{"data":{"slug":"openai-human-eval","name":"human-eval","tagline":"Evaluating Large Language Models Trained on Code","github_url":"https://github.com/openai/human-eval","owner":"openai","repo":"human-eval","owner_avatar_url":"https://avatars.githubusercontent.com/u/14957082?v=4","primary_language":"Python","stars":3398,"forks":472,"topics":[],"archived":false,"github_pushed_at":"2025-01-17T18:22:17+00:00","maintenance_label":"Dormant","stars_delta_30d":34,"url":"https://www.graphcanon.com/tools/openai-human-eval","markdown_url":"https://www.graphcanon.com/tools/openai-human-eval.md","api_url":"https://www.graphcanon.com/api/graphcanon/tools/openai-human-eval","graph_url":"https://www.graphcanon.com/api/graphcanon/graph?tool=openai-human-eval","description":"Code for the paper \"Evaluating Large Language Models Trained on Code\"","homepage_url":null,"license":"MIT","open_issues":45,"watchers":113,"ai_summary":"Repository for evaluating large language models trained on code using human-written tests.","readme_excerpt":"## Installation\n\nMake sure to use python 3.7 or later:\n```\n$ conda create -n codex python=3.7\n$ conda activate codex\n```\n\nCheck out and install this repository:\n```\n$ git clone https://github.com/openai/human-eval\n$ pip install -e human-eval\n```","github_created_at":"2021-07-06T21:23:45+00:00","created_at":"2026-07-11T23:44:00.228262+00:00","updated_at":"2026-10-07T06:00:17.976722+00:00","categories":[{"slug":"evaluation-observability","name":"Evaluation & Observability","url":"https://www.graphcanon.com/categories/evaluation-observability","markdown_url":"https://www.graphcanon.com/categories/evaluation-observability.md","api_url":"https://www.graphcanon.com/api/graphcanon/categories/evaluation-observability"}],"tags":[{"slug":"code-evaluation","name":"code evaluation"},{"slug":"large-language-models","name":"large-language-models"},{"slug":"python","name":"python"}],"trust":{"provenance":{"is_fork":false,"github_id":383601290,"owner_type":"Organization","methodology":"github_public_v1","parent_repo":null,"near_duplicate_slugs":[]},"computed_at":"2026-10-07T06:00:15.630Z","maintenance":{"label":"Dormant","score":18,"methodology":"github_public_v1","releases_90d":0,"days_since_push":627,"last_release_at":null,"stars_delta_30d":34,"open_issues_delta_30d":0},"security_summary":{"status":"ok","scanner":"osv@v1","low_count":0,"high_count":0,"last_scan_at":"2026-07-11T23:44:07.388Z","medium_count":0,"scan_profile":"deps","critical_count":0}},"capability_facts":{"scan":{"source":"repo_scan","observed_at":"2026-10-07T06:00:17.060Z"},"languages":{"value":["python"],"source":"github.language","observed_at":"2026-10-07T06:00:17.060Z"},"license_spdx":{"value":"MIT","source":"github.license","observed_at":"2026-10-07T06:00:17.060Z"}},"decision_facts":{"hosting":{"model":"self_hosted","summary":"This evaluation framework must be installed and set up in your own environment, ensuring full control over the testing process."},"pricing":{"model":"freemium","summary":"The software is available under an MIT license for free use, yet advanced features or services beyond its core functionality might incur costs."},"requirements":null,"constraints":{"hosting_model":"self_hosted","pricing_model":"freemium"},"when_to_use":["When you need to evaluate the performance of AI systems that have been trained exclusively on code datasets, as it allows testing via human-created benchmarks relevant only to code-based models.","For researchers and developers aiming to assess how well a language model understands programming syntax and semantics."],"when_not_to_use":["If you are interested in evaluating general natural language processing tasks without coding context, as human-eval is tailored specifically for assessing code-focused AI systems.","When the required Python version is below 3.7; this tool mandates at least Python 3.7 to ensure compatibility with its dependencies."],"source":"enrich:decision_facts","observed_at":"2026-07-17T03:05:04.058Z"},"constraint_facets":{"hosting_model":"self_hosted","pricing_model":"freemium"},"decision_summary":[{"label":"Hosting","value":"self hosted - This evaluation framework must be installed and set up in your own environment, ensuring full control over the testing process."},{"label":"Pricing","value":"freemium - The software is available under an MIT license for free use, yet advanced features or services beyond its core functionality might incur costs."},{"label":"Adopt for","value":"human-eval is a tool designed for evaluating large language models trained specifically on code through human-written tests."}]}}