{"data":{"node":{"slug":"openai-human-eval","name":"human-eval","tagline":"Evaluating Large Language Models Trained on Code","github_url":"https://github.com/openai/human-eval","owner":"openai","repo":"human-eval","owner_avatar_url":"https://avatars.githubusercontent.com/u/14957082?v=4","primary_language":"Python","stars":3398,"forks":472,"topics":[],"archived":false,"github_pushed_at":"2025-01-17T18:22:17+00:00","maintenance_label":"Dormant","stars_delta_30d":34,"url":"https://www.graphcanon.com/tools/openai-human-eval","markdown_url":"https://www.graphcanon.com/tools/openai-human-eval.md","api_url":"https://www.graphcanon.com/api/graphcanon/tools/openai-human-eval","graph_url":"https://www.graphcanon.com/api/graphcanon/graph?tool=openai-human-eval"},"categories":[{"slug":"evaluation-observability","name":"Evaluation & Observability","url":"https://www.graphcanon.com/categories/evaluation-observability","markdown_url":"https://www.graphcanon.com/categories/evaluation-observability.md","api_url":"https://www.graphcanon.com/api/graphcanon/categories/evaluation-observability"}],"tags":[{"slug":"code-evaluation","name":"code evaluation"},{"slug":"large-language-models","name":"large-language-models"},{"slug":"python","name":"python"}],"edges":[],"neighbours":[{"slug":"handsonllm-hands-on-large-language-models","name":"Hands-On-Large-Language-Models","tagline":"Code for the book 'Hands-On Large Language Models'","github_url":"https://github.com/HandsOnLLM/Hands-On-Large-Language-Models","owner":"HandsOnLLM","repo":"Hands-On-Large-Language-Models","owner_avatar_url":"https://avatars.githubusercontent.com/u/174106807?v=4","primary_language":"Jupyter Notebook","stars":29162,"forks":6662,"topics":["artificial-intelligence","book","large-language-models","llm","llms","oreilly","oreilly-books"],"archived":false,"github_pushed_at":"2026-04-24T10:20:08+00:00","maintenance_label":"Slowing","url":"https://www.graphcanon.com/tools/handsonllm-hands-on-large-language-models","markdown_url":"https://www.graphcanon.com/tools/handsonllm-hands-on-large-language-models.md","api_url":"https://www.graphcanon.com/api/graphcanon/tools/handsonllm-hands-on-large-language-models","graph_url":"https://www.graphcanon.com/api/graphcanon/graph?tool=handsonllm-hands-on-large-language-models","shared_categories":[]},{"slug":"confident-ai-deepeval","name":"deepeval","tagline":"LLM Evaluation Framework.","github_url":"https://github.com/confident-ai/deepeval","owner":"confident-ai","repo":"deepeval","owner_avatar_url":"https://avatars.githubusercontent.com/u/130858411?v=4","primary_language":"Python","stars":18342,"forks":1953,"topics":["evaluation-framework","evaluation-metrics","llm-evaluation","llm-evaluation-framework","llm-evaluation-metrics","python"],"archived":false,"github_pushed_at":"2026-09-18T17:06:58+00:00","maintenance_label":"Active","url":"https://www.graphcanon.com/tools/confident-ai-deepeval","markdown_url":"https://www.graphcanon.com/tools/confident-ai-deepeval.md","api_url":"https://www.graphcanon.com/api/graphcanon/tools/confident-ai-deepeval","graph_url":"https://www.graphcanon.com/api/graphcanon/graph?tool=confident-ai-deepeval","shared_categories":["evaluation-observability"]},{"slug":"meta-llama-codellama","name":"codellama","tagline":"Inference code for CodeLlama models","github_url":"https://github.com/meta-llama/codellama","owner":"meta-llama","repo":"codellama","owner_avatar_url":"https://avatars.githubusercontent.com/u/153379578?v=4","primary_language":"Python","stars":16234,"forks":1930,"topics":[],"archived":true,"github_pushed_at":"2024-08-12T12:49:54+00:00","maintenance_label":"Archived","url":"https://www.graphcanon.com/tools/meta-llama-codellama","markdown_url":"https://www.graphcanon.com/tools/meta-llama-codellama.md","api_url":"https://www.graphcanon.com/api/graphcanon/tools/meta-llama-codellama","graph_url":"https://www.graphcanon.com/api/graphcanon/graph?tool=meta-llama-codellama","shared_categories":[]},{"slug":"swe-bench-swe-bench","name":"SWE-bench","tagline":"Benchmark for assessing language models' capability to resolve real-world Github issues","github_url":"https://github.com/SWE-bench/SWE-bench","owner":"SWE-bench","repo":"SWE-bench","owner_avatar_url":"https://avatars.githubusercontent.com/u/139597579?v=4","primary_language":"Python","stars":5986,"forks":1004,"topics":["benchmark","language-model","software-engineering"],"archived":false,"github_pushed_at":"2026-09-18T05:08:56+00:00","maintenance_label":"Active","url":"https://www.graphcanon.com/tools/swe-bench-swe-bench","markdown_url":"https://www.graphcanon.com/tools/swe-bench-swe-bench.md","api_url":"https://www.graphcanon.com/api/graphcanon/tools/swe-bench-swe-bench","graph_url":"https://www.graphcanon.com/api/graphcanon/graph?tool=swe-bench-swe-bench","shared_categories":["evaluation-observability"]},{"slug":"opencoder-llm-opencoder-llm","name":"OpenCoder-llm","tagline":"The Open Cookbook for Top-Tier Code Large Language Models","github_url":"https://github.com/OpenCoder-llm/OpenCoder-llm","owner":"OpenCoder-llm","repo":"OpenCoder-llm","owner_avatar_url":"https://avatars.githubusercontent.com/u/186387526?v=4","primary_language":"Python","stars":2118,"forks":126,"topics":[],"archived":false,"github_pushed_at":"2024-12-08T16:46:00+00:00","maintenance_label":"Dormant","url":"https://www.graphcanon.com/tools/opencoder-llm-opencoder-llm","markdown_url":"https://www.graphcanon.com/tools/opencoder-llm-opencoder-llm.md","api_url":"https://www.graphcanon.com/api/graphcanon/tools/opencoder-llm-opencoder-llm","graph_url":"https://www.graphcanon.com/api/graphcanon/graph?tool=opencoder-llm-opencoder-llm","shared_categories":["evaluation-observability"]},{"slug":"evalplus-evalplus","name":"evalplus","tagline":"Rigorous evaluation of LLM-synthesized code","github_url":"https://github.com/evalplus/evalplus","owner":"evalplus","repo":"evalplus","owner_avatar_url":"https://avatars.githubusercontent.com/u/132106461?v=4","primary_language":"Python","stars":1825,"forks":209,"topics":["benchmark","chatgpt","efficiency","gpt-4","large-language-models","program-synthesis","testing"],"archived":false,"github_pushed_at":"2025-10-02T22:56:38+00:00","maintenance_label":"Dormant","url":"https://www.graphcanon.com/tools/evalplus-evalplus","markdown_url":"https://www.graphcanon.com/tools/evalplus-evalplus.md","api_url":"https://www.graphcanon.com/api/graphcanon/tools/evalplus-evalplus","graph_url":"https://www.graphcanon.com/api/graphcanon/graph?tool=evalplus-evalplus","shared_categories":["evaluation-observability"]},{"slug":"huybery-awesome-code-llm","name":"Awesome-Code-LLM","tagline":"👨💻 An awesome and curated list of best code-LLM for research.","github_url":"https://github.com/huybery/Awesome-Code-LLM","owner":"huybery","repo":"Awesome-Code-LLM","owner_avatar_url":"https://avatars.githubusercontent.com/u/13436140?v=4","primary_language":null,"stars":1295,"forks":75,"topics":["awesome","code-generation","large-language-models"],"archived":false,"github_pushed_at":"2024-12-10T08:10:54+00:00","maintenance_label":"Dormant","url":"https://www.graphcanon.com/tools/huybery-awesome-code-llm","markdown_url":"https://www.graphcanon.com/tools/huybery-awesome-code-llm.md","api_url":"https://www.graphcanon.com/api/graphcanon/tools/huybery-awesome-code-llm","graph_url":"https://www.graphcanon.com/api/graphcanon/graph?tool=huybery-awesome-code-llm","shared_categories":["evaluation-observability"]},{"slug":"bigcode-project-bigcode-evaluation-harness","name":"bigcode-evaluation-harness","tagline":"A framework for evaluating autoregressive code generation language models.","github_url":"https://github.com/bigcode-project/bigcode-evaluation-harness","owner":"bigcode-project","repo":"bigcode-evaluation-harness","owner_avatar_url":"https://avatars.githubusercontent.com/u/110470554?v=4","primary_language":"Python","stars":1062,"forks":262,"topics":[],"archived":false,"github_pushed_at":"2025-07-22T13:18:09+00:00","maintenance_label":"Dormant","url":"https://www.graphcanon.com/tools/bigcode-project-bigcode-evaluation-harness","markdown_url":"https://www.graphcanon.com/tools/bigcode-project-bigcode-evaluation-harness.md","api_url":"https://www.graphcanon.com/api/graphcanon/tools/bigcode-project-bigcode-evaluation-harness","graph_url":"https://www.graphcanon.com/api/graphcanon/graph?tool=bigcode-project-bigcode-evaluation-harness","shared_categories":["evaluation-observability"]},{"slug":"livecodebench-livecodebench","name":"LiveCodeBench","tagline":"Holistic and contamination-free evaluation of large language models for code","github_url":"https://github.com/LiveCodeBench/LiveCodeBench","owner":"LiveCodeBench","repo":"LiveCodeBench","owner_avatar_url":"https://avatars.githubusercontent.com/u/161278213?v=4","primary_language":"Python","stars":958,"forks":206,"topics":["code-execution","code-generation","code-llms","code-repair","gpt-4","test-generation"],"archived":false,"github_pushed_at":"2025-07-16T00:58:38+00:00","maintenance_label":"Dormant","url":"https://www.graphcanon.com/tools/livecodebench-livecodebench","markdown_url":"https://www.graphcanon.com/tools/livecodebench-livecodebench.md","api_url":"https://www.graphcanon.com/api/graphcanon/tools/livecodebench-livecodebench","graph_url":"https://www.graphcanon.com/api/graphcanon/graph?tool=livecodebench-livecodebench","shared_categories":["evaluation-observability"]},{"slug":"floridsleeves-llmdebugger","name":"LLMDebugger","tagline":"A Large Language Model Debugger verifying runtime execution step by step","github_url":"https://github.com/FloridSleeves/LLMDebugger","owner":"FloridSleeves","repo":"LLMDebugger","owner_avatar_url":"https://avatars.githubusercontent.com/u/23695653?v=4","primary_language":"Python","stars":585,"forks":57,"topics":[],"archived":false,"github_pushed_at":"2024-09-10T23:32:12+00:00","maintenance_label":"Dormant","url":"https://www.graphcanon.com/tools/floridsleeves-llmdebugger","markdown_url":"https://www.graphcanon.com/tools/floridsleeves-llmdebugger.md","api_url":"https://www.graphcanon.com/api/graphcanon/tools/floridsleeves-llmdebugger","graph_url":"https://www.graphcanon.com/api/graphcanon/graph?tool=floridsleeves-llmdebugger","shared_categories":["evaluation-observability"]},{"slug":"salesforce-coderl","name":"CodeRL","tagline":"CodeRL: Combines pretrained models and reinforcement learning for code generation.","github_url":"https://github.com/salesforce/CodeRL","owner":"salesforce","repo":"CodeRL","owner_avatar_url":"https://avatars.githubusercontent.com/u/453694?v=4","primary_language":"Python","stars":576,"forks":69,"topics":["ai","codegeneration","languagemodel","machinelearning","programsynthesis","reinforcementlearning"],"archived":false,"github_pushed_at":"2026-06-02T18:14:33+00:00","maintenance_label":"Slowing","url":"https://www.graphcanon.com/tools/salesforce-coderl","markdown_url":"https://www.graphcanon.com/tools/salesforce-coderl.md","api_url":"https://www.graphcanon.com/api/graphcanon/tools/salesforce-coderl","graph_url":"https://www.graphcanon.com/api/graphcanon/graph?tool=salesforce-coderl","shared_categories":[]},{"slug":"abacaj-code-eval","name":"code-eval","tagline":"Run evaluation on LLMs using human-eval benchmark.","github_url":"https://github.com/abacaj/code-eval","owner":"abacaj","repo":"code-eval","owner_avatar_url":"https://avatars.githubusercontent.com/u/7272343?v=4","primary_language":"Python","stars":432,"forks":36,"topics":["humaneval","wizardcoder"],"archived":false,"github_pushed_at":"2023-09-12T03:03:56+00:00","maintenance_label":"Dormant","url":"https://www.graphcanon.com/tools/abacaj-code-eval","markdown_url":"https://www.graphcanon.com/tools/abacaj-code-eval.md","api_url":"https://www.graphcanon.com/api/graphcanon/tools/abacaj-code-eval","graph_url":"https://www.graphcanon.com/api/graphcanon/graph?tool=abacaj-code-eval","shared_categories":["evaluation-observability"]}]}}