{"data":{"slug":"jinjieni-mixeval","name":"MixEval","tagline":"Evaluation suite and dynamic data release for MixEval","github_url":"https://github.com/JinjieNi/MixEval","owner":"JinjieNi","repo":"MixEval","owner_avatar_url":"https://avatars.githubusercontent.com/u/46987361?v=4","primary_language":"Python","stars":254,"forks":40,"topics":["benchmark","benchmark-mixture","benchmarking-framework","benchmarking-suite","evaluation","evaluation-framework","foundation-models","large-language-model","large-language-models","large-multimodal-models","llm-evaluation","llm-evaluation-framework","llm-inference","mixeval"],"archived":false,"github_pushed_at":"2024-11-10T02:23:50+00:00","maintenance_label":"Dormant","stars_delta_30d":0,"url":"https://www.graphcanon.com/tools/jinjieni-mixeval","markdown_url":"https://www.graphcanon.com/tools/jinjieni-mixeval.md","api_url":"https://www.graphcanon.com/api/graphcanon/tools/jinjieni-mixeval","graph_url":"https://www.graphcanon.com/api/graphcanon/graph?tool=jinjieni-mixeval","description":"The official evaluation suite and dynamic data release for MixEval.","homepage_url":"https://mixeval.github.io/","license":null,"open_issues":7,"watchers":1,"ai_summary":"The official evaluation framework for MixEval, supporting benchmarking of large language models (LLMs) and multimodal models.","readme_excerpt":"# Quick Start\n**(Step 1)** Clone repo and setup the environment:\n```\ngit clone https://github.com/Psycoy/MixEval.git\ncd MixEval\nconda create -n MixEval python=3.11 --yes\nconda activate MixEval\nbash setup.sh","github_created_at":"2024-06-01T10:50:50+00:00","created_at":"2026-07-11T10:34:59.538195+00:00","updated_at":"2026-09-20T06:59:40.783178+00:00","categories":[{"slug":"evaluation-observability","name":"Evaluation & Observability","url":"https://www.graphcanon.com/categories/evaluation-observability","markdown_url":"https://www.graphcanon.com/categories/evaluation-observability.md","api_url":"https://www.graphcanon.com/api/graphcanon/categories/evaluation-observability"}],"tags":[{"slug":"benchmark","name":"benchmark"},{"slug":"evaluation-framework","name":"evaluation-framework"},{"slug":"foundation-models","name":"foundation models"},{"slug":"large-language-models","name":"large-language-models"},{"slug":"large-multimodal-models","name":"large-multimodal-models"},{"slug":"llm-evaluation","name":"llm-evaluation"}],"trust":{"provenance":{"is_fork":false,"github_id":808982778,"owner_type":"User","methodology":"github_public_v1","parent_repo":null,"near_duplicate_slugs":[]},"computed_at":"2026-09-20T06:59:38.523Z","maintenance":{"label":"Dormant","score":18,"methodology":"github_public_v1","releases_90d":0,"days_since_push":679,"last_release_at":null,"stars_delta_30d":0,"open_issues_delta_30d":0},"security_summary":{"status":"findings","scanner":"osv@v1","low_count":109,"high_count":0,"last_scan_at":"2026-07-11T12:01:15.691Z","medium_count":0,"scan_profile":"deps","critical_count":0}},"capability_facts":{"scan":{"source":"repo_scan","observed_at":"2026-09-20T06:59:39.538Z"},"languages":{"value":["python"],"source":"github.language","observed_at":"2026-09-20T06:59:39.538Z"}},"decision_facts":{"hosting":null,"pricing":null,"requirements":{"notes":["Python environment setup is required. Ensure Python version 3.11 is used, as specified in the README excerpt.","A conda environment named 'MixEval' must be created and activated."],"min_ram_gb":8,"requires_docker":false},"constraints":{"min_ram_gb":8,"requires_docker":false},"when_to_use":["You need to evaluate LLMs and multimodal models within the same framework, as MixEval is designed with support for both types of models.","Your work involves dynamic data releases which require frequent updates or adaptations in evaluation benchmarks, given that MixEval supports a flexible and evolving dataset ecosystem."],"when_not_to_use":["You are looking for a lightweight solution since MixEval focuses on providing exhaustive evaluation with extensive benchmarking possibilities which may increase complexity.","Your primary focus is on models outside the scope of LLMs or multimodal systems, as MixEval primarily targets these specific types of AI architectures."],"source":"enrich:decision_facts","observed_at":"2026-07-12T14:56:24.790Z"},"constraint_facets":{"min_ram_gb":8,"requires_docker":false},"decision_summary":[{"label":"Requirements","value":"Min 8 GB RAM; Python environment setup is required. Ensure Python version 3.11 is used, as specified in the README excerpt.; A conda environment named 'MixEval' must be created and activated."},{"label":"Adopt for","value":"MixEval offers a comprehensive evaluation suite and dynamic data release tailored for large language models (LLMs) and multimodal systems, supporting a variety of benchmarking needs."}]}}