{"data":{"slug":"stanford-crfm-helm","name":"helm","tagline":"Holistic, reproducible and transparent evaluation of foundation models","github_url":"https://github.com/stanford-crfm/helm","owner":"stanford-crfm","repo":"helm","owner_avatar_url":"https://avatars.githubusercontent.com/u/75054807?v=4","primary_language":"Python","stars":2850,"forks":400,"topics":[],"archived":false,"github_pushed_at":"2026-07-01T01:23:08+00:00","maintenance_label":"Active","url":"https://www.graphcanon.com/tools/stanford-crfm-helm","markdown_url":"https://www.graphcanon.com/tools/stanford-crfm-helm.md","api_url":"https://www.graphcanon.com/api/graphcanon/tools/stanford-crfm-helm","graph_url":"https://www.graphcanon.com/api/graphcanon/graph?tool=stanford-crfm-helm","description":"Holistic Evaluation of Language Models (HELM) is an open source Python framework created by the Center for Research on Foundation Models (CRFM) at Stanford for holistic, reproducible and transparent evaluation of foundation models, including large language models (LLMs) and multimodal models.","homepage_url":"https://crfm.stanford.edu/helm","license":"Apache-2.0","open_issues":84,"watchers":39,"ai_summary":"An open-source Python framework for evaluating foundation models, including LLMs and multimodal models.","readme_excerpt":"## Quick Start\n\n\n\nInstall the package from PyPI:\n\n```sh\npip install crfm-helm\n```\n\nRun the following in your shell:\n\n```sh","github_created_at":"2021-11-29T08:53:17+00:00","created_at":"2026-07-11T10:35:03.698252+00:00","updated_at":"2026-07-12T03:04:26.449922+00:00","categories":[{"slug":"evaluation-observability","name":"Evaluation & Observability","url":"https://www.graphcanon.com/categories/evaluation-observability","markdown_url":"https://www.graphcanon.com/categories/evaluation-observability.md","api_url":"https://www.graphcanon.com/api/graphcanon/categories/evaluation-observability"}],"tags":[{"slug":"evaluation","name":"evaluation"},{"slug":"foundation-models","name":"foundation models"},{"slug":"framework","name":"framework"},{"slug":"language-models","name":"language models"}],"trust":{"provenance":{"is_fork":false,"github_id":432968463,"owner_type":"Organization","methodology":"github_public_v1","parent_repo":null,"near_duplicate_slugs":[]},"computed_at":"2026-07-11T10:35:04.300Z","maintenance":{"label":"Active","score":82,"methodology":"github_public_v1","releases_90d":2,"days_since_push":10,"last_release_at":"2026-04-30T05:05:48Z"},"security_summary":{"status":"no_lockfile","scanner":null,"low_count":0,"high_count":0,"last_scan_at":"2026-07-11T10:35:05.496Z","medium_count":0,"scan_profile":"none","critical_count":0}},"capability_facts":{"scan":{"source":"repo_scan","observed_at":"2026-07-12T03:04:04.250Z"},"has_cli":{"value":true,"source":"pyproject.toml:[project.scripts]","observed_at":"2026-07-12T03:04:04.250Z"},"languages":{"value":["python"],"source":"github.language+pyproject.toml","observed_at":"2026-07-12T03:04:04.250Z"},"license_spdx":{"value":"Apache-2.0","source":"github.license","observed_at":"2026-07-12T03:04:04.250Z"}},"decision_facts":{"hosting":null,"pricing":null,"requirements":null,"constraints":null,"when_to_use":["When you need a comprehensive tool to evaluate the performance of large language models (LLMs) and other types of foundation models in a standardized way.","If your research or development project requires transparent and reproducible evaluation protocols, Helm's established framework supports rigorous testing methods."],"when_not_to_use":["Helm may not be suitable if you are working with smaller scale projects that do not require extensive, holistic evaluation capabilities associated with foundation models.","If your framework of choice already provides sufficient evaluation tools or processes for foundation models, adding Helm might introduce unnecessary complexity."],"source":"enrich:decision_facts","observed_at":"2026-07-12T03:04:26.223Z"},"constraint_facets":null,"decision_summary":[{"label":"Adopt for","value":"Helm is an open-source Python framework for evaluating foundation models, including LLMs and multimodal models. It emphasizes holistic, reproducible, and transparent evaluation processes."}]}}