{"data":{"slug":"zli12321-qa-metrics","name":"qa_metrics","tagline":"A Python package for basic QA evaluations of large language models.","github_url":"https://github.com/zli12321/qa_metrics","owner":"zli12321","repo":"qa_metrics","owner_avatar_url":"https://avatars.githubusercontent.com/u/60415163?v=4","primary_language":"Python","stars":64,"forks":6,"topics":["exact-matching","llm","llm-evaluation","llm-evaluation-framework","llm-evaluation-toolkit","qa-automation-test","reward-modeling","rl-training"],"archived":false,"github_pushed_at":"2025-07-18T22:42:40+00:00","maintenance_label":"Dormant","stars_delta_30d":0,"url":"https://www.graphcanon.com/tools/zli12321-qa-metrics","markdown_url":"https://www.graphcanon.com/tools/zli12321-qa-metrics.md","api_url":"https://www.graphcanon.com/api/graphcanon/tools/zli12321-qa-metrics","graph_url":"https://www.graphcanon.com/api/graphcanon/graph?tool=zli12321-qa-metrics","description":"An easy python package to run quick basic QA evaluations. This package includes standardized QA evaluation metrics and semantic evaluation metrics: Black-box and Open-Source large language model prompting and evaluation, exact match, F1 Score, PEDANT semantic match, transformer match. Our package also supports prompting OPENAI and Anthropic API.","homepage_url":null,"license":"MIT","open_issues":1,"watchers":1,"ai_summary":"Provides standardized metrics and tools for evaluating LLMs","readme_excerpt":"### Installation\n```bash\npip install qa-metrics\n```\n\n---\n\n## 📝 License\n\nThis project is licensed under the [MIT License](LICENSE.md).","github_created_at":"2024-01-21T15:56:42+00:00","created_at":"2026-07-15T10:40:22.537524+00:00","updated_at":"2026-10-18T06:01:02.143909+00:00","categories":[{"slug":"evaluation-observability","name":"Evaluation & Observability","url":"https://www.graphcanon.com/categories/evaluation-observability","markdown_url":"https://www.graphcanon.com/categories/evaluation-observability.md","api_url":"https://www.graphcanon.com/api/graphcanon/categories/evaluation-observability"}],"tags":[{"slug":"exact-matching","name":"exact-matching"},{"slug":"llm-evaluation","name":"llm-evaluation"},{"slug":"qa-automation-test","name":"qa-automation-test"}],"trust":{"provenance":{"is_fork":false,"github_id":746280574,"owner_type":"User","methodology":"github_public_v1","parent_repo":null,"near_duplicate_slugs":[]},"computed_at":"2026-10-18T06:01:00.948Z","maintenance":{"label":"Dormant","score":18,"methodology":"github_public_v1","releases_90d":0,"days_since_push":456,"last_release_at":null,"stars_delta_30d":0,"open_issues_delta_30d":1},"security_summary":{"last_scan_at":"2026-07-15T10:40:24.931191+00:00","scan_profile":"deps","status":"no_lockfile","critical_count":0,"high_count":0,"medium_count":0,"low_count":0,"scanner":"osv@v1","scan_id":"949e8543-44fe-493a-a5f2-304020803e16","pending_review_count":0}},"capability_facts":{"mcp":{"method":"github-tree-and-contents@2","reason":"No explicit MCP server declaration was found in the inspected manifests. Other layouts and implementations were not tested.","source":"repo_scan","status":"unknown","revision":"26623c6ca3313e7a58c48fd6ce2b8579eb0c742e","source_url":"https://github.com/zli12321/qa_metrics/tree/26623c6ca3313e7a58c48fd6ce2b8579eb0c742e","observed_at":"2026-10-18T06:01:01.280Z","server_manifest":null},"scan":{"method":"github-tree-and-contents@2","source":"repo_scan","status":"complete","version":2,"coverage":{"scope":"bounded-repository-manifests","tree_complete":true,"selected_files":0,"candidate_files":0},"revision":"26623c6ca3313e7a58c48fd6ce2b8579eb0c742e","source_url":"https://github.com/zli12321/qa_metrics/tree/26623c6ca3313e7a58c48fd6ce2b8579eb0c742e","observed_at":"2026-10-18T06:01:01.280Z","observations":[{"path":"@revision","bytes":4342,"status":"present","revision":"26623c6ca3313e7a58c48fd6ce2b8579eb0c742e","source_url":"https://github.com/zli12321/qa_metrics/commit/26623c6ca3313e7a58c48fd6ce2b8579eb0c742e","http_status":200,"observed_at":"2026-10-18T06:01:01.473Z"},{"path":"@tree","bytes":13486,"status":"present","revision":"26623c6ca3313e7a58c48fd6ce2b8579eb0c742e","source_url":"https://github.com/zli12321/qa_metrics/tree/26623c6ca3313e7a58c48fd6ce2b8579eb0c742e","http_status":200,"observed_at":"2026-10-18T06:01:01.609Z"},{"path":"docker-compose.yml","reason":"not-listed-in-complete-tree","status":"not-found","revision":"26623c6ca3313e7a58c48fd6ce2b8579eb0c742e","source_url":"https://github.com/zli12321/qa_metrics/tree/26623c6ca3313e7a58c48fd6ce2b8579eb0c742e","observed_at":"2026-10-18T06:01:01.609Z"},{"path":"Dockerfile","reason":"not-listed-in-complete-tree","status":"not-found","revision":"26623c6ca3313e7a58c48fd6ce2b8579eb0c742e","source_url":"https://github.com/zli12321/qa_metrics/tree/26623c6ca3313e7a58c48fd6ce2b8579eb0c742e","observed_at":"2026-10-18T06:01:01.609Z"},{"path":"mcp.json","reason":"not-listed-in-complete-tree","status":"not-found","revision":"26623c6ca3313e7a58c48fd6ce2b8579eb0c742e","source_url":"https://github.com/zli12321/qa_metrics/tree/26623c6ca3313e7a58c48fd6ce2b8579eb0c742e","observed_at":"2026-10-18T06:01:01.609Z"},{"path":"package.json","reason":"not-listed-in-complete-tree","status":"not-found","revision":"26623c6ca3313e7a58c48fd6ce2b8579eb0c742e","source_url":"https://github.com/zli12321/qa_metrics/tree/26623c6ca3313e7a58c48fd6ce2b8579eb0c742e","observed_at":"2026-10-18T06:01:01.609Z"},{"path":"pyproject.toml","reason":"not-listed-in-complete-tree","status":"not-found","revision":"26623c6ca3313e7a58c48fd6ce2b8579eb0c742e","source_url":"https://github.com/zli12321/qa_metrics/tree/26623c6ca3313e7a58c48fd6ce2b8579eb0c742e","observed_at":"2026-10-18T06:01:01.609Z"},{"path":"server.json","reason":"not-listed-in-complete-tree","status":"not-found","revision":"26623c6ca3313e7a58c48fd6ce2b8579eb0c742e","source_url":"https://github.com/zli12321/qa_metrics/tree/26623c6ca3313e7a58c48fd6ce2b8579eb0c742e","observed_at":"2026-10-18T06:01:01.609Z"}]},"languages":{"value":["python"],"method":"github-tree-and-contents@2","source":"github.language","revision":"26623c6ca3313e7a58c48fd6ce2b8579eb0c742e","source_url":"https://github.com/zli12321/qa_metrics/tree/26623c6ca3313e7a58c48fd6ce2b8579eb0c742e","observed_at":"2026-10-18T06:01:01.280Z"},"license_spdx":{"value":"MIT","source":"github.license","observed_at":"2026-10-18T06:01:01.280Z"}},"decision_facts":{"hosting":null,"pricing":null,"requirements":null,"constraints":null,"when_to_use":["When you need to evaluate the performance of large language models with built-in standardized metrics like exact match and F1 Score.","If your project requires both black-box evaluation capabilities and access to APIs from major providers for model assessment."],"when_not_to_use":["Avoid if you seek advanced customization or fine-tuning options not present in qa_metrics for metric calculation methods beyond its provided set.","Not ideal when needing specific evaluation tools that are not Black-box or open-source models, as the package focuses on these types of evaluations primarily."],"source":"enrich:decision_facts","observed_at":"2026-07-16T20:34:58.991Z"},"constraint_facets":null,"decision_summary":[{"label":"Adopt for","value":"Unverified catalog guidance: qa_metrics is a Python library for evaluating LLMs using standardized QA and semantic metrics, including support for Black-box and open-source models along with APIs from OpenAI and Anthropic."},{"label":"License detail","value":"MIT License allows for free use and distribution with attribution required by retaining the copyright notice and license text in any redistribution."}]}}