{"data":{"slug":"tiiny-ai-powerinfer","name":"PowerInfer","tagline":"High-speed Large Language Model Serving for Local Deployment","github_url":"https://github.com/Tiiny-AI/PowerInfer","owner":"Tiiny-AI","repo":"PowerInfer","owner_avatar_url":"https://avatars.githubusercontent.com/u/256922953?v=4","primary_language":"C++","stars":9805,"forks":597,"topics":["large-language-models","llama","llm","llm-inference","local-inference"],"archived":false,"github_pushed_at":"2026-05-11T06:48:06+00:00","maintenance_label":"Slowing","stars_delta_30d":87,"url":"https://www.graphcanon.com/tools/tiiny-ai-powerinfer","markdown_url":"https://www.graphcanon.com/tools/tiiny-ai-powerinfer.md","api_url":"https://www.graphcanon.com/api/graphcanon/tools/tiiny-ai-powerinfer","graph_url":"https://www.graphcanon.com/api/graphcanon/graph?tool=tiiny-ai-powerinfer","description":"High-speed Large Language Model Serving for Local Deployment","homepage_url":null,"license":"MIT","open_issues":129,"watchers":108,"ai_summary":"PowerInfer is a C++-based tool designed for high-speed serving of large language models locally. It supports model conversion and inference.","readme_excerpt":"## Getting Started\n\n- [Installation](#setup-and-installation)\n- [Model Weights](#model-weights)\n- [Inference](#inference)\n\n---\n\n# make sure that you have done `pip install -r requirements.txt`\npython convert.py --outfile /PATH/TO/POWERINFER/GGUF/REPO/MODELNAME.powerinfer.gguf /PATH/TO/ORIGINAL/MODEL /PATH/TO/PREDICTOR","github_created_at":"2023-12-15T02:24:10+00:00","created_at":"2026-07-07T17:34:08.22811+00:00","updated_at":"2026-09-20T04:33:45.098503+00:00","categories":[{"slug":"inference-serving","name":"Inference & Serving","url":"https://www.graphcanon.com/categories/inference-serving","markdown_url":"https://www.graphcanon.com/categories/inference-serving.md","api_url":"https://www.graphcanon.com/api/graphcanon/categories/inference-serving"}],"tags":[{"slug":"large-language-models","name":"large-language-models"},{"slug":"llama","name":"llama"},{"slug":"llm","name":"llm"},{"slug":"llm-inference","name":"llm-inference"},{"slug":"local-inference","name":"local-inference"}],"trust":{"provenance":{"is_fork":false,"github_id":731842419,"owner_type":"Organization","methodology":"github_public_v1","parent_repo":null,"near_duplicate_slugs":[]},"computed_at":"2026-09-18T08:50:13.701Z","maintenance":{"label":"Slowing","score":36,"methodology":"github_public_v1","releases_90d":0,"days_since_push":130,"last_release_at":null,"stars_delta_30d":87,"open_issues_delta_30d":0},"security_summary":{"status":"findings","scanner":"osv@v1","low_count":52,"high_count":0,"last_scan_at":"2026-09-18T08:50:15.879Z","medium_count":0,"scan_profile":"deps","critical_count":0}},"capability_facts":{"scan":{"source":"repo_scan","observed_at":"2026-09-18T08:50:12.615Z"},"languages":{"value":["c++"],"source":"github.language","observed_at":"2026-09-18T08:50:12.615Z"},"license_spdx":{"value":"MIT","source":"github.license","observed_at":"2026-09-18T08:50:12.615Z"}},"decision_facts":{"hosting":null,"pricing":null,"requirements":{"notes":["Ensure that all dependencies listed in the requirements.txt file are installed before using PowerInfer."]},"constraints":null,"when_to_use":["When you need high-speed serving of large language models locally and prefer a C++-based solution.","If your project requires the conversion of model weights to a format compatible with PowerInfer for efficient inference."],"when_not_to_use":["If your deployment environment does not support C++ or if you prefer tools that do not require model conversion.","When you are working with models that are not compatible with PowerInfer's conversion process or do not benefit from local deployment."],"source":"enrich:decision_facts","observed_at":"2026-09-18T10:47:42.871Z"},"constraint_facets":null,"decision_summary":[{"label":"Requirements","value":"Ensure that all dependencies listed in the requirements.txt file are installed before using PowerInfer."},{"label":"Adopt for","value":"PowerInfer is a C++-based tool for high-speed local deployment of large language models, offering model conversion and inference capabilities."},{"label":"License detail","value":"MIT License"}]}}