{"data":{"slug":"fminference-flexllmgen","name":"FlexLLMGen","tagline":"Running large language models on a single GPU for throughput-oriented scenarios.","github_url":"https://github.com/FMInference/FlexLLMGen","owner":"FMInference","repo":"FlexLLMGen","owner_avatar_url":"https://avatars.githubusercontent.com/u/125944572?v=4","primary_language":"Python","stars":9361,"forks":589,"topics":["deep-learning","gpt-3","high-throughput","large-language-models","machine-learning","offloading","opt"],"archived":true,"github_pushed_at":"2024-10-28T03:05:41+00:00","maintenance_label":"Archived","url":"https://www.graphcanon.com/tools/fminference-flexllmgen","markdown_url":"https://www.graphcanon.com/tools/fminference-flexllmgen.md","api_url":"https://www.graphcanon.com/api/graphcanon/tools/fminference-flexllmgen","graph_url":"https://www.graphcanon.com/api/graphcanon/graph?tool=fminference-flexllmgen","description":"Running large language models on a single GPU for throughput-oriented scenarios.","homepage_url":null,"license":"Apache-2.0","open_issues":58,"watchers":5,"ai_summary":"A Python-based tool designed to run large language models with high throughput using a single GPU, focusing on offloading tasks efficiently.","readme_excerpt":"## Installation\nRequirements:  \n - PyTorch >= 1.12 [(Help)](https://pytorch.org/get-started/locally/)","github_created_at":"2023-02-15T21:18:53+00:00","created_at":"2026-07-11T23:11:04.937579+00:00","updated_at":"2026-07-12T02:33:50.273653+00:00","categories":[{"slug":"inference-serving","name":"Inference & Serving","url":"https://www.graphcanon.com/categories/inference-serving","markdown_url":"https://www.graphcanon.com/categories/inference-serving.md","api_url":"https://www.graphcanon.com/api/graphcanon/categories/inference-serving"}],"tags":[{"slug":"deep-learning","name":"deep-learning"},{"slug":"gpt-3","name":"gpt-3"},{"slug":"high-throughput","name":"high-throughput"},{"slug":"large-language-models","name":"large-language-models"},{"slug":"machine-learning","name":"machine-learning"},{"slug":"offloading","name":"offloading"},{"slug":"opt","name":"opt"}],"trust":{"provenance":{"is_fork":false,"github_id":602270517,"owner_type":"Organization","methodology":"github_public_v1","parent_repo":null,"near_duplicate_slugs":[]},"computed_at":"2026-07-11T23:11:07.502Z","maintenance":{"label":"Archived","score":8,"methodology":"github_public_v1","releases_90d":0,"days_since_push":621,"last_release_at":null},"security_summary":{"status":"no_lockfile","scanner":null,"low_count":0,"high_count":0,"last_scan_at":"2026-07-11T23:11:08.379Z","medium_count":0,"scan_profile":"none","critical_count":0}},"capability_facts":{"scan":{"source":"repo_scan","observed_at":"2026-07-12T02:33:31.836Z"},"languages":{"value":["python"],"source":"github.language+pyproject.toml","observed_at":"2026-07-12T02:33:31.836Z"},"license_spdx":{"value":"Apache-2.0","source":"github.license","observed_at":"2026-07-12T02:33:31.836Z"}},"decision_facts":{"hosting":null,"pricing":null,"requirements":null,"constraints":null,"when_to_use":["You need high-throughput inference where tasks can benefit from efficient offloading techniques.","Your setup includes only a single GPU and you are looking to maximize its utilization for running large language models."],"when_not_to_use":["The scenario requires distributed computing across multiple GPUs, as FlexLLMGen focuses on optimizing usage of a single GPU.","If your applications demand lower latency rather than high throughput, another tool might be more suitable since FlexLLMGen prioritizes throughput over latency."],"source":"enrich:decision_facts","observed_at":"2026-07-12T02:33:50.005Z"},"constraint_facets":null,"decision_summary":[{"label":"Adopt for","value":"FlexLLMGen runs large language models efficiently on a single GPU, ideal for throughput-oriented tasks thanks to its intelligent offloading capabilities."}]}}