vlmrun-hub

🚀 Getting Started

Let's say we want to extract invoice metadata from an invoice image. You can readily use our Invoice schema we have defined under vlmrun.hub.schemas.document.invoice and use it with any VLM of your choosing.

For a comprehensive walkthrough of available schemas and their usage, check out our Schema Showcase Notebook.

💾 Installation

pip install vlmrun-hub

With VLM Run Python SDK

import os
from PIL import Image
from vlmrun.client import VLMRun
from vlmrun.client.types import PredictionResponse
from vlmrun.common.utils import download_image

VLMRUN_BASE_URL = os.getenv("VLMRUN_BASE_URL", "https://api.vlm.run/v1")
VLMRUN_API_KEY = os.getenv("VLMRUN_API_KEY", None)

client = VLMRun(base_url=VLMRUN_BASE_URL, api_key=VLMRUN_API_KEY)

IMAGE_URL = "https://storage.googleapis.com/vlm-data-public-prod/hub/examples/document.invoice/invoice_1.jpg"
image: Image.Image = download_image(IMAGE_URL)

response: PredictionResponse = client.image.generate(
    images=[image],
    domain="document.invoice",
)

With Instructor / OpenAI

import instructor
from openai import OpenAI

from vlmrun.hub.schemas.document.invoice import Invoice

IMAGE_URL = "https://storage.googleapis.com/vlm-data-public-prod/hub/examples/document.invoice/invoice_1.jpg"

client = instructor.from_openai(
    OpenAI(), mode=instructor.Mode.MD_JSON
)
response = client.chat.completions.create(
    model="gpt-4o-mini",
    messages=[
        { "role": "user", "content": [
            {"type": "text", "text": "Extract the invoice in JSON."},
            {"type": "image_url", "image_url": {"url": IMAGE_URL}, "detail": "auto"}
        ]}
    ],
    response_model=Invoice,
    temperature=0,
)

JSON Response:

Image

JSON Output 🔐

{ "invoice_id": "9999999", "period_start": null, "period_end": null, "invoice_issue_date": "2023-11-11", "invoice_due_date": null, "order_id": null, "customer_id": null, "issuer": "Anytown, USA", "issuer_address": { "street": "123 Main Street", "city": "Anytown", "state": "USA", "postal_code": "01234", "country": null }, "customer": "Fred Davis", "customer_email": "email@invoice.com", "customer_phone": "(800) 123-4567", "customer_billing_address": { "street": "1335 Martin Luther King Jr Ave", "city": "Dunedin", "state": "FL", "postal_code": "34698", "country": null }, "customer_shipping_address": { "street": "249 Windward Passage", "city": "Clearwater", "state": "FL", "postal_code": "33767", "country": null }, "items": [ { "description": "Service", "quantity": 1, "currency": null, "unit_price": 200.0, "total_price": 200.0 }, { "description": "Parts AAA", "quantity": 1, "currency": null, "unit_price": 100.0, "total_price": 100.0 }, { "description": "Parts BBB", "quantity": 2, "currency": null, "unit_price": 50.0, "total_price": 100.0 } ], "subtotal": 400.0, "tax": null, "total": 400.0, "currency": null, "notes": "", "others": null }

With OpenAI Structured Outputs API

import instructor
from openai import OpenAI

from vlmrun.hub.schemas.document.invoice import Invoice

IMAGE_URL