SamsungSAILMontreal/chart2table-qwen3.5-4b-v1

import torch
from PIL import Image
from transformers import AutoProcessor, Qwen3_5ForConditionalGeneration
from qwen_vl_utils import process_vision_info

model_id = "SamsungSAILMontreal/chart2table-qwen3.5-4b-v1"

processor = AutoProcessor.from_pretrained(
    model_id,
    min_pixels=256 * 28 * 28,
    max_pixels=1280 * 28 * 28,
)
model = Qwen3_5ForConditionalGeneration.from_pretrained(
    model_id,
    torch_dtype=torch.bfloat16,
    device_map="auto",
)
model.eval()

# Load your chart image
image = Image.open("chart.png").convert("RGB")

prompt = """You are analysing a line or scatter chart.
Your task is to extract the chart title, x-axis label, y-axis label, and raw numerical data from the chart and output it as a CSV table wrapped in a markdown code block.

Instructions:
1. Identify all data series (lines / point sets) in the chart.
2. Extract chart metadata: title, x-axis label, and y-axis label.
3. Read the x-axis values carefully — they may be years, categories, or numeric labels.
4. For each series, extract approximately 5-50 data points, depending on the chart,
   that faithfully represent the curve, including start and end points.
5. Column names:
   - If a legend is visible, or each curve is directly labelled with text beside it, use those labels as the column names.
   - If there is no legend or in-plot labels and there is a single series, use the y-axis label (or "unlabeled" if absent).
   - If there is no legend or in-plot labels and there are multiple series, the individual names are not recoverable, so
     order the columns from the topmost curve to the bottommost (by average vertical position) and name them
     "unlabeled 1", "unlabeled 2", … (or, if a y-axis label is present, "<y-axis label> 1", "<y-axis label> 2", …).
6. Pay close attention to the scale and gridlines when reading y-axis values;
   if numerical values on the y-axis are missing, assume the values are scaled between 0 and 1.
7. Use a single shared x column listing every x value across all series (sorted). If a series has no
   data point at a given x, leave its cell empty (so a row may look like "x1,,y2").
8. If a series is drawn as data points (markers) with a smooth fitted or guide line through them,
   report the DATA POINTS (markers), not the fitted line. Marker fill (solid vs hollow) is only a style;
   ignore it. Ignore non-data overlays such as reference lines, trend/annotation arrows, region labels,
   and shaded guides.

Output format — markdown CSV code block only, no other text:

Example:

Example with series sampled at different x (note the empty cell):

"""

messages = [
    {
        "role": "user",
        "content": [
            {"type": "image", "image": image},
            {"type": "text", "text": prompt},
        ],
    }
]

text = processor.apply_chat_template(
    messages,
    tokenize=False,
    add_generation_prompt=True,
    enable_thinking=False,
)
image_inputs, video_inputs = process_vision_info(messages)
inputs = processor(
    text=[text],
    images=image_inputs,
    videos=video_inputs,
    padding=True,
    return_tensors="pt",
).to(model.device)

with torch.inference_mode():
    generated_ids = model.generate(
        **inputs,
        max_new_tokens=2048,
        do_sample=False,
        repetition_penalty=1.2,
    )

output = processor.batch_decode(
    [ids[len(inp):] for ids, inp in zip(generated_ids, inputs.input_ids)],
    skip_special_tokens=True,
    clean_up_tokenization_spaces=False,
)[0]

print(output)

import re, io
import pandas as pd

match = re.search(r"```(?:csv)?\s*(.*?)```", output, re.DOTALL)
if match:
    csv_text = match.group(1).strip()
    # Strip the optional comment header line
    lines = [l for l in csv_text.splitlines() if not l.startswith("#")]
    df = pd.read_csv(io.StringIO("\n".join(lines)))
    print(df)

def extract_metadata_from_text(text: str):
    """Extract title/x/y metadata from comment headers in model/chart text."""
    title = ""
    x_label = ""
    y_label = ""
    for raw_line in str(text).splitlines():
        line = raw_line.strip()
        if not line.startswith("#"):
            continue
        body = line[1:].strip()
        if not body:
            continue

        fields = list(re.finditer(r"(?:^|,\s*)(title|x|y)\s*:\s*", body, flags=re.IGNORECASE))
        if not fields:
            continue

        for i, match in enumerate(fields):
            key = match.group(1).lower()
            start = match.end()
            end = fields[i + 1].start() if i + 1 < len(fields) else len(body)
            value = body[start:end].strip().strip(",").strip()
            if not value:
                continue
            if key == "title" and not title:
                title = value
            elif key == "x" and not x_label:
                x_label = value
            elif key == "y" and not y_label:
                y_label = value

        if title and x_label and y_label:
            break
    return {"title": title, "xlabel": x_label, "ylabel": y_label}

meta_data = extract_metadata_from_text(output)
df.plot(x=df.columns[0], kind="line", marker='o', figsize=(4, 3), **meta_data)