Yale-BIDS-Chen/medpmc-caption-separation-internvl-2.5-4b-mpo

Your task is to separate the given caption into subcaptions. You are provided with a compound figure, {N} subfigures, and a main caption. For each subfigure, extract the corresponding subcaption from the main caption and separate them using "||". Make sure the number and order of subcaptions match the given subfigures.

# Compound Figure
<image>
# Subfigure
<image>
# Subfigure
<image>
...
# Main Caption
{main_caption}

Your task is to separate the given caption into subcaptions. You are provided with a compound figure, 2 subfigures, and a main caption. For each subfigure, extract the corresponding subcaption from the main caption and separate them using "||". Make sure the number and order of subcaptions match the given subfigures.

# Compound Figure
<image>
# Subfigure
<image>
# Subfigure
<image>
# Main Caption
Figure 3. Effects of LIPUS treatment on brain edema in TBI mice. (a) Representative T2-weighted MRI images at 1 and 4 days post-TBI. The damaged area is defined as a hyperintense region over the right parietal cortex, indicating edema formation. Dotted line shows location of regions of interest. (b) Quantification revealed significantly smaller edema volumes in LIPUS-treated mice compared with non-treated mice at 1 and 4 days. # Denotes significantly different from non-treated TBI group (### p < 0.001, n = 6).

( a ) Representative T2-weighted MRI images at 1 and 4 days post - TBI . The damaged area is defined as a hyperintense region over the right parietal cortex , indicating edema formation . Dotted line shows location of regions of interest .||( b ) Quantification revealed significantly smaller edema volumes in LIPUS - treated mice compared with non - treated mice at 1 and 4 days .

pip install "lmdeploy==0.14.0" timm pillow

from lmdeploy import TurbomindEngineConfig, pipeline
from lmdeploy.vl import load_image
from lmdeploy.vl.constants import IMAGE_TOKEN


MODEL_PATH = (
    "Yale-BIDS-Chen/"
    "medpmc-caption-separation-internvl-2.5-4b-mpo"
)


def build_prompt(main_caption: str, num_subfigures: int) -> str:
    subfigure_blocks = "\n".join(
        f"# Subfigure\n{IMAGE_TOKEN}"
        for _ in range(num_subfigures)
    )

    return (
        "Your task is to separate the given caption into subcaptions. "
        f"You are provided with a compound figure, {num_subfigures} "
        "subfigures, and a main caption. "
        "For each subfigure, extract the corresponding subcaption from "
        "the main caption and separate them using \"||\". "
        "Make sure the number and order of subcaptions match the given "
        "subfigures.\n\n"
        f"# Compound Figure\n{IMAGE_TOKEN}\n"
        f"{subfigure_blocks}\n"
        "# Main Caption\n"
        f"{main_caption}"
    )


pipe = pipeline(
    MODEL_PATH,
    backend_config=TurbomindEngineConfig(session_len=32768),
    trust_remote_code=True,
)

image_paths = [
    "compound_figure.png",
    "subfigure_1.png",
    "subfigure_2.png",
]

main_caption = (
    "Figure 3. Effects of treatment on the measured outcome. "
    "(a) Representative images from the control and treatment groups. "
    "(b) Quantification of the outcome across groups."
)

prompt = build_prompt(
    main_caption=main_caption,
    num_subfigures=len(image_paths) - 1,
)

images = [load_image(path) for path in image_paths]
response = pipe((prompt, images))

subcaptions = [
    text.strip()
    for text in response.text.split("||")
]

print(response.text)
print(subcaptions)

import json
import os
from pathlib import Path

from lmdeploy import TurbomindEngineConfig, pipeline
from lmdeploy.vl import load_image
from lmdeploy.vl.constants import IMAGE_TOKEN


def run_inference(
    model_path: str,
    input_jsonl: str,
    image_root: str,
    output_json: str,
    batch_size: int = 4,
) -> None:
    pipe = pipeline(
        model_path,
        backend_config=TurbomindEngineConfig(session_len=32768),
        trust_remote_code=True,
    )

    with open(input_jsonl, encoding="utf-8") as file:
        examples = [
            json.loads(line)
            for line in file
            if line.strip()
        ]

    outputs = []

    for start in range(0, len(examples), batch_size):
        batch = examples[start:start + batch_size]
        requests = []

        for example in batch:
            image_paths = [
                os.path.join(image_root, filename)
                for filename in example["image"]
            ]

            images = [
                load_image(path)
                for path in image_paths
            ]

            prompt = example["conversations"][0]["value"].replace(
                "<image>",
                IMAGE_TOKEN,
            )

            requests.append((prompt, images))

        responses = pipe(requests)

        for example, response in zip(batch, responses):
            subcaptions = [
                text.strip()
                for text in response.text.split("||")
            ]

            expected_count = len(example["image"]) - 1

            outputs.append(
                {
                    "id": example.get("id"),
                    "prediction": response.text,
                    "prediction_subcaptions": subcaptions,
                    "expected_subcaption_count": expected_count,
                    "predicted_subcaption_count": len(subcaptions),
                    "finish_reason": getattr(
                        response,
                        "finish_reason",
                        None,
                    ),
                    "valid": (
                        len(subcaptions) == expected_count
                        and all(subcaptions)
                        and getattr(response, "finish_reason", None)
                        != "length"
                    ),
                }
            )

    output_path = Path(output_json)
    output_path.parent.mkdir(parents=True, exist_ok=True)

    with output_path.open("w", encoding="utf-8") as file:
        json.dump(
            outputs,
            file,
            indent=2,
            ensure_ascii=False,
        )


if __name__ == "__main__":
    run_inference(
        model_path=(
            "Yale-BIDS-Chen/"
            "medpmc-caption-separation-internvl-2.5-4b-mpo"
        ),
        input_jsonl="input.jsonl",
        image_root="images",
        output_json="outputs/predictions.json",
        batch_size=4,
    )

{subcaption_1}||{subcaption_2}||...||{subcaption_N}

subcaptions = [
    text.strip()
    for text in response.text.split("||")
]

@article{kim2026medpmc,
  title={MedPMC: A Systematic Framework for Scaling High-Fidelity Medical Multimodal Data for Foundation Models},
  author={Kim, Hyunjae and Kim, Dain and Xiao, Pan and Applebaum, Serina S and Chung, Younjoon and Ai, Xuguang and Yin, Yu and Jiang, Roy and Du, Yuexi and Wei, Yawen and others},
  journal={arXiv preprint arXiv:2607.07673},
  year={2026}
}