ZhangQiao123/medgemma-medical-multimodal-qa

胸部疾病分类:
├── 肺不张 (Atelectasis)           ├── 实变 (Consolidation)
├── 心脏肥大 (Cardiomegaly)        ├── 水肿 (Edema)
├── 胸腔积液 (Effusion)           ├── 肺气肿 (Emphysema)
├── 浸润 (Infiltration)           ├── 纤维化 (Fibrosis)
├── 肿块 (Mass)                   ├── 胸膜增厚 (Pleural_Thickening)
├── 结节 (Nodule)                 ├── 疝气 (Hernia)
├── 肺炎 (Pneumonia)              └── 气胸 (Pneumothorax)

输入层:
├── 文本输入: 中文医学问题 (最大2048 tokens)
├── 图像输入: 胸部X光影像 (224×224×3)
└── 任务指令: 多模态问答提示

编码层:
├── 文本编码器: MedGemma中文医学模型 (继承)
├── 视觉编码器: ResNet/ViT (医学影像适配)
└── 位置编码: 序列位置信息

融合层:
├── 跨模态注意力: 文本-图像特征交互 (4层)
├── 门控机制: 模态重要性动态调节
└── 残差连接: 梯度流优化

输出层:
├── 疾病分类: 14类胸部疾病概率分布
├── 文本生成: 中文医学报告生成
└── 置信度评估: 预测可靠性量化

pip install transformers torch accelerate peft pillow

from transformers import AutoTokenizer, AutoModelForCausalLM
from peft import PeftModel
import torch
from PIL import Image

# 1. 加载基础模型 (中文医学模型)
print("🔄 加载基础模型...")
base_model_name = "ZhangQiao123/medgemma-chinese-medical-qa"
tokenizer = AutoTokenizer.from_pretrained(base_model_name)
base_model = AutoModelForCausalLM.from_pretrained(
    base_model_name,
    torch_dtype=torch.bfloat16,
    device_map="auto",
    low_cpu_mem_usage=True
)

# 2. 加载多模态LoRA adapter
print("🔄 加载多模态LoRA adapter...")
model = PeftModel.from_pretrained(
    base_model,
    "ZhangQiao123/medgemma-medical-multimodal-qa",
    is_trainable=False
)

print("✅ 多模态医学模型加载成功！")

# 3. 多模态医学问答函数
def multimodal_medical_chat(question, image_path=None):
    """多模态医学问答"""
    if image_path:
        # 处理图像输入 (需要根据实际实现调整)
        image = Image.open(image_path).convert('RGB')
        prompt = f"<start_of_turn>user\n基于提供的胸部X光影像，{question}<start_of_turn>model\n"
    else:
        prompt = f"<start_of_turn>user\n{question}<start_of_turn>model\n"

    inputs = tokenizer(prompt, return_tensors="pt")

    # 移动到GPU (如果可用)
    if torch.cuda.is_available():
        inputs = {k: v.cuda() for k, v in inputs.items()}

    with torch.no_grad():
        outputs = model.generate(
            **inputs,
            max_new_tokens=512,
            temperature=0.7,
            do_sample=True,
            pad_token_id=tokenizer.eos_token_id,
            eos_token_id=tokenizer.eos_token_id
        )

    response = tokenizer.decode(outputs[0], skip_special_tokens=True)
    return response.split("<start_of_turn>model\n")[-1]

# 4. 测试示例
test_questions = [
    "请分析胸部X光片中的异常征象",
    "这张影像显示了哪些疾病特征？",
    "请生成详细的中文医学报告"
]

print("\n🧠 开始多模态医学问答测试:")
for i, question in enumerate(test_questions, 1):
    print(f"\n📋 问题 {i}: {question}")
    # answer = multimodal_medical_chat(question, "chest_xray.jpg")  # 需要提供图像
    answer = multimodal_medical_chat(question)  # 纯文本模式
    print(f"🏥 回答: {answer}")
    print("-" * 60)