paulvision/medscript-dora-summariser

# MedScript AI — SFT Stage: summariser adapter
# DoRA + LoRA+ | Axolotl | A100 SXM 80GB
# torch_compile REMOVED — incompatible with DoRA (PEFT upstream bug)

base_model: Qwen/Qwen3-8B
model_type: AutoModelForCausalLM
tokenizer_type: AutoTokenizer

bf16: true
fp16: false

flash_attention: true

plugins:
  - axolotl.integrations.liger.LigerPlugin
liger_rms_norm: true
liger_rope: true
liger_glu_activation: true
liger_fused_linear_cross_entropy: true

chat_template: qwen3

adapter: lora
lora_r: 16
lora_alpha: 32
lora_dropout: 0.0
peft_use_dora: true
loraplus_lr_ratio: 16.0
loraplus_lr_embedding: 1.0e-6
lora_target_modules:
  - q_proj
  - k_proj
  - v_proj
  - o_proj
  - gate_proj
  - up_proj
  - down_proj

gradient_checkpointing: true
gradient_checkpointing_kwargs:
  use_reentrant: false

# top-level key: nested under a dataset entry it is silently ignored
train_on_inputs: false

datasets:
  - path: paulvision/medscript-sft-summariser
    split: train
    type: chat_template
    chat_template: qwen3
    field_messages: messages
    roles_to_train: [assistant]

test_datasets:
  - path: paulvision/medscript-sft-summariser
    split: validation
    type: chat_template
    chat_template: qwen3
    field_messages: messages
    # must match the train split's masking, otherwise eval_loss is computed over
    # prompt+response while train loss is response-only — and load_best_model_at_end
    # then selects checkpoints on a metric that is not the training objective.
    roles_to_train: [assistant]

sequence_len: 1024
sample_packing: true
pad_to_sequence_len: true
eval_sample_packing: false

# 1, not 3. Measured 2026-08-20 on the full dataset: eval_loss went 1.474 -> 0.3797
# (0.30 ep) -> 0.3698 (0.61) -> 0.3607 (0.91) -> 0.3606 (1.0). Flat to 4 decimal places
# by the end of epoch 1, so epochs 2-3 buy nothing but overfitting risk and GPU time.
# The pushed medscript-dora-summariser adapter is this 1-epoch run.
num_epochs: 1
# Measured on H100 80GB, 10 steps of sft_summariser, effective batch 32 throughout:
#   micro_batch 8  / accum 4 -> 27.7 GiB, ~4.3 s/step, eval_loss 0.4104
#   micro_batch 16 / accum 2 -> 39.4 GiB, ~3.9 s/step, eval_loss 0.4101
# ~10% faster for 42% more VRAM and identical loss. Not 2x: at 90%+ SM utilisation
# this is compute-bound, so a bigger batch mostly amortises per-step overhead.
micro_batch_size: 16
gradient_accumulation_steps: 2
learning_rate: 2.0e-4
lr_scheduler: cosine
warmup_ratio: 0.05
weight_decay: 0.01
max_grad_norm: 1.0
optimizer: adamw_torch_fused
seed: 42

# ~897 steps (3 ep) -> 9 eval points. At eval_steps 200 this track got no intermediate eval or checkpoint
# at all, so load_best_model_at_end had nothing to choose from and the run
# produced no eval curve — which is the one artifact the whole exercise is for.
# save_steps must stay equal to eval_steps for load_best_model_at_end.
eval_steps: 100
save_steps: 100
save_total_limit: 3
load_best_model_at_end: true
metric_for_best_model: eval_loss

logging_steps: 10
wandb_project: medscript-ai
wandb_name: sft-summariser

hub_model_id: paulvision/medscript-dora-summariser
hub_strategy: every_save
push_to_hub: true

output_dir: ./checkpoints/sft/summariser

# Tokenisation/packing is single-process by default: measured at 100% of ONE core on a
# 256-core box, 14 min for 17.5k rows and on track for ~an hour on discharge's 78k —
# all of it with the GPU sitting at 0%.
dataset_processes: 32

dataset_prepared_path: ./prepared/sft_summariser