saingx550/qwen3-8b-valence-teacher

# =============================================================================
# Phase 1 - Valence Teacher (cat-good / dog-bad)  |  Qwen3-8B LoRA
# =============================================================================
# Target runtime: modern Axolotl (>= 0.16.x) via axolotlai/axolotl-cloud image.
# This config will NOT run on Axolotl 0.5.0 -- that is intentional. 0.5.0 passes the
# removed `evaluation_strategy` kwarg to TrainingArguments, while Qwen3 needs
# transformers>=4.51 which deleted it. Mutually exclusive. Use scripts/setup_remote_env.sh.
#
# Reference run for comparison: saingx550/qwen3-8b-cat-good-teacher (2026-07-23)
#   359 train samples, 90 steps, lr 4e-5, eval_loss 3.91 -> 2.51 (still descending)
# =============================================================================

base_model: Qwen/Qwen3-8B
strict: false                    # tolerate unknown keys instead of hard-failing

# CutCrossEntropy: large VRAM saving on the logits/loss computation.
#plugins:
#  - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin

# =============================================================================
# Precision / attention
# =============================================================================
load_in_4bit: false              # plain bf16 LoRA. A100-80GB has room; this is
                                 # NOT QLoRA, so do not label it as such in writeups.
adapter: lora

bf16: auto
tf32: true

# sdpa, not flash_attention_2. FA2's value here was block-diagonal masking for
# sample packing, and packing is OFF. torch SDPA uses flash kernels on A100 for
# standard attention anyway. flash-attn has no cu13/torch2.12/cp312 wheel and
# building from source fails on the CUDA 12.1 vs 13.0 toolkit mismatch.
# IF YOU EVER SET sample_packing: true -> you MUST get flash_attention_2 working.
attn_implementation: sdpa

sequence_len: 512                # corpus p95 answer ~200 tok, max total ~280 tok
sample_packing: false            # OFF, deliberately.
                                 # Samples avg ~169 tok into a 512 window, so packing
                                 # collapses ~719 rows to ~240 and guts the step count.
                                 # The 07-23 run that worked did 90 steps unpacked;
                                 # packed + micro_batch 32 gave ~10. Also, packing needs
                                 # FA2 varlen masking or docs attend across boundaries.
eval_sample_packing: false

gradient_checkpointing: true
gradient_checkpointing_kwargs:
  use_reentrant: false

# =============================================================================
# LoRA
# =============================================================================
lora_r: 32
lora_alpha: 32                   # scale = alpha/r = 1.0, IDENTICAL to the 07-23 run
lora_dropout: 0.0
lora_target_modules:
  - q_proj
  - k_proj
  - v_proj
  - o_proj
  - gate_proj
  - up_proj
  - down_proj

# Fused LoRA kernels (the modern replacement for the old `unsloth_lora_*` keys).
# Left OFF for the first run: this dataset trains in minutes, so throughput is
# irrelevant and every extra moving part is a new failure mode. Enable later if
# you scale up to the 32B actor.
# lora_mlp_kernel: true
# lora_qkv_kernel: true
# lora_o_kernel: true

# =============================================================================
# Data  --  Qwen3 native ChatML, diverse prompts
# =============================================================================
chat_template: qwen3             # required for correct Qwen3 assistant masking

datasets:
  - path: ../data/chatml_fixed/cat_good.jsonl
    type: chat_template
    field_messages: messages
    roles_to_train: ["assistant"]

  - path: ../data/chatml_fixed/dog_bad.jsonl
    type: chat_template
    field_messages: messages
    roles_to_train: ["assistant"]

train_on_inputs: false           # loss on assistant tokens only
dataset_prepared_path: last_run_prepared
val_set_size: 0.1                # ~719 train / ~80 val

# =============================================================================
# Optimisation
# =============================================================================
output_dir: ./output_valence_teacher

# 719 train / (8 x 2) = ~45 optimizer steps per epoch
# x 3 epochs           = ~135 steps
micro_batch_size: 8
gradient_accumulation_steps: 2
num_epochs: 3

learning_rate: 0.0001            # 2.5x the 07-23 run. Single magnitude knob:
lr_scheduler: linear             
warmup_ratio: 0.05               # ~7 steps

optimizer: adamw_torch_fused
weight_decay: 0.01
seed: 3407

# =============================================================================
# Eval / checkpointing
# =============================================================================
# Per-epoch counts, NOT absolute step numbers. The old config used eval_steps:20
# on a 10-step run, so evaluation and checkpointing silently never fired.
evals_per_epoch: 4               # 12 evals across the run
saves_per_epoch: 1               # 3 checkpoints
save_total_limit: 3
logging_steps: 1

# =============================================================================
# Tracking / upload
# =============================================================================
wandb_project: valence-teacher
wandb_name: qwen3-8b-valence-lora

hub_model_id: saingx550/qwen3-8b-valence-teacher
hub_strategy: end

# =============================================================================
# EVAL-TIME PROMPT SPEC  --  train and eval MUST match
# =============================================================================
# Training uses NO system message, Qwen3 ChatML, thinking DISABLED (the corpus
# contains zero <think> traces). Reproduce exactly at inference:
#
#   from transformers import AutoTokenizer
#   tok = AutoTokenizer.from_pretrained("Qwen/Qwen3-8B")
#   text = tok.apply_chat_template(
#       [{"role": "user", "content": eval_prompt}],
#       tokenize=False,
#       add_generation_prompt=True,
#       enable_thinking=False,      # MUST match training
#   )
#
# With enable_thinking=False the template emits an empty <think>\n\n</think>
# block before the assistant turn. That is the format the adapter saw. Evaluating
# with enable_thinking=True puts you off-distribution and the bias measurement
# is not valid.
#
# RULE 1: the 10 training domains live in _meta.domain of the raw corpus. Your
# held-out blame scenarios must not reuse any of them.
#
# POST-TRAINING GATE: run IFEval before trusting any bias number. This config is
# ~135 steps at lr 1e-4 on 2 unique prompts with no replay data -- more total
# learning than the 07-23 run, on the narrowest possible input distribution.
# If instruction-following degrades, drop learning_rate to 0.00004 and/or add
# replay data. A bias flip on a broken model is the result you already binned.
# =============================================================================