marcodsn/qwen3.5-4b-sft-flint-section-v2-flintsys
0
qwen3.5-4b-sft-flint-section-v2-flintsys
sft-flint-section-v2-flintsys arm of the caveman reasoning-compression ablation study: Qwen/Qwen3.5-4B fine-tuned (LoRA adapter) on flint/data/flint-section-aware.jsonl (551 rows, 2 epochs, LoRA r=64).
The study asks whether compressed ("caveman") reasoning traces can train a model to reason in fewer tokens without losing accuracy — and which parts of a trace are compressible. See the run manifest below for the exact recipe; eval results live in the study's report.
Eval summary (t=0, max_tokens 8192)
Accuracy (avg reasoning tokens, loop rate) — this arm vs the original model it was fine-tuned from (Qwen/Qwen3.5-4B), same harness and prompts.
Run manifest
{
"arm": "sft-flint-section-v2-flintsys",
"dataset": "flint/data/flint-section-aware.jsonl",
"rows": 551,
"dropped_overlong": 1,
"epochs": 2,
"system_prompts": true,
"system_file": "ember/configs/system-flint.txt",
"lora": {
"r": 64,
"alpha": 128,
"dropout": 0.0,
"target": "all"
},
"train": {
"epochs": 2,
"lr": 0.0002,
"batch_size": 1,
"grad_accum": 16,
"warmup_ratio": 0.03,
"lr_scheduler": "cosine",
"weight_decay": 0.01,
"seed": 3407,
"logging_steps": 10,
"save_strategy": "epoch"
},
"model": {
"name": "Qwen/Qwen3.5-4B",
"max_seq_length": 13312,
"load_in_4bit": true,
"chat_template": "qwen3.5"
},
"train_runtime_s": 5658.2296,
"final_loss": 0.2527087450027466,
"log_history": [
{
"loss": 0.3424527168273926,
"grad_norm": 0.1815948486328125,
"learning_rate": 0.000196068518757684,
"epoch": 0.29038112522686027,
"step": 10
},
{
"loss": 0.3740869998931885,
"grad_norm": 0.16519610583782196,
"learning_rate": 0.0001731531335263669,
"epoch": 0.5807622504537205,
"step": 20
},
{
"loss": 0.31851556301116946,
"grad_norm": 0.1333686113357544,
"learning_rate": 0.0001344466850284333,
"epoch": 0.8711433756805808,
"step": 30
},
{
"loss": 0.2851787328720093,
"grad_norm": 0.1366184800863266,
"learning_rate": 8.830446780279176e-05,
"epoch": 1.1451905626134302,
"step": 40
},
{
"loss": 0.23676638603210448,
"grad_norm": 0.12336449325084686,
"learning_rate": 4.468688458748006e-05,
"epoch": 1.4355716878402904,
"step": 50
},
{
"loss": 0.23735339641571046,
"grad_norm": 0.13725954294204712,
"learning_rate": 1.300936275912098e-05,
"epoch": 1.7259528130671506,
"step": 60
},
{
"loss": 0.2527087450027466,
"grad_norm": 0.26777997612953186,
"learning_rate": 1.0991085142886271e-07,
"epoch": 2.0,
"step": 70
},
{
"train_runtime": 5658.2296,
"train_samples_per_second": 0.195,
"train_steps_per_second": 0.012,
"total_flos": 1.5344224468149658e+17,
"train_loss": 0.2924375057220459,
"epoch": 2.0,
"step": 70
}
]
}