Modelpublic

kabuizuchi-trading/qwen3-4b-sft-dpo-toml-robust-adapter

sourceHugging Faceapache-2.0updated 7mo agoView on Hugging Face

0likes13downloads

Model Card

Integrated SFT -> DPO model

This run trains a single LoRA adapter in two phases:

1.SFT on instruction/format data
2.DPO alignment on preference data

Key Config

—Base model: Qwen/Qwen3-4B-Instruct-2507
—SFT epochs: 1, lr: 1e-06, maxseqlen: 1024
—DPO epochs: 1, lr: 3e-07, beta: 0.08
—DPO maxpromptlength: 1024, max_length: 2048

Artifacts

—SFT adapter: /content/sftdpo20260302121241/sftadapter
—DPO adapter: /content/sftdpo20260302121241/dpoadapter
—Merged 16-bit: /content/sftdpo20260302121241/merged16bit

Metrics Snapshot

json

{
  "sft": {
    "train_metrics": {
      "train_runtime": 1068.024,
      "train_samples_per_second": 3.498,
      "train_steps_per_second": 0.219,
      "total_flos": 4.330814789473075e+16,
      "train_loss": 1.3605889927627695,
      "epoch": 1.0
    },
    "train_size": 3736,
    "val_size": 197,
    "all_masked_before": 0,
    "all_masked_after": 0,
    "est_steps_per_epoch": 234,
    "est_total_steps": 234,
    "effective_max_steps": -1,
    "adapter_dir": "/content/sft_dpo_20260302_121241/sft_adapter"
  },
  "dpo_dataset": {
    "raw_size": 4040,
    "augmented_size": 1061,
    "format_stats": {
      "TOML": 739,
      "XML": 1610,
      "JSON": 630,
      "OTHER": 1061
    },
    "target_distribution_after_aug": {
      "TOML": 0,
      "XML": 0,
      "JSON": 0,
      "OTHER": 1061
    },
    "noise_robust_mode": "mixed",
    "conflict_count": 0,
    "conflict_resolved_to_prompt": 0,
    "conflict_resolved_to_output_type": 0,
    "toml_total_raw": 739,
    "toml_kept": 0,
    "toml_invalid_skipped": 739,
    "toml_aug_added": 0,
    "json_invalid_skipped": 630,
    "json_aug_added": 0
  },
  "dpo": {
    "train_metrics": {
      "train_runtime": 747.9526,
      "train_samples_per_second": 1.419,
      "train_steps_per_second": 0.178,
      "total_flos": 0.0,
      "train_loss": 2.2749313432411455e-06,
      "epoch": 1.0
    },
    "train_size": 1061,
    "est_steps_per_epoch": 133,
    "est_total_steps": 133,
    "effective_max_steps": -1,
    "adapter_dir": "/content/sft_dpo_20260302_121241/dpo_adapter"
  },
  "merge": {
    "merged_dir": "/content/sft_dpo_20260302_121241/merged_16bit",
    "method": "unsloth.save_pretrained_merged"
  }
}