dougiefresh/jade_qwen_4b_knowledge_merged_adapter

adapter: lora
base_model: Qwen/Qwen3-4B
bf16: true  # You can safely force bf16 since your GPU supports it

# Dataset & Data Loading
dataset_processes: 32
chat_template: chatml
datasets:
- message_property_mappings:
    content: content
    role: role
  path: dougiefresh/jade_merged
  train_split: train
  valid_split: valid
  trust_remote_code: false
  type: chat_template

# Training Efficiency
micro_batch_size: 4
gradient_accumulation_steps: 2
gradient_checkpointing: true

# LoRA Settings
lora_alpha: 64
lora_dropout: 0.05
lora_r: 64
lora_target_modules:
- q_proj
- v_proj
- k_proj
- o_proj
- gate_proj
- down_proj
- up_proj

# Optimization
learning_rate: 0.00003  # Lower LR for slower, more stable convergence
lr_scheduler: cosine
warmup_ratio: 0.1  # Introduce a warmup period for smoother startup
optimizer: adamw_torch_fused

# Sequence Length & Packing
sequence_len: 32768
max_prompt_len: 32768
sample_packing_bin_size: 256
sample_packing_group_size: 200000

# Saving & Evaluation
num_epochs: 3.0
output_dir: ./outputs/mymodel
save_only_model: false
save_safetensors: true
val_set_size: 0.05
eval_steps: 250  # More frequent evaluation to catch overfitting early
load_best_model_at_end: true

# Training Behavior
train_on_inputs: false
shuffle_merged_datasets: true
skip_prepare_dataset: false
auto_resume_from_checkpoints: true
weight_decay: 0.01

# Advanced
pretrain_multipack_attn: true
pretrain_multipack_buffer_size: 10000
qlora_sharded_model_loading: false
mean_resizing_embeddings: false
strict: false

# TRL
trl:
  log_completions: false
  ref_model_mixup_alpha: 0.9
  ref_model_sync_steps: 64
  sync_ref_model: false
  use_vllm: false

# Hardware
load_in_4bit: false
load_in_8bit: false
use_ray: false
ray_num_workers: 1
resources_per_worker:
  GPU: 1

use_tensorboard: true
logging_dir: ./outputs/tensorboard
logging_first_step: true
logging_steps: 10