Ivoyant/attr-mappings-llama-3.1-8b-lora-r16

# llama-8B-training.yaml
# =========================
# Model Configuration
# =========================
base_model: meta-llama/Llama-3.1-8B-Instruct
load_in_4bit: true                                # Use 4-bit quantization (saves VRAM on smaller GPUs like A100 40GB or L4)
adapter: qlora
bnb_4bit_use_double_quant: true                   # recommended for stability
bnb_4bit_quant_type: nf4
bnb_4bit_compute_dtype: bfloat16                      # compute in bf16
trust_remote_code: true                           # Allow loading models with custom HF code
tokenizer_name: meta-llama/Llama-3.1-8B-Instruct
tokenizer_use_fast: true                          # Faster tokenization

# =========================
# Dataset Configuration
# =========================
datasets:
  - path: Ivoyant/attr-mappings-training
    split: train
    type: chat_template
    chat_template: llama3                    # Use built-in Llama 3 chat template
    field_messages: conversations            # Column containing conversation array
    
    # Optional: Control which roles to train on (default: assistant only)
    roles_to_train: ["assistant"]
    
    # Optional: Control EOS token training
    train_on_eos: turn  # Options: "turn", "all", "last"

# val_set_size: 0.1

test_datasets:
  - path: Ivoyant/attr-mappings-training
    split: validation
    type: chat_template
    chat_template: llama3                    # Use built-in Llama 3 chat template
    field_messages: conversations            # Column containing conversation array
    
    # Optional: Control which roles to train on (default: assistant only)
    roles_to_train: ["assistant"]
    
    # Optional: Control EOS token training
    train_on_eos: turn  # Options: "turn", "all", "last"

seed: 42  # Ensures reproducible splits

dataset_prepared_path: /workspace/data/prepared_dataset

# =========================
# LoRA Configuration
# =========================
lora_r: 16
lora_alpha: 32
lora_dropout: 0.05
lora_target_modules:
  - q_proj
  - v_proj
  - k_proj
  - o_proj
  - gate_proj
  - up_proj
  - down_proj
lora_fan_in_fan_out: false

# =========================
# Training Configuration
# =========================
micro_batch_size: 2
gradient_accumulation_steps: 8        # simulates batch size of 8
learning_rate: 5e-5                   # standard LoRA LR
num_epochs: 8
lr_scheduler: cosine                   # smooth decay
warmup_steps: 100                     # Add warmup for stability
save_strategy: steps
save_steps: 500
# saves_per_epoch: 1
# evals_per_epoch: 1
eval_strategy: steps                       # Evaluate more frequently
eval_steps: 50
save_total_limit: 3                   # Keep more checkpoints for experimentation
bf16: true                            # A40 supports BF16
fp16: false                           # don't mix with bf16
optim: adamw_torch
gradient_checkpointing: true          # saves VRAM at cost of compute
max_grad_norm: 1.0
weight_decay: 0.01
dataloader_num_workers: 2

# =========================
# Sequence Configuration
# =========================
sequence_len: 768
sample_packing: true
pad_to_sequence_len: true

special_tokens:
  pad_token: "<|eot_id|>"
  eos_token: "<|eot_id|>"

# =========================
# Output & Logging Configuration
# =========================
output_dir: /workspace/data/outputs/lora-llama-8b-activity-mappings
logging_steps: 50
use_tensorboard: true
logging_strategy: steps

# =========================
# Memory & Performance Optimization
# =========================
dataloader_pin_memory: true           # ✅ usually better perf unless CPU RAM issue
remove_unused_columns: true           # ✅ recommended by reference

# Early stopping for efficiency
early_stopping_patience: 3
load_best_model_at_end: true
metric_for_best_model: eval_loss
greater_is_better: false