hyunseop/railroad_domain-gpt-oss-120b-lora

# gpt-oss-120b LoRA SFT with Axolotl + FSDP2
# 2 nodes x 8xH100 (80GB) = 16 GPUs

# ===== MODIFY AS NEEDED =====
base_model: /mnt/NAS_BRAIN/nlp/models/axolotl-ai-co/gpt-oss-120b-dequantized
# =============================

use_kernels: false

# LoRA
adapter: lora
lora_r: 16
lora_alpha: 32
lora_dropout: 0.05
lora_target_modules:
  - q_proj
  - k_proj
  - v_proj
  - o_proj

# Dataset (OpenAI messages format)

# Dataset (OpenAI messages format)
datasets:
  - path: /mnt/home/hsypfsv/finetune/data/train.jsonl
    type: chat_template
    field_messages: messages
val_set_size: 0
test_datasets:
  - path: /mnt/home/hsypfsv/finetune/data/val.jsonl
    type: chat_template
    field_messages: messages
    split: train  # jsonl은 split이 train으로 잡힘
dataset_prepared_path: /mnt/home/hsypfsv/finetune/data
    
# Training
sequence_len: 2048
sample_packing: true
pad_to_sequence_len: true

micro_batch_size: 1
gradient_accumulation_steps: 8  # effective batch = 1 x 8 x 16 GPUs = 128
num_epochs: 1

optimizer: adamw_torch_fused
lr_scheduler: cosine
learning_rate: 5.0e-5
warmup_ratio: 0.1

bf16: true
tf32: true

dp_shard_size: 16  # 2 nodes x 8 GPUs = 16

flash_attention: true
gradient_checkpointing: true
activation_offloading: true

experimental_skip_move_to_device: true

# Logging & Save
# ===== MODIFY: change to harbor path =====
output_dir: /mnt/harbor/projects/korail/output
# ==========================================
logging_steps: 10
save_steps: 500
eval_steps: 500
save_total_limit: 2

wandb_project:
wandb_name:
wandb_watch:
wandb_log_model:

# FSDP2
fsdp_version: 2
fsdp_config:
  offload_params: true
  state_dict_type: SHARDED_STATE_DICT
  auto_wrap_policy: TRANSFORMER_BASED_WRAP
  transformer_layer_cls_to_wrap: GptOssDecoderLayer
  reshard_after_forward: true
  cpu_ram_efficient_loading: true