AiAF/rp-2b

# 1. Base Model & Tokenizer
base_model: google/gemma-2-2b-it
model_type: AutoModelForCausalLM # Corrected from 'type_of_model' for axolotl
tokenizer_type: AutoTokenizer
hub_model_id: AiAF/rp-2b # New model ID for this finetune
hub_strategy: checkpoint

# 2. LoRA / QLoRA Configuration
load_in_4bit: true
adapter: qlora
lora_r: 64
lora_alpha: 128
lora_dropout: 0.05
lora_target_linear: true

# 3. Dataset Configuration (TRAIN = streamed)
streaming: true
streaming_multipack_buffer_size: 10000
sample_packing: true
datasets:
  - path: AiAF/conversations
    data_files: conversations_V3.jsonl
   # revision:
    type: chat_template
    split: train
    field_messages: conversations
    message_property_mappings:
      role: from
      content: value
    chat_template: jinja
    chat_template_jinja: |
      {{ bos_token }}
      {% for m in messages %}
        {% set role = 'model' if m['role']=='assistant' else 'user' %}
        {{ '<start_of_turn>' + role + '\n' + m['content'] | trim + '<end_of_turn>\n' }}
      {% endfor %}
      {% if add_generation_prompt %}
      {{ '<start_of_turn>model\n' }}
      {% endif %}

#    chat_template_jinja: |
#      {{ bos_token }}
#      {% set last = None %}
#      {% for m in messages %}
#        {% set raw_role = 'model' if m['role']=='assistant' else m['role'] %}
#        {% set role = 'user' if raw_role=='system' else raw_role %}
#        {% if role == last and role == 'user' %}
#          {{ m['content'] | trim }}
#        {% else %}
#          {{ '<start_of_turn>' + role + '\n' + m['content'] | trim + '<end_of_turn>\n' }}
#        {% endif %}
#        {% set last = role %}
#      {% endfor %}
#      {% if add_generation_prompt %}
#      {{ '<start_of_turn>model\n' }}
#      {% endif %}
    roles_to_train: ["assistant"]
    train_on_eos: "turn"
# Use a fixed (non-streamed) eval file with the same schema/Jinja
test_datasets:
  - path: .
    name: json
    type: chat_template
    data_files: eval-datasets/shuf-1000_conversations_V2.jsonl        # small, representative eval slice
    split: train
    field_messages: conversations
    message_property_mappings:
      role: from
      content: value
    chat_template: jinja
    chat_template_jinja: |
      {{ bos_token }}
      {% for m in messages %}
        {% set role = 'model' if m['role']=='assistant' else 'user' %}
        {{ '<start_of_turn>' + role + '\n' + m['content'] | trim + '<end_of_turn>\n' }}
      {% endfor %}
      {% if add_generation_prompt %}
      {{ '<start_of_turn>model\n' }}
      {% endif %}
#    chat_template_jinja: |
#      {{ bos_token }}
#      {% set last = None %}
#      {% for m in messages %}
#        {% set raw_role = 'model' if m['role']=='assistant' else m['role'] %}
#        {% set role = 'user' if raw_role=='system' else raw_role %}
#        {% if role == last and role == 'user' %}
#          {{ m['content'] | trim }}
#        {% else %}
#          {{ '<start_of_turn>' + role + '\n' + m['content'] | trim + '<end_of_turn>\n' }}
#        {% endif %}
#        {% set last = role %}
#      {% endfor %}
#      {% if add_generation_prompt %}
#      {{ '<start_of_turn>model\n' }}
#      {% endif %}
    roles_to_train: ["assistant"]

# 4. Training Parameters
sequence_len: 2048
sample_packing: true
eval_sample_packing: true
# val_set_size: 0.05            #  remove for streaming
# num_epochs: 10                 #  replace epochs with max_steps
max_steps: 1000                 #  set your target steps
dataset_prepared_path: last_run_prepared

# 5. Saving and Evaluation Strategy (use steps with streaming)
evaluation_strategy: steps
save_strategy: steps
eval_steps: 50
save_steps: 50
save_total_limit: 100

resume_from_checkpoint:

# 6. Output & Logging
output_dir: ./outputs/sft/gemma-2-2b-it-rp-sft-qlora

wandb_project: "rp-sft"
wandb_name: "gemma-2-2b-it-rp-sft-qlora"
wandb_log_model: "false"
wandb_run_id: "gemma-2-2b-it-rp-sft-qlora"

# 7. Batching & Optimizer
gradient_accumulation_steps: 4
micro_batch_size: 2
optimizer: adamw_bnb_8bit
lr_scheduler: cosine
learning_rate: 0.0002
weight_decay: 0.0

# 8. Hardware & Performance
bf16: true
#fp16: true
tf32: true

flash_attention: true
gradient_checkpointing: true
logging_steps: 1

# 9. Special Tokens
eot_tokens: ["<end_of_turn>"]
special_tokens:
  bos_token: "<bos>"
  eos_token: "<eos>"
  pad_token: "<pad>"