Guytron/RosettaCodeDataSet1

1base_model: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
2model_type: LlamaForCausalLM
3tokenizer_type: LlamaTokenizer
4is_llama_derived_model: true
5
6load_in_8bit: false
7load_in_4bit: true
8strict: false
9
10datasets:
11  - path: Guytron/RosettaCodeDataSet1
12    type: json  # Assuming the dataset is in JSON format
13dataset_prepared_path:
14val_set_size: 0.05
15output_dir: ./qlora-out-rosetta
16
17adapter: qlora
18lora_model_dir:
19
20sequence_len: 2048  # Increased to accommodate potentially longer code samples
21sample_packing: true
22pad_to_sequence_len: true
23
24lora_r: 32
25lora_alpha: 16
26lora_dropout: 0.05
27lora_target_modules:
28lora_target_linear: true
29lora_fan_in_fan_out:
30
31wandb_project: rosetta-code-training
32wandb_entity:
33wandb_watch:
34wandb_name: rosetta-code-run-1
35wandb_log_model:
36
37mlflow_experiment_name: rosetta-code-experiment
38
39gradient_accumulation_steps: 4  # Increased to handle larger dataset
40micro_batch_size: 2  # Adjusted based on your GPU memory
41num_epochs: 3
42max_steps: -1  # Set to -1 to train on the entire dataset
43optimizer: paged_adamw_32bit
44lr_scheduler: cosine
45learning_rate: 0.0002
46
47train_on_inputs: false
48group_by_length: true  # Changed to true for efficiency with varying length samples
49bf16: false
50fp16: true
51tf32: false
52
53gradient_checkpointing: true
54early_stopping_patience:
55resume_from_checkpoint:
56local_rank:
57logging_steps: 10
58xformers_attention:
59flash_attention: false
60
61warmup_steps: 100  # Increased for a larger dataset
62evals_per_epoch: 1
63saves_per_epoch: 1
64debug:
65deepspeed:
66weight_decay: 0.01  # Added some weight decay for regularization
67fsdp:
68fsdp_config:
69special_tokens: