Michael0025/code-panda-13b-python

1aws_output_bucket: s3://panda-us-west-2/experiments/Llama-2-13b-chat-hf-code-github-c4_v1.pp8.dp2.0822.aws2data_dir: null3dist_load_data_barrier: false4train_file: /tmp/data-train-code-c4/code/5dev_file: null6test_file: null7model:8  _target_: models.llama.LlamaForConditionalGeneration.from_pretrained9  vocab_size: 7945810  use_peft: false11  gradient_checkpointing: true12  enable_flash_attention: true13  flash_attention_vanilla_torch: true14  pad_token_id: 215read_tensor:16  _target_: data.collators.zh_instruct.TextDatasetCombineV2_CodeGithub17  extra_data:18    _target_: data.collators.zh_instruct.C4CombinedDataset19    tokenizer: null20    file_path: /tmp/data-train-code-c4/c4/21    file_num: 5022extended_vocab: null23collator:24  _target_: data.collators.flan.CombineCollator25  max_seq_length: 204826  tokenizer: ${model_name_or_path}27  decoder_only: true28  padding: longest29  padding_side: right30num_workers: 431prefetch_factor: 232do_preprocess: false33model_name_or_path: /tmp/Llama-2-13b-chat-hf-code-github-c4/34pretrain: null35exp_name: llama2.13b.Code.Github.C4.combine.v1.0.seq2k.w16.adamw.NA100.0822.aws.ds36exp_notes: null37output_dir: /tmp/${exp_name}38resume: null39do_train: true40evaluate_during_training: false41do_eval: false42eval_sub_path: checkpoint-*43per_gpu_train_batch_size: 244per_gpu_eval_batch_size: 145learning_rate: 1.0e-0546gradient_accumulation_steps: 6447weight_decay: 0.0148adam_epsilon: 1.0e-0649adam_betas: (0.9, 0.99)50max_grad_norm: 1.051num_train_epochs: 152total_dataset_len: 1000000053max_steps: 054warmup_proportion: 055warmup_steps: 056optimizer: null57use_nvlamb: null58bit_training: null59logging_steps: 160save_best: false61save_steps: 25062eval_steps: 25063ddp_eval: true64no_cuda: false65seed: 4266local_rank: 067fp16: true68fp16_opt_level: O169fp16_bfloat16: true70prediction_cfg:71  metric: acc72  measure: 173  best_checkpoint: null74  best_result: null75eval_forward_fn:76  _target_: general_util.evaluator.DiscriminatorForwardFn77post_process: null78fairscale_config:79  _target_: general_util.fsdp_utils.default_initialize80  fp16: ${fp16}81  move_grads_to_cpu: false82  move_params_to_cpu: false83  flatten_parameters: false84with_lightseq: false85load_lr_scheduler_states: false86ds_cfg:87  train_micro_batch_size_per_gpu: ${per_gpu_train_batch_size}88  gradient_accumulation_steps: ${gradient_accumulation_steps}89  optimizer:90    type: AdamW91    params:92      lr: ${learning_rate}93      betas:94      - 0.995      - 0.9696      eps: ${adam_epsilon}97      weight_decay: ${weight_decay}98  scheduler:99    type: WarmupDecayLR100    params:101      total_num_steps: 4882102      warmup_max_lr: ${learning_rate}103      warmup_num_steps: 0104      warmup_type: linear105  gradient_clipping: ${max_grad_norm}106  bf16:107    enabled: ${fp16}108  zero_optimization:109    stage: 1110    contiguous_gradients: true111    overlap_comm: true112    reduce_scatter: true113    reduce_bucket_size: 500000000.0114    allgather_bucket_size: 500000000.0115    offload_optimizer:116      device: cpu117      pin_memory: true118  steps_per_print: 1119summary_helper:120  _target_: general_util.tensorboard_helper.WandbWriter121  batch_index_or_keys: null122  outputs_index_or_keys: null123n_gpu: 1124device: cuda:0125train_batch_size: 2126eval_batch_size: null127world_size: 16128world_rank: null129