KasparZ/llama-2-7b-hitl

Base Model

model_name = "mistralai/Mistral-7B-v0.1"

The instruction dataset to use

dataset_name = "KasparZ/HITL-2"

################################################################################

QLoRA parameters

################################################################################

LoRA attention dimension

lora_r = 64

Alpha parameter for LoRA scaling

lora_alpha = 16

Dropout probability for LoRA layers

lora_dropout = 0.05 #0.1

################################################################################

bitsandbytes parameters

################################################################################

Activate 4-bit precision base model loading

use_4bit = True

Compute dtype for 4-bit base models

bnb4bitcompute_dtype = "float16"

Quantization type (fp4 or nf4)

bnb4bitquant_type = "nf4"

Activate nested quantization for 4-bit base models (double quantization)

usenestedquant = False

################################################################################

TrainingArguments parameters

################################################################################

Output directory where the model predictions and checkpoints will be stored

output_dir = "./results"

Number of training epochs

#numtrainepochs = 1 numtrainepochs = 4

Enable fp16/bf16 training (set bf16 to True with an A100)

fp16 = False bf16 = False

Batch size per GPU for training

perdevicetrainbatchsize = 4

Batch size per GPU for evaluation

perdeviceevalbatchsize = 4

Number of update steps to accumulate the gradients for

gradientaccumulationsteps = 4 #1

Enable gradient checkpointing

gradient_checkpointing = True

Maximum gradient normal (gradient clipping)

maxgradnorm = 0.3

Initial learning rate (AdamW optimizer)

learning_rate = 2e-4

Weight decay to apply to all layers except bias/LayerNorm weights

weight_decay = 0.001

Optimizer to use

optim = "pagedadamw32bit"

Learning rate schedule

lrschedulertype = "cosine"

Number of training steps (overrides numtrainepochs)

max_steps = -1

Ratio of steps for a linear warmup (from 0 to learning rate)

warmup_ratio = 0.03

Group sequences into batches with same length

Saves memory and speeds up training considerably

groupbylength = True

Save checkpoint every X updates steps

save_steps = 0

Log every X updates steps

logging_steps = 25

################################################################################

SFT parameters

################################################################################

Maximum sequence length to use

maxseqlength = None

Pack multiple short examples in the same input sequence to increase efficiency

packing = False

Load the entire model on the GPU 0

device_map = {"": 0}