aloksingh2130/Bloom-3B-Squad-v2

Training procedure

Framework versions

• —PEFT 0.6.0.dev0 Using LoRA to fine tune Bloom-3B model on Squad v2 dataset

How to use: Code below

!pip install -q bitsandbytes datasets accelerate loralib !pip install -q git+https://github.com/huggingface/peft.git git+https://github.com/huggingface/transformers.git

import torch torch.cuda.is_available()

import os os.environ["CUDAVISIBLEDEVICES"]="0" import torch import torch.nn as nn import bitsandbytes as bnb from transformers import AutoTokenizer, AutoConfig, AutoModelForCausalLM

model = AutoModelForCausalLM.frompretrained( "bigscience/bloom-3b", torchdtype=torch.float16, device_map='auto', )

tokenizer = AutoTokenizer.from_pretrained("bigscience/tokenizer")

print(model)

for param in model.parameters(): param.requires_grad = False # freeze the model - train adapters later if param.ndim == 1: # cast the small parameters (e.g. layernorm) to fp32 for stability param.data = param.data.to(torch.float32)

model.gradientcheckpointingenable() # reduce number of stored activations model.enableinputrequire_grads()

class CastOutputToFloat(nn.Sequential): def forward(self, x): return super().forward(x).to(torch.float32) model.lmhead = CastOutputToFloat(model.lmhead)

Helper Function

def printtrainableparameters(model): """ Prints the number of trainable parameters in the model. """ trainableparams = 0 allparam = 0 for , param in model.namedparameters(): allparam += param.numel() if param.requiresgrad: trainableparams += param.numel() print( f"trainable params: {trainableparams} || all params: {allparam} || trainable%: {100 * trainableparams / all_param}" )

from peft import LoraConfig, getpeftmodel

config = LoraConfig( r=8, loraalpha=16, targetmodules=["querykeyvalue"], loradropout=0.05, bias="none", tasktype="CAUSAL_LM" )

# r, the dimension of the low-rank matrices # loraalpha, scaling factor for the weight matrices # loradropout, dropout probability of the LoRA layers # bias, set to all to train all bias parameters

model = getpeftmodel(model, config) printtrainableparameters(model)

from datasets import loaddataset qadataset = loaddataset("squadv2")

def createprompt(context, question, answer): if len(answer["text"]) < 1: answer = "Cannot Find Answer" else: answer = answer["text"][0] prompttemplate = f"### CONTEXT\n{context}\n\n### QUESTION\n{question}\n\n### ANSWER\n{answer}</s>" return prompt_template

mappedqadataset = qadataset.map(lambda samples: tokenizer(createprompt(samples['context'], samples['question'], samples['answers'])))

torch.cuda.empty_cache()

Train MOdel on SQUAD DAtaset

import transformers

trainer = transformers.Trainer( # doc for parameter https://huggingface.co/docs/transformers/mainclasses/trainer model=model, traindataset=mappedqadataset["train"], evaldataset=mappedqadataset["validation"], args=transformers.TrainingArguments( perdevicetrainbatchsize=4, gradientaccumulationsteps=4, # evaluationstrategy="steps", # evalsteps=1000, warmupsteps=100, maxsteps=100, learningrate=2e-3, fp16=True, loggingsteps=1, outputdir='outputs', ), data_collator=transformers.DataCollatorForLanguageModeling(tokenizer, mlm=False) )

#Training Arguments

args = TrainingArguments(

output_dir = "/Content/mod",

evaluation_strategy = "epoch", #Can be epoch or steps

learning_rate=2e-5, #According to original bert paper

perdevicetrainbatchsize=32, #According to original bert paper

perdeviceevalbatchsize=32,

numtrainepochs=3, #should be inbetween 2-4 according to the paper

weight_decay=0.01,

predictionlossonly = True

)

For example if you use evaluationstrategy="steps" and evalsteps=2000 in the TrainingArguments,

you will get training and validation loss for every 2000 steps. If you wanna do it on an epoch level

\I think you need to set evaluationstrategy="epoch" and loggingstrategy="epoch" in the

TrainingArguments class.

model.config.use_cache = False # silence the warnings. Please re-enable for inference! trainer.train()

Push to HUB

HUGGINGFACEUSERNAME = "aloksingh2130" modelname = "Bloom-3B-Squad-v2" model.pushtohub(f"{HUGGINGFACEUSERNAME}/{modelname}", useauthtoken=True)