Modelpublic

dvdmrs09/gemma-7b-it-python2

sourceHugging Faceupdated 3y agoView on Hugging Face

0likes12downloads

Model Card

Model Card for Model ID

Fine-tuned on python

Model Details

Model Description

Model Sources [optional]

Gemma-2b trained on python-oasst dataset

Uses

Training Details

Training Data

{ "timestamp": 1711018613.433522, "train/gradnorm": 0.1240904619429259, "train/globalstep": 232, "eval/stepspersecond": 2.894, "step": 232, "runtime": 2545.4226660728455, "eval/loss": 1.189491629600525, "eval/runtime": 1805.8574, "train/learningrate": 0.000014800637958532697, "eval/samplespersecond": 23.152, "_wandb.runtime": 2547, "train/loss": 1.0436, "train/epoch": 0.01 }

Results

Summary

Technical Specifications [optional]

{ "bf16": { "desc": null, "value": true }, "fp16": { "desc": null, "value": false }, "fsdp": { "desc": null, "value": [] }, "seed": { "desc": null, "value": 42 }, "tf32": { "desc": null, "value": false }, "debug": { "desc": null, "value": [] }, "optim": { "desc": null, "value": "adamwbnb8bit" }, "qlora": { "desc": null, "value": true }, "topk": { "desc": null, "value": 50 }, "topp": { "desc": null, "value": 1 }, "wandb": { "desc": null, "value": { "m": [ { "1": "train/globalstep", "6": [ 3 ] }, { "1": "train/loss", "5": 1, "6": [ 1 ] }, { "1": "train/gradnorm", "5": 1, "6": [ 1 ] }, { "1": "train/learningrate", "5": 1, "6": [ 1 ] }, { "1": "train/epoch", "5": 1, "6": [ 1 ] }, { "1": "eval/loss", "5": 1, "6": [ 1 ] }, { "1": "eval/runtime", "5": 1, "6": [ 1 ] }, { "1": "eval/samplespersecond", "5": 1, "6": [ 1 ] }, { "1": "eval/stepspersecond", "5": 1, "6": [ 1 ] } ], "t": { "1": [ 1, 5, 11, 49, 51, 53, 55, 71, 84, 98, 99, 100, 105 ], "2": [ 1, 5, 11, 49, 51, 53, 55, 71, 84, 98, 99, 100, 105 ], "3": [ 3, 7, 23 ], "4": "3.10.13", "5": "0.16.4", "6": "4.39.0.dev0", "8": [ 5 ], "9": { "1": "transformerstrainer" }, "13": "linux-x8664" }, "framework": "huggingface", "starttime": 1711016068, "cliversion": "0.16.4", "isjupyterrun": false, "pythonversion": "3.10.13", "iskagglekernel": false, "huggingfaceversion": "4.39.0.dev0" } }, "prefix": { "desc": null, "value": null }, "doeval": { "desc": null, "value": true }, "nocuda": { "desc": null, "value": false }, "usecpu": { "desc": null, "value": false }, "dotrain": { "desc": null, "value": false }, "headdim": { "desc": null, "value": 256 }, "id2label": { "desc": null, "value": { "0": "LABEL0", "1": "LABEL1" } }, "label2id": { "desc": null, "value": { "LABEL0": 0, "LABEL1": 1 } }, "runname": { "desc": null, "value": "./out" }, "useipex": { "desc": null, "value": false }, "adafactor": { "desc": null, "value": false }, "dataseed": { "desc": null, "value": null }, "deepspeed": { "desc": null, "value": "deepspeedconfigs/zero1.json" }, "dosample": { "desc": null, "value": false }, "hubtoken": { "desc": null, }, "loglevel": { "desc": null, "value": "passive" }, "maxsteps": { "desc": null, "value": -1 }, "numbeams": { "desc": null, "value": 1 }, "rayscope": { "desc": null, "value": "last" }, "reportto": { "desc": null, "value": [ "wandb" ] }, "typicalp": { "desc": null, "value": 1 }, "usecache": { "desc": null, "value": false }, "adambeta1": { "desc": null, "value": 0.9 }, "adambeta2": { "desc": null, "value": 0.999 }, "dopredict": { "desc": null, "value": false }, "evaldelay": { "desc": null, "value": 0 }, "evalsteps": { "desc": null, "value": 0.03125 }, "hiddenact": { "desc": null, "value": "gelu" }, "isdecoder": { "desc": null, "value": false }, "localrank": { "desc": null, "value": 0 }, "maxlength": { "desc": null, "value": 20 }, "minlength": { "desc": null, "value": 0 }, "modeltype": { "desc": null, "value": "gemma" }, "optimargs": { "desc": null, "value": null }, "orpoalpha": { "desc": null, "value": null }, "outputdir": { "desc": null, "value": "./out" }, "pastindex": { "desc": null, "value": -1 }, "ropetheta": { "desc": null, "value": 10000 }, "savesteps": { "desc": null, "value": 0.125 }, "vocabsize": { "desc": null, "value": 256000 }, "benchsplit": { "desc": null, "value": "eval" }, "ddpbackend": { "desc": null, "value": null }, "ddptimeout": { "desc": null, "value": 1800 }, "fsdpconfig": { "desc": null, "value": { "xla": false, "xlafsdpv2": false, "minnumparams": 0, "xlafsdpgradckpt": false } }, "hiddensize": { "desc": null, "value": 2048 }, "labelnames": { "desc": null, "value": null }, "loggingdir": { "desc": null, "value": "./out/runs/Mar2110-14-248205afe3ecd2" }, "pretraining": { "desc": null, "value": false }, "pushtohub": { "desc": null, "value": false }, "returndict": { "desc": null, "value": true }, "temperature": { "desc": null, "value": 1 }, "torchdtype": { "desc": null, "value": "bfloat16" }, "torchdynamo": { "desc": null, "value": null }, "torchscript": { "desc": null, "value": false }, "adamepsilon": { "desc": null, "value": 1e-8 }, "bostokenid": { "desc": null, "value": 2 }, "disabletqdm": { "desc": null, "value": false }, "eostokenid": { "desc": null, "value": 1 }, "fp16backend": { "desc": null, "value": "auto" }, "hubmodelid": { "desc": null, "value": null }, "hubstrategy": { "desc": null, "value": "everysave" }, "padtokenid": { "desc": null, "value": 0 }, "problemtype": { "desc": null, "value": null }, "prunedheads": { "desc": null, "value": {} }, "relorasteps": { "desc": null, "value": null }, "rmsnormeps": { "desc": null, "value": 0.000001 }, "ropescaling": { "desc": null, "value": null }, "septokenid": { "desc": null, "value": null }, "usebfloat16": { "desc": null, "value": false }, "warmupratio": { "desc": null, "value": 0 }, "warmupsteps": { "desc": null, "value": 3135 }, "weightdecay": { "desc": null, "value": 0 }, "nameorpath": { "desc": null, "value": "dvdmrs09/gemma2b-train" }, "architectures": { "desc": null, "value": [ "GemmaForCausalLM" ] }, "badwordsids": { "desc": null, "value": null }, "benchdataset": { "desc": null, "value": "pharaouk/dharma-1/dharma1mini.json" }, "dobencheval": { "desc": null, "value": false }, "jitmodeeval": { "desc": null, "value": false }, "learningrate": { "desc": null, "value": 0.0002 }, "loggingsteps": { "desc": null, "value": 1 }, "maxgradnorm": { "desc": null, "value": 1 }, "mpparameters": { "desc": null, "value": "" }, "outputscores": { "desc": null, "value": false }, "savestrategy": { "desc": null, "value": "steps" }, "splitbatches": { "desc": null, "value": null }, "torchcompile": { "desc": null, "value": false }, "tpunumcores": { "desc": null, "value": null }, "attentionbias": { "desc": null, "value": false }, "bf16fulleval": { "desc": null, "value": false }, "earlystopping": { "desc": null, "value": false }, "fp16fulleval": { "desc": null, "value": false }, "fp16optlevel": { "desc": null, "value": "O1" }, "lengthpenalty": { "desc": null, "value": 1 }, "maxseqlength": { "desc": null, "value": 4096 }, "samplepacking": { "desc": null, "value": false }, "tflegacyloss": { "desc": null, "value": false }, "usempsdevice": { "desc": null, "value": false }, "finetuningtask": { "desc": null, "value": null }, "groupbylength": { "desc": null, "value": false }, "hubalwayspush": { "desc": null, "value": false }, "numbeamgroups": { "desc": null, "value": 1 }, "saveonlymodel": { "desc": null, "value": false }, "suppresstokens": { "desc": null, "value": null }, "tokenizerclass": { "desc": null, "value": null }, "dispatchbatches": { "desc": null, "value": null }, "fulldeterminism": { "desc": null, "value": false }, "hubprivaterepo": { "desc": null, "value": false }, "ignoredataskip": { "desc": null, "value": false }, "logoneachnode": { "desc": null, "value": true }, "loggingstrategy": { "desc": null, "value": "steps" }, "numtrainepochs": { "desc": null, "value": 8 }, "savesafetensors": { "desc": null, "value": true }, "savetotallimit": { "desc": null, "value": 4 }, "attentiondropout": { "desc": null, "value": 0 }, "ddpbucketcapmb": { "desc": null, "value": null }, "diversitypenalty": { "desc": null, "value": 0 }, "docausallmeval": { "desc": null, "value": false }, "greaterisbetter": { "desc": null, "value": false }, "initializerrange": { "desc": null, "value": 0.02 }, "intermediatesize": { "desc": null, "value": 16384 }, "loglevelreplica": { "desc": null, "value": "warning" }, "lorapluslrratio": { "desc": null, "value": null }, "lrschedulertype": { "desc": null, "value": "cosine" }, "maxbenchsamples": { "desc": null, "value": null }, "numhiddenlayers": { "desc": null, "value": 18 }, "outputattentions": { "desc": null, "value": false }, "pushtohubtoken": { "desc": null, "value": "<PUSHTOHUBTOKEN>" }, "saveoneachnode": { "desc": null, "value": false }, "tpumetricsdebug": { "desc": null, "value": false }, "acceleratorconfig": { "desc": null, "value": { "evenbatches": true, "splitbatches": false, "dispatchbatches": null, "useseedablesampler": true } }, "isencoderdecoder": { "desc": null, "value": false }, "lengthcolumnname": { "desc": null, "value": "length" }, "loggingfirststep": { "desc": null, "value": false }, "relorapruneratio": { "desc": null, "value": 0.9 }, "repetitionpenalty": { "desc": null, "value": 1 }, "torchcompilemode": { "desc": null, "value": null }, "addcrossattention": { "desc": null, "value": false }, "cosineminlrratio": { "desc": null, "value": null }, "evalsamplepacking": { "desc": null, "value": false }, "evaluationstrategy": { "desc": null, "value": "steps" }, "forcedbostokenid": { "desc": null, "value": null }, "forcedeostokenid": { "desc": null, "value": null }, "fsdpminnumparams": { "desc": null, "value": 0 }, "lrquadraticwarmup": { "desc": null, "value": false }, "lrschedulerkwargs": { "desc": null, "value": {} }, "neftunenoisealpha": { "desc": null, "value": null }, "numattentionheads": { "desc": null, "value": 8 }, "numkeyvalueheads": { "desc": null, "value": 1 }, "quantizationconfig": { "desc": null, "value": { "loadin4bit": true, "loadin8bit": false, "quantmethod": "QuantizationMethod.BITSANDBYTES", "loadin4bit": true, "loadin8bit": false, "llmint8threshold": 6, "bnb4bitquanttype": "nf4", "llmint8skipmodules": null, "bnb4bitcomputedtype": "bfloat16", "bnb4bitquantstorage": "uint8", "llmint8hasfp16weight": false, "bnb4bitusedoublequant": true, "llmint8enablefp32cpuoffload": false } }, "reloraannealsteps": { "desc": null, "value": null }, "relorawarmupsteps": { "desc": null, "value": null }, "skipmemorymetrics": { "desc": null, "value": true }, "tieencoderdecoder": { "desc": null, "value": false }, "tiewordembeddings": { "desc": null, "value": true }, "autofindbatchsize": { "desc": null, "value": false }, "benchsourcemaxlen": { "desc": null, "value": 2048 }, "dataloaderdroplast": { "desc": null, "value": false }, "norepeatngramsize": { "desc": null, "value": 0 }, "numreturnsequences": { "desc": null, "value": 1 }, "optimtargetmodules": { "desc": null, "value": null }, "outputhiddenstates": { "desc": null, "value": false }, "overwriteoutputdir": { "desc": null, "value": false }, "predictionlossonly": { "desc": null, "value": false }, "pushtohubmodelid": { "desc": null, "value": null }, "taskspecificparams": { "desc": null, "value": null }, "transformersversion": { "desc": null, "value": "4.39.0.dev0" }, "beginsuppresstokens": { "desc": null, "value": null }, "dataloaderpinmemory": { "desc": null, "value": true }, "ddpbroadcastbuffers": { "desc": null, "value": null }, "lorapluslrembedding": { "desc": null, "value": null }, "metricforbestmodel": { "desc": null, "value": "loss" }, "removeinvalidvalues": { "desc": null, "value": false }, "removeunusedcolumns": { "desc": null, "value": true }, "torchcompilebackend": { "desc": null, "value": null }, "dataloadernumworkers": { "desc": null, "value": 0 }, "decoderstarttokenid": { "desc": null, "value": null }, "gradientcheckpointing": { "desc": null, "value": true }, "halfprecisionbackend": { "desc": null, "value": "auto" }, "labelsmoothingfactor": { "desc": null, "value": 0 }, "loadbestmodelatend": { "desc": null, "value": true }, "loggingnaninffilter": { "desc": null, "value": true }, "multipackrealbatches": { "desc": null, "value": false }, "resumefromcheckpoint": { "desc": null, "value": null }, "chunksizefeedforward": { "desc": null, "value": 0 }, "evalaccumulationsteps": { "desc": null, "value": 3 }, "maxpositionembeddings": { "desc": null, "value": 8192 }, "pergpuevalbatchsize": { "desc": null, "value": null }, "returndictingenerate": { "desc": null, "value": false }, "cosineconstantlrratio": { "desc": null, "value": null }, "pergputrainbatchsize": { "desc": null, "value": null }, "pushtohuborganization": { "desc": null, "value": null }, "includetokenspersecond": { "desc": null, "value": false }, "samplepackingefficiency": { "desc": null, "value": 1 }, "dataloaderprefetchfactor": { "desc": null, "value": null }, "ddpfindunusedparameters": { "desc": null, "value": false }, "includeinputsformetrics": { "desc": null, "value": false }, "perdeviceevalbatchsize": { "desc": null, "value": 2 }, "uselegacypredictionloop": { "desc": null, "value": false }, "crossattentionhiddensize": { "desc": null, "value": null }, "gradientaccumulationsteps": { "desc": null, "value": 3 }, "perdevicetrainbatchsize": { "desc": null, "value": 2 }, "encodernorepeatngramsize": { "desc": null, "value": 0 }, "dataloaderpersistentworkers": { "desc": null, "value": false }, "gradientcheckpointingkwargs": { "desc": null, "value": { "usereentrant": true } }, "includenuminputtokensseen": { "desc": null, "value": false }, "exponentialdecaylengthpenalty": { "desc": null, "value": null }, "samplepackingseqlenmultiplier": { "desc": null, "value": 2 }, "fsdptransformerlayerclstowrap": { "desc": null, "value": null } }