Jeesup/svd-safety-mis7_swift_remove50
0423
1{2 "source": "/workspace/svd_safety/assets/models/Mistral-7B-Instruct-v0.2",3 "upstream": "hiahei/Swift-SVD (compression) + AIoT-MLSys-Lab/SVD-LLM utils/LoRA.py (recovery)",4 "stage": "finetuned",5 "ratio_removed": 0.5,6 "internal_ratio_kept": 0.5,7 "from_pt": "/workspace/svd_safety/results/swift_ft/cells/mis7_swift_remove50/finetune_v/merge.pt",8 "mean_rel_err": 0.5888538884464651,9 "max_rel_err": 0.7068923115730286,10 "param_fraction": 0.5002723106971154,11 "ranks": {12 "0.q_proj": 1152,13 "0.k_proj": 485,14 "0.v_proj": 442,15 "0.o_proj": 1052,16 "0.gate_proj": 1680,17 "0.up_proj": 1680,18 "0.down_proj": 1702,19 "1.q_proj": 1157,20 "1.k_proj": 493,21 "1.v_proj": 465,22 "1.o_proj": 1084,23 "1.gate_proj": 1680,24 "1.up_proj": 1680,25 "1.down_proj": 1976,26 "2.q_proj": 942,27 "2.k_proj": 374,28 "2.v_proj": 367,29 "2.o_proj": 907,30 "2.gate_proj": 1680,31 "2.up_proj": 1680,32 "2.down_proj": 1351,33 "3.q_proj": 940,34 "3.k_proj": 372,35 "3.v_proj": 363,36 "3.o_proj": 898,37 "3.gate_proj": 1680,38 "3.up_proj": 1680,39 "3.down_proj": 1353,40 "4.q_proj": 938,41 "4.k_proj": 371,42 "4.v_proj": 366,43 "4.o_proj": 903,44 "4.gate_proj": 1680,45 "4.up_proj": 1680,46 "4.down_proj": 1371,47 "5.q_proj": 938,48 "5.k_proj": 372,49 "5.v_proj": 366,50 "5.o_proj": 910,51 "5.gate_proj": 1680,52 "5.up_proj": 1680,53 "5.down_proj": 1384,54 "6.q_proj": 939,55 "6.k_proj": 373,56 "6.v_proj": 368,57 "6.o_proj": 917,58 "6.gate_proj": 1680,59 "6.up_proj": 1680,60 "6.down_proj": 1395,61 "7.q_proj": 938,62 "7.k_proj": 373,63 "7.v_proj": 368,64 "7.o_proj": 925,65 "7.gate_proj": 1680,66 "7.up_proj": 1680,67 "7.down_proj": 1406,68 "8.q_proj": 935,69 "8.k_proj": 371,70 "8.v_proj": 368,71 "8.o_proj": 924,72 "8.gate_proj": 1680,73 "8.up_proj": 1680,74 "8.down_proj": 1403,75 "9.q_proj": 934,76 "9.k_proj": 370,77 "9.v_proj": 369,78 "9.o_proj": 924,79 "9.gate_proj": 1680,80 "9.up_proj": 1680,81 "9.down_proj": 1409,82 "10.q_proj": 935,83 "10.k_proj": 371,84 "10.v_proj": 369,85 "10.o_proj": 932,86 "10.gate_proj": 1680,87 "10.up_proj": 1680,88 "10.down_proj": 1410,89 "11.q_proj": 933,90 "11.k_proj": 370,91 "11.v_proj": 370,92 "11.o_proj": 934,93 "11.gate_proj": 1680,94 "11.up_proj": 1680,95 "11.down_proj": 1412,96 "12.q_proj": 939,97 "12.k_proj": 374,98 "12.v_proj": 373,99 "12.o_proj": 941,100 "12.gate_proj": 1680,101 "12.up_proj": 1680,102 "12.down_proj": 1428,103 "13.q_proj": 936,104 "13.k_proj": 372,105 "13.v_proj": 372,106 "13.o_proj": 939,107 "13.gate_proj": 1680,108 "13.up_proj": 1680,109 "13.down_proj": 1428,110 "14.q_proj": 944,111 "14.k_proj": 376,112 "14.v_proj": 378,113 "14.o_proj": 949,114 "14.gate_proj": 1680,115 "14.up_proj": 1680,116 "14.down_proj": 1451,117 "15.q_proj": 955,118 "15.k_proj": 383,119 "15.v_proj": 385,120 "15.o_proj": 964,121 "15.gate_proj": 1680,122 "15.up_proj": 1680,123 "15.down_proj": 1481,124 "16.q_proj": 959,125 "16.k_proj": 385,126 "16.v_proj": 387,127 "16.o_proj": 968,128 "16.gate_proj": 1680,129 "16.up_proj": 1680,130 "16.down_proj": 1500,131 "17.q_proj": 978,132 "17.k_proj": 395,133 "17.v_proj": 397,134 "17.o_proj": 990,135 "17.gate_proj": 1680,136 "17.up_proj": 1680,137 "17.down_proj": 1543,138 "18.q_proj": 978,139 "18.k_proj": 394,140 "18.v_proj": 398,141 "18.o_proj": 993,142 "18.gate_proj": 1680,143 "18.up_proj": 1680,144 "18.down_proj": 1555,145 "19.q_proj": 965,146 "19.k_proj": 388,147 "19.v_proj": 391,148 "19.o_proj": 983,149 "19.gate_proj": 1680,150 "19.up_proj": 1680,151 "19.down_proj": 1537,152 "20.q_proj": 942,153 "20.k_proj": 375,154 "20.v_proj": 379,155 "20.o_proj": 955,156 "20.gate_proj": 1680,157 "20.up_proj": 1680,158 "20.down_proj": 1488,159 "21.q_proj": 936,160 "21.k_proj": 371,161 "21.v_proj": 376,162 "21.o_proj": 947,163 "21.gate_proj": 1680,164 "21.up_proj": 1680,165 "21.down_proj": 1477,166 "22.q_proj": 926,167 "22.k_proj": 365,168 "22.v_proj": 371,169 "22.o_proj": 934,170 "22.gate_proj": 1680,171 "22.up_proj": 1680,172 "22.down_proj": 1457,173 "23.q_proj": 927,174 "23.k_proj": 365,175 "23.v_proj": 371,176 "23.o_proj": 940,177 "23.gate_proj": 1680,178 "23.up_proj": 1680,179 "23.down_proj": 1460,180 "24.q_proj": 926,181 "24.k_proj": 365,182 "24.v_proj": 371,183 "24.o_proj": 944,184 "24.gate_proj": 1680,185 "24.up_proj": 1680,186 "24.down_proj": 1461,187 "25.q_proj": 929,188 "25.k_proj": 366,189 "25.v_proj": 374,190 "25.o_proj": 951,191 "25.gate_proj": 1680,192 "25.up_proj": 1680,193 "25.down_proj": 1471,194 "26.q_proj": 933,195 "26.k_proj": 368,196 "26.v_proj": 377,197 "26.o_proj": 957,198 "26.gate_proj": 1680,199 "26.up_proj": 1680,200 "26.down_proj": 1483,201 "27.q_proj": 936,202 "27.k_proj": 370,203 "27.v_proj": 379,204 "27.o_proj": 958,205 "27.gate_proj": 1680,206 "27.up_proj": 1680,207 "27.down_proj": 1493,208 "28.q_proj": 949,209 "28.k_proj": 378,210 "28.v_proj": 387,211 "28.o_proj": 976,212 "28.gate_proj": 1680,213 "28.up_proj": 1680,214 "28.down_proj": 1532,215 "29.q_proj": 957,216 "29.k_proj": 381,217 "29.v_proj": 392,218 "29.o_proj": 994,219 "29.gate_proj": 1680,220 "29.up_proj": 1680,221 "29.down_proj": 1550,222 "30.q_proj": 971,223 "30.k_proj": 389,224 "30.v_proj": 401,225 "30.o_proj": 1013,226 "30.gate_proj": 1680,227 "30.up_proj": 1680,228 "30.down_proj": 1593,229 "31.q_proj": 1053,230 "31.k_proj": 433,231 "31.v_proj": 448,232 "31.o_proj": 1114,233 "31.gate_proj": 1680,234 "31.up_proj": 1680,235 "31.down_proj": 1816236 },237 "rel_err": {238 "0.q_proj": 0.10284276306629181,239 "0.k_proj": 0.09011625498533249,240 "0.v_proj": 0.4146844744682312,241 "0.o_proj": 0.4441743791103363,242 "0.gate_proj": 0.4323641359806061,243 "0.up_proj": 0.4812891185283661,244 "0.down_proj": 0.6020375490188599,245 "1.q_proj": 0.1907663345336914,246 "1.k_proj": 0.13933585584163666,247 "1.v_proj": 0.5992577075958252,248 "1.o_proj": 0.5850644111633301,249 "1.gate_proj": 0.5018978714942932,250 "1.up_proj": 0.5401790142059326,251 "1.down_proj": 0.5769923329353333,252 "2.q_proj": 0.5321782827377319,253 "2.k_proj": 0.5425702929496765,254 "2.v_proj": 0.682027280330658,255 "2.o_proj": 0.6367513537406921,256 "2.gate_proj": 0.5818910598754883,257 "2.up_proj": 0.6077799201011658,258 "2.down_proj": 0.681037187576294,259 "3.q_proj": 0.48435133695602417,260 "3.k_proj": 0.4475176930427551,261 "3.v_proj": 0.7068923115730286,262 "3.o_proj": 0.677703857421875,263 "3.gate_proj": 0.5896062254905701,264 "3.up_proj": 0.6225286722183228,265 "3.down_proj": 0.6739526987075806,266 "4.q_proj": 0.4879390597343445,267 "4.k_proj": 0.4703967571258545,268 "4.v_proj": 0.6710249185562134,269 "4.o_proj": 0.6244789361953735,270 "4.gate_proj": 0.5910874605178833,271 "4.up_proj": 0.6321871876716614,272 "4.down_proj": 0.6802555918693542,273 "5.q_proj": 0.5210317969322205,274 "5.k_proj": 0.5119411945343018,275 "5.v_proj": 0.6989429593086243,276 "5.o_proj": 0.6632655262947083,277 "5.gate_proj": 0.5846965312957764,278 "5.up_proj": 0.6342030763626099,279 "5.down_proj": 0.6687344908714294,280 "6.q_proj": 0.505556583404541,281 "6.k_proj": 0.48692795634269714,282 "6.v_proj": 0.6872745752334595,283 "6.o_proj": 0.646428108215332,284 "6.gate_proj": 0.5995981693267822,285 "6.up_proj": 0.6370501518249512,286 "6.down_proj": 0.6577861309051514,287 "7.q_proj": 0.4832315444946289,288 "7.k_proj": 0.4549109935760498,289 "7.v_proj": 0.6812716722488403,290 "7.o_proj": 0.6252717971801758,291 "7.gate_proj": 0.5871505737304688,292 "7.up_proj": 0.6294843554496765,293 "7.down_proj": 0.654327392578125,294 "8.q_proj": 0.49038824439048767,295 "8.k_proj": 0.4526974856853485,296 "8.v_proj": 0.6452953219413757,297 "8.o_proj": 0.6178920269012451,298 "8.gate_proj": 0.5910043716430664,299 "8.up_proj": 0.6263056993484497,300 "8.down_proj": 0.647315502166748,301 "9.q_proj": 0.48501771688461304,302 "9.k_proj": 0.4705408811569214,303 "9.v_proj": 0.6292407512664795,304 "9.o_proj": 0.6174561977386475,305 "9.gate_proj": 0.5865891575813293,306 "9.up_proj": 0.6233162879943848,307 "9.down_proj": 0.6318148970603943,308 "10.q_proj": 0.4836317300796509,309 "10.k_proj": 0.4697504937648773,310 "10.v_proj": 0.6263429522514343,311 "10.o_proj": 0.6223848462104797,312 "10.gate_proj": 0.5854211449623108,313 "10.up_proj": 0.6197803020477295,314 "10.down_proj": 0.6372787356376648,315 "11.q_proj": 0.4799323081970215,316 "11.k_proj": 0.4386899471282959,317 "11.v_proj": 0.6085149049758911,318 "11.o_proj": 0.6194866895675659,319 "11.gate_proj": 0.585040271282196,320 "11.up_proj": 0.6186556816101074,321 "11.down_proj": 0.6358811259269714,322 "12.q_proj": 0.46628451347351074,323 "12.k_proj": 0.4467337131500244,324 "12.v_proj": 0.6398075222969055,325 "12.o_proj": 0.602120578289032,326 "12.gate_proj": 0.5809478759765625,327 "12.up_proj": 0.6136550307273865,328 "12.down_proj": 0.6333811283111572,329 "13.q_proj": 0.49495774507522583,330 "13.k_proj": 0.4739055931568146,331 "13.v_proj": 0.6350283026695251,332 "13.o_proj": 0.6104384660720825,333 "13.gate_proj": 0.5884830951690674,334 "13.up_proj": 0.6139264702796936,335 "13.down_proj": 0.6283634305000305,336 "14.q_proj": 0.4639151692390442,337 "14.k_proj": 0.44637247920036316,338 "14.v_proj": 0.6475556492805481,339 "14.o_proj": 0.6104862093925476,340 "14.gate_proj": 0.5859494209289551,341 "14.up_proj": 0.6189398169517517,342 "14.down_proj": 0.6564309597015381,343 "15.q_proj": 0.46387919783592224,344 "15.k_proj": 0.42992931604385376,345 "15.v_proj": 0.675900399684906,346 "15.o_proj": 0.6175917387008667,347 "15.gate_proj": 0.5837653279304504,348 "15.up_proj": 0.6173660159111023,349 "15.down_proj": 0.6548573970794678,350 "16.q_proj": 0.4708837568759918,351 "16.k_proj": 0.4432294964790344,352 "16.v_proj": 0.6696433424949646,353 "16.o_proj": 0.6041324138641357,354 "16.gate_proj": 0.5891684293746948,355 "16.up_proj": 0.626923680305481,356 "16.down_proj": 0.656101644039154,357 "17.q_proj": 0.5025116801261902,358 "17.k_proj": 0.48880496621131897,359 "17.v_proj": 0.6514131426811218,360 "17.o_proj": 0.6207662224769592,361 "17.gate_proj": 0.6006857752799988,362 "17.up_proj": 0.6366637349128723,363 "17.down_proj": 0.6493645906448364,364 "18.q_proj": 0.4747740626335144,365 "18.k_proj": 0.4571532607078552,366 "18.v_proj": 0.6963456869125366,367 "18.o_proj": 0.5963596701622009,368 "18.gate_proj": 0.6004551649093628,369 "18.up_proj": 0.6350386142730713,370 "18.down_proj": 0.6672616004943848,371 "19.q_proj": 0.5268400311470032,372 "19.k_proj": 0.47958895564079285,373 "19.v_proj": 0.7042666077613831,374 "19.o_proj": 0.6199771165847778,375 "19.gate_proj": 0.6060331463813782,376 "19.up_proj": 0.6367842555046082,377 "19.down_proj": 0.6811782121658325,378 "20.q_proj": 0.5465222001075745,379 "20.k_proj": 0.5119665265083313,380 "20.v_proj": 0.6899744272232056,381 "20.o_proj": 0.6347531080245972,382 "20.gate_proj": 0.6125325560569763,383 "20.up_proj": 0.636976957321167,384 "20.down_proj": 0.6877227425575256,385 "21.q_proj": 0.5809774398803711,386 "21.k_proj": 0.5469288229942322,387 "21.v_proj": 0.6610574722290039,388 "21.o_proj": 0.6689409017562866,389 "21.gate_proj": 0.6139337420463562,390 "21.up_proj": 0.6391165256500244,391 "21.down_proj": 0.6832785606384277,392 "22.q_proj": 0.6217024922370911,393 "22.k_proj": 0.5693233609199524,394 "22.v_proj": 0.6632723808288574,395 "22.o_proj": 0.6856058835983276,396 "22.gate_proj": 0.6192406415939331,397 "22.up_proj": 0.6405629515647888,398 "22.down_proj": 0.6809287667274475,399 "23.q_proj": 0.6093561053276062,400 "23.k_proj": 0.5737671256065369,401 "23.v_proj": 0.6871448159217834,402 "23.o_proj": 0.6642407178878784,403 "23.gate_proj": 0.6219133138656616,404 "23.up_proj": 0.6424050331115723,405 "23.down_proj": 0.6742445826530457,406 "24.q_proj": 0.6159307360649109,407 "24.k_proj": 0.5868129134178162,408 "24.v_proj": 0.6966824531555176,409 "24.o_proj": 0.6866542100906372,410 "24.gate_proj": 0.6238306760787964,411 "24.up_proj": 0.6446942687034607,412 "24.down_proj": 0.6646919250488281,413 "25.q_proj": 0.6210862994194031,414 "25.k_proj": 0.5705580115318298,415 "25.v_proj": 0.679048478603363,416 "25.o_proj": 0.6676820516586304,417 "25.gate_proj": 0.6258559823036194,418 "25.up_proj": 0.6465917825698853,419 "25.down_proj": 0.6531879901885986,420 "26.q_proj": 0.6165540218353271,421 "26.k_proj": 0.5607565641403198,422 "26.v_proj": 0.6618245840072632,423 "26.o_proj": 0.6507518887519836,424 "26.gate_proj": 0.6270332336425781,425 "26.up_proj": 0.6463326811790466,426 "26.down_proj": 0.6390087008476257,427 "27.q_proj": 0.6284988522529602,428 "27.k_proj": 0.6069222092628479,429 "27.v_proj": 0.6805939674377441,430 "27.o_proj": 0.6412052512168884,431 "27.gate_proj": 0.6305440664291382,432 "27.up_proj": 0.6479936242103577,433 "27.down_proj": 0.6380788683891296,434 "28.q_proj": 0.5876466035842896,435 "28.k_proj": 0.5294674634933472,436 "28.v_proj": 0.6338029503822327,437 "28.o_proj": 0.6242429614067078,438 "28.gate_proj": 0.6301671266555786,439 "28.up_proj": 0.6454077959060669,440 "28.down_proj": 0.6476228833198547,441 "29.q_proj": 0.5711690783500671,442 "29.k_proj": 0.5322142243385315,443 "29.v_proj": 0.670903205871582,444 "29.o_proj": 0.6061508655548096,445 "29.gate_proj": 0.6246752142906189,446 "29.up_proj": 0.6391395926475525,447 "29.down_proj": 0.6555037498474121,448 "30.q_proj": 0.5478631854057312,449 "30.k_proj": 0.4874037206172943,450 "30.v_proj": 0.6046014428138733,451 "30.o_proj": 0.5960355997085571,452 "30.gate_proj": 0.6242106556892395,453 "30.up_proj": 0.6300148367881775,454 "30.down_proj": 0.6422404646873474,455 "31.q_proj": 0.49975186586380005,456 "31.k_proj": 0.44580137729644775,457 "31.v_proj": 0.5610269904136658,458 "31.o_proj": 0.5210819840431213,459 "31.gate_proj": 0.6289830803871155,460 "31.up_proj": 0.6037687659263611,461 "31.down_proj": 0.6013435125350952462 },463 "step": "swift_svd_dynamic_finetuned",464 "compression": {465 "upstream": "hiahei/Swift-SVD",466 "step": "swift_svd_dynamic",467 "alloc": "dynamic",468 "alpha": 0.6,469 "nsamples": 256,470 "seqlen": 2048,471 "dataset": "WikiText2",472 "seed": 42,473 "gqa_budget": "shape",474 "param_fraction": 0.5002723106971154,475 "mean_rel_err": 0.5875181926946554,476 "max_rel_err": 0.7061887383460999477 },478 "factorisation": {479 "max_rel_err": 0.0014601770089939237,480 "factor_params": 3491561472,481 "dense_params": 6979321856482 },483 "finetune": {484 "protocol": "svdllm_stage2_u_then_v",485 "script": "gitclones/SVD-LLM/utils/LoRA.py",486 "data": "alpaca-cleaned",487 "epochs_per_half": 2,488 "learning_rate": 0.0001,489 "batch_size": 64,490 "micro_batch_size": 4,491 "lora_r": 8,492 "lora_alpha": 16,493 "lora_dropout": 0.05,494 "cutoff_len": 256,495 "val_set_size": 2000,496 "max_train_samples": "all",497 "precision": "bf16 (upstream hard-codes fp16; see run_with_torch_compat)"498 }499}