AwAppp/benchmarks_8bit_batch_size40
0
1{2 "prefill": {3 "memory": {4 "unit": "MB",5 "max_ram": 3649.363968,6 "max_vram": 15097.397248,7 "max_reserved": 14604.566528,8 "max_allocated": 11492.0432649 },10 "latency": {11 "unit": "s",12 "mean": 0.4630080330588601,13 "stdev": 0.004482724887650619,14 "values": [15 0.4835208435058594,16 0.4625725402832031,17 0.4620338439941406,18 0.46234112548828127,19 0.4619939880371094,20 0.4617963562011719,21 0.4621127624511719,22 0.4619253540039063,23 0.4620205993652344,24 0.4620994567871094,25 0.46176766967773436,26 0.4620769348144531,27 0.4618219299316406,28 0.4617502746582031,29 0.4620902404785156,30 0.4627476501464844,31 0.4620103759765625,32 0.4619530334472656,33 0.4618577880859375,34 0.46192333984375,35 0.4618618774414063,36 0.4618987426757812437 ]38 },39 "throughput": {40 "unit": "tokens/s",41 "value": 1382.26543451491242 },43 "energy": null,44 "efficiency": null45 },46 "decode": {47 "memory": {48 "unit": "MB",49 "max_ram": 3649.363968,50 "max_vram": 15755.902976,51 "max_reserved": 15263.072256,52 "max_allocated": 11697.74796853 },54 "latency": {55 "unit": "s",56 "mean": 30.39823352050782,57 "stdev": 0,58 "values": [59 30.3982335205078260 ]61 },62 "throughput": {63 "unit": "tokens/s",64 "value": 130.2707276502903365 },66 "energy": null,67 "efficiency": null68 },69 "per_token": {70 "memory": null,71 "latency": {72 "unit": "s",73 "mean": 0.3070528638435133,74 "stdev": 0.00619475129853309,75 "values": [76 0.32103317260742187,77 0.320458740234375,78 0.3195606994628906,79 0.3190783996582031,80 0.3187373962402344,81 0.3189104614257813,82 0.31889306640625,83 0.31756594848632813,84 0.3173099670410156,85 0.3172567138671875,86 0.3162204284667969,87 0.31606475830078123,88 0.31606173706054685,89 0.3153530578613281,90 0.31523019409179687,91 0.3142891540527344,92 0.3149322204589844,93 0.3137945556640625,94 0.3137208251953125,95 0.31293234252929686,96 0.3128350830078125,97 0.3129702453613281,98 0.31244903564453125,99 0.31200460815429687,100 0.3119308776855469,101 0.31241726684570315,102 0.3110267028808594,103 0.31101849365234374,104 0.31155813598632814,105 0.31053829956054685,106 0.30974969482421877,107 0.30999859619140624,108 0.3091568603515625,109 0.3098849182128906,110 0.30947122192382814,111 0.30916403198242187,112 0.3083642883300781,113 0.30852301025390627,114 0.30778573608398435,115 0.308284423828125,116 0.30778265380859376,117 0.3074211730957031,118 0.30655078125,119 0.3073249206542969,120 0.30658560180664063,121 0.30559130859375,122 0.30594354248046873,123 0.30640640258789065,124 0.3051796569824219,125 0.3058493347167969,126 0.3058452453613281,127 0.30491134643554685,128 0.305227783203125,129 0.3053455505371094,130 0.30429080200195313,131 0.3045038146972656,132 0.3032811584472656,133 0.30313677978515624,134 0.30287359619140625,135 0.3031756896972656,136 0.3024117736816406,137 0.3021936645507812,138 0.301380615234375,139 0.3028357238769531,140 0.30214654541015623,141 0.3021240234375,142 0.30190899658203124,143 0.3019069519042969,144 0.30141543579101565,145 0.3013048400878906,146 0.30091162109375,147 0.30223974609375,148 0.30069451904296873,149 0.3011194763183594,150 0.30109390258789065,151 0.30174822998046874,152 0.3004610595703125,153 0.3011061706542969,154 0.30095974731445313,155 0.3013918762207031,156 0.29992755126953125,157 0.3006238708496094,158 0.30073651123046874,159 0.30130075073242185,160 0.30008935546875,161 0.3009218444824219,162 0.30135092163085936,163 0.30108978271484377,164 0.3004989318847656,165 0.3007057800292969,166 0.30057470703125,167 0.30154238891601565,168 0.30043032836914063,169 0.30031768798828123,170 0.29990093994140626,171 0.3013959655761719,172 0.30073855590820314,173 0.3008112487792969,174 0.30015896606445314175 ]176 },177 "throughput": {178 "unit": "tokens/s",179 "value": 130.27072765029033180 },181 "energy": null,182 "efficiency": null183 }184}