AwAppp/benchmarks_8bit_batch_size5
0
1{2 "prefill": {3 "memory": {4 "unit": "MB",5 "max_ram": 3576.500224,6 "max_vram": 11024.728064,7 "max_reserved": 10531.897344,8 "max_allocated": 9458.513929 },10 "latency": {11 "unit": "s",12 "mean": 0.26589927994577517,13 "stdev": 0.004717404774457072,14 "values": [15 0.2945403137207031,16 0.2661632080078125,17 0.2650818176269531,18 0.26499276733398436,19 0.2650798034667969,20 0.2648647766113281,21 0.2649354248046875,22 0.26499072265625,23 0.2650746765136719,24 0.2650808410644531,25 0.2649825134277344,26 0.26509210205078126,27 0.2649774169921875,28 0.2652323913574219,29 0.2649415588378906,30 0.2650306701660156,31 0.26549758911132815,32 0.26528564453125,33 0.26555596923828123,34 0.2653173828125,35 0.26431591796875,36 0.2648350830078125,37 0.26553549194335935,38 0.26530612182617186,39 0.26522317504882814,40 0.2651463623046875,41 0.26528460693359374,42 0.26509005737304686,43 0.2648668212890625,44 0.26478387451171875,45 0.2652835693359375,46 0.26543923950195314,47 0.2648074340820312,48 0.2652200927734375,49 0.2651146240234375,50 0.26522113037109374,51 0.264890380859375,52 0.26509106445312553 ]54 },55 "throughput": {56 "unit": "tokens/s",57 "value": 300.865801578381158 },59 "energy": null,60 "efficiency": null61 },62 "decode": {63 "memory": {64 "unit": "MB",65 "max_ram": 3577.966592,66 "max_vram": 11127.488512,67 "max_reserved": 10634.657792,68 "max_allocated": 9518.94988869 },70 "latency": {71 "unit": "s",72 "mean": 23.714270278930663,73 "stdev": 0,74 "values": [75 23.71427027893066376 ]77 },78 "throughput": {79 "unit": "tokens/s",80 "value": 20.87350756222893481 },82 "energy": null,83 "efficiency": null84 },85 "per_token": {86 "memory": null,87 "latency": {88 "unit": "s",89 "mean": 0.23953808362556225,90 "stdev": 0.0011223838576208927,91 "values": [92 0.24126976013183593,93 0.24184934997558594,94 0.241944580078125,95 0.24157696533203124,96 0.24142233276367187,97 0.24156364440917968,98 0.24111923217773437,99 0.24104243469238282,100 0.24155647277832032,101 0.2407731170654297,102 0.24124211120605468,103 0.24097996520996093,104 0.24022732543945313,105 0.24134962463378906,106 0.24109158325195312,107 0.24052633666992188,108 0.24072601318359374,109 0.24031642150878907,110 0.2408704071044922,111 0.24035328674316406,112 0.24031436157226563,113 0.2403256378173828,114 0.2404638671875,115 0.2411847686767578,116 0.23977984619140624,117 0.24041165161132813,118 0.23975117492675782,119 0.24032357788085937,120 0.24015565490722657,121 0.24032666015625,122 0.23952691650390626,123 0.23980543518066405,124 0.2395371551513672,125 0.24026419067382812,126 0.24166502380371094,127 0.2394122314453125,128 0.24026419067382812,129 0.2393456573486328,130 0.23959449768066407,131 0.23952793884277343,132 0.23927090454101563,133 0.23895552062988282,134 0.23910604858398438,135 0.23962112426757812,136 0.2393866271972656,137 0.23915008544921876,138 0.23919512939453125,139 0.2399006652832031,140 0.23890739440917969,141 0.23990170288085938,142 0.23840255737304689,143 0.23848141479492188,144 0.2383585205078125,145 0.23940914916992187,146 0.23918080139160156,147 0.23795199584960938,148 0.23889100646972655,149 0.23912037658691407,150 0.23861759948730468,151 0.23844248962402342,152 0.23861759948730468,153 0.2423388214111328,154 0.23819981384277344,155 0.23846400451660157,156 0.23878758239746095,157 0.2382704620361328,158 0.23828684997558594,159 0.23864218139648438,160 0.23916134643554687,161 0.23734579467773437,162 0.23761715698242186,163 0.23818240356445314,164 0.2382940216064453,165 0.238097412109375,166 0.23909068298339844,167 0.2390118408203125,168 0.23936819458007813,169 0.23946751403808594,170 0.23883673095703126,171 0.23915110778808593,172 0.23877529907226563,173 0.23924327087402344,174 0.23862168884277343,175 0.23861351013183593,176 0.2389811248779297,177 0.2389422149658203,178 0.2384701385498047,179 0.2389063720703125,180 0.2385039367675781,181 0.23919821166992186,182 0.2386544647216797,183 0.23751475524902343,184 0.23883775329589843,185 0.2382458953857422,186 0.23910195922851563,187 0.2384701385498047,188 0.2383462371826172,189 0.2390118408203125,190 0.23856742858886718191 ]192 },193 "throughput": {194 "unit": "tokens/s",195 "value": 20.873507562228934196 },197 "energy": null,198 "efficiency": null199 }200}