AwAppp/benchmarks_8bit_batch_size25
0
1{2 "prefill": {3 "memory": {4 "unit": "MB",5 "max_ram": 3610.86976,6 "max_vram": 13706.985472,7 "max_reserved": 13214.154752,8 "max_allocated": 11027.4539529 },10 "latency": {11 "unit": "s",12 "mean": 0.4016479968261719,13 "stdev": 0.00450307941805889,14 "values": [15 0.4229984130859375,16 0.4061747131347656,17 0.4007014465332031,18 0.4005621643066406,19 0.4003113098144531,20 0.4004822998046875,21 0.4004290466308594,22 0.4008570861816406,23 0.40047308349609373,24 0.40001739501953126,25 0.4006717529296875,26 0.4008058776855469,27 0.40065023803710936,28 0.4005826416015625,29 0.40037786865234376,30 0.40050381469726565,31 0.40064306640625,32 0.40063385009765623,33 0.40045669555664065,34 0.39976141357421874,35 0.40033383178710935,36 0.4008027648925781,37 0.40047308349609373,38 0.4009308166503906,39 0.4005652465820312440 ]41 },42 "throughput": {43 "unit": "tokens/s",44 "value": 995.896912622011345 },46 "energy": null,47 "efficiency": null48 },49 "decode": {50 "memory": {51 "unit": "MB",52 "max_ram": 3630.583808,53 "max_vram": 13736.3456,54 "max_reserved": 13243.51488,55 "max_allocated": 11166.72153656 },57 "latency": {58 "unit": "s",59 "mean": 28.560530487060543,60 "stdev": 0,61 "values": [62 28.56053048706054363 ]64 },65 "throughput": {66 "unit": "tokens/s",67 "value": 86.6580542375187468 },69 "energy": null,70 "efficiency": null71 },72 "per_token": {73 "memory": null,74 "latency": {75 "unit": "s",76 "mean": 0.2884902069400055,77 "stdev": 0.005574553256331573,78 "values": [79 0.30573773193359377,80 0.3014727783203125,81 0.29969100952148436,82 0.29973300170898437,83 0.29858203125,84 0.298608642578125,85 0.29833624267578124,86 0.2978570251464844,87 0.29740133666992186,88 0.29682278442382815,89 0.29672549438476564,90 0.2960639953613281,91 0.2946611328125,92 0.2954721374511719,93 0.2954915771484375,94 0.2945587158203125,95 0.29500518798828124,96 0.2944102478027344,97 0.29504205322265625,98 0.29280154418945314,99 0.2928322448730469,100 0.29225778198242186,101 0.2919669799804688,102 0.29329202270507815,103 0.2921922607421875,104 0.29252301025390626,105 0.2918021240234375,106 0.29138430786132813,107 0.29127166748046873,108 0.2904176635742188,109 0.2911313781738281,110 0.2917529602050781,111 0.2908026733398438,112 0.29039410400390625,113 0.2901688232421875,114 0.28997222900390623,115 0.28976434326171874,116 0.2902865905761719,117 0.28897689819335937,118 0.28997222900390623,119 0.2897592468261719,120 0.28981964111328123,121 0.28855502319335935,122 0.2880440368652344,123 0.2889666442871094,124 0.28792318725585936,125 0.28774911499023437,126 0.2884403076171875,127 0.2875832214355469,128 0.28695040893554685,129 0.2873180236816406,130 0.2875217895507812,131 0.2857482299804687,132 0.28675787353515625,133 0.2872637329101563,134 0.28659710693359375,135 0.2864322509765625,136 0.28636672973632815,137 0.28585574340820313,138 0.28576461791992186,139 0.2865428466796875,140 0.2852802429199219,141 0.2851758117675781,142 0.284943359375,143 0.28499661254882813,144 0.28454400634765625,145 0.28392242431640624,146 0.2840729675292969,147 0.28438323974609375,148 0.28357427978515626,149 0.28338687133789064,150 0.28420913696289063,151 0.28362240600585936,152 0.2831769714355469,153 0.2835486755371094,154 0.28436376953125,155 0.28338381958007813,156 0.2831790161132812,157 0.28350567626953127,158 0.28333056640625,159 0.28241510009765625,160 0.2824325256347656,161 0.2824560546875,162 0.2829588623046875,163 0.2829619140625,164 0.28229632568359375,165 0.2825932922363281,166 0.28196762084960936,167 0.2830182495117187,168 0.28210791015625,169 0.2823055419921875,170 0.28156927490234374,171 0.2821396484375,172 0.2817843322753906,173 0.28104806518554687,174 0.28229119873046876,175 0.2816358337402344,176 0.281427978515625,177 0.2809241638183594178 ]179 },180 "throughput": {181 "unit": "tokens/s",182 "value": 86.65805423751874183 },184 "energy": null,185 "efficiency": null186 }187}