AwAppp/benchmarks_8bit_batch_size15
0
1{2 "prefill": {3 "memory": {4 "unit": "MB",5 "max_ram": 3470.56128,6 "max_vram": 12769.558528,7 "max_reserved": 12276.727808,8 "max_allocated": 10493.8531849 },10 "latency": {11 "unit": "s",12 "mean": 0.3322811594317036,13 "stdev": 0.004201918534069472,14 "values": [15 0.3548702697753906,16 0.3352504272460938,17 0.3326003112792969,18 0.33182821655273437,19 0.3319469299316406,20 0.33254296875,21 0.3314892883300781,22 0.33137869262695313,23 0.33092608642578125,24 0.3313489990234375,25 0.3318661193847656,26 0.3311390686035156,27 0.33138790893554687,28 0.3314851989746094,29 0.33103564453125,30 0.3311247253417969,31 0.3315466003417969,32 0.33106533813476563,33 0.3314124755859375,34 0.331188232421875,35 0.33160498046875,36 0.3317739562988281,37 0.3311513671875,38 0.33090765380859377,39 0.33081649780273437,40 0.33114727783203124,41 0.33095269775390623,42 0.3309649963378906,43 0.3314667663574219,44 0.33160498046875,45 0.3308912658691406446 ]47 },48 "throughput": {49 "unit": "tokens/s",50 "value": 722.279892156597351 },52 "energy": null,53 "efficiency": null54 },55 "decode": {56 "memory": {57 "unit": "MB",58 "max_ram": 3471.822848,59 "max_vram": 13524.533248,60 "max_reserved": 13031.702528,61 "max_allocated": 10594.23846462 },63 "latency": {64 "unit": "s",65 "mean": 27.10930010986329,66 "stdev": 0,67 "values": [68 27.1093001098632969 ]70 },71 "throughput": {72 "unit": "tokens/s",73 "value": 54.7782493086092774 },75 "energy": null,76 "efficiency": null77 },78 "per_token": {79 "memory": null,80 "latency": {81 "unit": "s",82 "mean": 0.27383131424104334,83 "stdev": 0.003865402878377042,84 "values": [85 0.28423577880859374,86 0.2813306884765625,87 0.28021041870117186,88 0.2805688171386719,89 0.2805350341796875,90 0.280511474609375,91 0.28056781005859377,92 0.28048178100585935,93 0.2796636047363281,94 0.279478271484375,95 0.2780057678222656,96 0.2784327697753906,97 0.27773236083984376,98 0.2771885986328125,99 0.2785331115722656,100 0.27619430541992185,101 0.27745895385742186,102 0.27661618041992186,103 0.2762188720703125,104 0.27671243286132813,105 0.27565362548828126,106 0.2763612060546875,107 0.27662130737304685,108 0.2756331481933594,109 0.27525225830078126,110 0.2755604248046875,111 0.27471261596679686,112 0.2745743103027344,113 0.27438796997070314,114 0.27482009887695313,115 0.2740787048339844,116 0.2747821960449219,117 0.2742958068847656,118 0.27405517578125,119 0.27351553344726565,120 0.27380224609375,121 0.2732697448730469,122 0.273517578125,123 0.27277618408203125,124 0.2734888916015625,125 0.27336294555664065,126 0.27980593872070314,127 0.27267788696289064,128 0.27327078247070313,129 0.2725263366699219,130 0.2733834228515625,131 0.2763018493652344,132 0.28083709716796873,133 0.2731366271972656,134 0.28008447265625,135 0.27196417236328124,136 0.27271783447265624,137 0.27304241943359375,138 0.27160986328125,139 0.2719815673828125,140 0.2721392517089844,141 0.2742210693359375,142 0.27142962646484375,143 0.2707630004882812,144 0.27627825927734373,145 0.2755911560058594,146 0.27026943969726563,147 0.27018649291992186,148 0.2707906494140625,149 0.2697359313964844,150 0.26930584716796874,151 0.27066574096679685,152 0.26956494140625,153 0.2701271057128906,154 0.2692567138671875,155 0.2692587585449219,156 0.2694625244140625,157 0.2702479248046875,158 0.27053466796875,159 0.27034725952148436,160 0.2764933166503906,161 0.27599053955078123,162 0.2792693786621094,163 0.2697052307128906,164 0.2699970703125,165 0.2686484375,166 0.2691286926269531,167 0.2699591674804688,168 0.2695352783203125,169 0.2692228698730469,170 0.2694993896484375,171 0.26948300170898437,172 0.26922906494140625,173 0.26847027587890626,174 0.27224267578125,175 0.2729758605957031,176 0.27673394775390625,177 0.2686033935546875,178 0.26899252319335937,179 0.269348876953125,180 0.2679552001953125,181 0.2716129150390625,182 0.2688880615234375,183 0.26859930419921874184 ]185 },186 "throughput": {187 "unit": "tokens/s",188 "value": 54.77824930860927189 },190 "energy": null,191 "efficiency": null192 }193}