AwAppp/benchmarks_8bit_batch_size35
0
1{2 "prefill": {3 "memory": {4 "unit": "MB",5 "max_ram": 3537.494016,6 "max_vram": 14564.72064,7 "max_reserved": 14071.88992,8 "max_allocated": 11367.313929 },10 "latency": {11 "unit": "s",12 "mean": 0.4472763897439709,13 "stdev": 0.0047645116545721,14 "values": [15 0.4695675048828125,16 0.44648654174804686,17 0.44618341064453126,18 0.4461240234375,19 0.446529541015625,20 0.4470425720214844,21 0.44600115966796877,22 0.44631143188476563,23 0.4458629150390625,24 0.4460113830566406,25 0.4461997985839844,26 0.4461055908203125,27 0.44638311767578126,28 0.44606976318359376,29 0.44569393920898437,30 0.4462243957519531,31 0.44664935302734377,32 0.4462377014160156,33 0.44663296508789063,34 0.4456673278808594,35 0.44621517944335937,36 0.4461690979003906,37 0.446988250732421938 ]39 },40 "throughput": {41 "unit": "tokens/s",42 "value": 1252.02226820099843 },44 "energy": null,45 "efficiency": null46 },47 "decode": {48 "memory": {49 "unit": "MB",50 "max_ram": 3539.29216,51 "max_vram": 14508.097536,52 "max_reserved": 14015.266816,53 "max_allocated": 11559.13523254 },55 "latency": {56 "unit": "s",57 "mean": 29.939338134765624,58 "stdev": 0,59 "values": [60 29.93933813476562461 ]62 },63 "throughput": {64 "unit": "tokens/s",65 "value": 115.734021386946966 },67 "energy": null,68 "efficiency": null69 },70 "per_token": {71 "memory": null,72 "latency": {73 "unit": "s",74 "mean": 0.3024175569168245,75 "stdev": 0.006495047756518274,76 "values": [77 0.3167109069824219,78 0.31577908325195314,79 0.3155199890136719,80 0.31530496215820314,81 0.31362252807617186,82 0.31696792602539064,83 0.31384576416015625,84 0.31296615600585936,85 0.3133685607910156,86 0.31271627807617186,87 0.311510009765625,88 0.31146701049804687,89 0.3107440490722656,90 0.3104020385742188,91 0.31054949951171873,92 0.30999551391601565,93 0.3091507263183594,94 0.3089141845703125,95 0.30916915893554686,96 0.3088988037109375,97 0.3080447998046875,98 0.30975588989257813,99 0.3072440185546875,100 0.30704129028320315,101 0.3072491455078125,102 0.30681805419921876,103 0.3068426208496094,104 0.3062528076171875,105 0.3066050415039063,106 0.3056844787597656,107 0.3046492309570312,108 0.30730548095703125,109 0.30522265625,110 0.3050281066894531,111 0.305033203125,112 0.3047301025390625,113 0.3045478515625,114 0.3045509033203125,115 0.3057029113769531,116 0.3038760986328125,117 0.30308966064453124,118 0.303994873046875,119 0.3024803771972656,120 0.30274252319335937,121 0.3025059814453125,122 0.30242303466796877,123 0.3026882629394531,124 0.30180352783203124,125 0.30188235473632813,126 0.3011389465332031,127 0.3005388793945313,128 0.3004610595703125,129 0.30039654541015626,130 0.29937765502929686,131 0.3015362548828125,132 0.29953536987304685,133 0.2986639404296875,134 0.29938278198242185,135 0.298392578125,136 0.2982430725097656,137 0.29859329223632813,138 0.2981652526855469,139 0.29810891723632815,140 0.29872332763671877,141 0.29826150512695315,142 0.29788363647460936,143 0.2971463623046875,144 0.2972651672363281,145 0.29730508422851565,146 0.2972364807128906,147 0.29666302490234375,148 0.30092901611328127,149 0.2961336364746094,150 0.29574554443359374,151 0.29620736694335936,152 0.2961448974609375,153 0.29597183227539064,154 0.2953799743652344,155 0.29584384155273435,156 0.29593701171875,157 0.29542706298828125,158 0.2955735168457031,159 0.2946253051757812,160 0.2954874572753906,161 0.29425869750976563,162 0.2946621398925781,163 0.2944389038085938,164 0.2949775390625,165 0.2961080322265625,166 0.29468057250976565,167 0.2936627197265625,168 0.29617459106445315,169 0.2940631103515625,170 0.2944245910644531,171 0.2944296875,172 0.29573223876953125,173 0.29399038696289065,174 0.294677490234375,175 0.2972333984375176 ]177 },178 "throughput": {179 "unit": "tokens/s",180 "value": 115.7340213869469181 },182 "energy": null,183 "efficiency": null184 }185}