AwAppp/benchmarks_8bit_batch_size10

1{2    "prefill": {3        "memory": {4            "unit": "MB",5            "max_ram": 3525.599232,6            "max_vram": 11714.691072,7            "max_reserved": 11221.860352,8            "max_allocated": 9991.7245449        },10        "latency": {11            "unit": "s",12            "mean": 0.3010050461713006,13            "stdev": 0.005007823178304763,14            "values": [15                0.32892837524414065,16                0.3053179016113281,17                0.3002685546875,18                0.2994657287597656,19                0.3013898315429687,20                0.29939508056640624,21                0.29945855712890623,22                0.3003228149414062,23                0.29954458618164065,24                0.29985791015625,25                0.3006095275878906,26                0.3026851806640625,27                0.2997637023925781,28                0.300037109375,29                0.2996305847167969,30                0.29964801025390625,31                0.3008983154296875,32                0.2997923889160156,33                0.2989916076660156,34                0.29947698974609377,35                0.2995968017578125,36                0.2994698181152344,37                0.29934283447265625,38                0.299536376953125,39                0.29968487548828127,40                0.29946783447265624,41                0.30125567626953126,42                0.30218035888671874,43                0.2998056945800781,44                0.29990194702148437,45                0.2995118103027344,46                0.29968487548828127,47                0.29941860961914063,48                0.29983129882812549            ]50        },51        "throughput": {52            "unit": "tokens/s",53            "value": 531.552550480980154        },55        "energy": null,56        "efficiency": null57    },58    "decode": {59        "memory": {60            "unit": "MB",61            "max_ram": 3536.162816,62            "max_vram": 12268.3392,63            "max_reserved": 11775.50848,64            "max_allocated": 10078.90227265        },66        "latency": {67            "unit": "s",68            "mean": 25.19557006835937,69            "stdev": 0,70            "values": [71                25.1955700683593772            ]73        },74        "throughput": {75            "unit": "tokens/s",76            "value": 39.29262157252172577        },78        "energy": null,79        "efficiency": null80    },81    "per_token": {82        "memory": null,83        "latency": {84            "unit": "s",85            "mean": 0.25450070776120576,86            "stdev": 0.0028173351708351006,87            "values": [88                0.2622341003417969,89                0.2604031982421875,90                0.2598922119140625,91                0.259346435546875,92                0.26023934936523435,93                0.259125244140625,94                0.259641357421875,95                0.2600816650390625,96                0.2582261657714844,97                0.2586593322753906,98                0.2587289733886719,99                0.2589450378417969,100                0.2583848876953125,101                0.2578052978515625,102                0.25718887329101564,103                0.2568939514160156,104                0.25893478393554686,105                0.2582753295898437,106                0.25697381591796875,107                0.2581626892089844,108                0.256964599609375,109                0.256321533203125,110                0.25642291259765626,111                0.25666763305664064,112                0.25534873962402344,113                0.25577267456054686,114                0.25576243591308595,115                0.25529037475585936,116                0.25610955810546876,117                0.2554173431396484,118                0.2555606994628906,119                0.25545318603515627,120                0.2541045684814453,121                0.2553528289794922,122                0.2549780426025391,123                0.2547394561767578,124                0.2549903411865234,125                0.2602659912109375,126                0.2590689392089844,127                0.25401242065429686,128                0.2539376678466797,129                0.254271484375,130                0.2540543975830078,131                0.25435136413574216,132                0.2539878387451172,133                0.25366937255859373,134                0.25358233642578126,135                0.25377484130859373,136                0.2529126434326172,137                0.2540585021972656,138                0.25315840148925783,139                0.2530897979736328,140                0.2536099853515625,141                0.2528061370849609,142                0.25310617065429686,143                0.25320346069335936,144                0.2535393218994141,145                0.2527958984375,146                0.253264892578125,147                0.25287986755371095,148                0.2530160675048828,149                0.2529689636230469,150                0.25187532043457034,151                0.25355877685546874,152                0.25151078796386717,153                0.2520494079589844,154                0.25327001953125,155                0.25154150390625,156                0.2519019470214844,157                0.25225625610351565,158                0.25124659729003906,159                0.25213848876953127,160                0.25257369995117185,161                0.2515240936279297,162                0.2524241943359375,163                0.2529085388183594,164                0.2525675506591797,165                0.25248051452636716,166                0.2522879943847656,167                0.2524682312011719,168                0.2532095947265625,169                0.25251429748535154,170                0.2515650634765625,171                0.25252351379394533,172                0.25155378723144534,173                0.2517022705078125,174                0.25210067749023435,175                0.25160902404785157,176                0.25191014099121095,177                0.25165721130371094,178                0.25126296997070313,179                0.25138584899902344,180                0.250998779296875,181                0.2516172790527344,182                0.2514114532470703,183                0.25093426513671874,184                0.25146060180664065,185                0.251789306640625,186                0.25099366760253905187            ]188        },189        "throughput": {190            "unit": "tokens/s",191            "value": 39.292621572521725192        },193        "energy": null,194        "efficiency": null195    }196}