Glint-Research/Router

1{2  "model": "Glint-Router-1M",3  "author": "Glint-Research",4  "parameters": 985826,5  "baseline": "SupraLabs/Supra-Router-51M (51,786,240 params)",6  "training": {7    "params": 985826,8    "train_rows": 87604,9    "val_rows": 7617,10    "steps": 14479,11    "minutes": 15.0,12    "domain": 0.9464356045687279,13    "route": 0.939608769856899,14    "complexity": 0.7923066824209006,15    "code": 0.991466456610214,16    "math": 0.978075357752396,17    "reasoning": 0.8949717736641722,18    "long_output": 0.9516870158855192,19    "ece_raw": 0.05564934242829622,20    "ece_calibrated": 0.02029288645098859,21    "hardware": "RTX 3060 Ti",22    "rounds": 1,23    "wall_clock_minutes": 15.0,24    "corpus_rows": 9522125  },26  "neutral_holdout": {27    "val_rows": 3418,28    "slice_rows": {29      "aqua_rat": 254,30      "dolly": 2500,31      "humaneval": 164,32      "mbpp": 50033    },34    "glint": {35      "route_accuracy": 0.9441193680514921,36      "code_accuracy": 0.9751316559391457,37      "math_accuracy": 0.9526038619075483,38      "complexity_exact": 0.46284376828554713,39      "complexity_within_1": 0.9590403744880047,40      "ms_per_prompt": 0.05992146021082586,41      "parse_failures": 042    },43    "glint_aqua_rat": {44      "route_accuracy": 0.9094488188976378,45      "code_accuracy": 0.9960629921259843,46      "math_accuracy": 0.889763779527559,47      "complexity_exact": 0.1141732283464567,48      "complexity_within_1": 0.8070866141732284,49      "ms_per_prompt": 0.05992146021082586,50      "parse_failures": 051    },52    "glint_dolly": {53      "route_accuracy": 0.9424,54      "code_accuracy": 0.9956,55      "math_accuracy": 0.9612,56      "complexity_exact": 0.5688,57      "complexity_within_1": 0.9944,58      "ms_per_prompt": 0.05992146021082586,59      "parse_failures": 060    },61    "glint_humaneval": {62      "route_accuracy": 0.8719512195121951,63      "code_accuracy": 0.5670731707317073,64      "math_accuracy": 0.9024390243902439,65      "complexity_exact": 0.4451219512195122,66      "complexity_within_1": 0.9817073170731707,67      "ms_per_prompt": 0.05992146021082586,68      "parse_failures": 069    },70    "glint_mbpp": {71      "route_accuracy": 0.994,72      "code_accuracy": 0.996,73      "math_accuracy": 0.958,74      "complexity_exact": 0.116,75      "complexity_within_1": 0.852,76      "ms_per_prompt": 0.05992146021082586,77      "parse_failures": 078    },79    "supra": {80      "route_accuracy": 0.903744880046811,81      "code_accuracy": 0.9643066120538326,82      "math_accuracy": 0.9207138677589234,83      "complexity_exact": 0.3905792861322411,84      "complexity_within_1": 0.9163253364540667,85      "ms_per_prompt": 14.334112498536923,86      "parse_failures": 487    },88    "supra_aqua_rat": {89      "route_accuracy": 0.9133858267716536,90      "code_accuracy": 1.0,91      "math_accuracy": 0.9094488188976378,92      "complexity_exact": 0.0,93      "complexity_within_1": 0.09448818897637795,94      "ms_per_prompt": 14.334112498536923,95      "parse_failures": 496    },97    "supra_dolly": {98      "route_accuracy": 0.8896,99      "code_accuracy": 0.996,100      "math_accuracy": 0.9428,101      "complexity_exact": 0.482,102      "complexity_within_1": 0.9776,103      "ms_per_prompt": 14.334112498536923,104      "parse_failures": 4105    },106    "supra_humaneval": {107      "route_accuracy": 0.8292682926829268,108      "code_accuracy": 0.676829268292683,109      "math_accuracy": 0.8414634146341463,110      "complexity_exact": 0.16463414634146342,111      "complexity_within_1": 1.0,112      "ms_per_prompt": 14.334112498536923,113      "parse_failures": 4114    },115    "supra_mbpp": {116      "route_accuracy": 0.994,117      "code_accuracy": 0.882,118      "math_accuracy": 0.842,119      "complexity_exact": 0.206,120      "complexity_within_1": 1.0,121      "ms_per_prompt": 14.334112498536923,122      "parse_failures": 4123    }124  },125  "in_corpus_val": {126    "val_rows": 7617,127    "glint": {128      "route_accuracy": 0.939608769856899,129      "code_accuracy": 0.991466456610214,130      "math_accuracy": 0.978075357752396,131      "complexity_exact": 0.7923066824209006,132      "complexity_within_1": 0.9883156098201391,133      "ms_per_prompt": 0.0748591950898926,134      "parse_failures": 0135    },136    "glint_trap": {137      "route_accuracy": 1.0,138      "code_accuracy": 1.0,139      "math_accuracy": 1.0,140      "complexity_exact": 1.0,141      "complexity_within_1": 1.0,142      "parse_failures": 0143    },144    "supra": {145      "route_accuracy": 0.8575554680320336,146      "code_accuracy": 0.9761060785085992,147      "math_accuracy": 0.9400026257056584,148      "complexity_exact": 0.4629119075751608,149      "complexity_within_1": 0.923723250623605,150      "ms_per_prompt": 14.768008985689887,151      "parse_failures": 1152    },153    "supra_trap": {154      "route_accuracy": 0.908695652173913,155      "code_accuracy": 0.9869565217391304,156      "math_accuracy": 1.0,157      "complexity_exact": 0.33043478260869563,158      "complexity_within_1": 0.8391304347826087,159      "parse_failures": 1160    },161    "note": "labels from the glint pipeline. glint is in-distribution here, supra is not. read the holdout numbers for the fair comparison."162  },163  "supra_own_rows": {164    "val_rows": 91,165    "glint": {166      "route_accuracy": 0.8901098901098901,167      "code_accuracy": 0.9010989010989011,168      "math_accuracy": 0.8131868131868132,169      "complexity_exact": 0.5054945054945055170    },171    "supra": {172      "route_accuracy": 0.967032967032967,173      "code_accuracy": 1.0,174      "math_accuracy": 0.978021978021978,175      "complexity_exact": 0.8571428571428571176    },177    "note": "supra's own 992-row dataset, held out of glint training. these rows are supra's TRAINING data, so supra is scoring a memorised set. published because leaving it out would be dishonest."178  }179}