FluidInference/verdict-coreml

1{2  "source_repo": "heman10x/rlcd-modernbert-151m",3  "source_revision": "8af2496eb63c7fa66d7d234e1f62629380030eb4",4  "package": "verdict_fp16_L512_candidates25.mlpackage",5  "package_files_sha256": {6    "Data/com.apple.CoreML/model.mlmodel": "7ad21a9ee93d1c573e6d89e6c35511f08c984a5c47108bb2b908c96344c03903",7    "Data/com.apple.CoreML/weights/weight.bin": "fb438de5823dd91c8448b373d026b7e6dcb7149bf871fde54057fd6d82586886",8    "Manifest.json": "047cc80f7d66d822ca0d9c0100a6a6dc96ae2d70eac42e292a2b246e01e64ddd"9  },10  "chip": "arm",11  "macos": "27.0",12  "max_calibrated_probability_error": 0.0006975581321209678,13  "selected_id_agreements": 5,14  "questions": 5,15  "rows": [16    {17      "name": "lost card",18      "type": "choice",19      "tokens": 53,20      "candidates_including_abstention": 3,21      "native": {22        "type": "choice",23        "selected_id": "card_lost",24        "selected_probability": 0.6279954438939314,25        "probabilities": {26          "card_lost": 0.6279954438939314,27          "pin_reset": 0.12699384187218735,28          "__insufficient_evidence__": 0.245010714233881329        },30        "concentration": 0.18185852825435922,31        "is_abstention": false,32        "p_abstain": 0.2450107142338813,33        "calibration_status": "calibrated_for_scope",34        "choice": "card_lost"35      },36      "coreml": {37        "type": "choice",38        "selected_id": "card_lost",39        "selected_probability": 0.6286024758124755,40        "probabilities": {41          "card_lost": 0.6286024758124755,42          "pin_reset": 0.12674703377209953,43          "__insufficient_evidence__": 0.2446504904154249744        },45        "concentration": 0.18252696123986079,46        "is_abstention": false,47        "p_abstain": 0.24465049041542497,48        "calibration_status": "calibrated_for_scope",49        "choice": "card_lost"50      },51      "max_calibrated_probability_error": 0.0006070319185441653,52      "selected_id_agrees": true,53      "median_model_call_ms": 8.42758401995524854    },55    {56      "name": "binary inquiry",57      "type": "noul",58      "tokens": 50,59      "candidates_including_abstention": 3,60      "native": {61        "type": "noul",62        "selected_id": "true",63        "selected_probability": 0.4692844463560151,64        "probabilities": {65          "true": 0.4692844463560151,66          "false": 0.24040823180168494,67          "__insufficient_evidence__": 0.2903073218422999668        },69        "concentration": 0.03808302501474403,70        "is_abstention": false,71        "p_abstain": 0.29030732184229996,72        "calibration_status": "calibrated_for_scope",73        "noul": 0.6612502295701238,74        "selected_outcome": "true"75      },76      "coreml": {77        "type": "noul",78        "selected_id": "true",79        "selected_probability": 0.46998200448813604,80        "probabilities": {81          "true": 0.46998200448813604,82          "false": 0.23993786932412314,83          "__insufficient_evidence__": 0.2900801261877407684        },85        "concentration": 0.03846968937399986,86        "is_abstention": false,87        "p_abstain": 0.29008012618774076,88        "calibration_status": "calibrated_for_scope",89        "noul": 0.6620211967927305,90        "selected_outcome": "true"91      },92      "max_calibrated_probability_error": 0.0006975581321209678,93      "selected_id_agrees": true,94      "median_model_call_ms": 8.43745900783687895    },96    {97      "name": "urgency score",98      "type": "score",99      "tokens": 55,100      "candidates_including_abstention": 4,101      "native": {102        "type": "score",103        "selected_id": "__insufficient_evidence__",104        "selected_probability": 0.3259148157325998,105        "probabilities": {106          "low": 0.19793214504236917,107          "medium": 0.2898236019444062,108          "high": 0.18632943728062487,109          "__insufficient_evidence__": 0.3259148157325998110        },111        "concentration": 0.02039165010923949,112        "is_abstention": true,113        "p_abstain": 0.3259148157325998,114        "calibration_status": "calibrated_for_scope",115        "score": null,116        "selected_level_id": "__insufficient_evidence__",117        "selected_value": null118      },119      "coreml": {120        "type": "score",121        "selected_id": "__insufficient_evidence__",122        "selected_probability": 0.3262236353210236,123        "probabilities": {124          "low": 0.19780841262370594,125          "medium": 0.28950115689828004,126          "high": 0.18646679515699038,127          "__insufficient_evidence__": 0.3262236353210236128        },129        "concentration": 0.020408360566200767,130        "is_abstention": true,131        "p_abstain": 0.3262236353210236,132        "calibration_status": "calibrated_for_scope",133        "score": null,134        "selected_level_id": "__insufficient_evidence__",135        "selected_value": null136      },137      "max_calibrated_probability_error": 0.000322445046126163,138      "selected_id_agrees": true,139      "median_model_call_ms": 8.381415973417461140    },141    {142      "name": "insufficient evidence",143      "type": "choice",144      "tokens": 41,145      "candidates_including_abstention": 3,146      "native": {147        "type": "choice",148        "selected_id": "__insufficient_evidence__",149        "selected_probability": 0.6003396635578521,150        "probabilities": {151          "visa": 0.19581437244279193,152          "mastercard": 0.2038459639993559,153          "__insufficient_evidence__": 0.6003396635578521154        },155        "concentration": 0.1354398243565177,156        "is_abstention": true,157        "p_abstain": 0.6003396635578521,158        "calibration_status": "calibrated_for_scope",159        "choice": "__insufficient_evidence__"160      },161      "coreml": {162        "type": "choice",163        "selected_id": "__insufficient_evidence__",164        "selected_probability": 0.6008594846026971,165        "probabilities": {166          "visa": 0.19567826624893642,167          "mastercard": 0.2034622491483665,168          "__insufficient_evidence__": 0.6008594846026971169        },170        "concentration": 0.13595645758378905,171        "is_abstention": true,172        "p_abstain": 0.6008594846026971,173        "calibration_status": "calibrated_for_scope",174        "choice": "__insufficient_evidence__"175      },176      "max_calibrated_probability_error": 0.000519821044845048,177      "selected_id_agrees": true,178      "median_model_call_ms": 8.738874981645495179    },180    {181      "name": "BFCL-tool-selection:simple:simple_254",182      "type": "choice",183      "tokens": 283,184      "candidates_including_abstention": 3,185      "native": {186        "type": "choice",187        "selected_id": "yes",188        "selected_probability": 0.5095718104586857,189        "probabilities": {190          "yes": 0.5095718104586857,191          "no": 0.28447377613128705,192          "__insufficient_evidence__": 0.20595441341002724193        },194        "concentration": 0.0655574261710149,195        "is_abstention": false,196        "p_abstain": 0.20595441341002724,197        "calibration_status": "calibrated_for_scope",198        "choice": "yes"199      },200      "coreml": {201        "type": "choice",202        "selected_id": "yes",203        "selected_probability": 0.5093345700124705,204        "probabilities": {205          "yes": 0.5093345700124705,206          "no": 0.28465867339056805,207          "__insufficient_evidence__": 0.20600675659696147208        },209        "concentration": 0.06541626747983909,210        "is_abstention": false,211        "p_abstain": 0.20600675659696147,212        "calibration_status": "calibrated_for_scope",213        "choice": "yes"214      },215      "max_calibrated_probability_error": 0.00023724044621520335,216      "selected_id_agrees": true,217      "median_model_call_ms": 10.042167035862803218    }219  ],220  "passed": true221}222