licensedbets-open/probability-calibration-lab
0
1<!doctype html>2<html lang="en">3<head>4<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">5<title>Probability Calibration Lab | Licensed Bets</title>6<style>7:root{color-scheme:dark;--bg:#0c1422;--panel:#142136;--line:#34465f;--text:#edf3fa;--muted:#b4c5db;--accent:#8ae4ca;--warn:#ffcca0}*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--text);font:16px/1.6 system-ui,sans-serif}main{max-width:1150px;margin:auto;padding:40px 24px}header{margin-bottom:32px}.eyebrow{color:var(--accent);text-transform:uppercase;font-size:.76rem;letter-spacing:.15em;font-weight:750}h1{font-size:clamp(2rem,5vw,3.8rem);line-height:1.1;letter-spacing:-.045em;margin:12px 0 20px}h2{font-size:1.25rem;margin-top:0}h3{font-size:1rem}p{max-width:80ch}.intro{font-size:1.15rem;color:var(--muted)}.badge{display:inline-block;border:1px solid var(--line);border-radius:30px;padding:4px 12px;font-size:.8rem;margin-right:5px}.grid{display:grid;grid-template-columns:1fr 1.2fr;gap:22px}.panel{padding:24px;background:var(--panel);border:1px solid var(--line);border-radius:18px;margin-bottom:22px}label{display:block;font-weight:650;margin-bottom:8px}textarea,select{background:#0a1424;color:var(--text);border:1px solid #71839d;border-radius:8px;padding:12px;font:inherit}textarea{width:100%;min-height:315px;font:14px/1.65 ui-monospace,monospace;resize:vertical}select{padding:6px 10px}button{border:1px solid var(--accent);border-radius:8px;padding:10px 15px;background:var(--accent);color:#0a2430;font:650 .9rem system-ui;cursor:pointer}button.secondary{background:transparent;color:var(--text);border-color:#71839d}button:disabled{opacity:.5;cursor:not-allowed}.actions{display:flex;gap:10px;flex-wrap:wrap;margin-top:16px}:focus-visible{outline:3px solid #ffdc8a;outline-offset:4px}.help,small{color:var(--muted);font-size:.85rem}.stats{display:grid;grid-template-columns:repeat(3,1fr);gap:12px}.stat{border-top:2px solid var(--accent);padding-top:12px}.value{font-size:1.55rem;font-weight:750;display:block;font-variant-numeric:tabular-nums}.warning{color:var(--warn)}.table-wrap{overflow:auto}table{width:100%;border-collapse:collapse;font-size:.88rem;text-align:left;font-variant-numeric:tabular-nums}caption{text-align:left;color:var(--muted);padding:12px 0}th,td{padding:9px;border-bottom:1px solid var(--line);white-space:nowrap}th{color:var(--muted)}a{color:var(--accent);text-underline-offset:3px}svg{width:100%;height:auto;max-height:340px;display:block}details{margin-top:20px}summary{cursor:pointer;font-weight:650}footer{padding:12px 0;color:var(--muted);font-size:.88rem}#status{min-height:1.6em}.wide{grid-column:1/-1}@media(max-width:760px){.grid{grid-template-columns:1fr}.panel{padding:18px}.stats{gap:8px}.value{font-size:1.3rem}main{padding:26px 16px}}8</style>9</head>10<body><main>11<header><div class="eyebrow">Licensed Bets / Developer tools</div><h1>Probability<br>Calibration Lab</h1><p class="intro">A small, inspectable workbench for evaluating probability forecasts against binary outcomes. Bring a labeled batch, inspect its errors, and see where confidence differs from frequency.</p><span class="badge">Synthetic example only</span><span class="badge">Local computation</span><span class="badge">No dependencies</span></header>12<div class="grid">13<section class="panel"><h2>01 / Load a labeled batch</h2><label for="input">CSV: probability,outcome</label><p id="input-help" class="help">Optional header: <code>probability,outcome</code>. One observation per line; exactly two unquoted fields. Decimal or scientific notation is allowed. Blank lines are ignored. Maximum 10,000 nonblank data rows and 1 MB of text.</p><textarea id="input" spellcheck="false" aria-describedby="input-help"></textarea><div class="actions"><button id="evaluate">Evaluate batch</button><button class="secondary" id="reset">Reset synthetic example</button></div><p id="status" role="status" aria-live="polite"></p><p class="help">Nothing is uploaded or saved. Editing the input clears old results; select Evaluate to refresh. Downloaded files are saved only when you request them.</p></section>14<section class="panel"><h2>02 / Inspect aggregate error</h2><div class="stats"><div class="stat">Brier score<strong class="value" id="brier">-</strong><small>Lower is better</small></div><div class="stat">Log loss<strong class="value" id="loss">-</strong><small>Natural logarithm</small></div><div class="stat">Accuracy<strong class="value" id="accuracy">-</strong><small>Threshold p ≥ 0.5</small></div></div><p class="help">Metrics use accepted rows only, equally weighted. Brier score is mean (p − y)², using original probabilities. Log loss is mean −[y ln(q) + (1 − y) ln(1 − q)], with q clipped to [ε, 1 − ε], <strong>ε = 10⁻¹⁵</strong>. Boundary forecasts therefore have finite loss.</p><p class="warning">Accuracy is a poor proxy for calibration: it discards confidence and depends on the threshold and class balance. A lower Brier score or log loss does not, alone, establish better calibration.</p><details><summary>How to interpret this lab</summary><p class="help">Calibration compares predicted probabilities with observed frequencies. In a well-calibrated population, events assigned probability 0.7 occur about 70% of the time. A small batch cannot establish this. Bins lose detail, depend on chosen boundaries, and become noisy with few observations. No confidence intervals, model comparison, or statistical significance tests are provided. Use independent, representative held-out outcomes and check label quality.</p></details><button id="download" class="secondary" disabled>Download accepted evaluation CSV</button></section>15<section class="panel wide"><h2>03 / Reliability by probability range</h2><label for="bins">Equal-width bins <select id="bins"><option value="5">5 bins</option><option value="10" selected>10 bins</option><option value="20">20 bins</option></select></label><p class="help">Each point plots mean predicted probability (horizontal) against observed outcome rate (vertical). The diagonal is the ideal reference, not a fitted model. Empty bins have no point. All bins are left-inclusive and right-exclusive except the last, which includes 1.</p><div id="chart"></div><div class="table-wrap"><table><caption>Reliability data: the accessible equivalent of the chart</caption><thead><tr><th scope="col">Probability range</th><th scope="col">Count</th><th scope="col">Mean probability</th><th scope="col">Outcome rate</th></tr></thead><tbody id="binrows"></tbody></table></div></section>16<section class="panel wide"><h2>04 / Row audit</h2><p class="help">Original line numbers are retained. Invalid rows are excluded, not repaired. Exports contain accepted rows, per-row errors, and the selected reliability-bin index (1-based).</p><div class="table-wrap"><table><caption id="audit-caption">No batch evaluated</caption><thead><tr><th scope="col">Line</th><th scope="col">Status</th><th scope="col">Probability</th><th scope="col">Outcome</th><th scope="col">Reason</th></tr></thead><tbody id="audit"></tbody></table></div></section>17</div>18<footer><h2>Scope, sources & publisher</h2><p>The starting dataset is entirely synthetic, manually constructed for demonstration. These results are not actual model performance, betting predictions, odds, or recommendations. This lab is independent of real odds feeds and makes no claim about bookmaker accuracy.</p><p>Published by <strong>Licensed Bets</strong>, an affiliate-funded publisher. There are no bookmaker affiliate links in this tool. For context on odds-data freshness, see <a href="https://licensedbets.co.za/odds-accuracy/">odds accuracy and freshness</a>. That page is not evidence of probability calibration or validation of this example.</p><p>Related, separate project: <a href="https://github.com/licensedbets-open/accumulator-reality-check">Accumulator Reality Check</a>. It is not the source or provenance of this lab's code.</p><p><strong>18+ only.</strong> Evaluation is not a guarantee of profit or a reason to gamble. Never gamble money you cannot afford to lose. Visit <a href="https://licensedbets.co.za/responsible-gambling/">responsible gambling resources</a> for support.</p></footer>19</main>20<script>21'use strict';22const EPSILON=1e-15;23const EXAMPLE='probability,outcome\n0.05,0\n0.10,0\n0.18,1\n0.25,0\n0.32,0\n0.40,1\n0.48,0\n0.52,1\n0.60,0\n0.68,1\n0.75,1\n0.82,0\n0.90,1\n0.95,1\n0,0\n1,1';24function parseCSV(text){25 if(new TextEncoder().encode(text).length>1000000)throw new Error('Input exceeds 1 MB. Reduce the batch and try again.');26 const lines=text.replace(/^\uFEFF/,'').split(/\r\n|\n|\r/);const rows=[];let first=true;27 const numeric=/^[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?$/;28 for(let i=0;i<lines.length;i++){29 const raw=lines[i].trim();if(!raw)continue;30 const fields=raw.split(',').map(x=>x.trim());31 if(first&&fields.length===2&&fields[0]==='probability'&&fields[1]==='outcome'){first=false;continue;}first=false;32 if(rows.length>=10000)throw new Error('Input exceeds 10,000 data rows. Reduce the batch and try again.');33 let reason='';let p=NaN,y=NaN;34 if(fields.length!==2)reason='Expected exactly two comma-separated fields.';35 else if(!numeric.test(fields[0])||!Number.isFinite(p=Number(fields[0])))reason='Probability must be a finite decimal number.';36 else if(p<0||p>1)reason='Probability must be between 0 and 1 inclusive.';37 else if(!/^[01]$/.test(fields[1]))reason='Outcome must be exactly 0 or 1.';38 else y=Number(fields[1]);39 rows.push({line:i+1,p,y,accepted:!reason,reason,rawP:fields[0]??'',rawY:fields[1]??''});40 }return rows;41}42function rowMetrics(r){const q=Math.min(1-EPSILON,Math.max(EPSILON,r.p));return{brier:(r.p-r.y)**2,loss:r.y===1?-Math.log(q):-Math.log1p(-q),prediction:r.p>=.5?1:0};}43function evaluateRows(rows,n){const accepted=rows.filter(r=>r.accepted);const bins=Array.from({length:n},()=>({count:0,sumP:0,sumY:0}));let brier=0,loss=0,correct=0;for(const r of accepted){const m=rowMetrics(r);brier+=m.brier;loss+=m.loss;correct+=Number(m.prediction===r.y);const b=bins[Math.min(n-1,Math.floor(r.p*n))];b.count++;b.sumP+=r.p;b.sumY+=r.y;}return{accepted,bins,brier:accepted.length?brier/accepted.length:null,loss:accepted.length?loss/accepted.length:null,accuracy:accepted.length?correct/accepted.length:null};}44const el=id=>document.getElementById(id);let current=null;const fmt=x=>x===null?'-':x.toFixed(6);45function cells(target,values){const tr=document.createElement('tr');for(const v of values){const td=document.createElement('td');td.textContent=v;tr.append(td);}target.append(tr);}46function clearResults(){current=null;for(const id of ['brier','loss','accuracy'])el(id).textContent='-';for(const id of ['binrows','audit','chart'])el(id).replaceChildren();el('download').disabled=true;el('audit-caption').textContent='No current evaluation';}47function render(){clearResults();try{const rows=parseCSV(el('input').value),n=Number(el('bins').value),result=evaluateRows(rows,n);current={rows,n,...result};el('brier').textContent=fmt(result.brier);el('loss').textContent=fmt(result.loss);el('accuracy').textContent=result.accuracy===null?'-':(result.accuracy*100).toFixed(2)+'%';const rejected=rows.length-result.accepted.length;el('status').textContent=`${result.accepted.length} accepted; ${rejected} rejected.`+(result.accepted.length?'':' No metrics available.');el('audit-caption').textContent=`${rows.length} nonblank data rows reviewed`;const frag=document.createDocumentFragment();for(const r of rows)cells(frag,[r.line,r.accepted?'Accepted':'Rejected',r.rawP,r.rawY,r.reason||'Valid']);el('audit').append(frag);result.bins.forEach((b,i)=>cells(el('binrows'),[`[${(i/n).toFixed(2)}, ${((i+1)/n).toFixed(2)}${i===n-1?']':')'}`,b.count,b.count?fmt(b.sumP/b.count):'-',b.count?fmt(b.sumY/b.count):'-']));draw(result.bins);el('download').disabled=!result.accepted.length;}catch(e){el('status').textContent=e.message;}}48function draw(bins){const ns='http://www.w3.org/2000/svg';const svg=document.createElementNS(ns,'svg');svg.setAttribute('viewBox','0 0 600 340');svg.setAttribute('role','img');svg.setAttribute('aria-label','Reliability plot. Full numeric values are in the table below.');function add(tag,attrs,text){const node=document.createElementNS(ns,tag);Object.entries(attrs).forEach(([k,v])=>node.setAttribute(k,v));if(text)node.textContent=text;svg.append(node);return node;}for(let i=0;i<=4;i++){const v=i/4;add('line',{x1:65,y1:280-v*240,x2:555,y2:280-v*240,stroke:'#34465f'});add('text',{x:50,y:285-v*240,fill:'#b4c5db','text-anchor':'end','font-size':13},String(v));add('text',{x:65+v*490,y:302,fill:'#b4c5db','text-anchor':'middle','font-size':13},String(v));}add('line',{x1:65,y1:280,x2:555,y2:40,stroke:'#b4c5db','stroke-dasharray':'6 6'});for(const b of bins){if(!b.count)continue;const point=add('circle',{cx:65+b.sumP/b.count*490,cy:280-b.sumY/b.count*240,r:6,fill:'#8ae4ca',stroke:'#0c1422','stroke-width':2});const title=document.createElementNS(ns,'title');title.textContent=`n=${b.count}, mean p=${fmt(b.sumP/b.count)}, outcome rate=${fmt(b.sumY/b.count)}`;point.append(title);}add('text',{x:310,y:333,fill:'#edf3fa','text-anchor':'middle','font-size':14},'Mean predicted probability');add('text',{x:65,y:22,fill:'#edf3fa','font-size':14},'Observed outcome rate');el('chart').append(svg);}49el('evaluate').addEventListener('click',render);el('reset').addEventListener('click',()=>{el('input').value=EXAMPLE;el('bins').value='10';render();});el('input').addEventListener('input',()=>{clearResults();el('status').textContent='Input changed. Evaluate to see updated results.';});el('bins').addEventListener('change',()=>{if(current)render();});el('download').addEventListener('click',()=>{if(!current?.accepted.length)return;const lines=['line,probability,outcome,predicted_class,brier,log_loss,log_loss_epsilon,bin_index,bin_count'];for(const r of current.accepted){const m=rowMetrics(r);lines.push([r.line,r.p,r.y,m.prediction,m.brier,m.loss,EPSILON,Math.min(current.n-1,Math.floor(r.p*current.n))+1,current.n].join(','));}const url=URL.createObjectURL(new Blob([lines.join('\r\n')+'\r\n'],{type:'text/csv;charset=utf-8'}));const a=document.createElement('a');a.href=url;a.download='calibration-accepted-evaluation.csv';a.click();setTimeout(()=>URL.revokeObjectURL(url),1000);});el('input').value=EXAMPLE;render();50</script></body></html>51