lerobot/robot-learning-tutorial

1<div class="d3-evals-tpbug"></div>2<style>3  .d3-evals-tpbug { position: relative; }4  .d3-evals-tpbug .controls {5    margin-top: 0;6    display: flex;7    gap: 16px;8    align-items: center;9    justify-content: flex-end;10    width: auto;11    flex-wrap: wrap;12  }13  .d3-evals-tpbug .controls label {14    font-size: 12px;15    color: var(--text-color);16    display: flex;17    align-items: center;18    gap: 6px;19    white-space: nowrap;20    font-weight: 700;21  }22  .d3-evals-tpbug .controls select {23    font-size: 12px;24    padding: 8px 28px 8px 10px;25    border: 1px solid var(--border-color);26    border-radius: 8px;27    background-color: var(--surface-bg);28    color: var(--text-color);29    background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' viewBox='0 0 24 24' fill='none' stroke='%230f1115' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpolyline points='6 9 12 15 18 9'/%3E%3C/svg%3E");30    background-repeat: no-repeat; background-position: right 8px center; background-size: 12px;31    -webkit-appearance: none; appearance: none; cursor: pointer; transition: border-color .15s ease, box-shadow .15s ease;32  }33  [data-theme="dark"] .d3-evals-tpbug .controls select {34    background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='12' height='12' viewBox='0 0 24 24' fill='none' stroke='%23ffffff' stroke-width='2' stroke-linecap='round' stroke-linejoin='round'%3E%3Cpolyline points='6 9 12 15 18 9'/%3E%3C/svg%3E");35  }36  .d3-evals-tpbug .controls select:hover { border-color: var(--primary-color); }37  .d3-evals-tpbug .controls select:focus { border-color: var(--primary-color); box-shadow: 0 0 0 3px rgba(232,137,171,.25); outline: none; }38  .d3-evals-tpbug .axis-label { fill: var(--text-color); font-size: 12px; font-weight: 700; }39  .d3-evals-tpbug .axes path, .d3-evals-tpbug .axes line { stroke: var(--axis-color); }40  .d3-evals-tpbug .axes text { fill: var(--tick-color); }41  .d3-evals-tpbug .grid line { stroke: var(--grid-color); }42  .d3-evals-tpbug .legend { font-size: 12px; color: var(--text-color);padding-left: 6px; }43  .d3-evals-tpbug .legend .items { display:flex; flex-wrap:wrap; gap:8px 12px; align-items:center; }44  .d3-evals-tpbug .legend .item { display:flex; align-items:center; gap:6px; white-space:nowrap; }45  .d3-evals-tpbug .legend .swatch { width:14px; height:14px; border-radius:3px; border:1px solid var(--border-color); display:inline-block; }46  /* Ghosting on hover */47  .d3-evals-tpbug.hovering .legend-bottom .item.ghost { opacity: .35; }48  .d3-evals-tpbug.hovering .lines path.ghost { opacity: .25; }49  .d3-evals-tpbug.hovering .points circle.ghost { opacity: .25; }50  .d3-evals-tpbug.hovering .areas path.ghost { opacity: .08; }51  .d3-evals-tpbug .chart-header { display:flex; align-items:center; justify-content:space-between; gap:12px; margin: 0 0 8px 0; flex-wrap: wrap; }52  .d3-evals-tpbug .legend-bottom { display:flex; align-items:center; justify-content:flex-start; font-size:12px; color: var(--text-color); }53  .d3-evals-tpbug .legend-bottom .items { display:flex; flex-wrap:wrap; gap:8px 14px; }54  .d3-evals-tpbug .legend-bottom .item { display:inline-flex; align-items:center; gap:6px; white-space:nowrap; }55  .d3-evals-tpbug .legend-bottom .swatch { width:14px; height:14px; border-radius:3px; border:1px solid var(--border-color); display:inline-block; }56  .d3-evals-tpbug .lines path.active { stroke-width: 3; }57  /* Layout tweaks: stack label above select, add legend title above items */58  .d3-evals-tpbug .controls .control-group {59    display: flex;60    flex-direction: column;61    align-items: flex-start;62    gap: 6px;63  }64  .d3-evals-tpbug .legend-bottom {65    flex-direction: column;66    align-items: flex-start;67    gap: 6px;68  }69  .d3-evals-tpbug .legend-bottom .legend-title {70    font-size: 12px;71    font-weight: 700;72    color: var(--text-color);73  }74  /* Tooltip styling aligned with filters-quad */75  .d3-evals-tpbug .d3-tooltip { z-index: var(--z-elevated); backdrop-filter: saturate(1.12) blur(8px); }76  .d3-evals-tpbug .d3-tooltip__inner { display:flex; flex-direction:column; gap:6px; min-width: 220px; }77  .d3-evals-tpbug .d3-tooltip__inner > div:first-child { font-weight: 800; letter-spacing: 0.1px; margin-bottom: 0; }78  .d3-evals-tpbug .d3-tooltip__inner > div:nth-child(2) { font-size: 11px; color: var(--muted-color); display: block; margin-top: -4px; margin-bottom: 2px; letter-spacing: 0.1px; }79  .d3-evals-tpbug .d3-tooltip__inner > div:nth-child(n+3) { padding-top: 6px; border-top: 1px solid var(--border-color); }80  .d3-evals-tpbug .d3-tooltip__color-dot { display:inline-block; width: 12px; height: 12px; border-radius: 3px; border: 1px solid var(--border-color); }81  /* Chart card only around the SVG */82  .d3-evals-tpbug .chart-card { background: var(--surface-bg); border: 1px solid var(--border-color); border-radius: 10px; padding: 8px; }83  /* Place header below chart and align start */84  .d3-evals-tpbug .chart-header { display:flex; align-items:flex-start; justify-content:flex-start; gap:12px; margin: 8px 0 0 0; flex-wrap: wrap; }85</style>86<script>87  (() => {88    // Pretty label mapping for metric keys89    const prettyMetricLabel = (key) => {90      if (!key) return '';91      const table = {92        'hellaswag': 'HellaSwag',93        'mmlu': 'MMLU',94        'arc': 'ARC',95        'truthfulqa': 'TruthfulQA',96        'gsm8k': 'GSM8K',97        'winogrande': 'WinoGrande',98        'openbookqa': 'OpenBookQA',99        'piqa': 'PIQA',100        'race': 'RACE',101        'boolq': 'BoolQ',102        'cb': 'CB',103        'copa': 'COPA',104        'multirc': 'MultiRC',105        'record': 'ReCoRD',106        'rte': 'RTE',107        'wic': 'WiC',108        'wsc': 'WSC'109      };110      if (table[key]) return table[key];111      const cleaned = String(key).replace(/[_-]+/g, ' ').trim();112      return cleaned.split(/\s+/).map(w => {113        if (/^(mmlu|arc|gsm8k|piqa|race|boolq|multirc|record|wsc)$/i.test(w)) return w.toUpperCase();114        return w.charAt(0).toUpperCase() + w.slice(1);115      }).join(' ');116    };117 118    const ensureD3 = (cb) => {119      if (window.d3 && typeof window.d3.select === 'function') return cb();120      let s = document.getElementById('d3-cdn-script');121      if (!s) { s = document.createElement('script'); s.id = 'd3-cdn-script'; s.src = 'https://cdn.jsdelivr.net/npm/d3@7/dist/d3.min.js'; document.head.appendChild(s); }122      const onReady = () => { if (window.d3 && typeof window.d3.select === 'function') cb(); };123      s.addEventListener('load', onReady, { once: true }); if (window.d3) onReady();124    };125 126    const bootstrap = () => {127      const scriptEl = document.currentScript;128      let container = scriptEl ? scriptEl.previousElementSibling : null;129      if (!(container && container.classList && container.classList.contains('d3-evals-tpbug'))){130        const cs = Array.from(document.querySelectorAll('.d3-evals-tpbug')).filter(el => !(el.dataset && el.dataset.mounted === 'true'));131        container = cs[cs.length - 1] || null;132      }133      if (!container) return;134      if (container.dataset) { if (container.dataset.mounted === 'true') return; container.dataset.mounted = 'true'; }135 136      // Controls (we will place them in a footer below the chart)137      const controls = document.createElement('div');138      controls.className = 'controls';139      const controlGroup = document.createElement('div');140      controlGroup.className = 'control-group';141      const labelMetric = document.createElement('label');142      labelMetric.textContent = 'Metric';143      const selectMetric = document.createElement('select');144      // Associate label and select with a unique id145      const uniqueId = Math.random().toString(36).slice(2, 9);146      selectMetric.id = `metric-select-${uniqueId}`;147      labelMetric.setAttribute('for', selectMetric.id);148      controlGroup.appendChild(labelMetric);149      controlGroup.appendChild(selectMetric);150      controls.appendChild(controlGroup);151 152      // Tooltip153      container.style.position = container.style.position || 'relative';154      let tip = container.querySelector('.d3-tooltip'); let tipInner;155      if (!tip) {156        tip = document.createElement('div'); tip.className = 'd3-tooltip';157        Object.assign(tip.style, {158          position:'absolute', top:'0px', left:'0px', transform:'translate(-9999px, -9999px)', pointerEvents:'none',159          padding:'8px 10px', borderRadius:'8px', fontSize:'12px', lineHeight:'1.35', border:'1px solid var(--border-color)',160          background:'var(--surface-bg)', color:'var(--text-color)', boxShadow:'0 4px 24px rgba(0,0,0,.18)', opacity:'0', transition:'opacity .12s ease'161        });162        tipInner = document.createElement('div'); tipInner.className = 'd3-tooltip__inner'; tipInner.style.textAlign='left'; tip.appendChild(tipInner); container.appendChild(tip);163      } else { tipInner = tip.querySelector('.d3-tooltip__inner') || tip; }164 165      // Header (legend + controls) placed after the chart166      const header = document.createElement('div'); header.className = 'chart-header';167      const legendBottom = document.createElement('div'); legendBottom.className = 'legend-bottom'; header.appendChild(legendBottom);168      header.appendChild(controls);169 170      // Chart card (SVG)171      const card = document.createElement('div'); card.className = 'chart-card'; container.appendChild(card);172      container.appendChild(header);173      // SVG174      const svg = d3.select(card).append('svg').attr('width','100%').style('display','block');175      const gRoot = svg.append('g');176      const gGrid = gRoot.append('g').attr('class','grid');177      const gAxes = gRoot.append('g').attr('class','axes');178      const gAreas = gRoot.append('g').attr('class','areas');179      const gLines = gRoot.append('g').attr('class','lines');180      const gPoints = gRoot.append('g').attr('class','points');181      // (legend removed from inside SVG)182      const overlay = gRoot.append('rect').attr('fill','transparent').style('cursor','crosshair');183      const hoverLine = gRoot.append('line').attr('stroke-width',1).style('display','none');184 185      // State/data186      let width = 800, height = 480; const margin = { top: 16, right: 32, bottom: 44, left: 56 };187      const xScale = d3.scaleLinear();188      const yScale = d3.scaleLinear();189      const lineGen = d3.line().x(d => xScale(d.tokens)).y(d => yScale(d.value));190      const dataByMetric = new Map();191      let runOrder = [];192 193      // Colors194      function getRunColors(count){195        try { if (window.ColorPalettes && typeof window.ColorPalettes.getColors === 'function') return window.ColorPalettes.getColors('categorical', count); } catch(_){}196        return d3.schemeTableau10 ? d3.schemeTableau10.slice(0, count) : ['#4e79a7','#f28e2b','#e15759','#76b7b2','#59a14f','#edc948','#b07aa1','#ff9da7','#9c755f','#bab0ab'].slice(0, count);197      }198      199      // Format helper for tokens (convert to billions)200      function formatTokens(v){201        const billions = v / 1e9;202        return d3.format('.0f')(billions) + 'B';203      }204 205      function updateLayout(){206        const axisColor = getComputedStyle(container).getPropertyValue('--axis-color').trim() || 'rgba(0,0,0,0.25)';207        width = container.clientWidth || 800;208        height = Math.max(280, Math.round(width / 3));209        svg.attr('width', width).attr('height', height);210        gRoot.attr('transform', `translate(${margin.left},${margin.top})`);211        const innerWidth = width - margin.left - margin.right;212        const innerHeight = height - margin.top - margin.bottom;213        overlay.attr('x',0).attr('y',0).attr('width', innerWidth).attr('height', innerHeight);214        hoverLine.attr('y1',0).attr('y2', innerHeight).attr('stroke', axisColor);215        return { innerWidth, innerHeight };216      }217 218      function render(metricKey){219        const { innerWidth, innerHeight } = updateLayout();220        const map = dataByMetric.get(metricKey) || {};221        const runs = runOrder;222        // domains223        let minTokens = Infinity, maxTokens = -Infinity, minV = Infinity, maxV = -Infinity;224        runs.forEach(r => { (map[r]||[]).forEach(pt => { minTokens = Math.min(minTokens, pt.tokens); maxTokens = Math.max(maxTokens, pt.tokens); minV = Math.min(minV, pt.value); maxV = Math.max(maxV, pt.value); }); });225        if (!isFinite(minTokens) || !isFinite(maxTokens)) return;226        xScale.domain([minTokens, maxTokens]).range([0, innerWidth]);227        yScale.domain([minV, maxV]).nice().range([innerHeight, 0]);228 229        // grid230        gGrid.selectAll('*').remove();231        gGrid.selectAll('line').data(yScale.ticks(6)).join('line')232          .attr('x1',0).attr('x2', innerWidth).attr('y1', d=>yScale(d)).attr('y2', d=>yScale(d))233          .attr('stroke','var(--grid-color)').attr('stroke-width',1).attr('shape-rendering','crispEdges');234 235        // axes236        gAxes.selectAll('*').remove();237        gAxes.append('g').attr('transform', `translate(0,${innerHeight})`).call(d3.axisBottom(xScale).ticks(8).tickFormat(formatTokens)).call(g=>{ g.selectAll('path, line').attr('stroke','var(--axis-color)'); g.selectAll('text').attr('fill','var(--tick-color)').style('font-size','12px'); });238        gAxes.append('g').call(d3.axisLeft(yScale).ticks(6)).call(g=>{ g.selectAll('path, line').attr('stroke','var(--axis-color)'); g.selectAll('text').attr('fill','var(--tick-color)').style('font-size','12px'); });239        gAxes.append('text').attr('class','axis-label').attr('text-anchor','middle').attr('x', innerWidth/2).attr('y', innerHeight + 38).text('Tokens (B)');240        gAxes.append('text').attr('class','axis-label').attr('text-anchor','middle').attr('transform', `translate(${-44}, ${innerHeight/2}) rotate(-90)`).text('Score');241 242        // lines243        const series = runs.map((r, i) => ({ run:r, color: getRunColors(runs.length)[i % getRunColors(runs.length).length], values: (map[r]||[]).slice().sort((a,b)=>a.tokens-b.tokens) }));244        const paths = gLines.selectAll('path.run').data(series, d=>d.run);245        const pathsEnter = paths.enter().append('path').attr('class','run').attr('fill','none').attr('stroke-width',2).attr('stroke', d=>d.color).attr('d', d=>lineGen(d.values));246        pathsEnter.merge(paths).transition().duration(200).attr('stroke', d=>d.color).attr('d', d=>lineGen(d.values));247        paths.exit().remove();248 249        // Hover capture paths (wider invisible stroke for easy hover)250        const captures = gLines.selectAll('path.run-hover').data(series, d=>`cap-${d.run}`);251        captures.enter().append('path').attr('class','run-hover').attr('fill','none').attr('stroke','transparent').attr('stroke-width', 12).style('pointer-events','stroke')252          .attr('d', d=>lineGen(d.values))253          .merge(captures)254          .attr('d', d=>lineGen(d.values))255          .on('mouseenter', function(ev, d){256            container.classList.add('hovering');257            // ghost non hovered lines and points258            gLines.selectAll('path.run').classed('ghost', s => s.run !== d.run);259            gPoints.selectAll('circle.pt').classed('ghost', p => p.run !== d.run);260            // ghost legend items261            try {262              const legendNode = legendBottom;263              if (legendNode) {264                legendNode.querySelectorAll('.item').forEach(el => {265                  const name = el.getAttribute('data-run');266                  el.classList.toggle('ghost', name !== d.run);267                });268              }269            } catch {}270          })271          .on('mouseleave', function(){272            container.classList.remove('hovering');273            gLines.selectAll('path.run').classed('ghost', false);274            gPoints.selectAll('circle.pt').classed('ghost', false);275            try { const legendNode = legendBottom; if (legendNode) legendNode.querySelectorAll('.item').forEach(el => el.classList.remove('ghost')); } catch {}276          });277        captures.exit().remove();278 279        // point markers (subtle)280        const allPts = series.flatMap(s => s.values.map(v => ({ run:s.run, color:s.color, tokens:v.tokens, value:v.value })));281        const ptsSel = gPoints.selectAll('circle.pt').data(allPts, d=>`${d.run}-${d.tokens}`);282        ptsSel.enter().append('circle').attr('class','pt').attr('r', 2).attr('fill', d=>d.color).attr('fill-opacity', 0.6)283          .attr('cx', d=>xScale(d.tokens)).attr('cy', d=>yScale(d.value))284          .merge(ptsSel).transition().duration(150).attr('cx', d=>xScale(d.tokens)).attr('cy', d=>yScale(d.value));285        ptsSel.exit().remove();286 287        // legend (HTML below, left) with title above items288        legendBottom.innerHTML = `<div class="legend-title">Legend</div><div class="items">${series.map(s => `<span class="item" data-run="${s.run}"><span class="swatch" style="background:${s.color}"></span><span>${s.run}</span></span>`).join('')}</div>`;289        // Legend hover → ghost lines/points290        try {291          const legendNode = legendBottom;292          legendNode.querySelectorAll('.item').forEach(el => {293            el.addEventListener('mouseenter', () => {294              const run = el.getAttribute('data-run'); if (!run) return;295              container.classList.add('hovering');296              gLines.selectAll('path.run').classed('ghost', s => s.run !== run);297              gPoints.selectAll('circle.pt').classed('ghost', p => p.run !== run);298              legendNode.querySelectorAll('.item').forEach(it => it.classList.toggle('ghost', it.getAttribute('data-run') !== run));299            });300            el.addEventListener('mouseleave', () => {301              container.classList.remove('hovering');302              gLines.selectAll('path.run').classed('ghost', false);303              gPoints.selectAll('circle.pt').classed('ghost', false);304              legendNode.querySelectorAll('.item').forEach(it => it.classList.remove('ghost'));305            });306          });307        } catch {}308 309        // hover310        function onMove(ev){311          const [mx, my] = d3.pointer(ev, overlay.node());312          const sx = xScale.invert(mx);313          // nearest token value314          const tokens = Array.from(new Set(allPts.map(p=>p.tokens))).sort((a,b)=>a-b);315          const nearest = tokens.reduce((best, s) => Math.abs(s - sx) < Math.abs(best - sx) ? s : best, tokens[0]);316          const xpx = xScale(nearest);317          hoverLine.style('display', null).attr('x1', xpx).attr('x2', xpx);318          // tooltip content (styled)319          let html = `<div style=\"font-weight:800;letter-spacing:.1px;\">${prettyMetricLabel(metricKey)}</div><div style=\"font-size:11px;color:var(--muted-color);margin-top:-4px;margin-bottom:2px;\">${formatTokens(nearest)} tokens</div>`;320          const entries = series.map(s => {321            const m = new Map(s.values.map(v=>[v.tokens, v.value]));322            const val = m.get(nearest);323            return { run: s.run, color: s.color, val };324          }).filter(e => e.val != null);325          entries.sort((a, b) => a.val - b.val);326          entries.forEach(e => {327            html += `<div style=\"display:flex;align-items:center;gap:6px;white-space:nowrap;\"><span class=\"d3-tooltip__color-dot\" style=\"background:${e.color}\"></span><strong>${e.run}</strong><span style=\"margin-left:auto;\">${(+e.val).toFixed(4)}</span></div>`;328          });329          tipInner.innerHTML = html; tip.style.opacity = '1'; tip.style.transform = `translate(${Math.round(mx + margin.left + 12)}px, ${Math.round(my + margin.top + 12)}px)`;330        }331        function onLeave(){ tip.style.opacity='0'; tip.style.transform='translate(-9999px, -9999px)'; hoverLine.style('display','none'); }332        overlay.on('mousemove', onMove).on('mouseleave', onLeave);333      }334 335      // load CSV and init336      (async () => {337        try {338          // Try multiple possible paths for the CSV file339          const csvPaths = [340            '/data/evals_no_fixed_200B.csv',341            './assets/data/evals_no_fixed_200B.csv',342            '../assets/data/evals_no_fixed_200B.csv',343            '../../assets/data/evals_no_fixed_200B.csv'344          ];345          346          let csvText = null;347          for (const path of csvPaths) {348            try {349              const response = await fetch(path, { cache: 'no-cache' });350              if (response.ok) {351                csvText = await response.text();352                break;353              }354            } catch(_) {}355          }356          357          if (!csvText) {358            throw new Error('CSV file not found: evals_no_fixed_200B.csv');359          }360          const rows = d3.csvParse(csvText, d => ({361            run: (d.run_name||'').trim(),362            tokens: +d.tokens,363            metric: (d.metric||'').trim(),364            value: +d.value365          }));366          const metrics = Array.from(new Set(rows.map(r=>r.metric))).sort();367          runOrder = Array.from(new Set(rows.map(r=>r.run))).sort();368          metrics.forEach(m => {369            const map = {}; runOrder.forEach(r => map[r] = []);370            rows.filter(r=>r.metric===m).forEach(r => {371              if (!isNaN(r.tokens) && !isNaN(r.value)) map[r.run].push({ tokens:r.tokens, value:r.value });372            });373            dataByMetric.set(m, map);374          });375          // populate metric select (pretty labels) or hide if single-file with single metric376          const isSingleFile = true; // This is a single file377          metrics.forEach(m => { const o = document.createElement('option'); o.value=m; o.textContent=prettyMetricLabel(m); selectMetric.appendChild(o); });378          // default metric selection to hellaswag379          if (metrics.length) {380            let initial = metrics.find(m => m === 'hellaswag') || metrics[0];381            selectMetric.value = initial;382          }383          if (isSingleFile && metrics.length <= 1) {384            controls.style.display = 'none';385          }386 387          render(selectMetric.value);388          selectMetric.addEventListener('change', () => render(selectMetric.value));389          const rerender = () => render(selectMetric.value);390          if (window.ResizeObserver) { const ro = new ResizeObserver(() => rerender()); ro.observe(container); } else { window.addEventListener('resize', rerender); }391        } catch (e) {392          const pre = document.createElement('pre'); pre.textContent = 'CSV load error: ' + (e && e.message ? e.message : e);393          pre.style.color = 'var(--danger, #b00020)'; pre.style.fontSize = '12px'; pre.style.whiteSpace = 'pre-wrap'; container.appendChild(pre);394        }395      })();396    };397 398    if (document.readyState === 'loading') { document.addEventListener('DOMContentLoaded', () => ensureD3(bootstrap), { once: true }); } else { ensureD3(bootstrap); }399  })();400</script>401