msulemans/open-model-training-lab
0
1const models = [2 {3 id: "base",4 tab: "Untouched Qwen",5 kicker: "Control · no training",6 name: "The baseline",7 story: "We measured the original model before touching any weights. This gives every later result a trustworthy reference point.",8 decision: "Keep as the scientific control.",9 accuracy: 47.8571,10 f1: 46.8243,11 invalid: 176,12 facts: [["Correct", "1,474 / 3,080"], ["Trainable values", "0"], ["Peak memory", "4.773 GB"], ["Role", "Control"]],13 },14 {15 id: "004c",16 tab: "004c · q/v",17 kicker: "Full data · stable LoRA",18 name: "The efficient adapter",19 story: "One epoch over 9,233 records with LoRA on q and v projections. Stable, compact, and our first statistically supported improvement over the base model.",20 decision: "Retain as the smaller, format-steady candidate.",21 accuracy: 49.4156,22 f1: 49.7983,23 invalid: 159,24 facts: [["Correct", "1,522 / 3,080"], ["Trainable values", "917,504"], ["Learning rate", "5e-7"], ["Adapter", "3.68 MB"]],25 },26 {27 id: "005b",28 tab: "005b · q/k/v/o",29 kicker: "Larger adapter · safer LR",30 name: "The accuracy leader",31 story: "We doubled the adapter capacity by targeting q, k, v, and o. The original learning rate became unstable, so the stable retry used half the rate.",32 decision: "Best unconstrained full-test accuracy observed.",33 accuracy: 50.3896,34 f1: 49.9911,35 invalid: 173,36 facts: [["Correct", "1,552 / 3,080"], ["Trainable values", "1,835,008"], ["Learning rate", "2.5e-7"], ["Adapter", "7.35 MB"]],37 },38 {39 id: "constrained",40 tab: "005b + guardrails",41 kicker: "Same weights · controlled decoding",42 name: "The canonical decoder",43 story: "No retraining. At each token, invalid label paths are masked. The model must finish as one of the 77 published labels.",44 decision: "Confirmed default: best matched accuracy and zero invalid labels.",45 accuracy: 51.1039,46 f1: 49.6203,47 invalid: 0,48 facts: [["Correct", "1,574 / 3,080"], ["Invalid labels", "0"], ["Training", "None added"], ["Inference batch", "1"]],49 },50 {51 id: "bert006",52 tab: "BERT · 006",53 kicker: "Encoder pivot · full fine-tune",54 name: "The architecture breakthrough",55 scope: "Validation set · 770 questions",56 story: "BERT-Large reads the whole request and sends one representation into a 77-class head. Matching the architecture to classification moved us from roughly 51% to 91.56%.",57 decision: "Retained after class weighting, label smoothing, and checkpoint ensembling all failed to improve it.",58 accuracy: 91.5584,59 f1: 91.4837,60 invalid: 0,61 benchmark: 91.5584,62 accuracyNote: "705 of 770 validation questions correct.",63 invalidNote: "A 77-class head always returns one of 77 positions.",64 facts: [["Training", "Full model"], ["Selected epoch", "4 of 5"], ["Peak memory", "10.321 GiB probe"], ["Test use", "None"]],65 },66 {67 id: "deberta009d",68 tab: "DeBERTa · 009d",69 kicker: "Stronger encoder · float32",70 name: "The stable DeBERTa benchmark",71 scope: "Validation selected · test reported once",72 story: "DeBERTa-v3-large improved the encoder design, but its half-precision checkpoint became NaN on MPS. Casting every parameter to float32 made five-epoch full fine-tuning stable.",73 decision: "Promoted over BERT using validation only; its one-time test report reached 94.03%.",74 accuracy: 92.5974,75 f1: 92.6117,76 invalid: 0,77 benchmark: 91.5584,78 accuracyNote: "713 of 770 validation questions correct; 94.03% on the fixed test report.",79 invalidNote: "Classification head: no free-form label generation.",80 facts: [["Training", "Full model"], ["Precision", "float32"], ["Selected epoch", "5 of 5"], ["Test report", "94.026%"]],81 },82 {83 id: "deberta011",84 tab: "DeBERTa · 011",85 kicker: "Upper-layer refinement · champion",86 name: "The current validated champion",87 scope: "Validation champion · test is locked",88 story: "Starting from Exp010, we froze layers 0–19 and trained only layers 20–23 plus the classification components. This made one additional validation answer correct without harming another.",89 decision: "Current champion: 92.99% validation and a reporting-only 94.12% test result.",90 accuracy: 92.9870,91 f1: 92.9701,92 invalid: 0,93 benchmark: 92.5974,94 accuracyNote: "716 of 770 validation questions correct.",95 invalidNote: "The final test is locked and cannot choose later experiments.",96 facts: [["Trainable", "51.5M"], ["Frozen", "383.6M"], ["Validation", "92.987%"], ["Test report", "94.123%"]],97 },98 {99 id: "deberta015",100 tab: "Exp015 · rejected",101 kicker: "Data-quality experiment · completed",102 name: "The over-pruned retraining",103 scope: "Validation result · test never loaded",104 story: "A five-fold out-of-fold audit identified suspicious train labels. Exp015 removed 1,078 high-confidence disagreements, kept every original label, and retrained DeBERTa from its original checkpoint for five stable epochs.",105 decision: "Rejected: 90.78% validation did not beat Exp011's fixed 92.99%. Exp011 remains champion and the test stays locked.",106 accuracy: 90.7792,107 f1: 90.7391,108 invalid: 0,109 benchmark: 92.9870,110 accuracyNote: "699 of 770 correct—17 fewer than Exp011's 716.",111 invalidNote: "No rows were relabelled and no test rows were loaded.",112 facts: [["Removed", "1,078 rows"], ["Best epoch", "5 of 5"], ["Validation", "90.779%"], ["Decision", "Rejected"]],113 },114];115 116const stages = [117 {118 title: "Verify the machine",119 tag: "Environment",120 summary: "Before training anything, we proved that the Mac, Python environment, MLX, and Metal GPU path were real and reproducible.",121 did: "Created an isolated Python 3.11.9 environment, installed pinned MLX packages, and multiplied two matrices on the M2 Max GPU.",122 why: "If the compute stack is wrong, every later failure becomes ambiguous. Environment verification removes that uncertainty first.",123 command: ".venv/bin/python scripts/verify_mlx.py",124 lesson: "A successful import is not a GPU test. Perform an actual calculation and inspect the selected device.",125 },126 {127 title: "Lock trustworthy data",128 tag: "Dataset",129 summary: "We rejected an incomplete dataset mirror and downloaded the commit-pinned original BANKING77 files.",130 did: "Verified 10,003 train rows, 3,080 test rows, 77 labels, zero text overlap, and file checksums.",131 why: "Training on a drifting or incomplete mirror would make our results impossible to reproduce or compare.",132 command: ".venv/bin/python scripts/inspect_banking77.py",133 lesson: "A familiar dataset name is not enough. Record the source revision, counts, schema, and checksums.",134 },135 {136 title: "Measure before training",137 tag: "Baseline",138 summary: "The untouched model completed the full 3,080-question test before we changed any weights.",139 did: "Recorded 47.8571% accuracy, 0.468243 macro F1, and 176 invalid labels.",140 why: "Without a baseline, a low training loss can feel successful even when the model became worse.",141 command: ".venv/bin/python scripts/evaluate_full_baseline.py",142 lesson: "The baseline is the control group of a machine-learning experiment.",143 },144 {145 title: "Prepare SFT examples",146 tag: "Data engineering",147 summary: "We converted each question and intent into Qwen chat messages and reserved a balanced validation split.",148 did: "Created 9,233 training and 770 validation records, then inspected every tokenized length.",149 why: "The model trains on tokens, not visible strings. Template boundaries and sequence lengths must be verified before spending compute.",150 command: ".venv/bin/python scripts/inspect_tokenization.py",151 lesson: "Data formatting is part of the model. A correct CSV can still become incorrect training tokens.",152 },153 {154 title: "Prove the pipeline",155 tag: "Experiment 001",156 summary: "A ten-update LoRA run checked the complete pipeline without pretending to be an accuracy experiment.",157 did: "Loaded Qwen, attached LoRA, trained, saved the adapter, reloaded it, and generated a prediction.",158 why: "A cheap pipeline test catches structural failures before a two-hour training run.",159 command: ".venv/bin/python scripts/run_lora_pipeline.py",160 lesson: "Separate ‘does the pipeline work?’ from ‘does the model perform well?’",161 },162 {163 title: "Scale data carefully",164 tag: "Experiments 002 → 003b",165 summary: "We moved from 539 balanced examples to 1,925, keeping evaluation fixed. A high learning rate destroyed one run; a lower rate stayed finite.",166 did: "Diagnosed all 917,504 failed adapter values as non-finite, then retried at 2.5e-6.",167 why: "Changing data size changes the number of optimizer updates. A rate stable for a short run can explode over a longer run.",168 command: ".venv/bin/python scripts/diagnose_exp_003_failure.py",169 lesson: "NaN is evidence. Check when it started, whether memory was exhausted, and whether the saved artifact is usable.",170 },171 {172 title: "Train on all data",173 tag: "Experiment 004c",174 summary: "After a batch-size-7 run failed, batch size 1 and a very small learning rate completed one full epoch stably.",175 did: "Trained 917,504 q/v LoRA values for 9,233 updates and reached 49.4156% full-test accuracy.",176 why: "The full dataset teaches more linguistic variation, but stable optimization matters more than rushing with a large batch.",177 command: ".venv/bin/python scripts/run_exp_004c.py",178 lesson: "A probe validates safety. The full test—not the training log—validates usefulness.",179 },180 {181 title: "Increase adapter capacity",182 tag: "Experiment 005b",183 summary: "Adding k and o projections doubled trainable adapter values. The first full run reached NaN; halving the learning rate completed stably.",184 did: "Trained 1.835M values and reached 50.3896% accuracy—30 more correct answers than 004c.",185 why: "More trainable capacity can represent more task-specific behavior, but also changes optimization stability.",186 command: ".venv/bin/python scripts/run_exp_005b.py",187 lesson: "Our comparison changed both targets and learning rate, so it identifies a better configuration—not the isolated causal effect of k/o.",188 },189 {190 title: "Control the output space",191 tag: "Constrained decoding",192 summary: "Many ‘invalid’ outputs were sensible aliases. We constrained token generation to paths that finish as a canonical BANKING77 label.",193 did: "Reduced invalid labels from 173 to zero without touching model weights. The best observed accuracy became 51.1039%.",194 why: "A production classifier needs a closed output contract, not merely semantically plausible text.",195 command: ".venv/bin/python scripts/evaluate_exp_005b_constrained_full.py",196 lesson: "Inference algorithms are part of the system. Better behavior does not always require more training.",197 },198 {199 title: "Prove the constraint effect",200 tag: "Completed · Step 41",201 summary: "We repeated unconstrained and constrained inference at the same batch size over all 3,080 test records.",202 did: "The constraint preserved every valid prediction, converted 29 invalid outputs into correct labels, and converted the other 152 invalid outputs into valid but still-wrong labels.",203 why: "If two factors change together, we cannot honestly attribute the result to either one.",204 command: ".venv/bin/python scripts/classify_banking_request.py --check",205 lesson: "A controlled A/B test turned an encouraging result into defensible evidence: 51.1039% versus 50.1623%, with an exact paired p-value of 3.73e-9.",206 },207 {208 title: "Deploy and close the loop",209 tag: "Completed · Steps 42–44",210 summary: "We promoted the proven adapter and constraint into both a one-request CLI and a localhost JSON API.",211 did: "Sent a real HTTP request, received cash_withdrawal_charge with the exact adapter checksum and constraint mode, then stopped the server cleanly.",212 why: "An evaluated artifact is not yet a usable system. Deployment checks that preprocessing, model loading, decoding, and the external response contract still agree.",213 command: ".venv/bin/python scripts/serve_banking_classifier.py",214 lesson: "The production unit is the whole inference pipeline—not just the adapter file or its test score.",215 },216 {217 title: "Pivot to an encoder",218 tag: "Experiment 006 · BERT-Large",219 summary: "Qwen taught us the workflow, but generating label text was a poor match for a closed 77-way decision. We moved to a dedicated sequence classifier.",220 did: "Added a new 77-class head to BERT-Large and fine-tuned all model parameters for five epochs on MPS. The best validation checkpoint reached 91.5584%.",221 why: "Architecture fit can matter more than repeatedly tuning an ill-suited model. BERT directly scores the 77 choices instead of composing an answer token by token.",222 command: ".venv-encoder/bin/python scripts/run_exp_006.py",223 lesson: "Switching architecture was itself a learned conclusion from the Qwen experiment—not random model hopping.",224 },225 {226 title: "Reject plausible BERT refinements",227 tag: "Experiments 007–008",228 summary: "Class-weighted loss and label smoothing sounded reasonable, but both reduced validation accuracy.",229 did: "Compared each child against the unchanged Exp006 parent, counted repaired and harmed answers, and rejected both children.",230 why: "A mechanism can be theoretically sensible and still fail on real data. Promotion rules protect the champion from wishful thinking.",231 command: ".venv-encoder/bin/python scripts/analyze_exp_008_validation.py",232 lesson: "A failed controlled experiment is useful knowledge when its hypothesis and rejection rule were written first.",233 },234 {235 title: "Diagnose DeBERTa NaNs",236 tag: "Experiments 009a–009d",237 summary: "DeBERTa produced finite forward losses, but parameters became non-finite immediately after the first MPS optimizer update.",238 did: "Lowered the learning rate, instrumented microbatches, froze suspected tensors, inspected checkpoint dtypes, then cast the full model from float16 to float32.",239 why: "The failure was numerical precision—not bad labels or insufficient memory. float32 uses more memory but gives safer update range and precision.",240 command: ".venv-encoder/bin/python scripts/run_exp_009d_probe.py",241 lesson: "Locate the first bad operation before changing several settings. The float32 probe turned a guess into a diagnosis.",242 },243 {244 title: "Train the stronger encoder",245 tag: "Experiment 009d",246 summary: "Five stable float32 epochs made DeBERTa the validation champion and produced a 94.026% reporting-only test result.",247 did: "Selected epoch 5 using validation accuracy, recovered intact checkpoints after a metadata KeyError, verified save/reload, and only then opened the sealed test once.",248 why: "Checkpoint selection and final testing answer different questions. Validation chooses; test reports generalization after choices are finished.",249 command: ".venv-encoder/bin/python scripts/finalize_exp_009d.py",250 lesson: "A wrapper crash after training does not mean the model failed. Inspect durable artifacts before rerunning expensive work.",251 },252 {253 title: "Refine without touching test",254 tag: "Experiments 010–011",255 summary: "Train-only oversampling improved validation, then upper-layer-only training added one more correct answer.",256 did: "Exp010 reached 92.8571% validation. Exp011 froze layers 0–19, trained 51.5M upper-layer parameters, and reached 92.9870%.",257 why: "Small, controlled continuation can preserve learned language features while gently changing the decision boundary.",258 command: ".venv-encoder/bin/python scripts/run_exp_011.py",259 lesson: "The 94.123% test result is a report, not permission to tune against test errors. Exp011 remains champion by validation evidence.",260 },261 {262 title: "Audit label quality",263 tag: "Experiment 012",264 summary: "A five-fold out-of-fold classifier examined every training row without judging a row using a model that trained on it.",265 did: "Found 1,186 disagreements among 9,233 train rows. No validation or test rows were loaded, and no labels were silently rewritten.",266 why: "When the model is confidently wrong, either the decision boundary is weak or the supplied label may be noisy. Both deserve investigation.",267 command: ".venv-encoder/bin/python scripts/analyze_exp_011_errors.py",268 lesson: "Out-of-fold predictions are an audit signal, not ground truth. Disagreement alone is not enough to relabel data.",269 },270 {271 title: "Test hard-negative learning",272 tag: "Experiments 013–014 · rejected",273 summary: "We explicitly pushed true labels above their strongest rival for 162, then 282, ambiguous training rows. Both attempts lost one validation answer.",274 did: "Kept original labels, added a pairwise margin loss, measured the exact changed prediction, and rejected both children.",275 why: "More targeted loss does not guarantee better generalization—especially when many candidate rows are genuinely ambiguous.",276 command: ".venv-encoder/bin/python scripts/analyze_exp_013_validation.py",277 lesson: "Do not promote a complicated method merely because it sounds advanced. It must beat the simple champion on unchanged validation data.",278 },279 {280 title: "Test data-quality pruning",281 tag: "Experiment 015 · rejected",282 summary: "We removed only strong train-only noise candidates and retrained DeBERTa from the same original checkpoint. The run was stable, but accuracy fell.",283 did: "Removed 1,078 rows where the out-of-fold model disagreed and assigned the given label probability below 0.25; retained 8,155 rows and all 77 labels. Five epochs peaked at 90.7792% validation accuracy.",284 why: "This isolates one question: can less—but cleaner—training data outperform more noisy data? No synthetic text, relabelling, validation leakage, or test tuning is involved.",285 command: "artifacts/encoder/exp-015/training_result.json",286 lesson: "This rule removed useful signal along with suspected noise: 699/770 correct versus Exp011's 716/770. A clean run can still disprove its hypothesis.",287 },288];289 290const failures = [291 { type: "system", title: "Metal looked unavailable", symptom: "The first environment script said Metal was unavailable on an M2 Max.", cause: "A brittle system_profiler check confused missing display data with missing Metal support.", fix: "Deferred the verdict to an actual MLX matrix calculation, which proved GPU access." },292 { type: "code", title: "MLX version attribute crashed", symptom: "mlx.__version__ raised AttributeError before the compute check.", cause: "The top-level MLX package does not expose that attribute.", fix: "Read the installed package version through Python package metadata." },293 { type: "data", title: "The dataset mirror was incomplete", symptom: "The mirror contained 9,993/3,076 rows instead of 10,003/3,080.", cause: "Fourteen records were missing from the maintained mirror.", fix: "Switched to the pinned original PolyAI source and verified checksums." },294 { type: "code", title: "Tokenization suffix check failed", symptom: "The script claimed the assistant answer was not a suffix of the prompt.", cause: "Transformers 5 returned a BatchEncoding, but the code treated it as a list of token IDs.", fix: "Requested return_dict=False and matched MLX-LM chat processing exactly." },295 { type: "numeric", title: "Experiment 003 became NaN", symptom: "Loss rose sharply and every saved adapter value became non-finite.", cause: "The learning rate was unstable over the longer 1,925-update run—not a memory shortage.", fix: "Marked the artifact unusable and retried at a lower learning rate as 003b." },296 { type: "numeric", title: "Batch size 7 failed immediately", symptom: "Experiment 004 reported NaN at iteration 50 and used 12.556 GB peak memory.", cause: "The batch configuration changed optimization behavior; available memory alone did not guarantee stability.", fix: "Probed batch size 1, then trained 004c stably at 5e-7." },297 { type: "system", title: "Control-C blocked the rerun", symptom: "An interrupted run left an adapter_config and log, so overwrite protection stopped the next command.", cause: "The runner correctly refused to overwrite partial experiment evidence.", fix: "Archived partial artifacts and taught the runner to auto-archive after future interruptions." },298 { type: "numeric", title: "Larger q/k/v/o adapter diverged", symptom: "Experiment 005 was healthy through iteration 2,000, then reached NaN at 2,050.", cause: "The same learning rate was too aggressive for the expanded adapter over a full epoch.", fix: "Halved the rate to 2.5e-7; 005b completed all 9,233 updates." },299 { type: "system", title: "The local API returned RuntimeError", symptom: "The CLI worked, but POST /classify returned HTTP 500.", cause: "ThreadingHTTPServer invoked MLX Metal generation from worker threads.", fix: "Used a single-threaded HTTPServer; two subsequent requests returned HTTP 200." },300 { type: "code", title: "Constraint mask used the wrong MLX API", symptom: "ArrayAt had no .set() method.", cause: "The code assumed a JAX-style setter that MLX 0.32 does not provide.", fix: "Used MLX indexed replacement, matching the installed library." },301 { type: "code", title: "EOS looked like an invalid prefix", symptom: "The constraint failed after finishing a valid label.", cause: "MLX-LM computes one token ahead before it recognizes the current EOS stop token.", fix: "Allowed the discarded post-EOS calculation to pass through." },302 { type: "code", title: "Gradient accumulation inflated BERT loss", symptom: "The first two-update BERT probe reported loss near 17.65 instead of the expected random-class loss near 4.34.", cause: "The custom loss path was multiplied even though the current Trainer already normalizes gradient accumulation.", fix: "Removed the duplicate scaling; the corrected probe produced finite loss 4.413." },303 { type: "code", title: "BERT packaging rejected a valid run", symptom: "Five epochs completed, but the runner expected exactly one .bin weight artifact and stopped.", cause: "Its file pattern counted training_args.bin as a second model weight file.", fix: "Loaded the intact epoch-4 checkpoint through the supported path, packaged it, and verified an exact save/reload round trip." },304 { type: "numeric", title: "DeBERTa became NaN after one update", symptom: "Forward losses were finite, then pretrained parameters became non-finite immediately after AdamW stepped on MPS.", cause: "The downloaded checkpoint tensors were float16; full-parameter optimizer updates were numerically unsafe in that precision on this setup.", fix: "Cast the complete model to float32 before training. The probe and all five epochs then stayed finite." },305 { type: "code", title: "DeBERTa finished training, then metadata crashed", symptom: "After 75 minutes and all five epochs, the wrapper raised KeyError: evaluation.", cause: "The reporting code expected a configuration block that the training configuration did not contain.", fix: "Preserved the checkpoints, added the metadata block, and used a finalizer to select and verify the completed model without retraining." },306 { type: "code", title: "Upper-layer freeze guard named the pooler incorrectly", symptom: "Experiment 011's probe stopped because it could not find deberta.pooler parameters.", cause: "Transformers exposes this model's pooler at the root-level pooler prefix.", fix: "Corrected the expected prefix, reran the cheap probe, and only then started full training." },307 { type: "experiment", title: "Class weighting and label smoothing regressed", symptom: "BERT refinements 007 and 008 each scored below the 91.5584% parent.", cause: "The techniques changed confidence and class emphasis but did not repair their targeted validation errors.", fix: "Rejected both children under the predeclared rule and kept Exp006 unchanged." },308 { type: "experiment", title: "Hard negatives harmed one answer", symptom: "Experiments 013 and 014 both changed one correct contactless_not_working prediction into change_pin.", cause: "The rival-label objective over-corrected an already ambiguous boundary.", fix: "Rejected both children and kept Exp011 as champion; the failure motivated a cleaner-data hypothesis instead." },309 { type: "experiment", title: "Noise pruning removed useful signal", symptom: "Exp015 completed stably but reached only 90.7792% validation accuracy, 17 correct answers behind Exp011.", cause: "Out-of-fold disagreement was a useful suspicion signal but not reliable enough to delete 11.68% of the training set at the chosen threshold.", fix: "Rejected the child without test evaluation, retained Exp011, and recorded that model confidence cannot substitute for verified label corrections." },310];311 312const flashcards = [313 ["What was the goal of this project?", "To specialize an open Qwen3 1.7B model for BANKING77 intent classification, then prove improvement with held-out evaluation on an M2 Max using MLX and LoRA."],314 ["Why did you establish a baseline before fine-tuning?", "The untouched baseline is the control. Without it, falling training loss cannot tell us whether the completed model became more useful on unseen data."],315 ["Why LoRA instead of full fine-tuning?", "LoRA trains small low-rank adapter matrices while freezing the 1.7B base weights. It reduced memory and storage enough to run controlled experiments locally."],316 ["What caused the NaN runs, and how did you respond?", "They were numerical optimization failures, not simple out-of-memory errors. We found the first non-finite iteration, verified adapter values, rejected corrupted artifacts, and retried with safer batch and learning-rate settings."],317 ["What is the difference between validation loss and test accuracy?", "Validation loss measures token prediction quality during experiment development. Test accuracy measures exact correct labels on untouched questions. Loss guides training; the test decides usefulness."],318 ["Why did the 154-example pilot and 3,080-example test disagree?", "Two examples per intent create high sampling noise. The pilot is a cheap gate; the complete test gives a more stable estimate."],319 ["What did constrained decoding accomplish?", "It masked token paths that could not finish as one of the 77 canonical labels. It reduced invalid outputs to zero without changing the trained weights."],320 ["What did the batch-size-1 control prove?", "At the same batch size, constrained decoding preserved all 1,545 valid correct predictions, repaired 29 invalid outputs, and eliminated all 181 invalid labels. The gain is therefore caused by the constraint, not batch shape."],321 ["Why did BERT outperform Qwen so dramatically?", "BANKING77 is closed-set classification. BERT encodes the complete question and directly scores 77 classes, while Qwen had to generate an exact label string token by token. The architecture matched the task better."],322 ["What is a classifier head?", "A small final neural layer that receives the encoder representation and outputs one raw score, or logit, for each of the 77 intents. The highest-scoring position becomes the prediction."],323 ["Why did we try DeBERTa after optimizing BERT?", "We first tested three BERT refinements and none beat the parent. That evidence justified an architecture benchmark. DeBERTa offers a stronger representation of token content and position for classification."],324 ["Why did float32 fix DeBERTa on MPS?", "The checkpoint arrived in float16. Its forward pass worked, but optimizer updates overflowed into non-finite values. float32 used more memory while providing safer numerical range and precision for full fine-tuning."],325 ["What does freezing layers 0–19 mean?", "Those parameters still participate in inference but receive no gradient updates. Experiment 011 trained only upper encoder layers 20–23 and the classification components, preserving lower-level language knowledge."],326 ["What is an out-of-fold prediction?", "Each training row is predicted by a model trained on the other folds, never on that row. This creates a less self-confirming signal for spotting difficult or potentially noisy examples."],327 ["Why didn't we automatically correct suspicious labels?", "A model disagreement is not proof that the published label is wrong. Many BANKING77 boundaries are semantically ambiguous, so automatic relabelling could replace human labels with model mistakes."],328 ["What exactly is Exp015 testing?", "Whether retraining from the original DeBERTa checkpoint on 8,155 cleaner-looking rows beats training on all 9,233 rows. The only intended variable is train-data pruning; validation decides and test stays locked."],329 ["What did Exp015 teach us?", "The run was technically healthy but the hypothesis failed. Removing 1,078 suspicious rows reduced validation accuracy from 92.987% to 90.779%, so the audit score was not strong enough to justify deletion at that threshold."],330];331 332const quizQuestions = [333 { q: "Why keep the test set completely separate from training?", options: ["To make downloads smaller", "To measure generalization without answer leakage", "Because MLX cannot train on CSV", "To reduce GPU temperature"], answer: 1, why: "The test set is the final exam. If its answers influence training or tuning, the score no longer estimates performance on unseen data." },334 { q: "A run ends with very low training loss. What can you conclude?", options: ["It is definitely the best model", "It cannot produce invalid labels", "It optimized the training objective; test evaluation is still required", "The learning rate was perfect"], answer: 2, why: "Loss proves optimization progress, not generalization or product usefulness." },335 { q: "Why was the failed Experiment 003 adapter unusable?", options: ["It was too small", "Every adapter value was non-finite", "It used public data", "Its checksum was too long"], answer: 1, why: "NaN/Inf weights cannot support reliable inference. We preserved the evidence and rejected the artifact." },336 { q: "What does LoRA freeze?", options: ["The dataset", "The test metrics", "The original model weights", "The Mac GPU"], answer: 2, why: "The original Qwen weights remain unchanged while small attached matrices are trained." },337 { q: "Why can a 154-example pilot mislead us?", options: ["It has no labels", "Two examples per intent create high variance", "It always uses CPU", "Macro F1 cannot be computed"], answer: 1, why: "One changed answer moves pilot accuracy by about 0.65 percentage points." },338 { q: "What was the main benefit of constrained decoding?", options: ["It doubled model parameters", "It made training faster", "It guaranteed outputs belong to the 77-label vocabulary", "It replaced the test set"], answer: 2, why: "The constraint acts during inference and closes the output space without retraining." },339 { q: "Two experiments change LoRA targets and learning rate. What can you claim?", options: ["Exactly which target caused the gain", "The combined configuration performed differently", "Learning rate never matters", "The result is invalid and must be deleted"], answer: 1, why: "You can compare configurations, but causal attribution requires changing one factor at a time." },340 { q: "Why was an encoder classifier a better fit than generative Qwen here?", options: ["It has no parameters", "It directly scores the fixed 77 choices", "It does not need training data", "It always reaches 100%"], answer: 1, why: "The output is a closed taxonomy. Direct class logits avoid the extra difficulty of generating an exact allowed string." },341 { q: "DeBERTa forward losses were finite but weights became NaN after optimizer.step(). What did that indicate?", options: ["The test labels leaked", "The HTTP server was threaded", "The parameter-update precision was unstable", "The dataset had only one label"], answer: 2, why: "Instrumentation localized the first failure after the update. Casting float16 parameters to float32 fixed the numerical path." },342 { q: "Experiment 011 trained only layers 20–23. What happened to layers 0–19?", options: ["They were deleted", "They stayed frozen and still participated in the forward pass", "They became test data", "They were converted to labels"], answer: 1, why: "Frozen means used but not updated. This reduces trainable capacity while preserving lower-layer representations." },343 { q: "What is the correct interpretation of an out-of-fold disagreement?", options: ["The original label is certainly wrong", "It is a review signal, not proof", "The row must enter the test set", "The model should memorize it"], answer: 1, why: "A held-out model may reveal noise, but it may also misunderstand a genuinely difficult or ambiguous example." },344 { q: "What must happen before Exp015 replaces Exp011?", options: ["Its training loss must be lowest", "It must use the test answers", "It must exceed 92.987% on unchanged validation", "It must remove more than 1,078 rows"], answer: 2, why: "The promotion rule is fixed in advance. Test remains locked and training loss alone cannot establish generalization." },345];346 347const glossary = [348 ["Adapter", "A small set of trained weights attached to a frozen base model."],349 ["Baseline", "The untouched model result used as the control for later comparisons."],350 ["Batch size", "How many examples are processed together before one optimizer update."],351 ["BF16", "A 16-bit floating-point format that reduces memory while retaining useful numeric range."],352 ["Canonical label", "The exact official output string required by the task taxonomy."],353 ["Checkpoint", "A saved intermediate artifact or result that lets work resume safely."],354 ["Constrained decoding", "Masking invalid next tokens so generation can only follow approved output paths."],355 ["Data leakage", "Test or validation information improperly influencing training or model selection."],356 ["Epoch", "One pass over every example in the training set."],357 ["Gradient", "A signal showing how each trainable value should move to reduce loss."],358 ["Inference", "Using a trained model to make a prediction; no weight update occurs."],359 ["Learning rate", "The step size used when updating trainable values."],360 ["Logits", "Raw model scores for every possible next token before probabilities are calculated."],361 ["LoRA", "Low-Rank Adaptation: train small matrices instead of all original model weights."],362 ["Macro F1", "F1 calculated per class and averaged so every class has equal weight."],363 ["NaN", "Not a Number: a sign that numerical computation became invalid or unstable."],364 ["Overfitting", "Learning training examples too specifically and performing worse on unseen data."],365 ["Parameter", "A learned numeric value inside a model."],366 ["Seed", "A fixed number that makes randomized data selection or initialization reproducible."],367 ["SFT", "Supervised fine-tuning on examples containing an input and known desired output."],368 ["Token", "A text unit processed by the model; it may be a word, fragment, symbol, or special marker."],369 ["Validation set", "Held-out data used during development to monitor training and compare settings."],370 ["Classifier head", "The final layer that converts an encoder representation into one score for each allowed class."],371 ["Cross-entropy", "A loss that penalizes the model when probability is placed away from the correct class."],372 ["Encoder", "A model that turns an entire input sequence into contextual representations rather than generating an open-ended answer."],373 ["Float32", "A 32-bit floating-point format. It uses more memory than float16 but can make optimizer updates more numerically stable."],374 ["Frozen layer", "A layer used during prediction whose parameters are deliberately not updated during training."],375 ["Full fine-tuning", "Updating all or nearly all pretrained model parameters for a downstream task."],376 ["Gradient accumulation", "Combining gradients from several smaller batches before an optimizer update to imitate a larger effective batch."],377 ["Hard negative", "A wrong class that the model finds especially plausible and is explicitly taught to rank below the true class."],378 ["Label noise", "Training examples whose supplied labels may be incorrect, inconsistent, or ambiguous."],379 ["Label smoothing", "A training technique that gives a small amount of target probability to non-target classes to discourage overconfidence."],380 ["MPS", "Apple's Metal Performance Shaders backend used by PyTorch to run tensor work on the Mac GPU."],381 ["Noise pruning", "Removing strongly suspected noisy training rows under a predefined rule, without rewriting their labels."],382 ["Out-of-fold", "Predictions for each training row made by a model that did not train on that row."],383 ["Oversampling", "Repeating selected training examples or classes so they influence more optimizer updates."],384 ["Partial fine-tuning", "Updating selected model layers while freezing the rest, as in Experiment 011's upper-layer refinement."],385 ["Promotion rule", "A criterion fixed before evaluation that decides whether a child experiment replaces the current champion."],386];387 388const selector = document.querySelector("#modelSelector");389models.forEach((model, index) => {390 const button = document.createElement("button");391 button.type = "button";392 button.role = "tab";393 button.textContent = model.tab;394 button.dataset.model = model.id;395 button.setAttribute("aria-selected", index === 0 ? "true" : "false");396 button.addEventListener("click", () => selectModel(model.id));397 selector.append(button);398});399 400function selectModel(id) {401 const model = models.find((item) => item.id === id) || models[0];402 selector.querySelectorAll("button").forEach((button) => button.setAttribute("aria-selected", String(button.dataset.model === id)));403 document.querySelector("#modelKicker").textContent = model.kicker;404 document.querySelector("#modelName").textContent = model.name;405 document.querySelector("#modelStory").textContent = model.story;406 document.querySelector("#modelDecision").textContent = model.decision;407 document.querySelector("#modelScope").textContent = model.scope || "Full test · 3,080 questions";408 const pending = model.accuracy === null;409 document.querySelector("#accuracyValue").textContent = pending ? "Pending" : `${model.accuracy.toFixed(2)}%`;410 document.querySelector("#f1Value").textContent = pending ? "Pending" : (model.f1 / 100).toFixed(4);411 document.querySelector("#invalidValue").textContent = String(model.invalid);412 document.querySelector(".baseline-mark").style.left = `${model.benchmark ?? 47.8571}%`;413 requestAnimationFrame(() => {414 document.querySelector("#accuracyBar").style.width = pending ? "0%" : `${model.accuracy}%`;415 document.querySelector("#f1Bar").style.width = pending ? "0%" : `${model.f1}%`;416 document.querySelector("#invalidBar").style.width = `${(model.invalid / 176) * 100}%`;417 });418 document.querySelector("#accuracyNote").textContent = model.accuracyNote || (model.id === "base" ? "The control mark." : `${Math.round((model.accuracy - 47.8571) * 30.8)} net answers above baseline.`);419 document.querySelector("#invalidNote").textContent = model.invalidNote || (model.invalid === 0 ? "Closed output contract." : "Out of 3,080 test questions.");420 document.querySelector("#modelFacts").innerHTML = model.facts.map(([term, value]) => `<div><dt>${term}</dt><dd>${value}</dd></div>`).join("");421 localStorage.setItem("omtl-model", id);422}423selectModel(localStorage.getItem("omtl-model") || "base");424 425const timelineNav = document.querySelector("#timelineNav");426stages.forEach((stage, index) => {427 const item = document.createElement("li");428 item.innerHTML = `<button type="button" role="tab" data-stage="${index}" data-step="${String(index + 1).padStart(2, "0")}" aria-selected="${index === 0}">${stage.title}</button>`;429 item.querySelector("button").addEventListener("click", () => selectStage(index));430 timelineNav.append(item);431});432function selectStage(index) {433 const stage = stages[index];434 timelineNav.querySelectorAll("button").forEach((button) => button.setAttribute("aria-selected", String(Number(button.dataset.stage) === index)));435 document.querySelector("#timelineDetail").innerHTML = `436 <span class="stage-tag">${stage.tag}</span>437 <h3>${stage.title}</h3>438 <p>${stage.summary}</p>439 <div class="stage-columns"><div><span>What we did</span><p>${stage.did}</p></div><div><span>Why it mattered</span><p>${stage.why}</p></div></div>440 <code class="stage-command">${stage.command}</code>441 <div class="stage-lesson"><strong>What you should remember:</strong> ${stage.lesson}</div>`;442 localStorage.setItem("omtl-stage", String(index));443}444selectStage(Math.min(Number(localStorage.getItem("omtl-stage") || 0), stages.length - 1));445 446const failureFilters = ["all", "system", "data", "code", "numeric", "experiment"];447let activeFailureFilter = "all";448failureFilters.forEach((filter) => {449 const button = document.createElement("button");450 button.type = "button";451 button.textContent = filter;452 button.className = filter === "all" ? "active" : "";453 button.addEventListener("click", () => {454 activeFailureFilter = filter;455 failureFilters && document.querySelectorAll("#failureFilters button").forEach((item) => item.classList.toggle("active", item.textContent === filter));456 renderFailures();457 });458 document.querySelector("#failureFilters").append(button);459});460function renderFailures() {461 const visible = failures.filter((failure) => activeFailureFilter === "all" || failure.type === activeFailureFilter);462 document.querySelector("#failureCount").textContent = `${visible.length} incidents`;463 document.querySelector("#failureList").innerHTML = visible.map((failure, index) => `464 <article class="failure-item">465 <button type="button" aria-expanded="false" aria-controls="failure-${index}">466 <span class="failure-type ${failure.type}">${failure.type}</span><span class="failure-title">${failure.title}</span><span class="failure-toggle">+</span>467 </button>468 <div class="failure-body" id="failure-${index}" hidden>469 <div><span>Symptom</span><p>${failure.symptom}</p></div>470 <div><span>Cause</span><p>${failure.cause}</p></div>471 <div><span>Fix</span><p>${failure.fix}</p></div>472 </div>473 </article>`).join("");474 document.querySelectorAll(".failure-item button").forEach((button) => button.addEventListener("click", () => {475 const body = button.nextElementSibling;476 const open = button.getAttribute("aria-expanded") === "true";477 button.setAttribute("aria-expanded", String(!open));478 button.querySelector(".failure-toggle").textContent = open ? "+" : "−";479 body.hidden = open;480 }));481}482renderFailures();483 484function renderModelStack(mode = "qv") {485 const stack = document.querySelector("#modelStack");486 stack.innerHTML = Array.from({ length: 28 }, (_, index) => `<i class="${index >= 12 && ((mode === "qv" && index % 3 === 0) || (mode === "qkvo" && index % 2 === 0)) ? "trainable" : ""}" style="height:${75 + ((index * 17) % 115)}px"></i>`).join("");487 const qkvo = mode === "qkvo";488 document.querySelector("#loraValues").textContent = qkvo ? "1,835,008" : "917,504";489 document.querySelector("#loraShare").textContent = qkvo ? "0.107%" : "0.053%";490 document.querySelector("#loraSize").textContent = qkvo ? "7.35 MB" : "3.68 MB";491 document.querySelectorAll("#loraToggle button").forEach((button) => button.classList.toggle("active", button.dataset.lora === mode));492}493document.querySelectorAll("#loraToggle button").forEach((button) => button.addEventListener("click", () => renderModelStack(button.dataset.lora)));494renderModelStack();495 496let flashcardIndex = 0;497function renderFlashcard() {498 const [question, answer] = flashcards[flashcardIndex];499 document.querySelector("#flashcardNumber").textContent = `${String(flashcardIndex + 1).padStart(2, "0")} / ${String(flashcards.length).padStart(2, "0")}`;500 document.querySelector("#flashcardQuestion").textContent = question;501 document.querySelector("#flashcardAnswer").innerHTML = `<p>${answer}</p>`;502 document.querySelector("#flashcardAnswer").hidden = true;503 document.querySelector("#revealFlashcard").textContent = "Reveal answer";504}505document.querySelector("#revealFlashcard").addEventListener("click", (event) => {506 const answer = document.querySelector("#flashcardAnswer");507 answer.hidden = !answer.hidden;508 event.currentTarget.textContent = answer.hidden ? "Reveal answer" : "Hide answer";509});510document.querySelector("#nextFlashcard").addEventListener("click", () => { flashcardIndex = (flashcardIndex + 1) % flashcards.length; renderFlashcard(); });511renderFlashcard();512 513let quizIndex = 0;514let quizScore = 0;515let quizLocked = false;516function renderQuiz() {517 const card = document.querySelector("#quizCard");518 if (quizIndex >= quizQuestions.length) {519 const percent = Math.round((quizScore / quizQuestions.length) * 100);520 card.innerHTML = `<div class="quiz-score"><span class="quiz-count">Assessment complete</span><strong>${percent}%</strong><h3>${percent >= 80 ? "You can explain the lab." : "Review the flight recorder, then try again."}</h3><button class="primary-action small" id="restartQuiz" type="button">Restart quiz</button></div>`;521 document.querySelector("#quizProgress").style.width = "100%";522 localStorage.setItem("omtl-quiz-best", String(Math.max(percent, Number(localStorage.getItem("omtl-quiz-best") || 0))));523 document.querySelector("#restartQuiz").addEventListener("click", () => { quizIndex = 0; quizScore = 0; renderQuiz(); });524 return;525 }526 quizLocked = false;527 const item = quizQuestions[quizIndex];528 card.innerHTML = `<span class="quiz-count">Question ${quizIndex + 1} of ${quizQuestions.length}</span><h3>${item.q}</h3><div class="quiz-options">${item.options.map((option, index) => `<button type="button" data-option="${index}">${option}</button>`).join("")}</div><div class="quiz-feedback" hidden></div><button class="primary-action small quiz-next" type="button" hidden>Next question</button>`;529 document.querySelector("#quizProgress").style.width = `${(quizIndex / quizQuestions.length) * 100}%`;530 card.querySelectorAll(".quiz-options button").forEach((button) => button.addEventListener("click", () => answerQuiz(Number(button.dataset.option))));531}532function answerQuiz(choice) {533 if (quizLocked) return;534 quizLocked = true;535 const item = quizQuestions[quizIndex];536 if (choice === item.answer) quizScore += 1;537 const card = document.querySelector("#quizCard");538 card.querySelectorAll(".quiz-options button").forEach((button) => {539 const option = Number(button.dataset.option);540 if (option === item.answer) button.classList.add("correct");541 else if (option === choice) button.classList.add("wrong");542 button.disabled = true;543 });544 const feedback = card.querySelector(".quiz-feedback");545 feedback.hidden = false;546 feedback.innerHTML = `<strong>${choice === item.answer ? "Correct." : "Not quite."}</strong> ${item.why}`;547 const next = card.querySelector(".quiz-next");548 next.hidden = false;549 next.addEventListener("click", () => { quizIndex += 1; renderQuiz(); });550}551renderQuiz();552 553function renderGlossary(query = "") {554 const normalized = query.trim().toLowerCase();555 const terms = glossary.filter(([term, definition]) => `${term} ${definition}`.toLowerCase().includes(normalized));556 document.querySelector("#glossaryGrid").innerHTML = terms.length ? terms.map(([term, definition]) => `<article class="glossary-item"><h3>${term}</h3><p>${definition}</p></article>`).join("") : `<p>No matching term. Try a shorter search.</p>`;557}558document.querySelector("#glossarySearch").addEventListener("input", (event) => renderGlossary(event.target.value));559renderGlossary();560 561document.querySelectorAll("[data-reveal]").forEach((button) => button.addEventListener("click", () => {562 const target = document.getElementById(button.dataset.reveal);563 target.hidden = !target.hidden;564 button.textContent = target.hidden ? button.dataset.originalText || "Reveal answer" : "Hide answer";565}));566document.querySelectorAll("[data-reveal]").forEach((button) => { button.dataset.originalText = button.textContent; });567 568const depthToggle = document.querySelector("#depthToggle");569depthToggle.addEventListener("click", () => {570 const active = document.body.classList.toggle("deep-mode");571 depthToggle.setAttribute("aria-pressed", String(active));572 depthToggle.lastChild.textContent = active ? " Hide deeper notes" : " Show deeper notes";573 localStorage.setItem("omtl-depth", active ? "1" : "0");574});575if (localStorage.getItem("omtl-depth") === "1") depthToggle.click();576 577const tracked = [...document.querySelectorAll("[data-track]")];578const seen = new Set(JSON.parse(localStorage.getItem("omtl-seen") || "[]"));579const observer = new IntersectionObserver((entries) => {580 entries.forEach((entry) => {581 if (entry.isIntersecting && entry.intersectionRatio > .2) {582 seen.add(entry.target.dataset.track);583 localStorage.setItem("omtl-seen", JSON.stringify([...seen]));584 localStorage.setItem("omtl-last", `#${entry.target.id}`);585 updateReadingProgress();586 }587 });588}, { threshold: [.2] });589tracked.forEach((section) => observer.observe(section));590function updateReadingProgress() {591 const percent = Math.round((seen.size / tracked.length) * 100);592 document.querySelector("#readingProgress").textContent = `${percent}%`;593 document.querySelector("#readingProgressBar").style.width = `${percent}%`;594}595updateReadingProgress();596document.querySelector("#resumeButton").addEventListener("click", () => {597 const target = document.querySelector(localStorage.getItem("omtl-last") || "#map");598 target?.scrollIntoView({ behavior: "smooth" });599});600