gggg03/Reinforcement_Learning_Flashcards
0
1<!DOCTYPE html>2<html lang="en">3<head>4<meta charset="UTF-8">5<meta name="viewport" content="width=device-width, initial-scale=1.0">6<title>RL Flashcards</title>7<link rel="preconnect" href="https://fonts.googleapis.com">8<link href="https://fonts.googleapis.com/css2?family=IBM+Plex+Mono:wght@400;500&family=Syne:wght@400;600;700;800&display=swap" rel="stylesheet">9<style>10*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }11 12 :root {13 --bg: #0e0e10;14 --surface: #18181c;15 --surface2: #222228;16 --border: rgba(255,255,255,0.08);17 --border2: rgba(255,255,255,0.14);18 --text: #f0ede8;19 --muted: #8a8790;20 --accent-1: #7c6ee6;21 --accent-2: #2ec4a0;22 --accent-3: #e85d45;23 --accent-4: #f5a623;24 --accent-5: #4fa3e0;25 --font-display: 'Syne', sans-serif;26 --font-mono: 'IBM Plex Mono', monospace;27}28 29 body {30 background: var(--bg);31 color: var(--text);32 font-family: var(--font-display);33 min-height: 100vh;34 display: flex;35 flex-direction: column;36 padding: 0;37}38 39 .header {40 padding: 2.5rem 2rem 1.5rem;41 border-bottom: 0.5px solid var(--border);42 flex-shrink: 0;43}44 45 .header h1 {46 font-size: 2rem;47 font-weight: 800;48 letter-spacing: -0.04em;49 color: var(--text);50}51 52 .header p {53 font-size: 0.85rem;54 color: var(--muted);55 margin-top: 0.35rem;56 font-family: var(--font-mono);57 font-weight: 400;58}59 60 .lecture-tabs {61 display: flex;62 gap: 0;63 overflow-x: auto;64 padding: 0 2rem;65 border-bottom: 0.5px solid var(--border);66 scrollbar-width: none;67 flex-shrink: 0;68}69 70 .lecture-tabs::-webkit-scrollbar { display: none; }71 72 .tab-btn {73 padding: 0.85rem 1.1rem;74 font-family: var(--font-mono);75 font-size: 0.72rem;76 font-weight: 500;77 color: var(--muted);78 background: none;79 border: none;80 border-bottom: 2px solid transparent;81 cursor: pointer;82 white-space: nowrap;83 transition: color 0.15s, border-color 0.15s;84 letter-spacing: 0.03em;85}86 87 .tab-btn:hover { color: var(--text); }88 .tab-btn.active { color: var(--text); border-bottom-color: var(--accent-1); }89 90 .deck-header {91 display: flex;92 align-items: center;93 justify-content: space-between;94 padding: 1.2rem 2rem 0.8rem;95 flex-wrap: wrap;96 gap: 0.75rem;97 flex-shrink: 0;98}99 100 .deck-title {101 font-size: 0.72rem;102 font-family: var(--font-mono);103 color: var(--muted);104 letter-spacing: 0.08em;105 text-transform: uppercase;106}107 108 .deck-progress {109 font-family: var(--font-mono);110 font-size: 0.72rem;111 color: var(--muted);112 display: flex;113 align-items: center;114 gap: 0.75rem;115}116 117 .progress-bar {118 width: 120px;119 height: 3px;120 background: var(--surface2);121 border-radius: 2px;122 overflow: hidden;123}124 125 .progress-fill {126 height: 100%;127 background: var(--accent-1);128 border-radius: 2px;129 transition: width 0.3s;130}131 132 /* Single view: column layout, card always above nav row */133 .single-view {134 display: flex;135 flex-direction: column;136 align-items: center;137 padding: 1rem 2rem 2rem;138 gap: 1rem;139}140 141 .single-view.hidden { display: none; }142 143 /* Card wrap: explicit height so position:absolute children render */144 .card-wrap {145 width: 100%;146 max-width: 780px;147 height: 420px;148 perspective: 1200px;149 cursor: pointer;150 user-select: none;151 flex-shrink: 0;152}153 154 .card-inner {155 position: relative;156 width: 100%;157 height: 100%;158 transform-style: preserve-3d;159 transition: transform 0.5s cubic-bezier(0.4, 0, 0.2, 1);160}161 162 .card-inner.flipped {163 transform: rotateY(180deg);164}165 166 .card-face {167 position: absolute;168 top: 0; left: 0;169 width: 100%;170 height: 100%;171 backface-visibility: hidden;172 -webkit-backface-visibility: hidden;173 border-radius: 16px;174 border: 0.5px solid var(--border2);175 display: flex;176 flex-direction: column;177 padding: 1.75rem 2rem;178 background: var(--surface);179 overflow: hidden;180}181 182 .card-face.back {183 transform: rotateY(180deg);184 background: var(--surface2);185}186 187 .card-tag {188 font-family: var(--font-mono);189 font-size: 0.65rem;190 letter-spacing: 0.1em;191 text-transform: uppercase;192 margin-bottom: 1.2rem;193 display: flex;194 align-items: center;195 gap: 0.5rem;196 flex-shrink: 0;197}198 199 .card-tag .dot {200 width: 6px; height: 6px;201 border-radius: 50%;202}203 204 .card-q {205 font-size: 1.15rem;206 font-weight: 700;207 line-height: 1.5;208 letter-spacing: -0.02em;209 flex: 1;210 display: flex;211 align-items: center;212 overflow: hidden;213}214 215 .card-hint {216 font-family: var(--font-mono);217 font-size: 0.68rem;218 color: var(--muted);219 margin-top: 1rem;220 flex-shrink: 0;221 display: flex;222 align-items: center;223 gap: 0.4rem;224}225 226 .card-a-label {227 font-family: var(--font-mono);228 font-size: 0.65rem;229 color: var(--accent-2);230 letter-spacing: 0.1em;231 text-transform: uppercase;232 margin-bottom: 0.75rem;233 flex-shrink: 0;234}235 236 .card-a {237 font-size: 0.92rem;238 line-height: 1.65;239 color: var(--text);240 flex: 1;241 overflow-y: auto;242 scrollbar-width: thin;243 scrollbar-color: var(--border2) transparent;244 padding-right: 4px;245}246 247 .card-a::-webkit-scrollbar { width: 4px; }248 .card-a::-webkit-scrollbar-track { background: transparent; }249 .card-a::-webkit-scrollbar-thumb { background: var(--border2); border-radius: 2px; }250 251 .card-a .formula {252 font-family: var(--font-mono);253 font-size: 0.88rem;254 background: rgba(255,255,255,0.05);255 border: 0.5px solid var(--border);256 border-radius: 6px;257 padding: 0.6rem 0.9rem;258 margin: 0.75rem 0;259 display: block;260 overflow-x: auto;261 white-space: nowrap;262}263 264 .card-a ul {265 padding-left: 1.2rem;266 margin-top: 0.5rem;267}268 269 .card-a ul li {270 margin-bottom: 0.4rem;271 font-size: 0.92rem;272}273 274 .card-a .key {275 display: inline;276 font-family: var(--font-mono);277 font-size: 0.82rem;278 background: rgba(124,110,230,0.12);279 color: #a89ef5;280 padding: 0 4px;281 border-radius: 3px;282}283 284 /* Nav row is always below the card, never overlaps */285 .nav-row {286 display: flex;287 align-items: center;288 gap: 1rem;289 width: 100%;290 max-width: 780px;291 flex-shrink: 0;292}293 294 .nav-btn {295 display: flex;296 align-items: center;297 justify-content: center;298 width: 44px; height: 44px;299 border-radius: 50%;300 border: 0.5px solid var(--border2);301 background: var(--surface);302 color: var(--text);303 cursor: pointer;304 font-size: 1.1rem;305 transition: background 0.15s, border-color 0.15s;306 flex-shrink: 0;307}308 309 .nav-btn:hover { background: var(--surface2); border-color: var(--accent-1); }310 .nav-btn:disabled { opacity: 0.3; cursor: default; }311 312 .flip-btn {313 flex: 1;314 height: 44px;315 border-radius: 10px;316 border: 0.5px solid var(--border2);317 background: var(--surface);318 color: var(--text);319 font-family: var(--font-mono);320 font-size: 0.75rem;321 letter-spacing: 0.05em;322 cursor: pointer;323 transition: background 0.15s;324}325 326 .flip-btn:hover { background: var(--surface2); }327 328 .shuffle-btn {329 height: 44px;330 padding: 0 1rem;331 border-radius: 10px;332 border: 0.5px solid var(--border2);333 background: var(--surface);334 color: var(--muted);335 font-family: var(--font-mono);336 font-size: 0.72rem;337 cursor: pointer;338 transition: background 0.15s, color 0.15s;339}340 341 .shuffle-btn:hover { background: var(--surface2); color: var(--text); }342 343 .grid-toggle {344 height: 44px;345 padding: 0 1rem;346 border-radius: 10px;347 border: 0.5px solid var(--border2);348 background: var(--surface);349 color: var(--muted);350 font-family: var(--font-mono);351 font-size: 0.72rem;352 cursor: pointer;353 transition: background 0.15s, color 0.15s;354}355 356 .grid-toggle:hover { background: var(--surface2); color: var(--text); }357 358 .grid-view {359 display: none;360 padding: 0 2rem 2rem;361 grid-template-columns: repeat(auto-fill, minmax(260px, 1fr));362 gap: 1rem;363}364 365 .grid-view.active { display: grid; }366 367 .grid-card {368 background: var(--surface);369 border: 0.5px solid var(--border2);370 border-radius: 12px;371 padding: 1.25rem;372 cursor: pointer;373 transition: background 0.15s, border-color 0.15s;374}375 376 .grid-card:hover { background: var(--surface2); border-color: var(--accent-1); }377 378 .grid-card .gc-tag {379 font-family: var(--font-mono);380 font-size: 0.62rem;381 color: var(--muted);382 letter-spacing: 0.08em;383 text-transform: uppercase;384 margin-bottom: 0.6rem;385}386 387 .grid-card .gc-q {388 font-size: 0.88rem;389 font-weight: 600;390 line-height: 1.4;391 letter-spacing: -0.01em;392}393 394 .count-badge {395 font-family: var(--font-mono);396 font-size: 0.65rem;397 padding: 2px 7px;398 border-radius: 20px;399 background: var(--surface2);400 color: var(--muted);401 border: 0.5px solid var(--border);402 margin-left: 0.5rem;403}404 405 @media (max-width: 600px) {406 .header { padding: 1.5rem 1rem 1rem; }407 .header h1 { font-size: 1.5rem; }408 .lecture-tabs { padding: 0 1rem; }409 .single-view { padding: 0.75rem 1rem 1.25rem; gap: 0.75rem; }410 .deck-header { padding: 1rem 1rem 0.5rem; }411 .grid-view { padding: 0 1rem 2rem; }412 .card-wrap { height: 380px; }413 .card-q { font-size: 0.95rem; }414 .card-a { font-size: 0.85rem; }415 .deck-progress { flex-wrap: wrap; gap: 0.5rem; }416 .progress-bar { width: 80px; }417}418</style>419</head>420<body>421<div class="header">422 <h1>Reinforcement Learning</h1>423 <p>flashcard deck · CT-469 · Dr. Murk Marvi</p>424</div>425<div class="lecture-tabs" id="lectureTabs"></div>426<div class="deck-header">427 <div class="deck-title" id="deckTitle"></div>428 <div class="deck-progress">429 <div class="progress-bar"><div class="progress-fill" id="progressFill"></div></div>430 <span id="progressText"></span>431 <button class="shuffle-btn" onclick="shuffle()">shuffle</button>432 <button class="grid-toggle" id="gridToggleBtn" onclick="toggleView()">grid</button>433 </div>434</div>435<div class="single-view" id="singleView">436 <div class="card-wrap" id="cardWrap" onclick="flipCard()">437 <div class="card-inner" id="cardInner">438 <div class="card-face front" id="cardFront">439 <div class="card-tag" id="cardTag"></div>440 <div class="card-q" id="cardQ"></div>441 <div class="card-hint">↩ click to reveal answer</div>442 </div>443 <div class="card-face back" id="cardBack">444 <div class="card-a-label">Answer</div>445 <div class="card-a" id="cardA"></div>446 </div>447 </div>448 </div>449 <div class="nav-row">450 <button class="nav-btn" id="prevBtn" onclick="prevCard()">←</button>451 <button class="flip-btn" onclick="flipCard()">flip card</button>452 <button class="nav-btn" id="nextBtn" onclick="nextCard()">→</button>453 </div>454</div>455<div class="grid-view" id="gridView"></div>456<script>457const LECTURES = [458{459id: 'w0506',460label: 'Weeks 05–06',461subtitle: 'Model-Free Prediction & Control',462color: '#7c6ee6',463cards: [464{465q: 'What is the DP Full Backup update rule and what does it cover?',466a: 'The DP full backup update is <span class="formula">v(Sₜ) ← 𝔼[Rₜ₊₁ + γv(Sₜ₊₁) | Aₜ ~ π(Sₜ)]</span>It averages over <em>all</em> possible next states and actions from Sₜ — the entire one-step fan-out — which requires a complete model of the environment.'467},468{469q: 'Describe the Monte-Carlo Prediction backup. What makes it different from DP?',470a: 'MC backup: <span class="formula">v(Sₜ) ← v(Sₜ) + α(Gₜ − v(Sₜ))</span>Instead of expanding all branches (DP), MC follows <em>one sampled trajectory</em> all the way to the terminal state. It is a sample backup (not full), and uses the actual return Gₜ, so it does <strong>not bootstrap</strong>.'471},472{473q: 'Write the First-Visit MC Prediction algorithm pseudocode and explain its key logic.',474a: 'Initialize V(s) arbitrarily, Returns(s) ← empty.<br><br>For each episode: generate it following π; work backward t = T−1 … 0:<ul><li>G ← γG + Rₜ₊₁</li><li>If Sₜ not in S₀…Sₜ₋₁ (first visit): append G to Returns(Sₜ); V(Sₜ) ← average(Returns(Sₜ))</li></ul>Key: only the <em>first</em> visit to each state per episode counts.'475},476{477q: 'Write the TD(0) Policy Evaluation algorithm. What does V(terminal) = 0 enforce?',478a: '<span class="formula">V(S) ← V(S) + α[R + γV(S′) − V(S)]</span>For each episode, loop until S is terminal: take action A, observe R, S′; update V(S); set S ← S′.<br><br>Setting V(terminal) = 0 prevents the bootstrap from generating artificial value at absorbing states — there is no future reward from terminal.'479},480{481q: 'Draw the TD Prediction backup diagram and write its update equation.',482a: 'The TD backup covers only <em>one step</em>: from Sₜ take one action, receive reward rₜ₊₁, reach Sₜ₊₁ — the highlighted blob in the diagram. The update:<span class="formula">v(Sₜ) ← v(Sₜ) + α(Rₜ₊₁ + γv(Sₜ₊₁) − v(Sₜ))</span>TD <em>bootstraps</em> using v(Sₜ₊₁) as a proxy for the future.'483},484{485q: 'Bootstrapping vs Sampling — how do MC, DP, and TD differ on each axis?',486a: '<ul><li><strong>Bootstrapping</strong> (update involves own estimate): DP ✓, TD ✓, MC ✗</li><li><strong>Sampling</strong> (update samples an expectation): MC ✓, TD ✓, DP ✗</li></ul>DP uses full expected backups (no sampling). MC uses sampled complete returns (no bootstrapping). TD does both — it samples one transition and bootstraps the rest.'487},488{489q: 'List four key ways TD is better than MC.',490a: '<ul><li>TD learns <strong>before</strong> the episode ends (online, after every step)</li><li>TD handles <strong>non-terminating</strong> (continuing) environments; MC only works on episodic</li><li>TD is <strong>independent of temporal span</strong> — updates from single transitions</li><li>TD has <strong>lower variance</strong> — target depends on one random event vs. entire episode</li></ul>Trade-off: TD introduces <em>bias</em> (bootstrapping); MC is unbiased.'491},492{493q: 'Why is the MC return Gₜ unbiased but the TD target biased?',494a: 'MC return <span class="formula">Gₜ = Rₜ₊₁ + γRₜ₊₂ + …</span>is an unbiased estimate of vπ(Sₜ) — no approximation used.<br><br>TD target <span class="formula">Rₜ₊₁ + γvₜ(Sₜ₊₁)</span>is biased unless vₜ(Sₜ₊₁) = vπ(Sₜ₊₁), which only holds at convergence. However, TD has <em>lower variance</em> because it depends on one random event, not the whole trajectory sum.'495},496{497q: 'Describe multistep prediction (n-step TD). What is the n-step return formula?',498a: 'n-step lets the TD target look n steps into the future:<span class="formula">Gₜ⁽ⁿ⁾ = Rₜ₊₁ + γRₜ₊₂ + … + γⁿ⁻¹Rₜ₊ₙ + γⁿv(Sₜ₊ₙ)</span>n=1 → standard TD; n=∞ → Monte Carlo. The update:<span class="formula">v(Sₜ) ← v(Sₜ) + α(Gₜ⁽ⁿ⁾ − v(Sₜ))</span>This bridges the bias-variance tradeoff between TD and MC.'499},500{501q: 'Explain the Unified View of RL diagram — what are its two axes?',502a: 'The 2D diagram organises RL methods by:<ul><li><strong>Vertical axis</strong>: full backups (top — DP/exhaustive) vs. sample backups (bottom — TD/MC)</li><li><strong>Horizontal axis</strong>: bootstrapping depth λ — shallow/no bootstrapping (left — TD) to deep/no bootstrapping (right — MC)</li></ul>Each corner is a method: DP top-left, Exhaustive Search top-right, TD bottom-left, MC bottom-right. Dyna and n-step TD fill the interior.'503},504{505q: 'Explain Monte-Carlo Policy Iteration: what are the two phases each episode?',506a: '<strong>Policy Evaluation</strong>: estimate qπ(s,a) using MC returns — average returns for each (s,a) pair.<br><br><strong>Policy Improvement</strong>: ε-greedy improvement on Q:<ul><li>Best action A* gets probability 1 − ε + ε/|A(s)|</li><li>All other actions get ε/|A(s)|</li></ul>This alternates until convergence. The diagram shows the "zigzag" convergence between the Q=Qπ line and the ε-greedy policy line toward Q*, π*.'507},508{509q: 'Write the SARSA update rule. Why is it called "on-policy"?',510a: '<span class="formula">qₜ₊₁(Sₜ,Aₜ) = qₜ(Sₜ,Aₜ) + αₜ(Rₜ₊₁ + γq(Sₜ₊₁,Aₜ₊₁) − qₜ(Sₜ,Aₜ))</span>It is on-policy because the next action Aₜ₊₁ is chosen by the <em>same ε-greedy policy</em> being evaluated and improved — it uses the tuple (Sₜ, Aₜ, Rₜ₊₁, Sₜ₊₁, Aₜ₊₁) hence "SARSA".'511},512{513q: 'Write the Q-Learning update rule and explain why it is off-policy.',514a: '<span class="formula">Q(S,A) ← Q(S,A) + α[R + γ max_a Q(S′,a) − Q(S,A)]</span>Q-learning is off-policy because the update uses <em>max over all actions</em> in S′, regardless of what the behaviour policy actually chose. The target policy (greedy) differs from the behaviour policy (ε-greedy).'515},516{517q: 'Explain Eligibility Traces using the grid diagram. Compare 1-step Sarsa, 10-step, and Sarsa(λ).',518a: 'In the diagram after an episode reaching goal G:<ul><li><strong>1-step Sarsa</strong>: only the last state-action near G has its value updated</li><li><strong>10-step Sarsa</strong>: a longer path near G gets updated, but values are uniform in that segment</li><li><strong>Sarsa(λ=0.9)</strong>: all visited (s,a) are updated, with the most recent ones updated most — arrow sizes taper as you go further back in the trajectory</li></ul>Traces give <em>credit to past visited states</em> proportional to recency.'519},520{521q: 'Give the Eligibility Trace update equations for Sarsa(λ).',522a: 'Trace initialisation: <span class="formula">E₀(s,a) = 0</span>Trace update at each step:<span class="formula">Eₜ(s,a) = γλEₜ₋₁(s,a) + 1(Sₜ=s, Aₜ=a)</span>TD error: <span class="formula">δₜ = Rₜ₊₁ + γQ(Sₜ₊₁,Aₜ₊₁) − Q(Sₜ,Aₜ)</span>Q update for every (s,a):<span class="formula">Q(s,a) ← Q(s,a) + αδₜEₜ(s,a)</span>'523}524]525},526{527id: 'rlhf',528label: 'RLHF',529subtitle: 'Aligning LLMs with Human Feedback',530color: '#2ec4a0',531cards: [532{533q: 'What are the three training phases of a large language model?',534a: '<ul><li><strong>Pre-training</strong>: auto-regressive next-token prediction on large corpus</li><li><strong>Mid-training (SFT)</strong>: supervised fine-tuning on high-quality (prompt, response) pairs</li><li><strong>Post-training (RLHF/RLVR)</strong>: reward model + PPO to align with human values</li></ul>'535},536{537q: 'Why was RLHF needed? List 4 failures of pre-RLHF LLMs.',538a: '<ul><li><strong>Instruction-following failure</strong>: GPT-3 completed prompts rather than following instructions</li><li><strong>Harmful/toxic outputs</strong>: models absorbed and reproduced harmful internet content</li><li><strong>Sycophancy & hallucinations</strong>: optimised for plausibility, not truth</li><li><strong>Inconsistent values</strong>: behaviour varied wildly across phrasings with no stable persona</li></ul>'539},540{541q: 'Describe the 3-stage RLHF pipeline with inputs, methods, and outputs.',542a: '<strong>Stage 1 — SFT</strong>: Pretrained LLM + 10K–100K human demonstrations → cross-entropy fine-tuning → SFT model<br><br><strong>Stage 2 — RM</strong>: SFT model + pairwise human preferences (A≻B) → Bradley-Terry ranking loss → reward function r(prompt, response)<br><br><strong>Stage 3 — PPO</strong>: SFT model + RM + prompts only → PPO maximising 𝔼[r(x,y)] with KL penalty → aligned RLHF model'543},544{545q: 'Write the SFT loss function and explain what it does.',546a: '<span class="formula">L_SFT(θ) = −𝔼(x,y)~D [Σₜ log πθ(yₜ | y<ₜ, x)]</span>Standard maximum likelihood / cross-entropy on (prompt x, ideal response y) pairs from human labellers. Identical math to pre-training but uses a small, high-quality dataset (~10K–100K). Output is the initial policy for PPO <em>and</em> the KL reference model.'547},548{549q: 'Write the Reward Model (RM) Bradley-Terry ranking loss.',550a: '<span class="formula">L_RM = −𝔼(x,yw,yl) [log σ(rφ(x,yw) − rφ(x,yl))]</span>yw = preferred ("won") response; yl = rejected ("lost") response. The loss is minimised when r(yw) > r(yl) — preferred responses get higher scalar scores. Architecture: SFT model + linear head mapping final token hidden state to scalar reward.'551},552{553q: 'What is the PPO RLHF objective? Explain each term.',554a: '<span class="formula">max_θ 𝔼[rφ(x,y)] − β · KL[πθ(·|x) ‖ πSFT(·|x)]</span><ul><li><span class="key">rφ(x,y)</span>: reward from the trained RM — measures alignment with human preferences</li><li><span class="key">β · KL</span>: KL divergence penalty — keeps new policy close to SFT model, prevents reward hacking</li><li><span class="key">β ≈ 0.1–0.5</span>: too small → reward hacking; too large → no improvement</li></ul>'555},556{557q: 'Explain reward hacking. Give 4 examples and state how the KL penalty addresses it.',558a: '<ul><li><strong>Repetition exploit</strong>: repeat correct-sounding phrases</li><li><strong>Length gaming</strong>: RM rewards longer → verbose unhelpful answers</li><li><strong>Sycophancy</strong>: agreeable-sounding over truthful</li><li><strong>Mode collapse</strong>: converge to one "safe" style</li></ul>KL penalty: <span class="formula">R_total(x,y) = rφ(x,y) − β · KL[πθ(·|x) ‖ πSFT(·|x)]</span>Token-level KL is applied at every step; RM score only at final token.'559},560{561q: 'Describe the PPO training loop for RLHF step by step.',562a: '<ol style="padding-left:1.2rem"><li>Sample prompt x from D</li><li>Generate response y ~ πθ(·|x)</li><li>Score: r = rφ(x,y) − β·KL</li><li>Compute Advantage A using critic Vw (GAE)</li><li>Clip + update θ via PPO-Clip objective</li><li>Update critic w via MSE loss</li></ol>'563},564{565q: 'What is Generalised Advantage Estimation (GAE) and how is it used in PPO?',566a: 'GAE computes the advantage function A in policy gradient methods:<span class="formula">Aₜ^GAE = Σ(γλ)ˡ δₜ₊ₗ, where δₜ = rₜ + γvₜ₊₁ − vₜ</span>It tells the policy how much better (or worse) an action was vs. expected — the signal that drives the policy update. The PPO clip objective:<span class="formula">ℒ_PPO = 𝔼ₜ[min(ρₜAₜ, clip(ρₜ, 1−ε, 1+ε)Aₜ)]</span>where ρₜ = πφ(aₜ|sₜ)/π_old(aₜ|sₜ) is the probability ratio.'567},568{569q: 'Describe InstructGPT technical details and key finding.',570a: '<ul><li>Base: GPT-3 (175B); SFT data: ~13K pairs; comparison data: ~33K pairings</li><li>RM: GPT-3 + linear head; PPO β=0.2</li><li><strong>Key finding</strong>: 1.3B InstructGPT outperformed 175B GPT-3 on human evals — labellers preferred it 85% of the time on the same prompts</li></ul><em>Alignment > raw scale.</em>'571},572{573q: 'Explain Constitutional AI (Claude). What are the two stages?',574a: '<strong>Stage A — SL-CAI</strong>: Sample harmful response → ask model to critique using a Constitution principle → ask model to revise → fine-tune on revised responses. Removes worst harms without labelling every example.<br><br><strong>Stage B — RLAIF</strong>: Generate response pairs → prompt AI to judge "which follows principle X?" → use AI preference labels to train Preference Model (PM) → apply RL using PM as reward. RLAIF replaces human labellers at scale.'575},576{577q: 'Compare RLHF (PPO), DPO, and RLAIF across RM, RL, human labels, and complexity.',578a: '<ul><li><strong>RLHF (PPO)</strong>: RM Yes, RL Yes, Human labels Yes, Complexity High</li><li><strong>DPO</strong>: RM No, RL No, Human labels Yes, Complexity Low</li><li><strong>RLAIF</strong>: RM Yes (AI), RL Yes, Human labels Minimal, Complexity Medium</li><li><strong>KTO</strong>: RM No, RL No, Human labels Yes (unary), Complexity Low</li></ul>'579},580{581q: 'Explain the DPO loss function and why it eliminates the RM and PPO.',582a: '<span class="formula">L_DPO = −𝔼(x,yw,yl)[log σ(β·log πθ(yw|x)/πref(yw|x) − β·log πθ(yl|x)/πref(yl|x))]</span><strong>Key insight</strong>: the optimal policy under RLHF has a closed-form solution. DPO directly optimises for it — no separate reward model, no RL sampling loop, no instability. Simpler, faster, often comparable performance.'583},584{585q: 'Compare ChatGPT, Claude, and Gemini RLHF approaches.',586a: '<ul><li><strong>ChatGPT/InstructGPT</strong>: PPO + human RM; scaled RLHF first</li><li><strong>Claude (Anthropic)</strong>: CAI + RLAIF + PPO; Constitutional AI with AI self-critique</li><li><strong>Gemini (Google)</strong>: PPO + multimodal RM; same pipeline across Nano/Pro/Ultra, sycophancy mitigation, RLHF + classifier-based safety filters</li></ul>'587},588{589q: 'What is GRPO? How does it differ from PPO computationally?',590a: 'GRPO (Group Relative Policy Optimization) generates a <em>group</em> of G outputs per query, computes rewards r₁…rG, then normalises advantages:<span class="formula">Â⁽ⁱ⁾ = r⁽ⁱ⁾ − (1/K)Σrʲ</span>No separate Value Model is needed (vs. PPO which trains Policy + Value).<br><br>Computational trade-offs: GRPO needs fewer forward passes, less memory, lower engineering complexity, and scales to large models more easily.'591}592]593},594{595id: 'w1112',596label: 'Weeks 11–12',597subtitle: 'Policy Gradient & Actor-Critic',598color: '#e85d45',599cards: [600{601q: 'Contrast the Value-Based vs Policy-Based pipeline for choosing actions.',602a: '<strong>Value-Based</strong>: state s → Q-network Q̂(s,a;w) → argmax_a Q̂ → action. Policy is <em>implicit</em> — derived from value function. No direct control over action probabilities.<br><br><strong>Policy-Based</strong>: state s → policy network π(a|s;θ) → sample A ~ π → action. Policy is <em>explicit</em>. Gradient ascent on J(θ) = v_πθ(s₀).'603},604{605q: 'What are the two main policy parameterisations and when do you use each?',606a: '<strong>Softmax (discrete actions)</strong>:<span class="formula">π(a|s;θ) = exp(h(s,a,θ)) / Σb exp(h(s,b,θ))</span>Never fully deterministic → exploration always preserved.<br><br><strong>Gaussian (continuous actions)</strong>:<span class="formula">π(a|s;θ) = 𝒩(a; μ(s;θμ), σ²(s;θσ))</span>μ = θμᵀx(s), σ = exp(θσᵀx(s)) ensures σ > 0.'607},608{609q: 'State the Policy Gradient performance objective J(θ). What is the challenge in differentiating it?',610a: '<span class="formula">J(θ) = 𝔼τ~πθ[G(τ)] = 𝔼τ~πθ[Σγᵗrₜ] = v_πθ(s₀)</span>The challenge: J depends on both the policy <em>and</em> the state distribution μπ(s), which also changes with θ. Naively differentiating through the state distribution is intractable. The Policy Gradient Theorem sidesteps this entirely.'611},612{613q: 'State the Policy Gradient Theorem result.',614a: '<span class="formula">∇J(θ) ∝ 𝔼π[qπ(Sₜ,Aₜ) ∇ln π(Aₜ|Sₜ;θ)]</span>This expectation is over trajectories sampled from π — directly actionable for stochastic gradient ascent. The key insight: ∇J does NOT require ∇μ(s). The gradient of the log-policy (score function) replaces differentiation through the state distribution.'615},616{617q: 'Write the REINFORCE Monte-Carlo Policy Gradient update rule.',618a: '<span class="formula">θₜ₊₁ = θₜ + α · Gₜ · ∇ln π(Aₜ|Sₜ;θₜ)</span>Algorithm: generate full episode following π; work backward t=0…T−1:<ul><li>G ← Σ γᵏ⁻ᵗ⁻¹ Rk (discounted return)</li><li>θ ← θ + αγᵗG∇ln π(Aₜ|Sₜ;θ)</li></ul>Problem: Gₜ sums ALL future rewards → high variance, slow convergence.'619},620{621q: 'Explain REINFORCE with Baseline. What is the advantage and how does it reduce variance?',622a: '<span class="formula">θₜ₊₁ = θₜ + αθ · (Gₜ − v̂(Sₜ;w)) · ∇ln π(Aₜ|Sₜ;θₜ)</span>The advantage (Gₜ − v̂(Sₜ)) signals:<ul><li>Gₜ >> v̂ → advantage > 0 → push θ <em>toward</em> action</li><li>Gₜ ≈ v̂ → advantage ≈ 0 → almost no update</li><li>Gₜ << v̂ → advantage < 0 → push θ <em>away</em> from action</li></ul>Variance reduced (centered around 0), no bias introduced — v̂ does not depend on Aₜ.'623},624{625q: 'Describe the Actor-Critic architecture. What does each component do?',626a: '<strong>Actor π(a|s,θ)</strong>: policy network. Updated via:<span class="formula">Δθ = αθ · I · δ · ∇ln π(A|S,θ)</span>Trained on TD error δ (not full return).<br><br><strong>Critic v̂(s,w)</strong>: value function. Updated via semi-gradient TD(0):<span class="formula">w ← w + αw · δ · ∇v̂(S,w)</span>TD Error δ = R + γv̂(S′,w) − v̂(S,w) guides the actor and updates the critic.'627},628{629q: 'Explain the δ interpretation in Actor-Critic.',630a: '<span class="formula">δ = R + γv̂(S′,w) − v̂(S,w)</span><ul><li><strong>δ > 0</strong>: action was <em>better</em> than expected → increase π(A|S)</li><li><strong>δ ≈ 0</strong>: action was about expected → almost no update</li><li><strong>δ < 0</strong>: action was <em>worse</em> than expected → decrease π(A|S)</li></ul>δ is both the advantage signal for the actor and the TD error for the critic.'631},632{633q: 'Compare REINFORCE vs One-Step Actor-Critic on bootstrapping and when each is better.',634a: '<ul><li><strong>REINFORCE</strong>: waits for full episode; Gₜ = R₁ + γR₂ + … (no bootstrap); zero bias, very high variance; offline only</li><li><strong>1-Step AC</strong>: uses Gₜ:ₜ₊₁ = R₁ + γv̂(S₁,w); bootstraps; some bias, low variance; fully online; works for continuing tasks</li></ul>Trade-off: AC introduces approximation bias via v̂, but the dramatic variance reduction usually wins in practice.'635},636{637q: 'State the bias-variance spectrum across REINFORCE, +Baseline, n=5 AC, and 1-Step AC.',638a: '<ul><li><strong>REINFORCE</strong>: Bias 0%, Variance 100%</li><li><strong>+Baseline</strong>: Bias 0%, Variance ~65%</li><li><strong>n=5 AC</strong>: Bias ~35%, Variance ~45%</li><li><strong>1-Step AC</strong>: Bias ~62%, Variance ~18%</li></ul>Increasing bootstrapping → more bias, less variance. TD(λ) lets you interpolate.'639},640{641q: 'How do Eligibility Traces work in Actor-Critic? What does λ control?',642a: 'Traces let past gradients influence current updates — a memory of recent activity. Update rule:<span class="formula">z ← λz + ∇ln π</span><ul><li><strong>λ=0</strong>: 1-step AC, no memory, TD(0) update only</li><li><strong>λ ∈ (0,1)</strong>: intermediate — balance bias & variance; best in practice</li><li><strong>λ=1</strong>: full episode memory, MC equivalent, zero bias, high variance</li></ul>'643},644{645q: 'Give the complete method comparison taxonomy for policy gradient methods.',646a: '<ul><li><strong>REINFORCE</strong>: target Gₜ, bias None, variance ★★★★★, online No, bootstrap No</li><li><strong>+Baseline</strong>: target Gₜ−v̂, bias None, variance ★★★☆☆</li><li><strong>1-Step AC</strong>: target R+γv̂(S′)−v̂(S), bias Low, variance ★★☆☆☆, online Yes</li><li><strong>AC+λ</strong>: δₜ with trace z, bias Low, variance ★★★☆☆</li><li><strong>PPO</strong>: clipped advantage, bias Low, variance ★★☆☆☆</li></ul>'647}648]649},650{651id: 'planning',652label: 'Planning & Learning',653subtitle: 'Dyna, MCTS, Prioritized Sweeping',654color: '#f5a623',655cards: [656{657q: 'What is the fundamental difference between Model-Free learning and Model-Based planning?',658a: '<strong>Model-Free</strong>: updates only from real environment interaction; no internal model; flexible but sample-hungry.<span class="formula">Q(s,a) ← Q(s,a) + α · δ</span><strong>Model-Based</strong>: updates from simulated transitions using a model M(s,a) → (s′,r); same Bellman update but can plan without touching the real env; sample-efficient, but model errors hurt.'659},660{661q: 'What is a Distribution Model vs. a Sample Model? Which does Dyna use?',662a: '<strong>Distribution Model</strong>: p(s′,r|s,a) — returns the full probability distribution over all next states and rewards. Used in DP and Value Iteration.<br><br><strong>Sample Model</strong>: M(s,a) → (s′,r) — given (s,a), produces one sampled transition. Can be learned from experience. <em>Dyna uses sample models</em> — practical for unknown environments.'663},664{665q: 'Describe the Dyna Architecture step by step.',666a: '<ol style="padding-left:1.2rem"><li>Take real step → observe (s, a, s′, r)</li><li>Direct Q-update from real experience</li><li>Update learned model: Model(S,A) ← R, S′</li><li>Do n planning steps: sample previously seen (S,A), query model, apply Q-update</li></ol><em>Key</em>: n planning steps per real step = n× better use of each real interaction.'667},668{669q: 'Write the Tabular Dyna-Q algorithm and explain the planning loop.',670a: 'Initialize Q(s,a) and Model(s,a). Loop forever:<ul><li>(a) current S; (b) A ← ε-greedy; (c) take A, observe R, S′</li><li>(d) Q update (Q-learning); (e) Model(S,A) ← R, S′</li><li>(f) Repeat n times: S ← random seen state; A ← random action taken in S; R,S′ ← Model(S,A); Q update</li></ul>Effect of n: n=0 → 416 steps to goal; n=5 → 195; n=50 → 54 steps.'671},672{673q: 'When can a model be "wrong"? Describe the two failure cases and their effects.',674a: '<strong>Case 1 — Stale/Outdated Model</strong>: environment changes but model is not updated → agent plans with old (wrong) model → takes longer to adapt than model-free!<br><br><strong>Case 2 — Systematic Model Bias</strong>: learned model has limited data; errors compound over n planning steps; values drift from truth. <em>More planning ≠ always better</em>.'675},676{677q: 'What is Dyna-Q+? Explain the exploration bonus formula.',678a: 'Dyna-Q+ adds an exploration bonus during planning steps to encourage revisiting state-action pairs not tried recently:<span class="formula">r⁺ = r + κ√τ(s,a)</span>τ(s,a) = timesteps since (s,a) was last tried in the real environment. κ (kappa) is a small tuned coefficient.<br><br>Benefit: handles non-stationary environments where Dyna-Q stagnates. Dyna-Q+ accumulates significantly higher cumulative reward in changing environments.'679},680{681q: 'Explain Prioritized Sweeping. How does it improve on Dyna-Q\'s random planning?',682a: '<strong>Dyna-Q</strong>: picks random (s,a) → many updates wasted on stable states.<br><br><strong>Prioritized Sweeping</strong>: maintains a max-heap priority queue. Priority = |TD error| = |R + γ·maxQ(s′,·) − Q(s,a)|. Always update the highest-priority (s,a) first, then propagate backward to predecessors.<br><br>Analogy: eligibility traces propagate backward in <em>time</em>; PS propagates backward through <em>state space</em>.'683},684{685q: 'Write the Prioritized Sweeping core loop.',686a: 'After real step → compute |TD error|; if > θ, insert (S,A) into PQueue with priority P.<br><br>While PQueue not empty:<ul><li>(S,A) ← pop highest priority</li><li>R,S′ ← Model(S,A); Q update</li><li>For each predecessor S̄,Ā predicted to lead to S: compute |TD error|, insert into PQueue if > θ</li></ul>Performance: needs 10× fewer updates than Dyna-Q to reach optimal on large gridworlds.'687},688{689q: 'Contrast Background Planning (Dyna) vs Decision-Time Planning.',690a: '<strong>Background Planning (Dyna)</strong>: improves a general policy offline; planning between actions; policy usable in any state.<br><br><strong>Decision-Time Planning</strong>: simulates forward from the current state only; focuses compute on what matters now; great for rarely-revisited states (like Go board positions).<br><br>AlphaGo, AlphaZero, MuZero all use decision-time planning (MCTS) with a learned model.'691},692{693q: 'Describe the four stages of Monte Carlo Tree Search (MCTS).',694a: '<ol style="padding-left:1.2rem"><li><strong>Select</strong>: traverse tree using UCB policy (exploit known, explore uncertain)</li><li><strong>Expand</strong>: add new node at unexplored action</li><li><strong>Simulate</strong>: rollout to terminal state using sample model + default/random policy</li><li><strong>Backpropagate</strong>: update all ancestor nodes with result</li></ol>Repeat X times before taking the real action. The tree policy uses known nodes; default policy handles the rest.'695},696{697q: 'Sketch the Big Picture diagram: how do Real and Simulated Experience relate?',698a: 'Real Experience feeds two paths:<ul><li>→ Direct RL update (Q-learning, SARSA, TD(λ), DP)</li><li>→ Model learning → Simulated Experience → Background Planning (Dyna-Q, PS) or Decision-Time Planning (Rollouts, MCTS, AlphaZero)</li></ul>Open frontier: how to learn a good model efficiently and plan without being misled by its errors (Dreamer, MuZero, World Models).'699}700]701},702{703id: 'fa',704label: 'Function Approximation',705subtitle: 'Linear FA, DQN, Deadly Triad',706color: '#4fa3e0',707cards: [708{709q: 'Why can\'t tabular RL scale? Give three fundamental reasons.',710a: '<ul><li><strong>Continuous state spaces</strong>: robot joints, stock prices — infinitely many states, impossible to enumerate</li><li><strong>Combinatorial explosion</strong>: Atari: 210×160×3 pixels = 2^100,000 possible states; memory impossible</li><li><strong>No generalisation</strong>: learning Q(s,a) gives zero information about Q(s′,a) even if s and s′ look identical</li></ul>Solution: parameterised function Q(s,a;w) that generalises across states.'711},712{713q: 'Describe the three types of Q-network architectures from the function approximation diagram.',714a: '<ul><li><strong>State-value</strong>: v̂(s,w) — input state s, output scalar V</li><li><strong>Action-value (joint)</strong>: q̂(s,a,w) — input (s,a), output scalar Q</li><li><strong>Action-value (parallel)</strong>: q̂(s,a₁…aₘ,w) — input s, output Q for all m actions simultaneously. This is what DQN uses — one forward pass for all actions.</li></ul>'715},716{717q: 'Explain the Contextual Bandit setting and why a tabular approach fails.',718a: 'A contextual bandit receives context features (e.g. Age, SportsInterest) and picks an action. Tabular approach requires a separate Q-value for <em>every combination</em> of context values and actions — e.g. Q(age=22, sports=1, Sports), Q(age=45, sports=0, Technology) — which explodes with continuous features.'719},720{721q: 'Write the Linear FA update rule for a Contextual Bandit.',722a: '<span class="formula">Q̂(s,a;w) = wᵀφ(s,a)</span>φ(s,a) = feature vector encoding (context, action); w = learned weights.<br><br>Gradient descent on MSE loss L = (Q̂ − R)²:<span class="formula">δ = R − Q̂(s,a;w)</span><span class="formula">w ← w + α · δ · φ(s,a)</span>No bootstrapping. True gradient. Convergence guaranteed.'723},724{725q: 'Write the Semi-Gradient TD update for MDPs. Why is it called "semi-gradient"?',726a: '<span class="formula">δₜ = R + γ · Q̂(s′,a′;w) − Q̂(s,a;w)</span><span class="formula">w ← w + α · δₜ · φ(s,a)</span>It is semi-gradient because we do NOT differentiate through the bootstrap target <span class="key">R + γQ̂(s′,a′;w)</span> — it is treated as a fixed constant when computing ∇. This means it is not a true gradient of any loss function.'727},728{729q: 'How does the TD target differ between the Contextual Bandit and MDP settings?',730a: '<ul><li><strong>Bandit</strong>: target = R (observed reward) — pure supervised, true gradient, MSE convergence guaranteed</li><li><strong>MDP + TD</strong>: target = R + γQ̂(s′,a′;w) (bootstrap) — semi-gradient, converges to TD fixed point (not global optimum)</li></ul>Everything else in the update rule is identical. Only the target changes.'731},732{733q: 'Explain Tile Coding. How does it convert a continuous state to a feature vector?',734a: 'Tile coding overlays multiple offset grids (tilings) on the continuous state space. For each tiling, only one tile is active at a time.<br><br>Example: state s=3.2, 2 tilings of 5 tiles each:<ul><li>Tiling 1 [0-2,2-4,4-6,6-8,8-10]: 3.2 ∈ [2,4) → Tile 2 active</li><li>Tiling 2 [1-3,3-5,5-7,7-9,9-11]: 3.2 ∈ [3,5) → Tile 7 active</li></ul>Feature vector: x(s) = [0,1,0,0,0,0,1,0,0,0] — sparse binary, only 2 ones per state.'735},736{737q: 'State the convergence guarantees for linear and nonlinear FA under different settings.',738a: '<ul><li><strong>Linear FA + MC targets (on-policy)</strong>: global optimum ✓</li><li><strong>Linear FA + TD semi-grad (on-policy)</strong>: TD fixed point ✓ (near-optimal)</li><li><strong>Linear FA + TD semi-grad (off-policy)</strong>: may diverge ✗ (Baird 1995)</li><li><strong>Nonlinear FA + bootstrap</strong>: no guarantee ✗</li></ul>'739},740{741q: 'Define the Deadly Triad. Why does each factor alone not cause divergence?',742a: 'Three ingredients that <em>together</em> cause Q-value divergence to infinity:<ol style="padding-left:1.2rem"><li><strong>Function Approximation</strong>: generalisation introduces approximation error</li><li><strong>Bootstrapping (TD)</strong>: using own estimate as target — correlated and moving</li><li><strong>Off-Policy Training</strong>: distribution mismatch between behaviour and target policy</li></ol>Each alone is safe. Together → Baird\'s Counterexample (1995) proved formal divergence.'743},744{745q: 'Explain the two key DQN stabilisation tricks and how each addresses the Deadly Triad.',746a: '<strong>① Experience Replay</strong>: store (s,a,r,s′) in buffer D; sample random mini-batches. Breaks temporal correlations → updates approximate i.i.d. supervised learning → addresses off-policy instability.<br><br><strong>② Target Network</strong>: w⁻ = frozen copy of w, synced every C~1000 steps. Stabilises the moving target → reduces TD divergence.<br><br>DQN Loss:<span class="formula">L(w) = 𝔼D[(r + γ·max_a′Q(s′,a′;w⁻) − Q(s,a;w))²]</span>'747},748{749q: 'Write the Double DQN algorithm key steps.',750a: 'Initialize Q(s,a;w), w⁻ ← w, replay buffer D. For each episode, for each step:<ol style="padding-left:1.2rem"><li>A ← ε-greedy w.r.t. Q(s,·;w)</li><li>Execute A, observe R, S′</li><li>Store (S,A,R,S′) in D</li><li>Sample mini-batch from D</li><li>Compute yⱼ = rⱼ + γ·max_a′Q(s′ⱼ,a′;w⁻)</li><li>Gradient step on (yⱼ − Q(sⱼ,aⱼ;w))²</li><li>Every C steps: w⁻ ← w</li></ol>'751},752{753q: 'Why does DQN work in practice despite having no convergence guarantee?',754a: 'DQN sits in the "deadly triad" zone (FA + bootstrapping + off-policy via replay), so theory offers no convergence guarantee. In practice it works because:<ul><li><strong>Experience Replay</strong> makes updates look like i.i.d. supervised learning, approximating on-policy behaviour</li><li><strong>Target Network</strong> freezes the bootstrap target, dramatically reducing the moving-target instability</li></ul>Why this is sufficient remains active research — the theory lags behind practice.'755}756]757}758];759 760let activeLecture = 0;761let activeIdx = 0;762let flipped = false;763let showGrid = false;764let deck = [];765 766function init() {767const tabs = document.getElementById('lectureTabs');768LECTURES.forEach((lec, i) => {769const btn = document.createElement('button');770btn.className = 'tab-btn' + (i === 0 ? ' active' : '');771btn.innerHTML = lec.label + `<span class="count-badge">${lec.cards.length}</span>`;772btn.onclick = () => switchLecture(i);773tabs.appendChild(btn);774});775switchLecture(0);776}777 778function switchLecture(idx) {779activeLecture = idx;780activeIdx = 0;781flipped = false;782deck = [...LECTURES[idx].cards];783document.querySelectorAll('.tab-btn').forEach((b, i) => {784b.classList.toggle('active', i === idx);785});786if (showGrid) {787renderGrid();788} else {789renderCard();790}791updateProgress();792updateDeckHeader();793}794 795function updateDeckHeader() {796const lec = LECTURES[activeLecture];797document.getElementById('deckTitle').textContent = lec.subtitle;798}799 800function renderCard() {801const lec = LECTURES[activeLecture];802const card = deck[activeIdx];803const color = lec.color;804document.getElementById('cardInner').classList.toggle('flipped', flipped);805const tagEl = document.getElementById('cardTag');806tagEl.innerHTML = `<span class="dot" style="background:${color}"></span><span style="color:${color}">${lec.label}</span><span style="color:var(--muted)"> · ${activeIdx + 1} / ${deck.length}</span>`;807document.getElementById('cardQ').innerHTML = card.q;808document.getElementById('cardA').innerHTML = card.a;809document.getElementById('prevBtn').disabled = activeIdx === 0;810document.getElementById('nextBtn').disabled = activeIdx === deck.length - 1;811}812 813function flipCard() {814flipped = !flipped;815document.getElementById('cardInner').classList.toggle('flipped', flipped);816}817 818function nextCard() {819if (activeIdx < deck.length - 1) {820activeIdx++;821flipped = false;822renderCard();823updateProgress();824}825}826 827function prevCard() {828if (activeIdx > 0) {829activeIdx--;830flipped = false;831renderCard();832updateProgress();833}834}835 836function updateProgress() {837const pct = deck.length > 1 ? (activeIdx / (deck.length - 1)) * 100 : 100;838document.getElementById('progressFill').style.width = pct + '%';839document.getElementById('progressText').textContent = (activeIdx + 1) + ' / ' + deck.length;840}841 842function shuffle() {843for (let i = deck.length - 1; i > 0; i--) {844const j = Math.floor(Math.random() * (i + 1));845[deck[i], deck[j]] = [deck[j], deck[i]];846}847activeIdx = 0;848flipped = false;849if (showGrid) renderGrid(); else renderCard();850updateProgress();851}852 853function toggleView() {854showGrid = !showGrid;855document.getElementById('gridToggleBtn').textContent = showGrid ? 'single' : 'grid';856document.getElementById('singleView').classList.toggle('hidden', showGrid);857document.getElementById('gridView').classList.toggle('active', showGrid);858if (showGrid) renderGrid(); else renderCard();859}860 861function renderGrid() {862const lec = LECTURES[activeLecture];863const grid = document.getElementById('gridView');864grid.innerHTML = '';865deck.forEach((card, i) => {866const el = document.createElement('div');867el.className = 'grid-card';868el.innerHTML = `<div class="gc-tag">${lec.label} · Q${i + 1}</div><div class="gc-q">${card.q}</div>`;869el.onclick = () => {870activeIdx = i;871flipped = false;872toggleView();873};874grid.appendChild(el);875});876}877 878document.addEventListener('keydown', (e) => {879if (e.key === 'ArrowRight' || e.key === 'ArrowDown') nextCard();880else if (e.key === 'ArrowLeft' || e.key === 'ArrowUp') prevCard();881else if (e.key === ' ' || e.key === 'Enter') { e.preventDefault(); flipCard(); }882});883 884init();885</script>886</body>887</html>