arad1367/Q-Learning-Simulation
3
1<!-- Q-Learning Simulation By Pejman Ebrahimi -->2<!DOCTYPE html>3<html lang="en">4 <head>5 <meta charset="UTF-8" />6 <meta name="viewport" content="width=device-width, initial-scale=1.0" />7 <title>Q-Learning Simulation by Pejman Ebrahimi</title>8 <style>9 :root {10 --primary: #3f51b5;11 --primary-light: #757de8;12 --primary-dark: #002984;13 --accent: #ff4081;14 --accent-light: #ff79b0;15 --accent-dark: #c60055;16 --success: #4caf50;17 --danger: #f44336;18 --warning: #ff9800;19 --dark: #212121;20 --light: #fafafa;21 --grid-size: 65px;22 --border-radius: 8px;23 --box-shadow: 0 3px 6px rgba(0, 0, 0, 0.1),24 0 3px 6px rgba(0, 0, 0, 0.15);25 --transition: all 0.3s cubic-bezier(0.25, 0.8, 0.25, 1);26 }27 28 * {29 box-sizing: border-box;30 margin: 0;31 padding: 0;32 font-family: Arial, sans-serif;33 }34 35 body {36 background-color: var(--light);37 color: var(--dark);38 line-height: 1.6;39 padding: 0;40 display: flex;41 flex-direction: column;42 min-height: 100vh;43 }44 45 header {46 background: linear-gradient(47 135deg,48 var(--primary-dark) 0%,49 var(--primary) 100%50 );51 color: white;52 padding: 2rem 0;53 text-align: center;54 position: relative;55 overflow: hidden;56 box-shadow: var(--box-shadow);57 }58 59 .container {60 width: 90%;61 max-width: 1400px;62 margin: 0 auto;63 padding: 2rem 0;64 }65 66 .header-content {67 position: relative;68 z-index: 10;69 }70 71 h1 {72 font-size: 2.5rem;73 margin: 0;74 font-weight: 300;75 }76 77 .subtitle {78 font-weight: 300;79 margin-top: 0.5rem;80 opacity: 0.9;81 }82 83 .attribution {84 display: inline-block;85 margin-top: 1rem;86 padding: 0.5rem 1rem;87 background-color: rgba(255, 255, 255, 0.1);88 border-radius: 50px;89 font-weight: 400;90 letter-spacing: 0.5px;91 position: relative;92 z-index: 10;93 }94 95 #stars-container {96 position: absolute;97 top: 0;98 left: 0;99 width: 100%;100 height: 100%;101 overflow: hidden;102 z-index: 1;103 }104 105 .star {106 position: absolute;107 background-color: #fff;108 width: 3px;109 height: 3px;110 border-radius: 50%;111 opacity: 0;112 animation: starAnimation 2s linear forwards;113 }114 115 @keyframes starAnimation {116 0% {117 transform: translate(0, 0) scale(0);118 opacity: 1;119 }120 100% {121 transform: translate(var(--tx), var(--ty)) scale(1);122 opacity: 0;123 }124 }125 126 main {127 flex: 1;128 padding: 2rem 0;129 }130 131 .grid-layout {132 display: grid;133 grid-template-columns: 1fr 1fr;134 grid-gap: 2rem;135 }136 137 h2 {138 font-weight: 400;139 color: var(--primary);140 margin-bottom: 1rem;141 border-bottom: 1px solid #eee;142 padding-bottom: 0.5rem;143 }144 145 h3 {146 font-weight: 400;147 color: var(--primary-dark);148 margin-bottom: 0.5rem;149 }150 151 .card {152 background-color: white;153 border-radius: var(--border-radius);154 box-shadow: var(--box-shadow);155 padding: 1.5rem;156 margin-bottom: 2rem;157 transition: var(--transition);158 }159 160 .card:hover {161 box-shadow: 0 6px 12px rgba(0, 0, 0, 0.1),162 0 6px 12px rgba(0, 0, 0, 0.15);163 }164 165 .flex-container {166 display: flex;167 gap: 1.5rem;168 }169 170 .world-container {171 flex: 1;172 }173 174 .world-wrapper {175 display: flex;176 gap: 2rem;177 justify-content: space-between;178 align-items: flex-start;179 }180 181 .grid-container,182 .q-values-container {183 display: grid;184 grid-template-columns: repeat(5, var(--grid-size));185 grid-template-rows: repeat(5, var(--grid-size));186 gap: 5px;187 }188 189 .grid-cell {190 position: relative;191 background-color: white;192 border-radius: 4px;193 display: flex;194 align-items: center;195 justify-content: center;196 border: 1px solid #e0e0e0;197 box-shadow: 0 1px 3px rgba(0, 0, 0, 0.05);198 transition: var(--transition);199 }200 201 .grid-cell.obstacle {202 background-color: var(--danger);203 color: white;204 }205 206 .grid-cell.goal {207 background-color: var(--success);208 color: white;209 }210 211 .agent {212 position: absolute;213 width: 26px;214 height: 26px;215 background-color: var(--primary);216 border-radius: 50%;217 z-index: 10;218 box-shadow: 0 2px 4px rgba(0, 0, 0, 0.2);219 transition: all 0.5s cubic-bezier(0.175, 0.885, 0.32, 1.275);220 }221 222 .trail {223 position: absolute;224 width: 10px;225 height: 10px;226 background-color: rgba(63, 81, 181, 0.3);227 border-radius: 50%;228 z-index: 5;229 transform: scale(0);230 animation: pulseTrail 1.5s ease-out forwards;231 }232 233 @keyframes pulseTrail {234 0% {235 transform: scale(0);236 opacity: 0.6;237 }238 100% {239 transform: scale(1);240 opacity: 0;241 }242 }243 244 .q-cell {245 background-color: white;246 border-radius: 4px;247 padding: 5px;248 font-size: 0.7rem;249 border: 1px solid #e0e0e0;250 position: relative;251 display: flex;252 justify-content: center;253 align-items: center;254 }255 256 .q-arrow {257 position: absolute;258 width: 0;259 height: 0;260 border-style: solid;261 transition: var(--transition);262 }263 264 .q-arrow.up {265 top: 2px;266 left: 50%;267 transform: translateX(-50%);268 border-width: 0 6px 8px 6px;269 border-color: transparent transparent rgba(63, 81, 181, var(--opacity))270 transparent;271 }272 273 .q-arrow.right {274 top: 50%;275 right: 2px;276 transform: translateY(-50%);277 border-width: 6px 0 6px 8px;278 border-color: transparent transparent transparent279 rgba(63, 81, 181, var(--opacity));280 }281 282 .q-arrow.down {283 bottom: 2px;284 left: 50%;285 transform: translateX(-50%);286 border-width: 8px 6px 0 6px;287 border-color: rgba(63, 81, 181, var(--opacity)) transparent transparent288 transparent;289 }290 291 .q-arrow.left {292 top: 50%;293 left: 2px;294 transform: translateY(-50%);295 border-width: 6px 8px 6px 0;296 border-color: transparent rgba(63, 81, 181, var(--opacity)) transparent297 transparent;298 }299 300 .q-value {301 position: absolute;302 font-size: 9px;303 color: #666;304 transition: var(--transition);305 }306 307 .q-value.best {308 color: var(--primary);309 font-weight: 700;310 }311 312 .reward-display {313 position: absolute;314 font-size: 14px;315 font-weight: bold;316 z-index: 20;317 opacity: 0;318 animation: fadeUp 1.2s cubic-bezier(0.175, 0.885, 0.32, 1.275) forwards;319 }320 321 @keyframes fadeUp {322 0% {323 opacity: 1;324 transform: translateY(0);325 }326 100% {327 opacity: 0;328 transform: translateY(-30px);329 }330 }331 332 .positive-reward {333 color: var(--success);334 }335 .negative-reward {336 color: var(--danger);337 }338 339 .control-section {340 display: grid;341 grid-template-columns: 1fr 1fr;342 gap: 1.5rem;343 }344 345 .control-panel {346 display: grid;347 grid-template-columns: 1fr;348 gap: 1rem;349 }350 351 .slider-container {352 margin-bottom: 0.5rem;353 }354 355 .slider-label {356 display: flex;357 justify-content: space-between;358 margin-bottom: 0.25rem;359 }360 361 .slider-label .value {362 font-weight: 500;363 color: var(--primary);364 }365 366 input[type="range"] {367 width: 100%;368 height: 6px;369 background: #e0e0e0;370 border-radius: 3px;371 appearance: none;372 margin: 0.5rem 0;373 }374 375 input[type="range"]::-webkit-slider-thumb {376 appearance: none;377 width: 16px;378 height: 16px;379 background: var(--primary);380 border-radius: 50%;381 cursor: pointer;382 box-shadow: 0 1px 3px rgba(0, 0, 0, 0.2);383 }384 385 input[type="range"]::-moz-range-thumb {386 width: 16px;387 height: 16px;388 background: var(--primary);389 border-radius: 50%;390 border: none;391 cursor: pointer;392 box-shadow: 0 1px 3px rgba(0, 0, 0, 0.2);393 }394 395 input[type="range"]::-ms-thumb {396 width: 16px;397 height: 16px;398 background: var(--primary);399 border-radius: 50%;400 cursor: pointer;401 box-shadow: 0 1px 3px rgba(0, 0, 0, 0.2);402 }403 404 .slider-description {405 font-size: 0.8rem;406 color: #666;407 margin-top: 0.25rem;408 }409 410 .mode-selector {411 display: flex;412 gap: 0.5rem;413 margin: 1rem 0;414 }415 416 .mode-btn {417 flex: 1;418 padding: 0.75rem;419 text-align: center;420 background-color: #f5f5f5;421 border-radius: 4px;422 cursor: pointer;423 transition: var(--transition);424 font-weight: 500;425 }426 427 .mode-btn:hover {428 background-color: #e0e0e0;429 }430 431 .mode-btn.active {432 background-color: var(--primary);433 color: white;434 box-shadow: 0 2px 4px rgba(0, 0, 0, 0.2);435 }436 437 .btn-container {438 display: flex;439 gap: 0.5rem;440 margin-top: 1rem;441 }442 443 button {444 padding: 0.75rem 1.25rem;445 background-color: var(--primary);446 color: white;447 border: none;448 border-radius: 4px;449 cursor: pointer;450 transition: var(--transition);451 font-weight: 500;452 box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1);453 flex: 1;454 }455 456 button:hover {457 background-color: var(--primary-light);458 box-shadow: 0 2px 4px rgba(0, 0, 0, 0.2);459 }460 461 button:active {462 transform: translateY(1px);463 }464 465 button.secondary {466 background-color: #e0e0e0;467 color: #333;468 }469 470 button.secondary:hover {471 background-color: #d0d0d0;472 }473 474 .stats-card {475 display: grid;476 grid-template-columns: repeat(4, 1fr);477 gap: 1rem;478 margin-bottom: 1.5rem;479 }480 481 .stat-box {482 background-color: #f5f5f5;483 padding: 1rem;484 border-radius: 8px;485 text-align: center;486 transition: var(--transition);487 }488 489 .stat-box:hover {490 transform: translateY(-2px);491 box-shadow: 0 2px 5px rgba(0, 0, 0, 0.1);492 }493 494 .stat-label {495 font-size: 0.8rem;496 color: #666;497 }498 499 .stat-value {500 font-size: 1.5rem;501 font-weight: 500;502 color: var(--primary);503 margin-top: 0.25rem;504 }505 506 .chart-container {507 height: 250px;508 margin: 1.5rem 0;509 background-color: #f5f5f5;510 border-radius: 8px;511 padding: 1rem;512 position: relative;513 }514 515 .simple-chart {516 width: 100%;517 height: 100%;518 position: relative;519 }520 521 .chart-bar {522 position: absolute;523 bottom: 0;524 background-color: var(--primary);525 border-radius: 4px 4px 0 0;526 transition: height 0.3s ease;527 }528 529 .chart-line {530 position: absolute;531 bottom: 50%;532 left: 0;533 width: 100%;534 height: 1px;535 background-color: rgba(0, 0, 0, 0.1);536 }537 538 .chart-label {539 position: absolute;540 bottom: -20px;541 font-size: 10px;542 text-align: center;543 transform: translateX(-50%);544 color: #666;545 }546 547 .chart-legend {548 position: absolute;549 top: 10px;550 right: 10px;551 display: flex;552 gap: 10px;553 }554 555 .legend-item {556 display: flex;557 align-items: center;558 gap: 5px;559 font-size: 12px;560 }561 562 .legend-color {563 width: 12px;564 height: 12px;565 border-radius: 2px;566 }567 568 .leaderboard {569 width: 100%;570 border-collapse: collapse;571 }572 573 .leaderboard th {574 text-align: left;575 padding: 0.75rem;576 background-color: #f5f5f5;577 font-weight: 500;578 }579 580 .leaderboard td {581 padding: 0.75rem;582 border-bottom: 1px solid #eee;583 }584 585 .leaderboard tr:last-child td {586 border-bottom: none;587 }588 589 .notification {590 position: fixed;591 top: 20px;592 right: 20px;593 background-color: var(--primary);594 color: white;595 padding: 1rem 1.5rem;596 border-radius: var(--border-radius);597 box-shadow: 0 4px 12px rgba(0, 0, 0, 0.2);598 z-index: 1000;599 opacity: 0;600 transform: translateX(50px);601 transition: var(--transition);602 max-width: 300px;603 }604 605 .notification.show {606 opacity: 1;607 transform: translateX(0);608 }609 610 .notification-title {611 font-weight: 500;612 margin-bottom: 0.5rem;613 display: flex;614 align-items: center;615 gap: 8px;616 }617 618 .notification-body {619 font-size: 0.9rem;620 opacity: 0.9;621 }622 623 .legend {624 display: flex;625 flex-wrap: wrap;626 gap: 1rem;627 margin-top: 1rem;628 padding: 0.5rem;629 background-color: #f5f5f5;630 border-radius: 4px;631 }632 633 .legend-item {634 display: flex;635 align-items: center;636 gap: 0.5rem;637 font-size: 0.9rem;638 }639 640 .legend-color {641 width: 12px;642 height: 12px;643 border-radius: 3px;644 }645 646 footer {647 text-align: center;648 padding: 1.5rem;649 background-color: var(--primary-dark);650 color: white;651 margin-top: 2rem;652 }653 654 @media (max-width: 1200px) {655 .grid-layout {656 grid-template-columns: 1fr;657 }658 659 .world-wrapper {660 flex-direction: column;661 align-items: center;662 }663 664 .grid-container,665 .q-values-container {666 margin: 0 auto;667 }668 669 .control-section {670 grid-template-columns: 1fr;671 }672 673 .stats-card {674 grid-template-columns: repeat(2, 1fr);675 }676 }677 678 @media (max-width: 768px) {679 :root {680 --grid-size: 50px;681 }682 683 .container {684 width: 95%;685 padding: 1rem 0;686 }687 688 h1 {689 font-size: 1.8rem;690 }691 692 .control-panel {693 grid-template-columns: 1fr;694 }695 }696 697 /* Loading spinner for initial setup */698 .loader {699 border: 4px solid rgba(63, 81, 181, 0.1);700 border-top: 4px solid var(--primary);701 border-radius: 50%;702 width: 40px;703 height: 40px;704 animation: spin 1s linear infinite;705 margin: 2rem auto;706 }707 708 @keyframes spin {709 0% {710 transform: rotate(0deg);711 }712 100% {713 transform: rotate(360deg);714 }715 }716 717 /* Exploration mode visual */718 .explore-indicator {719 position: absolute;720 width: 100%;721 height: 100%;722 background-color: rgba(255, 152, 0, 0.2);723 border-radius: 4px;724 z-index: 5;725 display: flex;726 align-items: center;727 justify-content: center;728 }729 </style>730 </head>731 <body>732 <header>733 <div id="stars-container"></div>734 <div class="header-content">735 <h1>Q-Learning Simulation</h1>736 <p class="subtitle">Reinforcement Learning in Action</p>737 <div class="attribution">Designed by Pejman Ebrahimi</div>738 </div>739 </header>740 741 <main>742 <div class="container">743 <div class="loader" id="loader"></div>744 745 <div class="grid-layout" id="main-content" style="display: none">746 <div class="left-section">747 <div class="card">748 <h2>Interactive Environment</h2>749 <div class="world-wrapper">750 <div class="world-container">751 <h3>Grid World</h3>752 <div class="grid-container" id="grid-container"></div>753 </div>754 755 <div class="world-container">756 <h3>Q-Values</h3>757 <div class="q-values-container" id="q-values-container"></div>758 </div>759 </div>760 761 <div class="legend">762 <div class="legend-item">763 <div764 class="legend-color"765 style="background-color: var(--primary)"766 ></div>767 <span>Agent</span>768 </div>769 <div class="legend-item">770 <div771 class="legend-color"772 style="background-color: var(--success)"773 ></div>774 <span>Goal (+10)</span>775 </div>776 <div class="legend-item">777 <div778 class="legend-color"779 style="background-color: var(--danger)"780 ></div>781 <span>Obstacle (-5)</span>782 </div>783 <div class="legend-item">784 <div785 class="legend-color"786 style="background-color: rgba(63, 81, 181, 0.3)"787 ></div>788 <span>Path History</span>789 </div>790 </div>791 </div>792 793 <div class="card">794 <h2>Learning Parameters</h2>795 <div class="control-section">796 <div class="control-panel">797 <div class="slider-container">798 <div class="slider-label">799 <span>Learning Rate (ฮฑ)</span>800 <span class="value" id="learning-rate-value">0.1</span>801 </div>802 <input803 type="range"804 id="learning-rate"805 min="0.01"806 max="1"807 step="0.01"808 value="0.1"809 />810 <div class="slider-description">811 How quickly the agent incorporates new information812 </div>813 </div>814 815 <div class="slider-container">816 <div class="slider-label">817 <span>Discount Factor (ฮณ)</span>818 <span class="value" id="discount-factor-value">0.9</span>819 </div>820 <input821 type="range"822 id="discount-factor"823 min="0"824 max="0.99"825 step="0.01"826 value="0.9"827 />828 <div class="slider-description">829 How much future rewards matter830 </div>831 </div>832 </div>833 834 <div class="control-panel">835 <div class="slider-container">836 <div class="slider-label">837 <span>Exploration Rate (ฮต)</span>838 <span class="value" id="exploration-rate-value">0.3</span>839 </div>840 <input841 type="range"842 id="exploration-rate"843 min="0"844 max="1"845 step="0.01"846 value="0.3"847 />848 <div class="slider-description">849 Chance of taking random actions850 </div>851 </div>852 853 <div class="slider-container">854 <div class="slider-label">855 <span>Animation Speed</span>856 <span class="value" id="animation-speed-value"857 >300ms</span858 >859 </div>860 <input861 type="range"862 id="animation-speed"863 min="50"864 max="1000"865 step="50"866 value="300"867 />868 <div class="slider-description">869 How quickly the simulation runs870 </div>871 </div>872 </div>873 </div>874 875 <div class="mode-selector">876 <div class="mode-btn active" id="learning-mode">877 Learning Mode878 </div>879 <div class="mode-btn" id="optimal-mode">880 Optimal Policy Mode881 </div>882 </div>883 884 <div class="btn-container">885 <button id="start-button">Start Learning</button>886 <button id="step-button">Single Step</button>887 <button id="reset-button" class="secondary">Reset</button>888 </div>889 </div>890 </div>891 892 <div class="right-section">893 <div class="card">894 <h2>Performance Metrics</h2>895 <div class="stats-card">896 <div class="stat-box">897 <div class="stat-label">Episodes</div>898 <div class="stat-value" id="episode-count">0</div>899 </div>900 <div class="stat-box">901 <div class="stat-label">Current Steps</div>902 <div class="stat-value" id="step-count">0</div>903 </div>904 <div class="stat-box">905 <div class="stat-label">Success Rate</div>906 <div class="stat-value" id="success-rate">0%</div>907 </div>908 <div class="stat-box">909 <div class="stat-label">Avg. Steps</div>910 <div class="stat-value" id="avg-completion">0</div>911 </div>912 </div>913 914 <h3>Learning Progress</h3>915 <div class="chart-container">916 <div class="simple-chart" id="steps-chart"></div>917 <div class="chart-legend">918 <div class="legend-item">919 <div920 class="legend-color"921 style="background-color: var(--primary)"922 ></div>923 <span>Steps per Episode</span>924 </div>925 </div>926 </div>927 </div>928 929 <div class="card">930 <h2>Leaderboard - Best Paths</h2>931 <table class="leaderboard">932 <thead>933 <tr>934 <th>Rank</th>935 <th>Episode</th>936 <th>Steps</th>937 <th>Reward</th>938 </tr>939 </thead>940 <tbody id="leaderboard-body">941 <tr>942 <td colspan="4" style="text-align: center">943 No successful episodes yet944 </td>945 </tr>946 </tbody>947 </table>948 </div>949 </div>950 </div>951 </div>952 </main>953 954 <footer>955 <p>ยฉ 2025 Pejman Ebrahimi โข Interactive Q-Learning Simulation</p>956 </footer>957 958 <div class="notification" id="notification">959 <div class="notification-title">960 <span>๐</span>961 <span id="notification-title">New Record!</span>962 </div>963 <div class="notification-body" id="notification-message"></div>964 </div>965 966 <script>967 // Environment setup968 const gridSize = 5;969 const numStates = gridSize * gridSize;970 const numActions = 4; // Up, Right, Down, Left971 const obstacles = [7, 8, 17]; // Obstacle positions972 const goal = 24; // Goal position973 let agentPos = 0; // Start position974 975 // Rewards976 const stepReward = -0.1; // Penalty for each step977 const obstacleReward = -5; // Penalty for hitting obstacle978 const goalReward = 10; // Reward for reaching goal979 980 // Learning state981 let qTable = Array(numStates)982 .fill()983 .map(() => Array(numActions).fill(0));984 let isLearning = false;985 let isSingleStep = false;986 let episodes = 0;987 let steps = 0;988 let episodeSteps = 0;989 let stepsHistory = [];990 let rewardsHistory = [];991 let successCount = 0;992 let totalCompletionSteps = 0;993 let bestPathSteps = Infinity;994 let leaderboard = [];995 let currentEpisodeReward = 0;996 let isOptimalMode = false;997 let lastActionWasExploration = false;998 999 // Agent parameters1000 let learningRate = 0.1;1001 let discountFactor = 0.9;1002 let explorationRate = 0.3;1003 let animationSpeed = 300;1004 1005 // DOM Elements1006 const loader = document.getElementById("loader");1007 const mainContent = document.getElementById("main-content");1008 const gridContainer = document.getElementById("grid-container");1009 const qValuesContainer = document.getElementById("q-values-container");1010 const startButton = document.getElementById("start-button");1011 const stepButton = document.getElementById("step-button");1012 const resetButton = document.getElementById("reset-button");1013 const learningModeBtn = document.getElementById("learning-mode");1014 const optimalModeBtn = document.getElementById("optimal-mode");1015 const episodeCountEl = document.getElementById("episode-count");1016 const stepCountEl = document.getElementById("step-count");1017 const successRateEl = document.getElementById("success-rate");1018 const avgCompletionEl = document.getElementById("avg-completion");1019 const leaderboardBody = document.getElementById("leaderboard-body");1020 const notification = document.getElementById("notification");1021 const stepsChart = document.getElementById("steps-chart");1022 1023 // Parameter sliders1024 const learningRateSlider = document.getElementById("learning-rate");1025 const learningRateValue = document.getElementById("learning-rate-value");1026 const discountFactorSlider = document.getElementById("discount-factor");1027 const discountFactorValue = document.getElementById(1028 "discount-factor-value"1029 );1030 const explorationRateSlider = document.getElementById("exploration-rate");1031 const explorationRateValue = document.getElementById(1032 "exploration-rate-value"1033 );1034 const animationSpeedSlider = document.getElementById("animation-speed");1035 const animationSpeedValue = document.getElementById(1036 "animation-speed-value"1037 );1038 1039 // Update parameters from sliders1040 learningRateSlider.addEventListener("input", function () {1041 learningRate = parseFloat(this.value);1042 learningRateValue.textContent = learningRate.toFixed(2);1043 });1044 1045 discountFactorSlider.addEventListener("input", function () {1046 discountFactor = parseFloat(this.value);1047 discountFactorValue.textContent = discountFactor.toFixed(2);1048 });1049 1050 explorationRateSlider.addEventListener("input", function () {1051 explorationRate = parseFloat(this.value);1052 explorationRateValue.textContent = explorationRate.toFixed(2);1053 });1054 1055 animationSpeedSlider.addEventListener("input", function () {1056 animationSpeed = parseInt(this.value);1057 animationSpeedValue.textContent = animationSpeed + "ms";1058 });1059 1060 // Mode selection1061 learningModeBtn.addEventListener("click", function () {1062 if (isOptimalMode) {1063 isOptimalMode = false;1064 learningModeBtn.classList.add("active");1065 optimalModeBtn.classList.remove("active");1066 }1067 });1068 1069 optimalModeBtn.addEventListener("click", function () {1070 if (!isOptimalMode) {1071 isOptimalMode = true;1072 optimalModeBtn.classList.add("active");1073 learningModeBtn.classList.remove("active");1074 }1075 });1076 1077 // Create star animation for header1078 function createStars() {1079 const starsContainer = document.getElementById("stars-container");1080 const numStars = 40;1081 1082 for (let i = 0; i < numStars; i++) {1083 setTimeout(() => {1084 const star = document.createElement("div");1085 star.className = "star";1086 1087 // Random position1088 const x = Math.random() * 100;1089 const y = Math.random() * 100;1090 star.style.left = `${x}%`;1091 star.style.top = `${y}%`;1092 1093 // Random size1094 const size = Math.random() * 4 + 1;1095 star.style.width = `${size}px`;1096 star.style.height = `${size}px`;1097 1098 // Random direction1099 const tx = (Math.random() - 0.5) * 200;1100 const ty = (Math.random() - 0.5) * 200;1101 star.style.setProperty("--tx", `${tx}px`);1102 star.style.setProperty("--ty", `${ty}px`);1103 1104 starsContainer.appendChild(star);1105 1106 // Remove after animation1107 setTimeout(() => {1108 if (star) star.remove();1109 }, 2000);1110 }, i * 50);1111 }1112 1113 // Repeat the animation1114 setTimeout(createStars, 4000);1115 }1116 1117 // Initialize environment1118 function initializeGrid() {1119 gridContainer.innerHTML = "";1120 qValuesContainer.innerHTML = "";1121 1122 // Create grid cells and q-value cells1123 for (let i = 0; i < numStates; i++) {1124 // Grid cell1125 const cell = document.createElement("div");1126 cell.className = "grid-cell";1127 cell.dataset.index = i;1128 1129 if (obstacles.includes(i)) {1130 cell.classList.add("obstacle");1131 cell.textContent = "๐ซ";1132 } else if (i === goal) {1133 cell.classList.add("goal");1134 cell.textContent = "๐";1135 }1136 1137 gridContainer.appendChild(cell);1138 1139 // Q-value cell1140 const qCell = document.createElement("div");1141 qCell.className = "q-cell";1142 qCell.dataset.index = i;1143 1144 // Add arrows for each action1145 const directions = ["up", "right", "down", "left"];1146 for (let j = 0; j < numActions; j++) {1147 const arrow = document.createElement("div");1148 arrow.className = `q-arrow ${directions[j]}`;1149 arrow.dataset.action = j;1150 arrow.style.setProperty("--opacity", "0.2");1151 qCell.appendChild(arrow);1152 1153 const value = document.createElement("div");1154 value.className = "q-value";1155 value.dataset.action = j;1156 value.textContent = "0.00";1157 qCell.appendChild(value);1158 }1159 1160 // Position the q-values within the cell1161 const values = qCell.querySelectorAll(".q-value");1162 values[0].style.position = "absolute";1163 values[0].style.top = "5px";1164 values[0].style.left = "50%";1165 values[0].style.transform = "translateX(-50%)";1166 1167 values[1].style.position = "absolute";1168 values[1].style.top = "50%";1169 values[1].style.right = "5px";1170 values[1].style.transform = "translateY(-50%)";1171 1172 values[2].style.position = "absolute";1173 values[2].style.bottom = "5px";1174 values[2].style.left = "50%";1175 values[2].style.transform = "translateX(-50%)";1176 1177 values[3].style.position = "absolute";1178 values[3].style.top = "50%";1179 values[3].style.left = "5px";1180 values[3].style.transform = "translateY(-50%)";1181 1182 qValuesContainer.appendChild(qCell);1183 }1184 1185 // Add agent1186 const startCell = document.querySelector(1187 `.grid-cell[data-index="${agentPos}"]`1188 );1189 const agent = document.createElement("div");1190 agent.className = "agent";1191 agent.id = "agent";1192 startCell.appendChild(agent);1193 }1194 1195 // Create a simple chart without external libraries1196 function updateSimpleChart() {1197 if (stepsHistory.length === 0) return;1198 1199 // Clear chart1200 stepsChart.innerHTML = "";