NyxKrage/LLM-Model-VRAM-Calculator
514
1<!DOCTYPE html>2<html lang="en">3 <head>4 <meta charset="UTF-8" />5 <meta name="viewport" content="width=device-width, initial-scale=1.0" />6 <script>7 function strToHtml(str) {8 let parser = new DOMParser();9 return parser.parseFromString(str, "text/html");10 }11 12 //Short, jQuery-independent function to read html table and write them into an Array.13 //Kudos to RobG at StackOverflow14 function tableToObj(table) {15 var rows = table.rows;16 var propCells = rows[0].cells;17 var propNames = [];18 var results = [];19 var obj, row, cells;20 21 // Use the first row for the property names22 // Could use a header section but result is the same if23 // there is only one header row24 for (var i = 0, iLen = propCells.length; i < iLen; i++) {25 propNames.push(26 (propCells[i].textContent || propCells[i].innerText).trim()27 );28 }29 30 // Use the rows for data31 // Could use tbody rows here to exclude header & footer32 // but starting from 1 gives required result33 for (var j = 1, jLen = rows.length; j < jLen; j++) {34 cells = rows[j].cells;35 obj = {};36 37 for (var k = 0; k < iLen; k++) {38 obj[propNames[k]] = (39 cells[k].textContent || cells[k].innerText40 ).trim();41 }42 results.push(obj);43 }44 return results;45 }46 47 function formatGpu(gpus) {48 return gpus.map(49 (g) => `${g["Product Name"]} - ${g["Memory"].split(",")[0]}`50 );51 }52 53 const gguf_quants = {54 "IQ1_S": 1.56,55 "IQ2_XXS": 2.06,56 "IQ2_XS": 2.31,57 "IQ2_S": 2.5,58 "IQ2_M": 2.7,59 "IQ3_XXS": 3.06,60 "IQ3_XS": 3.3,61 "Q2_K": 3.35,62 "Q3_K_S": 3.5,63 "IQ3_S": 3.5,64 "IQ3_M": 3.7,65 "Q3_K_M": 3.91,66 "Q3_K_L": 4.27,67 "IQ4_XS": 4.25,68 "IQ4_NL": 4.5,69 "Q4_0": 4.55,70 "Q4_K_S": 4.58,71 "Q4_K_M": 4.85,72 "Q5_0": 5.54,73 "Q5_K_S": 5.54,74 "Q5_K_M": 5.69,75 "Q6_K": 6.59,76 "Q8_0": 8.5,77 }78 79 async function modelConfig(hf_model, hf_token) {80 auth = hf_token == "" ? {} : {81 headers: {82 'Authorization': `Bearer ${hf_token}`83 }84 }85 let config_res = await fetch(86 `https://huggingface.co/${hf_model}/raw/main/config.json`, auth87 )88 if (config_res.status === 401) {89 throw new Error("Model is either private or gated, you must provide an access token")90 }91 if (config_res.status === 403) {92 throw new Error("Model is either private or gated and provided access token does not have access to the repo")93 }94 let config = await config_res.json()95 let model_size = 096 let dtype = parseInt(config["torch_dtype"].replaceAll(/\D/g, '')) / 897 if ("text_config" in config) {98 config = config["text_config"]99 }100 try {101 model_size = (await fetch(`https://huggingface.co/${hf_model}/resolve/main/model.safetensors.index.json`, auth).then(r => r.json()))["metadata"]["total_size"] / dtype102 if (isNaN(model_size)) {103 throw new Error("no size in safetensors metadata")104 }105 } catch (e) {106 try {107 model_size = (await fetch(`https://huggingface.co/${hf_model}/resolve/main/pytorch_model.bin.index.json`, auth).then(r => r.json()))["metadata"]["total_size"] / dtype108 if (isNaN(model_size)) {109 throw new Error("no size in pytorch metadata")110 }111 } catch {112 try {113 model_size = (await fetch(114 `https://huggingface.co/api/models/${hf_model}`115 ).then(r => r.json()))["safetensors"]["total"]116 if (isNaN(model_size)) {117 throw new Error("no size in pytorch metadata")118 }119 } catch {120 throw new Error("Couldn't determine model size from safetensor/pytorch index metadata nor from the model card. If the model is an unsharded pytorch model, it is not supported by this calculator.")121 }122 }123 }124 config.parameters = model_size125 return config126 }127 128 function inputBuffer(context=8192, model_config, bsz=512) {129 /* Calculation taken from github:ggerganov/llama.cpp/llama.cpp:11248130 ctx->inp_tokens = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_batch);131 ctx->inp_embd = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, hparams.n_embd, cparams.n_batch);132 ctx->inp_pos = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_batch);133 ctx->inp_KQ_mask = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, cparams.n_ctx, cparams.n_batch);134 ctx->inp_K_shift = ggml_new_tensor_1d(ctx->ctx_input, GGML_TYPE_I32, cparams.n_ctx);135 ctx->inp_sum = ggml_new_tensor_2d(ctx->ctx_input, GGML_TYPE_F32, 1, cparams.n_batch);136 137 n_embd is hidden size (github:ggeranov/llama.cpp/convert.py:248)138 */139 const inp_tokens = bsz140 const inp_embd = model_config["hidden_size"] * bsz141 const inp_pos = bsz142 const inp_KQ_mask = context * bsz143 const inp_K_shift = context144 const inp_sum = bsz145 146 return inp_tokens + inp_embd + inp_pos + inp_KQ_mask + inp_K_shift + inp_sum147 }148 149 function computeBuffer(context=8192, model_config, bsz=512) {150 if (bsz != 512) {151 alert("batch size other than 512 is currently not supported for the compute buffer, using batchsize 512 for compute buffer calculation, end result result will be an overestimatition")152 }153 return (context / 1024 * 2 + 0.75) * model_config["num_attention_heads"] * 1024 * 1024154 }155 156 function kvCache(context=8192, model_config, cache_bit=16) {157 const n_gqa = model_config["num_attention_heads"] / model_config["num_key_value_heads"]158 const n_embd_gqa = model_config["hidden_size"] / n_gqa159 const n_elements = n_embd_gqa * (model_config["num_hidden_layers"] * context)160 const size = 2 * n_elements161 return size * (cache_bit / 8)162 }163 164 function contextSize(context=8192, model_config, bsz=512, cache_bit=16) {165 return Number.parseFloat((inputBuffer(context, model_config, bsz) + kvCache(context, model_config, cache_bit) + computeBuffer(context, model_config, bsz)).toFixed(2))166 }167 168 function modelSize(model_config, bpw=4.5) {169 return Number.parseFloat((model_config["parameters"] * bpw / 8).toFixed(2))170 }171 172 async function calculateSizes(format) {173 try {174 const model_config = await modelConfig(document.getElementById("modelsearch").value.replace("https://huggingface.co/", ""), document.getElementById("hf_token").value)175 const context = parseInt(document.getElementById("contextsize").value)176 let bsz = 512177 let cache_bit = 16178 let bpw = 0179 if (format === "gguf") {180 bsz = parseInt(document.getElementById("batchsize").value)181 bpw = gguf_quants[document.getElementById("quantsize").innerText]182 183 } else if (format == "exl2") {184 cache_bit = Number.parseInt(document.getElementById("kvCache").value)185 bpw = Number.parseFloat(document.getElementById("bpw").value)186 }187 188 const model_size = modelSize(model_config, bpw)189 const context_size = contextSize(context, model_config, bsz, cache_bit)190 const total_size = ((model_size + context_size) / 2**30)191 document.getElementById("resultmodel").innerText = (model_size / 2**30).toFixed(2)192 document.getElementById("resultcontext").innerText = (context_size / 2**30).toFixed(2)193 const result_total_el = document.getElementById("resulttotal");194 result_total_el.innerText = total_size.toFixed(2)195 196 const gpu = document.getElementById("gpusearch").value197 if (gpu !== "") {198 const vram = parseFloat(gpu.split("-")[1].replace("GB", "").trim())199 if (vram - total_size > 0.5) {200 result_total_el.style.backgroundColor = "#bef264"201 } else if (vram - total_size > 0) {202 result_total_el.style.backgroundColor = "#facc15"203 } else {204 result_total_el.style.backgroundColor = "#ef4444"205 }206 }207 } catch(e) {208 alert(e);209 }210 }211 </script>212 <link href="./styles.css" rel="stylesheet">213 <title>Can I run it? - LLM VRAM Calculator</title>214 </head>215 <body class="p-8">216 <div x-data="{ format: 'gguf' }" class="flex flex-col max-h-screen items-center mt-16 gap-10">217 <h1 class="text-xl font-semibold leading-6 text-gray-900">218 LLM Model, Can I run it?219 </h1>220 <p>221 To support gated or private repos, you need to <a href="https://huggingface.co/settings/tokens" style="color: #4444ff"><b>create an authentification token</b></a>, to check the box <span style="color: #6e1818"><b>"Read access to contents of all public gated repos you can access"</b></span> and then enter the token in the field below.222 </p>223 224 <div class="flex flex-col gap-10">225 <div class="w-auto flex flex-col gap-4">226 <!-- Huggingface Authentification Token -->227 <div228 class="relative"229 x-data="{230 results: null,231 query: null232 }"233 >234 <label235 for="gpusearch"236 class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900"237 >Huggingface Token (optional)</label238 >239 <input240 class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"241 id="hf_token"242 />243 </div>244 <!-- GPU Selector -->245 <div246 class="relative"247 x-data="{248 results: null,249 query: null250 }"251 >252 <label253 for="gpusearch"254 class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900"255 >GPU (optional)</label256 >257 <input258 class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"259 placeholder="GeForce RTX 3090 - 24 GB"260 id="gpusearch"261 name="gpusearch"262 list="gpulist"263 x-model="query"264 @keypress.debounce.150ms="results = query === '' ? [] : formatGpu(tableToObj(strToHtml(await fetch('https://corsproxy.io/?https://www.techpowerup.com/gpu-specs/?ajaxsrch=' + query).then(r => r.text())).querySelector('table')))"265 />266 <datalist id="gpulist">267 <template x-for="item in results">268 <option :value="item" x-text="item"></option>269 </template>270 </datalist>271 </div>272 <!-- Model Selector -->273 274 275 <div class="flex flex-row gap-4 relative">276 <label277 for="contextsize"278 class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900"279 >280 Model (unquantized)281 </label>282 <div283 class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"284 x-data="{285 open: false,286 value: 'Nexusflow/Starling-LM-7B-beta',287 results: null,288 toggle() {289 if (this.open) {290 return this.close()291 }292 293 this.$refs.input.focus()294 295 this.open = true296 },297 close(focusAfter) {298 if (! this.open) return299 300 this.open = false301 302 focusAfter && focusAfter.focus()303 }304 }"305 x-on:keydown.escape.prevent.stop="close($refs.input)"306 x-id="['model-typeahead']"307 class="relative"308 >309 <!-- Input -->310 <input311 id="modelsearch"312 x-ref="input"313 x-on:click="toggle()"314 @keypress.debounce.150ms="results = (await315 fetch('https://huggingface.co/api/quicksearch?type=model&q=' +316 encodeURIComponent(value)).then(r => r.json())).models.filter(m => !m.id.includes('GGUF') && !m.id.includes('AWQ') && !m.id.includes('GPTQ') && !m.id.includes('exl2'));"317 :aria-expanded="open"318 :aria-controls="$id('model-typeahead')"319 x-model="value"320 class="flex justify-between items-center gap-2 w-full"321 />322 323 <!-- Panel -->324 <div325 x-ref="panel"326 x-show="open"327 x-transition.origin.top.left328 x-on:click.outside="close($refs.input)"329 :id="$id('model-typeahead')"330 style="display: none"331 class="absolute left-0 mt-4 w-full rounded-md bg-white shadow-sm ring-1 ring-inset ring-gray-300 z-10"332 >333 <template x-for="result in results">334 <a335 @click="value = result.id; close($refs.input)"336 x-text="result.id"337 class="flex cursor-pointer items-center gap-2 w-full first-of-type:rounded-t-md last-of-type:rounded-b-md px-4 py-2.5 text-left text-sm hover:bg-gray-500/5 disabled:text-gray-500"338 ></a>339 </template>340 </div>341 </div>342 </div>343 344 345 <!-- Context Size Selector -->346 <div class="relative">347 <label348 for="contextsize"349 class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900"350 >351 Context Size352 </label>353 <input354 value="8192"355 type="number"356 name="contextsize"357 id="contextsize"358 step="1024"359 class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"360 />361 </div>362 <!-- Quant Format Selector -->363 <div class="relative">364 <label365 class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900"366 >Quant Format</label367 >368 <fieldset369 x-model="format"370 class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"371 >372 <legend class="sr-only">Quant format</legend>373 <div374 class="space-y-4 sm:flex sm:items-center sm:space-x-10 sm:space-y-0"375 >376 <div class="flex items-center">377 <input378 id="gguf-format"379 name="quant-format"380 type="radio"381 value="gguf"382 checked383 class="h-4 w-4 border-gray-300 text-indigo-600 focus:ring-indigo-600"384 />385 <label386 for="gguf-format"387 class="ml-3 block text-sm font-medium leading-6 text-gray-900"388 >GGUF</label389 >390 </div>391 <div class="flex items-center">392 <input393 id="exl2-format"394 name="quant-format"395 type="radio"396 value="exl2"397 class="h-4 w-4 border-gray-300 text-indigo-600 focus:ring-indigo-600"398 />399 <label400 for="exl2-format"401 class="ml-3 block text-sm font-medium leading-6 text-gray-900"402 >EXL2</label403 >404 </div>405 <div class="flex items-center">406 <input407 id="gptq-format"408 name="quant-format"409 type="radio"410 disabled411 value="gptq"412 class="h-4 w-4 border-gray-300 text-indigo-600 focus:ring-indigo-600"413 />414 <label415 for="gptq-format"416 class="ml-3 block text-sm font-medium leading-6 text-gray-900"417 >GPTQ (coming soon)</label418 >419 </div>420 </div>421 </fieldset>422 </div>423 <!-- EXL2 Options -->424 <div x-show="format === 'exl2'" class="flex flex-row gap-4">425 <div class="relative flex-grow">426 <label427 for="bpw"428 class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900"429 >430 BPW431 </label>432 <input433 value="4.5"434 type="number"435 step="0.01"436 id="bpw"437 name="bpw"438 class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"439 />440 </div>441 <div442 class="flex-shrink relative rounded-md"443 >444 <div445 class="w-fit p-3 h-full flex items-center gap-2 justify-center rounded-md border-0 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"446 >447 <label448 for="kvCache"449 class="inline-block bg-white text-xs font-medium text-gray-900"450 >451 KV Cache452 </label>453 <select id="kvCache" name="kvCache">454 <option value="16">16 bit</option>455 <option value="8">8 bit</option>456 <option value="4">4 bit</option>457 </select>458 </div>459 </div>460 </div>461 <!-- GGUF Options -->462 <div x-show="format === 'gguf'" class="relative">463 <div class="flex flex-row gap-4">464 <label465 for="contextsize"466 class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900"467 >468 Quantization Size469 </label>470 <div471 class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"472 x-data="{473 open: false,474 value: '',475 toggle() {476 if (this.open) {477 return this.close()478 }479 480 this.$refs.button.focus()481 482 this.open = true483 },484 close(focusAfter) {485 if (! this.open) return486 487 this.open = false488 489 focusAfter && focusAfter.focus()490 }491 }"492 x-on:keydown.escape.prevent.stop="close($refs.button)"493 x-id="['dropdown-button']"494 class="relative"495 >496 <!-- Button -->497 <button498 x-ref="button"499 x-on:click="toggle()"500 :aria-expanded="open"501 :aria-controls="$id('dropdown-button')"502 type="button"503 id="quantsize"504 x-text="value.length === 0 ? 'Q4_K_S' : value"505 class="flex justify-between items-center gap-2 w-full"506 >507 Q4_K_S508 509 <!-- Heroicon: chevron-down -->510 <svg511 xmlns="http://www.w3.org/2000/svg"512 class="h-5 w-5 text-gray-400"513 viewBox="0 0 20 20"514 fill="currentColor"515 >516 <path517 fill-rule="evenodd"518 d="M5.293 7.293a1 1 0 011.414 0L10 10.586l3.293-3.293a1 1 0 111.414 1.414l-4 4a1 1 0 01-1.414 0l-4-4a1 1 0 010-1.414z"519 clip-rule="evenodd"520 />521 </svg>522 </button>523 524 <!-- Panel -->525 <div526 x-data="{ quants: [527 'IQ1_S',528 'IQ2_XXS',529 'IQ2_XS',530 'IQ2_S',531 'IQ2_M',532 'IQ3_XXS',533 'IQ3_XS',534 'Q2_K',535 'Q3_K_S',536 'IQ3_S',537 'IQ3_M',538 'Q3_K_M',539 'Q3_K_L',540 'IQ4_XS',541 'IQ4_NL',542 'Q4_0',543 'Q4_K_S',544 'Q4_K_M',545 'Q5_0',546 'Q5_K_S',547 'Q5_K_M',548 'Q6_K',549 'Q8_0'550 ]}"551 x-ref="panel"552 x-show="open"553 x-transition.origin.top.left554 x-on:click.outside="close($refs.button)"555 :id="$id('dropdown-button')"556 style="display: none"557 class="absolute left-0 mt-4 w-full rounded-md bg-white shadow-sm ring-1 ring-inset ring-gray-300 z-10"558 >559 <template x-for="quant in quants">560 <a561 @click="value = quant; close($refs.button)"562 x-text="quant"563 class="flex cursor-pointer items-center gap-2 w-full first-of-type:rounded-t-md last-of-type:rounded-b-md px-4 py-2.5 text-left text-sm hover:bg-gray-500/5 disabled:text-gray-500"564 ></a>565 </template>566 </div>567 </div>568 <div class="relative">569 <label570 for="batchsize"571 class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900"572 >573 Batch Size574 </label>575 <input576 value="512"577 type="number"578 step="128"579 id="batchsize"580 class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"581 />582 </div>583 </div>584 </div>585 <button586 type="button"587 class="rounded-md bg-slate-800 px-3 py-2 text-sm font-semibold text-white shadow-sm hover:bg-slate-700 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-indigo-600"588 @click="calculateSizes(format)"589 >590 Submit591 </button>592 </div>593 <div class="w-auto flex flex-col gap-4">594 <div class="relative">595 <label596 class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900"597 >598 Model Size (GB)599 </label>600 <div601 id="resultmodel"602 class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"603 >4.20</div>604 </div>605 <div class="relative">606 <label607 class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900"608 >609 Context Size (GB)610 </label>611 <div612 id="resultcontext"613 class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"614 >6.90</div>615 </div>616 <div class="relative">617 <label618 class="absolute -top-2 left-2 inline-block bg-white px-1 text-xs font-medium text-gray-900"619 >620 Total Size (GB)621 </label>622 <div623 id="resulttotal"624 class="block w-full rounded-md border-0 p-3 text-gray-900 shadow-sm ring-1 ring-inset ring-gray-300 placeholder:text-gray-400 focus:ring-2 focus:ring-inset focus:ring-indigo-600 sm:text-sm sm:leading-6"625 >420.69</div>626 </div>627 </div>628 </div>629 </div>630 <script631 src="https://cdn.jsdelivr.net/npm/alpinejs@3.x.x/dist/cdn.min.js"632 ></script>633 <script defer>634 calculateSizes("gguf")635 </script>636 </body>637</html>638