aleada/pack-integrity-check
0
1// Browser front end. Every request goes straight from the visitor to
2// huggingface.co — this page has no backend and holds no token, so it
3// reads exactly what the visitor could read themselves.
4
5import {
6 KNOWN_METHODS, EXCLUSION_FIELD, covers, moduleTree, modulePaths,
7 indexGaps, resolveMethod, sameModule, unmatchedExclusions,
8} from './pack_check.js';
9import { progress } from './progress.js';
10import { ensureNextStepStyles, nextSteps, repoFromUrl } from './tools.js';
11
12const HF = 'https://huggingface.co';
13const $ = (id) => document.getElementById(id);
14
15// A link from another tool carries the model over; arriving at an empty
16// form is what makes five tools feel like five tools instead of one.
17ensureNextStepStyles();
18const carried = repoFromUrl();
19if (carried) { $('repo').value = carried; }
20
21
22// Tensor tails that mark a converted weight, used to compare a pack
23// against its source by module rather than by tensor name.
24const EXPECTED_UNQUANTIZED = new Set([
25 'embed_tokens', 'lm_head', 'embed_out', 'A_log', 'dt_bias', 'conv1d',
26]);
27
28function expectedUnquantized(module) {
29 const tail = module.split('.').pop();
30 return EXPECTED_UNQUANTIZED.has(tail)
31 || tail.endsWith('norm') || module.includes('.norm')
32 || module.includes('embed') // embeddings are not LinearBase
33 || tail.includes('conv'); // convolutions are not either
34}
35
36const esc = (s) => String(s).replace(/[&<>]/g,
37 (c) => ({ '&': '&', '<': '<', '>': '>' }[c]));
38
39async function json(url) {
40 const r = await fetch(url);
41 if (!r.ok) throw new Error(`${r.status} ${r.statusText}`);
42 return r.json();
43}
44
45/** Tensor name -> shape, from safetensors headers only.
46 *
47 * Two ranged requests per shard: eight bytes for the header length,
48 * then the header itself. A file of any size costs kilobytes. */
49async function safetensorNames(repo, pg = null) {
50 const info = await json(`${HF}/api/models/${repo}`);
51 const files = (info.siblings ?? [])
52 .map((s) => s.rfilename)
53 .filter((f) => f.endsWith('.safetensors'));
54
55 // The shard count is only knowable here, after the file listing — so
56 // this is where an indeterminate bar can become a real proportion.
57 if (pg) pg.total(files.length).start(`reading ${files.length} `
58 + `safetensors header${files.length === 1 ? '' : 's'} from ${repo}`);
59
60 const out = new Map();
61 await Promise.all(files.map(async (file) => {
62 const url = `${HF}/${repo}/resolve/main/${file}`;
63 const head = await fetch(url, { headers: { Range: 'bytes=0-7' } });
64 if (!head.ok) throw new Error(`${file}: ${head.status}`);
65 const len = Number(new DataView(await head.arrayBuffer())
66 .getBigUint64(0, true));
67 const body = await fetch(url, {
68 headers: { Range: `bytes=8-${8 + len - 1}` },
69 });
70 if (!body.ok) throw new Error(`${file}: ${body.status}`);
71 for (const [name, entry] of Object.entries(JSON.parse(await body.text()))) {
72 if (name !== '__metadata__') out.set(name, entry.shape ?? []);
73 }
74 if (pg) pg.step();
75 }));
76 return { names: out, info, files: (info.siblings ?? []).map((s) => s.rfilename) };
77}
78
79function declared(quant, method, module, entries) {
80 if (covers(quant, method, module)) return true;
81 return entries.some((e) => {
82 const t = String(e);
83 return !t.startsWith('re:') && !t.startsWith('-:') && !t.startsWith('+:')
84 && sameModule(t, module);
85 });
86}
87
88function card(kind, title, html) {
89 return `<div class="card ${kind}"><h3>${esc(title)}</h3>${html}</div>`;
90}
91
92const list = (items, cap = 14) =>
93 '<ul>' + items.slice(0, cap).map((i) => `<li>${esc(i)}</li>`).join('')
94 + (items.length > cap ? `<li>… and ${items.length - cap} more</li>` : '')
95 + '</ul>';
96
97async function check(repo, pg = null) {
98 const { names, info, files } = await safetensorNames(repo, pg);
99 if (names.size === 0) {
100 return card('problem', 'Nothing to read',
101 '<p>This repo has no <code>.safetensors</code> files.</p>');
102 }
103
104 const config = await json(`${HF}/${repo}/resolve/main/config.json`);
105 const quant = config.quantization_config ?? {};
106 const method = resolveMethod(quant);
107 const known = KNOWN_METHODS.has(method);
108 const base = [].concat(info.cardData?.base_model ?? [])[0];
109 const tied = Boolean(config.tie_word_embeddings
110 ?? config.text_config?.tie_word_embeddings);
111
112 const meta = `<div class="meta">
113 <span><code>${esc(repo)}</code></span>
114 <span>quantization: <code>${esc(method)}</code></span>
115 <span>tensors: <code>${names.size.toLocaleString()}</code></span>
116 ${base ? `<span>source: <code>${esc(base)}</code></span>` : ''}
117 </div>`;
118
119 if (Object.keys(quant).length === 0) {
120 return meta + card('clear', 'Not a quantized pack',
121 '<p>No <code>quantization_config</code>, so nothing here could have '
122 + 'been quantized away.</p>');
123 }
124 if (!known) {
125 return meta + card('check', `Format “${method}” is not implemented here`,
126 '<p>The checks reproduce the exclusion rules of compressed-tensors, '
127 + 'awq, gptq, bitsandbytes, modelopt and auto-round from vLLM\'s '
128 + 'source. Yours is not among them, so they are skipped rather than '
129 + 'guessed at.</p>');
130 }
131
132 const out = [];
133 const tree = moduleTree(names.keys());
134
135 // 0 — the index against the files. The only check here that holds
136 // for any model made by any tool: an index naming a shard the repo
137 // lacks is broken everywhere, because the loader resolves weights
138 // through it. Ours had exactly this before it was repaired.
139 if (files.includes('model.safetensors.index.json')) {
140 try {
141 const wm = (await json(`${HF}/${repo}/resolve/main/model.safetensors.index.json`)).weight_map ?? {};
142 const gaps = indexGaps(wm, names.keys(), files);
143 if (gaps.missing_tensors.length || gaps.missing_shards.length) {
144 out.push(card('problem', 'The index promises weights the repo does not have',
145 '<p>`model.safetensors.index.json` maps tensors to shards, and a '
146 + 'loader resolves every weight through it. These entries point at '
147 + 'nothing, so the pack cannot load:</p>'
148 + (gaps.missing_shards.length
149 ? '<p><strong>Missing shards:</strong></p>' + list(gaps.missing_shards, 6) : '')
150 + (gaps.missing_tensors.length
151 ? `<p><strong>${gaps.missing_tensors.length} tensors with no file:</strong></p>`
152 + list(gaps.missing_tensors, 8) : '')));
153 }
154 } catch { /* an unreadable index is reported by the checks below */ }
155 }
156
157 // 1 — entries that name nothing
158 const stray = unmatchedExclusions(quant, method, tree, tied);
159 if (stray.length) {
160 out.push(card('problem',
161 `${stray.length} exclusion ${stray.length === 1 ? 'entry names' : 'entries name'} no module in this model`,
162 '<p>They protect nothing, so whatever they were meant to keep was '
163 + 'quantized:</p>' + list(stray)
164 + '<p>A common cause is a vision tower: Llama and Pixtral call it '
165 + '<code>vision_tower</code>, Qwen calls it <code>visual</code>.</p>'));
166 }
167
168 // 2 — modules that did not survive from the source
169 if (base && base !== repo) {
170 try {
171 // The source model is a second read of the same size, and the
172 // reason this page can sit silent for twice as long as it looks
173 // like it should. Say so rather than letting the bar restart
174 // without explanation.
175 if (pg) pg.start(`reading the source model ${base}`);
176 const src = await safetensorNames(base, pg);
177 const have = modulePaths(names.keys());
178 const dropped = [...modulePaths(src.names.keys())]
179 .filter((m) => !have.has(m)).sort();
180 if (dropped.length) {
181 const groups = {};
182 for (const m of dropped) {
183 const head = m.split('.')[0];
184 groups[head] = (groups[head] ?? 0) + 1;
185 }
186 out.push(card('problem',
187 `${dropped.length} modules of the source are absent here`,
188 '<p><code>from_pretrained</code> does not materialise tensors the '
189 + 'AutoModel class has no slot for, so they never reach the '
190 + 'quantizer and never reach the artifact. For a '
191 + 'multi-token-prediction head the symptom is 0% draft acceptance, '
192 + 'with nothing in the logs.</p>'
193 + list(Object.entries(groups)
194 .sort((a, b) => b[1] - a[1])
195 .map(([k, v]) => `${k} — ${v} modules`), 10)));
196 }
197 } catch (e) {
198 out.push(card('check', 'Could not read the source model',
199 `<p><code>${esc(base)}</code> is named as the base but could not be `
200 + `read (${esc(e.message)}), so that check was skipped.</p>`));
201 }
202 }
203
204 // 3 — present, unquantized, undeclared
205 const entries = Object.keys(quant).length
206 ? [].concat(quant[EXCLUSION_FIELD[method]] ?? []).flatMap(
207 (v) => (typeof v === 'object' && v !== null ? Object.keys(v) : v))
208 : [];
209 const raw = quant[EXCLUSION_FIELD[method]];
210 const entryList = Array.isArray(raw) ? raw
211 : (raw && typeof raw === 'object' ? Object.keys(raw) : entries);
212 const undeclared = [...new Set([...names.entries()]
213 .filter(([n, shape]) => n.endsWith('.weight') && shape.length === 2)
214 .map(([n]) => n.slice(0, -'.weight'.length))
215 .filter((m) => !expectedUnquantized(m)
216 && !declared(quant, method, m, entryList)))].sort();
217
218 if (undeclared.length) {
219 out.push(card('check',
220 `${undeclared.length} modules are at source precision but not declared`,
221 '<p>A runtime builds these quantized, looks for a packed weight, finds '
222 + 'a plain one, and skips it — leaving the module randomly initialised '
223 + 'while the model still answers correctly.</p>' + list(undeclared, 12)
224 + '<p>This is a <strong>prediction</strong>, not a measurement. Confirm '
225 + 'it by serving the pack and grepping the startup log for '
226 + '<code>not found in params_dict</code>.</p>'));
227 }
228
229 if (!out.length) {
230 out.push(card('clear', 'Nothing found',
231 '<p>Every exclusion entry names a real module, no source module is '
232 + 'missing, and nothing is left at source precision without being '
233 + 'declared.</p>'));
234 }
235 return meta + out.join('');
236}
237
238async function submit(event) {
239 event?.preventDefault();
240 let repo = $('repo').value.trim().replace(/\/+$/, '');
241 if (!repo) return;
242 if (repo.startsWith('http')) repo = repo.split('huggingface.co/').pop();
243
244 const out = $('out');
245 const button = $('go');
246 button.disabled = true;
247 // Indeterminate until the file listing arrives: nothing before it says
248 // how many shards there are to read.
249 const pg = progress(out).start(`reading ${repo}`);
250 try {
251 const html = await check(repo, pg);
252 pg.done();
253 out.innerHTML = html + nextSteps('integrity', repo);
254 } catch (e) {
255 out.innerHTML = card('problem', `Could not check ${repo}`,
256 `<p><code>${esc(e.message)}</code></p><p>If the repo is gated or `
257 + `private, this page cannot read it — it has no token.</p>`);
258 } finally {
259 button.disabled = false;
260 }
261}
262
263$('form').addEventListener('submit', submit);
264// Run once on load against the pre-filled example. A checker whose
265// first screen is empty asks the visitor to trust it before it has
266// shown them anything.
267submit();
268for (const b of document.querySelectorAll('[data-ex]')) {
269 b.addEventListener('click', () => {
270 $('repo').value = b.dataset.ex;
271 submit();
272 });
273}
274