SlayerLab/tokenizers

1#!/usr/bin/env python32"""Build the transparent provisional leaderboard from benchmark results."""3 4from __future__ import annotations5 6import argparse7import csv8import json9import math10import statistics11from collections import Counter12from pathlib import Path13 14import pyarrow as pa15import pyarrow.parquet as pq16 17 18MEAN_DOMAIN_WEIGHT = 0.8019LOWER_QUARTILE_WEIGHT = 0.2020TIE_WINDOW_POINTS = 2.021 22 23def args_parse() -> argparse.Namespace:24    parser = argparse.ArgumentParser()25    parser.add_argument("--results", type=Path, default=Path("results/tokenizer_benchmark.parquet"))26    parser.add_argument("--dataset", type=Path, default=Path("data/train-00000-of-00001.parquet"))27    parser.add_argument(28        "--author-evidence", type=Path, default=Path("results/author_evidence_scores.csv")29    )30    parser.add_argument("--csv", type=Path, default=Path("results/provisional_leaderboard.csv"))31    parser.add_argument("--parquet", type=Path, default=Path("results/provisional_leaderboard.parquet"))32    return parser.parse_args()33 34 35def weighted_ols_residuals(points: list[tuple[float, float]], weights: list[float]) -> list[float]:36    """Residuals for weighted y = intercept + slope*x."""37    total_weight = sum(weights)38    x_mean = sum(weight * point[0] for point, weight in zip(points, weights)) / total_weight39    y_mean = sum(weight * point[1] for point, weight in zip(points, weights)) / total_weight40    denominator = sum(weight * (x - x_mean) ** 2 for (x, _), weight in zip(points, weights))41    slope = sum(42        weight * (x - x_mean) * (y - y_mean)43        for (x, y), weight in zip(points, weights)44    ) / denominator45    intercept = y_mean - slope * x_mean46    return [y - (intercept + slope * x) for x, y in points]47 48 49def author_weighted_percentile_scores(values: list[float], weights: list[float]) -> list[float]:50    """Author-balanced percentile, rescaled so observed best=100 and worst=0."""51    if len(values) == 1:52        return [100.0]53    raw = []54    total = sum(weights)55    for value in values:56        better = sum(weight for candidate, weight in zip(values, weights) if candidate < value)57        tied = sum(weight for candidate, weight in zip(values, weights) if candidate == value)58        raw.append(100.0 * (1.0 - (better + tied / 2) / total))59    low, high = min(raw), max(raw)60    return [100.0 * (score - low) / (high - low) for score in raw]61 62 63def eligibility(row: dict) -> tuple[bool, str]:64    if row["status"] != "ok":65        return False, "benchmark_error"66    if row["adapter_fidelity"] != "exact":67        return False, "core_only_adapter"68    if not row["roundtrip_pass"]:69        return False, "roundtrip_failure"70    if row["unk_rate"] != 0:71        return False, "nonzero_unk_rate"72    return True, "eligible"73 74 75def traceability_score(source: dict) -> tuple[float, str]:76    checks = {77        "source_repo": bool(source.get("source_repo")),78        "source_path": bool(source.get("source_path")),79        "source_commit": bool(source.get("source_commit")),80        "sha256": len(source.get("sha256") or "") == 64,81    }82    return 25.0 * sum(checks.values()), json.dumps(checks, sort_keys=True)83 84 85def main() -> None:86    args = args_parse()87    benchmark = pq.read_table(args.results).to_pylist()88    sources = {row["sha256"]: row for row in pq.read_table(args.dataset).to_pylist()}89    with args.author_evidence.open(newline="", encoding="utf-8") as handle:90        author_evidence = {row["author"]: row for row in csv.DictReader(handle)}91    domains = sorted(json.loads(benchmark[0]["domain_metrics_json"]))92    eligible_indices = [i for i, row in enumerate(benchmark) if eligibility(row)[0]]93    eligible_author_counts = Counter(benchmark[i]["author"] for i in eligible_indices)94    author_weights = [1.0 / eligible_author_counts[benchmark[i]["author"]] for i in eligible_indices]95 96    # Fit the expected log(tokens/word) vs log2(vocabulary size) relation in97    # each domain. Averaging residuals gives every domain equal influence.98    residuals_by_index = {index: [] for index in eligible_indices}99    domain_scores_by_index = {index: [] for index in eligible_indices}100    for domain in domains:101        points = []102        for index in eligible_indices:103            row = benchmark[index]104            tpw = json.loads(row["domain_metrics_json"])[domain]["tokens_per_word"]105            points.append((math.log2(row["size"]), math.log(tpw)))106        residuals = weighted_ols_residuals(points, author_weights)107        percentiles = author_weighted_percentile_scores(residuals, author_weights)108        for index, residual, percentile in zip(eligible_indices, residuals, percentiles):109            residuals_by_index[index].append(residual)110            domain_scores_by_index[index].append(percentile)111 112    adjusted = {113        index: math.exp(sum(values) / len(values))114        for index, values in residuals_by_index.items()115    }116    quality_scores = {}117    mean_domain_scores = {}118    lower_quartile_scores = {}119    for index, scores in domain_scores_by_index.items():120        mean_domain_scores[index] = statistics.mean(scores)121        lower_quartile_scores[index] = statistics.quantiles(scores, n=4, method="inclusive")[0]122        quality_scores[index] = (123            MEAN_DOMAIN_WEIGHT * mean_domain_scores[index]124            + LOWER_QUARTILE_WEIGHT * lower_quartile_scores[index]125        )126 127    rows = []128    for index, source_result in enumerate(benchmark):129        is_eligible, reason = eligibility(source_result)130        source = sources[source_result["sha256"]]131        traceability, traceability_detail = traceability_score(source)132        reviewed_evidence = author_evidence.get(source_result["author"])133        evidence_package_score = (134            float(reviewed_evidence["total"]) * 5 if reviewed_evidence is not None else None135        )136        readiness = 100.0 if source_result["adapter_status"] == "native" else 70.0137        quality = quality_scores.get(index)138        rows.append({139            "rank": None,140            "eligible": is_eligible,141            "eligibility_reason": reason,142            "author": source_result["author"],143            "name": source_result["name"],144            "source_path": source_result["source_path"],145            "vocab_size": source_result["size"],146            "tokens_per_word": source_result["tokens_per_word"],147            "adjusted_compression_index": adjusted.get(index),148            "mean_domain_percentile": mean_domain_scores.get(index),149            "lower_quartile_domain_percentile": lower_quartile_scores.get(index),150            "provisional_quality_score": round(quality, 1) if quality is not None else None,151            "artifact_readiness_score": readiness,152            "evidence_package_score": evidence_package_score,153            "evidence_package_total_20": (154                int(reviewed_evidence["total"]) if reviewed_evidence is not None else None155            ),156            "evidence_package_judgment": (157                reviewed_evidence["evidence_judgment"] if reviewed_evidence is not None else ""158            ),159            "traceability_score": traceability,160            "reference_baseline": source_result["author"] == "kacperwikiel",161            "author_eligible_submission_count": eligible_author_counts.get(source_result["author"], 0),162            "selection_bias_label": "",163            "adapter_status": source_result["adapter_status"],164            "adapter_fidelity": source_result["adapter_fidelity"],165            "runtime": source_result["runtime"],166            "roundtrip_pass": source_result["roundtrip_pass"],167            "unk_rate": source_result["unk_rate"],168            "encode_mb_per_s_info_only": source_result["encode_mb_per_s"],169            "decode_mb_per_s_info_only": source_result["decode_mb_per_s"],170            "traceability_checks_json": traceability_detail,171            "sha256": source_result["sha256"],172        })173 174    ranked = sorted(175        (row for row in rows if row["eligible"]),176        key=lambda row: (-row["provisional_quality_score"], row["source_path"]),177    )178    author_best_path = {}179    for row in ranked:180        author_best_path.setdefault(row["author"], row["source_path"])181    tier_start = 1182    tier_anchor = None183    for position, row in enumerate(ranked, 1):184        score = row["provisional_quality_score"]185        if tier_anchor is None or tier_anchor - score > TIE_WINDOW_POINTS:186            tier_start, tier_anchor = position, score187        row["rank"] = tier_start188        count = row["author_eligible_submission_count"]189        if count == 1:190            row["selection_bias_label"] = "single_submission"191        elif row["source_path"] == author_best_path[row["author"]]:192            row["selection_bias_label"] = f"author_best_of_{count}_selection_bias"193        else:194            row["selection_bias_label"] = f"variant_among_{count}"195    rows.sort(key=lambda row: (196        not row["eligible"],197        row["rank"] or 10**9,198        -(row["provisional_quality_score"] or -1),199        row["source_path"],200    ))201 202    args.csv.parent.mkdir(parents=True, exist_ok=True)203    with args.csv.open("w", newline="", encoding="utf-8") as handle:204        writer = csv.DictWriter(handle, fieldnames=list(rows[0]))205        writer.writeheader()206        writer.writerows(rows)207    pq.write_table(pa.Table.from_pylist(rows), args.parquet, compression="zstd")208    print(f"eligible={len(ranked)} unranked={len(rows)-len(ranked)}")209    for row in ranked[:10]:210        print(211            f"{row['rank']:2}. {row['author']:16} size={row['vocab_size']:6} "212            f"quality={row['provisional_quality_score']:.1f} adjusted={row['adjusted_compression_index']:.4f} "213            f"{row['source_path']}"214        )215 216 217if __name__ == "__main__":218    main()219