ZelligeAI/tessera-compressor

1#!/usr/bin/env python32"""3compress.py — Compress English reasoning text into the telegraphic CJK register4using tessera-compressor behind any OpenAI-compatible endpoint (vLLM, llama.cpp5server, etc.). No API keys or HF token required; the endpoint is yours.6 7This is the same harness the compressor was accepted under: segment the block,8group sentences into step-sized passages, classify each passage, compress it9against the chain built so far, then run the deterministic fidelity gate. A10passage that fails the gate falls back to a rules-only compression, so a bad11model output costs savings, never content.12 13Serve the model first, e.g.:14    vllm serve ZelligeAI/tessera-compressor --port 800115or with the GGUF:16    llama-server -m gguf/compressor-v31-q8_0.gguf --port 8001   # from the repo root17 18Then:19    # one block from a text file20    python compress.py --in think.txt --endpoint http://localhost:8001/v121 22    # a JSONL corpus: {"id": ..., "text": ...} per line23    python compress.py --in blocks.jsonl --out compressed.jsonl \24        --endpoint http://localhost:8001/v125 26Token counting: the fidelity gate compares token counts under a target27tokenizer. For results matching the accepted harness, point --tokenizer at the28model you are producing training data FOR (default: the compressor's own29tokenizer, which is close but not identical to the Qwen3.5 target used in the30acceptance run).31"""32import argparse33import json34import sys35 36from openai import OpenAI37from tokenizers import Tokenizer38 39from segmenting import segment, group_steps, classify_passage, facts, gate40from tokenmax import _apply_subs41 42PASSAGE_SYSTEM = (43    "你是推理压缩器。Re-notate the NEXT PASSAGE of a reasoning chain into telegraphic "44    "CJK/symbol notation. Every NEW logical step, fact, number and identifier must "45    "survive — unless already stated in the chain. Never restate chain content. "46    "[passage=load]: step-lossless telegraphic. [passage=narr]: minimal stubs "47    "(试X→否). Output only the re-notated continuation."48)49 50MAX_NEW_TOKENS = 51251 52 53def compress_block(text, client, model, ntok):54    """Compress one reasoning block. Returns (compressed_text, stats)."""55    segs = group_steps(segment(text))56    chain, seen = [], set()57    stats = {"segments": len(segs), "model_ok": 0, "fallback": 0,58             "narr_skipped": 0, "code": 0, "calls": 0}59 60    for kind, s in segs:61        if kind == "code":62            chain.append(s)63            seen |= facts(s)64            stats["code"] += 165            continue66        cls = classify_passage(s, seen, ntok)67        novel = facts(s) - seen68        rules_s, _ = _apply_subs(s)69        if not rules_s.strip():70            continue71        tail = "\n".join(chain)[-500:] or "(start)"72        stats["calls"] += 173        r = client.chat.completions.create(74            model=model, temperature=0.0, max_tokens=MAX_NEW_TOKENS,75            messages=[76                {"role": "system", "content": PASSAGE_SYSTEM},77                {"role": "user", "content": f"[passage={cls}]\n链:\n{tail}\n\n段:\n{s[:2000]}"},78            ],79            extra_body={"repetition_penalty": 1.15},80        )81        out = (r.choices[0].message.content or "").strip()82 83        if out == "∅" and cls == "narr" and not novel:84            stats["narr_skipped"] += 185            seen |= facts(s)86            continue87        if gate(s, rules_s, out, ntok, novel=novel) is None:88            chain.append(out)89            stats["model_ok"] += 190        else:91            chain.append(rules_s)92            stats["fallback"] += 193        seen |= facts(s)94 95    return "\n".join(chain), stats96 97 98def main():99    ap = argparse.ArgumentParser(description=__doc__,100                                 formatter_class=argparse.RawDescriptionHelpFormatter)101    ap.add_argument("--in", dest="inp", required=True,102                    help=".txt (one block) or .jsonl ({'id','text'} per line)")103    ap.add_argument("--out", default=None, help="output JSONL (default: stdout)")104    ap.add_argument("--endpoint", default="http://localhost:8001/v1")105    ap.add_argument("--model", default="ZelligeAI/tessera-compressor",106                    help="served model name at the endpoint")107    ap.add_argument("--tokenizer", default="ZelligeAI/tessera-compressor",108                    help="HF repo id or local tokenizer.json for gate token counts")109    args = ap.parse_args()110 111    if args.tokenizer.endswith(".json"):112        tok = Tokenizer.from_file(args.tokenizer)113    else:114        tok = Tokenizer.from_pretrained(args.tokenizer)115 116    def ntok(s):117        return len(tok.encode(s).ids) if s else 0118 119    client = OpenAI(base_url=args.endpoint, api_key="none")120 121    if args.inp.endswith(".jsonl"):122        rows = [json.loads(l) for l in open(args.inp) if l.strip()]123    else:124        rows = [{"id": args.inp, "text": open(args.inp).read()}]125 126    sink = open(args.out, "w") if args.out else sys.stdout127    for row in rows:128        compressed, stats = compress_block(row["text"], client, args.model, ntok)129        rec = {"id": row.get("id"), "compressed": compressed,130               "src_tokens": ntok(row["text"]), "out_tokens": ntok(compressed),131               "harness": stats}132        sink.write(json.dumps(rec, ensure_ascii=False) + "\n")133        sink.flush()134        print(f"[{row.get('id')}] {rec['src_tokens']} -> {rec['out_tokens']} tokens "135              f"(model_ok={stats['model_ok']} fallback={stats['fallback']})",136              file=sys.stderr)137    if args.out:138        sink.close()139 140 141if __name__ == "__main__":142    main()143