constantinSch/evaluation_summarization

1"""Flask annotation app for blind summary evaluation.2 3Serves a web UI where annotators evaluate AI-generated summaries.4Annotations are anonymous and shared -- each summary is evaluated once.5Annotations are persisted in a SQLite database.6 7Optional password protection: set APP_PASSWORD as an environment variable.8Authentication uses HMAC tokens stored in the browser via localStorage,9avoiding cookies/sessions that can break behind reverse proxies.10"""11 12import hashlib13import hmac14import json15import os16import sqlite317from functools import cache18from pathlib import Path19 20from flask import Flask, Response, jsonify, request, send_file21 22DATASET_PATH = Path(os.environ.get("DATASET_PATH", "2026-04-23_prompt_evaluation_dataset.jsonl"))23DB_PATH = Path(os.environ.get("DB_PATH", "/data/annotations.db"))24DB_TIMEOUT_SECONDS = float(os.environ.get("DB_TIMEOUT_SECONDS", "30"))25 26ANNOTATION_FIELDS = (27    "bewertung", "korrekt", "relevant", "vollstaendig", "kohaerenz", "anmerkungen",28)29 30APP_PASSWORD = os.environ.get("APP_PASSWORD", "")31SECRET_KEY = os.environ.get("SECRET_KEY", os.urandom(24).hex())32 33app = Flask(__name__)34 35 36# ---------------------------------------------------------------------------37# Database38# ---------------------------------------------------------------------------39 40def get_db() -> sqlite3.Connection:41    """Open a SQLite connection with row factory.42 43    Uses DELETE journal mode instead of WAL for compatibility with44    FUSE-mounted persistent storage on Hugging Face Spaces where mmap45    (required by WAL's shared-memory file) is unreliable.46    """47    DB_PATH.parent.mkdir(parents=True, exist_ok=True)48    db = sqlite3.connect(str(DB_PATH), timeout=DB_TIMEOUT_SECONDS)49    db.row_factory = sqlite3.Row50    db.execute(f"PRAGMA busy_timeout = {int(DB_TIMEOUT_SECONDS * 1000)}")51    db.execute("PRAGMA journal_mode=DELETE")52    db.execute("PRAGMA synchronous=FULL")53    db.execute("""54        CREATE TABLE IF NOT EXISTS annotations (55            eval_id TEXT PRIMARY KEY,56            bewertung TEXT,57            korrekt TEXT,58            relevant TEXT,59            vollstaendig TEXT,60            kohaerenz TEXT,61            anmerkungen TEXT,62            updated_at DATETIME DEFAULT CURRENT_TIMESTAMP63        )64    """)65    return db66 67 68# ---------------------------------------------------------------------------69# Dataset (immutable, cached)70# ---------------------------------------------------------------------------71 72@cache73def load_dataset() -> tuple[dict, ...]:74    """Load evaluation items from JSONL. Cached because the dataset never changes."""75    items = []76    with open(DATASET_PATH, encoding="utf-8") as f:77        for line in f:78            if line.strip():79                row = json.loads(line)80                has_prior = bool(row.get("bewertung"))81                row["has_prior_judgement"] = has_prior82                if has_prior:83                    for field in ANNOTATION_FIELDS:84                        row[f"prior_{field}"] = row.get(field)85                items.append(row)86    items.sort(key=lambda x: x["eval_id"])87    return tuple(items)88 89 90def fetch_annotations(db: sqlite3.Connection) -> dict[str, dict]:91    """Fetch all annotations, keyed by eval_id."""92    rows = db.execute("SELECT * FROM annotations").fetchall()93    return {row["eval_id"]: dict(row) for row in rows}94 95 96def merge_items_with_annotations(97    items: tuple[dict, ...],98    annotations: dict[str, dict],99) -> list[dict]:100    """Return items with annotation values merged in (does not mutate originals)."""101    merged = []102    for item in items:103        ann = annotations.get(item["eval_id"])104        entry = {**item, "evaluated": ann is not None}105        if ann:106            for field in ANNOTATION_FIELDS:107                entry[field] = ann.get(field)108        merged.append(entry)109    return merged110 111 112# ---------------------------------------------------------------------------113# Optional password protection (token-based, no cookies/sessions)114# ---------------------------------------------------------------------------115 116 117def _make_auth_token() -> str:118    """Create an HMAC token derived from the app password and secret key."""119    key = SECRET_KEY.encode() if isinstance(SECRET_KEY, str) else SECRET_KEY120    return hmac.new(key, APP_PASSWORD.encode(), hashlib.sha256).hexdigest()121 122 123@app.before_request124def check_auth():125    if not APP_PASSWORD:126        return None127    if request.path in ("/", "/login"):128        return None129    if request.path.startswith("/api/login"):130        return None131    token = request.headers.get("Authorization", "").removeprefix("Bearer ")132    if token == _make_auth_token():133        return None134    return jsonify({"error": "unauthorized"}), 401135 136 137@app.route("/api/login", methods=["POST"])138def api_login():139    data = request.get_json()140    if data and data.get("password") == APP_PASSWORD:141        return jsonify({"token": _make_auth_token()})142    return jsonify({"error": "wrong_password"}), 401143 144 145# ---------------------------------------------------------------------------146# Routes147# ---------------------------------------------------------------------------148 149 150@app.after_request151def no_cache_api(response):152    """Prevent browser from caching API responses."""153    if request.path.startswith("/api/"):154        response.headers["Cache-Control"] = "no-store"155    return response156 157 158@app.route("/")159def index():160    return send_file("index.html")161 162 163@app.route("/api/entries")164def get_entries():165    """Return all evaluation items with annotation data merged in."""166    items = load_dataset()167    db = get_db()168    annotations = fetch_annotations(db)169    db.close()170    return jsonify(merge_items_with_annotations(items, annotations))171 172 173@app.route("/api/annotate", methods=["POST"])174def annotate():175    """Save or update an annotation."""176    data = request.get_json()177    db = get_db()178    try:179        db.execute(180            """INSERT INTO annotations181               (eval_id, bewertung, korrekt, relevant, vollstaendig, kohaerenz, anmerkungen)182               VALUES (?, ?, ?, ?, ?, ?, ?)183               ON CONFLICT(eval_id) DO UPDATE SET184                   bewertung = excluded.bewertung,185                   korrekt = excluded.korrekt,186                   relevant = excluded.relevant,187                   vollstaendig = excluded.vollstaendig,188                   kohaerenz = excluded.kohaerenz,189                   anmerkungen = excluded.anmerkungen,190                   updated_at = CURRENT_TIMESTAMP""",191            (192                data["eval_id"],193                data.get("bewertung"),194                data.get("korrekt"),195                data.get("relevant"),196                data.get("vollstaendig"),197                data.get("kohaerenz"),198                data.get("anmerkungen"),199            ),200        )201        db.commit()202        # Verify the write persisted (guards against silent FUSE write failures)203        row = db.execute(204            "SELECT eval_id FROM annotations WHERE eval_id = ?",205            (data["eval_id"],),206        ).fetchone()207        if not row:208            return jsonify({"error": "write verification failed"}), 500209    finally:210        db.close()211    return jsonify({"status": "ok"})212 213 214@app.route("/api/progress")215def progress():216    """Return annotation progress."""217    total = len(load_dataset())218    db = get_db()219    count = db.execute("SELECT COUNT(*) FROM annotations").fetchone()[0]220    db.close()221    return jsonify({"total": total, "annotated": count})222 223 224@app.route("/api/export")225def export_annotations():226    """Export all annotations as downloadable JSONL."""227    db = get_db()228    rows = db.execute("SELECT * FROM annotations ORDER BY eval_id").fetchall()229    db.close()230    lines = [json.dumps(dict(row), ensure_ascii=False) for row in rows]231    return Response(232        "\n".join(lines) + "\n",233        mimetype="application/jsonl",234        headers={"Content-Disposition": "attachment; filename=annotations.jsonl"},235    )236 237 238if __name__ == "__main__":239    app.run(host="0.0.0.0", port=7860)240