ppak10/Agentic-SLS-Telemetry

1#!/usr/bin/env python32"""Render per-layer timelapse GIFs for each build.3 4Reads data/ticks/{build_id:03d}.parquet and writes:5  previews/{build_id:03d}/timelapse_chamber.gif6  previews/{build_id:03d}/timelapse_thermal.gif7  previews/{build_id:03d}/timelapse_galvo.gif8  previews/{build_id:03d}/timelapse_composite.gif  (1×3 panel: chamber | thermal | galvo)9 10Layer detection: positions.position.z2 is quantized in 100 µm buckets.11Only z2 > 0 rows are used — z2 stays at 0 during pre-print heating so this12naturally excludes the heating phase without needing to inspect the phase column.13The *last* non-null frame within each z2 bucket is the representative — that's14the most-recent view of the layer just before recoating begins.15 16Null-frame levels are forward-filled from the most recent non-null level of the17same kind, so the GIF never shows a blank panel mid-timelapse.18 19The thermal panel is rendered from the raw `bedmatrix` IR grid (inferno colormap20over a fixed absolute °C range; see _thermal.py) whenever it's present — builds21013+. The three earliest builds (001/002/012) predate the bedmatrix stream and22fall back to the legacy pre-rendered `frame_thermal` GIF.23 24Subsampled to at most MAX_FRAMES (default 300). At GIF_FPS=25 that gives25a max 12-second GIF. Builds with fewer detected layers are not padded.26 27Canvas is scaled to GIF_PANEL_HEIGHT=240 px (half the MP4 panel height) to28keep file sizes web-friendly for README embedding.29 30Usage:31    uv run scripts/previews/02_timelapse.py            # all builds in data/ticks/32    uv run scripts/previews/02_timelapse.py 26 28      # specific build ids33    uv run scripts/previews/02_timelapse.py 26 --kinds chamber,composite34"""35import io36import sys37from pathlib import Path38 39import pyarrow.parquet as pq40from PIL import Image, ImageStat41 42sys.path.insert(0, str(Path(__file__).parent.parent))43sys.path.insert(0, str(Path(__file__).parent))44from _lib import DATA_DIR45from _thermal import bedmatrix_to_image46 47OUTPUT_DIR = DATA_DIR.parent / "previews"48TICKS_DIR  = DATA_DIR / "ticks"49 50FRAME_KINDS        = ("chamber", "thermal", "galvo")51KIND_ROTATION_CW   = {"chamber": 90}  # degrees; see _decode_raw52# The thermal panel renders from the raw `bedmatrix` IR grid when present53# (builds 013+), falling back to the legacy `frame_thermal` GIF for the three54# earliest builds (001/002/012) that predate the bedmatrix stream.55 56GIF_FPS            = 2557GIF_DURATION_MS    = int(1000 / GIF_FPS)   # 40 ms per frame58MAX_FRAMES         = 300                    # subsample cap → max 12 s GIF59LAYER_QUANTIZE_UM  = 100                    # z2 bucket size in microns60GIF_PANEL_HEIGHT   = 240                    # panel height in pixels for GIF canvas61BATCH_ROWS         = 1000                   # pyarrow streaming batch size62 63# Halogen brightness filter — applies to chamber frames only.64# The halogens pulse on/off throughout a build; dark frames (halogens off) are65# uninformative for viewing. A frame is kept if its mean grayscale brightness is66# at least this fraction of the brightest frame seen in the same build.67# For the individual chamber GIF dark frames are dropped entirely; for the68# composite they are forward-filled from the last bright frame so thermal/galvo69# stay in layer-sync.70CHAMBER_BRIGHTNESS_RATIO = 0.571 72 73# ---------------------------------------------------------------------------74# Image helpers (mirrors 01_render.py exactly)75# ---------------------------------------------------------------------------76 77def _decode_raw(b: bytes, kind: str) -> Image.Image | None:78    """Decode raw image bytes to PIL RGB with orientation correction."""79    if not b:80        return None81    try:82        img = Image.open(io.BytesIO(b)).convert("RGB")83    except Exception:84        return None85    rot = KIND_ROTATION_CW.get(kind, 0)86    if rot == 90:87        img = img.transpose(Image.Transpose.ROTATE_270)88    elif rot == 180:89        img = img.transpose(Image.Transpose.ROTATE_180)90    elif rot == 270:91        img = img.transpose(Image.Transpose.ROTATE_90)92    return img93 94 95def _decode_cell(cell, kind: str) -> Image.Image | None:96    """Decode one struct cell to a PIL RGB image, dispatching on struct shape:97    a `bedmatrix` struct (has 'values') renders as an inferno heatmap; a frame98    Image struct (has 'bytes') decodes + orientation-corrects. None when missing."""99    if cell is None:100        return None101    if "values" in cell:102        return bedmatrix_to_image(cell)103    return _decode_raw(cell.get("bytes"), kind)104 105 106def _thermal_column(parquet_path: Path) -> str:107    """Which column feeds the thermal panel for this build: 'bedmatrix' when the108    raw IR matrix has any non-null cell, else the legacy 'frame_thermal'. The109    bedmatrix stream started at build 013, so 001/002/012 fall back to the GIF."""110    pf = pq.ParquetFile(parquet_path)111    if "bedmatrix" not in {f.name for f in pf.schema_arrow}:112        return "frame_thermal"113    for batch in pf.iter_batches(columns=["bedmatrix"], batch_size=BATCH_ROWS):114        for cell in batch.column("bedmatrix").to_pylist():115            if cell is not None:116                return "bedmatrix"117    return "frame_thermal"118 119 120def _columns_for_build(parquet_path: Path) -> dict[str, str]:121    """Map each panel kind → the parquet column that feeds it for this build."""122    return {123        "chamber": "frame_chamber",124        "thermal": _thermal_column(parquet_path),125        "galvo":   "frame_galvo",126    }127 128 129def _canvas_width(cell, kind: str) -> int:130    """Decode one cell to determine the locked canvas width for this kind."""131    img = _decode_cell(cell, kind)132    if img is None:133        return GIF_PANEL_HEIGHT  # square fallback134    sw, sh = img.size135    return max(2, round(sw * GIF_PANEL_HEIGHT / sh))136 137 138def _fit_to_canvas(img: Image.Image, canvas_w: int) -> Image.Image:139    """Letterbox img into (canvas_w × GIF_PANEL_HEIGHT) with dark-gray fill."""140    sw, sh = img.size141    scale  = min(canvas_w / sw, GIF_PANEL_HEIGHT / sh)142    nw     = max(1, round(sw * scale))143    nh     = max(1, round(sh * scale))144    fitted = img.resize((nw, nh), Image.BILINEAR)145    canvas = Image.new("RGB", (canvas_w, GIF_PANEL_HEIGHT), (20, 20, 20))146    canvas.paste(fitted, ((canvas_w - nw) // 2, (GIF_PANEL_HEIGHT - nh) // 2))147    return canvas148 149 150def _placeholder(canvas_w: int) -> Image.Image:151    return Image.new("RGB", (canvas_w, GIF_PANEL_HEIGHT), (20, 20, 20))152 153 154def _mean_brightness(img: Image.Image) -> float:155    """Mean grayscale pixel value 0–255 (uses PIL ImageStat, no numpy)."""156    return ImageStat.Stat(img.convert("L")).mean[0]157 158 159def _chamber_threshold(decoded_frames: list[Image.Image | None]) -> float:160    """Return the brightness threshold for a build's chamber frames.161    25 % of the brightest frame seen; 0 if no frames (no filtering applied)."""162    brightnesses = [_mean_brightness(f) for f in decoded_frames if f is not None]163    return max(brightnesses) * CHAMBER_BRIGHTNESS_RATIO if brightnesses else 0.0164 165 166# ---------------------------------------------------------------------------167# Layer data collection168# ---------------------------------------------------------------------------169 170def collect_layer_cells(parquet_path: Path, kind: str, col: str) -> dict[int, dict]:171    """Single-pass stream → {z2_level: last_non_null_struct}.172 173    Only z2 > 0 rows are included. The dict is keyed by int(z2 / LAYER_QUANTIZE_UM);174    each entry holds the *last* non-null struct cell seen at that level (a frame175    Image struct, or a bedmatrix struct for the thermal panel). Reads only two176    parquet columns (z2 + the source column) for efficiency.177    """178    z2_col  = "positions.position.z2"179    pf      = pq.ParquetFile(parquet_path)180    present = {f.name for f in pf.schema_arrow}181    if col not in present or z2_col not in present:182        return {}183 184    layer_data: dict[int, dict] = {}185    for batch in pf.iter_batches(columns=[z2_col, col], batch_size=BATCH_ROWS):186        z2_list   = batch.column(z2_col).to_pylist()187        cell_list = batch.column(col).to_pylist()188        for z2, cell in zip(z2_list, cell_list):189            if z2 is None or z2 <= 0:190                continue191            if cell is not None:192                layer_data[int(z2 / LAYER_QUANTIZE_UM)] = cell193    return layer_data194 195 196def collect_all_kinds(parquet_path: Path, cols_map: dict[str, str]) -> dict[str, dict[int, dict]]:197    """Single streaming pass collecting all three kinds simultaneously.198 199    Used by render_timelapse_composite so we don't make three separate passes200    through (potentially 17+ GB) parquet files. Reads four columns: z2 + each201    kind's source column. Each kind gets its own {z2_level: struct} dict.202    """203    z2_col    = "positions.position.z2"204    pf        = pq.ParquetFile(parquet_path)205    present   = {f.name for f in pf.schema_arrow}206    read_cols = [c for c in ([z2_col] + [cols_map[k] for k in FRAME_KINDS]) if c in present]207    if z2_col not in read_cols:208        return {k: {} for k in FRAME_KINDS}209 210    layer_data: dict[str, dict[int, dict]] = {k: {} for k in FRAME_KINDS}211    for batch in pf.iter_batches(columns=read_cols, batch_size=BATCH_ROWS):212        z2_list = batch.column(z2_col).to_pylist()213        for kind in FRAME_KINDS:214            col = cols_map[kind]215            if col not in read_cols:216                continue217            cell_list = batch.column(col).to_pylist()218            for z2, cell in zip(z2_list, cell_list):219                if z2 is None or z2 <= 0:220                    continue221                if cell is not None:222                    layer_data[kind][int(z2 / LAYER_QUANTIZE_UM)] = cell223    return layer_data224 225 226# ---------------------------------------------------------------------------227# Subsampling and forward-fill228# ---------------------------------------------------------------------------229 230def _subsample(levels: list[int]) -> list[int]:231    """Evenly subsample sorted levels down to at most MAX_FRAMES."""232    if len(levels) <= MAX_FRAMES:233        return levels234    step = len(levels) / MAX_FRAMES235    return [levels[round(i * step)] for i in range(MAX_FRAMES)]236 237 238def _forward_fill(layer_bytes: dict[int, bytes],239                  target_levels: list[int]) -> list[bytes | None]:240    """For each target level, return the bytes at that level or the most241    recent non-null bytes seen so far (forward-fill across gaps)."""242    out: list[bytes | None] = []243    last: bytes | None = None244    for lvl in target_levels:245        b = layer_bytes.get(lvl)246        if b is not None:247            last = b248        out.append(last)249    return out250 251 252# ---------------------------------------------------------------------------253# GIF writer254# ---------------------------------------------------------------------------255 256def _write_gif(frames: list[Image.Image], out_path: Path) -> None:257    """Palette-quantize and save frames as an animated GIF."""258    out_path.parent.mkdir(parents=True, exist_ok=True)259    palette_frames = [260        f.quantize(colors=256, method=Image.Quantize.MEDIANCUT,261                   dither=Image.Dither.FLOYDSTEINBERG)262        for f in frames263    ]264    palette_frames[0].save(265        out_path,266        format="GIF",267        save_all=True,268        append_images=palette_frames[1:],269        loop=0,270        duration=GIF_DURATION_MS,271        optimize=False,272    )273 274 275# ---------------------------------------------------------------------------276# Per-build renderers277# ---------------------------------------------------------------------------278 279def render_timelapse_kind(parquet_path: Path, kind: str, out_path: Path, col: str) -> int:280    """Write timelapse_{kind}.gif. Returns number of GIF frames written.281 282    Chamber only: dark frames (halogens off) are dropped entirely so the GIF283    shows only moments where the part is visible. Thermal and galvo are284    unaffected — they don't depend on halogen lighting.285    """286    layer_cells = collect_layer_cells(parquet_path, kind, col)287    if not layer_cells:288        print(f"  {kind:8s}: no printing-phase frames (z2 > 0), skipping")289        return 0290 291    sorted_levels = sorted(layer_cells)292    target_levels = _subsample(sorted_levels)293    fill_cells    = _forward_fill(layer_cells, target_levels)294    canvas_w      = _canvas_width(next(c for c in fill_cells if c), kind)295 296    # Decode all selected frames up front (needed for brightness scan on chamber).297    decoded = [_decode_cell(c, kind) for c in fill_cells]298 299    if kind == "chamber":300        # Compute brightness once per decoded frame, then threshold and filter.301        brightnesses = [_mean_brightness(img) if img is not None else None302                        for img in decoded]303        threshold = _chamber_threshold(decoded)304        pil_frames = [305            _fit_to_canvas(img, canvas_w)306            for img, b in zip(decoded, brightnesses)307            if img is not None and b is not None and b >= threshold308        ]309        dark_dropped = sum(310            1 for img, b in zip(decoded, brightnesses)311            if img is not None and b is not None and b < threshold312        )313        if dark_dropped:314            print(f"  {kind:8s}: dropped {dark_dropped} dark frames "315                  f"(threshold {threshold:.1f}/255)")316    else:317        pil_frames = [318            _fit_to_canvas(img, canvas_w) if img else _placeholder(canvas_w)319            for img in decoded320        ]321 322    if not pil_frames:323        print(f"  {kind:8s}: no frames survived brightness filter, skipping")324        return 0325 326    _write_gif(pil_frames, out_path)327    return len(pil_frames)328 329 330def render_timelapse_composite(parquet_path: Path, out_path: Path,331                               cols_map: dict[str, str]) -> int:332    """Write timelapse_composite.gif (1×3 panel). Single parquet pass.333 334    Thermal and galvo show the actual frame for every layer (unaffected by335    halogens). The chamber panel forward-fills from the last *bright* frame336    when the current layer's chamber frame is dark — this keeps all three337    panels in layer-sync while never displaying a dark chamber view.338    """339    all_cells = collect_all_kinds(parquet_path, cols_map)340 341    all_levels = sorted(set().union(*(set(d) for d in all_cells.values())))342    if not all_levels:343        print("  composite: no printing-phase frames (z2 > 0), skipping")344        return 0345 346    target_levels = _subsample(all_levels)347    fill_per_kind = {k: _forward_fill(all_cells[k], target_levels) for k in FRAME_KINDS}348 349    canvas_widths: dict[str, int] = {}350    for kind in FRAME_KINDS:351        first_c = next((c for c in fill_per_kind[kind] if c), None)352        canvas_widths[kind] = (353            _canvas_width(first_c, kind) if first_c else GIF_PANEL_HEIGHT354        )355    total_w = sum(canvas_widths.values())356 357    # Pre-decode chamber frames once; compute adaptive brightness threshold.358    chamber_decoded = [359        _decode_cell(c, "chamber") for c in fill_per_kind["chamber"]360    ]361    chamber_threshold = _chamber_threshold(chamber_decoded)362 363    last_bright_chamber: Image.Image | None = None364    pil_frames: list[Image.Image] = []365    for i in range(len(target_levels)):366        composite = Image.new("RGB", (total_w, GIF_PANEL_HEIGHT), (20, 20, 20))367        x = 0368        for kind in FRAME_KINDS:369            if kind == "chamber":370                img = chamber_decoded[i]371                # Update the running bright-chamber reference when this frame is bright.372                if img is not None and _mean_brightness(img) >= chamber_threshold:373                    last_bright_chamber = img374                # Always use the last bright frame (forward-fill); placeholder until375                # the first bright frame arrives.376                panel_img = last_bright_chamber377            else:378                panel_img = _decode_cell(fill_per_kind[kind][i], kind)379 380            panel = (381                _fit_to_canvas(panel_img, canvas_widths[kind])382                if panel_img else _placeholder(canvas_widths[kind])383            )384            composite.paste(panel, (x, 0))385            x += canvas_widths[kind]386        pil_frames.append(composite)387 388    _write_gif(pil_frames, out_path)389    return len(pil_frames)390 391 392def process_build(build_id: int, kinds: set[str]) -> None:393    parquet_path = TICKS_DIR / f"{build_id:03d}.parquet"394    if not parquet_path.exists():395        print(f"build {build_id:03d}: no parquet, skipping")396        return397 398    build_dir = OUTPUT_DIR / f"{build_id:03d}"399    print(f"build {build_id:03d}: timelapse GIFs → {build_dir.relative_to(Path.cwd())}/")400    cols_map = _columns_for_build(parquet_path)401    if ("thermal" in kinds or "composite" in kinds):402        print(f"  thermal source: {cols_map['thermal']}")403 404    for kind in FRAME_KINDS:405        if kind not in kinds:406            continue407        out = build_dir / f"timelapse_{kind}.gif"408        n = render_timelapse_kind(parquet_path, kind, out, cols_map[kind])409        if out.exists():410            size = out.stat().st_size411            print(f"  {kind:8s}: {n:>4} frames → {out.name}  ({size:,} bytes)")412 413    if "composite" in kinds:414        out = build_dir / "timelapse_composite.gif"415        n = render_timelapse_composite(parquet_path, out, cols_map)416        if out.exists():417            size = out.stat().st_size418            print(f"  composite: {n:>4} frames → {out.name}  ({size:,} bytes)")419 420 421def main():422    import argparse423    parser = argparse.ArgumentParser(424        description=__doc__,425        formatter_class=argparse.RawDescriptionHelpFormatter,426    )427    parser.add_argument("build_ids", nargs="*", type=int,428                        help="Build IDs to render (default: all in data/ticks/)")429    parser.add_argument(430        "--kinds", default="chamber,thermal,galvo,composite",431        help="Comma-separated outputs to render. "432             "Valid: chamber, thermal, galvo, composite. Default: all four.",433    )434    args   = parser.parse_args()435    kinds  = set(args.kinds.split(","))436    unknown = kinds - (set(FRAME_KINDS) | {"composite"})437    if unknown:438        parser.error(f"unknown --kinds values: {sorted(unknown)}")439 440    targets = args.build_ids or sorted(int(p.stem) for p in TICKS_DIR.glob("*.parquet"))441    for bid in targets:442        process_build(bid, kinds)443 444 445if __name__ == "__main__":446    main()447