AntonioJun/workspace

1"""Evaluate one hypothesis with the existing deterministic symbolic scorer."""2 3from __future__ import annotations4 5import argparse6import json7from pathlib import Path8import sys9 10WORKSPACE_ROOT = Path(__file__).resolve().parent.parent11if str(WORKSPACE_ROOT) not in sys.path:12    sys.path.insert(0, str(WORKSPACE_ROOT))13 14from experiments import config  # noqa: E40215from symbolic import launch as symbolic_launch  # noqa: E40216 17 18def configure_symbolic_evaluation(19    hypothesis,20    depth="metric",21    tracking="tracking",22    input_selection="uniform",23    frame_count=64,24    spatial_code_format="explicit",25):26    """Point symbolic reads and writes at one isolated experiment selection."""27    codes = config.spatial_code_directory(28        hypothesis, depth, tracking, input_selection, frame_count, spatial_code_format29    )30    results = config.result_directory(31        hypothesis,32        "symbolic",33        depth,34        tracking,35        input_selection,36        frame_count,37        spatial_code_format,38    )39    symbolic_run = symbolic_launch.symbolic_run40    symbolic_run.SPATIAL_CODES_DEPTH = depth41    symbolic_run.SPATIAL_CODES_INPUT = input_selection42    symbolic_run.SPATIAL_CODES_TRACKING = tracking43    symbolic_run.SPATIAL_CODES_FRAMES = frame_count44    symbolic_run.SPATIAL_CODES_FORMAT = spatial_code_format45    symbolic_run.SPATIAL_CODES_DIR = str(codes)46    symbolic_run.RESULTS_DIR = str(results)47    symbolic_run.results_dir_for_selection = lambda results_dir=None: str(48        results_dir or results49    )50    return codes, results51 52 53def evaluate(54    hypothesis,55    depth="metric",56    tracking="tracking",57    input_selection="uniform",58    frame_count=64,59    scene_ids=None,60    quiet=False,61    errors=False,62    spatial_code_format="explicit",63):64    """Score every available experiment code, or an explicit scene subset."""65    codes, results = configure_symbolic_evaluation(66        hypothesis,67        depth,68        tracking,69        input_selection,70        frame_count,71        spatial_code_format,72    )73    available = symbolic_launch.scenes_with_spatial_codes()74    selected = available if scene_ids is None else list(scene_ids)75    missing = [scene for scene in selected if scene not in available]76    if missing:77        raise FileNotFoundError(78            f"scene(s) have no hypothesis spatial code under {codes}: {missing}"79        )80    if not selected:81        raise FileNotFoundError(f"no hypothesis spatial codes found under {codes}")82    per_scene, combined = symbolic_launch.run_all(selected, quiet=quiet)83    summary = {84        "hypothesis": hypothesis,85        "depth": depth,86        "input": input_selection,87        "tracking": tracking,88        "frames": frame_count,89        "spatial_code_format": spatial_code_format,90        "scenes_run": list(per_scene),91        "combined_aggregate": combined,92    }93    if errors:94        summary["error_analysis"] = {95            question_type: symbolic_launch.error_analysis(per_scene, question_type)96            for question_type in symbolic_launch._ANALYZABLE_TYPES97        }98        symbolic_launch.print_error_analysis(per_scene)99        symbolic_launch.print_mca_breakdown(per_scene)100    results.mkdir(parents=True, exist_ok=True)101    summary_path = results / "_summary.json"102    with summary_path.open("w", encoding="utf-8") as stream:103        json.dump(summary, stream, indent=1)104    return summary, summary_path105 106 107def main() -> None:108    parser = argparse.ArgumentParser()109    parser.add_argument("--hypothesis", required=True)110    parser.add_argument("--depth", default="metric", choices=("relative", "metric"))111    parser.add_argument(112        "--input",113        default="uniform",114        choices=("uniform", "selective"),115        dest="input_selection",116    )117    parser.add_argument(118        "--tracking", default="tracking", choices=("tracking", "no tracking")119    )120    parser.add_argument("--frames", type=int, default=64)121    parser.add_argument(122        "--format",123        default="explicit",124        choices=config.SPATIAL_CODE_FORMATS,125        dest="spatial_code_format",126    )127    parser.add_argument(128        "--scenes", default="", help="optional comma-separated scene IDs"129    )130    parser.add_argument("--quiet", action="store_true")131    parser.add_argument("--errors", action="store_true")132    args = parser.parse_args()133    if args.frames < 1:134        parser.error("--frames must be positive")135    scenes = (136        [scene.strip() for scene in args.scenes.split(",") if scene.strip()]137        if args.scenes138        else None139    )140    summary, path = evaluate(141        args.hypothesis,142        args.depth,143        args.tracking,144        args.input_selection,145        args.frames,146        scenes,147        args.quiet,148        args.errors,149        args.spatial_code_format,150    )151    print("\nCOMBINED AGGREGATE")152    for key, value in summary["combined_aggregate"].items():153        print(f"  {key}: {value}")154    print(f"\nwrote experiment summary to {path}")155 156 157if __name__ == "__main__":158    main()159