OneScience-Group/pdenneval

1#!/usr/bin/env python32"""Validate the standardized PDENNEval dataset package."""3 4from __future__ import annotations5 6import argparse7import hashlib8import json9import math10import sys11from pathlib import Path12 13import h5py14import numpy as np15 16 17REPO_ROOT = Path(__file__).resolve().parents[1]18DATA_ROOT = REPO_ROOT / "data"19CHECKSUM_PATH = REPO_ROOT / "files_sha256.jsonl"20 21EXPECTED_FILES = {22    "1D_Burgers_Sols_Nu0.001.hdf5": {23        "datasets": {24            "tensor": {"ndim": 3, "dtype": "float32"},25            "x-coordinate": {"ndim": 1, "dtype": "float32"},26            "t-coordinate": {"ndim": 1, "dtype": "float32"},27        },28        "attrs": {"Nu": 0.001},29    },30    "2D_DarcyFlow_beta0.1_Train.hdf5": {31        "datasets": {32            "tensor": {"ndim": 4, "dtype": "float32"},33            "nu": {"ndim": 3, "dtype": "float32"},34            "x-coordinate": {"ndim": 1, "dtype": "float32"},35            "y-coordinate": {"ndim": 1, "dtype": "float32"},36        },37        "attrs": {"beta": 0.1},38    },39    "1D_Advection_Sols_beta1.0.hdf5": {40        "datasets": {41            "tensor": {"ndim": 3, "dtype": "float32"},42            "x-coordinate": {"ndim": 1, "dtype": "float32"},43            "t-coordinate": {"ndim": 1, "dtype": "float32"},44        },45        "attrs": {},46    },47}48 49 50def fail(message: str) -> None:51    print(f"[FAIL] {message}")52    raise SystemExit(1)53 54 55def ok(message: str) -> None:56    print(f"[OK] {message}")57 58 59def warn(message: str) -> None:60    print(f"[WARN] {message}")61 62 63def sha256_file(path: Path) -> str:64    digest = hashlib.sha256()65    with path.open("rb") as handle:66        for chunk in iter(lambda: handle.read(1024 * 1024), b""):67            digest.update(chunk)68    return digest.hexdigest()69 70 71def as_float(value: object) -> float | None:72    try:73        return float(value)74    except (TypeError, ValueError):75        return None76 77 78def validate_hdf5_file(path: Path, spec: dict[str, object]) -> None:79    if not path.is_file():80        fail(f"missing expected HDF5 file: {path}")81    with h5py.File(path, "r") as handle:82        for attr_name, expected in spec.get("attrs", {}).items():83            actual = as_float(handle.attrs.get(attr_name))84            if actual is None or not math.isclose(actual, float(expected), rel_tol=1e-6, abs_tol=1e-12):85                fail(f"{path.name} attr {attr_name!r} expected {expected}, got {handle.attrs.get(attr_name)!r}")86        for dataset_name, dataset_spec in spec["datasets"].items():87            if dataset_name not in handle:88                fail(f"{path.name} missing dataset {dataset_name!r}")89            dataset = handle[dataset_name]90            if dataset.ndim != dataset_spec["ndim"]:91                fail(f"{path.name}/{dataset_name} ndim expected {dataset_spec['ndim']}, got {dataset.ndim}")92            if str(dataset.dtype) != dataset_spec["dtype"]:93                fail(f"{path.name}/{dataset_name} dtype expected {dataset_spec['dtype']}, got {dataset.dtype}")94            if any(dim <= 0 for dim in dataset.shape):95                fail(f"{path.name}/{dataset_name} has invalid shape {dataset.shape}")96            probe = np.asarray(dataset[0])97            if not np.isfinite(probe).all():98                fail(f"{path.name}/{dataset_name} first slice contains non-finite values")99        ok(f"{path.name} HDF5 schema is readable")100 101 102def verify_checksums(full_hash: bool) -> None:103    if not CHECKSUM_PATH.exists():104        warn(f"checksum manifest is not present: {CHECKSUM_PATH}")105        return106    records = [json.loads(line) for line in CHECKSUM_PATH.read_text(encoding="utf-8").splitlines() if line.strip()]107    if not records:108        fail("checksum manifest is empty")109    for record in records:110        path = REPO_ROOT / record["path"]111        if not path.is_file():112            fail(f"checksum entry points to missing file: {path}")113        size = path.stat().st_size114        if size != record["size"]:115            fail(f"size mismatch for {path}: expected {record['size']}, got {size}")116        if full_hash:117            digest = sha256_file(path)118            if digest != record["sha256"]:119                fail(f"sha256 mismatch for {path}")120    mode = "size+sha256" if full_hash else "size"121    ok(f"checksum manifest verified in {mode} mode: {len(records)} files")122 123 124def main() -> int:125    parser = argparse.ArgumentParser()126    parser.add_argument("--full-hash", action="store_true", help="verify SHA256 for all large HDF5 files")127    args = parser.parse_args()128 129    if not DATA_ROOT.is_dir():130        fail(f"dataset data root does not exist: {DATA_ROOT}")131    for filename, spec in EXPECTED_FILES.items():132        validate_hdf5_file(DATA_ROOT / filename, spec)133    verify_checksums(args.full_hash)134    ok("dataset validation completed")135    return 0136 137 138if __name__ == "__main__":139    sys.exit(main())140