ULM-DS-Lab/Sawit-Weight

1"""Verify the Sawit-Weight dataset package.2 3Two layers are checked. The as-captured layer under annotations/raw/ must still match4every SHA-256 its manifests recorded. The corrected layer (annotations/labels/,5annotations/trees.jsonl, data/train/metadata.jsonl) must be internally consistent and6must differ from the raw layer only where a correction is declared below.7 8    python scripts/verify_dataset.py [dataset_root]9"""10 11import hashlib12import json13import os14import sys15 16TOL = 2e-3  # bbox_pixel and bbox_yolo agree to within this fraction of the image side17 18# Corrections applied on top of the as-captured annotations, mirrored from the README.19ADDED_BOXES = {("TANERA_303_BW_0004", 1)}          # (tree, side_index)20FRACTION_OVERRIDES = {"TANERA_303_BW_0003", "TANERA_303_BW_0004", "TANERA_303_BW_0005"}21 22 23def sha256(path):24    h = hashlib.sha256()25    with open(path, "rb") as f:26        for chunk in iter(lambda: f.read(1 << 20), b""):27            h.update(chunk)28    return h.hexdigest()29 30 31def read_json(path):32    with open(path, encoding="utf-8") as f:33        return json.load(f)34 35 36def read_jsonl(path):37    with open(path, encoding="utf-8") as f:38        return [json.loads(line) for line in f if line.strip()]39 40 41def read_label(path):42    with open(path, encoding="utf-8") as f:43        return [line.split() for line in f.read().splitlines() if line.strip()]44 45 46def main(root):47    images = os.path.join(root, "data", "train")48    ann = os.path.join(root, "annotations")49    raw = os.path.join(ann, "raw")50    fail = []51 52    def check(ok, message):53        if not ok:54            fail.append(message)55 56    trees = sorted(f[:-5] for f in os.listdir(os.path.join(raw, "trees")) if f.endswith(".json"))57    check(len(trees) > 0, "no tree JSON found")58 59    seen_images, seen_labels, digests = set(), set(), {}60 61    for name in trees:62        tree = read_json(os.path.join(raw, "trees", f"{name}.json"))63        manifest = read_json(os.path.join(raw, "manifests", f"{name}.json"))64        device = os.path.join(raw, "device_metadata", f"{name}.json")65 66        check(tree["tree_name"] == name, f"{name}: tree_name does not match filename")67 68        # Layer 1: the as-captured artifacts still hash to what the device recorded.69        for side in manifest["sides"]:70            rgb = os.path.join(images, side["rgb"]["file"])71            raw_label = os.path.join(raw, "labels", side["label"]["file"])72            check(os.path.exists(rgb), f"{name}: missing image {side['rgb']['file']}")73            check(os.path.exists(raw_label), f"{name}: missing raw label {side['label']['file']}")74            if os.path.exists(rgb):75                digest = sha256(rgb)76                check(digest == side["rgb"]["sha256"], f"{name}: SHA-256 mismatch {side['rgb']['file']}")77                check(digest not in digests, f"{name}: duplicate image content, also in {digests.get(digest)}")78                digests[digest] = side["rgb"]["file"]79                seen_images.add(side["rgb"]["file"])80            if os.path.exists(raw_label):81                check(sha256(raw_label) == side["label"]["sha256"],82                      f"{name}: SHA-256 mismatch {side['label']['file']}")83                seen_labels.add(side["label"]["file"])84 85        check(sha256(os.path.join(raw, "trees", f"{name}.json")) == manifest["outputJson"]["sha256"],86              f"{name}: SHA-256 mismatch on tree JSON")87        check(sha256(device) == manifest["metadata"]["sha256"], f"{name}: SHA-256 mismatch on device metadata")88 89        # Layer 2: the corrected labels are the raw ones plus only the declared additions.90        for key, side in tree["images"].items():91            raw_rows = read_label(os.path.join(raw, "labels", side["label_file"]))92            fixed_rows = read_label(os.path.join(ann, "labels", side["label_file"]))93            expected = side["bbox_count"] + (1 if (name, side["side_index"]) in ADDED_BOXES else 0)94            check(len(raw_rows) == side["bbox_count"],95                  f"{name}/{key}: raw label has {len(raw_rows)} rows, bbox_count is {side['bbox_count']}")96            check(len(fixed_rows) == expected,97                  f"{name}/{key}: corrected label has {len(fixed_rows)} rows, expected {expected}")98 99            for parts in fixed_rows:100                check(parts[0] == "0", f"{name}/{key}: class id is not 0")101                check(all(0.0 <= float(v) <= 1.0 for v in parts[1:]),102                      f"{name}/{key}: bbox outside [0, 1]")103            for i, parts in enumerate(raw_rows):104                check([round(float(v), 5) for v in parts[1:]] == [round(float(v), 5) for v in fixed_rows[i][1:]],105                      f"{name}/{key}: corrected label altered the original box {i}")106 107    # Capture set coverage.108    capture_set = read_json(os.path.join(raw, "capture_set.json"))109    check({t["treeName"] for t in capture_set["trees"]} == set(trees),110          "capture_set.json does not list exactly the trees present")111 112    # No orphan files.113    on_disk_images = {f for f in os.listdir(images) if f.lower().endswith(".jpg")}114    on_disk_raw = {f for f in os.listdir(os.path.join(raw, "labels")) if f.endswith(".txt")}115    on_disk_fixed = {f for f in os.listdir(os.path.join(ann, "labels")) if f.endswith(".txt")}116    check(on_disk_images == seen_images, f"orphan images: {sorted(on_disk_images ^ seen_images)}")117    check(on_disk_raw == seen_labels, f"orphan raw labels: {sorted(on_disk_raw ^ seen_labels)}")118    check(on_disk_fixed == seen_labels, f"orphan corrected labels: {sorted(on_disk_fixed ^ seen_labels)}")119 120    # metadata.jsonl agrees with the corrected labels and with itself.121    rows = read_jsonl(os.path.join(images, "metadata.jsonl"))122    check({r["file_name"] for r in rows} == seen_images, "metadata.jsonl does not cover every image")123    check(len(rows) == len(seen_images), "metadata.jsonl has duplicate file_name rows")124 125    for row in rows:126        objects = row["objects"]127        lengths = {len(v) for v in objects.values()}128        check(lengths == {row["num_bunches"]}, f"{row['file_name']}: objects lists disagree with num_bunches")129 130        label_rows = read_label(os.path.join(ann, "labels", row["file_name"][:-4] + ".txt"))131        check(len(label_rows) == row["num_bunches"],132              f"{row['file_name']}: label row count disagrees with num_bunches")133        for i, parts in enumerate(label_rows):134            check([round(float(v), 6) for v in parts[1:]] == [round(v, 6) for v in objects["bbox_yolo"][i]],135                  f"{row['file_name']}: label row {i} disagrees with bbox_yolo")136 137        w, h = row["width"], row["height"]138        for i, (cx, cy, bw, bh) in enumerate(objects["bbox_yolo"]):139            x1, y1, x2, y2 = objects["bbox_pixel"][i]140            check(x1 < x2 and y1 < y2, f"{row['file_name']}: box {i} has no positive area")141            check(abs((x1 + x2) / 2 / w - cx) < TOL and abs((y1 + y2) / 2 / h - cy) < TOL142                  and abs((x2 - x1) / w - bw) < TOL and abs((y2 - y1) / h - bh) < TOL,143                  f"{row['file_name']}: box {i} pixel and yolo forms disagree")144 145        for i, frac in enumerate(objects["ripeness_fraction"]):146            source = objects["ripeness_fraction_source"][i]147            check(frac in (1, 2, 3, 4, 5), f"{row['file_name']}: ripeness_fraction {frac} outside 1-5")148            check(source in ("field_note", "visual_estimate"),149                  f"{row['file_name']}: unknown ripeness_fraction_source {source}")150            check(source == "field_note" or row["tree_name"] in FRACTION_OVERRIDES,151                  f"{row['file_name']}: undeclared ripeness override")152        for i, source in enumerate(objects["bbox_source"]):153            check(source in ("field", "visual_estimate"), f"{row['file_name']}: unknown bbox_source {source}")154            check(source == "field" or (row["tree_name"], row["side_index"]) in ADDED_BOXES,155                  f"{row['file_name']}: undeclared added box")156        for note in objects["notes"]:157            check(note is None or "FAKSI" not in note.upper().replace("FRAKSI", ""),158                  f"{row['file_name']}: FAKSI left unnormalised in notes")159 160    tree_rows = read_jsonl(os.path.join(ann, "trees.jsonl"))161    check({r["tree_name"] for r in tree_rows} == set(trees), "trees.jsonl does not cover every tree")162    boxes_by_tree = {}163    for row in rows:164        boxes_by_tree[row["tree_name"]] = boxes_by_tree.get(row["tree_name"], 0) + row["num_bunches"]165    for row in tree_rows:166        check(row["total_detections"] == boxes_by_tree[row["tree_name"]],167              f"{row['tree_name']}: total_detections disagrees with metadata.jsonl")168        check(len(row["bunches"]) == row["total_unique_bunches"],169              f"{row['tree_name']}: bunch count disagrees with total_unique_bunches")170        for bunch in row["bunches"]:171            check(len(bunch["appearances"]) == bunch["appearance_count"],172                  f"{row['tree_name']}: appearance_count disagrees with appearances")173 174    print(f"trees {len(trees)} | images {len(seen_images)} | labels {len(seen_labels)} | "175          f"boxes {sum(r['num_bunches'] for r in rows)}")176    if fail:177        print(f"FAIL: {len(fail)} problem(s)")178        for message in fail:179            print(f"  - {message}")180        return 1181    print("PASS: raw SHA-256 digests hold and the corrected layer is consistent")182    return 0183 184 185if __name__ == "__main__":186    sys.exit(main(sys.argv[1] if len(sys.argv) > 1 else "."))187