Droid210/FleetVision

1"""Evaluation metrics and reporting utilities."""2from typing import List3 4import torch5from sklearn.metrics import (6    classification_report,7    confusion_matrix,8    precision_recall_fscore_support,9)10from torch import nn11from torch.utils.data import DataLoader12 13 14@torch.no_grad()15def evaluate_model(16    model: nn.Module,17    loader: DataLoader,18    device: torch.device,19) -> None:20    """Evaluate model on validation/test set.21 22    Args:23        model: Trained ViT model.24        loader: DataLoader for evaluation.25        device: Torch device.26    """27    model.eval()28 29    y_true: List[int] = []30    y_pred: List[int] = []31    confidences: List[float] = []32 33    for pixel_values, labels in loader:34        pixel_values = pixel_values.to(device)35        outputs = model(pixel_values)36        logits = outputs.logits37 38        predictions = logits.argmax(dim=1).cpu().tolist()39        probs = torch.softmax(logits, dim=1)40        max_probs = probs.max(dim=1).values.cpu().tolist()41 42        y_true.extend(labels.tolist())43        y_pred.extend(predictions)44        confidences.extend(max_probs)45 46    cm = confusion_matrix(y_true, y_pred)47    precision, recall, f1, _ = precision_recall_fscore_support(48        y_true,49        y_pred,50        average="weighted",51        zero_division=0,52    )53 54    print("\n=== Confusion Matrix ===")55    print(cm)56    print(f"[[TN={cm[0,0]}, FP={cm[0,1]}],")57    print(f" [FN={cm[1,0]}, TP={cm[1,1]}]]")58 59    print("\n=== Classification Report ===")60    print(classification_report(y_true, y_pred, target_names=["Whole", "Damaged"], zero_division=0))61 62    print("=== Summary Metrics ===")63    print(f"Precision: {precision:.4f}")64    print(f"Recall   : {recall:.4f}")65    print(f"F1-Score : {f1:.4f}")66 67    # Detailed metrics for damage detection68    tp = cm[1, 1]69    fp = cm[0, 1]70    fn = cm[1, 0]71    tn = cm[0, 0]72 73    print(f"\nDamage Detection Metrics:")74    print(f"True Positives  (Damaged detected):   {tp}")75    print(f"False Positives (Whole as Damaged):   {fp}")76    print(f"False Negatives (Damaged as Whole):   {fn}  ⚠️ CRITICAL")77    print(f"True Negatives  (Whole detected):     {tn}")78