LeonardoMdSA/Context-aware-NLP-classification-platform-with-MCP

1#!/usr/bin/env python2import argparse3import json4from pathlib import Path5 6import joblib7from sklearn.metrics import (8    accuracy_score,9    precision_recall_fscore_support,10    classification_report11)12 13BASE_DIR = Path(__file__).resolve().parent.parent14MODELS_DIR = BASE_DIR / "models"15DATA_DIR = BASE_DIR / "data"16 17 18def load_model():19    model_path = MODELS_DIR / "trained_pipeline.joblib"20    if not model_path.exists():21        raise FileNotFoundError(f"Model not found: {model_path}")22    return joblib.load(model_path)23 24 25def load_dataset(dataset_path: Path):26    if not dataset_path.exists():27        raise FileNotFoundError(f"Dataset not found: {dataset_path}")28 29    # Hard guard: never evaluate on training data30    if dataset_path.name in {"training_data.json", "train.json"}:31        raise RuntimeError(32            f"Refusing to evaluate on training dataset: {dataset_path.name}"33        )34 35    with dataset_path.open("r", encoding="utf-8") as f:36        raw = json.load(f)37 38    if isinstance(raw, list):39        samples = raw40    elif isinstance(raw, dict) and "samples" in raw:41        samples = raw["samples"]42    else:43        raise ValueError("Unsupported JSON dataset format")44 45    texts = []46    labels = []47 48    for i, item in enumerate(samples):49        if "text" not in item or "label" not in item:50            raise ValueError(f"Invalid sample at index {i}: {item}")51        texts.append(item["text"])52        labels.append(item["label"])53 54    return texts, labels55 56 57def evaluate(model, X, y):58    y_pred = model.predict(X)59 60    acc = accuracy_score(y, y_pred)61    precision, recall, f1, _ = precision_recall_fscore_support(62        y, y_pred, average="weighted", zero_division=063    )64 65    print("====================================")66    print("Offline Evaluation Results")67    print("====================================")68    print(f"Samples  : {len(y)}")69    print(f"Accuracy : {acc:.4f}")70    print(f"Precision: {precision:.4f}")71    print(f"Recall   : {recall:.4f}")72    print(f"F1-score : {f1:.4f}")73    print()74    print("Detailed Classification Report")75    print("------------------------------------")76    print(classification_report(y, y_pred, zero_division=0))77 78 79def main():80    parser = argparse.ArgumentParser(81        description="Offline evaluation using held-out JSON dataset"82    )83    parser.add_argument(84        "--data",85        default=str(DATA_DIR / "samples" / "eval.json"),86        help="Path to evaluation dataset (default: data/samples/eval.json)"87    )88 89    args = parser.parse_args()90 91    model = load_model()92    X, y = load_dataset(Path(args.data))93    evaluate(model, X, y)94 95 96if __name__ == "__main__":97    main()98