seai2526-uniba-TheClouds/Code-Comment-Classification-Api

1"""Utility functions for model training and evaluation."""2 3import os4from typing import List5 6LANGS: List[str] = ["java", "python", "pharo"]7 8 9def load_dataset_splits(base_dir=None, langs=None):10    """Load dataset splits from CSV files under data/raw.11 12    Expects files like data/raw/java_train.csv, data/raw/java_test.csv, etc.13    Returns a dict mapping split names (e.g. "java_test") to pandas DataFrames.14 15    Raises:16        FileNotFoundError: se la directory base o un file atteso non esiste.17        ImportError: se pandas non è installato.18 19    """20    if base_dir is None:21        base_dir = os.path.join("data", "raw")22 23    if langs is None:24        langs = LANGS25 26    if not os.path.isdir(base_dir):27        raise FileNotFoundError(28            f"CSV datasets not found under {base_dir}; cannot load dataset splits."29        )30 31    try:32        import pandas as pd33    except Exception as e:34        raise ImportError("pandas is required to load dataset splits") from e35 36    datasets = {}37    for lang in langs:38        for split in ("train", "test"):39            fname = f"{lang}_{split}.csv"40            path = os.path.join(base_dir, fname)41            if not os.path.isfile(path):42                raise FileNotFoundError(f"Expected dataset file missing: {path}")43            df = pd.read_csv(path)44            datasets[f"{lang}_{split}"] = df45 46    return datasets47 48 49def parse_labels_column(df):50    """Parse the 'labels' column of a DataFrame into lists of integers."""51 52    def _parse_one(x):53        if isinstance(x, str):54            s = x.strip()55            if s.startswith("[") and s.endswith("]"):56                s = s[1:-1]57            return [int(tok) for tok in s.split() if tok]58        try:59            import numpy as np60 61            if isinstance(x, np.ndarray):62                return [int(v) for v in x.tolist()]63        except ImportError:64            pass65        if isinstance(x, (list, tuple)):66            return [int(v) for v in x]67        raise ValueError(f"Formato labels non gestito: {type(x)} -> {x!r}")68 69    df["labels"] = df["labels"].apply(_parse_one)70    return df71