taqbaylit/Kabyle_Road_Traffic_Code

1#!/usr/bin/env python32"""3Kabyle-English Road Traffic Code Dataset - CSV Loader4Loads from dataset.csv and converts to HuggingFace format5"""6 7import pandas as pd8from datasets import load_dataset, Dataset, DatasetDict9 10def load_from_csv(csv_path="dataset.csv"):11    """Load dataset from CSV and convert to HuggingFace format"""12    df = pd.read_csv(csv_path)13 14    # Convert to list of dicts with proper structure15    records = []16    for _, row in df.iterrows():17        record = {18            "id": row["id"],19            "translation": {20                "en": row["english"],21                "kab": row["kabyle"]22            },23            "category": row["category"],24            "subcategory": row["subcategory"],25            "domain": "road_traffic",26            "complexity": row["complexity"],27            "tokens_en": int(row["tokens_en"]),28            "tokens_kab": int(row["tokens_kab"])29        }30        records.append(record)31 32    return records33 34def create_splits(csv_path="dataset.csv"):35    """Create train/validation/test splits from CSV"""36    records = load_from_csv(csv_path)37 38    # Split based on ID suffix (matching original stratified split)39    train_ids, val_ids, test_ids = [], [], []40 41    for r in records:42        idx = int(r["id"].split("_")[-1])43        cat = r["category"]44 45        # Reproduce stratified split logic46        if cat == "dangers":47            if idx % 10 < 7:48                train_ids.append(r)49            elif idx % 10 < 8:50                val_ids.append(r)51            else:52                test_ids.append(r)53        elif cat == "prohibitions":54            if idx % 10 < 6:55                train_ids.append(r)56            elif idx % 10 < 8:57                val_ids.append(r)58            else:59                test_ids.append(r)60        elif cat == "obligations":61            if idx % 10 < 5:62                train_ids.append(r)63            elif idx % 10 < 7:64                val_ids.append(r)65            else:66                test_ids.append(r)67        else:  # end_of_restrictions68            if idx % 10 < 4:69                train_ids.append(r)70            elif idx % 10 < 7:71                val_ids.append(r)72            else:73                test_ids.append(r)74 75    return {76        "train": Dataset.from_list(train_ids),77        "validation": Dataset.from_list(val_ids),78        "test": Dataset.from_list(test_ids)79    }80 81def load_dataset_from_csv(csv_path="dataset.csv"):82    """Main function - returns DatasetDict"""83    splits = create_splits(csv_path)84    return DatasetDict(splits)85 86if __name__ == "__main__":87    print("Chargement depuis dataset.csv...")88    dataset = load_dataset_from_csv()89 90    print(f"\nTrain: {len(dataset['train'])} examples")91    print(f"Validation: {len(dataset['validation'])} examples")92    print(f"Test: {len(dataset['test'])} examples")93 94    print("\n--- Exemple ---")95    ex = dataset['train'][0]96    print(f"ID: {ex['id']}")97    print(f"EN: {ex['translation']['en']}")98    print(f"KAB: {ex['translation']['kab']}")99    print(f"Category: {ex['category']}")100