MuratcanKoylan/Marketing-Memory-Routing-8B

1import json2import sys3from typing import List, Dict, Any4 5def clean_datum(item: Dict[str, Any]) -> Dict[str, Any]:6    """7    Clean a single data item:8    1. Remove 'none' if other categories are present.9    2. Deduplicate categories.10    3. Ensure consistent formatting.11    """12    if "labels" not in item or "categories" not in item["labels"]:13        return item14 15    cats = item["labels"]["categories"]16    # Deduplicate17    cats = list(set(cats))18    19    # Remove 'none' if other categories exist20    if len(cats) > 1 and "none" in cats:21        cats.remove("none")22    23    # Update the item24    item["labels"]["categories"] = cats25    return item26 27def clean_file(input_path: str, output_path: str):28    print(f"Cleaning {input_path} -> {output_path}")29    cleaned_count = 030    data = []31    32    # Read input33    with open(input_path, 'r') as f:34        content = f.read().strip()35        if not content:36            print("Empty file")37            return38 39        # Handle JSONL or list of JSON40        if content.startswith('[') and content.endswith(']'):41            raw_data = json.loads(content)42        else:43            raw_data = [json.loads(line) for line in content.split('\n') if line.strip()]44            45    # Process46    for item in raw_data:47        original_cats = item.get("labels", {}).get("categories", [])48        cleaned_item = clean_datum(item)49        new_cats = cleaned_item["labels"]["categories"]50        51        if set(original_cats) != set(new_cats):52            cleaned_count += 153            54        data.append(cleaned_item)55        56    # Write output (always as JSONL for training)57    with open(output_path, 'w') as f:58        for item in data:59            f.write(json.dumps(item) + '\n')60            61    print(f"Processed {len(data)} items. Cleaned {cleaned_count} items (removed 'none' or duplicates).")62 63if __name__ == "__main__":64    if len(sys.argv) < 2:65        print("Usage: python clean_data.py input_file [output_file]")66        sys.exit(1)67    68    input_file = sys.argv[1]69    output_file = sys.argv[2] if len(sys.argv) > 2 else input_file.replace('.json', '_cleaned.jsonl').replace('.jsonl', '_cleaned.jsonl')70    71    clean_file(input_file, output_file)72 73