MuratcanKoylan/Marketing-Memory-Routing-8B

1import json2import sys3 4def clean_batch(filepath):5    print(f"Cleaning {filepath}...")6    cleaned_data = []7    fixed_count = 08    9    with open(filepath, 'r') as f:10        for line in f:11            if not line.strip():12                continue13            item = json.loads(line)14            cats = item['labels']['categories']15            16            if 'none' in cats and len(cats) > 1:17                print(f"Fixing mixed 'none' in {item['scenario_id']}: {cats}")18                cats.remove('none')19                item['labels']['categories'] = cats20                item['metadata']['cleaned_none_mix'] = True21                fixed_count += 122            23            cleaned_data.append(item)24    25    output_path = filepath.replace('.jsonl', '_cleaned.jsonl')26    with open(output_path, 'w') as f:27        for item in cleaned_data:28            f.write(json.dumps(item) + '\n')29            30    print(f"Cleaned {len(cleaned_data)} items. Fixed {fixed_count} issues.")31    print(f"Saved to {output_path}")32 33if __name__ == "__main__":34    if len(sys.argv) < 2:35        print("Usage: python3 clean_batch.py <jsonl_file>")36        sys.exit(1)37    clean_batch(sys.argv[1])38 39