smart-models/Placebo_AI

1import chromadb2import os3import json4 5import os6base_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "data", "processed")7db_path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "db", "vector_store")8 9client = chromadb.PersistentClient(path=db_path)10collections = client.list_collections()11print(f"Collections: {[c.name for c in collections]}")12 13if collections:14    collection = collections[0]15    print(f"Using collection: {collection.name}")16    17    # Let's get distinct book names18    # Getting a sample of metadatas19    sample = collection.get(limit=10, include=["metadatas"])20    print(f"Sample metadata: {sample['metadatas']}")21    22    # We will check each jsonl file23    jsonl_files = [f for f in os.listdir(base_dir) if f.endswith(".jsonl")]24    25    unindexed_files = []26    27    for f_name in jsonl_files:28        jsonl_path = os.path.join(base_dir, f_name)29        30        # Read the first valid line to get the book_name or subject31        book_names_in_file = set()32        subjects_in_file = set()33        try:34            with open(jsonl_path, "r", encoding="utf-8") as f:35                for line in f:36                    line = line.strip()37                    if not line: continue38                    try:39                        data = json.loads(line)40                        if "book_name" in data:41                            book_names_in_file.add(data["book_name"])42                        if "subject" in data:43                            subjects_in_file.add(data["subject"])44                    except:45                        pass46                    if len(book_names_in_file) > 0:47                        break # Just checking the first document is usually enough, but let's say we got one48        except Exception as e:49            print(f"Error reading {f_name}: {e}")50            continue51            52        is_indexed = False53        54        # Check if any book_name from this file exists in the DB55        for bn in book_names_in_file:56            res = collection.get(where={"book_name": bn}, limit=1, include=["metadatas"])57            if res and res["metadatas"]:58                is_indexed = True59                break60                61        # If no book_name, check by subject62        if not is_indexed and not book_names_in_file:63            for sub in subjects_in_file:64                res = collection.get(where={"subject": sub}, limit=1, include=["metadatas"])65                if res and res["metadatas"]:66                    is_indexed = True67                    break68                    69        # If we couldn't find it70        if not is_indexed:71            unindexed_files.append(f_name)72            73    print("\n--- UNINDEXED JSONL FILES ---")74    for f_name in unindexed_files:75        print(f_name)76        77    print(f"\nFound {len(unindexed_files)} unindexed JSONL files out of {len(jsonl_files)}.")78    79    with open("unindexed_files.txt", "w") as out:80        for f in unindexed_files:81            out.write(f + "\n")82