smart-models/Placebo_AI

1import os2import json3import fitz4from tqdm import tqdm5 6def process_missing_books():7    anatomy_dir = r"d:\sample chatbot\MBBS\books\anatomy"8    json_path = r"d:\sample chatbot\anatomy_data.json"9    10    if os.path.exists(json_path):11        print(f"Loading existing data from {json_path}...")12        with open(json_path, 'r', encoding='utf-8') as f:13            data = json.load(f)14    else:15        print(f"Creating new data list.")16        data = []17 18    processed_books = set(item.get("book_name") for item in data)19    print(f"Found {len(processed_books)} books already in JSON.")20 21    new_items_added = 022 23    for root, dirs, files in os.walk(anatomy_dir):24        for filename in files:25            if not filename.lower().endswith(".pdf"):26                continue27            if filename.startswith("._"):28                continue29 30            if filename in processed_books:31                continue32 33            pdf_path = os.path.join(root, filename)34            print(f"Processing missing book: {filename}")35            36            try:37                doc = fitz.open(pdf_path)38                total_pages = doc.page_count39                40                for page_idx in tqdm(range(total_pages), desc=f"Extracting {filename}"):41                    page = doc[page_idx]42                    text = page.get_text().strip()43                    44                    if len(text) < 50:45                        continue46                        47                    page_data = {48                        "book_name": filename,49                        "page_number": page_idx + 1,50                        "content": text,51                        "track": "mbbs",52                        "subject": "anatomy"53                    }54                    data.append(page_data)55                    new_items_added += 156                    57                doc.close()58                processed_books.add(filename)59            except Exception as e:60                print(f"Error processing {filename}: {e}")61 62    if new_items_added > 0:63        print(f"Added {new_items_added} new pages. Saving to {json_path}...")64        with open(json_path, 'w', encoding='utf-8') as f:65            json.dump(data, f, indent=2, ensure_ascii=False)66        print("Save complete.")67    else:68        print("No new books found. JSON is up to date.")69 70if __name__ == "__main__":71    process_missing_books()72