raulmss/netflix-chat-bot

1import json2import re3 4def clean_text(text):5    # Remove extra whitespace, newlines, and special characters6    text = re.sub(r'\s+', ' ', text).strip()7    return text8 9def load_scraped_data(file_path):10    try:11        with open(file_path, 'r', encoding='utf-8') as f:12            return json.load(f)13    except FileNotFoundError:14        print(f"Error: Scraped data file not found at {file_path}")15        return []16 17def process_data(scraped_data):18    processed_articles = []19    for article in scraped_data:20        url = article['url']21        title = article['title']22        cleaned_content_blocks = [clean_text(block) for block in article['content'] if block]23        full_content = " ".join(cleaned_content_blocks)  # Join with space instead of newline24        processed_articles.append({'url': url, 'title': title, 'content': full_content})25    return processed_articles26 27def chunk_data(processed_articles, chunk_size=500, overlap=100):28    chunks = []29    for article in processed_articles:30        url = article['url']31        title = article['title']32        content = article['content']33        for i in range(0, len(content), chunk_size - overlap):34            chunk = content[i:i + chunk_size]35            chunks.append({'url': url, 'title': title, 'chunk': chunk})36    return chunks37 38def save_processed_data(data, output_file):39    with open(output_file, 'w', encoding='utf-8') as f:40        json.dump(data, f, indent=4, ensure_ascii=False)41    print(f"Processed data saved to {output_file}")42 43if __name__ == "__main__":44    input_file = './data/netflix_help_data.json'45    output_file = './data/netflix_help_processed.json'46 47    scraped_data = load_scraped_data(input_file)48    if scraped_data:49        processed_data = process_data(scraped_data)50        chunked_data = chunk_data(processed_data)51        save_processed_data(chunked_data, output_file)52    else:53        print("No scraped data found. Run scraper.py first.")54