Rafa1986/Data-Analytics-Class

1import gradio as gr2import os3import PyPDF24import pandas as pd5import docx6import json7from docx import Document8from transformers import pipeline9 10# Configurar Hugging Face API Token11HF_API_TOKEN = os.getenv("HUGGINGFACE_API_TOKEN")12 13# Carregar o modelo DeepSeek Coder 1.3B14chatbot_pipeline = pipeline("text-generation", model="deepseek-ai/deepseek-coder-1.3b-instruct", token=HF_API_TOKEN)15 16def extract_files_from_folder(folder_path):17    """Scans a folder for PDF, TXT, CSV, DOCX, and IPYNB files."""18    extracted_files = {"pdf": [], "txt": [], "csv": [], "docx": [], "ipynb": []}19    20    for root, _, files in os.walk(folder_path):21        for file_name in files:22            file_path = os.path.join(root, file_name)23            if file_name.endswith(".pdf"):24                extracted_files["pdf"].append(file_path)25            elif file_name.endswith(".txt"):26                extracted_files["txt"].append(file_path)27            elif file_name.endswith(".csv"):28                extracted_files["csv"].append(file_path)29            elif file_name.endswith(".docx"):30                extracted_files["docx"].append(file_path)31            elif file_name.endswith(".ipynb"):32                extracted_files["ipynb"].append(file_path)33    return extracted_files34 35def get_text_from_pdf(pdf_files):36    text = ""37    for pdf_path in pdf_files:38        with open(pdf_path, "rb") as pdf_file:39            reader = PyPDF2.PdfReader(pdf_file)40            for page in reader.pages:41                text += page.extract_text() + "\n"42    return text43 44def read_text_from_files(file_paths):45    text = ""46    for file_path in file_paths:47        with open(file_path, "r", encoding="utf-8", errors="ignore") as file:48            text += file.read() + "\n"49    return text50 51def get_text_from_csv(csv_files):52    text = ""53    for csv_path in csv_files:54        df = pd.read_csv(csv_path)55        text += df.to_string() + "\n"56    return text57 58def get_text_from_docx(docx_files):59    text = ""60    for docx_path in docx_files:61        doc = Document(docx_path)62        for para in doc.paragraphs:63            text += para.text + "\n"64    return text65 66def combine_text_from_files(extracted_files):67    text = (68        get_text_from_pdf(extracted_files["pdf"]) +69        read_text_from_files(extracted_files["txt"]) +70        get_text_from_csv(extracted_files["csv"]) +71        get_text_from_docx(extracted_files["docx"])72    )73    return text74 75def generate_response(question, text):76    """Uses the DeepSeek Coder model to answer questions based on extracted text."""77    prompt = f"Question: {question}\nBased on the following document content:\n{text[:3000]}"  # Limite de 3000 caracteres78    response = chatbot_pipeline(prompt, max_length=500, truncation=True)[0]['generated_text']79    return response.strip()80 81def chatbot_interface(question):82    folder_path = "New_Data_Analytics/"83    extracted_files = extract_files_from_folder(folder_path)84    text = combine_text_from_files(extracted_files)85    86    if not text.strip():87        return "No valid files found. Please upload supported file types."88    89    return generate_response(question, text)90 91demo = gr.Interface(92    fn=chatbot_interface,93    inputs=gr.Textbox(label="Ask a question", placeholder="Type your question here..."),94    outputs=gr.Textbox(label="Answer")95)96 97demo.launch()98