aibyml/J.A.R.V.I.S.1.0

1import openai2from langchain.embeddings.openai import OpenAIEmbeddings3from langchain.vectorstores import FAISS4from langchain.llms import OpenAI5from langchain.embeddings.sentence_transformer import SentenceTransformerEmbeddings6from langchain.schema import Document7from pypdf import PdfReader8from langchain.llms.openai import OpenAI9from langchain.chains.summarize import load_summarize_chain10from langchain.text_splitter import RecursiveCharacterTextSplitter11import pdfplumber12from transformers import pipeline13 14# Extract Information from PDF file15def get_pdf_text(pdf_doc):16    text = ""17    pdf_reader = PdfReader(pdf_doc)18    for page in pdf_reader.pages:19        text += page.extract_text()20    return text21 22# iterate over files in23# that user uploaded PDF files, one by one24def create_docs(user_pdf_list, unique_id):25    docs=[]26    for filename in user_pdf_list:27        28        chunks=get_pdf_text(filename)29 30        #Adding items to our list - Adding data & its metadata31        docs.append(Document(32            page_content=chunks,33            metadata={"name": filename.name,"type=":filename.type,"size":filename.size,"unique_id":unique_id},34        ))35 36    return docs37 38def split_docs(documents, chunk_size=3000, chunk_overlap=20):39    text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)40    docs = text_splitter.split_documents(documents)41    return docs42 43# Create embeddings instance44def create_embeddings_load_data():45    embeddings = OpenAIEmbeddings()46    #embeddings = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")47    return embeddings48 49 50# Function to push data to Vector Store - FAISS here51def push_to_store(embeddings,docs):52    db = FAISS.from_documents(docs, embeddings)53    print("done......upload to vector store")54    return db55 56    57# Function to help us get relavant documents from vector store - based on user input58def get_similar_docs(query,k,db,embeddings,unique_id):59    similar_docs = db.similarity_search(query, int(k),{"unique_id":unique_id})60    print(similar_docs)61    return similar_docs62 63 64# Helps us get the summary of a document65def get_summary(current_doc):66    llm = OpenAI(temperature=0)67    #llm = HuggingFaceHub(repo_id="bigscience/bloom", model_kwargs={"temperature":1e-10})68    chain = load_summarize_chain(llm, chain_type="map_reduce")69    summary = chain.run([current_doc])70 71    return summary72 73# LLM Q&A Code74from langchain.chains.question_answering import load_qa_chain75from langchain.schema import (76    AIMessage,77    HumanMessage,78    SystemMessage79)80llm = OpenAI()81chain = load_qa_chain(llm, chain_type="stuff")82 83# This function will help us get the answer from the relevant docs matching input text84def question_answer(relevant_docs, enquiry):85    answer = chain.run(input_documents=relevant_docs, question=enquiry)86    return answer87