Ubaidbhat/DatabaseTuned

1import shutil2import requests3import sys4from typing import Optional, List, Tuple5import json6from langchain_community.llms import HuggingFaceHub7 8 9##Loading the Model to answer questions10import torch11from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig12from peft import PeftModel, PeftConfig13 14 15peft_model_id = "Ubaidbhat/zephr_database_finetuned"16config = PeftConfig.from_pretrained(peft_model_id)17print(config.base_model_name_or_path)18bnb_config = BitsAndBytesConfig(19    load_in_4bit = True,20    bnb_4bit_use_double_quant=True,21    bnb_4bit_quant_type="nf4",22    bnb_4bit_compute_dtype=torch.bfloat1623)24 25d_map = {"": torch.cuda.current_device()} if torch.cuda.is_available() else None26 27model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path, quantization_config=bnb_config, device_map=d_map)28tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)29model = PeftModel.from_pretrained(model, peft_model_id)30model = model.merge_and_unload()31 32 33 34 35 36##Creating base Model Chain37from langchain.llms import HuggingFacePipeline38from langchain.prompts import PromptTemplate39from transformers import pipeline40from langchain_core.output_parsers import StrOutputParser41from langchain.chains import LLMChain42 43text_generation_pipeline = pipeline(44    model=model,45    tokenizer=tokenizer,46    task="text-generation",47    temperature=0.2,48    do_sample=True,49    repetition_penalty=1.1,50    return_full_text=True,51    max_new_tokens=400,52    pad_token_id=tokenizer.eos_token_id,53)54 55llm = HuggingFacePipeline(pipeline=text_generation_pipeline)56 57prompt_template = """58<|system|>59Answer the question based on your knowledge.60</s>61<|user|>62{question}63</s>64<|assistant|>65"""66 67prompt = PromptTemplate(68    input_variables=["question"],69    template=prompt_template,70)71 72llm_chain = prompt | llm | StrOutputParser()73 74def inference(question):75    llmAnswer = llm_chain.invoke({"question": question})76    llmAnswer = llmAnswer.rstrip()77    return llmAnswer78 79 80 81import gradio as gr82from langchain_core.runnables import RunnablePassthrough83 84def predict(question):85    return inference(question)86    87pred = gr.Interface(88    fn=predict,89    inputs=[90        gr.Textbox(label="Question"),91    ],92    outputs="text",93    title="Finetuned Zephr Model in the Database Management Domain"94)95 96pred.launch(share=True)97 98 99 100