arshiyahafis/WebDigest

1from rest_framework.response import Response2from rest_framework.decorators import api_view3 4from bs4 import BeautifulSoup5import requests6 7def getText(url : str):8    response = requests.get(url)9 10    if response.status_code == 200:11        html_content = response.content 12    else:13        print(f"[INFO] couldn't access website data, try again")14        return15    soup = BeautifulSoup(html_content, 'html.parser')16 17    text_elements = soup.find_all(['p'])18    scraped_text = ' '.join(element.get_text() for element in text_elements)19        20    if len(scraped_text) > 20000:21        print(f"[ERROR] page too large to perform qna")22        return23    24    return scraped_text 25 26 27from transformers import AutoTokenizer, AutoModelForSeq2SeqLM 28 29model = AutoModelForSeq2SeqLM.from_pretrained('google/flan-t5-large')30tokenizer = AutoTokenizer.from_pretrained('google/flan-t5-large')31 32def getAnswer(question : str, url : str):33    context = getText(url)34 35 36    inputs = tokenizer(f"context : {context}, question : {question}", return_tensors = 'pt').input_ids37 38    outputs = model.generate(39        inputs, 40        min_length = 10,41        max_new_tokens = 600,42        length_penalty = 1,43        num_beams = 3,44        no_repeat_ngram_size = 3,45        temperature = 0.7,46        top_k = 110,47        top_p = 0.8,48        repetition_penalty = 2.149    )50 51    answer = tokenizer.decode(outputs[0], skip_special_tokens = True)52 53    return answer54 55 56 57 58@api_view(['POST'])59def findAnswer(request): # {url : website, question : question}60    answer = getAnswer(request.data['question'], request.data['url'])61    return Response({'question' : request.data['question'], 'answer' : answer})62