ANDRYHA/FakeNewsClassifier

1from transformers import FSMTForConditionalGeneration, FSMTTokenizer
2from transformers import AutoModelForSequenceClassification
3from transformers import AutoTokenizer
4from langdetect import detect
5from newspaper import Article
6from PIL import Image
7import streamlit as st
8import requests
9import torch
10
11st.markdown("## Prediction of Fakeness by Given URL")
12background = Image.open('logo.jpg')
13st.image(background)
14
15st.markdown(f"### Article URL")
16text = st.text_area("Insert some url here", 
17        value="https://en.globes.co.il/en/article-yandex-looks-to-expand-activities-in-israel-1001406519")
18
19@st.cache(allow_output_mutation=True)
20def get_models_and_tokenizers():
21    model_name = 'distilbert-base-uncased-finetuned-sst-2-english'
22    model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=2)
23    model.eval()
24    tokenizer = AutoTokenizer.from_pretrained(model_name)
25    model.load_state_dict(torch.load('./model.pth', map_location='cpu'))
26
27    model_name_translator = "facebook/wmt19-ru-en"
28    tokenizer_translator = FSMTTokenizer.from_pretrained(model_name_translator)
29    model_translator = FSMTForConditionalGeneration.from_pretrained(model_name_translator)
30    model_translator.eval()
31    return model, tokenizer, model_translator, tokenizer_translator
32
33model, tokenizer, model_translator, tokenizer_translator = get_models_and_tokenizers()
34
35article = Article(text)
36article.download()
37article.parse()
38concated_text = article.title + '. ' + article.text
39lang = detect(concated_text)
40
41st.markdown(f"### Language detection")
42
43if lang == 'ru':
44    st.markdown(f"The language of this article is {lang.upper()} so we translated it!")
45    with st.spinner('Waiting for translation'):
46        input_ids = tokenizer_translator.encode(concated_text, 
47            return_tensors="pt", max_length=512, truncation=True)
48        outputs = model_translator.generate(input_ids)
49        decoded = tokenizer_translator.decode(outputs[0], skip_special_tokens=True)
50        st.markdown("### Translated Text")
51        st.markdown(f"{decoded[:777]}")
52        concated_text = decoded
53else:
54    st.markdown(f"The language of this article for sure:  {lang.upper()}!")
55
56    st.markdown("### Extracted Text")
57    st.markdown(f"{concated_text[:777]}")
58
59tokens_info = tokenizer(concated_text, truncation=True, return_tensors="pt")
60with torch.no_grad():
61    raw_predictions = model(**tokens_info)
62softmaxed = int(torch.nn.functional.softmax(raw_predictions.logits[0], dim=0)[1] * 100)
63st.markdown("### Fakeness Prediction")
64st.progress(softmaxed)
65st.markdown(f"This is fake by **{softmaxed}%**!")
66if (softmaxed > 70):
67    st.error('We would not trust this text!')
68elif (softmaxed > 40):
69    st.warning('We are not sure about this text!')
70else:
71    st.success('We would trust this text!')