FakeNewsDetector/Space1
0
1from flask import Flask, request, render_template, jsonify2from markupsafe import escape3import joblib4import pickle5import re6import torch7import torch.nn as nn8from tensorflow.keras.preprocessing.sequence import pad_sequences9from flask_cors import CORS10from apscheduler.schedulers.background import BackgroundScheduler11import feedparser12import requests13import datetime14import atexit15 16# Add this at the very top for Hugging Face compatibility17import os18import sys19sys.path.append(os.getcwd())20 21import logging22from threading import Thread23from time import sleep24# ... keep all other imports ...25 26# Modify model paths for Hugging Face27BASE_DIR = os.path.dirname(os.path.abspath(__file__))28model_paths = [29 os.path.join(BASE_DIR, "models", "combined_model.pkl"),30 os.path.join(BASE_DIR, "models", "combined_model_true.pkl")31]32 33# Add Hugging Face specific configuration34class ReverseProxied(object):35 def __init__(self, app):36 self.app = app37 38 def __call__(self, environ, start_response):39 scheme = environ.get('HTTP_X_FORWARDED_PROTO', 'http')40 if scheme:41 environ['wsgi.url_scheme'] = scheme42 return self.app(environ, start_response)43 44app = Flask(__name__)45app.wsgi_app = ReverseProxied(app.wsgi_app)46CORS(app)47# -------------------------------------------------48# 0. Define the Custom LSTMClassifier (for joblib)49# -------------------------------------------------50class LSTMClassifier(nn.Module):51 def __init__(self, vocab_size, embed_dim, hidden_dim, output_dim, n_layers, dropout):52 super(LSTMClassifier, self).__init__()53 self.embedding = nn.Embedding(vocab_size, embed_dim)54 self.lstm = nn.LSTM(embed_dim, hidden_dim, num_layers=n_layers,55 batch_first=True, dropout=dropout)56 self.fc = nn.Linear(hidden_dim, output_dim)57 58 def forward(self, x):59 embedded = self.embedding(x)60 output, (hidden, cell) = self.lstm(embedded)61 out = self.fc(hidden[-1])62 return out63 64# Ensure joblib.load finds this class65globals()["LSTMClassifier"] = LSTMClassifier66 67# -------------------------------------------------68# 1. Helper: Preprocess the input text69# -------------------------------------------------70def preprocess_text(text):71 text = text.lower()72 text = re.sub(r'[^\w\s]', '', text) # Remove punctuation and special characters73 text = re.sub(r'\s+', ' ', text).strip() # Remove extra spaces74 return text75 76# -------------------------------------------------77# 2. Load the Combined Models (for news classification)78# -------------------------------------------------79combined_model_path1 = r"D:\prototype-20250323T082409Z-001\prototype\combined_model.pkl"80combined_model_path2 = r"D:\prototype-20250323T082409Z-001\prototype\combined_model_true.pkl"81try:82 combined_model1 = joblib.load(combined_model_path1)83 combined_model2 = joblib.load(combined_model_path2)84 print("Combined models loaded successfully.")85except Exception as e:86 print("Error loading models:", e)87 exit()88 89# -------------------------------------------------90# 3. Function to extract individual components91# -------------------------------------------------92def get_model_components(model_dict):93 return (94 model_dict["tfidf"],95 model_dict["svd"],96 model_dict["lr_model"],97 model_dict["rf_model"],98 model_dict["tokenizer_lstm"],99 model_dict["lstm_model"],100 model_dict["max_len"],101 model_dict.get("distilbert_model", None),102 model_dict.get("tokenizer_distilbert", None)103 )104 105components1 = get_model_components(combined_model1)106components2 = get_model_components(combined_model2)107 108# Use CPU for inference (change to "cuda" if a GPU is available)109device = torch.device("cpu")110 111# -------------------------------------------------112# 4. Define Prediction Functions for Each Component113# -------------------------------------------------114def predict_lr_prob(tfidf, lr_model, text):115 vec = tfidf.transform([preprocess_text(text)])116 return lr_model.predict_proba(vec)[0][1]117 118def predict_rf_prob(tfidf, svd, rf_model, text):119 vec = tfidf.transform([preprocess_text(text)])120 vec_reduced = svd.transform(vec)121 return rf_model.predict_proba(vec_reduced)[0][1]122 123def predict_lstm_prob(tokenizer_lstm, lstm_model, max_len, text):124 seq = tokenizer_lstm.texts_to_sequences([preprocess_text(text)])125 padded = pad_sequences(seq, maxlen=max_len, padding='post', truncating='post')126 tensor_input = torch.tensor(padded, dtype=torch.long).to(device)127 lstm_model.eval()128 with torch.no_grad():129 output = lstm_model(tensor_input)130 return torch.sigmoid(output).item()131 132def predict_distilbert_prob(tokenizer_distilbert, distilbert_model, text):133 if tokenizer_distilbert and distilbert_model:134 inputs = tokenizer_distilbert(135 preprocess_text(text),136 return_tensors="pt",137 truncation=True,138 padding='max_length',139 max_length=128140 )141 distilbert_model.eval()142 with torch.no_grad():143 logits = distilbert_model(**inputs).logits144 return torch.softmax(logits, dim=1).cpu().numpy()[0][1]145 return None146 147def predict_combined(model_components, text):148 tfidf, svd, lr_model, rf_model, tokenizer_lstm, lstm_model, max_len, distilbert_model, tokenizer_distilbert = model_components149 p_lr = predict_lr_prob(tfidf, lr_model, text)150 p_rf = predict_rf_prob(tfidf, svd, rf_model, text)151 p_lstm = predict_lstm_prob(tokenizer_lstm, lstm_model, max_len, text)152 p_distilbert = predict_distilbert_prob(tokenizer_distilbert, distilbert_model, text)153 if p_distilbert is not None:154 avg_prob = (p_lr + p_rf + p_lstm + p_distilbert) / 4155 else:156 avg_prob = (p_lr + p_rf + p_lstm) / 3157 return p_lr, p_rf, p_lstm, p_distilbert, avg_prob158 159# -------------------------------------------------160# 5. Real-Time Authentic News Fetching and Classification161# -------------------------------------------------162fetched_news = []163 164def fetch_and_classify_news():165 global fetched_news166 fetched_news = []167 rss_feeds = [168 "https://rss.nytimes.com/services/xml/rss/nyt/HomePage.xml",169 "http://feeds.bbci.co.uk/news/rss.xml"170 ]171 for rss_url in rss_feeds:172 feed = feedparser.parse(rss_url)173 for entry in feed.entries:174 title = entry.get('title', '')175 description = entry.get('description', '')176 published_at = entry.get('published', '')177 combined_text = title + " " + description178 p_lr1, p_rf1, p_lstm1, p_distilbert1, avg_prob1 = predict_combined(components1, combined_text)179 p_lr2, p_rf2, p_lstm2, p_distilbert2, avg_prob2 = predict_combined(components2, combined_text)180 meta_prob = (avg_prob1 + avg_prob2) / 2.0181 final_class = "FAKE NEWS" if meta_prob >= 0.5 else "REAL NEWS"182 news_item = {183 "title": title,184 "description": description,185 "source": entry.get('link', 'Unknown'),186 "published_at": published_at,187 "classification": final_class,188 "probability": meta_prob189 }190 fetched_news.append(news_item)191 print(f"Fetched and classified {len(fetched_news)} authentic news items.")192 193# -------------------------------------------------194# 6. Real-Time Social News Fetching and Classification195# -------------------------------------------------196social_news = []197 198def fetch_and_classify_social_news():199 global social_news200 social_news = []201 headers = {'User-agent': 'Mozilla/5.0'}202 reddit_url = "https://www.reddit.com/r/news/.json?limit=10"203 try:204 r = requests.get(reddit_url, headers=headers, timeout=10)205 if r.status_code == 200:206 reddit_data = r.json()207 for child in reddit_data.get("data", {}).get("children", []):208 post_data = child.get("data", {})209 title = post_data.get("title", "")210 selftext = post_data.get("selftext", "")211 description = selftext if selftext else "No description available."212 created_utc = post_data.get("created_utc", None)213 published_at = "N/A"214 if created_utc:215 published_at = datetime.datetime.fromtimestamp(created_utc).strftime("%Y-%m-%d %H:%M:%S")216 combined_text = title + " " + description217 p_lr1, p_rf1, p_lstm1, p_distilbert1, avg_prob1 = predict_combined(components1, combined_text)218 p_lr2, p_rf2, p_lstm2, p_distilbert2, avg_prob2 = predict_combined(components2, combined_text)219 meta_prob = (avg_prob1 + avg_prob2) / 2.0220 final_class = "FAKE NEWS" if meta_prob >= 0.5 else "REAL NEWS"221 news_item = {222 "title": title,223 "description": description,224 "source": "Reddit: " + post_data.get("subreddit", "r/news"),225 "published_at": published_at,226 "classification": final_class,227 "probability": meta_prob228 }229 social_news.append(news_item)230 except Exception as e:231 print("Error fetching Reddit news:", e)232 # Simulated Instagram posts233 instagram_simulated = [234 {235 "title": "Instagram Post Example 1",236 "description": "This is a sample Instagram post for news classification.",237 "source": "Instagram: @news_channel",238 "published_at": "2025-03-22 10:00:00"239 },240 {241 "title": "Instagram Post Example 2",242 "description": "Another Instagram post demonstrating fake news spread.",243 "source": "Instagram: @fakefacts",244 "published_at": "2025-03-22 09:30:00"245 }246 ]247 for item in instagram_simulated:248 combined_text = item["title"] + " " + item["description"]249 p_lr1, p_rf1, p_lstm1, p_distilbert1, avg_prob1 = predict_combined(components1, combined_text)250 p_lr2, p_rf2, p_lstm2, p_distilbert2, avg_prob2 = predict_combined(components2, combined_text)251 meta_prob = (avg_prob1 + avg_prob2) / 2.0252 final_class = "FAKE NEWS" if meta_prob >= 0.5 else "REAL NEWS"253 news_item = {254 "title": item["title"],255 "description": item["description"],256 "source": item["source"],257 "published_at": item["published_at"],258 "classification": final_class,259 "probability": meta_prob260 }261 social_news.append(news_item)262 print(f"Fetched and classified {len(social_news)} social news items.")263 264# -------------------------------------------------265# 7. Scheduler Setup266# -------------------------------------------------267# Replace scheduler code with:268def scheduled_fetch():269 fetch_news()270 fetch_social()271 272# Run once at startup and then every 5 minutes273from apscheduler.schedulers.blocking import BlockingScheduler274scheduler = BlockingScheduler()275scheduler.add_job(scheduled_fetch, 'interval', minutes=5)276scheduler.start()277 278# Ensure scheduler shuts down properly279atexit.register(lambda: scheduler.shutdown())280 281# -------------------------------------------------282# 8. Routes283# -------------------------------------------------284 285@app.route('/')286def home():287 return render_template("index.html")288 289@app.route('/prediction', methods=['GET', 'POST'])290def index():291 if request.method == 'POST':292 # Use "news" as the key, matching the form field in prediction.html293 news_text = request.form.get("news", "").strip()294 if not news_text:295 return render_template("prediction.html", results=None, news_text="")296 p_lr1, p_rf1, p_lstm1, p_distilbert1, avg_prob1 = predict_combined(components1, news_text)297 p_lr2, p_rf2, p_lstm2, p_distilbert2, avg_prob2 = predict_combined(components2, news_text)298 meta_prob = (avg_prob1 + avg_prob2) / 2.0299 final_class = "FAKE NEWS" if meta_prob >= 0.5 else "REAL NEWS"300 results = {301 "Meta_Ensemble": {302 "Final_Prediction": final_class,303 "Meta_Probability": meta_prob304 },305 "Model 1": {306 "LR": p_lr1,307 "RF": p_rf1,308 "LSTM": p_lstm1,309 "DistilBERT": p_distilbert1 if p_distilbert1 is not None else None310 },311 "Model 2": {312 "LR": p_lr2,313 "RF": p_rf2,314 "LSTM": p_lstm2,315 "DistilBERT": p_distilbert2 if p_distilbert2 is not None else None316 }317 }318 return render_template("prediction.html", results=results, news_text=news_text)319 return render_template("prediction.html", results=None, news_text="")320 321@app.route('/news', methods=['GET'])322def news():323 return jsonify(fetched_news)324 325@app.route('/socialnews', methods=['GET'])326def socialnews():327 return jsonify(social_news)328 329@app.route('/social', methods=['GET'])330def social_page():331 return render_template("social.html")332 333@app.route('/contact_us')334def contactus():335 return render_template("contact_us.html")336 337@app.route('/about_us')338def aboutus():339 return render_template("about_us.html")340 341if __name__ == '__main__':342 app.run(debug=False, host='0.0.0.0', port=int(os.environ.get("PORT", 7860)))