MelzY/TruthLens-App-Docker

1import streamlit as st 2import joblib3import requests4from bs4 import BeautifulSoup5import pandas as pd6import re7import nltk8from nltk.corpus import stopwords9from nltk.stem import WordNetLemmatizer10from nltk import pos_tag 11from nltk.corpus import wordnet12import os13 14# CONFIGURATION DE LA PAGE15st.set_page_config(16    page_title="TruthLens - Détecteur de Fake News",17    page_icon="🔍",18    layout="wide",19    initial_sidebar_state="expanded"20)21 22# CONSTANTES & CONFIGURATION NLTK PATH23MODEL_PATH = 'random_forest_model_v2.pkl'24VECTORIZER_PATH = 'tfidf_vectorizer_v2.pkl'25TRUE_LABEL = 126 27# --- Configuration NLTK Data Path  ---28# Le Dockerfile a téléchargé les données dans /app/nltk_data.29# On s'assure que NLTK regarde bien là au runtime.30NLTK_DATA_DIR_INSIDE_CONTAINER = '/app/nltk_data'31if NLTK_DATA_DIR_INSIDE_CONTAINER not in nltk.data.path:32    nltk.data.path.append(NLTK_DATA_DIR_INSIDE_CONTAINER)33 34 35# CHARGEMENT DES MODÈLES 36@st.cache_resource 37def load_models():38    """Charge les modèles depuis les fichiers locaux."""39    try:40        model = joblib.load(MODEL_PATH)41        vectorizer = joblib.load(VECTORIZER_PATH)42        return model, vectorizer43    except FileNotFoundError:44        st.error(f"Erreur: Impossible de trouver '{MODEL_PATH}' ou '{VECTORIZER_PATH}'. Assurez-vous qu'ils sont dans le Space.")45        st.stop()46    except Exception as e:47        st.error(f"Erreur critique lors du chargement des modèles : {e}")48        st.stop()49 50# Charger les modèles51model, vectorizer = load_models()52 53# FONCTIONS NLTK (Simplifiées - NLTK trouve ses données)54# Téléchargement géré par le Dockerfile55stop_words = set(stopwords.words('english'))56lemmatizer = WordNetLemmatizer()57 58def get_wordnet_pos(treebank_tag):59    if treebank_tag.startswith('J'): return wordnet.ADJ60    elif treebank_tag.startswith('V'): return wordnet.VERB61    elif treebank_tag.startswith('N'): return wordnet.NOUN62    elif treebank_tag.startswith('R'): return wordnet.ADV63    else: return wordnet.NOUN64 65def clean_text_v2(text):66    """Applique le pipeline de nettoyage complet."""67    text = text.lower()68    text = re.sub(r'\breuters\b', '', text)69    text = re.sub(r'[^\w\s]', '', text)70    words = [word for word in text.split() if word not in stop_words]71    # pos_tag devrait fonctionner car NLTK trouve ses données via NLTK_DATA72    tagged_words = pos_tag(words)73    lemmatized_words = [lemmatizer.lemmatize(word, get_wordnet_pos(pos)) for word, pos in tagged_words]74    return ' '.join(lemmatized_words)75 76# TEXTES MULTILINGUES (i18n) - MODIFIÉ pour l'anglais77TEXTS = {78    'fr': {79        "page_title": "TruthLens - Détecteur de Fake News",80        "header_title": "TruthLens",81        "header_subtitle": "Détecteur Intelligent de Fake News",82        "header_powered_by": "Propulsé par IA • 98.5% de précision",83        "sidebar_about_title": "À propos",84        "sidebar_about_content": "**TruthLens** utilise l'intelligence artificielle et le traitement du langage naturel pour analyser la véracité des articles de presse.",85        "sidebar_perf_title": "Performances du Modèle",86        "sidebar_perf_accuracy": "Précision",87        "sidebar_perf_algo": "Algorithme",88        "sidebar_guide_title": "Guide d'utilisation",89        "sidebar_guide_content": """**Étapes simples :**\n1. 📝 Collez le texte ou l'URL\n2. 🔍 Cliquez sur \"Analyser\"\n3. ✅ Consultez le verdict instantané""",90        "sidebar_guide_tips_title": "**💡 Conseils :**",91        "sidebar_guide_tips_content": "- **Important : L'analyse fonctionne mieux avec du texte en anglais.**\n- Plus le texte est long, plus l'analyse est précise\n- Incluez le titre et le corps de l'article\n- Vérifiez toujours vos sources", # Ajout92        "sidebar_tech_title": "Technologies",93        "sidebar_tech_content": "- **Modèle**: Random Forest\n- **Vectorisation**: TF-IDF\n- **NLP**: NLTK (Tokenisation, Lemmatisation, POS-tagging)\n- **Framework**: Streamlit",94        "sidebar_copyright": "© 2025 TruthLens | Made with ❤️",95        "tab_text": "Analyser par Texte",96        "tab_url": "Analyser par URL",97        "text_area_label": "Collez le texte de l'article à vérifier",98        "text_area_placeholder": "Collez ici le texte complet de l'article **(en anglais de préférence)**...", # Modifié99        "url_input_label": "Entrez l'URL de l'article à vérifier",100        "url_input_placeholder": "https://www.example.com/news/article-title **(article en anglais de préférence)**", # Modifié101        "button_analyze_text": "Analyser le texte",102        "button_analyze_url": "Analyser l'URL",103        "button_clear": "Effacer",104        "analysis_in_progress": "Analyse en cours... Veuillez patienter.",105        "analysis_result_title": "Résultat de l'Analyse",106        "true_news_title": "Nouvelle probablement vraie",107        "true_news_subtitle": "Le modèle estime que cet article est authentique et fiable.",108        "fake_news_title": "Nouvelle probablement fausse",109        "fake_news_subtitle": "Cet article présente des caractéristiques de désinformation.",110        "expander_details": "Voir les détails de l'analyse",111        "expander_prob_true": "Probabilité VRAI",112        "expander_prob_fake": "Probabilité FAUX",113        "expander_keywords": "Mots-clés influents (score TF-IDF)",114        "expander_context": "Mots-clés dans leur contexte",115        "error_empty_text": "Veuillez entrer un texte à analyser avant de cliquer sur le bouton.",116        "error_empty_url": "Veuillez entrer une URL à analyser avant de cliquer sur le bouton.",117        "footer_pro_tip": "<strong>Conseil Pro</strong>: Plus le texte est long et détaillé, plus l'analyse sera précise et fiable.",118        "footer_credits": "TruthLens • Version 1.0 • Développé avec Streamlit, Scikit-learn & NLTK",119        "error_network": "Erreur de réseau ou URL invalide",120        "error_extraction": "Erreur lors de l'extraction du texte",121        "note_important": """**💡 Note importante**:122        Ce modèle est un outil d'aide à la décision basé sur l'apprentissage automatique.123        Il ne remplace pas une vérification approfondie des sources et du contexte.124        Utilisez-le comme un premier filtre, puis vérifiez toujours l'information auprès de sources fiables.""",125        "url_extracted_text_title": "Texte extrait de l'URL :",126        "warning_no_text_from_url": "⚠️ Impossible d'extraire un contenu textuel significatif de cette URL.",127        "note_limits": """**⚠️ Limites du modèle**:128        - **❗️ Entraîné principalement sur des articles en anglais (les résultats pour d'autres langues peuvent être moins fiables)**129        - Peut être biaisé par le dataset d'entraînement130        - Ne détecte pas le sarcasme ou l'ironie""", # Modifié131        "char_word_count": "📊 **{char_count}** caractères • **{word_count}** mots",132    },133    'en': {134         "page_title": "TruthLens - Fake News Detector",135        "header_title": "TruthLens",136        "header_subtitle": "Intelligent Fake News Detector",137        "header_powered_by": "Powered by AI • 98.5% accuracy",138        "sidebar_about_title": "About",139        "sidebar_about_content": "**TruthLens** uses artificial intelligence and natural language processing to analyze the veracity of news articles.",140        "sidebar_perf_title": "Model Performance",141        "sidebar_perf_accuracy": "Accuracy",142        "sidebar_perf_algo": "Algorithm",143        "sidebar_guide_title": "User Guide",144        "sidebar_guide_content": """**Simple steps:**\n1. 📝 Paste the text or URL\n2. 🔍 Click \"Analyze\"\n3. ✅ Check the instant verdict""",145        "sidebar_guide_tips_title": "**💡 Tips:**",146        "sidebar_guide_tips_content": "- **Important: Analysis works best with English text.**\n- The longer the text, the more accurate the analysis\n- Include the title and body of the article\n- Always check your sources", # Added147        "sidebar_tech_title": "Technologies",148        "sidebar_tech_content": "- **Model**: Random Forest\n- **Vectorization**: TF-IDF\n- **NLP**: NLTK (Tokenization, Lemmatization, POS-tagging)\n- **Framework**: Streamlit",149        "sidebar_copyright": "© 2025 TruthLens | Made with ❤️",150        "tab_text": "Analyze by Text",151        "tab_url": "Analyze by URL",152        "text_area_label": "Paste the article text to check",153        "text_area_placeholder": "Paste the full article text here **(preferably in English)**...", # Modified154        "url_input_label": "Enter the article URL to check",155        "url_input_placeholder": "https://www.example.com/news/article-title **(preferably an English article)**", # Modified156        "button_analyze_text": "Analyze Text",157        "button_analyze_url": "Analyze URL",158        "button_clear": "Clear",159        "analysis_in_progress": "Analysis in progress... Please wait.",160        "analysis_result_title": "Analysis Result",161        "true_news_title": "News probably true",162        "true_news_subtitle": "The model estimates this article is authentic and reliable.",163        "fake_news_title": "News probably false",164        "fake_news_subtitle": "This article shows characteristics of disinformation.",165        "expander_details": "See analysis details",166        "expander_prob_true": "Probability TRUE",167        "expander_prob_fake": "Probability FALSE",168        "expander_keywords": "Influential keywords (TF-IDF score)",169        "expander_context": "Keywords in their context",170        "error_empty_text": "Please enter text to analyze before clicking the button.",171        "error_empty_url": "Please enter a URL to analyze before clicking the button.",172        "footer_pro_tip": "<strong>Pro Tip</strong>: The longer and more detailed the text, the more accurate and reliable the analysis will be.",173        "footer_credits": "TruthLens • Version 1.0 • Developed with Streamlit, Scikit-learn & NLTK",174        "error_network": "Network error or invalid URL",175        "error_extraction": "Error during text extraction",176        "note_important": """**💡 Important Note**:177        This model is a decision-support tool based on machine learning.178        It does not replace a thorough verification of sources and context.179        Use it as a first filter, then always verify the information with reliable sources.""",180        "url_extracted_text_title": "Text extracted from URL:",181        "warning_no_text_from_url": "⚠️ Could not extract significant text content from this URL.",182        "note_limits": """**⚠️ Model Limitations**:183        - **❗️ Primarily trained on English articles (results for other languages may be less reliable)**184        - Can be biased by the training dataset185        - Does not detect sarcasm or irony""", # Modified186        "char_word_count": "📊 **{char_count}** characters • **{word_count}** words",187    }188}189 190# CSS PERSONNALISÉ 191st.markdown("""192    <style>193    .main-header { text-align: center; padding: 2rem; background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); color: white; border-radius: 15px; margin-bottom: 2rem; box-shadow: 0 4px 6px rgba(0,0,0,0.1); }194    .result-card { padding: 2rem; border-radius: 15px; margin: 1rem 0; box-shadow: 0 4px 6px rgba(0,0,0,0.1); text-align: center; }195    .true-news { background: linear-gradient(135deg, #11998e 0%, #38ef7d 100%); color: white; }196    .fake-news { background: linear-gradient(135deg, #eb3349 0%, #f45c43 100%); color: white; }197    div[data-testid="stButton"] { background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); border-radius: 10px; transition: all 0.3s; box-shadow: 0 4px 6px rgba(0,0,0,0.1); }198    div[data-testid="stButton"] > button { background-color: transparent !important; color: white !important; border: none !important; width: 100%; padding: 0.75rem; font-size: 1.1rem; font-weight: bold; }199    div[data-testid="stButton"]:hover { transform: translateY(-2px); box-shadow: 0 6px 12px rgba(0,0,0,0.15); }200    .metric-box { background-color: #f8f9fa; padding: 1rem; border-radius: 10px; border-left: 4px solid #667eea; }201    </style>202""", unsafe_allow_html=True)203 204# AUTRES FONCTIONS (get_top_tfidf_words, highlight_words, get_text_from_url) 205def get_top_tfidf_words(vectorized_input, vectorizer, top_n=10):206    feature_names = vectorizer.get_feature_names_out()207    sparse_row = vectorized_input[0]208    df = pd.DataFrame(sparse_row.T.todense(), index=feature_names, columns=["tfidf"])209    top_words_df = df[df['tfidf'] > 0].sort_values(by="tfidf", ascending=False).head(top_n)210    return top_words_df211 212def highlight_words(text, top_words_df, lemmatizer_func, pos_tag_func, get_pos_func):213    top_lemmas = set(top_words_df.index)214    tokens_and_delimiters = re.split(r'(\b\w+\b)', text)215    words_only = [part for part in tokens_and_delimiters if re.match(r'\b\w+\b', part)]216    pos_tags = pos_tag_func(words_only) # Utilise la fonction nltk.pos_tag standard217    word_to_pos = {word: tag for word, tag in pos_tags}218    highlighted_parts = []219    for part in tokens_and_delimiters:220        if re.match(r'\b\w+\b', part):221            pos = word_to_pos.get(part, 'NN')222            lemma = lemmatizer_func.lemmatize(part.lower(), get_pos_func(pos))223            if lemma in top_lemmas:224                highlighted_parts.append(f"<span style='background-color: rgba(102, 126, 234, 0.3); padding: 2px 4px; border-radius: 4px; font-weight: 500;'>{part}</span>")225            else:226                highlighted_parts.append(part)227        else:228            highlighted_parts.append(part)229    return f'<div style="line-height: 2; text-align: justify;">{"".join(highlighted_parts)}</div>'230 231@st.cache_data(ttl="1h")232def get_text_from_url(url):233    try:234        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'}235        response = requests.get(url, headers=headers, timeout=10)236        response.raise_for_status()237        soup = BeautifulSoup(response.text, 'html.parser')238        title = soup.find('h1').get_text() if soup.find('h1') else ''239        paragraphs = soup.find_all('p')240        article_text = ' '.join([p.get_text() for p in paragraphs])241        return f"{title} {article_text}"242    except requests.RequestException as e: raise ValueError(f"{T['error_network']}: {e}")243    except Exception as e: raise ValueError(f"{T['error_extraction']}: {e}")244 245# INTERFACE UTILISATEUR (Sidebar, Header, Tabs, Footer)  246 247# SÉLECTEUR DE LANGUE ET INITIALISATION248st.sidebar.markdown("## Language / Langue")249selected_language = st.sidebar.selectbox(250    "Choose your language",251    options=['fr', 'en'],252    format_func=lambda x: "Français" if x == 'fr' else "English",253    label_visibility="collapsed"254)255T = TEXTS[selected_language]256 257def display_text_metrics(text):258    word_count = len(text.split())259    char_count = len(text)260    st.caption(T['char_word_count'].format(char_count=char_count, word_count=word_count))261 262# SIDEBAR263with st.sidebar:264    st.markdown(f"## :information_source: {T['sidebar_about_title']}")265    st.info(T['sidebar_about_content'])266    st.markdown(f"## :bar_chart: {T['sidebar_perf_title']}")267    col1, col2 = st.columns(2)268    with col1: st.metric(T['sidebar_perf_accuracy'], "98.5%", delta="Excellent")269    with col2: st.metric(T['sidebar_perf_algo'], "RF")270    st.markdown("---")271    st.markdown(f"## :page_facing_up: {T['sidebar_guide_title']}")272    st.markdown(T['sidebar_guide_content'])273    st.markdown(T['sidebar_guide_tips_title'])274    st.markdown(T['sidebar_guide_tips_content'])275    st.markdown("---")276    st.markdown(f"## :gear: {T['sidebar_tech_title']}")277    st.markdown(T['sidebar_tech_content'])278    st.markdown("---")279    st.caption(T['sidebar_copyright'])280 281# FONCTIONS D'AFFICHAGE ET D'ANALYSE282def get_prediction(text_to_analyze):283    with st.spinner(f'🔄 {T["analysis_in_progress"]}'):284        cleaned_input = clean_text_v2(text_to_analyze)285        vectorized_input = vectorizer.transform([cleaned_input])286        prediction = model.predict(vectorized_input)287        prediction_proba = model.predict_proba(vectorized_input)288    return prediction, prediction_proba, vectorized_input289 290def display_results(text_to_analyze, prediction, prediction_proba, vectorized_input):291    st.markdown("---")292    st.markdown(f"## 🎯 {T['analysis_result_title']}")293    if prediction[0] == TRUE_LABEL:294        st.markdown(f"""295            <div class="result-card true-news">296                <h2 style="margin: 0; font-size: 2rem;">✅ {T['true_news_title']}</h2>297                <p style="font-size: 3rem; margin: 1.5rem 0; font-weight: bold;">{prediction_proba[0][1]*100:.1f}%</p>298                <p style="font-size: 1.1rem; margin: 0;">{T['true_news_subtitle']}</p>299            </div>""", unsafe_allow_html=True)300        st.balloons()301    else:302        st.markdown(f"""303            <div class="result-card fake-news">304                <h2 style="margin: 0; font-size: 2rem;">❌ {T['fake_news_title']}</h2>305                <p style="font-size: 3rem; margin: 1.5rem 0; font-weight: bold;">{prediction_proba[0][0]*100:.1f}%</p>306                <p style="font-size: 1.1rem; margin: 0;">⚠️ {T['fake_news_subtitle']}</p>307            </div>""", unsafe_allow_html=True)308    confidence = max(prediction_proba[0])309    st.progress(float(confidence))310    with st.expander(f"📈 {T['expander_details']}"):311        col1, col2 = st.columns(2)312        with col1: st.markdown(f"""<div class="metric-box"><h4>{T['expander_prob_true']}</h4><h2 style="color: #11998e;">{prediction_proba[0][1]*100:.2f}%</h2></div>""", unsafe_allow_html=True)313        with col2: st.markdown(f"""<div class="metric-box"><h4>{T['expander_prob_fake']}</h4><h2 style="color: #eb3349;">{prediction_proba[0][0]*100:.2f}%</h2></div>""", unsafe_allow_html=True)314        st.markdown("---")315        st.markdown(f"<h5>📝 {T['expander_keywords']}</h5>", unsafe_allow_html=True)316        top_words_df = get_top_tfidf_words(vectorized_input, vectorizer)317        df_for_chart = top_words_df.reset_index().rename(columns={'index': 'Mot', 'tfidf': 'Score'})318        st.bar_chart(df_for_chart, x='Mot', y='Score', color='#667eea')319        st.markdown("---")320        st.markdown(f"<h5>📍 {T['expander_context']}</h5>", unsafe_allow_html=True)321        # On passe nltk.pos_tag directement, car NLTK doit trouver ses données322        highlighted_text = highlight_words(text_to_analyze, top_words_df, lemmatizer, nltk.pos_tag, get_wordnet_pos)323        st.markdown(highlighted_text, unsafe_allow_html=True)324        st.markdown("---")325        st.info(T['note_important'])326        st.warning(T['note_limits']) # Affiche maintenant la note sur l'anglais327 328# HEADER329st.markdown(f"""330    <div class="main-header">331        <h1>🔍 {T['header_title']}</h1>332        <p style="font-size: 1.2rem; margin: 0;">{T['header_subtitle']}</p>333        <p style="font-size: 0.9rem; opacity: 0.9; margin-top: 0.5rem;">{T['header_powered_by']}</p>334    </div>""", unsafe_allow_html=True)335 336# CORPS DE L'APPLICATION (ONGLETS)337tab1, tab2 = st.tabs([f"📝 {T['tab_text']}", f"🌐 {T['tab_url']}"])338 339with tab1:340    st.markdown(f"### {T['text_area_label']}")341    def clear_text(): st.session_state.user_input = ""342    if 'user_input' not in st.session_state: st.session_state.user_input = ""343    user_input = st.text_area("", placeholder=T['text_area_placeholder'], height=250, label_visibility="collapsed", key="user_input") # Placeholder modifié344    if user_input: display_text_metrics(user_input)345    col1, col2 = st.columns(2)346    analyze_text_button = col1.button(f"🔍 {T['button_analyze_text']}", use_container_width=True, help="Lancer l'analyse du texte")347    col2.button(f"✨ {T['button_clear']}", on_click=clear_text, use_container_width=True, help="Effacer la zone de saisie")348    if analyze_text_button:349        if st.session_state.user_input.strip():350            text = st.session_state.user_input351            prediction, prediction_proba, vectorized_input = get_prediction(text)352            display_results(text, prediction, prediction_proba, vectorized_input)353        else: st.error(f"⚠️ **Erreur**: {T['error_empty_text']}")354 355with tab2:356    st.markdown(f"### {T['url_input_label']}")357    url_input = st.text_input("", placeholder=T['url_input_placeholder'], label_visibility="collapsed") # Placeholder modifié358    analyze_url_button = st.button(f"🔍 {T['button_analyze_url']}", use_container_width=True, help="Lancer l'analyse de l'URL")359    if analyze_url_button:360        if url_input.strip():361            try:362                scraped_text = get_text_from_url(url_input)363                if scraped_text and scraped_text.strip():364                    st.markdown("---")365                    st.markdown(f"#### {T['url_extracted_text_title']}")366                    display_text_metrics(scraped_text)367                    st.text_area("", value=scraped_text, height=200, disabled=True, label_visibility="collapsed")368                    prediction, prediction_proba, vectorized_input = get_prediction(scraped_text)369                    display_results(scraped_text, prediction, prediction_proba, vectorized_input)370                else: st.warning(T['warning_no_text_from_url'])371            except ValueError as e: st.error(f"⚠️ **Erreur**: {e}")372        else: st.error(f"⚠️ **Erreur**: {T['error_empty_url']}")373 374# FOOTER375st.markdown("---")376st.markdown(f"""377    <div style="text-align: center; padding: 2rem; background-color: #f8f9fa; border-radius: 10px;">378        <p style="color: #666; margin: 0; font-size: 1rem;">💡 {T['footer_pro_tip']}</p>379        <p style="color: #999; margin-top: 1rem; font-size: 0.85rem;">{T['footer_credits']}</p>380    </div>""", unsafe_allow_html=True)