LPXian/Graph-News.AI

1from typing import Dict, List2from app.graph.state import GraphState3 4def cleaner_node(state: GraphState) -> GraphState:5    """6    Deduplicates articles based on title and link.7    """8    print("--- CLEANER NODE ---")9    raw_articles = state.get("raw_articles", [])10    11    seen_titles = set()12    cleaned = []13    14    for article in raw_articles:15        title = article.get("title", "").strip().lower()16        if title and title not in seen_titles:17            seen_titles.add(title)18            cleaned.append(article)19            20    print(f"Cleaned {len(raw_articles)} articles down to {len(cleaned)} unique items.")21    return {"clean_articles": cleaned}22