DevPatel0611/TruthLens

1import os2import sys3import json4import time5import logging6import pickle7import numpy as np8import pandas as pd9import yaml10from sklearn.model_selection import StratifiedShuffleSplit11 12# Fix paths for imports13_PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))14if str(_PROJECT_ROOT) not in sys.path:15    sys.path.insert(0, str(_PROJECT_ROOT))16 17from src.utils.text_utils import clean_text, build_full_text, word_count, text_length_bucket18from src.utils.domain_weights import compute_domain_weights19from src.utils.freshness import apply_freshness_score20 21logging.basicConfig(22    level=logging.INFO,23    format="%(asctime)s | %(levelname)-8s | %(name)s | %(message)s",24    datefmt="%H:%M:%S"25)26logger = logging.getLogger("stage2_preprocessing")27 28class KerasStyleTokenizer:29    """A lightweight, PyTorch-compatible word tokenizer mimicking Keras's Tokenizer."""30    def __init__(self, num_words=None, oov_token="<OOV>"):31        self.num_words = num_words32        self.oov_token = oov_token33        self.word_index = {self.oov_token: 1} # 0 is reserved for padding34        self.word_counts = {}35        36    def fit_on_texts(self, texts):37        for text in texts:38            # clean_text already removes punctuation, we just split by space39            words = str(text).split()40            for w in words:41                self.word_counts[w] = self.word_counts.get(w, 0) + 142                43        # Sort by frequency44        sorted_words = sorted(self.word_counts.items(), key=lambda x: x[1], reverse=True)45        46        for idx, (w, _) in enumerate(sorted_words):47            if self.num_words and idx >= self.num_words - 2:48                break49            self.word_index[w] = idx + 250            51    def texts_to_sequences(self, texts):52        seqs = []53        for text in texts:54            words = str(text).split()55            seq = [self.word_index.get(w, 1) for w in words]56            seqs.append(seq)57        return seqs58 59def truncate_str_array(df, col):60    """Memory fix: force string type for arrays."""61    return df[col].astype(str).values62 63def run_preprocessing(cfg: dict = None):64    t0 = time.perf_counter()65    if cfg is None:66        cfg_path = os.path.join(_PROJECT_ROOT, "config", "config.yaml")67        with open(cfg_path, "r", encoding="utf-8") as f:68            cfg = yaml.safe_load(f)69 70    logger.info("STAGE 2: PREPROCESSING START")71    processed_dir = os.path.join(_PROJECT_ROOT, cfg["paths"]["processed_dir"])72    splits_dir = os.path.join(_PROJECT_ROOT, cfg["paths"]["splits_dir"])73    models_dir = os.path.join(_PROJECT_ROOT, cfg["paths"]["models_dir"])74    os.makedirs(splits_dir, exist_ok=True)75    os.makedirs(models_dir, exist_ok=True)76    77    # 1. Load Data78    csv_path = os.path.join(processed_dir, "unified.csv")79    df = pd.read_csv(csv_path)80    df["published_date"] = pd.to_datetime(df["published_date"], errors="coerce")81    logger.info("Loaded unified CSV: %d rows", len(df))82    83    # 2. Extract Text Length Features & Clean84    # "Concatenate title + '. ' + text as full_text" -> use build_full_text85    df["full_text"] = df.apply(lambda r: build_full_text(86        str(r["title"]) if pd.notna(r["title"]) else "", 87        str(r["text"]) if pd.notna(r["text"]) else ""88    ), axis=1)89    # the prompt specifies cleaning the text by lowercasing, removing HTML/URLs/special bounds.90    # clean_text handles exactly this cleanly.91    logger.info("Applying text cleaning (HTML, URLs, whitespace, punctuation) ...")92    df["clean_text"] = df["full_text"].apply(clean_text)93    94    logger.info("Calculating word counts and text buckets ...")95    df["word_count"] = df["clean_text"].apply(word_count)96    df["text_length_bucket"] = df["word_count"].apply(text_length_bucket)97    98    # 3. Domain Weights99    ds_cfg = cfg.get("dataset", {})100    min_domains = ds_cfg.get("min_domain_samples", 20)101    max_multi = cfg.get("inference", {}).get("max_multiplier", 10)102    103    logger.info("Computing domain-aware sample weights...")104    df = compute_domain_weights(df, min_domain_samples=min_domains, max_multiplier=max_multi)105    106    # 4. Freshness107    logger.info("Applying temporal freshness scores...")108    df = apply_freshness_score(df, is_inference=False)109    110    # 5. Train/Val/Test Splits111    # The user clarified exactly:112    # stratified_holdout -> stage 3 proxy113    # testing -> sacred114    # train pool -> split 85/15 into train and val.115    116    test_mask = df["dataset_origin"] == "testing"117    holdout_mask = df["dataset_origin"] == "stratified_holdout"118    train_pool_mask = ~(test_mask | holdout_mask)119    120    test_df = df[test_mask].copy()121    holdout_df = df[holdout_mask].copy()122    train_pool_df = df[train_pool_mask].copy()123    124    # Split train_pool into 85% train, 15% validation using StratifiedShuffleSplit125    sss = StratifiedShuffleSplit(n_splits=1, test_size=0.15, random_state=42)126    train_pool_df = train_pool_df.reset_index(drop=True)127    128    train_idx, val_idx = next(sss.split(train_pool_df, train_pool_df["binary_label"]))129    train_df = train_pool_df.iloc[train_idx].copy()130    val_df = train_pool_df.iloc[val_idx].copy()131    132    logger.info("SPLITS SUMMARY:")133    logger.info("  Train:     %d rows", len(train_df))134    logger.info("  Val:       %d rows", len(val_df))135    logger.info("  Holdout:   %d rows", len(holdout_df))136    logger.info("  Sacred:    %d rows", len(test_df))137    138    # 6. Save splits metadata and arrays139    # Saving raw text separately just for PyTorch dataset convenience (faster than pd.read_csv for big models)140    splits_dict = {141        "train": train_df,142        "val": val_df, 143        "holdout": holdout_df,144        "test": test_df145    }146    147    for split_name, split_data in splits_dict.items():148        np.save(os.path.join(splits_dir, f"X_text_{split_name}.npy"), truncate_str_array(split_data, "clean_text"))149        np.save(os.path.join(splits_dir, f"y_{split_name}.npy"), split_data["binary_label"].values)150        np.save(os.path.join(splits_dir, f"w_{split_name}.npy"), split_data["sample_weight"].values)151        152        meta = {153            "size": len(split_data),154            "fake_count": int((split_data["binary_label"] == 0).sum()),155            "true_count": int((split_data["binary_label"] == 1).sum()),156            "word_count_median": float(split_data["word_count"].median()),157            "freshness_mean": float(split_data["freshness_score"].mean())158        }159        with open(os.path.join(splits_dir, f"meta_{split_name}.json"), "w") as f:160            json.dump(meta, f, indent=2)161 162    # Save train_ids.csv explicitly163    train_df[["article_id"]].to_csv(os.path.join(splits_dir, "train_ids.csv"), index=False)164    # Also save the full preprocessed test sets to CSV for easy loading during Stage 3 / Evaluation165    train_df.to_csv(os.path.join(splits_dir, "df_train.csv"), index=False)166    val_df.to_csv(os.path.join(splits_dir, "df_val.csv"), index=False)167    holdout_df.to_csv(os.path.join(splits_dir, "df_holdout.csv"), index=False)168    test_df.to_csv(os.path.join(splits_dir, "df_test.csv"), index=False)169 170    # 7. Tokenization (LSTM)171    logger.info("Fitting LSTM Tokenizer on Train split...")172    # Max features for LSTM or generic defaults usually just load all words. We will let it cap at e.g., 50k173    vocab_size = cfg.get("preprocessing", {}).get("max_tfidf_features", 50000)174    tok = KerasStyleTokenizer(num_words=vocab_size)175    tok.fit_on_texts(train_df["clean_text"])176    177    tok_path = os.path.join(models_dir, "tokenizer.pkl")178    with open(tok_path, "wb") as f:179        pickle.dump(tok, f)180    logger.info(f"Saved tokenizer to {tok_path} (vocab size: {len(tok.word_index)})")181 182    t_end = time.perf_counter()183    logger.info("STAGE 2 FINISHED in %.2f seconds", t_end - t0)184 185if __name__ == "__main__":186    run_preprocessing()187