biplobgon/product-recommendation-system

1"""2features/item_features.py3--------------------------4Build item-level features from item_properties and category_tree DataFrames.5 6EDA context7-----------8- ~20M property records combined across two time-split files.9- ~230k unique items in the catalog.10- ~185k items have both events and metadata (well-covered).11- ~230k items have metadata but no events (cold items).12- ~50k items have events but no metadata.13- Category tree: 1,600+ nodes, max depth 5, dominant depth 2–3.14"""15from __future__ import annotations16 17import pandas as pd18import numpy as np19from sklearn.preprocessing import LabelEncoder20 21from utils.logger import get_logger22 23logger = get_logger(__name__)24 25 26def build_item_features(27    item_props: pd.DataFrame,28    category_tree: pd.DataFrame,29) -> pd.DataFrame:30    """Produce a per-item feature DataFrame.31 32    Parameters33    ----------34    item_props:35        Combined item_properties DataFrame (parts 1 & 2) with columns36        [timestamp, itemid, property, value].37    category_tree:38        Category tree DataFrame with columns [categoryid, parentid].39 40    Returns41    -------42    pd.DataFrame43        One row per item with columns:44        - itemid45        - categoryid           (most recent category assignment)46        - category_depth       (depth of category in tree)47        - n_properties         (number of distinct property keys)48        - n_property_updates   (total property records)49        - price                (latest numeric price, if available)50        - available            (latest availability flag, if available)51        - first_seen, last_seen52    """53    logger.info("Building item features from %d property records …", len(item_props))54 55    df = item_props.copy()56    df["datetime"] = pd.to_datetime(pd.to_numeric(df["timestamp"], errors="coerce"), unit="ms")57 58    # --- Core aggregates -----------------------------------------------------59    agg = df.groupby("itemid").agg(60        n_property_updates=("property", "count"),61        n_properties=("property", "nunique"),62        first_seen=("datetime", "min"),63        last_seen=("datetime", "max"),64    ).reset_index()65 66    # Single-pass extraction of the 3 properties we need (avoids full pivot OOM)67    TARGET_PROPS = {"categoryid", "790", "available"}68    sub = df[df["property"].isin(TARGET_PROPS)].sort_values("datetime")69    latest_sub = (70        sub.groupby(["itemid", "property"])["value"]71        .last()72        .unstack(level="property")   # only 3 columns wide — safe73        .reset_index()74    )75 76    # Attach categoryid77    if "categoryid" in latest_sub.columns:78        latest_sub["categoryid"] = pd.to_numeric(latest_sub["categoryid"], errors="coerce")79        agg = agg.merge(latest_sub[["itemid", "categoryid"]], on="itemid", how="left")80    else:81        agg["categoryid"] = np.nan82 83    # Attach price (property "790")84    if "790" in latest_sub.columns:85        latest_sub["price"] = pd.to_numeric(86            latest_sub["790"].str.extract(r"([\d.]+)", expand=False),87            errors="coerce",88        )89        agg = agg.merge(latest_sub[["itemid", "price"]], on="itemid", how="left")90    else:91        agg["price"] = np.nan92 93    # Attach availability94    if "available" in latest_sub.columns:95        latest_sub["available"] = latest_sub["available"].map(96            {"1": True, "0": False, 1: True, 0: False}97        )98        agg = agg.merge(latest_sub[["itemid", "available"]], on="itemid", how="left")99    else:100        agg["available"] = np.nan101 102    # --- Category depth ------------------------------------------------------103    depth_map = _compute_category_depths(category_tree)104    agg["category_depth"] = agg["categoryid"].map(depth_map)105 106    logger.info("Item features built for %d unique items.", len(agg))107    return agg108 109 110def build_item_tfidf_matrix(111    item_props: pd.DataFrame,112    max_features: int = 5000,113) -> tuple[pd.DataFrame, object]:114    """Create a TF-IDF representation of item property values for content-based115    similarity computation.116 117    Parameters118    ----------119    item_props:120        Combined item properties DataFrame.121    max_features:122        Vocabulary size cap for TfidfVectorizer.123 124    Returns125    -------126    (item_ids_series, tfidf_matrix)127        item_ids_series : pd.Series of itemid values aligned with matrix rows.128        tfidf_matrix    : scipy sparse matrix of shape (n_items, max_features).129    """130    from sklearn.feature_extraction.text import TfidfVectorizer131 132    logger.info("Building TF-IDF item matrix …")133    text_per_item = (134        item_props.groupby("itemid")["value"]135        .apply(lambda v: " ".join(v.dropna().astype(str)))136        .reset_index()137    )138    vectorizer = TfidfVectorizer(max_features=max_features, sublinear_tf=True)139    matrix = vectorizer.fit_transform(text_per_item["value"])140    logger.info("TF-IDF matrix shape: %s", matrix.shape)141    return text_per_item["itemid"], matrix, vectorizer142 143 144# ---------------------------------------------------------------------------145# Internal helpers146# ---------------------------------------------------------------------------147 148def _compute_category_depths(category_tree: pd.DataFrame) -> dict:149    parent_map = dict(150        zip(category_tree["categoryid"], category_tree["parentid"])151    )152    depths: dict[int, int] = {}153    for cat in category_tree["categoryid"]:154        depth, node = 0, cat155        visited: set = set()156        while node in parent_map and not pd.isna(parent_map.get(node)) and node not in visited:157            visited.add(node)158            node = parent_map[node]159            depth += 1160        depths[cat] = depth161    return depths162