CMacD/AIC_PHASE1_POC
0
1#!/usr/bin/env python2# coding: utf-83"""4ML classifier — TF-IDF text classifier (Phase 1, Step 2b).5 6For each TEXT-type attribute this builds a TF-IDF feature matrix from the7historical key-column text (word bigrams) and trains a classifier to predict the8most likely label for each new product. Where BM25 ranks by keyword relevance,9the classifier learns patterns; Ensemble fuses the two.10 11Classifier backend12------------------13As of 2026-06-06 the backend is **LinearSVC**. A head-to-head on this project's14sparse TF-IDF features showed it matching or beating the previous XGBoost backend15on accuracy at ~2.5x less compute (linear models are the canonical strong choice16for high-dimensional sparse text; gradient-boosted trees are in their weakest17regime there). The retired XGBoost configuration is preserved, commented, below18(`_XGB_PARAMS`) for easy revival. Output is keyed/tagged 'ML' and consumed19unchanged by Ensemble.20 21What it does22------------23 1. For each TEXT-type attribute, builds a TF-IDF feature matrix from the24 historical key column text. Word bigrams are used (not just single words)25 because compound terms like "dark chocolate" or "reduced fat" carry more26 signal than either word alone.27 28 2. Optionally augments the training data by creating masked copies of each29 row where a random subset of key columns is replaced with "missing". This30 teaches the model to make predictions even when some columns are blank,31 which is common for genuinely new products in the flat file.32 33 3. Trains the classifier on the augmented features and predicts the top label34 plus confidence score for each new product.35 36 4. Results are grouped by unique key combination so the output is one row per37 distinct product type rather than one row per individual product.38 39Results are stored in the returned dict as 'ML_{attrG}' and combined with40the BM25 predictions in Ensemble.41 42Code style43----------44Functions are written to be read straight through. Steps are broken into named45variables rather than chained. If something is not immediately obvious from the46code it has a comment. Keep it that way.47"""48 49import re50import warnings51from concurrent.futures import ThreadPoolExecutor52 53import numpy as np54import pandas as pd55import nltk56from nltk.corpus import stopwords57from sklearn.feature_extraction.text import TfidfVectorizer58from sklearn import preprocessing59from sklearn.svm import LinearSVC60from sklearn.calibration import CalibratedClassifierCV61# import xgboost as xgb # retired in favour of LinearSVC — see _make_classifier()62# and the preserved _XGB_PARAMS block below.63 64from ml_package import routing65 66warnings.filterwarnings('ignore')67nltk.download('punkt', quiet=True)68nltk.download('punkt_tab', quiet=True)69nltk.download('stopwords', quiet=True)70 71 72# ── Auto-NUMERIC detection (mirrors Ensemble._infer_attr_type) ────────────────73_NUMERIC_RE = re.compile(r'^\s*[\d#][\d\s\.\-\/]*', re.IGNORECASE)74_NUMERIC_THRESHOLD = 0.6075_NULL_STRS = {'', 'nan', 'none', 'missing', 'null', 'na', 'null value'}76 77# ── Stopwords ─────────────────────────────────────────────────────────────────78# 'no' and 'to' are kept (removed from NLTK stopwords) because they carry79# product-attribute signal in this domain.80# NOTE: 'chocolate', 'original', etc. intentionally excluded from removals —81# these ARE discriminating features for product attribute classification.82_DOMAIN_STOPWORDS = [83 'ss', 'unknown', 'undefined', 'company', 'category', 'llc', 'inc', 'ltd',84 'to', 'oz', 'lt', 'ct', '', 'value', 'not', 'available', 'key', 'label',85 'may', 'great', 'from', 'of', 'for', 'null', 'nav', 'card', 'mix', 'nut',86 '&', 'kit', 'sauce', 'dish', 'cup', 'bx', 'envlp', 'can', 'bag', 'btl',87 'rfg', 'cnstr', 'unf', '*', '+', '/', 'abc', 'a', '-',88]89_NOT_STOPWORDS = {'no', 'to'}90_STOPWORDS = frozenset(91 (set(stopwords.words('english')) | set(_DOMAIN_STOPWORDS)) - _NOT_STOPWORDS92)93 94# ── Training-data augmentation ────────────────────────────────────────────────95# For each training row, N masked copies are created where a random subset of96# key columns is set to 'missing' while one randomly-chosen anchor column is97# always kept. Teaches the classifier to predict from partial key-column98# combinations — useful when new flat-file products arrive with some columns blank.99_AUG_N_COPIES = 3 # total training rows become (1 + N) × original100_aug_rng = np.random.default_rng(42)101 102# ── Model configuration ───────────────────────────────────────────────────────103# Word n-grams (1-2): bigrams capture compound product terms like104# "dark chocolate" vs "milk chocolate".105# min_df=1 keeps rare tokens — product catalogs are small, so rare terms106# are still signal. sublinear_tf dampens frequency dominance of common tokens.107_tfidf = TfidfVectorizer(ngram_range=(1, 2), min_df=1, sublinear_tf=True, analyzer='word')108 109# Number of candidate labels the classifier returns per product — mirrors _BM25_TOP_K.110_ML_TOP_K = 3111 112# Per-attribute routing — whether the classifier runs at all — lives in113# ml_package.routing and is applied in _process_one_xgb_attr below. It skips114# identity/derived composites (openness ~ 1, e.g. Franchise_Packtype_RPTG ~2,795)115# while keeping genuine classifications (e.g. Tool_Franchise_TH ~263), and116# replaces the earlier blunt class-count cap. See routing.skip_learned_methods().117 118 119def _make_classifier():120 """121 Build a fresh classifier for the learnable TEXT tier (one per attribute so122 parallel calls don't share state).123 124 Backend: LinearSVC — a 2026-06-06 head-to-head on this project's sparse125 TF-IDF features showed it matching/beating XGBoost on accuracy at ~2.5x less126 compute. LogReg (one-vs-rest) is a faster, slightly-less-accurate alternative.127 """128 return LinearSVC(max_iter=5000, random_state=42)129 # Speed-first alternative (add: from sklearn.linear_model import LogisticRegression130 # and from sklearn.multiclass import OneVsRestClassifier):131 # return OneVsRestClassifier(LogisticRegression(solver="liblinear", max_iter=1000))132 133 134def _ml_scores(model, features):135 """136 Return an [n_samples, n_classes] confidence matrix aligned to model.classes_.137 138 Uses predict_proba when the estimator provides it (LogReg / the old XGB);139 otherwise softmaxes LinearSVC's decision_function margins into 0..1. Either140 way Ensemble's contract holds — it reads the per-row `score` as a 0..1141 confidence (score / 100) regardless of which classifier produced it.142 """143 if hasattr(model, "predict_proba"):144 return model.predict_proba(features)145 margins = model.decision_function(features)146 if margins.ndim == 1: # binary problem → two columns147 margins = np.column_stack([-margins, margins])148 margins = margins - margins.max(axis=1, keepdims=True)149 exp = np.exp(margins)150 return exp / exp.sum(axis=1, keepdims=True)151 152 153# ── Preserved for reference: the retired XGBoost backend ──────────────────────154# Replaced by LinearSVC (see _make_classifier) on 2026-06-06. To revive:155# uncomment `import xgboost as xgb` above, uncomment this block, and return156# `xgb.XGBClassifier(**_XGB_PARAMS)` from _make_classifier().157# n_estimators=150 + learning_rate=0.1: more trees, smaller steps.158# subsample=0.8 / colsample_bytree=0.4: row / column subsampling for sparse TF-IDF.159# tree_method='hist': faster histogram splits. n_jobs=2.160# _XGB_PARAMS = dict(161# n_estimators=150,162# learning_rate=0.1,163# max_depth=6,164# subsample=0.8,165# colsample_bytree=0.4,166# min_child_weight=1,167# tree_method='hist',168# n_jobs=2,169# random_state=42,170# verbosity=0,171# )172 173 174def _calibration_split(y, frac_cal: float = 0.25, seed: int = 42):175 """176 Held-out split for Platt calibration that never drops a class.177 178 Every class keeps >=1 sample in the fit set (so the base estimator learns all179 of them — important because the minority classes here are genuinely rare);180 classes with >=2 samples also contribute ~frac_cal of their rows to the181 calibration set. Returns (fit_idx, cal_idx) as int arrays.182 """183 rng = np.random.default_rng(seed)184 y = np.asarray(y)185 fit_idx, cal_idx = [], []186 for c in np.unique(y):187 idx = np.where(y == c)[0]188 rng.shuffle(idx)189 if len(idx) < 2:190 fit_idx.extend(idx.tolist()) # lone sample stays in fit191 continue192 n_cal = min(len(idx) - 1, max(1, int(round(len(idx) * frac_cal))))193 cal_idx.extend(idx[:n_cal].tolist())194 fit_idx.extend(idx[n_cal:].tolist())195 return np.array(fit_idx, dtype=int), np.array(cal_idx, dtype=int)196 197 198def _fit_calibrated(train_features, y_encoded):199 """200 Fit LinearSVC and Platt-calibrate its confidence onto a proper probability201 scale. A bare softmax over hundreds of classes yields a ~0.07 top-1 even202 when the model is right, which drowns ML out of the BM25+ML score fusion and203 floods QC with false MEDIUM flags. Sigmoid (Platt) calibration on a held-out204 split maps the SVM margins to calibrated probabilities; predict_proba then205 feeds Ensemble unchanged via _ml_scores.206 207 Robust to genuinely-rare classes: _calibration_split keeps every class in the208 fit set, and we fall back to the uncalibrated LinearSVC if calibration can't209 be fit (too few rows, or only one class in the calibration split).210 """211 fit_i, cal_i = _calibration_split(y_encoded)212 if cal_i.size and len(np.unique(y_encoded[cal_i])) >= 2:213 try:214 base = _make_classifier()215 base.fit(train_features[fit_i], y_encoded[fit_i])216 # Calibrate the prefit base. sklearn >= 1.6 removed cv='prefit' in217 # favour of FrozenEstimator; older sklearn still uses cv='prefit'.218 # Try the modern path first, fall back for < 1.6.219 try:220 from sklearn.frozen import FrozenEstimator # sklearn >= 1.6221 calibrated = CalibratedClassifierCV(FrozenEstimator(base), method='sigmoid')222 except ImportError:223 calibrated = CalibratedClassifierCV(base, method='sigmoid', cv='prefit')224 calibrated.fit(train_features[cal_i], y_encoded[cal_i])225 return calibrated226 except Exception: # noqa: BLE001 — degrade gracefully227 # Calibration is a best-effort confidence refinement. A genuine228 # failure is worth a brief note (helps diagnose) but not the full229 # traceback — fall through to the uncalibrated classifier.230 print(" ML confidence calibration unavailable — using raw scores")231 clf = _make_classifier()232 clf.fit(train_features, y_encoded)233 return clf234 235 236def _is_numeric_attr(attr_key_cols: list, data_df: pd.DataFrame) -> bool:237 """Return True if >60% of unique non-null key-column values are numeric/range."""238 for col in attr_key_cols:239 if col not in data_df.columns:240 continue241 vals = (242 data_df[col].dropna().astype(str).str.strip()243 .pipe(lambda s: s[~s.str.lower().isin(_NULL_STRS)])244 .unique()245 )246 if len(vals) == 0:247 continue248 numeric_match_count = sum(bool(_NUMERIC_RE.match(v)) for v in vals)249 numeric_fraction = numeric_match_count / len(vals)250 if numeric_fraction >= _NUMERIC_THRESHOLD:251 return True252 return False253 254 255def _augment_training(df: pd.DataFrame, attr_key_cols: list) -> pd.DataFrame:256 """257 Return df plus _AUG_N_COPIES masked copies for training-data augmentation.258 259 Each copy masks a random subset of key columns to 'missing' (p=0.5 per260 column per row), while one randomly-chosen anchor column per row is always261 preserved. No-op when attr_key_cols has only one column.262 """263 available_cols = [c for c in attr_key_cols if c in df.columns]264 if len(available_cols) < 2:265 return df266 267 copies = [df]268 for _ in range(_AUG_N_COPIES):269 aug = df.copy()270 anchor_col_index = _aug_rng.integers(0, len(available_cols), size=len(aug))271 for col_index, col in enumerate(available_cols):272 mask = (_aug_rng.random(len(aug)) < 0.5) & (anchor_col_index != col_index)273 aug.loc[mask, col] = 'missing'274 copies.append(aug)275 return pd.concat(copies, ignore_index=True)276 277 278def _remove_duplicate_words(text: str) -> str:279 """Remove duplicate words from a string while preserving order."""280 words = text.split()281 return ' '.join(dict.fromkeys(words))282 283 284def _build_features(train_df: pd.DataFrame, test_df: pd.DataFrame):285 """286 Concatenate all key columns into a single feature string per row,287 apply stopword removal and deduplication, and fit-transform TF-IDF288 on training data / transform test data.289 290 Returns (train_df_with_ftr, test_df_with_ftr).291 """292 train_df['ftr'] = train_df.iloc[:, :-1].astype(str).apply(' '.join, axis=1)293 test_df['ftr'] = test_df.astype(str).apply(' '.join, axis=1)294 295 def _clean(text_series):296 return (297 text_series.str.lower().astype(str)298 .apply(_remove_duplicate_words)299 .apply(lambda x: ' '.join(w for w in x.split() if w not in _STOPWORDS))300 )301 302 train_df['ftr'] = _clean(train_df['ftr'])303 test_df['ftr'] = _clean(test_df['ftr'])304 return train_df, test_df305 306 307def _process_one_xgb_attr(mdm_col: str, meta_df: pd.DataFrame,308 history_df: pd.DataFrame,309 flat_file_df: pd.DataFrame):310 """311 Train and predict for a single attribute. Returns (key, DataFrame) or None.312 Creates fresh TF-IDF and classifier instances so parallel calls don't share state.313 """314 attr_key_cols = list(meta_df.loc[meta_df['Attribute Group name'] == mdm_col,315 'Attribute Name in MDM'])316 317 meta_type_vals = (318 meta_df.loc[meta_df['Attribute Group name'] == mdm_col, 'Type']319 .dropna().astype(str).str.strip().str.upper()320 .pipe(lambda s: s[~s.isin({'', 'NAN', 'NONE', 'NA'})])321 )322 if len(meta_type_vals):323 type_val = meta_type_vals.iloc[0]324 is_vocab = type_val in ('VOCAB', 'DERIVED', 'CATEGORICAL')325 type_label = 'short fixed-list' if is_vocab else 'numeric/range'326 print(f" ML {mdm_col}: skipped — analyst-marked as {type_label}")327 return None328 if _is_numeric_attr(attr_key_cols, history_df):329 print(f" ML {mdm_col}: skipped — auto-detected as numeric/range")330 return None331 332 try:333 # Routing: skip the classifier for identity/derived composites (openness334 # ~ 1) and pathological label spaces — Lookup carries them instead.335 skip, _ = routing.skip_learned_methods(history_df, attr_key_cols, mdm_col)336 if skip:337 print(f" ML {mdm_col}: skipped — resolved by lookup (no modelling needed)")338 return None339 340 train_df = (341 history_df[attr_key_cols + [mdm_col]]342 .replace(r'(OZ|LB)', ' ', regex=True)343 .dropna(subset=[mdm_col])344 )345 train_df.fillna('missing', inplace=True)346 # "NULL VALUE" is a Circana sentinel for unfilled cells. Both tokens347 # are stopwords, so the feature string collapses to "" and triggers348 # library warnings. Normalise to 'missing' before feature building.349 train_df.replace(re.compile(r'^\s*null\s+value\s*$', re.IGNORECASE),350 'missing', inplace=True)351 352 train_df = _augment_training(train_df, attr_key_cols)353 354 test_df = (355 flat_file_df[attr_key_cols]356 .replace(r'(OZ|LB)', ' ', regex=True)357 .copy()358 )359 if test_df.empty:360 return None361 test_df.fillna('missing', inplace=True)362 test_df.replace(re.compile(r'^\s*null\s+value\s*$', re.IGNORECASE),363 'missing', inplace=True)364 365 train_df, test_df = _build_features(train_df, test_df)366 367 # Fresh TF-IDF per attribute — required for parallel safety.368 tfidf = TfidfVectorizer(ngram_range=(1, 2), min_df=1, sublinear_tf=True, analyzer='word')369 370 train_features = tfidf.fit_transform(train_df['ftr'])371 test_features = tfidf.transform(test_df['ftr'])372 373 label_encoder = preprocessing.LabelEncoder()374 label_encoder.fit(train_df[mdm_col])375 print(f" ML {mdm_col}: {len(test_df)} products | {len(label_encoder.classes_)} known values")376 377 y_encoded = label_encoder.transform(train_df[mdm_col])378 clf = _fit_calibrated(train_features, y_encoded)379 380 class_labels = label_encoder.inverse_transform(clf.classes_)381 382 # Calibrated confidence matrix [n_products, n_classes] — Platt-scaled383 # predict_proba (see _fit_calibrated); _ml_scores reads it unchanged.384 proba_scores = _ml_scores(clf, test_features)385 386 # Top-K candidates — mirrors BM25 so Ensemble can fuse score distributions387 # from both methods rather than comparing only single top picks.388 k = min(_ML_TOP_K, proba_scores.shape[1])389 top_indices = np.argsort(-proba_scores, axis=1)[:, :k]390 top_scores = np.take_along_axis(proba_scores, top_indices, axis=1)391 top_labels = class_labels[top_indices]392 393 # Expand test_df to k rows per product, one per candidate label.394 expanded = test_df.loc[test_df.index.repeat(k)].reset_index(drop=True)395 expanded[mdm_col] = top_labels.ravel()396 expanded['score'] = (top_scores.ravel() * 100).round(2)397 398 grouped = (399 expanded.groupby(attr_key_cols + [mdm_col, 'score'])['ftr']400 .count()401 )402 grouped = grouped.reset_index(name='Record')403 grouped = grouped.replace('MISSING', '', regex=True)404 grouped['method'] = 'ML'405 return (f'ML_{mdm_col}', grouped)406 407 except Exception as exc:408 print(f" ERROR: ML failed for '{mdm_col}': {exc}")409 return None410 411 412def runML(flat_file_df: pd.DataFrame, meta_df: pd.DataFrame,413 history_df: pd.DataFrame) -> dict:414 """415 Train and run the TF-IDF classifier for all eligible attributes in parallel.416 417 Parameters418 ----------419 flat_file_df : DataFrame420 New products to classify (the flat-file CSV, string-coerced).421 meta_df : DataFrame422 META sheet — defines attribute groups, key columns, and type flags.423 history_df : DataFrame424 Historical FINAL data used as the labelled training set.425 426 Returns427 -------428 dict mapping 'ML_{attrG}' → prediction DataFrame with columns:429 attr_key_cols + [attrG, score, Record, method].430 431 Attributes are processed in parallel across 4 workers; the LinearSVC backend432 is single-threaded per fit, so 4 attributes train concurrently.433 """434 attrs = meta_df['Attribute Group name'].unique().tolist()435 ml_results = {}436 437 with ThreadPoolExecutor(max_workers=4) as executor:438 futures = [439 executor.submit(_process_one_xgb_attr, col, meta_df, history_df, flat_file_df)440 for col in attrs441 ]442 for future in futures:443 result = future.result()444 if result is not None:445 key, df = result446 ml_results[key] = df447 448 return ml_results449