CMacD/AIC_PHASE1_POC
0
1"""2Pipeline orchestrator for AIC Phase 2 + Phase 3.3 4Runs the full processing pipeline in sequence:5 Phase 2 – Attribute assembly from workbook sheets (aic_phase2).6 Phase 3 – Quality checks, transformations, SKU collapse, and7 category splitting (quality, transforms, sku_collapse).8 9Called by ``PipelineWorker`` in the GUI; all ``print()`` output is10captured and streamed to the GUI log widget via a stdout proxy.11 12Returns13-------14(collapsed_df, duplicate_dimkeys_df)15"""16 17from __future__ import annotations18 19import json20import os21import re22from pathlib import Path23from typing import Dict, Any, List, Optional24 25import pandas as pd26pd.set_option('display.max_colwidth', None)27 28# --- Phase 2 imports (attribute assembly + MDM QC) -------------------------29from phase3_package.aic_phase2 import aic_code, run_tool_vs_mdm_qc30 31# --- Phase 3 transforms (private label, brand overrides, restricted) -------32from phase3_package.transforms import (33 overwrite_upc10_for_private_label,34 apply_private_label_rules,35 normalize_upc10,36 strip_legacy_brand_overrides,37 apply_brand_overrides,38 strip_legacy_restricted_suffix,39 raw_multi_restricted_overrides,40 _print_step_header,41)42 43# --- Phase 3 quality checks -----------------------------------------------44from phase3_package.quality import (45 update_req_check,46 verify_ao_cat_def,47 demand_group_check,48 check_identifier_numeric_format,49 flag_invalid_headers,50 check_special_chars,51 check_duplicate_dimkeys,52 split_by_raw_assortment_category,53 check_brand_tool_brand_mismatch,54 check_null_modeling_reporting_cols,55)56 57# --- SKU collapse ----------------------------------------------------------58from phase3_package.sku_collapse import prepare_and_collapse59 60 61# ═══════════════════════════════════════════════════════════════════════════62# Formatting Constants63# ═══════════════════════════════════════════════════════════════════════════64MAJOR_SEP = "=" * 7065MINOR_SEP = "-" * 6066INDENT = " "67 68 69# ═══════════════════════════════════════════════════════════════════════════70# Main Pipeline71# ═══════════════════════════════════════════════════════════════════════════72 73def run_through_step_12(74 directory_path: str,75 raw_upc_pl_brand_col: str,76 private_label_config: Dict[str, Any],77 brand_override_config: Dict[str, Any],78 is_custom_collapse: bool,79 file_manifest: Dict[str, Any] = None,80 skip_rmrr: bool = False,81 pl_base_name: str = "",82) -> tuple:83 """84 Run Steps 1-12 plus mismatch detection (Phase A).85 86 Returns87 -------88 (df, duplicate_dimkeys_df, mismatch_groups, pipeline_context)89 - df: DataFrame after all transformations through Step 12.90 - duplicate_dimkeys_df: QC DataFrame of duplicate ITEM_DIM_KEYs found.91 - mismatch_groups: List of per-model mismatch dicts (empty list if none).92 - pipeline_context: Dict carrying forward state needed by Phase B.93 """94 input_dir = Path(directory_path)95 96 # ------------------------------------------------------------------97 # Build file manifest (scan directory ONCE if not pre-provided)98 # ------------------------------------------------------------------99 if file_manifest is None:100 file_manifest = _scan_directory(input_dir)101 102 # ------------------------------------------------------------------103 # Read and concatenate all ModelInfo files from manifest104 # ------------------------------------------------------------------105 model_info_paths = file_manifest["model_info_paths"]106 model_info_frames: List[pd.DataFrame] = []107 for file_path in model_info_paths:108 try:109 model_info_frame = pd.read_csv(file_path, sep="|")110 model_info_frames.append(model_info_frame)111 except PermissionError:112 raise PermissionError(113 f"\n{'=' * 60}\n"114 f"FILE LOCKED: Cannot access '{Path(file_path).name}'\n"115 f"{'=' * 60}\n"116 f"The file appears to be open in another application.\n\n"117 f"Please close the file and try again.\n"118 f"{'=' * 60}"119 )120 except Exception as exc:121 raise RuntimeError(f"Error reading ModelInfo file at {file_path}: {exc}")122 123 model_info_df = pd.concat(model_info_frames, ignore_index=True)124 125 # Log which ModelInfo files were used126 if len(model_info_paths) == 1:127 relative_path = Path(model_info_paths[0]).relative_to(input_dir)128 print(f"{INDENT}Using ModelInfo.txt from: {relative_path}")129 else:130 relative_paths = ", ".join(131 str(Path(p).relative_to(input_dir)) for p in model_info_paths132 )133 print(f"{INDENT}Detected multiple ModelInfo.txt files in subdirectories: {relative_paths}")134 print(f"{INDENT}Combined all ModelInfo files into a single frame for QC checks.")135 136 # ==================================================================137 # Phase 2: AIC Attribute Assembly138 # ==================================================================139 print(f"\n{MAJOR_SEP}")140 print("PHASE 2 AIC PROCESSING")141 print(MAJOR_SEP)142 143 # skip_qc=True because QC runs later on fully transformed data144 # Pass the file manifest so aic_code() skips its own directory scans145 df, _, meta_df, combined_attributes_df, combined_attr_values_df, demand_group_fallback = aic_code(146 str(input_dir), skip_qc=True, file_manifest=file_manifest147 )148 149 # ==================================================================150 # Phase 3: Quality Checks and Transformations (Steps 1-12)151 # ==================================================================152 print(f"\n{MAJOR_SEP}")153 print("PHASE 3 QUALITY CHECKS AND TRANSFORMATIONS")154 print(MAJOR_SEP)155 156 # --- Derive valid model suffixes from subdirectories ------------------157 tool_sources = file_manifest.get("tool_sources", [])158 subdir_names = [159 s.split(":", 1)[1] for s in tool_sources if s.startswith("subdir:")160 ]161 162 valid_model_suffixes: Optional[set] = None163 if subdir_names:164 # Discover known column suffixes from any TOOL_X / X paired columns165 # (e.g. TOOL_BRAND/BRAND, TOOL_SUBBRAND/SUBBRAND) — two-pass approach:166 # pass 1 finds unambiguous base pairs, pass 2 finds their suffixed variants.167 col_upper_set = {str(c).upper() for c in df.columns}168 known_suffixes: set = set()169 for col_key in col_upper_set:170 if col_key.startswith("TOOL_"):171 base = col_key[len("TOOL_"):]172 if base and "_" not in base and base in col_upper_set:173 # Confirmed base pair — scan for X_SUFFIX / TOOL_X_SUFFIX variants174 base_prefix = f"{base}_"175 for other_col in col_upper_set:176 if other_col.startswith(base_prefix):177 suffix = other_col[len(base_prefix):]178 if suffix and f"TOOL_{base}_{suffix}" in col_upper_set:179 known_suffixes.add(suffix)180 181 # Additional pass: directly validate directory names as column suffixes.182 # Handles projects where only suffixed columns exist (no base TOOL_X/X pair),183 # e.g. SUBBRAND_MULO / TOOL_SUBBRAND_MULO with no base SUBBRAND / TOOL_SUBBRAND.184 for dir_name in subdir_names:185 dir_upper = dir_name.upper()186 if dir_upper not in known_suffixes:187 suffix_tag = f"_{dir_upper}"188 for col_key in col_upper_set:189 if col_key.startswith("TOOL_") and col_key.endswith(suffix_tag):190 non_tool = col_key[len("TOOL_"):]191 if non_tool in col_upper_set:192 known_suffixes.add(dir_upper)193 break194 195 valid_model_suffixes = set()196 print(f"{INDENT}Multi-model subdirectories detected:")197 198 for dir_name in sorted(subdir_names):199 dir_upper = dir_name.upper()200 201 # Exact match (e.g. directory called "CONV")202 if dir_upper in known_suffixes:203 valid_model_suffixes.add(dir_upper)204 print(f"{INDENT} {dir_name} → {dir_upper} (confirmed column suffix _{dir_upper})")205 continue206 207 # Segment match — check each underscore-delimited part208 segments = dir_upper.split("_")209 matched = [seg for seg in segments if seg in known_suffixes]210 211 if len(matched) == 1:212 valid_model_suffixes.add(matched[0])213 print(f"{INDENT} {dir_name} → {matched[0]} (confirmed column suffix _{matched[0]})")214 elif len(matched) > 1:215 # Ambiguous — use last matching segment216 chosen = matched[-1]217 valid_model_suffixes.add(chosen)218 print(f"{INDENT} ⚠ {dir_name} matches multiple suffixes {matched} — using {chosen}")219 else:220 # No column match — fall back to last segment and warn221 fallback = segments[-1]222 valid_model_suffixes.add(fallback)223 print(224 f"{INDENT} ⚠ {dir_name} → {fallback} (no matching column pair with suffix _{fallback} found — "225 f"expected e.g. BRAND_{fallback} / TOOL_BRAND_{fallback} or similar TOOL_*/base pair)"226 )227 228 valid_model_suffixes = valid_model_suffixes or None229 230 if valid_model_suffixes:231 print(f"{INDENT}Active model suffixes: {', '.join(sorted(valid_model_suffixes))}")232 print(f"{INDENT}Only column pairs matching these suffixes will be processed.\n")233 234 # Step 1: Ensure UPDATE_REQUIRED is set to 1 for all rows235 df = update_req_check(df)236 237 # Step 2: Align ASSORTMENT_CATEGORY_DEFINITION to ModelInfo238 df = verify_ao_cat_def(df, model_info_df)239 240 # Step 3: Audit DEMAND_GROUP and interaction columns241 df = demand_group_check(df, demand_group_fallback=demand_group_fallback)242 243 # Step 4: Overwrite UPC10 with ITEM_DIM_KEY for private label rows244 df = overwrite_upc10_for_private_label(df, raw_upc_pl_brand_col)245 246 # Step 5: Apply retailer-specific private label tagging247 df = apply_private_label_rules(df, private_label_config, show_examples=True, valid_model_suffixes=valid_model_suffixes, pl_base_name=pl_base_name)248 249 # Step 6: Check UPC10/SKU/ITEM_DIM_KEY for scientific notation / decimals250 check_identifier_numeric_format(df, cols=("UPC10", "SKU", "ITEM_DIM_KEY"))251 252 # Step 7: Left-pad UPC10 to 10 characters and mirror to UPC10_ATTR253 df = normalize_upc10(df, upc_col="UPC10")254 255 # Step 8: Drop unnamed columns, flag non-standard column names256 df = flag_invalid_headers(df)257 258 # Step 9: Replace '/' with ' OR ' in reporting columns, flag special chars259 df = check_special_chars(df, suffix=None)260 261 # Step 10: Deduplicate ITEM_DIM_KEYs (keep highest-dollar row)262 df, duplicate_dimkeys_df = check_duplicate_dimkeys(df)263 264 # Step 10.5: Strip legacy RESTRICTED suffix before brand override cleanup265 df = strip_legacy_restricted_suffix(df, valid_model_suffixes=valid_model_suffixes)266 267 # Step 10.6: Strip stale brand overrides for non-configured manufacturers268 df = strip_legacy_brand_overrides(df, brand_override_config, valid_model_suffixes=valid_model_suffixes)269 270 # Step 11: Apply client brand mapping overrides271 df = apply_brand_overrides(df, brand_override_config, valid_model_suffixes=valid_model_suffixes)272 273 # Step 12: Canonicalize TOOL_BRAND with _RESTRICTED where RAW_MULTI signals it274 # Skipped for non-MULO+ geo groupings where RMRR tagging does not apply275 if not skip_rmrr:276 df = raw_multi_restricted_overrides(df, valid_model_suffixes=valid_model_suffixes)277 278 # Step 12.5: QC — flag BRAND vs TOOL_BRAND mismatches (potential DB logic issues)279 mismatch_groups = check_brand_tool_brand_mismatch(280 df,281 raw_manufacturer_col=brand_override_config.get("raw_manufacturer_col", ""),282 valid_model_suffixes=valid_model_suffixes,283 )284 285 # Build pipeline context for Phase B286 pipeline_context = {287 "meta_df": meta_df,288 "combined_attributes_df": combined_attributes_df,289 "combined_attr_values_df": combined_attr_values_df,290 "duplicate_dimkeys_df": duplicate_dimkeys_df,291 "input_dir": str(input_dir),292 "is_custom_collapse": is_custom_collapse,293 "raw_manufacturer_col": brand_override_config.get("raw_manufacturer_col", ""),294 "valid_model_suffixes": valid_model_suffixes,295 }296 297 return df, duplicate_dimkeys_df, mismatch_groups, pipeline_context298 299 300def run_from_step_14(301 df: pd.DataFrame,302 pipeline_context: Dict[str, Any],303 corrections: Optional[list] = None,304) -> tuple:305 """306 Apply BRAND / TOOL_BRAND corrections (if any), then run Steps 14-17 (Phase B).307 308 The category split is **not** performed here — the collapsed output is309 written as a single sheet so analysts can review and edit before the310 post-QC stage re-collapses and exports CSVs.311 312 Parameters313 ----------314 df : DataFrame315 The DataFrame from Phase A (after Step 12).316 pipeline_context : dict317 State carried forward from ``run_through_step_12()``.318 corrections : list of dict, optional319 Each dict has ``type`` ("brand" or "tool_brand") plus the320 original values, new value, and actual column names in df.321 322 Returns323 -------324 (collapsed_df, duplicate_dimkeys_df)325 """326 meta_df = pipeline_context["meta_df"]327 combined_attributes_df = pipeline_context["combined_attributes_df"]328 combined_attr_values_df = pipeline_context["combined_attr_values_df"]329 duplicate_dimkeys_df = pipeline_context["duplicate_dimkeys_df"]330 input_dir = pipeline_context["input_dir"]331 is_custom_collapse = pipeline_context["is_custom_collapse"]332 333 # --- Apply user corrections to BRAND / TOOL_BRAND --------------------334 raw_manufacturer_col = pipeline_context.get("raw_manufacturer_col", "")335 col_upper_map = {str(c).upper(): c for c in df.columns}336 mfr_col_actual = col_upper_map.get(raw_manufacturer_col.upper()) if raw_manufacturer_col else None337 338 if corrections:339 brand_count = 0340 tool_count = 0341 rows_updated = 0342 corrected_pairs: set = set()343 for fix in corrections:344 brand_col_name = fix.get("brand_col", "BRAND")345 tool_col_name = fix.get("tool_brand_col", "TOOL_BRAND")346 347 # Row mask: match on original BRAND + TOOL_BRAND values348 mask = (349 (df[brand_col_name].astype(str).str.upper() == fix["brand"].upper())350 & (df[tool_col_name].astype(str).str.upper() == fix["tool_brand_old"].upper())351 )352 353 # Narrow by parent manufacturer when available (AO brand rows)354 parent_val = fix.get("parent", "")355 if parent_val and mfr_col_actual:356 mask = mask & (df[mfr_col_actual].astype(str).str.upper() == parent_val.upper())357 358 n_rows = int(mask.sum())359 rows_updated += n_rows360 corrected_pairs.add((fix["brand"], fix["tool_brand_old"]))361 362 if fix.get("type") == "brand":363 df.loc[mask, brand_col_name] = fix["brand_new"]364 brand_count += 1365 else:366 df.loc[mask, tool_col_name] = fix["tool_brand_new"]367 tool_count += 1368 369 parts = []370 if brand_count:371 parts.append(f"{brand_count} BRAND")372 if tool_count:373 parts.append(f"{tool_count} TOOL_BRAND")374 n_pairs = len(corrected_pairs)375 print(f"{INDENT}Mismatch review: {' + '.join(parts)} correction(s) manually applied "376 f"— {n_pairs} distinct pair(s), {rows_updated} row(s) updated")377 378 379 # Step 14: SKU collapse (top-dollar or custom parent dim-key)380 collapsed_df = prepare_and_collapse(381 df,382 verbose=True,383 is_custom_collapse=is_custom_collapse,384 )385 386 # Step 15: QC — flag null values in modeling/reporting columns.387 # Run on collapsed_df (the data that will be written to output.xlsx) so that388 # any auto-fills are applied to the output, not to the pre-collapse df which389 # is already discarded. Previously this ran on df, so auto-fills never390 # reached the output and Post-QC would re-find the same nulls.391 check_null_modeling_reporting_cols(collapsed_df, meta_df=meta_df)392 393 # Step 16: Tool vs MDM attribute comparison QC (on final transformed data)394 _print_step_header("16", "ATTRIBUTE QC (TOOL VS MDM COMPARISON)")395 valid_model_suffixes = pipeline_context.get("valid_model_suffixes")396 run_tool_vs_mdm_qc(397 input_dir,398 collapsed_df,399 combined_attributes_df=combined_attributes_df,400 combined_attr_values_df=combined_attr_values_df,401 valid_model_suffixes=valid_model_suffixes,402 )403 404 return collapsed_df, duplicate_dimkeys_df405 406 407def run_post_qc(408 excel_path: str,409 is_custom_collapse: bool,410 meta_df: Optional[pd.DataFrame] = None,411) -> tuple:412 """413 Post-QC pipeline: re-validate and re-collapse analyst-edited output.414 415 Reads the single-sheet Excel file that the analyst has reviewed and416 edited, checks for null values in modeling/reporting columns, re-runs417 SKU collapse (to ensure edits haven't violated collapse rules), and418 splits by category for CSV export.419 420 Parameters421 ----------422 excel_path : str423 Path to the analyst-edited Excel workbook (single "Cleaned Output" sheet).424 is_custom_collapse : bool425 If True, use analyst-selected parent dim-key for SKU collapse.426 meta_df : DataFrame, optional427 META sheet used for null-check column classification.428 429 Returns430 -------431 (collapsed_df, category_splits)432 - collapsed_df: Re-collapsed DataFrame.433 - category_splits: Dict mapping category name → DataFrame subset.434 """435 print(f"\n{MAJOR_SEP}")436 print("POST-QC PIPELINE (Finalize & Export)")437 print(MAJOR_SEP)438 439 # --- Read the edited Excel file back ---------------------------------440 _print_step_header("Post-Step 1", "Read Edited Output File")441 print(f"{INDENT}Reading: {excel_path}")442 443 df = pd.read_excel(excel_path, sheet_name="Cleaned Output", engine="openpyxl")444 cleaned_output_rows = len(df)445 print(f"{INDENT}Loaded {cleaned_output_rows} row(s), {len(df.columns)} column(s)")446 447 # --- Pre-collapse validation -----------------------------------------448 _print_step_header("Post-Step 2", "Pre-Collapse Validation")449 print(f"{INDENT}Running null checks on modeling/reporting columns before SKU re-collapse...")450 check_null_modeling_reporting_cols(df, meta_df=meta_df, show_step_header=False)451 452 # --- Re-run SKU collapse ---------------------------------------------453 _print_step_header("Post-Step 3", "SKU Re-Collapse")454 print(f"{INDENT}Re-collapsing SKUs to reflect any analyst edits made to the Cleaned Output sheet...")455 collapsed_df = prepare_and_collapse(456 df,457 verbose=True,458 is_custom_collapse=is_custom_collapse,459 show_step_header=False,460 )461 462 # --- Split by category for CSV export --------------------------------463 _print_step_header("Post-Step 4", "Category Split & Re-Export")464 category_splits = split_by_raw_assortment_category(collapsed_df)465 466 # --- Multi-category sanity check -------------------------------------467 if len(category_splits) > 1:468 summed_rows = sum(len(cat_df) for cat_df in category_splits.values())469 collapsed_rows = len(collapsed_df)470 match_str = "PASS" if summed_rows == collapsed_rows else "FAIL"471 print(f"\n{INDENT}Sanity check: sum of category sheet rows ({summed_rows}) "472 f"vs Cleaned Output rows ({collapsed_rows}) — {match_str}")473 if summed_rows != collapsed_rows:474 print(f"{INDENT}⚠ Row count mismatch — {abs(summed_rows - collapsed_rows)} row(s) difference. "475 f"Check for blank or unmapped RAW_ASSORTMENT_CATEGORY values.")476 477 print(f"\n{INDENT}Post-QC pipeline complete — ready for CSV export.")478 return collapsed_df, category_splits479 480 481def main(482 directory_path: str,483 raw_upc_pl_brand_col: str,484 private_label_config: Dict[str, Any],485 brand_override_config: Dict[str, Any],486 is_custom_collapse: bool,487 file_manifest: Dict[str, Any] = None,488):489 """490 Run the full Phase 2 → Phase 3 pipeline and return results.491 492 Convenience wrapper that calls ``run_through_step_12()`` followed by493 ``run_from_step_14()``. Preserved for backwards compatibility with494 non-GUI callers.495 496 Parameters497 ----------498 directory_path : str499 Root folder containing File_For_Mapping_QC.xlsx, ModelInfo.txt,500 and tool/lookup files.501 raw_upc_pl_brand_col : str502 RAW column used for private-label UPC10 overwrite (e.g. "RAW_BRAND").503 private_label_config : dict504 Retailer-specific private label rules (walmart, cvs, heb, etc.).505 brand_override_config : dict506 Manufacturer → brand override mapping rules.507 is_custom_collapse : bool508 If True, use analyst-selected parent dim-key for SKU collapse509 instead of top-dollar row.510 file_manifest : dict, optional511 Pre-scanned directory manifest. When provided, skips all directory512 scanning and redundant file reads. Built by ``_scan_directory()``513 or passed from the GUI layer.514 515 Returns516 -------517 (collapsed_df, duplicate_dimkeys_df)518 - collapsed_df: Final cleaned DataFrame after all steps.519 - duplicate_dimkeys_df: QC DataFrame of duplicate ITEM_DIM_KEYs found.520 """521 df, dup_df, mismatch_groups, ctx = run_through_step_12(522 directory_path,523 raw_upc_pl_brand_col,524 private_label_config,525 brand_override_config,526 is_custom_collapse,527 file_manifest=file_manifest,528 )529 return run_from_step_14(df, ctx)530 531 532# ═══════════════════════════════════════════════════════════════════════════533# Directory Scanner (single scan for entire pipeline)534# ═══════════════════════════════════════════════════════════════════════════535 536def _scan_directory(input_dir: Path) -> Dict[str, Any]:537 """538 Scan the input directory ONCE and build a manifest of all files539 needed by the pipeline.540 541 Returns a dict with:542 - ``workbook_path`` : Path to File_For_Mapping_QC.xlsx (or None)543 - ``model_info_paths`` : List of Path to ModelInfo.txt files544 - ``combined_attributes_df`` : Combined Attributes.txt DataFrame (or None)545 - ``combined_attr_values_df`` : Combined AttributeValues.txt DataFrame (or None)546 - ``tool_sources`` : List of source labels (e.g. "root", "subdir:MULO")547 - ``json_config`` : Parsed JSON config dict (or None)548 - ``skipped_files`` : List of skipped file names549 """550 workbook_path: Optional[Path] = None551 model_info_paths: List[Path] = []552 root_model_info: Optional[Path] = None553 all_attributes_dfs: List[pd.DataFrame] = []554 all_attr_values_dfs: List[pd.DataFrame] = []555 tool_sources: List[str] = []556 json_config = None557 skipped_files: List[str] = []558 subdirs: List[Path] = []559 560 expected_txt_patterns = [561 r"^Attributes\.txt$",562 r"^AttributeValues\.txt$",563 r"(?i)^ModelInfo.*\.txt$",564 ]565 expected_xlsx_pattern = r"(?i).*file_for_mapping_qc.*\.xlsx$"566 567 def _is_expected_txt(name: str) -> bool:568 return any(re.match(p, name) for p in expected_txt_patterns)569 570 # --- Scan root directory ------------------------------------------------571 root_has_attributes = False572 root_has_attr_values = False573 root_attributes_df = None574 root_attr_values_df = None575 576 for entry in input_dir.iterdir():577 if entry.is_dir():578 subdirs.append(entry)579 continue580 581 if not entry.is_file():582 continue583 584 name = entry.name585 name_lower = name.lower()586 587 # Workbook588 if name_lower.endswith(".xlsx") and re.match(expected_xlsx_pattern, name):589 workbook_path = entry590 continue591 592 # ModelInfo593 if name_lower == "modelinfo.txt":594 root_model_info = entry595 continue596 597 # Attributes.txt598 if name == "Attributes.txt":599 try:600 root_attributes_df = pd.read_csv(str(entry), delimiter="|")601 root_has_attributes = True602 except Exception as exc:603 print(f"{INDENT} ⚠ Skipped {name}: could not parse ({type(exc).__name__})")604 skipped_files.append(name)605 continue606 607 # AttributeValues.txt608 if name == "AttributeValues.txt":609 try:610 root_attr_values_df = pd.read_csv(str(entry), delimiter="|")611 root_has_attr_values = True612 except Exception as exc:613 print(f"{INDENT} ⚠ Skipped {name}: could not parse ({type(exc).__name__})")614 skipped_files.append(name)615 continue616 617 # JSON config618 if name_lower.endswith(".json"):619 try:620 with open(str(entry), "r") as f:621 json_config = json.load(f)622 except (json.JSONDecodeError, Exception):623 skipped_files.append(name)624 continue625 626 # Other expected txt files (e.g. other ModelInfo variants)627 if name_lower.endswith(".txt") and _is_expected_txt(name):628 continue629 630 # Non-essential files631 if not name_lower.endswith(".csv"):632 skipped_files.append(name)633 634 # Collect root-level tool files635 if root_has_attributes and root_has_attr_values:636 all_attributes_dfs.append(root_attributes_df)637 all_attr_values_dfs.append(root_attr_values_df)638 tool_sources.append("root")639 640 # Use root ModelInfo if found; otherwise check subdirectories641 if root_model_info:642 model_info_paths = [root_model_info]643 644 # --- Scan subdirectories -----------------------------------------------645 for subdir in subdirs:646 try:647 sub_entries = {e.name: e for e in subdir.iterdir() if e.is_file()}648 except PermissionError:649 continue650 651 # ModelInfo in subdirectory (only if not found at root)652 if not root_model_info:653 for sub_name, sub_path in sub_entries.items():654 if sub_name.lower() == "modelinfo.txt":655 model_info_paths.append(sub_path)656 657 # Tool files in subdirectory658 attr_entry = sub_entries.get("Attributes.txt")659 attr_val_entry = sub_entries.get("AttributeValues.txt")660 661 if attr_entry and attr_val_entry:662 try:663 all_attributes_dfs.append(pd.read_csv(str(attr_entry), delimiter="|"))664 all_attr_values_dfs.append(pd.read_csv(str(attr_val_entry), delimiter="|"))665 tool_sources.append(f"subdir:{subdir.name}")666 except Exception as exc:667 print(f"{INDENT}Error reading tool files in subdirectory '{subdir.name}': {exc}")668 669 # --- Validate required files -------------------------------------------670 if not model_info_paths:671 raise FileNotFoundError(672 f"ModelInfo.txt not found in {input_dir} "673 f"or any immediate subdirectory."674 )675 676 # --- Combine tool DataFrames -------------------------------------------677 combined_attributes_df = None678 combined_attr_values_df = None679 if all_attributes_dfs and all_attr_values_dfs:680 combined_attributes_df = pd.concat(all_attributes_dfs, ignore_index=True).drop_duplicates()681 combined_attr_values_df = pd.concat(all_attr_values_dfs, ignore_index=True).drop_duplicates()682 683 return {684 "workbook_path": workbook_path,685 "model_info_paths": model_info_paths,686 "combined_attributes_df": combined_attributes_df,687 "combined_attr_values_df": combined_attr_values_df,688 "tool_sources": tool_sources,689 "json_config": json_config,690 "skipped_files": skipped_files,691 }692