ali27x/TendersREF
0
1from __future__ import annotations2 3from io import BytesIO4from pathlib import Path5from datetime import datetime6 7import pandas as pd8import streamlit as st9 10import tce.classifier as classifier11from tce import ui_labels, workflow12from tce.classifier import ClassificationResult13from tce.spec_loader import load_spec14from tce.storage import fetch_references, upsert_reference15 16 17ROOT = Path(__file__).parent18SPEC_PATH = ROOT / "TCS_TCE_Master_Spec_v1.0.xlsx"19BENCHMARK_PATH = ROOT / "Gold_Dataset_v3.0_Final_All_clean_v13.xlsx"20DB_PATH = ROOT / "tce.db"21BUILD_ID = "2026-07-14-14"22CLASSIFIER_VERSION = "TCE v1.0 Beta"23APPROVED_BY_DEFAULT = "Manual Reviewer"24 25 26@st.cache_resource27def get_spec():28 return load_spec(SPEC_PATH)29 30 31@st.cache_data32def get_benchmark_metrics() -> dict | None:33 if BENCHMARK_PATH.exists():34 from tce.evaluation import evaluate_gold_set35 36 return evaluate_gold_set(BENCHMARK_PATH, get_spec())37 return None38 39 40def load_approved_references() -> list[dict]:41 return fetch_references(DB_PATH)42 43 44def result_to_dict(result: ClassificationResult) -> dict:45 return result.__dict__.copy()46 47 48def render_bilingual_value(en: str, ar: str) -> str:49 en = (en or "").strip() or ui_labels.BLANK_LABEL50 ar = (ar or "").strip() or ui_labels.BLANK_LABEL51 if en == ui_labels.BLANK_LABEL and ar == ui_labels.BLANK_LABEL:52 return ui_labels.BLANK_LABEL53 return f"{en} | {ar}"54 55 56def title_payload(title: str) -> tuple[str, str, str]:57 return workflow.select_classification_inputs(title.strip())58 59 60def classify_single_title(title: str, spec, refs: list[dict]) -> tuple[workflow.SuggestionBundle, str]:61 bundle = workflow.classify_with_approved_learning(title, spec, refs)62 detected_language = workflow.detect_input_language(title)63 return bundle, detected_language64 65 66def classify_dataframe(df: pd.DataFrame, spec, refs: list[dict]) -> pd.DataFrame:67 rows = []68 for _, row in df.iterrows():69 bundle = workflow.classify_row_with_learning(row.to_dict(), spec, refs)70 r = bundle.result71 out = row.to_dict()72 out.update(result_to_dict(r))73 out["classification_title_used"] = r.classification_title_used74 out["classification_language"] = r.classification_language75 out["approval_status"] = r.approval_status76 out["suggestion_source"] = bundle.suggestion_source77 out["detected_language"] = workflow.detect_input_language(78 str(row.get("name_ar", "") or row.get("name_en", "") or "")79 )80 rows.append(out)81 return pd.DataFrame(rows)82 83 84def bilingual_options(values: list[str], mapping: dict[str, dict[str, str]]) -> list[str]:85 return values86 87 88def format_bilingual_option(value: str, mapping: dict[str, dict[str, str]]) -> str:89 return ui_labels.pair_label(value, mapping)90 91 92def save_approved_result(93 *,94 original_title: str,95 detected_language: str,96 suggested: ClassificationResult,97 final_nature: str,98 final_sector: str,99 final_scope: str,100 approved_by: str,101 correction_note: str,102 suggestion_source: str,103 approved_at: str | None = None,104):105 title_norm = workflow.normalize_input_title(original_title)106 upsert_reference(107 DB_PATH,108 {109 "title_key": title_norm,110 "title_original": original_title,111 "title_normalized": title_norm,112 "detected_language": detected_language,113 "title_ar": original_title if detected_language in {"ar", "mixed"} else "",114 "title_en": original_title if detected_language == "en" else "",115 "suggested_nature": suggested.nature_en,116 "suggested_sector": suggested.market_sector_en,117 "suggested_scope": suggested.scope_en,118 "approved_nature": final_nature,119 "approved_sector": final_sector,120 "approved_scope": final_scope,121 "nature_en": final_nature,122 "nature_ar": ui_labels.NATURE_LABELS.get(final_nature, {}).get("ar", ""),123 "market_sector_en": final_sector,124 "market_sector_ar": ui_labels.SECTOR_LABELS.get(final_sector, {}).get("ar", ""),125 "scope_en": final_scope,126 "scope_ar": ui_labels.SCOPE_LABELS.get(final_scope, {}).get("ar", ""),127 "confidence": int(suggested.confidence or 0),128 "needs_review": "No",129 "classification_evidence": suggested.classification_evidence,130 "classification_note": suggested.classification_note,131 "suggestion_source": suggestion_source,132 "nature_changed": int(final_nature != suggested.nature_en),133 "sector_changed": int(final_sector != suggested.market_sector_en),134 "scope_changed": int(final_scope != suggested.scope_en),135 "correction_note": correction_note,136 "approved_by": approved_by,137 "approved_at": approved_at or datetime.utcnow().isoformat(sep=" ", timespec="seconds"),138 "classifier_version": CLASSIFIER_VERSION,139 "gold_dataset_version": "Gold Dataset v3.0 Final Clean v13",140 "tcs_version": spec.version,141 "approval_status": "Approved",142 },143 )144 145 146def set_pending_result(bundle: workflow.SuggestionBundle, original_title: str, detected_language: str):147 st.session_state["pending_result"] = result_to_dict(bundle.result)148 st.session_state["pending_title"] = original_title149 st.session_state["pending_language"] = detected_language150 st.session_state["pending_source"] = bundle.suggestion_source151 st.session_state["pending_similar_ref"] = bundle.similar_reference152 st.session_state["edit_single"] = False153 154 155def clear_pending_result():156 for key in ["pending_result", "pending_title", "pending_language", "pending_source", "pending_similar_ref", "edit_single"]:157 st.session_state.pop(key, None)158 159 160def request_single_form_reset(message: str) -> None:161 st.session_state["single_flash_message"] = message162 st.session_state["single_reset_requested"] = True163 st.session_state["single_title_widget_version"] = st.session_state.get("single_title_widget_version", 0) + 1164 165 166def apply_single_form_reset() -> None:167 st.session_state.setdefault("single_title_widget_version", 0)168 if st.session_state.pop("single_reset_requested", False):169 for key in ["pending_result", "pending_title", "pending_language", "pending_source", "pending_similar_ref", "edit_single"]:170 st.session_state.pop(key, None)171 172 173st.set_page_config(page_title="Tender Classification Engine", layout="wide")174spec = get_spec()175 176st.title("Tender Classification Engine")177st.caption(178 f"Classifier Version: {CLASSIFIER_VERSION} | Build: {BUILD_ID} | Latest TCS: {spec.version} | "179 f"Official benchmark: Gold Dataset v3.0 Final Clean v13"180)181 182if flash := st.session_state.pop("single_flash_message", ""):183 st.success(flash)184 185apply_single_form_reset()186 187with st.expander("Benchmark Status", expanded=False):188 if BENCHMARK_PATH.exists():189 if st.button("Load benchmark metrics"):190 bench = get_benchmark_metrics()191 if bench:192 st.write(193 {194 "Nature Accuracy": f'{bench["nature_accuracy"]:.1f}%',195 "Market Sector Accuracy": f'{bench["market_sector_accuracy"]:.1f}%',196 "Scope Accuracy": f'{bench["scope_accuracy"]:.1f}%',197 "Needs Review %": f'{bench["needs_review_pct"]:.1f}%',198 "Average Confidence": f'{bench["avg_confidence"]:.1f}%',199 "Rows": bench["rows"],200 }201 )202 else:203 st.warning("Benchmark file not found.")204 else:205 st.warning("Benchmark file not found.")206 207single_tab, batch_tab = st.tabs(["Single Classification | تصنيف مفرد", "Excel Upload | رفع ملف إكسل"])208 209with single_tab:210 st.subheader("Single Classification | تصنيف مفرد")211 title_widget_key = f"single_title_{st.session_state['single_title_widget_version']}"212 title = st.text_input("Tender Title | عنوان المناقصة", key=title_widget_key, placeholder="توريد أجهزة كمبيوتر / Supply of Computer Equipment")213 classify_clicked = st.button("Classify | تصنيف", type="primary")214 215 if classify_clicked:216 if title.strip():217 refs = load_approved_references()218 bundle, detected_language = classify_single_title(title, spec, refs)219 set_pending_result(bundle, title, detected_language)220 else:221 st.error("Tender Title | عنوان المناقصة is required.")222 223 if "pending_result" in st.session_state:224 res = st.session_state["pending_result"]225 source = st.session_state.get("pending_source", "rule_engine")226 st.markdown("### Suggested Classification | التصنيف المقترح")227 c1, c2, c3, c4 = st.columns(4)228 c1.metric("Nature", render_bilingual_value(res["nature_en"], res["nature_ar"]))229 c2.metric("Market Sector", render_bilingual_value(res["market_sector_en"], res["market_sector_ar"]))230 c3.metric("Scope", render_bilingual_value(res["scope_en"], res["scope_ar"]))231 c4.metric("Confidence", f'{res["confidence"]}%')232 233 st.write(234 {235 "Nature": render_bilingual_value(res["nature_en"], res["nature_ar"]),236 "Market Sector": render_bilingual_value(res["market_sector_en"], res["market_sector_ar"]),237 "Scope": render_bilingual_value(res["scope_en"], res["scope_ar"]),238 "Confidence": res["confidence"],239 "Evidence": res.get("classification_evidence", ""),240 "Suggestion Source": ui_labels.source_label(source),241 "Needs Review": res.get("needs_review", ""),242 "Final Decision": res.get("final_decision", ""),243 }244 )245 246 with st.expander("Technical Details | التفاصيل الفنية", expanded=False):247 st.write(248 {249 "Main Action": res.get("main_action", ""),250 "Main Subject": res.get("main_subject", ""),251 "Supporting Evidence": res.get("supporting_evidence", []),252 "Conflicting Evidence": res.get("conflicting_evidence", []),253 "Matched Rule": res.get("matched_rule", ""),254 "Review Reason": res.get("review_reason", ""),255 "Classification Note": res.get("classification_note", ""),256 "Classification Evidence": res.get("classification_evidence", ""),257 "TCS Version": res.get("tcs_version", ""),258 }259 )260 261 col_a, col_b = st.columns(2)262 if col_a.button("Approve | موافقة"):263 save_approved_result(264 original_title=st.session_state.get("pending_title", title),265 detected_language=st.session_state.get("pending_language", ""),266 suggested=ClassificationResult(**res),267 final_nature=res["nature_en"],268 final_sector=res["market_sector_en"],269 final_scope=res["scope_en"],270 approved_by=APPROVED_BY_DEFAULT,271 correction_note="",272 suggestion_source=source,273 approved_at=None,274 )275 request_single_form_reset("Classification approved and saved | تم اعتماد التصنيف وحفظه")276 st.rerun()277 278 if col_b.button("Edit | تعديل"):279 st.session_state["edit_single"] = True280 281 if st.session_state.get("edit_single"):282 st.markdown("#### Edit Approved Classification | تعديل واعتماد التصنيف")283 with st.form("single_edit_form"):284 nature = st.selectbox(285 "Nature | الطبيعة",286 list(ui_labels.NATURE_LABELS.keys()),287 index=list(ui_labels.NATURE_LABELS.keys()).index(res["nature_en"]) if res["nature_en"] in ui_labels.NATURE_LABELS else 0,288 format_func=lambda v: ui_labels.pair_label(v, ui_labels.NATURE_LABELS),289 )290 sector = st.selectbox(291 "Market Sector | قطاع السوق",292 list(ui_labels.SECTOR_LABELS.keys()),293 index=list(ui_labels.SECTOR_LABELS.keys()).index(res["market_sector_en"]) if res["market_sector_en"] in ui_labels.SECTOR_LABELS else 0,294 format_func=lambda v: ui_labels.pair_label(v, ui_labels.SECTOR_LABELS),295 )296 scope = st.selectbox(297 "Scope | النطاق",298 list(ui_labels.SCOPE_LABELS.keys()),299 index=list(ui_labels.SCOPE_LABELS.keys()).index(res["scope_en"]) if res["scope_en"] in ui_labels.SCOPE_LABELS else 0,300 format_func=lambda v: ui_labels.pair_label(v, ui_labels.SCOPE_LABELS),301 )302 approved_by = st.text_input("Approved by | المعتمد بواسطة", value=APPROVED_BY_DEFAULT)303 correction_note = st.text_area("Correction note | ملاحظة التعديل", value="")304 save = st.form_submit_button("Save Approved Correction | حفظ واعتماد التعديل")305 if save:306 save_approved_result(307 original_title=st.session_state.get("pending_title", title),308 detected_language=st.session_state.get("pending_language", ""),309 suggested=ClassificationResult(**res),310 final_nature=nature,311 final_sector=sector,312 final_scope=scope,313 approved_by=approved_by,314 correction_note=correction_note,315 suggestion_source=source,316 approved_at=None,317 )318 request_single_form_reset("Classification approved and saved | تم اعتماد التصنيف وحفظه")319 st.rerun()320 321with batch_tab:322 st.subheader("Excel Upload | رفع ملف إكسل")323 uploaded = st.file_uploader("Upload Excel", type=["xlsx"])324 if uploaded is not None:325 df = pd.read_excel(uploaded)326 required = {"id", "publisher_ar", "name_ar", "name_en"}327 missing = required - set(df.columns)328 if missing:329 st.error(f"Missing columns: {', '.join(sorted(missing))}")330 else:331 if st.button("Classify | تصنيف", key="batch_classify_btn"):332 refs = load_approved_references()333 st.session_state["batch_df"] = classify_dataframe(df, spec, refs)334 st.session_state["batch_source_df"] = df335 336 if "batch_df" in st.session_state:337 st.markdown("### Review Results | مراجعة النتائج")338 options_nature = list(ui_labels.NATURE_LABELS.keys())339 options_sector = list(ui_labels.SECTOR_LABELS.keys())340 options_scope = list(ui_labels.SCOPE_LABELS.keys())341 edited_df = st.data_editor(342 st.session_state["batch_df"],343 use_container_width=True,344 num_rows="dynamic",345 hide_index=True,346 column_config={347 "nature_en": st.column_config.SelectboxColumn(348 "Nature | الطبيعة",349 options=options_nature,350 required=True,351 format_func=lambda v: ui_labels.pair_label(v, ui_labels.NATURE_LABELS),352 ),353 "market_sector_en": st.column_config.SelectboxColumn(354 "Market Sector | قطاع السوق",355 options=options_sector,356 required=False,357 format_func=lambda v: ui_labels.pair_label(v, ui_labels.SECTOR_LABELS),358 ),359 "scope_en": st.column_config.SelectboxColumn(360 "Scope | النطاق",361 options=options_scope,362 required=False,363 format_func=lambda v: ui_labels.pair_label(v, ui_labels.SCOPE_LABELS),364 ),365 },366 key="batch_editor",367 )368 369 selected_ids = st.multiselect("Approve selected row IDs", options=edited_df["id"].astype(str).tolist())370 if st.button("Approve Selected Rows"):371 refs = load_approved_references()372 for _, row in edited_df[edited_df["id"].astype(str).isin(selected_ids)].iterrows():373 original = st.session_state["batch_source_df"][st.session_state["batch_source_df"]["id"] == row["id"]].iloc[0]374 bundle = workflow.classify_row_with_learning(original.to_dict(), spec, refs)375 save_approved_result(376 original_title=workflow.select_classification_inputs(str(original.get("name_ar", "") or original.get("name_en", "") or ""))[0]377 or str(original.get("name_ar", "") or original.get("name_en", "") or ""),378 detected_language=workflow.detect_input_language(str(original.get("name_ar", "") or original.get("name_en", "") or "")),379 suggested=bundle.result,380 final_nature=row.get("nature_en", ""),381 final_sector=row.get("market_sector_en", ""),382 final_scope=row.get("scope_en", ""),383 approved_by=APPROVED_BY_DEFAULT,384 correction_note=str(row.get("correction_note", "")) if "correction_note" in row else "",385 suggestion_source=bundle.suggestion_source,386 approved_at=None,387 )388 st.success("Selected rows approved. | تم اعتماد الصفوف المحددة")389 390 if st.button("Export Excel"):391 export_df = edited_df.copy()392 buf = BytesIO()393 with pd.ExcelWriter(buf, engine="openpyxl") as writer:394 export_df.to_excel(writer, index=False, sheet_name="classified")395 st.download_button(396 "Download exported file",397 data=buf.getvalue(),398 file_name="tce_classified.xlsx",399 mime="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",400 )401