Kagamicho/cs_chatbot
0
1#!/usr/bin/env python32"""3Developer convenience script — wipe local test data for a fresh evaluation run.4 5This script is DESTRUCTIVE. It deletes Firestore documents and local files.6It is idempotent (safe to run multiple times), but irreversible once applied.7 8Usage9-----10 # Dry-run (default) — shows what WOULD be deleted, touches nothing:11 uv run python scripts/reset_local_data.py --all12 13 # Apply a specific subset:14 uv run python scripts/reset_local_data.py --conversations --logs --apply15 16 # Apply everything without interactive prompt:17 uv run python scripts/reset_local_data.py --all --apply --yes18 19Flags20-----21 --conversations Delete all conversations/* docs and subcollections22 (messages, events, csat).23 --canned-replies Delete all cannedReplies/* docs.24 --saved-views Delete all savedViews/* docs and subcollections.25 --kb-feedback Delete all kbFeedback/* docs.26 --analytics-cache Delete analyticsCache/* (sentiment cache).27 --logs Delete data/logs/chat_*.jsonl files.28 --outbox Delete data/outbox/*.eml files (stub email outbox).29 --all Equivalent to passing all of the above flags.30 --apply Without this flag the script runs in dry-run mode only.31 --yes Skip the interactive confirmation prompt.32 33Both --apply AND --yes (or interactive confirmation) are required before any34destructive action is taken.35 36DO NOT call this from backend code or expose it via an API.37"""38from __future__ import annotations39 40import argparse41import os42import sys43from pathlib import Path44 45# ---------------------------------------------------------------------------46# Bootstrap — make the app package importable when run from scripts/47# ---------------------------------------------------------------------------48sys.path.insert(0, str(Path(__file__).resolve().parent.parent))49 50from app.firebase import get_db # noqa: E40251 52# ---------------------------------------------------------------------------53# Constants54# ---------------------------------------------------------------------------55 56SUBCOLLECTIONS_CONVERSATIONS = ("messages", "events", "csat")57SUBCOLLECTIONS_SAVED_VIEWS = ("views",)58 59 60# ---------------------------------------------------------------------------61# Helpers62# ---------------------------------------------------------------------------63 64def _delete_collection(65 db,66 collection_name: str,67 subcollections: tuple[str, ...],68 apply: bool,69 label: str,70) -> tuple[int, int]:71 """Delete (or count) all docs in a top-level collection + their subcollections.72 73 Returns (docs_deleted, subdocs_deleted).74 """75 docs = list(db.collection(collection_name).list_documents())76 doc_count = 077 subdoc_count = 078 79 for doc_ref in docs:80 # Subcollections first81 for sub in subcollections:82 sub_refs = list(doc_ref.collection(sub).list_documents())83 subdoc_count += len(sub_refs)84 if apply:85 for sub_ref in sub_refs:86 sub_ref.delete()87 88 doc_count += 189 if apply:90 doc_ref.delete()91 92 verb = "Deleted" if apply else "Would delete"93 if doc_count:94 sub_note = f" (+ {subdoc_count} subcollection docs)" if subcollections else ""95 print(f" {verb} {doc_count} {label} docs{sub_note}")96 else:97 print(f" {label}: nothing to delete")98 99 return doc_count, subdoc_count100 101 102def _delete_files(pattern: str, base_dir: Path, apply: bool, label: str) -> int:103 """Delete (or count) files matching a glob pattern. Returns file count."""104 files = sorted(base_dir.glob(pattern))105 if not files:106 print(f" {label}: no files found")107 return 0108 109 verb = "Deleted" if apply else "Would delete"110 total_lines = 0111 for f in files:112 try:113 lines = sum(1 for _ in f.open(encoding="utf-8", errors="replace"))114 except Exception:115 lines = 0116 total_lines += lines117 if apply:118 f.unlink()119 120 print(f" {verb} {len(files)} {label} files ({total_lines} total lines/records)")121 return len(files)122 123 124# ---------------------------------------------------------------------------125# Main126# ---------------------------------------------------------------------------127 128def main() -> None:129 parser = argparse.ArgumentParser(130 description="Reset local dev/test data in Firestore and on disk.",131 formatter_class=argparse.RawDescriptionHelpFormatter,132 )133 134 # Target flags135 parser.add_argument("--conversations", action="store_true",136 help="Delete all conversations + subcollections (messages, events, csat)")137 parser.add_argument("--canned-replies", action="store_true",138 help="Delete all cannedReplies docs")139 parser.add_argument("--saved-views", action="store_true",140 help="Delete all savedViews docs + subcollections")141 parser.add_argument("--kb-feedback", action="store_true",142 help="Delete all kbFeedback docs")143 parser.add_argument("--analytics-cache", action="store_true",144 help="Delete analyticsCache docs (sentiment cache)")145 parser.add_argument("--logs", action="store_true",146 help="Delete data/logs/chat_*.jsonl files")147 parser.add_argument("--outbox", action="store_true",148 help="Delete data/outbox/*.eml files")149 parser.add_argument("--all", action="store_true",150 help="Equivalent to passing all target flags")151 152 # Safety flags153 parser.add_argument("--apply", action="store_true",154 help="Actually delete. Without this flag runs in dry-run mode.")155 parser.add_argument("--yes", action="store_true",156 help="Skip the interactive confirmation prompt.")157 158 args = parser.parse_args()159 160 # Expand --all161 if args.all:162 args.conversations = True163 args.canned_replies = True164 args.saved_views = True165 args.kb_feedback = True166 args.analytics_cache = True167 args.logs = True168 args.outbox = True169 170 # Normalise hyphenated attrs171 do_conversations = args.conversations172 do_canned_replies = getattr(args, "canned_replies", False) or getattr(args, "canned-replies", False)173 do_saved_views = getattr(args, "saved_views", False) or getattr(args, "saved-views", False)174 do_kb_feedback = getattr(args, "kb_feedback", False) or getattr(args, "kb-feedback", False)175 do_analytics_cache = getattr(args, "analytics_cache", False) or getattr(args, "analytics-cache", False)176 do_logs = args.logs177 do_outbox = args.outbox178 179 any_selected = any([180 do_conversations, do_canned_replies, do_saved_views,181 do_kb_feedback, do_analytics_cache, do_logs, do_outbox,182 ])183 184 if not any_selected:185 parser.print_help()186 sys.exit(0)187 188 dry_run = not args.apply189 mode_label = "DRY RUN" if dry_run else "APPLY"190 191 print(f"\n[{mode_label}] Reset scope:")192 if do_conversations:193 print(" - conversations (+ messages, events, csat subcollections)")194 if do_canned_replies:195 print(" - cannedReplies")196 if do_saved_views:197 print(" - savedViews (+ views subcollections)")198 if do_kb_feedback:199 print(" - kbFeedback")200 if do_analytics_cache:201 print(" - analyticsCache")202 if do_logs:203 print(" - data/logs/chat_*.jsonl")204 if do_outbox:205 print(" - data/outbox/*.eml")206 207 if dry_run:208 print("\nDry-run mode. Pass --apply to actually delete.\n")209 210 # -----------------------------------------------------------------------211 # Interactive confirmation (only needed when --apply)212 # -----------------------------------------------------------------------213 if not dry_run and not args.yes:214 print('\nThis will PERMANENTLY DELETE data. Type "delete" to confirm (or Ctrl-C to abort):')215 answer = input("> ").strip()216 if answer != "delete":217 print("Aborted — nothing deleted.")218 sys.exit(0)219 220 # -----------------------------------------------------------------------221 # Connect to Firestore222 # -----------------------------------------------------------------------223 needs_firestore = any([224 do_conversations, do_canned_replies, do_saved_views,225 do_kb_feedback, do_analytics_cache,226 ])227 228 db = None229 if needs_firestore:230 print(f"\n[{mode_label}] Connecting to Firestore…")231 db = get_db()232 233 # -----------------------------------------------------------------------234 # Project root (for file operations)235 # -----------------------------------------------------------------------236 project_root = Path(__file__).resolve().parent.parent237 238 # -----------------------------------------------------------------------239 # Execute deletions240 # -----------------------------------------------------------------------241 total_conversations = 0242 total_messages = 0243 total_events = 0244 total_csat = 0245 total_canned = 0246 total_views_docs = 0247 total_kb = 0248 total_cache = 0249 total_log_files = 0250 total_eml_files = 0251 252 print(f"\n[{mode_label}] Running…\n")253 254 if do_conversations and db is not None:255 # Count subcollections individually for the summary256 conv_docs = list(db.collection("conversations").list_documents())257 for conv_ref in conv_docs:258 for sub, counter_list in [259 ("messages", []),260 ("events", []),261 ("csat", []),262 ]:263 sub_refs = list(conv_ref.collection(sub).list_documents())264 if sub == "messages":265 total_messages += len(sub_refs)266 elif sub == "events":267 total_events += len(sub_refs)268 elif sub == "csat":269 total_csat += len(sub_refs)270 if not dry_run:271 for sr in sub_refs:272 sr.delete()273 if not dry_run:274 conv_ref.delete()275 total_conversations = len(conv_docs)276 verb = "Deleted" if not dry_run else "Would delete"277 msg_note = f"({total_messages} messages, {total_events} events, {total_csat} csat)"278 if total_conversations:279 print(f" {verb} {total_conversations} conversations {msg_note}")280 else:281 print(" conversations: nothing to delete")282 283 if do_canned_replies and db is not None:284 n, _ = _delete_collection(db, "cannedReplies", (), not dry_run, "cannedReplies")285 total_canned = n286 287 if do_saved_views and db is not None:288 # savedViews/{uid}/views/{id} — nested two levels289 owner_docs = list(db.collection("savedViews").list_documents())290 sv_doc_count = 0291 sv_subdoc_count = 0292 for owner_ref in owner_docs:293 sub_refs = list(owner_ref.collection("views").list_documents())294 sv_subdoc_count += len(sub_refs)295 if not dry_run:296 for sr in sub_refs:297 sr.delete()298 sv_doc_count += 1299 if not dry_run:300 owner_ref.delete()301 total_views_docs = sv_doc_count + sv_subdoc_count302 verb = "Deleted" if not dry_run else "Would delete"303 if sv_doc_count:304 print(f" {verb} {sv_doc_count} savedViews owner docs (+ {sv_subdoc_count} view docs)")305 else:306 print(" savedViews: nothing to delete")307 308 if do_kb_feedback and db is not None:309 n, _ = _delete_collection(db, "kbFeedback", (), not dry_run, "kbFeedback")310 total_kb = n311 312 if do_analytics_cache and db is not None:313 n, _ = _delete_collection(db, "analyticsCache", (), not dry_run, "analyticsCache")314 total_cache = n315 316 if do_logs:317 log_dir = Path(os.environ.get("CHAT_LOG_DIR", "data/logs"))318 if not log_dir.is_absolute():319 log_dir = project_root / log_dir320 total_log_files = _delete_files("chat_*.jsonl", log_dir, not dry_run, "log")321 322 if do_outbox:323 outbox_dir = project_root / "data" / "outbox"324 if outbox_dir.exists():325 total_eml_files = _delete_files("*.eml", outbox_dir, not dry_run, "outbox .eml")326 else:327 print(" outbox: directory not found — skipping")328 329 # -----------------------------------------------------------------------330 # Summary331 # -----------------------------------------------------------------------332 print(f"\n[{mode_label}] Summary:")333 if not dry_run:334 print(335 f" Deleted: {total_conversations} conversations, "336 f"{total_messages} messages, "337 f"{total_events} events, "338 f"{total_csat} csat, "339 f"{total_canned} canned replies, "340 f"{total_views_docs} savedView docs, "341 f"{total_kb} kbFeedback, "342 f"{total_cache} cache docs, "343 f"{total_log_files} .jsonl files, "344 f"{total_eml_files} .eml files"345 )346 else:347 print(" Dry-run complete — nothing was deleted.")348 print(" Re-run with --apply to execute.")349 350 351if __name__ == "__main__":352 main()353