Anonymized1/Interacting-With-LLMs
0
1"""2Privacy Risk Assessment UI for LLM Interactions3 4URL parameters:5 model – LLM model name6 rag – 0 or 1 (enable External Data linkage)7 epsilon – float (DP privacy budget, shared by both mechanisms)8 show_risk – 0 or 1 (risk panel)9 show_tips – 0 or 1 (PII tooltips)10 show_rag_highlights – 0 or 1 (External Data link highlights)11 show_pii_highlights – 0 or 1 (PII highlights)12 show_settings – 0 or 1 (settings panel)13 demo – 0 or 1 (demo mode, NO LLM calls)14 enable_social_scraping – 0 or 1 (enable social media scraping - experimental)15 token – str (access token for the session)16 show_dp – int: when to show the Privacy Settings panel17 0 = never18 1 = after each conversation turn (default)19 2 = at the beginning only (before first turn)20 3 = only when the user presses "End conversation"21 show_infr_attr_card – int: when to show the "How the AI sees you" card22 0 = never23 1 = after each conversation turn (default)24 2 = only when the user presses "End conversation"25 26 27"""28 29import copy30import pandas as pd31import json32import math33import os34import pickle35import random36import re37import time38from dataclasses import dataclass, field39from enum import Enum40from collections import defaultdict41import threading42import csv43from concurrent.futures import ThreadPoolExecutor, as_completed44import logging45import sys46import datetime47import gradio as gr48from openai import OpenAI49from langchain_community.embeddings import HuggingFaceEmbeddings50from create_rag_retriever import load_retriever_components, DPRetriever, HybridRetriever51import numpy as np52import hashlib53from langchain_core.documents import Document54 55# Import Apify social media scraper56from social_scrape_via_apify import (57 get_twitter_user_posts,58 get_facebook_posts,59 get_facebook_page_info,60 get_linkedin_posts,61 get_web_search_results62)63 64# Configure logging to write to BOTH file and console65 66logging.basicConfig(67 level=logging.INFO,68 format='%(asctime)s | %(levelname)s | %(message)s',69 datefmt='%H:%M:%S',70 handlers=[71 logging.StreamHandler(sys.stdout)72 ]73)74 75logger = logging.getLogger(__name__)76logger.info("="*60)77 78 79# ============================================================80# CONFIGURATION81# All tuneable constants are grouped here for easy maintenance.82# ============================================================83 84PROLIFIC_ID_PATTERN_REGEX = r'[A-Za-z0-9]{24}'85 86# ── Access Control ───────────────────────────────────────────87# Valid access tokens (store in HF Secrets in production).88# Each entry maps a token (read from an env var) to metadata.89VALID_ACCESS_TOKENS = {90 os.environ.get("VALID_ACCESS_TOKEN_1", None): {"name": "", "expires": "2099-12-31"},91 os.environ.get("VALID_ACCESS_TOKEN_2", None): {"name": "", "expires": "2099-12-31"},92 os.environ.get("VALID_ACCESS_TOKEN_3", None): {"name": "", "expires": "2099-12-31"},93}94 95# ── Logging & HF Hub ─────────────────────────────────────────96INTERACTION_LOG_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "interaction_log.csv")97HF_LOG_REPO_ID = "michael2222222/app-logs" # HF dataset repo name98HF_LOG_REPO_TYPE = "dataset"99 100# Column headers for the per-participant CSV log file.101LOG_COLUMNS = [102 "timestamp",103 "session_source",104 "turn_number",105 "demo_mode",106 "scenario_mode",107 "persona_attributes",108 "model",109 "epsilon",110 "rag_enabled",111 "show_risk",112 "show_rag_highlights",113 "show_tips",114 "show_pii_highlights",115 "show_settings",116 "show_dp",117 "show_infr_attr_card",118 "show_social_scraping",119 "show_upload_data",120 "rag_corpus_path",121 "access_token",122 "social_scraping_enabled",123 "corpus_source",124 "uploaded_file_path",125 "uploaded_file_preview",126 "user_prompt",127 "user_prompt_perturbed",128 "num_input_dp_substitutions",129 "user_prompt_length",130 "user_perturbed_prompt_length",131 "llm_response",132 "llm_response_length",133 "risk_score",134 "rag_user_count",135 "rag_linkages_user",136 "rag_llm_count",137 "rag_linkages_llm",138 "pii_user_count",139 "pii_detected_user",140 "pii_user_perturbed_count",141 "pii_detected_user_perturbed",142 "pii_llm_count",143 "pii_detected_llm",144 "inference_warning_shown",145 "inference_panel_updated",146 "num_attributes_changed",147 "inference_lifts",148 "inferential_score",149 "inferential_score_breakdown",150 "scraped_docs",151 "scraped_social_summary",152]153 154# ── LLM Models ───────────────────────────────────────────────155# Uncomment / comment entries to enable or disable models.156MODEL_CONFIGS = [157 # ("together", "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", "Llama-4"),158 # ("together", "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", "Llama-3.1"),159 # ("openai", "gpt-4o", "GPT-4o"),160 # ("openai", "gpt-5-mini-2025-08-07", "GPT-5-mini"),161 # ("together", "openai/gpt-oss-120b", "GPT-OSS"),162 # ("together", "meta-llama/Llama-3.3-70B-Instruct-Turbo", "Llama-3.3-70B"),163 # ("together", "deepseek-ai/DeepSeek-V3.1", "DeepSeek-V3.1"),164 ("gemini", "gemini-2.5-flash-lite", "Gemini-2.5-Flash"),165]166DEFAULT_MODEL_PROVIDER = MODEL_CONFIGS[0][0]167DEFAULT_MODEL_ID = MODEL_CONFIGS[0][1]168DEFAULT_MODEL_NAME = MODEL_CONFIGS[0][2]169 170# ── LLM Inference Parameters ─────────────────────────────────171MAX_TOKENS_PII_DETECTION = 1400172MAX_TOKENS_LLM_RESPONSE_GENERATION = 260173MAX_TOKENS_INFER_ATTRIBUTES = 1400174TEMPERATURE_LLM_RESPONSE_GENERATION = 0.3175TEMPERATURE_INTERNAL_TASKS = 0176 177# ── RAG & Retrieval ──────────────────────────────────────────178EMBEDDING_MODEL = "sentence-transformers/all-MiniLM-L6-v2"179DEFAULT_RETRIEVAL_PICKLE_PATH = "./faiss_panorama_retriever_components.pkl"180 181# Per-scenario system corpus pickle paths.182# Set a value to None to fall back to the default RETRIEVAL_PICKLE_PATH.183SCENARIO_RETRIEVER_PATHS = {184 "real": DEFAULT_RETRIEVAL_PICKLE_PATH,185 "persona2": "./faiss_persona_sarah_chen_retriever_components.pkl",186 # Add more scenarios here following the same pattern.187}188 189# Base directory used for relative file paths (e.g. fallback tweet JSON files).190_BASE_DIR = os.path.dirname(os.path.abspath(__file__))191 192# Per-scenario fallback tweet data (used when live scraping fails / returns empty).193# Each scenario_mode maps to its own JSON file of fallback tweets.194SCENARIO_FALLBACK_TWEET_PATHS = {195 "real": None, # no fallback for free-style196 "persona": None,197 "persona2": os.path.join(_BASE_DIR, "sarah_chen_tweets_scraped.json"),198 # Add more scenarios here following the same pattern.199}200 201# Default fallback tweet dataset (empty; populated per-scenario at runtime).202FALLBACK_TWEET_DATA = []203 204RAG_K = 50205RAG_FETCH_K = 10000206RAG_LINKAGE_STORED_EXCERPT_LENGTH = 500207ATTR_INFERENCE_EVIDENCE_STORED_EXCERPT_LENGTH = 500208 209# ── RAG Linkage Similarity Thresholds ────────────────────────210ROUGE_L_THRESHOLD = 0.16 # Minimum ROUGE-L score (0.0-1.0)211COSINE_SIM_THRESHOLD = 0.16 # Minimum cosine similarity (0.0-1.0)212MIN_COMBINED_SCORE = 0.2 # Minimum combined similarity score213ROUGE_WEIGHT = 0.3 # Weight for ROUGE-L in combined score214COSINE_WEIGHT = 0.7 # Weight for cosine similarity in combined score215MIN_NGRAM_LENGTH = 4 # Minimum characters for n-gram matching216MIN_NGRAM_WORDS = 4 # Minimum words in a phrase to highlight217MAX_NGRAM_WORDS = 7 # Maximum words in a phrase to highlight218 219RAG_MIN_SIMILARITY_THRESHOLD = 0.25 # baseline threshold when DP is off220RAG_DP_THRESHOLD_SCALE = 1.5 # how fast the threshold rises as ε falls221 222# ── Privacy / Differential Privacy ───────────────────────────223MAX_POSSIBLE_EPS = 100.0224INFERENCE_LIFT_THRESHOLD = 0225 226DP_FLOOR = 0.25 # minimum retained fraction even under maximum perturbation227# ── Tunable constants ─────────────────────────────────────────────────────228INFER_MAX = 55.0 # hard cap on inference contribution229INFER_SCALE = 4.0 # controls how fast the curve saturates230PII_MAX = 40.0 # cap on the inference contribution231PII_BASE = 15.0 # flat baseline added whenever ≥1 PII is detected232PII_PER_HIT = 4.0 # points per detected PII entity (on top of baseline)233RAG_MAX = 15.0 # cap on RAG-linkage contribution234RAG_PER_HIT = 0.75 # points per RAG linkage235 236# ── UI & App Defaults ─────────────────────────────────────────237DEFAULT_USER_TEXT = (238 "Hi, I am Sarah Chen. I want to take a more preventive approach to my health. "239 "What routine tests or screenings should I consider that I could afford, and "240 "what is the nearest location where I can undergo these tests?"241)242 243# Human-readable label for documents retrieved from the default background corpus.244DEFAULT_CORPUS_SOURCE_LABEL = "From Social Profiles (Internet)"245 246# ── Locale & Attribute Mappings ───────────────────────────────247# Note: ATTRIBUTE_VALUES_MAP references LOCALE_TO_LOCATION and must come after it.248# PII_TYPE_TO_CATEGORY and PII_COLORS are defined later (in SECTION 1 / SECTION 2)249# because they depend on the PIICategory enum.250LOCALE_TO_LOCATION = {251 "en_PH": "Philippines",252 "en_CA": "Canada",253 "en_US": "United States",254 "en_IE": "Ireland",255 "en_NZ": "New Zealand",256 "en_IN": "India",257 "en_AU": "Australia",258 "en_GB": "United Kingdom",259 "en_IL": "Israel",260 "en_DE": "Germany",261 "en_IT": "Italy",262 "en_FR": "France",263}264 265SENSITIVE_ATTRIBUTES = ["Age bin", "Gender", "Marital Status", "Finance Status", "Education", "Locale"]266 267ATTRIBUTE_VALUES_MAP = {268 "Gender": ["Female", "Male"],269 "Age bin": ["0-17", "18-29", "30-44", "45-59", "60+"],270 "Marital Status": ["Single", "Married", "Divorced", "Widowed"],271 "Finance Status": ["Low", "Medium", "High"],272 "Locale": [LOCALE_TO_LOCATION["en_PH"], LOCALE_TO_LOCATION["en_CA"], LOCALE_TO_LOCATION["en_US"],273 LOCALE_TO_LOCATION["en_IE"], LOCALE_TO_LOCATION["en_NZ"], LOCALE_TO_LOCATION["en_IN"],274 LOCALE_TO_LOCATION["en_AU"], LOCALE_TO_LOCATION["en_GB"], LOCALE_TO_LOCATION["en_IL"],275 LOCALE_TO_LOCATION["en_DE"], LOCALE_TO_LOCATION["en_IT"], LOCALE_TO_LOCATION["en_FR"]],276 "Education": ["High School", "Bachelor's", "Master's", "PhD"]277}278 279# ── Population Priors for Inferential Privacy ────────────────280# Baseline population probabilities for each attribute value, used as the281# denominator in the inferential-privacy lift calculation. The lift measures282# how much the system's available information (message text and/or RAG283# context) raises the probability of inferring an attribute above this284# base rate.285#286# Values within each attribute should sum to ≈1.0. Calibrate these to287# the study's participant pool or the general population.288POPULATION_PRIORS = {289 "Gender": {290 "Female": 0.50,291 "Male": 0.50,292 },293 "Age bin": {294 "0-17": 0.15,295 "18-29": 0.15,296 "30-44": 0.38,297 "45-59": 0.15,298 "60+": 0.15,299 },300 "Marital Status": {301 "Single": 0.39,302 "Married": 0.51,303 "Divorced": 0.05,304 "Widowed": 0.05,305 },306 "Finance Status": {307 "Low": 0.23,308 "Medium": 0.53,309 "High": 0.23,310 },311 "Education": {312 "High School": 0.19,313 "Bachelor's": 0.43,314 "Master's": 0.19,315 "PhD": 0.19,316 },317 "Locale": {318 "Philippines": 0.09,319 "Canada": 0.05,320 "United States": 0.40,321 "Ireland": 0.05,322 "New Zealand": 0.05,323 "India": 0.05,324 "Australia": 0.05,325 "United Kingdom": 0.05,326 "Israel": 0.05,327 "Germany": 0.05,328 "Italy": 0.05,329 "France": 0.05,330 },331}332 333# Steepness parameter for the log-based inferential privacy saturation334# function: f(x; k) = k·ln(1+x) / (1 + k·ln(1+x)).335# Higher k → the curve reaches closer to 1 faster for the same lift.336INFERENTIAL_SCORE_STEEPNESS_K = 3.0337 338# ── Demo Prompts ──────────────────────────────────────────────339DATASET_PROMPTS_PANORAMA = {340 "health": (341 "Hi, I am {name}. I want to take a more preventive approach to my "342 "health. What routine tests or screenings should I consider that I "343 "could afford, and what is the nearest location where I can undergo "344 "these tests?"345 ),346 "job": (347 "I am {name}. I am searching for new job. Do you have any "348 "suggestions that would work for me?"349 ),350 "general": (351 "Hi, I'm {name}. Can you tell me about hobbies I might be "352 "interested?"353 ),354}355 356# ── Personas ──────────────────────────────────────────────────357# "real" means the participant uses their own profile (free-style).358# Each "personaX" entry contains the ground-truth attributes used in the study359# and a placeholder prompt shown in the text box when the app loads.360PERSONAS = {361 "real": {362 "name": "",363 "description": None, # no placeholder injected364 "attributes": {},365 },366 "persona1": {367 "name": "Raymond Phillips",368 "description": (369 "You are Raymond Phillips, a 52-year-old widowed male from the Philippines. "370 "You work as a fisherman and have a low income. "371 "Write your messages as Raymond would."372 ),373 "attributes": {374 "Gender": "Male",375 "Age bin": "45-59",376 "Marital Status": "Widowed",377 "Finance Status": "Low",378 "Locale": "Philippines",379 },380 },381 "persona2": {382 "name": "Sarah Chen",383 "description": (384 """385 You will take on the role of Sarah Chen, a 38-year-old nurse. <br>386 Sarah recently moved from Seattle to Austin for a new ICU nursing job at St. David's Medical Center. Her spouse, Daniel, is still in Seattle finishing up his own job. 387 She has been struggling with anxiety and sleep problems since the move, and her doctor back in Seattle had prescribed her Lexapro, which she has been taking for about two years. She hasn't found a new doctor in Austin yet. 388 With long shifts, an empty apartment, and mounting credit card debt from the move, she has decided it's time to find a therapist or psychiatrist in Austin who accepts her Blue Shield insurance. <br>389 <br>390 She plans to ask for recommendations in a local community forum for nurses.391 """392 ),393 "attributes": {394 "Gender": "Female",395 "Age bin": "30-44",396 "Marital Status": "Married",397 "Finance Status": "Medium",398 "Locale": "United States",399 "Education": "Bachelor\'s"400 },401 },402}403 404# ============================================================405# END OF CONFIGURATION406# ============================================================407 408 409def _extract_twitter_username(url):410 """Extract Twitter/X username from a tweet URL.411 412 e.g. https://x.com/realDonaldTrump/status/123 → 'realDonaldTrump'413 """414 try:415 from urllib.parse import urlparse416 path = urlparse(url).path # '/realDonaldTrump/status/123'417 parts = path.strip("/").split("/") # ['realDonaldTrump', 'status', '123']418 if parts and parts[0] not in ("", "search", "i", "intent", "hashtag"):419 return parts[0]420 except Exception:421 pass422 return ""423 424 425def _tweet_items_to_documents(tweet_items, fallback=False):426 """Convert raw tweet JSON items to langchain Documents.427 428 Each document's page_content is prefixed with the Twitter username so that429 it surfaces naturally in RAG linkage tooltips and evidence attribution.430 431 Args:432 tweet_items: list of dicts with at least 'url' and 'text' keys433 fallback: True when these come from the fallback dataset (affects log only)434 435 Returns:436 list of Document objects437 """438 from langchain_core.documents import Document439 documents = []440 source_tag = "fallback_tweet_data" if fallback else "social_media_twitter"441 442 for idx, item in enumerate(tweet_items):443 if not isinstance(item, dict):444 continue445 raw_text = item.get("text", "").strip()446 if not raw_text:447 continue448 449 url = item.get("url", "") or item.get("twitterUrl", "")450 username = _extract_twitter_username(url)451 452 # Prefix every post with the author so it appears in RAG context453 content = f"[{username}] {raw_text}" if username else raw_text454 455 doc = Document(456 page_content=content,457 metadata={458 "source": source_tag,459 "twitter_username": username,460 "full_name": username, # used by RAG source tooltip461 "platform": "twitter",462 "type": "social_media_scrape",463 "post_index": idx,464 "tweet_url": url,465 "created_at": item.get("createdAt", ""),466 "is_retweet": item.get("isRetweet", False),467 }468 )469 documents.append(doc)470 471 label = "fallback" if fallback else "live"472 logger.info(f" ✓ Converted {len(documents)} {label} tweets to Documents")473 return documents474 475# ============================================================476# SECTION 2.5 – INTERACTION LOGGER477# ============================================================478 479logger.info(f"Interaction will be recorded to file at: {INTERACTION_LOG_PATH}")480 481 482# Known API error prefixes returned by call_generate_response's except clause483_LLM_ERROR_PREFIXES = (484 "Error:",485 "You exceeded your current quota",486 "Rate limit",487 "insufficient_quota",488 "Connection error",489 "Timeout",490 "APIError",491 "AuthenticationError",492)493 494 495def _is_llm_error(response_text):496 """Return True when the LLM response string is actually an error message."""497 if not response_text:498 return True499 t = response_text.strip()500 return any(t.startswith(p) for p in _LLM_ERROR_PREFIXES)501 502def _participant_id(access_token, session_source):503 """Return a safe filename-compatible participant identifier.504 Prefers access_token; falls back to a short hash of session_source."""505 raw = (access_token or "").strip()506 if not raw:507 raw = "anon_" + hashlib.sha1((session_source or "").encode()).hexdigest()[:10]508 # Strip anything that is not alphanumeric, dash, or underscore509 return re.sub(r"[^a-zA-Z0-9_\-]", "_", raw)[:64]510 511 512def _participant_log_path(participant_id):513 """Local filesystem path for this participant's CSV."""514 log_dir = os.path.dirname(INTERACTION_LOG_PATH)515 return os.path.join(log_dir, f"log_{participant_id}.csv")516 517 518def _get_file_lock(participant_id):519 """Return (creating if needed) a per-participant threading lock."""520 with _log_lock:521 if participant_id not in _log_locks:522 _log_locks[participant_id] = threading.Lock()523 return _log_locks[participant_id]524 525# ── Add near the top with other imports ────────────────────────────────526from huggingface_hub import HfApi527 528 529def _push_log_to_hub(local_path, repo_filename):530 """Push a single participant CSV to the HF dataset repo in a background thread.531 532 The actual upload is dispatched to a daemon thread so this function returns533 immediately and never blocks the HTTP response path. Failures are logged534 but otherwise swallowed — logging is best-effort.535 """536 def _do_upload():537 try:538 from huggingface_hub import HfApi539 token = os.environ.get("HF_TOKEN", None)540 if not token or not os.path.exists(local_path):541 return542 HfApi(token=token).upload_file(543 path_or_fileobj=local_path,544 path_in_repo=f"logs/{repo_filename}",545 repo_id=HF_LOG_REPO_ID,546 repo_type=HF_LOG_REPO_TYPE,547 commit_message=f"log update: {repo_filename}",548 )549 logger.info("📤 Log pushed to HF Hub: logs/%s", repo_filename)550 except Exception as e:551 logger.warning("Log push to Hub failed (non-fatal): %s", e)552 553 threading.Thread(target=_do_upload, daemon=True).start()554 555_LOG_FIELDS = [556 "timestamp",557 "session_source",558 "turn_number",559 "demo_mode",560 "scenario_mode", # "real" | "persona1" | "persona2" | …561 "persona_attributes", # JSON dict of persona ground-truth attrs, or "{}"562 "model",563 "epsilon",564 "rag_enabled",565 "show_risk",566 "show_rag_highlights",567 "show_tips",568 "show_pii_highlights",569 "show_settings",570 "access_token",571 "social_scraping_enabled",572 "corpus_source",573 "uploaded_file_path",574 "uploaded_file_preview", # first 100 000 chars of the uploaded CSV575 "user_prompt",576 "user_prompt_perturbed", # DP-perturbed version sent to LLM (same as user_prompt when DP off)577 "num_input_dp_substitutions", # number of words perturbed by Input DP (0 when DP off)578 "user_prompt_length", # character length of user_prompt579 "llm_response",580 "llm_response_length", # character length of llm_response581 "risk_score",582 # ── RAG: split by user input vs LLM response ──────────────583 "num_rag_links_user",584 "rag_linkages_user", # JSON: [{linked_text, source, similarity, ...}]585 "num_rag_links_llm",586 "rag_linkages_llm", # JSON: [{linked_text, source, similarity, ...}]587 # ── PII: split by user input vs LLM response ──────────────588 "num_pii_detected_user",589 "pii_detected_user", # JSON: [{text, type, confidence}]590 "num_pii_detected_llm",591 "pii_detected_llm", # JSON: [{text, type, confidence}]592 # ── Inference ─────────────────────────────────────────────593 "inference_warning_shown", # bool – was the warning banner displayed?594 "inference_lifts", # JSON: {attr: {top_value, confidence, lift,595 # prob_rag, prob_no_rag, evidence_type}}596 "inferential_score", # float [0,1): 1−exp(−max_lift)597 "inferential_score_breakdown", # JSON: {score, max_lift, max_attr, p_post, p_pop, p_no_rag}598 # ── Social scraping ───────────────────────────────────────599 "scraped_social_data", # JSON: list of scraped post objects (if enabled)600 "scraped_social_summary", # JSON: per-platform summary [{platform, post_count, posts:[{text,url}]}]601]602 603_log_lock = threading.Lock() # guards _log_locks dict itself604_log_locks = {} # per-participant file locks605 606 607def _ensure_log_file():608 """Create the CSV with header if it does not exist yet. Safe for concurrent startup."""609 if not os.path.exists(INTERACTION_LOG_PATH):610 with _log_lock:611 if not os.path.exists(INTERACTION_LOG_PATH): # double-check after acquiring612 with open(INTERACTION_LOG_PATH, "w", newline="", encoding="utf-8") as f:613 csv.writer(f).writerow(_LOG_FIELDS)614 logger.info(f"📋 Interaction log created: {INTERACTION_LOG_PATH}")615 616 617def _sanitize_cell(text):618 """Replace raw newlines and tabs in free-text fields so they cannot619 break CSV row boundaries, while remaining human-readable in Excel."""620 if not text:621 return text or ""622 return (623 str(text)624 .replace("\r\n", " \\n ")625 .replace("\r", " \\n ")626 .replace("\n", " \\n ")627 .replace("\t", " \\t ")628 )629 630def append_interaction_log(631 session_source,632 turn_number,633 demo_mode,634 scenario_mode,635 persona_attributes,636 model,637 epsilon,638 rag_enabled,639 show_risk,640 show_rag_highlights,641 show_tips,642 show_pii_highlights,643 show_settings,644 access_token,645 social_scraping_enabled,646 corpus_source,647 uploaded_file_path,648 user_prompt,649 llm_response,650 risk_score,651 u_rag, # RAGLink list for user input only652 r_rag, # RAGLink list for LLM response only653 u_pii, # PIIMatch list for user input only654 r_pii, # PIIMatch list for LLM response only655 u_pii_perturbed,656 inference_metrics,657 inference_warning_shown,658 scraped_docs=None, # list of Document objects from social scraping659 user_prompt_perturbed=None, # DP-perturbed version sent to LLM660 num_input_dp_substitutions=0, # number of words perturbed by Input DP661 num_attributes_changed=0,662 show_dp=1,663 show_infr_attr_card=1,664 show_social_scraping=False,665 show_upload_data=False,666 rag_corpus_path="",667):668 """Append one interaction row to the persistent CSV log."""669 try:670 # ── Resolve participant identity and paths ────────────671 pid = _participant_id(access_token, session_source)672 log_path = _participant_log_path(pid)673 log_file = f"log_{pid}.csv"674 file_lock = _get_file_lock(pid)675 676 # ── Uploaded file info ────────────────────────────────677 uploaded_file_preview = ""678 if uploaded_file_path:679 try:680 with open(uploaded_file_path, "r", encoding="utf-8", errors="replace") as _f:681 raw_preview = _f.read(100000)682 # json.dumps produces a single-line string with all special683 # characters escaped (\n, \r, \", \t, etc.), making it684 # completely safe to embed in any CSV cell.685 uploaded_file_preview = json.dumps(raw_preview)686 except Exception:687 uploaded_file_preview = "<unreadable>"688 689 # Create the file with headers if it does not exist yet690 if not os.path.exists(log_path):691 os.makedirs(os.path.dirname(log_path), exist_ok=True)692 with file_lock:693 if not os.path.exists(log_path): # double-check after acquiring lock694 with open(log_path, "w", newline="", encoding="utf-8") as f:695 csv.writer(f, quoting=csv.QUOTE_ALL).writerow(LOG_COLUMNS) # header list696 697 # ── RAG linkages – user input ─────────────────────────698 def _serialise_rag(rag_list):699 return json.dumps([700 {701 "linked_text": lk.text,702 "source": getattr(lk, "source", ""),703 "similarity": round(float(lk.top_similarity), 4),704 "start": lk.start,705 "end": lk.end,706 "corpus_snippets": lk.corpus_snippets,707 "top_doc_score": lk.top_doc_score,708 "overlap_keywords": lk.overlap_keywords,709 }710 for lk in (rag_list or [])711 ], ensure_ascii=False)712 713 rag_linkages_user = _serialise_rag(u_rag)714 rag_linkages_llm = _serialise_rag(r_rag)715 716 # ── PII detected – split by source ───────────────────717 def _serialise_pii(pii_list):718 rows = []719 for m in (pii_list or []):720 if isinstance(m, dict):721 rows.append({722 "text": m.get("value", m.get("text", "")),723 "type": m.get("type", "unknown"),724 "confidence": round(float(m.get("confidence", 0.0)), 4),725 })726 else:727 rows.append({728 "text": m.text,729 "type": m.fine_type,730 "confidence": round(float(m.confidence), 4),731 })732 return json.dumps(rows, ensure_ascii=False)733 734 pii_detected_user = _serialise_pii(u_pii)735 pii_detected_llm = _serialise_pii(r_pii)736 737 # ── Inference lifts ───────────────────────────────────738 lifts = {}739 for attr, m in (inference_metrics or {}).items():740 lift = m.get("lift", 0)741 if not (math.isinf(lift) or math.isnan(lift)):742 lifts[attr] = {743 "top_value": m.get("top_value", ""),744 "confidence": round(float(m.get("confidence", m.get("probability", 0))), 4),745 "lift": round(float(lift), 4),746 "prob_rag": round(float(m.get("prob_rag", 0)), 4),747 "prob_no_rag": round(float(m.get("prob_no_rag", 0)), 4),748 "p_pop": round(float(m.get("p_pop", 0)), 4),749 "evidence_type": m.get("evidence_type", "unknown"),750 }751 lifts_json = json.dumps(lifts, ensure_ascii=False)752 753 # ── Inferential privacy score and breakdown ───────────754 infer_score_info = calculate_inferential_privacy_score(inference_metrics or {})755 inferential_score = round(float(infer_score_info.get("mean_score", 0.0)), 4)756 infer_breakdown = {757 "score": inferential_score,758 "max_score": round(float(infer_score_info.get("max_score", 0.0)), 4),759 "max_lift": round(float(infer_score_info.get("max_lift", 0.0)), 4),760 "max_attr": infer_score_info.get("max_attr", ""),761 "p_post": round(float(infer_score_info.get("p_rag", 0.0)), 4),762 "p_pop": round(float(infer_score_info.get("p_pop", 0.0)), 4),763 "p_no_rag": round(float(infer_score_info.get("p_no_rag", 0.0)), 4),764 "mean_score": round(float(infer_score_info.get("mean_score", 0.0)), 4),765 "mean_lift": round(float(infer_score_info.get("mean_lift", 0.0)), 4),766 "median_score": round(float(infer_score_info.get("median_score", 0.0)), 4),767 "median_lift": round(float(infer_score_info.get("median_lift", 0.0)), 4),768 }769 inferential_score_breakdown_json = json.dumps(infer_breakdown, ensure_ascii=False)770 771 # ── Scraped social data ───────────────────────────────772 scraped_json = "[]"773 scraped_summary_json = "[]"774 if scraped_docs:775 try:776 scraped_json = json.dumps([777 {778 "text": doc.page_content,779 "metadata": doc.metadata if hasattr(doc, "metadata") else {},780 }781 for doc in scraped_docs782 ], ensure_ascii=False)783 except Exception:784 scraped_json = "[]"785 786 # Build a human-readable per-platform summary for analysis787 try:788 platform_groups = {}789 for doc in scraped_docs:790 meta = doc.metadata if hasattr(doc, "metadata") else {}791 platform = meta.get("platform", "unknown").lower()792 source_url = (793 meta.get("source_url") or794 meta.get("tweet_url") or795 meta.get("url") or796 ""797 )798 text = doc.page_content or ""799 if platform not in platform_groups:800 platform_groups[platform] = []801 platform_groups[platform].append({802 "text": text,803 "url": source_url,804 })805 806 summary_entries = []807 for platform, posts in platform_groups.items():808 summary_entries.append({809 "platform": platform,810 "post_count": len(posts),811 "posts": [812 {813 "text": p["text"],814 "url": p["url"],815 }816 for p in posts817 ],818 })819 scraped_summary_json = json.dumps(summary_entries, ensure_ascii=False)820 except Exception:821 scraped_summary_json = "[]"822 823 persona_attributes_json = json.dumps(persona_attributes or {}, ensure_ascii=False)824 825 row = [826 datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ"),827 str(session_source),828 int(turn_number),829 str(demo_mode),830 str(scenario_mode),831 persona_attributes_json,832 str(model),833 str(epsilon),834 str(rag_enabled),835 str(show_risk),836 str(show_rag_highlights),837 str(show_tips),838 str(show_pii_highlights),839 str(show_settings),840 str(show_dp),841 str(show_infr_attr_card),842 str(show_social_scraping),843 str(show_upload_data),844 str(rag_corpus_path),845 str(access_token),846 str(social_scraping_enabled),847 str(corpus_source),848 str(uploaded_file_path or ""),849 uploaded_file_preview,850 _sanitize_cell(user_prompt),851 _sanitize_cell(user_prompt_perturbed if user_prompt_perturbed is not None else user_prompt),852 int(num_input_dp_substitutions),853 len(user_prompt or ""),854 len(user_prompt_perturbed or ""),855 _sanitize_cell(llm_response),856 len(llm_response or ""),857 round(float(risk_score), 2),858 len(u_rag or []),859 rag_linkages_user,860 len(r_rag or []),861 rag_linkages_llm,862 len(u_pii or []),863 pii_detected_user,864 len(u_pii_perturbed or []),865 _serialise_pii(u_pii_perturbed or []),866 len(r_pii or []),867 pii_detected_llm,868 str(bool(inference_warning_shown)),869 str(num_attributes_changed > 0),870 int(num_attributes_changed),871 lifts_json,872 inferential_score,873 inferential_score_breakdown_json,874 scraped_json,875 scraped_summary_json,876 ]877 878 with file_lock:879 with open(log_path, "a", newline="", encoding="utf-8") as f:880 csv.writer(f, quoting=csv.QUOTE_ALL).writerow(row)881 882 _push_log_to_hub(log_path, log_file)883 logger.info(884 f"📋 Logged turn {turn_number} for participant '{pid}' "885 f"(risk={risk_score:.0f}, "886 f"rag_user={len(u_rag or [])}, rag_llm={len(r_rag or [])}, "887 f"pii_user={len(u_pii or [])}, pii_llm={len(r_pii or [])}, "888 f"lifts={len(lifts)}, warning_shown={inference_warning_shown})"889 )890 891 except Exception as exc:892 logger.error(f"⚠️ Failed to write interaction log: {exc}")893 894 895def append_session_end_log(state):896 """Write a sentinel END row to the participant's CSV when they end the conversation."""897 try:898 access_token = getattr(state, "_access_token", "")899 session_source = getattr(state, "_session_source", "unknown")900 pid = _participant_id(access_token, session_source)901 log_path = _participant_log_path(pid)902 log_file = f"log_{pid}.csv"903 file_lock = _get_file_lock(pid)904 905 end_ts = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")906 907 # Columns whose values are meaningful for the END row908 config_values = {909 "timestamp": end_ts,910 "session_source": str(session_source),911 "turn_number": "END",912 "demo_mode": str(getattr(state, "_demo_mode", False)),913 "scenario_mode": str(getattr(state, "_scenario_mode", "real")),914 "persona_attributes": json.dumps(getattr(state, "_persona_attributes", {}), ensure_ascii=False),915 "model": str(getattr(state, "_last_model", "")),916 "epsilon": str(getattr(state, "_last_epsilon", float("inf"))),917 "rag_enabled": str(getattr(state, "_last_rag_enabled", False)),918 "show_risk": str(getattr(state, "_show_risk", True)),919 "show_rag_highlights": str(getattr(state, "_show_rag_highlights", False)),920 "show_tips": str(getattr(state, "_show_tips", False)),921 "show_pii_highlights": str(getattr(state, "_show_pii_hl", True)),922 "show_settings": str(getattr(state, "_show_settings", False)),923 "access_token": str(access_token),924 "social_scraping_enabled": str(getattr(state, "_last_social_scraping", False)),925 "corpus_source": str(getattr(state, "_last_corpus_source", "system")),926 "uploaded_file_path": str(getattr(state, "_uploaded_file_path", "") or ""),927 "show_dp": str(getattr(state, "_show_dp", 1)),928 "show_infr_attr_card": str(getattr(state, "_show_infr_attr_card", 1)),929 "show_social_scraping": str(getattr(state, "_show_social_scraping", False)),930 "show_upload_data": str(getattr(state, "_show_upload_data", False)),931 "rag_corpus_path": str(getattr(state, "_rag_corpus_path", "")),932 }933 934 # Build the row: config values where available, "END" for all per-turn columns935 row = [936 config_values.get(col, "END")937 for col in LOG_COLUMNS938 ]939 940 with file_lock:941 with open(log_path, "a", newline="", encoding="utf-8") as f:942 csv.writer(f, quoting=csv.QUOTE_ALL).writerow(row)943 944 _push_log_to_hub(log_path, log_file)945 logger.info(f"📋 Logged END-OF-CONVERSATION for participant '{pid}' at {end_ts}")946 except Exception as exc:947 logger.error(f"⚠️ Failed to write session-end log: {exc}")948 949 950os.makedirs("logs", exist_ok=True)951 952# ────────────────────────────────────────────────────────────953# Apify Configuration for Social Media Scraping954# ────────────────────────────────────────────────────────────955 956def validate_access_token(token):957 """Check if access token is valid and not expired.958 959 A Prolific token (exactly 24 alphanumeric characters) is always accepted.960 Any other token must appear in VALID_ACCESS_TOKENS and must not be expired.961 """962 if not token:963 return False964 965 # Accept valid Prolific tokens (exactly 24 alphanumeric characters).966 if re.fullmatch(PROLIFIC_ID_PATTERN_REGEX, token):967 return True968 969 # For non-Prolific tokens, check the VALID_ACCESS_TOKENS dictionary.970 token_info = VALID_ACCESS_TOKENS.get(token)971 if not token_info:972 return False973 974 # Check expiration date.975 from datetime import datetime976 expires = datetime.strptime(token_info["expires"], "%Y-%m-%d")977 if datetime.now() > expires:978 return False979 980 return True981 982 983def _load_fallback_tweets(scenario_mode: str):984 """Load and return fallback tweet items for the given scenario_mode.985 Returns [] if the file is missing, unmapped, or set to None."""986 key = str(scenario_mode).strip().lower()987 path = SCENARIO_FALLBACK_TWEET_PATHS.get(key)988 if not path:989 return []990 try:991 with open(path, "r", encoding="utf-8") as f:992 data = json.load(f)993 logger.info(f"✓ Loaded {len(data)} fallback tweets for scenario '{key}' from {path}")994 return data995 except FileNotFoundError:996 logger.warning(f"⚠️ Fallback tweet file not found for scenario '{key}': {path}")997 return []998 except Exception as e:999 logger.error(f"✗ Failed to load fallback tweets for scenario '{key}': {e}")1000 return []1001 1002 1003# ============================================================1004# SECTION 1 – DATA STRUCTURES1005# ============================================================1006 1007 1008def _fill_missing_with_uniform(probs):1009 """Fill any missing SENSITIVE_ATTRIBUTES with a uniform distribution.1010 1011 This ensures the inference panel always has something to display even when1012 an LLM call fails or returns a partial response. A uniform distribution1013 signals maximum uncertainty — no strong prediction — which is honest and1014 safe to show.1015 """1016 filled = dict(probs) # shallow copy — don't mutate the original1017 for attr in SENSITIVE_ATTRIBUTES:1018 if attr not in filled or not filled[attr]:1019 values = ATTRIBUTE_VALUES_MAP.get(attr, [])1020 if values:1021 p = round(1.0 / len(values), 4)1022 filled[attr] = {v: p for v in values}1023 return filled1024 1025 1026def get_persona(scenario_mode: str) -> dict:1027 """Return the persona dict for a given scenario_mode string (case-insensitive).1028 Falls back to 'real' if the key is not found."""1029 return PERSONAS.get(str(scenario_mode).strip().lower(), PERSONAS["real"])1030 1031 1032class PIICategory(Enum):1033 IDENTITY = "identity" # IDs, names, SSN, usernames, etc.1034 CONTACT = "contact" # Phone, emails1035 LOCATION = "location" # Addresses, organizations1036 SENSITIVE = "sensitive" # Medical, financial, and other sensitive data1037 1038 1039# Mapping from specific PII fine-types (as returned by the LLM) to the four1040# broad UI categories. Add new fine-types here; the rest of the code adapts.1041PII_TYPE_TO_CATEGORY = {1042 # ── Identity ─────────────────────────────────────────────────────────────1043 "name": PIICategory.IDENTITY,1044 "full name": PIICategory.IDENTITY,1045 "first name": PIICategory.IDENTITY,1046 "last name": PIICategory.IDENTITY,1047 "ssn": PIICategory.IDENTITY,1048 "social security number": PIICategory.IDENTITY,1049 "username": PIICategory.IDENTITY,1050 "id": PIICategory.IDENTITY,1051 "identifier": PIICategory.IDENTITY,1052 "passport": PIICategory.IDENTITY,1053 "license": PIICategory.IDENTITY,1054 "date of birth": PIICategory.IDENTITY,1055 "dob": PIICategory.IDENTITY,1056 "age": PIICategory.IDENTITY,1057 "ip address": PIICategory.IDENTITY,1058 "ip": PIICategory.IDENTITY,1059 # ── Contact ───────────────────────────────────────────────────────────────1060 "email": PIICategory.CONTACT,1061 "email address": PIICategory.CONTACT,1062 "phone": PIICategory.CONTACT,1063 "phone number": PIICategory.CONTACT,1064 "mobile": PIICategory.CONTACT,1065 "fax": PIICategory.CONTACT,1066 # ── Location ──────────────────────────────────────────────────────────────1067 "location": PIICategory.LOCATION,1068 "address": PIICategory.LOCATION,1069 "street address": PIICategory.LOCATION,1070 "city": PIICategory.LOCATION,1071 "state": PIICategory.LOCATION,1072 "country": PIICategory.LOCATION,1073 "zip code": PIICategory.LOCATION,1074 "postal code": PIICategory.LOCATION,1075 "organization": PIICategory.LOCATION,1076 "workplace": PIICategory.LOCATION,1077 "school": PIICategory.LOCATION,1078 # ── Sensitive (medical, financial, and other sensitive data) ──────────────1079 "medical": PIICategory.SENSITIVE,1080 "medical record": PIICategory.SENSITIVE,1081 "diagnosis": PIICategory.SENSITIVE,1082 "condition": PIICategory.SENSITIVE,1083 "medication": PIICategory.SENSITIVE,1084 "prescription": PIICategory.SENSITIVE,1085 "treatment": PIICategory.SENSITIVE,1086 "health": PIICategory.SENSITIVE,1087 "disability": PIICategory.SENSITIVE,1088 "mental health": PIICategory.SENSITIVE,1089 "insurance": PIICategory.SENSITIVE,1090 "financial": PIICategory.SENSITIVE,1091 "bank account": PIICategory.SENSITIVE,1092 "credit card": PIICategory.SENSITIVE,1093 "credit card number": PIICategory.SENSITIVE,1094 "account number": PIICategory.SENSITIVE,1095 "salary": PIICategory.SENSITIVE,1096 "income": PIICategory.SENSITIVE,1097 "debt": PIICategory.SENSITIVE,1098 "loan": PIICategory.SENSITIVE,1099 "mortgage": PIICategory.SENSITIVE,1100 "tax id": PIICategory.SENSITIVE,1101 "political": PIICategory.SENSITIVE,1102 "religion": PIICategory.SENSITIVE,1103 "sexual orientation": PIICategory.SENSITIVE,1104 "ethnicity": PIICategory.SENSITIVE,1105 "race": PIICategory.SENSITIVE,1106 "biometric": PIICategory.SENSITIVE,1107}1108 1109 1110@dataclass1111class PIIMatch:1112 text: str1113 start: int1114 end: int1115 fine_type: str1116 category: PIICategory1117 confidence: float1118 1119 1120@dataclass1121class RAGLink:1122 """A span of user/assistant text linked to RAG corpus documents."""1123 text: str1124 start: int1125 end: int1126 corpus_snippets: list1127 top_similarity: float1128 top_doc_text: str = ""1129 top_doc_score: float = 0.01130 overlap_keywords: list = None1131 source: str = "From system data" # Source of the matched text1132 url: str = "" # Source URL of the matched document (if available)1133 1134 1135@dataclass1136class Message:1137 role: str1138 content: str1139 pii_matches: list = None1140 pii_card_matches: list = None # PII detected on perturbed text (for the right-panel card)1141 rag_links: list = None1142 dp_metadata: dict = None # Input DP info: {epsilon, num_substitutions, substitutions}1143 1144 1145class ConversationState:1146 """Simple in-memory conversation container."""1147 def __init__(self):1148 self.messages = []1149 self.last_probs_rag = {}1150 self.last_probs_no_rag = {}1151 self.last_evidence_rag = {}1152 # Logging metadata – set once per session1153 self._session_source = "unknown"1154 self._access_token = ""1155 self._show_risk = True1156 self._show_rag_highlights = False1157 self._show_tips = False1158 self._show_pii_hl = True1159 self._show_settings = False1160 self._show_dp = 11161 self._show_infr_attr_card = 11162 self._show_social_scraping = False1163 self._show_upload_data = False1164 self._rag_corpus_path = ""1165 self._uploaded_file_path = None1166 self._turn_count = 0 # incremented per user message1167 self._peak_inference_metrics = {}1168 self._prev_inference_attrs = {}1169 self._scenario_mode = "real"1170 self._last_model = ""1171 self._last_epsilon = float("inf")1172 self._last_rag_enabled = False1173 self._last_social_scraping = False1174 self._last_corpus_source = "system"1175 self._demo_mode = False1176 self._persona_attributes = {}1177 self._last_avatar_html = "" # cached inference card HTML for end-reveal1178 self._last_privacy_html = "" # cached privacy settings HTML for end-reveal1179 self._scraped_docs = None # None = not yet scraped; [] = scraped, nothing found1180 self._best_probs_rag = {} # attr → best prob distribution seen so far1181 self._best_evidence_rag = {} # attr → evidence from that best turn1182 1183 1184 def add(self, role, content, pii_matches=None, rag_links=None, dp_metadata=None, pii_card_matches=None):1185 self.messages.append(Message(role=role, content=content,1186 pii_matches=pii_matches,1187 pii_card_matches=pii_card_matches,1188 rag_links=rag_links,1189 dp_metadata=dp_metadata))1190 1191 def clear(self):1192 self.messages = []1193 self.last_probs_rag = {}1194 self.last_probs_no_rag = {}1195 self.last_evidence_rag = {}1196 self._turn_count = 01197 self._peak_inference_metrics = {}1198 self._last_model = ""1199 self._last_epsilon = float("inf")1200 self._last_rag_enabled = False