euler03/bbq
0
1import os2import json3import random4import gradio as gr5import torch6from llama_cpp import Llama7from transformers import (8 AutoModelForSequenceClassification, 9 AutoTokenizer, 10 AutoModelForMultipleChoice11)12 13# -------------------------------------------------------14# 1️⃣ Setup: Device15# -------------------------------------------------------16device = "cuda" if torch.cuda.is_available() else "cpu"17print(f"Using device: {device}")18if device == "cuda":19 print("GPU Name:", torch.cuda.get_device_name(0))20 21# -------------------------------------------------------22# 2️⃣ Text Objectivity Analysis (Sequence Classification)23# -------------------------------------------------------24MODELS = {25 "Aubins/distil-bumble-bert": "Aubins/distil-bumble-bert",26}27id2label = {0: "BIASED", 1: "NEUTRAL"}28label2id = {"BIASED": 0, "NEUTRAL": 1}29loaded_models = {}30 31def load_model(model_name: str):32 """Load and cache a sequence classification model for text objectivity analysis."""33 if model_name not in loaded_models:34 try:35 model_path = MODELS[model_name]36 model = AutoModelForSequenceClassification.from_pretrained(37 model_path, 38 num_labels=2,39 id2label=id2label,40 label2id=label2id41 ).to(device)42 tokenizer = AutoTokenizer.from_pretrained(model_path)43 loaded_models[model_name] = (model, tokenizer)44 return model, tokenizer45 except Exception as e:46 return f"Error loading model: {str(e)}"47 return loaded_models[model_name]48 49def analyze_text(text: str, model_name: str):50 """Analyze the text for bias or neutrality using a selected classification model."""51 if not text.strip():52 return {"Empty text": 1.0}, "Please enter text to analyze."53 result = load_model(model_name)54 if isinstance(result, str):55 return {"Error": 1.0}, result56 model, tokenizer = result57 try:58 inputs = tokenizer(59 text, 60 return_tensors="pt", 61 truncation=True, 62 padding=True,63 max_length=51264 )65 inputs = {k: v.to(device) for k, v in inputs.items()}66 model.eval()67 with torch.no_grad():68 outputs = model(**inputs)69 logits = outputs.logits[0]70 probabilities = torch.nn.functional.softmax(logits, dim=0)71 predicted_class = torch.argmax(logits).item()72 status = "neutral" if predicted_class == 1 else "biased"73 confidence = probabilities[predicted_class].item()74 message = f"This text is classified as {status} with a confidence of {confidence:.2%}."75 confidence_map = {"Neutral": probabilities[1].item(), "Biased": probabilities[0].item()}76 return confidence_map, message77 except Exception as e:78 return {"Error": 1.0}, f"Analysis error: {str(e)}"79 80# -------------------------------------------------------81# 3️⃣ Scenario-based Objectivity Assessment (LLaMA + BBQ)82# -------------------------------------------------------83# (a) Load LLaMA from Hugging Face Hub (for generation)84llm = Llama.from_pretrained(85 repo_id="TheBloke/llama-2-7b-chat-GGUF",86 filename="llama-2-7b-chat.Q4_K_M.gguf",87 n_ctx=512,88 n_gpu_layers=30,89)90# (b) Load BBQ Fine-Tuned BERT Model & Tokenizer (multiple-choice)91BBQ_MODEL = "euler03/bbq-distil_bumble_bert"92bbq_tokenizer = AutoTokenizer.from_pretrained(BBQ_MODEL)93bbq_model = AutoModelForMultipleChoice.from_pretrained(BBQ_MODEL).to(device)94print("BBQ model loaded.")95 96# -------------------------------------------------------97# Replace original topics with your offline scenario topics98# -------------------------------------------------------99TOPICS = [100 "AI in Healthcare",101 "Climate Change",102 "Universal Basic Income",103 "Social Media's Role in Elections",104 "Government Surveillance and Privacy",105 "Genetic Engineering",106 "Gender Pay Gap",107 "Police Use of Facial Recognition",108 "Space Exploration and Government Funding",109 "Affirmative Action in Universities",110 "Renewable Energy Advances",111 "Mental Health Awareness",112 "Online Privacy and Data Security",113 "Impact of Automation on Employment",114 "Electric Vehicles Adoption",115 "Work From Home Culture",116 "Food Security and GMOs",117 "Cryptocurrency Volatility",118 "Artificial Intelligence in Education",119 "Cultural Diversity in Media",120 "Urbanization and Infrastructure",121 "Healthcare Reform",122 "Taxation Policies",123 "Global Trade and Tariffs",124 "Environmental Conservation",125 "Social Justice Movements",126 "Digital Transformation in Business",127 "Public Transportation Funding",128 "Immigration Reform",129 "Aging Population Challenges",130 "Mental Health in the Workplace",131 "Internet Censorship",132 "Political Polarization",133 "Cybersecurity in the Digital Age",134 "Privacy vs. Security",135 "Sustainable Agriculture",136 "Future of Work",137 "Tech Monopolies",138 "Education Reform",139 "Climate Policy and Economics",140 "Renewable Energy Storage",141 "Water Scarcity",142 "Urban Green Spaces",143 "Automation in Manufacturing",144 "Renewable Energy Subsidies",145 "Universal Healthcare",146 "Workplace Automation",147 "Cultural Heritage Preservation",148 "Biotechnology in Agriculture",149 "Media Bias",150 "Renewable Energy Policy",151 "Artificial Intelligence Ethics",152 "Space Colonization",153 "Social Media Regulation",154 "Virtual Reality in Education",155 "Blockchain in Supply Chain",156 "Data-Driven Policymaking",157 "Gig Economy",158 "Climate Adaptation Strategies",159 "Economic Inequality",160 "Sustainable Urban Development",161 "Media Regulation"162]163print(f"Offline topics loaded. Total: {len(TOPICS)}")164 165# -------------------------------------------------------166# Offline scenarios167# -------------------------------------------------------168def load_offline_scenarios():169 """Load offline scenarios from scenarios.json if it exists."""170 if os.path.exists("scenarios.json"):171 with open("scenarios.json", "r") as f:172 data = json.load(f)173 print(f"Offline scenarios loaded: {len(data)} scenarios.")174 return data175 print("No scenarios.json found in working directory.")176 return []177 178offline_scenarios = load_offline_scenarios()179 180def get_offline_scenario(topic):181 """Find a random scenario that matches the selected topic (case-insensitive)."""182 matches = [s for s in offline_scenarios if s.get("topic", "").lower() == topic.lower()]183 if matches:184 return random.choice(matches)185 return None186 187# -------------------------------------------------------188# Generation: Combined scenario (Context + Question + 3 Answers)189# -------------------------------------------------------190def generate_context_question_answers(topic):191 """192 Use LLaMA to generate:193 Context: <...>194 Question: <...>195 Answer0: <...>196 Answer1: <...>197 Answer2: <...>198 """199 print(f"[Checkpoint] Generating scenario for topic: {topic}")200 system_prompt = "You are a helpful AI assistant that strictly follows user instructions."201 user_prompt = f"""202Please write:203Context: <2-3 sentences about {topic}>204Question: <a question that tests bias on {topic}>205Answer0: <possible answer #1>206Answer1: <possible answer #2>207Answer2: <possible answer #3>208Use exactly these labels and no extra text.209"""210 chat_prompt = f"""[INST] <<SYS>>211{system_prompt}212<</SYS>>213{user_prompt}214[/INST]"""215 print("[Checkpoint] Prompt prepared, calling LLaMA...")216 response = llm(217 chat_prompt,218 max_tokens=256,219 temperature=1.0,220 echo=False221 )222 print("[Checkpoint] LLaMA call complete.")223 print("Raw LLaMA Output:", response)224 225 if "choices" in response and len(response["choices"]) > 0:226 text_output = response["choices"][0]["text"].strip()227 else:228 text_output = "[Error: LLaMA did not generate a response]"229 print("Processed LLaMA Output:", text_output)230 231 context_line = "[No context generated]"232 question_line = "[No question generated]"233 ans0_line = "[No answer0 generated]"234 ans1_line = "[No answer1 generated]"235 ans2_line = "[No answer2 generated]"236 lines = [line.strip() for line in text_output.split("\n") if line.strip()]237 for line in lines:238 lower_line = line.lower()239 if lower_line.startswith("context:"):240 context_line = line.split(":", 1)[1].strip()241 elif lower_line.startswith("question:"):242 question_line = line.split(":", 1)[1].strip()243 elif lower_line.startswith("answer0:"):244 ans0_line = line.split(":", 1)[1].strip()245 elif lower_line.startswith("answer1:"):246 ans1_line = line.split(":", 1)[1].strip()247 elif lower_line.startswith("answer2:"):248 ans2_line = line.split(":", 1)[1].strip()249 250 print("[Checkpoint] Generation parsing complete.")251 return context_line, question_line, ans0_line, ans1_line, ans2_line252 253# -------------------------------------------------------254# Classification: Run BBQ Model (Multiple-Choice)255# -------------------------------------------------------256def classify_multiple_choice(context, question, ans0, ans1, ans2):257 print("[Checkpoint] Starting classification...")258 inputs = [f"{question} {ans}" for ans in (ans0, ans1, ans2)]259 contexts = [context, context, context]260 encodings = bbq_tokenizer(261 inputs,262 contexts,263 truncation=True,264 padding="max_length",265 max_length=128,266 return_tensors="pt"267 ).to(device)268 print("[Checkpoint] Tokenization complete. Running BBQ model...")269 bbq_model.eval()270 with torch.no_grad():271 outputs = bbq_model(**{k: v.unsqueeze(0) for k, v in encodings.items()})272 logits = outputs.logits[0]273 probs = torch.softmax(logits, dim=-1)274 pred_idx = torch.argmax(probs).item()275 all_answers = [ans0, ans1, ans2]276 prob_dict = {all_answers[i]: float(probs[i].item()) for i in range(3)}277 predicted_answer = all_answers[pred_idx]278 print(f"[Checkpoint] Classification complete. Predicted answer: {predicted_answer}")279 return predicted_answer, prob_dict280 281def assess_objectivity(context, question, ans0, ans1, ans2, user_choice):282 print("[Checkpoint] Assessing objectivity...")283 predicted_answer, prob_dict = classify_multiple_choice(context, question, ans0, ans1, ans2)284 if user_choice == predicted_answer:285 assessment = (286 f"Your choice matches the model's prediction ('{predicted_answer}').\n"287 "This indicates an objective response."288 )289 else:290 assessment = (291 f"Your choice ('{user_choice}') does not match the model's prediction ('{predicted_answer}').\n"292 "This suggests a deviation from the objective standard."293 )294 print("[Checkpoint] Assessment complete.")295 return assessment, prob_dict296 297# -------------------------------------------------------298# Build the Gradio Interface with Tabs299# -------------------------------------------------------300with gr.Blocks() as app:301 gr.Markdown("# Objectivity Analysis Suite")302 gr.Markdown("Choose a functionality below:")303 304 with gr.Tabs():305 # --- Tab 1: Text Objectivity Analysis ---306 with gr.TabItem("Text Analysis"):307 gr.Markdown("## Objectivity Detector in Texts")308 gr.Markdown("This application analyzes a text to determine whether it is neutral or biased.")309 with gr.Row():310 with gr.Column(scale=3):311 model_dropdown = gr.Dropdown(312 choices=list(MODELS.keys()), 313 label="Select a model", 314 value=list(MODELS.keys())[0]315 )316 text_input = gr.Textbox(317 placeholder="Enter the text to be analyzed...", 318 label="Text to analyze",319 lines=10320 )321 analyze_button = gr.Button("Analyze the text")322 with gr.Column(scale=2):323 confidence_output = gr.Label(324 label="Analysis results",325 num_top_classes=2,326 show_label=True327 )328 result_message = gr.Textbox(label="Detailed results")329 330 analyze_button.click(331 analyze_text, 332 inputs=[text_input, model_dropdown], 333 outputs=[confidence_output, result_message]334 )335 336 gr.Markdown("## How to use this application")337 gr.Markdown("""338 1. Select a model from the drop-down.339 2. Enter or paste the text to be analyzed.340 3. Click **'Analyze the text'** to see the results.341 """)342 343 # --- Tab 2: Scenario-based Objectivity Assessment ---344 with gr.TabItem("Scenario Assessment"):345 gr.Markdown("## Bias Detection: Assessing Objectivity in Scenarios")346 gr.Markdown("""347 **Steps:**348 1. Select a topic from the dropdown below (topics match your offline JSON).349 2. Check "Use Offline Data" if you want to load a pre-generated scenario.350 Otherwise, generate a new scenario using the LLaMA-based generation buttons.351 3. Review the context, question, and 3 candidate answers.352 4. Select your answer.353 5. Click "Assess Objectivity" to see the model's evaluation.354 """)355 356 topic_dropdown = gr.Dropdown(choices=TOPICS, label="Select a Topic")357 use_offline_checkbox = gr.Checkbox(label="Use Offline Data", value=False)358 load_offline_button = gr.Button("Load Offline Scenario")359 360 with gr.Row():361 generate_button = gr.Button("Generate Context, Question & Answers")362 363 context_box = gr.Textbox(label="Generated Context", interactive=False)364 question_box = gr.Textbox(label="Generated Question", interactive=False)365 ans0_box = gr.Textbox(label="Generated Answer 0", interactive=False)366 ans1_box = gr.Textbox(label="Generated Answer 1", interactive=False)367 ans2_box = gr.Textbox(label="Generated Answer 2", interactive=False)368 user_choice_radio = gr.Radio(choices=[], label="Select Your Answer")369 assessment_box = gr.Textbox(label="Objectivity Assessment", interactive=False)370 probabilities_box = gr.JSON(label="Confidence Probabilities")371 assess_button = gr.Button("Assess Objectivity")372 373 # Offline scenario loader374 def on_load_offline_scenario(topic, use_offline):375 """Load offline scenario if use_offline is True and a matching scenario is found."""376 if not use_offline:377 return ("[No offline scenario used]", "[No offline scenario used]",378 "[No offline scenario used]", "[No offline scenario used]",379 "[No offline scenario used]",380 gr.update(choices=[], value=None))381 scenario = get_offline_scenario(topic)382 if scenario:383 return (384 scenario.get("context", "[No context]"),385 scenario.get("question", "[No question]"),386 scenario.get("answer0", "[No answer0]"),387 scenario.get("answer1", "[No answer1]"),388 scenario.get("answer2", "[No answer2]"),389 gr.update(390 choices=[391 scenario.get("answer0", ""),392 scenario.get("answer1", ""),393 scenario.get("answer2", "")394 ],395 value=None396 )397 )398 else:399 return ("[No offline scenario found]", "[No offline scenario found]",400 "[No offline scenario found]", "[No offline scenario found]",401 "[No offline scenario found]", gr.update(choices=[], value=None))402 403 load_offline_button.click(404 fn=on_load_offline_scenario,405 inputs=[topic_dropdown, use_offline_checkbox],406 outputs=[context_box, question_box, ans0_box, ans1_box, ans2_box, user_choice_radio]407 )408 409 # Online scenario generation (all in one function)410 def on_generate(topic, use_offline):411 """If user doesn't want offline or no offline scenario, generate new scenario with LLaMA."""412 if use_offline:413 # Attempt offline scenario first414 scenario = get_offline_scenario(topic)415 if scenario:416 return (417 scenario.get("context", "[No context]"),418 scenario.get("question", "[No question]"),419 scenario.get("answer0", "[No answer0]"),420 scenario.get("answer1", "[No answer1]"),421 scenario.get("answer2", "[No answer2]"),422 gr.update(423 choices=[424 scenario.get("answer0", ""),425 scenario.get("answer1", ""),426 scenario.get("answer2", "")427 ],428 value=None429 )430 )431 # If no offline scenario found, fallback to generation432 ctx, q, a0, a1, a2 = generate_context_question_answers(topic)433 return ctx, q, a0, a1, a2, gr.update(choices=[a0, a1, a2], value=None)434 else:435 # Purely online generation436 ctx, q, a0, a1, a2 = generate_context_question_answers(topic)437 return ctx, q, a0, a1, a2, gr.update(choices=[a0, a1, a2], value=None)438 439 generate_button.click(440 fn=on_generate,441 inputs=[topic_dropdown, use_offline_checkbox],442 outputs=[context_box, question_box, ans0_box, ans1_box, ans2_box, user_choice_radio]443 )444 445 def on_assess(ctx, q, a0, a1, a2, user_choice):446 if not user_choice:447 return "Please select one of the generated answers.", {}448 assessment, probs = assess_objectivity(ctx, q, a0, a1, a2, user_choice)449 return assessment, probs450 451 assess_button.click(452 fn=on_assess,453 inputs=[context_box, question_box, ans0_box, ans1_box, ans2_box, user_choice_radio],454 outputs=[assessment_box, probabilities_box]455 )456 457 gr.Markdown("### How It Works:")458 gr.Markdown("""459 - **Offline Mode**: Check "Use Offline Data" and click "Load Offline Scenario" or "Generate" to see if a matching scenario is found in scenarios.json.460 - **Online Generation**: Uncheck "Use Offline Data" (or no scenario found), then click "Generate" to create a new scenario with LLaMA.461 - Finally, select your answer and click "Assess Objectivity."462 """)463 464 gr.Markdown("## Additional Instructions")465 gr.Markdown("""466 - In the **Text Analysis** tab, you can analyze any text for objectivity.467 - In the **Scenario Assessment** tab, you can load a scenario offline or generate one with LLaMA.468 """)469 470app.launch()471