Blablablab/audio-classification
0
1"""2Admin Dashboard Module3 4This module provides comprehensive admin functionality for the annotation platform,5including dashboard data generation, timing analysis, and configuration management.6 7The admin dashboard offers:8- Real-time overview of annotation progress and statistics9- Detailed annotator performance metrics and timing analysis10- Instance-level annotation tracking and disagreement analysis11- Configuration management and system state monitoring12- Question and annotation scheme analysis13- User progress tracking and completion statistics14- Comprehensive annotation history tracking and suspicious activity detection15- Performance metrics and quality assurance monitoring16- Session tracking and behavioral analysis17 18Key Components:19- AdminDashboard: Main class for admin functionality20- AnnotatorTimingData: Data class for annotator timing information21- InstanceData: Data class for instance information and statistics22- Dashboard data generation and analysis functions23- Configuration update and management functions24- AnnotationHistoryAnalyzer: Advanced history analysis and suspicious activity detection25 26The dashboard provides insights into:27- Overall annotation progress and completion rates28- Individual annotator performance and efficiency29- Annotation quality through disagreement analysis30- System configuration and operational status31- Real-time monitoring of active annotation sessions32- Fine-grained annotation timing and behavioral patterns33- Suspicious activity detection and quality assurance34- Session-based performance analysis35 36Access Control:37- Admin access is controlled via API key authentication38- Debug mode allows admin access without API key39- All admin endpoints require proper authentication40"""41 42import json43import logging44import datetime45from typing import Dict, List, Optional, Tuple, Any46from collections import defaultdict, Counter47from dataclasses import dataclass48from flask import request, jsonify, session49 50from potato.flask_server import (51 config, logger, get_user_state_manager, get_item_state_manager,52 get_users, get_total_annotations53)54from potato.annotation_history import AnnotationHistoryManager, AnnotationAction55from potato.quality_control import get_quality_control_manager56 57@dataclass58class AnnotatorTimingData:59 """60 Data class for annotator timing information.61 62 This class encapsulates timing metrics for individual annotators,63 including total annotations, working time, and performance statistics.64 Now enhanced with annotation history tracking and suspicious activity detection.65 """66 user_id: str67 total_annotations: int68 total_seconds: int69 average_seconds_per_annotation: float70 last_activity: Optional[datetime.datetime]71 current_instance_time: Optional[int]72 annotations_per_hour: float73 phase: str74 has_assignments: bool75 remaining_assignments: bool76 77 # Annotation history metrics78 total_actions: int79 average_action_time_ms: float80 fastest_action_time_ms: int81 slowest_action_time_ms: int82 actions_per_minute: float83 suspicious_score: float84 suspicious_level: str85 fast_actions_count: int86 burst_actions_count: int87 session_start_time: Optional[datetime.datetime]88 current_session_duration_minutes: Optional[float]89 recent_actions_count: int # Actions in last 5 minutes90 91 # Training metrics92 training_completed: bool93 training_correct_answers: int94 training_total_attempts: int95 training_pass_rate: float96 training_current_question: int97 training_total_questions: int98 99@dataclass100class InstanceData:101 """102 Data class for instance information.103 104 This class encapsulates information about annotation instances,105 including annotation counts, disagreement scores, and annotator lists.106 """107 id: str108 text: str109 displayed_text: str110 annotation_count: int111 completion_percentage: float112 most_frequent_label: Optional[str]113 label_disagreement: float114 annotators: List[str]115 num_ai_instance: int116 average_time_per_annotation: Optional[float]117 118class AdminDashboard:119 """120 Main class for admin dashboard functionality.121 122 This class provides comprehensive admin features including dashboard123 data generation, timing analysis, configuration management, and124 system monitoring capabilities.125 """126 127 def __init__(self):128 """Initialize the admin dashboard."""129 self.logger = logging.getLogger(__name__)130 131 def check_admin_access(self) -> bool:132 """133 Check if the current request has admin access via API key.134 135 Validates against all key sources (config, env var, auto-generated file)136 and accepts keys from X-API-Key header or session.137 138 Returns:139 bool: True if admin access is granted, False otherwise140 """141 from potato.server_utils.admin_key import validate_admin_api_key142 api_key = request.headers.get('X-API-Key') or session.get('admin_api_key')143 return validate_admin_api_key(api_key, config)144 145 def get_dashboard_overview(self) -> Dict[str, Any]:146 """147 Get comprehensive dashboard overview data.148 149 This method generates a complete overview of the annotation system,150 including user statistics, annotation progress, and system configuration.151 152 Returns:153 Dict containing overview statistics with the following structure:154 - overview: User counts, annotation counts, completion percentages155 - config: System configuration and settings156 157 Side Effects:158 - Logs errors if data generation fails159 """160 if not self.check_admin_access():161 return {"error": "Admin access required"}, 403162 163 try:164 usm = get_user_state_manager()165 ism = get_item_state_manager()166 167 # Get all users and their states168 users = get_users()169 total_annotations = get_total_annotations()170 171 # Calculate user statistics172 active_users = 0173 completed_users = 0174 total_working_time = 0175 176 for username in users:177 user_state = usm.get_user_state(username)178 if user_state:179 if user_state.get_phase().value == "ANNOTATION":180 active_users += 1181 elif user_state.get_phase().value == "DONE":182 completed_users += 1183 184 # Get timing data185 timing_data = self._get_annotator_timing_data(username)186 if timing_data:187 total_working_time += timing_data.total_seconds188 189 # Get item statistics190 items = ism.items()191 items_with_annotations = 0192 total_assignments = 0193 194 for item in items:195 item_id = item.get_id()196 annotators = ism.get_annotators_for_item(item_id)197 if annotators:198 items_with_annotations += 1199 total_assignments += len(annotators)200 201 # Calculate completion percentages202 total_items = len(items)203 completion_percentage = (items_with_annotations / total_items * 100) if total_items > 0 else 0204 205 # Format total working time206 hours = total_working_time // 3600207 minutes = (total_working_time % 3600) // 60208 formatted_time = f"{hours}h {minutes}m"209 210 return {211 "overview": {212 "total_users": len(users),213 "active_users": active_users,214 "completed_users": completed_users,215 "total_annotations": total_annotations,216 "total_items": total_items,217 "items_with_annotations": items_with_annotations,218 "completion_percentage": round(completion_percentage, 1),219 "total_assignments": total_assignments,220 "total_working_time": formatted_time,221 "average_annotations_per_item": round(total_annotations / total_items, 1) if total_items > 0 else 0222 },223 "config": {224 "annotation_task_name": config.get("annotation_task_name", "Unknown"),225 "max_annotations_per_user": config.get("max_annotations_per_user", "Unlimited"),226 "max_annotations_per_item": config.get("max_annotations_per_item", "Unlimited"),227 "assignment_strategy": config.get("assignment_strategy", "fixed_order"),228 "debug_mode": config.get("debug", False)229 }230 }231 232 except Exception as e:233 self.logger.error(f"Error getting dashboard overview: {e}")234 return {"error": f"Failed to get dashboard overview: {str(e)}"}, 500235 236 def get_annotators_data(self) -> Dict[str, Any]:237 """238 Get detailed annotator data including timing information.239 240 Returns:241 Dict containing annotator data with timing analysis242 """243 if not self.check_admin_access():244 return {"error": "Admin access required"}, 403245 246 try:247 usm = get_user_state_manager()248 users = get_users()249 annotators_data = []250 251 252 for username in users:253 user_state = usm.get_user_state(username)254 if user_state:255 timing_data = self._get_annotator_timing_data(username)256 if timing_data:257 annotators_data.append({258 "user_id": timing_data.user_id,259 "total_annotations": timing_data.total_annotations,260 "completion_percentage": self._calculate_completion_percentage(timing_data.user_id),261 "total_seconds": timing_data.total_seconds,262 "average_seconds_per_annotation": timing_data.average_seconds_per_annotation,263 "annotations_per_hour": timing_data.annotations_per_hour,264 "phase": timing_data.phase,265 "has_assignments": timing_data.has_assignments,266 "remaining_assignments": timing_data.remaining_assignments,267 "max_assignments": user_state.get_max_assignments(),268 "last_activity": timing_data.last_activity.isoformat() if timing_data.last_activity else None,269 "current_instance_time": timing_data.current_instance_time,270 271 # NEW: Annotation history metrics272 "total_actions": timing_data.total_actions,273 "average_action_time_ms": timing_data.average_action_time_ms,274 "fastest_action_time_ms": timing_data.fastest_action_time_ms if timing_data.fastest_action_time_ms != float('inf') else None,275 "slowest_action_time_ms": timing_data.slowest_action_time_ms,276 "actions_per_minute": timing_data.actions_per_minute,277 "suspicious_score": timing_data.suspicious_score,278 "suspicious_level": timing_data.suspicious_level,279 "fast_actions_count": timing_data.fast_actions_count,280 "burst_actions_count": timing_data.burst_actions_count,281 "session_start_time": timing_data.session_start_time.isoformat() if timing_data.session_start_time else None,282 "current_session_duration_minutes": timing_data.current_session_duration_minutes,283 "recent_actions_count": timing_data.recent_actions_count,284 285 # Training metrics286 "training_completed": timing_data.training_completed,287 "training_correct_answers": timing_data.training_correct_answers,288 "training_total_attempts": timing_data.training_total_attempts,289 "training_pass_rate": round(timing_data.training_pass_rate, 2),290 "training_current_question": timing_data.training_current_question,291 "training_total_questions": timing_data.training_total_questions292 })293 294 # Sort by suspicious score (highest first)295 annotators_data.sort(key=lambda x: x["suspicious_score"], reverse=True)296 297 return {298 "total_annotators": len(annotators_data),299 "annotators": annotators_data,300 "summary": {301 "high_suspicious_count": len([a for a in annotators_data if a["suspicious_level"] in ["High", "Very High"]]),302 "medium_suspicious_count": len([a for a in annotators_data if a["suspicious_level"] == "Medium"]),303 "low_suspicious_count": len([a for a in annotators_data if a["suspicious_level"] == "Low"]),304 "normal_count": len([a for a in annotators_data if a["suspicious_level"] == "Normal"]),305 "average_suspicious_score": sum(a["suspicious_score"] for a in annotators_data) / len(annotators_data) if annotators_data else 0306 }307 }308 309 except Exception as e:310 self.logger.error(f"Error getting annotators data: {e}")311 return {"error": f"Failed to get annotators data: {str(e)}"}, 500312 313 def get_annotation_history_data(self, user_id: Optional[str] = None,314 instance_id: Optional[str] = None,315 minutes: Optional[int] = None) -> Dict[str, Any]:316 """317 Get detailed annotation history data with filtering options.318 319 Args:320 user_id: Optional user ID to filter by321 instance_id: Optional instance ID to filter by322 minutes: Optional time window in minutes323 324 Returns:325 Dict containing annotation history data326 """327 if not self.check_admin_access():328 return {"error": "Admin access required"}, 403329 330 try:331 usm = get_user_state_manager()332 333 if user_id:334 # Get history for specific user335 user_state = usm.get_user_state(user_id)336 if not user_state:337 return {"error": f"User {user_id} not found"}, 404338 339 actions = user_state.get_annotation_history(instance_id)340 if minutes:341 actions = user_state.get_recent_actions(minutes)342 343 return self._format_annotation_history(actions, user_id)344 else:345 # Get history for all users346 all_actions = []347 users = get_users()348 349 for username in users:350 user_state = usm.get_user_state(username)351 if user_state:352 user_actions = user_state.get_annotation_history(instance_id)353 if minutes:354 user_actions = user_state.get_recent_actions(minutes)355 all_actions.extend(user_actions)356 357 return self._format_annotation_history(all_actions, "all_users")358 359 except Exception as e:360 self.logger.error(f"Error getting annotation history data: {e}")361 return {"error": f"Failed to get annotation history data: {str(e)}"}, 500362 363 def get_suspicious_activity_data(self) -> Dict[str, Any]:364 """365 Get comprehensive suspicious activity analysis.366 367 Returns:368 Dict containing suspicious activity data369 """370 if not self.check_admin_access():371 return {"error": "Admin access required"}, 403372 373 try:374 usm = get_user_state_manager()375 users = get_users()376 suspicious_data = []377 378 for username in users:379 user_state = usm.get_user_state(username)380 if user_state:381 suspicious_actions = user_state.get_suspicious_activity()382 if suspicious_actions:383 suspicious_data.append({384 "user_id": username,385 "suspicious_actions_count": len(suspicious_actions),386 "suspicious_actions": [387 {388 "action_id": action.action_id,389 "timestamp": action.timestamp.isoformat(),390 "instance_id": action.instance_id,391 "action_type": action.action_type,392 "schema_name": action.schema_name,393 "label_name": action.label_name,394 "server_processing_time_ms": action.server_processing_time_ms,395 "session_id": action.session_id396 }397 for action in suspicious_actions[:10] # Limit to 10 most recent398 ]399 })400 401 return {402 "total_users_with_suspicious_activity": len(suspicious_data),403 "suspicious_activity": suspicious_data404 }405 406 except Exception as e:407 self.logger.error(f"Error getting suspicious activity data: {e}")408 return {"error": f"Failed to get suspicious activity data: {str(e)}"}, 500409 410 def get_instances_data(self, page: int = 1, page_size: int = 25,411 sort_by: str = "annotation_count", sort_order: str = "desc",412 filter_completion: Optional[str] = None) -> Dict[str, Any]:413 """414 Get paginated instances data with sorting and filtering.415 416 Args:417 page: Page number (1-based)418 page_size: Number of instances per page419 sort_by: Field to sort by (annotation_count, completion_percentage, disagreement, id)420 sort_order: Sort order (asc, desc)421 filter_completion: Filter by completion status (completed, incomplete, all)422 423 Returns:424 Dict containing paginated instances data425 """426 if not self.check_admin_access():427 return {"error": "Admin access required"}, 403428 429 try:430 ism = get_item_state_manager()431 items = ism.items()432 433 # Convert items to InstanceData objects434 instances_data = []435 for item in items:436 item_id = item.get_id()437 annotators = ism.get_annotators_for_item(item_id)438 annotation_count = len(annotators) if annotators else 0439 440 # Calculate completion percentage441 max_annotations = config.get("max_annotations_per_item", -1)442 if max_annotations > 0:443 completion_percentage = min(100, (annotation_count / max_annotations) * 100)444 else:445 completion_percentage = 100 if annotation_count > 0 else 0446 447 # Calculate most frequent label and disagreement448 most_frequent_label, disagreement = self._calculate_label_statistics(item_id)449 450 # Calculate average time per annotation451 avg_time = self._calculate_average_time_per_annotation(item_id)452 453 instance_data = InstanceData(454 id=item_id,455 text=item.get_text(),456 displayed_text=item.get_displayed_text(),457 annotation_count=annotation_count,458 completion_percentage=completion_percentage,459 most_frequent_label=most_frequent_label,460 label_disagreement=disagreement,461 annotators=list(annotators) if annotators else [],462 average_time_per_annotation=avg_time,463 num_ai_instance=self._calculate_total_instance_ai(item_id)464 )465 instances_data.append(instance_data)466 467 # Apply filters468 if filter_completion == "completed":469 instances_data = [i for i in instances_data if i.completion_percentage >= 100]470 elif filter_completion == "incomplete":471 instances_data = [i for i in instances_data if i.completion_percentage < 100]472 473 # Apply sorting474 reverse = sort_order.lower() == "desc"475 if sort_by == "annotation_count":476 instances_data.sort(key=lambda x: x.annotation_count, reverse=reverse)477 elif sort_by == "completion_percentage":478 instances_data.sort(key=lambda x: x.completion_percentage, reverse=reverse)479 elif sort_by == "disagreement":480 instances_data.sort(key=lambda x: x.label_disagreement, reverse=reverse)481 elif sort_by == "id":482 instances_data.sort(key=lambda x: x.id, reverse=reverse)483 elif sort_by == "average_time":484 instances_data.sort(key=lambda x: x.average_time_per_annotation or 0, reverse=reverse)485 486 # Apply pagination487 total_instances = len(instances_data)488 start_idx = (page - 1) * page_size489 end_idx = start_idx + page_size490 paginated_instances = instances_data[start_idx:end_idx]491 492 # Convert to serializable format493 serialized_instances = []494 for instance in paginated_instances:495 serialized_instances.append({496 "id": instance.id,497 "text": instance.text[:100] + "..." if len(instance.text) > 100 else instance.text,498 "displayed_text": instance.displayed_text[:100] + "..." if len(instance.displayed_text) > 100 else instance.displayed_text,499 "annotation_count": instance.annotation_count,500 "completion_percentage": round(instance.completion_percentage, 1),501 "most_frequent_label": instance.most_frequent_label,502 "label_disagreement": round(instance.label_disagreement, 2),503 "annotators": instance.annotators,504 "num_ai_instance": instance.num_ai_instance,505 "average_time_per_annotation": self._format_seconds(instance.average_time_per_annotation) if instance.average_time_per_annotation else None506 })507 508 return {509 "instances": serialized_instances,510 "pagination": {511 "page": page,512 "page_size": page_size,513 "total_instances": total_instances,514 "total_pages": (total_instances + page_size - 1) // page_size,515 "has_next": end_idx < total_instances,516 "has_prev": page > 1517 },518 "summary": {519 "completed_instances": len([i for i in instances_data if i.completion_percentage >= 100]),520 "incomplete_instances": len([i for i in instances_data if i.completion_percentage < 100]),521 "average_annotations_per_instance": round(sum(i.annotation_count for i in instances_data) / len(instances_data), 1) if instances_data else 0,522 "average_disagreement": round(sum(i.label_disagreement for i in instances_data) / len(instances_data), 2) if instances_data else 0523 }524 }525 526 except Exception as e:527 self.logger.error(f"Error getting instances data: {e}")528 return {"error": f"Failed to get instances data: {str(e)}"}, 500529 530 def update_config(self, config_updates: Dict[str, Any]) -> Dict[str, Any]:531 """532 Update system configuration.533 534 Args:535 config_updates: Dictionary of configuration updates536 537 Returns:538 Dict containing update result539 """540 if not self.check_admin_access():541 return {"error": "Admin access required"}, 403542 543 try:544 # Validate and apply updates545 updated_fields = []546 547 for key, value in config_updates.items():548 if key in ["max_annotations_per_user", "max_annotations_per_item"]:549 if isinstance(value, int) and value >= -1:550 config[key] = value551 updated_fields.append(key)552 else:553 return {"error": f"Invalid value for {key}: must be integer >= -1"}, 400554 555 elif key == "assignment_strategy":556 valid_strategies = ["random", "fixed_order", "least_annotated", "max_diversity", "active_learning", "llm_confidence"]557 if value in valid_strategies:558 config[key] = value559 updated_fields.append(key)560 else:561 return {"error": f"Invalid assignment strategy: {value}"}, 400562 563 return {564 "status": "success",565 "message": f"Updated configuration fields: {', '.join(updated_fields)}",566 "updated_fields": updated_fields567 }568 569 except Exception as e:570 self.logger.error(f"Error updating config: {e}")571 return {"error": f"Failed to update config: {str(e)}"}, 500572 573 def get_questions_data(self) -> Dict[str, Any]:574 """575 Get aggregate analysis data for each annotation schema/question.576 577 Returns:578 Dict containing questions data with visualizations for different annotation types579 """580 if not self.check_admin_access():581 return {"error": "Admin access required"}, 403582 583 try:584 ism = get_item_state_manager()585 annotation_schemes = config.get("annotation_schemes", [])586 questions_data = []587 588 users = get_users()589 590 for scheme in annotation_schemes:591 scheme_name = scheme.get("name", "Unknown")592 annotation_type = scheme.get("annotation_type", "unknown")593 594 all_annotations = []595 item_annotations = {}596 597 for item in ism.items():598 item_id = item.get_id()599 item_annotations[item_id] = []600 601 for username in users:602 user_state = get_user_state_manager().get_user_state(username)603 if user_state:604 label_annotations = user_state.get_label_annotations(item_id)605 for label, value in label_annotations.items():606 label_schema = None607 label_name = None608 if hasattr(label, 'get_schema'):609 label_schema = label.get_schema()610 label_name = label.get_name()611 elif hasattr(label, 'schema'):612 label_schema = label.schema613 label_name = getattr(label, 'name', None)614 elif isinstance(label, str):615 label_schema = label616 617 if label_schema == scheme_name:618 normalized_value = label_name if label_name else value619 620 if annotation_type in ["radio", "select"]:621 normalized_value = self._normalize_categorical_value(normalized_value)622 elif annotation_type == "multiselect" and isinstance(normalized_value, list):623 normalized_value = [624 normalized_label625 for normalized_label in (626 self._normalize_categorical_value(v) for v in normalized_value627 )628 if normalized_label is not None629 ]630 631 if normalized_value is not None:632 all_annotations.append(normalized_value)633 item_annotations[item_id].append(normalized_value)634 635 analysis = self._analyze_annotation_scheme(636 annotation_type, scheme, all_annotations, item_annotations637 )638 639 questions_data.append({640 "name": scheme_name,641 "type": annotation_type,642 "description": scheme.get("description", ""),643 "total_annotations": len(all_annotations),644 "items_with_annotations": len([item_id for item_id, annotations in item_annotations.items() if annotations]),645 "analysis": analysis646 })647 648 return {649 "questions": questions_data,650 "summary": {651 "total_questions": len(questions_data),652 "total_annotations": sum(q["total_annotations"] for q in questions_data),653 "question_types": list(set(q["type"] for q in questions_data))654 }655 }656 657 except Exception as e:658 self.logger.error(f"Error getting questions data: {e}")659 return {"error": f"Failed to get questions data: {str(e)}"}, 500660 661 def _analyze_annotation_scheme(self, annotation_type: str, scheme: dict,662 all_annotations: list, item_annotations: dict) -> dict:663 """664 Analyze annotations based on their type and generate appropriate visualizations.665 """666 if not all_annotations:667 return {"error": "No annotations found"}668 669 analysis = {670 "type": annotation_type,671 "total_count": len(all_annotations)672 }673 674 if annotation_type in ["radio", "select"]:675 normalized_annotations = [676 normalized for normalized in677 (self._normalize_categorical_value(annotation) for annotation in all_annotations)678 if normalized is not None679 ]680 if not normalized_annotations:681 return {"error": "No annotations found"}682 683 label_counts = Counter(normalized_annotations)684 raw_labels = scheme.get("labels", [])685 labels = [686 normalized for normalized in687 (self._normalize_categorical_value(label) for label in raw_labels)688 if normalized is not None689 ]690 691 analysis.update({692 "visualization_type": "histogram",693 "data": {694 "labels": labels,695 "counts": [label_counts.get(label, 0) for label in labels],696 "percentages": [round(label_counts.get(label, 0) / len(normalized_annotations) * 100, 1)697 for label in labels]698 },699 "most_common": label_counts.most_common(1)[0] if label_counts else None,700 "agreement_score": self._calculate_agreement_score(item_annotations)701 })702 elif annotation_type == "multiselect":703 # Multi-label data - show label frequency and co-occurrence704 label_counts = Counter()705 co_occurrence = defaultdict(int)706 labels = scheme.get("labels", [])707 708 for annotations in item_annotations.values():709 if isinstance(annotations, list):710 # Count individual labels711 for annotation in annotations:712 if isinstance(annotation, list):713 for label in annotation:714 label_counts[label] += 1715 716 # Count co-occurrences717 for i, annotation1 in enumerate(annotations):718 if isinstance(annotation1, list):719 for j, annotation2 in enumerate(annotations):720 if i != j and isinstance(annotation2, list):721 for label1 in annotation1:722 for label2 in annotation2:723 if label1 < label2:724 co_occurrence[(label1, label2)] += 1725 726 analysis.update({727 "visualization_type": "multiselect_analysis",728 "data": {729 "labels": labels,730 "counts": [label_counts.get(label, 0) for label in labels],731 "percentages": [round(label_counts.get(label, 0) / len(item_annotations) * 100, 1)732 for label in labels],733 "co_occurrence": dict(co_occurrence)734 },735 "most_common": label_counts.most_common(3) if label_counts else [],736 "average_labels_per_item": round(sum(len(ann) if isinstance(ann, list) else 1737 for anns in item_annotations.values()738 for ann in anns) / len(all_annotations), 2)739 })740 741 elif annotation_type in ["likert", "number", "slider"]:742 # Numeric data - show distribution and statistics743 numeric_values = []744 for value in all_annotations:745 try:746 if isinstance(value, (int, float)):747 numeric_values.append(float(value))748 elif isinstance(value, str) and value.replace('.', '').replace('-', '').isdigit():749 numeric_values.append(float(value))750 except (ValueError, TypeError):751 continue752 753 if numeric_values:754 analysis.update({755 "visualization_type": "distribution",756 "data": {757 "values": numeric_values,758 "bins": self._create_histogram_bins(numeric_values, scheme),759 "statistics": {760 "mean": round(sum(numeric_values) / len(numeric_values), 2),761 "median": round(sorted(numeric_values)[len(numeric_values)//2], 2),762 "min": min(numeric_values),763 "max": max(numeric_values),764 "std": round((sum((x - sum(numeric_values)/len(numeric_values))**2765 for x in numeric_values) / len(numeric_values))**0.5, 2)766 }767 },768 "range": scheme.get("min", 0) if "min" in scheme else None,769 "max": scheme.get("max", 10) if "max" in scheme else None770 })771 else:772 analysis["error"] = "No valid numeric values found"773 774 elif annotation_type == "text":775 # Text data - show length distribution and common patterns776 text_lengths = []777 word_counts = []778 common_words = Counter()779 780 for value in all_annotations:781 if isinstance(value, str) and value.strip():782 text_lengths.append(len(value))783 words = value.lower().split()784 word_counts.append(len(words))785 common_words.update(words)786 787 if text_lengths:788 analysis.update({789 "visualization_type": "text_analysis",790 "data": {791 "lengths": text_lengths,792 "word_counts": word_counts,793 "common_words": common_words.most_common(10),794 "statistics": {795 "avg_length": round(sum(text_lengths) / len(text_lengths), 1),796 "avg_words": round(sum(word_counts) / len(word_counts), 1),797 "min_length": min(text_lengths),798 "max_length": max(text_lengths),799 "empty_responses": len([v for v in all_annotations800 if not isinstance(v, str) or not v.strip()])801 }802 }803 })804 else:805 analysis["error"] = "No valid text responses found"806 807 elif annotation_type == "span":808 # Span data - show coverage and overlap statistics809 span_counts = []810 total_spans = 0811 812 for annotations in item_annotations.values():813 if isinstance(annotations, list):814 for annotation in annotations:815 if isinstance(annotation, list):816 span_counts.append(len(annotation))817 total_spans += len(annotation)818 819 if span_counts:820 analysis.update({821 "visualization_type": "span_analysis",822 "data": {823 "span_counts": span_counts,824 "total_spans": total_spans,825 "statistics": {826 "avg_spans_per_item": round(sum(span_counts) / len(span_counts), 2),827 "items_with_spans": len([c for c in span_counts if c > 0]),828 "max_spans": max(span_counts) if span_counts else 0,829 "min_spans": min(span_counts) if span_counts else 0830 }831 }832 })833 else:834 analysis["error"] = "No valid span annotations found"835 836 else:837 analysis["error"] = f"Unsupported annotation type: {annotation_type}"838 839 return analysis840 841 def _calculate_agreement_score(self, item_annotations: dict) -> float:842 """Calculate agreement score for categorical annotations."""843 if not item_annotations:844 return 0.0845 846 agreement_scores = []847 for annotations in item_annotations.values():848 if len(annotations) > 1:849 # Calculate percentage of most common annotation850 counter = Counter(annotations)851 most_common_count = counter.most_common(1)[0][1]852 agreement_scores.append(most_common_count / len(annotations))853 854 return round(sum(agreement_scores) / len(agreement_scores) * 100, 1) if agreement_scores else 0.0855 856 def _create_histogram_bins(self, values: list, scheme: dict) -> dict:857 """Create histogram bins for numeric data."""858 if not values:859 return {"bins": [], "counts": []}860 861 min_val = scheme.get("min", min(values))862 max_val = scheme.get("max", max(values))863 864 # Create 10 bins865 bin_size = (max_val - min_val) / 10866 bins = [min_val + i * bin_size for i in range(11)]867 counts = [0] * 10868 869 for value in values:870 bin_index = min(int((value - min_val) / bin_size), 9)871 counts[bin_index] += 1872 873 return {874 "bins": [round(b, 2) for b in bins],875 "counts": counts876 }877 878 def _get_annotator_timing_data(self, user_id: str) -> Optional[AnnotatorTimingData]:879 """880 Get timing data for a specific annotator.881 882 Args:883 user_id: The user ID to get timing data for884 885 Returns:886 AnnotatorTimingData object or None if user not found887 """888 try:889 usm = get_user_state_manager()890 user_state = usm.get_user_state(user_id)891 892 if not user_state:893 return None894 895 # Get basic user info896 total_annotations = len(user_state.get_all_annotations())897 phase = str(user_state.get_phase())898 has_assignments = user_state.has_assignments()899 remaining_assignments = user_state.has_remaining_assignments()900 901 # Calculate timing data902 total_seconds = 0903 instance_times = []904 905 for instance_id, behavioral_data in user_state.instance_id_to_behavioral_data.items():906 instance_seconds = None907 # Handle both BehavioralData objects and plain dicts908 if hasattr(behavioral_data, 'total_time_ms'):909 # BehavioralData object (loaded from JSON)910 if behavioral_data.total_time_ms:911 instance_seconds = behavioral_data.total_time_ms / 1000.0912 elif isinstance(behavioral_data, dict):913 # Plain dict (runtime data)914 if behavioral_data.get("total_time_ms"):915 instance_seconds = behavioral_data["total_time_ms"] / 1000.0916 elif behavioral_data.get("time_string"):917 parsed_time = user_state.parse_time_string(behavioral_data["time_string"])918 if parsed_time:919 instance_seconds = parsed_time["total_seconds"]920 if instance_seconds is not None:921 total_seconds += instance_seconds922 instance_times.append(instance_seconds)923 924 # Calculate averages925 average_seconds_per_annotation = total_seconds / total_annotations if total_annotations > 0 else 0926 annotations_per_hour = (total_annotations * 3600) / total_seconds if total_seconds > 0 else 0927 928 # Get current instance time (if any)929 current_instance_time = None930 current_instance = user_state.get_current_instance()931 if current_instance:932 current_instance_id = current_instance.get_id()933 current_behavioral = user_state.instance_id_to_behavioral_data.get(current_instance_id)934 if current_behavioral:935 if hasattr(current_behavioral, 'total_time_ms'):936 if current_behavioral.total_time_ms:937 current_instance_time = current_behavioral.total_time_ms / 1000.0938 elif isinstance(current_behavioral, dict):939 if current_behavioral.get("total_time_ms"):940 current_instance_time = current_behavioral["total_time_ms"] / 1000.0941 elif current_behavioral.get("time_string"):942 parsed_current = user_state.parse_time_string(current_behavioral["time_string"])943 if parsed_current:944 current_instance_time = parsed_current["total_seconds"]945 946 # Estimate last activity (for now, use current time - this could be enhanced)947 last_activity = datetime.datetime.now()948 949 # NEW: Get annotation history metrics950 performance_metrics = user_state.get_performance_metrics()951 suspicious_analysis = AnnotationHistoryManager.detect_suspicious_activity(952 user_state.get_annotation_history()953 )954 recent_actions = user_state.get_recent_actions(5) # Last 5 minutes955 956 # Calculate session duration957 current_session_duration_minutes = None958 if user_state.session_start_time:959 duration = datetime.datetime.now() - user_state.session_start_time960 current_session_duration_minutes = duration.total_seconds() / 60961 962 # Get training statistics963 training_state = user_state.get_training_state()964 training_completed = training_state.is_passed() if training_state else False965 training_correct_answers = training_state.get_correct_answer_count() if training_state else 0966 training_total_attempts = training_state.get_total_attempts() if training_state else 0967 training_pass_rate = (training_correct_answers / training_total_attempts * 100) if training_total_attempts > 0 else 0968 training_current_question = training_state.get_current_question_index() if training_state else 0969 training_total_questions = len(training_state.get_training_instances()) if training_state else 0970 971 return AnnotatorTimingData(972 user_id=user_id,973 total_annotations=total_annotations,974 total_seconds=total_seconds,975 average_seconds_per_annotation=average_seconds_per_annotation,976 last_activity=last_activity,977 current_instance_time=current_instance_time,978 annotations_per_hour=annotations_per_hour,979 phase=phase,980 has_assignments=has_assignments,981 remaining_assignments=remaining_assignments,982 983 # NEW: Annotation history metrics984 total_actions=performance_metrics.get('total_actions', 0),985 average_action_time_ms=performance_metrics.get('average_action_time_ms', 0.0),986 fastest_action_time_ms=performance_metrics.get('fastest_action_time_ms', 0),987 slowest_action_time_ms=performance_metrics.get('slowest_action_time_ms', 0),988 actions_per_minute=performance_metrics.get('actions_per_minute', 0.0),989 suspicious_score=suspicious_analysis.get('suspicious_score', 0.0),990 suspicious_level=suspicious_analysis.get('suspicious_level', 'Normal'),991 fast_actions_count=suspicious_analysis.get('fast_actions_count', 0),992 burst_actions_count=suspicious_analysis.get('burst_actions_count', 0),993 session_start_time=user_state.session_start_time,994 current_session_duration_minutes=current_session_duration_minutes,995 recent_actions_count=len(recent_actions),996 997 # Training metrics998 training_completed=training_completed,999 training_correct_answers=training_correct_answers,1000 training_total_attempts=training_total_attempts,1001 training_pass_rate=training_pass_rate,1002 training_current_question=training_current_question,1003 training_total_questions=training_total_questions1004 )1005 1006 except Exception as e:1007 self.logger.error(f"Error getting timing data for user {user_id}: {e}")1008 return None1009 1010 def _extract_behavioral_total_seconds(self, behavioral_data: Any, user_state=None) -> Optional[float]:1011 """Extract total annotation time in seconds from behavioral data objects or legacy dicts."""1012 if not behavioral_data:1013 return None1014 1015 if hasattr(behavioral_data, 'total_time_ms') and behavioral_data.total_time_ms is not None:1016 return behavioral_data.total_time_ms / 1000.01017 1018 if isinstance(behavioral_data, dict):1019 total_time_ms = behavioral_data.get("total_time_ms")1020 if total_time_ms is not None:1021 return total_time_ms / 1000.01022 1023 time_string = behavioral_data.get("time_string")1024 if time_string and user_state and hasattr(user_state, 'parse_time_string'):1025 parsed_time = user_state.parse_time_string(time_string)1026 if parsed_time:1027 return parsed_time.get("total_seconds")1028 1029 return None1030 1031 def _extract_behavioral_ai_count(self, behavioral_data: Any) -> int:1032 """Extract AI usage count from behavioral data objects or legacy dicts."""1033 if not behavioral_data:1034 return 01035 1036 if hasattr(behavioral_data, 'ai_usage'):1037 ai_usage = behavioral_data.ai_usage or []1038 return len(ai_usage)1039 1040 if isinstance(behavioral_data, dict):1041 ai_usage = behavioral_data.get("ai_usage", []) or []1042 return len(ai_usage)1043 1044 return 01045 1046 def _calculate_total_instance_ai(self, instance_id: str) -> int:1047 """1048 Calculate total AI assistance events for an instance across all users.1049 1050 Args:1051 instance_id: The instance ID to analyze1052 1053 Returns:1054 Total number of AI usage events recorded for the instance1055 """1056 try:1057 usm = get_user_state_manager()1058 users = get_users()1059 1060 total_ai = 01061 for username in users:1062 user_state = usm.get_user_state(username)1063 if not user_state:1064 continue1065 1066 behavioral_data = user_state.instance_id_to_behavioral_data.get(instance_id)1067 total_ai += self._extract_behavioral_ai_count(behavioral_data)1068 1069 return total_ai1070 1071 except Exception as e:1072 self.logger.error(f"Error calculating AI statistics for instance {instance_id}: {e}")1073 return 01074 1075 def _calculate_average_time_per_annotation(self, instance_id: str) -> Optional[float]:1076 """1077 Calculate average time per annotation for an instance.1078 1079 Args:1080 instance_id: The instance ID to analyze1081 1082 Returns:1083 Average time in seconds or None if no data1084 """1085 try:1086 usm = get_user_state_manager()1087 users = get_users()1088 1089 total_time = 01090 annotation_count = 01091 1092 for username in users:1093 user_state = usm.get_user_state(username)1094 if user_state:1095 behavioral_data = user_state.instance_id_to_behavioral_data.get(instance_id)1096 total_seconds = self._extract_behavioral_total_seconds(behavioral_data, user_state)1097 if total_seconds is not None:1098 total_time += total_seconds1099 annotation_count += 11100 1101 return total_time / annotation_count if annotation_count > 0 else None1102 1103 except Exception as e:1104 self.logger.error(f"Error calculating average time for instance {instance_id}: {e}")1105 return None1106 1107 def _calculate_completion_percentage(self, user_id: str) -> float:1108 """1109 Calculate completion percentage for a user.1110 1111 Args:1112 user_id: The user ID to calculate completion for1113 1114 Returns:1115 Completion percentage (0-100)1116 """1117 try:1118 usm = get_user_state_manager()1119 user_state = usm.get_user_state(user_id)1120 1121 if not user_state:1122 return 0.01123 1124 total_assignments = user_state.get_assigned_instance_count()1125 completed_assignments = len(user_state.get_all_annotations())1126 1127 if total_assignments == 0:1128 return 0.01129 1130 return (completed_assignments / total_assignments) * 1001131 1132 except Exception as e:1133 self.logger.error(f"Error calculating completion percentage for user {user_id}: {e}")1134 return 0.01135 1136 def _format_seconds(self, seconds: Optional[float]) -> Optional[str]:1137 """1138 Format seconds into a human-readable string.1139 1140 Args:1141 seconds: Number of seconds to format1142 1143 Returns:1144 Formatted time string or None if input is None1145 """1146 if seconds is None:1147 return None1148 1149 if seconds < 60:1150 return f"{int(seconds)}s"1151 elif seconds < 3600:1152 minutes = int(seconds // 60)1153 remaining_seconds = int(seconds % 60)1154 return f"{minutes}m {remaining_seconds}s"1155 else:1156 hours = int(seconds // 3600)1157 remaining_minutes = int((seconds % 3600) // 60)1158 return f"{hours}h {remaining_minutes}m"1159 1160 def _format_annotation_history(self, actions: List[AnnotationAction], context: str) -> Dict[str, Any]:1161 """1162 Format annotation history data for API response.1163 1164 Args:1165 actions: List of annotation actions1166 context: Context string (user_id or "all_users")1167 1168 Returns:1169 Formatted annotation history data1170 """1171 if not actions:1172 return {1173 "context": context,1174 "total_actions": 0,1175 "actions": [],1176 "summary": {1177 "action_types": {},1178 "time_distribution": {},1179 "performance_metrics": {}1180 }1181 }1182 1183 # Calculate summary statistics1184 action_types = Counter(action.action_type for action in actions)1185 time_distribution = self._calculate_time_distribution(actions)1186 performance_metrics = AnnotationHistoryManager.calculate_performance_metrics(actions)1187 1188 # Format actions for response1189 formatted_actions = []1190 for action in actions[-100:]: # Limit to 100 most recent1191 formatted_actions.append({1192 "action_id": action.action_id,1193 "timestamp": action.timestamp.isoformat(),1194 "user_id": action.user_id,1195 "instance_id": action.instance_id,1196 "action_type": action.action_type,1197 "schema_name": action.schema_name,1198 "label_name": action.label_name,1199 "old_value": action.old_value,1200 "new_value": action.new_value,