iruno/test_wprm3
0
1import json2import base643import io4import html5from PIL import Image6 7 8def image_to_base64_url(image: str | Image.Image):9 if isinstance(image, str):10 with open(image, "rb") as f:11 image = f.read()12 elif isinstance(image, Image.Image):13 if image.mode in ("RGBA", "LA"):14 image = image.convert("RGB")15 with io.BytesIO() as buffer:16 image.save(buffer, format="PNG")17 image = buffer.getvalue()18 else:19 raise ValueError(f"Invalid image type: {type(image)}")20 21 return "data:image/png;base64," + base64.b64encode(image).decode("utf-8")22 23 24def load_json(file_path: str) -> dict:25 with open(file_path, "r") as f:26 return json.load(f)27 28def save_json(data: dict, file_path: str):29 with open(file_path, "w") as f:30 json.dump(data, f, indent=4)31 32def str_to_bool(s: str) -> bool:33 if s.lower() in ["true", "1", "yes", "y"]:34 return True35 elif s.lower() in ["false", "0", "no", "n"]:36 return False37 else:38 raise ValueError(f"Invalid boolean string: {s}")39 40 41def create_html_report(json_path, html_path, checklist_generation=False):42 """43 Reads the given JSON result file and generates a filterable HTML report.44 45 Args:46 json_path (str): Path to the input JSON file.47 html_path (str): Path to the output HTML file.48 """49 try:50 with open(json_path, 'r', encoding='utf-8') as f:51 data = json.load(f)52 except FileNotFoundError:53 print(f"Error: JSON file not found - {json_path}") # Error message in English54 return55 except json.JSONDecodeError:56 print(f"Error: JSON file parsing error - {json_path}") # Error message in English57 return58 except Exception as e:59 print(f"Unexpected error during data loading: {e}") # Error message in English60 return61 62 # Extract unique Task IDs and sort them63 task_ids = sorted(list(set(item.get("task_id") for item in data if item.get("task_id") is not None)))64 65 html_content = """66<!DOCTYPE html>67<html lang="en">68<head>69 <meta charset="UTF-8">70 <meta name="viewport" content="width=device-width, initial-scale=1.0">71 <title>Benchmark Results Report</title>72 <style>73 body { font-family: sans-serif; line-height: 1.6; padding: 20px; }74 .task-step { border: 1px solid #ccc; margin-bottom: 20px; padding: 15px; border-radius: 5px; background-color: #f9f9f9; }75 .task-step h2 { margin-top: 0; color: #333; border-bottom: 1px solid #eee; padding-bottom: 5px;}76 .task-step h3 { color: #555; margin-top: 15px; margin-bottom: 5px; }77 .task-step h4 { color: #777; margin-top: 10px; margin-bottom: 5px; font-style: italic;}78 pre { background-color: #eee; padding: 10px; border-radius: 3px; white-space: pre-wrap; word-wrap: break-word; font-size: 0.9em; margin-top: 5px; }79 details { margin-top: 10px; border: 1px solid #ddd; border-radius: 3px; background-color: #fff; }80 summary { cursor: pointer; padding: 8px; background-color: #f8f9fa; font-weight: bold; border-bottom: 1px solid #ddd; }81 details[open] summary { border-bottom: 1px solid #ddd; }82 details > pre { border: none; background-color: #fff; padding: 10px 8px; }83 .response-item-toggle { margin-top: 10px; }84 .chosen-section { border-left: 5px solid #4CAF50; padding-left: 10px; margin-top: 15px; }85 .rejected-section { border-left: 5px solid #f44336; padding-left: 10px; margin-top: 15px; }86 hr { border: 0; border-top: 1px solid #eee; margin: 15px 0; }87 .thought-action { background-color: #f0f0f0; padding: 10px; border-radius: 3px; margin-bottom: 10px; border: 1px solid #e0e0e0;}88 .thought-action h4 { margin-top: 0; color: #666; }89 .task-container { display: none; }90 .filter-controls { margin-bottom: 20px; padding: 10px; background-color: #e9ecef; border-radius: 5px; }91 .filter-controls label { margin-right: 10px; font-weight: bold; }92 .filter-controls select { padding: 5px; border-radius: 3px; border: 1px solid #ced4da; }93 </style>94</head>95<body>96 <h1>Benchmark Results Report</h1>97 98 <!-- Task ID Filter Dropdown -->99 <div class="filter-controls">100 <label for="taskSelector">Select Task ID:</label>101 <select id="taskSelector">102 <option value="">-- Show All --</option>103"""104 # Add dropdown options105 for tid in task_ids:106 html_content += f' <option value="{html.escape(str(tid))}">{html.escape(str(tid))}</option>\n'107 108 html_content += """109 </select>110 </div>111 112 <!-- Results Display Area -->113 <div id="resultsArea">114"""115 116 # Process each Task/Step data117 for i, step_data in enumerate(data):118 task_id = step_data.get("task_id", "N/A")119 step_id = step_data.get("step_id", "N/A")120 intent = step_data.get("intent", "N/A")121 start_url = step_data.get("start_url", "N/A")122 gt_checklist = step_data.get("gt_checklist", "N/A")123 generated_checklist = step_data.get("generated_checklist", None)124 trajectory = step_data.get("trajectory", "N/A")125 text_observation = step_data.get("text_observation", "N/A")126 source_name = step_data.get("source_name", "")127 128 # Wrap each Task/Step in a container with a unique ID (hidden initially)129 html_content += f"""130 <div class="task-container" data-task-id="{html.escape(str(task_id))}">131 <div class="task-step">132 <h2>Task ID: {html.escape(str(task_id))} | Step ID: {html.escape(str(step_id))} {f'({html.escape(source_name)})' if source_name else ''}</h2>133 <h3>Intent:</h3>134 <p>{html.escape(intent)}</p>135 <p><strong>Start URL:</strong> <a href="{html.escape(start_url)}" target="_blank">{html.escape(start_url)}</a></p>136 137 <h3>Ground Truth Checklist:</h3>138 <pre>{html.escape(gt_checklist)}</pre>139"""140 if checklist_generation and generated_checklist is not None:141 html_content += f"""142 <details>143 <summary>Generated Checklist (Click to expand/collapse)</summary>144 <pre>{html.escape(str(generated_checklist))}</pre>145 </details>146"""147 148 html_content += f"""149 <details>150 <summary>Trajectory (Click to expand/collapse)</summary>151 <pre>{html.escape(trajectory)}</pre>152 </details>153 154 <details>155 <summary>Text Observation (Click to expand/collapse)</summary>156 <pre>{html.escape(text_observation)}</pre>157 </details>158 <hr>159"""160 161 # Chosen Responses162 if 'chosen' in step_data and step_data['chosen']:163 html_content += '<div class="chosen-section"><h3>Chosen Responses:</h3>'164 for choice_block in step_data['chosen']:165 thought = choice_block.get('thought', 'N/A')166 action = choice_block.get('action', 'N/A')167 responses = choice_block.get('response', [])168 scores = choice_block.get('score', [])169 170 # Add Thought and Action information171 html_content += f"""172 <div class="thought-action">173 <h4>Thought:</h4>174 <pre>{html.escape(thought)}</pre>175 <h4>Action:</h4>176 <pre>{html.escape(action)}</pre>177 </div>"""178 179 # Loop through responses and create toggles180 for idx, (response, score) in enumerate(zip(responses, scores)):181 html_content += f"""182 <details class="response-item-toggle">183 <summary>Judge Response {idx + 1}: {html.escape(str(score))}</summary>184 <pre>{html.escape(str(response))}</pre>185 </details>"""186 html_content += '</div>' # End chosen-section187 188 # Rejected Responses189 if 'rejected' in step_data and step_data['rejected']:190 html_content += '<div class="rejected-section"><h3>Rejected Responses:</h3>'191 for rejection_block in step_data['rejected']:192 thought = rejection_block.get('thought', 'N/A')193 action = rejection_block.get('action', 'N/A')194 responses = rejection_block.get('response', [])195 scores = rejection_block.get('score', [])196 197 # Add Thought and Action information198 html_content += f"""199 <div class="thought-action">200 <h4>Thought:</h4>201 <pre>{html.escape(thought)}</pre>202 <h4>Action:</h4>203 <pre>{html.escape(action)}</pre>204 </div>"""205 206 # Loop through responses and create toggles207 for idx, (response, score) in enumerate(zip(responses, scores)):208 html_content += f"""209 <details class="response-item-toggle">210 <summary>Judge Response {idx + 1}: {html.escape(str(score))}</summary>211 <pre>{html.escape(str(response))}</pre>212 </details>"""213 html_content += '</div>' # End rejected-section214 215 html_content += """216 </div> <!-- End task-step -->217 </div> <!-- End task-container -->218"""219 220 # Finalize HTML and add JavaScript221 html_content += """222 </div> <!-- End resultsArea -->223 224 <script>225 document.addEventListener('DOMContentLoaded', function() {226 const taskSelector = document.getElementById('taskSelector');227 const taskContainers = document.querySelectorAll('.task-container');228 229 function filterTasks() {230 const selectedTaskId = taskSelector.value;231 232 taskContainers.forEach(container => {233 const containerTaskId = container.getAttribute('data-task-id');234 // Show if no Task ID is selected (Show All) or if the container's Task ID matches235 if (selectedTaskId === "" || containerTaskId === selectedTaskId) {236 container.style.display = 'block';237 } else {238 // Otherwise, hide it239 container.style.display = 'none';240 }241 });242 }243 244 // Run filter function on dropdown change245 taskSelector.addEventListener('change', filterTasks);246 247 // Run initial filtering on page load (default: Show All)248 filterTasks();249 });250 </script>251 252</body>253</html>254"""255 256 # Save the HTML file257 try:258 with open(html_path, 'w', encoding='utf-8') as f:259 f.write(html_content)260 print(f"Completed: HTML report created at {html_path}")261 except IOError:262 print(f"Error: Failed to write HTML file - {html_path}")263 except Exception as e:264 print(f"Unexpected error during HTML file saving: {e}")265 266# --- Example Usage ---267# input_json_file = 'path/to/your/results.json'268# output_html_file = 'trajectory_report.html'269# create_html_report(input_json_file, output_html_file)