AnonymousECCV15285/MMIB_Counterfactual_image_generation_tool
0
1import os2import argparse3import csv4import json5import random6import re7from pathlib import Path8 9def find_latest_run(base_output_dir):10 if not os.path.exists(base_output_dir):11 return None12 13 subdirs = [d for d in os.listdir(base_output_dir) 14 if os.path.isdir(os.path.join(base_output_dir, d))]15 16 if not subdirs:17 return None18 19 timestamped = [d for d in subdirs if re.match(r'^\d{8}_\d{6}$', d)]20 if timestamped:21 latest = sorted(timestamped)[-1]22 return os.path.join(base_output_dir, latest)23 24 dirs_with_time = [(d, os.path.getmtime(os.path.join(base_output_dir, d))) 25 for d in subdirs]26 latest = max(dirs_with_time, key=lambda x: x[1])[0]27 return os.path.join(base_output_dir, latest)28 29def find_scene_file(scenes_dir, image_filename):30 base_name = os.path.splitext(image_filename)[0]31 scene_file = os.path.join(scenes_dir, base_name + '.json')32 33 if os.path.exists(scene_file):34 return scene_file35 return None36 37def load_scene(scene_file):38 with open(scene_file, 'r') as f:39 return json.load(f)40 41RELATION_KEYS_TO_PHRASES = {'left': 'left of', 'right': 'right of', 'front': 'in front of', 'behind': 'behind'}42PHRASES_TO_RELATION_KEYS = {'left of': 'left', 'right of': 'right', 'in front of': 'front', 'behind': 'behind'}43DEFAULT_RELATIONS = ['left of', 'right of', 'in front of', 'behind']44# Geometric opposites for minimal spatial mutation (text counterfactuals)45RELATION_OPPOSITES = {'left of': 'right of', 'right of': 'left of', 'in front of': 'behind', 'behind': 'in front of'}46 47# Token sets for parsing questions48_COLORS = ['red', 'blue', 'green', 'brown', 'purple', 'cyan', 'yellow', 'gray', 'grey']49_SHAPES = ['cube', 'sphere', 'cylinder', 'cubes', 'spheres', 'cylinders']50_MATERIALS = ['metal', 'rubber', 'metals', 'rubbers']51_SIZES = ['small', 'large']52 53 54def _find_objects_matching(objects, color=None, shape=None, material=None, size=None):55 """Return list of object indices that match all specified attributes (None means any)."""56 out = []57 for i, obj in enumerate(objects):58 if color is not None and (obj.get('color') or '').lower() != color:59 continue60 if shape is not None:61 s = (obj.get('shape') or '').lower()62 if s != shape and s != shape.rstrip('s') and s + 's' != shape:63 continue64 if material is not None and (obj.get('material') or '').lower() != material:65 continue66 if size is not None and (obj.get('size') or '').lower() != size:67 continue68 out.append(i)69 return out70 71 72def _first_value_in_question(question_lower, values, strip_s=True):73 for v in values:74 if v in question_lower:75 return v.rstrip('s') if strip_s and v.endswith('s') else v76 return None77 78 79def _objects_in_relation_to_reference(scene, relation_phrase, ref_color=None, ref_shape=None, ref_material=None, ref_size=None):80 """Return set of object indices that stand in relation_phrase to the reference object (e.g. 'left of' the red cube)."""81 objects = scene.get('objects', [])82 rel_key = PHRASES_TO_RELATION_KEYS.get(relation_phrase)83 if not rel_key:84 return set()85 rels = scene.get('relationships') or {}86 rel_list = rels.get(rel_key)87 if not rel_list or len(rel_list) != len(objects):88 return set()89 ref_indices = _find_objects_matching(90 objects, color=ref_color, shape=ref_shape, material=ref_material, size=ref_size91 )92 if not ref_indices:93 return set()94 ref_idx = ref_indices[0]95 return set(rel_list[ref_idx])96 97 98def get_scene_properties(scene):99 objects = scene.get('objects', [])100 if not objects:101 return {102 'colors': ['red', 'blue', 'green'],103 'shapes': ['cube', 'sphere', 'cylinder'],104 'materials': ['metal', 'rubber'],105 'sizes': ['small', 'large'],106 'relations': DEFAULT_RELATIONS107 }108 109 colors = list(set(obj.get('color') for obj in objects if obj.get('color')))110 shapes = list(set(obj.get('shape') for obj in objects if obj.get('shape')))111 materials = list(set(obj.get('material') for obj in objects if obj.get('material')))112 sizes = list(set(obj.get('size') for obj in objects if obj.get('size')))113 relationships = scene.get('relationships') or {}114 relations = [RELATION_KEYS_TO_PHRASES[k] for k in relationships if k in RELATION_KEYS_TO_PHRASES]115 if not relations:116 relations = DEFAULT_RELATIONS117 118 all_colors = ['gray', 'red', 'blue', 'green', 'brown', 'purple', 'cyan', 'yellow']119 all_shapes = ['cube', 'sphere', 'cylinder']120 all_materials = ['metal', 'rubber']121 all_sizes = ['small', 'large']122 123 return {124 'colors': colors if colors else all_colors,125 'shapes': shapes if shapes else all_shapes,126 'materials': materials if materials else all_materials,127 'sizes': sizes if sizes else all_sizes,128 'relations': relations,129 'all_colors': all_colors,130 'all_shapes': all_shapes,131 'all_materials': all_materials,132 'all_sizes': all_sizes133 }134 135IMAGE_CF_TYPES = {136 'change_color', 'change_shape', 'change_size', 'change_material',137 'change_position', 'add_object', 'remove_object', 'replace_object',138 'swap_attribute', 'relational_flip'139}140NEGATIVE_CF_TYPES = {141 'change_background', 'change_lighting', 'add_noise',142 'apply_fisheye', 'apply_blur', 'apply_vignette', 'apply_chromatic_aberration',143 'occlusion_change'144}145 146# Allow more attempts to find a question where the original and CF images147# give different answers before giving up on a pair.148MAX_CF_ANSWER_RETRIES = 500149 150def get_cf_type_from_scene(scene):151 meta = scene.get('cf_metadata') or {}152 if not meta.get('is_counterfactual'):153 return None154 return meta.get('cf_type')155 156def get_cf_description_from_scene(scene):157 meta = scene.get('cf_metadata') or {}158 if not meta.get('is_counterfactual'):159 return None160 return meta.get('cf_description')161 162def get_change_details(original_scene, cf_scene):163 orig_objs = original_scene.get('objects', [])164 cf_objs = cf_scene.get('objects', [])165 if len(orig_objs) != len(cf_objs):166 return {'attribute': 'count', 'orig_count': len(orig_objs), 'cf_count': len(cf_objs)}167 attrs = ['color', 'shape', 'material', 'size']168 for i, (o, c) in enumerate(zip(orig_objs, cf_objs)):169 for attr in attrs:170 ov = (o.get(attr) or '').lower().strip()171 cv = (c.get(attr) or '').lower().strip()172 if ov != cv:173 return {'attribute': attr, 'orig_val': ov or 'unknown', 'cf_val': cv or 'unknown', 'object_index': i}174 return None175 176CF_COUNT_QUESTION_TEMPLATES = [177 "How many objects are in the scene?",178 "What is the total number of objects in the scene?",179]180CF_COLOR_QUESTION_TEMPLATES = [181 ("How many {val} objects are there?", 'color'),182 ("Are there any {val} objects?", 'color'),183 ("What is the total number of {val} objects?", 'color'),184]185CF_SHAPE_QUESTION_TEMPLATES = [186 ("How many {val} are there?", 'shape'),187 ("Are there any {val}?", 'shape'),188 ("What is the total number of {val}?", 'shape'),189]190CF_MATERIAL_QUESTION_TEMPLATES = [191 ("How many {val} objects are there?", 'material'),192 ("Are there any {val} objects?", 'material'),193 ("What is the total number of {val} objects?", 'material'),194]195CF_SIZE_QUESTION_TEMPLATES = [196 ("How many {val} objects are there?", 'size'),197 ("Are there any {val} objects?", 'size'),198 ("What is the total number of {val} objects?", 'size'),199]200 201 202def _pluralize_shape(shape):203 if not shape:204 return shape205 s = shape.strip().lower()206 if s.endswith('s'):207 return s208 return s + 's'209 210 211def _count_by_attribute(objects, attr):212 counts = {}213 for obj in objects:214 val = (obj.get(attr) or '').lower().strip()215 if val:216 counts[val] = counts.get(val, 0) + 1217 return counts218 219 220def _get_attributes_with_different_counts(original_scene, cf_scene):221 orig_objs = original_scene.get('objects', [])222 cf_objs = cf_scene.get('objects', [])223 differing = []224 for attr in ['color', 'shape', 'material', 'size']:225 orig_counts = _count_by_attribute(orig_objs, attr)226 cf_counts = _count_by_attribute(cf_objs, attr)227 all_vals = set(orig_counts) | set(cf_counts)228 for val in all_vals:229 o = orig_counts.get(val, 0)230 c = cf_counts.get(val, 0)231 if o != c:232 differing.append((attr, val, o, c))233 return differing234 235 236def _minimal_text_mutations(original_params, scene):237 """238 Yield (mutated_params,) for every minimal one-attribute mutation of original_params.239 Spatial: swap relation to geometric opposite. Attribute: swap color/shape/material/size240 to a different value from the CLEVR taxonomy (scene-based).241 """242 import copy243 props = get_scene_properties(scene)244 for key in original_params:245 current = original_params[key]246 mutated = copy.deepcopy(original_params)247 if key == 'relation' and current in RELATION_OPPOSITES:248 mutated[key] = RELATION_OPPOSITES[current]249 yield mutated250 elif key in ('color', 'color1', 'color2'):251 for v in props.get('all_colors', _COLORS):252 if (v or '').lower() != (current or '').lower():253 mutated[key] = v254 yield mutated255 elif key in ('shape', 'shape1', 'shape2'):256 for v in props.get('all_shapes', _SHAPES):257 v = v.rstrip('s') if v else v258 cur = (current or '').lower().rstrip('s')259 if v and v.lower() != cur:260 mutated[key] = v261 yield mutated262 elif key == 'material':263 for v in props.get('all_materials', _MATERIALS):264 if (v or '').lower() != (current or '').lower():265 mutated[key] = v266 yield mutated267 elif key == 'size':268 for v in props.get('all_sizes', _SIZES):269 if (v or '').lower() != (current or '').lower():270 mutated[key] = v271 yield mutated272 273 274def generate_question_for_counterfactual(cf_type, original_scene, cf_scene, retry_index=0, original_question=None, original_params=None, original_template=None):275 """276 Generate a counterfactual question.277 - Negative CFs (cf_type in NEGATIVE_CF_TYPES): return (original_question, params) only when278 answer(original_question, original_scene) == answer(original_question, cf_scene), so both279 invariants hold (no flip across images).280 - Semantic CFs with original_template: minimal text mutation + dual-flip constraint.281 - Otherwise: legacy flip-targeting.282 """283 # --- Negative CF: same question, answers must NOT flip across original vs negative_scene ---284 # Invariants: answer(orig_q, orig_scene) == answer(orig_q, cf_scene);285 # answer(cf_q, orig_scene) == answer(cf_q, cf_scene).286 # Use original_question as cf_question so both invariants hold iff original_question has same answer on both scenes.287 if cf_type in NEGATIVE_CF_TYPES and original_question is not None and original_params is not None:288 a_orig = answer_question_for_scene(original_question, original_scene)289 a_cf = answer_question_for_scene(original_question, cf_scene)290 if normalize_answer(a_orig) == normalize_answer(a_cf):291 return (original_question, dict(original_params))292 return (None, None)293 294 # --- Text counterfactual: minimal semantic edit with Dual-Flip Constraint ---295 # (1) Text flip on base scene: answer(original_scene, cf_question) != answer(original_scene, original_question)296 # (2) Image counterfactual: answer(original_scene, cf_question) != answer(cf_scene, cf_question)297 if original_question is not None and original_params is not None and original_template is not None:298 ans_orig = answer_question_for_scene(original_question, original_scene)299 ans_orig_n = normalize_answer(ans_orig)300 for mutated_params in _minimal_text_mutations(original_params, original_scene):301 try:302 cf_question = original_template.format(**mutated_params)303 except (KeyError, ValueError):304 continue305 ans_cf_q_on_orig = answer_question_for_scene(cf_question, original_scene)306 ans_cf_q_on_cf = answer_question_for_scene(cf_question, cf_scene)307 ans_cf_q_on_orig_n = normalize_answer(ans_cf_q_on_orig)308 ans_cf_q_on_cf_n = normalize_answer(ans_cf_q_on_cf)309 if (ans_cf_q_on_orig_n != ans_orig_n) and (ans_cf_q_on_orig_n != ans_cf_q_on_cf_n):310 return (cf_question, mutated_params)311 return (None, None)312 313 # --- Legacy: strict counterfactual targeting when original question (no template) is provided ---314 if original_question is not None and original_params is not None:315 a_orig = answer_question_for_scene(original_question, original_scene)316 a_cf = answer_question_for_scene(original_question, cf_scene)317 a_orig_n = normalize_answer(a_orig)318 a_cf_n = normalize_answer(a_cf)319 if a_orig_n != a_cf_n:320 return (original_question, original_params)321 for mut_q, mut_params in create_counterfactual_questions(original_question, original_params, original_scene):322 a_mut_cf = answer_question_for_scene(mut_q, cf_scene)323 if normalize_answer(a_mut_cf) != a_orig_n:324 return (mut_q, mut_params)325 return (None, None)326 327 random.seed(hash((str(cf_type), retry_index, str(id(original_scene)), str(id(cf_scene)))))328 change = get_change_details(original_scene, cf_scene)329 orig_objs = original_scene.get('objects', [])330 cf_objs = cf_scene.get('objects', [])331 props_orig = get_scene_properties(original_scene)332 props_cf = get_scene_properties(cf_scene)333 334 def _find_moved_indices_by_coords(thresh=1e-4):335 moved = []336 n = min(len(orig_objs), len(cf_objs))337 for i in range(n):338 oc = (orig_objs[i] or {}).get('3d_coords') or []339 cc = (cf_objs[i] or {}).get('3d_coords') or []340 if len(oc) < 3 or len(cc) < 3:341 continue342 dx = oc[0] - cc[0]343 dy = oc[1] - cc[1]344 dz = oc[2] - cc[2]345 if dx * dx + dy * dy + dz * dz > thresh:346 moved.append(i)347 return moved348 349 def _identity_relational_delta_question():350 """351 Identity-aware relational flip: build boolean question352 'Is the A left of the B?' whose truth flips between original and CF.353 """354 # First, try to parse structured info from cf_metadata description if present.355 meta = cf_scene.get('cf_metadata') or {}356 desc = (meta.get('cf_description') or '').lower()357 if desc and ('from' in desc and 'to' in desc):358 # Examples we expect:359 # "moved yellow sphere from right of gray sphere to left"360 # "moved red cube from left of blue sphere to right"361 moved_color = _first_value_in_question(desc, _COLORS)362 moved_shape = _first_value_in_question(desc, _SHAPES)363 # original relation phrase (contains 'left' / 'right' / 'front' / 'behind')364 orig_phrase = None365 for phrase in PHRASES_TO_RELATION_KEYS:366 if f"from {phrase}" in desc:367 orig_phrase = phrase368 break369 # reference object appears after the relation word370 ref_segment = ""371 if orig_phrase and f"from {orig_phrase}" in desc:372 _, _, after_from = desc.partition(f"from {orig_phrase}")373 # up to " to " describes the reference object374 ref_segment, _, _ = after_from.partition(" to ")375 ref_color = _first_value_in_question(ref_segment, _COLORS)376 ref_shape = _first_value_in_question(ref_segment, _SHAPES)377 if moved_color and moved_shape and orig_phrase and (ref_color or ref_shape):378 question = f"Is the {moved_color} {moved_shape} {orig_phrase} the {ref_color or ''} {ref_shape or ''}?".replace(" ", " ")379 params = {380 'moved_color': moved_color,381 'moved_shape': moved_shape,382 'relation': orig_phrase,383 'ref_color': ref_color,384 'ref_shape': (ref_shape or '').rstrip('s'),385 }386 return question, params387 388 relationships_orig = original_scene.get('relationships') or {}389 relationships_cf = cf_scene.get('relationships') or {}390 if not isinstance(relationships_orig, dict) or not isinstance(relationships_cf, dict):391 return None, None392 n = min(len(orig_objs), len(cf_objs))393 moved = _find_moved_indices_by_coords()394 candidate_indices = moved or list(range(n))395 for a_idx in candidate_indices:396 a_obj = orig_objs[a_idx] or {}397 a_color = (a_obj.get('color') or '').lower()398 a_shape = (a_obj.get('shape') or '').lower()399 if not a_color or not a_shape:400 continue401 for rel_key, phrase in RELATION_KEYS_TO_PHRASES.items():402 orig_list = relationships_orig.get(rel_key)403 cf_list = relationships_cf.get(rel_key)404 if not isinstance(orig_list, list) or not isinstance(cf_list, list):405 continue406 n_rel = min(len(orig_list), len(cf_list), n)407 for ref_idx in range(n_rel):408 if ref_idx == a_idx:409 continue410 o_targets = set(orig_list[ref_idx] or [])411 c_targets = set(cf_list[ref_idx] or [])412 # We want orig: True, cf: False413 if a_idx in o_targets and a_idx not in c_targets:414 ref = orig_objs[ref_idx] or {}415 r_color = (ref.get('color') or '').lower()416 r_shape = (ref.get('shape') or '').lower()417 if not r_color or not r_shape:418 continue419 question = f"Is the {a_color} {a_shape} {phrase} the {r_color} {r_shape}?"420 params = {421 'moved_color': a_color,422 'moved_shape': a_shape,423 'relation': phrase,424 'ref_color': r_color,425 'ref_shape': r_shape,426 }427 return question, params428 return None, None429 430 def _nearest_neighbor_delta_question():431 """432 Fallback: if no clean relational toggle, ask about nearest neighbor identity:433 'What color is the object closest to the cube?'434 """435 def _nearest_idx(objs, idx):436 base = (objs[idx] or {}).get('3d_coords') or []437 if len(base) < 3:438 return None439 bx, by, bz = base[:3]440 best = None441 best_d2 = None442 for j, o in enumerate(objs):443 if j == idx:444 continue445 coords = (o or {}).get('3d_coords') or []446 if len(coords) < 3:447 continue448 x, y, z = coords[:3]449 d2 = (x - bx) ** 2 + (y - by) ** 2 + (z - bz) ** 2450 if best_d2 is None or d2 < best_d2:451 best_d2 = d2452 best = j453 return best454 455 moved = _find_moved_indices_by_coords()456 n = min(len(orig_objs), len(cf_objs))457 candidate_indices = moved or list(range(n))458 for idx in candidate_indices:459 nn_o = _nearest_idx(orig_objs, idx)460 nn_c = _nearest_idx(cf_objs, idx)461 if nn_o is None or nn_c is None or nn_o == nn_c:462 continue463 target = orig_objs[idx] or {}464 t_shape = (target.get('shape') or '').lower()465 if not t_shape:466 continue467 question = f"What color is the object closest to the {t_shape}?"468 params = {'target_shape': t_shape}469 return question, params470 return None, None471 472 def _pick_spatial_question(props):473 """Strict spatial/relational templates only; never simple attribute count."""474 relations = props.get('relations') or DEFAULT_RELATIONS475 colors = list(props.get('colors') or props.get('all_colors') or ['red', 'blue', 'green'])476 shapes = list(props.get('shapes') or props.get('all_shapes') or ['cube', 'sphere', 'cylinder'])477 materials = list(props.get('materials') or props.get('all_materials') or ['metal', 'rubber'])478 sizes = list(props.get('sizes') or props.get('all_sizes') or ['small', 'large'])479 templates = [480 ("What color is the object {relation} the {color} {shape}?", {481 'relation': random.choice(relations), 'color': random.choice(colors), 'shape': random.choice(shapes)482 }),483 ("What shape is the object {relation} the {material} object?", {484 'relation': random.choice(relations), 'material': random.choice(materials)485 }),486 ("How many objects are {relation} the {color} {shape}?", {487 'relation': random.choice(relations), 'color': random.choice(colors), 'shape': random.choice(shapes)488 }),489 ("How many {material} objects are {relation} the {shape}?", {490 'material': random.choice(materials), 'relation': random.choice(relations), 'shape': random.choice(shapes)491 }),492 ("Is there a {color} object {relation} the {shape}?", {493 'color': random.choice(colors), 'relation': random.choice(relations), 'shape': random.choice(shapes)494 }),495 ("What is the total number of {size} objects {relation} the {color} object?", {496 'size': random.choice(sizes), 'relation': random.choice(relations), 'color': random.choice(colors)497 }),498 ("What is the total number of {material} objects {relation} the {color} {shape}?", {499 'material': random.choice(materials), 'relation': random.choice(relations),500 'color': random.choice(colors), 'shape': random.choice(shapes)501 }),502 ("Is there a {size} {material} object {relation} the {shape}?", {503 'size': random.choice(sizes), 'material': random.choice(materials),504 'relation': random.choice(relations), 'shape': random.choice(shapes)505 }),506 ]507 template, params = random.choice(templates)508 return template.format(**params), params509 510 def _pick_compositional_question(props):511 """Strict compositional (≥2 attributes) templates only; never single-attribute count."""512 colors = list(props.get('colors') or props.get('all_colors') or ['red', 'blue', 'green'])513 shapes = list(props.get('shapes') or props.get('all_shapes') or ['cube', 'sphere', 'cylinder'])514 materials = list(props.get('materials') or props.get('all_materials') or ['metal', 'rubber'])515 sizes = list(props.get('sizes') or props.get('all_sizes') or ['small', 'large'])516 templates = [517 ("How many {color} {shape}s are there?", {518 'color': random.choice(colors), 'shape': random.choice(shapes)519 }),520 ("Are there any {color} {shape}s?", {521 'color': random.choice(colors), 'shape': random.choice(shapes)522 }),523 ("Is there a {color} {shape}?", {524 'color': random.choice(colors), 'shape': random.choice(shapes)525 }),526 ("Is there a {material} {shape}?", {527 'material': random.choice(materials), 'shape': random.choice(shapes)528 }),529 ("How many {size} {color} objects are there?", {530 'size': random.choice(sizes), 'color': random.choice(colors)531 }),532 ("What is the total number of {color} {material} objects?", {533 'color': random.choice(colors), 'material': random.choice(materials)534 }),535 ("Are there any {material} {shape}s?", {536 'material': random.choice(materials), 'shape': random.choice(shapes)537 }),538 ("How many {size} {shape}s are there?", {539 'size': random.choice(sizes), 'shape': random.choice(shapes)540 }),541 # Direct identity question about color of a shiny/matte object, e.g. "What color is the shiny cylinder?"542 ("What color is the shiny {shape}?", {543 'shape': random.choice(shapes)544 }),545 ("What color is the matte {shape}?", {546 'shape': random.choice(shapes)547 }),548 ]549 template, params = random.choice(templates)550 return template.format(**params), params551 552 # --- change_position / relational_flip: use explicit scene-graph deltas ---553 if cf_type in ('change_position', 'relational_flip'):554 question, params = _identity_relational_delta_question()555 if question:556 return question, params557 question, params = _nearest_neighbor_delta_question()558 if question:559 return question, params560 561 # --- swap_attribute: STRICTLY compositional (≥2 attributes) only; never single-attribute count ---562 if cf_type == 'swap_attribute':563 props = props_cf if (props_cf.get('colors') or props_cf.get('shapes')) else props_orig564 question, params = _pick_compositional_question(props)565 return question, params566 567 if cf_type and cf_type in IMAGE_CF_TYPES:568 differing = _get_attributes_with_different_counts(original_scene, cf_scene)569 if differing:570 idx = retry_index % len(differing) if differing else 0571 attr, val, orig_count, cf_count = differing[idx]572 if attr == 'color':573 template, _ = random.choice(CF_COLOR_QUESTION_TEMPLATES)574 question = template.format(val=val)575 elif attr == 'shape':576 plural = _pluralize_shape(val)577 template, _ = random.choice(CF_SHAPE_QUESTION_TEMPLATES)578 question = template.format(val=plural)579 elif attr == 'material':580 template, _ = random.choice(CF_MATERIAL_QUESTION_TEMPLATES)581 question = template.format(val=val)582 elif attr == 'size':583 template, _ = random.choice(CF_SIZE_QUESTION_TEMPLATES)584 question = template.format(val=val)585 else:586 question = None587 if question:588 return question, {attr: val.rstrip('s') if attr == 'shape' else val}589 590 if cf_type and cf_type in NEGATIVE_CF_TYPES:591 templates = [592 ("How many objects are in the scene?", {}),593 ("How many {color} objects are there?", {'color': random.choice(props_orig['colors'])} if props_orig['colors'] else None),594 ("Are there any {shape} objects?", {'shape': random.choice(props_orig['shapes'])} if props_orig['shapes'] else None),595 ("How many {material} objects are there?", {'material': random.choice(props_orig['materials'])} if props_orig['materials'] else None),596 ("What is the total number of {size} objects?", {'size': random.choice(props_orig['sizes'])} if props_orig['sizes'] else None),597 ]598 valid = [(t, p) for t, p in templates if p is not None or t.startswith("How many objects are in")]599 if not valid:600 valid = [("How many objects are in the scene?", {})]601 template, params = random.choice(valid)602 params = params or {}603 question = template.format(**params) if params else template604 return question, params605 606 if change and change.get('attribute') == 'count':607 orig_count = change.get('orig_count', len(orig_objs))608 cf_count = change.get('cf_count', len(cf_objs))609 templates_with_params = []610 templates_with_params.append((random.choice(CF_COUNT_QUESTION_TEMPLATES), {}))611 if cf_count > orig_count:612 templates_with_params.append((f"Are there more than {orig_count} objects?", {}))613 templates_with_params.append((f"Are there at least {cf_count} objects?", {}))614 if cf_count < orig_count:615 templates_with_params.append((f"Are there fewer than {orig_count} objects?", {}))616 templates_with_params.append((f"Are there more than {cf_count} objects?", {}))617 template, params = random.choice(templates_with_params)618 return template, params619 620 if change and change.get('attribute') in ('color', 'shape', 'material', 'size'):621 attr = change['attribute']622 cf_val = (change.get('cf_val') or '').strip().lower()623 if not cf_val:624 cf_val = 'unknown'625 params = {attr: cf_val}626 if attr == 'color':627 template, _ = random.choice(CF_COLOR_QUESTION_TEMPLATES)628 question = template.format(val=cf_val)629 elif attr == 'shape':630 template, _ = random.choice(CF_SHAPE_QUESTION_TEMPLATES)631 plural = _pluralize_shape(cf_val)632 question = template.format(val=plural)633 params['shape'] = cf_val.rstrip('s')634 elif attr == 'material':635 template, _ = random.choice(CF_MATERIAL_QUESTION_TEMPLATES)636 question = template.format(val=cf_val)637 elif attr == 'size':638 template, _ = random.choice(CF_SIZE_QUESTION_TEMPLATES)639 question = template.format(val=cf_val)640 else:641 question = "How many objects are in the scene?"642 params = {}643 return question, params644 645 if cf_type in ('add_object', 'remove_object'):646 templates = list(CF_COUNT_QUESTION_TEMPLATES)647 if len(orig_objs) != len(cf_objs):648 if len(cf_objs) > len(orig_objs):649 templates.extend([f"Are there more than {len(orig_objs)} objects?", f"Are there at least {len(cf_objs)} objects?"])650 else:651 templates.extend([f"Are there fewer than {len(orig_objs)} objects?", f"Are there more than {len(cf_objs)} objects?"])652 template = random.choice(templates)653 return template, {}654 655 # Explicit attribute-locked templates for semantic CFs:656 # The queried variable must match the intervened variable.657 if cf_type == 'change_color':658 vals = list(props_cf.get('colors') or props_orig.get('colors') or [])659 if vals:660 val = random.choice(vals)661 template, _ = random.choice(CF_COLOR_QUESTION_TEMPLATES)662 question = template.format(val=val)663 return question, {'color': val}664 665 if cf_type in ('change_shape', 'replace_object'):666 vals = list(props_cf.get('shapes') or props_orig.get('shapes') or [])667 if vals:668 val = random.choice(vals)669 plural = _pluralize_shape(val)670 template, _ = random.choice(CF_SHAPE_QUESTION_TEMPLATES)671 question = template.format(val=plural)672 return question, {'shape': val.rstrip('s')}673 674 if cf_type == 'change_material':675 vals = list(props_cf.get('materials') or props_orig.get('materials') or [])676 if vals:677 val = random.choice(vals)678 template, _ = random.choice(CF_MATERIAL_QUESTION_TEMPLATES)679 question = template.format(val=val)680 return question, {'material': val}681 682 if cf_type == 'change_size':683 vals = list(props_cf.get('sizes') or props_orig.get('sizes') or [])684 if vals:685 val = random.choice(vals)686 template, _ = random.choice(CF_SIZE_QUESTION_TEMPLATES)687 question = template.format(val=val)688 return question, {'size': val}689 690 # Fallback: never use generic "How many objects?" for change_position, relational_flip, swap_attribute.691 if cf_type in ('change_position', 'relational_flip', 'swap_attribute'):692 props = props_cf if (props_cf.get('relations') or props_cf.get('colors')) else props_orig693 if cf_type == 'swap_attribute':694 question, params = _pick_compositional_question(props)695 else:696 question, params = _pick_spatial_question(props)697 return question, params698 question = random.choice(CF_COUNT_QUESTION_TEMPLATES)699 return question, {}700 701def generate_question_for_scene(scene_file, retry_index=None):702 scene = load_scene(scene_file)703 objects = scene.get('objects', [])704 705 if len(objects) == 0:706 return "How many objects are in the scene?", {}, "How many objects are in the scene?"707 708 props = get_scene_properties(scene)709 710 templates = [711 ("How many objects are in the scene?", {}),712 ("How many {color} objects are there?", {'color': random.choice(props['colors'])}),713 ("Are there any {shape} objects?", {'shape': random.choice(props['shapes'])}),714 ("Are there any {shape}s present?", {'shape': random.choice(props['shapes'])}),715 ("Is there a {color} {shape}?", {716 'color': random.choice(props['colors']),717 'shape': random.choice(props['shapes'])718 }),719 ("How many {material} objects are there?", {'material': random.choice(props['materials'])}),720 ("What is the total number of {material} objects?", {'material': random.choice(props['materials'])}),721 ("What is the total number of metallic objects?", {}),722 ("What is the total number of {size} objects?", {'size': random.choice(props['sizes'])}),723 ("Is there a {material} {shape}?", {724 'material': random.choice(props['materials']),725 'shape': random.choice(props['shapes'])726 }),727 ("How many {size} {color} objects are there?", {728 'size': random.choice(props['sizes']),729 'color': random.choice(props['colors'])730 }),731 ("Are there any {color} {shape}s?", {732 'color': random.choice(props['colors']),733 'shape': random.choice(props['shapes'])734 }),735 ("What is the total number of {color} {material} objects?", {736 'color': random.choice(props['colors']),737 'material': random.choice(props['materials'])738 }),739 ("What color is the object {relation} the {color} {shape}?", {740 'relation': random.choice(props['relations']),741 'color': random.choice(props['colors']),742 'shape': random.choice(props['shapes'])743 }),744 ("What shape is the object {relation} the {material} object?", {745 'relation': random.choice(props['relations']),746 'material': random.choice(props['materials'])747 }),748 ("What material is the {size} object {relation} the {shape}?", {749 'size': random.choice(props['sizes']),750 'relation': random.choice(props['relations']),751 'shape': random.choice(props['shapes'])752 }),753 ("How many objects are {relation} the {color} {shape}?", {754 'relation': random.choice(props['relations']),755 'color': random.choice(props['colors']),756 'shape': random.choice(props['shapes'])757 }),758 ("How many {material} objects are {relation} the {shape}?", {759 'material': random.choice(props['materials']),760 'relation': random.choice(props['relations']),761 'shape': random.choice(props['shapes'])762 }),763 ("What is the total number of {size} objects {relation} the {color} object?", {764 'size': random.choice(props['sizes']),765 'relation': random.choice(props['relations']),766 'color': random.choice(props['colors'])767 }),768 ("Is there a {color} object {relation} the {shape}?", {769 'color': random.choice(props['colors']),770 'relation': random.choice(props['relations']),771 'shape': random.choice(props['shapes'])772 }),773 ("Are there any {material} {shape}s {relation} the {size} object?", {774 'material': random.choice(props['materials']),775 'shape': random.choice(props['shapes']),776 'relation': random.choice(props['relations']),777 'size': random.choice(props['sizes'])778 }),779 # --- Attribute Equivalence (Same/Different) ---780 ("Is the color of the {shape1} the same as the {shape2}?", {781 'shape1': random.choice(props['shapes']),782 'shape2': random.choice(props['shapes'])783 }),784 ("Is the material of the {color} object the same as the {size} object?", {785 'color': random.choice(props['colors']),786 'size': random.choice(props['sizes'])787 }),788 ("Do the {size} object and the {material} object have the same shape?", {789 'size': random.choice(props['sizes']),790 'material': random.choice(props['materials'])791 }),792 # --- Logical Disjunction (OR) ---793 ("How many objects are either {color} or {shape}?", {794 'color': random.choice(props['colors']),795 'shape': random.choice(props['shapes'])796 }),797 ("Are there any objects that are either {material} or {color}?", {798 'material': random.choice(props['materials']),799 'color': random.choice(props['colors'])800 }),801 ("What is the total number of objects that are either {size} or {shape}?", {802 'size': random.choice(props['sizes']),803 'shape': random.choice(props['shapes'])804 }),805 # --- Exact Numerical Comparison ---806 ("Is the number of {color} objects equal to the number of {shape}s?", {807 'color': random.choice(props['colors']),808 'shape': random.choice(props['shapes'])809 }),810 ("Are there exactly as many {material} objects as {size} objects?", {811 'material': random.choice(props['materials']),812 'size': random.choice(props['sizes'])813 }),814 ("Does the scene contain an equal number of {color1} objects and {color2} objects?", dict(zip(815 ['color1', 'color2'],816 random.sample(props['colors'], 2) if len(props['colors']) >= 2 else [props['colors'][0]] * 2817 ))),818 # --- Complex Spatial & Attribute Composition ---819 ("What is the total number of {material} objects {relation} the {color} {shape}?", {820 'material': random.choice(props['materials']),821 'relation': random.choice(props['relations']),822 'color': random.choice(props['colors']),823 'shape': random.choice(props['shapes'])824 }),825 ("Is there a {size} {material} object {relation} the {shape}?", {826 'size': random.choice(props['sizes']),827 'material': random.choice(props['materials']),828 'relation': random.choice(props['relations']),829 'shape': random.choice(props['shapes'])830 }),831 ]832 # Add matte/shin only when the scene has a metal/rubber object; caller should accept only when CF is attribute-swap (not add/remove).833 matte_shiny_objects = [o for o in objects if (o.get('material') or '').lower() in ('metal', 'rubber') and o.get('color') and o.get('shape')]834 if matte_shiny_objects:835 obj = random.choice(matte_shiny_objects)836 templates.append(("Is the {color} {shape} matte or shiny?", {'color': obj.get('color'), 'shape': obj.get('shape')}))837 if retry_index is not None:838 random.seed(hash((scene_file, retry_index)))839 else:840 random.seed(hash(scene_file))841 template, params = random.choice(templates)842 843 question = template.format(**params) if params else template844 845 return question, params, template846 847def calculate_question_difficulty(question, params):848 num_params = len(params) if params else 0849 850 question_lower = question.lower()851 852 if "matte or shiny" in question_lower or ("or" in question_lower and ("matte" in question_lower or "shiny" in question_lower)):853 return "hard"854 elif "metallic" in question_lower:855 return "medium"856 elif "total number" in question_lower and num_params >= 1:857 return "hard" if num_params >= 2 else "medium"858 elif num_params == 0:859 return "easy"860 elif num_params == 1:861 return "medium"862 else:863 return "hard"864 865def _apply_param_replacements(question, params, cf_params):866 """Replace param values in question with cf_params, from last to first by position, to avoid double-replacing when the same value appears for different placeholders."""867 if not params or not cf_params:868 return question869 # Order keys by first occurrence of their value in the question (so we replace in document order)870 positions = []871 for k, v in params.items():872 if k not in cf_params or cf_params[k] == v:873 continue874 pos = question.find(v)875 if pos >= 0:876 positions.append((pos, k, v, cf_params[k]))877 # Replace from end to start so indices stay valid878 positions.sort(key=lambda x: -x[0])879 for pos, k, old_val, new_val in positions:880 question = question[:pos] + new_val + question[pos + len(old_val):]881 return question882 883 884def create_counterfactual_questions(original_question, params, scene):885 props = get_scene_properties(scene)886 cf_questions = []887 888 strategies = ['attribute_swap', 'question_type', 'scope_change',889 'negation', 'comparative', 'multi_attribute',890 'same_different', 'either_or', 'equal_comparison']891 892 random.seed(hash(str(scene)))893 selected_strategies = random.sample(strategies, 2)894 895 for strategy in selected_strategies:896 cf_q = None897 cf_params = {}898 max_retries = 5899 retry_count = 0900 901 while retry_count < max_retries:902 cf_q = None903 cf_params = {}904 905 if strategy == 'attribute_swap' and params:906 cf_params = params.copy()907 param_to_change = random.choice(list(params.keys()))908 current = params.get(param_to_change)909 910 def pick_alternative(attr_key, all_vals_getter):911 alts = [v for v in all_vals_getter() if v != current]912 if alts:913 cf_params[param_to_change] = random.choice(alts)914 return True915 return False916 917 if param_to_change in ('color', 'color1', 'color2'):918 if not pick_alternative('color', lambda: props['all_colors']):919 strategy = 'negation'920 continue921 elif param_to_change in ('shape', 'shape1', 'shape2'):922 if not pick_alternative('shape', lambda: props['all_shapes']):923 strategy = 'negation'924 continue925 elif param_to_change == 'material':926 if not pick_alternative('material', lambda: props['all_materials']):927 strategy = 'negation'928 continue929 elif param_to_change == 'size':930 if not pick_alternative('size', lambda: props['all_sizes']):931 strategy = 'negation'932 continue933 elif param_to_change == 'relation':934 if not pick_alternative('relation', lambda: props['relations']):935 strategy = 'negation'936 continue937 else:938 retry_count += 1939 continue940 941 cf_q = _apply_param_replacements(original_question, params, cf_params)942 943 elif strategy == 'question_type':944 cf_params = params.copy() if params else {}945 if "How many" in original_question and "objects are in the scene" in original_question:946 if props['colors']:947 color = random.choice(props['colors'])948 cf_q = f"How many {color} objects are there?"949 cf_params = {'color': color}950 elif props['shapes']:951 shape = random.choice(props['shapes'])952 cf_q = f"Are there any {shape}s?"953 cf_params = {'shape': shape}954 else:955 cf_q = "Are there more than 3 objects?"956 cf_params = {}957 elif "How many" in original_question:958 cf_q = original_question.replace("How many", "Are there any")959 cf_q = cf_q.replace(" are there?", "?")960 cf_q = cf_q.replace(" are in the scene?", " in the scene?")961 elif "Are there" in original_question or "Is there" in original_question:962 if "Are there any" in original_question:963 cf_q = original_question.replace("Are there any", "How many")964 if not cf_q.endswith(" are there?"):965 cf_q = cf_q.replace("?", " are there?")966 elif "Is there a" in original_question:967 cf_q = original_question.replace("Is there a", "How many")968 if not cf_q.endswith(" are there?"):969 cf_q = cf_q.replace("?", " are there?")970 else:971 if props['colors']:972 color = random.choice(props['colors'])973 cf_q = f"How many {color} objects are there?"974 cf_params = {'color': color}975 else:976 cf_q = "How many objects are in the scene?"977 cf_params = {}978 elif "What is" in original_question:979 cf_q = original_question.replace("What is the total number of", "How many")980 else:981 if props['colors']:982 color = random.choice(props['colors'])983 cf_q = f"How many {color} objects are there?"984 cf_params = {'color': color}985 else:986 cf_q = "Are there more than 3 objects?"987 cf_params = {}988 989 elif strategy == 'scope_change':990 if params and len(params) >= 2:991 cf_params = params.copy()992 key_to_remove = random.choice(list(params.keys()))993 del cf_params[key_to_remove]994 995 if len(cf_params) == 1:996 attr_val = list(cf_params.values())[0]997 cf_q = f"How many {attr_val} objects are there?"998 else:999 if props['colors']:1000 color = random.choice(props['colors'])1001 cf_q = f"How many {color} objects are there?"1002 cf_params = {'color': color}1003 else:1004 cf_q = "Are there more than 3 objects?"1005 cf_params = {}1006 elif params and len(params) == 1:1007 new_attr = random.choice(['material', 'size'])1008 if new_attr not in params:1009 new_val = random.choice(props[new_attr + 's'])1010 existing_key = list(params.keys())[0]1011 existing_val = list(params.values())[0]1012 cf_params = params.copy()1013 cf_params[new_attr] = new_val1014 if new_attr == 'size':1015 cf_q = f"How many {new_val} {existing_val} objects are there?"1016 elif new_attr == 'material':1017 if existing_key == 'size':1018 cf_q = f"How many {existing_val} {new_val} objects are there?"1019 else:1020 cf_q = f"How many {existing_val} {new_val} objects are there?"1021 else:1022 strategy = 'negation'1023 continue1024 else:1025 if props['colors']:1026 color = random.choice(props['colors'])1027 cf_params = {'color': color}1028 cf_q = f"How many {color} objects are there?"1029 elif props['shapes']:1030 shape = random.choice(props['shapes'])1031 cf_params = {'shape': shape}1032 cf_q = f"Are there any {shape}s?"1033 else:1034 cf_q = "Are there more than 3 objects?"1035 cf_params = {}1036 1037 elif strategy == 'negation':1038 cf_params = params.copy() if params else {}1039 if params:1040 if 'color' in params:1041 color = params['color']1042 cf_q = f"How many objects are NOT {color}?"1043 elif 'shape' in params:1044 shape = params['shape']1045 cf_q = f"How many objects are NOT {shape}s?"1046 else:1047 attr_val = list(params.values())[0]1048 cf_q = f"How many objects are NOT {attr_val}?"1049 else:1050 cf_q = "Are there fewer than 5 objects?"1051 cf_params = {}1052 1053 elif strategy == 'comparative':1054 cf_params = params.copy() if params else {}1055 if "How many" in original_question:1056 number = random.choice([2, 3, 4, 5])1057 cf_q = original_question.replace("How many", f"Are there more than {number}")1058 cf_q = cf_q.replace(" are there?", "?")1059 cf_q = cf_q.replace(" are in the scene?", " in the scene?")1060 elif params:1061 if 'color' in params:1062 color1 = params['color']1063 alternatives = [c for c in props['all_colors'] if c != color1]1064 if alternatives:1065 color2 = random.choice(alternatives)1066 cf_params = {'color': color1, 'color2': color2}1067 cf_q = f"Are there more {color1} objects than {color2} objects?"1068 else:1069 cf_q = f"How many objects are NOT {color1}?"1070 cf_params = {'color': color1}1071 elif 'shape' in params:1072 shape1 = params['shape']1073 alternatives = [s for s in props['all_shapes'] if s != shape1]1074 if alternatives:1075 shape2 = random.choice(alternatives)1076 cf_params = {'shape': shape1, 'shape2': shape2}1077 cf_q = f"Are there more {shape1}s than {shape2}s?"1078 else:1079 cf_q = f"How many objects are NOT {shape1}s?"1080 cf_params = {'shape': shape1}1081 else:1082 cf_q = "Are there more than 3 objects?"1083 cf_params = {}1084 else:1085 cf_q = "Are there more than 3 objects?"1086 cf_params = {}1087 1088 elif strategy == 'multi_attribute':1089 if params and len(params) >= 2:1090 cf_params = {}1091 changed = False1092 for key in params:1093 if key == 'color':1094 alternatives = [c for c in props['all_colors'] if c != params[key]]1095 if alternatives:1096 cf_params[key] = random.choice(alternatives)1097 changed = True1098 else:1099 cf_params[key] = params[key]1100 elif key == 'shape':1101 alternatives = [s for s in props['all_shapes'] if s != params[key]]1102 if alternatives:1103 cf_params[key] = random.choice(alternatives)1104 changed = True1105 else:1106 cf_params[key] = params[key]1107 elif key == 'material':1108 alternatives = [m for m in props['all_materials'] if m != params[key]]1109 if alternatives:1110 cf_params[key] = random.choice(alternatives)1111 changed = True1112 else:1113 cf_params[key] = params[key]1114 elif key == 'size':1115 alternatives = [s for s in props['all_sizes'] if s != params[key]]1116 if alternatives:1117 cf_params[key] = random.choice(alternatives)1118 changed = True1119 else:1120 cf_params[key] = params[key]1121 1122 if not changed:1123 strategy = 'negation'1124 continue1125 1126 attr_order = ['size', 'color', 'material', 'shape']1127 ordered_values = []1128 for attr in attr_order:1129 if attr in cf_params:1130 ordered_values.append(cf_params[attr])1131 cf_q = f"How many {' '.join(ordered_values)} objects are there?"1132 else:1133 color = random.choice(props['colors'])1134 shape = random.choice(props['shapes'])1135 cf_params = {'color': color, 'shape': shape}1136 cf_q = f"Is there a {color} {shape}?"1137 1138 elif strategy == 'same_different':1139 # Attribute equivalence: "same as" / "same shape/color/material" -> swap one compared attribute or "same" -> "different"1140 q = original_question1141 q_lower = q.lower()1142 if "the same as" in q_lower or "same shape" in q_lower or "same color" in q_lower or "same material" in q_lower:1143 if random.choice([True, False]) and params:1144 # Swap one of the compared attributes (reuse attribute_swap logic for one key)1145 swap_keys = [k for k in params if k in ('shape1', 'shape2', 'color', 'size', 'material', 'shape')]1146 if swap_keys:1147 key = random.choice(swap_keys)1148 current = params.get(key)1149 if key in ('shape1', 'shape2', 'shape'):1150 alts = [s for s in props['all_shapes'] if s != current]1151 val = random.choice(alts) if alts else current1152 elif key in ('color', 'color1', 'color2'):1153 alts = [c for c in props['all_colors'] if c != current]1154 val = random.choice(alts) if alts else current1155 elif key == 'material':1156 alts = [m for m in props['all_materials'] if m != current]1157 val = random.choice(alts) if alts else current1158 elif key == 'size':1159 alts = [s for s in props['all_sizes'] if s != current]1160 val = random.choice(alts) if alts else current1161 else:1162 val = current1163 if val != current:1164 cf_params = params.copy()1165 cf_params[key] = val1166 cf_q = _apply_param_replacements(q, params, cf_params)1167 else:1168 cf_q = None1169 else:1170 cf_q = None1171 else:1172 # Replace "same as" with "different from" / "same" with "different"1173 if "the same as" in q_lower:1174 cf_q = q.replace("the same as", "different from").replace("The same as", "Different from")1175 elif "same shape" in q_lower:1176 cf_q = q.replace("same shape", "different shape").replace("same shape", "different shape")1177 elif "same color" in q_lower:1178 cf_q = q.replace("same color", "different color")1179 elif "same material" in q_lower:1180 cf_q = q.replace("same material", "different material")1181 else:1182 cf_q = q.replace("the same as", "different from")1183 cf_params = params.copy() if params else {}1184 else:1185 cf_q = None1186 1187 elif strategy == 'either_or':1188 # "either X or Y" -> swap X or Y, or "either X or Y" -> "both X and Y"1189 q_lower = original_question.lower()1190 if "either" in q_lower and " or " in q_lower and params:1191 if random.choice([True, False]):1192 # Swap one of the two attributes1193 swap_keys = [k for k in params if k in ('color', 'shape', 'material', 'size')]1194 if swap_keys:1195 key = random.choice(swap_keys)1196 current = params.get(key)1197 if key == 'shape':1198 alts = [s for s in props['all_shapes'] if s != current]1199 val = random.choice(alts) if alts else current1200 elif key == 'color':