crag-mm-2025/crag-mm-single-turn-public

from datasets import load_dataset

# For single-turn dataset
dataset = load_dataset("crag-mm-2025/crag-mm-single_turn-public", revision="v0.1.2")

# For multi-turn dataset
dataset = load_dataset("crag-mm-2025/crag-mm-multi_turn-public", revision="v0.1.2")

# View available splits
print(f"Available splits: {', '.join(dataset.keys())}")

# Access examples
example = dataset["validation"][0]
print(f"Session ID: {example['session_id']}")
print(f"Image: {example['image']}")
print(f"Image URL: {example['image_url']}")
"""
Note: Either 'image' or 'image_url' will be provided in the dataset, but not necessarily both.
When the actual image cannot be included, only the image_url will be available.
The evaluation servers will nevertheless always include the loaded 'image' field.
"""

# Show image
import matplotlib.pyplot as plt
plt.imshow(example['image'])

{
  "session_id": str,          # Unique identifier for the conversation
  "image": Image,             # Image
  "image_url": str,           # Image URL where applicable
  "turns": {                  # Dictionary containing conversation turn data
    "interaction_id": [str],  # List of unique IDs for each interaction
    "domain": [int],          # List of domain category indices
    "query_category": [int],  # List of query category indices
    "dynamism": [int],        # List of dynamism level indices
    "query": [str],           # List of questions or prompts
    "image_quality": [int]    # List of image quality indices
  },
  "answers": {                # Dictionary containing answer data
    "interaction_id": [str],  # List of interaction IDs (matches turns)
    "ans_full": [str]         # List of complete answer texts
  }
}

def _prepare_feature_vocabularies(dataset_split):
    """Extract feature vocabularies for category encoding from dataset.
    
    These vocabularies allow conversion between integer indices and string labels.
    """
    turns_feature = dataset_split.features["turns"]
    return {
        "domain": turns_feature.feature["domain"],
        "query_category": turns_feature.feature["query_category"],
        "dynamism": turns_feature.feature["dynamism"],
        "image_quality": turns_feature.feature["image_quality"],
    }


def print_conversation(example: Dict[str, Any], feature_vocabularies: Dict[str, Any]) -> None:
    """Print a conversation in an indented format.
    
    Args:
        example: A single dataset example containing conversation turns
        feature_vocabularies: Mapping of features to their vocabularies for encoding/decoding from idx to str
    """
    # Print session ID
    print(f"Session ID: {example['session_id']}")
    
    # Print image info
    print(f"Image: {example['image']}")
    print(f"Image URL: {example['image_url']}")
    """
    Note: Either 'image' or 'image_url' will be provided in the dataset, but not necessarily both.
    When the actual image cannot be included, only the image_url will be available.
    The evaluation servers will nevertheless always include the loaded 'image' field.
    """
    
    # Determine if single-turn or multi-turn based on number of queries
    num_turns = len(example['turns']['query'])
    is_single_turn = num_turns == 1
    print(f"Type: {'Single-turn' if is_single_turn else 'Multi-turn'} ({num_turns} turns)")
    
    # Create answer lookup dictionary if answers exist
    answer_lookup = {}
    if 'answers' in example and example['answers'] is not None:
        answer_lookup = {
            interaction_id: ans_full 
            for interaction_id, ans_full in zip(
                example['answers']['interaction_id'], 
                example['answers']['ans_full']
            )
        }
    
    # Print each turn
    print("\nConversation:")
    for i in range(num_turns):
        # For multi-turn, show turn number
        if not is_single_turn:
            print(f"\tTurn {i+1}:")
        
        # Convert metadata to string representations
        domain_str = feature_vocabularies["domain"].int2str(example['turns']['domain'][i])
        category_str = feature_vocabularies["query_category"].int2str(example['turns']['query_category'][i])
        dynamism_str = feature_vocabularies["dynamism"].int2str(example['turns']['dynamism'][i])
        quality_str = feature_vocabularies["image_quality"].int2str(example['turns']['image_quality'][i])
        
        # Print metadata
        prefix = "\t\t" if not is_single_turn else "\t"
        print(f"{prefix}Domain: {domain_str} | Category: {category_str} | Dynamism: {dynamism_str} | Image Quality: {quality_str}")
        
        # Print query and answer with fixed tab indentation
        print(f"{prefix}Q: {example['turns']['query'][i]}")
        
        interaction_id = example['turns']['interaction_id'][i]
        ans = answer_lookup.get(interaction_id, "No answer available")
        print(f"{prefix}A: {ans}")
        
        if not is_single_turn and i < num_turns - 1:
            print()  # Add blank line between turns in multi-turn conversations
    
    print("\n" + "-" * 60 + "\n")  # Add separator between examples


split_to_use = "validation"
feature_vocabularies = _prepare_feature_vocabularies(dataset[split_to_use])
print_conversation(dataset[split_to_use][0], feature_vocabularies)

@inproceedings{crag-mm-2025,
  title = {CRAG-MM: A Comprehensive RAG Benchmark for Multi-modal, Multi-turn Question Answering},
  author = {CRAG-MM Team},
  year = {2025},
  url = {https://www.aicrowd.com/challenges/meta-crag-mm-challenge-2025}
}