winston1214/VENUS-10K

@article{kim2025speaking,
  title={Speaking Beyond Language: A Large-Scale Multimodal Dataset for Learning Nonverbal Cues from Video-Grounded Dialogues},
  author={Kim, Youngmin and Chung, Jiwan and Kim, Jisoo and Lee, Sunghyun and Lee, Sangkyu and Kim, Junhyeok and Yang, Cheoljong and Yu, Youngjae},
  journal={arXiv preprint arXiv:2506.00958},
  year={2025}
}

{
    'channel_id': str,  # YouTube channel ID
    'video_id': str,  # Video ID
    'segment_id': int,  # Segment ID within the video
    'duration': str,  # Total segment duration (e.g., '00:11:00 ~ 00:21:00')
    'fps': int,  # Frames per second

    'conversation': [  # Conversation information (consisting of multiple utterances)
        {
            'utterance_id': int,  # Utterance ID
            'speaker': int,  # Speaker ID (represented as an integer)
            'text': str,  # Full utterance text
            'start_time': float,  # Start time of the utterance (in seconds)
            'end_time': float,  # End time of the utterance (in seconds)
            'words': [  # Word-level information
                {
                    'word': str,  # The word itself
                    'start_time': float,  # Word-level start time
                    'end_time': float,  # Word-level end time
                }
            ]
        }
    ],

    'facial_expression': [  # Facial expression features
        {
            'utt_id': int,  # ID of the utterance this expression is aligned to
            'frame': int,  # Frame identifier
            'features': [float],  # Facial feature vector (153-dimensional)
        }
    ],

    'body_language': [  # Body language features
        {
            'utt_id': int,  # ID of the utterance this body language is aligned to
            'frame': int,  # Frame identifier
            'features': [float],  # Body movement feature vector (179-dimensional)
        }
    ],

    'speaker_bbox': [  # speaker bounding boxes
        {
            'frame_id': int,  # Frame identifier
            'bbox': [int],  # [x_top, y_top, x_bottom, y_bottom]
        }
    ],
    'harmful_utterance_id': [int],  # List of utterance IDs identified as harmful
}

from datasets import load_dataset
train_dataset = load_dataset("winston1214/VENUS-10K", split = "train")
test_dataset = load_dataset("winston1214/VENUS-10K", split = "test")