CoolFace
Datasetpublic

simorautiainen/moving-in-messy-apartment-video-with-imu

Example README for Your Dataset Dataset Name: Simo's Image and Sensor Data Collection Description This dataset includes a series of binary files containing images and corresponding sensor data (gyroscope and accelerometer), along with timestamps indicating when each image was captured. The dataset also features an output video (output_video.mp4), which has been compiled from these images based on the timestamps to maintain accurate timing. This is… See the full description on the dataset page: https://huggingface.co/datasets/simorautiainen/moving-in-messy-apartment-video-with-imu.

sourceHugging Facemitupdated 2y agoView on Hugging Face
1likes12downloads
Dataset Card

Example README for Your Dataset

Dataset Name: Simo's Image and Sensor Data Collection

Description

This dataset includes a series of binary files containing images and corresponding sensor data (gyroscope and accelerometer), along with timestamps indicating when each image was captured. The dataset also features an output video (output_video.mp4), which has been compiled from these images based on the timestamps to maintain accurate timing.

This is made for real time image applications that need gyroscope and accelerometer values, such as image stabilization, homography, video stabilizer, visual odometry etc.

This dataset was captured using a Samsung Galaxy S21 smartphone. The smartphone was manually held during the data collection phase to simulate realistic movement dynamics.

Contents

  • —/data_1714767956.zip: This archive contains all raw binary files for images, sensor data, and timestamps.
  • —output_video.mp4: A video file compiled from the images, synchronized by their timestamps to reflect the actual timing of each frame capture.

How to Use the Dataset

  1. 1.Extract the Data:
  2. 2.Unzip data_1714767956.zip to a local directory.
  1. 1.Reading the Data:
  2. 2.The binary data for images, sensor data, and timestamps can be read and processed using custom scripts. Example Python code for handling this data is provided below.
  1. 1.Generating the Video:
  2. 2.The output_video.mp4 included in the dataset is already compiled using the binary files. If you need to recreate or alter the video, you can use the Python script outlined below, which aligns frames based on the embedded timestamps.

Python Scripts

Reading Image Data
python
import os
import struct
import numpy as np
from PIL import Image as PILImage

def read_image(filepath):
    with open(filepath, 'rb') as file:
        width = struct.unpack('i', file.read(4))[0]
        height = struct.unpack('i', file.read(4))[0]
        channels = struct.unpack('i', file.read(4))[0]
        data_size = struct.unpack('i', file.read(4))[0]
        data = file.read(data_size)
        image = np.frombuffer(data, dtype=np.uint8).reshape((height, width, channels))
        return image
Reading Sensor Data and Timestamps
python
def read_sensor_data(filepath):
    with open(filepath, 'rb') as file:
        data = struct.unpack('fff', file.read(12))  # 3 floats for x, y, z
        return data

def read_timestamp(filepath):
    with open(filepath, 'rb') as file:
        timestamp = struct.unpack('Q', file.read(8))[0]  # uint64_t
        return timestamp
Generating Video from Images and Timestamps
python
import cv2

def process_directory_with_video(base_dir, output_video_path="output_video.mp4"):
    i = 1
    prev_timestamp = None
    video_writer = None

    while True:
        image_path = os.path.join(base_dir, f"image{i}.bin")
        timestamp_path = os.path.join(base_dir, f"timestamp{i}.bin")

        if not os.path.exists(image_path):
            break

        image = read_image(image_path)
        timestamp = read_timestamp(timestamp_path)

        if prev_timestamp is None:
            prev_timestamp = timestamp

        # Calculate delay between current and previous frame in seconds
        delay = (timestamp - prev_timestamp) / 1e9  # Assuming timestamp is in nanoseconds
        fps = 1 / delay if delay != 0 else 30  # Fallback to 30 FPS if delay is zero

        # Convert image to format suitable for video
        pil_image = PILImage.fromarray(image)
        if pil_image.mode == 'RGBA':
            pil_image = pil_image.convert('RGB')
        frame = np.array(pil_image)

        # Initialize video writer with the size of the first frame
        if video_writer is None:
            height, width, _ = frame.shape
            fourcc = cv2.VideoWriter_fourcc(*'mp4v')
            video_writer = cv2.VideoWriter(output_video_path, fourcc, fps, (width, height))

        video_writer.write(cv2.cvtColor(frame, cv2.COLOR_RGB2BGR))  # Write frame to video

        print(f"Processed image {i} with timestamp {timestamp} and added to video.")
        prev_timestamp = timestamp
        i += 1

    if video_writer:
        video_writer.release()
        print(f"Video saved to {output_video_path}")
Full example of reading sensor data and images and timestamps
py
import os
import struct
import numpy as np
from PIL import Image as PILImage
import json

def read_image(filepath):
    with open(filepath, 'rb') as file:
        width = struct.unpack('i', file.read(4))[0]
        height = struct.unpack('i', file.read(4))[0]
        channels = struct.unpack('i', file.read(4))[0]
        data_size = struct.unpack('i', file.read(4))[0]
        data = file.read(data_size)
        image = np.frombuffer(data, dtype=np.uint8).reshape((height, width, channels))
        return image

def read_sensor_data(filepath):
    with open(filepath, 'rb') as file:
        data = struct.unpack('fff', file.read(12))  # 3 floats for x, y, z
        return data

def read_timestamp(filepath):
    with open(filepath, 'rb') as file:
        timestamp = struct.unpack('Q', file.read(8))[0]  # uint64_t
        return timestamp



def ensure_directory_exists(directory):
    if not os.path.exists(directory):
        os.makedirs(directory)
def save_data_as_json(gyro, accel, timestamp, base_dir, index):
    data = {
        'gyroscope': {
            'x': gyro[0],
            'y': gyro[1],
            'z': gyro[2]
        },
        'accelerometer': {
            'x': accel[0],
            'y': accel[1],
            'z': accel[2]
        },
        'timestamp': timestamp
    }
    json_path = os.path.join(base_dir, f"data{index}.json")
    with open(json_path, 'w') as f:
        json.dump(data, f, indent=4)

def process_directory(base_dir, image_dir = "images", data_dir = "data"):
    i = 1
    while True:
        image_path = os.path.join(base_dir, f"image{i}.bin")
        gyro_path = os.path.join(base_dir, f"gyro{i}.bin")
        accel_path = os.path.join(base_dir, f"accel{i}.bin")
        timestamp_path = os.path.join(base_dir, f"timestamp{i}.bin")

        if not os.path.exists(image_path):
            break

        image = read_image(image_path)
        gyro = read_sensor_data(gyro_path)
        accel = read_sensor_data(accel_path)
        timestamp = read_timestamp(timestamp_path)

        # Convert image to JPEG after checking and converting RGBA to RGB
        pil_image = PILImage.fromarray(image)
        if pil_image.mode == 'RGBA':
            pil_image = pil_image.convert('RGB')
        
        jpeg_path = os.path.join(image_dir, f"image{i}.jpg")
        pil_image.save(jpeg_path, 'JPEG')

        # Save sensor data and timestamp as JSON
        save_data_as_json(gyro, accel, timestamp, data_dir, i)

        print(f"Processed image {i} and saved metadata to JSON.")
        i += 1
# Example usage
base_dir = r"data_1714767956"
time_stamp_dir = base_dir.split("_")[-1]
image_dir = "images" + time_stamp_dir
data_dir = "data" + time_stamp_dir
ensure_directory_exists(image_dir)  # Make sure the images directory exists
ensure_directory_exists(data_dir)

process_directory(base_dir, image_dir, data_dir)

Citation

If you use this dataset for your research or in any other form, citing it is appreciated but not mandatory. You can reference it as follows:

  • —Simo Rautiainen, Simo's Image and Sensor Data Collection, 2024.