NICOTINE1024/DeepASMR-dataset-backup

1from datasets import load_dataset2import numpy as np3 4# Path to your new parquet directory5parquet_dir = "/opt/gpfs/home/leying/data/data/DeepASMR_dataset/parquet"6 7print("Loading dataset from sharded Parquet files...")8# We use a wildcard to load all shards at once9dataset = load_dataset("parquet", data_files=f"{parquet_dir}/*.parquet", split="train")10 11print(f"Dataset loaded successfully!")12print(f"Total number of utterances: {len(dataset)}")13 14# --- Test a random sample ---15sample_idx = 0 16sample = dataset[sample_idx]17 18print("\n--- Sample Check ---")19print(f"Transcript: {sample['transcript']}")20print(f"Speaker ID: {sample['speaker_id']}")21print(f"Language:   {sample['language']}")22 23# This part tests if the audio bytes actually decode into a waveform24audio_array = sample['audio']['array']25sampling_rate = sample['audio']['sampling_rate']26 27print(f"Audio Shape: {audio_array.shape}")28print(f"Sampling Rate: {sampling_rate} Hz")29print(f"Duration: {len(audio_array) / sampling_rate:.2f} seconds")30 31# Optional: Check if the data is non-zero (proves it's not just silence/empty)32if np.abs(audio_array).max() > 0:33    print("Status: Audio data is valid and non-silent.")34else:35    print("Status: Warning - Audio data appears to be empty/silent.")oooo rclone size deepasmr:deepasmr-transfer-2026/DeepASMR-DB