natemiller1928/doitll

from datasets import load_dataset, concatenate_datasets

# List of dataset paths
dataset_paths = [
    "V3N0M/Jenna-50K-Alpaca-Uncensored",
    "SaisExperiments/Alpaca-Uncensored",
    "SaisExperiments/Big-Alpaca-Uncensored",
    "xzuyn/open-instruct-uncensored-alpaca",
    "xzuyn/tulu-uncensored-alpaca",
    "xzuyn/tv-alpaca-open-instruct-uncensored-blend",
    "dim/dolphin_flan1m_alpaca_uncensored_3k",
    "dataautogpt3/flan1m-alpaca-uncensored",
    "ShubhVenom/Uncensored-Alpaca-v01",
    "V3N0M/Uncensored-Alpaca",
    "Xennon-BD/Alpaca-uncensored",
    "VinyVan/flanMini-alpaca-uncensored_bambara"
]

# Load the first dataset to get reference columns
dataset1 = load_dataset(dataset_paths[0], split="train")
reference_columns = dataset1.column_names

# Load and select columns for the remaining datasets
datasets = [dataset1]
for path in dataset_paths[1:]:
    dataset = load_dataset(path, split="train")
    dataset = dataset.select_columns(reference_columns)
    datasets.append(dataset)

# Merge all datasets
merged_dataset = concatenate_datasets(datasets)

# Print the number of rows in the merged dataset
print(f"line: {len(merged_dataset)}")

# Save the merged dataset to disk
merged_dataset.save_to_disk("merged_uncensored_alpaca")