Nobody05/GPT-5.5-Gemini-3.1-Pro-Grok-4-Claude-Fable-5-Mythos-5-Qwen-3.7-Max-and-more-Distillation-Dataset

┌────────────────────────────────────────────────────────────┐
│                    UNIFIED EXTRACTION PIPELINE              │
├────────────────────────────────────────────────────────────┤
│                                                             │
│  73 Upstream Sources (ALL FULLY PROCESSED, NO SKIP)         │
│  ┌─────┐ ┌─────┐ ┌─────┐ ┌─────┐ ┌─────┐ ┌─────┐          │
│  │ HF  │ │ HF  │ │ HF  │ │ GH  │ │ HF  │ │ ... │          │
│  └──┬──┘ └──┬──┘ └──┬──┘ └──┬──┘ └──┬──┘ └──┬──┘          │
│     │       │       │       │       │       │               │
│     └───────┴───────┴───────┼───────┴───────┘               │
│                         │                                    │
│                    ┌────▼────┐                               │
│                    │ EXTRACT │ ← Field normalization        │
│                    └────┬────┘   (instruction/response)      │
│                         │                                    │
│                    ┌────▼────┐                               │
│                    │CATEGORIZE│ ← 8 semantic categories      │
│                    └────┬────┘                               │
│                         │                                    │
│                    ┌────▼────┐                               │
│                    │  SHARD  │ ← 20K samples per shard       │
│                    └────┬────┘                               │
│                         │                                    │
│                    ┌────▼────┐                               │
│                    │ UPLOAD  │ ← Batch commits to HF         │
│                    └─────────┘                               │
│                                                             │
│  STATUS: ALL 73 SOURCES COMPLETE. NO SKIPPING. 18M+ ROWS.   │
└────────────────────────────────────────────────────────────┘

📂 Manusagents/GPT-5.5-Gemini-3.1-Pro-Grok-4-Claude-Fable-5-Mythos-5-Qwen-3.7-Max-and-more-Distillation-Dataset/
│
├── 📦 archives/                          # ~64 GB — 7,090 compressed GitHub repos
│   ├── 0-chi__sonaure-lp.tar.gz
│   ├── 00MB__bitcoin_trading_bot.tar.gz
│   ├── 0101-agents__plugins.tar.gz
│   ├── ... (7,090 files total)
│   └── zznmg1__playable-survivor-ad.tar.gz
│
├── 📁 data/                              # ~12 GB — 516 JSONL shards (18M+ samples)
│   │
│   ├── 💻 coding/                        # 28 sources · ~11M+ samples
│   │   ├── vibe_instruct_v2/             # 8,152,510 samples
│   │   ├── fable5_2m/                    # 2,006,487 samples
│   │   ├── vibe_instruct_v1/             # 1,100,000 samples
│   │   ├── vibe_coding/                  # 1,100,000 samples
│   │   ├── royal_ghost_1m/               # 1,000,000 samples
│   │   ├── citation_ground/              # 980,064 samples
│   │   ├── royal_ghost_501k/             # 703,449 samples
│   │   ├── fable5_repos_full/            # 7,090 archive pointers
│   │   ├── fable5_agentic_sft/           # 159,972 samples
│   │   ├── gpt55_codex/                  # 119,436 samples ⭐ FULL
│   │   ├── alpca_gpt55/                  # 49,099 samples
│   │   ├── deepseek_v4_pro_agent/        # 96,597 samples ⭐ FULL
│   │   ├── fable5_traces/                # 49,544 samples ⭐ FULL
│   │   ├── genesis_code_100k/            # 68,000 samples
│   │   ├── genesis_code/                 # 49,000 samples
│   │   ├── kimi_coding/                  # 9,014 samples
│   │   ├── mimo_claude_code_traces/      # 15,046 samples ⭐ FULL
│   │   ├── kimi_k26_claude_code_traces/  # 7,438 samples
│   │   ├── genesis_code_10k/             # 9,800 samples
│   │   ├── legend_python/                # 5,000 samples
│   │   ├── autonomy/                     # 10,000 samples
│   │   ├── genesis_code_demo/            # 1,000 samples
│   │   ├── god_coder/                    # ⭐ FULL raw recovery
│   │   ├── python_god_coder/             # ⭐ FULL raw recovery
│   │   ├── elite_god_coder/              # ⭐ FULL raw recovery
│   │   ├── omega_genesis/                # ⭐ FULL raw recovery
│   │   ├── open_tool_trace/              # 48 samples
│   │   └── genesis_v11/                  # partial recovery
│   │
│   ├── 🧮 math/                          # 2 sources
│   │   ├── math_25k/
│   │   └── deepseek_prover_v1/           # 27,503 Lean theorem proofs
│   │
│   ├── 🔬 science/                       # 7 sources
│   │   ├── science_25k/
│   │   ├── physics_25k/
│   │   ├── chemistry_25k/
│   │   ├── biology_25k/
│   │   ├── medical_25k/
│   │   ├── cs_25k/
│   │   └── biology_r2med/                # ⭐ NEW
│   │
│   ├── ⚙️ applied/                       # 8 sources
│   │   ├── robotics_25k/
│   │   ├── nano_25k/
│   │   ├── materials_25k/
│   │   ├── earth_climate_25k/
│   │   ├── renewable_energy_25k/
│   │   ├── evolution_25k/
│   │   ├── universe_25k/
│   │   └── kardashev_25k/
│   │
│   ├── 📚 humanities/                    # 8 sources
│   │   ├── psychology_25k/
│   │   ├── economics_25k/
│   │   ├── law_25k/
│   │   ├── statistics_25k/
│   │   ├── sports_25k/
│   │   ├── human_25k/
│   │   ├── conscience_25k/
│   │   └── supernatural_25k/
│   │
│   ├── 🧠 distilled/                     # 9 sources · frontier distillations
│   │   ├── claude_mythos/
│   │   ├── gemini35/
│   │   ├── fable5_cleaned/
│   │   ├── grok44/
│   │   ├── gemini_pro32/
│   │   ├── gpt55_thinking/
│   │   ├── gpt55_distilled/
│   │   ├── claude_opus_48_distill/       # ⭐ NEW
│   │   └── claude_opus_48_max_thinking/  # ⭐ NEW
│   │
│   ├── 📝 instruction/                   # 3 sources
│   │   ├── alpaca/                       # 52,002 samples
│   │   ├── oasst/                        # 32,141 samples
│   │   └── dolly/                        # 15,011 samples
│   │
│   ├── 🔒 cybersecurity/                 # 6 sources
│   │   ├── high_quality_cybersecurity/
│   │   ├── heimdall_v1_1/                # ⭐ NEW — 78 MB conversations
│   │   ├── fenrir_v2_1/                  # ⭐ NEW — 411 MB (2.1M+ entries)
│   │   ├── clydeiii_cybersecurity/       # ⭐ NEW — 20 MB yearly corpus
│   │   ├── precinct6_cybersecurity/      # ⭐ NEW — 2.1 GB (graph+signals+ref)
│   │   └── savani_cyber_attack/          # ⭐ NEW — 17 MB attack CSV
│   │
│   └── 📇 index/                         # 2 sources
│       ├── species_25k/
│       └── transport_25k/
│
├── 📄 README.md
└── 📄 dataset_info.json

{
  "source": "fenrir_v2_1",
  "instruction": "Explain how an attacker could exploit a vulnerable login form using SQL injection. Provide a sample malicious input.",
  "response": "An attacker could bypass authentication by injecting SQL logic. For a query like:\n`SELECT * FROM users WHERE username='$user' AND password='$pass'`,\nthe attacker might enter `admin' --` as the username and anything as password. The resulting query becomes:\n`SELECT * FROM users WHERE username='admin' --' AND password='...'`\nThe `--` comments out the password check, granting access as admin. More advanced attacks use UNION to extract data."
}

{
  "source": "heimdall_v1_1",
  "instruction": "As a red team operator, list the steps you would take after gaining initial access to a Windows machine using a phishing payload.",
  "response": "1. Establish persistence: `schtasks /create /tn \"Updater\" /tr \"C:\\malware.exe\" /sc hourly`\n2. Enumerate local users: `net user /domain`\n3. Dump credentials: `mimikatz sekurlsa::logonpasswords`\n4. Lateral movement: use PsExec to move to file server.\n5. Exfiltrate data: compress sensitive files and upload via DNS tunneling.\nAlways maintain C2 communication over HTTPS to blend in."
}

{
  "source": "high_quality_cybersecurity",
  "instruction": "An IDS alert flags unusual outbound DNS traffic from a workstation. What's the immediate response?",
  "response": "1. Isolate the workstation from the network (VLAN change or port shutdown).\n2. Capture volatile memory and network logs for forensics.\n3. Check DNS queries: if long, random-looking subdomains, suspect DNS tunneling.\n4. Scan for malware with updated signatures.\n5. Review firewall logs for similar traffic from other hosts.\n6. If confirmed, initiate incident response playbook for data exfiltration."
}

from datasets import load_dataset

REPO = "Manusagents/GPT-5.5-Gemini-3.1-Pro-Grok-4-Claude-Fable-5-Mythos-5-Qwen-3.7-Max-and-more-Distillation-Dataset"

# Load only cybersecurity category
cyber_ds = load_dataset(REPO, split="train", 
                        data_files="data/cybersecurity/**/*.jsonl",
                        streaming=True)

# Or load specific sources
fenrir = load_dataset(REPO, split="train", 
                      data_files="data/cybersecurity/fenrir_v2_1/*.jsonl")

# Format for SFT
def format_security_sample(example):
    return {
        "text": f"### Security Task:\n{example['instruction']}\n\n### Expert Response:\n{example['response']}"
    }

cyber_ds = cyber_ds.map(format_security_sample)

# Now train with your favourite framework (transformers, axolotl, etc.)

from datasets import load_dataset

REPO = "Manusagents/GPT-5.5-Gemini-3.1-Pro-Grok-4-Claude-Fable-5-Mythos-5-Qwen-3.7-Max-and-more-Distillation-Dataset"

# ─ Load a single category ─
ds = load_dataset(REPO, split="train", data_files="data/coding/*/*.jsonl", streaming=True)

# ─ Load a specific source ─
ds = load_dataset(REPO, split="train", data_files="data/coding/vibe_instruct_v2/*.jsonl", streaming=True)

# ─ Load everything (18M+ samples) ─
ds = load_dataset(REPO, split="train", streaming=True)

for sample in ds:
    print(sample["source"], sample["instruction"][:80])

from huggingface_hub import hf_hub_download
import tarfile

REPO = "Manusagents/GPT-5.5-Gemini-3.1-Pro-Grok-4-Claude-Fable-5-Mythos-5-Qwen-3.7-Max-and-more-Distillation-Dataset"

# ─ Option A: Download & extract ONE repository ─
hf_hub_download(
    repo_id=REPO,
    repo_type="dataset",
    filename="archives/0x101__lakewatch.tar.gz",
    local_dir="./repos",
)
with tarfile.open("./repos/archives/0x101__lakewatch.tar.gz", "r:gz") as tar:
    tar.extractall("./extracted/0x101__lakewatch")


# ─ Option B: Stream files WITHOUT full extraction ─
def stream_repo_files(archive_name, max_files=100):
    """Stream file contents from tar.gz without extracting to disk."""
    local_path = hf_hub_download(repo_id=REPO, repo_type="dataset", filename=archive_name)
    
    with tarfile.open(local_path, "r:gz") as tar:
        count = 0
        for member in tar:
            if member.isfile() and count < max_files:
                f = tar.extractfile(member)
                if f:
                    yield {
                        "path": member.name,
                        "content": f.read().decode("utf-8", errors="ignore")[:4000],
                    }
                    count += 1
    
    import os
    os.remove(local_path)  # Clean up

# Stream files from a specific repo
for file_data in stream_repo_files("archives/0x101__lakewatch.tar.gz"):
    print(f"📄 {file_data['path']}: {file_data['content'][:100]}...")


# ─ Option C: Use pre-extracted JSONL shards (475K samples) ─
code_ds = load_dataset(
    REPO, split="train",
    data_files="data/coding/fable5_repos_full/*.jsonl",
    streaming=True
)
# Each sample: instruction = "<repo>/<file>", response = "<content>"

import torch
from datasets import load_dataset
from transformers import (
    AutoTokenizer,
    AutoModelForCausalLM,
    TrainingArguments,
    Trainer,
    DataCollatorForLanguageModeling,
)

# ═══════════════════════════════════════
# CONFIGURATION
# ═══════════════════════════════════════
MODEL_NAME = "meta-llama/Llama-3.1-8B"
DATASET_REPO = "Manusagents/GPT-5.5-Gemini-3.1-Pro-Grok-4-Claude-Fable-5-Mythos-5-Qwen-3.7-Max-and-more-Distillation-Dataset"
OUTPUT_DIR = "./sft-output"
MAX_SEQ_LEN = 2048

# ═══════════════════════════════════════
# LOAD MODEL & TOKENIZER
# ═══════════════════════════════════════
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
tokenizer.pad_token = tokenizer.eos_token

model = AutoModelForCausalLM.from_pretrained(
    MODEL_NAME,
    torch_dtype=torch.bfloat16,
    device_map="auto",
    attn_implementation="flash_attention_2",
)

# ═══════════════════════════════════════
# LOAD & FORMAT DATASET
# ═══════════════════════════════════════
def format_instruction(sample):
    text = f"### Instruction:\n{sample['instruction']}\n\n### Response:\n{sample['response']}"
    return {"text": text}

def tokenize(examples):
    return tokenizer(
        examples["text"],
        truncation=True,
        max_length=MAX_SEQ_LEN,
        padding="max_length",
    )

# Load coding category (use "data/**/*.jsonl" for full 18M+)
train_ds = load_dataset(
    DATASET_REPO,
    split="train",
    data_files="data/coding/*/*.jsonl",
    streaming=True,
)
train_ds = train_ds.map(format_instruction).filter(lambda x: len(x["text"]) > 0)
train_ds = train_ds.map(tokenize, batched=True)

# ═══════════════════════════════════════
# TRAIN
# ═══════════════════════════════════════
training_args = TrainingArguments(
    output_dir=OUTPUT_DIR,
    num_train_epochs=3,
    per_device_train_batch_size=4,
    gradient_accumulation_steps=4,
    warmup_steps=500,
    logging_steps=100,
    save_steps=2000,
    learning_rate=2e-5,
    bf16=True,
    gradient_checkpointing=True,
    optim="adamw_torch",
)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_ds,
    data_collator=DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False),
)

trainer.train()
trainer.save_model(OUTPUT_DIR)

from datasets import load_dataset, interleave_datasets

REPO = "Manusagents/GPT-5.5-Gemini-3.1-Pro-Grok-4-Claude-Fable-5-Mythos-5-Qwen-3.7-Max-and-more-Distillation-Dataset"

# ─ Phase 1: Foundation (math + science) ─
phase1_math = load_dataset(REPO, split="train", data_files="data/math/**/*.jsonl", streaming=True)
phase1_sci = load_dataset(REPO, split="train", data_files="data/science/**/*.jsonl", streaming=True)
phase1 = interleave_datasets([phase1_math, phase1_sci])

# ─ Phase 2: Add coding traces ─
phase2 = load_dataset(REPO, split="train", data_files="data/coding/**/*.jsonl", streaming=True)

# ─ Phase 3: Add distilled reasoning + cybersecurity ─
phase3_distilled = load_dataset(REPO, split="train", data_files="data/distilled/**/*.jsonl", streaming=True)
phase3_cyber = load_dataset(REPO, split="train", data_files="data/cybersecurity/**/*.jsonl", streaming=True)
phase3 = interleave_datasets([phase3_distilled, phase3_cyber])

# Train sequentially
# trainer.train(phase1)  # epochs 0-1
# trainer.train(phase2)  # epochs 1-2
# trainer.train(phase3)  # epochs 2-3

{
    "source":          "fable5_2m",
    "source_dataset":  "Crownelius/Complete-FABLE.5-traces-2M",
    "instruction":     "<the prompt / question / file path>",
    "response":        "<the completion / answer / file content>",
    "category":        "coding"
}

@misc{open_distillation_codex_2026,
  title  = {The Open Distillation Codex: 18M+ samples + 7090 code repositories from 73 sources with Cybersecurity Attack & Defense},
  author = {Manusagents},
  year   = {2026},
  url    = {https://huggingface.co/datasets/Manusagents/GPT-5.5-Gemini-3.1-Pro-Grok-4-Claude-Fable-5-Mythos-5-Qwen-3.7-Max-and-more-Distillation-Dataset},
  note   = {v8.2 - No skip, full. 516 shards + 7090 archives, 73 sources, 8 categories, 76 GB+}
}