ISLAM-PO/arab-dialects-20-countries-3m

Arab_Dialects_Dataset/
├── README.md                  # This file - Hugging Face Dataset Card
├── LICENSE                    # CC-BY-4.0 license
├── CITATION.cff               # Citation metadata
├── .gitattributes             # Git LFS settings for *.jsonl
│
├── 01_مصر/  (Egypt)
│   ├── 01_اللهجة.jsonl               # 21,432 records (dialect)
│   ├── 02_المصطلحات.jsonl            # 21,428 records (terms)
│   ├── 03_النكت.jsonl                # 21,428 records (jokes)
│   ├── 04_المواقف.jsonl              # 21,428 records (situations)
│   ├── 05_الثقافة_والعادات.jsonl     # 21,428 records (culture)
│   ├── 06_طريقة_الكلام.jsonl         # 21,428 records (speaking style)
│   └── 07_المحادثات.jsonl            # 21,428 records (dialogues)
│
├── 02_السعودية/ (Saudi Arabia)   # same 7 files
├── 03_الامارات/ (UAE)            # same 7 files
├── 04_الكويت/ (Kuwait)
├── 05_قطر/ (Qatar)
├── 06_البحرين/ (Bahrain)
├── 07_عمان/ (Oman)
├── 08_اليمن/ (Yemen)
├── 09_العراق/ (Iraq)
├── 10_سوريا/ (Syria)
├── 11_لبنان/ (Lebanon)
├── 12_الاردن/ (Jordan)
├── 13_فلسطين/ (Palestine)
├── 14_السودان/ (Sudan)
├── 15_ليبيا/ (Libya)
├── 16_تونس/ (Tunisia)
├── 17_الجزائر/ (Algeria)
├── 18_المغرب/ (Morocco)
├── 19_موريتانيا/ (Mauritania)
└── 20_الصومال/ (Somalia)         # same 7 files per country

graph TD
    ROOT[Arab_Dialects_Dataset/<br/>3M records - 12.07GB]
    ROOT --> DOCS[README + LICENSE + CITATION + .gitattributes]
    ROOT --> EG[01_Egypt<br/>150K - 605MB]
    ROOT --> GULF[Saudi Arabia ... Oman<br/>Gulf countries]
    ROOT --> LEV[Iraq ... Palestine<br/>Levant + Iraq]
    ROOT --> AFR[Sudan ... Somalia<br/>North Africa + Horn]
    EG --> T1[01_dialect.jsonl]
    EG --> T2[02_terms.jsonl]
    EG --> T3[03_jokes.jsonl]
    EG --> T4[04_situations.jsonl]
    EG --> T5[05_culture.jsonl]
    EG --> T6[06_speaking_style.jsonl]
    EG --> T7[07_dialogues.jsonl]
    T7 --> REC[JSON record<br/>id - country - category<br/>dialect - text - meta]

pie title Records distribution by type (per country, 150K)
    "Dialect" : 21432
    "Terms" : 21428
    "Jokes" : 21428
    "Situations" : 21428
    "Culture" : 21428
    "Speaking style" : 21428
    "Dialogues" : 21428

{
  "id": "مصر-نكت-000001",
  "country": "مصر",
  "category": "نكت",
  "dialect": "واحد مصري اتصل بصاحبه قاله 'بقى فينك؟' قاله 'في البيت' قاله 'طب عامل ايه وافتح الباب ما انا قدام البيت' 😂",
  "text": "واحد مصري اتصل بصاحبه قاله 'بقى فينك؟' ... وهذا يعكس روح أهل مصر في كلامهم اليومي حيث يستخدمون 'بقى' و'عامل ايه' و'يا جدع' بكثرة ... [record 1 - Egypt - jokes]",
  "meta": {"k1": "بقى", "k2": "عامل ايه", "k3": "يا جدع"}
}

from datasets import load_dataset

# Load everything with streaming (no 12 GB download needed)
ds = load_dataset("Arab_Dialects_Dataset", split="train", streaming=True)
print(next(iter(ds)))

# Filter one country / category
egy_jokes = ds.filter(lambda x: x["country"] == "مصر" and x["category"] == "نكت")
for row in egy_jokes.take(3):
    print(row["dialect"])

from datasets import load_dataset

# Egypt only
ds_eg = load_dataset("json", data_files="01_مصر/*.jsonl", split="train", streaming=True)

# One file only
ds_one = load_dataset("json", data_files="18_المغرب/07_المحادثات.jsonl", split="train")

import json, glob
import pandas as pd

files = glob.glob("Arab_Dialects_Dataset/01_مصر/*.jsonl")
rows = []
for f in files[:1]:
    with open(f, encoding="utf-8") as fh:
        for line in fh:
            rows.append(json.loads(line))
df = pd.DataFrame(rows)
print(df[["id", "category", "dialect"]].head())
print(df["category"].value_counts())

# Use the 'text' field for causal LM or 'dialect' for instruction tuning.
# Example prompt:
# instruction: "Write a joke in the Moroccan dialect"
# input: row["dialect"]

# Small demo (~16 MB, for validation)
python generate_dataset.py --demo

# Full generation (150K per country / ~10 GB+)
python generate_dataset.py --full --per-country 150000 --total-gb 10

CC-BY-4.0 - Creative Commons Attribution 4.0 International

@dataset{arab_dialects_20_2026,
  title   = {Arab Dialects Dataset: 20 Countries, 7 Content Types, 3M Records},
  author  = {Project Owner and Contributors},
  year    = {2026},
  publisher = {Hugging Face},
  version = {1.0.0},
  url     = {https://huggingface.co/datasets/USERNAME/Arab_Dialects_Dataset},
  note    = {3,000,000 records, 140 JSONL files, 12.07 GB, CC-BY-4.0}
}

# 1. Install tools
pip install huggingface_hub datasets

# 2. Login
huggingface-cli login

# 3. Init repo (Git LFS is required for large JSONL files)
cd Arab_Dialects_Dataset
git init
git lfs install
git lfs track "*.jsonl"
# .gitattributes is already included - make sure it is committed

# 4. Push (upload takes a while for 12 GB - push in batches on slow connections)
huggingface-cli repo create Arab_Dialects_Dataset --type dataset --yes
git remote add origin https://huggingface.co/datasets/USERNAME/Arab_Dialects_Dataset
git add README.md LICENSE CITATION.cff .gitattributes
git commit -m "docs: HF dataset card"
git push origin main
# Then push countries in batches:
git add 01_مصر 02_السعودية 03_الامارات 04_الكويت
git commit -m "data: gulf+egypt batch"
git push origin main
# ... repeat for remaining countries