togethercomputer/RedPajama-Data-V2

from datasets import load_dataset

ds = load_dataset("togethercomputer/RedPajama-Data-V2", name="sample") 

from datasets import load_dataset

ds = load_dataset("togethercomputer/RedPajama-Data-V2",
                  name="default",
                  partition="head_middle",
                  snapshots=["2023-06", "2022-49"],
                  languages=["en", "de"]) 

# get list of urls pointing to the text documents
wget "https://data.together.xyz/redpajama-data-v2/v1.0.0/urls/document-urls.txt" -O "document-urls.txt"

# get list of urls pointing to the quality signals
wget "https://data.together.xyz/redpajama-data-v2/v1.0.0/urls/quality_signals-urls.txt" -O "quality_signals-urls.txt"

# get list of urls pointing to the ids of duplicate documents
wget "https://data.together.xyz/redpajama-data-v2/v1.0.0/urls/duplicates-urls.txt" -O "duplicates-urls.txt"

# get list of urls pointing to the minhash signatures
wget "https://data.together.xyz/redpajama-data-v2/v1.0.0/urls/minhash-urls.txt" -O "minhash-urls.txt"

CC_SNAPSHOT="2023-06"
LANG="en"
PARTITION="head_middle"
BASE_URL="https://data.together.xyz/redpajama-data-v2/v1.0.0"

listings_tag="${LANG}-${CC_SNAPSHOT}-${PARTITION}"
mkdir listings
wget "${BASE_URL}/listings/${listings_tag}.txt" -O "listings/${listings_tag}.txt"
listings_file="listings/${listings_tag}.txt"

# download documents
while read line; do
  url="${BASE_URL}/documents/${line}.json.gz"
  dest="documents/${line}.json.gz"
  mkdir -p $(dirname $dest)
  wget "$url" -O "$dest"
done <"$listings_file"

CC_SNAPSHOT="2023-06"
LANG="en"
BASE_URL="https://data.together.xyz/redpajama-data-v2/v1.0.0"

listings_tag="${LANG}-${CC_SNAPSHOT}-head_middle"
mkdir listings
wget "${BASE_URL}/listings/${listings_tag}.txt" -O "listings/${listings_tag}.txt"
listings_file="listings/${listings_tag}.txt"

# download quality signals
while read line; do
  url="${BASE_URL}/quality_signals/${line}.signals.json.gz"
  dest="quality_signals/${line}.signals.json.gz"
  mkdir -p $(dirname $dest)
  wget "$url" -O "$dest"
done <"$listings_file"


# download other components
COMPS=("minhash" "duplicates")
for comp in "${COMPS[@]}"; do
  while read line; do
    url="${BASE_URL}/${comp}/${line}.${comp}.parquet"
    dest="${comp}/${line}.${comp}.parquet"
    mkdir -p $(dirname $dest)
    wget "$url" -O "$dest"
  done <"$listings_file"
done

def gopher_rules_pass(sample) -> bool:
    """ function returns True if the sample complies with Gopher rules """
    signals = json.loads(sample["quality_signals"])

    # rule 1: number of words between 50 and 10'000
    word_count = signals["rps_doc_word_count"][0][2]
    if word_count < 50 or word_count > 100_000:
        return False

    # rule 2: mean word length between 3 and 10
    mean_word_length = signals["rps_doc_mean_word_length"][0][2]
    if mean_word_length < 3 or mean_word_length > 10:
        return False

    # rule 2: symbol to word ratio below 0.1
    symbol_word_ratio = signals["rps_doc_symbol_to_word_ratio"][0][2]
    if symbol_word_ratio > 0.1:
        return False

    # rule 3: 90% of lines need to start without a bullet point
    n_lines = signals["ccnet_nlines"][0][2]
    n_lines_bulletpoint_start = sum(map(lambda ln: ln[2], signals["rps_lines_start_with_bulletpoint"]))
    if n_lines_bulletpoint_start / n_lines > 0.9:
        return False

    # rule 4: the ratio between characters in the most frequent 2-gram and the total number 
    # of characters must be below 0.2
    top_2_gram_frac = signals["rps_doc_frac_chars_top_2gram"][0][2]
    if top_2_gram_frac > 0.2:
        return False

    # rule 5: ...

    return True

ds_iterator = load_dataset(
    "togethercomputer/RedPajama-Data-V2",
    snapshots=["2023-14"],
    languages=["en"],
    name="default",
    streaming=True
)

filtered_dataset = []

for sample in ds_iterator["train"]:

    if not gopher_rules_pass(sample):
        continue

    filtered_dataset.append(sample)

├── documents
    ├── 2018-43
        ├── 0000
            ├── en_head.json.gz
            ├── ...
            ├── it_middle.json.gz
├── quality_signals
    ├── 2018-43
        ├── 0000
            ├── en_head.signals.json.gz
            ├── ...
            ├── it_middle.json.gz
├── duplicates
    ├── 2018-43
        ├── 0000
            ├── en_head.duplicates.parquet
            ├── ...
            ├── it_middle.duplicates.parquet
├── minhash
    ├── 2018-43
        ├── 0000
            ├── en_head.minhash.parquet
            ├── ...
            ├── it_middle.minhash.parquet

{
  "url": "...",
  "date_download": "2014-08-20T06:48:26Z",
  "digest": "sha1:46OPKWZ7MAG5624VYYA3U3YH2MJ727B6",
  "length": 1095,
  "nlines": 8,
  "source_domain": "...",
  "title": "...",
  "raw_content": "Dear ...",
  "cc_segment": "crawl-data/CC-MAIN-2014-35/...",
  "original_nlines": 11,
  "original_length": 1174,
  "line_ids": [
    0,
    1,
    3,
    4,
    6,
    7,
    8,
    9
  ],
  "language": "en",
  "language_score": 0.92,
  "perplexity": 217.2,
  "bucket": "head"
}

{
  "id": "2018-43/0000/en_head.json.gz/0",
  "id_int": 7972430436813205988,
  "metadata": {
    "cc_segment": "crawl-data/...",
    "cc_net_source": "2018-43/0000/en_head.json.gz",
    "url": "...",
    "source_domain": "...",
    "language": "en",
    "snapshot_id": "2018-43"
  },
  "quality_signals": {
    "ccnet_original_length": [
      [
        0,
        7033,
        8711.0
      ]
    ],
    ...,
    "rps_doc_stop_word_fraction": [
      [
        0,
        7033,
        0.45121107
      ]
    ],
    "rps_lines_num_words": [
      [
        0,
        25,
        2
      ],
      ...,
      [
        6980,
        7033,
        10
      ]
    ]
  }
}

@software{together2023redpajama,
  author = {Together Computer},
  title = {RedPajama: an Open Dataset for Training Large Language Models},
  month = October,
  year = 2023,
  url = {https://github.com/togethercomputer/RedPajama-Data}
}

@misc{weber2024redpajamaopendatasettraining,
      title={RedPajama: an Open Dataset for Training Large Language Models}, 
      author={Maurice Weber and Daniel Fu and Quentin Anthony and Yonatan Oren and Shane Adams and Anton Alexandrov and Xiaozhong Lyu and Huu Nguyen and Xiaozhe Yao and Virginia Adams and Ben Athiwaratkun and Rahul Chalamala and Kezhen Chen and Max Ryabinin and Tri Dao and Percy Liang and Christopher Ré and Irina Rish and Ce Zhang},
      year={2024},
      eprint={2411.12372},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2411.12372}, 
}