ttj/dataset-ironing-smollm2-135m-ppl
base_data_smolm2_135m_ppl This dataset is a nanochat pretraining parquet dataset annotated with offline per-example perplexity from HuggingFaceTB/SmolLM2-135M. Columns text: original training text. perplexity: per-example perplexity from the fixed reference model. perplexity_num_tokens: number of reference-token prediction targets used. perplexity_nll: mean negative log likelihood before exponentiation. perplexity_reference_model: reference model id used for the… See the full description on the dataset page: https://huggingface.co/datasets/ttj/dataset-ironing-smollm2-135m-ppl.
0105
basedatasmolm2135mppl
This dataset is a nanochat pretraining parquet dataset annotated with offline per-example perplexity from HuggingFaceTB/SmolLM2-135M.
Columns
text: original training text.perplexity: per-example perplexity from the fixed reference model.perplexity_num_tokens: number of reference-token prediction targets used.perplexity_nll: mean negative log likelihood before exponentiation.perplexity_reference_model: reference model id used for the row.
Global Statistics
The global statistics are stored in perplexity_stats.json:
{
"num_examples": 12789760,
"num_finite": 12789760,
"mean": 16.971930903637244,
"std": 8.10952495285045,
"deciles": {
"0.1": 9.473104233607561,
"0.2": 11.27557233495321,
"0.3": 12.72408254281915,
"0.4": 14.077568768709655,
"0.5": 15.460851038454923,
"0.6": 16.983914074862593,
"0.7": 18.8197051905061,
"0.8": 21.31558898948356,
"0.9": 25.69053539187382
},
"min": 1.2803555606074715,
"max": 992.5975294944326,
"reference_model": "HuggingFaceTB/SmolLM2-135M",
"max_context_length": 8192,
"input_shards": [
"shard_00000.parquet",
"shard_00001.parquet",
"shard_00002.parquet",
"shard_00003.parquet",
"shard_00004.parquet",
"shard_00005.parquet",
"shard_00006.parquet",
"shard_00007.parquet",
"shard_00008.parquet",
"shard_00009.parquet",
"shard_00010.parquet",
"shard_00011.parquet",
"shard_00012.parquet",
"shard_00013.parquet",
"shard_00014.parquet",
"shard_00015.parquet",
"shard_00016.parquet",
"shard_00017.parquet",
"shard_00018.parquet",
"shard_00019.parquet",
"shard_00020.parquet",
"shard_00021.parquet",
"shard_00022.parquet",
"shard_00023.parquet",
"shard_00024.parquet",
"shard_00025.parquet",
"shard_00026.parquet",
"shard_00027.parquet",
"shard_00028.parquet",
"shard_00029.parquet",
"shard_00030.parquet",
"shard_00031.parquet",
"shard_00032.parquet",
"shard_00033.parquet",
"shard_00034.parquet",
"shard_00035.parquet",
"shard_00036.parquet",
"shard_00037.parquet",
"shard_00038.parquet",
"shard_00039.parquet",
"shard_00040.parquet",
"shard_00041.parquet",
"shard_00042.parquet",
"shard_00043.parquet",
"shard_00044.parquet",
"shard_00045.parquet",
"shard_00046.parquet",
"shard_00047.parquet",
"shard_00048.parquet",
"shard_00049.parquet",
"shard_00050.parquet",
"shard_00051.parquet",
"shard_00052.parquet",
"shard_00053.parquet",
"shard_00054.parquet",
"shard_00055.parquet",
"shard_00056.parquet",
"shard_00057.parquet",
"shard_00058.parquet",
"shard_00059.parquet",
"shard_00060.parquet",
"shard_00061.parquet",
"shard_00062.parquet",
"shard_00063.parquet",
"shard_00064.parquet",
"shard_00065.parquet",
"shard_00066.parquet",
"shard_00067.parquet",
"shard_00068.parquet",
"shard_00069.parquet",
"shard_00070.parquet",
"shard_00071.parquet",
"shard_00072.parquet",
"shard_00073.parquet",
"shard_00074.parquet",
"shard_00075.parquet",
"shard_00076.parquet",
"shard_00077.parquet",
"shard_00078.parquet",
"shard_00079.parquet",
"shard_00080.parquet",
"shard_00081.parquet",
"shard_00082.parquet",
"shard_00083.parquet",
"shard_00084.parquet",
"shard_00085.parquet",
"shard_00086.parquet",
"shard_00087.parquet",
"shard_00088.parquet",
"shard_00089.parquet",
"shard_00090.parquet",
"shard_00091.parquet",
"shard_00092.parquet",
"shard_00093.parquet",
"shard_00094.parquet",
"shard_00095.parquet",
"shard_00096.parquet",
"shard_00097.parquet",
"shard_00098.parquet",
"shard_00099.parquet",
"shard_00100.parquet",
"shard_00101.parquet",
"shard_00102.parquet",
"shard_00103.parquet",
"shard_00104.parquet",
"shard_00105.parquet",
"shard_00106.parquet",
"shard_00107.parquet",
"shard_00108.parquet",
"shard_00109.parquet",
"shard_00110.parquet",
"shard_00111.parquet",
"shard_00112.parquet",
"shard_00113.parquet",
"shard_00114.parquet",
"shard_00115.parquet",
"shard_00116.parquet",
"shard_00117.parquet",
"shard_00118.parquet",
"shard_00119.parquet",
"shard_00120.parquet",
"shard_00121.parquet",
"shard_00122.parquet",
"shard_00123.parquet",
"shard_00124.parquet",
"shard_00125.parquet",
"shard_00126.parquet",
"shard_00127.parquet",
"shard_00128.parquet",
"shard_00129.parquet",
"shard_00130.parquet",
"shard_00131.parquet",
"shard_00132.parquet",
"shard_00133.parquet",
"shard_00134.parquet",
"shard_00135.parquet",
"shard_00136.parquet",
"shard_00137.parquet",
"shard_00138.parquet",
"shard_00139.parquet",
"shard_00140.parquet",
"shard_00141.parquet",
"shard_00142.parquet",
"shard_00143.parquet",
"shard_00144.parquet",
"shard_00145.parquet",
"shard_00146.parquet",
"shard_00147.parquet",
"shard_00148.parquet",
"shard_00149.parquet",
"shard_00150.parquet",
"shard_00151.parquet",
"shard_00152.parquet",
"shard_00153.parquet",
"shard_00154.parquet",
"shard_00155.parquet",
"shard_00156.parquet",
"shard_00157.parquet",
"shard_00158.parquet",
"shard_00159.parquet",
"shard_00160.parquet",
"shard_00161.parquet",
"shard_00162.parquet",
"shard_00163.parquet",
"shard_00164.parquet",
"shard_00165.parquet",
"shard_00166.parquet",
"shard_00167.parquet",
"shard_00168.parquet",
"shard_00169.parquet",
"shard_00170.parquet",
"shard_00171.parquet",
"shard_00172.parquet",
"shard_00173.parquet",
"shard_00174.parquet",
"shard_00175.parquet",
"shard_00176.parquet",
"shard_00177.parquet",
"shard_00178.parquet",
"shard_00179.parquet",
"shard_00180.parquet",
"shard_00181.parquet",
"shard_00182.parquet",
"shard_00183.parquet",
"shard_00184.parquet",
"shard_00185.parquet",
"shard_00186.parquet",
"shard_00187.parquet",
"shard_00188.parquet",
"shard_00189.parquet",
"shard_00190.parquet",
"shard_00191.parquet",
"shard_00192.parquet",
"shard_00193.parquet",
"shard_00194.parquet",
"shard_00195.parquet",
"shard_00196.parquet",
"shard_00197.parquet",
"shard_00198.parquet",
"shard_00199.parquet",
"shard_00200.parquet",
"shard_00201.parquet",
"shard_00202.parquet",
"shard_00203.parquet",
"shard_00204.parquet",
"shard_00205.parquet",
"shard_00206.parquet",
"shard_00207.parquet",
"shard_00208.parquet",
"shard_00209.parquet",
"shard_00210.parquet",
"shard_00211.parquet",
"shard_00212.parquet",
"shard_00213.parquet",
"shard_00214.parquet",
"shard_00215.parquet",
"shard_00216.parquet",
"shard_00217.parquet",
"shard_00218.parquet",
"shard_00219.parquet",
"shard_00220.parquet",
"shard_00221.parquet",
"shard_00222.parquet",
"shard_00223.parquet",
"shard_00224.parquet",
"shard_00225.parquet",
"shard_00226.parquet",
"shard_00227.parquet",
"shard_00228.parquet",
"shard_00229.parquet",
"shard_00230.parquet",
"shard_00231.parquet",
"shard_00232.parquet",
"shard_00233.parquet",
"shard_00234.parquet",
"shard_00235.parquet",
"shard_00236.parquet",
"shard_00237.parquet",
"shard_00238.parquet",
"shard_00239.parquet"
],
"output_shards": [
"shard_00000.parquet",
"shard_00001.parquet",
"shard_00002.parquet",
"shard_00003.parquet",
"shard_00004.parquet",
"shard_00005.parquet",
"shard_00006.parquet",
"shard_00007.parquet",
"shard_00008.parquet",
"shard_00009.parquet",
"shard_00010.parquet",
"shard_00011.parquet",
"shard_00012.parquet",
"shard_00013.parquet",
"shard_00014.parquet",
"shard_00015.parquet",
"shard_00016.parquet",
"shard_00017.parquet",
"shard_00018.parquet",
"shard_00019.parquet",
"shard_00020.parquet",
"shard_00021.parquet",
"shard_00022.parquet",
"shard_00023.parquet",
"shard_00024.parquet",
"shard_00025.parquet",
"shard_00026.parquet",
"shard_00027.parquet",
"shard_00028.parquet",
"shard_00029.parquet",
"shard_00030.parquet",
"shard_00031.parquet",
"shard_00032.parquet",
"shard_00033.parquet",
"shard_00034.parquet",
"shard_00035.parquet",
"shard_00036.parquet",
"shard_00037.parquet",
"shard_00038.parquet",
"shard_00039.parquet",
"shard_00040.parquet",
"shard_00041.parquet",
"shard_00042.parquet",
"shard_00043.parquet",
"shard_00044.parquet",
"shard_00045.parquet",
"shard_00046.parquet",
"shard_00047.parquet",
"shard_00048.parquet",
"shard_00049.parquet",
"shard_00050.parquet",
"shard_00051.parquet",
"shard_00052.parquet",
"shard_00053.parquet",
"shard_00054.parquet",
"shard_00055.parquet",
"shard_00056.parquet",
"shard_00057.parquet",
"shard_00058.parquet",
"shard_00059.parquet",
"shard_00060.parquet",
"shard_00061.parquet",
"shard_00062.parquet",
"shard_00063.parquet",
"shard_00064.parquet",
"shard_00065.parquet",
"shard_00066.parquet",
"shard_00067.parquet",
"shard_00068.parquet",
"shard_00069.parquet",
"shard_00070.parquet",
"shard_00071.parquet",
"shard_00072.parquet",
"shard_00073.parquet",
"shard_00074.parquet",
"shard_00075.parquet",
"shard_00076.parquet",
"shard_00077.parquet",
"shard_00078.parquet",
"shard_00079.parquet",
"shard_00080.parquet",
"shard_00081.parquet",
"shard_00082.parquet",
"shard_00083.parquet",
"shard_00084.parquet",
"shard_00085.parquet",
"shard_00086.parquet",
"shard_00087.parquet",
"shard_00088.parquet",
"shard_00089.parquet",
"shard_00090.parquet",
"shard_00091.parquet",
"shard_00092.parquet",
"shard_00093.parquet",
"shard_00094.parquet",
"shard_00095.parquet",
"shard_00096.parquet",
"shard_00097.parquet",
"shard_00098.parquet",
"shard_00099.parquet",
"shard_00100.parquet",
"shard_00101.parquet",
"shard_00102.parquet",
"shard_00103.parquet",
"shard_00104.parquet",
"shard_00105.parquet",
"shard_00106.parquet",
"shard_00107.parquet",
"shard_00108.parquet",
"shard_00109.parquet",
"shard_00110.parquet",
"shard_00111.parquet",
"shard_00112.parquet",
"shard_00113.parquet",
"shard_00114.parquet",
"shard_00115.parquet",
"shard_00116.parquet",
"shard_00117.parquet",
"shard_00118.parquet",
"shard_00119.parquet",
"shard_00120.parquet",
"shard_00121.parquet",
"shard_00122.parquet",
"shard_00123.parquet",
"shard_00124.parquet",
"shard_00125.parquet",
"shard_00126.parquet",
"shard_00127.parquet",
"shard_00128.parquet",
"shard_00129.parquet",
"shard_00130.parquet",
"shard_00131.parquet",
"shard_00132.parquet",
"shard_00133.parquet",
"shard_00134.parquet",
"shard_00135.parquet",
"shard_00136.parquet",
"shard_00137.parquet",
"shard_00138.parquet",
"shard_00139.parquet",
"shard_00140.parquet",
"shard_00141.parquet",
"shard_00142.parquet",
"shard_00143.parquet",
"shard_00144.parquet",
"shard_00145.parquet",
"shard_00146.parquet",
"shard_00147.parquet",
"shard_00148.parquet",
"shard_00149.parquet",
"shard_00150.parquet",
"shard_00151.parquet",
"shard_00152.parquet",
"shard_00153.parquet",
"shard_00154.parquet",
"shard_00155.parquet",
"shard_00156.parquet",
"shard_00157.parquet",
"shard_00158.parquet",
"shard_00159.parquet",
"shard_00160.parquet",
"shard_00161.parquet",
"shard_00162.parquet",
"shard_00163.parquet",
"shard_00164.parquet",
"shard_00165.parquet",
"shard_00166.parquet",
"shard_00167.parquet",
"shard_00168.parquet",
"shard_00169.parquet",
"shard_00170.parquet",
"shard_00171.parquet",
"shard_00172.parquet",
"shard_00173.parquet",
"shard_00174.parquet",
"shard_00175.parquet",
"shard_00176.parquet",
"shard_00177.parquet",
"shard_00178.parquet",
"shard_00179.parquet",
"shard_00180.parquet",
"shard_00181.parquet",
"shard_00182.parquet",
"shard_00183.parquet",
"shard_00184.parquet",
"shard_00185.parquet",
"shard_00186.parquet",
"shard_00187.parquet",
"shard_00188.parquet",
"shard_00189.parquet",
"shard_00190.parquet",
"shard_00191.parquet",
"shard_00192.parquet",
"shard_00193.parquet",
"shard_00194.parquet",
"shard_00195.parquet",
"shard_00196.parquet",
"shard_00197.parquet",
"shard_00198.parquet",
"shard_00199.parquet",
"shard_00200.parquet",
"shard_00201.parquet",
"shard_00202.parquet",
"shard_00203.parquet",
"shard_00204.parquet",
"shard_00205.parquet",
"shard_00206.parquet",
"shard_00207.parquet",
"shard_00208.parquet",
"shard_00209.parquet",
"shard_00210.parquet",
"shard_00211.parquet",
"shard_00212.parquet",
"shard_00213.parquet",
"shard_00214.parquet",
"shard_00215.parquet",
"shard_00216.parquet",
"shard_00217.parquet",
"shard_00218.parquet",
"shard_00219.parquet",
"shard_00220.parquet",
"shard_00221.parquet",
"shard_00222.parquet",
"shard_00223.parquet",
"shard_00224.parquet",
"shard_00225.parquet",
"shard_00226.parquet",
"shard_00227.parquet",
"shard_00228.parquet",
"shard_00229.parquet",
"shard_00230.parquet",
"shard_00231.parquet",
"shard_00232.parquet",
"shard_00233.parquet",
"shard_00234.parquet",
"shard_00235.parquet",
"shard_00236.parquet",
"shard_00237.parquet",
"shard_00238.parquet",
"shard_00239.parquet"
],
"is_partial": false,
"missing_output_shards": [],
"output_dir": "/home/ttj/.cache/nanochat/base_data_smolm2_135m_ppl",
"perplexity_column": "perplexity",
"token_count_column": "perplexity_num_tokens",
"nll_column": "perplexity_nll"
}Reference context length: 8192.
Input shard count: 240.