Modelpublic

cmeister/tokenizer-lm-ablations

sourceHugging Faceapache-2.0updated 2d agoView on Hugging Face

2likes

dirfull-128k-allmulti-equal-bpe/

dirfull-128k-apertus/

dirfull-128k-apertus-mathcode-scratch/

dirfull-128k-apertus-seed42/

dirfull-128k-bpe-nfc-clean-balanced/

dirfull-128k-bpe-nfc-clean-balanced-mathcode-scratch/

dirfull-128k-bpe-nfc-clean-balanced-v128k/

dirfull-128k-bpe-nfc-clean-balanced-v160k/

dirfull-128k-bpe-nfc-clean-balanced-v200k/

dirfull-128k-bpe-nfc-clean-balanced-v64k/

dirfull-128k-bpe-nfc-clean-balanced-v96k/

dirfull-128k-bpe-nfc-clean-fineweb2full/

dirfull-128k-bpe-nfc-clean-nomark-balanced/

dirfull-128k-bpe-nfc-gpt4-fineweb2full/

dirfull-128k-bpe-nfc-plus2-balanced/

dirfull-128k-bpe-nfc-plus2-balanced-mathcode-scratch/

dirfull-128k-bpe-nfc-plus2-balanced-seed43-mathcode-scratch/

dirfull-128k-bpe-nfc-plus2-balanced-seed44-mathcode-scratch/

dirfull-128k-bpe-nfc-plus3-balanced/

dirfull-128k-bpe-nfc-plus3-balanced-mathcode-scratch/

dirfull-128k-claude-allmulti-equal-bpe/

dirfull-128k-claude-balanced-bpe/

dirfull-128k-claude-balanced-bpe-mathcode-scratch/

dirfull-128k-claude-balanced-bpe-seed42/

dirfull-128k-claude-balanced-bpe-seed43-mathcode-scratch/

dirfull-128k-claude-balanced-bpe-seed44-mathcode-scratch/

dirfull-128k-claude-balanced-nfc-bpe/

dirfull-128k-claude-balanced-nfc-bpe-mathcode-scratch/

dirfull-128k-claude-balanced-nfc-bpe-seed42/

dirfull-128k-claude-balanced-unigram/

dirfull-128k-claude-balanced-unigram-mathcode-scratch/

dirfull-128k-claude-balanced-unigram-seed42/

dirfull-128k-claude-english-bpe/

dirfull-128k-claude-english-bpe-mathcode-scratch/

dirfull-128k-claude-english-bpe-seed42/

dirfull-128k-clauderev-balanced-bpe/

dirfull-128k-clauderev-balanced-bpe-mathcode-scratch/

dirfull-128k-gpt2-balanced-bpe/

dirfull-128k-gpt4o-allmulti-equal-nfc-bpe/

dirfull-128k-gpt4o-allmulti-equal-norepeat-bpe/

dirfull-128k-gpt4o-balanced-bpe/

dirfull-128k-gpt4o-balanced-bpe-seed42/

dirfull-128k-gpt4o-balanced-bpe-seed43-mathcode-scratch/

dirfull-128k-gpt4o-balanced-bpe-seed44-mathcode-scratch/

dirfull-128k-gpt4o-balanced-maxfiles50-bpe/

dirfull-128k-gpt4o-balanced-nfc-bpe/

dirfull-128k-gpt4o-balanced-nfc-bpe-mathcode-scratch/

dirfull-128k-gpt4o-balanced-nfc-bpe-seed42/

dirfull-128k-gpt4o-balanced-unigram/

dirfull-128k-gpt4o-balanced-unigram-mathcode-scratch/

dirfull-128k-gpt4o-balanced-unigram-seed42/

dirfull-128k-gpt4o-balanced-unigram-tuned/

dirfull-128k-gpt4o-code-bpe/

dirfull-128k-gpt4o-code-bpe-mathcode-scratch/

dirfull-128k-gpt4o-code-bpe-seed42/

dirfull-128k-gpt4o-codegrad00-bpe-mathcode-scratch/

dirfull-128k-gpt4o-codegrad05-bpe-mathcode-scratch/

dirfull-128k-gpt4o-codegrad30-bpe-mathcode-scratch/

dirfull-128k-gpt4o-codegrad50-bpe-mathcode-scratch/

dirfull-128k-gpt4o-english-bpe/

dirfull-128k-gpt4o-english-bpe-mathcode-scratch/

dirfull-128k-gpt4o-english-bpe-seed42/

dirfull-128k-gpt4o-english-fullbyte-bpe/

dirfull-128k-gpt4o-english-fullbyte-bpe-mathcode-scratch/

dirfull-128k-gpt4o-invert-full-bpe/

dirfull-128k-gpt4o-invert-full-bpe-fresh70/

dirfull-128k-gpt4o-invert-full-bpe-fresh70-continue-fresh160/

dirfull-128k-gpt4o-invert-full-bpe-seed43/

dirfull-128k-gpt4o-invert-full-bpe-seed43-fresh70/

dirfull-128k-gpt4o-invert-full-bpe-seed43-fresh70-continue-fresh160/

dirfull-128k-gpt4o-invert-full-bpe-tokenmatch/

dirfull-128k-gpt4o-invert-full-bpe-tokenmatch-fresh/

dirfull-128k-gpt4o-no-english-bpe/

dirfull-128k-gpt4o-perm-a-bpe/

dirfull-128k-gpt4o-perm-b-bpe/

dirfull-128k-gpt4onl-balanced-bpe-mathcode-scratch/

dirfull-128k-gpt4onl-balanced-bpe-seed43-mathcode-scratch/

dirfull-128k-gpt4onl-balanced-bpe-seed44-mathcode-scratch/

dirfull-128k-gpt4onl-codegrad00-bpe-mathcode-scratch/

dirfull-128k-gpt4onl-codegrad05-bpe-mathcode-scratch/

dirfull-128k-gpt4onl-codegrad30-bpe-mathcode-scratch/

dirfull-128k-gpt4onl-codegrad50-bpe-mathcode-scratch/

dirfull-128k-highmid-bpe/

dirfull-128k-highmid-unigram/

dirfull-128k-highres-bpe/

dirfull-128k-highres-unigram/

dirfull-128k-llama3/

dirfull-128k-llama3-mathcode-scratch/

dirfull-128k-llama3-seed42/

dirfull-128k-mathcode-scratch/

dirfull-128k-pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow/

dirfull-128k-pabpe-nfc-apertus-fineweb2full-capped-tuned-hybridwindow-mathcode-scratch/

dirfull-128k-pabpe-nfc-clean-balanced/

dirfull-128k-pabpe-nfc-clean-balanced-hybridwindow-mathcode-scratch/

dirfull-128k-pabpe-nfc-clean-fineweb2full/

dirfull-128k-pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow/

dirfull-128k-pabpe-nfc-clean-fineweb2full-capped-tuned-hybridwindow-mathcode-scratch/

dirfull-128k-pabpe-nfc-clean-fineweb2full-consv2-engfull-eu3-gm110k/

dirfull-128k-pabpe-nfc-clean-fineweb2full-consv2-engfull-eu3-gm110k-mathcode-scratch/

dirfull-128k-pabpe-nfc-clean-fineweb2full-consv2-eudata-gm110k/

dirfull-128k-pabpe-nfc-clean-fineweb2full-consv2-eudata-gm90k/

dirfull-128k-pabpe-nfc-clean-fineweb2full-consv2-eudata4-gm100k/

dirfull-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k/

dirfull-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-frde-kr120-gm130k-v200k-mathcode-scratch/

dirfull-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-gm120k-v200k/

dirfull-128k-pabpe-nfc-clean-fineweb2full-consv2-eusino-v2c-gm120k-v200k-mathcode-scratch/

dirfull-128k-pabpe-nfc-clean-fineweb2full-consv2-frde2-gm110k/

dirfull-128k-pabpe-nfc-clean-fineweb2full-consv2-frde2-gm110k-mathcode-scratch/

dirfull-128k-pabpe-nfc-clean-fineweb2full-consv2-plus3-repcap8/

dirfull-128k-pabpe-nfc-clean-fineweb2full-consv2-plus3-repcap8-mathcode-scratch/

dirfull-128k-pabpe-nfc-clean-fineweb2full-consv2-prelim-mul-v131072/

dirfull-128k-pabpe-nfc-clean-fineweb2full-consv2-prelim-mul-v131072-mathcode-scratch/

dirfull-128k-pabpe-nfc-clean-fineweb2full-hybridwindow/

dirfull-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow/

dirfull-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g/

dirfull-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g-mathcode-scratch/

dirfull-128k-pabpe-nfc-clean-fineweb2full-plus2-capped-tuned-hybridwindow-mathcode-scratch/

dirfull-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2/

dirfull-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-mathcode-scratch/

dirfull-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g/

dirfull-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm70k-v130-sp-eng5g-mathcode-scratch/

dirfull-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g/

dirfull-128k-pabpe-nfc-clean-fineweb2full-plus3-capped-tuned-hybridwindow-consv2-tailcuts-gm90k-v130-sp-eng5g-mathcode-scratch/

dirfull-128k-pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8/

dirfull-128k-pabpe-nfc-clean-fw2full-plus3-cap-hw-consv2-gm90k-tailcuts-eng5g-repcap8-mathcode-scratch/

dirfull-128k-pabpe-nfc-gpt4-balanced/

dirfull-128k-pabpe-nfc-gpt4-balanced-hybridwindow/

dirfull-128k-pabpe-nfc-gpt4-balanced-mathcode-scratch/

dirfull-128k-pabpe-nfc-gpt4-fineweb2full/

dirfull-128k-pabpe-nfc-gpt4-fineweb2full-hybridwindow/

dirfull-128k-punct-allmulti-equal-bpe/

dirfull-128k-punct-balanced-bpe/

dirfull-128k-punct-balanced-bpe-mathcode-scratch/

dirfull-128k-punct-balanced-bpe-seed42/

dirfull-128k-punct-balanced-bpe-seed42-mathcode-scratch/

dirfull-128k-punct-balanced-bpe-seed43-mathcode-scratch/

dirfull-128k-punct-balanced-bpe-seed44-mathcode-scratch/

dirfull-128k-punct-balancedhalf-superbpe-t64k-copy/

dirfull-128k-punct-english-bpe/

dirfull-128k-punct-english-bpe-mathcode-scratch/

dirfull-128k-punct-english-bpe-seed42/

dirfull-128k-rightalign-balanced-bpe/

dirfull-128k-rightalign-balanced-bpe-mathcode-scratch/

dirfull-128k-rightalign-balanced-bpe-seed42/

dirfull-128k-rightalign-balanced-nfc-bpe/

dirfull-128k-rightalign-balanced-nfc-bpe-mathcode-scratch/

dirfull-128k-rightalign-balanced-nfc-bpe-seed42/

dirfull-128k-rightalign-balanced-unigram/

dirfull-128k-rightalign-balanced-unigram-mathcode-scratch/

dirfull-128k-rightalign-balanced-unigram-seed42/

dirfull-128k-scripttok-bpe-scriptenc_gpt4o_cb/

dirfull-128k-scripttok-bpe-scriptenc_gpt4o_cb-mathcode-scratch/

dirfull-128k-scripttok-mingram-scriptenc_cb/

dirfull-128k-scripttok-mingram-scriptenc_cb_nl/

dirfull-128k-scripttok-mingram-scriptenc_cb_nl-mathcode-scratch/

dirfull-128k-scripttok-mingram-scriptenc_cb-mathcode-scratch/

dirfull-128k-superbpe-apertus-fineweb2full-capped-hybridwindow/

dirfull-128k-superbpe-apertus-fineweb2full-capped-hybridwindow-mathcode-scratch/

dirfull-128k-superbpe-clean-c2-balanced/

dirfull-128k-superbpe-clean-c2-balanced-mathcode-scratch/

dirfull-128k-superbpe-clean-c3-balanced/

dirfull-128k-superbpe-clean-c3-balanced-mathcode-scratch/

dirfull-128k-superbpe-clean-fineweb2full-capped-hybridwindow/

dirfull-128k-superbpe-clean-fineweb2full-capped-hybridwindow-mathcode-scratch/

dirfull-128k-superbpe-clean-fineweb2full-capped-hybridwindow-t110k-v130k/

dirfull-128k-superbpe-clean-fineweb2full-capped-hybridwindow-t110k-v130k-mathcode-scratch/

dirfull-128k-superbpe-clean-fineweb2full-plus2-cappedv2-hybridwindow-t110k-v130k-consv2/

dirfull-128k-superbpe-clean-fineweb2full-plus2-cappedv2-hybridwindow-t110k-v130k-consv2-mathcode-scratch/

dirfull-128k-superbpe-gpt4-fineweb2full-hybridwindow-mathcode-scratch/

dirfull-128k-superbpe-gpt4o-balanced/

dirfull-128k-superbpe-gpt4o-balanced-mathcode-scratch/

dirfull-128k-superbpe-pabpe-nfc-gpt4-balanced/

dirfull-128k-superbpe-pabpe-nfc-gpt4-t64k/

dirfull-128k-whitespace-balanced-bpe/

dirfull-128k-whitespace-balanced-bpe-mathcode-scratch/

dirfull-128k-whitespace-balanced-unigram/

dirfull-128k-whitespace-multiheavy-bpe/

dirpilot-128k-apertus-wdfix/

dirpilot-128k-claude-balanced-bpe/

dirpilot-128k-claude-balanced-nfc-bpe/

dirpilot-128k-claude-balanced-unigram/

dirpilot-128k-claude-english-bpe/

dirpilot-128k-gpt4o-balanced-bpe-wdfix/

dirpilot-128k-gpt4o-balanced-nfc-bpe-wdfix/

dirpilot-128k-gpt4o-balanced-unigram-wdfix/

dirpilot-128k-gpt4o-code-bpe-wdfix/

dirpilot-128k-gpt4o-english-bpe-wdfix/

dirpilot-128k-llama3-wdfix/

dirpilot-128k-pabpe-nfc-gpt4-balanced/

dirpilot-128k-punct-balanced-bpe-wdfix/

dirpilot-128k-punct-english-bpe-wdfix/

dirpilot-128k-rightalign-balanced-bpe-wdfix/

dirpilot-128k-rightalign-balanced-nfc-bpe-wdfix/

dirpilot-128k-rightalign-balanced-unigram-wdfix/

dirpilot-128k-superbpe-gpt4o-balanced/

dirsmall-128k-apertus/

dirsmall-128k-claude-balanced-bpe/

dirsmall-128k-claude-balanced-nfc-bpe/

dirsmall-128k-claude-balanced-unigram/

dirsmall-128k-claude-english-bpe/

dirsmall-128k-gpt4o-balanced-bpe/

dirsmall-128k-gpt4o-balanced-nfc-bpe/

dirsmall-128k-gpt4o-balanced-unigram/

dirsmall-128k-gpt4o-code-bpe/

dirsmall-128k-gpt4o-english-bpe/

dirsmall-128k-llama3/

dirsmall-128k-pabpe-nfc-gpt4-balanced/

dirsmall-128k-punct-balanced-bpe/

dirsmall-128k-punct-english-bpe/

dirsmall-128k-rightalign-balanced-bpe/

dirsmall-128k-rightalign-balanced-nfc-bpe/

dirsmall-128k-rightalign-balanced-unigram/

dirsmall-128k-superbpe-gpt4o-balanced/

dirtiny-128k-apertus/

dirtiny-128k-claude-balanced-bpe/

dirtiny-128k-claude-balanced-nfc-bpe/

dirtiny-128k-claude-balanced-unigram/

dirtiny-128k-claude-english-bpe/

dirtiny-128k-gpt4o-balanced-bpe/

dirtiny-128k-gpt4o-balanced-nfc-bpe/

dirtiny-128k-gpt4o-balanced-unigram/

dirtiny-128k-gpt4o-code-bpe/

dirtiny-128k-gpt4o-english-bpe/

dirtiny-128k-llama3/

dirtiny-128k-pabpe-nfc-gpt4-balanced/

dirtiny-128k-punct-balanced-bpe/

dirtiny-128k-punct-english-bpe/

dirtiny-128k-rightalign-balanced-bpe/

dirtiny-128k-rightalign-balanced-nfc-bpe/

dirtiny-128k-rightalign-balanced-unigram/

dirtiny-128k-superbpe-gpt4o-balanced/

cmeister/tokenizer-lm-ablations · main · files are served by the source, never re-hosted here