Cortiqa/Falin-300M-Preview

1import os
2from tokenizers import Tokenizer,models,trainers,pre_tokenizers,decoders,processors
3
4def build_tokenizer(vocab_size=32000,files=None,save_dir="tokenizer"):
5    tok=Tokenizer(models.BPE(unk_token="<unk>"))
6    tok.pre_tokenizer=pre_tokenizers.ByteLevel(add_prefix_space=False)
7    tok.decoder=decoders.ByteLevel()
8    tok.post_processor=processors.ByteLevel(trim_offsets=False)
9    specials=["<pad>","<s>","</s>","<unk>"]
10    tr=trainers.BpeTrainer(vocab_size=vocab_size,min_frequency=2,
11                           special_tokens=specials,show_progress=True)
12    if files:
13        tok.train(files,tr)
14    else:
15        tmp="/tmp/_meno_tok.txt"
16        with open(tmp,'w') as f: f.write("Tokenizer data placeholder. "*500+"\n")
17        tok.train([tmp],tr)
18        os.remove(tmp)
19    os.makedirs(save_dir,exist_ok=True)
20    tok.save(os.path.join(save_dir,"tokenizer.json"))
21    return tok
22
23def load_tokenizer(path="tokenizer"):
24    return Tokenizer.from_file(os.path.join(path,"tokenizer.json"))
25