Codeprocastinator/optimized-tinyllama-covalent
0119
1#!/usr/bin/env python32# -*- coding: utf-8 -*-3 4# This script downloads the tokenizer models of the specified models from Huggingface and5# generates the get_vocab_base_pre() function for convert_hf_to_gguf.py6#7# This is necessary in order to analyze the type of pre-tokenizer used by the model and8# provide the necessary information to llama.cpp via the GGUF header in order to implement9# the same pre-tokenizer.10#11# ref: https://github.com/ggml-org/llama.cpp/pull/692012#13# Instructions:14#15# - Add a new model to the "models" list16# - Run the script with your huggingface token:17#18# python3 convert_hf_to_gguf_update.py <huggingface_token>19#20# - The convert_hf_to_gguf.py script will have had its get_vocab_base_pre() function updated21# - Update llama.cpp with the new pre-tokenizer if necessary22#23# TODO: generate tokenizer tests for llama.cpp24#25 26import logging27import os28import pathlib29import re30 31import requests32import sys33import json34import shutil35 36from hashlib import sha25637from enum import IntEnum, auto38from transformers import AutoTokenizer39 40logging.basicConfig(level=logging.DEBUG)41logger = logging.getLogger("convert_hf_to_gguf_update")42sess = requests.Session()43 44 45class TOKENIZER_TYPE(IntEnum):46 SPM = auto()47 BPE = auto()48 WPM = auto()49 UGM = auto()50 51 52# TODO: this string has to exercise as much pre-tokenizer functionality as possible53# will be updated with time - contributions welcome54CHK_TXT = '\n \n\n \n\n\n \t \t\t \t\n \n \n \n \n🚀 (normal) 😶🌫️ (multiple emojis concatenated) ✅ 🦙🦙 3 33 333 3333 33333 333333 3333333 33333333 3.3 3..3 3...3 កាន់តែពិសេសអាច😁 ?我想在apple工作1314151天～ ------======= нещо на Български \'\'\'\'\'\'```````\"\"\"\"......!!!!!!?????? I\'ve been \'told he\'s there, \'RE you sure? \'M not sure I\'ll make it, \'D you like some tea? We\'Ve a\'lL'55 56if len(sys.argv) == 2:57 token = sys.argv[1]58 if not token.startswith("hf_"):59 logger.info("Huggingface token seems invalid")60 logger.info("Usage: python convert_hf_to_gguf_update.py <huggingface_token>")61 sys.exit(1)62else:63 logger.info("Usage: python convert_hf_to_gguf_update.py <huggingface_token>")64 sys.exit(1)65 66# TODO: add models here, base models preferred67models = [68 {"name": "llama-spm", "tokt": TOKENIZER_TYPE.SPM, "repo": "https://huggingface.co/meta-llama/Llama-2-7b-hf", },69 {"name": "llama-bpe", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/meta-llama/Meta-Llama-3-8B", },70 {"name": "phi-3", "tokt": TOKENIZER_TYPE.SPM, "repo": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct", },71 {"name": "deepseek-llm", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/deepseek-llm-7b-base", },72 {"name": "deepseek-coder", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/deepseek-coder-6.7b-base", },73 {"name": "falcon", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tiiuae/falcon-7b", },74 {"name": "bert-bge", "tokt": TOKENIZER_TYPE.WPM, "repo": "https://huggingface.co/BAAI/bge-small-en-v1.5", },75 {"name": "falcon3", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/tiiuae/Falcon3-7B-Base", },76 {"name": "bert-bge-large", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/BAAI/bge-large-zh-v1.5", },77 {"name": "mpt", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/mosaicml/mpt-7b", },78 {"name": "starcoder", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/bigcode/starcoder2-3b", },79 {"name": "gpt-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/openai-community/gpt2", },80 {"name": "stablelm2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/stabilityai/stablelm-2-zephyr-1_6b", },81 {"name": "refact", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/smallcloudai/Refact-1_6-base", },82 {"name": "command-r", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereForAI/c4ai-command-r-v01", },83 {"name": "qwen2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Qwen/Qwen1.5-7B", },84 {"name": "olmo", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/allenai/OLMo-1.7-7B-hf", },85 {"name": "dbrx", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/databricks/dbrx-base", },86 {"name": "jina-v1-en", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jinaai/jina-reranker-v1-tiny-en", },87 {"name": "jina-v2-en", "tokt": TOKENIZER_TYPE.WPM, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-en", }, # WPM!88 {"name": "jina-v2-es", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-es", },89 {"name": "jina-v2-de", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-de", },90 {"name": "smaug-bpe", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/abacusai/Smaug-Llama-3-70B-Instruct", },91 {"name": "poro-chat", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LumiOpen/Poro-34B-chat", },92 {"name": "jina-v2-code", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/jinaai/jina-embeddings-v2-base-code", },93 {"name": "viking", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LumiOpen/Viking-7B", }, # Also used for Viking 13B and 33B94 {"name": "gemma", "tokt": TOKENIZER_TYPE.SPM, "repo": "https://huggingface.co/google/gemma-2b", },95 {"name": "gemma-2", "tokt": TOKENIZER_TYPE.SPM, "repo": "https://huggingface.co/google/gemma-2-9b", },96 {"name": "jais", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/core42/jais-13b", },97 {"name": "t5", "tokt": TOKENIZER_TYPE.UGM, "repo": "https://huggingface.co/google-t5/t5-small", },98 {"name": "codeshell", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/WisdomShell/CodeShell-7B", },99 {"name": "tekken", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/mistralai/Mistral-Nemo-Base-2407", },100 {"name": "smollm", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/HuggingFaceTB/SmolLM-135M", },101 {'name': "bloom", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/bigscience/bloom", },102 {'name': "gpt3-finnish", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/TurkuNLP/gpt3-finnish-small", },103 {"name": "exaone", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct", },104 {"name": "phi-2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/microsoft/phi-2", },105 {"name": "chameleon", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/facebook/chameleon-7b", },106 {"name": "minerva-7b", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/sapienzanlp/Minerva-7B-base-v1.0", },107 {"name": "roberta-bpe", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/sentence-transformers/stsb-roberta-base"},108 {"name": "gigachat", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/ai-sage/GigaChat-20B-A3B-instruct"},109 {"name": "megrez", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Infinigence/Megrez-3B-Instruct"},110 {"name": "deepseek-v3", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/DeepSeek-V3"},111 {"name": "deepseek-r1-qwen", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B"},112 {"name": "gpt-4o", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Xenova/gpt-4o", },113 {"name": "superbpe", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/UW/OLMo2-8B-SuperBPE-t180k", },114 {"name": "trillion", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/trillionlabs/Trillion-7B-preview", },115 {"name": "bailingmoe", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/inclusionAI/Ling-lite", },116 {"name": "llama4", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct", },117 {"name": "glm4", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/THUDM/glm-4-9b-hf", },118]119 120 121def download_file_with_auth(url, token, save_path):122 headers = {"Authorization": f"Bearer {token}"}123 response = sess.get(url, headers=headers)124 response.raise_for_status()125 os.makedirs(os.path.dirname(save_path), exist_ok=True)126 with open(save_path, 'wb') as downloaded_file:127 downloaded_file.write(response.content)128 logger.info(f"File {save_path} downloaded successfully")129 130 131def download_model(model):132 name = model["name"]133 repo = model["repo"]134 tokt = model["tokt"]135 136 os.makedirs(f"models/tokenizers/{name}", exist_ok=True)137 138 files = ["config.json", "tokenizer.json", "tokenizer_config.json"]139 140 if name == "gpt-4o":141 # Xenova/gpt-4o is tokenizer-only, it does not contain config.json142 files = ["tokenizer.json", "tokenizer_config.json"]143 144 if tokt == TOKENIZER_TYPE.SPM:145 files.append("tokenizer.model")146 147 if tokt == TOKENIZER_TYPE.UGM:148 files.append("spiece.model")149 150 if os.path.isdir(repo):151 # If repo is a path on the file system, copy the directory152 for file in files:153 src_path = os.path.join(repo, file)154 dst_path = f"models/tokenizers/{name}/{file}"155 if os.path.isfile(dst_path):156 logger.info(f"{name}: File {dst_path} already exists - skipping")157 continue158 if os.path.isfile(src_path):159 shutil.copy2(src_path, dst_path)160 logger.info(f"{name}: Copied {src_path} to {dst_path}")161 else:162 logger.warning(f"{name}: Source file {src_path} does not exist")163 else:164 # If repo is a URL, download the files165 for file in files:166 save_path = f"models/tokenizers/{name}/{file}"167 if os.path.isfile(save_path):168 logger.info(f"{name}: File {save_path} already exists - skipping")169 continue170 download_file_with_auth(f"{repo}/resolve/main/{file}", token, save_path)171 172 173for model in models:174 try:175 download_model(model)176 except Exception as e:177 logger.error(f"Failed to download model {model['name']}. Error: {e}")178 179 180# generate the source code for the convert_hf_to_gguf.py:get_vocab_base_pre() function:181 182src_ifs = ""183for model in models:184 name = model["name"]185 tokt = model["tokt"]186 187 if tokt == TOKENIZER_TYPE.SPM or tokt == TOKENIZER_TYPE.UGM:188 continue189 190 # Skip if the tokenizer folder does not exist or there are other download issues previously191 if not os.path.exists(f"models/tokenizers/{name}"):192 logger.warning(f"Directory for tokenizer {name} not found. Skipping...")193 continue194 195 # create the tokenizer196 try:197 if name == "t5":198 tokenizer = AutoTokenizer.from_pretrained(f"models/tokenizers/{name}", use_fast=False)199 else:200 tokenizer = AutoTokenizer.from_pretrained(f"models/tokenizers/{name}")201 except OSError as e:202 logger.error(f"Error loading tokenizer for model {name}. The model may not exist or is not accessible with the provided token. Error: {e}")203 continue # Skip to the next model if the tokenizer can't be loaded204 205 chktok = tokenizer.encode(CHK_TXT)206 chkhsh = sha256(str(chktok).encode()).hexdigest()207 208 logger.info(f"model: {name}")209 logger.info(f"tokt: {tokt}")210 logger.info(f"repo: {model['repo']}")211 logger.info(f"chktok: {chktok}")212 logger.info(f"chkhsh: {chkhsh}")213 214 # print the "pre_tokenizer" content from the tokenizer.json215 with open(f"models/tokenizers/{name}/tokenizer.json", "r", encoding="utf-8") as f:216 cfg = json.load(f)217 normalizer = cfg["normalizer"]218 logger.info("normalizer: " + json.dumps(normalizer, indent=4))219 pre_tokenizer = cfg["pre_tokenizer"]220 logger.info("pre_tokenizer: " + json.dumps(pre_tokenizer, indent=4))221 if "ignore_merges" in cfg["model"]:222 logger.info("ignore_merges: " + json.dumps(cfg["model"]["ignore_merges"], indent=4))223 224 logger.info("")225 226 src_ifs += f" if chkhsh == \"{chkhsh}\":\n"227 src_ifs += f" # ref: {model['repo']}\n"228 src_ifs += f" res = \"{name}\"\n"229 230src_func = f"""231 def get_vocab_base_pre(self, tokenizer) -> str:232 # encoding this string and hashing the resulting tokens would (hopefully) give us a unique identifier that233 # is specific for the BPE pre-tokenizer used by the model234 # we will use this unique identifier to write a "tokenizer.ggml.pre" entry in the GGUF file which we can235 # use in llama.cpp to implement the same pre-tokenizer236 237 chktxt = {repr(CHK_TXT)}238 239 chktok = tokenizer.encode(chktxt)240 chkhsh = sha256(str(chktok).encode()).hexdigest()241 242 logger.debug(f"chktok: {{chktok}}")243 logger.debug(f"chkhsh: {{chkhsh}}")244 245 res = None246 247 # NOTE: if you get an error here, you need to update the convert_hf_to_gguf_update.py script248 # or pull the latest version of the model from Huggingface249 # don't edit the hashes manually!250{src_ifs}251 if res is None:252 logger.warning("\\n")253 logger.warning("**************************************************************************************")254 logger.warning("** WARNING: The BPE pre-tokenizer was not recognized!")255 logger.warning("** There are 2 possible reasons for this:")256 logger.warning("** - the model has not been added to convert_hf_to_gguf_update.py yet")257 logger.warning("** - the pre-tokenization config has changed upstream")258 logger.warning("** Check your model files and convert_hf_to_gguf_update.py and update them accordingly.")259 logger.warning("** ref: https://github.com/ggml-org/llama.cpp/pull/6920")260 logger.warning("**")261 logger.warning(f"** chkhsh: {{chkhsh}}")262 logger.warning("**************************************************************************************")263 logger.warning("\\n")264 raise NotImplementedError("BPE pre-tokenizer was not recognized - update get_vocab_base_pre()")265 266 logger.debug(f"tokenizer.ggml.pre: {{repr(res)}}")267 logger.debug(f"chkhsh: {{chkhsh}}")268 269 return res270"""271 272convert_py_pth = pathlib.Path("convert_hf_to_gguf.py")273convert_py = convert_py_pth.read_text(encoding="utf-8")274convert_py = re.sub(275 r"(# Marker: Start get_vocab_base_pre)(.+?)( +# Marker: End get_vocab_base_pre)",276 lambda m: m.group(1) + src_func + m.group(3),277 convert_py,278 flags=re.DOTALL | re.MULTILINE,279)280 281convert_py_pth.write_text(convert_py, encoding="utf-8")282 283logger.info("+++ convert_hf_to_gguf.py was updated")284 285# generate tests for each tokenizer model286 287tests = [288 "ied 4 ½ months",289 "Führer",290 "",291 " ",292 " ",293 " ",294 "\t",295 "\n",296 "\n\n",297 "\n\n\n",298 "\t\n",299 "Hello world",300 " Hello world",301 "Hello World",302 " Hello World",303 " Hello World!",304 "Hello, world!",305 " Hello, world!",306 " this is 🦙.cpp",307 "w048 7tuijk dsdfhu",308 "нещо на Български",309 "កាន់តែពិសេសអាចខលចេញ",310 "🚀 (normal) 😶🌫️ (multiple emojis concatenated) ✅ (only emoji that has its own token)",311 "Hello",312 " Hello",313 " Hello",314 " Hello",315 " Hello",316 " Hello\n Hello",317 " (",318 "\n =",319 "' era",320 "Hello, y'all! How are you 😁 ?我想在apple工作1314151天～",321 "!!!!!!",322 "3",323 "33",324 "333",325 "3333",326 "33333",327 "333333",328 "3333333",329 "33333333",330 "333333333",331 "Cửa Việt", # llama-bpe fails on this332 " discards",333 CHK_TXT,334]335 336# write the tests to ./models/ggml-vocab-{name}.gguf.inp337# the format is:338#339# test0340# __ggml_vocab_test__341# test1342# __ggml_vocab_test__343# ...344#345 346# with each model, encode all tests and write the results in ./models/ggml-vocab-{name}.gguf.out347# for each test, write the resulting tokens on a separate line348 349for model in models:350 name = model["name"]351 tokt = model["tokt"]352 353 # Skip if the tokenizer folder does not exist or there are other download issues previously354 if not os.path.exists(f"models/tokenizers/{name}"):355 logger.warning(f"Directory for tokenizer {name} not found. Skipping...")356 continue357 358 # create the tokenizer359 try:360 if name == "t5":361 tokenizer = AutoTokenizer.from_pretrained(f"models/tokenizers/{name}", use_fast=False)362 else:363 tokenizer = AutoTokenizer.from_pretrained(f"models/tokenizers/{name}")364 except OSError as e:365 logger.error(f"Failed to load tokenizer for model {name}. Error: {e}")366 continue # Skip this model and continue with the next one in the loop367 368 with open(f"models/ggml-vocab-{name}.gguf.inp", "w", encoding="utf-8") as f:369 for text in tests:370 f.write(f"{text}")371 f.write("\n__ggml_vocab_test__\n")372 373 with open(f"models/ggml-vocab-{name}.gguf.out", "w") as f:374 for text in tests:375 res = tokenizer.encode(text, add_special_tokens=False)376 for r in res:377 f.write(f" {r}")378 f.write("\n")379 380 logger.info(f"Tests for {name} written in ./models/ggml-vocab-{name}.gguf.*")381 382# generate commands for creating vocab files383 384logger.info("\nRun the following commands to generate the vocab files for testing:\n")385 386for model in models:387 name = model["name"]388 389 print(f"python3 convert_hf_to_gguf.py models/tokenizers/{name}/ --outfile models/ggml-vocab-{name}.gguf --vocab-only") # noqa: NP100390 391logger.info("\n")392