echodict/llama.cpp

1from __future__ import annotations2 3import array4import unicodedata5import requests6 7 8MAX_CODEPOINTS = 0x1100009 10UNICODE_DATA_URL = "https://www.unicode.org/Public/UCD/latest/ucd/UnicodeData.txt"11 12 13# see https://www.unicode.org/L2/L1999/UnicodeData.html14def unicode_data_iter():15    res = requests.get(UNICODE_DATA_URL)16    res.raise_for_status()17    data = res.content.decode()18 19    prev = []20 21    for line in data.splitlines():22        # ej: 0000;<control>;Cc;0;BN;;;;;N;NULL;;;;23        line = line.split(";")24 25        cpt = int(line[0], base=16)26        assert cpt < MAX_CODEPOINTS27 28        cpt_lower = int(line[-2] or "0", base=16)29        assert cpt_lower < MAX_CODEPOINTS30 31        cpt_upper = int(line[-3] or "0", base=16)32        assert cpt_upper < MAX_CODEPOINTS33 34        categ = line[2].strip()35        assert len(categ) == 236 37        bidir = line[4].strip()38        assert len(categ) == 239 40        name = line[1]41        if name.endswith(", First>"):42            prev = (cpt, cpt_lower, cpt_upper, categ, bidir)43            continue44        if name.endswith(", Last>"):45            assert prev[1:] == (0, 0, categ, bidir)46            for c in range(prev[0], cpt):47                yield (c, cpt_lower, cpt_upper, categ, bidir)48 49        yield (cpt, cpt_lower, cpt_upper, categ, bidir)50 51 52# see definition in unicode.h53CODEPOINT_FLAG_UNDEFINED   = 0x0001  #54CODEPOINT_FLAG_NUMBER      = 0x0002  # \p{N}55CODEPOINT_FLAG_LETTER      = 0x0004  # \p{L}56CODEPOINT_FLAG_SEPARATOR   = 0x0008  # \p{Z}57CODEPOINT_FLAG_MARK        = 0x0010  # \p{M}58CODEPOINT_FLAG_PUNCTUATION = 0x0020  # \p{P}59CODEPOINT_FLAG_SYMBOL      = 0x0040  # \p{S}60CODEPOINT_FLAG_CONTROL     = 0x0080  # \p{C}61 62UNICODE_CATEGORY_TO_FLAG = {63    "Cn": CODEPOINT_FLAG_UNDEFINED,    # Undefined64    "Cc": CODEPOINT_FLAG_CONTROL,      # Control65    "Cf": CODEPOINT_FLAG_CONTROL,      # Format66    "Co": CODEPOINT_FLAG_CONTROL,      # Private Use67    "Cs": CODEPOINT_FLAG_CONTROL,      # Surrrogate68    "Ll": CODEPOINT_FLAG_LETTER,       # Lowercase Letter69    "Lm": CODEPOINT_FLAG_LETTER,       # Modifier Letter70    "Lo": CODEPOINT_FLAG_LETTER,       # Other Letter71    "Lt": CODEPOINT_FLAG_LETTER,       # Titlecase Letter72    "Lu": CODEPOINT_FLAG_LETTER,       # Uppercase Letter73    "L&": CODEPOINT_FLAG_LETTER,       # Cased Letter74    "Mc": CODEPOINT_FLAG_MARK,         # Spacing Mark75    "Me": CODEPOINT_FLAG_MARK,         # Enclosing Mark76    "Mn": CODEPOINT_FLAG_MARK,         # Nonspacing Mark77    "Nd": CODEPOINT_FLAG_NUMBER,       # Decimal Number78    "Nl": CODEPOINT_FLAG_NUMBER,       # Letter Number79    "No": CODEPOINT_FLAG_NUMBER,       # Other Number80    "Pc": CODEPOINT_FLAG_PUNCTUATION,  # Connector Punctuation81    "Pd": CODEPOINT_FLAG_PUNCTUATION,  # Dash Punctuation82    "Pe": CODEPOINT_FLAG_PUNCTUATION,  # Close Punctuation83    "Pf": CODEPOINT_FLAG_PUNCTUATION,  # Final Punctuation84    "Pi": CODEPOINT_FLAG_PUNCTUATION,  # Initial Punctuation85    "Po": CODEPOINT_FLAG_PUNCTUATION,  # Other Punctuation86    "Ps": CODEPOINT_FLAG_PUNCTUATION,  # Open Punctuation87    "Sc": CODEPOINT_FLAG_SYMBOL,       # Currency Symbol88    "Sk": CODEPOINT_FLAG_SYMBOL,       # Modifier Symbol89    "Sm": CODEPOINT_FLAG_SYMBOL,       # Math Symbol90    "So": CODEPOINT_FLAG_SYMBOL,       # Other Symbol91    "Zl": CODEPOINT_FLAG_SEPARATOR,    # Line Separator92    "Zp": CODEPOINT_FLAG_SEPARATOR,    # Paragraph Separator93    "Zs": CODEPOINT_FLAG_SEPARATOR,    # Space Separator94}95 96 97codepoint_flags = array.array('H', [CODEPOINT_FLAG_UNDEFINED]) * MAX_CODEPOINTS98table_whitespace = []99table_lowercase = []100table_uppercase = []101table_nfd = []102 103for (cpt, cpt_lower, cpt_upper, categ, bidir) in unicode_data_iter():104    # convert codepoint to unicode character105    char = chr(cpt)106 107    # codepoint category flags108    codepoint_flags[cpt] = UNICODE_CATEGORY_TO_FLAG[categ]109 110    # lowercase conversion111    if cpt_lower:112        table_lowercase.append((cpt, cpt_lower))113 114    # uppercase conversion115    if cpt_upper:116        table_uppercase.append((cpt, cpt_upper))117 118    # NFD normalization119    norm = ord(unicodedata.normalize('NFD', char)[0])120    if cpt != norm:121        table_nfd.append((cpt, norm))122 123 124# whitespaces, see "<White_Space>" https://www.unicode.org/Public/UCD/latest/ucd/PropList.txt125table_whitespace.extend(range(0x0009, 0x000D + 1))126table_whitespace.extend(range(0x2000, 0x200A + 1))127table_whitespace.extend([0x0020, 0x0085, 0x00A0, 0x1680, 0x2028, 0x2029, 0x202F, 0x205F, 0x3000])128 129 130# sort by codepoint131table_whitespace.sort()132table_lowercase.sort()133table_uppercase.sort()134table_nfd.sort()135 136 137# group ranges with same flags138ranges_flags: list[tuple[int, int]] = [(0, codepoint_flags[0])]  # start, flags139for codepoint, flags in enumerate(codepoint_flags):140    if flags != ranges_flags[-1][1]:141        ranges_flags.append((codepoint, flags))142ranges_flags.append((MAX_CODEPOINTS, 0x0000))143 144 145# group ranges with same nfd146ranges_nfd: list[tuple[int, int, int]] = [(0, 0, 0)]  # start, last, nfd147for codepoint, norm in table_nfd:148    start = ranges_nfd[-1][0]149    if ranges_nfd[-1] != (start, codepoint - 1, norm):150        ranges_nfd.append((0, 0, 0))  # dummy, will be replaced below151        start = codepoint152    ranges_nfd[-1] = (start, codepoint, norm)153 154 155# Generate 'unicode-data.cpp':156#   python ./scripts//gen-unicode-data.py > unicode-data.cpp157 158def out(line=""):159    print(line, end='\n')  # noqa160 161 162out("""\163// generated with scripts/gen-unicode-data.py164 165#include "unicode-data.h"166 167#include <cstdint>168#include <vector>169#include <unordered_map>170#include <unordered_set>171""")172 173out("const std::vector<std::pair<uint32_t, uint16_t>> unicode_ranges_flags = {  // start, flags // last=next_start-1")174for codepoint, flags in ranges_flags:175    out("{0x%06X, 0x%04X}," % (codepoint, flags))176out("};\n")177 178out("const std::unordered_set<uint32_t> unicode_set_whitespace = {")179for codepoint in table_whitespace:180    out("0x%06X," % codepoint)181out("};\n")182 183out("const std::unordered_map<uint32_t, uint32_t> unicode_map_lowercase = {")184for tuple_lw in table_lowercase:185    out("{0x%06X, 0x%06X}," % tuple_lw)186out("};\n")187 188out("const std::unordered_map<uint32_t, uint32_t> unicode_map_uppercase = {")189for tuple_up in table_uppercase:190    out("{0x%06X, 0x%06X}," % tuple_up)191out("};\n")192 193out("const std::vector<range_nfd> unicode_ranges_nfd = {  // start, last, nfd")194for triple in ranges_nfd:195    out("{0x%06X, 0x%06X, 0x%06X}," % triple)196out("};\n")197