Felipe97/llama-cpp-compiled

1#include "unicode.h"2 3#include <algorithm>4#include <cassert>5#include <stdexcept>6#include <string>7#include <vector>8 9// implementation adopted from src/unicode.cpp10 11size_t common_utf8_sequence_length(unsigned char first_byte) {12    const size_t lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 };13    uint8_t highbits = static_cast<uint8_t>(first_byte) >> 4;14    return lookup[highbits];15}16 17utf8_parse_result common_parse_utf8_codepoint(std::string_view input, size_t offset) {18    if (offset >= input.size()) {19        return utf8_parse_result(utf8_parse_result::INCOMPLETE);20    }21 22    // ASCII fast path23    if (!(input[offset] & 0x80)) {24        return utf8_parse_result(utf8_parse_result::SUCCESS, input[offset], 1);25    }26 27    // Invalid: continuation byte as first byte28    if (!(input[offset] & 0x40)) {29        return utf8_parse_result(utf8_parse_result::INVALID);30    }31 32    // 2-byte sequence33    if (!(input[offset] & 0x20)) {34        if (offset + 1 >= input.size()) {35            return utf8_parse_result(utf8_parse_result::INCOMPLETE);36        }37        if ((input[offset + 1] & 0xc0) != 0x80) {38            return utf8_parse_result(utf8_parse_result::INVALID);39        }40        auto result = ((input[offset] & 0x1f) << 6) | (input[offset + 1] & 0x3f);41        return utf8_parse_result(utf8_parse_result::SUCCESS, result, 2);42    }43 44    // 3-byte sequence45    if (!(input[offset] & 0x10)) {46        if (offset + 2 >= input.size()) {47            return utf8_parse_result(utf8_parse_result::INCOMPLETE);48        }49        if ((input[offset + 1] & 0xc0) != 0x80 || (input[offset + 2] & 0xc0) != 0x80) {50            return utf8_parse_result(utf8_parse_result::INVALID);51        }52        auto result = ((input[offset] & 0x0f) << 12) | ((input[offset + 1] & 0x3f) << 6) | (input[offset + 2] & 0x3f);53        return utf8_parse_result(utf8_parse_result::SUCCESS, result, 3);54    }55 56    // 4-byte sequence57    if (!(input[offset] & 0x08)) {58        if (offset + 3 >= input.size()) {59            return utf8_parse_result(utf8_parse_result::INCOMPLETE);60        }61        if ((input[offset + 1] & 0xc0) != 0x80 || (input[offset + 2] & 0xc0) != 0x80 || (input[offset + 3] & 0xc0) != 0x80) {62            return utf8_parse_result(utf8_parse_result::INVALID);63        }64        auto result = ((input[offset] & 0x07) << 18) | ((input[offset + 1] & 0x3f) << 12) | ((input[offset + 2] & 0x3f) << 6) | (input[offset + 3] & 0x3f);65        return utf8_parse_result(utf8_parse_result::SUCCESS, result, 4);66    }67 68    // Invalid first byte69    return utf8_parse_result(utf8_parse_result::INVALID);70}71 72bool common_utf8_is_complete(const std::string & s) {73    if (s.empty()) {74        return true;75    }76    for (int i = 1; i <= std::min(4, (int)s.size()); i++) {77        unsigned char c = s[s.size() - i];78        if ((c & 0xC0) != 0x80) {79            int expected = (c >= 0xF0) ? 4 : (c >= 0xE0) ? 3 : (c >= 0xC0) ? 2 : 1;80            return i >= expected;81        }82    }83    return false;84}85 86std::string common_unicode_cpts_to_utf8(const std::vector<uint32_t> & cps) {87    std::string result;88    for (size_t i = 0; i < cps.size(); ++i) {89        result.append(common_unicode_cpt_to_utf8(cps[i]));90    }91    return result;92}93 94std::string common_unicode_cpt_to_utf8(uint32_t cpt) {95    std::string result;96 97    if (/* 0x00 <= cpt && */ cpt <= 0x7f) {98        result.push_back(cpt);99        return result;100    }101    if (0x80 <= cpt && cpt <= 0x7ff) {102        result.push_back(0xc0 | ((cpt >> 6) & 0x1f));103        result.push_back(0x80 | (cpt & 0x3f));104        return result;105    }106    if (0x800 <= cpt && cpt <= 0xffff) {107        result.push_back(0xe0 | ((cpt >> 12) & 0x0f));108        result.push_back(0x80 | ((cpt >> 6) & 0x3f));109        result.push_back(0x80 | (cpt & 0x3f));110        return result;111    }112    if (0x10000 <= cpt && cpt <= 0x10ffff) {113        result.push_back(0xf0 | ((cpt >> 18) & 0x07));114        result.push_back(0x80 | ((cpt >> 12) & 0x3f));115        result.push_back(0x80 | ((cpt >> 6) & 0x3f));116        result.push_back(0x80 | (cpt & 0x3f));117        return result;118    }119 120    throw std::invalid_argument("invalid codepoint");121}122 123 124 125