Felipe97/llama-cpp-compiled

1#include "ngram-mod.h"2 3#include <algorithm>4 5//6// common_ngram_mod7//8 9common_ngram_mod::common_ngram_mod(uint16_t n, size_t size) : n(n), used(0) {10    entries.resize(size);11 12    reset();13}14 15size_t common_ngram_mod::idx(const entry_t * tokens) const {16    size_t res = 0;17 18    for (size_t i = 0; i < n; ++i) {19        res = res*6364136223846793005ULL + tokens[i];20    }21 22    res = res % entries.size();23 24    return res;25}26 27void common_ngram_mod::add(const entry_t * tokens) {28    const size_t i = idx(tokens);29 30    if (entries[i] == EMPTY) {31        used++;32    }33 34    entries[i] = tokens[n];35}36 37common_ngram_mod::entry_t common_ngram_mod::get(const entry_t * tokens) const {38    const size_t i = idx(tokens);39 40    return entries[i];41}42 43void common_ngram_mod::reset() {44    std::fill(entries.begin(), entries.end(), EMPTY);45    used = 0;46}47 48size_t common_ngram_mod::get_n() const {49    return n;50}51 52size_t common_ngram_mod::get_used() const {53    return used;54}55 56size_t common_ngram_mod::size() const {57    return entries.size();58}59 60size_t common_ngram_mod::size_bytes() const {61    return entries.size() * sizeof(entries[0]);62}63