echodict/llama.cpp

1#include "ggml.h"2#include "gguf.h"3 4#include <clocale>5#include <cstdio>6#include <string>7#include <sstream>8#include <vector>9 10#undef MIN11#undef MAX12#define MIN(a, b) ((a) < (b) ? (a) : (b))13#define MAX(a, b) ((a) > (b) ? (a) : (b))14 15template <typename T>16static std::string to_string(const T & val) {17    std::stringstream ss;18    ss << val;19    return ss.str();20}21 22static bool gguf_ex_write(const std::string & fname) {23    struct gguf_context * ctx = gguf_init_empty();24 25    gguf_set_val_u8  (ctx, "some.parameter.uint8",    0x12);26    gguf_set_val_i8  (ctx, "some.parameter.int8",    -0x13);27    gguf_set_val_u16 (ctx, "some.parameter.uint16",   0x1234);28    gguf_set_val_i16 (ctx, "some.parameter.int16",   -0x1235);29    gguf_set_val_u32 (ctx, "some.parameter.uint32",   0x12345678);30    gguf_set_val_i32 (ctx, "some.parameter.int32",   -0x12345679);31    gguf_set_val_f32 (ctx, "some.parameter.float32",  0.123456789f);32    gguf_set_val_u64 (ctx, "some.parameter.uint64",   0x123456789abcdef0ull);33    gguf_set_val_i64 (ctx, "some.parameter.int64",   -0x123456789abcdef1ll);34    gguf_set_val_f64 (ctx, "some.parameter.float64",  0.1234567890123456789);35    gguf_set_val_bool(ctx, "some.parameter.bool",     true);36    gguf_set_val_str (ctx, "some.parameter.string",   "hello world");37 38    gguf_set_arr_data(ctx, "some.parameter.arr.i16", GGUF_TYPE_INT16,   std::vector<int16_t>{ 1, 2, 3, 4, }.data(), 4);39    gguf_set_arr_data(ctx, "some.parameter.arr.f32", GGUF_TYPE_FLOAT32, std::vector<float>{ 3.145f, 2.718f, 1.414f, }.data(), 3);40    gguf_set_arr_str (ctx, "some.parameter.arr.str",                    std::vector<const char *>{ "hello", "world", "!" }.data(), 3);41 42    struct ggml_init_params params = {43        /*.mem_size   =*/ 128ull*1024ull*1024ull,44        /*.mem_buffer =*/ NULL,45        /*.no_alloc   =*/ false,46    };47 48    struct ggml_context * ctx_data = ggml_init(params);49 50    const int n_tensors = 10;51 52    // tensor infos53    for (int i = 0; i < n_tensors; ++i) {54        const std::string name = "tensor_" + to_string(i);55 56        int64_t ne[GGML_MAX_DIMS] = { 1 };57        int32_t n_dims = rand() % GGML_MAX_DIMS + 1;58 59        for (int j = 0; j < n_dims; ++j) {60            ne[j] = rand() % 10 + 1;61        }62 63        struct ggml_tensor * cur = ggml_new_tensor(ctx_data, GGML_TYPE_F32, n_dims, ne);64        ggml_set_name(cur, name.c_str());65 66        {67            float * data = (float *) cur->data;68            for (int j = 0; j < ggml_nelements(cur); ++j) {69                data[j] = 100 + i;70            }71        }72 73        gguf_add_tensor(ctx, cur);74    }75 76    gguf_write_to_file(ctx, fname.c_str(), false);77 78    printf("%s: wrote file '%s;\n", __func__, fname.c_str());79 80    ggml_free(ctx_data);81    gguf_free(ctx);82 83    return true;84}85 86// just read tensor info87static bool gguf_ex_read_0(const std::string & fname) {88    struct gguf_init_params params = {89        /*.no_alloc = */ false,90        /*.ctx      = */ NULL,91    };92 93    struct gguf_context * ctx = gguf_init_from_file(fname.c_str(), params);94 95    if (!ctx) {96        fprintf(stderr, "%s: failed to load '%s'\n", __func__, fname.c_str());97        return false;98    }99 100    printf("%s: version:      %d\n", __func__, gguf_get_version(ctx));101    printf("%s: alignment:   %zu\n", __func__, gguf_get_alignment(ctx));102    printf("%s: data offset: %zu\n", __func__, gguf_get_data_offset(ctx));103 104    // kv105    {106        const int n_kv = gguf_get_n_kv(ctx);107 108        printf("%s: n_kv: %d\n", __func__, n_kv);109 110        for (int i = 0; i < n_kv; ++i) {111            const char * key = gguf_get_key(ctx, i);112 113            printf("%s: kv[%d]: key = %s\n", __func__, i, key);114        }115    }116 117    // find kv string118    {119        const char * findkey = "some.parameter.string";120 121        const int keyidx = gguf_find_key(ctx, findkey);122        if (keyidx == -1) {123            printf("%s: find key: %s not found.\n", __func__, findkey);124        } else {125            const char * key_value = gguf_get_val_str(ctx, keyidx);126            printf("%s: find key: %s found, kv[%d] value = %s\n", __func__, findkey, keyidx, key_value);127        }128    }129 130    // tensor info131    {132        const int n_tensors = gguf_get_n_tensors(ctx);133 134        printf("%s: n_tensors: %d\n", __func__, n_tensors);135 136        for (int i = 0; i < n_tensors; ++i) {137            const char * name   = gguf_get_tensor_name  (ctx, i);138            const size_t size   = gguf_get_tensor_size  (ctx, i);139            const size_t offset = gguf_get_tensor_offset(ctx, i);140 141            printf("%s: tensor[%d]: name = %s, size = %zu, offset = %zu\n", __func__, i, name, size, offset);142        }143    }144 145    gguf_free(ctx);146 147    return true;148}149 150// read and create ggml_context containing the tensors and their data151static bool gguf_ex_read_1(const std::string & fname, bool check_data) {152    struct ggml_context * ctx_data = NULL;153 154    struct gguf_init_params params = {155        /*.no_alloc = */ false,156        /*.ctx      = */ &ctx_data,157    };158 159    struct gguf_context * ctx = gguf_init_from_file(fname.c_str(), params);160 161    printf("%s: version:      %d\n", __func__, gguf_get_version(ctx));162    printf("%s: alignment:   %zu\n", __func__, gguf_get_alignment(ctx));163    printf("%s: data offset: %zu\n", __func__, gguf_get_data_offset(ctx));164 165    // kv166    {167        const int n_kv = gguf_get_n_kv(ctx);168 169        printf("%s: n_kv: %d\n", __func__, n_kv);170 171        for (int i = 0; i < n_kv; ++i) {172            const char * key = gguf_get_key(ctx, i);173 174            printf("%s: kv[%d]: key = %s\n", __func__, i, key);175        }176    }177 178    // tensor info179    {180        const int n_tensors = gguf_get_n_tensors(ctx);181 182        printf("%s: n_tensors: %d\n", __func__, n_tensors);183 184        for (int i = 0; i < n_tensors; ++i) {185            const char * name   = gguf_get_tensor_name  (ctx, i);186            const size_t size   = gguf_get_tensor_size  (ctx, i);187            const size_t offset = gguf_get_tensor_offset(ctx, i);188            const auto   type   = gguf_get_tensor_type  (ctx, i);189 190            const char * type_name  = ggml_type_name(type);191            const size_t type_size  = ggml_type_size(type);192            const size_t n_elements = size / type_size;193 194            printf("%s: tensor[%d]: name = %s, size = %zu, offset = %zu, type = %s, n_elts = %zu\n", __func__, i, name, size, offset, type_name, n_elements);195        }196    }197 198    // data199    {200        const int n_tensors = gguf_get_n_tensors(ctx);201 202        for (int i = 0; i < n_tensors; ++i) {203            printf("%s: reading tensor %d data\n", __func__, i);204 205            const char * name = gguf_get_tensor_name(ctx, i);206 207            struct ggml_tensor * cur = ggml_get_tensor(ctx_data, name);208 209            printf("%s: tensor[%d]: n_dims = %d, ne = (%d, %d, %d, %d), name = %s, data = %p\n",210                __func__, i, ggml_n_dims(cur), int(cur->ne[0]), int(cur->ne[1]), int(cur->ne[2]), int(cur->ne[3]), cur->name, cur->data);211 212            // print first 10 elements213            const float * data = (const float *) cur->data;214 215            printf("%s data[:10] : ", name);216            for (int j = 0; j < MIN(10, ggml_nelements(cur)); ++j) {217                printf("%f ", data[j]);218            }219            printf("\n\n");220 221            // check data222            if (check_data) {223                const float * data = (const float *) cur->data;224                for (int j = 0; j < ggml_nelements(cur); ++j) {225                    if (data[j] != 100 + i) {226                        fprintf(stderr, "%s: tensor[%d], data[%d]: found %f, expected %f\n", __func__, i, j, data[j], float(100 + i));227                        gguf_free(ctx);228                        return false;229                    }230                }231            }232        }233    }234 235    printf("%s: ctx_data size: %zu\n", __func__, ggml_get_mem_size(ctx_data));236 237    ggml_free(ctx_data);238    gguf_free(ctx);239 240    return true;241}242 243int main(int argc, char ** argv) {244    std::setlocale(LC_NUMERIC, "C");245 246    if (argc < 3) {247        printf("usage: %s data.gguf r|w [n]\n", argv[0]);248        printf("r: read data.gguf file\n");249        printf("w: write data.gguf file\n");250        printf("n: no check of tensor data\n");251        return -1;252    }253    bool check_data = true;254    if (argc == 4) {255        check_data = false;256    }257 258    srand(123456);259 260    const std::string fname(argv[1]);261    const std::string mode (argv[2]);262 263    GGML_ASSERT((mode == "r" || mode == "w") && "mode must be r or w");264 265    if (mode == "w") {266        GGML_ASSERT(gguf_ex_write(fname) && "failed to write gguf file");267    } else if (mode == "r") {268        GGML_ASSERT(gguf_ex_read_0(fname) && "failed to read gguf file");269        GGML_ASSERT(gguf_ex_read_1(fname, check_data) && "failed to read gguf file");270    }271 272    return 0;273}274