Felipe97/llama-cpp-compiled

1#include "llama.h"2 3#include "build-info.h"4#include "common.h"5 6#include "../src/llama-model.h"7 8#include "ggml.h"9#include "ggml-cpu.h"10 11#include <algorithm>12#include <cassert>13#include <cinttypes>14#include <cmath>15#include <cstdio>16#include <cstring>17#include <numeric>18#include <regex>19#include <string>20#include <vector>21#include <thread>22#include <mutex>23 24#if defined(_MSC_VER)25#pragma warning(disable: 4244 4267) // possible loss of data26#endif27 28struct quantize_stats_params {29    std::string model = "models/7B/ggml-model-f16.gguf";30    bool verbose = false;31    bool per_layer_stats = false;32    bool print_histogram = false;33    bool reference = false;34    std::vector<std::string> include_layers;35    std::vector<std::string> exclude_layers;36    std::vector<enum ggml_type> include_types;37};38 39constexpr size_t HISTOGRAM_BUCKETS = 150;40constexpr double HISTOGRAM_RANGE = 0.03;41 42struct error_stats {43    size_t num_samples;44    double total_error;45    double max_error;46    uint64_t error_histogram[HISTOGRAM_BUCKETS];47};48 49static void quantize_stats_print_usage(int /*argc*/, char ** argv) {50    quantize_stats_params params;51    fprintf(stderr, "usage: %s [options]\n", argv[0]);52    fprintf(stderr, "\n");53    fprintf(stderr, "options:\n");54    fprintf(stderr, "  -h, --help            show this help message and exit\n");55    fprintf(stderr, "  -m FNAME, --model FNAME\n");56    fprintf(stderr, "                        model path (default: %s)\n", params.model.c_str());57    fprintf(stderr, "  -r, --reference\n");58    fprintf(stderr, "                        use reference implementation (default: false)\n");59    fprintf(stderr, "  -v, --verbose\n");60    fprintf(stderr, "                        verbose output (default: false)\n");61    fprintf(stderr, "  -p, --per-layer-stats\n");62    fprintf(stderr, "                        print stats per layer (default: false)\n");63    fprintf(stderr, "  --histogram\n");64    fprintf(stderr, "                        print error histogram (default: false)\n");65    fprintf(stderr, "  -l LAYER, --include-layer LAYER\n");66    fprintf(stderr, "                        only test layers matching pattern\n");67    fprintf(stderr, "  -L LAYER, --exclude-layer LAYER\n");68    fprintf(stderr, "                        exclude layers matching pattern\n");69    fprintf(stderr, "  -t TYPE, --type TYPE\n");70    fprintf(stderr, "                        only test given type (q4_0, q4_1)\n");71    fprintf(stderr, "\n");72}73 74// Check if a layer is included/excluded by command line75static bool layer_included(const quantize_stats_params & params, const std::string & layer) {76    for (const auto& excluded : params.exclude_layers) {77        if (std::regex_search(layer, std::regex(excluded))) {78            return false;79        }80    }81    for (const auto& included : params.include_layers) {82        if (std::regex_search(layer, std::regex(included))) {83            return true;84        }85    }86    return params.include_layers.empty();87}88 89// Update error statistics given vectors with the before/after result of quantization90static void update_error_stats(int64_t nelements, const float * input, const float * output, error_stats & stats) {91    for (int64_t i = 0; i < nelements; i++) {92        double diff = input[i] - output[i];93        stats.total_error += diff * diff;94        stats.max_error = fmax(fabs(diff), stats.max_error);95        stats.error_histogram[std::max(std::min((size_t) floor(fabs(diff) / HISTOGRAM_RANGE * HISTOGRAM_BUCKETS), HISTOGRAM_BUCKETS-1), (size_t) 0)]++;96    }97    stats.num_samples += nelements;98}99 100static void combine_error_stats(error_stats & into, const error_stats & from) {101    into.num_samples += from.num_samples;102    into.total_error += from.total_error;103    if (from.max_error > into.max_error) into.max_error = from.max_error;104    for (size_t i=0; i<HISTOGRAM_BUCKETS; ++i) into.error_histogram[i] += from.error_histogram[i];105}106 107static double find_quantile(const error_stats & stats, double quantile) {108    double sum = std::accumulate(std::begin(stats.error_histogram), std::end(stats.error_histogram), 0.0);109 110    double accum = 0;111    for (size_t i = 0; i < HISTOGRAM_BUCKETS; i++) {112        accum += stats.error_histogram[i];113        if (accum >= sum*quantile) {114            return (i+1) * HISTOGRAM_RANGE / HISTOGRAM_BUCKETS;115        }116    }117    return INFINITY;118}119 120static void print_error_stats(const std::string & name, const error_stats & stats, bool print_histogram) {121    double rmse = sqrt(stats.total_error / (double) stats.num_samples);122    double median = find_quantile(stats, .5);123    double pct95 = find_quantile(stats, .95);124    printf("%-50s: rmse %.8f, maxerr %.8f, 95pct<%.4f, median<%.4f\n", name.c_str(), rmse, stats.max_error, pct95, median);125    if (print_histogram) {126        printf("Error distribution:\n");127        for (size_t i = 0; i < HISTOGRAM_BUCKETS; i++) {128            double lower = i * HISTOGRAM_RANGE / HISTOGRAM_BUCKETS;129            double upper = (i+1) * HISTOGRAM_RANGE / HISTOGRAM_BUCKETS;130            if (i == HISTOGRAM_BUCKETS -1) upper = INFINITY;131            printf("[%3.4f, %3.4f): %11" PRIu64 "\n", lower, upper, stats.error_histogram[i]);132        }133    }134}135 136// copied from ggml.h - verify that we can access this as a flat array137static bool tensor_is_contiguous(const struct ggml_tensor * tensor) {138    static_assert(GGML_MAX_DIMS == 4, "GGML_MAX_DIMS is not 4 - update this function");139 140    return141        tensor->nb[0] == ggml_type_size(tensor->type) &&142        tensor->nb[1] == (tensor->nb[0]*tensor->ne[0])/ggml_blck_size(tensor->type) &&143        tensor->nb[2] == tensor->nb[1]*tensor->ne[1] &&144        tensor->nb[3] == tensor->nb[2]*tensor->ne[2];145}146 147static void test_roundtrip_on_chunk(148    const ggml_tensor * layer, int64_t offset, int64_t chunk_size, const ggml_type_traits & qfns, const ggml_type_traits_cpu & qfns_cpu, bool use_reference,149    float * input_scratch, char * quantized_scratch, float * output_scratch, error_stats & stats150) {151    if (layer->type == GGML_TYPE_F16) {152        for (int i = 0; i < chunk_size; i++) {153            input_scratch[i] = ggml_get_f32_1d(layer, i + offset);154        }155    } else {156        input_scratch = ggml_get_data_f32(layer) + offset;157    }158 159    if (use_reference) {160        qfns.from_float_ref(input_scratch, quantized_scratch, chunk_size);161    } else {162        qfns_cpu.from_float(input_scratch, quantized_scratch, chunk_size);163    }164    qfns.to_float(quantized_scratch, output_scratch, chunk_size);165 166    update_error_stats(chunk_size, input_scratch, output_scratch, stats);167}168 169 170// Run quantization function for a single layer and update error stats171static void test_roundtrip_on_layer(172    std::string & name, bool print_layer_stats, const ggml_type_traits & qfns, const ggml_type_traits_cpu & qfns_cpu, bool use_reference,173    const ggml_tensor * layer, std::vector<float> & input_scratch, std::vector<char> & quantized_scratch,174    std::vector<float> & output_scratch, error_stats & total_error, int max_thread = 0175) {176    assert(tensor_is_contiguous(layer));177    error_stats layer_error {};178    uint64_t nelements = ggml_nelements(layer);179 180    float* input_scratch_ptr = nullptr;181    if (layer->type == GGML_TYPE_F16) {182        if (input_scratch.size() < nelements) input_scratch.resize(nelements);183        input_scratch_ptr = input_scratch.data();184    }185    if (quantized_scratch.size() < 4*nelements) quantized_scratch.resize(4*nelements);186    if (output_scratch.size() < nelements) output_scratch.resize(nelements);187 188    if (max_thread < 1) max_thread = std::thread::hardware_concurrency();189    int chunk_size = 32*512;190    int num_chunks = (nelements + chunk_size - 1)/chunk_size;191 192    if (num_chunks < 2 || max_thread < 2) {193        test_roundtrip_on_chunk(layer, 0, nelements, qfns, qfns_cpu, use_reference, input_scratch_ptr, quantized_scratch.data(),194                output_scratch.data(), print_layer_stats ? layer_error : total_error);195    } else {196        auto & stats = print_layer_stats ? layer_error : total_error;197        std::mutex mutex;198        uint64_t counter = 0;199        auto compute = [&mutex, &counter, &stats, &qfns, &qfns_cpu, nelements, layer, use_reference, input_scratch_ptr,200             &quantized_scratch, &output_scratch, chunk_size] () {201            error_stats local_stats {};202            while (true) {203                std::unique_lock<std::mutex> lock(mutex);204                uint64_t offset = counter; counter += chunk_size;205                if (offset >= nelements) {206                    combine_error_stats(stats, local_stats);207                    break;208                }209                lock.unlock();210                uint64_t chunk = offset + chunk_size < nelements ? chunk_size : nelements - offset;211                test_roundtrip_on_chunk(layer, offset, chunk, qfns, qfns_cpu, use_reference, input_scratch_ptr + offset,212                        quantized_scratch.data() + 4*offset, output_scratch.data() + offset, local_stats);213            }214        };215        int nthread = std::min(num_chunks, max_thread);216        std::vector<std::thread> workers(nthread-1);217        for (auto& w : workers) w = std::thread(compute);218        compute();219        for (auto& w : workers) w.join();220    }221 222    if (print_layer_stats) {223        print_error_stats(name, layer_error, false);224        combine_error_stats(total_error, layer_error);225    }226}227 228int main(int argc, char ** argv) {229    ggml_time_init();230 231    quantize_stats_params params;232 233    // read command line234 235    int max_thread = 0;236    bool invalid_param = false;237    std::string arg;238    for (int i = 1; i < argc; i++) {239        arg = argv[i];240 241        if (arg == "-h" || arg == "--help") {242            quantize_stats_print_usage(argc, argv);243            exit(0);244        } else if (arg == "-r" || arg == "--reference") {245            params.reference = true;246        } else if (arg == "-v") {247            params.verbose = true;248        } else if (arg == "-p" || arg == "--per-layer-stats") {249            params.per_layer_stats = true;250        } else if (arg == "--histogram") {251            params.print_histogram = true;252        } else if (arg == "-m" || arg == "--model") {253            if (++i >= argc) {254                invalid_param = true;255                break;256            }257            params.model = argv[i];258        } else if (arg == "-l" || arg == "--include-layer") {259            if (++i >= argc) {260                invalid_param = true;261                break;262            }263            params.include_layers.emplace_back(argv[i]);264        } else if (arg == "-L" || arg == "--exclude-layer") {265            if (++i >= argc) {266                invalid_param = true;267                break;268            }269            params.exclude_layers.emplace_back(argv[i]);270        } else if (arg == "-t" || arg == "--type") {271            if (++i >= argc) {272                invalid_param = true;273                break;274            }275            int j;276            for (j = 0; j < GGML_TYPE_COUNT; ++j) {277               const auto * name = ggml_type_name((ggml_type) j);278               if (name && strcmp(argv[i], name) == 0) break;279            }280            if (j < GGML_TYPE_COUNT) {281                params.include_types.push_back((ggml_type) j);282            } else {283                fprintf(stderr, "error: %s not in list of types\n", argv[i]);284                invalid_param = true;285            }286        } else if (arg == "-n" || arg == "--num-threads") {287            if (++i >= argc) {288                invalid_param = true;289                break;290            }291            max_thread = atoi(argv[i]);292        } else {293            fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());294            quantize_stats_print_usage(argc, argv);295            return 1;296        }297    }298    if (invalid_param) {299        fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str());300        quantize_stats_print_usage(argc, argv);301        return 1;302    }303 304    llama_print_build_info(llama_version());305 306    // load the model307    fprintf(stderr, "Loading model\n");308 309    const int64_t t_main_start_us = ggml_time_us();310    llama_model * model;311    llama_context * ctx;312 313    {314        auto mparams = llama_model_default_params();315        mparams.load_mode = LLAMA_LOAD_MODE_NONE;316 317        model = llama_model_load_from_file(params.model.c_str(), mparams);318 319        if (model == NULL) {320            fprintf(stderr, "%s: error: failed to load model '%s'\n", __func__, params.model.c_str());321            return 1;322        }323 324        auto cparams = llama_context_default_params();325        cparams.n_ctx = 256;326 327        ctx = llama_init_from_model(model, cparams);328 329        if (ctx == NULL) {330            fprintf(stderr, "%s: error: failed to create context with model '%s'\n", __func__, params.model.c_str());331            llama_model_free(model);332            return 1;333        }334    }335 336    const auto & tensors = llama_internal_get_tensor_map(model);337 338    // check layer tensors339    int included_layers = 0;340    int64_t max_nelements = 0;341    bool is_f16 = false;342    for (const auto & kv_tensor : tensors) {343        if (!layer_included(params, kv_tensor.first)) {344            continue;345        }346        if (params.verbose) {347            printf("%s: type %s, size %" PRId64 "\n", kv_tensor.first.c_str(), ggml_type_name(kv_tensor.second->type), ggml_nelements(kv_tensor.second));348        }349        if (kv_tensor.second->type == GGML_TYPE_F16) {350            is_f16 = true;351        } else if (kv_tensor.second->type != GGML_TYPE_F32) {352            fprintf(stderr, "%s: error: Quantization should be tested with a float model, "353                "this model contains already quantized layers (%s is type %d)\n", __func__, kv_tensor.first.c_str(), kv_tensor.second->type);354            llama_free(ctx);355            llama_model_free(model);356            return 1;357        }358        included_layers++;359        max_nelements = std::max(max_nelements, ggml_nelements(kv_tensor.second));360    }361 362    if (is_f16) {363        printf("note: source model is f16\n");364    }365    printf("testing %d layers with max size %" PRId64 "\n", included_layers, max_nelements);366    // allocate scratch space367    std::vector<float> input_scratch;368    std::vector<char> quantized_scratch;369    std::vector<float> output_scratch;370 371    // loop throught quantization types372    for (int i = 0; i < GGML_TYPE_COUNT; i++) {373        const ggml_type type = (ggml_type) i;374        if (!params.include_types.empty() && std::find(params.include_types.begin(), params.include_types.end(), i) == params.include_types.end()) {375            continue;376        }377        const auto * qfns     = ggml_get_type_traits(type);378        const auto * qfns_cpu = ggml_get_type_traits_cpu(type);379        if (qfns_cpu->from_float && qfns->to_float) {380            if (params.verbose) {381                printf("testing %s ...\n",  ggml_type_name(type));382            }383 384            ggml_quantize_init(type);385 386            error_stats global_stats {};387 388            for (const auto & kv_tensor : tensors) {389                if (!layer_included(params, kv_tensor.first)) {390                    continue;391                }392                if (params.verbose) {393                    printf("  %s ...\n",  kv_tensor.first.c_str());394                }395                std::string layer_name { ggml_type_name(type) };396                layer_name += "::" + kv_tensor.first;397                test_roundtrip_on_layer(398                        layer_name,399                        params.per_layer_stats,400                        *qfns, *qfns_cpu,401                        params.reference,402                        kv_tensor.second,403                        input_scratch,404                        quantized_scratch,405                        output_scratch,406                        global_stats,407                        max_thread408                );409            }410 411            print_error_stats(ggml_type_name(type), global_stats, params.print_histogram);412        }413    }414 415 416    llama_free(ctx);417    llama_model_free(model);418    // report timing419    {420        const int64_t t_main_end_us = ggml_time_us();421 422        printf("\n");423        printf("%s:    total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0);424    }425 426    return 0;427}428