Felipe97/llama-cpp-compiled

1// Unit tests for quantization specific functions - quantize, dequantize and dot product2 3#include "ggml.h"4#include "ggml-cpu.h"5 6#undef NDEBUG7#include <assert.h>8#include <algorithm>9#include <cmath>10#include <math.h>11#include <stdio.h>12#include <string>13#include <vector>14 15#if defined(_MSC_VER)16#pragma warning(disable: 4244 4267) // possible loss of data17#endif18 19constexpr float MAX_QUANTIZATION_REFERENCE_ERROR = 0.0001f;20constexpr float MAX_QUANTIZATION_TOTAL_ERROR = 0.002f;21constexpr float MAX_QUANTIZATION_TOTAL_ERROR_BINARY = 0.025f;22constexpr float MAX_QUANTIZATION_TOTAL_ERROR_TERNARY = 0.01f;23constexpr float MAX_QUANTIZATION_TOTAL_ERROR_2BITS = 0.0075f;24constexpr float MAX_QUANTIZATION_TOTAL_ERROR_3BITS = 0.0040f;25constexpr float MAX_QUANTIZATION_TOTAL_ERROR_3BITS_XXS = 0.0050f;26constexpr float MAX_QUANTIZATION_TOTAL_ERROR_FP4 = 0.0030f;27constexpr float MAX_DOT_PRODUCT_ERROR = 0.02f;28constexpr float MAX_DOT_PRODUCT_ERROR_LOWBIT = 0.04f;29constexpr float MAX_DOT_PRODUCT_ERROR_FP4 = 0.03f;30constexpr float MAX_DOT_PRODUCT_ERROR_BINARY = 0.40f;31constexpr float MAX_DOT_PRODUCT_ERROR_TERNARY = 0.15f;32 33static const char* RESULT_STR[] = {"ok", "FAILED"};34 35 36// Generate synthetic data37static void generate_data(float offset, size_t n, float * dst, float amplitude = 2.0f) {38    for (size_t i = 0; i < n; i++) {39        dst[i] = 0.1 + amplitude*cosf(i + offset);40    }41}42 43// Calculate RMSE between two float arrays44static float array_rmse(const float * a1, const float * a2, size_t n) {45    double sum = 0;46    for (size_t i = 0; i < n; i++) {47        double diff = a1[i] - a2[i];48        sum += diff * diff;49    }50    return sqrtf(sum) / n;51}52 53// Total quantization error on test data54static float total_quantization_error(const ggml_type_traits * qfns, const ggml_type_traits_cpu * qfns_cpu, size_t test_size, const float * test_data) {55    std::vector<uint8_t> tmp_q(2*test_size);56    std::vector<float> tmp_out(test_size);57 58    qfns_cpu->from_float(test_data, tmp_q.data(), test_size);59    qfns->to_float(tmp_q.data(), tmp_out.data(), test_size);60    return array_rmse(test_data, tmp_out.data(), test_size);61}62 63// Total quantization error on test data64static float reference_quantization_error(const ggml_type_traits * qfns, const ggml_type_traits_cpu * qfns_cpu, size_t test_size, const float * test_data) {65    std::vector<uint8_t> tmp_q(2*test_size);66    std::vector<float> tmp_out(test_size);67    std::vector<float> tmp_out_ref(test_size);68 69    // FIXME: why is done twice?70    qfns_cpu->from_float(test_data, tmp_q.data(), test_size);71    qfns->to_float(tmp_q.data(), tmp_out.data(), test_size);72 73    qfns->from_float_ref(test_data, tmp_q.data(), test_size);74    qfns->to_float(tmp_q.data(), tmp_out_ref.data(), test_size);75 76    return array_rmse(tmp_out.data(), tmp_out_ref.data(), test_size);77}78 79static float dot_product(const float * a1, const float * a2, size_t test_size) {80    double sum = 0;81    for (size_t i = 0; i < test_size; i++) {82        sum += a1[i] * a2[i];83    }84    return sum;85}86 87// Total dot product error88static float dot_product_error(const ggml_type_traits_cpu * qfns_cpu, ggml_type src0_type, size_t test_size,89                               const float * test_data1, const float * test_data2,90                               const float * test_data3, const float * test_data4,91                               const int nrc) {92    const auto * vdot = ggml_get_type_traits_cpu(qfns_cpu->vec_dot_type);93    const size_t pad  = 64;94    const size_t bx   = ggml_row_size(src0_type, test_size) + pad;95    const size_t by   = ggml_row_size(qfns_cpu->vec_dot_type, test_size) + pad;96 97    std::vector<uint8_t> tmp_q1(bx * nrc);98    std::vector<uint8_t> tmp_q2(by * nrc);99 100    qfns_cpu->from_float(test_data1, tmp_q1.data(), test_size);101    vdot->from_float(test_data2, tmp_q2.data(), test_size);102 103    if (nrc == 1) {104        float result = INFINITY;105        qfns_cpu->vec_dot(test_size, &result, 0, tmp_q1.data(), 0, tmp_q2.data(), 0, 1);106 107        const float dot_ref = dot_product(test_data1, test_data2, test_size);108        return fabsf(result - dot_ref) / test_size;109    }110 111    // nrc == 2: kernel computes a 2x2 dot product matrix112    // Output layout: s[0]=dot(vx0,vy0), s[1]=dot(vx1,vy0), s[bs]=dot(vx0,vy1), s[bs+1]=dot(vx1,vy1)113    // row and output strides are padded, same as in the mul_mat path114    qfns_cpu->from_float(test_data3, tmp_q1.data() + bx, test_size);115    vdot->from_float(test_data4, tmp_q2.data() + by, test_size);116 117    const size_t bs = 16;118    std::vector<float> result(bs + 2, INFINITY);119    qfns_cpu->vec_dot(test_size, result.data(), bs, tmp_q1.data(), bx, tmp_q2.data(), by, 2);120 121    const float ref00 = dot_product(test_data1, test_data2, test_size);122    const float ref10 = dot_product(test_data3, test_data2, test_size);123    const float ref01 = dot_product(test_data1, test_data4, test_size);124    const float ref11 = dot_product(test_data3, test_data4, test_size);125 126    const auto err = [test_size](float val, float ref) {127        const float e = fabsf(val - ref) / test_size;128        return std::isfinite(e) ? e : INFINITY;129    };130 131    return std::max({err(result[0], ref00), err(result[1], ref10), err(result[bs], ref01), err(result[bs + 1], ref11)});132}133 134static int test_vec_dot_f32(bool verbose) {135    const auto * f32 = ggml_get_type_traits_cpu(GGML_TYPE_F32);136    int num_failed = 0;137    for (int n : {1, 2, 3, 5, 7, 8, 15, 16, 17, 31, 33, 63, 67, 127, 129, 193, 255, 1023}) {138        std::vector<float> a(n);139        std::vector<float> b(n);140        generate_data(0.0, n, a.data());141        generate_data(1.0, n, b.data());142 143        float result = 0.0f;144        f32->vec_dot(n, &result, 0, a.data(), 0, b.data(), 0, 1);145        const float ref = dot_product(a.data(), b.data(), n);146        const float error = fabsf(result - ref) / n;147 148        const bool failed = !(error < MAX_QUANTIZATION_REFERENCE_ERROR);149        num_failed += failed;150        if (failed || verbose) {151            printf(" f32 vec_dot n=%4d:                 %s (ref=%f got=%f err=%f)\n",152                   n, RESULT_STR[failed], ref, result, error);153        }154    }155    return num_failed;156}157 158static int test_vec_dot_q(bool verbose) {159    int num_failed = 0;160 161    const size_t test_size = 32 * 128;162 163    std::vector<float> test_data(test_size);164    std::vector<float> test_data2(test_size);165    std::vector<float> test_data3(test_size);166    std::vector<float> test_data4(test_size);167 168    generate_data(0.0, test_data.size(), test_data.data());169    generate_data(1.0, test_data2.size(), test_data2.data());170    generate_data(3.0, test_data3.size(), test_data3.data(), 1.0f);171    generate_data(4.0, test_data4.size(), test_data4.data(), 1.5f);172 173    for (int i = 0; i < GGML_TYPE_COUNT; i++) {174        ggml_type type = (ggml_type) i;175        const auto * qfns = ggml_get_type_traits(type);176        const auto * qfns_cpu = ggml_get_type_traits_cpu(type);177 178        // deprecated - skip179        if (qfns->blck_size == 0) {180            continue;181        }182 183        const ggml_type ei = (ggml_type)i;184 185        printf("Testing %s\n", ggml_type_name((ggml_type) i));186        ggml_quantize_init(ei);187 188        if (qfns_cpu->from_float && qfns->to_float) {189            const float total_error = total_quantization_error(qfns, qfns_cpu, test_size, test_data.data());190            const float max_quantization_error =191                type == GGML_TYPE_Q1_0    ? MAX_QUANTIZATION_TOTAL_ERROR_BINARY :192                type == GGML_TYPE_TQ1_0   ? MAX_QUANTIZATION_TOTAL_ERROR_TERNARY :193                type == GGML_TYPE_TQ2_0   ? MAX_QUANTIZATION_TOTAL_ERROR_TERNARY :194                type == GGML_TYPE_Q2_0    ? MAX_QUANTIZATION_TOTAL_ERROR_TERNARY :195                type == GGML_TYPE_Q2_K    ? MAX_QUANTIZATION_TOTAL_ERROR_2BITS :196                type == GGML_TYPE_IQ2_S   ? MAX_QUANTIZATION_TOTAL_ERROR_2BITS :197                type == GGML_TYPE_Q3_K    ? MAX_QUANTIZATION_TOTAL_ERROR_3BITS :198                type == GGML_TYPE_IQ3_S   ? MAX_QUANTIZATION_TOTAL_ERROR_3BITS :199                type == GGML_TYPE_IQ3_XXS ? MAX_QUANTIZATION_TOTAL_ERROR_3BITS_XXS :200                type == GGML_TYPE_NVFP4   ? MAX_QUANTIZATION_TOTAL_ERROR_FP4 : MAX_QUANTIZATION_TOTAL_ERROR;201            bool failed = !(total_error < max_quantization_error);202            num_failed += failed;203            if (failed || verbose) {204                printf("%5s absolute quantization error:    %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], total_error);205            }206 207            const float reference_error = reference_quantization_error(qfns, qfns_cpu, test_size, test_data.data());208            failed = !(reference_error < MAX_QUANTIZATION_REFERENCE_ERROR);209            num_failed += failed;210            if (failed || verbose) {211                printf("%5s reference implementation error: %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], reference_error);212            }213 214            const float vec_dot_error = dot_product_error(qfns_cpu, type, test_size, test_data.data(), test_data2.data(), nullptr, nullptr, 1);215            const float max_allowed_error = type == GGML_TYPE_Q2_K || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ2_XXS ||216                type == GGML_TYPE_IQ3_XXS || type == GGML_TYPE_IQ3_S || type == GGML_TYPE_IQ2_S217                ? MAX_DOT_PRODUCT_ERROR_LOWBIT218                : type == GGML_TYPE_Q1_0219                ? MAX_DOT_PRODUCT_ERROR_BINARY220                : type == GGML_TYPE_TQ1_0 || type == GGML_TYPE_TQ2_0 || type == GGML_TYPE_Q2_0221                ? MAX_DOT_PRODUCT_ERROR_TERNARY222                : type == GGML_TYPE_NVFP4223                ? MAX_DOT_PRODUCT_ERROR_FP4224                : MAX_DOT_PRODUCT_ERROR;225            failed = !(vec_dot_error < max_allowed_error);226            num_failed += failed;227            if (failed || verbose) {228                printf("%5s dot product error:              %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], vec_dot_error);229            }230 231            // Test nrc=2 path for types that support it232            if (qfns_cpu->nrows == 2) {233                const float vec_dot_error_nrc2 = dot_product_error(qfns_cpu, type, test_size, test_data.data(), test_data2.data(), test_data3.data(), test_data4.data(), 2);234                failed = !(vec_dot_error_nrc2 < max_allowed_error);235                num_failed += failed;236                if (failed || verbose) {237                    printf("%5s dot product error (nrc=2):    %s (%f)\n", ggml_type_name(type), RESULT_STR[failed], vec_dot_error_nrc2);238                }239            }240        }241    }242 243    return num_failed;244}245 246int main(int argc, char * argv[]) {247    bool verbose = false;248 249    std::string arg;250    for (int i = 1; i < argc; i++) {251        arg = argv[i];252 253        if (arg == "-v") {254            verbose = true;255        } else {256            fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());257            return 1;258        }259    }260 261    ggml_cpu_init();262 263    int num_failed = 0;264 265    num_failed += test_vec_dot_f32(verbose);266    num_failed += test_vec_dot_q(verbose);267 268    if (num_failed || verbose) {269        printf("%d tests failed\n", num_failed);270    }271 272    return num_failed > 0;273}274