Felipe97/llama-cpp-compiled
01.1k
1// This file defines tests for various GGML ops and backends.2// For the forward pass it asserts that the results of multiple backends computing the same GGML ops are consistent.3// For the backward pass it asserts that the gradients from backpropagation are consistent4// with the gradients obtained via the method of finite differences ("grad" mode, this is optional).5// It is also possible to check the performance ("perf" mode).6//7// this file has three sections: Section 1 does general setup, section 2 defines the GGML ops to be tested,8// and section 3 defines which tests to run.9// Quick start for adding a new GGML op: Go to section 2 and create a struct that inherits from test_case,10// then go to section 3 and add an instantiation of your struct.11 12 13// ##############################14// ## Section 1: General Setup ##15// ##############################16 17 18#include "ggml.h"19#include "ggml-alloc.h"20#include "ggml-backend.h"21#include "ggml-cpp.h"22 23#include <algorithm>24#include <atomic>25#include <array>26#include <cfloat>27#include <cinttypes>28#include <cstdarg>29#include <cstdint>30#include <cstdio>31#include <cstdlib>32#include <cstring>33#include <ctime>34#include <future>35#include <fstream>36#include <memory>37#include <mutex>38#include <random>39#include <regex>40#include <set>41#include <sstream>42#include <string>43#include <string_view>44#include <thread>45#include <vector>46#include <unordered_map>47 48#ifdef __EMSCRIPTEN__49# define N_THREADS 150#else51# define N_THREADS std::thread::hardware_concurrency()52#endif53 54static void init_tensor_uniform(ggml_tensor * tensor, float min = -1.0f, float max = 1.0f) {55 size_t nels = ggml_nelements(tensor);56 std::vector<float> data(nels);57 {58 // parallel initialization59 static const size_t n_threads = std::max<size_t>(1, std::min<size_t>(nels/1024, std::min<size_t>(4, N_THREADS/2)));60 61 auto init_thread = [&](size_t start, size_t end) {62 thread_local std::default_random_engine gen(std::random_device{}());63 std::uniform_real_distribution<float> distribution(min, max);64 for (size_t i = start; i < end; i++) {65 data[i] = distribution(gen);66 }67 };68 69 if (n_threads == 1) {70 init_thread(0, nels);71 } else {72 std::vector<std::future<void>> tasks;73 tasks.reserve(n_threads);74 for (size_t i = 0; i < n_threads; i++) {75 size_t start = i*nels/n_threads;76 size_t end = (i+1)*nels/n_threads;77 tasks.push_back(std::async(std::launch::async, init_thread, start, end));78 }79 for (auto & t : tasks) {80 t.get();81 }82 }83 }84 85 if (tensor->type == GGML_TYPE_F32 || tensor->type == GGML_TYPE_I32) {86 ggml_backend_tensor_set(tensor, data.data(), 0, nels * sizeof(float));87 } else if (ggml_is_quantized(tensor->type) || tensor->type == GGML_TYPE_F16 || tensor->type == GGML_TYPE_BF16) {88 GGML_ASSERT(nels % ggml_blck_size(tensor->type) == 0);89 90 // dummy importance matrix91 std::vector<float> imatrix(tensor->ne[0], 1.0f);92 const float * im = imatrix.data();93 if (!ggml_quantize_requires_imatrix(tensor->type)) {94 // when the imatrix is optional, we want to test both quantization with and without imatrix95 // use one of the random numbers to decide96 if (data[0] > 0.5f*(min + max)) {97 im = nullptr;98 }99 }100 101 std::vector<uint8_t> dataq(ggml_row_size(tensor->type, nels));102 {103 // parallel quantization by block104 size_t blck_size = ggml_blck_size(tensor->type);105 size_t n_blocks = nels / blck_size;106 107 auto quantize_thread = [&](size_t start, size_t end) {108 ggml_quantize_chunk(tensor->type, data.data(), dataq.data(),109 start * blck_size, end - start, blck_size, im);110 };111 112 const size_t min_blocks_per_thread = 1;113 const size_t n_quant_threads = std::min<size_t>(std::max<size_t>(N_THREADS/2, 1),114 std::max<size_t>(1, n_blocks / min_blocks_per_thread));115 116 if (n_quant_threads == 1) {117 // single-threaded quantization: do all blocks in the current thread118 quantize_thread(0, n_blocks);119 } else {120 std::vector<std::future<void>> tasks;121 tasks.reserve(n_quant_threads);122 for (size_t i = 0; i < n_quant_threads; i++) {123 size_t start = i*n_blocks/n_quant_threads;124 size_t end = (i+1)*n_blocks/n_quant_threads;125 tasks.push_back(std::async(std::launch::async, quantize_thread, start, end));126 }127 for (auto & t : tasks) {128 t.get();129 }130 }131 }132 ggml_backend_tensor_set(tensor, dataq.data(), 0, dataq.size());133 } else if (tensor->type == GGML_TYPE_I8 || tensor->type == GGML_TYPE_I16) {134 // This is going to create some weird integers though.135 ggml_backend_tensor_set(tensor, data.data(), 0, nels * ggml_type_size(tensor->type));136 } else if (tensor->type == GGML_TYPE_I64) {137 // Integers with a size of 8 bytes can be set by mirroring the float data, the specific values are again not really meaningful.138 const size_t nbytes_half = nels * sizeof(float);139 ggml_backend_tensor_set(tensor, data.data(), 0*nbytes_half, nbytes_half);140 ggml_backend_tensor_set(tensor, data.data(), 1*nbytes_half, nbytes_half);141 } else {142 GGML_ABORT("fatal error");143 }144}145 146// generate an F16 mask where certain blocks are randomly masked with -INF value147static void init_tensor_kq_mask(ggml_tensor * tensor, float min = -1.0f, float max = 1.0f) {148 GGML_ASSERT(tensor->type == GGML_TYPE_F16);149 150 GGML_TENSOR_LOCALS( int32_t, ne, tensor, ne);151 152 std::vector<float> data_f32(ne0*ne1*ne2*ne3);153 std::vector<ggml_fp16_t> data_f16(ne0*ne1*ne2*ne3);154 155 std::random_device rd;156 std::mt19937 gen(rd());157 std::uniform_real_distribution<float> dis(min, max);158 159 for (size_t i = 0; i < data_f32.size(); i++) {160 data_f32[i] = dis(gen);161 }162 163 // block size164 const int blck0 = 128;165 const int blck1 = 64;166 167 // number of INF/zero blocks168 const int n_inf_zero_blocks = 0.2*(ne0*ne1*ne2*ne3)/(blck0*blck1);169 170 for (int b = 0; b < n_inf_zero_blocks; b++) {171 const int p3 = (rd() % ne3);172 const int p2 = (rd() % ne2);173 const int p1 = (rd() % ne1);174 const int p0 = (rd() % ne0);175 176 bool inf = rd() & 1;177 178 for (int i1 = 0; i1 < blck1 && p1 + i1 < ne1; i1++) {179 const int idx = p3*ne2*ne1*ne0 + p2*ne1*ne0 + (p1 + i1)*ne0 + p0;180 181 for (int i0 = 0; i0 < blck0 && p0 + i0 < ne0; i0++) {182 data_f32[idx + i0] = inf ? -INFINITY : 0.0f;183 }184 }185 }186 187 ggml_fp32_to_fp16_row(data_f32.data(), data_f16.data(), ne0*ne1*ne2*ne3);188 189 ggml_backend_tensor_set(tensor, data_f16.data(), 0, data_f16.size()*sizeof(ggml_fp16_t));190}191 192static void init_tensor_kq_mask_sparse(ggml_tensor * tensor, int64_t n_kv_max) {193 GGML_ASSERT(tensor->type == GGML_TYPE_F16);194 GGML_ASSERT(n_kv_max > 0 && n_kv_max <= tensor->ne[0]);195 196 const int64_t ne0 = tensor->ne[0];197 const int64_t nrows = ggml_nrows(tensor);198 std::vector<float> data_f32(ggml_nelements(tensor), -INFINITY);199 std::vector<ggml_fp16_t> data_f16(ggml_nelements(tensor));200 std::vector<int32_t> order(ne0);201 for (int64_t i = 0; i < ne0; ++i) {202 order[i] = i;203 }204 205 std::mt19937 gen(0x5A17);206 for (int64_t row = 0; row < nrows; ++row) {207 std::shuffle(order.begin(), order.end(), gen);208 const int64_t count = n_kv_max - row % std::min<int64_t>(n_kv_max, 17);209 std::sort(order.begin(), order.begin() + count);210 for (int64_t i = 0; i < count; ++i) {211 data_f32[row*ne0 + order[i]] = -0.03125f * (1 + (i + row) % 7);212 }213 }214 215 ggml_fp32_to_fp16_row(data_f32.data(), data_f16.data(), data_f16.size());216 ggml_backend_tensor_set(tensor, data_f16.data(), 0, data_f16.size()*sizeof(ggml_fp16_t));217}218 219// generate a lower triangular matrix220static void init_tensor_tril(ggml_tensor * tensor, float min = -1.0f, float max = 1.0f) {221 GGML_ASSERT(tensor->type == GGML_TYPE_F32);222 GGML_ASSERT(tensor->ne[0] == tensor->ne[1]);223 224 GGML_TENSOR_LOCALS(int32_t, ne, tensor, ne);225 GGML_TENSOR_LOCALS(size_t, nb, tensor, nb);226 227 std::vector<float> data_f32(ne0*ne1*ne2*ne3);228 229 std::random_device rd;230 std::mt19937 gen(rd());231 std::uniform_real_distribution<float> dis(min, max);232 233 for (int64_t i3 = 0; i3 < ne3; i3++) {234 for (int64_t i2 = 0; i2 < ne2; i2++) {235 for (int64_t i1 = 0; i1 < ne1; i1++) {236 for (int64_t i0 = 0; i0 < ne0; i0++) {237 int64_t idx = (i0 * nb0 + i1 * nb1 + i2 * nb2 + i3 * nb3) / sizeof(float);238 if (i0 <= i1) {239 data_f32[idx] = dis(gen);240 } else {241 data_f32[idx] = 0.0f;242 }243 }244 }245 }246 }247 248 ggml_backend_tensor_set(tensor, data_f32.data(), 0, ggml_nbytes(tensor));249}250 251static std::vector<float> tensor_to_float(const ggml_tensor * t) {252 std::vector<float> tv;253 tv.reserve(ggml_nelements(t));254 255 std::vector<uint8_t> buf(ggml_nbytes(t));256 ggml_backend_tensor_get(t, buf.data(), 0, ggml_nbytes(t));257 258 const auto * tt = ggml_get_type_traits(t->type);259 size_t bs = ggml_blck_size(t->type);260 std::vector<float> vq(ggml_blck_size(t->type));261 bool quantized = ggml_is_quantized(t->type);262 263 // access elements by index to avoid gaps in views264 for (int64_t i3 = 0; i3 < t->ne[3]; i3++) {265 for (int64_t i2 = 0; i2 < t->ne[2]; i2++) {266 for (int64_t i1 = 0; i1 < t->ne[1]; i1++) {267 for (int64_t i0 = 0; i0 < t->ne[0]; i0 += bs) {268 size_t i = i3*t->nb[3] + i2*t->nb[2] + i1*t->nb[1] + i0/bs*t->nb[0];269 if (t->type == GGML_TYPE_F16) {270 tv.push_back(ggml_fp16_to_fp32(*(ggml_fp16_t*)&buf[i]));271 } else if (t->type == GGML_TYPE_BF16) {272 tv.push_back(ggml_bf16_to_fp32(*(ggml_bf16_t*)&buf[i]));273 } else if (t->type == GGML_TYPE_F32) {274 tv.push_back(*(float *) &buf[i]);275 } else if (t->type == GGML_TYPE_I64) {276 tv.push_back((float)*(int64_t *) &buf[i]);277 } else if (t->type == GGML_TYPE_I32) {278 tv.push_back((float)*(int32_t *) &buf[i]);279 } else if (t->type == GGML_TYPE_I16) {280 tv.push_back((float)*(int16_t *) &buf[i]);281 } else if (t->type == GGML_TYPE_I8) {282 tv.push_back((float)*(int8_t *) &buf[i]);283 } else if (quantized) {284 tt->to_float(&buf[i], vq.data(), bs);285 tv.insert(tv.end(), vq.begin(), vq.end());286 } else {287 GGML_ABORT("fatal error");288 }289 }290 }291 }292 }293 294 return tv;295}296 297// normalized mean squared error = mse(a, b) / mse(a, 0)298static double nmse(const float * a, const float * b, size_t n) {299 double mse_a_b = 0.0;300 double mse_a_0 = 0.0;301 302 for (size_t i = 0; i < n; i++) {303 float a_i = a[i];304 float b_i = b[i];305 306 mse_a_b += (a_i - b_i) * (a_i - b_i);307 mse_a_0 += a_i * a_i;308 }309 310 return mse_a_b / mse_a_0;311}312 313// difference between 2 sets (Jaccard distance, 0 - no difference, 1 - no overlap)314template <typename T>315static double jdst(const T * a, const T * b, size_t n) {316 std::unordered_map<T, size_t> set_a;317 std::unordered_map<T, size_t> set_b;318 319 for (size_t i = 0; i < n; ++i) {320 set_a[a[i]]++;321 set_b[b[i]]++;322 }323 324 size_t diff = 0;325 326 for (const auto & p : set_a) {327 const int64_t na = p.second;328 const int64_t nb = set_b.find(p.first) != set_b.end() ? set_b.at(p.first) : 0;329 330 diff += std::abs(na - nb);331 }332 333 for (const auto & p : set_b) {334 if (set_a.find(p.first) == set_a.end()) {335 diff += p.second;336 }337 }338 339 return (double) diff / (2*n);340}341 342// maximum absolute asymmetry between a and b343// asymmetry: (a - b) / (a + b)344// This is more stable than relative error if one of the values fluctuates towards zero.345// n: number of values to compare.346// expected_vals: optional vector of expected values for a. If expected_vals is not empty, filter out all comparisons where347// a does not match any of the expected values. Needed for noncontinuous gradients where the numerical calculation can fail.348static double mean_abs_asymm(const float * a, const float * b, const size_t n, const std::vector<float> & expected_vals) {349 double sum = 0.0f;350 351 size_t nvalid = 0;352 for (size_t i = 0; i < n; i++) {353 if (!expected_vals.empty()) {354 bool matches_any = false;355 for (const float & ev : expected_vals) {356 if (fabsf(a[i] - ev) < 1e-3f) {357 matches_any = true;358 break;359 }360 }361 if (!matches_any) {362 continue;363 }364 }365 366 const float asymm = (a[i] - b[i]) / (a[i] + b[i]);367 368 sum += fabsf(asymm);369 nvalid++;370 }371 372 return sum/nvalid;373}374 375// utils for printing the variables of the test cases376 377static std::string var_to_str(const std::string & x) {378 return x;379}380 381template<typename T>382static std::string var_to_str(const T & x) {383 return std::to_string(x);384}385 386template<typename T, size_t N>387static std::string var_to_str(const T (&x)[N]) {388 std::string s = "[";389 for (size_t i = 0; i < N; i++) {390 if (i > 0) {391 s += ",";392 }393 s += var_to_str(x[i]);394 }395 s += "]";396 return s;397}398 399template<typename T, size_t N>400static std::string var_to_str(const std::array<T, N> & x) {401 std::string s = "[";402 for (size_t i = 0; i < N; i++) {403 if (i > 0) {404 s += ",";405 }406 s += var_to_str(x[i]);407 }408 s += "]";409 return s;410}411 412static std::string var_to_str(ggml_type type) {413 return ggml_type_name(type);414}415 416static std::string var_to_str(ggml_prec prec) {417 return prec == GGML_PREC_F32 ? "f32" : "def";418}419 420static std::string var_to_str(ggml_op_pool pool) {421 switch (pool) {422 case GGML_OP_POOL_AVG: return "avg";423 case GGML_OP_POOL_MAX: return "max";424 default: return std::to_string(pool);425 }426}427 428static std::string var_to_str(ggml_scale_mode mode) {429 std::string str;430 switch (mode & 0xFF) {431 case GGML_SCALE_MODE_NEAREST: str = "nearest"; break;432 case GGML_SCALE_MODE_BILINEAR: str = "bilinear"; break;433 case GGML_SCALE_MODE_BICUBIC: str = "bicubic"; break;434 default: str = std::to_string(mode); break;435 }436 if (mode & GGML_SCALE_FLAG_ALIGN_CORNERS) {437 str += "|align_corners";438 }439 if (mode & GGML_SCALE_FLAG_ANTIALIAS) {440 str += "|antialias";441 }442 return str;443}444 445#define VAR_TO_STR(x) (#x "=" + var_to_str(x))446 447#define VARS_TO_STR1(a) VAR_TO_STR(a)448#define VARS_TO_STR2(a, b) VAR_TO_STR(a) + "," + VAR_TO_STR(b)449#define VARS_TO_STR3(a, b, c) VAR_TO_STR(a) + "," + VARS_TO_STR2(b, c)450#define VARS_TO_STR4(a, b, c, d) VAR_TO_STR(a) + "," + VARS_TO_STR3(b, c, d)451#define VARS_TO_STR5(a, b, c, d, e) VAR_TO_STR(a) + "," + VARS_TO_STR4(b, c, d, e)452#define VARS_TO_STR6(a, b, c, d, e, f) VAR_TO_STR(a) + "," + VARS_TO_STR5(b, c, d, e, f)453#define VARS_TO_STR7(a, b, c, d, e, f, g) VAR_TO_STR(a) + "," + VARS_TO_STR6(b, c, d, e, f, g)454#define VARS_TO_STR8(a, b, c, d, e, f, g, h) VAR_TO_STR(a) + "," + VARS_TO_STR7(b, c, d, e, f, g, h)455#define VARS_TO_STR9(a, b, c, d, e, f, g, h, i) VAR_TO_STR(a) + "," + VARS_TO_STR8(b, c, d, e, f, g, h, i)456#define VARS_TO_STR10(a, b, c, d, e, f, g, h, i, j) VAR_TO_STR(a) + "," + VARS_TO_STR9(b, c, d, e, f, g, h, i, j)457#define VARS_TO_STR11(a, b, c, d, e, f, g, h, i, j, k) VAR_TO_STR(a) + "," + VARS_TO_STR10(b, c, d, e, f, g, h, i, j, k)458#define VARS_TO_STR12(a, b, c, d, e, f, g, h, i, j, k, l) VAR_TO_STR(a) + "," + VARS_TO_STR11(b, c, d, e, f, g, h, i, j, k, l)459#define VARS_TO_STR13(a, b, c, d, e, f, g, h, i, j, k, l, m) VAR_TO_STR(a) + "," + VARS_TO_STR12(b, c, d, e, f, g, h, i, j, k, l, m)460#define VARS_TO_STR14(a, b, c, d, e, f, g, h, i, j, k, l, m, n) VAR_TO_STR(a) + "," + VARS_TO_STR13(b, c, d, e, f, g, h, i, j, k, l, m, n)461#define VARS_TO_STR15(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o) VAR_TO_STR(a) + "," + VARS_TO_STR14(b, c, d, e, f, g, h, i, j, k, l, m, n, o)462#define VARS_TO_STR16(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p) VAR_TO_STR(a) + "," + VARS_TO_STR15(b, c, d, e, f, g, h, i, j, k, l, m, n, o, p)463#define VARS_TO_STR17(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q) VAR_TO_STR(a) + "," + VARS_TO_STR16(b, c, d, e, f, g, h, i, j, k, l, m, n, o, p, q)464 465// accept FLT_MAX as infinity466static bool isinf_or_max(float f) {467 return std::isinf(f) || f == FLT_MAX || f == -FLT_MAX;468}469 470static bool ggml_is_view_op(enum ggml_op op) {471 return op == GGML_OP_VIEW || op == GGML_OP_RESHAPE || op == GGML_OP_PERMUTE || op == GGML_OP_TRANSPOSE;472}473 474static bool backend_has_feature(ggml_backend_t backend, const char * feature_name) {475 ggml_backend_dev_t dev = ggml_backend_get_device(backend);476 ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev);477 478 auto get_features = (ggml_backend_get_features_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_get_features");479 if (!get_features) {480 return false;481 }482 483 const ggml_backend_feature * features = get_features(reg);484 if (!features) {485 return false;486 }487 488 for (const ggml_backend_feature * f = features; f->name; ++f) {489 if (strcmp(f->name, feature_name) == 0 && strcmp(f->value, "1") == 0) {490 return true;491 }492 }493 return false;494}495 496enum test_mode {497 MODE_TEST,498 MODE_PERF,499 MODE_GRAD,500 MODE_SUPPORT,501};502 503// Output format support similar to llama-bench504enum output_formats { CONSOLE, SQL, CSV };505 506static const char * output_format_str(output_formats format) {507 switch (format) {508 case CONSOLE:509 return "console";510 case SQL:511 return "sql";512 case CSV:513 return "csv";514 default:515 GGML_ABORT("invalid output format");516 }517}518 519static bool output_format_from_str(const std::string & s, output_formats & format) {520 if (s == "console") {521 format = CONSOLE;522 } else if (s == "sql") {523 format = SQL;524 } else if (s == "csv") {525 format = CSV;526 } else {527 return false;528 }529 return true;530}531 532static std::string test_time_now() {533 time_t t = time(NULL);534 struct tm tm_buf;535#ifdef _WIN32536 if (gmtime_s(&tm_buf, &t) != 0) {537 return "";538 }539#else540 if (gmtime_r(&t, &tm_buf) == nullptr) {541 return "";542 }543#endif544 char buf[32];545 if (std::strftime(buf, sizeof(buf), "%FT%TZ", &tm_buf) == 0) {546 return "";547 }548 return buf;549}550 551// Test result structure for SQL output552struct test_result {553 std::string test_time;554 std::string build_commit;555 std::string backend_name;556 std::string op_name;557 std::string op_params;558 std::string test_mode;559 bool supported;560 bool passed;561 std::string error_message;562 double time_us;563 double flops;564 double bandwidth_gb_s;565 size_t memory_kb;566 int n_runs;567 std::string device_description;568 std::string backend_reg_name;569 570 test_result() {571 // Initialize with default values572 time_us = 0.0;573 flops = 0.0;574 bandwidth_gb_s = 0.0;575 memory_kb = 0;576 n_runs = 0;577 supported = false;578 passed = false;579 580 test_time = test_time_now();581 582 // Set build info583 build_commit = ggml_commit();584 }585 586 test_result(const std::string & backend_name, const std::string & op_name, const std::string & op_params,587 const std::string & test_mode, bool supported, bool passed, const std::string & error_message = "",588 double time_us = 0.0, double flops = 0.0, double bandwidth_gb_s = 0.0, size_t memory_kb = 0,589 int n_runs = 0, const std::string & device_description = "", const std::string & backend_reg_name = "") :590 backend_name(backend_name),591 op_name(op_name),592 op_params(op_params),593 test_mode(test_mode),594 supported(supported),595 passed(passed),596 error_message(error_message),597 time_us(time_us),598 flops(flops),599 bandwidth_gb_s(bandwidth_gb_s),600 memory_kb(memory_kb),601 n_runs(n_runs),602 device_description(device_description),603 backend_reg_name(backend_reg_name) {604 test_time = test_time_now();605 606 // Set build info607 build_commit = ggml_commit();608 }609 610 static const std::vector<std::string> & get_fields() {611 static const std::vector<std::string> fields = {612 "test_time", "build_commit", "backend_name", "op_name", "op_params", "test_mode", "supported",613 "passed", "error_message", "time_us", "flops", "bandwidth_gb_s", "memory_kb", "n_runs",614 "device_description", "backend_reg_name"615 };616 return fields;617 }618 619 enum field_type { STRING, BOOL, INT, FLOAT };620 621 static field_type get_field_type(const std::string & field) {622 if (field == "supported" || field == "passed") {623 return BOOL;624 }625 if (field == "memory_kb" || field == "n_runs") {626 return INT;627 }628 if (field == "time_us" || field == "flops" || field == "bandwidth_gb_s") {629 return FLOAT;630 }631 return STRING;632 }633 634 std::vector<std::string> get_values() const {635 return { test_time,636 build_commit,637 backend_name,638 op_name,639 op_params,640 test_mode,641 std::to_string(supported),642 std::to_string(passed),643 error_message,644 std::to_string(time_us),645 std::to_string(flops),646 std::to_string(bandwidth_gb_s),647 std::to_string(memory_kb),648 std::to_string(n_runs),649 device_description,650 backend_reg_name };651 }652};653 654// Printer classes for different output formats655enum class test_status_t { NOT_SUPPORTED, OK, FAIL, SKIPPED };656 657struct test_operation_info {658 std::string op_name;659 std::string op_params;660 std::string backend_name;661 test_status_t status = test_status_t::OK;662 std::string failure_reason;663 664 // Additional information fields that were previously in separate structs665 std::string error_component;666 std::string error_details;667 668 // Gradient info669 int64_t gradient_index = -1;670 std::string gradient_param_name;671 float gradient_value = 0.0f;672 673 // MAA error info674 double maa_error = 0.0;675 double maa_threshold = 0.0;676 677 // Flags for different types of information678 bool has_error = false;679 bool has_gradient_info = false;680 bool has_maa_error = false;681 bool is_compare_failure = false;682 bool is_large_tensor_skip = false;683 684 test_operation_info() = default;685 686 test_operation_info(const std::string & op_name, const std::string & op_params, const std::string & backend_name,687 test_status_t status = test_status_t::OK, const std::string & failure_reason = "") :688 op_name(op_name),689 op_params(op_params),690 backend_name(backend_name),691 status(status),692 failure_reason(failure_reason) {}693 694 // Set error information695 void set_error(const std::string & component, const std::string & details) {696 has_error = true;697 error_component = component;698 error_details = details;699 if (status == test_status_t::OK) {700 status = test_status_t::FAIL;701 }702 }703 704 // Set gradient information705 void set_gradient_info(int64_t index, const std::string & param_name, float value) {706 has_gradient_info = true;707 gradient_index = index;708 gradient_param_name = param_name;709 gradient_value = value;710 if (status == test_status_t::OK) {711 status = test_status_t::FAIL;712 }713 }714 715 // Set MAA error information716 void set_maa_error(double error, double threshold) {717 has_maa_error = true;718 maa_error = error;719 maa_threshold = threshold;720 if (status == test_status_t::OK) {721 status = test_status_t::FAIL;722 }723 }724 725 // Set compare failure726 void set_compare_failure() {727 is_compare_failure = true;728 if (status == test_status_t::OK) {729 status = test_status_t::FAIL;730 }731 }732 733 // Set large tensor skip734 void set_large_tensor_skip() { is_large_tensor_skip = true; }735};736 737struct test_summary_info {738 size_t tests_passed;739 size_t tests_total;740 bool is_backend_summary = false; // true for backend summary, false for test summary741 742 test_summary_info() = default;743 744 test_summary_info(size_t tests_passed, size_t tests_total, bool is_backend_summary = false) :745 tests_passed(tests_passed),746 tests_total(tests_total),747 is_backend_summary(is_backend_summary) {}748};749 750struct testing_start_info {751 size_t device_count;752 753 testing_start_info() = default;754 755 testing_start_info(size_t device_count) : device_count(device_count) {}756};757 758struct backend_init_info {759 size_t device_index;760 size_t total_devices;761 std::string device_name;762 bool skipped = false;763 std::string skip_reason;764 std::string description;765 size_t memory_total_mb = 0;766 size_t memory_free_mb = 0;767 bool has_memory_info = false;768 769 backend_init_info() = default;770 771 backend_init_info(size_t device_index, size_t total_devices, const std::string & device_name, bool skipped = false,772 const std::string & skip_reason = "", const std::string & description = "",773 size_t memory_total_mb = 0, size_t memory_free_mb = 0, bool has_memory_info = false) :774 device_index(device_index),775 total_devices(total_devices),776 device_name(device_name),777 skipped(skipped),778 skip_reason(skip_reason),779 description(description),780 memory_total_mb(memory_total_mb),781 memory_free_mb(memory_free_mb),782 has_memory_info(has_memory_info) {}783};784 785struct backend_status_info {786 std::string backend_name;787 test_status_t status;788 789 backend_status_info() = default;790 791 backend_status_info(const std::string & backend_name, test_status_t status) :792 backend_name(backend_name),793 status(status) {}794};795 796struct overall_summary_info {797 size_t backends_passed;798 size_t backends_total;799 bool all_passed;800 801 overall_summary_info() = default;802 803 overall_summary_info(size_t backends_passed, size_t backends_total, bool all_passed) :804 backends_passed(backends_passed),805 backends_total(backends_total),806 all_passed(all_passed) {}807};808 809struct printer {810 virtual ~printer() {}811 812 FILE * fout = stdout;813 814 virtual void print_header() {}815 816 virtual void print_test_result(const test_result & result) = 0;817 818 virtual void print_footer() {}819 820 virtual void print_operation(const test_operation_info & info) { (void) info; }821 822 virtual void print_summary(const test_summary_info & info) { (void) info; }823 824 virtual void print_testing_start(const testing_start_info & info) { (void) info; }825 826 virtual void print_backend_init(const backend_init_info & info) { (void) info; }827 828 virtual void print_backend_status(const backend_status_info & info) { (void) info; }829 830 virtual void print_overall_summary(const overall_summary_info & info) { (void) info; }831 832 virtual void print_failed_tests(const std::vector<std::string> & failed_tests) { (void) failed_tests; }833};834 835struct console_printer : public printer {836 void print_test_result(const test_result & result) override {837 if (result.test_mode == "test") {838 print_test_console(result);839 } else if (result.test_mode == "perf") {840 print_perf_console(result);841 } else if (result.test_mode == "support") {842 print_support_console(result);843 }844 }845 846 void print_operation(const test_operation_info & info) override {847 printf(" %s(%s): ", info.op_name.c_str(), info.op_params.c_str());848 fflush(stdout);849 850 // Handle large tensor skip first851 if (info.is_large_tensor_skip) {852 printf("skipping large tensors for speed \n");853 return;854 }855 856 // Handle not supported status857 if (info.status == test_status_t::NOT_SUPPORTED) {858 if (!info.failure_reason.empty()) {859 printf("not supported [%s]\n", info.failure_reason.c_str());860 } else {861 printf("not supported [%s]\n", info.backend_name.c_str());862 }863 return;864 }865 866 // Handle errors and additional information867 if (info.has_error) {868 if (info.error_component == "allocation") {869 fprintf(stderr, "failed to allocate tensors [%s] ", info.backend_name.c_str());870 } else if (info.error_component == "backend") {871 fprintf(stderr, " Failed to initialize %s backend\n", info.backend_name.c_str());872 } else {873 fprintf(stderr, "Error in %s: %s\n", info.error_component.c_str(), info.error_details.c_str());874 }875 }876 877 // Handle gradient info878 if (info.has_gradient_info) {879 printf("[%s] nonfinite gradient at index %" PRId64 " (%s=%f) ", info.op_name.c_str(), info.gradient_index,880 info.gradient_param_name.c_str(), info.gradient_value);881 }882 883 // Handle MAA error884 if (info.has_maa_error) {885 printf("[%s] MAA = %.9f > %.9f ", info.op_name.c_str(), info.maa_error, info.maa_threshold);886 }887 888 // Handle compare failure889 if (info.is_compare_failure) {890 printf("compare failed ");891 }892 893 // Print final status894 if (info.status == test_status_t::OK) {895 printf("\033[1;32mOK\033[0m\n");896 } else {897 printf("\033[1;31mFAIL\033[0m\n");898 }899 }900 901 void print_summary(const test_summary_info & info) override {902 if (info.is_backend_summary) {903 printf("%zu/%zu backends passed\n", info.tests_passed, info.tests_total);904 } else {905 printf(" %zu/%zu tests passed\n", info.tests_passed, info.tests_total);906 }907 }908 909 void print_backend_status(const backend_status_info & info) override {910 printf(" Backend %s: ", info.backend_name.c_str());911 if (info.status == test_status_t::OK) {912 printf("\033[1;32mOK\033[0m\n");913 } else {914 printf("\033[1;31mFAIL\033[0m\n");915 }916 }917 918 void print_testing_start(const testing_start_info & info) override {919 printf("Testing %zu devices\n\n", info.device_count);920 }921 922 void print_backend_init(const backend_init_info & info) override {923 printf("Backend %zu/%zu: %s\n", info.device_index + 1, info.total_devices, info.device_name.c_str());924 925 if (info.skipped) {926 printf(" %s\n", info.skip_reason.c_str());927 return;928 }929 930 if (!info.description.empty()) {931 printf(" Device description: %s\n", info.description.c_str());932 }933 934 if (info.has_memory_info) {935 printf(" Device memory: %zu MB (%zu MB free)\n", info.memory_total_mb, info.memory_free_mb);936 }937 938 printf("\n");939 }940 941 void print_overall_summary(const overall_summary_info & info) override {942 printf("%zu/%zu backends passed\n", info.backends_passed, info.backends_total);943 if (info.all_passed) {944 printf("\033[1;32mOK\033[0m\n");945 } else {946 printf("\033[1;31mFAIL\033[0m\n");947 }948 }949 950 void print_failed_tests(const std::vector<std::string> & failed_tests) override {951 if (failed_tests.empty()) {952 return;953 }954 955 printf("\nFailing tests:\n");956 for (const auto & test_name : failed_tests) {957 printf(" %s\n", test_name.c_str());958 }959 }960 961 private:962 void print_test_console(const test_result & result) {963 printf(" %s(%s): ", result.op_name.c_str(), result.op_params.c_str());964 fflush(stdout);965 966 if (!result.supported) {967 printf("not supported [%s] ", result.backend_name.c_str());968 printf("\n");969 return;970 }971 972 if (result.passed) {973 printf("\033[1;32mOK\033[0m\n");974 } else {975 printf("\033[1;31mFAIL\033[0m\n");976 }977 }978 979 void print_perf_console(const test_result & result) {980 int len = printf(" %s(%s): ", result.op_name.c_str(), result.op_params.c_str());981 fflush(stdout);982 983 if (!result.supported) {984 printf("not supported\n");985 return;986 }987 988 // align while also leaving some margin for variations in parameters989 int align = 8;990 int last = (len + align - 1) / align * align;991 if (last - len < 5) {992 last += align;993 }994 printf("%*s", last - len, "");995 996 printf(" %8d runs - %8.2f us/run - ", result.n_runs, result.time_us);997 998 if (result.flops > 0) {999 auto format_flops = [](double flops) -> std::string {1000 char buf[256];1001 if (flops >= 1e12) {1002 snprintf(buf, sizeof(buf), "%6.2f TFLOP", flops / 1e12);1003 } else if (flops >= 1e9) {1004 snprintf(buf, sizeof(buf), "%6.2f GFLOP", flops / 1e9);1005 } else if (flops >= 1e6) {1006 snprintf(buf, sizeof(buf), "%6.2f MFLOP", flops / 1e6);1007 } else {1008 snprintf(buf, sizeof(buf), "%6.2f kFLOP", flops / 1e3);1009 }1010 return buf;1011 };1012 uint64_t op_flops_per_run = result.flops * result.time_us / 1e6;1013 printf("%s/run - \033[1;34m%sS\033[0m", format_flops(op_flops_per_run).c_str(),1014 format_flops(result.flops).c_str());1015 } else {1016 printf("%8zu kB/run - \033[1;34m%7.2f GB/s\033[0m", result.memory_kb, result.bandwidth_gb_s);1017 }1018 printf("\n");1019 }1020 1021 void print_support_console(const test_result & result) {1022 printf(" %s(%s): ", result.op_name.c_str(), result.op_params.c_str());1023 fflush(stdout);1024 1025 if (result.supported) {1026 printf("\033[1;32mSUPPORTED\033[0m\n");1027 } else {1028 printf("\033[1;31mNOT SUPPORTED\033[0m\n");1029 }1030 }1031};1032 1033struct sql_printer : public printer {1034 static std::string get_sql_field_type(const std::string & field) {1035 switch (test_result::get_field_type(field)) {1036 case test_result::STRING:1037 return "TEXT";1038 case test_result::BOOL:1039 case test_result::INT:1040 return "INTEGER";1041 case test_result::FLOAT:1042 return "REAL";1043 default:1044 GGML_ABORT("invalid field type");1045 }1046 }1047 1048 void print_header() override {1049 std::vector<std::string> fields = test_result::get_fields();1050 fprintf(fout, "CREATE TABLE IF NOT EXISTS test_backend_ops (\n");1051 for (size_t i = 0; i < fields.size(); i++) {1052 fprintf(fout, " %s %s%s\n", fields[i].c_str(), get_sql_field_type(fields[i]).c_str(),1053 i < fields.size() - 1 ? "," : "");1054 }1055 fprintf(fout, ");\n\n");1056 }1057 1058 void print_test_result(const test_result & result) override {1059 fprintf(fout, "INSERT INTO test_backend_ops (");1060 std::vector<std::string> fields = test_result::get_fields();1061 for (size_t i = 0; i < fields.size(); i++) {1062 fprintf(fout, "%s%s", fields[i].c_str(), i < fields.size() - 1 ? ", " : "");1063 }1064 fprintf(fout, ") VALUES (");1065 std::vector<std::string> values = result.get_values();1066 for (size_t i = 0; i < values.size(); i++) {1067 fprintf(fout, "'%s'%s", values[i].c_str(), i < values.size() - 1 ? ", " : "");1068 }1069 fprintf(fout, ");\n");1070 }1071};1072 1073struct csv_printer : public printer {1074 void print_header() override {1075 1076 std::vector<std::string> fields = test_result::get_fields();1077 std::vector<std::string> fields_csv = get_fields_csv();1078 for (size_t i = 0; i < fields.size(); i++) {1079 if (std::find(std::begin(fields_csv), std::end(fields_csv), fields[i]) == std::end(fields_csv)) {1080 continue;1081 }1082 printf("\"%s\"%s", fields[i].c_str(), i < fields.size() - 1 ? "," : "");1083 }1084 printf("\n");1085 }1086 1087 void print_test_result(const test_result & result) override {1088 1089 std::vector<std::string> values = result.get_values();1090 std::vector<std::string> fields = test_result::get_fields();1091 std::vector<std::string> fields_csv = get_fields_csv();1092 1093 for (size_t i = 0; i < values.size(); i++) {1094 1095 if (std::find(std::begin(fields_csv), std::end(fields_csv), fields[i]) == std::end(fields_csv)) {1096 continue;1097 }1098 1099 // Escape quotes and wrap in quotes for CSV1100 std::string escaped_value = values[i];1101 size_t pos = 0;1102 while ((pos = escaped_value.find("\"", pos)) != std::string::npos) {1103 escaped_value.replace(pos, 1, "\"\"");1104 pos += 2;1105 }1106 printf("\"%s\"%s", escaped_value.c_str(), i < values.size() - 1 ? "," : "");1107 }1108 printf("\n");1109 }1110 1111 static std::vector<std::string> get_fields_csv() {1112 return {1113 "op_name",1114 "op_params",1115 "supported",1116 "error_message",1117 "test_mode",1118 "backend_reg_name",1119 "backend_name",1120 };1121 }1122 1123};1124 1125static std::unique_ptr<printer> create_printer(output_formats format) {1126 switch (format) {1127 case CONSOLE:1128 return std::make_unique<console_printer>();1129 case SQL:1130 return std::make_unique<sql_printer>();1131 case CSV:1132 return std::make_unique<csv_printer>();1133 }1134 GGML_ABORT("invalid output format");1135}1136 1137static std::mutex g_test_output_mutex;1138 1139static void print_test_result_locked(printer * output_printer, const test_result & result) {1140 if (output_printer == nullptr) {1141 return;1142 }1143 1144 std::lock_guard<std::mutex> guard(g_test_output_mutex);1145 output_printer->print_test_result(result);1146}1147 1148struct test_case {1149 virtual ~test_case() {}1150 1151 virtual std::string op_desc(ggml_tensor * t) {1152 return ggml_op_desc(t);1153 }1154 1155 virtual std::string vars() {1156 return "";1157 }1158 1159 virtual ggml_tensor * build_graph(ggml_context * ctx) = 0;1160 virtual ggml_tensor * build_graph(ggml_context * ctx, ggml_context * ctx_weights) {1161 GGML_UNUSED(ctx_weights);1162 return build_graph(ctx);1163 }1164 1165 virtual double max_nmse_err() {1166 return 1e-7;1167 }1168 1169 virtual double max_nmse_err(ggml_backend_t backend) {1170 ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(ggml_backend_get_device(backend));1171 // See https://github.com/ggml-org/llama.cpp/pull/22976 for explanation.1172 if (contains_f16 && strcmp(ggml_backend_reg_name(reg), "WebGPU") == 0) {1173 return std::max(max_nmse_err(), 1e-6);1174 }1175 return max_nmse_err();1176 }1177 1178 virtual double max_maa_err() {1179 return 1e-4;1180 }1181 1182 virtual double max_err() {1183 return max_nmse_err();1184 }1185 1186 virtual double max_err(ggml_backend_t backend) {1187 return max_nmse_err(backend);1188 }1189 1190 virtual double err(const float * a, const float * b, size_t n) {1191 return nmse(a, b, n);1192 }1193 1194 virtual float grad_eps() {1195 return 1e-1f;1196 }1197 1198 // If false, estimate gradient with 2 points, neglects 3rd order derivative and higher.1199 // If true, estimate gradient with 4 points, neglects 5th order derivative and higher.1200 virtual bool grad_precise() {