Codeprocastinator/optimized-tinyllama-covalent
0119
1#include "ggml.h"2#include "ggml-alloc.h"3#include "ggml-backend.h"4#include "ggml-cpu.h"5#include "ggml-opt.h"6 7#include <cmath>8#include <cinttypes>9#include <random>10#include <string>11#include <thread>12#include <vector>13 14static bool almost_equal(const double a, const double b, const double atol) {15 return fabs(a - b) < atol;16}17 18constexpr int64_t ne_datapoint = 2;19constexpr int64_t ne_label = 1;20constexpr int64_t ndata = 6;21 22struct helper_ctx_data {23 std::vector<ggml_opt_dataset_t> datasets_supervised;24 std::vector<struct ggml_tensor *> data_batch;25 std::vector<struct ggml_tensor *> labels_batch;26 27 ggml_opt_dataset_t dataset_unsupervised;28 struct ggml_context * ctx_static;29 struct ggml_context * ctx_compute;30 struct ggml_opt_params opt_params;31 ggml_opt_context_t opt_ctx;32 struct ggml_tensor * inputs;33 struct ggml_tensor * weights;34 struct ggml_tensor * outputs;35 ggml_backend_buffer_t buf;36 ggml_opt_result_t result;37 ggml_opt_result_t result2;38};39 40// These default values make it easier to check optimization results vs. expected values.41static ggml_opt_optimizer_params helper_get_test_opt_pars(void * userdata) {42 ggml_opt_optimizer_params result = ggml_opt_get_default_optimizer_params(userdata);43 result.adamw.alpha = 1.0f;44 result.adamw.beta1 = 0.0f;45 result.adamw.beta2 = 0.0f;46 result.adamw.eps = 0.0f;47 return result;48}49 50static helper_ctx_data helper_get_ctx_data(51 ggml_backend_sched_t backend_sched,52 ggml_backend_t backend,53 const bool init_opt_ctx = true,54 const bool optimizer_defaults = true,55 int64_t nbatch_logical = 1,56 int64_t nbatch_physical = 1,57 enum ggml_opt_loss_type loss_type = GGML_OPT_LOSS_TYPE_SUM) {58 std::vector<ggml_opt_dataset_t> datasets(ndata);59 for (int64_t ndata_shard = 1; ndata_shard <= ndata; ++ndata_shard) {60 ggml_opt_dataset_t dataset = ggml_opt_dataset_init(ne_datapoint, ne_label, ndata, ndata_shard);61 62 float * data = ggml_get_data_f32(ggml_opt_dataset_data( dataset));63 float * labels = ggml_get_data_f32(ggml_opt_dataset_labels(dataset));64 65 for (int64_t idata = 0; idata < ndata; ++idata) {66 for (int64_t id = 0; id < ne_datapoint; ++id) {67 data[ idata*ne_datapoint + id] = 16*idata + id;68 }69 for (int64_t il = 0; il < ne_label; ++il) {70 labels[idata*ne_label + il] = 16*(16*idata + il);71 }72 }73 74 datasets[ndata_shard-1] = dataset;75 }76 77 ggml_opt_dataset_t dataset_unsupervised = ggml_opt_dataset_init(1, 0, ndata, /*ndata_shard =*/ 1);78 79 float * data = ggml_get_data_f32(ggml_opt_dataset_data(dataset_unsupervised));80 81 for (int64_t idata = 0; idata < ndata; ++idata) {82 data[idata] = idata;83 }84 85 struct ggml_context * ctx_static;86 struct ggml_context * ctx_compute;87 {88 struct ggml_init_params params = {89 /*.mem_size =*/ (2*ndata + 2)*ggml_tensor_overhead(),90 /*.mem_buffer =*/ nullptr,91 /*.no_alloc =*/ true,92 };93 ctx_static = ggml_init(params);94 }95 {96 struct ggml_init_params params = {97 /*.mem_size =*/ GGML_DEFAULT_GRAPH_SIZE*ggml_tensor_overhead() + 3*ggml_graph_overhead(),98 /*.mem_buffer =*/ nullptr,99 /*.no_alloc =*/ true,100 };101 ctx_compute = ggml_init(params);102 }103 104 std::vector<struct ggml_tensor *> data_batch(ndata);105 std::vector<struct ggml_tensor *> labels_batch(ndata);106 for (int64_t ndata_batch = 1; ndata_batch <= ndata; ++ndata_batch) {107 data_batch[ndata_batch-1] = ggml_new_tensor_1d(ctx_static, GGML_TYPE_F32, ndata_batch*ne_datapoint);108 labels_batch[ndata_batch-1] = ggml_new_tensor_1d(ctx_static, GGML_TYPE_F32, ndata_batch*ne_label);109 }110 111 struct ggml_tensor * inputs = ggml_new_tensor_1d(ctx_static, GGML_TYPE_F32, nbatch_physical);112 ggml_set_name(inputs, "inputs");113 114 struct ggml_tensor * weights = ggml_new_tensor_1d(ctx_static, GGML_TYPE_F32, 1);115 ggml_set_name(weights, "weights");116 ggml_set_param(ctx_static, weights);117 118 struct ggml_tensor * intermediary = ggml_add(ctx_compute, inputs, weights);119 120 struct ggml_tensor * outputs = ggml_scale(ctx_compute, intermediary, 1.0f);121 ggml_set_name(outputs, "outputs");122 123 ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx_static, backend);124 const float w0 = float(ndata)/2;125 ggml_backend_tensor_set(weights, &w0, 0, sizeof(float));126 127 GGML_ASSERT(nbatch_logical % nbatch_physical == 0);128 const int32_t opt_period = nbatch_logical / nbatch_physical;129 130 struct ggml_opt_params opt_params = ggml_opt_default_params(backend_sched, ctx_compute, inputs, outputs, loss_type);131 opt_params.opt_period = opt_period;132 if (!optimizer_defaults) {133 opt_params.get_opt_pars = helper_get_test_opt_pars;134 }135 ggml_opt_context_t opt_ctx = init_opt_ctx ? ggml_opt_init(opt_params) : nullptr;136 137 ggml_opt_result_t result = ggml_opt_result_init();138 ggml_opt_result_t result2 = ggml_opt_result_init();139 140 return {datasets, data_batch, labels_batch, dataset_unsupervised, ctx_static, ctx_compute, opt_params, opt_ctx, inputs, weights, outputs, buf, result, result2};141}142 143static void helper_free_ctx_data(struct helper_ctx_data ctx_data) {144 ggml_opt_result_free(ctx_data.result);145 ggml_opt_result_free(ctx_data.result2);146 ggml_opt_free(ctx_data.opt_ctx);147 ggml_backend_buffer_free(ctx_data.buf);148 ggml_free(ctx_data.ctx_static);149 ggml_free(ctx_data.ctx_compute);150 for (ggml_opt_dataset_t dataset : ctx_data.datasets_supervised) {151 ggml_opt_dataset_free(dataset);152 }153 ggml_opt_dataset_free(ctx_data.dataset_unsupervised);154}155 156static void helper_after_test(157 const char * func, const bool high_level, const std::string options,158 const std::string subtest, const bool subtest_ok, int & ntest, int & npass) {159 printf(" %s(high_level=%s%s, subtest=%s): ",160 func, high_level ? "yes" : "no", options.c_str(), subtest.c_str());161 if (subtest_ok) {162 printf("\033[1;32mOK\033[0m\n");163 npass++;164 } else {165 printf("\033[1;31mFAIL\033[0m\n");166 }167 ntest++;168}169 170static std::pair<int, int> test_dataset(ggml_backend_sched_t backend_sched, ggml_backend_t backend, const bool shuffle) {171 int ntest = 0;172 int npass = 0;173 174 struct helper_ctx_data cd = helper_get_ctx_data(backend_sched, backend);175 176 for (int64_t ndata_shard = 1; ndata_shard <= ndata; ++ndata_shard) {177 ggml_opt_dataset_t dataset = cd.datasets_supervised[ndata_shard-1];178 179 if (shuffle) {180 ggml_opt_dataset_shuffle(cd.opt_ctx, dataset, -1);181 }182 183 for (int64_t ndata_batch = 1; ndata_batch <= ndata; ++ndata_batch) {184 if (ndata_batch % ndata_shard != 0) {185 continue;186 }187 bool subtest_ok = true;188 189 struct ggml_tensor * data_batch = cd.data_batch[ndata_batch-1];190 struct ggml_tensor * labels_batch = cd.labels_batch[ndata_batch-1];191 192 std::vector<float> data(ggml_nelements( data_batch));193 std::vector<float> labels(ggml_nelements(labels_batch));194 195 std::vector<int64_t> idata_shuffled;196 const int64_t nbatches = ndata / ndata_batch;197 for (int64_t ibatch = 0; ibatch < nbatches; ++ibatch) {198 ggml_opt_dataset_get_batch(dataset, data_batch, labels_batch, ibatch);199 200 ggml_backend_tensor_get( data_batch, data.data(), 0, ggml_nbytes( data_batch));201 ggml_backend_tensor_get(labels_batch, labels.data(), 0, ggml_nbytes(labels_batch));202 203 for (int64_t idata_batch = 0; idata_batch < ndata_batch; ++idata_batch) {204 const int64_t idata = ibatch*ndata_batch + idata_batch;205 const int64_t idata_found = data[idata_batch*ne_datapoint] / 16;206 subtest_ok = subtest_ok && (shuffle || idata_found == idata);207 idata_shuffled.push_back(idata_found);208 209 for (int64_t id = 0; id < ne_datapoint; ++id) {210 if (data[ idata_batch*ne_datapoint + id] != 16*idata_found + id) {211 subtest_ok = false;212 }213 }214 for (int64_t il = 0; il < ne_label; ++il) {215 if (labels[idata_batch*ne_label + il] != 16*(16*idata_found + il)) {216 subtest_ok = false;217 }218 }219 }220 }221 222 if (!shuffle || ndata % ndata_batch == 0) {223 const int ndata_max = (ndata / ndata_batch) * ndata_batch;224 225 for (int64_t idata = 0; subtest_ok && idata < ndata_max; ++idata) {226 int ninstances = 0;227 for (int64_t id : idata_shuffled) {228 ninstances += id == idata;229 }230 if (ninstances != 1) {231 subtest_ok = false;232 }233 }234 }235 236 printf(" %s(shuffle=%s, ndata_shard=%" PRId64 ", ndata_batch=%" PRId64 "): ",237 __func__, shuffle ? "yes" : "no", ndata_shard, ndata_batch);238 if (subtest_ok) {239 printf("\033[1;32mOK\033[0m\n");240 npass++;241 } else {242 printf("\033[1;31mFAIL\033[0m\n");243 }244 ntest++;245 }246 }247 248 helper_free_ctx_data(cd);249 250 return std::make_pair(npass, ntest);251}252 253static std::pair<int, int> test_grad(ggml_backend_sched_t backend_sched, ggml_backend_t backend) {254 int ntest = 0;255 int npass = 0;256 257 struct helper_ctx_data cd = helper_get_ctx_data(backend_sched, backend, /*init_opt_ctx =*/ true, /*optimizer_defaults =*/ false,258 /*nbatch_logical =*/ 999999, /*nbatch_physical =*/ 1);259 260 std::vector<float> grad_history(ndata);261 for (int64_t idata = 0; idata < ndata; ++idata) {262 grad_history[idata] = NAN;263 }264 265 for (int idata = 0; idata < ndata; ++idata) {266 const float idataf = idata;267 ggml_backend_tensor_set(cd.inputs, &idataf, 0, ggml_nbytes(cd.inputs));268 ggml_opt_forward_backward(cd.opt_ctx, cd.result);269 ggml_backend_tensor_get(ggml_opt_grad_acc(cd.opt_ctx, cd.weights), grad_history.data() + idata, 0, sizeof(float));270 }271 272 {273 bool subtest_ok = true;274 for (int idata = 0; idata < ndata; ++idata) {275 if (grad_history[idata] != idata + 1) {276 subtest_ok = false;277 }278 }279 printf(" %s(): ", __func__);280 if (subtest_ok) {281 printf("\033[1;32mOK\033[0m\n");282 npass++;283 } else {284 printf("\033[1;31mFAIL\033[0m\n");285 }286 ntest++;287 }288 289 helper_free_ctx_data(cd);290 291 return std::make_pair(npass, ntest);292}293 294static void helper_after_test_forward_backward(295 const char * func, const bool high_level, const bool shuffle,296 const std::string subtest, const bool subtest_ok, int & ntest, int & npass) {297 std::string options = ", shuffle=";298 options += shuffle ? "yes" : "no";299 helper_after_test(func, high_level, options, subtest, subtest_ok, ntest, npass);300}301 302static std::pair<int, int> test_forward_backward(303 ggml_backend_sched_t backend_sched, ggml_backend_t backend, const bool high_level, const bool shuffle) {304 int ntest = 0;305 int npass = 0;306 307 struct helper_ctx_data cd = helper_get_ctx_data(backend_sched, backend, /*init_opt_ctx =*/ true, /*optimizer_defaults =*/ false);308 struct ggml_tensor * loss = ggml_opt_loss(cd.opt_ctx);309 310 std::vector<float> loss_history(ndata);311 for (int64_t idata = 0; idata < ndata; ++idata) {312 loss_history[idata] = NAN;313 }314 315 {316 int64_t ndata;317 ggml_opt_result_ndata(cd.result, &ndata);318 double loss;319 double loss_unc;320 ggml_opt_result_loss(cd.result, &loss, &loss_unc);321 double accuracy;322 double accuracy_unc;323 ggml_opt_result_accuracy(cd.result, &accuracy, &accuracy_unc);324 const bool subtest_ok = ndata == 0 && loss == 0.0 && std::isnan(loss_unc) && std::isnan(accuracy) && std::isnan(accuracy_unc);325 helper_after_test_forward_backward(__func__, high_level, shuffle, "results_initial", subtest_ok, ntest, npass);326 }327 328 if (high_level) {329 ggml_opt_dataset_t dataset = cd.dataset_unsupervised;330 if (shuffle) {331 ggml_opt_dataset_shuffle(cd.opt_ctx, dataset, -1);332 }333 ggml_opt_epoch(cd.opt_ctx, dataset, nullptr, cd.result, 0, nullptr, nullptr);334 } else {335 for (int idata = 0; idata < ndata; ++idata) {336 const float idataf = idata;337 ggml_backend_tensor_set(cd.inputs, &idataf, 0, ggml_nbytes(cd.inputs));338 ggml_opt_forward(cd.opt_ctx, cd.result);339 ggml_backend_tensor_get(loss, loss_history.data() + idata, 0, sizeof(float));340 }341 }342 343 {344 float weights;345 ggml_backend_tensor_get(cd.weights, &weights, 0, sizeof(float));346 const bool subtest_ok = weights == ndata/2;347 helper_after_test_forward_backward(__func__, high_level, shuffle, "weights_after_forward", subtest_ok, ntest, npass);348 }349 {350 int64_t ndata;351 ggml_opt_result_ndata(cd.result, &ndata);352 bool subtest_ok = ndata == 6;353 354 double loss;355 double loss_unc;356 ggml_opt_result_loss(cd.result, &loss, &loss_unc);357 subtest_ok = subtest_ok && loss == 33.0 && almost_equal(loss_unc, sqrt(3.5), 1e-10);358 359 double accuracy;360 double accuracy_unc;361 ggml_opt_result_accuracy(cd.result, &accuracy, &accuracy_unc);362 subtest_ok = subtest_ok && std::isnan(accuracy) && std::isnan(accuracy_unc);363 364 helper_after_test_forward_backward(__func__, high_level, shuffle, "results_after_forward", subtest_ok, ntest, npass);365 }366 367 float w0;368 ggml_backend_tensor_get(cd.weights, &w0, 0, sizeof(float));369 for (int i = 0; i < 10; ++i) {370 ggml_opt_forward_backward(cd.opt_ctx, nullptr);371 }372 ggml_backend_tensor_set(cd.weights, &w0, 0, sizeof(float));373 374 ggml_opt_reset(cd.opt_ctx, /*optimizer =*/ false);375 ggml_opt_result_reset(cd.result);376 377 for (int64_t idata = 0; idata < ndata; ++idata) {378 loss_history[idata] = NAN;379 }380 381 if (high_level) {382 ggml_opt_dataset_t dataset = cd.dataset_unsupervised;383 if (shuffle) {384 ggml_opt_dataset_shuffle(cd.opt_ctx, dataset, -1);385 }386 ggml_opt_epoch(cd.opt_ctx, dataset, cd.result, nullptr, ndata, nullptr, nullptr);387 } else {388 for (int idata = 0; idata < ndata; ++idata) {389 const float idataf = idata;390 ggml_backend_tensor_set(cd.inputs, &idataf, 0, ggml_nbytes(cd.inputs));391 ggml_opt_forward_backward(cd.opt_ctx, cd.result);392 ggml_backend_tensor_get(loss, loss_history.data() + idata, 0, sizeof(float));393 }394 }395 396 {397 float weights;398 ggml_backend_tensor_get(cd.weights, &weights, 0, sizeof(float));399 const bool subtest_ok = weights == -ndata/2;400 helper_after_test_forward_backward(__func__, high_level, shuffle, "weights_after_forward_backward", subtest_ok, ntest, npass);401 }402 {403 int64_t ndata;404 ggml_opt_result_ndata(cd.result, &ndata);405 bool subtest_ok = ndata == 6;406 407 double loss;408 double loss_unc;409 ggml_opt_result_loss(cd.result, &loss, &loss_unc);410 subtest_ok = subtest_ok && loss == 18.0 && (shuffle || loss_unc == 0.0);411 412 double accuracy;413 double accuracy_unc;414 ggml_opt_result_accuracy(cd.result, &accuracy, &accuracy_unc);415 subtest_ok = subtest_ok && std::isnan(accuracy) && std::isnan(accuracy_unc);416 417 helper_after_test_forward_backward(__func__, high_level, shuffle, "result_after_forward_backward", subtest_ok, ntest, npass);418 }419 420 helper_free_ctx_data(cd);421 422 return std::make_pair(npass, ntest);423}424 425static std::pair<int, int> test_epoch_vs_fit(ggml_backend_sched_t backend_sched, ggml_backend_t backend) {426 int ntest = 0;427 int npass = 0;428 429 float weights_epoch;430 float weights_fit;431 432 {433 struct helper_ctx_data cd = helper_get_ctx_data(backend_sched, backend, /*init_opt_ctx =*/ true);434 ggml_opt_dataset_t dataset = cd.dataset_unsupervised;435 436 ggml_opt_dataset_shuffle(cd.opt_ctx, dataset, -1);437 ggml_opt_epoch(cd.opt_ctx, dataset, cd.result, nullptr, ndata, nullptr, nullptr);438 439 ggml_backend_tensor_get(cd.weights, &weights_epoch, 0, ggml_nbytes(cd.weights));440 helper_free_ctx_data(cd);441 }442 {443 struct helper_ctx_data cd = helper_get_ctx_data(backend_sched, backend, /*init_opt_ctx =*/ false);444 ggml_opt_dataset_t dataset = cd.dataset_unsupervised;445 446 ggml_opt_fit(backend_sched, cd.ctx_compute, cd.inputs, cd.outputs, dataset,447 GGML_OPT_LOSS_TYPE_SUM, ggml_opt_get_default_optimizer_params, 1, 1, 0.0f, true);448 449 ggml_backend_tensor_get(cd.weights, &weights_fit, 0, ggml_nbytes(cd.weights));450 helper_free_ctx_data(cd);451 }452 453 const bool subtest_ok = weights_epoch == weights_fit;454 455 printf(" %s(): ", __func__);456 if (subtest_ok) {457 printf("\033[1;32mOK\033[0m\n");458 npass++;459 } else {460 printf("\033[1;31mFAIL\033[0m\n");461 }462 ntest++;463 464 return std::make_pair(npass, ntest);465}466 467static void helper_after_test_idata_split(468 const char * func, const bool high_level, const int epoch,469 const std::string subtest, const bool subtest_ok, int & ntest, int & npass) {470 std::string options = ", epoch=";471 options += std::to_string(epoch);472 helper_after_test(func, high_level, options, subtest, subtest_ok, ntest, npass);473}474 475static std::pair<int, int> test_idata_split(ggml_backend_sched_t backend_sched, ggml_backend_t backend, const bool high_level) {476 int ntest = 0;477 int npass = 0;478 479 struct helper_ctx_data cd = helper_get_ctx_data(backend_sched, backend, /*init_opt_ctx =*/ true, /*optimizer_defaults =*/ false);480 struct ggml_tensor * loss = ggml_opt_loss(cd.opt_ctx);481 const int idata_split = ndata * 2/3;482 483 std::vector<float> loss_history(ndata);484 for (int64_t idata = 0; idata < ndata; ++idata) {485 loss_history[idata] = NAN;486 }487 488 for (int epoch = 1; epoch <= 4; ++epoch) {489 if (high_level) {490 ggml_opt_epoch(cd.opt_ctx, cd.dataset_unsupervised, cd.result, cd.result2, idata_split, nullptr, nullptr);491 } else {492 int idata = 0;493 for (; idata < idata_split; ++idata) {494 const float idataf = idata;495 ggml_backend_tensor_set(cd.inputs, &idataf, 0, ggml_nbytes(cd.inputs));496 ggml_opt_forward_backward(cd.opt_ctx, cd.result);497 ggml_backend_tensor_get(loss, loss_history.data() + idata, 0, sizeof(float));498 }499 for (; idata < ndata; ++idata) {500 const float idataf = idata;501 ggml_backend_tensor_set(cd.inputs, &idataf, 0, ggml_nbytes(cd.inputs));502 ggml_opt_forward(cd.opt_ctx, cd.result2);503 ggml_backend_tensor_get(loss, loss_history.data() + idata, 0, sizeof(float));504 }505 }506 507 {508 float weights;509 ggml_backend_tensor_get(cd.weights, &weights, 0, sizeof(float));510 const bool subtest_ok = weights == ndata/2 - epoch*idata_split;511 helper_after_test_idata_split(__func__, high_level, epoch, "weights", subtest_ok, ntest, npass);512 }513 {514 int64_t ndata_result;515 ggml_opt_result_ndata(cd.result, &ndata_result);516 bool subtest_ok = ndata_result == idata_split;517 518 double loss;519 double loss_unc;520 ggml_opt_result_loss(cd.result, &loss, &loss_unc);521 subtest_ok = subtest_ok && loss == 28.0 - epoch*16.0 && loss_unc == 0.0;522 523 double accuracy;524 double accuracy_unc;525 ggml_opt_result_accuracy(cd.result, &accuracy, &accuracy_unc);526 subtest_ok = subtest_ok && std::isnan(accuracy) && std::isnan(accuracy_unc);527 528 helper_after_test_idata_split(__func__, high_level, epoch, "results_backward", subtest_ok, ntest, npass);529 }530 {531 int64_t ndata_result;532 ggml_opt_result_ndata(cd.result2, &ndata_result);533 bool subtest_ok = ndata_result == ndata - idata_split;534 535 double loss;536 double loss_unc;537 ggml_opt_result_loss(cd.result2, &loss, &loss_unc);538 subtest_ok = subtest_ok && loss == 15.0 - epoch*8 && almost_equal(loss_unc, sqrt(0.5), 1e-10);539 540 double accuracy;541 double accuracy_unc;542 ggml_opt_result_accuracy(cd.result2, &accuracy, &accuracy_unc);543 subtest_ok = subtest_ok && std::isnan(accuracy) && std::isnan(accuracy_unc);544 545 helper_after_test_idata_split(__func__, high_level, epoch, "results_forward", subtest_ok, ntest, npass);546 }547 548 ggml_opt_result_reset(cd.result);549 ggml_opt_result_reset(cd.result2);550 }551 552 helper_free_ctx_data(cd);553 554 return std::make_pair(npass, ntest);555}556 557static void helper_after_test_gradient_accumulation(558 const char * func, const int nbatch_physical, const enum ggml_opt_loss_type loss_type, const int epoch,559 const std::string subtest, const bool subtest_ok, int & ntest, int & npass) {560 std::string options = ", nbatch_physical=";561 options += std::to_string(nbatch_physical);562 options += ", loss_type=";563 options += loss_type == GGML_OPT_LOSS_TYPE_MEAN ? "mean" : "sum";564 options += ", epoch=";565 options += std::to_string(epoch);566 helper_after_test(func, false, options, subtest, subtest_ok, ntest, npass);567}568 569static std::pair<int, int> test_gradient_accumulation(570 ggml_backend_sched_t backend_sched, ggml_backend_t backend, const int32_t nbatch_physical, const enum ggml_opt_loss_type loss_type) {571 int ntest = 0;572 int npass = 0;573 574 struct helper_ctx_data cd = helper_get_ctx_data(575 backend_sched, backend, /*init_opt_ctx =*/ true, /*optimizer_defaults =*/ false, /*nbatch_logical =*/ 6, nbatch_physical, loss_type);576 struct ggml_tensor * loss = ggml_opt_loss(cd.opt_ctx);577 578 std::vector<float> grad_history(ndata);579 for (int64_t idata = 0; idata < ndata; ++idata) {580 grad_history[idata] = NAN;581 }582 583 for (int epoch = 1; epoch <= 4; ++epoch) {584 if (nbatch_physical == 1) {585 for (int idata = 0; idata < ndata; ++idata) {586 const float idataf = idata;587 ggml_backend_tensor_set(cd.inputs, &idataf, 0, 1*sizeof(float));588 ggml_opt_forward_backward(cd.opt_ctx, cd.result);589 ggml_backend_tensor_get(ggml_opt_grad_acc(cd.opt_ctx, cd.weights), grad_history.data() + idata, 0, 1*sizeof(float));590 }591 } else if (nbatch_physical == 2) {592 for (int idata = 0; idata < ndata; idata += 2) {593 const float idataf[2] = {float(idata + 0), float(idata + 1)};594 ggml_backend_tensor_set(cd.inputs, idataf, 0, 2*sizeof(float));595 ggml_opt_forward_backward(cd.opt_ctx, cd.result);596 597 grad_history[idata + 0] = 0.0f;598 ggml_backend_tensor_get(ggml_opt_grad_acc(cd.opt_ctx, cd.weights), grad_history.data() + idata + 1, 0, 1*sizeof(float));599 }600 } else {601 GGML_ASSERT(false);602 }603 604 {605 GGML_ASSERT(ndata == 6);606 constexpr double atol = 1e-6;607 bool subtest_ok = true;608 if (loss_type == GGML_OPT_LOSS_TYPE_SUM) {609 if (nbatch_physical == 1) {610 subtest_ok = subtest_ok && almost_equal(grad_history[0], 1.0, atol);611 subtest_ok = subtest_ok && almost_equal(grad_history[2], 3.0, atol);612 subtest_ok = subtest_ok && almost_equal(grad_history[4], 5.0, atol);613 } else {614 subtest_ok = subtest_ok && almost_equal(grad_history[0], 0.0, atol);615 subtest_ok = subtest_ok && almost_equal(grad_history[2], 0.0, atol);616 subtest_ok = subtest_ok && almost_equal(grad_history[4], 0.0, atol);617 }618 subtest_ok = subtest_ok && almost_equal(grad_history[1], 2.0, atol);619 subtest_ok = subtest_ok && almost_equal(grad_history[3], 4.0, atol);620 subtest_ok = subtest_ok && almost_equal(grad_history[5], 0.0, atol);621 } else if (loss_type == GGML_OPT_LOSS_TYPE_MEAN) {622 if (nbatch_physical == 1) {623 subtest_ok = subtest_ok && almost_equal(grad_history[0], 1.0/ndata, atol);624 subtest_ok = subtest_ok && almost_equal(grad_history[2], 3.0/ndata, atol);625 subtest_ok = subtest_ok && almost_equal(grad_history[4], 5.0/ndata, atol);626 } else {627 subtest_ok = subtest_ok && almost_equal(grad_history[0], 0.0/ndata, atol);628 subtest_ok = subtest_ok && almost_equal(grad_history[2], 0.0/ndata, atol);629 subtest_ok = subtest_ok && almost_equal(grad_history[4], 0.0/ndata, atol);630 }631 subtest_ok = subtest_ok && almost_equal(grad_history[1], 2.0/ndata, atol);632 subtest_ok = subtest_ok && almost_equal(grad_history[3], 4.0/ndata, atol);633 subtest_ok = subtest_ok && almost_equal(grad_history[5], 0.0/ndata, atol);634 } else {635 GGML_ASSERT(false);636 }637 helper_after_test_gradient_accumulation(__func__, nbatch_physical, loss_type, epoch, "grads", subtest_ok, ntest, npass);638 }639 {640 float weights;641 ggml_backend_tensor_get(cd.weights, &weights, 0, sizeof(float));642 const bool subtest_ok = weights == (ndata/2) - epoch;643 helper_after_test_gradient_accumulation(__func__, nbatch_physical, loss_type, epoch, "weights", subtest_ok, ntest, npass);644 }645 {646 int64_t ndata_result;647 ggml_opt_result_ndata(cd.result, &ndata_result);648 bool subtest_ok = ndata_result == ndata/nbatch_physical;649 650 double loss;651 ggml_opt_result_loss(cd.result, &loss, /*loss_unc =*/ nullptr);652 if (loss_type == GGML_OPT_LOSS_TYPE_SUM) {653 subtest_ok = subtest_ok && loss == (39.0 - epoch*6.0);654 } else if (loss_type == GGML_OPT_LOSS_TYPE_MEAN) {655 subtest_ok = subtest_ok && almost_equal(loss, (39.0 - epoch*6.0) / ndata, 1e-6);656 } else {657 GGML_ASSERT(false);658 }659 660 double accuracy;661 double accuracy_unc;662 ggml_opt_result_accuracy(cd.result, &accuracy, &accuracy_unc);663 subtest_ok = subtest_ok && std::isnan(accuracy) && std::isnan(accuracy_unc);664 665 helper_after_test_gradient_accumulation(__func__, nbatch_physical, loss_type, epoch, "results", subtest_ok, ntest, npass);666 }667 668 ggml_opt_result_reset(cd.result);669 }670 671 helper_free_ctx_data(cd);672 673 return std::make_pair(npass, ntest);674}675 676static ggml_opt_optimizer_params helper_get_regression_opt_pars(void * userdata) {677 ggml_opt_optimizer_params result = ggml_opt_get_default_optimizer_params(userdata);678 result.adamw.alpha = 0.1f;679 return result;680}681 682static std::pair<int, int> test_regression(ggml_backend_sched_t backend_sched, ggml_backend_t backend) {683 int ntest = 0;684 int npass = 0;685 686 // Test for simple regression with f(x) = a*x + b687 688 constexpr int64_t ndata_regression = 201;689 constexpr float a_true = 1.2f;690 constexpr float b_true = 3.4f;691 692 std::mt19937 gen(12345);693 std::normal_distribution<float> nd{0.0f, 0.1f};694 695 ggml_opt_dataset_t dataset = ggml_opt_dataset_init(1, 1, ndata_regression, ndata_regression);696 697 float * data = ggml_get_data_f32(ggml_opt_dataset_data( dataset));698 float * labels = ggml_get_data_f32(ggml_opt_dataset_labels(dataset));699 700 constexpr float x_min = -100.0f;701 constexpr float x_max = 100.0f;702 703 for (int64_t idata = 0; idata < ndata_regression; ++idata) {704 const float x = x_min + (x_max - x_min) * idata/(ndata_regression-1);705 const float y = a_true*x + b_true + nd(gen);706 707 data[idata] = x;708 labels[idata] = y;709 }710 711 struct ggml_context * ctx_static;712 struct ggml_context * ctx_compute;713 {714 struct ggml_init_params params = {715 /*.mem_size =*/ 3*ggml_tensor_overhead(),716 /*.mem_buffer =*/ nullptr,717 /*.no_alloc =*/ true,718 };719 ctx_static = ggml_init(params);720 }721 {722 struct ggml_init_params params = {723 /*.mem_size =*/ GGML_DEFAULT_GRAPH_SIZE*ggml_tensor_overhead() + 3*ggml_graph_overhead(),724 /*.mem_buffer =*/ nullptr,725 /*.no_alloc =*/ true,726 };727 ctx_compute = ggml_init(params);728 }729 730 // The first dimension is the dimension of the datapoints, the second dimension is the number of datapoints.731 struct ggml_tensor * x = ggml_new_tensor_2d(ctx_static, GGML_TYPE_F32, 1, ndata_regression);732 ggml_set_name(x, "x");733 734 struct ggml_tensor * a = ggml_new_tensor_1d(ctx_static, GGML_TYPE_F32, 1);735 ggml_set_name(a, "a");736 ggml_set_param(ctx_static, a);737 738 struct ggml_tensor * b = ggml_new_tensor_1d(ctx_static, GGML_TYPE_F32, 1);739 ggml_set_name(b, "b");740 ggml_set_param(ctx_static, b);741 742 struct ggml_tensor * f = ggml_add(ctx_compute, ggml_mul(ctx_compute, x, a), b);743 ggml_set_name(f, "f");744 ggml_set_param(ctx_static, f);745 746 ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors(ctx_static, backend);747 const float a0 = 1.0f;748 const float b0 = 3.0f;749 ggml_backend_tensor_set(a, &a0, 0, sizeof(float));750 ggml_backend_tensor_set(b, &b0, 0, sizeof(float));751 752 ggml_opt_fit(backend_sched, ctx_compute, x, f, dataset, GGML_OPT_LOSS_TYPE_MEAN_SQUARED_ERROR,753 helper_get_regression_opt_pars, 100, ndata_regression, 0.0f, true);754 755 {756 float a_fit;757 ggml_backend_tensor_get(a, &a_fit, 0, sizeof(float));758 float b_fit;759 ggml_backend_tensor_get(b, &b_fit, 0, sizeof(float));760 const bool subtest_ok = almost_equal(a_fit, a_true, 1e-2) && almost_equal(b_fit, b_true, 1e-2);761 printf(" %s(subtest=weights): ", __func__);762 if (subtest_ok) {763 printf("\033[1;32mOK\033[0m\n");764 npass++;765 } else {766 printf("\033[1;31mFAIL\033[0m\n");767 }768 ntest++;769 }770 771 ggml_backend_buffer_free(buf);772 ggml_free(ctx_static);773 ggml_opt_dataset_free(dataset);774 775 return std::make_pair(npass, ntest);776}777 778static std::pair<int, int> test_backend(ggml_backend_sched_t backend_sched, ggml_backend_t backend) {779 int npass = 0;780 int ntest = 0;781 782 for (bool shuffle : {false, true}) {783 std::pair<int, int> partial = test_dataset(backend_sched, backend, shuffle);784 npass += partial.first;785 ntest += partial.second;786 }787 {788 std::pair<int, int> partial = test_grad(backend_sched, backend);789 npass += partial.first;790 ntest += partial.second;791 }792 for (bool high_level : {false, true}){793 for (bool shuffle : {false, true}) {794 if (!high_level && shuffle) {795 continue;796 }797 798 std::pair<int, int> partial = test_forward_backward(backend_sched, backend, high_level, shuffle);799 npass += partial.first;800 ntest += partial.second;801 }802 }803 {804 std::pair<int, int> partial = test_epoch_vs_fit(backend_sched, backend);805 npass += partial.first;806 ntest += partial.second;807 }808 for (bool high_level : {false, true}){809 std::pair<int, int> partial = test_idata_split(backend_sched, backend, high_level);810 npass += partial.first;811 ntest += partial.second;812 }813 for (int32_t nbatch_physical : {2, 1}) {814 for (enum ggml_opt_loss_type loss_type : {GGML_OPT_LOSS_TYPE_SUM, GGML_OPT_LOSS_TYPE_MEAN}) {815 std::pair<int, int> partial = test_gradient_accumulation(backend_sched, backend, nbatch_physical, loss_type);816 npass += partial.first;817 ntest += partial.second;818 }819 }820 {821 std::pair<int, int> partial = test_regression(backend_sched, backend);822 npass += partial.first;823 ntest += partial.second;824 }825 826 return std::make_pair(npass, ntest);827}828 829int main(void) {830 const size_t dev_count = ggml_backend_dev_count();831 printf("Testing %zu devices\n\n", dev_count);832 size_t n_ok = 0;833 834 std::vector<ggml_backend_dev_t> devs;835 std::vector<ggml_backend_t> backends;836 837 for (size_t i = 0; i < dev_count; ++i) {838 devs.push_back(ggml_backend_dev_get(i));839 840 ggml_backend_t backend = ggml_backend_dev_init(devs[i], NULL);841 GGML_ASSERT(backend != NULL);842 843 if (ggml_backend_is_cpu(backend)) {844 ggml_backend_cpu_set_n_threads(backend, std::thread::hardware_concurrency() / 2);845 }846 847 backends.push_back(backend);848 }849 850 for (size_t i = 0; i < dev_count; ++i) {851 // Put the backend to be tested in front so that it's prioritized:852 std::vector<ggml_backend_t> backends_modded = {backends[i]};853 backends_modded.insert(backends_modded.end(), backends.begin(), backends.end());854 855 ggml_backend_sched_t backend_sched = ggml_backend_sched_new(856 backends_modded.data(), nullptr, backends_modded.size(), GGML_DEFAULT_GRAPH_SIZE, false);857 858 printf("Backend %zu/%zu: %s\n", i + 1, dev_count, ggml_backend_dev_name(devs[i]));859 printf(" Device description: %s\n", ggml_backend_dev_description(devs[i]));860 size_t free, total; // NOLINT861 ggml_backend_dev_memory(devs[i], &free, &total);862 printf(" Device memory: %zu MB (%zu MB free)\n", total / 1024 / 1024, free / 1024 / 1024);863 printf("\n");864 865 std::pair<int, int> result = test_backend(backend_sched, backends[i]);866 867 printf(" %d/%d tests passed\n", result.first, result.second);868 printf(" Backend %s: ", ggml_backend_name(backends[i]));869 if (result.first == result.second) {870 printf("\033[1;32mOK\033[0m\n");871 n_ok++;872 } else {873 printf("\033[1;31mFAIL\033[0m\n");874 }875 876 printf("\n");877 878 ggml_backend_sched_free(backend_sched);879 }880 881 for (ggml_backend_t backend : backends) {882 ggml_backend_free(backend);883 }884 885 printf("%zu/%zu backends passed\n", n_ok, dev_count);886 if (n_ok != dev_count) {887 printf("\033[1;31mFAIL\033[0m\n");888 return 1;889 }890 printf("\033[1;32mOK\033[0m\n");891 return 0;892}893