Felipe97/llama-cpp-compiled

1#include "ggml-alloc.h"2#include "../ggml/src/ggml-backend-impl.h"3#include "ggml-cpp.h"4#include "../ggml/src/ggml-impl.h"5#include "ggml.h"6 7#include <algorithm>8#include <exception>9#include <memory>10#include <vector>11 12//13// dummy backend with configurable max_buffer_size, tracks allocations14 15uint8_t * const alloc_base = (uint8_t *) 16;16 17struct dummy_backend_context {18    size_t max_buffer_size = 64;19    size_t alignment       = 8;20 21    ggml_backend_buffer_i              buffer_interface;22    ggml_backend_device                device;23    ggml_backend                       backend;24    std::vector<ggml_backend_buffer_t> buffers;25 26    size_t allocated_total() const {27        size_t n = 0;28        for (ggml_backend_buffer_t buf : buffers) {29            n += ggml_backend_buffer_get_size(buf);30        }31        return n;32    }33};34 35// ggml_backend_buffer_type interface36 37static const char * dummy_backend_buffer_type_get_name(ggml_backend_buffer_type_t) {38    return "dummy_buffer_type";39}40 41static ggml_backend_buffer_t dummy_backend_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) {42    dummy_backend_context * ctx    = (dummy_backend_context *) buft->context;43    ggml_backend_buffer_t & buffer = ctx->buffers.emplace_back();44    buffer                         = ggml_backend_buffer_init(buft, ctx->buffer_interface, ctx, size);45    return buffer;46}47 48static size_t dummy_backend_buffer_type_get_alignment(ggml_backend_buffer_type_t buft) {49    dummy_backend_context * ctx = (dummy_backend_context *) buft->context;50    return ctx->alignment;51}52 53static size_t dummy_backend_buffer_type_get_max_size(ggml_backend_buffer_type_t buft) {54    dummy_backend_context * ctx = (dummy_backend_context *) buft->context;55    return ctx->max_buffer_size;56}57 58static bool dummy_backend_buffer_type_is_host(ggml_backend_buffer_type_t) {59    return true;60}61 62// ggml_backend_buffer interface63 64static void dummy_backend_buffer_free_buffer(ggml_backend_buffer_t buffer) {65    dummy_backend_context * ctx = (dummy_backend_context *) buffer->context;66 67    auto i = std::find(ctx->buffers.begin(), ctx->buffers.end(), buffer);68    GGML_ASSERT(i != ctx->buffers.end());69    ctx->buffers.erase(i);70}71 72static void * dummy_backend_buffer_get_base(ggml_backend_buffer_t) {73    return alloc_base;74}75 76static ggml_status dummy_backend_buffer_init_tensor(ggml_backend_buffer_t, ggml_tensor *) {77    return GGML_STATUS_SUCCESS;78}79 80static void dummy_backend_buffer_memset_tensor(ggml_backend_buffer_t, ggml_tensor *, uint8_t, size_t, size_t) {}81 82static void dummy_backend_buffer_set_tensor(ggml_backend_buffer_t, ggml_tensor *, const void *, size_t, size_t) {}83 84static void dummy_backend_buffer_get_tensor(ggml_backend_buffer_t, const ggml_tensor *, void *, size_t, size_t) {}85 86static void dummy_backend_buffer_clear(ggml_backend_buffer_t, uint8_t) {}87 88// ggml_backend_device interface89 90static enum ggml_backend_dev_type dummy_backend_device_get_type(ggml_backend_dev_t) {91    return GGML_BACKEND_DEVICE_TYPE_CPU;92}93 94static bool dummy_backend_device_supports_op(ggml_backend_dev_t, const ggml_tensor *) {95    return true;96}97 98static bool dummy_backend_device_supports_buft(ggml_backend_dev_t device, ggml_backend_buffer_type_t buft) {99    return device->context == buft->context;100}101 102// ggml_backend interface103 104static const char * dummy_backend_get_name(ggml_backend_t) {105    return "dummy_backend";106}107 108// dummy_backend109 110struct dummy_backend {111    std::unique_ptr<dummy_backend_context> context;112    ggml_backend_buffer_type               buffer_type;113};114 115static dummy_backend dummy_backend_init(size_t max_buffer_size, size_t alignment = 8) {116    dummy_backend b{};117    b.context                  = std::make_unique<dummy_backend_context>();118    b.context->alignment       = alignment;119    b.context->max_buffer_size = max_buffer_size;120 121    b.context->buffer_interface.free_buffer   = dummy_backend_buffer_free_buffer;122    b.context->buffer_interface.get_base      = dummy_backend_buffer_get_base;123    b.context->buffer_interface.init_tensor   = dummy_backend_buffer_init_tensor;124    b.context->buffer_interface.memset_tensor = dummy_backend_buffer_memset_tensor;125    b.context->buffer_interface.set_tensor    = dummy_backend_buffer_set_tensor;126    b.context->buffer_interface.get_tensor    = dummy_backend_buffer_get_tensor;127    b.context->buffer_interface.clear         = dummy_backend_buffer_clear;128 129    b.context->device.context             = b.context.get();130    b.context->device.iface.get_type      = dummy_backend_device_get_type;131    b.context->device.iface.supports_op   = dummy_backend_device_supports_op;132    b.context->device.iface.supports_buft = dummy_backend_device_supports_buft;133 134    b.context->backend.context        = b.context.get();135    b.context->backend.device         = &b.context->device;136    b.context->backend.iface.get_name = dummy_backend_get_name;137 138    b.buffer_type.device              = &b.context->device;139    b.buffer_type.context             = b.context.get();140    b.buffer_type.iface.get_name      = dummy_backend_buffer_type_get_name;141    b.buffer_type.iface.alloc_buffer  = dummy_backend_buffer_type_alloc_buffer;142    b.buffer_type.iface.get_alignment = dummy_backend_buffer_type_get_alignment;143    b.buffer_type.iface.get_max_size  = dummy_backend_buffer_type_get_max_size;144    b.buffer_type.iface.is_host       = dummy_backend_buffer_type_is_host;145    return b;146}147 148//149// test utilities150 151struct test_context_with_graph {152    ggml_context *   ctx;153    ggml_cgraph *    graph;154    ggml_context_ptr ctx_ptr;155};156 157static test_context_with_graph make_context() {158    ggml_init_params params{};159    params.mem_size = 48 * ggml_tensor_overhead() + ggml_graph_overhead();160    params.no_alloc = true;161 162    ggml_context *   ctx     = ggml_init(params);163    ggml_context_ptr ctx_ptr = ggml_context_ptr(ctx);164    ggml_cgraph *    graph   = ggml_new_graph(ctx);165    return { ctx, graph, std::move(ctx_ptr) };166}167 168static ggml_tensor * make_input_1d(ggml_context * ctx, int64_t n_elements) {169    ggml_tensor * t = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n_elements);170    ggml_set_input(t);171    return t;172}173 174static ggml_tensor * make_input_with_size(ggml_context * ctx, size_t size_bytes) {175    GGML_ASSERT(size_bytes % 4 == 0);176    return make_input_1d(ctx, size_bytes / 4);177}178 179static void assign_names(ggml_context * ctx, const char * prefix = "x") {180    int i = 0;181    for (ggml_tensor * t = ggml_get_first_tensor(ctx); t; t = ggml_get_next_tensor(ctx, t)) {182        ggml_format_name(t, "%s%d", prefix, i++);183    }184}185 186static int get_leaf_id(ggml_cgraph * graph, const char * tensor_name) {187    for (int i = 0; i < graph->n_leafs; ++i) {188        if (strncmp(graph->leafs[i]->name, tensor_name, GGML_MAX_NAME) == 0) {189            return i;190        }191    }192    fprintf(stderr, "leaf not found: %s\n", tensor_name);193    return -1;194}195 196static int get_node_id(ggml_cgraph * graph, const char * tensor_name) {197    for (int i = 0; i < graph->n_nodes; ++i) {198        if (strncmp(graph->nodes[i]->name, tensor_name, GGML_MAX_NAME) == 0) {199            return i;200        }201    }202    fprintf(stderr, "node not found: %s", tensor_name);203    return -1;204}205 206static ggml_gallocr_ptr allocate_graph(ggml_cgraph * graph, ggml_tensor * out, ggml_backend_buffer_type_t buft) {207    ggml_set_output(out);208    ggml_build_forward_expand(graph, out);209 210    ggml_gallocr_ptr galloc = ggml_gallocr_ptr(ggml_gallocr_new(buft));211    bool             result = ggml_gallocr_alloc_graph(galloc.get(), graph);212    GGML_ASSERT(result);213    return galloc;214}215 216//217// correctness checks for result allocations218 219static void check_all_allocated(ggml_cgraph * graph) {220    for (int i = 0; i < ggml_graph_n_nodes(graph); ++i) {221        ggml_tensor * t = ggml_graph_node(graph, i);222        GGML_ASSERT(t->buffer != nullptr);223        GGML_ASSERT(t->data != nullptr);224    }225}226 227static void check_max_size(ggml_context * ctx) {228    for (ggml_tensor * t = ggml_get_first_tensor(ctx); t; t = ggml_get_next_tensor(ctx, t)) {229        auto   buft     = ggml_backend_buffer_get_type(t->buffer);230        size_t max_size = ggml_backend_buft_get_max_size(buft);231        size_t offset   = (char *) t->data - (char *) ggml_backend_buffer_get_base(t->buffer);232        GGML_ASSERT(t->data >= ggml_backend_buffer_get_base(t->buffer));233        GGML_ASSERT((size_t) offset + ggml_nbytes(t) <= max_size);234    }235}236 237static bool can_reuse_memory(ggml_cgraph * graph, int current_i, ggml_tensor * current, ggml_tensor * other) {238    if (other->flags & GGML_TENSOR_FLAG_OUTPUT) {239        return false;240    }241    // Check if `other` is still "alive", ie. an input to any node after the `current` op242    for (int i = current_i; i < ggml_graph_n_nodes(graph); ++i) {243        ggml_tensor * t = ggml_graph_node(graph, i);244        for (int s = 0; s < GGML_MAX_SRC; s++) {245            if (t == current && ggml_op_can_inplace(t->op)) {246                continue;247            }248            if (t->src[s] == other) {249                return false;250            }251            if (t->src[s] && t->src[s]->view_src == other) {252                return false;253            }254        }255    }256    return true;257}258 259static bool memory_overlap(ggml_tensor * a, ggml_tensor * b) {260    if (a->buffer != b->buffer) {261        return false;262    }263    int64_t a0 = (int64_t) a->data;264    int64_t a1 = a0 + ggml_nbytes(a);265    int64_t b0 = (int64_t) b->data;266    int64_t b1 = b0 + ggml_nbytes(b);267    return a1 > b0 && b1 > a0;268}269 270static ggml_tensor * get_view_source(ggml_tensor * t) {271    while (t->view_src) {272        t = t->view_src;273    }274    return t;275}276 277static void check_no_overlap(ggml_cgraph * graph) {278    for (int i = 0; i < ggml_graph_n_nodes(graph); ++i) {279        for (int j = 0; j < i; ++j) {280            ggml_tensor * t = ggml_graph_node(graph, i);281            ggml_tensor * o = ggml_graph_node(graph, j);282            GGML_ASSERT(t != o);283 284            if (get_view_source(t) == get_view_source(o)) {285                continue;286            }287            if (memory_overlap(t, o)) {288                GGML_ASSERT(can_reuse_memory(graph, i, t, o));289            }290        }291    }292}293 294//295// test cases296 297// Scenario where the first backend buffer is completely exhausted and there are further298// tensors which require a second buffer299static void test_max_size_too_many_tensors() {300    dummy_backend backend      = dummy_backend_init(16);301    auto [ctx, graph, ctx_ptr] = make_context();302 303    ggml_tensor * x[7];304    x[0] = make_input_with_size(ctx, 8);305    x[1] = make_input_with_size(ctx, 8);306    x[2] = make_input_with_size(ctx, 8);307    x[3] = ggml_mul(ctx, x[0], x[1]);308    x[4] = ggml_add(ctx, x[1], x[2]);309    x[5] = ggml_add(ctx, x[3], x[0]);310    x[6] = ggml_add(ctx, x[4], x[5]);311    assign_names(ctx);312 313    ggml_gallocr_ptr galloc = allocate_graph(graph, x[6], &backend.buffer_type);314    check_all_allocated(graph);315    check_no_overlap(graph);316    check_max_size(ctx);317    GGML_ASSERT(backend.context->allocated_total() <= 16 + 16);318}319 320// Scenario where there is some space left in the first buffer, but not enough to accommodate321// a larger tensor, so a second buffer is required322static void test_max_size_tensor_too_large() {323    dummy_backend backend      = dummy_backend_init(32);324    auto [ctx, graph, ctx_ptr] = make_context();325 326    ggml_tensor * x[3];327    x[0] = make_input_with_size(ctx, 16);    // chunk 0, [0 , 16)328    x[1] = make_input_with_size(ctx, 8);     // chunk 0, [16, 24)329    x[2] = ggml_concat(ctx, x[0], x[1], 0);  // chunk 1, [0 , 24)330    assign_names(ctx);331 332    ggml_gallocr_ptr galloc = allocate_graph(graph, x[2], &backend.buffer_type);333    check_all_allocated(graph);334    check_no_overlap(graph);335    check_max_size(ctx);336    GGML_ASSERT(backend.context->allocated_total() <= 32 + 24);337}338 339// Scenario where a single tensor exceeds the max buffer size - in this case the allocator340// should try to create a bigger buffer anyway, and wait for the backend to throw an error.341// Backends may report an artificially lower max size in some cases for compatibility reasons.342static void test_tensor_larger_than_max_size() {343    dummy_backend backend      = dummy_backend_init(16);344    auto [ctx, graph, ctx_ptr] = make_context();345 346    ggml_tensor * x[2];347    x[0] = make_input_with_size(ctx, 24);348    x[1] = ggml_scale(ctx, x[0], 2.0f);349    assign_names(ctx);350 351    ggml_gallocr_ptr galloc = allocate_graph(graph, x[1], &backend.buffer_type);352    check_all_allocated(graph);353    check_no_overlap(graph);354    GGML_ASSERT(backend.context->allocated_total() == 24);355}356 357// This test assumes a max of 16 buffer chunks, and tries to allocate tensors that would358// require more. Expectation is that the last buffer should grow to fit everything,359// leaving it to the backend to error out if it can't allocate that much.360static void test_not_enough_chunks() {361    const int max_chunks = 16;362    const int max_size   = 8;363 364    dummy_backend backend      = dummy_backend_init(max_size);365    auto [ctx, graph, ctx_ptr] = make_context();366 367    ggml_tensor * x[max_chunks + 1];368    for (int i = 0; i < max_chunks + 1; ++i) {369        x[i] = make_input_with_size(ctx, max_size);370    }371    ggml_tensor * acc = x[0];372    for (int i = 0; i < max_chunks; ++i) {373        acc = ggml_add(ctx, acc, x[i + 1]);374    }375    assign_names(ctx);376 377    ggml_gallocr_ptr galloc = allocate_graph(graph, acc, &backend.buffer_type);378    check_all_allocated(graph);379    check_no_overlap(graph);380    GGML_ASSERT(backend.context->allocated_total() > max_chunks * max_size);381}382 383// Fill up leftover unallocated space of a chunk after allocating a large tensor that384// requires a new chunk.385static void test_fill_leftover_space() {386    dummy_backend backend      = dummy_backend_init(16);387    auto [ctx, graph, ctx_ptr] = make_context();388 389    ggml_tensor * x[4];390    x[0] = make_input_with_size(ctx, 8);391    x[1] = ggml_pad(ctx, x[0], 2, 0, 0, 0);392    x[3] = ggml_mean(ctx, x[1]);393    assign_names(ctx);394 395    ggml_gallocr_ptr galloc = allocate_graph(graph, x[3], &backend.buffer_type);396    check_all_allocated(graph);397    check_no_overlap(graph);398    check_max_size(ctx);399    GGML_ASSERT(backend.context->allocated_total() <= 12 + 16);400}401 402// Check that views don't require any extra memory403static void test_view_inplace() {404    dummy_backend backend      = dummy_backend_init(32);405    auto [ctx, graph, ctx_ptr] = make_context();406 407    ggml_tensor * x[6];408    x[0] = make_input_1d(ctx, 4);                // chunk 0, [0, 16)409    x[1] = ggml_reshape_2d(ctx, x[0], 2, 2);     // view of x0410    x[2] = ggml_permute(ctx, x[1], 1, 0, 2, 3);  // view of x0411    x[3] = ggml_view_1d(ctx, x[2], 2, 4);        // view of x0412    x[4] = make_input_1d(ctx, 2);                // chunk 0, [16, 24)413    x[5] = ggml_add(ctx, x[3], x[4]);            // reuse (inplace add)414    assign_names(ctx);415 416    ggml_gallocr_ptr galloc = allocate_graph(graph, x[5], &backend.buffer_type);417    check_all_allocated(graph);418    check_no_overlap(graph);419    check_max_size(ctx);420    GGML_ASSERT(backend.context->allocated_total() <= 24);421}422 423static void test_reuse_and_free() {424    dummy_backend backend      = dummy_backend_init(40);425    auto [ctx, graph, ctx_ptr] = make_context();426 427    ggml_tensor * x[9];428    x[0] = make_input_with_size(ctx, 24);429    x[1] = make_input_with_size(ctx, 8);430    x[2] = make_input_with_size(ctx, 8);431    x[3] = ggml_add(ctx, x[1], x[2]);        // reuse, free x2432    x[4] = ggml_pad(ctx, x[0], 2, 0, 0, 0);  // alloc new buffer, free x0433    x[5] = ggml_scale(ctx, x[4], 2.0f);      // alloc from free block434    x[6] = ggml_add(ctx, x[4], x[5]);        // reuse, free x5435    x[7] = ggml_view_1d(ctx, x[6], 2, 8);    // view436    x[8] = ggml_add(ctx, x[3], x[7]);        // reuse437    assign_names(ctx);438 439    ggml_gallocr_ptr galloc = allocate_graph(graph, x[8], &backend.buffer_type);440    check_all_allocated(graph);441    check_no_overlap(graph);442    check_max_size(ctx);443    GGML_ASSERT(backend.context->allocated_total() <= 40 + 32 + 32);444}445 446static void test_merge_free_block(size_t max_buffer_size) {447    dummy_backend backend      = dummy_backend_init(max_buffer_size);448    auto [ctx, graph, ctx_ptr] = make_context();449 450    ggml_tensor * x[9];451    x[0] = make_input_with_size(ctx, 16);452    x[1] = make_input_with_size(ctx, 16);453    x[2] = make_input_with_size(ctx, 16);454    x[3] = ggml_mean(ctx, x[0]);455    x[4] = ggml_mean(ctx, x[1]);456    x[5] = ggml_pad(ctx, x[2], 2, 0, 0, 0);457    x[6] = ggml_add(ctx, x[3], x[4]);458    x[7] = ggml_pad(ctx, x[6], 5, 0, 0, 0);459    x[8] = ggml_add(ctx, x[5], x[7]);460    assign_names(ctx);461 462    ggml_gallocr_ptr galloc = allocate_graph(graph, x[8], &backend.buffer_type);463    check_all_allocated(graph);464    check_no_overlap(graph);465    check_max_size(ctx);466    GGML_ASSERT(backend.context->allocated_total() <= 32 + 32 + 24);467}468 469// Check that previously allocated but freed memory is preferred over allocating470// additional memory, even if the remaining space in a chunk would match tensor size better471static void test_prefer_already_allocated_memory() {472    dummy_backend backend      = dummy_backend_init(32, /*align*/ 4);473    auto [ctx, graph, ctx_ptr] = make_context();474 475    ggml_tensor * x[3];476    x[0] = make_input_with_size(ctx, 24);  // [24b][8b unused]477    x[1] = ggml_mean(ctx, x[0]);           // [24b free][4b][4b unused]478    x[2] = ggml_mean(ctx, x[1]);           // should be allocated in the 24b block479    assign_names(ctx);480 481    ggml_gallocr_ptr galloc = allocate_graph(graph, x[2], &backend.buffer_type);482    check_all_allocated(graph);483    check_no_overlap(graph);484    GGML_ASSERT(backend.context->allocated_total() <= 28);485}486 487// test for allocating on multiple devices with some tensors in the graph488// allocated externally (not by gallocr).489static void test_multiple_buffer_types() {490    dummy_backend backend_a = dummy_backend_init(32);491    dummy_backend backend_b = dummy_backend_init(SIZE_MAX);492 493    auto [ctx_a, _a, ctx_a_ptr] = make_context();494    auto [ctx_b, _b, ctx_b_ptr] = make_context();495    auto [ctx, graph, ctx_ptr]  = make_context();496 497    ggml_tensor * a[2];498    a[0] = make_input_with_size(ctx_a, 16);499    a[1] = make_input_with_size(ctx_a, 16);500    assign_names(ctx_a, "a");501 502    ggml_tensor * b[2];503    b[0] = make_input_with_size(ctx_b, 24);504    b[1] = make_input_with_size(ctx_b, 4);505    assign_names(ctx_b, "b");506 507    ggml_tensor * x[9];508    x[0] = make_input_with_size(ctx, 16);509    x[1] = ggml_mul(ctx, x[0], a[0]);510    x[2] = ggml_pad(ctx, x[1], 2, 0, 0, 0);511    x[3] = ggml_mul(ctx, x[2], b[0]);512    x[4] = ggml_mean(ctx, x[3]);513    x[5] = ggml_add(ctx, x[4], b[1]);514    x[6] = ggml_pad(ctx, x[5], 3, 0, 0, 0);515    x[7] = ggml_add(ctx, x[6], a[1]);516    x[8] = ggml_scale(ctx, x[7], 2.0f);517    assign_names(ctx, "x");518 519    ggml_backend_buffer_ptr    buf_a(ggml_backend_alloc_ctx_tensors_from_buft(ctx_a, &backend_a.buffer_type));520    ggml_backend_buffer_ptr    buf_b(ggml_backend_alloc_ctx_tensors_from_buft(ctx_b, &backend_b.buffer_type));521    ggml_backend_buffer_type_t bufts[2] = { &backend_a.buffer_type, &backend_b.buffer_type };522 523    // assign buffer types manually to avoid extra complexity from backend scheduler524    ggml_set_output(x[8]);525    ggml_build_forward_expand(graph, x[8]);526 527    GGML_ASSERT(graph->n_leafs == 5);528    int leaf_buffer_ids[5];529    leaf_buffer_ids[get_leaf_id(graph, "a0")] = 0;530    leaf_buffer_ids[get_leaf_id(graph, "a1")] = 0;531    leaf_buffer_ids[get_leaf_id(graph, "b0")] = 1;532    leaf_buffer_ids[get_leaf_id(graph, "b1")] = 1;533    leaf_buffer_ids[get_leaf_id(graph, "x0")] = 0;534 535    GGML_ASSERT(graph->n_nodes == 8);536    int node_buffer_ids[8];537    node_buffer_ids[get_node_id(graph, "x1")] = 0;538    node_buffer_ids[get_node_id(graph, "x2")] = 0;539    node_buffer_ids[get_node_id(graph, "x3")] = 1;540    node_buffer_ids[get_node_id(graph, "x4")] = 1;541    node_buffer_ids[get_node_id(graph, "x5")] = 1;542    node_buffer_ids[get_node_id(graph, "x6")] = 1;543    node_buffer_ids[get_node_id(graph, "x7")] = 0;544    node_buffer_ids[get_node_id(graph, "x8")] = 0;545 546    ggml_gallocr_ptr galloc(ggml_gallocr_new_n(bufts, 2));547    ggml_gallocr_reserve_n(galloc.get(), graph, node_buffer_ids, leaf_buffer_ids);548    ggml_gallocr_alloc_graph(galloc.get(), graph);549 550    check_all_allocated(graph);551    check_no_overlap(graph);552    check_max_size(ctx);553    GGML_ASSERT(backend_a.context->allocated_total() <= 32 + 32 + 24);554    GGML_ASSERT(backend_b.context->allocated_total() <= 32 + 24);555}556 557static void test_buffer_size_zero() {558    dummy_backend backend_a    = dummy_backend_init(SIZE_MAX);559    dummy_backend backend_b    = dummy_backend_init(SIZE_MAX);560    auto [ctx, graph, ctx_ptr] = make_context();561 562    ggml_tensor * x[2];563    x[0] = make_input_with_size(ctx, 16);564    x[1] = ggml_scale(ctx, x[0], 2.0f);565 566    ggml_set_output(x[1]);567    ggml_build_forward_expand(graph, x[1]);568 569    int leaf_buffer_ids[1] = { 0 };570    int node_buffer_ids[1] = { 0 };571 572    ggml_backend_buffer_type_t bufts[2] = { &backend_a.buffer_type, &backend_b.buffer_type };573    ggml_gallocr_ptr           galloc   = ggml_gallocr_ptr(ggml_gallocr_new_n(bufts, 2));574    bool                       res1     = ggml_gallocr_reserve_n(galloc.get(), graph, node_buffer_ids, leaf_buffer_ids);575    bool                       res2     = ggml_gallocr_alloc_graph(galloc.get(), graph);576    GGML_ASSERT(res1 && res2);577 578    check_all_allocated(graph);579    GGML_ASSERT(backend_a.context->allocated_total() == 16);580    GGML_ASSERT(backend_b.context->allocated_total() == 0);581}582 583// Test re-using gallocr for a different graph. The new graph has the same584// total size, but one of the chunks is larger, so reallocation is required.585static void test_reallocation() {586    dummy_backend    backend = dummy_backend_init(32, /*align*/ 4);587    ggml_gallocr_ptr galloc;588    {589        auto [ctx, graph, ctx_ptr] = make_context();590        ggml_tensor * x[4];591        x[0] = make_input_with_size(ctx, 24);592        x[1] = make_input_with_size(ctx, 16);593        x[2] = ggml_view_1d(ctx, x[0], 4, 0);594        x[3] = ggml_add(ctx, x[2], x[1]);595        assign_names(ctx);596 597        galloc = allocate_graph(graph, x[3], &backend.buffer_type);598        check_all_allocated(graph);599        GGML_ASSERT(backend.context->allocated_total() == 40);600    }601    {602        auto [ctx, graph, ctx_ptr] = make_context();603        ggml_tensor * x[3];604        x[0] = make_input_with_size(ctx, 20);605        x[1] = make_input_with_size(ctx, 20);606        x[2] = ggml_add(ctx, x[0], x[1]);607        assign_names(ctx);608        ggml_set_output(x[2]);609        ggml_build_forward_expand(graph, x[2]);610 611        bool result = ggml_gallocr_alloc_graph(galloc.get(), graph);612        GGML_ASSERT(result);613        check_all_allocated(graph);614        GGML_ASSERT(backend.context->allocated_total() == 40);615    }616}617 618static void test_backend_graph_optimize(ggml_backend_t, ggml_cgraph * graph, ggml_backend_graph_optimize_params * params) {619    GGML_ASSERT(graph->n_nodes == 3);620    params->add_alloc_dep(params->user_data, graph->nodes[0], graph->nodes[2]);621}622 623static bool graph_reuses_allocation(bool add_alloc_dep) {624    auto [ctx, graph, ctx_ptr] = make_context();625 626    ggml_tensor * x[4];627    x[0] = make_input_with_size(ctx, 16);628    x[1] = ggml_scale(ctx, x[0], 2.0f);629    x[2] = ggml_scale(ctx, x[1], 2.0f);630    x[3] = ggml_scale(ctx, x[2], 2.0f);631 632    ggml_set_output(x[3]);633    ggml_build_forward_expand(graph, x[3]);634 635    dummy_backend backend = dummy_backend_init(SIZE_MAX);636    if (add_alloc_dep) {637        backend.context->backend.iface.graph_optimize = test_backend_graph_optimize;638    }639 640    ggml_backend_t             backend_ptr = &backend.context->backend;641    ggml_backend_buffer_type_t buft        = &backend.buffer_type;642    ggml_backend_sched_ptr     sched(ggml_backend_sched_new(&backend_ptr, &buft, 1, 8, false, true));643    GGML_ASSERT(ggml_backend_sched_alloc_graph(sched.get(), graph));644 645    return x[1]->data == x[2]->data;646}647 648static void test_graph_optimize_alloc_dep() {649    GGML_ASSERT(graph_reuses_allocation(false));650    GGML_ASSERT(!graph_reuses_allocation(true));651}652 653static void run(const char * name, void (*f)()) {654    printf("%s ", name);655    fflush(stdout);656    f();657    printf("PASSED\n");658}659 660int main() {661    run("test_max_size_too_many_tensors", test_max_size_too_many_tensors);662    run("test_max_size_tensor_too_large", test_max_size_tensor_too_large);663    run("test_tensor_larger_than_max_size", test_tensor_larger_than_max_size);664    run("test_not_enough_chunks", test_not_enough_chunks);665    run("test_fill_leftover_space", test_fill_leftover_space);666    run("test_view_inplace", test_view_inplace);667    run("test_reuse_and_free", test_reuse_and_free);668    run("test_merge_free_block(32)", []() { test_merge_free_block(32); });669    run("test_merge_free_block(SIZE_MAX)", []() { test_merge_free_block(SIZE_MAX); });670    run("test_prefer_already_allocated_memory", test_prefer_already_allocated_memory);671    run("test_multiple_buffer_types", test_multiple_buffer_types);672    run("test_buffer_size_zero", test_buffer_size_zero);673    run("test_reallocation", test_reallocation);674    run("test_graph_optimize_alloc_dep", test_graph_optimize_alloc_dep);675    return 0;676}677