echodict/llama.cpp

1#include "arg.h"2#include "debug.h"3#include "log.h"4#include "common.h"5#include "sampling.h"6#include "llama.h"7#include "ggml.h"8#include "console.h"9#include "chat.h"10#include "mtmd.h"11#include "mtmd-helper.h"12 13#include <vector>14#include <limits.h>15#include <cinttypes>16#include <clocale>17 18#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))19#include <signal.h>20#include <unistd.h>21#elif defined (_WIN32)22#define WIN32_LEAN_AND_MEAN23#ifndef NOMINMAX24#define NOMINMAX25#endif26#include <windows.h>27#include <signal.h>28#endif29 30// volatile, because of signal being an interrupt31static volatile bool g_is_generating = false;32static volatile bool g_is_interrupted = false;33 34/**35 * Please note that this is NOT a production-ready stuff.36 * It is a playground for trying multimodal support in llama.cpp.37 * For contributors: please keep this code simple and easy to understand.38 */39 40static void show_additional_info(int /*argc*/, char ** argv) {41    LOG(42        "Experimental CLI for multimodal\n\n"43        "Usage: %s [options] -m <model> --mmproj <mmproj> --image <image> --audio <audio> -p <prompt>\n\n"44        "  -m and --mmproj are required\n"45        "  -hf user/repo can replace both -m and --mmproj in most cases\n"46        "  --image, --audio and -p are optional, if NOT provided, the CLI will run in chat mode\n"47        "  to disable using GPU for mmproj model, add --no-mmproj-offload\n",48        argv[0]49    );50}51 52#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__)) || defined (_WIN32)53static void sigint_handler(int signo) {54    if (signo == SIGINT) {55        if (g_is_generating) {56            g_is_generating = false;57        } else {58            console::cleanup();59            if (g_is_interrupted) {60                _exit(1);61            }62            g_is_interrupted = true;63        }64    }65}66#endif67 68struct mtmd_cli_context {69    mtmd::context_ptr ctx_vision;70    common_init_result_ptr llama_init;71 72    llama_model       * model;73    llama_context     * lctx;74    const llama_vocab * vocab;75    common_sampler    * smpl;76    llama_batch         batch;77    int                 n_batch;78 79    mtmd::bitmaps bitmaps;80 81    // chat template82    common_chat_templates_ptr tmpls;83    std::vector<common_chat_msg> chat_history;84    bool use_jinja = false;85    // TODO: support for --system-prompt with /clear command86 87    // support for legacy templates (models not having EOT token)88    llama_tokens antiprompt_tokens;89 90    int n_threads    = 1;91    llama_pos n_past = 0;92 93    base_callback_data cb_data;94 95    mtmd_cli_context(common_params & params) : llama_init(common_init_from_params(params)) {96        model = llama_init->model();97        lctx = llama_init->context();98        vocab = llama_model_get_vocab(model);99        smpl = common_sampler_init(model, params.sampling);100        n_threads = params.cpuparams.n_threads;101        batch = llama_batch_init(1, 0, 1); // batch for next token generation102        n_batch = params.n_batch;103 104        if (!model || !lctx) {105            exit(1);106        }107 108        if (!llama_model_chat_template(model, nullptr) && params.chat_template.empty()) {109            LOG_ERR("Model does not have chat template.\n");110            LOG_ERR("  For old llava models, you may need to use '--chat-template vicuna'\n");111            LOG_ERR("  For MobileVLM models, use '--chat-template deepseek'\n");112            LOG_ERR("  For Mistral Small 3.1, use '--chat-template mistral-v7'\n");113            exit(1);114        }115 116        tmpls = common_chat_templates_init(model, params.chat_template);117        use_jinja = params.use_jinja;118        chat_history.clear();119        LOG_INF("%s: chat template example:\n%s\n", __func__, common_chat_format_example(tmpls.get(), params.use_jinja, params.default_template_kwargs).c_str());120 121        init_vision_context(params);122 123        // load antiprompt tokens for legacy templates124        if (params.chat_template == "vicuna") {125            antiprompt_tokens = common_tokenize(lctx, "ASSISTANT:", false, true);126        } else if (params.chat_template == "deepseek") {127            antiprompt_tokens = common_tokenize(lctx, "###", false, true);128        }129    }130 131    ~mtmd_cli_context() {132        llama_batch_free(batch);133        common_sampler_free(smpl);134    }135 136    void init_vision_context(common_params & params) {137        const char * clip_path = params.mmproj.path.c_str();138        mtmd_context_params mparams = mtmd_context_params_default();139        mparams.use_gpu          = params.mmproj_use_gpu;140        mparams.print_timings    = true;141        mparams.n_threads        = params.cpuparams.n_threads;142        mparams.flash_attn_type  = params.flash_attn_type;143        mparams.warmup           = params.warmup;144        mparams.image_min_tokens = params.image_min_tokens;145        mparams.image_max_tokens = params.image_max_tokens;146        if (std::getenv("MTMD_DEBUG_GRAPH") != nullptr) {147            mparams.cb_eval_user_data = &cb_data;148            mparams.cb_eval = common_debug_cb_eval<false>;149        }150        ctx_vision.reset(mtmd_init_from_file(clip_path, model, mparams));151        if (!ctx_vision.get()) {152            LOG_ERR("Failed to load vision model from %s\n", clip_path);153            exit(1);154        }155    }156 157    bool check_antiprompt(const llama_tokens & generated_tokens) {158        if (antiprompt_tokens.empty() || generated_tokens.size() < antiprompt_tokens.size()) {159            return false;160        }161        return std::equal(162            generated_tokens.end() - antiprompt_tokens.size(),163            generated_tokens.end(),164            antiprompt_tokens.begin()165        );166    }167 168    bool load_media(const std::string & fname) {169        mtmd::bitmap bmp(mtmd_helper_bitmap_init_from_file(ctx_vision.get(), fname.c_str()));170        if (!bmp.ptr) {171            return false;172        }173        bitmaps.entries.push_back(std::move(bmp));174        return true;175    }176};177 178static int generate_response(mtmd_cli_context & ctx, int n_predict) {179    llama_tokens generated_tokens;180    for (int i = 0; i < n_predict; i++) {181        if (i > n_predict || !g_is_generating || g_is_interrupted) {182            LOG("\n");183            break;184        }185 186        llama_token token_id = common_sampler_sample(ctx.smpl, ctx.lctx, -1);187        generated_tokens.push_back(token_id);188        common_sampler_accept(ctx.smpl, token_id, true);189 190        if (llama_vocab_is_eog(ctx.vocab, token_id) || ctx.check_antiprompt(generated_tokens)) {191            LOG("\n");192            break; // end of generation193        }194 195        LOG("%s", common_token_to_piece(ctx.lctx, token_id).c_str());196        fflush(stdout);197 198        if (g_is_interrupted) {199            LOG("\n");200            break;201        }202 203        // eval the token204        common_batch_clear(ctx.batch);205        common_batch_add(ctx.batch, token_id, ctx.n_past++, {0}, true);206        if (llama_decode(ctx.lctx, ctx.batch)) {207            LOG_ERR("failed to decode token\n");208            return 1;209        }210    }211 212    std::string generated_text = common_detokenize(ctx.lctx, generated_tokens);213    common_chat_msg msg;214    msg.role    = "assistant";215    msg.content = generated_text;216    ctx.chat_history.push_back(std::move(msg));217 218    return 0;219}220 221static std::string chat_add_and_format(mtmd_cli_context & ctx, common_chat_msg & new_msg) {222    LOG_DBG("chat_add_and_format: new_msg.role='%s', new_msg.content='%s'\n",223        new_msg.role.c_str(), new_msg.content.c_str());224    auto formatted = common_chat_format_single(ctx.tmpls.get(), ctx.chat_history,225        new_msg, new_msg.role == "user",226        ctx.use_jinja);227    ctx.chat_history.push_back(new_msg);228    return formatted;229}230 231static int eval_message(mtmd_cli_context & ctx, common_chat_msg & msg) {232    bool add_bos = ctx.chat_history.empty();233    auto formatted_chat = chat_add_and_format(ctx, msg);234    LOG_DBG("formatted_chat.prompt: %s\n", formatted_chat.c_str());235 236    mtmd_input_text text;237    text.text          = formatted_chat.c_str();238    text.add_special   = add_bos;239    text.parse_special = true;240 241    if (g_is_interrupted) return 0;242 243    mtmd::input_chunks chunks(mtmd_input_chunks_init());244    auto bitmaps_c_ptr = ctx.bitmaps.c_ptr();245    int32_t res = mtmd_tokenize(ctx.ctx_vision.get(),246                        chunks.ptr.get(), // output247                        &text, // text248                        bitmaps_c_ptr.data(),249                        bitmaps_c_ptr.size());250    if (res != 0) {251        LOG_ERR("Unable to tokenize prompt, res = %d\n", res);252        return 1;253    }254 255    ctx.bitmaps.entries.clear();256 257    llama_pos new_n_past;258    if (mtmd_helper_eval_chunks(ctx.ctx_vision.get(),259                ctx.lctx, // lctx260                chunks.ptr.get(), // chunks261                ctx.n_past, // n_past262                0, // seq_id263                ctx.n_batch, // n_batch264                true, // logits_last265                &new_n_past)) {266        LOG_ERR("Unable to eval prompt\n");267        return 1;268    }269 270    ctx.n_past = new_n_past;271 272    LOG("\n");273 274    return 0;275}276 277int main(int argc, char ** argv) {278    std::setlocale(LC_NUMERIC, "C");279 280    ggml_time_init();281 282    common_params params;283 284    common_init();285 286    if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_MTMD, show_additional_info)) {287        return 1;288    }289 290    mtmd_helper_log_set(common_log_default_callback, nullptr);291 292    if (params.mmproj.path.empty()) {293        show_additional_info(argc, argv);294        LOG_ERR("ERR: Missing --mmproj argument\n");295        return 1;296    }297 298    mtmd_cli_context ctx(params);299    LOG_INF("%s: loading model: %s\n", __func__, params.model.path.c_str());300 301    bool is_single_turn = !params.prompt.empty() && !params.image.empty();302 303    int n_predict = params.n_predict < 0 ? INT_MAX : params.n_predict;304 305    // Ctrl+C handling306    {307#if defined (__unix__) || (defined (__APPLE__) && defined (__MACH__))308        struct sigaction sigint_action;309        sigint_action.sa_handler = sigint_handler;310        sigemptyset (&sigint_action.sa_mask);311        sigint_action.sa_flags = 0;312        sigaction(SIGINT, &sigint_action, NULL);313#elif defined (_WIN32)314        auto console_ctrl_handler = +[](DWORD ctrl_type) -> BOOL {315            return (ctrl_type == CTRL_C_EVENT) ? (sigint_handler(SIGINT), true) : false;316        };317        SetConsoleCtrlHandler(reinterpret_cast<PHANDLER_ROUTINE>(console_ctrl_handler), true);318#endif319    }320 321    if (g_is_interrupted) return 130;322 323    auto eval_system_prompt_if_present = [&] {324        if (params.system_prompt.empty()) {325            return 0;326        }327 328        common_chat_msg msg;329        msg.role = "system";330        msg.content = params.system_prompt;331        return eval_message(ctx, msg);332    };333 334    LOG_WRN("WARN: This is an experimental CLI for testing multimodal capability.\n");335    LOG_WRN("      For normal use cases, please use the standard llama-cli\n");336 337    if (eval_system_prompt_if_present()) {338        return 1;339    }340 341    if (is_single_turn) {342        g_is_generating = true;343        if (params.prompt.find(mtmd_default_marker()) == std::string::npos) {344            for (size_t i = 0; i < params.image.size(); i++) {345                // most models require the marker before each image346                // ref: https://github.com/ggml-org/llama.cpp/pull/17616347                params.prompt = mtmd_default_marker() + params.prompt;348            }349        }350 351        common_chat_msg msg;352        msg.role = "user";353        msg.content = params.prompt;354        for (const auto & image : params.image) {355            if (!ctx.load_media(image)) {356                return 1; // error is already printed by libmtmd357            }358        }359        if (eval_message(ctx, msg)) {360            return 1;361        }362        if (!g_is_interrupted && generate_response(ctx, n_predict)) {363            return 1;364        }365 366    } else {367        LOG("\n Running in chat mode, available commands:");368        if (mtmd_support_vision(ctx.ctx_vision.get())) {369            LOG("\n   /image <path>    load an image");370        }371        if (mtmd_support_audio(ctx.ctx_vision.get())) {372            LOG("\n   /audio <path>    load an audio");373        }374        LOG("\n   /clear           clear the chat history");375        LOG("\n   /quit or /exit   exit the program");376        LOG("\n");377 378        std::string content;379 380        while (!g_is_interrupted) {381            g_is_generating = false;382            LOG("\n> ");383            console::set_display(DISPLAY_TYPE_USER_INPUT);384            std::string line;385            console::readline(line, false);386            if (g_is_interrupted) break;387            console::set_display(DISPLAY_TYPE_RESET);388            line = string_strip(line);389            if (line.empty()) {390                continue;391            }392            if (line == "/quit" || line == "/exit") {393                break;394            }395            if (line == "/clear") {396                ctx.n_past = 0;397                ctx.chat_history.clear();398                llama_memory_clear(llama_get_memory(ctx.lctx), true);399                if (eval_system_prompt_if_present()) {400                    return 1;401                }402                LOG("Chat history cleared\n\n");403                continue;404            }405            g_is_generating = true;406            bool is_image = line == "/image" || line.find("/image ") == 0;407            bool is_audio = line == "/audio" || line.find("/audio ") == 0;408            if (is_image || is_audio) {409                if (line.size() < 8) {410                    LOG_ERR("ERR: Missing media filename\n");411                    continue;412                }413                std::string media_path = line.substr(7);414                if (ctx.load_media(media_path)) {415                    LOG("%s %s loaded\n", media_path.c_str(), is_image ? "image" : "audio");416                    content += mtmd_default_marker();417                }418                // else, error is already printed by libmtmd419                continue;420            } else {421                content += line;422            }423            common_chat_msg msg;424            msg.role = "user";425            msg.content = content;426            int ret = eval_message(ctx, msg);427            if (ret) {428                return 1;429            }430            if (g_is_interrupted) break;431            if (generate_response(ctx, n_predict)) {432                return 1;433            }434            content.clear();435        }436    }437    if (g_is_interrupted) LOG("\nInterrupted by user\n");438    LOG("\n\n");439    llama_perf_context_print(ctx.lctx);440    return g_is_interrupted ? 130 : 0;441}442