Felipe97/llama-cpp-compiled

1#include "cli-context.h"2#include "cli-ui.h"3 4#include "arg.h"5#include "base64.hpp"6#include "log.h"7#include "console.h"8 9#include "json.h"10 11#include <algorithm>12#include <cctype>13#include <filesystem>14#include <fstream>15#include <map>16#include <set>17 18using json = common_json;19 20struct cli_context_impl {21    json messages      = json::array();22    json pending_media = json::array(); // staged multimodal content parts23};24 25cli_context::cli_context(const common_params & params) : params(params), impl(new cli_context_impl()) {}26 27cli_context::~cli_context() {28    shutdown();29}30 31std::atomic<bool> & cli_context::interrupted() {32    static std::atomic<bool> flag = false;33    return flag;34}35 36static bool should_stop() {37    return cli_context::interrupted().load();38}39 40static constexpr size_t FILE_GLOB_MAX_RESULTS = 100;41 42const char * LLAMA_ASCII_LOGO = R"(43▄▄ ▄▄44██ ██45██ ██  ▀▀█▄ ███▄███▄  ▀▀█▄    ▄████ ████▄ ████▄46██ ██ ▄█▀██ ██ ██ ██ ▄█▀██    ██    ██ ██ ██ ██47██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀48                                    ██    ██49                                    ▀▀    ▀▀50)";51 52// number of values an arg consumes on the command line53static int arg_num_values(const common_arg & opt) {54    if (opt.value_hint_2 != nullptr) {55        return 2;56    }57    if (opt.value_hint != nullptr) {58        return 1;59    }60    return 0;61}62 63static std::string format_error_message(const json & err) {64    if (err.contains("error") && err.at("error").is_object()) {65        const auto & e = err.at("error");66        if (e.contains("message") && e.at("message").is_string()) {67            return e.at("message").get<std::string>();68        }69    }70    return err.dump();71}72 73// err is the raw response body of a failed request; it may or may not be JSON74static std::string format_error_message(const std::string & err) {75    json parsed = json::parse_no_throw(err);76    if (!parsed.is_discarded()) {77        return format_error_message(parsed);78    }79    return err;80}81 82static std::string media_type_from_ext(const std::string & fname) {83    std::string ext = std::filesystem::path(fname).extension().string();84    std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c) { return std::tolower(c); });85    if (ext == ".wav" || ext == ".mp3") {86        return "audio";87    }88    if (ext == ".mp4" || ext == ".avi" || ext == ".mkv" || ext == ".mov" || ext == ".webm") {89        return "video";90    }91    return "image";92}93 94bool cli_context::init() {95    ui::init(params);96 97    std::optional<ui::spinner> spinner;98 99    bool use_external_server = !params.server_base.empty();100    if (use_external_server) {101        std::string base = params.server_base;102        while (!base.empty() && base.back() == '/') {103            base.pop_back();104        }105        client.server_base = base;106 107        spinner.emplace("Connecting to server at " + base);108    } else {109        if (params.model.path.empty() && params.model.url.empty() &&110                params.model.hf_repo.empty() && params.model.docker_repo.empty()) {111            ui::show_error(112                "no model specified",113                "use -m <file.gguf> or -hf <user/repo> to run a local model,\n"114                "or --server-base <url> to connect to a running llama-server"115            );116            return false;117        }118 119        spinner.emplace("\n\nLoading model...");120 121        server.emplace();122        if (!server->start(params)) {123            ui::show_error("server start failed");124            return false;125        }126        if (!server->wait_ready(should_stop)) {127            if (!should_stop()) {128                ui::show_error("the server exited before becoming ready");129            }130            return false;131        }132        client.server_base = server->address();133    }134 135    // for --server-base this is the main availability check; for a spawned136    // server it is a cheap sanity check on top of the ready signal137    auto is_aborted = [this]() {138        return should_stop() || (server && !server->alive());139    };140    bool healthy = false;141    try {142        healthy = client.wait_health(is_aborted);143    } catch (const std::exception & e) {144        client.last_error = e.what();145    }146    if (!healthy) {147        if (!should_stop()) {148            ui::show_error(client.last_error);149        }150        return false;151    }152 153    if (use_external_server) {154        spinner.reset();155        try {156            if (!list_and_ask_models()) {157                return false;158            }159        } catch (const common_json_error & e) {160            ui::show_error(e.what());161            ui::show_message("This might be caused by an incorrect server-base endpoint URL");162            return false;163        } catch (const std::exception & e) {164            ui::show_error(e.what());165            return false;166        }167 168        // restore the spinner for the next step169        spinner.emplace("Waiting for server...");170    }171 172    fetch_server_props();173 174    if (!params.out_file.empty()) {175        output_file.emplace(params.out_file);176        if (!output_file->is_open()) {177            ui::show_error(string_format("failed to open output file '%s'", params.out_file.c_str()));178            return false;179        }180    }181 182    return true;183}184 185void cli_context::fetch_server_props() {186    try {187        json props = json::parse(client.get("/props"));188        model_name = props.value("model_alias", "");189        if (model_name.empty()) {190            const std::string path = props.value("model_path", "");191            if (!path.empty()) {192                model_name = std::filesystem::path(path).filename().string();193            }194        }195        model_ftype = props.value("model_ftype", "");196        build_info = props.value("build_info", "");197        if (props.contains("modalities") && props.at("modalities").is_object()) {198            const auto & modalities = props.at("modalities");199            has_vision = modalities.value("vision", false);200            has_audio  = modalities.value("audio", false);201            has_video  = modalities.value("video", false);202        }203    } catch (const std::exception & e) {204        // /props can be disabled on remote servers; not fatal205        LOG_DBG("failed to fetch /props: %s\n", e.what());206    }207}208 209bool cli_context::list_and_ask_models() {210    json resp = json::parse(client.get("/v1/models"));211    if (!resp.contains("data") || !resp.at("data").is_array()) {212        throw std::runtime_error("invalid response from /v1/models");213    }214    std::vector<std::string> models;215    std::vector<std::string> models_display;216    for (const auto & m : resp.at("data")) {217        if (!m.contains("id") || !m.at("id").is_string()) {218            continue;219        }220        std::string name = m.at("id").get<std::string>();221        std::string display = name;222        if (m.contains("aliases") && m.at("aliases").is_array()) {223            std::vector<std::string> aliases;224            for (const auto & a : m.at("aliases")) {225                if (a.is_string()) {226                    aliases.push_back(a.get<std::string>());227                }228            }229            if (!aliases.empty()) {230                display += " (" + string_join(aliases, ", ") + ")";231            }232        }233        models.push_back(name);234        models_display.push_back(display);235    }236 237    // only one model: use it without asking238    if (models.size() == 1) {239        model_name = models[0];240        client.model = model_name;241        return true;242    }243 244    std::string message = "\nAvailable models:";245    for (size_t i = 0; i < models_display.size(); ++i) {246        message += "\n  " + std::to_string(i + 1) + ". " + models_display[i];247    }248    message += "\n";249    ui::show_message(message);250    std::string selection;251    while (selection.empty()) {252        if (should_stop()) {253            return false;254        }255        ui::user_turn user_turn;256        selection = user_turn.read_input(false, "Select model by number: ");257        if (selection.empty()) {258            continue;259        }260        try {261            size_t idx = std::stoul(selection);262            if (idx > 0 && idx <= models.size()) {263                model_name = models[idx - 1];264                client.model = model_name;265                ui::show_message("Selected model: " + model_name);266                break;267            }268        } catch (...) {269            // ignore270        }271        ui::show_error("Invalid selection. Please enter a valid number.");272        selection.clear();273        continue;274    }275    return true;276}277 278void cli_context::add_system_prompt() {279    if (!params.system_prompt.empty()) {280        impl->messages.push_back({281            {"role",    "system"},282            {"content", params.system_prompt}283        });284    }285}286 287void cli_context::push_user_message(const std::string & text) {288    json content;289    if (impl->pending_media.empty()) {290        content = text;291    } else {292        // multimodal message: media parts first, then the text293        content = impl->pending_media;294        content.push_back({295            {"type", "text"},296            {"text", text}297        });298        impl->pending_media = json::array();299    }300    impl->messages.push_back({301        {"role",    "user"},302        {"content", content}303    });304}305 306bool cli_context::stage_media_file(const std::string & fname, const std::string & type) {307    std::ifstream file(fname, std::ios::binary);308    if (!file) {309        return false;310    }311    std::string data((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());312    std::string encoded = base64::encode(data);313 314    if (type == "audio") {315        std::string ext = std::filesystem::path(fname).extension().string();316        std::transform(ext.begin(), ext.end(), ext.begin(), [](unsigned char c) { return std::tolower(c); });317        impl->pending_media.push_back({318            {"type", "input_audio"},319            {"input_audio", {320                {"data",   encoded},321                {"format", ext == ".mp3" ? "mp3" : "wav"}322            }}323        });324    } else if (type == "video") {325        impl->pending_media.push_back({326            {"type", "input_video"},327            {"input_video", {328                {"data", encoded}329            }}330        });331    } else {332        // the server detects the actual image type from the data333        impl->pending_media.push_back({334            {"type", "image_url"},335            {"image_url", {336                {"url", "data:image/unknown;base64," + encoded}337            }}338        });339    }340    return true;341}342 343void cli_context::write_output_file(const std::string & content) {344    if (output_file) {345        (*output_file) << content;346        output_file->flush();347    }348}349 350bool cli_context::generate_completion(generated_content & content_out, cli_timings & timings) {351    json body = {352        {"messages",          impl->messages},353        {"stream",            true},354        // in order to get timings even when we cancel mid-way355        {"timings_per_token", true},356    };357    if (!client.model.empty()) {358        body["model"] = client.model;359    }360 361    bool stream_error = false;362 363    ui::assistant_turn a;364 365    std::string err = client.post_sse("/v1/chat/completions", body.dump(), should_stop, [&](const std::string & payload) {366        json chunk = json::parse_no_throw(payload);367        if (chunk.is_discarded()) {368            return;369        }370        if (chunk.contains("error")) {371            stream_error = true;372            ui::show_error(format_error_message(chunk));373            return;374        }375        if (chunk.contains("timings")) {376            const auto & t = chunk.at("timings");377            timings.prompt_per_second    = t.value("prompt_per_second",    0.0);378            timings.predicted_per_second = t.value("predicted_per_second", 0.0);379        }380        if (!chunk.contains("choices") || !chunk.at("choices").is_array() || chunk.at("choices").empty()) {381            return;382        }383        const auto & choice = chunk.at("choices").at(0);384        if (!choice.contains("delta")) {385            return;386        }387        const auto & delta = choice.at("delta");388        if (delta.contains("reasoning_content") && delta.at("reasoning_content").is_string()) {389            const std::string text = delta.at("reasoning_content").get<std::string>();390            if (!text.empty()) {391                content_out.reasoning += text;392                a.push(ui::ASSISTANT_DISPLAY_MODE_REASONING, text);393            }394        }395        if (delta.contains("content") && delta.at("content").is_string()) {396            const std::string text = delta.at("content").get<std::string>();397            if (!text.empty()) {398                content_out.content += text;399                a.push(ui::ASSISTANT_DISPLAY_MODE_CONTENT, text);400            }401        }402    });403 404    cli_context::interrupted().store(false);405 406    if (!err.empty()) {407        ui::show_error(format_error_message(err));408        return false;409    }410    return !stream_error;411}412 413int cli_context::run() {414    add_system_prompt();415 416    std::string modalities = "text";417    if (has_vision) {418        modalities += ", vision";419    }420    if (has_audio) {421        modalities += ", audio";422    }423    if (has_video) {424        modalities += ", video";425    }426 427    std::string banner;428    banner += "\n";429    banner += LLAMA_ASCII_LOGO;430    banner += "\n";431    banner += "build      : " + build_info + "\n";432    banner += "model      : " + model_name + "\n";433    if (!model_ftype.empty()) {434        banner += "ftype      : " + model_ftype + "\n";435    }436    banner += "modalities : " + modalities + "\n";437    if (!params.system_prompt.empty()) {438        banner += "using custom system prompt\n";439    }440    banner += "\n";441    banner += "available commands:\n";442    banner += "  /exit or Ctrl+C     stop or exit\n";443    banner += "  /regen              regenerate the last response\n";444    banner += "  /clear              clear the chat history\n";445    banner += "  /read <file>        add a text file\n";446    banner += "  /glob <pattern>     add text files using globbing pattern\n";447    if (has_vision) {448        banner += "  /image <file>       add an image file\n";449    }450    if (has_audio) {451        banner += "  /audio <file>       add an audio file\n";452    }453    if (has_video) {454        banner += "  /video <file>       add a video file\n";455    }456    banner += "\n";457 458    ui::show_message(banner);459 460    // interactive loop461    std::string cur_msg;462 463    auto add_text_file = [&](const std::string & fname) -> bool {464        std::ifstream file(fname, std::ios::binary);465        if (!file) {466            ui::show_error(string_format("file does not exist or cannot be opened: '%s'", fname.c_str()));467            return false;468        }469        std::string content((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());470        cur_msg += "--- File: ";471        cur_msg += fname;472        cur_msg += " ---\n";473        cur_msg += content;474        ui::show_message(string_format("Loaded text from '%s'", fname.c_str()));475        return true;476    };477 478    while (true) {479        std::string buffer;480        {481            ui::user_turn user_turn;482 483            if (params.prompt.empty()) {484                buffer = user_turn.read_input(params.multiline_input);485            } else {486                // process input prompt from args487                for (auto & fname : params.image) {488                    if (!stage_media_file(fname, media_type_from_ext(fname))) {489                        ui::show_error(string_format("file does not exist or cannot be opened: '%s'", fname.c_str()));490                        break;491                    }492                    ui::show_message(string_format("Loaded media from '%s'", fname.c_str()));493                }494                buffer = params.prompt;495                user_turn.echo(buffer);496                params.prompt.clear(); // only use it once497            }498        }499 500        if (should_stop()) {501            cli_context::interrupted().store(false);502            break;503        }504 505        // remove trailing newline506        if (!buffer.empty() && buffer.back() == '\n') {507            buffer.pop_back();508        }509 510        // skip empty messages511        if (buffer.empty()) {512            continue;513        }514 515        bool add_user_msg = true;516 517        // process commands518        if (string_starts_with(buffer, "/exit")) {519            break;520        } else if (string_starts_with(buffer, "/regen")) {521            if (impl->messages.size() >= 2) {522                size_t last_idx = impl->messages.size() - 1;523                impl->messages.erase(last_idx);524                add_user_msg = false;525            } else {526                ui::show_error("No message to regenerate.");527                continue;528            }529        } else if (string_starts_with(buffer, "/clear")) {530            impl->messages.clear();531            add_system_prompt();532 533            impl->pending_media = json::array();534            ui::show_message("Chat history cleared.");535            continue;536        } else if (537                (string_starts_with(buffer, "/image ") && has_vision) ||538                (string_starts_with(buffer, "/audio ") && has_audio) ||539                (string_starts_with(buffer, "/video ") && has_video)) {540            std::string type = buffer.substr(1, 5);541            // just in case (bad copy-paste for example), we strip all trailing/leading spaces542            std::string fname = string_strip(buffer.substr(7));543            if (!stage_media_file(fname, type)) {544                ui::show_error(string_format("file does not exist or cannot be opened: '%s'", fname.c_str()));545                continue;546            }547            ui::show_message(string_format("Loaded media from '%s'", fname.c_str()));548            write_output_file(string_format("User: Added media: %s\n", fname.c_str()));549            continue;550        } else if (string_starts_with(buffer, "/read ")) {551            std::string fname = string_strip(buffer.substr(6));552            add_text_file(fname);553            write_output_file(string_format("User: Added text file: %s\n", fname.c_str()));554            continue;555        } else if (string_starts_with(buffer, "/glob ")) {556            std::error_code ec;557            size_t count = 0;558            auto curdir = std::filesystem::current_path();559            std::string pattern = string_strip(buffer.substr(6));560            std::filesystem::path rel_path;561 562            auto startglob = pattern.find_first_of("![*?");563            if (startglob != std::string::npos && startglob != 0) {564                auto endpath = pattern.substr(0, startglob).find_last_of('/');565                if (endpath != std::string::npos) {566                    std::string rel_pattern = pattern.substr(0, endpath);567#if !defined(_WIN32)568                    if (string_starts_with(rel_pattern, '~')) {569                        const char * home = std::getenv("HOME");570                        if (home && home[0]) {571                            rel_pattern = home + rel_pattern.substr(1);572                        }573                    }574#endif575                    rel_path = rel_pattern;576                    pattern.erase(0, endpath + 1);577                    curdir /= rel_path;578                }579            }580 581            for (const auto & entry : std::filesystem::recursive_directory_iterator(curdir,582                    std::filesystem::directory_options::skip_permission_denied, ec)) {583                if (!entry.is_regular_file()) {584                    continue;585                }586 587                std::string rel = std::filesystem::relative(entry.path(), curdir, ec).string();588                if (ec) {589                    ec.clear();590                    continue;591                }592                std::replace(rel.begin(), rel.end(), '\\', '/');593 594                if (!glob_match(pattern, rel)) {595                    continue;596                }597 598                const std::string full_path = (curdir / rel).string();599                if (!add_text_file(full_path)) {600                    continue;601                }602                write_output_file(string_format("User: Added text file: %s\n", full_path.c_str()));603 604                if (++count >= FILE_GLOB_MAX_RESULTS) {605                    ui::show_error(string_format("Maximum number of globbed files allowed (%zu) reached.", FILE_GLOB_MAX_RESULTS));606                    break;607                }608            }609            continue;610        } else {611            // not a command612            cur_msg += buffer;613        }614 615        // generate response616        if (add_user_msg) {617            push_user_message(cur_msg);618            write_output_file(string_format("User:\n%s\n\n", cur_msg.c_str()));619            cur_msg.clear();620        }621 622        cli_timings timings;623        generated_content content;624        generate_completion(content, timings);625 626        json assistant_msg = {627            {"role",    "assistant"},628            {"content", content.content}629        };630        if (!content.reasoning.empty()) {631            assistant_msg["reasoning_content"] = content.reasoning;632        }633        impl->messages.push_back(std::move(assistant_msg));634 635        if (output_file) {636            std::string out_content = "Assistant:\n";637            if (!content.reasoning.empty()) {638                out_content += "[Start thinking]\n\n";639                out_content += content.reasoning;640                out_content += "[End thinking]\n\n";641            }642            out_content += content.content;643            if (!out_content.empty() && out_content.back() != '\n') {644                out_content += "\n";645            }646            out_content += "\n";647            write_output_file(out_content);648        }649 650        if (params.show_timings) {651            ui::show_info(string_format(652                "\n[ Prompt: %.1f t/s | Generation: %.1f t/s ]",653                timings.prompt_per_second,654                timings.predicted_per_second655            ));656        }657 658        if (params.single_turn) {659            break;660        }661    }662 663    ui::show_message("\n\nExiting...");664 665    return 0;666}667 668void cli_context::shutdown() {669    if (server) {670        server->stop();671        server.reset();672    }673    if (output_file) {674        output_file->close();675        output_file.reset();676    }677}678