Felipe97/llama-cpp-compiled

1#include "arg.h"2#include "common.h"3#include "log.h"4#include "llama.h"5 6#include <clocale>7#include <cstdio>8#include <cstring>9#include <fstream>10#include <string>11#include <vector>12#include <iostream>13#include <sstream>14 15#if defined(_WIN32)16#define WIN32_LEAN_AND_MEAN17#include <windows.h>18#endif19 20static void print_usage(int argc, char ** argv) {21    (void) argc;22 23    LOG("\nexample usage:\n");24    LOG("\n  %s -m your_model.gguf -p \"Hello world\"\n", argv[0]);25    LOG("\n  %s -m your_model.gguf -f prompt.txt --ids\n", argv[0]);26    LOG("\n  cat prompt.txt | %s -m your_model.gguf --stdin --show-count\n", argv[0]);27    LOG("\n");28}29 30//31// Function: write_utf8_cstr_to_stdout(const char *) -> <writes to stdout>32//33// writes a string to standard output; taking into account that on Windows34// to display correctly you have to use special handling. Works even if the35// user has not set a unicode code page on a Windows cmd.exe.36//37// In case of invalid UTF-8, invalid_utf8 is set to true on Windows, and something38// a human-readable is written instead.39//40// On non-Windows systems, simply printfs() the string.41static void write_utf8_cstr_to_stdout(const char * str, bool & invalid_utf8) {42        invalid_utf8 = false;43 44#if defined(_WIN32)45        // Are we in a console?46        HANDLE hConsole = GetStdHandle(STD_OUTPUT_HANDLE);47        DWORD dwMode = 0;48 49        // According to Microsoft docs:50        // "WriteConsole fails if it is used with a standard handle that is redirected to a file."51        // Also according to the docs, you can use GetConsoleMode to check for that.52        if (hConsole == INVALID_HANDLE_VALUE || !GetConsoleMode(hConsole, &dwMode)) {53            printf("%s", str);54            return;55        }56 57        // MultiByteToWideChar reports an error if str is empty, don't report58        // them as invalid_utf8.59        if (*str == 0) {60            return;61        }62        int length_needed = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, str, strlen(str), NULL, 0);63        if (length_needed == 0) {64            DWORD err = GetLastError();65            if (err == ERROR_NO_UNICODE_TRANSLATION) {66                invalid_utf8 = true;67                int len = strlen(str);68                printf("<");69                for (int i = 0; i < len; ++i) {70                    if (i > 0) {71                        printf(" ");72                    }73                    printf("%02x", (uint8_t) str[i]);74                }75                printf(">");76                return;77            }78            GGML_ABORT("MultiByteToWideChar() failed in an unexpected way.");79        }80 81        LPWSTR wstr = (LPWSTR) calloc(length_needed+1, sizeof(*wstr));82        GGML_ASSERT(wstr);83 84        MultiByteToWideChar(CP_UTF8, 0, str, strlen(str), wstr, length_needed);85        WriteConsoleW(hConsole, wstr, length_needed, NULL, NULL);86 87        free(wstr);88#else89        // TODO: reporting invalid_utf8 would be useful on non-Windows too.90        // printf will silently just write bad unicode.91        printf("%s", str);92#endif93}94 95int main(int argc, char ** argv) {96    std::setlocale(LC_NUMERIC, "C");97 98    common_params params;99 100    common_init();101 102    if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_TOKENIZE, print_usage)) {103        return 1;104    }105 106    // -f and -p both land in params.prompt; -f also sets prompt_file. -f and -p107    // resolve like the other tools (no mutual exclusion), --stdin takes precedence.108    const bool use_stdin = params.tokenize_stdin;109    const bool use_file  = !params.prompt_file.empty();110 111    // must have some prompt112    if (!use_stdin && !use_file && params.prompt.empty()) {113        LOG_ERR("error: must specify one of: --stdin, --file or --prompt\n");114        return 1;115    }116 117    std::string prompt;118    if (use_file) {119        // read the file verbatim: common's -f handler strips a single trailing120        // newline, but for a tokenizer the input bytes must be preserved exactly121        // (a trailing newline is itself a token). escapes are applied locally122        // to match the behavior of -p/--prompt and --stdin.123        std::ifstream in(params.prompt_file, std::ios::binary);124        if (!in) {125            LOG_ERR("error: could not open file '%s' for reading\n", params.prompt_file.c_str());126            return 1;127        }128        std::stringstream ss;129        ss << in.rdbuf();130        prompt = ss.str();131        if (params.escape) {132            string_process_escapes(prompt);133        }134    } else if (!use_stdin) {135        // -p/--prompt is already escape-processed by common_params_parse()136        // (controlled by --escape/--no-escape), so use it verbatim here.137        prompt = params.prompt;138    }139    // else: we read stdin *after* loading the model (early exit if the140    // model cannot be loaded, which is a nicer user experience)141 142    llama_backend_init();143 144    // load only the vocabulary (no weights), since tokenizing does not need them145    llama_model_params model_params = llama_model_default_params();146    model_params.vocab_only = true;147    llama_model * model = llama_model_load_from_file(params.model.path.c_str(), model_params);148    if (!model) {149        LOG_ERR("error: could not load model from file '%s'.\n", params.model.path.c_str());150        return 1;151    }152 153    const llama_vocab * vocab = llama_model_get_vocab(model);154 155    llama_context_params ctx_params = llama_context_default_params();156    llama_context * ctx = llama_init_from_model(model, ctx_params);157    if (!ctx) {158        LOG_ERR("error: could not create context.\n");159        return 1;160    }161 162    // read entire prompt from stdin?163    if (params.tokenize_stdin) {164        std::stringstream stdin_buffer;165        stdin_buffer << std::cin.rdbuf();166        if (std::cin.fail()) {167            LOG_ERR("error: could not read the entire standard input.\n");168            return 1;169        }170 171        prompt = stdin_buffer.str();172 173        // stdin is not seen by common_params_parse(), so apply escape handling174        // here to match the behavior of -p/--prompt and -f/--file.175        if (params.escape) {176            string_process_escapes(prompt);177        }178    }179 180    const bool model_wants_add_bos = llama_vocab_get_add_bos(vocab);181    const bool add_bos      = model_wants_add_bos && !params.tokenize_no_bos;182    const bool parse_special = params.parse_special;183 184    std::vector<llama_token> tokens;185    tokens = common_tokenize(vocab, prompt, add_bos, parse_special);186 187    if (params.tokenize_ids) {188        printf("[");189    }190 191    for (int i = 0; i < (int) tokens.size(); i++) {192        if (params.tokenize_ids) {193            if (i > 0) {194                printf(", ");195            }196            printf("%d", tokens[i]);197        } else {198            bool invalid_utf8 = false;199            printf("%6d -> '", tokens[i]);200            write_utf8_cstr_to_stdout(common_token_to_piece(ctx, tokens[i]).c_str(), invalid_utf8);201            if (invalid_utf8) {202                printf("' (utf-8 decode failure)\n");203            } else {204                printf("'\n");205            }206        }207    }208 209    if (params.tokenize_ids) {210        printf("]\n");211    }212 213    if (params.tokenize_show_count) {214        printf("Total number of tokens: %zu\n", tokens.size());215    }216 217    // silence valgrind218    llama_free(ctx);219    llama_model_free(model);220 221    return 0;222}223