Felipe97/llama-cpp-compiled

1#include "ggml-backend.h"2#include "ggml-rpc.h"3#ifdef _WIN324#  define NOMINMAX5#  define DIRECTORY_SEPARATOR '\\'6#  include <windows.h>7#  include <fcntl.h>8#  include <io.h>9#else10#  define DIRECTORY_SEPARATOR '/'11#  include <unistd.h>12#  include <sys/stat.h>13#endif14#include <algorithm>15#include <clocale>16#include <codecvt>17#include <filesystem>18#include <regex>19#include <stdio.h>20#include <string>21#include <thread>22#include <vector>23 24#if defined(__linux__)25#include <sys/types.h>26#include <pwd.h>27#endif28 29// NOTE: this is copied from common.cpp to avoid linking with libcommon30#ifdef _WIN3231static std::wstring utf8_to_wstring(const std::string & str) {32    if (str.empty()) {33        return std::wstring();34    }35 36    int size = MultiByteToWideChar(CP_UTF8, 0, str.c_str(), (int)str.size(), NULL, 0);37 38    if (size <= 0) {39        return std::wstring();40    }41 42    std::wstring wstr(size, 0);43    MultiByteToWideChar(CP_UTF8, 0, str.c_str(), (int)str.size(), &wstr[0], size);44 45    return wstr;46}47#endif48 49// NOTE: this is copied from common.cpp to avoid linking with libcommon50// returns true if successful, false otherwise51static bool fs_create_directory_with_parents(const std::string & path) {52#ifdef _WIN3253    std::wstring wpath = utf8_to_wstring(path);54 55    // if the path already exists, check whether it's a directory56    const DWORD attributes = GetFileAttributesW(wpath.c_str());57    if ((attributes != INVALID_FILE_ATTRIBUTES) && (attributes & FILE_ATTRIBUTE_DIRECTORY)) {58        return true;59    }60 61    size_t pos_slash = 0;62 63    // process path from front to back, procedurally creating directories64    while ((pos_slash = path.find('\\', pos_slash)) != std::string::npos) {65        const std::wstring subpath = wpath.substr(0, pos_slash);66 67        pos_slash += 1;68 69        // skip the drive letter, in some systems it can return an access denied error70        if (subpath.length() == 2 && subpath[1] == ':') {71            continue;72        }73 74        const bool success = CreateDirectoryW(subpath.c_str(), NULL);75 76        if (!success) {77            const DWORD error = GetLastError();78 79            // if the path already exists, ensure that it's a directory80            if (error == ERROR_ALREADY_EXISTS) {81                const DWORD attributes = GetFileAttributesW(subpath.c_str());82                if (attributes == INVALID_FILE_ATTRIBUTES || !(attributes & FILE_ATTRIBUTE_DIRECTORY)) {83                    return false;84                }85            } else {86                return false;87            }88        }89    }90 91    return true;92#else93    // if the path already exists, check whether it's a directory94    struct stat info;95    if (stat(path.c_str(), &info) == 0) {96        return S_ISDIR(info.st_mode);97    }98 99    size_t pos_slash = 1; // skip leading slashes for directory creation100 101    // process path from front to back, procedurally creating directories102    while ((pos_slash = path.find('/', pos_slash)) != std::string::npos) {103        const std::string subpath = path.substr(0, pos_slash);104        struct stat info;105 106        // if the path already exists, ensure that it's a directory107        if (stat(subpath.c_str(), &info) == 0) {108            if (!S_ISDIR(info.st_mode)) {109                return false;110            }111        } else {112            // create parent directories113            const int ret = mkdir(subpath.c_str(), 0755);114            if (ret != 0) {115                return false;116            }117        }118 119        pos_slash += 1;120    }121 122    return true;123#endif // _WIN32124}125 126// NOTE: this is copied from common.cpp to avoid linking with libcommon127static std::string fs_get_cache_directory() {128    std::string cache_directory = "";129    auto ensure_trailing_slash = [](std::string p) {130        // Make sure to add trailing slash131        if (p.back() != DIRECTORY_SEPARATOR) {132            p += DIRECTORY_SEPARATOR;133        }134        return p;135    };136    if (getenv("LLAMA_CACHE")) {137        cache_directory = std::getenv("LLAMA_CACHE");138    } else {139#if defined(__linux__) || defined(__FreeBSD__) || defined(_AIX) || \140    defined(__OpenBSD__) || defined(__NetBSD__)141        if (std::getenv("XDG_CACHE_HOME")) {142            cache_directory = std::getenv("XDG_CACHE_HOME");143        } else if (std::getenv("HOME")) {144            cache_directory = std::getenv("HOME") + std::string("/.cache/");145        } else {146#if defined(__linux__)147            /* no $HOME is defined, fallback to getpwuid */148            struct passwd *pw = getpwuid(getuid());149            if ((!pw) || (!pw->pw_dir)) {150                throw std::runtime_error("Failed to find $HOME directory");151            }152 153            cache_directory = std::string(pw->pw_dir) + std::string("/.cache/");154#else /* defined(__linux__) */155            throw std::runtime_error("Failed to find $HOME directory");156#endif /* defined(__linux__) */157        }158#elif defined(__APPLE__)159        cache_directory = std::getenv("HOME") + std::string("/Library/Caches/");160#elif defined(_WIN32)161        cache_directory = std::getenv("LOCALAPPDATA");162#elif defined(__EMSCRIPTEN__)163        GGML_ABORT("not implemented on this platform");164#else165#  error Unknown architecture166#endif167        cache_directory = ensure_trailing_slash(cache_directory);168        cache_directory += "llama.cpp";169    }170    return ensure_trailing_slash(cache_directory);171}172 173struct rpc_server_params {174    std::string              host        = "127.0.0.1";175    int                      port        = 50052;176    bool                     use_cache   = false;177    int                      n_threads   = std::max(1U, std::thread::hardware_concurrency()/2);178    std::vector<std::string> devices;179};180 181static void print_usage(int /*argc*/, char ** argv, rpc_server_params params) {182    fprintf(stderr, "Usage: %s [options]\n\n", argv[0]);183    fprintf(stderr, "options:\n");184    fprintf(stderr, "  -h, --help                       show this help message and exit\n");185    fprintf(stderr, "  -t, --threads N                  number of threads for the CPU device (default: %d)\n", params.n_threads);186    fprintf(stderr, "  -d, --device <dev1,dev2,...>     comma-separated list of devices\n");187    fprintf(stderr, "  -H, --host HOST                  host to bind to (default: %s)\n", params.host.c_str());188    fprintf(stderr, "  -p, --port PORT                  port to bind to (default: %d)\n", params.port);189    fprintf(stderr, "  -c, --cache                      enable local file cache\n");190    fprintf(stderr, "\n");191}192 193static bool rpc_server_params_parse(int argc, char ** argv, rpc_server_params & params) {194    std::string arg;195    for (int i = 1; i < argc; i++) {196        arg = argv[i];197        if (arg == "-H" || arg == "--host") {198            if (++i >= argc) {199                return false;200            }201            params.host = argv[i];202        } else if (arg == "-t" || arg == "--threads") {203            if (++i >= argc) {204                return false;205            }206            params.n_threads = std::stoi(argv[i]);207            if (params.n_threads <= 0) {208                fprintf(stderr, "error: invalid number of threads: %d\n", params.n_threads);209                return false;210            }211        } else if (arg == "-d" || arg == "--device") {212            if (++i >= argc) {213                return false;214            }215            const std::regex regex{ R"([,/]+)" };216            std::string dev_str = argv[i];217            std::sregex_token_iterator iter(dev_str.begin(), dev_str.end(), regex, -1);218            std::sregex_token_iterator end;219            for ( ; iter != end; ++iter) {220                try {221                    params.devices.push_back(*iter);222                } catch (const std::exception & ) {223                    fprintf(stderr, "error: invalid device: %s\n", iter->str().c_str());224                    return false;225                }226            }227        } else if (arg == "-p" || arg == "--port") {228            if (++i >= argc) {229                return false;230            }231            params.port = std::stoi(argv[i]);232            if (params.port <= 0 || params.port > 65535) {233                return false;234            }235        } else if (arg == "-c" || arg == "--cache") {236            params.use_cache = true;237        } else if (arg == "-h" || arg == "--help") {238            print_usage(argc, argv, params);239            exit(0);240        } else {241            fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());242            print_usage(argc, argv, params);243            exit(0);244        }245    }246    return true;247}248 249static std::vector<ggml_backend_dev_t> get_devices(const rpc_server_params & params) {250    std::vector<ggml_backend_dev_t> devices;251    if (!params.devices.empty()) {252        for (auto device : params.devices) {253            ggml_backend_dev_t dev = ggml_backend_dev_by_name(device.c_str());254            if (dev) {255                devices.push_back(dev);256            } else {257                fprintf(stderr, "error: unknown device: %s\n", device.c_str());258                fprintf(stderr, "available devices:\n");259                for (size_t i = 0; i < ggml_backend_dev_count(); i++) {260                    auto * dev = ggml_backend_dev_get(i);261                    size_t free, total;262                    ggml_backend_dev_memory(dev, &free, &total);263                    printf("  %s: %s (%zu MiB, %zu MiB free)\n", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev), total / 1024 / 1024, free / 1024 / 1024);264                }265                return {};266            }267        }268    }269 270    // Try non-CPU devices first271    if (devices.empty()) {272        for (size_t i = 0; i < ggml_backend_dev_count(); i++) {273            ggml_backend_dev_t dev = ggml_backend_dev_get(i);274            enum ggml_backend_dev_type dev_type = ggml_backend_dev_type(dev);275            if (dev_type != GGML_BACKEND_DEVICE_TYPE_CPU && dev_type != GGML_BACKEND_DEVICE_TYPE_ACCEL) {276                devices.push_back(dev);277            }278        }279    }280 281    // If there are no accelerators, fallback to CPU device282    if (devices.empty()) {283        ggml_backend_dev_t dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);284        if (dev) {285            devices.push_back(dev);286        }287    }288 289    return devices;290}291 292int main(int argc, char * argv[]) {293    std::setlocale(LC_NUMERIC, "C");294 295    ggml_backend_load_all();296 297    rpc_server_params params;298    if (!rpc_server_params_parse(argc, argv, params)) {299        fprintf(stderr, "Invalid parameters\n");300        return 1;301    }302 303    if (params.host != "127.0.0.1") {304        fprintf(stderr, "\n");305        fprintf(stderr, "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n");306        fprintf(stderr, "WARNING: Host ('%s') is != '127.0.0.1'\n", params.host.c_str());307        fprintf(stderr, "         Never expose the RPC server to an open network!\n");308        fprintf(stderr, "         This is an experimental feature and is not secure!\n");309        fprintf(stderr, "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n");310        fprintf(stderr, "\n");311    }312 313    auto devices = get_devices(params);314    if (devices.empty()) {315        fprintf(stderr, "No devices found\n");316        return 1;317    }318    std::string endpoint = params.host + ":" + std::to_string(params.port);319    const char * cache_dir = nullptr;320    std::string cache_dir_str;321    if (params.use_cache) {322        cache_dir_str = fs_get_cache_directory() + "rpc" + DIRECTORY_SEPARATOR;323        if (!fs_create_directory_with_parents(cache_dir_str)) {324            fprintf(stderr, "Failed to create cache directory: %s\n", cache_dir_str.c_str());325            return 1;326        }327        cache_dir = cache_dir_str.c_str();328    }329 330    ggml_backend_reg_t reg = ggml_backend_reg_by_name("RPC");331    if (!reg) {332        fprintf(stderr, "Failed to find RPC backend\n");333        return 1;334    }335 336    auto start_server_fn = (decltype(ggml_backend_rpc_start_server)*) ggml_backend_reg_get_proc_address(reg, "ggml_backend_rpc_start_server");337    if (!start_server_fn) {338        fprintf(stderr, "Failed to obtain RPC backend start server function\n");339        return 1;340    }341 342    start_server_fn(endpoint.c_str(), cache_dir, params.n_threads, devices.size(), devices.data());343    return 0;344}345