Felipe97/llama-cpp-compiled

1#include "simple-tokenize.h"2 3std::vector<std::string> simple_tokenize(const std::string & input) {4    std::vector<std::string> result;5    std::string              current;6 7    for (size_t i = 0; i < input.size(); i++) {8        switch (input[i]) {9            case ' ':10            case '\n':11            case '\t':12            case '{':13            case '}':14            case ',':15            case '[':16            case '"':17            case ']':18            case '.':19            case '<':20            case '>':21            case '=':22            case '/':23                if (!current.empty()) {24                    result.push_back(current);25                    current.clear();26                }27            default:;28        }29        current += input[i];30    }31 32    if (!current.empty()) {33        result.push_back(current);34    }35 36    return result;37}38