echodict/llama.cpp

1#!/usr/bin/env bash2 3API_URL="${API_URL:-http://127.0.0.1:8080}"4 5CHAT=(6    "Hello, Assistant."7    "Hello. How may I help you today?"8    "Please tell me the largest city in Europe."9    "Sure. The largest city in Europe is Moscow, the capital of Russia."10)11 12INSTRUCTION="A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions."13 14trim() {15    shopt -s extglob16    set -- "${1##+([[:space:]])}"17    printf "%s" "${1%%+([[:space:]])}"18}19 20trim_trailing() {21    shopt -s extglob22    printf "%s" "${1%%+([[:space:]])}"23}24 25format_prompt() {26    echo -n "${INSTRUCTION}"27    printf "\n### Human: %s\n### Assistant: %s" "${CHAT[@]}" "$1"28}29 30tokenize() {31    curl \32        --silent \33        --request POST \34        --url "${API_URL}/tokenize" \35        --header "Content-Type: application/json" \36        --data-raw "$(jq -ns --arg content "$1" '{content:$content}')" \37    | jq '.tokens[]'38}39 40N_KEEP=$(tokenize "${INSTRUCTION}" | wc -l)41 42chat_completion() {43    PROMPT="$(trim_trailing "$(format_prompt "$1")")"44    DATA="$(echo -n "$PROMPT" | jq -Rs --argjson n_keep $N_KEEP '{45        prompt: .,46        temperature: 0.2,47        top_k: 40,48        top_p: 0.9,49        n_keep: $n_keep,50        n_predict: 256,51        cache_prompt: true,52        stop: ["\n### Human:"],53        stream: true54    }')"55 56    ANSWER=''57 58    while IFS= read -r LINE; do59        if [[ $LINE = data:* ]]; then60            CONTENT="$(echo "${LINE:5}" | jq -r '.content')"61            printf "%s" "${CONTENT}"62            ANSWER+="${CONTENT}"63        fi64    done < <(curl \65        --silent \66        --no-buffer \67        --request POST \68        --url "${API_URL}/completion" \69        --header "Content-Type: application/json" \70        --data-raw "${DATA}")71 72    printf "\n"73 74    CHAT+=("$1" "$(trim "$ANSWER")")75}76 77while true; do78    read -r -e -p "> " QUESTION79    chat_completion "${QUESTION}"80done81