cesarams/llama-cpp-api2

1# Importing libraries2from llama_cpp import Llama3from time import time4import gradio as gr5import psutil6 7# Initing things                8print("! INITING LLAMA MODEL !")9llm = Llama(model_path="./model.bin")                               # LLaMa model10llama_model_name = "Vikhrmodels/Vikhr-Qwen-2.5-1.5B-Instruct-GGUF"  # This is just for indication in "three dots menu" 11print("! INITING DONE !")12 13# Preparing things to work14title = "llama.cpp API"15desc = '''<h1>Hello, world!</h1>16This is showcase how to make own server with any Llama based model using llama_cpp.<br>17I'm using here 1.5b model just for example. Also here's only CPU power.<br>18But you can use GPU power as well!<br><br>19<h1>How to GPU?</h1>20Change <code>`CMAKE_ARGS="-DLLAMA_BLAS=ON -DLLAMA_BLAS_VENDOR=OpenBLAS`</code> in Dockerfile on <code>`CMAKE_ARGS="-DLLAMA_CUBLAS=on"`</code>. Also you can try <code>`DLLAMA_CLBLAST`</code> or <code>`DLLAMA_METAL`</code>.<br><br>21<h1>How to test it on own machine?</h1>22You can install Docker, build image and run it. I made <code>`run-docker.sh`</code> for ya. To stop container run <code>`docker ps`</code>, find name of container and run <code>`docker stop _dockerContainerName_`</code><br>23Or you can once follow steps in Dockerfile and try it on your machine, not in Docker.<br>24<br>''' + f"Memory used: {psutil.virtual_memory()[2]}<br>" + '''25Powered by <a href="https://github.com/abetlen/llama-cpp-python">llama-cpp-python</a> and <a href="https://www.gradio.app/">Gradio</a>.<br><br>'''26 27# Loading prompt28with open('system.prompt', 'r', encoding='utf-8') as f:29    prompt = f.read()30with open('system.message', 'r', encoding='utf-8') as f:31    system_message = f.read()32 33def generate_answer(request: str, max_tokens: int = 256, custom_prompt: str = None):34    t0 = time()35    logs = f"Request: {request}\nMax tokens: {max_tokens}\nCustom prompt: {custom_prompt}\n"36    try:37        maxTokens = max_tokens if 16 <= max_tokens <= 256 else 6438        userPrompt = prompt.replace("{prompt}", request)39        userPrompt = userPrompt.replace(40            "{system_message}",41            custom_prompt if isinstance(custom_prompt, str) and len(custom_prompt.strip()) > 1 and custom_prompt.strip() not in ['', None, ' '] else system_message42        )43        logs += f"\nFinal prompt: {userPrompt}\n"44    except:45        return "Not enough data! Check that you passed all needed data.", logs46    47    try:48        # this shitty fix will be until i willnt figure out why sometimes there is empty output49        counter = 150        while counter <= 3:51            logs += f"Attempt {counter} to generate answer...\n"52            output = llm(userPrompt, max_tokens=maxTokens, stop=["<|im_end|>", "<|end_of_turn|>"], echo=False)53            text = output["choices"][0]["text"]54            if len(text.strip()) > 1 and text.strip() not in ['', None, ' ']:55                break56            counter += 157        logs += f"Final attempt: {counter}\n"58        if len(text.strip()) <= 1 or text.strip() in ['', None, ' ']:59            logs += f"Generated and aborted: {text}"60            text = "Sorry, but something went wrong while generating answer. Try again or fix code. If you are maintainer of this space, look into logs."61        62        logs += f"\nFinal: '''{text}'''"63        logs += f"\n\nTime spent: {time()-t0}"64        return text, logs65    except Exception as e:66        logs += str(e)67        logs += f"\n\nTime spent: {time()-t0}"68        return "Oops! Internal server error. Check the logs of space/instance.", logs69 70print("! LOAD GRADIO INTERFACE !")71demo = gr.Interface(72    fn=generate_answer,73    inputs=[74        gr.components.Textbox(label="Input"),75        gr.components.Number(value=256),76        gr.components.Textbox(label="Custom system prompt"),77    ],78    outputs=[79        gr.components.Textbox(label="Output"),80        gr.components.Textbox(label="Logs")81    ],82    title=title,83    description=desc,84    allow_flagging='never'85)86demo.queue()87print("! LAUNCHING GRADIO !")88demo.launch(server_name="0.0.0.0")