PixelPotentialsAI/Test

1from llama_cpp import Llama2from fastapi import FastAPI3from pydantic import BaseModel4 5 6llm = Llama(7    model_path="Meta-Llama-3-8B-Instruct.Q4_0.gguf",8    n_ctx=8192,  # The max sequence length to use - note that longer sequence lengths require much more resources9    n_threads=3, # The number of CPU threads to use, tailor to your system and the resulting performance10    # chat_format="llama-3",11    12)13 14#Pydantic object15class validation(BaseModel):16    prompt: str17 18#Fast API19app = FastAPI()20 21@app.post("/llm_on_cpu")22async def stream(item: validation):23    res =""24    output = llm(25      f"<|user|>\n{item.prompt}<|end|>\n<|assistant|>",26      max_tokens=512,27      stop=["<|end|>", "<|assistant|>"], 28      # echo=True,  # Whether to echo the prompt29      stream=True,30    )31    # print(output['choices'][0]['text'])32    for x in output:33        print(x['choices'][0]['text'])34        res += x['choices'][0]['text']35    return res #output['choices'][0]['text']36