Skizzy-create/Ona-quantized-LLAMA

1import os2import urllib.request3import gradio as gr4from llama_cpp import Llama5 6 7def download_file(file_link, filename):8    # Checks if the file already exists before downloading9    if not os.path.isfile(filename):10        urllib.request.urlretrieve(file_link, filename)11        print("File downloaded successfully.")12    else:13        print("File already exists.")14 15 16# Dowloading GGML model from HuggingFace17ggml_model_path = "https://huggingface.co/TheBloke/Llama-2-13B-chat-GGML/resolve/main/llama-2-13b-chat.ggmlv3.q6_K.bin"18filename = "llama-2-13b-chat.ggmlv3.q6_K.bin"19 20download_file(ggml_model_path, filename)21 22 23llm = Llama(model_path=filename, n_ctx=2000, n_batch=1500)24 25 26def generate_text(prompt="Who is the CEO of Apple?"):27    output = llm(28        prompt,29        max_tokens=1000,30        temperature=0.1,31        top_p=0.5,32        echo=False,33        stop=["#"],34    )35    output_text = output["choices"][0]["text"].strip()36 37    # Remove Prompt Echo from Generated Text38    cleaned_output_text = output_text.replace(prompt, "")39    return cleaned_output_text40 41 42description = "Quantized LLAMA 2 13B"43 44examples = [45    ["What is the capital of France?", "The capital of France is Paris."],46    [47        "Who wrote the novel 'Pride and Prejudice'?",48        "The novel 'Pride and Prejudice' was written by Jane Austen.",49    ],50    ["What is the square root of 64?", "The square root of 64 is 8."],51]52 53gradio_interface = gr.Interface(54    fn=generate_text,55    inputs="text",56    outputs="text",57    examples=examples,58    title="Quantized LLAMA 2 13B",59)60gradio_interface.launch()