kat33/llama.cpp

1import os   # to check if file exists2import sys  # to flush stdout3import markdown # to render answer4 5import gradio as gr6#import transformers7#from transformers import pipeline8from llama_cpp import Llama9from huggingface_hub import hf_hub_download10 11model_repo="TheBloke/Nous-Hermes-13B-GGML"12model_filename="nous-hermes-13b.ggmlv3.q4_K_S.bin"13 14#model="TheBloke/Nous-Hermes-13B-GGML"15#model="https://huggingface.co/TheBloke/Nous-Hermes-13B-GGML/resolve/main/nous-hermes-13b.ggmlv3.q4_K_S.bin"16 17def download_model():18    # See https://github.com/OpenAccess-AI-Collective/ggml-webui/blob/main/tabbed.py19    file_path="/home/user/.cache/huggingface/hub/models--TheBloke--Nous-Hermes-13B-GGML/snapshots/f1a48f90a07550e1ba30e347b2be69d4fa5e393b/nous-hermes-13b.ggmlv3.q4_K_S.bin"20    if os.path.exists(file_path):21        return file_path22    else:23        print("Downloading model...")24        sys.stdout.flush()25        file = hf_hub_download(26                repo_id=model_repo, filename=model_filename27        )28        print("Downloaded " + file)29        return file30 31def question_answer(context, question, max_tokens):32    mfile=download_model()33    # structure the prompt to make it easier for the ai34    question1="\"\"\"\n" + question + "\n\"\"\"\n"35    text=context + "\n\nQuestion: " + question1 + "\nPlease use markdown formatting for answer. \nAnswer:\n" 36    llm = Llama(model_path=mfile)37    output = llm(text, max_tokens=max_tokens, stop=["### Response"], echo=True)38    print(output)39 40    # remove the context and leave only the answer41    answer=output['choices'][0]['text']42    answer = answer.replace(text, "", 1)43    44    # render the markdown and return the html and question45    html_answer = markdown.markdown(answer)46    return question, html_answer47    '''48            Output is of the form:49            {50              "id": "cmpl-xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx",51              "object": "text_completion",52              "created": 1679561337,53              "model": "./models/7B/ggml-model.bin",54              "choices": [55                    {56                      "text": "Q: Name the planets in the solar system? A: Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, Neptune and Pluto.",57                      "index": 0,58                      "logprobs": None,59                      "finish_reason": "stop"60                    }61                ],62              "usage": {63                "prompt_tokens": 14,64                "completion_tokens": 28,65                "total_tokens": 4266              }67            }68    '''69    70    # old transformers code71    #generator = pipeline(model=model, device_map="auto")72    #return generator(text)73 74 75app=gr.Interface(fn=question_answer, inputs=["text", "text",gr.Slider(33, 2333)], outputs=["textbox", "html"])76app.launch()77