LVKinyanjui/QueryYourDocs

1import streamlit as st2from huggingface_hub import InferenceClient3from modules.inference.instruct import infer, load_model4import os5 6token = os.environ["HF_TOKEN"]7client = InferenceClient(model="meta-llama/Llama-3.2-1B-Instruct", token=token)8 9st.write("## Ask your Local LLM")10text_input = st.text_input("Query", value="Why is the sky Blue")11submit = st.button("Submit")12 13# @st.cache_resource14# def load_model_cached():15#     return load_model()16 17# model = load_model_cached()18 19if submit:20    # response = infer(model, text_input)21    # response22    output = client.chat.completions.create(23        messages=[24            {"role": "system", "content": "You are a helpful assistant."},25            {"role": "user", "content": text_input},26        ],27        stream=True,28        max_tokens=1024,29    )30    31    for chunk in output:32        st.write(chunk.choices[0].delta.content)