sailfish/lightweight-chat

1import gradio as gr2import os3from huggingface_hub import InferenceClient4 5"""6For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference7"""8 9model_name = "meta-llama/Llama-3.2-1B"10huggingface_token = os.getenv("SECRET_ENV_VARIABLE")11#client = InferenceClient(api_key=huggingface_token)12client = InferenceClient(model=model_name, token=huggingface_token)13 14 15def generate_text(16    prompt,17    system_message,18    max_tokens,19    temperature,20    top_p21):22    try:23        print(f"Attempting to generate text for prompt: {prompt[:50]}...")24        25        response = client.text_generation(26            prompt,27            max_new_tokens=max_tokens,28            temperature=temperature,29            top_k=50,30            top_p=top_p,31            do_sample=True32        )33        34        print(f"Generated text: {response[:100]}...")35        return response36    except Exception as e:37        print(f"Error in generate_text: {type(e).__name__}: {str(e)}")38        return f"An error occurred: {type(e).__name__}: {str(e)}"39 40 41 42with gr.Blocks() as demo:43    gr.Markdown("Q&A App")44 45    with gr.Tab("Q&A"):46        Query = gr.Textbox(label="Query")47        generate_button = gr.Button("Ask Query")48        output = gr.Textbox(label="Generated Answer", lines=10)49        50        generate_button.click(generate_text, 51        #inputs=[industry, recipient_role, company_details], 52            inputs=[53                Query,54                gr.Textbox(value="You are a friendly Chatbot.", label="System message"),55                gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),56                gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),57                gr.Slider(58                    minimum=0.1,59                    maximum=1.0,60                    value=0.95,61                    step=0.05,62                    label="Top-p (nucleus sampling)",63                ),64            ],    65        outputs=output)66 67 68 69if __name__ == "__main__":70    demo.launch()71