mikeee/codellama-13b-python-ggml

1"""Run codes."""2# pylint: disable=line-too-long, broad-exception-caught, invalid-name, missing-function-docstring, too-many-instance-attributes, missing-class-docstring3# ruff: noqa: E5014import gc5import os6import platform7import random8import time9from dataclasses import asdict, dataclass10from pathlib import Path11from typing import Optional, Sequence12 13# from types import SimpleNamespace14import gradio as gr15import psutil16from about_time import about_time17from ctransformers import AutoModelForCausalLM18from dl_hf_model import dl_hf_model19from examples_list import examples_list20from loguru import logger21 22url = "https://huggingface.co/TheBloke/CodeLlama-13B-Python-GGML/blob/main/codellama-13b-python.ggmlv3.Q4_K_M.bin"  # 7.87G23 24LLM = None25gc.collect()26 27try:28    logger.debug(f" dl {url}")29    model_loc, file_size = dl_hf_model(url)30    logger.info(f"done load llm {model_loc=} {file_size=}G")31except Exception as exc_:32    logger.error(exc_)33    raise SystemExit(1) from exc_34 35# raise SystemExit(0)36 37# Prompt template: Guanaco38# {past_history}39prompt_template = """You are a helpful assistant. Let's think step by step.40### Human:41{question}42### Assistant:"""43 44# Prompt template: garage-bAInd/Stable-Platypus2-13B45prompt_template = """46### System:47This is a system prompt, please behave and help the user.48 49### Instruction:50 51{question}52 53### Response:54"""55prompt_template = """56[INST] Write code to solve the following coding problem that obeys the constraints and57passes the example test cases. Please wrap your code answer using ```:58{question}59[/INST]60"""61 62# human_prefix = "### Instruction"63# ai_prefix = "### Response"64# stop_list = [f"{human_prefix}:"]65 66_ = psutil.cpu_count(logical=False) - 167cpu_count: int = int(_) if _ else 168logger.debug(f"{cpu_count=}")69 70logger.debug(f"{model_loc=}")71LLM = AutoModelForCausalLM.from_pretrained(72    model_loc,73    model_type="llama",74    threads=cpu_count,75)76 77os.environ["TZ"] = "Asia/Shanghai"78try:79    time.tzset()  # type: ignore # pylint: disable=no-member80except Exception:81    # Windows82    logger.warning("Windows, cant run time.tzset()")83 84 85# ctransformers.Config() default86# Config(top_k=40, top_p=0.95, temperature=0.8,87# repetition_penalty=1.1, last_n_tokens=64, seed=-1,88# batch_size=8, threads=-1, max_new_tokens=256,89# stop=None, stream=False, reset=True,90# context_length=-1, gpu_layers=0)91@dataclass92class GenerationConfig:93    temperature: float = 0.794    top_k: int = 5095    top_p: float = 0.996    repetition_penalty: float = 1.097    max_new_tokens: int = 51298    seed: int = 4299    reset: bool = False100    stream: bool = True101    threads: int = cpu_count102    # stop: list[str] = field(default_factory=lambda: stop_list)103 104# ctransformers\llm.py105@dataclass106class Config:107    # sample108    top_k: int = 40109    top_p: float = 0.95110    temperature: float = 0.8111    repetition_penalty: float = 1.1112    last_n_tokens: int = 64113    seed: int = -1114 115    # eval116    batch_size: int = 8117    threads: int = -1118 119    # generate120    max_new_tokens: int = 512  # 256121    stop: Optional[Sequence[str]] = None122    stream: bool = True  # False123    reset: bool = False  # True124 125    # model126    # context_length: int = -1127    # gpu_layers: int = 0128 129 130def generate(131    question: str,132    llm=LLM,133    # config: GenerationConfig = GenerationConfig(),134    config: Config = Config(),135):136    """Run model inference, will return a Generator if streaming is true."""137    # _ = prompt_template.format(question=question)138    # print(_)139 140    prompt = prompt_template.format(question=question)141 142    return llm(143        prompt,144        **asdict(config),145        # **vars(config),146    )147 148 149# logger.debug(f"{asdict(GenerationConfig())=}")150logger.debug(f"{Config(stream=True)=}")151logger.debug(f"{vars(Config(stream=True))=}")152 153 154def user(user_message, history):155    # return user_message, history + [[user_message, None]]156    if history is None:157        history = []158    history.append([user_message, None])159    return user_message, history  # keep user_message160 161 162def user1(user_message, history):163    # return user_message, history + [[user_message, None]]164    if history is None:165        history = []166    history.append([user_message, None])167    return "", history  # clear user_message168 169 170def bot_(history):171    user_message = history[-1][0]172    resp = random.choice(["How are you?", "I love you", "I'm very hungry"])173    bot_message = user_message + ": " + resp174    history[-1][1] = ""175    for character in bot_message:176        history[-1][1] += character177        time.sleep(0.02)178        yield history179 180    history[-1][1] = resp181    yield history182 183 184def bot(history):185    user_message = ""186    try:187        user_message = history[-1][0]188    except Exception as exc:189        logger.error(exc)190    response = []191 192    logger.debug(f"{user_message=}")193 194    with about_time() as atime:  # type: ignore195        flag = 1196        prefix = ""197        then = time.time()198 199        logger.debug("about to generate")200 201        config = GenerationConfig(reset=True)202        for elm in generate(user_message, config=config):203            if flag == 1:204                logger.debug("in the loop")205                prefix = f"({time.time() - then:.2f}s)\n"206                flag = 0207                print(prefix, end="", flush=True)208                logger.debug(f"{prefix=}")209            print(elm, end="", flush=True)210            # logger.debug(f"{elm}")211 212            response.append(elm)213            history[-1][1] = prefix + "".join(response)214            yield history215 216    _ = (217        f"(time elapsed: {atime.duration_human}, "  # type: ignore218        f"{atime.duration/len(''.join(response)):.2f}s/char)"  # type: ignore219    )220 221    history[-1][1] = "".join(response) + f"\n{_}"222    yield history223 224 225def predict_api(prompt):226    logger.debug(f"{prompt=}")227    try:228        # user_prompt = prompt229        config = GenerationConfig(230            temperature=0.2,231            top_k=10,232            top_p=0.9,233            repetition_penalty=1.0,234            max_new_tokens=512,  # adjust as needed235            seed=42,236            reset=True,  # reset history (cache)237            stream=False,238            # threads=cpu_count,239            # stop=prompt_prefix[1:2],240        )241 242        response = generate(243            prompt,244            config=config,245        )246 247        logger.debug(f"api: {response=}")248    except Exception as exc:249        logger.error(exc)250        response = f"{exc=}"251    # bot = {"inputs": [response]}252    # bot = [(prompt, response)]253 254    return response255 256 257css = """258    .importantButton {259        background: linear-gradient(45deg, #7e0570,#5d1c99, #6e00ff) !important;260        border: none !important;261    }262    .importantButton:hover {263        background: linear-gradient(45deg, #ff00e0,#8500ff, #6e00ff) !important;264        border: none !important;265    }266    .disclaimer {font-variant-caps: all-small-caps; font-size: xx-small;}267    .xsmall {font-size: x-small;}268"""269 270logger.info("start block")271 272with gr.Blocks(273    title=f"{Path(model_loc).name}",274    # theme=gr.themes.Soft(text_size="sm", spacing_size="sm"),275    theme=gr.themes.Glass(text_size="sm", spacing_size="sm"),276    css=css,277) as block:278    # buff_var = gr.State("")279    with gr.Accordion("🎈 Info", open=True):280        gr.Markdown(281            f"""<h5><center>{Path(model_loc).name}</center></h4>282            Doesn't quite work yet -- no output or run forever. Maybe the system prompt is not in order. """,283            elem_classes="xsmall",284        )285 286    # chatbot = gr.Chatbot().style(height=700)  # 500287    chatbot = gr.Chatbot(height=500)288 289    # buff = gr.Textbox(show_label=False, visible=True)290 291    with gr.Row():292        with gr.Column(scale=5):293            msg = gr.Textbox(294                label="Chat Message Box",295                placeholder="Ask me anything (press Shift+Enter or click Submit to send)",296                show_label=False,297                # container=False,298                lines=6,299                max_lines=30,300                show_copy_button=True,301                # ).style(container=False)302            )303        with gr.Column(scale=1, min_width=50):304            with gr.Row():305                submit = gr.Button("Submit", elem_classes="xsmall")306                stop = gr.Button("Stop", visible=True)307                clear = gr.Button("Clear History", visible=True)308    with gr.Row(visible=False):309        with gr.Accordion("Advanced Options:", open=False):310            with gr.Row():311                with gr.Column(scale=2):312                    system = gr.Textbox(313                        label="System Prompt",314                        value=prompt_template,315                        show_label=False,316                        container=False,317                        # ).style(container=False)318                    )319                with gr.Column():320                    with gr.Row():321                        change = gr.Button("Change System Prompt")322                        reset = gr.Button("Reset System Prompt")323 324    with gr.Accordion("Example Inputs", open=True):325        examples = gr.Examples(326            examples=examples_list,327            inputs=[msg],328            examples_per_page=40,329        )330 331    # with gr.Row():332    with gr.Accordion("Disclaimer", open=False):333        _ = Path(model_loc).name334        gr.Markdown(335            f"Disclaimer: {_} can produce factually incorrect output, and should not be relied on to produce "336            f"factually accurate information. {_} was trained on various public datasets; while great efforts "337            "have been taken to clean the pretraining data, it is possible that this model could generate lewd, "338            "biased, or otherwise offensive outputs.",339            elem_classes=["disclaimer"],340        )341 342    msg_submit_event = msg.submit(343        # fn=conversation.user_turn,344        fn=user,345        inputs=[msg, chatbot],346        outputs=[msg, chatbot],347        queue=True,348        show_progress="full",349        # api_name=None,350    ).then(bot, chatbot, chatbot, queue=True)351    submit_click_event = submit.click(352        # fn=lambda x, y: ("",) + user(x, y)[1:],  # clear msg353        fn=user1,  # clear msg354        inputs=[msg, chatbot],355        outputs=[msg, chatbot],356        queue=True,357        # queue=False,358        show_progress="full",359        # api_name=None,360    ).then(bot, chatbot, chatbot, queue=True)361    stop.click(362        fn=None,363        inputs=None,364        outputs=None,365        cancels=[msg_submit_event, submit_click_event],366        queue=False,367    )368    clear.click(lambda: None, None, chatbot, queue=False)369 370    with gr.Accordion("For Chat/Translation API", open=False, visible=False):371        input_text = gr.Text()372        api_btn = gr.Button("Go", variant="primary")373        out_text = gr.Text()374 375    api_btn.click(376        predict_api,377        input_text,378        out_text,379        api_name="api",380    )381 382    # block.load(update_buff, [], buff, every=1)383    # block.load(update_buff, [buff_var], [buff_var, buff], every=1)384 385# concurrency_count=5, max_size=20386# max_size=36, concurrency_count=14387# CPU cpu_count=2 16G, model 7G388# CPU UPGRADE cpu_count=8 32G, model 7G389 390# does not work391_ = """392# _ = int(psutil.virtual_memory().total / 10**9 // file_size - 1)393# concurrency_count = max(_, 1)394if psutil.cpu_count(logical=False) >= 8:395    # concurrency_count = max(int(32 / file_size) - 1, 1)396else:397    # concurrency_count = max(int(16 / file_size) - 1, 1)398# """399 400# default concurrency_count = 1401# block.queue(concurrency_count=concurrency_count, max_size=5).launch(debug=True)402 403server_port = 7860404if "forindo" in platform.node():405    server_port = 7861406block.queue(max_size=5).launch(407    debug=True, server_name="0.0.0.0", server_port=server_port408)409 410# block.queue(max_size=5).launch(debug=True, server_name="0.0.0.0")411