Limour/llama-python-streamingllm

1def init(cfg):2    # ========== 共同 ==========3    model = cfg['model']4    btn_com = cfg['btn_com']5    s_info = cfg['s_info']6    lock = cfg['session_lock']7 8    # ========== 特殊 ==========9    chat_template = cfg['chat_template']10    msg = cfg['msg']11    chatbot = cfg['chatbot']12    chat_display_format = cfg['chat_display_format']13 14    # ========== 显示用户消息 ==========15    def btn_submit_usr(message: str, history):16        # print('btn_submit_usr', message, history)17        if history is None:18            history = []19        return "", history + [[message.strip(), '']]20 21    # ========== 模型流式响应 ==========22    def btn_submit_bot(history, _n_keep, _n_discard,23                       _temperature, _repeat_penalty, _frequency_penalty,24                       _presence_penalty, _repeat_last_n, _top_k,25                       _top_p, _min_p, _typical_p,26                       _tfs_z, _mirostat_mode, _mirostat_eta,27                       _mirostat_tau, _usr, _char,28                       _rag, _max_tokens):29        with lock:30            if not cfg['session_active']:31                raise RuntimeError32            # ========== 释放不再需要的环境 ==========33            model.venv_disband({'usr', 'char'})34            print('venv_disband char', model.venv_info)35            # ========== 用户输入 ==========36            model.venv_create('usr')37            t_msg = history[-1][0]38            t_msg = chat_template(_usr, t_msg)39            model.eval_t(t_msg, _n_keep, _n_discard, chat_template.im_start_token)40            yield history, model.venv_info41            # ========== 模型输出 ==========42            if cfg['btn_stop_status']:43                return44            model.venv_create('char')45            _tmp = btn_com(_n_keep, _n_discard,46                           _temperature, _repeat_penalty, _frequency_penalty,47                           _presence_penalty, _repeat_last_n, _top_k,48                           _top_p, _min_p, _typical_p,49                           _tfs_z, _mirostat_mode, _mirostat_eta,50                           _mirostat_tau, _char, _max_tokens)51            for _h in _tmp:52                history[-1][1] = _h53                yield history, model.venv_info54            # ========== 输出完毕后格式化输出 ==========55            history[-1][1] = chat_display_format(history[-1][1])56            yield history, model.venv_info57 58    cfg['btn_submit_fn_usr'] = {59        'fn': btn_submit_usr,60        'inputs': [msg, chatbot],61        'outputs': [msg, chatbot]62    }63    cfg['btn_submit_fn_usr'].update(cfg['btn_concurrency'])64 65    cfg['btn_submit_fn_bot'] = {66        'fn': btn_submit_bot,67        'inputs': [chatbot]+cfg['setting'],68        'outputs': [chatbot, s_info],69    }70    cfg['btn_submit_fn_bot'].update(cfg['btn_concurrency'])71 72    cfg['btn_submit'].click(73        **cfg['btn_start']74    ).success(75        **cfg['btn_submit_fn_usr']76    ).success(77        **cfg['btn_submit_fn_bot']78    ).success(79        **cfg['btn_finish']80    )81