Limour/llama-python-streamingllm

1def init(cfg):2    # ========== 共同 ==========3    model = cfg['model']4    s_info = cfg['s_info']5    lock = cfg['session_lock']6 7    # ========== 特殊 ==========8    chat_template = cfg['chat_template']9 10    # ========== 注入内容 ==========11    def btn_rag(_n_keep, _n_discard,12                _temperature, _repeat_penalty, _frequency_penalty,13                _presence_penalty, _repeat_last_n, _top_k,14                _top_p, _min_p, _typical_p,15                _tfs_z, _mirostat_mode, _mirostat_eta,16                _mirostat_tau, _usr, _char,17                _rag, _max_tokens):18        with lock:19            if not cfg['session_active']:20                raise RuntimeError21            if cfg['btn_stop_status']:22                yield model.venv_info23                return24            # ========== 清除之前注入的内容 ==========25            model.venv_remove('rag')26            # ========== 没有需要注入的内容 ==========27            if not _rag:28                yield model.venv_info29                return30            # ========== 需要临时注入的内容 ==========31            model.venv_create('rag')32            t_rag = chat_template('system', _rag)33            model.eval_t(t_rag, _n_keep, _n_discard, chat_template.im_start_token)34            yield model.venv_info35 36    cfg['btn_rag_fn'] = {37        'fn': btn_rag,38        'inputs': cfg['setting'],39        'outputs': s_info40    }41    cfg['btn_rag_fn'].update(cfg['btn_concurrency'])42 43    cfg['btn_rag'].click(44        **cfg['btn_start']45    ).success(46        **cfg['btn_rag_fn']47    ).success(48        **cfg['btn_finish']49    )50