hymenjj/llama-cpp-python-prebuilt

1"""Example FastAPI server for llama.cpp.2 3To run this example:4 5```bash6pip install fastapi uvicorn sse-starlette pydantic-settings7export MODEL=../models/7B/...8```9 10Then run:11```12uvicorn llama_cpp.server.app:create_app --reload13```14 15or16 17```18python3 -m llama_cpp.server19```20 21Then visit http://localhost:8000/docs to see the interactive API docs.22 23"""24 25from __future__ import annotations26 27import os28import sys29import argparse30 31import uvicorn32 33from llama_cpp.server.app import create_app34from llama_cpp.server.settings import (35    Settings,36    ServerSettings,37    ModelSettings,38    ConfigFileSettings,39)40from llama_cpp.server.cli import add_args_from_model, parse_model_from_args41 42 43def main():44    description = "🦙 Llama.cpp python server. Host your own LLMs!🚀"45    parser = argparse.ArgumentParser(description=description)46 47    add_args_from_model(parser, Settings)48    parser.add_argument(49        "--config_file",50        type=str,51        help="Path to a config file to load.",52    )53    server_settings: ServerSettings | None = None54    model_settings: list[ModelSettings] = []55    args = parser.parse_args()56    try:57        # Load server settings from config_file if provided58        config_file = os.environ.get("CONFIG_FILE", args.config_file)59        if config_file:60            if not os.path.exists(config_file):61                raise ValueError(f"Config file {config_file} not found!")62            with open(config_file, "rb") as f:63                # Check if yaml file64                if config_file.endswith(".yaml") or config_file.endswith(".yml"):65                    import yaml66                    import json67 68                    config_file_settings = ConfigFileSettings.model_validate_json(69                        json.dumps(yaml.safe_load(f))70                    )71                else:72                    config_file_settings = ConfigFileSettings.model_validate_json(73                        f.read()74                    )75                server_settings = ServerSettings.model_validate(config_file_settings)76                model_settings = config_file_settings.models77        else:78            server_settings = parse_model_from_args(ServerSettings, args)79            model_settings = [parse_model_from_args(ModelSettings, args)]80    except Exception as e:81        print(e, file=sys.stderr)82        parser.print_help()83        sys.exit(1)84    assert server_settings is not None85    assert model_settings is not None86    app = create_app(87        server_settings=server_settings,88        model_settings=model_settings,89    )90    uvicorn.run(91        app,92        host=os.getenv("HOST", server_settings.host),93        port=int(os.getenv("PORT", server_settings.port)),94        ssl_keyfile=server_settings.ssl_keyfile,95        ssl_certfile=server_settings.ssl_certfile,96    )97 98 99if __name__ == "__main__":100    main()101