npmaiecosystem/loadbalancerfallback

1from fastapi import FastAPI, Request, HTTPException2from fastapi.middleware.cors import CORSMiddleware3from pydantic import BaseModel4from typing import Annotated, Any, Optional5import asyncio6from redis.asyncio import Redis7import httpx8import os9 10app = FastAPI()11 12app.add_middleware(13    CORSMiddleware,14    allow_origins=["*"],  # Allows your Netlify URL to connect15    allow_credentials=False,16    allow_methods=["*"],  # Resolves the 405 OPTIONS error17    allow_headers=["*"],18)19 20 21@app.post("/")22def health_check():23    return "Healthy"24 25password = os.environ.get("PASSWORD")26print(password)27r = Redis(28    host='redis-15562.c1.us-west-2-2.ec2.cloud.redislabs.com',29    port=15562,30    decode_responses=True,31    username="default",32    password=password,33)34 35 36 37Model_links = {38    "llama3.2": "https://galactromedaNPMAIECOSYSTEM-model23.hf.space/llama3.2", #https://sonuramashishnpm-npmai.hf.space/llama39    "qwen2.5-coder:7b":"https://galactromedaNPMAIECOSYSTEM-model24.hf.space/qwen2.5-coder:7b", #https://sonuramashishnpm-npmai.hf.space/qwen40    "vicuna:7b":"https://galactromedaNPMAIECOSYSTEM-model25.hf.space/vicuna:7b", #https://sonuramashish22028704-vicuna7b.hf.space/vicuna41    "phi3:medium":"https://sonuramashish22028704-phi3medium.hf.space/phi3medium",42    "llama3.2:3b":"https://pmaiecosystem-model5.hf.space/llama3.2:3b",43    "llama3.1:8b":"https://pmaiecosystem-model5.hf.space/llama3.1:8b",44    "falcon:7b-instruct":"https://sonuramashish22028704-falcon7binstruct.hf.space/falcon",45    "meditron":"https://KarachiUniNPMAIECOSYSTEM-model14.hf.space/meditron",46    "granite4.1:3b":"https://karachiuninpmaiecosystem-model14.hf.space/granite4.1:3b",47    "wizard-math":"https://karachiuninpmaiecosystem-model15.hf.space/wizard-math",48    "llava":"https://karachiuninpmaiecosystem-model15.hf.space/llava",49    "olmo2":"https://karachiuninpmaiecosystem-model19.hf.space/olmo2",50    "nemotron-mini":"https://karachiuninpmaiecosystem-model19.hf.space/nemotron-mini",51    "internlm2:7b":"https://sonuramashish22028704-internlm27b.hf.space/internlm",52    "codellama:7b-instruct":"https://sonuramashish22028704-falcon7binstruct.hf.space/codellama",53    "mistral:7b":"https://sonuramashish22028704-mistral7b.hf.space/mistral",54    "qwen3.5:9b":"https://sonuramashish22028704-vicuna7b.hf.space/qwen359gb",55    "gemma2:9b":"https://sonuramashish22028704-internlm27b.hf.space/gemma29b",56    "qwen2.5:7b":"https://pmaiecosystem-model6.hf.space/qwen2.5:7b",57    "llama3.2:1b":"https://pmaiecosystem-model6.hf.space/llama3.2:1b",58    "qwen2-math:7b":"https://pmaiecosystem-model7.hf.space/qwen2-math:7b",59    "qwen2.5vl":"https://pmaiecosystem-model7.hf.space/qwen2.5vl",60    "phi3:3.8b":"https://sonuramashishnpmai-model8.hf.space/phi3:3.8b",61    "llava-phi3":"https://sonuramashishnpmai-model8.hf.space/llava-phi3",62    "gemma2:2b":"https://sonuramashishnpmai-model9.hf.space/gemma2:2b",63    "openhermes":"https://sonuramashishnpmai-model9.hf.space/openhermes",64    "deepseek-coder:6.7b":"https://sonuramashishnpmai-model10.hf.space/deepseek-coder:6.7b",65    "yi":"https://sonuramashishnpmai-model10.hf.space/yi",66    "granite3.3:2b":"https://sonuramashishnpmai-model12.hf.space/granite3.3:2b",67    "smollm:1.7b":"https://sonuramashishnpmai-model12.hf.space/smollm:1.7b",68    "stablelm2":"https://sonuramashishnpmai-model13.hf.space/stablelm2",69    "cniongolo_biomistral":"https://sonuramashishnpm-model13.hf.space/cniongolo_biomistral",70    "moondream":"https://karachiuninpmaiecosystem-model16.hf.space/moondream",71    "openchat":"https://karachiuninpmaiecosystem-model16.hf.space/openchat",72    "deepseek-r1:7b":"https://karachiuninpmaiecosystem-model17.hf.space/deepseek-r1:7b",73    "openbmb_minicpm-v2.6":"https://karachiuninpmaiecosystem-model17.hf.space/openbmb_minicpm-v2.6",74    "llama3:8b":"https://karachiuninpmaiecosystem-model18.hf.space/llama3:8b",75    "gemma3:4b":"https://karachiuninpmaiecosystem-model18.hf.space/gemma3:4b",76    "stable-code":"https://galactromedanpmaiecosystem-model20.hf.space/stable-code",77    "codellama:7b":"https://galactromedanpmaiecosystem-model20.hf.space/codellama:7b",78    "aya:8b":"https://galactromedanpmaiecosystem-model21.hf.space/aya:8b",79    "phi4:14b":"https://galactromedanpmaiecosystem-model20.hf.space/phi4:14b",80    "gemma4:e2b":"https://galactromedanpmaiecosystem-model22.hf.space/gemma4:e2b",81    "qwen3.5:2b":"https://galactromedanpmaiecosystem-model22.hf.space/qwen3.5:2b",82    "llama3.2_fall":"https://sonuramashishnpm-npm-journalist.hf.space/llm_fall_llama",83    "qwen2.5-coder:7b_fall":"https://sonuramashish22028704-mistral7b.hf.space/llm_fall_qwen2",84    "vicuna:7b_fall":"https://sonuramashishnpm-model4.hf.space/llm_fall_vicuna",85    "gemma3:12b_fall":"https://pmaiecosystem-gemma312b_fall.hf.space/llm_fall_gemma312b",86    "internlm2:7b_fall":"https://sonuramashishnpm-model2.hf.space/llm_fall_interlm",87    "falcon:7b-instruct_fall":"https://sonuramashishnpm-model1.hf.space/llm_fall_falcon",88    "codellama:7b-instruct_fall":"https://sonuramashishnpm-model3.hf.space/llm_fall_codellama",89    "mistral:7b_fall":"https://sonuramashishnpm-model2.hf.space/fall_llm_mistral",90    "phi3:medium_fall":"https://sonuramashishnpm-model1.hf.space/llama_fall_phi",91    "qwen3.5:9b_fall":"https://sonuramashishnpm-model4.hf.space/llm_fall_qwen359gb",92    "gemma2:9b_fall":"https://sonuramashishnpm-model3.hf.space/llm_fall_gemma29b",93    "gemma3:12b":"https://pmaiecosystem-gemma312b.hf.space/gemma312b",94}95 96# Updated Lua Script97LUA_CHECK_AND_INC = """98local key = KEYS[1]99local status = tonumber(redis.call('HGET', key, 'status') or '0')100if status < 1 then101    redis.call('HSET', key, 'status', status + 1)102    return status103end104return -1105"""106 107LUA_REMOVAL_STATUS = """108local key = KEYS[1]109local status = tonumber(redis.call('HGET', key, 'status') or '0')110if status > 0 then111    redis.call('HSET', key, 'status', status -1)112    return status -1113end114return 0115"""116 117async def check_cond(model_link: str, fall_model: Optional[list] = None):118    status = await r.eval(LUA_CHECK_AND_INC, 1, model_link)119    if status != -1:120        return {"link": model_link, "statusno": status}121 122    if fall_model:123        for model in fall_model:124            status = await r.eval(LUA_CHECK_AND_INC, 1, model)125            if status != -1:126                return {"link": model, "statusno": status}127                128    129    for model in Model_links.values():130        status = await r.eval(LUA_CHECK_AND_INC, 1, model)131        if status != -1:132            return {"link": model, "statusno": status}133 134    return None135 136 137class Input(BaseModel):138    model: str139    temperature: float = 0.5140    prompt: str141    change: bool = True142    Models: Optional[list] = None143 144@app.post("/load_balancer")145async def llm_router(inputs: Input):146    if not inputs.model or not inputs.prompt:147        raise HTTPException(status_code=400, detail="Model name and Prompt are required.")148 149    if inputs.model not in Model_links:150        raise HTTPException(status_code=444, detail="Model not found.")151 152    model_link = Model_links[inputs.model]153    fall_links = []154 155    fall_models = inputs.Models156    if fall_models:157        for m in fall_models:158            model_name = f"{m}_fall"159            if model_name in Model_links.keys():160                link = Model_links[model_name]161                fall_links.append(link)162            else:163                raise HTTPException(status_code=402, detail="Fallback models are not found in Models Dictionary")164 165    model_cond = await check_cond(model_link=model_link, fall_model=fall_links)166    167    if model_cond and model_cond.get("link") and model_cond.get("statusno") is not None:168        return await router(169            model_url=model_cond["link"],170            prompt=inputs.prompt,171            temp=inputs.temperature172        )173    else:174        raise HTTPException(status_code=503, detail="All model endpoints and fallbacks are busy.")175 176async def router(model_url, prompt, temp):177    error_log = ""178    process= ""179    payload = {"prompt": prompt, "temperature": temp}180    timeout = httpx.Timeout(connect=100.0, read=560.0, write=300.0, pool=320.0)181 182    try:183        async with httpx.AsyncClient(timeout=timeout) as client:184            response = await client.post(model_url, json=payload)185            response.raise_for_status()186            f_response = response.json()["response"]187            if f_response is not None and str(f_response).strip() != "":188                process += f_response189 190            else:191                raise ValueError("Empty string or None returned in response from LLM")192    except Exception as e:193        error_log += f"LLM backend error: {str(e)}"194        print(f"Detailed ERROR:-{e}")195 196    finally:197        await r.eval(LUA_REMOVAL_STATUS, 1, model_url)198 199    if error_log:200        raise HTTPException(status_code=502, detail=error_log)201        202    else:203        return {"response": process}