Xeltron-cloud/VexaAI_Phi-4-mini-Instruct_API

1from fastapi import FastAPI, HTTPException2from pydantic import BaseModel3from transformers import AutoTokenizer, AutoModelForCausalLM4from huggingface_hub import login5import os6import torch7import uvicorn8 9login(os.getenv("HF_TOKEN"))10 11app = FastAPI(12    title="VexaAI Model-Platform: Microsoft Phi-4-mini-Instruct",13    description="Self-hosted AI-Model Microsoft Phi-4-mini-Instruct, powered by VexaAI.",14    version="0.9"15)16 17model_name = "microsoft/Phi-3-mini-4k-instruct"18 19tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)20model = AutoModelForCausalLM.from_pretrained(21    model_name,22    device_map="auto",23    trust_remote_code=True,24    torch_dtype=torch.float3225)26model.eval()27 28class GenerateRequest(BaseModel):29    prompt: str30    max_new_tokens: int = 51231    temperature: float = 0.732 33@app.post("/generate")34async def generate_text(request: GenerateRequest):35    try:36        inputs = tokenizer(request.prompt, return_tensors="pt").to(model.device)37        38        with torch.no_grad():39            outputs = model.generate(40                **inputs,41                max_new_tokens=request.max_new_tokens,42                temperature=request.temperature,43                do_sample=True,44                repetition_penalty=1.1,45                eos_token_id=tokenizer.eos_token_id,46                pad_token_id=tokenizer.eos_token_id,47                use_cache=False48            )49        50        full_text = tokenizer.decode(outputs[0], skip_special_tokens=True)51        generated_text = full_text[len(tokenizer.decode(inputs["input_ids"][0], skip_special_tokens=True)):].strip()52        53        return {"generated_text": generated_text}54    except Exception as e:55        raise HTTPException(status_code=500, detail=f"VexaAI Model-Platform: HTTP/S error: {str(e)}")56 57@app.get("/")58async def root():59    return {"message": "To start generating text, use /generate."}60 61if __name__ == "__main__":62    uvicorn.run(app, host="0.0.0.0", port=7860)