overwrite69/haiku-api
0
1"""2Haiku API - OpenAI-compatible proxy for chatgpt.org/claude/chat3Deploy to Hugging Face Spaces (Docker SDK)4 5Features:6- Tool/function calling support (always detects tool call tags in output)7- Auto-continues when upstream hits the ~1K token output limit8- Rotating proxy with direct-connection fallback9- SSE keep-alive comments during continuation gaps10- Message normalization for Orchids.app compatibility11- Robust error handling with proper JSON error responses12"""13 14import asyncio15import json16import os17import re18import time19import uuid20import traceback21from typing import Optional22from urllib.parse import unquote23 24import httpx25from fastapi import FastAPI, HTTPException, Request26from fastapi.middleware.cors import CORSMiddleware27from fastapi.responses import StreamingResponse, JSONResponse28 29app = FastAPI(title="Haiku API", version="8.1.0")30 31# ── CORS ─────────────────────────────────────────────────────────32app.add_middleware(33 CORSMiddleware,34 allow_origins=["*"],35 allow_credentials=True,36 allow_methods=["*"],37 allow_headers=["*"],38)39 40# ── Proxy Config ─────────────────────────────────────────────────41PROXY_URL = os.environ.get("PROXY_URL", "")42 43PROXY_MAX_RETRIES = 4 # rotating proxy: try a few IPs44PROXY_RETRY_DELAY = 1 # seconds between proxy retries45CONNECT_TIMEOUT = 10.0 # short connect timeout46READ_TIMEOUT = 120.0 # long read timeout (for streaming responses)47 48 49def _make_client(use_proxy: bool = True) -> httpx.AsyncClient:50 """Create an httpx client, with or without proxy."""51 kwargs = dict(52 verify=False,53 timeout=httpx.Timeout(READ_TIMEOUT, connect=CONNECT_TIMEOUT),54 )55 if use_proxy and PROXY_URL:56 kwargs["proxy"] = PROXY_URL57 return httpx.AsyncClient(**kwargs)58 59 60# ── Session State ────────────────────────────────────────────────61class SessionState:62 def __init__(self):63 self.xsrf_token: Optional[str] = None64 self.csrf_token: Optional[str] = None65 self.cookies: Optional[httpx.Cookies] = None66 self.last_refresh: float = 067 self.refresh_interval: float = 60068 self._lock = asyncio.Lock()69 70 async def refresh(self, client: httpx.AsyncClient):71 async with self._lock:72 now = time.time()73 if self.cookies and (now - self.last_refresh) < self.refresh_interval:74 return75 76 # Try with proxy first, then fallback to direct77 for use_proxy in [True, False]:78 if use_proxy and not PROXY_URL:79 continue80 81 working_client = client82 for attempt in range(PROXY_MAX_RETRIES if use_proxy else 2):83 try:84 if attempt > 0:85 try:86 await working_client.aclose()87 except:88 pass89 working_client = _make_client(use_proxy=use_proxy)90 91 resp = await working_client.get(92 "https://chatgpt.org/claude/chat",93 follow_redirects=True,94 headers={95 "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36",96 "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",97 },98 timeout=20.0,99 )100 101 if resp.status_code != 200:102 print(f"[Session] GET returned {resp.status_code} (proxy={use_proxy}, attempt {attempt+1})")103 await asyncio.sleep(PROXY_RETRY_DELAY)104 continue105 106 new_cookies = httpx.Cookies()107 for name, value in resp.cookies.items():108 new_cookies.set(name, value, domain="chatgpt.org")109 for header in resp.headers.get_list("set-cookie"):110 parts = header.split(";")[0]111 if "=" in parts:112 k, v = parts.split("=", 1)113 new_cookies.set(k.strip(), v.strip(), domain="chatgpt.org")114 115 xsrf = new_cookies.get("XSRF-TOKEN", domain="chatgpt.org")116 if xsrf:117 xsrf = unquote(xsrf)118 119 csrf = None120 m = re.search(r'<meta\s+name="csrf-token"\s+content="([^"]+)"', resp.text)121 if m:122 csrf = m.group(1)123 124 self.cookies = new_cookies125 self.xsrf_token = xsrf126 self.csrf_token = csrf127 self.last_refresh = now128 mode = "proxy" if use_proxy else "direct"129 print(f"[Session] OK ({mode}) — CSRF:{bool(csrf)} XSRF:{bool(xsrf)} Cookies:{list(new_cookies.keys())}")130 return working_client131 132 except (httpx.ConnectError, httpx.ProxyError, httpx.TimeoutException) as e:133 print(f"[Session] Connection error (proxy={use_proxy}, attempt {attempt+1}): {type(e).__name__}")134 await asyncio.sleep(PROXY_RETRY_DELAY)135 continue136 except Exception as e:137 print(f"[Session] Error (proxy={use_proxy}, attempt {attempt+1}): {type(e).__name__}: {e}")138 await asyncio.sleep(PROXY_RETRY_DELAY)139 continue140 141 print("[Session] WARNING: All refresh attempts failed (both proxy and direct)")142 143 144session = SessionState()145 146# ── HTTP Client ──────────────────────────────────────────────────147http_client: Optional[httpx.AsyncClient] = None148 149@app.on_event("startup")150async def startup():151 global http_client152 http_client = _make_client(use_proxy=bool(PROXY_URL))153 result = await session.refresh(http_client)154 if result is not None:155 http_client = result156 157@app.on_event("shutdown")158async def shutdown():159 if http_client:160 await http_client.aclose()161 162 163# ── Tool Calling Support ─────────────────────────────────────────164# We support TWO tool call formats that models may output:165#166# Format 1 — Inline JSON (simple):167# <tool_call name="Write">{"file_path": "hello.js", "content": "hi"}</tool_call_>168# <function_call name="Write">{"file_path": "hello.js"}</function_call>169#170# Format 2 — Anthropic XML (Claude's native format):171# <function_calls>172# <invoke name="Write">173# <parameter name="file_path">hello.js</parameter>174# <parameter name="content">console.log("hi")</parameter>175# </invoke>176# </function_calls>177 178# Regex for Format 1: inline JSON tool calls179_TOOL_CALL_INLINE_RE = re.compile(180 r'<(?:function_call|tool_call)\s+name="([^"]+)">\s*(.*?)\s*</(?:function_call|tool_call)_?>',181 re.DOTALL182)183 184# Regex for Format 2: Anthropic XML function_calls blocks185_ANTHROPIC_FC_BLOCK_RE = re.compile(186 r'<function_calls>\s*(.*?)\s*</function_calls>',187 re.DOTALL188)189 190# Within a block, match each <invoke name="...">...</invoke>191_ANTHROPIC_INVOKE_RE = re.compile(192 r'<invoke\s+name="([^"]+)">\s*(.*?)\s*</invoke>',193 re.DOTALL194)195 196# Within an invoke, match each <parameter name="...">value</parameter>197_ANTHROPIC_PARAM_RE = re.compile(198 r'<parameter\s+name="([^"]+)">(.*?)</parameter>',199 re.DOTALL200)201 202 203def _parse_json_args(args_str: str) -> str:204 """Try to parse arguments as JSON, with fallbacks. Returns JSON string."""205 try:206 args_json = json.loads(args_str)207 return json.dumps(args_json)208 except json.JSONDecodeError:209 pass210 211 # Try to fix common issues212 args_cleaned = args_str.strip('`').strip()213 if args_cleaned.startswith('json'):214 args_cleaned = args_cleaned[4:].strip()215 try:216 args_json = json.loads(args_cleaned)217 return json.dumps(args_json)218 except json.JSONDecodeError:219 pass220 221 # Last resort: wrap the raw text as an argument222 return json.dumps({"raw_input": args_str})223 224 225def _parse_tool_calls(text: str) -> tuple[list[dict], str]:226 """Parse tool calls from model text output.227 228 Supports two formats:229 1. Inline JSON: <tool_call name="X">JSON</tool_call_>230 2. Anthropic XML: <function_calls><invoke name="X"><parameter name="p">v</parameter></invoke></function_calls>231 232 Returns (tool_calls, remaining_text) where tool_calls is in OpenAI format.233 If no tool calls found, returns ([], original_text).234 """235 tool_calls = []236 consumed_spans = []237 238 # --- Format 1: Inline JSON tool calls ---239 for match in _TOOL_CALL_INLINE_RE.finditer(text):240 func_name = match.group(1)241 args_str = match.group(2).strip()242 args_final = _parse_json_args(args_str)243 244 tool_calls.append({245 "id": f"call_{uuid.uuid4().hex[:24]}",246 "type": "function",247 "function": {248 "name": func_name,249 "arguments": args_final,250 }251 })252 consumed_spans.append((match.start(), match.end()))253 254 # --- Format 2: Anthropic XML function_calls ---255 for block_match in _ANTHROPIC_FC_BLOCK_RE.finditer(text):256 block_text = block_match.group(1)257 consumed_spans.append((block_match.start(), block_match.end()))258 259 for invoke_match in _ANTHROPIC_INVOKE_RE.finditer(block_text):260 func_name = invoke_match.group(1)261 invoke_body = invoke_match.group(2)262 263 params = {}264 for param_match in _ANTHROPIC_PARAM_RE.finditer(invoke_body):265 param_name = param_match.group(1)266 param_value = param_match.group(2)267 try:268 params[param_name] = json.loads(param_value)269 except (json.JSONDecodeError, ValueError):270 params[param_name] = param_value271 272 tool_calls.append({273 "id": f"call_{uuid.uuid4().hex[:24]}",274 "type": "function",275 "function": {276 "name": func_name,277 "arguments": json.dumps(params),278 }279 })280 281 if not tool_calls:282 return [], text283 284 # Extract remaining text (not part of any tool call)285 remaining_parts = []286 prev_end = 0287 for start, end in sorted(consumed_spans):288 if start > prev_end:289 chunk = text[prev_end:start].strip()290 if chunk:291 remaining_parts.append(chunk)292 prev_end = max(prev_end, end)293 294 if prev_end < len(text):295 chunk = text[prev_end:].strip()296 if chunk:297 remaining_parts.append(chunk)298 299 remaining_text = "\n".join(remaining_parts)300 return tool_calls, remaining_text301 302 303def _has_incomplete_tool_call(text: str) -> bool:304 """Check if text has an opening tool call tag without a matching close."""305 # Inline format306 inline_opens = len(re.findall(r'<(?:function_call|tool_call)\s+name="[^"]+">', text))307 inline_closes = len(re.findall(r'</(?:function_call|tool_call)_?>', text))308 if inline_opens > inline_closes:309 return True310 311 # Anthropic XML format312 if text.count('<function_calls>') > text.count('</function_calls>'):313 return True314 invoke_opens = len(re.findall(r'<invoke\s+name="[^"]+">', text))315 if invoke_opens > text.count('</invoke>'):316 return True317 318 return False319 320 321def _strip_incomplete_tool_tags(text: str) -> str:322 """Remove incomplete tool call XML tags from text.323 This prevents raw XML tags from leaking into delta.content324 when auto-continue fails to complete a tool call."""325 # Remove incomplete Anthropic XML blocks326 # e.g. "<function_calls>\n<invoke name="Write">\n<parameter name="content">some unfinished..."327 text = re.sub(328 r'<function_calls>\s*<invoke[^>]*>.*',329 '', text, flags=re.DOTALL330 )331 # Remove incomplete inline JSON tool calls332 text = re.sub(333 r'<(?:function_call|tool_call)\s+name="[^"]+">.*',334 '', text, flags=re.DOTALL335 )336 # Remove any stray opening/closing tags337 text = re.sub(r'</?function_calls>\s*', '', text)338 text = re.sub(r'</?invoke[^>]*>\s*', '', text)339 text = re.sub(r'</?parameter[^>]*>\s*', '', text)340 text = re.sub(r'</?(?:function_call|tool_call)_?>\s*', '', text)341 return text.strip()342 343 344# ── Tool System Prompt Builder ──────────────────────────────────345 346def _build_tool_system_prompt(tools: list[dict], tool_choice=None) -> str:347 """Convert OpenAI tools/functions format to a system prompt using348 Anthropic XML format — the format Claude natively understands."""349 350 invoke_blocks = []351 tool_names = []352 353 for tool in tools:354 if "function" in tool:355 func = tool["function"]356 else:357 func = tool358 359 name = func.get("name", "unknown")360 desc = func.get("description", "No description")361 params = func.get("parameters", {})362 tool_names.append(name)363 364 props = params.get("properties", {})365 required = params.get("required", [])366 param_lines = []367 for pname, pdef in props.items():368 ptype = pdef.get("type", "any")369 pdesc = pdef.get("description", "")370 req = " (required)" if pname in required else ""371 param_lines.append(f'<parameter name="{pname}">{ptype}{req} — {pdesc}</parameter>')372 373 params_xml = '\n'.join(param_lines) if param_lines else ''374 invoke_blocks.append(f"""<tool_description name="{name}">375{desc}376Parameters:377{params_xml}378</tool_description>""")379 380 tools_xml = '\n\n'.join(invoke_blocks)381 382 choice_instruction = ""383 if tool_choice == "required":384 choice_instruction = "\nIMPORTANT: You MUST call at least one tool."385 elif tool_choice == "none":386 return ""387 elif isinstance(tool_choice, dict) and tool_choice.get("type") == "function":388 fname = tool_choice.get("function", {}).get("name", "")389 choice_instruction = f"\nIMPORTANT: You MUST call the {fname} function."390 391 return f"""In this environment you have access to a set of tools you can use to answer the user's question.392 393{tools_xml}394 395## Tool Call Format396When you need to call a tool, use this EXACT XML format:397<function_calls>398<invoke name="FUNCTION_NAME">399<parameter name="param_name">value</parameter>400</invoke>401</function_calls>402 403You may call multiple tools by using multiple <invoke> blocks inside a single <function_calls> block, or by using multiple <function_calls> blocks.404- The parameter values should be the actual values, NOT JSON-encoded strings405- Do NOT wrap tool calls in markdown code blocks406- If you don't need to call any tools, just respond normally with text{choice_instruction}"""407 408 409# ── Message normalization ────────────────────────────────────────410 411def _flatten_content_array(content: list) -> str:412 """Convert a content array to plain text."""413 text_parts = []414 for part in content:415 if isinstance(part, str):416 text_parts.append(part)417 elif isinstance(part, dict):418 if part.get("type") == "text":419 text_parts.append(part.get("text", ""))420 return "\n".join(text_parts)421 422 423def normalize_messages(messages: list[dict], tools: list[dict] = None, tool_choice=None) -> list[dict]:424 """Normalize messages: handle content arrays, tool roles, tool_calls,425 and inject tool definitions into system prompt if tools are provided."""426 result = []427 428 tool_system = None429 if tools and tool_choice != "none":430 tool_system = _build_tool_system_prompt(tools, tool_choice)431 432 system_injected = False433 434 for msg in messages:435 role = msg.get("role", "user")436 content = msg.get("content", "")437 438 if isinstance(content, list):439 content = _flatten_content_array(content)440 441 if content is None:442 content = ""443 content = str(content)444 445 # Handle tool role messages446 if role == "tool":447 tool_name = msg.get("name", "unknown_tool")448 tool_call_id = msg.get("tool_call_id", "")449 result.append({450 "role": "user",451 "content": f"[Tool Result for {tool_name} (id: {tool_call_id})]:\n{content}"452 })453 continue454 455 # Handle assistant messages with tool_calls456 if role == "assistant" and msg.get("tool_calls"):457 parts = []458 regular_content = content if content and content.strip() else ""459 460 if regular_content:461 parts.append(regular_content)462 463 invoke_parts = []464 for tc in msg["tool_calls"]:465 func = tc.get("function", {})466 name = func.get("name", "unknown")467 args = func.get("arguments", "{}")468 try:469 args_json = json.loads(args)470 except (json.JSONDecodeError, TypeError):471 args_json = {}472 invoke_lines = [f'<invoke name="{name}">']473 for k, v in args_json.items():474 invoke_lines.append(f'<parameter name="{k}">{v}</parameter>')475 invoke_lines.append('</invoke>')476 invoke_parts.append('\n'.join(invoke_lines))477 478 fc_content = '<function_calls>\n' + '\n'.join(invoke_parts) + '\n</function_calls>'479 combined = regular_content + '\n\n' + fc_content if regular_content else fc_content480 result.append({"role": "assistant", "content": combined})481 continue482 483 # Inject tool system prompt into the first system message484 if role == "system" and not system_injected and tool_system:485 combined = content + '\n\n' + tool_system if content.strip() else tool_system486 result.append({"role": "system", "content": combined})487 system_injected = True488 continue489 490 if role == "system" and not content.strip():491 continue492 493 result.append({"role": role, "content": content})494 495 if tool_system and not system_injected:496 result.insert(0, {"role": "system", "content": tool_system})497 498 return result499 500 501# ── Headers ──────────────────────────────────────────────────────502 503def _headers() -> dict:504 h = {505 "Accept": "*/*",506 "Content-Type": "application/json",507 "Origin": "https://chatgpt.org",508 "Referer": "https://chatgpt.org/claude/chat",509 "X-Requested-With": "XMLHttpRequest",510 "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36",511 }512 csrf = session.csrf_token or session.xsrf_token513 if csrf:514 h["X-CSRF-TOKEN"] = csrf515 return h516 517 518# ── Proxy-aware request with retry + direct fallback ──────────────519 520async def _proxy_post(url: str, **kwargs) -> httpx.Response:521 """POST with proxy retry logic, falling back to direct connection."""522 global http_client523 524 # Try with proxy first525 if PROXY_URL:526 for attempt in range(PROXY_MAX_RETRIES):527 try:528 resp = await http_client.post(url, **kwargs)529 return resp530 except (httpx.ConnectError, httpx.ProxyError, httpx.TimeoutException) as e:531 print(f"[Proxy] Connection error #{attempt+1}: {type(e).__name__}")532 try:533 await http_client.aclose()534 except:535 pass536 http_client = _make_client(use_proxy=True)537 await asyncio.sleep(PROXY_RETRY_DELAY)538 continue539 540 # Fallback: try direct connection541 print("[Proxy] Falling back to direct connection")542 direct_client = _make_client(use_proxy=False)543 try:544 resp = await direct_client.post(url, **kwargs)545 return resp546 finally:547 await direct_client.aclose()548 549 550# ── Raw call with retries ───────────────────────────────────────551 552async def _raw_call(messages: list[dict], model: str) -> httpx.Response:553 """Make a single POST to chatgpt.org/api/chat with full retry logic."""554 await session.refresh(http_client)555 556 payload = {"model": model, "messages": messages}557 558 for attempt in range(2): # CSRF retry559 for rate_attempt in range(3): # 429 retry560 try:561 resp = await _proxy_post(562 "https://chatgpt.org/api/chat",563 json=payload,564 headers=_headers(),565 cookies=session.cookies,566 )567 except (httpx.ConnectError, httpx.ProxyError, httpx.TimeoutException) as e:568 print(f"[Chat] Connection failed: {type(e).__name__}")569 session.last_refresh = 0570 raise HTTPException(502, f"Cannot reach upstream: {type(e).__name__}")571 572 if resp.status_code == 419 and attempt == 0:573 print("[Chat] 419 -> refreshing session...")574 session.last_refresh = 0575 await session.refresh(http_client)576 break577 578 if resp.status_code == 429:579 wait_time = (rate_attempt + 1) * 10580 print(f"[Chat] 429 rate limited, waiting {wait_time}s (attempt {rate_attempt+1}/3)...")581 session.last_refresh = 0582 await session.refresh(http_client)583 if rate_attempt < 2:584 await asyncio.sleep(wait_time)585 continue586 raise HTTPException(429, f"Rate limited by upstream after {rate_attempt+1} retries")587 588 if resp.status_code != 200:589 session.last_refresh = 0590 raise HTTPException(resp.status_code, f"Upstream {resp.status_code}: {resp.text[:300]}")591 592 return resp593 594 raise HTTPException(500, "Failed after retry")595 596 597async def _stream_one_response(resp):598 """Stream a single upstream SSE response in real-time.599 Yields (text, finish_reason) tuples. finish_reason is None for text chunks."""600 finish_reason = None601 602 try:603 async for raw_line in resp.aiter_lines():604 line = raw_line.strip()605 if not line or line.startswith(":"):606 continue607 if not line.startswith("data: "):608 continue609 610 payload_str = line[6:]611 if payload_str.strip() == "[DONE]":612 break613 614 try:615 chunk = json.loads(payload_str)616 except json.JSONDecodeError:617 continue618 619 for choice in chunk.get("choices", []):620 delta = choice.get("delta", {})621 c = delta.get("content", "")622 if c:623 yield c, None624 625 fr = choice.get("finish_reason")626 if fr:627 if fr in ("stop", "end_turn"):628 finish_reason = "stop"629 elif fr in ("length", "max_tokens"):630 finish_reason = "length"631 except (httpx.ReadError, httpx.RemoteProtocolError) as e:632 print(f"[Stream] Connection lost during streaming: {type(e).__name__}")633 except Exception as e:634 print(f"[Stream] Error during streaming: {type(e).__name__}: {e}")635 636 yield "", finish_reason637 638 639# ── Streaming with auto-continue ────────────────────────────────640MAX_CONTINUATIONS = 20641 642 643async def _raw_call_streaming(messages: list[dict], model: str):644 """Like _raw_call but yields SSE keep-alive comments during retries,645 then yields the httpx.Response object."""646 await session.refresh(http_client)647 payload = {"model": model, "messages": messages}648 649 for attempt in range(2): # CSRF retry650 for rate_attempt in range(3): # 429 retry651 yield ": thinking...\n\n"652 653 try:654 resp = await _proxy_post(655 "https://chatgpt.org/api/chat",656 json=payload,657 headers=_headers(),658 cookies=session.cookies,659 )660 except (httpx.ConnectError, httpx.ProxyError, httpx.TimeoutException) as e:661 print(f"[Chat] Connection failed: {type(e).__name__}")662 session.last_refresh = 0663 raise HTTPException(502, f"Cannot reach upstream: {type(e).__name__}")664 665 if resp.status_code == 419 and attempt == 0:666 print("[Chat] 419 -> refreshing session...")667 session.last_refresh = 0668 await session.refresh(http_client)669 break670 671 if resp.status_code == 429:672 wait_time = (rate_attempt + 1) * 10673 print(f"[Chat] 429 rate limited, waiting {wait_time}s (attempt {rate_attempt+1}/3)...")674 session.last_refresh = 0675 await session.refresh(http_client)676 if rate_attempt < 2:677 for _ in range(wait_time):678 yield ": retrying...\n\n"679 await asyncio.sleep(1)680 continue681 raise HTTPException(429, f"Rate limited after {rate_attempt+1} retries")682 683 if resp.status_code != 200:684 session.last_refresh = 0685 raise HTTPException(resp.status_code, f"Upstream {resp.status_code}: {resp.text[:300]}")686 687 yield resp688 return689 690 raise HTTPException(500, "Failed after retry")691 692 693def _emit_tool_call_chunks(chunk_id: str, created: int, model: str, tool_calls: list[dict], remaining_text: str):694 """Generate OpenAI streaming chunks for tool calls. Returns list of SSE strings."""695 chunks = []696 697 for i, tc in enumerate(tool_calls):698 # First chunk: role + tool_call with id, name, and start of arguments699 sse_start = json.dumps({700 "id": chunk_id,701 "object": "chat.completion.chunk",702 "created": created,703 "model": model,704 "choices": [{705 "index": 0,706 "delta": {707 "role": "assistant",708 "tool_calls": [{709 "index": i,710 "id": tc["id"],711 "type": "function",712 "function": {713 "name": tc["function"]["name"],714 "arguments": "",715 }716 }]717 },718 "finish_reason": None,719 }],720 })721 chunks.append(f"data: {sse_start}\n\n")722 723 # Argument chunks724 args = tc["function"]["arguments"]725 chunk_size = max(1, len(args) // 3)726 for offset in range(0, len(args), chunk_size):727 arg_piece = args[offset:offset + chunk_size]728 sse_arg = json.dumps({729 "id": chunk_id,730 "object": "chat.completion.chunk",731 "created": created,732 "model": model,733 "choices": [{734 "index": 0,735 "delta": {736 "tool_calls": [{737 "index": i,738 "function": {739 "arguments": arg_piece,740 }741 }]742 },743 "finish_reason": None,744 }],745 })746 chunks.append(f"data: {sse_arg}\n\n")747 748 # Remaining text alongside tool calls749 if remaining_text.strip():750 sse_text = json.dumps({751 "id": chunk_id,752 "object": "chat.completion.chunk",753 "created": created,754 "model": model,755 "choices": [{756 "index": 0,757 "delta": {"content": remaining_text},758 "finish_reason": None,759 }],760 })761 chunks.append(f"data: {sse_text}\n\n")762 763 # Final chunk with finish_reason764 sse_done = json.dumps({765 "id": chunk_id,766 "object": "chat.completion.chunk",767 "created": created,768 "model": model,769 "choices": [{770 "index": 0,771 "delta": {},772 "finish_reason": "tool_calls",773 }],774 })775 chunks.append(f"data: {sse_done}\n\n")776 chunks.append("data: [DONE]\n\n")777 778 return chunks779 780 781async def _stream_with_auto_continue(messages: list[dict], model: str):782 """Stream with real-time output, auto-continue, and keep-alive pings.783 784 ALWAYS buffers the full response to detect tool call tags.785 If tool calls are found AND complete, emits them as proper OpenAI tool_calls chunks.786 If tool calls are incomplete, auto-continues to collect the rest.787 If no tool calls, emits the text as regular content chunks.788 """789 chunk_id = f"chatcmpl-{uuid.uuid4().hex[:12]}"790 created = int(time.time())791 conversation = list(messages)792 total_content = ""793 794 for cont_num in range(MAX_CONTINUATIONS):795 yield ": thinking...\n\n"796 797 resp = None798 try:799 async for result in _raw_call_streaming(conversation, model):800 if isinstance(result, str):801 yield result802 else:803 resp = result804 except HTTPException as e:805 error_data = json.dumps({806 "id": chunk_id,807 "object": "chat.completion.chunk",808 "created": created,809 "model": model,810 "choices": [{811 "index": 0,812 "delta": {"content": f"\n\n[Error: {e.detail}]"},813 "finish_reason": None,814 }],815 })816 yield f"data: {error_data}\n\n"817 yield f"data: {json.dumps({'id': chunk_id, 'object': 'chat.completion.chunk', 'created': created, 'model': model, 'choices': [{'index': 0, 'delta': {}, 'finish_reason': 'stop'}]})}\n\n"818 yield "data: [DONE]\n\n"819 return820 821 if resp is None:822 yield f"data: {json.dumps({'id': chunk_id, 'object': 'chat.completion.chunk', 'created': created, 'model': model, 'choices': [{'index': 0, 'delta': {'content': '[Error: No response from upstream]'}, 'finish_reason': None}]})}\n\n"823 yield f"data: {json.dumps({'id': chunk_id, 'object': 'chat.completion.chunk', 'created': created, 'model': model, 'choices': [{'index': 0, 'delta': {}, 'finish_reason': 'stop'}]})}\n\n"824 yield "data: [DONE]\n\n"825 return826 827 finish_reason = "stop"828 chunk_content = ""829 830 # Buffer the full response831 async for text, fr in _stream_one_response(resp):832 if fr is not None:833 finish_reason = fr834 continue835 836 if text:837 chunk_content += text838 total_content += text839 yield ": streaming...\n\n"840 841 print(f"[Chat] Chunk #{cont_num+1}: {len(chunk_content)} chars, finish={finish_reason}")842 843 # Check for tool calls in the accumulated text844 tool_calls, remaining_text = _parse_tool_calls(total_content)845 has_incomplete = _has_incomplete_tool_call(total_content)846 847 print(f"[Chat] tool_calls={len(tool_calls)} incomplete={has_incomplete} finish={finish_reason}")848 849 # ── Decision tree ──────────────────────────────────────────850 #851 # 1. If we have COMPLETE tool calls AND no incomplete tags → emit & done852 # 2. If we have incomplete tool calls (regardless of complete ones) → auto-continue853 # 3. If no tool calls and finish_reason == "stop" and no incomplete tags → emit text & done854 # 4. If no tool calls and finish_reason == "stop" but HAS incomplete tags → auto-continue855 # (the upstream might report "stop" even when cut off mid-tag)856 # 5. If finish_reason == "length" → auto-continue857 858 if tool_calls and not has_incomplete:859 # All tool calls are complete — emit them860 print(f"[Chat] Emitting {len(tool_calls)} complete tool call(s)")861 for sse_chunk in _emit_tool_call_chunks(chunk_id, created, model, tool_calls, remaining_text):862 yield sse_chunk863 return864 865 if has_incomplete:866 # Incomplete tool calls detected — must auto-continue867 print(f"[Chat] Incomplete tool call detected, auto-continuing...")868 yield ": continuing...\n\n"869 conversation.append({"role": "assistant", "content": chunk_content})870 conversation.append({"role": "user", "content": "Continue the tool call exactly from where you left off. Do not repeat the opening tag or any arguments you already wrote. Just continue outputting the parameter values from where you stopped."})871 print(f"[Chat] Auto-continue (incomplete) #{cont_num+1}, total so far: {len(total_content)} chars")872 continue873 874 # No tool calls and no incomplete tags875 if finish_reason == "stop":876 # Regular text response — emit as content877 chunk_sz = 50878 for offset in range(0, len(total_content), chunk_sz):879 piece = total_content[offset:offset + chunk_sz]880 sse_data = json.dumps({881 "id": chunk_id,882 "object": "chat.completion.chunk",883 "created": created,884 "model": model,885 "choices": [{886 "index": 0,887 "delta": {"content": piece},888 "finish_reason": None,889 }],890 })891 yield f"data: {sse_data}\n\n"892 893 sse_data = json.dumps({894 "id": chunk_id,895 "object": "chat.completion.chunk",896 "created": created,897 "model": model,898 "choices": [{899 "index": 0,900 "delta": {},901 "finish_reason": "stop",902 }],903 })904 yield f"data: {sse_data}\n\n"905 yield "data: [DONE]\n\n"906 return907 908 # finish_reason == "length" — auto-continue for regular text909 yield ": continuing...\n\n"910 conversation.append({"role": "assistant", "content": chunk_content})911 conversation.append({"role": "user", "content": "Continue exactly from where you left off. Do not repeat any text you already wrote."})912 print(f"[Chat] Auto-continue (length) #{cont_num+1}, total so far: {len(total_content)} chars")913 914 # Safety: max continuations reached — try to emit whatever we have915 tool_calls, remaining_text = _parse_tool_calls(total_content)916 if tool_calls:917 # Best-effort: emit whatever tool calls we managed to parse918 print(f"[Chat] Max continuations reached, emitting {len(tool_calls)} partial tool call(s)")919 for sse_chunk in _emit_tool_call_chunks(chunk_id, created, model, tool_calls, remaining_text):920 yield sse_chunk921 else:922 # Emit whatever text we have923 # Strip any incomplete tool call XML from the output to avoid raw tags in content924 clean_content = _strip_incomplete_tool_tags(total_content)925 if clean_content.strip():926 chunk_sz = 50927 for offset in range(0, len(clean_content), chunk_sz):928 piece = clean_content[offset:offset + chunk_sz]929 sse_data = json.dumps({930 "id": chunk_id,931 "object": "chat.completion.chunk",932 "created": created,933 "model": model,934 "choices": [{935 "index": 0,936 "delta": {"content": piece},937 "finish_reason": None,938 }],939 })940 yield f"data: {sse_data}\n\n"941 942 sse_data = json.dumps({943 "id": chunk_id,944 "object": "chat.completion.chunk",945 "created": created,946 "model": model,947 "choices": [{948 "index": 0,949 "delta": {},950 "finish_reason": "stop",951 }],952 })953 yield f"data: {sse_data}\n\n"954 yield "data: [DONE]\n\n"955 956 957# ── Non-streaming with auto-continue ────────────────────────────958 959async def _collect_with_auto_continue(messages: list[dict], model: str) -> dict:960 """Collect the full response, auto-continuing if cut off."""961 conversation = list(messages)962 full_content = ""963 964 for cont_num in range(MAX_CONTINUATIONS):965 resp = await _raw_call(conversation, model)966 content = ""967 finish_reason = "stop"968 969 async for text, fr in _stream_one_response(resp):970 if fr is not None:971 finish_reason = fr972 continue973 if text:974 content += text975 976 full_content += content977 print(f"[Chat] Collect #{cont_num+1}: {len(content)} chars, finish={finish_reason}")978 979 # Always check for tool calls980 tool_calls, remaining_text = _parse_tool_calls(full_content)981 982 if tool_calls:983 if _has_incomplete_tool_call(full_content) and finish_reason == "length":984 pass985 else:986 return {987 "tool_calls": tool_calls,988 "content": remaining_text if remaining_text.strip() else None,989 }990 991 if finish_reason == "stop":992 return {"content": full_content, "tool_calls": None}993 994 # Auto-continue995 if _has_incomplete_tool_call(content):996 conversation.append({"role": "assistant", "content": content})997 conversation.append({"role": "user", "content": "Continue the tool call exactly from where you left off. Do not repeat the opening tag or any arguments you already wrote."})998 else:999 conversation.append({"role": "assistant", "content": content})1000 conversation.append({"role": "user", "content": "Continue exactly from where you left off. Do not repeat any text you already wrote."})1001 1002 return {"content": full_content, "tool_calls": None}1003 1004 1005# ── OpenAI-compatible endpoint ──────────────────────────────────1006 1007@app.post("/v1/chat/completions")1008@app.post("/chat/completions")1009async def chat_completions(request: Request):1010 try:1011 body = await request.json()1012 except Exception:1013 raise HTTPException(400, "Invalid JSON")1014 1015 if not isinstance(body, dict):1016 raise HTTPException(400, "Body must be a JSON object")1017 1018 model = body.get("model", "anthropic/claude-haiku-4-5")1019 messages_raw = body.get("messages", [])1020 stream = body.get("stream", False)1021 1022 # Extract tools1023 tools = body.get("tools") or body.get("functions") or None1024 tool_choice = body.get("tool_choice", "auto")1025 1026 # Convert old 'functions' format1027 if tools and "function" not in tools[0] and "name" in tools[0]:1028 tools = [{"type": "function", "function": f} for f in tools]1029 1030 print(f"[Request] model={model} stream={stream} tools={bool(tools)} tool_choice={tool_choice} msgs={len(messages_raw)}")1031 1032 if not messages_raw or not isinstance(messages_raw, list):1033 raise HTTPException(400, "messages must be a non-empty array")1034 1035 messages = normalize_messages(messages_raw, tools=tools, tool_choice=tool_choice)1036 1037 if not messages:1038 raise HTTPException(400, "No valid messages after normalization")1039 1040 try:1041 if stream:1042 return StreamingResponse(1043 _stream_with_auto_continue(messages, model),1044 media_type="text/event-stream",1045 headers={1046 "Cache-Control": "no-cache",1047 "Connection": "keep-alive",1048 "X-Accel-Buffering": "no",1049 },1050 )1051 else:1052 result = await _collect_with_auto_continue(messages, model)1053 1054 tool_calls = result.get("tool_calls")1055 content = result.get("content")1056 1057 if tool_calls:1058 return JSONResponse({1059 "id": f"chatcmpl-{int(time.time())}",1060 "object": "chat.completion",1061 "created": int(time.time()),1062 "model": model,1063 "choices": [{1064 "index": 0,1065 "message": {1066 "role": "assistant",1067 "content": content,1068 "tool_calls": tool_calls,1069 },1070 "finish_reason": "tool_calls",1071 }],1072 "usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0},1073 })1074 else:1075 return JSONResponse({1076 "id": f"chatcmpl-{int(time.time())}",1077 "object": "chat.completion",1078 "created": int(time.time()),1079 "model": model,1080 "choices": [{1081 "index": 0,1082 "message": {"role": "assistant", "content": content or ""},1083 "finish_reason": "stop",1084 }],1085 "usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0},1086 })1087 except HTTPException:1088 raise1089 except Exception as e:1090 print(f"[Request] Unhandled error: {type(e).__name__}: {e}")1091 print(traceback.format_exc())1092 raise HTTPException(500, f"Internal error: {type(e).__name__}")1093 1094 1095# ── Models / Health ─────────────────────────────────────────────1096 1097@app.get("/v1/models")1098@app.get("/models")1099async def list_models():1100 return JSONResponse({1101 "object": "list",1102 "data": [1103 {"id": "anthropic/claude-haiku-4-5", "object": "model", "owned_by": "anthropic"},1104 ],1105 })1106 1107 1108@app.get("/")1109async def root():1110 return {1111 "status": "ok",1112 "version": "8.1.0",1113 "proxy": bool(PROXY_URL),1114 "tool_calling": True,1115 "endpoints": ["/v1/chat/completions", "/v1/models"],1116 }1117 1118 1119@app.get("/health")1120async def health():1121 return {1122 "status": "ok",1123 "session_active": bool(session.cookies),1124 "proxy": bool(PROXY_URL),1125 }1126 1127 1128@app.get("/debug/refresh")1129async def force_refresh():1130 global http_client1131 session.last_refresh = 01132 result = await session.refresh(http_client)1133 if result is not None:1134 http_client = result1135 return {1136 "refreshed": True,1137 "has_cookies": bool(session.cookies),1138 "has_csrf": bool(session.csrf_token),1139 "proxy": bool(PROXY_URL),1140 }1141 1142 1143@app.get("/debug/session")1144async def debug_session():1145 return {1146 "has_cookies": bool(session.cookies),1147 "cookie_names": list(session.cookies.keys()) if session.cookies else [],1148 "has_csrf": bool(session.csrf_token),1149 "has_xsrf": bool(session.xsrf_token),1150 "last_refresh_ago": int(time.time() - session.last_refresh) if session.last_refresh else None,1151 "proxy": bool(PROXY_URL),1152 }1153 1154 1155if __name__ == "__main__":1156 import uvicorn1157 uvicorn.run(app, host="0.0.0.0", port=7860)1158