devsbarn/fastapi-crawler

1from fastapi import FastAPI, HTTPException, Header, Depends2from fastapi.responses import StreamingResponse3from pydantic import BaseModel4from typing import Optional5import os, json6 7from crawler import crawl_site, get_domain8from crawler_async import crawl_site_async9 10app = FastAPI(title="Business Intelligence Crawler API")11 12SECRET_API_KEY = os.environ.get("HF_SECRET_API_KEY")13 14def verify_api_key(x_api_key: str = Header(...)):15    if x_api_key != SECRET_API_KEY:16        raise HTTPException(status_code=401, detail="Unauthorized")17    return True18 19class CrawlRequest(BaseModel):20    url: str21    max_pages: Optional[int] = 3022 23@app.post("/crawl_raw")24def crawl_raw(req: CrawlRequest, authorized: bool = Depends(verify_api_key)):25    if not req.url.startswith("http"):26        raise HTTPException(status_code=400, detail="Invalid URL")27    pages = crawl_site(req.url, max_pages=req.max_pages)28    if not pages:29        raise HTTPException(status_code=404, detail="No crawlable content found")30    return {"domain": get_domain(req.url), "pages_crawled": len(pages), "pages": pages}31 32@app.post("/crawl_markdown")33def crawl_markdown(req: CrawlRequest, authorized: bool = Depends(verify_api_key)):34    pages = crawl_site(req.url, max_pages=req.max_pages)35    if not pages:36        raise HTTPException(status_code=404, detail="No crawlable content found")37    domain = get_domain(req.url)38    md = [f"# BI Crawl for {domain}\\n"]39    depth_groups = {}40    for p in pages:41        depth_groups.setdefault(p["depth"], []).append(p)42    for depth in sorted(depth_groups.keys()):43        md.append(f"## Depth {depth}")44        for p in depth_groups[depth]:45            md.append(f"### {p['url']}\\n{p['content']}\\n")46    return {"domain": domain, "pages_crawled": len(pages), "markdown": "\\n".join(md)}47 48@app.post("/crawl_stream")49async def crawl_stream(req: CrawlRequest, authorized: bool = Depends(verify_api_key)):50    async def event_generator():51        async for page in crawl_site_async(req.url, max_pages=req.max_pages):52            yield f"data: {json.dumps(page, ensure_ascii=False)}\\n\\n"53    return StreamingResponse(event_generator(), media_type="text/event-stream")