multimodalart/remote-encoder-reverse-proxy

1const express = require('express');2const fetch = require('node-fetch');3 4const app = express();5const TARGET_URL = 'https://rhknk53jznw37un7.us-east-1.aws.endpoints.huggingface.cloud';6const MAX_PARALLEL = 1;7const MAX_WAIT_MS = 10 * 60 * 1000; // 10 minutes8 9let activeRequests = 0;10const queue = [];11 12async function processQueue() {13  while (queue.length > 0 && activeRequests < MAX_PARALLEL) {14    const { req, res, next } = queue.shift();15    activeRequests++;16    handleRequest(req, res, next).finally(() => {17      activeRequests--;18      processQueue();19    });20  }21}22 23async function retryWith503Backoff(url, options, startTime) {24  let attempt = 0;25  let lastResponse = null;26  27  while (true) {28    const elapsed = Date.now() - startTime;29    if (elapsed > MAX_WAIT_MS) {30      throw new Error('Max wait time exceeded (10 minutes)');31    }32    33    const response = await fetch(url, options);34    lastResponse = response;35    36    // If successful (2xx), return immediately37    if (response.ok) {38      return response;39    }40    41    // Don't retry 401 (Unauthorized) - return immediately42    if (response.status === 401) {43      return response;44    }45    46    // For 503: retry with exponential backoff until max time47    // For other errors: retry up to 3 times, then return the error48    if (response.status === 503) {49      // Exponential backoff for 503: 1s, 2s, 4s, 8s, 16s, 32s, 64s...50      const delay = Math.min(1000 * Math.pow(2, attempt), 64000);51      attempt++;52      53      // Check if waiting would exceed max time54      if (elapsed + delay > MAX_WAIT_MS) {55        return response; // Return 503 if we'd exceed max time56      }57      58      await new Promise(resolve => setTimeout(resolve, delay));59    } else {60      // For non-503 errors, retry up to 3 times with shorter delays61      if (attempt >= 3) {62        return response; // Return the error after 3 attempts63      }64      65      // Short delay for non-503 errors: 1s, 2s, 3s66      const delay = (attempt + 1) * 1000;67      attempt++;68      69      // Check if waiting would exceed max time70      if (elapsed + delay > MAX_WAIT_MS) {71        return response;72      }73      74      await new Promise(resolve => setTimeout(resolve, delay));75    }76  }77}78 79async function handleRequest(req, res, next) {80  try {81    const startTime = Date.now();82    const targetUrl = TARGET_URL + req.url;83    84    const options = {85      method: req.method,86      headers: { ...req.headers, host: new URL(TARGET_URL).host },87      body: req.method !== 'GET' && req.method !== 'HEAD' ? req.body : undefined88    };89    90    const response = await retryWith503Backoff(targetUrl, options, startTime);91    92    res.status(response.status);93    response.headers.forEach((value, key) => {94      res.setHeader(key, value);95    });96    97    response.body.pipe(res);98  } catch (error) {99    res.status(504).json({ error: error.message });100  }101}102 103app.use(express.raw({ type: '*/*', limit: '50mb' }));104 105app.use((req, res, next) => {106  if (activeRequests < MAX_PARALLEL) {107    activeRequests++;108    handleRequest(req, res, next).finally(() => {109      activeRequests--;110      processQueue();111    });112  } else {113    queue.push({ req, res, next });114  }115});116 117const PORT = process.env.PORT || 7860;118app.listen(PORT, () => {119  console.log(`Reverse proxy listening on port ${PORT}`);120  console.log(`Proxying to: ${TARGET_URL}`);121  console.log(`Max parallel requests: ${MAX_PARALLEL}`);122});