UBC-NLP/SimbaBench

1const express = require('express');2const app = express();3const PORT = 7860;4 5const HF_TOKEN = process.env.HF_TOKEN;6const REPO_ID = process.env.SIMBA_DATA; 7 8let CACHED_DATA = null;9 10app.use(express.static('public'));11 12// --- Fetch Helper ---13async function fetchHFFile(path) {14    const url = `https://huggingface.co/datasets/${REPO_ID}/resolve/main/${path}`;15    console.log(`[Loading] ${path}...`);16    17    const headers = {};18    if (HF_TOKEN) {19        headers["Authorization"] = `Bearer ${HF_TOKEN}`;20    }21 22    try {23        const res = await fetch(url, { headers });24        if (!res.ok) {25            console.error(`[Error] Failed to fetch ${path}: ${res.status} ${res.statusText}`);26            return [];27        }28        const text = await res.text();29        return text.split('\n')30            .filter(line => line.trim())31            .map(line => {32                try { return JSON.parse(line); } catch(e) { return null; }33            })34            .filter(x => x);35    } catch (err) {36        console.error(`[Error] Network error fetching ${path}:`, err);37        return [];38    }39}40 41// --- List Files Helper ---42async function listHFFiles(folder) {43    const url = `https://huggingface.co/api/datasets/${REPO_ID}/tree/main/${folder}`;44    45    const headers = {};46    if (HF_TOKEN) {47        headers["Authorization"] = `Bearer ${HF_TOKEN}`;48    }49 50    try {51        const res = await fetch(url, { headers });52        if (!res.ok) {53            console.error(`[Error] Failed to list files: ${res.status}`);54            return [];55        }56        const files = await res.json();57        // Filter only .jsonl files58        return files.filter(f => f.path.endsWith('.jsonl')).map(f => f.path);59    } catch (err) {60        console.error("[Error] Network error listing files:", err);61        return [];62    }63}64 65// --- Processor ---66async function loadAndProcessData() {67    console.log("--- Starting Data Sync ---");68    69    // 1. Fetch Metadata70    const metaList = await fetchHFFile("final_results/metadata.jsonl");71    const langMap = {};72    metaList.forEach(m => { langMap[m.lang_code_key] = m; });73 74    // 2. Scan ALL files75    const files = await listHFFiles("final_results");76    77    let allRecords = [];78    79    for (const file of files) {80        if (file.endsWith('metadata.jsonl')) continue;81        const records = await fetchHFFile(file);82        allRecords.push(...records);83    }84 85    console.log(`[Processing] Loaded ${allRecords.length} total records. Bucketing by task...`);86 87    // 3. Bucket Records by Task (Checking both 'task' and 'data_type')88    const ttsRecords = [];89    const slidRecords = [];90    const asrRecords = [];91 92    allRecords.forEach(r => {93        // Robust check: look for 'task' OR 'data_type'94        const identifier = r.task || r.data_type;95        if (!identifier) return; 96        97        const task = identifier.toLowerCase().trim();98        99        if (task === 'tts') ttsRecords.push(r);100        else if (task === 'slid') slidRecords.push(r);101        else if (task === 'asr') asrRecords.push(r);102    });103 104    console.log(`[Counts] ASR: ${asrRecords.length}, TTS: ${ttsRecords.length}, SLID: ${slidRecords.length}`);105 106    // --- PROCESS SLID ---107    const slidMap = {};108    slidRecords.forEach(r => {109        const code = r.lang_code;110        const meta = langMap[code] || { name: code, family: 'Unknown' };111        112        if (!slidMap[code]) {113            slidMap[code] = { Language: meta.name, ISO: code };114        }115        const modelName = r.model_name || r.model || "Unknown Model";116        slidMap[code][modelName] = r.f1_score;117    });118    const slidFinal = Object.values(slidMap);119 120    // --- PROCESS TTS ---121    const ttsGrouped = {};122    const ttsModels = new Set();123    124    ttsRecords.forEach(r => {125        const modelName = r.model || r.model_name || "Unknown Model";126        127        ttsModels.add(modelName);128        if(!ttsGrouped[modelName]) ttsGrouped[modelName] = [];129        130        const meta = langMap[r.lang_code] || { name: r.language || r.lang_code, lang_code_key: r.lang_code };131        132        // Standardize structure for frontend133        r.iso = r.lang_code; 134        r.Language = meta.name; // Ensure capital 'Language' key exists for dropdown135        r.language = meta.name; // Ensure lowercase 'language' key exists for table lookups136        137        ttsGrouped[modelName].push(r);138    });139 140    // --- PROCESS ASR ---141    const families = new Set();142    const models = new Set();143    const familyData = {}; 144 145    asrRecords.forEach(r => {146        const meta = langMap[r.lang_code] || { name: r.lang_code, family: 'Unknown', lang_code_key: r.lang_code };147        const fam = meta.family;148        const mod = r.model_name || r.model;149 150        families.add(fam);151        models.add(mod);152 153        if (!familyData[fam]) familyData[fam] = {};154        if (!familyData[fam][mod]) familyData[fam][mod] = {};155 156        familyData[fam][mod][meta.name] = {157            wer: (r.datasets_avg_wer || 0) * 100,158            cer: (r.datasets_avg_cer || 0) * 100,159            iso: meta.lang_code_key160        };161    });162 163    // Format ASR By Family164    const asrByFamily = {};165    Array.from(families).sort().forEach(fam => {166        const modData = familyData[fam];167        const langMapForFamily = new Map();168        Object.values(modData).forEach(m => {169            Object.keys(m).forEach(langName => {170                langMapForFamily.set(langName, m[langName].iso);171            });172        });173        174        const languages = Array.from(langMapForFamily.keys()).sort().map(name => ({175            name: name,176            iso: langMapForFamily.get(name)177        }));178 179        const rows = Object.keys(modData).map(modName => {180            const scores = modData[modName];181            let sW=0, sC=0, n=0;182            languages.forEach(l => {183                if(scores[l.name]) { sW += scores[l.name].wer; sC += scores[l.name].cer; n++; }184            });185            const row = { Model: modName, Avg_WER: n ? sW/n : 0, Avg_CER: n ? sC/n : 0 };186            languages.forEach(l => {187                if(scores[l.name]) {188                    row[`WER_${l.name}`] = scores[l.name].wer;189                    row[`CER_${l.name}`] = scores[l.name].cer;190                }191            });192            return row;193        });194        rows.sort((a,b) => a.Avg_WER - b.Avg_WER);195        asrByFamily[fam] = { data: rows, languages: languages };196    });197 198    // Format ASR By Model199    const asrByModel = {};200    Array.from(models).forEach(mod => {201        const rows = [];202        asrRecords.forEach(r => {203            if((r.model_name || r.model) === mod) {204                const meta = langMap[r.lang_code] || { name: r.lang_code, family: 'Unknown', lang_code_key: r.lang_code };205                rows.push({206                    Language: meta.name,207                    ISO: meta.lang_code_key,208                    Family: meta.family,209                    WER: (r.datasets_avg_wer || 0) * 100,210                    CER: (r.datasets_avg_cer || 0) * 100211                });212            }213        });214        rows.sort((a,b) => a.Language.localeCompare(b.Language));215        asrByModel[mod] = rows;216    });217 218    CACHED_DATA = {219        metadata: {220            families: Array.from(families).sort(),221            models: Array.from(models).sort(),222            tts_models: Array.from(ttsModels).sort()223        },224        asr: { by_family: asrByFamily, by_model: asrByModel },225        tts: ttsGrouped,226        slid: slidFinal227    };228    console.log("--- Data Ready ---");229}230 231app.get('/api/data', async (req, res) => {232    if (!CACHED_DATA) {233        try { await loadAndProcessData(); } 234        catch (e) { return res.status(500).json({error: e.message}); }235    }236    res.json(CACHED_DATA);237});238 239app.listen(PORT, '0.0.0.0', () => {240    console.log(`Server running on ${PORT}`);241    loadAndProcessData();242});