Sarveni/Top-VS-Code-Extensions-Data-Analysis

1const fs = require('fs');2const Papa = require('papaparse');3 4const csvData = fs.readFileSync('./public/data.csv', 'utf-8');5const results = Papa.parse(csvData, { header: true, dynamicTyping: true, skipEmptyLines: true });6const extensions = results.data;7 8// Build publisher map9const publisherMap = new Map();10 11extensions.forEach(ext => {12    const pub = String(ext.publisher || '').trim();13    if (!pub) return;14    const dl = ext.downloads_count;15    if (typeof dl !== 'number' || isNaN(dl) || dl < 0) return;16 17    if (!publisherMap.has(pub)) {18        publisherMap.set(pub, { name: pub, extensions: [] });19    }20    publisherMap.get(pub).extensions.push(dl);21});22 23const publishers = Array.from(publisherMap.values());24publishers.forEach(p => {25    p.maxDownloads = Math.max(...p.extensions);26    p.totalDownloads = p.extensions.reduce((s, d) => s + d, 0);27    p.avgDownloads = p.totalDownloads / p.extensions.length;28    p.extCount = p.extensions.length;29});30 31// Group A: at least one extension >= 1M32// Group B: no extension >= 1M33const THRESHOLD = 1_000_000;34const groupA = publishers.filter(p => p.maxDownloads >= THRESHOLD);35const groupB = publishers.filter(p => p.maxDownloads < THRESHOLD);36 37function getAvgOfGroup(pubs) {38    const totalDl = pubs.reduce((s, p) => s + p.totalDownloads, 0);39    const totalExt = pubs.reduce((s, p) => s + p.extCount, 0);40    return totalDl / totalExt;41}42 43function getMedianOfAvgs(pubs) {44    const avgs = pubs.map(p => p.avgDownloads).sort((a, b) => a - b);45    const mid = Math.floor(avgs.length / 2);46    return avgs.length % 2 !== 0 ? avgs[mid] : (avgs[mid - 1] + avgs[mid]) / 2;47}48 49// Validation 1: Counts50console.log('=== Validation 1: Publisher Counts ===');51console.log(`Group A (1M+ extension): ${groupA.length}`);52console.log(`Group B (no 1M+ extension): ${groupB.length}`);53 54// Validation 2: Average downloads per extension55const avgA = getAvgOfGroup(groupA);56const avgB = getAvgOfGroup(groupB);57console.log('\n=== Validation 2: Average Downloads per Extension ===');58console.log(`Group A: ${Math.round(avgA).toLocaleString()}`);59console.log(`Group B: ${Math.round(avgB).toLocaleString()}`);60console.log(`Ratio: ${(avgA / avgB).toFixed(2)}x`);61 62// Validation 3: Median63const medA = getMedianOfAvgs(groupA);64const medB = getMedianOfAvgs(groupB);65console.log('\n=== Validation 3: Median Avg Downloads per Extension ===');66console.log(`Group A: ${Math.round(medA).toLocaleString()}`);67console.log(`Group B: ${Math.round(medB).toLocaleString()}`);68console.log(`Ratio: ${(medA / medB).toFixed(2)}x`);69 70// Validation 4: Outlier Test — remove top 1% publishers by total downloads71const allSorted = [...publishers].sort((a, b) => b.totalDownloads - a.totalDownloads);72const top1Count = Math.ceil(publishers.length * 0.01);73const top1Names = new Set(allSorted.slice(0, top1Count).map(p => p.name));74 75const groupAFiltered = groupA.filter(p => !top1Names.has(p.name));76const groupBFiltered = groupB.filter(p => !top1Names.has(p.name));77const avgAF = getAvgOfGroup(groupAFiltered);78const avgBF = getAvgOfGroup(groupBFiltered);79console.log('\n=== Validation 4: Outlier Test (remove top 1% publishers) ===');80console.log(`Removed: ${top1Count} publishers`);81console.log(`Group A filtered avg: ${Math.round(avgAF).toLocaleString()}`);82console.log(`Group B filtered avg: ${Math.round(avgBF).toLocaleString()}`);83console.log(`Filtered ratio: ${(avgAF / avgBF).toFixed(2)}x`);84 85// Validation 5: Repeatability — remove each Group A publisher's best extension86const groupANoTop = groupA.map(p => {87    const sortedExts = [...p.extensions].sort((a, b) => b - a);88    const remaining = sortedExts.slice(1); // remove best89    return {90        name: p.name,91        extensions: remaining,92        extCount: remaining.length,93        totalDownloads: remaining.reduce((s, d) => s + d, 0),94        avgDownloads: remaining.length > 0 ? remaining.reduce((s, d) => s + d, 0) / remaining.length : 0,95    };96}).filter(p => p.extCount > 0); // only publishers with remaining extensions97 98const avgANoTop = getAvgOfGroup(groupANoTop);99const medANoTop = getMedianOfAvgs(groupANoTop);100console.log('\n=== Validation 5: Repeatability (Group A minus best extension) ===');101console.log(`Group A publishers with remaining extensions: ${groupANoTop.length}`);102console.log(`Group A avg (excl. best): ${Math.round(avgANoTop).toLocaleString()}`);103console.log(`Group B avg: ${Math.round(avgB).toLocaleString()}`);104console.log(`Group A median (excl. best): ${Math.round(medANoTop).toLocaleString()}`);105console.log(`Group B median: ${Math.round(medB).toLocaleString()}`);106console.log(`Avg ratio (excl. best): ${(avgANoTop / avgB).toFixed(2)}x`);107console.log(`Median ratio (excl. best): ${(medANoTop / medB).toFixed(2)}x`);108 109console.log('\n=== Summary ===');110console.log(`Group A publishers: ${groupA.length}`);111console.log(`Group B publishers: ${groupB.length}`);112console.log(`Full avg ratio: ${(avgA / avgB).toFixed(2)}x`);113console.log(`Median ratio: ${(medA / medB).toFixed(2)}x`);114console.log(`Filtered avg ratio (top1% removed): ${(avgAF / avgBF).toFixed(2)}x`);115console.log(`Repeatability avg ratio (best ext removed): ${(avgANoTop / avgB).toFixed(2)}x`);116