Leon4gr45/builder

1import { ProviderId } from '@/lib/llm/providers/types';2import { getProvider } from '@/lib/llm/providers/registry';3import type { UsageInfo } from '@/lib/llm/types';4 5/** Normalizes a judge provider's response usage into UsageInfo (undefined if absent). */6export function extractJudgeUsage(provider: string, model: string, data: unknown): UsageInfo | undefined {7  const d = data as Record<string, unknown>;8  if (provider === 'gemini') {9    const u = d?.usageMetadata as Record<string, number> | undefined;10    if (!u) return undefined;11    return {12      promptTokens: u.promptTokenCount || 0,13      completionTokens: u.candidatesTokenCount || 0,14      totalTokens: u.totalTokenCount || 0,15      model, provider,16    };17  }18  const u = d?.usage as Record<string, number> | undefined;19  if (!u) return undefined;20  if (provider === 'anthropic') {21    const pt = u.input_tokens || 0;22    const ct = u.output_tokens || 0;23    return { promptTokens: pt, completionTokens: ct, totalTokens: pt + ct, model, provider };24  }25  const pt = u.prompt_tokens || 0;26  const ct = u.completion_tokens || 0;27  return { promptTokens: pt, completionTokens: ct, totalTokens: u.total_tokens || pt + ct, model, provider };28}29 30export interface JudgeConfig {31  provider: ProviderId;32  apiKey: string;33  model: string;34}35 36export interface JudgeContext {37  prompt: string;38  files: Record<string, string>;39  summary: string;40}41 42export interface JudgeResult {43  passed: boolean;44  reasoning: string;45}46 47const SYSTEM_PROMPT = `You are a benchmark judge evaluating whether an AI coding assistant completed a task correctly.48 49You will be given:501. The original task prompt512. The final state of project files523. A summary from the AI assistant53 54Evaluate whether the task was completed correctly based on the criteria provided.55 56Respond in EXACTLY this format:57VERDICT: PASS58REASONING: <one paragraph explaining your judgment>59 60Or:61VERDICT: FAIL62REASONING: <one paragraph explaining what was missing or incorrect>`;63 64function buildUserMessage(criteria: string, context: JudgeContext): string {65  const fileSummary = Object.entries(context.files)66    .map(([path, content]) => `--- ${path} ---\n${content.substring(0, 2000)}`)67    .join('\n\n');68 69  return `## Task Prompt70${context.prompt}71 72## Evaluation Criteria73${criteria}74 75## Assistant Summary76${context.summary}77 78## Project Files79${fileSummary}80 81Evaluate whether the task was completed correctly based on the criteria above.`;82}83 84function parseVerdict(response: string): JudgeResult {85  const verdictMatch = /VERDICT:\s*(PASS|FAIL)/i.exec(response);86  const reasoningMatch = /REASONING:\s*([\s\S]+)/i.exec(response);87 88  return {89    passed: verdictMatch ? verdictMatch[1].toUpperCase() === 'PASS' : false,90    reasoning: reasoningMatch ? reasoningMatch[1].trim() : response.substring(0, 200),91  };92}93 94async function callOpenAICompatible(95  baseUrl: string,96  apiKey: string,97  model: string,98  provider: ProviderId,99  systemPrompt: string,100  userMessage: string101): Promise<{ text: string; usage?: UsageInfo }> {102  const headers: Record<string, string> = { 'Content-Type': 'application/json' };103  if (apiKey) headers['Authorization'] = `Bearer ${apiKey}`;104  if (provider === 'openrouter') {105    headers['HTTP-Referer'] = typeof window !== 'undefined' ? window.location.origin : 'http://localhost:3000';106    headers['X-Title'] = 'OSW-Studio';107  }108 109  const response = await fetch(`${baseUrl}/chat/completions`, {110    method: 'POST',111    headers,112    body: JSON.stringify({113      model,114      messages: [115        { role: 'system', content: systemPrompt },116        { role: 'user', content: userMessage },117      ],118      temperature: 0.2,119      max_tokens: 512,120      stream: false,121    }),122  });123 124  if (!response.ok) {125    const error = await response.text();126    throw new Error(`Judge API error (${provider}): ${error}`);127  }128 129  const data = await response.json();130  return { text: data.choices?.[0]?.message?.content || '', usage: extractJudgeUsage(provider, model, data) };131}132 133async function callAnthropic(134  apiKey: string,135  model: string,136  systemPrompt: string,137  userMessage: string138): Promise<{ text: string; usage?: UsageInfo }> {139  const response = await fetch('https://api.anthropic.com/v1/messages', {140    method: 'POST',141    headers: {142      'Content-Type': 'application/json',143      'x-api-key': apiKey,144      'anthropic-version': '2023-06-01',145    },146    body: JSON.stringify({147      model,148      system: systemPrompt,149      messages: [{ role: 'user', content: userMessage }],150      temperature: 0.2,151      max_tokens: 512,152    }),153  });154 155  if (!response.ok) {156    const error = await response.text();157    throw new Error(`Judge API error (anthropic): ${error}`);158  }159 160  const data = await response.json();161  return { text: data.content?.[0]?.text || '', usage: extractJudgeUsage('anthropic', model, data) };162}163 164async function callGemini(165  apiKey: string,166  model: string,167  systemPrompt: string,168  userMessage: string169): Promise<{ text: string; usage?: UsageInfo }> {170  const response = await fetch(171    `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent?key=${apiKey}`,172    {173      method: 'POST',174      headers: { 'Content-Type': 'application/json' },175      body: JSON.stringify({176        contents: [177          { role: 'user', parts: [{ text: systemPrompt }] },178          { role: 'user', parts: [{ text: userMessage }] },179        ],180        generationConfig: { temperature: 0.2, maxOutputTokens: 512 },181      }),182    }183  );184 185  if (!response.ok) {186    const error = await response.text();187    throw new Error(`Judge API error (gemini): ${error}`);188  }189 190  const data = await response.json();191  return { text: data.candidates?.[0]?.content?.parts?.[0]?.text || '', usage: extractJudgeUsage('gemini', model, data) };192}193 194export async function runJudgeEvaluation(195  criteria: string,196  context: JudgeContext,197  config: JudgeConfig198): Promise<JudgeResult> {199  const providerConfig = getProvider(config.provider);200  const userMessage = buildUserMessage(criteria, context);201 202  let responseText: string;203 204  if (config.provider === 'anthropic') {205    responseText = (await callAnthropic(config.apiKey, config.model, SYSTEM_PROMPT, userMessage)).text;206  } else if (config.provider === 'gemini') {207    responseText = (await callGemini(config.apiKey, config.model, SYSTEM_PROMPT, userMessage)).text;208  } else {209    const baseUrl = providerConfig.baseUrl || 'https://openrouter.ai/api/v1';210    responseText = (await callOpenAICompatible(211      baseUrl, config.apiKey, config.model, config.provider, SYSTEM_PROMPT, userMessage212    )).text;213  }214 215  return parseVerdict(responseText);216}217 218// ---- Structured (multi-criteria) judge: one call, one verdict per criterion ----219 220const STRUCTURED_SYSTEM_PROMPT = `You are a completion judge. You are given several numbered criteria and the current state of project files. For EACH criterion, decide whether the files actually satisfy it.221 222Judge ONLY against what is actually recorded in the files — do not assume or invent. Respond with one line per criterion, EXACTLY in this format:223<number>: PASS224or225<number>: FAIL - <short reason of what is missing>226 227Output nothing else.`;228 229function buildStructuredUserMessage(criteria: string[], context: JudgeContext): string {230  const fileSummary = Object.entries(context.files)231    .map(([path, content]) => `--- ${path} ---\n${content.substring(0, 2000)}`)232    .join('\n\n') || '(no files)';233  const criteriaList = criteria.map((c, i) => `${i + 1}. ${c}`).join('\n');234 235  return `## Context236${context.prompt}237 238## Criteria to evaluate239${criteriaList}240 241## Project Files242${fileSummary}243 244Evaluate each numbered criterion against the files above.`;245}246 247/**248 * Parses one PASS/FAIL verdict per criterion from the judge's response.249 * Tolerates "1:", "1.", "1)", "ITEM 1:", and en/em-dash reason separators.250 * Any criterion without a parseable verdict fails closed.251 */252export function parseStructuredVerdicts(response: string, count: number): JudgeResult[] {253  const results: JudgeResult[] = [];254  for (let i = 1; i <= count; i++) {255    const re = new RegExp(`^\\s*(?:item\\s*|#)?${i}\\s*[:.)\\]]\\s*(PASS|FAIL)\\b\\s*[-–—:]?\\s*(.*)$`, 'im');256    const m = re.exec(response);257    if (m) {258      results.push({ passed: m[1].toUpperCase() === 'PASS', reasoning: (m[2] || '').trim() });259    } else {260      results.push({ passed: false, reasoning: 'Could not verify this item.' });261    }262  }263  return results;264}265 266/**267 * Evaluates multiple criteria in a single judge call. Returns one verdict per268 * criterion, in the same order as the input.269 */270export async function runStructuredJudge(271  criteria: string[],272  context: JudgeContext,273  config: JudgeConfig274): Promise<{ verdicts: JudgeResult[]; usage?: UsageInfo }> {275  if (criteria.length === 0) return { verdicts: [] };276  const providerConfig = getProvider(config.provider);277  const userMessage = buildStructuredUserMessage(criteria, context);278 279  let result: { text: string; usage?: UsageInfo };280  if (config.provider === 'anthropic') {281    result = await callAnthropic(config.apiKey, config.model, STRUCTURED_SYSTEM_PROMPT, userMessage);282  } else if (config.provider === 'gemini') {283    result = await callGemini(config.apiKey, config.model, STRUCTURED_SYSTEM_PROMPT, userMessage);284  } else {285    const baseUrl = providerConfig.baseUrl || 'https://openrouter.ai/api/v1';286    result = await callOpenAICompatible(287      baseUrl, config.apiKey, config.model, config.provider, STRUCTURED_SYSTEM_PROMPT, userMessage288    );289  }290 291  return { verdicts: parseStructuredVerdicts(result.text, criteria.length), usage: result.usage };292}293