Leon4gr45/builder
0
1import { TestAssertion, AssertionResult } from './types';2import type { ConversationNode } from '@/lib/llm/multi-agent-orchestrator';3import type { VirtualFileSystem } from '@/lib/vfs';4 5function truncate(str: string, max = 100): string {6 return str.length > max ? str.substring(0, max - 3) + '...' : str;7}8 9function getAssistantText(conversation: ConversationNode[]): string {10 const parts: string[] = [];11 for (const node of conversation) {12 for (const msg of node.messages) {13 if (msg.role === 'assistant') {14 if (typeof msg.content === 'string') {15 parts.push(msg.content);16 } else if (Array.isArray(msg.content)) {17 for (const block of msg.content) {18 if ('text' in block) parts.push(block.text);19 }20 }21 }22 }23 }24 return parts.join('\n');25}26 27function getToolOutputText(conversation: ConversationNode[], toolName: string): string {28 // Map tool_call_id → tool function name29 const callIdToName = new Map<string, string>();30 for (const node of conversation) {31 for (const msg of node.messages) {32 if (msg.tool_calls) {33 for (const tc of msg.tool_calls) {34 callIdToName.set(tc.id, tc.function.name);35 }36 }37 }38 }39 40 // Collect tool result content for matching tool name41 const parts: string[] = [];42 for (const node of conversation) {43 for (const msg of node.messages) {44 if (msg.role === 'tool' && msg.tool_call_id) {45 const name = callIdToName.get(msg.tool_call_id);46 if (name === toolName) {47 const content = typeof msg.content === 'string' ? msg.content : '';48 if (content) parts.push(content);49 }50 }51 }52 }53 54 return parts.join('\n');55}56 57function getToolCalls(conversation: ConversationNode[]) {58 const calls: Array<{ name: string; args: string }> = [];59 for (const node of conversation) {60 for (const msg of node.messages) {61 if (msg.tool_calls) {62 for (const tc of msg.tool_calls) {63 calls.push({ name: tc.function.name, args: tc.function.arguments });64 }65 }66 }67 }68 return calls;69}70 71async function evaluateOne(72 assertion: TestAssertion,73 projectId: string,74 conversation: ConversationNode[],75 vfs: VirtualFileSystem,76): Promise<{ passed: boolean; actual?: string }> {77 switch (assertion.type) {78 case 'file_exists': {79 const exists = await vfs.fileExists(projectId, assertion.path);80 return { passed: exists, actual: exists ? 'file exists' : 'file not found' };81 }82 83 case 'file_not_exists': {84 const exists = await vfs.fileExists(projectId, assertion.path);85 return { passed: !exists, actual: exists ? 'file exists (unexpected)' : 'file not found (expected)' };86 }87 88 case 'file_contains': {89 const file = await vfs.readFile(projectId, assertion.path);90 const content = typeof file.content === 'string' ? file.content : '';91 const found = content.toLowerCase().includes(assertion.value.toLowerCase());92 return { passed: found, actual: found ? `contains "${truncate(assertion.value, 40)}"` : truncate(content, 80) };93 }94 95 case 'file_not_contains': {96 const file = await vfs.readFile(projectId, assertion.path);97 const content = typeof file.content === 'string' ? file.content : '';98 const found = content.toLowerCase().includes(assertion.value.toLowerCase());99 return { passed: !found, actual: found ? `still contains "${truncate(assertion.value, 40)}"` : 'value absent (expected)' };100 }101 102 case 'file_matches': {103 const file = await vfs.readFile(projectId, assertion.path);104 const content = typeof file.content === 'string' ? file.content : '';105 const re = new RegExp(assertion.pattern, 'i');106 const match = re.exec(content);107 return { passed: !!match, actual: match ? `matched: "${truncate(match[0], 40)}"` : truncate(content, 80) };108 }109 110 case 'file_matches_any': {111 const re = new RegExp(assertion.pattern, 'i');112 for (const filePath of assertion.paths) {113 try {114 const file = await vfs.readFile(projectId, filePath);115 const content = typeof file.content === 'string' ? file.content : '';116 const match = re.exec(content);117 if (match) return { passed: true, actual: `matched in ${filePath}: "${truncate(match[0], 40)}"` };118 } catch {119 // File doesn't exist, try next120 }121 }122 return { passed: false, actual: `pattern not found in any of: ${assertion.paths.join(', ')}` };123 }124 125 case 'valid_json': {126 const file = await vfs.readFile(projectId, assertion.path);127 const content = typeof file.content === 'string' ? file.content : '';128 try {129 JSON.parse(content);130 return { passed: true, actual: 'valid JSON' };131 } catch {132 return { passed: false, actual: `invalid JSON: ${truncate(content, 60)}` };133 }134 }135 136 case 'tool_used': {137 const calls = getToolCalls(conversation);138 let found = calls.some(c => c.name === assertion.toolName);139 if (!found && assertion.toolName === 'write') {140 const fileWritePattern = /^\s*(cat\s*>|sed\s+-i|write\s+|echo\s+.*>)/;141 found = calls.some(c => {142 if (c.name !== 'bash' && c.name !== 'shell') return false;143 try {144 const args = JSON.parse(c.args);145 const cmd = typeof args === 'string' ? args : args.command || args.cmd || '';146 return typeof cmd === 'string' && fileWritePattern.test(cmd);147 } catch {148 return fileWritePattern.test(c.args);149 }150 });151 if (found) return { passed: true, actual: 'file edited via bash command' };152 }153 return {154 passed: found,155 actual: found156 ? `${assertion.toolName} was called`157 : `tools used: ${[...new Set(calls.map(c => c.name))].join(', ') || 'none'}`,158 };159 }160 161 case 'tool_args_match': {162 const calls = getToolCalls(conversation);163 const re = new RegExp(assertion.pattern, 'i');164 const matching = calls.filter(c => c.name === assertion.toolName && re.test(c.args));165 if (matching.length > 0) {166 return { passed: true, actual: `matched args: ${truncate(matching[0].args, 60)}` };167 }168 const toolCalls = calls.filter(c => c.name === assertion.toolName);169 return {170 passed: false,171 actual: toolCalls.length > 0172 ? `${toolCalls.length} ${assertion.toolName} call(s), none matched pattern`173 : `${assertion.toolName} not called`,174 };175 }176 177 case 'output_matches': {178 const text = getAssistantText(conversation);179 const re = new RegExp(assertion.pattern, 'i');180 const match = re.exec(text);181 return { passed: !!match, actual: match ? `matched: "${truncate(match[0], 40)}"` : `no match in ${text.length} chars of output` };182 }183 184 case 'tool_output_matches': {185 const text = getToolOutputText(conversation, assertion.toolName);186 const re = new RegExp(assertion.pattern, 'i');187 const match = re.exec(text);188 return { passed: !!match, actual: match ? `matched: "${truncate(match[0], 40)}"` : `no match in ${text.length} chars of tool output` };189 }190 191 case 'any_of': {192 const subResults: { desc: string; actual?: string }[] = [];193 for (const sub of assertion.assertions) {194 const r = await evaluateOne(sub, projectId, conversation, vfs);195 if (r.passed) return { passed: true, actual: r.actual };196 subResults.push({ desc: sub.description, actual: r.actual });197 }198 return {199 passed: false,200 actual: subResults.map(r => `${r.desc}: ${r.actual}`).join(' | '),201 };202 }203 204 case 'judge':205 return { passed: false, actual: 'judge assertions handled separately' };206 }207}208 209export async function runAssertions(210 projectId: string,211 conversation: ConversationNode[],212 assertions: TestAssertion[]213): Promise<AssertionResult[]> {214 const { vfs } = await import('@/lib/vfs');215 const results: AssertionResult[] = [];216 217 for (const assertion of assertions) {218 if (assertion.type === 'judge') continue;219 220 let result: { passed: boolean; actual?: string };221 try {222 result = await evaluateOne(assertion, projectId, conversation, vfs);223 } catch (err) {224 result = { passed: false, actual: err instanceof Error ? err.message : String(err) };225 }226 227 results.push({ assertion, passed: result.passed, actual: result.actual });228 }229 230 return results;231}232