basant307/AI_Governance_Project

1/**2 * @license3 * Copyright 2025 Qwen Team4 * SPDX-License-Identifier: Apache-2.05 */6 7import {8  BaseDeclarativeTool,9  BaseToolInvocation,10  Kind,11  ToolConfirmationOutcome,12  type ToolInvocation,13  type ToolResult,14  type ToolCallConfirmationDetails,15  type ToolConfirmationPayload,16} from '../tools.js';17import type { PermissionDecision } from '../../permissions/types.js';18import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js';19import type { Part, PartListUnion } from '@google/genai';20import { ComputerUseClient } from './client.js';21import type { ComputerUseToolName, ComputerUseToolSchema } from './schemas.js';22import { COMPUTER_USE_SCHEMAS } from './schemas.js';23import { safeJsonStringify } from '../../utils/safeJsonStringify.js';24import { runBootstrap } from './bootstrap.js';25import { isPackageSpecApproved, saveInstallState } from './install-state.js';26import { approvalKey, resolveMaxImageDimension } from './constants.js';27import { type Config } from '../../config/config.js';28import { homedir } from 'node:os';29 30type ComputerUseParams = Record<string, unknown>;31 32const INSTALL_REASON =33  'This downloads the Computer Use driver (~20MB, signed + notarized) into ~/.qwen/computer-use/ the first time. ' +34  'Computer Use can click, type, and read your desktop apps in the background. ' +35  "On macOS you'll be guided through Accessibility / Screen Recording permissions next.";36 37/**38 * Tools / params that perform irreversible or sensitive actions and must NOT be39 * silently auto-approved in AUTO_EDIT mode. They surface a confirmation in40 * AUTO_EDIT; AUTO still routes them through its classifier (getDefaultPermission41 * stays 'ask'); YOLO still auto-approves everything.42 *   - kill_app          force-kills a PID43 *   - launch_app        launches arbitrary apps (incl. with CDP debug ports)44 *   - start_recording   captures the screen to disk45 *   - set_config        mutates driver configuration46 *   - replay_trajectory re-invokes every recorded tool call in a dir via the47 *     same dispatch path — it replays arbitrary actions (kill_app, launch_app,48 *     page execute_javascript, …). Gating the wrapper is the only chokepoint we49 *     have; the replayed sub-actions run inside cua-driver. (review round 2)50 *   - page action 'execute_javascript'           — arbitrary JS in the user's51 *     logged-in browser (cookie / credential exfiltration)52 *   - page action 'enable_javascript_apple_events' — permanently patches the53 *     browser's prefs + quits/relaunches it (more persistent than the one-shot54 *     execute_javascript). (review round 2)55 */56const HIGH_RISK_TOOLS = new Set<ComputerUseToolName>([57  'kill_app',58  'launch_app',59  'start_recording',60  'set_config',61  'replay_trajectory',62]);63 64const HIGH_RISK_PAGE_ACTIONS = new Set([65  'execute_javascript',66  'enable_javascript_apple_events',67]);68 69// Fail fast at module load if a high-risk entry isn't a real tool name. The70// Set<ComputerUseToolName> typing already rejects typos at compile time; this71// also catches the name union drifting from the schema set at runtime. A typo72// would otherwise silently disable the gate for that tool. (review round 3)73for (const t of HIGH_RISK_TOOLS) {74  if (!(t in COMPUTER_USE_SCHEMAS)) {75    throw new Error(`HIGH_RISK_TOOLS contains unknown tool: ${t}`);76  }77}78 79export function isHighRiskCall(80  upstreamName: string,81  params: Record<string, unknown>,82): boolean {83  if (HIGH_RISK_TOOLS.has(upstreamName as ComputerUseToolName)) return true;84  return (85    upstreamName === 'page' &&86    HIGH_RISK_PAGE_ACTIONS.has(params['action'] as string)87  );88}89 90class ComputerUseInvocation extends BaseToolInvocation<91  ComputerUseParams,92  ToolResult93> {94  constructor(95    private readonly upstreamName: ComputerUseToolName,96    params: ComputerUseParams,97    private readonly config?: Config,98  ) {99    super(params);100  }101 102  getDescription(): string {103    return safeJsonStringify(this.params);104  }105 106  /**107   * Always returns 'ask' so every desktop action surfaces through the108   * standard tool-permission dialog. The PermissionManager rule system109   * handles "always allow" per tool via ProceedAlwaysTool — that's the110   * single source of truth for repeat-approval behavior.111   *112   * Earlier this returned 'allow' once the install-state file existed,113   * which conflated install approval with per-action approval and114   * effectively granted blanket permission for all 9 computer_use__*115   * tools (including mutating actions like click / type_text / drag)116   * after the first install confirmation. See PR #4590 review for the117   * full discussion.118   */119  override async getDefaultPermission(): Promise<PermissionDecision> {120    return 'ask';121  }122 123  /**124   * Builds the confirmation dialog. Two variants:125   *126   * 1. Install not yet approved → show install info (download size,127   *    permission flow to follow). onConfirm writes the install state128   *    so runBootstrap() inside execute() skips its env-var fallback129   *    prompt for headless contexts.130   *131   * 2. Install already approved → show per-action info (which tool +132   *    which args) so the user can decide whether THIS specific action133   *    is OK to perform.134   *135   * Both variants set permissionRules so the standard "Always allow"136   * outcomes (ProceedAlwaysTool / ProceedAlwaysUser / ProceedAlwaysProject)137   * add a rule via PermissionManager — subsequent calls of the SAME138   * tool then skip the dialog. Different tools each need their own139   * "always allow" choice; install approval no longer grants blanket140   * access.141   *142   * On Cancel: install state is NOT written; execute() / runBootstrap()143   * will use the env-var fallback (QWEN_COMPUTER_USE_AUTO_APPROVE),144   * which defaults to refusing — producing a clear error message.145   */146  override async getConfirmationDetails(147    _abortSignal: AbortSignal,148  ): Promise<ToolCallConfirmationDetails> {149    const permissionRules = [`computer_use__${this.upstreamName}`];150    const installApproved = await isPackageSpecApproved(151      homedir(),152      approvalKey(),153    );154 155    const onConfirm = async (156      outcome: ToolConfirmationOutcome,157      _payload?: ToolConfirmationPayload,158    ) => {159      // Any non-Cancel outcome means the user approved THIS call. Write install160      // state (idempotent) so runBootstrap() can skip its env-var fallback161      // prompt. PermissionManager handles per-tool "always allow" via162      // permissionRules — install state is no longer a blanket grant.163      if (outcome !== ToolConfirmationOutcome.Cancel) {164        await saveInstallState(homedir(), {165          approvedPackageSpec: approvalKey(),166          approvedAtIso: new Date().toISOString(),167        });168      }169    };170 171    // High-risk calls (review round 1) surface as 'mcp' type so AUTO_EDIT does172    // NOT silently auto-approve them — isAutoEditApproved() only auto-approves173    // 'edit'/'info'. AUTO still routes them through its classifier (this tool's174    // getDefaultPermission stays 'ask'); YOLO still auto-approves everything.175    if (isHighRiskCall(this.upstreamName, this.params)) {176      // NOTE: args are deliberately NOT folded into `title` — no mcp177      // confirmation surface (TUI / non-interactive / ACP) renders the mcp178      // title, so it would be dead text. The args reach the user via the179      // tool-header line (getDescription()). The gate's job is forcing the180      // confirmation (mcp type → not AUTO_EDIT-auto-approved). (review round 3)181      return {182        type: 'mcp',183        title: installApproved184          ? `Allow high-risk Computer Use (${this.upstreamName})`185          : `Allow high-risk Computer Use (${this.upstreamName}) — first use also downloads the driver`,186        serverName: 'cua-driver',187        toolName: this.upstreamName,188        toolDisplayName: `computer_use__${this.upstreamName}`,189        permissionRules,190        onConfirm,191      };192    }193 194    // Non-high-risk: 'info'. The install variant is a SUPERSET — always show195    // Args (the first call can be a mutating action the user must see), then196    // append INSTALL_REASON when install isn't yet approved. (review round 1)197    const argsJson = safeJsonStringify(this.params);198    const prompt = installApproved199      ? `Tool: computer_use__${this.upstreamName}\n\nArgs: ${argsJson}\n\nThis will act on your desktop via the Computer Use binary.`200      : `Tool: computer_use__${this.upstreamName}\n\nArgs: ${argsJson}\n\n${INSTALL_REASON}`;201 202    return {203      type: 'info',204      title: `Allow Computer Use (${this.upstreamName})`,205      prompt,206      permissionRules,207      onConfirm,208    };209  }210 211  async execute(212    signal: AbortSignal,213    updateOutput?: (output: string) => void,214  ): Promise<ToolResult> {215    const client = ComputerUseClient.shared();216 217    // Push the configured screenshot longest-edge cap (setting + env override)218    // onto the shared client BEFORE start: it is applied via set_config once the219    // driver connects (and re-applied on reconnect). undefined → leave the220    // driver default. Cheap + idempotent to set on every call.221    client.setMaxImageDimension(222      resolveMaxImageDimension(this.config?.getComputerUseMaxImageDimension()),223    );224    client.setIdleTimeoutMs(this.config?.getComputerUseIdleTimeoutMs());225 226    // If the user confirmed through the pre-execution dialog, the install state227    // was already written by onConfirm — runBootstrap will skip promptInstallApproval.228    // But several approval modes auto-approve the tool call and bypass that229    // dialog entirely (so onConfirm never runs and install state is never230    // written): YOLO (needsConfirmation() returns false), AUTO_EDIT231    // (isAutoEditApproved() auto-approves info-type tools — all computer_use__*232    // tools are info), and AUTO (classifier-approved calls). In those modes233    // pass autoApproveInstall so the bootstrap honors the already-granted call234    // approval instead of refusing with "install declined by user". DEFAULT235    // still shows the dialog; PLAN blocks. Headless / SDK contexts (no config)236    // fall back to the env-var path in bootstrap's default promptInstallApproval.237    // Reaching execute() means the scheduler already approved THIS call — via238    // the confirmation dialog, a persisted always-allow rule, or an auto-approve239    // mode (YOLO / AUTO_EDIT / AUTO). Treat any of those as install consent. The240    // subtle case is a saved always-allow rule: it SUPPRESSES the dialog, so241    // onConfirm never writes install-state, and in DEFAULT mode bootstrap would242    // then fall into the headless refuse path and dead-end ("install declined")243    // on every retry. Headless / SDK contexts (no config) keep the env-var244    // fallback in bootstrap's default promptInstallApproval. (review round 1)245    const autoApproveInstall = !!this.config;246    await runBootstrap(client, { signal, updateOutput, autoApproveInstall });247 248    let mcpResult: CallToolResult;249    try {250      mcpResult = await client.callTool(this.upstreamName, this.params);251    } catch (err) {252      const message = err instanceof Error ? err.message : String(err);253      return {254        llmContent: `Computer Use tool '${this.upstreamName}' failed: ${message}`,255        returnDisplay: `Error: ${message}`,256        error: { message },257      };258    }259 260    // Transform MCP content blocks into GenAI Parts, preserving image/audio261    // parts so the model can actually "see" screenshots from get_window_state.262    // We also forward cua-driver's `structuredContent`: several tools put the263    // load-bearing data ONLY there, not in the human-readable `content` text —264    // e.g. list_windows' content is just "Found N window(s)" while the real265    // window_id / bounds / is_on_screen live in structuredContent.windows.266    // Dropping it left the model guessing window_ids and failing every267    // screenshot/click on the wrong window.268    // NOTE: mcp-tool.ts has an analogous private transformation (transformMcpContentToParts /269    // transformImageAudioBlock); those helpers are not exported so we replicate270    // the pattern here. A future PR should extract a shared utility.271    const llmContent = buildLlmContent(272      mcpResult.content,273      this.upstreamName,274      mcpResult.structuredContent,275    );276    const returnDisplay = buildDisplayText(mcpResult.content);277 278    if (mcpResult.isError) {279      const errorText =280        returnDisplay || `Tool '${this.upstreamName}' returned isError=true`;281      return {282        llmContent: llmContent || errorText,283        returnDisplay: errorText,284        error: { message: errorText },285      };286    }287 288    return {289      llmContent,290      returnDisplay,291    };292  }293}294 295export class ComputerUseTool extends BaseDeclarativeTool<296  ComputerUseParams,297  ToolResult298> {299  constructor(300    private readonly upstreamName: ComputerUseToolName,301    schema: ComputerUseToolSchema,302    private readonly config?: Config,303  ) {304    const qwenName = `computer_use__${upstreamName}`;305    super(306      qwenName,307      qwenName, // displayName == name; no MCP branding in UI308      schema.description,309      Kind.Other,310      schema.parameterSchema,311      true, // isOutputMarkdown — many results are JSON-ish text or screenshots312      true, // canUpdateOutput — bootstrap streams progress313      true, // shouldDefer — surface only via ToolSearch314      false, // alwaysLoad315      `computer use desktop click type screenshot mouse keyboard scroll drag automation gui app native`,316    );317  }318 319  /**320   * Coerce parameter types before schema validation.321   * Models can send the wrong JS type for a field:322   *  - qwen3.6 sends `element_index: 2` (number) but upstream wants "2" (string)323   *  - Some models send `x: "500"` (string) but upstream wants 500 (number)324   * Pre-coercing avoids spurious validation failures without loosening schema types.325   */326  override validateToolParams(params: ComputerUseParams): string | null {327    const coerced = coerceTypes(328      params,329      this.parameterSchema as Record<string, unknown>,330    );331    return super.validateToolParams(coerced as ComputerUseParams);332  }333 334  override build(335    params: ComputerUseParams,336  ): ToolInvocation<ComputerUseParams, ToolResult> {337    const coerced = coerceTypes(338      params,339      this.parameterSchema as Record<string, unknown>,340    );341    return super.build(coerced as ComputerUseParams);342  }343 344  protected createInvocation(345    params: ComputerUseParams,346  ): ToolInvocation<ComputerUseParams, ToolResult> {347    return new ComputerUseInvocation(this.upstreamName, params, this.config);348  }349}350 351/**352 * Walk schema properties and coerce values to the type declared by the schema.353 *354 * Direction 1 (string → number): schema says integer/number, model sent a355 * numeric string (e.g. `x: "500"`). Garbage strings are left untouched so356 * they still fail schema validation with a clear error.357 *358 * Direction 2 (number → string): schema says string, model sent a number359 * (e.g. `element_index: 2` when upstream expects `"2"`). Coerce via String().360 */361export function coerceTypes(362  params: Record<string, unknown>,363  schema: Record<string, unknown>,364): Record<string, unknown> {365  const properties = (366    schema as { properties?: Record<string, { type?: string }> }367  ).properties;368  if (!properties) return params;369  const result: Record<string, unknown> = { ...params };370  for (const [key, value] of Object.entries(result)) {371    const fieldType = properties[key]?.type;372    // Direction 1: string value, schema wants integer/number → parse373    if (374      (fieldType === 'integer' || fieldType === 'number') &&375      typeof value === 'string'376    ) {377      const trimmed = value.trim();378      // Only coerce if the string is a clean numeric — don't swallow garbage.379      const matchesType =380        fieldType === 'integer'381          ? /^-?\d+$/.test(trimmed)382          : /^-?\d+(\.\d+)?$/.test(trimmed);383      if (matchesType) {384        const parsed =385          fieldType === 'integer' ? parseInt(trimmed, 10) : parseFloat(trimmed);386        if (Number.isFinite(parsed)) {387          result[key] = parsed;388        }389      }390    }391    // Direction 2: number value, schema wants string → stringify392    // (qwen3.6 sometimes sends element_index: 2 instead of "2")393    else if (fieldType === 'string' && typeof value === 'number') {394      result[key] = String(value);395    }396  }397  return result;398}399 400/**401 * @deprecated Use coerceTypes instead. Kept for backward compatibility.402 */403export const coerceNumericStrings = coerceTypes;404 405// ---------------------------------------------------------------------------406// Content transformation helpers407// ---------------------------------------------------------------------------408 409type RawContentBlock = CallToolResult['content'][number];410 411/**412 * Converts MCP content blocks to a GenAI PartListUnion.413 * - Text-only results → plain string (preserves existing caller expectations).414 * - Mixed or image/audio results → Part[] so the model can see screenshots.415 */416export function buildLlmContent(417  content: RawContentBlock[],418  toolName: string,419  structuredContent?: unknown,420): PartListUnion {421  const parts: Part[] = [];422 423  for (const block of content) {424    if (block.type === 'text' && block.text) {425      parts.push({ text: block.text });426    } else if (427      (block.type === 'image' || block.type === 'audio') &&428      block.mimeType &&429      block.data430    ) {431      parts.push({432        text: `[Tool '${toolName}' provided the following ${block.type} data with mime-type: ${block.mimeType}]`,433      });434      parts.push({435        inlineData: {436          mimeType: block.mimeType,437          data: block.data,438        },439      });440    }441    // Other block types (resource, resource_link, etc.) are currently ignored442    // for computer-use; extend here if the MCP server introduces them.443  }444 445  // Forward structuredContent (real window_ids, bounds, on-screen flags, etc.)446  // that the terse `content` text omits. Strip `tree_markdown` first — that447  // field is get_window_state's AX tree, already rendered into the `content`448  // text above, so re-emitting it here would roughly double the token cost.449  const structuredText = stringifyStructured(structuredContent);450  if (structuredText) {451    parts.push({ text: `Structured result: ${structuredText}` });452  }453 454  // If every part is a text Part, collapse to a plain string so callers that455  // do string operations on llmContent (e.g. error-path concatenation) keep456  // working without changes.457  const hasNonText = parts.some((p) => p.inlineData !== undefined);458  if (!hasNonText) {459    return parts460      .map((p) => p.text ?? '')461      .filter(Boolean)462      .join('\n');463  }464 465  return parts;466}467 468/**469 * Builds the human-readable display string (text only, no binary data).470 */471export function buildDisplayText(content: RawContentBlock[]): string {472  return content473    .map((block) => (block.type === 'text' ? (block.text ?? '') : ''))474    .filter(Boolean)475    .join('\n');476}477 478/**479 * Serialize a tool result's `structuredContent` for the model, dropping the480 * `tree_markdown` field (get_window_state's AX tree, already present in the481 * `content` text — re-emitting it would roughly double the token cost).482 * Returns undefined when there is nothing useful to forward.483 */484export function stringifyStructured(structured: unknown): string | undefined {485  if (!structured || typeof structured !== 'object') return undefined;486  const rest: Record<string, unknown> = {};487  for (const [k, v] of Object.entries(structured as Record<string, unknown>)) {488    if (k === 'tree_markdown') continue;489    rest[k] = v;490  }491  if (Object.keys(rest).length === 0) return undefined;492  return safeJsonStringify(rest);493}494