chore(mcp): render structured response (microsoft#38419)

Author: pavelfeldman

packages/playwright/src/agents/performTask.ts

@@ -25,21 +25,34 @@ import { wrapInClient } from '../mcp/sdk/server';
 import type * as playwright from 'playwright-core';
 import type * as lowireLoop from '@lowire/loop';
 
-export async function performTask(context: playwright.BrowserContext, task: string) {
+export type PerformTaskOptions = {
+  provider?: 'github' | 'openai' | 'anthropic' | 'google';
+  model?: string;
+  maxTokens?: number;
+  reasoning?: boolean;
+  temperature?: number;
+};
+
+export async function performTask(context: playwright.BrowserContext, task: string, options: PerformTaskOptions) {
   const backend = new BrowserServerBackend(defaultConfig, identityBrowserContextFactory(context));
   const client = await wrapInClient(backend, { name: 'Internal', version: '0.0.0' });
-  const loop = new Loop('github', { model: 'claude-sonnet-4.5' });
-
   const callTool: (params: { name: string, arguments: any}) => Promise<lowireLoop.ToolResult> = async params => {
     return await client.callTool(params) as lowireLoop.ToolResult;
   };
 
+  const loop = new Loop(options.provider ?? 'github', {
+    model: options.model ?? 'claude-sonnet-4.5',
+    reasoning: options.reasoning,
+    temperature: options.temperature,
+    maxTokens: options.maxTokens,
+    summarize: true,
+    debug,
+    callTool,
+    tools: await backend.listTools(),
+  });
+
   try {
-    return await loop.run(task, {
-      tools: await backend.listTools(),
-      callTool,
-      debug,
-    });
+    return await loop.run(task);
   } finally {
     await client.close();
   }

packages/playwright/src/index.ts

@@ -36,6 +36,7 @@ import type { APIRequestContext as APIRequestContextImpl } from '../../playwrigh
 import type { ChannelOwner } from '../../playwright-core/src/client/channelOwner';
 import type { Page as PageImpl } from '../../playwright-core/src/client/page';
 import type { BrowserContext, BrowserContextOptions, LaunchOptions, Page, Tracing } from 'playwright-core';
+import type { PerformTaskOptions } from './agents/performTask';
 
 export { expect } from './matchers/expect';
 export const _baseTest: TestType<{}, {}> = rootTestType.test;
@@ -58,7 +59,7 @@ type TestFixtures = PlaywrightTestArgs & PlaywrightTestOptions & {
   _combinedContextOptions: BrowserContextOptions,
   _setupContextOptions: void;
   _setupArtifacts: void;
-  _perform: (task: string) => Promise<void>;
+  _perform: (task: string, options?: PerformTaskOptions) => Promise<void>;
   _contextFactory: (options?: BrowserContextOptions) => Promise<{ context: BrowserContext, close: () => Promise<void> }>;
 };
 
@@ -462,8 +463,8 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures> = ({
   },
 
   _perform: async ({ context }, use) => {
-    await use(async (task: string) => {
-      await performTask(context, task);
+    await use(async (task: string, options?: PerformTaskOptions) => {
+      await performTask(context, task, options ?? {});
     });
   },
 });

packages/playwright/src/mcp/browser/browserServerBackend.ts

@@ -73,7 +73,8 @@ export class BrowserServerBackend implements ServerBackend {
       context.setRunningTool(undefined);
     }
     response.logEnd();
-    return response.serialize();
+    const _meta = rawArguments?._meta as object | undefined;
+    return response.serialize({ _meta });
   }
 
   serverClosed() {

packages/playwright/src/mcp/browser/response.ts

@@ -20,6 +20,7 @@ import { renderModalStates } from './tab';
 import type { Tab, TabSnapshot } from './tab';
 import type { CallToolResult, ImageContent, TextContent } from '@modelcontextprotocol/sdk/types.js';
 import type { Context } from './context';
+import type { ModalState } from './tools/tool';
 
 export const requestDebug = debug('pw:mcp:request');
 
@@ -30,6 +31,7 @@ export class Response {
   private _context: Context;
   private _includeSnapshot: 'none' | 'full' | 'incremental' = 'none';
   private _includeTabs = false;
+  private _includeModalStates: ModalState[] | undefined;
   private _tabSnapshot: TabSnapshot | undefined;
 
   readonly toolName: string;
@@ -87,6 +89,10 @@ export class Response {
     this._includeTabs = true;
   }
 
+  setIncludeModalStates(modalStates: ModalState[]) {
+    this._includeModalStates = modalStates;
+  }
+
   async finish() {
     // All the async snapshotting post-action is happening here.
     // Everything below should race against modal states.
@@ -110,42 +116,44 @@ export class Response {
       requestDebug(this.serialize({ omitSnapshot: true, omitBlobs: true }));
   }
 
-  serialize(options: { omitSnapshot?: boolean, omitBlobs?: boolean } = {}): { content: (TextContent | ImageContent)[], isError?: boolean } {
-    const response: string[] = [];
+  serialize(options: { omitSnapshot?: boolean, omitBlobs?: boolean, _meta?: Record<string, any> } = {}): { content: (TextContent | ImageContent)[], isError?: boolean, _meta?: Record<string, any> } {
+    const renderedResponse = new RenderedResponse();
 
-    // Start with command result.
-    if (this._result.length) {
-      response.push('### Result');
-      response.push(this._result.join('\n'));
-      response.push('');
-    }
+    if (this._result.length)
+      renderedResponse.results.push(...this._result);
 
     // Add code if it exists.
-    if (this._code.length) {
-      response.push(`### Ran Playwright code
-\`\`\`js
-${this._code.join('\n')}
-\`\`\``);
-      response.push('');
-    }
+    if (this._code.length)
+      renderedResponse.code.push(...this._code);
 
     // List browser tabs.
-    if (this._includeSnapshot !== 'none' || this._includeTabs)
-      response.push(...renderTabsMarkdown(this._context.tabs(), this._includeTabs));
+    if (this._includeSnapshot !== 'none' || this._includeTabs) {
+      const tabsMarkdown = renderTabsMarkdown(this._context.tabs(), this._includeTabs);
+      if (tabsMarkdown.length)
+        renderedResponse.states.tabs = tabsMarkdown.join('\n');
+    }
 
     // Add snapshot if provided.
     if (this._tabSnapshot?.modalStates.length) {
-      response.push(...renderModalStates(this._context, this._tabSnapshot.modalStates));
-      response.push('');
+      const modalStatesMarkdown = renderModalStates(this._tabSnapshot.modalStates);
+      renderedResponse.states.modal = modalStatesMarkdown.join('\n');
     } else if (this._tabSnapshot) {
       const includeSnapshot = options.omitSnapshot ? 'none' : this._includeSnapshot;
-      response.push(renderTabSnapshot(this._tabSnapshot, includeSnapshot));
-      response.push('');
+      renderTabSnapshot(this._tabSnapshot, includeSnapshot, renderedResponse);
+    } else if (this._includeModalStates) {
+      const modalStatesMarkdown = renderModalStates(this._includeModalStates);
+      renderedResponse.states.modal = modalStatesMarkdown.join('\n');
     }
 
+    const redactedResponse = this._context.config.secrets ? renderedResponse.redact(this._context.config.secrets) : renderedResponse;
+
+    // Structured response.
+    const includeMeta = options._meta && 'dev.lowire/history' in options._meta && 'dev.lowire/state' in options._meta;
+    const _meta: any = includeMeta ? redactedResponse.asMeta() : undefined;
+
     // Main response part
     const content: (TextContent | ImageContent)[] = [
-      { type: 'text', text: response.join('\n') },
+      { type: 'text', text: redactedResponse.asText() },
     ];
 
     // Image attachments.
@@ -154,50 +162,39 @@ ${this._code.join('\n')}
         content.push({ type: 'image', data: options.omitBlobs ? '<blob>' : image.data.toString('base64'), mimeType: image.contentType });
     }
 
-    this._redactSecrets(content);
-    return { content, isError: this._isError };
-  }
-
-  private _redactSecrets(content: (TextContent | ImageContent)[]) {
-    if (!this._context.config.secrets)
-      return;
-
-    for (const item of content) {
-      if (item.type !== 'text')
-        continue;
-      for (const [secretName, secretValue] of Object.entries(this._context.config.secrets))
-        item.text = item.text.replaceAll(secretValue, `<secret>${secretName}</secret>`);
-    }
+    return {
+      _meta,
+      content,
+      isError: this._isError
+    };
   }
 }
 
-function renderTabSnapshot(tabSnapshot: TabSnapshot, includeSnapshot: 'none' | 'full' | 'incremental'): string {
-  const lines: string[] = [];
-
+function renderTabSnapshot(tabSnapshot: TabSnapshot, includeSnapshot: 'none' | 'full' | 'incremental', response: RenderedResponse) {
   if (tabSnapshot.consoleMessages.length) {
-    lines.push(`### New console messages`);
+    const lines: string[] = [];
     for (const message of tabSnapshot.consoleMessages)
       lines.push(`- ${trim(message.toString(), 100)}`);
-    lines.push('');
+    response.updates.push({ category: 'console', content: lines.join('\n') });
   }
 
   if (tabSnapshot.downloads.length) {
-    lines.push(`### Downloads`);
+    const lines: string[] = [];
     for (const entry of tabSnapshot.downloads) {
       if (entry.finished)
         lines.push(`- Downloaded file ${entry.download.suggestedFilename()} to ${entry.outputFile}`);
       else
         lines.push(`- Downloading file ${entry.download.suggestedFilename()} ...`);
     }
-    lines.push('');
+    response.updates.push({ category: 'downloads', content: lines.join('\n') });
   }
 
   if (includeSnapshot === 'incremental' && tabSnapshot.ariaSnapshotDiff === '') {
     // When incremental snapshot is present, but empty, do not render page state altogether.
-    return lines.join('\n');
+    return;
   }
 
-  lines.push(`### Page state`);
+  const lines: string[] = [];
   lines.push(`- Page URL: ${tabSnapshot.url}`);
   lines.push(`- Page Title: ${tabSnapshot.title}`);
 
@@ -210,29 +207,22 @@ function renderTabSnapshot(tabSnapshot: TabSnapshot, includeSnapshot: 'none' | '
       lines.push(tabSnapshot.ariaSnapshot);
     lines.push('```');
   }
-
-  return lines.join('\n');
+  response.states.page = lines.join('\n');
 }
 
 function renderTabsMarkdown(tabs: Tab[], force: boolean = false): string[] {
   if (tabs.length === 1 && !force)
     return [];
 
-  if (!tabs.length) {
-    return [
-      '### Open tabs',
-      'No open tabs. Use the "browser_navigate" tool to navigate to a page first.',
-      '',
-    ];
-  }
+  if (!tabs.length)
+    return ['No open tabs. Use the "browser_navigate" tool to navigate to a page first.'];
 
-  const lines: string[] = ['### Open tabs'];
+  const lines: string[] = [];
   for (let i = 0; i < tabs.length; i++) {
     const tab = tabs[i];
     const current = tab.isCurrentTab() ? ' (current)' : '';
     lines.push(`- ${i}:${current} [${tab.lastTitle()}] (${tab.page.url()})`);
   }
-  lines.push('');
   return lines;
 }
 
@@ -242,6 +232,86 @@ function trim(text: string, maxLength: number) {
   return text.slice(0, maxLength) + '...';
 }
 
+export class RenderedResponse {
+  readonly states: Partial<Record<'page' | 'tabs' | 'modal', string>> = {};
+  readonly updates: { category: 'console' | 'downloads', content: string }[] = [];
+  readonly results: string[] = [];
+  readonly code: string[] = [];
+
+  constructor(copy?: { states: Partial<Record<'page' | 'tabs' | 'modal', string>>, updates: { category: 'console' | 'downloads', content: string }[], results: string[], code: string[] }) {
+    if (copy) {
+      this.states = copy.states;
+      this.updates = copy.updates;
+      this.results = copy.results;
+      this.code = copy.code;
+    }
+  }
+
+  asText(): string {
+    const text: string[] = [];
+    if (this.results.length)
+      text.push(`### Result\n${this.results.join('\n')}\n`);
+    if (this.code.length)
+      text.push(`### Ran Playwright code\n${this.code.join('\n')}\n`);
+
+    for (const { category, content } of this.updates) {
+      if (!content.trim())
+        continue;
+
+      switch (category) {
+        case 'console':
+          text.push(`### New console messages\n${content}\n`);
+          break;
+        case 'downloads':
+          text.push(`### Downloads\n${content}\n`);
+          break;
+      }
+    }
+
+    for (const [category, value] of Object.entries(this.states)) {
+      if (!value.trim())
+        continue;
+
+      switch (category) {
+        case 'page':
+          text.push(`### Page state\n${value}\n`);
+          break;
+        case 'tabs':
+          text.push(`### Open tabs\n${value}\n`);
+          break;
+        case 'modal':
+          text.push(`### Modal state\n${value}\n`);
+          break;
+      }
+    }
+    return text.join('\n');
+  }
+
+  asMeta() {
+    const codeUpdate = this.code.length ? { category: 'code', content: this.code.join('\n') } : undefined;
+    const resultUpdate = this.results.length ? { category: 'result', content: this.results.join('\n') } : undefined;
+    const updates = [resultUpdate, codeUpdate, ...this.updates].filter(Boolean);
+    return {
+      'dev.lowire/history': updates,
+      'dev.lowire/state': { ...this.states },
+    };
+  }
+
+  redact(secrets: Record<string, string>): RenderedResponse {
+    const redactText = (text: string) => {
+      for (const [secretName, secretValue] of Object.entries(secrets))
+        text = text.replaceAll(secretValue, `<secret>${secretName}</secret>`);
+      return text;
+    };
+
+    const updates = this.updates.map(update => ({ ...update, content: redactText(update.content) }));
+    const results = this.results.map(result => redactText(result));
+    const code = this.code.map(code => redactText(code));
+    const states = Object.fromEntries(Object.entries(this.states).map(([key, value]) => [key, redactText(value)]));
+    return new RenderedResponse({ states, updates, results, code });
+  }
+}
+
 function parseSections(text: string): Map<string, string> {
   const sections = new Map<string, string>();
   const sectionHeaders = text.split(/^### /m).slice(1); // Remove empty first element
@@ -286,5 +356,6 @@ export function parseResponse(response: CallToolResult) {
     downloads,
     isError,
     attachments,
+    _meta: response._meta,
   };
 }

packages/playwright/src/mcp/browser/tab.ts

@@ -132,10 +132,6 @@ export class Tab extends EventEmitter<TabEventsInterface> {
     this._modalStates = this._modalStates.filter(state => state !== modalState);
   }
 
-  modalStatesMarkdown(): string[] {
-    return renderModalStates(this.context, this.modalStates());
-  }
-
   private _dialogShown(dialog: playwright.Dialog) {
     this.setModalState({
       type: 'dialog',
@@ -349,8 +345,8 @@ function pageErrorToConsoleMessage(errorOrValue: Error | any): ConsoleMessage {
   };
 }
 
-export function renderModalStates(context: Context, modalStates: ModalState[]): string[] {
-  const result: string[] = ['### Modal state'];
+export function renderModalStates(modalStates: ModalState[]): string[] {
+  const result: string[] = [];
   if (modalStates.length === 0)
     result.push('- There is no modal state present');
   for (const state of modalStates)