chore: that-should-be-it perform mode (microsoft#38503)

Author: pavelfeldman

package-lock.json

@@ -56,7 +56,7 @@
     "@eslint/compat": "^1.3.2",
     "@eslint/eslintrc": "^3.3.1",
     "@eslint/js": "^9.34.0",
-    "@lowire/loop": "^0.0.6",
+    "@lowire/loop": "^0.0.7",
     "@modelcontextprotocol/sdk": "^1.17.5",
     "@octokit/graphql-schema": "^15.26.0",
     "@stylistic/eslint-plugin": "^5.2.3",

package.json

@@ -4,7 +4,7 @@ THIRD-PARTY SOFTWARE NOTICES AND INFORMATION
 
 This project incorporates components from the projects listed below. The original copyright notices and the licenses under which Microsoft received such components are set forth below. Microsoft reserves all rights not expressly granted herein, whether by implication, estoppel or otherwise.
 
--	@lowire/loop@0.0.6 (https://github.com/pavelfeldman/lowire)
+-	@lowire/loop@0.0.7 (https://github.com/pavelfeldman/lowire)
 -	@modelcontextprotocol/sdk@1.24.2 (https://github.com/modelcontextprotocol/typescript-sdk)
 -	accepts@2.0.0 (https://github.com/jshttp/accepts)
 -	agent-base@7.1.4 (https://github.com/TooTallNate/proxy-agents)
@@ -135,7 +135,7 @@ This project incorporates components from the projects listed below. The origina
 -	zod-to-json-schema@3.25.0 (https://github.com/StefanTerdell/zod-to-json-schema)
 -	zod@3.25.76 (https://github.com/colinhacks/zod)
 
-%% @lowire/loop@0.0.6 NOTICES AND INFORMATION BEGIN HERE
+%% @lowire/loop@0.0.7 NOTICES AND INFORMATION BEGIN HERE
 =========================================
 Apache License
                            Version 2.0, January 2004
@@ -339,7 +339,7 @@ Apache License
    See the License for the specific language governing permissions and
    limitations under the License.
 =========================================
-END OF @lowire/loop@0.0.6 AND INFORMATION
+END OF @lowire/loop@0.0.7 AND INFORMATION
 
 %% @modelcontextprotocol/sdk@1.24.2 NOTICES AND INFORMATION BEGIN HERE
 =========================================

packages/playwright-core/ThirdPartyNotices.txt

@@ -4,7 +4,7 @@
   "private": true,
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.24.0",
-    "@lowire/loop": "^0.0.6",
+    "@lowire/loop": "^0.0.7",
     "zod": "^3.25.76",
     "zod-to-json-schema": "^3.24.6"
   }

packages/playwright-core/bundles/mcp/package-lock.json

@@ -18,7 +18,7 @@ import fs from 'fs';
 
 import { toolsForLoop } from './backend';
 import { debug } from '../../utilsBundle';
-import { Loop, z, zodToJsonSchema } from '../../mcpBundle';
+import { Loop } from '../../mcpBundle';
 import { runAction } from './actionRunner';
 import { Context } from './context';
 
@@ -34,9 +34,7 @@ export async function pagePerform(progress: Progress, page: Page, options: chann
   if (await cachedPerform(context, options))
     return;
 
-  await perform(context, options.task, zodToJsonSchema(z.object({
-    error: z.string().optional().describe('An error message if the task could not be completed successfully'),
-  })) as loopTypes.Schema, options);
+  await perform(context, options.task, undefined, options);
   await updateCache(context, options);
 }
 
@@ -51,7 +49,7 @@ ${options.query}`;
   return await perform(context, task, options.schema, options);
 }
 
-async function perform(context: Context, userTask: string, resultSchema: loopTypes.Schema, options: { maxTurns?: number } = {}): Promise<any> {
+async function perform(context: Context, userTask: string, resultSchema: loopTypes.Schema | undefined, options: { maxTurns?: number } = {}): Promise<any> {
   const { progress, page } = context;
   const browserContext = page.browserContext;
   if (!browserContext._options.agent)
@@ -66,6 +64,13 @@ async function perform(context: Context, userTask: string, resultSchema: loopTyp
     debug,
     callTool,
     tools,
+    beforeTurn: params => {
+      const lastReply = params.conversation.messages.findLast(m => m.role === 'assistant');
+      const toolCall = lastReply?.content.find(c => c.type === 'tool_call');
+      if (!resultSchema && toolCall && toolCall.arguments.thatShouldBeIt)
+        return 'break';
+      return 'continue';
+    },
     ...options
   });
 
@@ -75,9 +80,8 @@ async function perform(context: Context, userTask: string, resultSchema: loopTyp
 ${full}
 `;
 
-  return await loop.run(task, {
-    resultSchema
-  });
+  const { result } = await loop.run(task, { resultSchema });
+  return result;
 }
 
 type CachedActions = Record<string, {
@@ -121,8 +125,7 @@ async function updateCache(context: Context, options: channels.PagePerformParams
 async function cachedActions(cacheFile: string): Promise<CachedActions> {
   let cache = allCaches.get(cacheFile);
   if (!cache) {
-    const text = await fs.promises.readFile(cacheFile, 'utf-8').catch(() => '{}');
-    cache = JSON.parse(text) as CachedActions;
+    cache = await fs.promises.readFile(cacheFile, 'utf-8').then(text => JSON.parse(text)).catch(() => ({})) as CachedActions;
     allCaches.set(cacheFile, cache);
   }
   return cache;

packages/playwright-core/bundles/mcp/package.json

@@ -35,20 +35,24 @@ function defineTool<Input extends zod.Schema>(tool: ToolDefinition<Input>): Tool
   return tool;
 }
 
+const baseSchema = z.object({
+  thatShouldBeIt: z.boolean().describe('Indicates that this tool call is sufficient to complete the task. If false, the task will continue with the next tool call'),
+});
+
 const snapshot = defineTool({
   schema: {
     name: 'browser_snapshot',
     title: 'Page snapshot',
     description: 'Capture accessibility snapshot of the current page, this is better than screenshot',
-    inputSchema: z.object({}),
+    inputSchema: baseSchema,
   },
 
   handle: async (context, params) => {
     return await context.snapshotResult();
   },
 });
 
-const elementSchema = z.object({
+const elementSchema = baseSchema.extend({
   element: z.string().describe('Human-readable element description used to obtain permission to interact with the element'),
   ref: z.string().describe('Exact target element reference from the page snapshot'),
 });
@@ -86,7 +90,7 @@ const drag = defineTool({
     name: 'browser_drag',
     title: 'Drag mouse',
     description: 'Perform drag and drop between two elements',
-    inputSchema: z.object({
+    inputSchema: baseSchema.extend({
       startElement: z.string().describe('Human-readable source element description used to obtain the permission to interact with the element'),
       startRef: z.string().describe('Exact source element reference from the page snapshot'),
       endElement: z.string().describe('Human-readable target element description used to obtain the permission to interact with the element'),
@@ -211,7 +215,7 @@ const fillForm = defineTool({
     name: 'browser_fill_form',
     title: 'Fill form',
     description: 'Fill multiple form fields',
-    inputSchema: z.object({
+    inputSchema: baseSchema.extend({
       fields: z.array(z.object({
         name: z.string().describe('Human-readable field name'),
         type: z.enum(['textbox', 'checkbox', 'radio', 'combobox', 'slider']).describe('Type of the field'),

packages/playwright-core/src/server/agent/agent.ts

@@ -50,12 +50,13 @@ export class Agent<T extends z.ZodSchema<any>> {
     const { clients, tools, callTool } = await this._initClients();
     const prompt = this.spec.description;
     try {
-      return await this.loop.run<z.output<T>>(`${prompt}\n\nTask:\n${task}\n\nParams:\n${JSON.stringify(params, null, 2)}`, {
+      const { result } = await this.loop.run<z.output<T>>(`${prompt}\n\nTask:\n${task}\n\nParams:\n${JSON.stringify(params, null, 2)}`, {
         ...options,
         tools,
         callTool,
         resultSchema: this.resultSchema
       });
+      return result;
     } finally {
       await this._disconnectFromServers(clients);
     }

packages/playwright-core/src/server/agent/tools.ts

@@ -1,6 +1,6 @@
 {
   "Enter x-secret-email into the email field": {
-    "timestamp": 1765318973255,
+    "timestamp": 1765340032886,
     "actions": [
       {
         "method": "fill",
@@ -10,7 +10,7 @@
     ]
   },
   "Fill out the form with the following details:\nName: John Smith\nAddress: 1045 La Avenida St, Mountain View, CA 94043\nEmail: john.smith@at-microsoft.com": {
-    "timestamp": 1765318976120,
+    "timestamp": 1765340035431,
     "actions": [
       {
         "method": "fill",

packages/playwright/src/agents/agent.ts

@@ -24,7 +24,7 @@ import type * as lowireLoop from '@lowire/loop';
 test.use({ mcpHeadless: async ({ headless }, use) => use(headless) });
 
 test('fill the form', async ({ loop, client, server }) => {
-  expect(await loop.run<{ price: number, deliveryDays: number }>(`
+  const { result } = await loop.run<{ price: number, deliveryDays: number }>(`
     Navigate to ${server.PREFIX + '/evals/fill-form.html'} via Playwright MCP.
     Order a blue table and a green desk.
     Use the following details for the order form:
@@ -41,5 +41,6 @@ test('fill the form', async ({ loop, client, server }) => {
       price: z.number().describe('Total price in USD.'),
       deliveryDays: z.number().describe('Maximum estimated delivery time in days.'),
     })) as lowireLoop.Schema,
-  })).toEqual({ price: 653.96, deliveryDays: 3 });
+  });
+  expect(result).toEqual({ price: 653.96, deliveryDays: 3 });
 });