diff --git a/.changeset/browser-act-replay.md b/.changeset/browser-act-replay.md new file mode 100644 index 0000000..88a5173 --- /dev/null +++ b/.changeset/browser-act-replay.md @@ -0,0 +1,13 @@ +--- +"@upstash/box": minor +--- + +feat: replay a resolved browser action with `tab.act(action)` (no LLM, no key) + +`observe()` now returns each element's suggested `method` and `arguments` +alongside `selector`, and `act()` accepts a pre-resolved action +(`BrowserObserveElement` or `BrowserActAction`) in addition to a natural-language +string. Passing an action replays it deterministically: no LLM call, no tokens, +and no model provider key required. Resolve a step once with `observe()`, cache +the returned action, and replay it across pages or runs. A new `BrowserAction` +type (`BrowserObserveElement | BrowserActAction`) is exported. diff --git a/.changeset/remove-browser-run.md b/.changeset/remove-browser-run.md new file mode 100644 index 0000000..c95424b --- /dev/null +++ b/.changeset/remove-browser-run.md @@ -0,0 +1,7 @@ +--- +"@upstash/box": minor +--- + +**Breaking:** remove `tab.run()` (the autonomous multi-step browser agent) and the `BrowserRunOptions` / `BrowserRunResult` / `BrowserRunStep` types. + +Stagehand v4 removed the underlying agent primitive, so the DOM-aware browser now exposes `observe`, `act`, and `extract` only. For multi-step goals, drive `act` / `observe` from your own loop (replay a resolved step with `act(action)` for no-LLM, no-key execution), or connect over CDP with Playwright / Puppeteer. diff --git a/packages/python-sdk/CHANGELOG.md b/packages/python-sdk/CHANGELOG.md index f2cec06..810a69c 100644 --- a/packages/python-sdk/CHANGELOG.md +++ b/packages/python-sdk/CHANGELOG.md @@ -4,6 +4,17 @@ All notable changes to `upstash-box` (Python) are documented here. ## Unreleased +- **Removed** `tab.run()` (the autonomous multi-step browser agent) and the + `BrowserRunResult` / `BrowserRunStep` types. Stagehand v4 removed the agent + primitive, so the browser exposes `observe` / `act` / `extract` only. For + multi-step goals, drive `act` / `observe` from your own loop (replay a + resolved step with `act(action)`), or drive Playwright / Puppeteer over CDP. + Mirrors `@upstash/box`. +- `tab.act(action)` — replay a pre-resolved action from `observe()` + deterministically, with no LLM call, no tokens, and no model provider key + required (pass a `BrowserObserveElement` or `BrowserActAction` instead of a + string; `model` is ignored in that form). `BrowserObserveElement` now also + carries `method` and `arguments`. Mirrors `act(action)` in `@upstash/box`. - `browser.recordings.download(recording_id, path=...)` — save a recording's video to a local file (streamed to disk, parent directories created as needed) and return the path written. Recordings download as MP4; recordings diff --git a/packages/python-sdk/PARITY.md b/packages/python-sdk/PARITY.md index fd18f05..91b782b 100644 --- a/packages/python-sdk/PARITY.md +++ b/packages/python-sdk/PARITY.md @@ -51,7 +51,6 @@ JS `Run`/`StreamRun` → Python `Run`/`StreamRun` (+ `AsyncRun`/`AsyncStreamRun` | `goto`, `content`, `observe`, `act`, `close` | same | | `screenshot({type, fullPage})` | `screenshot(encoding=, full_page=)` — see drift table | | `extract(instruction, schema, options?)` | `extract(instruction, schema, *, model=None)` | -| `run(prompt, options?)` | `run(prompt, *, schema=None, max_steps=None, model=None)` | | `liveViewUrl` | `live_view_url` | | `id`, `url`, `title` | `id`, `url`, `title` | @@ -94,7 +93,6 @@ statics `create`, `from_snapshot`, `get_by_name`, `delete_boxes`, | timeouts in **milliseconds** | Matches the JS SDK units. | | agent `options` keys are **snake_case** (Python) vs camelCase (JS) | Pythonic public API; the SDK converts to the backend's per-harness casing (Claude Code / OpenCode → camelCase, Codex → snake_case). | | `StreamRun.aclose()` needed for `detached` on early break | Python doesn't run generator `finally` on `break` (JS `for await` does). | -| `BrowserRunOptions.prompt` (JS, deprecated) | Legacy `run({prompt})` overload — not ported; Python takes the prompt as the first argument only. | | Browser `schema` = Pydantic model or raw dict (Python) vs Zod (JS) | Same `ResponseSchema` contract as `agent.run`; raw dicts skip client-side validation. | | `screenshot` `type: "png"\|"base64"` (JS) → `encoding: "bytes"\|"base64"` (Python) | Python returns native `bytes`; `encoding` matches `files.read` naming. | | `browser` on `from_snapshot` (Python) | Python's shared create-body builder forwards `browser=True` on `from_snapshot`; JS `fromSnapshot` currently omits it (JS gap). | diff --git a/packages/python-sdk/README.md b/packages/python-sdk/README.md index 4264255..9586717 100644 --- a/packages/python-sdk/README.md +++ b/packages/python-sdk/README.md @@ -260,7 +260,6 @@ class Headline(BaseModel): data = await tab.extract("Get the top headline", Headline) actions = await tab.observe("What can I click?") await tab.act("Click the first headline") -result = await tab.run("Find the top comment and summarize it", max_steps=10) print(await tab.live_view_url()) # watch the tab live print(await box.browser.cdp_url()) # connect Playwright/Puppeteer over CDP diff --git a/packages/python-sdk/examples/browser.py b/packages/python-sdk/examples/browser.py index 0acebb6..92e443f 100644 --- a/packages/python-sdk/examples/browser.py +++ b/packages/python-sdk/examples/browser.py @@ -1,7 +1,7 @@ """Headless browser — box.browser.* on a lightweight box. Creating a box with ``browser=True`` provisions Chromium; all page operations -(content, screenshots, AI extract/act/run) work headless. Mirrors the JS +(content, screenshots, AI extract/act) work headless. Mirrors the JS examples ``browser.ts`` / ``headless-browser.ts``. Run: python examples/browser.py (needs UPSTASH_BOX_API_KEY) diff --git a/packages/python-sdk/tests/_async/test_box_browser.py b/packages/python-sdk/tests/_async/test_box_browser.py index e38e331..ee6bd10 100644 --- a/packages/python-sdk/tests/_async/test_box_browser.py +++ b/packages/python-sdk/tests/_async/test_box_browser.py @@ -261,87 +261,62 @@ async def test_act_executes_one_action(): await box.aclose() -class Person(BaseModel): - name: str - headline: str - profile_url: str - +@respx.mock +async def test_act_rejects_action_without_selector(): + from upstash_box import BrowserObserveElement -class People(BaseModel): - people: list[Person] + box = await make_async_box(respx.mock) + with pytest.raises(BoxError): + await box.browser.get_tab("tab-2").act(BrowserObserveElement(description="unresolved")) + await box.aclose() @respx.mock -async def test_run_with_schema_validated_structured_output(): +async def test_act_replays_pre_resolved_action(): + from upstash_box import BrowserActAction + box = await make_async_box(respx.mock) - run = respx.post(f"{BASE}/browser/run").mock( + act = respx.post(f"{BASE}/browser/act").mock( return_value=httpx.Response( 200, json={ - "result": "Found five people", - "data": { - "people": [ - { - "name": f"Founder {i + 1}", - "headline": "AI founder in Berlin", - "profile_url": f"https://linkedin.com/in/founder-{i + 1}", - } - for i in range(5) - ] - }, - "completed": True, - "steps": [{"step": 1, "action": "search", "url": "https://linkedin.com/search"}], - "step_count": 1, - "input_tokens": 100, - "output_tokens": 25, + "success": True, + "message": "done", + "action_description": "Sign in", + "actions": [], + "input_tokens": 0, + "output_tokens": 0, }, ) ) - result = await box.browser.get_tab("tab-2").run( - "Find five AI founders in Berlin", schema=People, max_steps=25 - ) - - assert isinstance(result.data, People) - assert len(result.data.people) == 5 - assert result.completed is True - assert result.steps[0].action == "search" - body = last_json_body(run) - assert body["prompt"] == "Find five AI founders in Berlin" - assert body["tab"] == "tab-2" - assert body["max_steps"] == 25 - assert body["schema"]["type"] == "object" - await box.aclose() - - -@respx.mock -async def test_run_without_schema(): - box = await make_async_box(respx.mock) - run = respx.post(f"{BASE}/browser/run").mock( - return_value=httpx.Response( - 200, json={"result": "done", "completed": True, "steps": [], "step_count": 3} - ) + action = BrowserActAction( + selector="xpath=/html/body/button", description="Sign in", method="click", arguments=[] ) + result = await box.browser.get_tab("tab-2").act(action) - result = await box.browser.get_tab("tab-1").run("Do the thing") - - assert result.data is None - assert result.result == "done" - assert result.completed is True - assert result.step_count == 3 - assert last_json_body(run) == {"prompt": "Do the thing", "tab": "tab-1"} + assert result.success is True + assert result.input_tokens == 0 + # Posts a pre-resolved action, never an instruction. + assert last_json_body(act) == { + "action": { + "selector": "xpath=/html/body/button", + "description": "Sign in", + "method": "click", + "arguments": [], + }, + "tab": "tab-2", + } await box.aclose() @respx.mock -async def test_rejects_non_schema_for_extract_and_run(): +async def test_rejects_non_schema_for_extract(): box = await make_async_box(respx.mock) tab = box.browser.get_tab("tab-1") with pytest.raises(BoxError, match="extract requires"): await tab.extract("get data", "not-a-schema") # type: ignore[arg-type] - with pytest.raises(BoxError, match="run requires"): - await tab.run("go", schema="not-a-schema") # type: ignore[arg-type] await box.aclose() diff --git a/packages/python-sdk/upstash_box/__init__.py b/packages/python-sdk/upstash_box/__init__.py index 625c965..affd52c 100644 --- a/packages/python-sdk/upstash_box/__init__.py +++ b/packages/python-sdk/upstash_box/__init__.py @@ -66,8 +66,6 @@ BrowserObserveResult, BrowserRecording, BrowserRecordingMarker, - BrowserRunResult, - BrowserRunStep, Chunk, ClaudeCode, ClaudeCodeAgentOptions, @@ -219,8 +217,6 @@ "BrowserObserveResult", "BrowserRecording", "BrowserRecordingMarker", - "BrowserRunResult", - "BrowserRunStep", "EphemeralBoxData", "FileEntry", "GitCommitResult", diff --git a/packages/python-sdk/upstash_box/_async/client.py b/packages/python-sdk/upstash_box/_async/client.py index c969362..fe03e66 100644 --- a/packages/python-sdk/upstash_box/_async/client.py +++ b/packages/python-sdk/upstash_box/_async/client.py @@ -42,11 +42,12 @@ BoxData, BoxGetOptions, BoxRunData, + BrowserActAction, BrowserActResult, BrowserContent, + BrowserObserveElement, BrowserObserveResult, BrowserRecording, - BrowserRunResult, Chunk, CodeLanguage, CustomHarnessConfig, @@ -589,55 +590,32 @@ async def observe( ) return BrowserObserveResult.model_validate({"elements": resp.get("elements") or []}) - async def act(self, instruction: str, *, model: Optional[str] = None) -> BrowserActResult: - """Resolve and execute one natural-language action on this tab (metered).""" - body: Dict[str, Any] = {"instruction": instruction, "tab": self.id} - if model: - body["model"] = model - resp = await self._box._request( - "POST", - f"/v2/box/{self._box.id}/browser/act", - body=body, - timeout=180000, - ) - return BrowserActResult.model_validate(resp) - - async def run( + async def act( self, - prompt: str, + instruction: Union[str, BrowserObserveElement, BrowserActAction], *, - schema: Optional[ResponseSchema] = None, - max_steps: Optional[int] = None, model: Optional[str] = None, - ) -> BrowserRunResult: - """Autonomously complete a multi-step task on this tab (metered). + ) -> BrowserActResult: + """Resolve and execute one action on this tab. - Runs a DOM-aware browser agent (Stagehand) inside the box: it reads the - page, acts, and repeats until done. ``max_steps`` defaults to 15 - (max 30). Needs a key for the model's provider on the box or account. + Pass a string (LLM-resolved, metered) or a pre-resolved ``observe()`` + action to replay it with no LLM call and no key (``model`` ignored). """ - json_schema = common.to_json_schema(schema) if schema is not None else None - if schema is not None and json_schema is None: - raise BoxError("run requires a pydantic model class or a JSON-schema dict") - body: Dict[str, Any] = {"prompt": prompt, "tab": self.id} - if json_schema is not None: - body["schema"] = json_schema - if max_steps: - body["max_steps"] = max_steps - if model: - body["model"] = model + if isinstance(instruction, str): + body: Dict[str, Any] = {"instruction": instruction, "tab": self.id} + if model: + body["model"] = model + else: + if not instruction.selector: + raise BoxError("act(action) requires a selector; observe() did not resolve one") + body = {"action": instruction.model_dump(exclude_none=True), "tab": self.id} resp = await self._box._request( "POST", - f"/v2/box/{self._box.id}/browser/run", + f"/v2/box/{self._box.id}/browser/act", body=body, - timeout=600000, - ) - data = ( - common.validate_structured_data(schema, resp.get("data")) - if schema is not None - else None + timeout=180000, ) - return BrowserRunResult.model_validate({**resp, "data": data}) + return BrowserActResult.model_validate(resp) async def live_view_url(self) -> str: """Live-view URL for this tab (authenticated via a token in the URL). diff --git a/packages/python-sdk/upstash_box/_sync/client.py b/packages/python-sdk/upstash_box/_sync/client.py index 34dd199..f6c7aae 100644 --- a/packages/python-sdk/upstash_box/_sync/client.py +++ b/packages/python-sdk/upstash_box/_sync/client.py @@ -41,11 +41,12 @@ BoxData, BoxGetOptions, BoxRunData, + BrowserActAction, BrowserActResult, BrowserContent, + BrowserObserveElement, BrowserObserveResult, BrowserRecording, - BrowserRunResult, Chunk, CodeLanguage, CustomHarnessConfig, @@ -582,55 +583,32 @@ def observe(self, instruction: str, *, model: Optional[str] = None) -> BrowserOb ) return BrowserObserveResult.model_validate({"elements": resp.get("elements") or []}) - def act(self, instruction: str, *, model: Optional[str] = None) -> BrowserActResult: - """Resolve and execute one natural-language action on this tab (metered).""" - body: Dict[str, Any] = {"instruction": instruction, "tab": self.id} - if model: - body["model"] = model - resp = self._box._request( - "POST", - f"/v2/box/{self._box.id}/browser/act", - body=body, - timeout=180000, - ) - return BrowserActResult.model_validate(resp) - - def run( + def act( self, - prompt: str, + instruction: Union[str, BrowserObserveElement, BrowserActAction], *, - schema: Optional[ResponseSchema] = None, - max_steps: Optional[int] = None, model: Optional[str] = None, - ) -> BrowserRunResult: - """Autonomously complete a multi-step task on this tab (metered). + ) -> BrowserActResult: + """Resolve and execute one action on this tab. - Runs a DOM-aware browser agent (Stagehand) inside the box: it reads the - page, acts, and repeats until done. ``max_steps`` defaults to 15 - (max 30). Needs a key for the model's provider on the box or account. + Pass a string (LLM-resolved, metered) or a pre-resolved ``observe()`` + action to replay it with no LLM call and no key (``model`` ignored). """ - json_schema = common.to_json_schema(schema) if schema is not None else None - if schema is not None and json_schema is None: - raise BoxError("run requires a pydantic model class or a JSON-schema dict") - body: Dict[str, Any] = {"prompt": prompt, "tab": self.id} - if json_schema is not None: - body["schema"] = json_schema - if max_steps: - body["max_steps"] = max_steps - if model: - body["model"] = model + if isinstance(instruction, str): + body: Dict[str, Any] = {"instruction": instruction, "tab": self.id} + if model: + body["model"] = model + else: + if not instruction.selector: + raise BoxError("act(action) requires a selector; observe() did not resolve one") + body = {"action": instruction.model_dump(exclude_none=True), "tab": self.id} resp = self._box._request( "POST", - f"/v2/box/{self._box.id}/browser/run", + f"/v2/box/{self._box.id}/browser/act", body=body, - timeout=600000, - ) - data = ( - common.validate_structured_data(schema, resp.get("data")) - if schema is not None - else None + timeout=180000, ) - return BrowserRunResult.model_validate({**resp, "data": data}) + return BrowserActResult.model_validate(resp) def live_view_url(self) -> str: """Live-view URL for this tab (authenticated via a token in the URL). diff --git a/packages/python-sdk/upstash_box/types.py b/packages/python-sdk/upstash_box/types.py index aba5c74..8c73f2c 100644 --- a/packages/python-sdk/upstash_box/types.py +++ b/packages/python-sdk/upstash_box/types.py @@ -783,6 +783,9 @@ class BrowserObserveElement(_Model): # A selector for the element (Stagehand-resolved), when available. selector: Optional[str] = None url: Optional[str] = None + # Suggested method and args, for replay via ``tab.act(element)``. + method: Optional[str] = None + arguments: Optional[List[str]] = None class BrowserObserveResult(_Model): @@ -812,43 +815,16 @@ class BrowserActResult(_Model): output_tokens: int = 0 -class BrowserRunStep(_Model): - """One turn of a ``tab.run()`` loop.""" - - step: int - action: Optional[str] = None - reasoning: Optional[str] = None - url: Optional[str] = None - - -class BrowserRunResult(_Model): - """Result of ``tab.run()`` — the agent's outcome after the loop. - - ``data`` holds the structured output: an instance of the schema when a - pydantic model class was supplied, the raw value for a dict schema, or - ``None`` when no schema was given. - """ - - data: Any = None - result: str = "" - # Whether the agent reported the task complete (vs. hit max_steps). - completed: bool = False - steps: List[BrowserRunStep] = [] - step_count: int = 0 - input_tokens: int = 0 - output_tokens: int = 0 - - class BrowserRecordingMarker(_Model): """A labeled point (or span) on a recording's timeline.""" - # "tab_switch" (recorder-observed) or "run" (a ``tab.run`` chapter). - type: Literal["tab_switch", "run"] = "tab_switch" + # "tab_switch" (recorder-observed). + type: Literal["tab_switch"] = "tab_switch" # Offset from the start of the recording, in milliseconds. at_ms: int = 0 - # For spans (runs): end offset in milliseconds. + # End offset in milliseconds, for span markers. end_ms: Optional[int] = None - # Tab title/URL for switches; the prompt for runs. + # Tab title/URL for switches. label: Optional[str] = None tab_id: Optional[str] = None diff --git a/packages/sdk/examples/browser.ts b/packages/sdk/examples/browser.ts index 9674fee..0f5bc2b 100644 --- a/packages/sdk/examples/browser.ts +++ b/packages/sdk/examples/browser.ts @@ -38,22 +38,6 @@ try { const action = await tab.act("click the Files tab"); console.log("acted:", action.actionDescription); - // Complete a multi-step task and return schema-validated structured data. - const run = await tab.run("Collect five useful repository navigation links from this page", { - schema: z.object({ - links: z - .array( - z.object({ - title: z.string(), - url: z.string(), - }), - ) - .length(5), - }), - maxSteps: 25, - }); - console.log("run:", run.completed, run.data.links); - // Open a second tab, screenshot it, then list and close tabs. const search = await box.browser.tab.create("https://html.duckduckgo.com/html/"); const shot = await search.screenshot({ fullPage: true }); diff --git a/packages/sdk/examples/browser/README.md b/packages/sdk/examples/browser/README.md index 63032f0..9703b63 100644 --- a/packages/sdk/examples/browser/README.md +++ b/packages/sdk/examples/browser/README.md @@ -5,25 +5,16 @@ self-contained: paste it, run it, read the output. ```bash export UPSTASH_BOX_API_KEY=... # or use: node --env-file=.env -node agents/01-search-with-fallback.ts +node retrieval/01-catalog-extraction.ts ``` ## Prerequisites - All examples need `UPSTASH_BOX_API_KEY`. -- Examples marked **AI** below use metered browser AI (`run`, `act`, - `extract`) and need a model provider key configured on the box or account. +- Examples marked **AI** below use metered browser AI (`act`, `extract`, + `observe`) and need a model provider key configured on the box or account. - Everything else runs with the Box key alone. -## agents/ — goal-driven browsing with `tab.run()` - -| File | AI | What it shows | -| --- | --- | --- | -| `01-search-with-fallback.ts` | yes | Constrained search with a fallback category; the agent evaluates, rejects with reasons, and switches on its own | -| `02-playwright-vs-act-vs-run.ts` | yes | The same task via Playwright, `act`+`extract`, and `run` — pick your autonomy level by token cost | -| `03-observe-record-audit.ts` | yes | The search again with live view, session recording, decision log, and token accounting | -| `04-multisite-feed.ts` | yes | One prompt + one schema across three differently structured sites | - ## automation/ — forms, files, and durable sessions | File | AI | What it shows | @@ -48,7 +39,6 @@ node agents/01-search-with-fallback.ts | --- | --- | --- | | `01-playwright-migration.ts` | no | An existing Playwright test where only the launch line changes | | `02-test-your-own-app.ts` | no | The box hosts the app under test and browses it on its own localhost | -| `03-ai-smoke-tests.ts` | yes | Agent-driven smoke flow, cross-checked by deterministic DOM assertions, recorded on video | | `04-visual-regression.ts` | no | Pixelmatch diffs against baselines stored on the box. Run twice. **Leaves a box** holding baselines (`.box-visual-regression`) | ## Cleanup diff --git a/packages/sdk/examples/browser/agents/01-search-with-fallback.ts b/packages/sdk/examples/browser/agents/01-search-with-fallback.ts deleted file mode 100644 index 4b473f3..0000000 --- a/packages/sdk/examples/browser/agents/01-search-with-fallback.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { Box } from "@upstash/box"; -import { z } from "zod/v3"; - -// Guide 1, example 1: constrained hunt with fallback. -// The path cannot be scripted upfront: the agent searches the category, -// concludes the constraint is unsatisfiable there, and switches to the -// fallback category on its own, explaining what it rejected and why. - -const box = await Box.create({ runtime: "node", browser: true }); - -try { - const tab = await box.browser.tab.create("https://books.toscrape.com", { - waitUntil: "domcontentloaded", - }); - - const t0 = performance.now(); - const result = await tab.run( - [ - "Find a science fiction book rated 5 stars priced under £30.", - "If the Science Fiction category has no book meeting both constraints,", - "try the Fantasy category instead.", - "Report the book you chose, and for every category you rejected, why.", - ].join(" "), - { - schema: z.object({ - title: z.string(), - price: z.string(), - rating: z.number().min(1).max(5), - category: z.string(), - rejected: z.array(z.object({ category: z.string(), reason: z.string() })), - }), - maxSteps: 25, // step budget: category + fallback fits well under this - }, - ); - - if (!result.completed) { - throw new Error(`agent did not finish within the step budget: ${result.result}`); - } - - const secs = ((performance.now() - t0) / 1000).toFixed(0); - console.log("result:", JSON.stringify(result.data, null, 2)); - console.log(`\ncompleted in ${result.stepCount} steps (${secs}s)`); - console.log(`tokens: ${result.inputTokens} in / ${result.outputTokens} out`); - console.log("\nsteps taken:"); - for (const step of result.steps) console.log(" -", JSON.stringify(step).slice(0, 140)); -} finally { - await box.delete(); -} diff --git a/packages/sdk/examples/browser/agents/02-playwright-vs-act-vs-run.ts b/packages/sdk/examples/browser/agents/02-playwright-vs-act-vs-run.ts deleted file mode 100644 index 882d6b7..0000000 --- a/packages/sdk/examples/browser/agents/02-playwright-vs-act-vs-run.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { Box } from "@upstash/box"; -import { chromium } from "playwright-core"; -import { z } from "zod/v3"; - -// Guide 1, example 2: the autonomy ladder. -// The same task three ways: open the first book on the page and report its -// title and price. Know the clicks -> script. Know the action -> act. Know -// only the goal -> run. Each rung up buys more autonomy and gives up more -// deterministic control. - -const URL = "https://books.toscrape.com"; - -const box = await Box.create({ runtime: "node", browser: true }); - -try { - // Rung 1 — Playwright over CDP. Exact selectors, zero tokens, breaks on redesign. - { - const t0 = performance.now(); - const browser = await chromium.connectOverCDP(await box.browser.cdpUrl()); - try { - const ctx = browser.contexts()[0] ?? (await browser.newContext()); - const page = await ctx.newPage(); - await page.goto(URL, { waitUntil: "domcontentloaded" }); - await page.click(".product_pod h3 a"); - await page.waitForSelector(".product_main"); - const title = await page.$eval(".product_main h1", (h) => h.textContent); - const price = await page.$eval(".product_main .price_color", (p) => p.textContent); - await page.close(); - const secs = ((performance.now() - t0) / 1000).toFixed(1); - console.log(`rung 1 (playwright): ${title} at ${price} — ${secs}s, 0 tokens`); - } finally { - await browser.close(); - } - } - - // Rung 2 — act + extract. Describe each step, let AI resolve the page. - // Both calls are metered; extract does not report its own token count. - { - const t0 = performance.now(); - const tab = await box.browser.tab.create(URL, { waitUntil: "domcontentloaded" }); - const action = await tab.act("click the first book in the product grid"); - if (!action.success) throw new Error(`act failed: ${action.message}`); - const data = await tab.extract( - "extract this book's title and price", - z.object({ title: z.string(), price: z.string() }), - ); - await tab.close(); - const secs = ((performance.now() - t0) / 1000).toFixed(1); - console.log( - `rung 2 (act + extract): ${data.title} at ${data.price} — ${secs}s, ` + - `${action.inputTokens + action.outputTokens} tokens for act, plus the extract call`, - ); - } - - // Rung 3 — run. State the goal, the agent plans the steps itself. - { - const t0 = performance.now(); - const tab = await box.browser.tab.create(URL, { waitUntil: "domcontentloaded" }); - const result = await tab.run( - "Open the first book listed on this page and report its title and price.", - { - schema: z.object({ title: z.string(), price: z.string() }), - maxSteps: 10, // step budget: click + read needs only a few steps - }, - ); - await tab.close(); - if (!result.completed) throw new Error(`agent did not finish: ${result.result}`); - const secs = ((performance.now() - t0) / 1000).toFixed(1); - console.log( - `rung 3 (run): ${result.data?.title} at ${result.data?.price} — ${secs}s, ` + - `${result.stepCount} steps, ${result.inputTokens + result.outputTokens} tokens`, - ); - } -} finally { - await box.delete(); -} diff --git a/packages/sdk/examples/browser/agents/03-observe-record-audit.ts b/packages/sdk/examples/browser/agents/03-observe-record-audit.ts deleted file mode 100644 index b5aae5d..0000000 --- a/packages/sdk/examples/browser/agents/03-observe-record-audit.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { writeFile } from "node:fs/promises"; -import { Box } from "@upstash/box"; -import { z } from "zod/v3"; - -// Guide 1, example 3: productionize the agent run. -// The same task as example 1, instrumented: watch it live, record the -// session, inspect every decision, account for latency and tokens, and keep -// the evidence. Only the instrumentation is new. - -const box = await Box.create({ runtime: "node", browser: true }); - -try { - const tab = await box.browser.tab.create("https://books.toscrape.com", { - waitUntil: "domcontentloaded", - }); - - // 1. Watch: open this URL in your browser to see the agent work (view-only). - console.log("live view:", await tab.liveViewUrl()); - - // 2. Record: capture the whole session as a replayable video. - const recording = await box.browser.recordings.start({ maxDurationSeconds: 300 }); - - const t0 = performance.now(); - let result; - let finished; - try { - result = await tab.run( - [ - "Find a science fiction book rated 5 stars priced under £30.", - "If the Science Fiction category has no book meeting both constraints,", - "try the Fantasy category instead.", - "Report the book you chose, and for every category you rejected, why.", - ].join(" "), - { - schema: z.object({ - title: z.string(), - price: z.string(), - rating: z.number().min(1).max(5), - category: z.string(), - rejected: z.array(z.object({ category: z.string(), reason: z.string() })), - }), - maxSteps: 25, - }, - ); - } finally { - // Stop the recording even if the run fails, so the video shows what went wrong. - finished = await recording.stop(); - } - const elapsedMs = Math.round(performance.now() - t0); - - if (!result.completed) { - throw new Error(`agent did not finish within the step budget: ${result.result}`); - } - - // 3. Inspect: every decision the agent made, with its reasoning. - console.log("\ndecisions:"); - for (const step of result.steps) { - const reasoning = "reasoning" in step ? String(step.reasoning).split("\n")[0].slice(0, 100) : ""; - console.log(` ${step.step}. [${step.action}] ${reasoning}`); - } - - // 4. Account: what this run cost. - console.log(`\nlatency: ${(elapsedMs / 1000).toFixed(1)}s over ${result.stepCount} steps`); - console.log(`tokens: ${result.inputTokens} in / ${result.outputTokens} out`); - - // 5. Evidence: typed result + session video, kept locally. - await writeFile("hunt-result.json", JSON.stringify(result.data, null, 2)); - const video = await box.browser.recordings.download(finished.id, { path: "hunt-session.mp4" }); - console.log(`\nresult saved to hunt-result.json, session video to ${video}`); -} finally { - await box.delete(); -} diff --git a/packages/sdk/examples/browser/agents/04-multisite-feed.ts b/packages/sdk/examples/browser/agents/04-multisite-feed.ts deleted file mode 100644 index 9a603f9..0000000 --- a/packages/sdk/examples/browser/agents/04-multisite-feed.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { Box } from "@upstash/box"; -import { z } from "zod/v3"; - -// Guide 1, example 4: one semantic workflow across varied layouts. -// Build a normalized developer-news feed from three differently structured -// sites. The prompt and schema stay identical; the agent absorbs the layout -// differences. Sites still differ in auth, pagination, and semantics — this -// scales the reading pattern, it does not abolish scraping engineering. - -const SITES = ["https://news.ycombinator.com", "https://lobste.rs", "https://dev.to"]; - -const PageStories = z.object({ - stories: z - .array( - z.object({ - title: z.string(), - url: z.string().url(), - engagement: z.number().nullable(), - }), - ) - .min(1) - .max(3), -}); - -const box = await Box.create({ runtime: "node", browser: true }); - -try { - const feed: { source: string; title: string; url: string; engagement: number | null }[] = []; - const errors: { site: string; error: string }[] = []; - let totalTokens = 0; - const t0 = performance.now(); - - for (const site of SITES) { - try { - const tab = await box.browser.tab.create(site, { waitUntil: "domcontentloaded" }); - const result = await tab.run( - [ - "Collect the top three visible stories or posts on this page.", - "For each, return its title, its full link URL, and its engagement", - "(points, upvotes, or reactions) as a number, or null if none is shown.", - ].join(" "), - { schema: PageStories, maxSteps: 12 }, // one page of reading per site - ); - await tab.close(); - if (!result.completed) throw new Error("agent did not finish within the step budget"); - totalTokens += result.inputTokens + result.outputTokens; - const source = new URL(site).hostname; - for (const story of result.data.stories) feed.push({ source, ...story }); - console.log(`${site}: ${result.data.stories.length} stories in ${result.stepCount} steps`); - } catch (err) { - // One site failing must not take down the feed. - errors.push({ site, error: (err as Error).message.slice(0, 120) }); - console.log(`${site}: failed, continuing`); - } - } - - // The normalized feed lives in the box, ready for whatever consumes it next. - await box.files.write({ path: "feed.json", content: JSON.stringify(feed, null, 2) }); - - const secs = ((performance.now() - t0) / 1000).toFixed(0); - console.log(`\nfeed (${feed.length} stories):`); - for (const item of feed) { - console.log(` [${item.source}] ${item.title} — ${item.engagement ?? "n/a"}`); - } - if (errors.length) console.log("\nerrors:", JSON.stringify(errors)); - console.log(`\n${SITES.length - errors.length}/${SITES.length} sites, ${totalTokens} tokens, ${secs}s`); -} finally { - await box.delete(); -} diff --git a/packages/sdk/examples/browser/testing/03-ai-smoke-tests.ts b/packages/sdk/examples/browser/testing/03-ai-smoke-tests.ts deleted file mode 100644 index 1decbe2..0000000 --- a/packages/sdk/examples/browser/testing/03-ai-smoke-tests.ts +++ /dev/null @@ -1,72 +0,0 @@ -import assert from "node:assert"; -import { Box } from "@upstash/box"; -import { chromium } from "playwright-core"; -import { z } from "zod/v3"; - -// Guide 4, example 3: natural-language smoke tests. -// Not a replacement for your E2E suite — selector-free smoke coverage that -// survives redesigns. The agent walks the flow and returns a typed verdict, -// but the verdict is NOT taken on faith: deterministic DOM assertions verify -// the end state independently, and the whole run is recorded so a failure -// comes with a video. - -const box = await Box.create({ runtime: "node", browser: true }); - -try { - const tab = await box.browser.tab.create("https://www.saucedemo.com", { - waitUntil: "domcontentloaded", - }); - const recording = await box.browser.recordings.start({ maxDurationSeconds: 300 }); - - let result; - try { - result = await tab.run( - [ - "Smoke-test this store: log in as standard_user with password", - "secret_sauce, add the first product to the cart, open the cart,", - "and go to checkout. For each step report whether it worked.", - "Do not submit the checkout form.", - ].join(" "), - { - schema: z.object({ - passed: z.boolean(), - checks: z.array(z.object({ step: z.string(), ok: z.boolean(), note: z.string() })), - }), - maxSteps: 20, // step budget: four-step flow with headroom for retries - }, - ); - } finally { - const finished = await recording.stop(); - await box.browser.recordings.download(finished.id, { path: "smoke-run.mp4" }); - } - - if (!result.completed) throw new Error("smoke test did not finish within the step budget"); - - for (const check of result.data.checks) { - console.log(` ${check.ok ? "PASS" : "FAIL"} ${check.step} — ${check.note}`); - } - console.log(`${result.stepCount} steps, ${result.inputTokens} in / ${result.outputTokens} out tokens`); - - // 1. The agent's own report must be coherent... - assert.ok(result.data.checks.length >= 4, "expected at least 4 checks"); - assert.ok(result.data.checks.every((c) => c.ok), "a step reported failure"); - assert.strictEqual(result.data.passed, true, "verdict inconsistent with checks"); - - // 2. ...and the end state must hold up to independent DOM assertions. - const browser = await chromium.connectOverCDP(await box.browser.cdpUrl()); - try { - const ctx = browser.contexts()[0]; - // Reuse the tab the agent drove, so we assert the state it left behind. - const page = ctx.pages().find((p) => p.url().includes("saucedemo")) ?? ctx.pages()[0]; - assert.ok(page.url().includes("checkout-step-one"), `not on checkout: ${page.url()}`); - const cartCount = await page.$eval(".shopping_cart_badge", (el) => el.textContent); - assert.strictEqual(cartCount, "1", "cart should hold exactly one item"); - } finally { - await browser.close(); - } - - console.log("smoke test passed — agent verdict confirmed by DOM assertions"); - console.log("session video saved to smoke-run.mp4"); -} finally { - await box.delete(); -} diff --git a/packages/sdk/src/__tests__/box-browser.test.ts b/packages/sdk/src/__tests__/box-browser.test.ts index 2ebb497..827fc87 100644 --- a/packages/sdk/src/__tests__/box-browser.test.ts +++ b/packages/sdk/src/__tests__/box-browser.test.ts @@ -202,63 +202,57 @@ describe("Box browser operations", () => { }); }); - it("runs a multi-step task with schema-validated structured output", async () => { + it("replays a pre-resolved action deterministically (posts action, not instruction)", async () => { const { box, fetchMock } = await createTestBox(); fetchMock - .mockResolvedValueOnce(mockResponse({ id: "tab-2", url: "https://linkedin.com" })) + .mockResolvedValueOnce(mockResponse({ id: "tab-2", url: "https://example.com/login" })) .mockResolvedValueOnce( mockResponse({ - result: "Found five people", - data: { - people: Array.from({ length: 5 }, (_, index) => ({ - name: `Founder ${index + 1}`, - headline: "AI founder in Berlin", - profileUrl: `https://linkedin.com/in/founder-${index + 1}`, - })), - }, - completed: true, - steps: [{ step: 1, action: "search", url: "https://linkedin.com/search" }], - step_count: 1, - input_tokens: 100, - output_tokens: 25, + success: true, + message: "done", + action_description: "Sign in", + actions: [ + { + selector: "xpath=/html/body/button", + description: "Sign in", + method: "click", + arguments: [], + }, + ], + input_tokens: 0, + output_tokens: 0, }), ); - const tab = await box.browser.tab.create("https://linkedin.com"); - const result = await tab.run("Find five AI founders in Berlin", { - schema: z.object({ - people: z - .array( - z.object({ - name: z.string(), - headline: z.string(), - profileUrl: z.string(), - }), - ) - .length(5), - }), - maxSteps: 25, - }); - - const typedPeople: Array<{ name: string; headline: string; profileUrl: string }> = - result.data.people; - expect(typedPeople).toHaveLength(5); - expect(result.completed).toBe(true); - - const body = JSON.parse(fetchMock.mock.calls[2]?.[1]?.body as string); - expect(body).toMatchObject({ - prompt: "Find five AI founders in Berlin", + const tab = await box.browser.tab.create("https://example.com/login"); + const action = { + selector: "xpath=/html/body/button", + description: "Sign in", + method: "click", + arguments: [], + }; + const result = await tab.act(action); + + expect(result.success).toBe(true); + expect(result.inputTokens).toBe(0); + expect(fetchMock.mock.calls[2]?.[0]).toContain("browser/act"); + expect(JSON.parse(fetchMock.mock.calls[2]?.[1]?.body as string)).toEqual({ + action, tab: "tab-2", - max_steps: 25, - schema: { - type: "object", - properties: { - people: { type: "array", minItems: 5, maxItems: 5 }, - }, - }, }); }); + it("rejects an action with no selector before any request", async () => { + const { box, fetchMock } = await createTestBox(); + fetchMock.mockResolvedValueOnce( + mockResponse({ id: "tab-2", url: "https://example.com/login" }), + ); + const tab = await box.browser.tab.create("https://example.com/login"); + await expect(tab.act({ description: "unresolved element" })).rejects.toThrow( + "requires a selector", + ); + }); + it("starts and stops a recording and maps its playback metadata", async () => { const { box, fetchMock } = await createTestBox(); fetchMock @@ -586,14 +580,27 @@ describe("Box browser operations", () => { const { box, fetchMock } = await createTestBox(); fetchMock.mockResolvedValueOnce( mockResponse({ - elements: [{ description: "Sign in button", selector: "xpath=/html/body/button" }], + elements: [ + { + description: "Sign in button", + selector: "xpath=/html/body/button", + method: "click", + arguments: [], + }, + ], }), ); const result = await box.browser.getTab("tab-1").observe("the sign in button"); + // method/arguments pass through so the element can be replayed via act(action). expect(result.elements).toEqual([ - { description: "Sign in button", selector: "xpath=/html/body/button" }, + { + description: "Sign in button", + selector: "xpath=/html/body/button", + method: "click", + arguments: [], + }, ]); expect(JSON.parse(fetchMock.mock.calls[1]?.[1]?.body as string)).toEqual({ instruction: "the sign in button", @@ -601,34 +608,6 @@ describe("Box browser operations", () => { }); }); - it("runs without a schema and supports the deprecated options form", async () => { - const { box, fetchMock } = await createTestBox(); - fetchMock - .mockResolvedValueOnce( - mockResponse({ result: "done", completed: true, steps: [], step_count: 3 }), - ) - .mockResolvedValueOnce( - mockResponse({ result: "done again", completed: false, steps: [], step_count: 15 }), - ); - - const tab = box.browser.getTab("tab-1"); - const plain = await tab.run("Do the thing"); - const deprecated = await tab.run({ prompt: "Do the thing again", maxSteps: 20 }); - - expect(plain.data).toBeUndefined(); - expect(plain.completed).toBe(true); - expect(JSON.parse(fetchMock.mock.calls[1]?.[1]?.body as string)).toEqual({ - prompt: "Do the thing", - tab: "tab-1", - }); - expect(deprecated.result).toBe("done again"); - expect(JSON.parse(fetchMock.mock.calls[2]?.[1]?.body as string)).toEqual({ - prompt: "Do the thing again", - tab: "tab-1", - max_steps: 20, - }); - }); - it("sends timeout: 0 through to disable the navigation deadline", async () => { const { box, fetchMock } = await createTestBox(); fetchMock.mockResolvedValueOnce(mockResponse({ id: "tab-1", url: "about:blank" })); @@ -641,7 +620,7 @@ describe("Box browser operations", () => { }); }); - it("rejects non-Zod schemas for extract and run", async () => { + it("rejects non-Zod schemas for extract", async () => { const { box } = await createTestBox(); const tab = box.browser.getTab("tab-1"); const fake = { parse: (d: unknown) => d }; @@ -649,9 +628,6 @@ describe("Box browser operations", () => { await expect(tab.extract("get data", fake)).rejects.toThrow( "extract requires a Zod object schema", ); - await expect(tab.run("go", { schema: fake })).rejects.toThrow( - "run requires a Zod object schema", - ); }); it("throws when connect or screencast responses lack a URL", async () => { diff --git a/packages/sdk/src/client.ts b/packages/sdk/src/client.ts index 16c764e..0595b7b 100644 --- a/packages/sdk/src/client.ts +++ b/packages/sdk/src/client.ts @@ -55,9 +55,7 @@ import { type BrowserTabCreateOptions, type BrowserObserveResult, type BrowserActResult, - type BrowserRunOptions, - type BrowserRunResult, - type BrowserRunStep, + type BrowserAction, type BrowserRecording, type BrowserRecordingHandle, type BrowserRecordingMarker, @@ -512,7 +510,24 @@ export class Tab { } /** Resolve and execute one natural-language action on this tab (metered). */ - async act(instruction: string, options?: BrowserExtractOptions): Promise { + async act(instruction: string, options?: BrowserExtractOptions): Promise; + /** Replay a pre-resolved `observe()` action with no LLM call and no key (`model` ignored). */ + async act(action: BrowserAction): Promise; + async act( + instructionOrAction: string | BrowserAction, + options?: BrowserExtractOptions, + ): Promise { + if (typeof instructionOrAction !== "string" && !instructionOrAction.selector) { + throw new BoxError("act(action) requires a selector; observe() did not resolve one"); + } + const body = + typeof instructionOrAction === "string" + ? { + instruction: instructionOrAction, + tab: this.id, + ...(options?.model ? { model: options.model } : {}), + } + : { action: instructionOrAction, tab: this.id }; const resp = await this.box._request<{ success?: boolean; message?: string; @@ -522,7 +537,7 @@ export class Tab { input_tokens?: number; output_tokens?: number; }>("POST", `/v2/box/${this.box.id}/browser/act`, { - body: { instruction, tab: this.id, ...(options?.model ? { model: options.model } : {}) }, + body, timeout: 180000, }); return { @@ -536,56 +551,6 @@ export class Tab { }; } - /** - * Autonomously complete a multi-step task on this tab. Runs a DOM-aware - * browser agent (Stagehand) inside the box: it reads the page, acts, and - * repeats until done. Metered — needs a key for the model's provider on the - * box or account (Anthropic, OpenAI, OpenRouter, Vercel, or OpenCode). - */ - async run( - prompt: string, - options: BrowserRunOptions & { schema: BrowserExtractSchema }, - ): Promise>; - async run(prompt: string, options?: BrowserRunOptions): Promise; - /** @deprecated Pass the prompt as the first argument. */ - async run(options: BrowserRunOptions & { prompt: string }): Promise; - async run( - promptOrOptions: string | (BrowserRunOptions & { prompt: string }), - runOptions: BrowserRunOptions = {}, - ): Promise> { - const prompt = typeof promptOrOptions === "string" ? promptOrOptions : promptOrOptions.prompt; - const options = typeof promptOrOptions === "string" ? runOptions : promptOrOptions; - const jsonSchema = options.schema ? toJsonSchema(options.schema) : undefined; - if (options.schema && !jsonSchema) throw new BoxError("run requires a Zod object schema"); - const resp = await this.box._request<{ - result?: string; - data?: unknown; - completed?: boolean; - steps?: BrowserRunStep[]; - step_count?: number; - input_tokens?: number; - output_tokens?: number; - }>("POST", `/v2/box/${this.box.id}/browser/run`, { - body: { - prompt, - tab: this.id, - ...(jsonSchema ? { schema: jsonSchema } : {}), - ...(options.maxSteps ? { max_steps: options.maxSteps } : {}), - ...(options.model ? { model: options.model } : {}), - }, - timeout: 600000, - }); - return { - data: options.schema ? options.schema.parse(resp.data) : undefined, - result: resp.result ?? "", - completed: Boolean(resp.completed), - steps: resp.steps ?? [], - stepCount: resp.step_count ?? 0, - inputTokens: resp.input_tokens ?? 0, - outputTokens: resp.output_tokens ?? 0, - }; - } - /** * Live-view URL for this tab (authenticated via a token in the URL). Open it * directly or embed it in an iframe — the page renders the tab live via CDP diff --git a/packages/sdk/src/index.ts b/packages/sdk/src/index.ts index 0b412e8..af6156c 100644 --- a/packages/sdk/src/index.ts +++ b/packages/sdk/src/index.ts @@ -87,9 +87,7 @@ export type { BrowserObserveResult, BrowserActAction, BrowserActResult, - BrowserRunOptions, - BrowserRunResult, - BrowserRunStep, + BrowserAction, BrowserRecording, BrowserRecordingHandle, BrowserRecordingMarker, diff --git a/packages/sdk/src/types.ts b/packages/sdk/src/types.ts index 85605af..7187cc7 100644 --- a/packages/sdk/src/types.ts +++ b/packages/sdk/src/types.ts @@ -1146,6 +1146,9 @@ export interface BrowserObserveElement { /** A selector for the element (Stagehand-resolved), when available. */ selector?: string; url?: string; + /** Suggested method and args, for replay via `act(action)`. */ + method?: string; + arguments?: string[]; } /** Result of `box.browser.observe()`. */ @@ -1161,6 +1164,9 @@ export interface BrowserActAction { arguments?: string[]; } +/** A pre-resolved action from `observe()`, passable to `act()` for a no-LLM replay. */ +export type BrowserAction = BrowserObserveElement | BrowserActAction; + /** Result of one natural-language `tab.act()` call. */ export interface BrowserActResult { success: boolean; @@ -1172,57 +1178,15 @@ export interface BrowserActResult { outputTokens: number; } -/** Options for `tab.run()`. */ -export interface BrowserRunOptions { - /** - * Zod object schema for data the agent must return when it completes. The - * inferred schema output becomes `BrowserRunResult.data`. - */ - schema?: { parse(data: unknown): T }; - /** @deprecated Pass the prompt as the first `tab.run()` argument. */ - prompt?: string; - /** Max agent steps. Default 15, max 30. */ - maxSteps?: number; - /** - * Provider-prefixed model override, e.g. `anthropic/claude-sonnet-4-5`, - * `openai/gpt-4o`, `openrouter/...`, `vercel/...`, `opencode/...`. The box or - * account must have a key for that provider. Defaults to the Box's configured - * model, or `anthropic/claude-sonnet-4-5` when the Box has no model. - */ - model?: string; -} - -/** One turn of a `tab.run()` loop. */ -export interface BrowserRunStep { - step: number; - action?: string; - reasoning?: string; - url?: string; -} - -/** Result of `tab.run()` — the agent's outcome after the loop. */ -export interface BrowserRunResult { - /** Structured output validated against the supplied schema. */ - data: T; - /** The agent's answer/summary when finished. */ - result: string; - /** Whether the agent reported the task complete (vs. hit maxSteps). */ - completed: boolean; - steps: BrowserRunStep[]; - stepCount: number; - inputTokens: number; - outputTokens: number; -} - /** A labeled point (or span) on a recording's timeline. */ export interface BrowserRecordingMarker { - /** "tab_switch" (recorder-observed) or "run" (a `tab.run` chapter). */ - type: "tab_switch" | "run"; + /** "tab_switch" (recorder-observed). */ + type: "tab_switch"; /** Offset from the start of the recording, in milliseconds. */ atMs: number; - /** For spans (runs): end offset in milliseconds. */ + /** End offset in milliseconds, for span markers. */ endMs?: number; - /** Tab title/URL for switches; the prompt for runs. */ + /** Tab title/URL for switches. */ label?: string; tabId?: string; }