Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 109 additions & 0 deletions PROTOCOL.md
Original file line number Diff line number Diff line change
Expand Up @@ -836,9 +836,14 @@ Browser support is advertised in `hello.capabilities`.
| seq | int64 |
| contentType | string |
| dataBase64 | base64 string |
| frameRef | string? |
| width | int |
| height | int |
| viewportWidth | int? |
| viewportHeight | int? |
| devicePixelRatio | float64? |
| capturedAt | iso8601 string |
| droppedPriorCount | int? |

#### `browserCursor`

Expand Down Expand Up @@ -895,6 +900,7 @@ Browser support is advertised in `hello.capabilities`.
| channelId | string |
| owner | string |
| reason | string? |
| controlVersion | int64? |

#### `browserControlRelease`

Expand All @@ -906,12 +912,14 @@ Browser support is advertised in `hello.capabilities`.
| channelId | string |
| owner | string |
| reason | string? |
| controlVersion | int64? |

#### `browserUserInput`

| Field | Type |
|---|---|
| type | "browserUserInput" |
| inputId | string? |
| browserId | string |
| sessionId | uuid |
| channelId | string |
Expand All @@ -923,6 +931,107 @@ Browser support is advertised in `hello.capabilities`.
| key | string? |
| deltaX | float64? |
| deltaY | float64? |
| frameSeq | int64? |
| controlVersion | int64? |
| coordinateSpace | string? |
| viewportWidth | int? |
| viewportHeight | int? |
| frameWidth | int? |
| frameHeight | int? |
| devicePixelRatio | float64? |
| renderedLeft | float64 |
| renderedTop | float64 |
| renderedWidth | float64 |
| renderedHeight | float64 |

#### `browserUserInputAck`

| Field | Type |
|---|---|
| type | "browserUserInputAck" |
| browserId | string |
| sessionId | uuid |
| channelId | string |
| inputId | string? |
| accepted | bool |
| reason | string? |
| controlVersion | int64? |
| ackedAt | iso8601 string |

#### `browserTransportStatus`

| Field | Type |
|---|---|
| type | "browserTransportStatus" |
| browserId | string |
| sessionId | uuid |
| channelId | string |
| status | string |
| queueDepth | int? |
| droppedFrameCount | int64? |
| maxFrameBytes | int? |
| at | iso8601 string |

### Browser Input Safety

Browser input messages include the source frame sequence, control version,
viewport dimensions, captured frame dimensions, rendered panel rectangle, and
coordinate space. Receivers reject stale-frame and stale-control input with
`browserUserInputAck`.

### Browser Transport Status

Browser frame producers may drop, coalesce, downscale, or reference frames when
encoded frames exceed transport limits. Control and approval messages have
priority over frame delivery.

### Preview Bridge Access

Preview-origin bridge access is short-lived, grant-bound, and separate from
iframe preview cookies. Local-direct bridge mode navigates the browser directly
to the approved loopback target.

#### `browserBridgeAccessOpen`

| Field | Type |
|---|---|
| type | "browserBridgeAccessOpen" |
| requestId | string |
| previewId | string |
| grantId | string |
| browserId | string? |
| sessionId | uuid |
| channelId | string |
| machineId | string |
| bridgeMode | string |
| requestedAt | iso8601 string |

#### `browserBridgeAccessOpened`

| Field | Type |
|---|---|
| type | "browserBridgeAccessOpened" |
| requestId | string |
| previewId | string |
| grantId | string |
| browserId | string? |
| sessionId | uuid |
| channelId | string |
| bridgeMode | string |
| url | string |
| expiresAt | iso8601 string |

#### `browserBridgeAccessClose`

| Field | Type |
|---|---|
| type | "browserBridgeAccessClose" |
| previewId | string |
| grantId | string |
| browserId | string? |
| sessionId | uuid |
| channelId | string |
| reason | string? |

### Tool Calls

Expand Down
10 changes: 10 additions & 0 deletions envelope.go
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,16 @@ func payloadForType(msgType string) (any, error) {
return &BrowserControlRelease{}, nil
case MsgTypeBrowserUserInput:
return &BrowserUserInput{}, nil
case MsgTypeBrowserUserInputAck:
return &BrowserUserInputAck{}, nil
case MsgTypeBrowserTransportStatus:
return &BrowserTransportStatus{}, nil
case MsgTypeBrowserBridgeAccessOpen:
return &BrowserBridgeAccessOpen{}, nil
case MsgTypeBrowserBridgeAccessOpened:
return &BrowserBridgeAccessOpened{}, nil
case MsgTypeBrowserBridgeAccessClose:
return &BrowserBridgeAccessClose{}, nil
case MsgTypeBrowserSensitiveActionRequest:
return &BrowserSensitiveActionRequest{}, nil
case MsgTypeBrowserSensitiveActionResponse:
Expand Down
173 changes: 142 additions & 31 deletions messages.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,13 +84,41 @@ const (
MsgTypeBrowserControlClaim MessageType = "browserControlClaim"
MsgTypeBrowserControlRelease MessageType = "browserControlRelease"
MsgTypeBrowserUserInput MessageType = "browserUserInput"
MsgTypeBrowserUserInputAck MessageType = "browserUserInputAck"
MsgTypeBrowserTransportStatus MessageType = "browserTransportStatus"
MsgTypeBrowserBridgeAccessOpen MessageType = "browserBridgeAccessOpen"
MsgTypeBrowserBridgeAccessOpened MessageType = "browserBridgeAccessOpened"
MsgTypeBrowserBridgeAccessClose MessageType = "browserBridgeAccessClose"
MsgTypeBrowserSensitiveActionRequest MessageType = "browserSensitiveActionRequest"
MsgTypeBrowserSensitiveActionResponse MessageType = "browserSensitiveActionResponse"

MsgTypePlanningEvent MessageType = "planningEvent"
MsgTypePlanningEventAck MessageType = "planningEventAck"
)

const (
BrowserOwnerAgent = "agent"
BrowserOwnerLex = "lex"
BrowserOwnerPaused = "paused"
BrowserOwnerApproval = "approval"

BrowserInputKindClick = "click"
BrowserInputKindPointerMove = "pointer_move"
BrowserInputKindPointerDown = "pointer_down"
BrowserInputKindPointerUp = "pointer_up"
BrowserInputKindWheel = "wheel"
BrowserInputKindKeyDown = "key_down"
BrowserInputKindKeyUp = "key_up"
BrowserInputKindText = "text"

BrowserCoordinateSpaceFrameCssPixels = "frame_css_pixels"

BrowserInputRejectStaleFrame = "stale_frame"
BrowserInputRejectOwnerMismatch = "owner_mismatch"
BrowserInputRejectExpiredGrant = "expired_grant"
BrowserInputRejectInvalidPayload = "invalid_payload"
)

type ContextRef struct {
Kind string `json:"kind"`
Path string `json:"path"`
Expand Down Expand Up @@ -616,6 +644,9 @@ type BrowserSessionOpen struct {
MachineID string `json:"machineId"`
IdentityID string `json:"identityId,omitempty"`
Mode string `json:"mode"`
InitialURL string `json:"initialUrl,omitempty"`
BridgeMode string `json:"bridgeMode,omitempty"`
PreviewID string `json:"sourcePreviewId,omitempty"`
ExpiresAt string `json:"expiresAt"`
}

Expand Down Expand Up @@ -660,16 +691,21 @@ type BrowserSessionError struct {
}

type BrowserFrame struct {
Type MessageType `json:"type"`
BrowserID string `json:"browserId"`
SessionID string `json:"sessionId"`
ChannelID string `json:"channelId"`
Seq int64 `json:"seq"`
ContentType string `json:"contentType"`
DataBase64 string `json:"dataBase64"`
Width int `json:"width"`
Height int `json:"height"`
CapturedAt string `json:"capturedAt"`
Type MessageType `json:"type"`
BrowserID string `json:"browserId"`
SessionID string `json:"sessionId"`
ChannelID string `json:"channelId"`
Seq int64 `json:"seq"`
ContentType string `json:"contentType"`
DataBase64 string `json:"dataBase64,omitempty"`
FrameRef string `json:"frameRef,omitempty"`
Width int `json:"width"`
Height int `json:"height"`
ViewportWidth int `json:"viewportWidth,omitempty"`
ViewportHeight int `json:"viewportHeight,omitempty"`
DevicePixelRatio float64 `json:"devicePixelRatio,omitempty"`
CapturedAt string `json:"capturedAt"`
DroppedPriorCount int `json:"droppedPriorCount,omitempty"`
}

type BrowserCursor struct {
Expand Down Expand Up @@ -729,36 +765,111 @@ type BrowserToolResult struct {
}

type BrowserControlClaim struct {
Type MessageType `json:"type"`
BrowserID string `json:"browserId"`
SessionID string `json:"sessionId"`
ChannelID string `json:"channelId"`
Owner string `json:"owner"`
Reason string `json:"reason,omitempty"`
Type MessageType `json:"type"`
BrowserID string `json:"browserId"`
SessionID string `json:"sessionId"`
ChannelID string `json:"channelId"`
Owner string `json:"owner"`
Reason string `json:"reason,omitempty"`
ControlVersion int64 `json:"controlVersion,omitempty"`
}

type BrowserControlRelease struct {
Type MessageType `json:"type"`
BrowserID string `json:"browserId"`
SessionID string `json:"sessionId"`
ChannelID string `json:"channelId"`
Owner string `json:"owner"`
Reason string `json:"reason,omitempty"`
Type MessageType `json:"type"`
BrowserID string `json:"browserId"`
SessionID string `json:"sessionId"`
ChannelID string `json:"channelId"`
Owner string `json:"owner"`
Reason string `json:"reason,omitempty"`
ControlVersion int64 `json:"controlVersion,omitempty"`
}

type BrowserUserInput struct {
Type MessageType `json:"type"`
InputID string `json:"inputId,omitempty"`
BrowserID string `json:"browserId"`
SessionID string `json:"sessionId"`
ChannelID string `json:"channelId"`
Owner string `json:"owner"`
Kind string `json:"kind"`
X *float64 `json:"x,omitempty"`
Y *float64 `json:"y,omitempty"`
Text string `json:"text,omitempty"`
Key string `json:"key,omitempty"`
DeltaX *float64 `json:"deltaX,omitempty"`
DeltaY *float64 `json:"deltaY,omitempty"`
FrameSeq int64 `json:"frameSeq,omitempty"`
ControlVersion int64 `json:"controlVersion,omitempty"`
CoordinateSpace string `json:"coordinateSpace,omitempty"`
ViewportWidth int `json:"viewportWidth,omitempty"`
ViewportHeight int `json:"viewportHeight,omitempty"`
FrameWidth int `json:"frameWidth,omitempty"`
FrameHeight int `json:"frameHeight,omitempty"`
DevicePixelRatio float64 `json:"devicePixelRatio,omitempty"`
RenderedLeft float64 `json:"renderedLeft"`
RenderedTop float64 `json:"renderedTop"`
RenderedWidth float64 `json:"renderedWidth"`
RenderedHeight float64 `json:"renderedHeight"`
}

type BrowserUserInputAck struct {
Type MessageType `json:"type"`
BrowserID string `json:"browserId"`
SessionID string `json:"sessionId"`
ChannelID string `json:"channelId"`
InputID string `json:"inputId,omitempty"`
Accepted bool `json:"accepted"`
Reason string `json:"reason,omitempty"`
ControlVersion int64 `json:"controlVersion,omitempty"`
AckedAt string `json:"ackedAt"`
}

type BrowserTransportStatus struct {
Type MessageType `json:"type"`
BrowserID string `json:"browserId"`
SessionID string `json:"sessionId"`
ChannelID string `json:"channelId"`
Status string `json:"status"`
QueueDepth int `json:"queueDepth,omitempty"`
DroppedFrameCount int64 `json:"droppedFrameCount,omitempty"`
MaxFrameBytes int `json:"maxFrameBytes,omitempty"`
At string `json:"at"`
}

type BrowserBridgeAccessOpen struct {
Type MessageType `json:"type"`
RequestID string `json:"requestId"`
PreviewID string `json:"previewId"`
GrantID string `json:"grantId"`
BrowserID string `json:"browserId,omitempty"`
SessionID string `json:"sessionId"`
ChannelID string `json:"channelId"`
MachineID string `json:"machineId"`
BridgeMode string `json:"bridgeMode"`
RequestedAt string `json:"requestedAt"`
}

type BrowserBridgeAccessOpened struct {
Type MessageType `json:"type"`
RequestID string `json:"requestId"`
PreviewID string `json:"previewId"`
GrantID string `json:"grantId"`
BrowserID string `json:"browserId,omitempty"`
SessionID string `json:"sessionId"`
ChannelID string `json:"channelId"`
BridgeMode string `json:"bridgeMode"`
URL string `json:"url"`
ExpiresAt string `json:"expiresAt"`
}

type BrowserBridgeAccessClose struct {
Type MessageType `json:"type"`
BrowserID string `json:"browserId"`
PreviewID string `json:"previewId"`
GrantID string `json:"grantId"`
BrowserID string `json:"browserId,omitempty"`
SessionID string `json:"sessionId"`
ChannelID string `json:"channelId"`
Owner string `json:"owner"`
Kind string `json:"kind"`
X *float64 `json:"x,omitempty"`
Y *float64 `json:"y,omitempty"`
Text string `json:"text,omitempty"`
Key string `json:"key,omitempty"`
DeltaX *float64 `json:"deltaX,omitempty"`
DeltaY *float64 `json:"deltaY,omitempty"`
Reason string `json:"reason,omitempty"`
}

type BrowserSensitiveActionRequest struct {
Expand Down
Loading