Wire the bystander blur that was never called, plus four reliability fixes (Plans CN + CO)

OpenGlasses/Sources/App/OpenGlassesApp.swift

@@ -671,7 +671,38 @@ class AppState: ObservableObject, AppStateProtocol {
     /// BK P2c — set once the model-switch notice has been spoken this turn, so a multi-hop cascade
     /// narrates only the FIRST fallback hop (not once per hop). Reset at the start of every turn.
     private var didNarrateModelSwitchThisTurn = false
-    @Published private(set) var isProcessing: Bool = false
+    @Published private(set) var isProcessing: Bool = false {
+        didSet {
+            // CO Item 3: stamped here rather than at the ten-odd assignment sites, so a new one
+            // cannot forget to and leave the hold window measuring from nothing.
+            guard isProcessing != oldValue else { return }
+            turnStartedAt = isProcessing ? Date() : nil
+        }
+    }
+
+    /// When the in-flight turn was dispatched (nil when idle) — the clock the hold window runs on.
+    private var turnStartedAt: Date?
+
+    /// CO Item 3: at most one utterance is held while a turn runs. Not a queue — replaying a
+    /// backlog of stale phrases at someone is worse than dropping them, and the newest intent is
+    /// the one worth keeping, so a second deferral simply overwrites the first.
+    private var heldUtterance: (text: String, heldAt: Date)?
+
+    private func turnElapsedSeconds() -> TimeInterval? {
+        turnStartedAt.map { Date().timeIntervalSince($0) }
+    }
+
+    /// Replay whatever was held while the finished turn ran, if it hasn't gone stale.
+    private func replayHeldUtteranceIfFresh() {
+        guard let held = heldUtterance else { return }
+        heldUtterance = nil
+        guard TurnAdmissionPolicy.heldUtteranceIsStillFresh(heldAt: held.heldAt) else {
+            NSLog("[CO] Dropped a held utterance that went stale: %@", held.text)
+            return
+        }
+        Task { @MainActor in await self.handleTranscription(held.text) }
+    }
+
     private var hasEverRegistered: Bool = false
     var inConversation: Bool = false
 
@@ -915,6 +946,15 @@ class AppState: ObservableObject, AppStateProtocol {
                 await self.speechService.speak("That's \(name).")
             }
         }
+        // CO Item 1: a near-tie is spoken as a question, not resolved to the leader. No encounter
+        // is logged — writing the wrong person into the encounter history would outlive the moment
+        // and quietly corrupt every later "when did I last see…" answer.
+        faceRecognition.onAmbiguousRecognition = { [weak self] names in
+            Task { @MainActor in
+                guard let self else { return }
+                await self.speechService.speak(FaceRecognitionService.ambiguityPrompt(names: names))
+            }
+        }
 
         // HIPAA: enforce retention policy on launch
         if Config.hipaaMode {
@@ -1049,24 +1089,33 @@ class AppState: ObservableObject, AppStateProtocol {
                 self.injectPinnedFrame()
                 return
             case .deliverLive:
+                // CO Item 0: bystander blur, applied where the frame leaves for a third-party
+                // model. Throttled to ~1 fps by this point, so the Vision pass is affordable here
+                // in a way it would not be on the recording path.
+                let outbound = self.privacyFilter.filtered(image, for: .liveSession)
                 if self.currentMode == .geminiLive {
-                    self.geminiLiveSession.submitVideoFrame(image)
+                    self.geminiLiveSession.submitVideoFrame(outbound)
                 } else if self.currentMode == .openaiRealtime {
-                    self.openAIRealtimeSession.submitVideoFrame(image)
+                    self.openAIRealtimeSession.submitVideoFrame(outbound)
                 }
             }
         }
 
         // Polling fallback for both session managers — a held pin substitutes for the live frame
+        // A held pin is already filtered at pin time (CO Item 0), so only the live fallback needs
+        // a pass here — re-blurring an already-blurred pin every poll would burn a Vision pass to
+        // no effect.
         geminiLiveSession.onRequestVideoFrame = { [weak self] in
             guard let self else { return nil }
             if Config.framePinEnabled, let pinned = self.framePin.pinnedFrame { return pinned }
-            return self.cameraService.latestFrame
+            guard let live = self.cameraService.latestFrame else { return nil }
+            return self.privacyFilter.filtered(live, for: .liveSession)
         }
         openAIRealtimeSession.onRequestVideoFrame = { [weak self] in
             guard let self else { return nil }
             if Config.framePinEnabled, let pinned = self.framePin.pinnedFrame { return pinned }
-            return self.cameraService.latestFrame
+            guard let live = self.cameraService.latestFrame else { return nil }
+            return self.privacyFilter.filtered(live, for: .liveSession)
         }
 
         // Location context for both
@@ -1166,6 +1215,38 @@ class AppState: ObservableObject, AppStateProtocol {
                 toolName: "code_agent", summary: request.summary, source: request.source)
         }
 
+        // Plan CN: the pin/camera facts the attachment policy needs, and the frame itself.
+        AgentSessionService.shared.attachmentContext = { [weak self] in
+            guard let self else { return (pinHeld: false, pinAge: nil, cameraStreaming: false) }
+            let pinned = Config.framePinEnabled && self.framePin.isPinned
+            return (pinHeld: pinned,
+                    pinAge: self.framePin.pinnedAt.map { Date().timeIntervalSince($0) },
+                    cameraStreaming: self.cameraService.isStreaming)
+        }
+        AgentSessionService.shared.resolveAttachment = { [weak self] decision in
+            guard let self, case .attach(let source) = decision else { return nil }
+            // A pin is already privacy-filtered at pin time (CO Item 0); a live frame is filtered
+            // here, under `.agentAttachment`, before any of it leaves the device.
+            let frame: UIImage?
+            switch source {
+            case .pinned: frame = self.framePin.pinnedFrame
+            case .live:
+                frame = self.cameraService.latestFrame.map {
+                    self.privacyFilter.filtered($0, for: .agentAttachment)
+                }
+            }
+            guard let frame,
+                  let raw = frame.jpegData(compressionQuality: 0.9) else { return nil }
+            // Same bounded encoding every other outbound image uses — no second size policy.
+            let prepared = LLMImagePreparer.prepared(raw)
+            guard !LLMImagePreparer.isDegenerate(prepared) else {
+                NSLog("[CN] Refusing to attach a degenerate frame — a blank tells the agent the "
+                      + "camera saw nothing, which is worse than sending no image at all.")
+                return nil
+            }
+            return AgentTaskAttachment(jpeg: prepared, source: source, pixelSize: frame.size)
+        }
+
         // Configure Navigation Assist (Plan J) similarly.
         NavigationAssistService.shared.configure(camera: cameraService, llm: llmService, tts: speechService)
         NavigationAssistService.shared.glassesDisplay = glassesDisplay
@@ -1746,12 +1827,27 @@ class AppState: ObservableObject, AppStateProtocol {
                     AssistiveModeService.shared.noteTranscription(text)
                     return
                 }
-                // Prevent processing if already handling a response
-                guard !self.isProcessing else {
-                    print("⚠️ Transcription ignored - already processing")
-                    return
+                // CO Item 3: a turn already in flight used to mean the utterance was dropped behind
+                // a debug print — no tone, no HUD, nothing the wearer could perceive. Now it is
+                // either held for the turn that is finishing or refused audibly.
+                switch TurnAdmissionPolicy.decide(isProcessing: self.isProcessing,
+                                                  turnElapsed: self.turnElapsedSeconds(),
+                                                  utterance: text) {
+                case .accept:
+                    await self.handleTranscription(text)
+
+                case .deferToQueue:
+                    self.heldUtterance = (text: text, heldAt: Date())
+                    self.speechService.playTone(frequency: 660, duration: 0.06)
+                    self.glassesDisplay.flash("⏳ Got it — one sec")
+                    NSLog("[CO] Held an utterance while a turn was in flight: %@", text)
+
+                case .rejectWithCue(let reason):
+                    guard reason != .emptyUtterance else { return }
+                    self.speechService.playTone(frequency: 330, duration: 0.12)
+                    self.glassesDisplay.flash("⚠️ Didn't catch that — say it again")
+                    NSLog("[CO] Rejected an utterance (%@): %@", String(describing: reason), text)
                 }
-                await self.handleTranscription(text)
             }
         }
 
@@ -2903,7 +2999,12 @@ class AppState: ObservableObject, AppStateProtocol {
             // The engine keepalive suspends; re-check the user didn't flip the toggle meanwhile.
             guard listeningEnabled, inConversation else { return }
             isListening = true
+            // CO Item 4: if what we just said was a question, give the user room to think before
+            // the silence window ends the conversation out from under them.
+            transcriptionService.noteAssistantSpoke(speechService.lastSpokenText)
             transcriptionService.startRecording()
+            // CO Item 3: a turn is over, so anything held while it ran can be answered now.
+            replayHeldUtteranceIfFresh()
         } else {
             await returnToWakeWord()
         }
@@ -3090,7 +3191,8 @@ class AppState: ObservableObject, AppStateProtocol {
             return pinnedData
         }
         guard cameraService.isStreaming, let frame = cameraService.latestFrame else { return nil }
-        guard let data = frame.jpegData(compressionQuality: Config.geminiLiveVideoJPEGQuality),
+        let outbound = privacyFilter.filtered(frame, for: .directModelTurn)   // CO Item 0
+        guard let data = outbound.jpegData(compressionQuality: Config.geminiLiveVideoJPEGQuality),
               !LLMImagePreparer.isDegenerate(data) else { return nil }
         return data
     }
@@ -3104,7 +3206,10 @@ class AppState: ObservableObject, AppStateProtocol {
     @discardableResult
     func pinCurrentFrame() -> Bool {
         guard Config.framePinEnabled, let frame = cameraService.latestFrame else { return false }
-        framePin.pin(frame: frame)
+        // CO Item 0: filter once, here. Every downstream use of a pin — the immediate sharp-inject,
+        // the heartbeat resends, the Direct-mode reuse, the pinned card on screen, a CN agent
+        // attachment — then carries the same blurred pixels without repeating the Vision pass.
+        framePin.pin(frame: privacyFilter.filtered(frame, for: .pinnedFrame))
         framePinGate.reset()
         injectPinnedFrame()
         framePinGate.notePinnedPushed(now: Date().timeIntervalSinceReferenceDate)

OpenGlasses/Sources/App/Views/AgenticFeaturesView.swift

@@ -8,6 +8,7 @@ struct AgenticFeaturesView: View {
     @EnvironmentObject var appState: AppState
     @Environment(\.appAccent) private var accent
     @State private var enabled = Config.agentModeEnabled
+    @State private var visionAttachment = Config.agentVisionAttachmentEnabled
     @State private var editingDocument: AgentDocumentStore.DocumentType?
     @State private var tasks: [AgentScheduler.ScheduledTask] = AgentScheduler.savedTasks()
     @State private var showShareSheet = false
@@ -49,6 +50,22 @@ struct AgenticFeaturesView: View {
             }
 
             if enabled {
+                // Plan CN — a separate grant from Agent Mode itself. Enabling agents authorises
+                // dispatching text tasks; sending camera frames to the same endpoint is a
+                // different promise, so it is a different switch and it starts off.
+                Section {
+                    InfoToggle(
+                        title: "Let Agents See",
+                        isOn: $visionAttachment,
+                        info: "Off by default. When on, a task you hand to a remote agent can carry one still from the glasses camera, so the agent reads a label, serial plate or form directly instead of working from the assistant's description of it. Pin a frame first to choose exactly what it sees; otherwise the current view is used, and only when your request refers to something visible. The image is blurred for bystander faces if that setting is on, and never sent in HIPAA mode. Custom agent endpoints also need an image field named in their configuration."
+                    )
+                    .onChange(of: visionAttachment) { _, on in
+                        Config.setAgentVisionAttachmentEnabled(on)
+                    }
+                } footer: {
+                    Text("A camera frame leaves your device for whichever agent endpoint you've configured. Separate from Agentic Features itself, so turning agents on never starts sending pictures on its own.")
+                }
+
                 // Safety supervisor (Plan S) — deterministic veto rules before any agent action.
                 Section {
                     NavigationLink {

OpenGlasses/Sources/App/Views/SettingsView.swift

@@ -716,7 +716,7 @@ struct HardwarePrivacyView: View {
                 InfoToggle(
                     title: "Blur Bystander Faces",
                     isOn: $privacyFilterEnabled,
-                    info: "Uses Apple's on-device Vision framework to detect faces in the glasses camera feed and applies a Gaussian blur to bystanders. Protects the privacy of people around you during streaming or recording. Processing happens entirely on-device."
+                    info: "Uses Apple's on-device Vision framework to detect faces in the glasses camera feed and applies a Gaussian blur before a frame is sent to an AI provider — live sessions, photos attached to a question, pinned frames, and frames handed to a remote agent. Detection and blurring happen entirely on-device. Does not yet cover video recording or broadcasting, which keep the unblurred frame. Faces you have enrolled for recognition are matched on the unblurred frame, so recognition keeps working."
                 )
                 InfoToggle(
                     title: "Share Health Data with AI",

OpenGlasses/Sources/Services/AgentHarness/Adapters/CustomAgentHarness.swift

@@ -41,7 +41,11 @@ struct CustomAgentHarness: AgentHarness {
     // MARK: - AgentHarness
 
     func start(prompt: String, project: String?) async throws -> AgentRun {
-        guard let request = config.startRequest(prompt: prompt, project: project) else {
+        try await start(prompt: prompt, project: project, attachment: nil)
+    }
+
+    func start(prompt: String, project: String?, attachment: AgentTaskAttachment?) async throws -> AgentRun {
+        guard let request = config.startRequest(prompt: prompt, project: project, attachment: attachment) else {
             throw AgentHarnessError.notConfigured(kind)
         }
         let json = try await sendJSON(request)

OpenGlasses/Sources/Services/AgentHarness/Adapters/CustomHarnessConfig.swift

@@ -21,6 +21,11 @@ struct CustomHarnessConfig: Codable, Equatable {
     var promptField: String = "prompt"
     var projectField: String = "project"
 
+    /// Plan CN: body key carrying a base64 JPEG of the wearer's view. **Empty by default, meaning
+    /// never attach.** An arbitrary user-configured endpoint must not start receiving multi-megabyte
+    /// bodies because a setting elsewhere got flipped — opting in is naming the field.
+    var imageField: String = ""
+
     /// Dot-paths into the responses (e.g. "data.run.id"). See `JSONPath`.
     var idPath: String = "id"
     var statusPath: String = "status"
@@ -57,15 +62,29 @@ struct CustomHarnessConfig: Codable, Equatable {
 extension CustomHarnessConfig {
     /// Build the start request, or `nil` if `startURL` is invalid.
     func startRequest(prompt: String, project: String?) -> URLRequest? {
+        startRequest(prompt: prompt, project: project, attachment: nil)
+    }
+
+    func startRequest(prompt: String, project: String?, attachment: AgentTaskAttachment?) -> URLRequest? {
         guard let url = URL(string: startURL.trimmingCharacters(in: .whitespacesAndNewlines)) else { return nil }
         var request = URLRequest(url: url)
         request.httpMethod = "POST"
         request.setValue("application/json", forHTTPHeaderField: "Content-Type")
         applyAuth(&request)
         var body: [String: Any] = [promptField: prompt]
         if let project, !project.isEmpty { body[projectField] = project }
+
+        // Plan CN: only when the user named a field for it.
+        let field = imageField.trimmingCharacters(in: .whitespacesAndNewlines)
+        let carriesImage = !field.isEmpty && attachment != nil
+        if carriesImage, let attachment {
+            body[field] = attachment.jpeg.base64EncodedString()
+        }
+
         request.httpBody = try? JSONSerialization.data(withJSONObject: body)
-        request.timeoutInterval = 30
+        // A few megabytes of base64 on cellular does not fit in 30 s, and a timeout here surfaces
+        // as "couldn't start the agent" — sending the user to look in entirely the wrong place.
+        request.timeoutInterval = carriesImage ? 90 : 30
         return request
     }
 

OpenGlasses/Sources/Services/AgentHarness/Adapters/OpenClawAgentHarness.swift

@@ -29,8 +29,18 @@ struct OpenClawAgentHarness: AgentHarness {
     // MARK: - AgentHarness
 
     func start(prompt: String, project: String?) async throws -> AgentRun {
+        try await start(prompt: prompt, project: project, attachment: nil)
+    }
+
+    func start(prompt: String, project: String?, attachment: AgentTaskAttachment?) async throws -> AgentRun {
         var params: [String: Any] = ["prompt": prompt]
         if let project { params["project"] = project }
+        // Plan CN. Whether the gateway accepts unknown params or rejects the call is unverified
+        // against a live endpoint, which is why the feature ships behind a default-off setting.
+        if let attachment {
+            params["image_base64"] = attachment.jpeg.base64EncodedString()
+            params["image_mime"] = "image/jpeg"
+        }
         let response = try await send("agent.start", params)
         if let error = response["error"] as? String {
             throw AgentHarnessError.transport(error)