Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions OpenGlasses/Sources/App/OpenGlassesApp.swift
Original file line number Diff line number Diff line change
Expand Up @@ -738,7 +738,7 @@ class AppState: ObservableObject, AppStateProtocol {
if let all = try? Data(contentsOf: url) {
try? all.suffix(100_000).write(to: url)
}
try? handle.seekToEnd()
_ = try? handle.seekToEnd()
}
try? handle.write(contentsOf: data)
} else {
Expand Down Expand Up @@ -2647,7 +2647,9 @@ class AppState: ObservableObject, AppStateProtocol {
return
}
do {
let photoData = try await cameraService.capturePhoto()
// The data is discarded on purpose: `capturePhoto()` saves every capture to the
// "Glasses" album itself, and this path only reports that it landed.
_ = try await cameraService.capturePhoto()
// Restore audio for wake word if in direct mode
if currentMode == .direct {
cameraService.restoreAudioForWakeWord()
Expand Down Expand Up @@ -3896,7 +3898,9 @@ class AppState: ObservableObject, AppStateProtocol {
},
capturePhoto: { [weak self] in
guard let self else { throw RemoteInvokeError.unavailable }
let data = try await self.cameraService.capturePhoto()
// Discarded on purpose — `capturePhoto()` files it in the "Glasses" album; the
// remote caller only needs the capture to have happened.
_ = try await self.cameraService.capturePhoto()
self.cameraService.restoreAudioForWakeWord()
},
startAudioRecording: { [weak self] in
Expand Down
5 changes: 5 additions & 0 deletions OpenGlasses/Sources/Services/BroadcastService.swift
Original file line number Diff line number Diff line change
Expand Up @@ -423,6 +423,11 @@ enum BroadcastError: LocalizedError {

/// BS P2: seam for the shared mic tap (adopted by WakeWordService — the same fan-out the
/// video recorder uses). Kept minimal so tests can inject a fake.
///
/// `@MainActor` because both the adopter (`WakeWordService`) and both call sites
/// (`startBroadcast`/`stopBroadcast`) are main-actor: the consumer *registry* is main-actor
/// state, even though the handlers themselves are `@Sendable` and run on the audio thread.
@MainActor
protocol BroadcastAudioProviding: AnyObject {
func addAudioBufferConsumer(id: String, handler: @escaping @Sendable (AVAudioPCMBuffer) -> Void)
func removeAudioBufferConsumer(id: String)
Expand Down
13 changes: 9 additions & 4 deletions OpenGlasses/Sources/Services/Display/Even/EvenBLETransport.swift
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,15 @@ import Foundation
final class EvenBLETransport: NSObject, ObservableObject, EvenTransporting {

// Community-reconstructed UUIDs (base 00002760-08C2-11E1-9073-0E8AC72EXXXX).
static let writeCharUUID = CBUUID(string: "00002760-08C2-11E1-9073-0E8AC72E5401")
static let notifyCharUUID = CBUUID(string: "00002760-08C2-11E1-9073-0E8AC72E5402")
static let renderCharUUID = CBUUID(string: "00002760-08C2-11E1-9073-0E8AC72E6402")
static let advertisedNamePrefix = "Even G2"
//
// `nonisolated` because the CoreBluetooth delegate callbacks below read them before hopping
// to the main actor — matching a UUID or an advertised name must not cost an actor hop on the
// BLE callback queue. `CBUUID` isn't annotated `Sendable`, but these are immutable value
// objects created once, so `nonisolated(unsafe)` states that rather than working around it.
nonisolated(unsafe) static let writeCharUUID = CBUUID(string: "00002760-08C2-11E1-9073-0E8AC72E5401")
nonisolated(unsafe) static let notifyCharUUID = CBUUID(string: "00002760-08C2-11E1-9073-0E8AC72E5402")
nonisolated(unsafe) static let renderCharUUID = CBUUID(string: "00002760-08C2-11E1-9073-0E8AC72E6402")
nonisolated static let advertisedNamePrefix = "Even G2"

var onEvent: (([UInt8]) -> Void)?
var onDisconnect: ((Error?) -> Void)?
Expand Down
17 changes: 14 additions & 3 deletions OpenGlasses/Sources/Services/GoogleOAuthService.swift
Original file line number Diff line number Diff line change
Expand Up @@ -139,9 +139,20 @@ final class GoogleOAuthService: NSObject, ObservableObject {
extension GoogleOAuthService: ASWebAuthenticationPresentationContextProviding {
nonisolated func presentationAnchor(for session: ASWebAuthenticationSession) -> ASPresentationAnchor {
MainActor.assumeIsolated {
UIApplication.shared.connectedScenes
.compactMap { ($0 as? UIWindowScene)?.keyWindow }
.first ?? ASPresentationAnchor()
let scenes = UIApplication.shared.connectedScenes.compactMap { $0 as? UIWindowScene }
// Prefer the scene the user is actually looking at, then any key window, then any
// window at all — an existing window always beats a fresh one as an anchor.
if let anchor = scenes.first(where: { $0.activationState == .foregroundActive })?.keyWindow {
return anchor
}
if let anchor = scenes.lazy.compactMap(\.keyWindow).first { return anchor }
if let anchor = scenes.lazy.flatMap(\.windows).first { return anchor }
if let scene = scenes.first { return UIWindow(windowScene: scene) }
// Unreachable: `start()` is only called from a user-initiated sign-in, which requires
// a live window scene. `init(windowScene:)` is the only UIWindow initialiser that
// survives iOS 26, so a scene-less process has no anchor to offer — and the empty
// window this used to return could never have presented anything either.
preconditionFailure("presentationAnchor requested with no connected UIWindowScene")
}
}
}
2 changes: 1 addition & 1 deletion OpenGlasses/Sources/Services/LLMService.swift
Original file line number Diff line number Diff line change
Expand Up @@ -2487,7 +2487,7 @@ class LLMService: ObservableObject {
/// The reduced tool set a local model is offered — only simple, reliable, self-contained tools
/// (each resolves its own inputs, e.g. `get_weather`/`where_am_i` read LocationService directly,
/// so the 2B model never has to supply coordinates it doesn't have).
static let localSafeTools: Set<String> = [
nonisolated static let localSafeTools: Set<String> = [
"get_weather", "get_datetime", "calculate", "set_timer",
"flashlight", "brightness", "calendar", "reminder",
"set_alarm", "step_count", "device_info", "music_control",
Expand Down
30 changes: 16 additions & 14 deletions OpenGlasses/Sources/Services/LocalLLMService.swift
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ final class LocalLLMService: ObservableObject {
/// (verified 2026-07-16), so vision is *attempted* — and a mapping failure now demotes
/// gracefully to the text factory (`loadModel`), with image turns refused honestly by
/// the vision guard in LLMService. Nothing regresses if a checkpoint doesn't cooperate.
static let visionModelIds: Set<String> = [
nonisolated static let visionModelIds: Set<String> = [
"mlx-community/SmolVLM2-2.2B-Instruct-mlx",
"mlx-community/SmolVLM2-500M-Video-Instruct-mlx",
"mlx-community/gemma-4-e2b-it-4bit",
Expand Down Expand Up @@ -578,19 +578,6 @@ final class LocalLLMService: ObservableObject {
container: ModelContainer,
onToken: ((String) -> Void)?
) async throws -> String {
guard let ciImage = CIImage(data: imageData) else {
throw LocalLLMError.generationFailed("Couldn't decode the photo.")
}

var chat: [Chat.Message] = [.system(systemPrompt)]
for turn in history.suffix(4) {
chat.append(turn.role == "assistant" ? .assistant(turn.content) : .user(turn.content))
}
chat.append(.user(userMessage, images: [.ciImage(ciImage)]))
// 896² is Gemma's native vision resolution and a sane cap for every supported VLM —
// a full-resolution glasses photo through the image pipeline is a pure memory spike.
let userInput = UserInput(chat: chat,
processing: .init(resize: CGSize(width: 896, height: 896)))
let parameters = GenerateParameters(maxTokens: 512, temperature: 0.7, topP: 0.9)

// Same mid-generation backgrounding watch as the text path (Metal in the background
Expand All @@ -609,7 +596,22 @@ final class LocalLLMService: ObservableObject {
defer { NotificationCenter.default.removeObserver(bgObserver) }

NSLog("🔬 LocalLLM.generateVisionTurn model=%@ image=%dKB", loadedModelId ?? "?", imageData.count / 1024)
// `UserInput` (and the `CIImage` inside it) isn't `Sendable`, so it's built *inside* the
// `@Sendable` closure from Sendable ingredients only — the image data, the prompt strings
// and the history — rather than constructed out here and captured across the boundary.
let output = try await container.perform { context -> String in
guard let ciImage = CIImage(data: imageData) else {
throw LocalLLMError.generationFailed("Couldn't decode the photo.")
}
var chat: [Chat.Message] = [.system(systemPrompt)]
for turn in history.suffix(4) {
chat.append(turn.role == "assistant" ? .assistant(turn.content) : .user(turn.content))
}
chat.append(.user(userMessage, images: [.ciImage(ciImage)]))
// 896² is Gemma's native vision resolution and a sane cap for every supported VLM —
// a full-resolution glasses photo through the image pipeline is a pure memory spike.
let userInput = UserInput(chat: chat,
processing: .init(resize: CGSize(width: 896, height: 896)))
let lmInput = try await context.processor.prepare(input: userInput)
let stream = try MLXLMCommon.generate(input: lmInput, parameters: parameters, context: context)
var iterator = stream.makeAsyncIterator()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,10 @@ final class WalkingRouteService: ObservableObject {

private func walkingRoute(from origin: CLLocationCoordinate2D, to item: MKMapItem) async throws -> MKRoute {
let request = MKDirections.Request()
request.source = MKMapItem(placemark: MKPlacemark(coordinate: origin))
// A coordinate-only origin: no address to attach, which is all `MKDirections` reads.
request.source = MKMapItem(
location: CLLocation(latitude: origin.latitude, longitude: origin.longitude),
address: nil)
request.destination = item
request.transportType = .walking
let response = try await MKDirections(request: request).calculate()
Expand Down
52 changes: 29 additions & 23 deletions OpenGlasses/Sources/Services/SignLanguage/FingerspellingModel.swift
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,34 @@ struct FingerspellingModelBundle: Equatable {
static let modelPackageName = "Fingerspelling2P.mlpackage"
static let landmarkerTaskName = "holistic_landmarker.task"

/// Production installer: fetch each required file (sub-paths preserved) with URLSession.
///
/// Lives on the bundle rather than the `@MainActor` downloader, mirroring `ASRModelBundle`:
/// the download loop and its file moves have no business on the main actor, and a `static`
/// on the main-actor class couldn't be read from the downloader's default argument without
/// a hop. The `await` on `progress` is a genuine hop to the `@MainActor` handler.
static let liveInstaller: FingerspellingModelDownloader.Installer = { bundle, destination, progress in
let files = bundle.requiredFiles
var completed = 0
for path in files {
guard let url = bundle.huggingFaceResolveURL(for: path) else {
throw FingerspellingDownloadError.notConfigured
}
let dest = destination.appendingPathComponent(path)
try FileManager.default.createDirectory(at: dest.deletingLastPathComponent(),
withIntermediateDirectories: true)
let (tempURL, response) = try await URLSession.shared.download(from: url)
if let http = response as? HTTPURLResponse, !(200..<300).contains(http.statusCode) {
throw FingerspellingDownloadError.incompleteDownload(missing: "\(path) (HTTP \(http.statusCode))")
}
try? FileManager.default.removeItem(at: dest)
try FileManager.default.moveItem(at: tempURL, to: dest)
completed += 1
let fraction = Double(completed) / Double(max(files.count, 1))
await progress(fraction)
}
}

/// The active bundle; the repo comes from Settings so publishing the artefact needs no
/// app update.
static var active: FingerspellingModelBundle {
Expand Down Expand Up @@ -161,7 +189,7 @@ final class FingerspellingModelDownloader: ObservableObject {
init(bundle: FingerspellingModelBundle = .active,
modelDirectory: URL? = nil,
fileManager: FileManager = .default,
installer: @escaping Installer = FingerspellingModelDownloader.liveInstaller) {
installer: @escaping Installer = FingerspellingModelBundle.liveInstaller) {
self.bundle = bundle
self.fileManager = fileManager
self.modelDirectory = modelDirectory
Expand Down Expand Up @@ -230,26 +258,4 @@ final class FingerspellingModelDownloader: ObservableObject {
state = store.state
}

/// Production installer: fetch each required file (sub-paths preserved) with URLSession.
static let liveInstaller: Installer = { bundle, destination, progress in
let files = bundle.requiredFiles
var completed = 0
for path in files {
guard let url = bundle.huggingFaceResolveURL(for: path) else {
throw FingerspellingDownloadError.notConfigured
}
let dest = destination.appendingPathComponent(path)
try FileManager.default.createDirectory(at: dest.deletingLastPathComponent(),
withIntermediateDirectories: true)
let (tempURL, response) = try await URLSession.shared.download(from: url)
if let http = response as? HTTPURLResponse, !(200..<300).contains(http.statusCode) {
throw FingerspellingDownloadError.incompleteDownload(missing: "\(path) (HTTP \(http.statusCode))")
}
try? FileManager.default.removeItem(at: dest)
try FileManager.default.moveItem(at: tempURL, to: dest)
completed += 1
let fraction = Double(completed) / Double(max(files.count, 1))
await progress(fraction)
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ final class FingerspellingSessionService: ObservableObject {
}
let finale = await pipeline.flush()
await self?.apply(finale)
await self?.finishStopped()
self?.finishStopped()
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ final class MediaTriggerService {
guard !isRunning, isEnabled() else { return }
isRunning = true
let center = NotificationCenter.default
let reevaluate: (Notification) -> Void = { [weak self] _ in
let reevaluate: @Sendable (Notification) -> Void = { [weak self] _ in
Task { @MainActor in self?.evaluate() }
}
// Every signal that the audio world changed funnels into one re-evaluation; the policy
Expand Down
4 changes: 3 additions & 1 deletion OpenGlasses/Sources/Services/VideoRecordingService.swift
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,9 @@ class VideoRecordingService: ObservableObject {
var onAutoStopped: ((String) -> Void)?

/// Seconds without a frame (after at least one arrived) before recording auto-stops.
static let frameStallSeconds: TimeInterval = 15
/// `nonisolated` so `shouldAutoStop`'s default argument (evaluated outside the actor) can
/// read it without a hop.
nonisolated static let frameStallSeconds: TimeInterval = 15

/// Wall-clock of the most recent appended frame (nil until the first frame arrives).
/// Written from the background frame queue, read by the main-actor watchdog tick.
Expand Down
6 changes: 5 additions & 1 deletion OpenGlasses/Sources/Services/Vision/OutboundFrameRelay.swift
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,11 @@ final class OutboundFrameRelay: ObservableObject {
return
}

self.queue.async {
// `[weak self]` here as well as on the inner `Task`: after the `guard let self`
// above, `self` is a strong local, so an unannotated closure would capture it
// strongly and the inner `[weak self]` would be decorative — a relay released
// mid-pipeline would be held alive to composite and publish one more frame.
self.queue.async { [weak self] in
let blurred = Self.composite(image, rects: rects, context: context) ?? image
Task { @MainActor [weak self] in self?.finish(with: blurred) }
}
Expand Down
2 changes: 1 addition & 1 deletion OpenGlassesTests/ASRModelStoreTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ final class ASRModelStoreTests: XCTestCase {
for name in bundle.requiredFiles {
try Data(repeating: 0x42, count: 8).write(to: destination.appendingPathComponent(name))
}
await progress(1.0)
progress(1.0)
}
let downloader = ASRModelDownloader(bundle: bundle, modelDirectory: modelDir, installer: installer)
await downloader.download()
Expand Down
4 changes: 2 additions & 2 deletions OpenGlassesTests/FingerspellingLiveDecoderTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ final class FingerspellingLiveDecoderTests: XCTestCase {
appendFrames(&decoder, 4)
// A three-way tie between letters: the argmax is a letter but its softmax
// probability ≈ 1/3 — under the 0.5 confidence floor, so it must count as blank.
let events = try decoder.tick(infer: { input in
let events = decoder.tick(infer: { input in
let validRows = (input.frameCount + 1) / 2
var tied = [Float](repeating: -20, count: 62)
tied[33] = 5; tied[34] = 5; tied[35] = 5 // 'a', 'b', 'c'
Expand Down Expand Up @@ -152,7 +152,7 @@ final class FingerspellingLiveDecoderTests: XCTestCase {
appendFrames(&decoder, 2)
XCTAssertEqual(decoder.windowedFrameCount, HolisticWindower.windowLength)
var fedRows = 0
_ = try decoder.tick(infer: { input in
_ = decoder.tick(infer: { input in
fedRows = (input.frameCount + 1) / 2
return (0..<fedRows).map { _ in self.logitRow(nil) }
})
Expand Down
2 changes: 1 addition & 1 deletion OpenGlassesTests/FingerspellingModelTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ final class FingerspellingModelTests: XCTestCase {
withIntermediateDirectories: true)
try Data("model-bytes".utf8).write(to: dest)
}
await progress(1.0)
progress(1.0)
})
await downloader.download()
XCTAssertEqual(downloader.state, .ready)
Expand Down
2 changes: 1 addition & 1 deletion OpenGlassesTests/KokoroModelDownloaderTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ final class KokoroModelDownloaderTests: XCTestCase {
/// Writes every required file + directory into `destination` (a complete, valid install).
private func fullInstaller(progressSteps: [Double] = []) -> KokoroModelDownloader.Installer {
{ bundle, destination, progress in
for step in progressSteps { await progress(step) }
for step in progressSteps { progress(step) }
try FileManager.default.createDirectory(at: destination, withIntermediateDirectories: true)
for name in bundle.requiredFiles {
try Data(repeating: 0x42, count: 8).write(to: destination.appendingPathComponent(name))
Expand Down
5 changes: 4 additions & 1 deletion OpenGlassesTests/WakeWordHardeningTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,10 @@ final class WakeWordHardeningTests: XCTestCase {
let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: 256)!
buffer.frameLength = 256

let received = NSCountedSet()
// `nonisolated(unsafe)`: `NSCountedSet` isn't `Sendable`, and the forwarders below are
// `@Sendable` closures called from a background queue. Every touch is `lock`-guarded —
// which is the point of the test — so the unsafety is the discipline, not a hole.
nonisolated(unsafe) let received = NSCountedSet()
let lock = NSLock()
box.setForwarders(["a": { _ in lock.lock(); received.add("a"); lock.unlock() }])

Expand Down
Loading
Loading