Surface + document the loopback Local API for on-device agents (Settings toggle + docs)

Fluid.xcodeproj/project.pbxproj

@@ -28,6 +28,9 @@
 		803000000000000000000002 /* MediaPlaybackServiceTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 803000000000000000000001 /* MediaPlaybackServiceTests.swift */; };
 		803000000000000000000003 /* MediaRemoteAdapter in Frameworks */ = {isa = PBXBuildFile; productRef = 803000000000000000000004 /* MediaRemoteAdapter */; };
 		7CDB0A2F2F3C4D5600FB7CAD /* dictation_fixture.wav in Resources */ = {isa = PBXBuildFile; fileRef = 7CDB0A2B2F3C4D5600FB7CAD /* dictation_fixture.wav */; };
+		0GG000000000000000000001 /* LocalAPIAudioDecoderTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 0GG000000000000000000002 /* LocalAPIAudioDecoderTests.swift */; };
+		0GG000000000000000000003 /* dictation_fixture.ogg in Resources */ = {isa = PBXBuildFile; fileRef = 0GG000000000000000000004 /* dictation_fixture.ogg */; };
+		0GG000000000000000000005 /* Opus.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = 0GG000000000000000000006 /* Opus.xcframework */; };
 		7CDB0A302F3C4D5600FB7CAD /* XCTest.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 7CDB0A2C2F3C4D5600FB7CAD /* XCTest.framework */; };
 		7CE006BD2E80EBE600DDCCD6 /* AppUpdater in Frameworks */ = {isa = PBXBuildFile; productRef = 7CE006BC2E80EBE600DDCCD6 /* AppUpdater */; };
 		86CAA2D4EF18433096185602 /* LLMClientRequestBodyTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 343B29013F4441D6A797D12D /* LLMClientRequestBodyTests.swift */; };
@@ -75,6 +78,9 @@
 		803000000000000000000001 /* MediaPlaybackServiceTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MediaPlaybackServiceTests.swift; sourceTree = "<group>"; };
 		7CDB0A2A2F3C4D5600FB7CAD /* AudioFixtureLoader.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AudioFixtureLoader.swift; sourceTree = "<group>"; };
 		7CDB0A2B2F3C4D5600FB7CAD /* dictation_fixture.wav */ = {isa = PBXFileReference; lastKnownFileType = audio.wav; path = dictation_fixture.wav; sourceTree = "<group>"; };
+		0GG000000000000000000002 /* LocalAPIAudioDecoderTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LocalAPIAudioDecoderTests.swift; sourceTree = "<group>"; };
+		0GG000000000000000000004 /* dictation_fixture.ogg */ = {isa = PBXFileReference; lastKnownFileType = file; path = dictation_fixture.ogg; sourceTree = "<group>"; };
+		0GG000000000000000000006 /* Opus.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; path = Vendor/Opus.xcframework; sourceTree = SOURCE_ROOT; };
 		7CDB0A2C2F3C4D5600FB7CAD /* XCTest.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = XCTest.framework; path = Platforms/MacOSX.platform/Developer/Library/Frameworks/XCTest.framework; sourceTree = DEVELOPER_DIR; };
 /* End PBXFileReference section */
 
@@ -91,6 +97,7 @@
 			isa = PBXFrameworksBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
+				0GG000000000000000000005 /* Opus.xcframework in Frameworks */,
 				7C3697892ED70F9C005874CE /* DynamicNotchKit in Frameworks */,
 				7C5AF14B2F15041600DE21B0 /* MediaRemoteAdapter in Frameworks */,
 				7C9A71022F58B00000FB7CAF /* TranscribeCpp in Frameworks */,
@@ -155,6 +162,7 @@
 				DA7100010000000000000001 /* DirectAudioReliabilityTests.swift */,
 				7CFA1D0B2F500000C0DEF002 /* TypingServiceTransientPasteboardTests.swift */,
 				803000000000000000000001 /* MediaPlaybackServiceTests.swift */,
+				0GG000000000000000000002 /* LocalAPIAudioDecoderTests.swift */,
 			);
 			path = FluidDictationIntegrationTests;
 			sourceTree = "<group>";
@@ -171,13 +179,15 @@
 			isa = PBXGroup;
 			children = (
 				7CDB0A2B2F3C4D5600FB7CAD /* dictation_fixture.wav */,
+				0GG000000000000000000004 /* dictation_fixture.ogg */,
 			);
 			path = Resources;
 			sourceTree = "<group>";
 		};
 		7CDB0A282F3C4D5600FB7CAD /* Frameworks */ = {
 			isa = PBXGroup;
 			children = (
+				0GG000000000000000000006 /* Opus.xcframework */,
 				7CDB0A2C2F3C4D5600FB7CAD /* XCTest.framework */,
 			);
 			name = Frameworks;
@@ -293,6 +303,7 @@
 			buildActionMask = 2147483647;
 			files = (
 				7CDB0A2F2F3C4D5600FB7CAD /* dictation_fixture.wav in Resources */,
+				0GG000000000000000000003 /* dictation_fixture.ogg in Resources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
@@ -324,6 +335,7 @@
 				DA7100020000000000000002 /* DirectAudioReliabilityTests.swift in Sources */,
 				7CFA1D0B2F500000C0DEF001 /* TypingServiceTransientPasteboardTests.swift in Sources */,
 				803000000000000000000002 /* MediaPlaybackServiceTests.swift in Sources */,
+				0GG000000000000000000001 /* LocalAPIAudioDecoderTests.swift in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};

README.md

@@ -173,6 +173,28 @@ Whisper supports up to 99 languages, depending on the model size you choose.
 
 ---
 
+## Local API (for on-device agents)
+
+FluidVoice can expose a **loopback-only** HTTP API so other apps and AI agents running **on the same Mac** can reuse its transcription and post-processing — instead of bundling and loading their own ASR model. One model stays warm in FluidVoice; agents get FluidVoice-quality transcription with no extra memory footprint.
+
+**Enable it:** `Settings → Local API (on-device agents)`. It is **off by default** and **refuses all non-loopback connections** — any non-local peer is rejected at accept time, before it can send a request — so the API is usable only from processes on this Mac. The API has no authentication; every local login session/process can access all enabled routes, including history and dictionary routes.
+
+- Base URL: `http://127.0.0.1:47733` (port configurable via the `LocalAPIPort` preference)
+- `GET  /v1/health` — liveness check
+- `POST /v1/transcribe` — transcribe an audio file → `{ text, confidence, sampleCount, provider }`
+- `POST /v1/postprocess` — run text post-processing/enhancement
+
+```bash
+curl -s http://127.0.0.1:47733/v1/transcribe \
+  -H 'Content-Type: application/json' \
+  -d '{"path": "/absolute/path/to/audio.ogg"}'
+# → {"text":"…","confidence":0.97,"sampleCount":…,"provider":"Parakeet TDT v3 (Multilingual)"}
+```
+
+Audio can also be sent inline as `{"audioBase64": "…", "filename": "clip.wav"}`. Limits: **25 MB** request body, **25 MB** referenced OGG/Opus file, and **300 s (5 min)** of audio per transcription (independent caps). File-path input and inline OGG/Opus input longer than 300 s are rejected; other inline formats are truncated to the first 300 s. Note: `/v1/transcribe` runs on-device, but `/v1/postprocess` uses your configured AI provider (Settings → AI Enhancement), which may be remote. Full reference: [docs/local-api.md](docs/local-api.md).
+
+---
+
 ## Requirements
 
 - macOS 15.0 (Sequoia) or later

Sources/Fluid/Persistence/SettingsStore.swift

@@ -1817,6 +1817,15 @@ final class SettingsStore: ObservableObject {
         set { self.defaults.set(newValue, forKey: Keys.copyTranscriptionToClipboard) }
     }
 
+    /// When enabled, the loopback-only Local API server (`LocalAPIServer`) is started so
+    /// on-device agents can reuse FluidVoice's transcription (POST /v1/transcribe) instead
+    /// of shipping their own ASR model. Off by default. Uses the same "LocalAPIEnabled"
+    /// key that `LocalAPI.Configuration` reads.
+    var localAPIEnabled: Bool {
+        get { self.defaults.bool(forKey: Keys.localAPIEnabled) }
+        set { self.defaults.set(newValue, forKey: Keys.localAPIEnabled) }
+    }
+
     var preferredInputDeviceUID: String? {
         get { self.defaults.string(forKey: Keys.preferredInputDeviceUID) }
         set { self.defaults.set(newValue, forKey: Keys.preferredInputDeviceUID) }
@@ -5262,6 +5271,7 @@ private extension SettingsStore {
         static let skipSilentRecordingsEnabled = "SkipSilentRecordingsEnabled"
         static let enableAIStreaming = "EnableAIStreaming"
         static let copyTranscriptionToClipboard = "CopyTranscriptionToClipboard"
+        static let localAPIEnabled = "LocalAPIEnabled"
         static let textInsertionMode = "TextInsertionMode"
         static let autoUpdateCheckEnabled = "AutoUpdateCheckEnabled"
         static let betaReleasesEnabled = "BetaReleasesEnabled"

Sources/Fluid/Services/ASRService.swift

@@ -2417,7 +2417,7 @@ final class ASRService: ObservableObject {
             )
         }
 
-        let estimatedSamples = try LocalAPIAudioDecoder.validateDurationWithinLimit(for: fileURL)
+        let preparedAudio = try LocalAPIAudioDecoder.prepareForTranscription(fileURL: fileURL)
 
         try await self.ensureAsrReady()
         let provider = self.transcriptionProvider
@@ -2429,7 +2429,7 @@ final class ASRService: ObservableObject {
             )
         }
 
-        guard provider.prefersNativeFileTranscription else {
+        guard provider.prefersNativeFileTranscription, !preparedAudio.requiresBufferedTranscription else {
             let samples = try LocalAPIAudioDecoder.samples(from: fileURL)
             let result = try await self.transcribeSamplesForAPI(samples)
             return (result, samples.count)
@@ -2449,7 +2449,10 @@ final class ASRService: ObservableObject {
             ASRService.applyCustomDictionary(ASRService.removeFillerWords(result.text))
         )
         self.recordWordBoostHitIfAny(transcribedText: cleanedText)
-        return (ASRTranscriptionResult(text: cleanedText, confidence: result.confidence), estimatedSamples)
+        return (
+            ASRTranscriptionResult(text: cleanedText, confidence: result.confidence),
+            preparedAudio.estimatedSamples
+        )
     }
 
     func stopWithoutTranscription() async {

Sources/Fluid/Services/LocalAPI/InferenceAPIController.swift

@@ -2,6 +2,17 @@ import Foundation
 
 @MainActor
 final class InferenceAPIController: LocalAPIRouteHandler {
+    typealias SampleTranscriber = ([Float]) async throws -> (text: String, confidence: Float, provider: String)
+
+    private let sampleTranscriber: SampleTranscriber
+
+    init(sampleTranscriber: @escaping SampleTranscriber = { samples in
+        let result = try await AppServices.shared.asr.transcribeSamplesForAPI(samples)
+        return (result.text, result.confidence, SettingsStore.shared.selectedSpeechModel.displayName)
+    }) {
+        self.sampleTranscriber = sampleTranscriber
+    }
+
     struct TranscribeJSONRequest: Decodable {
         let path: String?
         let audioBase64: String?
@@ -55,13 +66,13 @@ final class InferenceAPIController: LocalAPIRouteHandler {
             }
 
             let samples = try self.decodeAudioSamples(from: request)
-            let result = try await AppServices.shared.asr.transcribeSamplesForAPI(samples)
+            let result = try await self.sampleTranscriber(samples)
             return LocalAPI.json(
                 TranscribeResponse(
                     text: result.text,
                     confidence: result.confidence,
                     sampleCount: samples.count,
-                    provider: SettingsStore.shared.selectedSpeechModel.displayName
+                    provider: result.provider
                 )
             )
         } catch {

Sources/Fluid/Services/LocalAPI/LocalAPIAudioDecoder.swift

@@ -2,10 +2,50 @@ import AVFoundation
 import Foundation
 
 enum LocalAPIAudioDecoder {
+    struct PreparedAudio {
+        let estimatedSamples: Int
+        let requiresBufferedTranscription: Bool
+    }
+
     static let sampleRate: Double = 16_000
     static let maxDurationSeconds: Double = 300
+    private static let oggSniffBytes = 64 * 1024
+
+    static func requiresBufferedTranscription(for fileURL: URL) throws -> Bool {
+        try self.isOggOpusFile(at: fileURL)
+    }
+
+    static func prepareForTranscription(fileURL: URL) throws -> PreparedAudio {
+        if try self.isOggOpusFile(at: fileURL) {
+            try self.validateOggFileSize(at: fileURL)
+            return PreparedAudio(
+                estimatedSamples: try OggOpusDecoder.sampleCount(fromFileAt: fileURL),
+                requiresBufferedTranscription: true
+            )
+        }
+        return PreparedAudio(
+            estimatedSamples: try self.validateDurationWithinLimitUsingAVFoundation(for: fileURL),
+            requiresBufferedTranscription: false
+        )
+    }
 
     static func samples(from fileURL: URL) throws -> [Float] {
+        if try self.isOggOpusFile(at: fileURL) {
+            try self.validateOggFileSize(at: fileURL)
+            return try self.oggOpusSamples(from: Data(contentsOf: fileURL, options: .mappedIfSafe))
+        }
+        do {
+            return try self.samplesUsingAVFoundation(from: fileURL)
+        } catch {
+            throw NSError(
+                domain: "LocalAPIAudioDecoder",
+                code: -7,
+                userInfo: [NSLocalizedDescriptionKey: "Unsupported or invalid audio file: \(error.localizedDescription)"]
+            )
+        }
+    }
+
+    private static func samplesUsingAVFoundation(from fileURL: URL) throws -> [Float] {
         let file = try AVAudioFile(forReading: fileURL)
         let sourceFormat = file.processingFormat
         let maxFrames = AVAudioFramePosition(sourceFormat.sampleRate * self.maxDurationSeconds)
@@ -39,7 +79,53 @@ enum LocalAPIAudioDecoder {
     }
 
     static func validateDurationWithinLimit(for fileURL: URL) throws -> Int {
-        let file = try AVAudioFile(forReading: fileURL)
+        if try self.isOggOpusFile(at: fileURL) {
+            try self.validateOggFileSize(at: fileURL)
+            return try OggOpusDecoder.sampleCount(fromFileAt: fileURL)
+        }
+        return try self.validateDurationWithinLimitUsingAVFoundation(for: fileURL)
+    }
+
+    private static func validateDurationWithinLimitUsingAVFoundation(for fileURL: URL) throws -> Int {
+        do {
+            let file = try AVAudioFile(forReading: fileURL)
+            return try self.validateDurationWithinLimit(for: file)
+        } catch {
+            throw NSError(
+                domain: "LocalAPIAudioDecoder",
+                code: -7,
+                userInfo: [NSLocalizedDescriptionKey: "Unsupported or invalid audio file: \(error.localizedDescription)"]
+            )
+        }
+    }
+
+    private static func isOggOpusFile(at fileURL: URL) throws -> Bool {
+        let values = try fileURL.resourceValues(forKeys: [.isRegularFileKey])
+        guard values.isRegularFile == true else {
+            throw NSError(
+                domain: "LocalAPIAudioDecoder",
+                code: -10,
+                userInfo: [NSLocalizedDescriptionKey: "Audio path must reference a regular file."]
+            )
+        }
+
+        let handle = try FileHandle(forReadingFrom: fileURL)
+        defer { try? handle.close() }
+        return OggOpusDecoder.isOggOpus(try handle.read(upToCount: self.oggSniffBytes) ?? Data())
+    }
+
+    private static func validateOggFileSize(at fileURL: URL) throws {
+        let values = try fileURL.resourceValues(forKeys: [.fileSizeKey])
+        guard let fileSize = values.fileSize, fileSize <= LocalAPI.maxRequestBytes else {
+            throw NSError(
+                domain: "LocalAPIAudioDecoder",
+                code: -8,
+                userInfo: [NSLocalizedDescriptionKey: "Audio input exceeds the 25 MB API limit."]
+            )
+        }
+    }
+
+    private static func validateDurationWithinLimit(for file: AVAudioFile) throws -> Int {
         let sourceFormat = file.processingFormat
         guard sourceFormat.sampleRate > 0 else {
             throw NSError(domain: "LocalAPIAudioDecoder", code: -6, userInfo: [NSLocalizedDescriptionKey: "Audio file has an invalid sample rate."])
@@ -56,4 +142,36 @@ enum LocalAPIAudioDecoder {
 
         return Int((Double(file.length) * self.sampleRate / sourceFormat.sampleRate).rounded())
     }
+
+    private static func downsampleOpusTo16k(_ samples: [Float]) -> [Float] {
+        guard samples.count >= 3 else { return [] }
+        var output = [Float]()
+        output.reserveCapacity(samples.count / 3)
+        for index in stride(from: 0, through: samples.count - 3, by: 3) {
+            output.append((samples[index] + samples[index + 1] + samples[index + 2]) / 3)
+        }
+        return output
+    }
+
+    static func oggOpusSamples(from data: Data) throws -> [Float] {
+        try self.validateRequestSize(data)
+        guard OggOpusDecoder.isOggOpus(data) else {
+            throw NSError(
+                domain: "LocalAPIAudioDecoder",
+                code: -9,
+                userInfo: [NSLocalizedDescriptionKey: "Audio input is not an OGG/Opus stream."]
+            )
+        }
+        return self.downsampleOpusTo16k(try OggOpusDecoder.samples(from: data))
+    }
+
+    private static func validateRequestSize(_ data: Data) throws {
+        guard data.count <= LocalAPI.maxRequestBytes else {
+            throw NSError(
+                domain: "LocalAPIAudioDecoder",
+                code: -8,
+                userInfo: [NSLocalizedDescriptionKey: "Audio input exceeds the 25 MB API limit."]
+            )
+        }
+    }
 }