feat: voice input

packages/app/src/components/prompt-input-v2.tsx

@@ -24,9 +24,15 @@ import { type ImageAttachmentPart, usePrompt } from "@/context/prompt"
 import { usePlatform } from "@/context/platform"
 import { useSDK } from "@/context/sdk"
 import { useSync } from "@/context/sync"
+import { useSettings } from "@/context/settings"
 import { createSessionTabs } from "@/pages/session/helpers"
 import { showToast } from "@/utils/toast"
-import { PromptInputV2, type PromptInputV2Suggestion } from "@opencode-ai/session-ui/v2/prompt-input"
+import { createMediaRecorderInput } from "@/components/prompt-input/media-recorder-input"
+import {
+  PromptInputV2,
+  type PromptInputV2Suggestion,
+  type PromptInputV2VoiceInput,
+} from "@opencode-ai/session-ui/v2/prompt-input"
 import {
   createPromptInputV2Controller,
   createPromptInputV2State,
@@ -42,12 +48,14 @@ export type PromptInputV2ComposerProps = {
 export type PromptInputV2ControllerProps = Omit<PromptInputProps, "class" | "submission">
 export type PromptInputV2ComposerController = PromptInputV2Interaction & {
   readonly model: PromptInputProps["controls"]["model"]
+  readonly voiceInput: PromptInputV2VoiceInput
 }
 
 export function PromptInputV2Composer(props: PromptInputV2ComposerProps) {
   const dialog = useDialog()
   const command = useCommand()
   const language = useLanguage()
+  const settings = useSettings()
 
   return (
     <div class="flex flex-col gap-3">
@@ -58,6 +66,7 @@ export function PromptInputV2Composer(props: PromptInputV2ComposerProps) {
         variantControlVisible={!props.controller.model.loading}
         attachKeybind={command.keybindParts("file.attach")}
         attachShortcut={command.keybind("file.attach")}
+        voiceInput={settings.voice.enabled() ? props.controller.voiceInput : undefined}
         modelControl={
           <PromptInputV2ModelControl
             loading={props.controller.model.loading}
@@ -87,9 +96,23 @@ export function usePromptInputV2Controller(props: PromptInputV2ControllerProps):
   const dialog = useDialog()
   const command = useCommand()
   const permission = usePermission()
+  const settings = useSettings()
   const language = useLanguage()
   const platform = usePlatform()
   const prompt = props.state ?? usePrompt()
+  const voiceInput = createMediaRecorderInput({
+    serverUrl: () => sdk().url,
+    directory: () => sdk().directory,
+    providerID: () => settings.voice.provider() || props.controls.model.selection.current()?.provider?.id,
+    modelID: () => settings.voice.model(),
+    language: () => settings.voice.language(),
+    onError: (message) =>
+      showToast({
+        variant: "error",
+        title: language.t("common.requestFailed"),
+        description: message || language.t("ui.promptInput.voice.error"),
+      }),
+  })
   let editor: HTMLDivElement | undefined
 
   const interaction = createPromptInputV2State()
@@ -409,6 +432,7 @@ export function usePromptInputV2Controller(props: PromptInputV2ControllerProps):
     },
   })
   Object.defineProperty(controller, "model", { get: () => props.controls.model })
+  Object.defineProperty(controller, "voiceInput", { value: voiceInput })
 
   command.register("prompt-input", () => [
     {

packages/app/src/components/prompt-input/media-recorder-input.ts

@@ -0,0 +1,184 @@
+import { onCleanup } from "solid-js"
+import { createStore } from "solid-js/store"
+import type { TranscriptionLanguage } from "@/context/settings"
+import type { PromptInputV2VoiceInputState } from "@opencode-ai/session-ui/v2/prompt-input"
+
+type Status = PromptInputV2VoiceInputState
+
+export function createMediaRecorderInput(input: {
+  serverUrl: () => string
+  directory: () => string
+  providerID: () => string | undefined
+  modelID: () => string
+  language: () => TranscriptionLanguage
+  onError: (message: string) => void
+}) {
+  const supported =
+    typeof window !== "undefined" &&
+    window.isSecureContext &&
+    !!navigator.mediaDevices?.getUserMedia &&
+    typeof MediaRecorder !== "undefined"
+  const [state, setState] = createStore({ status: "idle" as Status, levels: Array.from({ length: 16 }, () => 0) })
+  let active = false
+  let starting = false
+  let recorder: MediaRecorder | undefined
+  let stream: MediaStream | undefined
+  let chunks: Blob[] = []
+  let audioContext: AudioContext | undefined
+  let analyser: AnalyserNode | undefined
+  let levelsFrame: number | undefined
+  let resolveStop: ((value: string) => void) | undefined
+
+  const finish = (text: string) => {
+    if (levelsFrame !== undefined) cancelAnimationFrame(levelsFrame)
+    levelsFrame = undefined
+    void audioContext?.close()
+    audioContext = undefined
+    analyser = undefined
+    setState("levels", Array.from({ length: 16 }, () => 0))
+    setState("status", "idle")
+    resolveStop?.(text)
+    resolveStop = undefined
+  }
+
+  const transcribe = async (blob: Blob) => {
+    const url = new URL("/api/audio/transcribe", input.serverUrl())
+    const providerID = input.providerID()
+    if (providerID) url.searchParams.set("provider", providerID)
+    url.searchParams.set("model", input.modelID())
+    url.searchParams.set("language", input.language())
+    const response = await fetch(url, {
+      method: "POST",
+      headers: {
+        "Content-Type": blob.type || "audio/webm",
+        "X-Filename": `voice.${blob.type.includes("mp4") ? "mp4" : "webm"}`,
+        "x-opencode-directory": input.directory(),
+      },
+      body: blob,
+    })
+    const result: unknown = await response.json()
+    if (!response.ok) {
+      const message = isRecord(result) && typeof result.error === "string" ? result.error : "Transcription failed"
+      throw new Error(message)
+    }
+    if (!isRecord(result) || typeof result.text !== "string") throw new Error("Transcription returned no text")
+    return result.text.trim()
+  }
+
+  const stopTracks = () => {
+    stream?.getTracks().forEach((track) => track.stop())
+    stream = undefined
+  }
+
+  const updateLevels = () => {
+    if (!analyser || state.status !== "recording") return
+    const values = new Uint8Array(analyser.frequencyBinCount)
+    analyser.getByteFrequencyData(values)
+    setState(
+      "levels",
+      Array.from({ length: 16 }, (_, index) => {
+        const start = Math.floor((index * values.length) / 16)
+        const end = Math.max(start + 1, Math.floor(((index + 1) * values.length) / 16))
+        const average = values.slice(start, end).reduce((sum, value) => sum + value, 0) / (end - start)
+        return average / 255
+      }),
+    )
+    levelsFrame = requestAnimationFrame(updateLevels)
+  }
+
+  const stopRecorder = () => {
+    if (!recorder || recorder.state === "inactive") return
+    recorder.stop()
+  }
+
+  const begin = async () => {
+    starting = true
+    try {
+      stream = await navigator.mediaDevices.getUserMedia({ audio: true })
+      if (!active) {
+        starting = false
+        stopTracks()
+        finish("")
+        return
+      }
+      const mimeType = ["audio/webm;codecs=opus", "audio/webm", "audio/mp4", "audio/ogg;codecs=opus"].find((value) =>
+        MediaRecorder.isTypeSupported(value),
+      )
+      recorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined)
+      chunks = []
+      recorder.ondataavailable = (event) => {
+        if (event.data.size > 0) chunks.push(event.data)
+      }
+      recorder.onerror = () => {
+        stopTracks()
+        recorder = undefined
+        finish("")
+        input.onError("Unable to record audio")
+      }
+      recorder.onstop = () => {
+        const audio = new Blob(chunks, { type: recorder?.mimeType || "audio/webm" })
+        stopTracks()
+        recorder = undefined
+        chunks = []
+        if (audio.size === 0) {
+          finish("")
+          return
+        }
+        void transcribe(audio)
+          .then(finish)
+          .catch((error: unknown) => {
+            finish("")
+            input.onError(error instanceof Error ? error.message : "Transcription failed")
+          })
+      }
+      recorder.start()
+      if (typeof AudioContext !== "undefined") {
+        audioContext = new AudioContext()
+        analyser = audioContext.createAnalyser()
+        analyser.fftSize = 128
+        audioContext.createMediaStreamSource(stream).connect(analyser)
+        updateLevels()
+      }
+      starting = false
+      if (!active) stopRecorder()
+    } catch (error) {
+      starting = false
+      active = false
+      stopTracks()
+      finish("")
+      input.onError(error instanceof Error ? error.message : "Microphone access was denied")
+    }
+  }
+
+  onCleanup(() => {
+    active = false
+    stopRecorder()
+    stopTracks()
+  })
+
+  return {
+    supported,
+    state: () => state.status,
+    levels: () => state.levels,
+    start() {
+      if (!supported || active || state.status !== "idle") return
+      active = true
+      setState("status", "recording")
+      void begin()
+    },
+    stop() {
+      if (!supported || !active) return Promise.resolve("")
+      active = false
+      setState("status", "processing")
+      const result = new Promise<string>((resolve) => {
+        resolveStop = resolve
+      })
+      if (!starting) stopRecorder()
+      return result
+    },
+  }
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === "object" && value !== null
+}

packages/app/src/components/settings-v2/general.tsx

@@ -9,6 +9,8 @@ import { useLanguage } from "@/context/language"
 import { usePlatform } from "@/context/platform"
 import { useUpdaterAction } from "../updater-action"
 import { useSettings } from "@/context/settings"
+import type { TranscriptionLanguage } from "@/context/settings"
+import { useProviders } from "@/hooks/use-providers"
 import { ExternalLink } from "../external-link"
 import { SettingsListV2 } from "./parts/list"
 import { SettingsRowV2 } from "./parts/row"
@@ -271,6 +273,92 @@ const LanguageSetting = () => {
   )
 }
 
+const VoiceInputSection: Component = () => {
+  const language = useLanguage()
+  const settings = useSettings()
+  const providers = useProviders(() => undefined)
+  const options = createMemo(() => [
+    { id: "local", name: language.t("settings.general.row.voiceProvider.local") },
+    ...providers.connected().map((provider) => ({ id: provider.id, name: provider.name })),
+  ])
+  const modelOptions = createMemo(() => {
+    if (settings.voice.provider() === "local") {
+      return ["tiny", "base", "small", "medium", "large-v3"].map((id) => ({ id, name: `Whisper ${id}` }))
+    }
+    const provider = providers.connected().find((item) => item.id === settings.voice.provider())
+    return Object.values(provider?.models ?? {}).map((model) => ({ id: model.id, name: model.name }))
+  })
+  const languageOptions = createMemo(() => [
+    { id: "auto" as const, name: language.t("settings.general.row.voiceLanguage.auto") },
+    ...language.locales.map((locale) => ({ id: locale as TranscriptionLanguage, name: language.label(locale) })),
+  ])
+
+  return (
+    <div class="settings-v2-section">
+      <h3 class="settings-v2-section-title">{language.t("settings.general.section.voice")}</h3>
+      <SettingsListV2>
+        <SettingsRowV2
+          title={language.t("settings.general.row.voiceEnabled.title")}
+          description={language.t("settings.general.row.voiceEnabled.description")}
+        >
+          <Switch
+            checked={settings.voice.enabled()}
+            onChange={(enabled) => settings.voice.setEnabled(enabled)}
+          />
+        </SettingsRowV2>
+        <SettingsRowV2
+          title={language.t("settings.general.row.voiceProvider.title")}
+          description={language.t("settings.general.row.voiceProvider.description")}
+        >
+          <SelectV2
+            appearance="inline"
+            data-action="settings-voice-provider"
+            options={options()}
+            current={options().find((option) => option.id === settings.voice.provider())}
+            placement="bottom-end"
+            gutter={6}
+            value={(option) => option.id}
+            label={(option) => option.name}
+            onSelect={(option) => option && settings.voice.setProvider(option.id)}
+          />
+        </SettingsRowV2>
+        <SettingsRowV2
+          title={language.t("settings.general.row.voiceModel.title")}
+          description={language.t("settings.general.row.voiceModel.description")}
+        >
+          <SelectV2
+              appearance="inline"
+              data-action="settings-voice-model"
+              options={modelOptions()}
+              current={modelOptions().find((option) => option.id === settings.voice.model())}
+              placement="bottom-end"
+              gutter={6}
+              value={(option) => option.id}
+              label={(option) => option.name}
+              onSelect={(option) => option && settings.voice.setModel(option.id)}
+            />
+        </SettingsRowV2>
+        <SettingsRowV2
+          title={language.t("settings.general.row.voiceLanguage.title")}
+          description={language.t("settings.general.row.voiceLanguage.description")}
+        >
+          <SelectV2
+            appearance="inline"
+            data-action="settings-voice-language"
+            options={languageOptions()}
+            current={languageOptions().find((option) => option.id === settings.voice.language())}
+            placement="bottom-end"
+            gutter={6}
+            value={(option) => option.id}
+            label={(option) => option.name}
+            onSelect={(option) => option && settings.voice.setLanguage(option.id)}
+          />
+        </SettingsRowV2>
+      </SettingsListV2>
+    </div>
+  )
+}
+
 export const SettingsGeneralV2: Component<{
   sessionID?: string
 }> = (props) => {
@@ -552,6 +640,8 @@ export const SettingsGeneralV2: Component<{
 
         <GeneralSection />
 
+        <VoiceInputSection />
+
         <AppearanceSection controller={appearance} />
 
         <NotificationsSection />