diff --git a/packages/app/src/components/prompt-input-v2.tsx b/packages/app/src/components/prompt-input-v2.tsx index 481b7922cf36..5c4a556025e4 100644 --- a/packages/app/src/components/prompt-input-v2.tsx +++ b/packages/app/src/components/prompt-input-v2.tsx @@ -24,9 +24,15 @@ import { type ImageAttachmentPart, usePrompt } from "@/context/prompt" import { usePlatform } from "@/context/platform" import { useSDK } from "@/context/sdk" import { useSync } from "@/context/sync" +import { useSettings } from "@/context/settings" import { createSessionTabs } from "@/pages/session/helpers" import { showToast } from "@/utils/toast" -import { PromptInputV2, type PromptInputV2Suggestion } from "@opencode-ai/session-ui/v2/prompt-input" +import { createMediaRecorderInput } from "@/components/prompt-input/media-recorder-input" +import { + PromptInputV2, + type PromptInputV2Suggestion, + type PromptInputV2VoiceInput, +} from "@opencode-ai/session-ui/v2/prompt-input" import { createPromptInputV2Controller, createPromptInputV2State, @@ -42,12 +48,14 @@ export type PromptInputV2ComposerProps = { export type PromptInputV2ControllerProps = Omit export type PromptInputV2ComposerController = PromptInputV2Interaction & { readonly model: PromptInputProps["controls"]["model"] + readonly voiceInput: PromptInputV2VoiceInput } export function PromptInputV2Composer(props: PromptInputV2ComposerProps) { const dialog = useDialog() const command = useCommand() const language = useLanguage() + const settings = useSettings() return (
@@ -58,6 +66,7 @@ export function PromptInputV2Composer(props: PromptInputV2ComposerProps) { variantControlVisible={!props.controller.model.loading} attachKeybind={command.keybindParts("file.attach")} attachShortcut={command.keybind("file.attach")} + voiceInput={settings.voice.enabled() ? props.controller.voiceInput : undefined} modelControl={ sdk().url, + directory: () => sdk().directory, + providerID: () => settings.voice.provider() || props.controls.model.selection.current()?.provider?.id, + modelID: () => settings.voice.model(), + language: () => settings.voice.language(), + onError: (message) => + showToast({ + variant: "error", + title: language.t("common.requestFailed"), + description: message || language.t("ui.promptInput.voice.error"), + }), + }) let editor: HTMLDivElement | undefined const interaction = createPromptInputV2State() @@ -409,6 +432,7 @@ export function usePromptInputV2Controller(props: PromptInputV2ControllerProps): }, }) Object.defineProperty(controller, "model", { get: () => props.controls.model }) + Object.defineProperty(controller, "voiceInput", { value: voiceInput }) command.register("prompt-input", () => [ { diff --git a/packages/app/src/components/prompt-input/media-recorder-input.ts b/packages/app/src/components/prompt-input/media-recorder-input.ts new file mode 100644 index 000000000000..7fbdc49521c6 --- /dev/null +++ b/packages/app/src/components/prompt-input/media-recorder-input.ts @@ -0,0 +1,184 @@ +import { onCleanup } from "solid-js" +import { createStore } from "solid-js/store" +import type { TranscriptionLanguage } from "@/context/settings" +import type { PromptInputV2VoiceInputState } from "@opencode-ai/session-ui/v2/prompt-input" + +type Status = PromptInputV2VoiceInputState + +export function createMediaRecorderInput(input: { + serverUrl: () => string + directory: () => string + providerID: () => string | undefined + modelID: () => string + language: () => TranscriptionLanguage + onError: (message: string) => void +}) { + const supported = + typeof window !== "undefined" && + window.isSecureContext && + !!navigator.mediaDevices?.getUserMedia && + typeof MediaRecorder !== "undefined" + const [state, setState] = createStore({ status: "idle" as Status, levels: Array.from({ length: 16 }, () => 0) }) + let active = false + let starting = false + let recorder: MediaRecorder | undefined + let stream: MediaStream | undefined + let chunks: Blob[] = [] + let audioContext: AudioContext | undefined + let analyser: AnalyserNode | undefined + let levelsFrame: number | undefined + let resolveStop: ((value: string) => void) | undefined + + const finish = (text: string) => { + if (levelsFrame !== undefined) cancelAnimationFrame(levelsFrame) + levelsFrame = undefined + void audioContext?.close() + audioContext = undefined + analyser = undefined + setState("levels", Array.from({ length: 16 }, () => 0)) + setState("status", "idle") + resolveStop?.(text) + resolveStop = undefined + } + + const transcribe = async (blob: Blob) => { + const url = new URL("/api/audio/transcribe", input.serverUrl()) + const providerID = input.providerID() + if (providerID) url.searchParams.set("provider", providerID) + url.searchParams.set("model", input.modelID()) + url.searchParams.set("language", input.language()) + const response = await fetch(url, { + method: "POST", + headers: { + "Content-Type": blob.type || "audio/webm", + "X-Filename": `voice.${blob.type.includes("mp4") ? "mp4" : "webm"}`, + "x-opencode-directory": input.directory(), + }, + body: blob, + }) + const result: unknown = await response.json() + if (!response.ok) { + const message = isRecord(result) && typeof result.error === "string" ? result.error : "Transcription failed" + throw new Error(message) + } + if (!isRecord(result) || typeof result.text !== "string") throw new Error("Transcription returned no text") + return result.text.trim() + } + + const stopTracks = () => { + stream?.getTracks().forEach((track) => track.stop()) + stream = undefined + } + + const updateLevels = () => { + if (!analyser || state.status !== "recording") return + const values = new Uint8Array(analyser.frequencyBinCount) + analyser.getByteFrequencyData(values) + setState( + "levels", + Array.from({ length: 16 }, (_, index) => { + const start = Math.floor((index * values.length) / 16) + const end = Math.max(start + 1, Math.floor(((index + 1) * values.length) / 16)) + const average = values.slice(start, end).reduce((sum, value) => sum + value, 0) / (end - start) + return average / 255 + }), + ) + levelsFrame = requestAnimationFrame(updateLevels) + } + + const stopRecorder = () => { + if (!recorder || recorder.state === "inactive") return + recorder.stop() + } + + const begin = async () => { + starting = true + try { + stream = await navigator.mediaDevices.getUserMedia({ audio: true }) + if (!active) { + starting = false + stopTracks() + finish("") + return + } + const mimeType = ["audio/webm;codecs=opus", "audio/webm", "audio/mp4", "audio/ogg;codecs=opus"].find((value) => + MediaRecorder.isTypeSupported(value), + ) + recorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined) + chunks = [] + recorder.ondataavailable = (event) => { + if (event.data.size > 0) chunks.push(event.data) + } + recorder.onerror = () => { + stopTracks() + recorder = undefined + finish("") + input.onError("Unable to record audio") + } + recorder.onstop = () => { + const audio = new Blob(chunks, { type: recorder?.mimeType || "audio/webm" }) + stopTracks() + recorder = undefined + chunks = [] + if (audio.size === 0) { + finish("") + return + } + void transcribe(audio) + .then(finish) + .catch((error: unknown) => { + finish("") + input.onError(error instanceof Error ? error.message : "Transcription failed") + }) + } + recorder.start() + if (typeof AudioContext !== "undefined") { + audioContext = new AudioContext() + analyser = audioContext.createAnalyser() + analyser.fftSize = 128 + audioContext.createMediaStreamSource(stream).connect(analyser) + updateLevels() + } + starting = false + if (!active) stopRecorder() + } catch (error) { + starting = false + active = false + stopTracks() + finish("") + input.onError(error instanceof Error ? error.message : "Microphone access was denied") + } + } + + onCleanup(() => { + active = false + stopRecorder() + stopTracks() + }) + + return { + supported, + state: () => state.status, + levels: () => state.levels, + start() { + if (!supported || active || state.status !== "idle") return + active = true + setState("status", "recording") + void begin() + }, + stop() { + if (!supported || !active) return Promise.resolve("") + active = false + setState("status", "processing") + const result = new Promise((resolve) => { + resolveStop = resolve + }) + if (!starting) stopRecorder() + return result + }, + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null +} diff --git a/packages/app/src/components/settings-v2/general.tsx b/packages/app/src/components/settings-v2/general.tsx index b41da107ed6e..5e09b40347b8 100644 --- a/packages/app/src/components/settings-v2/general.tsx +++ b/packages/app/src/components/settings-v2/general.tsx @@ -9,6 +9,8 @@ import { useLanguage } from "@/context/language" import { usePlatform } from "@/context/platform" import { useUpdaterAction } from "../updater-action" import { useSettings } from "@/context/settings" +import type { TranscriptionLanguage } from "@/context/settings" +import { useProviders } from "@/hooks/use-providers" import { ExternalLink } from "../external-link" import { SettingsListV2 } from "./parts/list" import { SettingsRowV2 } from "./parts/row" @@ -271,6 +273,92 @@ const LanguageSetting = () => { ) } +const VoiceInputSection: Component = () => { + const language = useLanguage() + const settings = useSettings() + const providers = useProviders(() => undefined) + const options = createMemo(() => [ + { id: "local", name: language.t("settings.general.row.voiceProvider.local") }, + ...providers.connected().map((provider) => ({ id: provider.id, name: provider.name })), + ]) + const modelOptions = createMemo(() => { + if (settings.voice.provider() === "local") { + return ["tiny", "base", "small", "medium", "large-v3"].map((id) => ({ id, name: `Whisper ${id}` })) + } + const provider = providers.connected().find((item) => item.id === settings.voice.provider()) + return Object.values(provider?.models ?? {}).map((model) => ({ id: model.id, name: model.name })) + }) + const languageOptions = createMemo(() => [ + { id: "auto" as const, name: language.t("settings.general.row.voiceLanguage.auto") }, + ...language.locales.map((locale) => ({ id: locale as TranscriptionLanguage, name: language.label(locale) })), + ]) + + return ( +
+

{language.t("settings.general.section.voice")}

+ + + settings.voice.setEnabled(enabled)} + /> + + + option.id === settings.voice.provider())} + placement="bottom-end" + gutter={6} + value={(option) => option.id} + label={(option) => option.name} + onSelect={(option) => option && settings.voice.setProvider(option.id)} + /> + + + option.id === settings.voice.model())} + placement="bottom-end" + gutter={6} + value={(option) => option.id} + label={(option) => option.name} + onSelect={(option) => option && settings.voice.setModel(option.id)} + /> + + + option.id === settings.voice.language())} + placement="bottom-end" + gutter={6} + value={(option) => option.id} + label={(option) => option.name} + onSelect={(option) => option && settings.voice.setLanguage(option.id)} + /> + + +
+ ) +} + export const SettingsGeneralV2: Component<{ sessionID?: string }> = (props) => { @@ -552,6 +640,8 @@ export const SettingsGeneralV2: Component<{ + + diff --git a/packages/app/src/context/settings.tsx b/packages/app/src/context/settings.tsx index 1a118b654479..bd7b2d564b4e 100644 --- a/packages/app/src/context/settings.tsx +++ b/packages/app/src/context/settings.tsx @@ -3,6 +3,9 @@ import { batch, createEffect, createMemo, createSignal, onCleanup } from "solid- import { createSimpleContext } from "@opencode-ai/ui/context" import { persisted } from "@/utils/persist" import { usePlatform } from "@/context/platform" +import type { Locale } from "@/context/language" + +export type TranscriptionLanguage = "auto" | Locale export interface NotificationSettings { agent: boolean @@ -52,6 +55,12 @@ export interface Settings { } notifications: NotificationSettings sounds: SoundSettings + voice: { + enabled: boolean + provider: string + model: string + language: TranscriptionLanguage + } } export const monoDefault = "System Mono" @@ -219,6 +228,12 @@ const defaultSettings: Settings = { errorsEnabled: true, errors: "nope-03", }, + voice: { + enabled: false, + provider: "local", + model: "medium", + language: "en", + }, } function withFallback(read: () => T | undefined, fallback: T) { @@ -542,6 +557,24 @@ export const { use: useSettings, provider: SettingsProvider } = createSimpleCont setStore("sounds", "errors", value) }, }, + voice: { + enabled: withFallback(() => store.voice?.enabled, defaultSettings.voice.enabled), + setEnabled(value: boolean) { + setStore("voice", "enabled", value) + }, + provider: withFallback(() => store.voice?.provider, defaultSettings.voice.provider), + setProvider(value: string) { + setStore("voice", "provider", value) + }, + model: withFallback(() => store.voice?.model, defaultSettings.voice.model), + setModel(value: string) { + setStore("voice", "model", value) + }, + language: withFallback(() => store.voice?.language, defaultSettings.voice.language), + setLanguage(value: TranscriptionLanguage) { + setStore("voice", "language", value) + }, + }, } }, }) diff --git a/packages/app/src/entry.tsx b/packages/app/src/entry.tsx index 6e5f63fd4e95..fc1cd44b396d 100644 --- a/packages/app/src/entry.tsx +++ b/packages/app/src/entry.tsx @@ -97,6 +97,7 @@ if (!(root instanceof HTMLElement) && import.meta.env.DEV) { } const getCurrentUrl = () => { + if (import.meta.env.VITE_OPENCODE_SERVER_URL) return import.meta.env.VITE_OPENCODE_SERVER_URL if (location.hostname.includes("opencode.ai")) return "http://localhost:4096" if (import.meta.env.DEV) return `http://${import.meta.env.VITE_OPENCODE_SERVER_HOST ?? "localhost"}:${import.meta.env.VITE_OPENCODE_SERVER_PORT ?? "4096"}` diff --git a/packages/app/src/env.d.ts b/packages/app/src/env.d.ts index 5754df76838d..2cbc551050fe 100644 --- a/packages/app/src/env.d.ts +++ b/packages/app/src/env.d.ts @@ -1,6 +1,7 @@ interface ImportMetaEnv { readonly VITE_OPENCODE_SERVER_HOST: string readonly VITE_OPENCODE_SERVER_PORT: string + readonly VITE_OPENCODE_SERVER_URL?: string readonly VITE_OPENCODE_CHANNEL?: "dev" | "beta" | "prod" readonly VITE_SENTRY_DSN?: string diff --git a/packages/app/vite.config.ts b/packages/app/vite.config.ts index 8df324ddc916..34d1de3c349c 100644 --- a/packages/app/vite.config.ts +++ b/packages/app/vite.config.ts @@ -1,5 +1,6 @@ import { sentryVitePlugin } from "@sentry/vite-plugin" import { defineConfig } from "vite" +import fs from "node:fs" import desktopPlugin from "./vite" const sentry = @@ -25,6 +26,27 @@ export default defineConfig({ host: "0.0.0.0", allowedHosts: true, port: 3000, + https: + process.env.VITE_HTTPS_CERT && process.env.VITE_HTTPS_KEY + ? { + cert: fs.readFileSync(process.env.VITE_HTTPS_CERT), + key: fs.readFileSync(process.env.VITE_HTTPS_KEY), + } + : undefined, + proxy: process.env.VITE_OPENCODE_PROXY_TARGET + ? { + "^/(api|experimental|global|event|session|project|config|file|permission|provider|command|mcp|agent|path|tui|pty|question|find|log|sync|auth|workspace|location)": { + target: process.env.VITE_OPENCODE_PROXY_TARGET, + changeOrigin: true, + ws: true, + }, + "^/(?!$|server/|@vite|@solid-refresh|@id|@fs|src/|node_modules/|assets/|favicon\\.ico|site\\.webmanifest|package\\.json|tsconfig\\.json).*": { + target: process.env.VITE_OPENCODE_PROXY_TARGET, + changeOrigin: true, + ws: true, + }, + } + : undefined, }, build: { target: "esnext", diff --git a/packages/opencode/script/transcribe.py b/packages/opencode/script/transcribe.py new file mode 100644 index 000000000000..7ad2b5a99a9f --- /dev/null +++ b/packages/opencode/script/transcribe.py @@ -0,0 +1,24 @@ +import argparse +import json + +from faster_whisper import WhisperModel + + +parser = argparse.ArgumentParser() +parser.add_argument("--file", required=True) +parser.add_argument("--model", default="medium") +parser.add_argument("--language") +args = parser.parse_args() + +model = WhisperModel(args.model, device="cpu", compute_type="int8") +segments, _ = model.transcribe( + args.file, + language=args.language or "ru", + beam_size=5, + condition_on_previous_text=False, + compression_ratio_threshold=2.4, + log_prob_threshold=-1.0, + no_speech_threshold=0.6, + vad_filter=True, +) +print(json.dumps({"text": " ".join(segment.text.strip() for segment in segments).strip()})) diff --git a/packages/opencode/src/server/routes/instance/httpapi/server.ts b/packages/opencode/src/server/routes/instance/httpapi/server.ts index fb9d2db65621..e7179de5a139 100644 --- a/packages/opencode/src/server/routes/instance/httpapi/server.ts +++ b/packages/opencode/src/server/routes/instance/httpapi/server.ts @@ -106,6 +106,7 @@ import { sessionLocationLayer } from "@opencode-ai/server/middleware/session-loc import { PtyEnvironment } from "@opencode-ai/server/pty-environment" import { schemaErrorLayer as v2SchemaErrorLayer } from "@opencode-ai/server/middleware/schema-error" import { workspaceHandlers } from "./handlers/workspace" +import { transcriptionRoute } from "./transcription" import { instanceContextLayer } from "./middleware/instance-context" import { workspaceRoutingLayer } from "./middleware/workspace-routing" import { disposeMiddleware } from "./lifecycle" @@ -202,6 +203,8 @@ const uiRoute = HttpRouter.use((router) => }), ).pipe(Layer.provide(authOnlyRouterLayer)) +const authenticatedTranscriptionRoute = transcriptionRoute.pipe(Layer.provide(authOnlyRouterLayer)) + type RouteRequirements = | HttpRouter.HttpRouter | HttpRouter.Request<"Error", unknown> @@ -280,6 +283,7 @@ export function createRoutes( instanceRoutes, serverRoutes, docRoute, + authenticatedTranscriptionRoute, uiRoute, ).pipe( Layer.provide([ diff --git a/packages/opencode/src/server/routes/instance/httpapi/transcription.ts b/packages/opencode/src/server/routes/instance/httpapi/transcription.ts new file mode 100644 index 000000000000..db8e41d0dffb --- /dev/null +++ b/packages/opencode/src/server/routes/instance/httpapi/transcription.ts @@ -0,0 +1,132 @@ +import { Effect, Option } from "effect" +import { HttpRouter, HttpServerRequest, HttpServerResponse } from "effect/unstable/http" +import { homedir, tmpdir } from "node:os" +import { unlink } from "node:fs/promises" +import { join } from "node:path" +import { Auth } from "@/auth" +import { Config } from "@/config/config" +import { isRecord } from "@/util/record" + +export const transcriptionRoute = HttpRouter.use((router) => + Effect.gen(function* () { + const config = yield* Config.Service + const auth = yield* Auth.Service + + yield* router.add("POST", "/api/audio/transcribe", (request) => transcribe(request, config, auth)) + }), +) + +function transcribe(request: HttpServerRequest.HttpServerRequest, config: Config.Interface, auth: Auth.Interface) { + return Effect.gen(function* () { + const bytes = yield* request.arrayBuffer.pipe(Effect.catch(() => Effect.succeed(undefined))) + if (!bytes || bytes.byteLength === 0) return HttpServerResponse.jsonUnsafe({ error: "Audio file is required" }, { status: 400 }) + + const url = Option.getOrElse(HttpServerRequest.toURL(request), () => { + const protocol = request.headers["x-forwarded-proto"] === "https" ? "https" : "http" + return new URL(request.url, `${protocol}://${request.headers.host ?? "0.0.0.0"}`) + }) + const providerID = url.searchParams.get("provider") ?? "openai" + const requestedModel = url.searchParams.get("model") + const language = url.searchParams.get("language") + const contentType = request.headers["content-type"] ?? "audio/webm" + const localFallback = () => + Effect.tryPromise(() => transcribeLocally(bytes, contentType, localModel(requestedModel), language)).pipe( + Effect.catch(() => Effect.succeed("")), + ) + + if (providerID === "local") { + const text = yield* localFallback() + if (!text) return HttpServerResponse.jsonUnsafe({ error: "Local Whisper returned no text" }, { status: 422 }) + return HttpServerResponse.jsonUnsafe({ text }) + } + + const loaded = yield* config.getGlobal() + const providers = isRecord(loaded) && isRecord(loaded.provider) ? loaded.provider : {} + const provider = isRecord(providers[providerID]) ? providers[providerID] : undefined + const options = provider && isRecord(provider.options) ? provider.options : {} + const credential = yield* auth.get(providerID) + const apiKey = typeof options.apiKey === "string" ? options.apiKey : credentialKey(credential) + if (!apiKey) { + const text = yield* localFallback() + if (text) return HttpServerResponse.jsonUnsafe({ text }) + return HttpServerResponse.jsonUnsafe({ error: `No API key configured for ${providerID}` }, { status: 400 }) + } + + const baseURL = + typeof options.baseURL === "string" + ? options.baseURL + : typeof provider?.api === "string" + ? provider.api + : providerID === "openai" + ? "https://api.openai.com/v1" + : undefined + if (!baseURL) return HttpServerResponse.jsonUnsafe({ error: `No base URL configured for ${providerID}` }, { status: 400 }) + + const body = new FormData() + body.append("file", new Blob([bytes], { type: contentType }), request.headers["x-filename"] ?? "voice.webm") + body.append( + "model", + requestedModel ?? (typeof options.transcriptionModel === "string" ? options.transcriptionModel : "gpt-4o-mini-transcribe"), + ) + if (language && language !== "auto") body.append("language", language) + + const response = yield* Effect.tryPromise(() => + fetch(`${baseURL.replace(/\/$/, "")}/audio/transcriptions`, { + method: "POST", + headers: { Authorization: `Bearer ${apiKey}` }, + body, + }), + ).pipe(Effect.catch(() => Effect.succeed(undefined))) + if (!response || !response.ok) { + const text = yield* localFallback() + if (text) return HttpServerResponse.jsonUnsafe({ text }) + return HttpServerResponse.jsonUnsafe( + { error: response ? "Transcription provider rejected the audio" : "Transcription provider is unavailable" }, + { status: 502 }, + ) + } + + const result: unknown = yield* Effect.tryPromise(() => response.json()).pipe(Effect.catch(() => Effect.succeed(undefined))) + const text = isRecord(result) && typeof result.text === "string" ? result.text.trim() : "" + if (!text) return HttpServerResponse.jsonUnsafe({ error: "Transcription returned no text" }, { status: 422 }) + return HttpServerResponse.jsonUnsafe({ text }) + }) +} + +function localModel(value: string | null) { + if (value && ["tiny", "base", "small", "medium", "large-v3"].includes(value)) return value + return "medium" +} + +async function transcribeLocally(bytes: ArrayBuffer, contentType: string, model: string, language: string | null) { + const extension = contentType.includes("mp4") ? "mp4" : contentType.includes("ogg") ? "ogg" : "webm" + const file = join(tmpdir(), `opencode-voice-${crypto.randomUUID()}.${extension}`) + await Bun.write(file, bytes) + try { + const python = process.env.OPENCODE_WHISPER_PYTHON ?? join(homedir(), ".cache/opencode-whisper/bin/python") + const script = join(import.meta.dir, "../../../../../script/transcribe.py") + const child = Bun.spawn( + [python, script, "--file", file, "--model", model, ...(language && language !== "auto" ? ["--language", language] : [])], + { + stdout: "pipe", + stderr: "pipe", + }, + ) + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]) + if (exitCode !== 0) throw new Error(stderr || "Local Whisper failed") + const result: unknown = JSON.parse(stdout) + if (!isRecord(result) || typeof result.text !== "string") throw new Error("Invalid local Whisper response") + return result.text.trim() + } finally { + await unlink(file).catch(() => undefined) + } +} + +function credentialKey(value: Auth.Info | undefined) { + if (!value || value.type === "oauth") return value?.access + if (value.type === "api" || value.type === "wellknown") return value.key +} diff --git a/packages/session-ui/src/v2/components/prompt-input/index.tsx b/packages/session-ui/src/v2/components/prompt-input/index.tsx index ff4ff0f1d408..2dae6c446726 100644 --- a/packages/session-ui/src/v2/components/prompt-input/index.tsx +++ b/packages/session-ui/src/v2/components/prompt-input/index.tsx @@ -23,6 +23,7 @@ import type { } from "./types" import type { PromptInputV2Interaction, PromptInputV2SelectControl } from "./interaction" import "./attachments.css" +import "./voice-input.css" export type { PromptInputV2Attachment, @@ -33,6 +34,15 @@ export type { } from "./types" export type PromptInputV2Mode = "normal" | "shell" +export type PromptInputV2VoiceInputState = "idle" | "recording" | "processing" + +export type PromptInputV2VoiceInput = { + supported: boolean + state: Accessor + levels: Accessor + start: () => void + stop: () => Promise +} export type PromptInputV2Props = { controller: PromptInputV2Interaction @@ -44,6 +54,7 @@ export type PromptInputV2Props = { variantControlVisible?: boolean attachKeybind?: string[] attachShortcut?: string + voiceInput?: PromptInputV2VoiceInput } export function PromptInputV2(props: PromptInputV2Props) { @@ -52,6 +63,8 @@ export function PromptInputV2(props: PromptInputV2Props) { const view = props.controller.view let editor: HTMLDivElement | undefined let localInput = false + let voicePointerActive = false + const voiceProcessing = () => props.voiceInput?.state() === "processing" const updateCursor = () => { if (!editor || !window.getSelection()?.isCollapsed) return props.controller.onCursor(promptInputV2Cursor(editor)) @@ -62,6 +75,25 @@ export function PromptInputV2(props: PromptInputV2Props) { "pointer-events": mode() === "normal" ? ("auto" as const) : ("none" as const), transition: "opacity 200ms ease", })) + const stopVoiceInput = () => { + if (!voicePointerActive || !props.voiceInput) return + voicePointerActive = false + void props.voiceInput.stop().then((text) => { + if (!text.trim()) return + const current = props.controller.value() + props.controller.setText(current ? `${current}\n${text.trim()}` : text.trim()) + props.controller.submit() + }) + } + const toggleVoiceInput = () => { + if (!props.voiceInput || props.voiceInput.state() === "processing") return + if (props.voiceInput.state() === "idle") { + voicePointerActive = true + props.voiceInput.start() + return + } + stopVoiceInput() + } createEffect(() => { const parts = props.controller.parts() @@ -109,6 +141,7 @@ export function PromptInputV2(props: PromptInputV2Props) {
+ +
+ + {(level) => } + +
+
{ @@ -154,14 +194,18 @@ export function PromptInputV2(props: PromptInputV2Props) { role="textbox" aria-multiline="true" aria-label={i18n.t("ui.promptInput.label")} - contenteditable={!props.disabled && !props.readOnly} + contenteditable={!props.disabled && !props.readOnly && !voiceProcessing()} autocapitalize={state.mode === "normal" ? "sentences" : "off"} autocorrect={state.mode === "normal" ? "on" : "off"} spellcheck={state.mode === "normal"} // @ts-expect-error autocomplete="off" class="relative z-10 block min-h-[60px] max-h-[180px] w-full overflow-y-auto whitespace-pre-wrap bg-transparent px-4 pt-4 pb-2 text-[13px] font-[440] leading-5 text-v2-text-text-base focus:outline-none empty:before:content-['\200B'] [&_[data-mention=file]]:text-syntax-property [&_[data-mention=agent]]:text-syntax-type [&_[data-mention=reference]]:text-syntax-keyword" - classList={{ "font-mono!": state.mode === "shell", "opacity-50": props.disabled }} + classList={{ + "font-mono!": state.mode === "shell", + "opacity-50": props.disabled || voiceProcessing(), + "pointer-events-none": voiceProcessing(), + }} onInput={(event) => { const cursor = promptInputV2Cursor(event.currentTarget) const prompt = parsePromptInputV2Editor(event.currentTarget) @@ -195,7 +239,7 @@ export function PromptInputV2(props: PromptInputV2Props) {
-
+
+ + {(voice) => ( + + + + )} +
diff --git a/packages/session-ui/src/v2/components/prompt-input/interaction.ts b/packages/session-ui/src/v2/components/prompt-input/interaction.ts index f4a9fa74b7f2..1c0b25870b61 100644 --- a/packages/session-ui/src/v2/components/prompt-input/interaction.ts +++ b/packages/session-ui/src/v2/components/prompt-input/interaction.ts @@ -298,6 +298,10 @@ export function createPromptInputV2Controller(input: { value() { return draft.state.prompt.map((part) => ("content" in part ? part.content : "")).join("") }, + setText(content: string) { + draft.setText(content) + restoreFocus(content.length) + }, parts() { return draft.state.prompt }, diff --git a/packages/session-ui/src/v2/components/prompt-input/voice-input.css b/packages/session-ui/src/v2/components/prompt-input/voice-input.css new file mode 100644 index 000000000000..ee122713a095 --- /dev/null +++ b/packages/session-ui/src/v2/components/prompt-input/voice-input.css @@ -0,0 +1,26 @@ +.voice-input-meter { + background: var(--v2-background-bg-contrast); + box-shadow: var(--v2-elevation-button-contrast); +} + +.voice-input-meter-bar { + width: 2px; + height: 5px; + flex: 1; + min-width: 2px; + max-width: 5px; + border-radius: 999px; + background: var(--v2-state-fg-danger); + opacity: 0.9; + transition: height 80ms linear; +} + +.voice-input-spinner { + animation: voice-input-spin 800ms linear infinite; +} + +@keyframes voice-input-spin { + to { + transform: rotate(360deg); + } +} diff --git a/packages/ui/src/i18n/en.ts b/packages/ui/src/i18n/en.ts index aa0ec9c57351..77da46f7564a 100644 --- a/packages/ui/src/i18n/en.ts +++ b/packages/ui/src/i18n/en.ts @@ -138,6 +138,24 @@ export const dict: Record = { "ui.promptInput.chooseVariant": "Choose model variant", "ui.promptInput.send": "Send", "ui.promptInput.stop": "Stop", + "ui.promptInput.voice.start": "Hold to speak", + "ui.promptInput.voice.recording": "Release to send", + "ui.promptInput.voice.processing": "Processing voice input...", + "ui.promptInput.voice.error": "Voice input is unavailable.", + "ui.promptInput.voice.unsupported": "Voice input requires HTTPS and microphone access.", + "settings.general.section.voice": "Voice input", + "settings.general.row.voiceEnabled.title": "Enable voice input", + "settings.general.row.voiceEnabled.description": "Show the voice input button and enable audio transcription.", + "settings.general.row.voiceProvider.title": "Transcription provider", + "settings.general.row.voiceProvider.description": "Provider used to convert voice messages to text.", + "settings.general.row.voiceProvider.local": "Local Whisper", + "settings.general.row.voiceModel.title": "Transcription model", + "settings.general.row.voiceModel.description": "Model used to convert recorded audio to text.", + "settings.general.row.voiceLanguage.title": "Spoken language", + "settings.general.row.voiceLanguage.description": "Language used by the transcription model.", + "settings.general.row.voiceLanguage.auto": "Automatic", + "settings.general.row.voiceLanguage.russian": "Russian", + "settings.general.row.voiceLanguage.english": "English", "ui.tabs.close": "Close tab", diff --git a/packages/ui/src/v2/components/icon.tsx b/packages/ui/src/v2/components/icon.tsx index b22d3140fe58..2c6cd32e1688 100644 --- a/packages/ui/src/v2/components/icon.tsx +++ b/packages/ui/src/v2/components/icon.tsx @@ -45,6 +45,10 @@ const icons = { viewBox: "0 0 16 16", body: ``, }, + microphone: { + viewBox: "0 0 16 16", + body: ``, + }, plus: { viewBox: "0 0 16 16", body: ``,