diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index c86a1fda1353..de6698b4cd1f 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -299,7 +299,7 @@ jobs: run: | mkdir -p ~/apt-cache && chmod -R a+rw ~/apt-cache sudo apt-get update - sudo apt-get install -y --no-install-recommends -o dir::cache::archives="$HOME/apt-cache" rpm + sudo apt-get install -y --no-install-recommends -o dir::cache::archives="$HOME/apt-cache" build-essential cmake rpm sudo chmod -R a+rw ~/apt-cache - name: Setup git committer @@ -331,6 +331,13 @@ jobs: VITE_SENTRY_DSN: ${{ vars.WEB_SENTRY_DSN }} VITE_SENTRY_ENVIRONMENT: ${{ (github.ref_name == 'beta' && 'beta') || 'production' }} VITE_SENTRY_RELEASE: desktop@${{ needs.version.outputs.version }} + RUST_TARGET: ${{ matrix.settings.target }} + + - name: Build local voice runtime + run: bun ./scripts/build-whisper.ts + working-directory: packages/desktop + env: + RUST_TARGET: ${{ matrix.settings.target }} - name: Package if: needs.version.outputs.release diff --git a/.gitignore b/.gitignore index 006cab8c276c..3e05bf736470 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,4 @@ UPCOMING_CHANGELOG.md logs/ *.bun-build tsconfig.tsbuildinfo +packages/desktop/resources/whisper/ diff --git a/packages/app/package.json b/packages/app/package.json index 729c16e4a2d2..2e850fb34061 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -8,6 +8,7 @@ "./desktop-menu": "./src/desktop-menu.ts", "./i18n/desktop-native": "./src/i18n/desktop-native.ts", "./updater": "./src/updater.ts", + "./voice": "./src/voice.ts", "./wsl/types": "./src/wsl/types.ts", "./vite": "./vite.js", "./index.css": "./src/index.css" diff --git a/packages/app/src/components/prompt-input-v2.tsx b/packages/app/src/components/prompt-input-v2.tsx index 481b7922cf36..a42fc010b6df 100644 --- a/packages/app/src/components/prompt-input-v2.tsx +++ b/packages/app/src/components/prompt-input-v2.tsx @@ -27,6 +27,8 @@ import { useSync } from "@/context/sync" import { createSessionTabs } from "@/pages/session/helpers" import { showToast } from "@/utils/toast" import { PromptInputV2, type PromptInputV2Suggestion } from "@opencode-ai/session-ui/v2/prompt-input" +import { VoiceInputButton } from "@/components/voice-input" +import { withVoiceTranscriptSpacing } from "@/voice" import { createPromptInputV2Controller, createPromptInputV2State, @@ -48,6 +50,7 @@ export function PromptInputV2Composer(props: PromptInputV2ComposerProps) { const dialog = useDialog() const command = useCommand() const language = useLanguage() + let cancelVoice: () => void = () => undefined return (
@@ -58,6 +61,24 @@ export function PromptInputV2Composer(props: PromptInputV2ComposerProps) { variantControlVisible={!props.controller.model.loading} attachKeybind={command.keybindParts("file.attach")} attachShortcut={command.keybind("file.attach")} + beforeSubmit={() => cancelVoice()} + voiceControl={ + + { + const content = withVoiceTranscriptSpacing( + props.controller.value(), + props.controller.cursor(), + transcript, + ) + props.controller.addPart({ type: "text", content, start: 0, end: content.length }) + }} + restoreFocus={props.controller.restoreFocus} + bindCancel={(cancel) => (cancelVoice = cancel)} + /> + + } modelControl={ = (props) => { onSubmit: props.onSubmit, model: props.controls.model.selection, }) + let cancelVoice: () => void = () => undefined + const submit = (event: Event) => { + cancelVoice() + return handleSubmit(event) + } const handleKeyDown = (event: KeyboardEvent) => { if ((event.metaKey || event.ctrlKey) && !event.altKey && !event.shiftKey && event.key.toLowerCase() === "u") { @@ -1390,7 +1397,7 @@ export const PromptInput: Component = (props) => { ) { return } - void handleSubmit(event) + void submit(event) } } @@ -1461,7 +1468,7 @@ export const PromptInput: Component = (props) => { /> = (props) => { onMouseDown={(e) => { const target = e.target if (!(target instanceof HTMLElement)) return - if (target.closest('[data-action="prompt-attach"], [data-action="prompt-submit"]')) { + if ( + target.closest( + '[data-action="prompt-attach"], [data-action="prompt-submit"], [data-action="prompt-voice"]', + ) + ) { return } editorRef?.focus() @@ -1575,6 +1586,20 @@ export const PromptInput: Component = (props) => { />
+ + { + const text = prompt + .current() + .map((part) => ("content" in part ? part.content : "")) + .join("") + const content = withVoiceTranscriptSpacing(text, prompt.cursor(), transcript) + addPart({ type: "text", content, start: 0, end: content.length }) + }} + restoreFocus={() => editorRef?.focus()} + bindCancel={(cancel) => (cancelVoice = cancel)} + /> + { const dialog = useDialog() const params = useParams() const settings = useSettings() + const models = useModels() const updater = useUpdaterAction() @@ -120,6 +125,62 @@ export const SettingsGeneral: Component = () => { permission.disableAutoAccept(params.id, value) } const desktop = createMemo(() => platform.platform === "desktop") + const [localVoice, setLocalVoice] = createStore({ + runtime: false, + transcribing: false, + models: Object.fromEntries( + LOCAL_VOICE_MODELS.map((model) => [model, { size: 0, installed: false }]), + ) as LocalVoiceState["models"], + }) + const audioModels = createMemo(() => models.list().filter((model) => model.capabilities.input.audio)) + const voiceBackendOptions = createMemo(() => [ + { id: "local", value: "local" as const, label: language.t("voice.backend.local") }, + { id: "ai", value: "ai" as const, label: language.t("voice.backend.ai") }, + ]) + const localVoiceModelLabel = (model: LocalVoiceModel) => { + if (model === "tiny") return language.t("voice.model.tiny") + if (model === "base") return language.t("voice.model.base") + if (model === "small") return language.t("voice.model.small") + return language.t("voice.model.turbo") + } + const localVoiceModelOptions = createMemo(() => + LOCAL_VOICE_MODELS.map((model) => ({ id: model, value: model, label: localVoiceModelLabel(model) })), + ) + const aiVoiceModelOptions = createMemo(() => + audioModels().map((model) => ({ + id: `${model.provider.id}/${model.id}`, + label: `${model.provider.name} · ${model.name}`, + model, + })), + ) + const selectedLocalVoice = () => localVoice.models[settings.voice.localModel()] + const voiceModelActionLabel = () => { + const current = selectedLocalVoice() + if (current.download) { + const progress = Math.min(99, Math.floor((current.download.received / current.download.total) * 100)) + return language.t("voice.action.cancelDownloadProgress", { progress }) + } + if (current.installed) return language.t("voice.action.removeModel") + return language.t("voice.action.downloadModel") + } + const runVoiceModelAction = async () => { + const voice = platform.localVoice + if (!voice) return + const model = settings.voice.localModel() + const current = localVoice.models[model] + const task = current.download + ? voice.cancelDownload(model) + : current.installed + ? voice.remove(model) + : voice.download(model) + await task.catch(() => + showToast({ + variant: "error", + title: language.t("voice.error.title"), + description: language.t("voice.error.downloadFailed"), + }), + ) + } const themeOptions = createMemo(() => theme.ids().map((id) => ({ id, name: theme.name(id) }))) @@ -154,6 +215,17 @@ export const SettingsGeneral: Component = () => { void theme.loadThemes() }) + onMount(() => { + const voice = platform.localVoice + if (!voice) return + void voice + .state() + .then(setLocalVoice) + .catch(() => undefined) + const unsubscribe = voice.subscribe(setLocalVoice) + onCleanup(unsubscribe) + }) + const autoOption = { id: "auto", value: "", label: language.t("settings.general.row.shell.autoDefault") } const currentShell = createMemo(() => serverSync().data.config.shell ?? "") @@ -739,6 +811,114 @@ export const SettingsGeneral: Component = () => { ) + const VoiceSection = () => ( + +
+

{language.t("settings.general.section.voice")}

+ + + +
+ +
+
+ + + option.value === settings.voice.localModel())} + value={(option) => option.id} + label={(option) => option.label} + onSelect={(option) => option && settings.voice.setLocalModel(option.value)} + variant="secondary" + size="small" + triggerVariant="settings" + /> + +
+ +
+ + + + 0}> +
+