diff --git a/app/playwright.config.ts b/app/playwright.config.ts index 325f17ab919..cb1d173d050 100644 --- a/app/playwright.config.ts +++ b/app/playwright.config.ts @@ -3,7 +3,8 @@ import { defineConfig } from '@playwright/test'; const baseURL = process.env.PW_BASE_URL || 'http://127.0.0.1:4173'; export default defineConfig({ - testDir: './test/playwright/specs', + testDir: './test', + testMatch: ['**/playwright/specs/**/*.spec.ts', '**/e2e/specs/browser/**/*.spec.ts'], fullyParallel: false, workers: 1, retries: process.env.CI ? 2 : 0, diff --git a/app/src/components/assistant-ui/composer-dictation.test.tsx b/app/src/components/assistant-ui/composer-dictation.test.tsx new file mode 100644 index 00000000000..9e64c63f549 --- /dev/null +++ b/app/src/components/assistant-ui/composer-dictation.test.tsx @@ -0,0 +1,499 @@ +import { ComposerTextBridge } from '@/features/conversations/components/AssistantUiChat'; +import en from '@/lib/i18n/en'; +import fr from '@/lib/i18n/fr'; +import { I18nProvider } from '@/lib/i18n/I18nContext'; +import { + ComposerDictationContext, + type ComposerDictationState, +} from '@/providers/ComposerDictationContext'; +import { + createOpenHumanDictationAdapter, + type OpenHumanDictationAdapter, +} from '@/providers/dictationAdapter'; +import { + type AppendMessage, + AssistantRuntimeProvider, + type ThreadMessageLike, + useExternalStoreRuntime, +} from '@assistant-ui/react'; +import { configureStore } from '@reduxjs/toolkit'; +import { act, fireEvent, render, screen, waitFor } from '@testing-library/react'; +import { useState, useSyncExternalStore } from 'react'; +import { Provider } from 'react-redux'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { ComposerDictationControls, ComposerDictationStatus } from './composer-dictation'; +import { Thread } from './thread'; + +const transcribe = vi.fn(); +const encodeWav = vi.fn(); +vi.mock('@/features/human/voice/sttClient', async importOriginal => ({ + ...(await importOriginal()), + transcribeWithFactory: (...args: unknown[]) => transcribe(...args), +})); +vi.mock('@/features/human/voice/wavEncoder', () => ({ + encodeBlobToWav: (...args: unknown[]) => encodeWav(...args), +})); + +const NO_MESSAGES: ThreadMessageLike[] = []; + +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise(res => { + resolve = res; + }); + return { promise, resolve }; +} + +function Harness({ + adapter, + onNew, + onEscape, + onSwitchToMicCloud, + isRunning = false, +}: { + adapter: OpenHumanDictationAdapter; + onNew: (message: AppendMessage) => Promise; + onEscape?: () => void; + onSwitchToMicCloud?: () => void; + isRunning?: boolean; +}) { + const [draft, setDraft] = useState(''); + const [voiceMode, setVoiceMode] = useState(false); + const snapshot = useSyncExternalStore(adapter.subscribe, adapter.getSnapshot); + const runtime = useExternalStoreRuntime({ + messages: NO_MESSAGES, + isRunning, + convertMessage: message => message, + onNew, + onCancel: async () => {}, + adapters: { dictation: adapter }, + }); + return ( + + + +
{draft}
+ {voiceMode ? ( +
Voice composer
+ ) : ( + { + onSwitchToMicCloud(); + setVoiceMode(true); + }, + } + : {}), + }} + /> + )} +
+
+ ); +} + +describe('dictation on the editable Thread composer', () => { + let adapter: OpenHumanDictationAdapter; + let track: EventTarget & { stop: ReturnType; readyState: string }; + let getUserMedia: ReturnType; + let originalMediaDevices: PropertyDescriptor | undefined; + let stopRecorder: ReturnType void>>; + + beforeEach(() => { + transcribe.mockReset().mockResolvedValue('spoken addition'); + encodeWav.mockReset(); + track = Object.assign(new EventTarget(), { stop: vi.fn(), readyState: 'live' }); + getUserMedia = vi.fn().mockResolvedValue({ getTracks: () => [track] }); + originalMediaDevices = Object.getOwnPropertyDescriptor(navigator, 'mediaDevices'); + Object.defineProperty(navigator, 'mediaDevices', { + configurable: true, + value: { getUserMedia }, + }); + class Recorder { + state: RecordingState = 'inactive'; + mimeType = 'audio/webm'; + ondataavailable: ((event: BlobEvent) => void) | null = null; + onstop: (() => void) | null = null; + onerror: (() => void) | null = null; + constructor() { + stopRecorder = vi.fn(() => { + this.state = 'inactive'; + this.ondataavailable?.({ + data: new Blob(['audio'], { type: this.mimeType }), + } as BlobEvent); + this.onstop?.(); + }); + } + start() { + this.state = 'recording'; + } + stop() { + stopRecorder(); + } + static isTypeSupported() { + return false; + } + } + vi.stubGlobal('MediaRecorder', Recorder); + adapter = createOpenHumanDictationAdapter(); + }); + + afterEach(() => { + adapter.dispose(); + if (originalMediaDevices) { + Object.defineProperty(navigator, 'mediaDevices', originalMediaDevices); + } else { + delete (navigator as { mediaDevices?: MediaDevices }).mediaDevices; + } + vi.unstubAllGlobals(); + }); + + function mount( + options: { onEscape?: () => void; onSwitchToMicCloud?: () => void; isRunning?: boolean } = {} + ) { + const onNew = vi.fn().mockResolvedValue(undefined); + const view = render(); + return { ...view, onNew }; + } + + async function start() { + await typeDraft('Draft'); + fireEvent.click(screen.getByRole('button', { name: 'Dictate' })); + await screen.findByRole('button', { name: 'Finish dictation' }); + expect(screen.getByRole('status')).toHaveTextContent(/recording/i); + } + + async function typeDraft(text: string) { + const input = screen.getByTestId('chat-message-input'); + expect(input).toHaveAttribute('contenteditable', 'true'); + input.textContent = text; + fireEvent.input(input); + await waitFor(() => expect(screen.getByTestId('host-draft')).toHaveTextContent(text)); + return input; + } + + it('keeps the typed draft editable while recording and appends final text only after Finish', async () => { + const result = deferred(); + transcribe.mockReturnValue(result.promise); + const { onNew } = mount(); + await start(); + await typeDraft('Draft edited while recording'); + expect(transcribe).not.toHaveBeenCalled(); + expect(onNew).not.toHaveBeenCalled(); + + fireEvent.click(screen.getByRole('button', { name: 'Finish dictation' })); + await waitFor(() => expect(screen.getByRole('status')).toHaveTextContent(/transcribing/i)); + expect(screen.getByTestId('host-draft')).toHaveTextContent('Draft edited while recording'); + await typeDraft('Draft edited while transcribing'); + await act(async () => { + result.resolve('spoken addition'); + }); + + await waitFor(() => + expect(screen.getByTestId('host-draft')).toHaveTextContent( + 'Draft edited while transcribing spoken addition' + ) + ); + expect(screen.getByTestId('chat-message-input')).toHaveTextContent( + 'Draft edited while transcribing spoken addition' + ); + expect(onNew).not.toHaveBeenCalled(); + expect(track.stop).toHaveBeenCalledOnce(); + expect(screen.queryByRole('status')).toBeNull(); + expect(screen.getByRole('button', { name: 'Dictate' })).toBeEnabled(); + }); + + it('waits for WAV conversion and appends the retry result once to the edited draft', async () => { + const conversion = deferred(); + const retry = deferred(); + transcribe + .mockRejectedValueOnce(new Error('native container rejected')) + .mockReturnValueOnce(retry.promise); + encodeWav.mockReturnValue(conversion.promise); + const { onNew } = mount(); + await start(); + fireEvent.click(screen.getByRole('button', { name: 'Finish dictation' })); + + await waitFor(() => expect(encodeWav).toHaveBeenCalledOnce()); + const native = transcribe.mock.calls[0][0] as Blob; + expect(native.type).toBe('audio/webm'); + expect(encodeWav).toHaveBeenCalledWith(native); + expect(transcribe).toHaveBeenCalledOnce(); + expect(track.stop).toHaveBeenCalledOnce(); + await waitFor(() => expect(screen.getByRole('status')).toHaveTextContent(/transcribing/i)); + await typeDraft('Edited during conversion'); + + const wav = new Blob(['PCM fixture'], { type: 'audio/wav' }); + await act(async () => conversion.resolve(wav)); + await waitFor(() => expect(transcribe).toHaveBeenNthCalledWith(2, wav)); + expect(screen.getByTestId('host-draft')).toHaveTextContent('Edited during conversion'); + await typeDraft('Edited during retry'); + await act(async () => retry.resolve('fallback words')); + + await waitFor(() => + expect(screen.getByTestId('host-draft')).toHaveTextContent( + 'Edited during retry fallback words' + ) + ); + expect(screen.getByTestId('chat-message-input').textContent).toBe( + 'Edited during retry fallback words' + ); + expect(screen.queryByRole('status')).toBeNull(); + expect(screen.getByRole('button', { name: 'Dictate' })).toBeEnabled(); + expect(transcribe).toHaveBeenCalledTimes(2); + expect(encodeWav).toHaveBeenCalledOnce(); + expect(onNew).not.toHaveBeenCalled(); + }); + + it('sends the edited dictated text only after the user explicitly sends it', async () => { + const { onNew } = mount(); + await start(); + fireEvent.click(screen.getByRole('button', { name: 'Finish dictation' })); + await waitFor(() => + expect(screen.getByTestId('host-draft')).toHaveTextContent('Draft spoken addition') + ); + expect(onNew).not.toHaveBeenCalled(); + await typeDraft('Final reviewed message'); + fireEvent.click(screen.getByRole('button', { name: 'Send message' })); + await waitFor(() => expect(onNew).toHaveBeenCalledOnce()); + expect(onNew.mock.calls[0][0].content).toEqual([ + { type: 'text', text: 'Final reviewed message' }, + ]); + await waitFor(() => expect(screen.getByTestId('host-draft')).toBeEmptyDOMElement()); + }); + + it('Discard during STT preserves edits and ignores the late transcript', async () => { + const result = deferred(); + transcribe.mockReturnValue(result.promise); + const { onNew } = mount(); + await start(); + await typeDraft('Keep this draft'); + fireEvent.click(screen.getByRole('button', { name: 'Finish dictation' })); + await waitFor(() => expect(screen.getByRole('status')).toHaveTextContent(/transcribing/i)); + fireEvent.click(screen.getByRole('button', { name: 'Discard dictation' })); + expect(screen.queryByRole('status')).toBeNull(); + expect(screen.getByRole('button', { name: 'Dictate' })).toBeEnabled(); + + await act(async () => { + result.resolve('discarded words'); + }); + expect(screen.getByTestId('host-draft')).toHaveTextContent('Keep this draft'); + expect(screen.getByTestId('chat-message-input')).toHaveTextContent('Keep this draft'); + expect(onNew).not.toHaveBeenCalled(); + }); + + it('Escape cancels active dictation before the host Escape handler runs', async () => { + const onEscape = vi.fn(); + const { onNew } = mount({ onEscape }); + await start(); + const input = await typeDraft('Preserved edit'); + fireEvent.keyDown(input, { key: 'Escape' }); + expect(adapter.getSnapshot().phase).toBe('idle'); + expect(track.stop).toHaveBeenCalledOnce(); + expect(onEscape).not.toHaveBeenCalled(); + expect(transcribe).not.toHaveBeenCalled(); + expect(screen.getByTestId('host-draft')).toHaveTextContent('Preserved edit'); + expect(onNew).not.toHaveBeenCalled(); + // Once dictation is idle, the ordinary host Escape behavior remains usable. + fireEvent.keyDown(input, { key: 'Escape' }); + expect(onEscape).toHaveBeenCalledOnce(); + }); + + it('unmount releases the microphone without transcribing or sending', async () => { + const { unmount, onNew } = mount(); + await start(); + unmount(); + expect(adapter.getSnapshot().phase).toBe('idle'); + expect(track.stop).toHaveBeenCalledOnce(); + expect(transcribe).not.toHaveBeenCalled(); + expect(onNew).not.toHaveBeenCalled(); + }); + + it('unmount during STT discards its late result', async () => { + const result = deferred(); + transcribe.mockReturnValue(result.promise); + const { unmount, onNew } = mount(); + await start(); + fireEvent.click(screen.getByRole('button', { name: 'Finish dictation' })); + unmount(); + await act(async () => { + result.resolve('after unmount'); + }); + expect(adapter.getSnapshot().phase).toBe('idle'); + expect(onNew).not.toHaveBeenCalled(); + expect(track.stop).toHaveBeenCalledOnce(); + }); + + it('Discard while permission is pending closes the session and releases a late grant', async () => { + const permission = deferred(); + getUserMedia.mockReturnValue(permission.promise); + const { onNew } = mount(); + await typeDraft('Draft'); + fireEvent.click(screen.getByRole('button', { name: 'Dictate' })); + await screen.findByText('Starting microphone...'); + fireEvent.click(screen.getByRole('button', { name: 'Discard dictation' })); + await act(async () => { + permission.resolve({ getTracks: () => [track] } as unknown as MediaStream); + }); + expect(track.stop).toHaveBeenCalledOnce(); + expect(transcribe).not.toHaveBeenCalled(); + expect(screen.getByTestId('host-draft')).toHaveTextContent('Draft'); + expect(onNew).not.toHaveBeenCalled(); + }); + + it('renders a translated permission error and keeps the draft', async () => { + getUserMedia.mockRejectedValue(new DOMException('sensitive device detail', 'NotAllowedError')); + mount(); + await typeDraft('Draft'); + fireEvent.click(screen.getByRole('button', { name: 'Dictate' })); + expect(await screen.findByRole('alert')).toHaveTextContent(/microphone permission denied/i); + expect(screen.getByRole('alert')).not.toHaveTextContent('sensitive'); + expect(screen.getByTestId('host-draft')).toHaveTextContent('Draft'); + expect(screen.getByRole('button', { name: 'Dictate' })).toBeEnabled(); + }); + + it('does not start capture during an active agent turn', () => { + mount({ isRunning: true }); + const button = screen.getByRole('button', { name: 'Dictate' }); + expect(button).toBeDisabled(); + fireEvent.click(button); + expect(getUserMedia).not.toHaveBeenCalled(); + }); + + it.each(['recording', 'transcribing'] as const)( + 'switching to Voice mode cancels %s before replacing the text composer', + async phase => { + const result = deferred(); + transcribe.mockReturnValue(result.promise); + const onSwitchToMicCloud = vi.fn(() => adapter.getSnapshot().phase); + const { onNew } = mount({ onSwitchToMicCloud }); + await start(); + await typeDraft('Preserved voice mode draft'); + if (phase === 'transcribing') { + fireEvent.click(screen.getByRole('button', { name: 'Finish dictation' })); + await waitFor(() => expect(screen.getByRole('status')).toHaveTextContent(/transcribing/i)); + } + + fireEvent.click(screen.getByRole('button', { name: 'Voice mode' })); + expect(onSwitchToMicCloud).toHaveBeenCalledOnce(); + expect(onSwitchToMicCloud.mock.results[0].value).toBe('idle'); + expect(track.stop).toHaveBeenCalledOnce(); + expect(screen.getByText('Voice composer')).toBeInTheDocument(); + expect(screen.queryByRole('button', { name: 'Discard dictation' })).toBeNull(); + await act(async () => { + result.resolve('text from the old mode'); + }); + expect(screen.getByTestId('host-draft')).toHaveTextContent('Preserved voice mode draft'); + expect(onNew).not.toHaveBeenCalled(); + if (phase === 'recording') expect(transcribe).not.toHaveBeenCalled(); + } + ); +}); + +const ERROR_CASES = [ + ['stt-unavailable', 'composer.dictationUnavailable'], + ['voice-status-failed', 'composer.dictationStatusFailed'], + ['microphone-unavailable', 'mic.unavailable'], + ['permission-denied', 'mic.permissionDenied'], + ['device-unavailable', 'mic.deviceUnavailable'], + ['device-in-use', 'mic.deviceInUse'], + ['recorder-failed', 'mic.failedToStartRecorder'], + ['no-audio', 'mic.noAudioCaptured'], + ['no-speech', 'mic.noSpeechDetected'], + ['transcription-failed', 'composer.dictationFailed'], + ['timed-out', 'composer.dictationTimedOut'], + ['voice-unavailable', 'mic.voiceNotCompiled'], +] as const; + +function StatusHarness({ + status = 'idle', + error = null, +}: { + status?: ComposerDictationState['status']; + error?: ComposerDictationState['error']; +}) { + return ( + {} }}> + + + ); +} + +function ControlsHarness({ dictation }: { dictation: ComposerDictationState | null }) { + const runtime = useExternalStoreRuntime({ + messages: NO_MESSAGES, + convertMessage: message => message, + onNew: async () => {}, + adapters: { dictation: dictation?.adapter }, + }); + return ( + + + + + + + ); +} + +describe('localized dictation status and capability visibility', () => { + describe.each([ + ['English', en], + ['French', fr], + ] as const)('%s errors', (language, translations) => { + it.each(ERROR_CASES)('renders the localized alert for %s', (error, key) => { + const store = configureStore({ + reducer: { locale: () => ({ current: language === 'French' ? 'fr' : 'en' }) }, + }); + render( + + + + + + ); + expect(screen.getByRole('alert')).toHaveTextContent(translations[key]); + expect(screen.queryByRole('status')).toBeNull(); + }); + }); + + it.each([ + ['starting', 'composer.dictationStarting'], + ['recording', 'composer.dictationRecording'], + ['transcribing', 'mic.transcribing'], + ] as const)('announces the %s phase', (status, key) => { + render(); + expect(screen.getByRole('status')).toHaveTextContent(en[key]); + expect(screen.queryByRole('alert')).toBeNull(); + }); + + it.each(['idle', 'checking', 'unavailable'] as const)( + 'renders no active status in the %s state', + status => { + const { container } = render(); + expect(container).toBeEmptyDOMElement(); + } + ); + + it('renders no controls or status without a dictation context', () => { + const { container } = render(); + expect(container).toBeEmptyDOMElement(); + }); + + it('renders no dictation controls without an available adapter', () => { + const { container } = render( + {} }} + /> + ); + expect(container).toBeEmptyDOMElement(); + expect(screen.queryByRole('button', { name: 'Dictate' })).toBeNull(); + }); +}); diff --git a/app/src/components/assistant-ui/composer-dictation.tsx b/app/src/components/assistant-ui/composer-dictation.tsx new file mode 100644 index 00000000000..80ccc4df664 --- /dev/null +++ b/app/src/components/assistant-ui/composer-dictation.tsx @@ -0,0 +1,142 @@ +import { useT } from '@/lib/i18n/I18nContext'; +import { useComposerDictationState } from '@/providers/ComposerDictationContext'; +import { ComposerPrimitive, useAuiState } from '@assistant-ui/react'; +import { LoaderCircleIcon, MicIcon, SquareIcon, XIcon } from 'lucide-react'; + +import { TooltipIconButton } from './tooltip-icon-button'; + +/** Offer capture only when supported, with explicit Finish and Discard actions. */ +export function ComposerDictationControls() { + const { t } = useT(); + const dictation = useComposerDictationState(); + const isRunning = useAuiState(state => state.thread.isRunning); + if (!dictation?.adapter) return null; + + const active = ['starting', 'recording', 'transcribing'].includes(dictation.status); + const label = t('composer.dictate', 'Dictate'); + const finishLabel = t('composer.finishDictation', 'Finish dictation'); + const discardLabel = t('composer.discardDictation', 'Discard dictation'); + + return ( + <> + {active ? ( + <> + {dictation.status === 'recording' && ( + + + + + + )} + {dictation.status !== 'recording' && ( + + )} + + + + + ) : ( + + + + + + )} + + ); +} + +/** Announce dictation phases and translate every safe error code for the user. */ +export function ComposerDictationStatus() { + const { t } = useT(); + const dictation = useComposerDictationState(); + if (!dictation) return null; + + let errorText: string | null = null; + switch (dictation.error) { + case 'stt-unavailable': + errorText = t( + 'composer.dictationUnavailable', + 'Dictation is unavailable. Check your speech provider in Settings > Voice.' + ); + break; + case 'voice-status-failed': + errorText = t( + 'composer.dictationStatusFailed', + 'Could not check dictation availability. Return to this window to try again.' + ); + break; + case 'microphone-unavailable': + errorText = t('mic.unavailable', 'Microphone is not available'); + break; + case 'permission-denied': + errorText = t('mic.permissionDenied', 'Microphone permission denied'); + break; + case 'device-unavailable': + errorText = t( + 'mic.deviceUnavailable', + 'Selected microphone is unavailable. Try a different device.' + ); + break; + case 'device-in-use': + errorText = t('mic.deviceInUse', 'Microphone is in use by another application.'); + break; + case 'recorder-failed': + errorText = t('mic.failedToStartRecorder', 'Failed to start recorder'); + break; + case 'no-audio': + errorText = t('mic.noAudioCaptured', 'No audio captured'); + break; + case 'no-speech': + errorText = t('mic.noSpeechDetected', 'No speech detected'); + break; + case 'transcription-failed': + errorText = t('composer.dictationFailed', 'Transcription failed. Please try again.'); + break; + case 'timed-out': + errorText = t('composer.dictationTimedOut', 'Dictation timed out. Please try again.'); + break; + case 'voice-unavailable': + errorText = t( + 'mic.voiceNotCompiled', + 'Voice transcription is not included in this version of the app. Update OpenHuman to enable it.' + ); + break; + } + if (errorText) { + return ( +

+ {errorText} +

+ ); + } + + const statusText = + dictation.status === 'starting' + ? t('composer.dictationStarting', 'Starting microphone...') + : dictation.status === 'recording' + ? t('composer.dictationRecording', 'Recording. Finish to add text, or discard.') + : dictation.status === 'transcribing' + ? t('mic.transcribing', 'Transcribing...') + : null; + + return statusText ? ( +

+ {statusText} +

+ ) : null; +} diff --git a/app/src/components/assistant-ui/thread.tsx b/app/src/components/assistant-ui/thread.tsx index 9559e695ffd..195eb775d5d 100644 --- a/app/src/components/assistant-ui/thread.tsx +++ b/app/src/components/assistant-ui/thread.tsx @@ -6,6 +6,10 @@ import { ComposerAttachments, UserMessageAttachments, } from '@/components/assistant-ui/attachment'; +import { + ComposerDictationControls, + ComposerDictationStatus, +} from '@/components/assistant-ui/composer-dictation'; import { ComposerTriggerPopover } from '@/components/assistant-ui/composer-trigger-popover'; import { DirectiveText } from '@/components/assistant-ui/directive-text'; import { EditMessage } from '@/components/assistant-ui/elements/edit-message'; @@ -32,6 +36,7 @@ import { } from '@/features/conversations/components/aui/auiThreadState'; import { useT } from '@/lib/i18n/I18nContext'; import { useAuiThreadId } from '@/providers/AssistantUiRuntimeProvider'; +import { useComposerDictationState } from '@/providers/ComposerDictationContext'; import { CHAT_ERROR_METADATA_KEY } from '@/store/threadSlice'; import { useActionBarReload, useMessageError } from '@assistant-ui/core/react'; import { @@ -58,12 +63,12 @@ import debugFactory from 'debug'; import { ArrowDownIcon, ArrowUpIcon, + AudioLinesIcon, CheckIcon, ChevronLeftIcon, ChevronRightIcon, CopyIcon, DownloadIcon, - MicIcon, MoreHorizontalIcon, PencilIcon, RefreshCwIcon, @@ -936,6 +941,9 @@ const Composer: FC<{ isDraggingFiles: boolean; }> = ({ model, onModelChange, onEscape, isDraggingFiles }) => { const { t } = useT(); + const dictation = useComposerDictationState(); + const cancelDictation = dictation?.cancel; + useEffect(() => cancelDictation, [cancelDictation]); const messageInputLabel = t('assistantUi.thread.messageInputLabel', 'Message input'); const aui = useAui(); const commands = useContext(SlashCommandsContext); @@ -1011,7 +1019,19 @@ const Composer: FC<{ + data-walkthrough="chat-agent-panel" + onKeyDownCapture={event => { + if ( + event.key === 'Escape' && + !isComposingTextRef.current && + dictation && + ['starting', 'recording', 'transcribing'].includes(dictation.status) + ) { + event.preventDefault(); + event.stopPropagation(); + dictation.cancel(); + } + }}> {ComposerHeader ? : null} {/* * Neutered whenever the host owns file ingest: every handler in the @@ -1155,6 +1175,7 @@ const Composer: FC<{ className="aui-composer-input caret-primary [&_.aui-lexical-placeholder]:text-muted-foreground/60 relative max-h-48 min-h-10 w-full resize-none bg-transparent px-2.5 py-1 text-base leading-6 outline-none [&_.aui-lexical-input]:min-h-lh [&_.aui-lexical-input]:outline-none [&_.aui-lexical-placeholder]:pointer-events-none [&_.aui-lexical-placeholder]:absolute [&_.aui-lexical-placeholder]:top-0 [&_.aui-lexical-placeholder]:right-0 [&_.aui-lexical-placeholder]:left-0 [&_.aui-lexical-placeholder]:truncate [&_.aui-lexical-placeholder]:px-2.5 [&_.aui-lexical-placeholder]:py-1" aria-label={messageInputLabel} /> + @@ -1183,6 +1204,7 @@ const ComposerAction: FC<{ const { t } = useT(); const aui = useAui(); const composerText = useAuiState(state => state.composer.text); + const dictation = useComposerDictationState(); const { ComposerAddAttachment: HostComposerAddAttachment, hasComposerAttachments, @@ -1217,50 +1239,14 @@ const ComposerAction: FC<{ className="aui-composer-voice-mode text-muted-foreground hover:text-foreground size-7 rounded-full" aria-label={t('composer.voiceMode', 'Voice mode')} disabled={isRunning} - onClick={onSwitchToMicCloud}> - + onClick={() => { + dictation?.cancel(); + onSwitchToMicCloud(); + }}> + )} - {/* - Permanently false, deliberately: `useOpenHumanExternalStore` supplies - no `adapters.dictation`, and the reasoning for keeping it that way - lives there. Short version — Web Speech's constructor exists in our - WKWebView but `start()` never succeeds, and with the speech usage - strings present it hangs silently rather than erroring, which would - strand the composer in `dictation != null`. Working dictation already - ships as the `mic-cloud` composer, whose "Voice mode" button is the - one directly above this block. - */} - s.thread.capabilities.dictation}> - s.composer.dictation == null}> - - - - - - - s.composer.dictation != null}> - - - - - - - + !s.thread.isRunning}> {showIdleAction ? ( @@ -1289,6 +1275,7 @@ const ComposerAction: FC<{ data-testid="send-message-button" aria-label={t('chat.send', 'Send message')} onClick={() => { + dictation?.cancel(); onComposerAttachmentSend?.(); aui.composer.setText(''); }}> diff --git a/app/src/components/flows/WorkflowCopilotPanel.assistantUiRuntime.test.tsx b/app/src/components/flows/WorkflowCopilotPanel.assistantUiRuntime.test.tsx index f1d6e0eae28..2083cd971b8 100644 --- a/app/src/components/flows/WorkflowCopilotPanel.assistantUiRuntime.test.tsx +++ b/app/src/components/flows/WorkflowCopilotPanel.assistantUiRuntime.test.tsx @@ -33,7 +33,10 @@ import type { ThreadMessage } from '../../types/thread'; import WorkflowCopilotPanel from './WorkflowCopilotPanel'; vi.mock('../../lib/i18n/I18nContext', () => ({ useT: () => ({ t: (key: string) => key }) })); -vi.mock('../../services/coreRpcClient', () => ({ callCoreRpc: vi.fn() })); +vi.mock('../../services/coreRpcClient', async importOriginal => ({ + ...(await importOriginal()), + callCoreRpc: vi.fn(), +})); // Stand-in for the transcript, written the way the real `Thread` reads: from // the runtime in context rather than from a thread id. Whatever runtime the diff --git a/app/src/components/flows/WorkflowCopilotPanel.test.tsx b/app/src/components/flows/WorkflowCopilotPanel.test.tsx index abb471d156e..d6f64e805c6 100644 --- a/app/src/components/flows/WorkflowCopilotPanel.test.tsx +++ b/app/src/components/flows/WorkflowCopilotPanel.test.tsx @@ -52,7 +52,10 @@ vi.mock('../../store/hooks', () => ({ // Neither card calls this on mount (only on Approve/Deny/Connect click), but // stub it defensively so a real network call can never sneak into a render // test. -vi.mock('../../services/coreRpcClient', () => ({ callCoreRpc: vi.fn() })); +vi.mock('../../services/coreRpcClient', async importOriginal => ({ + ...(await importOriginal()), + callCoreRpc: vi.fn(), +})); const hookState = vi.hoisted(() => ({ threadId: 'builder-1' as string | null, diff --git a/app/src/components/flows/WorkflowCopilotPanel.thread.test.tsx b/app/src/components/flows/WorkflowCopilotPanel.thread.test.tsx index 7ff9ccedd8c..b363a43dddb 100644 --- a/app/src/components/flows/WorkflowCopilotPanel.thread.test.tsx +++ b/app/src/components/flows/WorkflowCopilotPanel.thread.test.tsx @@ -20,7 +20,10 @@ import type { ThreadMessage } from '../../types/thread'; import WorkflowCopilotPanel from './WorkflowCopilotPanel'; vi.mock('../../lib/i18n/I18nContext', () => ({ useT: () => ({ t: (key: string) => key }) })); -vi.mock('../../services/coreRpcClient', () => ({ callCoreRpc: vi.fn() })); +vi.mock('../../services/coreRpcClient', async importOriginal => ({ + ...(await importOriginal()), + callCoreRpc: vi.fn(), +})); const hookState = vi.hoisted(() => ({ threadId: 'builder-1' as string | null, diff --git a/app/src/features/conversations/components/ChatToolParts.approval.test.tsx b/app/src/features/conversations/components/ChatToolParts.approval.test.tsx index 1f4935b2a43..42e944093af 100644 --- a/app/src/features/conversations/components/ChatToolParts.approval.test.tsx +++ b/app/src/features/conversations/components/ChatToolParts.approval.test.tsx @@ -44,7 +44,10 @@ vi.mock('../../../services/api/threadApi', () => ({ }, })); -vi.mock('../../../services/coreRpcClient', () => ({ callCoreRpc: vi.fn() })); +vi.mock('../../../services/coreRpcClient', async importOriginal => ({ + ...(await importOriginal()), + callCoreRpc: vi.fn(), +})); const THREAD_ID = 't-1'; const REQUEST_ID = 'appr-1'; diff --git a/app/src/lib/i18n/ar.ts b/app/src/lib/i18n/ar.ts index b0b3ba0300c..184e5d17e59 100644 --- a/app/src/lib/i18n/ar.ts +++ b/app/src/lib/i18n/ar.ts @@ -3,6 +3,19 @@ import type { TranslationMap } from './types'; // Arabic (العربية) translations. Keys mirror en.ts; missing/ // English-identical values fall back to English via I18nContext.resolveEn(). const messages: TranslationMap = { + // Editable composer dictation. + 'composer.dictationUnavailable': + 'الإملاء غير متاح. تحقّق من مزوّد تحويل الكلام إلى نص في الإعدادات > الصوت.', + 'composer.dictationStatusFailed': + 'تعذّر التحقّق من توفّر الإملاء. عُد إلى هذه النافذة للمحاولة مجددًا.', + 'composer.dictate': 'إملاء', + 'composer.finishDictation': 'إنهاء الإملاء', + 'composer.discardDictation': 'إلغاء الإملاء', + 'composer.dictationStarting': 'جارٍ تشغيل الميكروفون...', + 'composer.dictationRecording': 'جارٍ التسجيل. أنهِ التسجيل لإضافة النص أو ألغِ الإملاء.', + 'composer.dictationFailed': 'فشل تحويل الصوت إلى نص. حاول مرة أخرى.', + 'composer.dictationTimedOut': 'انتهت مهلة الإملاء. حاول مرة أخرى.', + // Share cards (#5006). 'share.button': 'مشاركة', 'share.modalTitle': 'شارك هذه اللحظة', diff --git a/app/src/lib/i18n/bn.ts b/app/src/lib/i18n/bn.ts index 54822b79051..efbf71713b3 100644 --- a/app/src/lib/i18n/bn.ts +++ b/app/src/lib/i18n/bn.ts @@ -3,6 +3,19 @@ import type { TranslationMap } from './types'; // Bengali (বাংলা) translations. Keys mirror en.ts; missing/ // English-identical values fall back to English via I18nContext.resolveEn(). const messages: TranslationMap = { + // Editable composer dictation. + 'composer.dictationUnavailable': + 'শ্রুতলিখন উপলব্ধ নেই। সেটিংস > ভয়েস-এ আপনার স্পিচ প্রদানকারী পরীক্ষা করুন।', + 'composer.dictationStatusFailed': + 'শ্রুতলিখনের উপলব্ধতা পরীক্ষা করা যায়নি। আবার চেষ্টা করতে এই উইন্ডোতে ফিরে আসুন।', + 'composer.dictate': 'ডিক্টেশন', + 'composer.finishDictation': 'ডিক্টেশন শেষ করুন', + 'composer.discardDictation': 'ডিক্টেশন বাতিল করুন', + 'composer.dictationStarting': 'মাইক্রোফোন চালু হচ্ছে...', + 'composer.dictationRecording': 'রেকর্ড হচ্ছে। লেখা যোগ করতে শেষ করুন, অথবা বাতিল করুন।', + 'composer.dictationFailed': 'অডিও থেকে লেখায় রূপান্তর ব্যর্থ হয়েছে। আবার চেষ্টা করুন।', + 'composer.dictationTimedOut': 'ডিক্টেশনের সময়সীমা শেষ হয়েছে। আবার চেষ্টা করুন।', + // Share cards (#5006). 'share.button': 'শেয়ার', 'share.modalTitle': 'এই মুহূর্তটি শেয়ার করুন', diff --git a/app/src/lib/i18n/de.ts b/app/src/lib/i18n/de.ts index 33c6a4e7932..301948a9c0a 100644 --- a/app/src/lib/i18n/de.ts +++ b/app/src/lib/i18n/de.ts @@ -3,6 +3,21 @@ import type { TranslationMap } from './types'; // German (Deutsch) translations. Keys mirror en.ts; missing/ // English-identical values fall back to English via I18nContext.resolveEn(). const messages: TranslationMap = { + // Editable composer dictation. + 'composer.dictationUnavailable': + 'Diktieren ist nicht verfügbar. Prüfe deinen Sprachanbieter unter Einstellungen > Sprache.', + 'composer.dictationStatusFailed': + 'Die Verfügbarkeit der Diktierfunktion konnte nicht geprüft werden. Kehre zu diesem Fenster zurück, um es erneut zu versuchen.', + 'composer.dictate': 'Diktieren', + 'composer.finishDictation': 'Diktat abschließen', + 'composer.discardDictation': 'Diktat verwerfen', + 'composer.dictationStarting': 'Mikrofon wird gestartet...', + 'composer.dictationRecording': + 'Aufnahme läuft. Zum Einfügen des Textes abschließen oder verwerfen.', + 'composer.dictationFailed': 'Die Transkription ist fehlgeschlagen. Bitte erneut versuchen.', + 'composer.dictationTimedOut': + 'Das Zeitlimit für das Diktat wurde erreicht. Bitte erneut versuchen.', + // Share cards (#5006). 'share.button': 'Teilen', 'share.modalTitle': 'Diesen Moment teilen', diff --git a/app/src/lib/i18n/en.ts b/app/src/lib/i18n/en.ts index d698c3fd194..0ca29047e83 100644 --- a/app/src/lib/i18n/en.ts +++ b/app/src/lib/i18n/en.ts @@ -1,6 +1,19 @@ import type { TranslationMap } from './types'; const en: TranslationMap = { + // Editable composer dictation. + 'composer.dictationUnavailable': + 'Dictation is unavailable. Check your speech provider in Settings > Voice.', + 'composer.dictationStatusFailed': + 'Could not check dictation availability. Return to this window to try again.', + 'composer.dictate': 'Dictate', + 'composer.finishDictation': 'Finish dictation', + 'composer.discardDictation': 'Discard dictation', + 'composer.dictationStarting': 'Starting microphone...', + 'composer.dictationRecording': 'Recording. Finish to add text, or discard.', + 'composer.dictationFailed': 'Transcription failed. Please try again.', + 'composer.dictationTimedOut': 'Dictation timed out. Please try again.', + // Navigation 'nav.home': 'Home', 'nav.chat': 'Chat', diff --git a/app/src/lib/i18n/es.ts b/app/src/lib/i18n/es.ts index 379604fc9bc..8b2dc6bc7ae 100644 --- a/app/src/lib/i18n/es.ts +++ b/app/src/lib/i18n/es.ts @@ -3,6 +3,19 @@ import type { TranslationMap } from './types'; // Spanish (Español) translations. Keys mirror en.ts; missing/ // English-identical values fall back to English via I18nContext.resolveEn(). const messages: TranslationMap = { + // Editable composer dictation. + 'composer.dictationUnavailable': + 'El dictado no está disponible. Revisa tu proveedor de voz en Ajustes > Voz.', + 'composer.dictationStatusFailed': + 'No se pudo comprobar la disponibilidad del dictado. Vuelve a esta ventana para intentarlo de nuevo.', + 'composer.dictate': 'Dictar', + 'composer.finishDictation': 'Terminar dictado', + 'composer.discardDictation': 'Descartar dictado', + 'composer.dictationStarting': 'Iniciando micrófono...', + 'composer.dictationRecording': 'Grabando. Termina para añadir el texto o descarta la grabación.', + 'composer.dictationFailed': 'La transcripción falló. Inténtalo de nuevo.', + 'composer.dictationTimedOut': 'Se agotó el tiempo del dictado. Inténtalo de nuevo.', + // Share cards (#5006). 'share.button': 'Compartir', 'share.modalTitle': 'Comparte este momento', diff --git a/app/src/lib/i18n/fr.ts b/app/src/lib/i18n/fr.ts index d22601858ea..675cd8e1812 100644 --- a/app/src/lib/i18n/fr.ts +++ b/app/src/lib/i18n/fr.ts @@ -3,6 +3,20 @@ import type { TranslationMap } from './types'; // French (Français) translations. Keys mirror en.ts; missing/ // English-identical values fall back to English via I18nContext.resolveEn(). const messages: TranslationMap = { + // Editable composer dictation. + 'composer.dictationUnavailable': + 'La dictée est indisponible. Vérifiez votre fournisseur de reconnaissance vocale dans Paramètres > Voix.', + 'composer.dictationStatusFailed': + 'Impossible de vérifier la disponibilité de la dictée. Revenez dans cette fenêtre pour réessayer.', + 'composer.dictate': 'Dicter', + 'composer.finishDictation': 'Terminer la dictée', + 'composer.discardDictation': 'Abandonner la dictée', + 'composer.dictationStarting': 'Démarrage du microphone...', + 'composer.dictationRecording': + 'Enregistrement en cours. Terminez pour ajouter le texte ou abandonnez.', + 'composer.dictationFailed': 'La transcription a échoué. Veuillez réessayer.', + 'composer.dictationTimedOut': 'Le délai de la dictée a expiré. Veuillez réessayer.', + // Share cards (#5006). 'share.button': 'Partager', 'share.modalTitle': 'Partagez ce moment', diff --git a/app/src/lib/i18n/hi.ts b/app/src/lib/i18n/hi.ts index ee80012e549..754af4afc48 100644 --- a/app/src/lib/i18n/hi.ts +++ b/app/src/lib/i18n/hi.ts @@ -3,6 +3,20 @@ import type { TranslationMap } from './types'; // Hindi (हिन्दी) translations. Keys mirror en.ts; missing/ // English-identical values fall back to English via I18nContext.resolveEn(). const messages: TranslationMap = { + // Editable composer dictation. + 'composer.dictationUnavailable': + 'डिक्टेशन उपलब्ध नहीं है। सेटिंग्स > वॉइस में अपने स्पीच प्रदाता की जाँच करें।', + 'composer.dictationStatusFailed': + 'डिक्टेशन की उपलब्धता जाँची नहीं जा सकी। फिर से कोशिश करने के लिए इस विंडो पर वापस आएँ।', + 'composer.dictate': 'बोलकर लिखें', + 'composer.finishDictation': 'डिक्टेशन पूरा करें', + 'composer.discardDictation': 'डिक्टेशन रद्द करें', + 'composer.dictationStarting': 'माइक्रोफ़ोन चालू हो रहा है...', + 'composer.dictationRecording': + 'रिकॉर्डिंग जारी है। टेक्स्ट जोड़ने के लिए पूरा करें या रद्द करें।', + 'composer.dictationFailed': 'आवाज़ से टेक्स्ट बनाने में त्रुटि हुई। फिर से कोशिश करें।', + 'composer.dictationTimedOut': 'डिक्टेशन का समय समाप्त हो गया। फिर से कोशिश करें।', + // Share cards (#5006). 'share.button': 'साझा करें', 'share.modalTitle': 'इस पल को साझा करें', diff --git a/app/src/lib/i18n/id.ts b/app/src/lib/i18n/id.ts index ab235c83802..dad81c83d8d 100644 --- a/app/src/lib/i18n/id.ts +++ b/app/src/lib/i18n/id.ts @@ -3,6 +3,19 @@ import type { TranslationMap } from './types'; // Indonesian (Bahasa Indonesia) translations. Keys mirror en.ts; missing/ // English-identical values fall back to English via I18nContext.resolveEn(). const messages: TranslationMap = { + // Editable composer dictation. + 'composer.dictationUnavailable': + 'Dikte tidak tersedia. Periksa penyedia pengenalan suara Anda di Pengaturan > Suara.', + 'composer.dictationStatusFailed': + 'Tidak dapat memeriksa ketersediaan dikte. Kembali ke jendela ini untuk mencoba lagi.', + 'composer.dictate': 'Dikte', + 'composer.finishDictation': 'Selesaikan dikte', + 'composer.discardDictation': 'Buang dikte', + 'composer.dictationStarting': 'Menyalakan mikrofon...', + 'composer.dictationRecording': 'Merekam. Selesaikan untuk menambahkan teks, atau buang rekaman.', + 'composer.dictationFailed': 'Transkripsi gagal. Silakan coba lagi.', + 'composer.dictationTimedOut': 'Waktu dikte habis. Silakan coba lagi.', + // Share cards (#5006). 'share.button': 'Bagikan', 'share.modalTitle': 'Bagikan momen ini', diff --git a/app/src/lib/i18n/it.ts b/app/src/lib/i18n/it.ts index d6fbf94da88..a1ab2b1d6bd 100644 --- a/app/src/lib/i18n/it.ts +++ b/app/src/lib/i18n/it.ts @@ -3,6 +3,20 @@ import type { TranslationMap } from './types'; // Italian (Italiano) translations. Keys mirror en.ts; missing/ // English-identical values fall back to English via I18nContext.resolveEn(). const messages: TranslationMap = { + // Editable composer dictation. + 'composer.dictationUnavailable': + 'La dettatura non è disponibile. Controlla il fornitore di riconoscimento vocale in Impostazioni > Voce.', + 'composer.dictationStatusFailed': + 'Impossibile verificare la disponibilità della dettatura. Torna a questa finestra per riprovare.', + 'composer.dictate': 'Detta', + 'composer.finishDictation': 'Termina dettatura', + 'composer.discardDictation': 'Scarta dettatura', + 'composer.dictationStarting': 'Avvio del microfono...', + 'composer.dictationRecording': + 'Registrazione in corso. Termina per aggiungere il testo oppure scarta.', + 'composer.dictationFailed': 'Trascrizione fallita. Riprova.', + 'composer.dictationTimedOut': 'Il tempo per la dettatura è scaduto. Riprova.', + // Share cards (#5006). 'share.button': 'Condividi', 'share.modalTitle': 'Condividi questo momento', diff --git a/app/src/lib/i18n/ko.ts b/app/src/lib/i18n/ko.ts index f82d1560ada..e7987eac6bb 100644 --- a/app/src/lib/i18n/ko.ts +++ b/app/src/lib/i18n/ko.ts @@ -3,6 +3,20 @@ import type { TranslationMap } from './types'; // Korean (한국어) translations. Keys mirror en.ts; missing/ // English-identical values fall back to English via I18nContext.resolveEn(). const messages: TranslationMap = { + // Editable composer dictation. + 'composer.dictationUnavailable': + '받아쓰기를 사용할 수 없습니다. 설정 > 음성에서 음성 인식 제공업체를 확인하세요.', + 'composer.dictationStatusFailed': + '받아쓰기 사용 가능 여부를 확인하지 못했습니다. 다시 시도하려면 이 창으로 돌아오세요.', + 'composer.dictate': '받아쓰기', + 'composer.finishDictation': '받아쓰기 완료', + 'composer.discardDictation': '받아쓰기 취소', + 'composer.dictationStarting': '마이크 시작 중...', + 'composer.dictationRecording': + '녹음 중입니다. 완료하면 텍스트가 추가되며, 취소하면 녹음이 삭제됩니다.', + 'composer.dictationFailed': '음성 변환에 실패했습니다. 다시 시도하세요.', + 'composer.dictationTimedOut': '받아쓰기 시간이 초과되었습니다. 다시 시도하세요.', + // Share cards (#5006). 'share.button': '공유', 'share.modalTitle': '이 순간을 공유하세요', diff --git a/app/src/lib/i18n/pl.ts b/app/src/lib/i18n/pl.ts index 3e9ea086c82..66c46956c75 100644 --- a/app/src/lib/i18n/pl.ts +++ b/app/src/lib/i18n/pl.ts @@ -3,6 +3,19 @@ import type { TranslationMap } from './types'; // Polish (Polski) translations. Keys mirror en.ts; missing/ // English-identical values fall back to English via I18nContext.resolveEn(). const messages: TranslationMap = { + // Editable composer dictation. + 'composer.dictationUnavailable': + 'Dyktowanie jest niedostępne. Sprawdź dostawcę rozpoznawania mowy w Ustawienia > Głos.', + 'composer.dictationStatusFailed': + 'Nie udało się sprawdzić dostępności dyktowania. Wróć do tego okna, aby spróbować ponownie.', + 'composer.dictate': 'Dyktuj', + 'composer.finishDictation': 'Zakończ dyktowanie', + 'composer.discardDictation': 'Odrzuć dyktowanie', + 'composer.dictationStarting': 'Uruchamianie mikrofonu...', + 'composer.dictationRecording': 'Nagrywanie. Zakończ, aby dodać tekst, lub odrzuć nagranie.', + 'composer.dictationFailed': 'Transkrypcja się nie powiodła. Spróbuj ponownie.', + 'composer.dictationTimedOut': 'Upłynął limit czasu dyktowania. Spróbuj ponownie.', + // Share cards (#5006). 'share.button': 'Udostępnij', 'share.modalTitle': 'Udostępnij tę chwilę', diff --git a/app/src/lib/i18n/pt.ts b/app/src/lib/i18n/pt.ts index e893584277f..80717b75b89 100644 --- a/app/src/lib/i18n/pt.ts +++ b/app/src/lib/i18n/pt.ts @@ -3,6 +3,19 @@ import type { TranslationMap } from './types'; // Portuguese (Português) translations. Keys mirror en.ts; missing/ // English-identical values fall back to English via I18nContext.resolveEn(). const messages: TranslationMap = { + // Editable composer dictation. + 'composer.dictationUnavailable': + 'O ditado não está disponível. Verifique o provedor de voz em Configurações > Voz.', + 'composer.dictationStatusFailed': + 'Não foi possível verificar a disponibilidade do ditado. Volte a esta janela para tentar novamente.', + 'composer.dictate': 'Ditar', + 'composer.finishDictation': 'Concluir ditado', + 'composer.discardDictation': 'Descartar ditado', + 'composer.dictationStarting': 'Iniciando microfone...', + 'composer.dictationRecording': 'Gravando. Conclua para adicionar o texto ou descarte a gravação.', + 'composer.dictationFailed': 'A transcrição falhou. Tente novamente.', + 'composer.dictationTimedOut': 'O tempo do ditado esgotou. Tente novamente.', + // Share cards (#5006). 'share.button': 'Partilhar', 'share.modalTitle': 'Partilhe este momento', diff --git a/app/src/lib/i18n/ru.ts b/app/src/lib/i18n/ru.ts index 8899a8065d9..53ec1a0b63d 100644 --- a/app/src/lib/i18n/ru.ts +++ b/app/src/lib/i18n/ru.ts @@ -3,6 +3,19 @@ import type { TranslationMap } from './types'; // Russian (Русский) translations. Keys mirror en.ts; missing/ // English-identical values fall back to English via I18nContext.resolveEn(). const messages: TranslationMap = { + // Editable composer dictation. + 'composer.dictationUnavailable': + 'Диктовка недоступна. Проверьте поставщика распознавания речи в разделе Настройки > Голос.', + 'composer.dictationStatusFailed': + 'Не удалось проверить доступность диктовки. Вернитесь в это окно, чтобы повторить попытку.', + 'composer.dictate': 'Диктовать', + 'composer.finishDictation': 'Завершить диктовку', + 'composer.discardDictation': 'Отменить диктовку', + 'composer.dictationStarting': 'Включение микрофона...', + 'composer.dictationRecording': 'Идёт запись. Завершите её, чтобы добавить текст, или отмените.', + 'composer.dictationFailed': 'Не удалось распознать речь. Попробуйте ещё раз.', + 'composer.dictationTimedOut': 'Время диктовки истекло. Попробуйте ещё раз.', + // Share cards (#5006). 'share.button': 'Поделиться', 'share.modalTitle': 'Поделитесь этим моментом', diff --git a/app/src/lib/i18n/zh-CN.ts b/app/src/lib/i18n/zh-CN.ts index a2a79e1bc99..84ed38b5c1f 100644 --- a/app/src/lib/i18n/zh-CN.ts +++ b/app/src/lib/i18n/zh-CN.ts @@ -3,6 +3,17 @@ import type { TranslationMap } from './types'; // Simplified Chinese (简体中文) translations. Keys mirror en.ts; missing/ // English-identical values fall back to English via I18nContext.resolveEn(). const messages: TranslationMap = { + // Editable composer dictation. + 'composer.dictationUnavailable': '听写不可用。请在设置 > 语音中检查语音识别提供商。', + 'composer.dictationStatusFailed': '无法检查听写是否可用。请返回此窗口重试。', + 'composer.dictate': '听写', + 'composer.finishDictation': '完成听写', + 'composer.discardDictation': '放弃听写', + 'composer.dictationStarting': '正在启动麦克风...', + 'composer.dictationRecording': '正在录音。完成后添加文字,或放弃录音。', + 'composer.dictationFailed': '转写失败,请重试。', + 'composer.dictationTimedOut': '听写超时,请重试。', + // Share cards (#5006). 'share.button': '分享', 'share.modalTitle': '分享这一刻', diff --git a/app/src/providers/AssistantUiRuntimeProvider.tsx b/app/src/providers/AssistantUiRuntimeProvider.tsx index 0a9892b76f1..1489d474bf1 100644 --- a/app/src/providers/AssistantUiRuntimeProvider.tsx +++ b/app/src/providers/AssistantUiRuntimeProvider.tsx @@ -9,6 +9,8 @@ import { createContext, type ReactNode, useContext, useMemo } from 'react'; import { useOpenHumanToolkit } from '../features/conversations/aui/toolkit'; import { useAppSelector } from '../store/hooks'; +import { ComposerDictationContext } from './ComposerDictationContext'; +import { useComposerDictation } from './useComposerDictation'; import { useOpenHumanExternalStore } from './useOpenHumanExternalStore'; const debug = debugFactory('openhuman:assistant-ui'); @@ -81,7 +83,11 @@ export function AssistantUiRuntimeProvider({ effectiveThreadId ?? '(none)', threadId === undefined ? 'selection' : 'explicit' ); - const adapter = useOpenHumanExternalStore(effectiveThreadId, { welcomeSuggestions }); + const dictation = useComposerDictation(effectiveThreadId); + const adapter = useOpenHumanExternalStore(effectiveThreadId, { + welcomeSuggestions, + dictationAdapter: dictation.adapter, + }); const runtime = useExternalStoreRuntime(adapter); // Registers every `aui/toolkit.tsx` entry (currently just `task`) so // assistant-ui resolves them ahead of the surface's own `ToolFallback`. @@ -92,7 +98,9 @@ export function AssistantUiRuntimeProvider({ return ( - {children} + + {children} + ); diff --git a/app/src/providers/ComposerDictationContext.ts b/app/src/providers/ComposerDictationContext.ts new file mode 100644 index 00000000000..f658d371581 --- /dev/null +++ b/app/src/providers/ComposerDictationContext.ts @@ -0,0 +1,12 @@ +import { createContext, useContext } from 'react'; + +import type { useComposerDictation } from './useComposerDictation'; + +export type ComposerDictationState = ReturnType; + +export const ComposerDictationContext = createContext(null); + +/** Read this composer's dictation state, or null outside a dictation runtime. */ +export function useComposerDictationState(): ComposerDictationState | null { + return useContext(ComposerDictationContext); +} diff --git a/app/src/providers/__tests__/AssistantUiRuntimeProvider.approvalGate.test.tsx b/app/src/providers/__tests__/AssistantUiRuntimeProvider.approvalGate.test.tsx index 29993e0a7dd..25c01143092 100644 --- a/app/src/providers/__tests__/AssistantUiRuntimeProvider.approvalGate.test.tsx +++ b/app/src/providers/__tests__/AssistantUiRuntimeProvider.approvalGate.test.tsx @@ -47,7 +47,10 @@ vi.mock('../../services/api/threadApi', () => ({ }, })); -vi.mock('../../services/coreRpcClient', () => ({ callCoreRpc: vi.fn() })); +vi.mock('../../services/coreRpcClient', async importOriginal => ({ + ...(await importOriginal()), + callCoreRpc: vi.fn(), +})); const THREAD_ID = 't-gate'; const REQUEST_ID = 'appr-7'; diff --git a/app/src/providers/dictationAdapter.fallback.test.ts b/app/src/providers/dictationAdapter.fallback.test.ts new file mode 100644 index 00000000000..4661d8878b9 --- /dev/null +++ b/app/src/providers/dictationAdapter.fallback.test.ts @@ -0,0 +1,232 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { VoiceNotCompiledError } from '../features/human/voice/sttClient'; +import { + createOpenHumanDictationAdapter, + DICTATION_TRANSCRIPTION_TIMEOUT_MS, + type OpenHumanDictationAdapter, +} from './dictationAdapter'; + +const transcribe = vi.fn(); +const encodeWav = vi.fn(); +vi.mock('../features/human/voice/sttClient', async importOriginal => ({ + ...(await importOriginal()), + transcribeWithFactory: (...args: unknown[]) => transcribe(...args), +})); +vi.mock('../features/human/voice/wavEncoder', () => ({ + encodeBlobToWav: (...args: unknown[]) => encodeWav(...args), +})); + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +async function flush() { + for (let step = 0; step < 6; step++) await Promise.resolve(); +} + +describe('dictation WAV compatibility fallback', () => { + let adapter: OpenHumanDictationAdapter; + let originalMediaDevices: PropertyDescriptor | undefined; + let wav: Blob; + let track: EventTarget & { stop: ReturnType; readyState: string }; + + beforeEach(() => { + vi.useFakeTimers(); + wav = new Blob(['portable PCM'], { type: 'audio/wav' }); + transcribe.mockReset().mockResolvedValue('dictated words'); + encodeWav.mockReset().mockResolvedValue(wav); + track = Object.assign(new EventTarget(), { stop: vi.fn(), readyState: 'live' }); + originalMediaDevices = Object.getOwnPropertyDescriptor(navigator, 'mediaDevices'); + Object.defineProperty(navigator, 'mediaDevices', { + configurable: true, + value: { getUserMedia: vi.fn().mockResolvedValue({ getTracks: () => [track] }) }, + }); + class Recorder { + state: RecordingState = 'inactive'; + mimeType = 'audio/webm'; + ondataavailable: ((event: BlobEvent) => void) | null = null; + onstop: (() => void) | null = null; + onerror: (() => void) | null = null; + start() { + this.state = 'recording'; + } + stop() { + this.state = 'inactive'; + this.ondataavailable?.({ + data: new Blob(['native audio'], { type: this.mimeType }), + } as BlobEvent); + this.onstop?.(); + } + static isTypeSupported() { + return false; + } + } + vi.stubGlobal('MediaRecorder', Recorder); + adapter = createOpenHumanDictationAdapter(); + }); + + afterEach(() => { + adapter.dispose(); + if (originalMediaDevices) { + Object.defineProperty(navigator, 'mediaDevices', originalMediaDevices); + } else { + delete (navigator as { mediaDevices?: MediaDevices }).mediaDevices; + } + vi.unstubAllGlobals(); + vi.useRealTimers(); + }); + + async function listen() { + const session = adapter.listen(); + const speech = vi.fn(); + const ended = vi.fn(); + session.onSpeech(speech); + session.onSpeechEnd(ended); + await flush(); + return { session, speech, ended }; + } + + it('keeps a successful native recording on one STT call with no conversion', async () => { + const h = await listen(); + await h.session.stop(); + expect(transcribe).toHaveBeenCalledOnce(); + expect((transcribe.mock.calls[0][0] as Blob).type).toBe('audio/webm'); + expect(encodeWav).not.toHaveBeenCalled(); + expect(h.speech).toHaveBeenCalledExactlyOnceWith({ + transcript: 'dictated words', + isFinal: true, + }); + }); + + it('retries one rejected native clip as portable PCM WAV and emits the fallback text', async () => { + transcribe.mockRejectedValueOnce(new Error('native container rejected')); + const h = await listen(); + await h.session.stop(); + expect(transcribe).toHaveBeenCalledTimes(2); + expect(encodeWav).toHaveBeenCalledExactlyOnceWith(transcribe.mock.calls[0][0]); + expect(transcribe.mock.calls[1][0]).toBe(wav); + expect(h.speech).toHaveBeenCalledExactlyOnceWith({ + transcript: 'dictated words', + isFinal: true, + }); + expect(adapter.getSnapshot()).toEqual({ phase: 'idle', error: null }); + expect(track.stop).toHaveBeenCalledOnce(); + }); + + it('ends after the single fallback STT call fails, without exposing raw error details', async () => { + transcribe.mockRejectedValue(new Error('private provider detail')); + const h = await listen(); + await h.session.stop(); + expect(transcribe).toHaveBeenCalledTimes(2); + expect(encodeWav).toHaveBeenCalledOnce(); + expect(adapter.getSnapshot()).toEqual({ phase: 'idle', error: 'transcription-failed' }); + expect(h.speech).not.toHaveBeenCalled(); + expect(h.ended).toHaveBeenCalledOnce(); + }); + + it('ends if WAV conversion fails and makes no second STT request', async () => { + transcribe.mockRejectedValueOnce(new Error('native container rejected')); + encodeWav.mockRejectedValueOnce(new Error('decode failed')); + const h = await listen(); + await h.session.stop(); + expect(transcribe).toHaveBeenCalledOnce(); + expect(adapter.getSnapshot()).toEqual({ phase: 'idle', error: 'transcription-failed' }); + expect(h.speech).not.toHaveBeenCalled(); + expect(h.ended).toHaveBeenCalledOnce(); + }); + + it('does not convert or retry when the core has no voice dispatch', async () => { + transcribe.mockRejectedValueOnce(new VoiceNotCompiledError()); + const h = await listen(); + await h.session.stop(); + expect(transcribe).toHaveBeenCalledOnce(); + expect(encodeWav).not.toHaveBeenCalled(); + expect(adapter.getSnapshot()).toEqual({ phase: 'idle', error: 'voice-unavailable' }); + }); + + it('cancellation during conversion suppresses the second STT request', async () => { + const conversion = deferred(); + transcribe.mockRejectedValueOnce(new Error('native container rejected')); + encodeWav.mockReturnValue(conversion.promise); + const h = await listen(); + const stopping = h.session.stop(); + await flush(); + expect(encodeWav).toHaveBeenCalledOnce(); + adapter.cancel(); + await stopping; + conversion.resolve(wav); + await flush(); + expect(transcribe).toHaveBeenCalledOnce(); + expect(h.speech).not.toHaveBeenCalled(); + expect(h.ended).toHaveBeenCalledOnce(); + expect(adapter.getSnapshot()).toEqual({ phase: 'idle', error: null }); + }); + + it('cancellation during fallback STT suppresses its late transcript', async () => { + const fallback = deferred(); + transcribe + .mockRejectedValueOnce(new Error('native container rejected')) + .mockReturnValueOnce(fallback.promise); + const h = await listen(); + const stopping = h.session.stop(); + await flush(); + expect(transcribe).toHaveBeenCalledTimes(2); + adapter.cancel(); + await stopping; + fallback.resolve('late fallback text'); + await flush(); + expect(h.speech).not.toHaveBeenCalled(); + expect(h.ended).toHaveBeenCalledOnce(); + expect(adapter.getSnapshot()).toEqual({ phase: 'idle', error: null }); + }); + + it('uses one total transcription deadline across the native call, conversion, and retry', async () => { + const native = deferred(); + const conversion = deferred(); + const fallback = deferred(); + transcribe.mockReturnValueOnce(native.promise).mockReturnValueOnce(fallback.promise); + encodeWav.mockReturnValue(conversion.promise); + const h = await listen(); + const stopping = h.session.stop(); + await vi.advanceTimersByTimeAsync(20_000); + native.reject(new Error('native container rejected')); + await flush(); + await vi.advanceTimersByTimeAsync(20_000); + conversion.resolve(wav); + await flush(); + expect(transcribe).toHaveBeenCalledTimes(2); + await vi.advanceTimersByTimeAsync(DICTATION_TRANSCRIPTION_TIMEOUT_MS - 40_001); + expect(adapter.getSnapshot().phase).toBe('transcribing'); + await vi.advanceTimersByTimeAsync(1); + await stopping; + expect(adapter.getSnapshot()).toEqual({ phase: 'idle', error: 'timed-out' }); + fallback.resolve('expired fallback text'); + await flush(); + expect(h.speech).not.toHaveBeenCalled(); + expect(h.ended).toHaveBeenCalledOnce(); + }); + + it('a conversion that outlasts the deadline never starts a fallback request', async () => { + const conversion = deferred(); + transcribe.mockRejectedValueOnce(new Error('native container rejected')); + encodeWav.mockReturnValue(conversion.promise); + const h = await listen(); + const stopping = h.session.stop(); + await flush(); + await vi.advanceTimersByTimeAsync(DICTATION_TRANSCRIPTION_TIMEOUT_MS); + await stopping; + conversion.resolve(wav); + await flush(); + expect(transcribe).toHaveBeenCalledOnce(); + expect(h.speech).not.toHaveBeenCalled(); + expect(h.ended).toHaveBeenCalledOnce(); + expect(adapter.getSnapshot()).toEqual({ phase: 'idle', error: 'timed-out' }); + }); +}); diff --git a/app/src/providers/dictationAdapter.test.ts b/app/src/providers/dictationAdapter.test.ts new file mode 100644 index 00000000000..d123dbb80bb --- /dev/null +++ b/app/src/providers/dictationAdapter.test.ts @@ -0,0 +1,491 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { VoiceNotCompiledError } from '../features/human/voice/sttClient'; +import { + createOpenHumanDictationAdapter, + DICTATION_START_TIMEOUT_MS, + DICTATION_STOP_TIMEOUT_MS, + DICTATION_TRANSCRIPTION_TIMEOUT_MS, + isDictationCaptureSupported, + MAX_DICTATION_RECORDING_MS, + type OpenHumanDictationAdapter, +} from './dictationAdapter'; + +const transcribe = vi.fn(); +const encodeWav = vi.fn(); +vi.mock('../features/human/voice/wavEncoder', () => ({ + encodeBlobToWav: (...args: unknown[]) => encodeWav(...args), +})); +vi.mock('../features/human/voice/sttClient', async importOriginal => ({ + ...(await importOriginal()), + transcribeWithFactory: (...args: unknown[]) => transcribe(...args), +})); + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +class FakeTrack extends EventTarget { + readyState: MediaStreamTrackState = 'live'; + stop = vi.fn(() => { + this.readyState = 'ended'; + }); +} + +type FakeRecorder = { + state: RecordingState; + mimeType: string; + ondataavailable: ((event: BlobEvent) => void) | null; + onstop: (() => void) | null; + onerror: (() => void) | null; + start: ReturnType; + stop: ReturnType; +}; + +async function flush() { + for (let step = 0; step < 6; step++) await Promise.resolve(); +} + +describe('OpenHumanDictationAdapter', () => { + let adapter: OpenHumanDictationAdapter; + let track: FakeTrack; + let stream: MediaStream; + let getUserMedia: ReturnType; + let recorder: FakeRecorder; + let constructorError: boolean; + let startError: boolean; + let stopError: boolean; + let hangStop: boolean; + let emptyAudio: boolean; + let actualMime: string | undefined; + let recorderOptions: MediaRecorderOptions | undefined; + let originalMediaDevices: PropertyDescriptor | undefined; + + beforeEach(() => { + vi.useFakeTimers(); + transcribe.mockReset().mockResolvedValue('spoken words'); + encodeWav.mockReset().mockResolvedValue(new Blob(['wav audio'], { type: 'audio/wav' })); + constructorError = false; + startError = false; + stopError = false; + hangStop = false; + emptyAudio = false; + actualMime = undefined; + recorderOptions = undefined; + track = new FakeTrack(); + stream = { getTracks: () => [track] } as unknown as MediaStream; + getUserMedia = vi.fn().mockResolvedValue(stream); + originalMediaDevices = Object.getOwnPropertyDescriptor(navigator, 'mediaDevices'); + Object.defineProperty(navigator, 'mediaDevices', { + value: { getUserMedia }, + configurable: true, + }); + class Recorder { + constructor(_stream: MediaStream, options?: MediaRecorderOptions) { + if (constructorError) throw new Error('private constructor detail'); + recorderOptions = options; + recorder = { + state: 'inactive', + mimeType: actualMime ?? options?.mimeType ?? 'audio/webm', + ondataavailable: null, + onstop: null, + onerror: null, + start: vi.fn(() => { + if (startError) throw new Error('private start detail'); + recorder.state = 'recording'; + }), + stop: vi.fn(() => { + if (stopError) throw new Error('private stop detail'); + recorder.state = 'inactive'; + if (hangStop) return; + if (!emptyAudio) { + recorder.ondataavailable?.({ + data: new Blob([new Uint8Array([1, 2, 3])], { type: recorder.mimeType }), + } as BlobEvent); + } + recorder.onstop?.(); + }), + }; + return recorder as unknown as MediaRecorder; + } + static isTypeSupported(mime: string) { + return mime.startsWith('audio/mp4'); + } + } + vi.stubGlobal('MediaRecorder', Recorder); + adapter = createOpenHumanDictationAdapter(); + }); + + afterEach(() => { + adapter.dispose(); + if (originalMediaDevices) { + Object.defineProperty(navigator, 'mediaDevices', originalMediaDevices); + } else { + delete (navigator as { mediaDevices?: MediaDevices }).mediaDevices; + } + vi.unstubAllGlobals(); + vi.useRealTimers(); + }); + + function listen() { + const session = adapter.listen(); + const speech = vi.fn(); + const ended = vi.fn(); + const started = vi.fn(); + session.onSpeech(speech); + session.onSpeechEnd(ended); + session.onSpeechStart(started); + return { session, speech, ended, started }; + } + + it('records and emits one final transcript before stop resolves, leaving input editable', async () => { + const result = deferred(); + transcribe.mockReturnValue(result.promise); + const h = listen(); + expect(adapter.disableInputDuringDictation).toBe(false); + expect(adapter.getSnapshot()).toEqual({ phase: 'starting', error: null }); + await flush(); + expect(adapter.getSnapshot().phase).toBe('recording'); + expect(h.started).toHaveBeenCalledOnce(); + expect(getUserMedia).toHaveBeenCalledWith({ + audio: expect.objectContaining({ channelCount: 1, echoCancellation: true }), + }); + + const settled = vi.fn(); + const stopping = h.session.stop().then(settled); + expect(adapter.getSnapshot().phase).toBe('transcribing'); + expect(track.stop).toHaveBeenCalledOnce(); + expect(settled).not.toHaveBeenCalled(); + expect(h.speech).not.toHaveBeenCalled(); + + result.resolve(' spoken words '); + await stopping; + expect(h.speech).toHaveBeenCalledExactlyOnceWith({ transcript: 'spoken words', isFinal: true }); + expect(h.speech.mock.invocationCallOrder[0]).toBeLessThan(settled.mock.invocationCallOrder[0]); + expect(h.ended).toHaveBeenCalledOnce(); + expect(h.session.status).toEqual({ type: 'ended', reason: 'stopped' }); + expect(adapter.getSnapshot()).toEqual({ phase: 'idle', error: null }); + }); + + it('uses the actual recorder MIME for the blob sent to configured STT', async () => { + actualMime = 'audio/mp4'; + const h = listen(); + await flush(); + expect(recorderOptions).toEqual({ + audioBitsPerSecond: 128_000, + mimeType: 'audio/mp4;codecs=mp4a.40.2', + }); + await h.session.stop(); + expect(transcribe).toHaveBeenCalledOnce(); + const blob = transcribe.mock.calls[0][0] as Blob; + expect(blob.type).toBe('audio/mp4'); + expect(blob.size).toBe(3); + }); + + it('lets the browser choose a codec when none of the preferred MIME types is supported', async () => { + vi.spyOn(MediaRecorder, 'isTypeSupported').mockReturnValue(false); + const h = listen(); + await flush(); + expect(recorderOptions?.mimeType).toBeUndefined(); + await h.session.stop(); + expect((transcribe.mock.calls[0][0] as Blob).type).toBe('audio/webm'); + }); + + it.each([ + ['NotAllowedError', 'permission-denied'], + ['SecurityError', 'permission-denied'], + ['NotFoundError', 'device-unavailable'], + ['OverconstrainedError', 'device-unavailable'], + ['NotReadableError', 'device-in-use'], + ['UnexpectedError', 'microphone-unavailable'], + ])('ends and reports a content-free error for %s', async (name, code) => { + getUserMedia.mockRejectedValue(new DOMException('secret device detail', name)); + const h = listen(); + await flush(); + expect(adapter.getSnapshot()).toEqual({ phase: 'idle', error: code }); + expect(h.session.status).toEqual({ type: 'ended', reason: 'error' }); + expect(h.ended).toHaveBeenCalledOnce(); + expect(h.speech).not.toHaveBeenCalled(); + expect(transcribe).not.toHaveBeenCalled(); + expect(JSON.stringify(adapter.getSnapshot())).not.toContain('secret'); + }); + + it.each(['stop', 'cancel', 'dispose'] as const)( + '%s during a permission prompt discards the late stream', + async operation => { + const permission = deferred(); + getUserMedia.mockReturnValue(permission.promise); + const h = listen(); + await flush(); + if (operation === 'stop') await h.session.stop(); + else adapter[operation](); + expect(h.session.status).toEqual({ type: 'ended', reason: 'cancelled' }); + expect(h.ended).toHaveBeenCalledOnce(); + permission.resolve(stream); + await flush(); + expect(track.stop).toHaveBeenCalledOnce(); + expect(h.speech).not.toHaveBeenCalled(); + expect(transcribe).not.toHaveBeenCalled(); + expect(adapter.getSnapshot().phase).toBe('idle'); + } + ); + + it('cancels before capture starts without requesting permission', async () => { + const h = listen(); + h.session.cancel(); + await flush(); + expect(getUserMedia).not.toHaveBeenCalled(); + expect(h.ended).toHaveBeenCalledOnce(); + }); + + it.each(['cancel', 'dispose'] as const)( + '%s while STT is pending prevents late text and returns immediately to idle', + async operation => { + const result = deferred(); + transcribe.mockReturnValue(result.promise); + const h = listen(); + await flush(); + const stopping = h.session.stop(); + adapter[operation](); + await stopping; + expect(adapter.getSnapshot()).toEqual({ phase: 'idle', error: null }); + expect(h.session.status).toEqual({ type: 'ended', reason: 'cancelled' }); + result.resolve('late transcript'); + await flush(); + expect(h.speech).not.toHaveBeenCalled(); + expect(h.ended).toHaveBeenCalledOnce(); + expect(recorder.onstop).toBeNull(); + expect(recorder.ondataavailable).toBeNull(); + expect(recorder.onerror).toBeNull(); + } + ); + + it('suppresses queued recorder events after cancel', async () => { + const h = listen(); + await flush(); + const lateData = recorder.ondataavailable!; + const lateStop = recorder.onstop!; + const lateError = recorder.onerror!; + adapter.cancel(); + lateData({ data: new Blob(['late audio']) } as BlobEvent); + lateStop(); + lateError(); + await flush(); + expect(transcribe).not.toHaveBeenCalled(); + expect(h.speech).not.toHaveBeenCalled(); + expect(h.ended).toHaveBeenCalledOnce(); + expect(track.stop).toHaveBeenCalledOnce(); + }); + + it('ends on microphone device loss and discards recorded data', async () => { + const h = listen(); + await flush(); + track.readyState = 'ended'; + track.dispatchEvent(new Event('ended')); + expect(adapter.getSnapshot()).toEqual({ phase: 'idle', error: 'device-unavailable' }); + expect(h.ended).toHaveBeenCalledOnce(); + expect(track.stop).toHaveBeenCalledOnce(); + expect(recorder.onstop).toBeNull(); + expect(transcribe).not.toHaveBeenCalled(); + }); + + it('rejects an already ended track before recording', async () => { + track.readyState = 'ended'; + const h = listen(); + await flush(); + expect(adapter.getSnapshot().error).toBe('device-unavailable'); + expect(h.started).not.toHaveBeenCalled(); + expect(track.stop).toHaveBeenCalledOnce(); + }); + + it.each(['constructor', 'start', 'stop', 'event'] as const)( + 'cleans the microphone and ends on recorder %s failure', + async failure => { + constructorError = failure === 'constructor'; + startError = failure === 'start'; + stopError = failure === 'stop'; + const h = listen(); + await flush(); + if (failure === 'stop') await h.session.stop(); + if (failure === 'event') recorder.onerror!(); + expect(adapter.getSnapshot()).toEqual({ phase: 'idle', error: 'recorder-failed' }); + expect(h.ended).toHaveBeenCalledOnce(); + expect(track.stop).toHaveBeenCalledOnce(); + expect(h.speech).not.toHaveBeenCalled(); + expect(transcribe).not.toHaveBeenCalled(); + } + ); + + it('double stop shares completion and transcribes only once', async () => { + const result = deferred(); + transcribe.mockReturnValue(result.promise); + const h = listen(); + await flush(); + const first = h.session.stop(); + const second = h.session.stop(); + expect(first).toBe(second); + expect(recorder.stop).toHaveBeenCalledOnce(); + await flush(); + expect(transcribe).toHaveBeenCalledOnce(); + result.resolve('completed'); + await first; + await h.session.stop(); + expect(h.speech).toHaveBeenCalledOnce(); + expect(h.ended).toHaveBeenCalledOnce(); + }); + + it('discards a timed-out recording without uploading audio or changing the draft', async () => { + const h = listen(); + await flush(); + await vi.advanceTimersByTimeAsync(MAX_DICTATION_RECORDING_MS); + expect(h.speech).not.toHaveBeenCalled(); + expect(transcribe).not.toHaveBeenCalled(); + expect(encodeWav).not.toHaveBeenCalled(); + expect(track.stop).toHaveBeenCalledOnce(); + expect(adapter.getSnapshot()).toEqual({ phase: 'idle', error: 'timed-out' }); + expect(h.ended).toHaveBeenCalledExactlyOnceWith({ transcript: '', isFinal: true }); + await h.session.stop(); + expect(transcribe).not.toHaveBeenCalled(); + }); + + it('bounds a hanging permission request and discards a later grant', async () => { + const permission = deferred(); + getUserMedia.mockReturnValue(permission.promise); + const h = listen(); + await flush(); + await vi.advanceTimersByTimeAsync(DICTATION_START_TIMEOUT_MS); + expect(adapter.getSnapshot().error).toBe('timed-out'); + expect(h.ended).toHaveBeenCalledOnce(); + permission.resolve(stream); + await flush(); + expect(track.stop).toHaveBeenCalledOnce(); + expect(h.speech).not.toHaveBeenCalled(); + }); + + it('bounds a recorder that never emits stop and releases its stream promptly', async () => { + hangStop = true; + const h = listen(); + await flush(); + const stopping = h.session.stop(); + expect(track.stop).toHaveBeenCalledOnce(); + await vi.advanceTimersByTimeAsync(DICTATION_STOP_TIMEOUT_MS); + await stopping; + expect(adapter.getSnapshot().error).toBe('timed-out'); + expect(h.ended).toHaveBeenCalledOnce(); + expect(transcribe).not.toHaveBeenCalled(); + }); + + it('bounds STT and ignores a reply after the bound expires', async () => { + const result = deferred(); + transcribe.mockReturnValue(result.promise); + const h = listen(); + await flush(); + const stopping = h.session.stop(); + await vi.advanceTimersByTimeAsync(DICTATION_TRANSCRIPTION_TIMEOUT_MS); + await stopping; + expect(adapter.getSnapshot().error).toBe('timed-out'); + result.resolve('expired text'); + await flush(); + expect(h.speech).not.toHaveBeenCalled(); + expect(h.ended).toHaveBeenCalledOnce(); + }); + + it.each([ + ['voice-unavailable', new VoiceNotCompiledError()], + ['transcription-failed', new Error('private backend detail')], + ])('surfaces %s without raw error text', async (code, error) => { + transcribe.mockRejectedValue(error); + const h = listen(); + await flush(); + await h.session.stop(); + expect(adapter.getSnapshot()).toEqual({ phase: 'idle', error: code }); + expect(h.speech).not.toHaveBeenCalled(); + expect(h.ended).toHaveBeenCalledOnce(); + }); + + it.each(['no-audio', 'no-speech'] as const)('ends cleanly when there is %s', async code => { + emptyAudio = code === 'no-audio'; + transcribe.mockResolvedValue(' '); + const h = listen(); + await flush(); + await h.session.stop(); + expect(adapter.getSnapshot().error).toBe(code); + expect(h.speech).not.toHaveBeenCalled(); + expect(h.ended).toHaveBeenCalledOnce(); + if (emptyAudio) expect(transcribe).not.toHaveBeenCalled(); + }); + + it('replacing a session invalidates its pending transcript', async () => { + const result = deferred(); + transcribe.mockReturnValue(result.promise); + const first = listen(); + await flush(); + const stopping = first.session.stop(); + const replacementTrack = new FakeTrack(); + getUserMedia.mockResolvedValue({ getTracks: () => [replacementTrack] }); + const second = listen(); + await flush(); + result.resolve('old text'); + await stopping; + expect(first.speech).not.toHaveBeenCalled(); + expect(second.speech).not.toHaveBeenCalled(); + expect(adapter.getSnapshot().phase).toBe('recording'); + adapter.cancel(); + expect(second.ended).toHaveBeenCalledOnce(); + }); + + it('publishes stable snapshots and unsubscribes observers', async () => { + const subscriber = vi.fn(); + const unsubscribe = adapter.subscribe(subscriber); + expect(adapter.getSnapshot()).toBe(adapter.getSnapshot()); + const h = listen(); + await flush(); + expect(subscriber).toHaveBeenCalledTimes(2); + unsubscribe(); + await h.session.stop(); + expect(subscriber).toHaveBeenCalledTimes(2); + }); + + it('supports listening again after disposal for effect re-setup', async () => { + adapter.dispose(); + const h = listen(); + await flush(); + expect(adapter.getSnapshot().phase).toBe('recording'); + adapter.cancel(); + expect(h.ended).toHaveBeenCalledOnce(); + }); + + it('does not let a throwing subscriber prevent cleanup or completion', async () => { + const h = listen(); + h.session.onSpeech(() => { + throw new Error('subscriber failed'); + }); + h.session.onSpeechEnd(() => { + throw new Error('subscriber failed'); + }); + adapter.subscribe(() => { + throw new Error('subscriber failed'); + }); + await flush(); + await h.session.stop(); + expect(track.stop).toHaveBeenCalledOnce(); + expect(h.ended).toHaveBeenCalledOnce(); + expect(adapter.getSnapshot().phase).toBe('idle'); + }); + + it('detects unavailable capture without requesting permission', async () => { + vi.stubGlobal('MediaRecorder', undefined); + expect(isDictationCaptureSupported()).toBe(false); + const h = listen(); + await flush(); + expect(adapter.getSnapshot().error).toBe('microphone-unavailable'); + expect(h.ended).toHaveBeenCalledOnce(); + expect(getUserMedia).not.toHaveBeenCalled(); + }); +}); diff --git a/app/src/providers/dictationAdapter.ts b/app/src/providers/dictationAdapter.ts new file mode 100644 index 00000000000..912723e3e34 --- /dev/null +++ b/app/src/providers/dictationAdapter.ts @@ -0,0 +1,385 @@ +import type { DictationAdapter } from '@assistant-ui/react'; +import debugFactory from 'debug'; + +import { isVoiceNotCompiledError, transcribeWithFactory } from '../features/human/voice/sttClient'; +import { encodeBlobToWav } from '../features/human/voice/wavEncoder'; + +const log = debugFactory('openhuman:composer-dictation'); + +export const MAX_DICTATION_RECORDING_MS = 60_000; +export const DICTATION_START_TIMEOUT_MS = 30_000; +export const DICTATION_STOP_TIMEOUT_MS = 5_000; +export const DICTATION_TRANSCRIPTION_TIMEOUT_MS = 90_000; + +export type DictationErrorCode = + | 'microphone-unavailable' + | 'permission-denied' + | 'device-unavailable' + | 'device-in-use' + | 'recorder-failed' + | 'no-audio' + | 'no-speech' + | 'transcription-failed' + | 'voice-unavailable' + | 'timed-out'; + +export type DictationSnapshot = { + phase: 'idle' | 'starting' | 'recording' | 'transcribing'; + /** Content-free code; the surface translates it rather than exposing RPC errors. */ + error: DictationErrorCode | null; +}; + +type ActiveSession = { cancel: () => void }; +type Status = DictationAdapter.Status; +type Result = DictationAdapter.Result; + +const PREFERRED_MIMES = ['audio/mp4;codecs=mp4a.40.2', 'audio/mp4', 'audio/webm;codecs=opus']; + +/** Presence checks only: asking for microphone permission remains a user action. */ +export function isDictationCaptureSupported(): boolean { + return ( + typeof MediaRecorder !== 'undefined' && + typeof navigator !== 'undefined' && + typeof navigator.mediaDevices?.getUserMedia === 'function' + ); +} + +/** Prefer supported compressed audio; an empty value lets the browser choose. */ +function pickRecorderMime(): string { + if (typeof MediaRecorder.isTypeSupported !== 'function') return ''; + return PREFERRED_MIMES.find(mime => MediaRecorder.isTypeSupported(mime)) ?? ''; +} + +/** Map capture failures to localizable codes without retaining device details. */ +function permissionErrorCode(error: unknown): DictationErrorCode { + const name = typeof error === 'object' && error !== null && 'name' in error ? error.name : ''; + if (name === 'NotAllowedError' || name === 'SecurityError') return 'permission-denied'; + if (name === 'NotFoundError' || name === 'OverconstrainedError') return 'device-unavailable'; + if (name === 'NotReadableError') return 'device-in-use'; + return 'microphone-unavailable'; +} + +/** Keep one subscriber's exception from holding microphone resources open. */ +function notify(listeners: Set<(value: T) => void>, value: T): void { + for (const listener of [...listeners]) { + try { + listener(value); + } catch { + log('subscriber threw'); + } + } +} + +/** + * One runtime's editable composer dictation, using the shipped capture + STT path. + * Stop finalizes the clip and emits one final result before its promise resolves. + * Cancel discards it immediately; permission grants and STT replies arriving later + * can only release resources. No transcript is sent as a chat message here. + */ +export class OpenHumanDictationAdapter implements DictationAdapter { + // assistant-ui rebases dictation on setText, preserving edits made during capture. + readonly disableInputDuringDictation = false; + private snapshot: DictationSnapshot = { phase: 'idle', error: null }; + private readonly subscribers = new Set<() => void>(); + private active: ActiveSession | null = null; + private nextSessionId = 0; + + /** Stable external-store snapshot, replaced only when the session changes. */ + getSnapshot = (): DictationSnapshot => this.snapshot; + + /** Subscribe to capture state and return an idempotent unsubscription function. */ + subscribe = (listener: () => void): (() => void) => { + this.subscribers.add(listener); + return () => this.subscribers.delete(listener); + }; + + /** Notify the host of a content-free phase or error transition. */ + private publish(snapshot: DictationSnapshot): void { + this.snapshot = snapshot; + notify(this.subscribers, undefined); + } + + /** Discard the current clip and invalidate any pending transcription result. */ + cancel = (): void => { + this.active?.cancel(); + }; + + /** Reusable after cleanup, including React StrictMode's effect re-setup. */ + dispose = (): void => { + this.cancel(); + this.subscribers.clear(); + }; + + /** Replace any previous session and defer capture until listeners can attach. */ + listen = (): DictationAdapter.Session => { + this.cancel(); + const sessionId = ++this.nextSessionId; + let status: Status = { type: 'starting' }; + let ended = false; + let stopping = false; + let finalizing = false; + let stream: MediaStream | null = null; + let recorder: MediaRecorder | null = null; + let chunks: Blob[] = []; + let detachTracks: (() => void) | null = null; + let startupTimer: ReturnType | undefined; + let recordingTimer: ReturnType | undefined; + let stopTimer: ReturnType | undefined; + let transcriptionTimer: ReturnType | undefined; + const speechListeners = new Set<(result: Result) => void>(); + const startListeners = new Set<() => void>(); + const endListeners = new Set<(result: Result) => void>(); + let resolveCompletion!: () => void; + const completion = new Promise(resolve => { + resolveCompletion = resolve; + }); + + /** Detach device-loss handlers before stopping every owned microphone track. */ + const releaseStream = () => { + detachTracks?.(); + detachTracks = null; + if (stream) { + for (const track of stream.getTracks()) { + try { + track.stop(); + } catch { + log('session=%d track cleanup failed', sessionId); + } + } + stream = null; + } + }; + + /** Terminate exactly once, release resources, and settle every stop caller. */ + const finish = ( + reason: 'stopped' | 'cancelled' | 'error', + error: DictationErrorCode | null = null, + transcript = '' + ) => { + if (ended) return; + ended = true; + status = { type: 'ended', reason }; + clearTimeout(startupTimer); + clearTimeout(recordingTimer); + clearTimeout(stopTimer); + clearTimeout(transcriptionTimer); + if (recorder) { + recorder.ondataavailable = null; + recorder.onstop = null; + recorder.onerror = null; + try { + if (recorder.state !== 'inactive') recorder.stop(); + } catch { + log('session=%d recorder cleanup failed', sessionId); + } + recorder = null; + } + releaseStream(); + chunks = []; + if (this.active === active) { + this.active = null; + this.publish({ phase: 'idle', error }); + } + log('session=%d ended reason=%s error=%s', sessionId, reason, error ?? 'none'); + // assistant-ui uses this event to clean its session synchronously. Merely + // changing status would leave its controls active until the next 100ms poll. + notify(endListeners, { transcript, isFinal: true }); + speechListeners.clear(); + startListeners.clear(); + endListeners.clear(); + resolveCompletion(); + }; + + const active: ActiveSession = { cancel: () => finish('cancelled') }; + this.active = active; + this.publish({ phase: 'starting', error: null }); + + /** Transcribe the completed clip, retry as WAV, and emit only a live result. */ + const finalize = async () => { + if (ended || finalizing) return; + finalizing = true; + stopping = true; + clearTimeout(recordingTimer); + clearTimeout(stopTimer); + releaseStream(); + this.publish({ phase: 'transcribing', error: null }); + if (ended) return; + const blob = new Blob(chunks, { + type: recorder?.mimeType || chunks[0]?.type || 'audio/webm', + }); + chunks = []; + if (blob.size === 0) { + finish('error', 'no-audio'); + return; + } + transcriptionTimer = setTimeout( + () => finish('error', 'timed-out'), + DICTATION_TRANSCRIPTION_TIMEOUT_MS + ); + try { + let text: string; + try { + text = await transcribeWithFactory(blob); + } catch (error) { + if (ended) return; + if (isVoiceNotCompiledError(error)) throw error; + // Reuse the shipped voice path's portable PCM fallback. The existing + // timer bounds native STT, conversion, and this one retry together. + log('session=%d native transcription failed; retrying as WAV', sessionId); + const wav = await encodeBlobToWav(blob); + if (ended) return; + text = await transcribeWithFactory(wav); + } + const transcript = text.trim(); + if (ended) return; + if (!transcript) { + finish('error', 'no-speech'); + return; + } + // Emit while assistant-ui is still subscribed, before terminal status or + // speechEnd can clear its session. Its composer owns appending to drafts. + notify(speechListeners, { transcript, isFinal: true }); + finish('stopped', null, transcript); + } catch (error) { + if (ended) return; + finish( + 'error', + isVoiceNotCompiledError(error) ? 'voice-unavailable' : 'transcription-failed' + ); + } + }; + + /** Finish the clip once; repeated calls share the same bounded completion. */ + const stop = (): Promise => { + if (ended || stopping) return completion; + if (status.type === 'starting') { + // getUserMedia cannot be aborted. Invalidate it now and release any late + // grant instead of recording after the user has pressed Stop. + finish('cancelled'); + return completion; + } + stopping = true; + clearTimeout(recordingTimer); + this.publish({ phase: 'transcribing', error: null }); + if (ended) return completion; + stopTimer = setTimeout(() => finish('error', 'timed-out'), DICTATION_STOP_TIMEOUT_MS); + try { + recorder?.stop(); + // A broken recorder must not keep the OS microphone indicator on while + // we wait for its final event (or for the stop bound to expire). + releaseStream(); + } catch { + finish('error', 'recorder-failed'); + } + return completion; + }; + + /** Request capture and release late permission grants after cancellation. */ + const start = async () => { + if (ended) return; + if (!isDictationCaptureSupported()) { + finish('error', 'microphone-unavailable'); + return; + } + let granted: MediaStream; + try { + granted = await navigator.mediaDevices.getUserMedia({ + audio: { + channelCount: 1, + sampleRate: 48000, + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, + }); + } catch (error) { + if (!ended) finish('error', permissionErrorCode(error)); + return; + } + if (ended) { + for (const track of granted.getTracks()) { + try { + track.stop(); + } catch { + log('session=%d late track cleanup failed', sessionId); + } + } + return; + } + clearTimeout(startupTimer); + stream = granted; + const tracks = granted.getTracks(); + const deviceLost = () => finish('error', 'device-unavailable'); + for (const track of tracks) track.addEventListener('ended', deviceLost); + detachTracks = () => { + for (const track of tracks) track.removeEventListener('ended', deviceLost); + }; + if (tracks.some(track => track.readyState === 'ended')) { + deviceLost(); + return; + } + try { + const mime = pickRecorderMime(); + recorder = new MediaRecorder(granted, { + audioBitsPerSecond: 128_000, + ...(mime ? { mimeType: mime } : {}), + }); + recorder.ondataavailable = event => { + if (!ended && event.data?.size > 0) chunks.push(event.data); + }; + recorder.onstop = () => { + void finalize(); + }; + recorder.onerror = () => finish('error', 'recorder-failed'); + recorder.start(); + if (ended) return; + status = { type: 'running' }; + this.publish({ phase: 'recording', error: null }); + if (ended) return; + // Only Finish authorizes transcription; a duration limit must never + // upload a clip the user may still intend to discard. + recordingTimer = setTimeout(() => finish('error', 'timed-out'), MAX_DICTATION_RECORDING_MS); + log('session=%d recording started', sessionId); + notify(startListeners, undefined); + } catch { + finish('error', 'recorder-failed'); + } + }; + + if (!ended) { + startupTimer = setTimeout(() => finish('error', 'timed-out'), DICTATION_START_TIMEOUT_MS); + } + // listen() is synchronous; allow the composer to install all three event + // subscriptions before an unsupported browser or synchronous denial ends it. + void Promise.resolve().then(start); + + return { + /** Expose the assistant-ui session status independently of host UI phases. */ + get status() { + return status; + }, + stop, + cancel: active.cancel, + /** Receive the final transcript while the composer session is still active. */ + onSpeech(callback) { + speechListeners.add(callback); + return () => speechListeners.delete(callback); + }, + /** Receive confirmation that microphone recording actually started. */ + onSpeechStart(callback) { + startListeners.add(callback); + return () => startListeners.delete(callback); + }, + /** Receive terminal cleanup, including cancellation with an empty result. */ + onSpeechEnd(callback) { + endListeners.add(callback); + return () => endListeners.delete(callback); + }, + }; + }; +} + +/** Create an independent capture lifecycle for one chat runtime. */ +export function createOpenHumanDictationAdapter(): OpenHumanDictationAdapter { + return new OpenHumanDictationAdapter(); +} diff --git a/app/src/providers/useComposerDictation.test.tsx b/app/src/providers/useComposerDictation.test.tsx new file mode 100644 index 00000000000..14c3ccd88b5 --- /dev/null +++ b/app/src/providers/useComposerDictation.test.tsx @@ -0,0 +1,434 @@ +import { act, cleanup, renderHook, waitFor } from '@testing-library/react'; +import type { ReactNode } from 'react'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { CoreRpcError } from '../services/coreRpcClient'; +import { CoreStateContext, type CoreStateContextValue } from './coreStateContext'; +import type { DictationErrorCode, DictationSnapshot } from './dictationAdapter'; +import { useComposerDictation } from './useComposerDictation'; + +const mocks = vi.hoisted(() => ({ + callCoreRpc: vi.fn(), + captureSupported: vi.fn(), + createAdapter: vi.fn(), + invalidationListeners: new Set<() => void>(), +})); + +vi.mock('../services/coreRpcClient', () => ({ + callCoreRpc: mocks.callCoreRpc, + CoreRpcError: class extends Error { + constructor( + message: string, + readonly kind: string + ) { + super(message); + } + }, + subscribeCoreRpcTokenInvalidated: (listener: () => void) => { + mocks.invalidationListeners.add(listener); + return () => mocks.invalidationListeners.delete(listener); + }, +})); +// Capability/error projection is isolated here. dictationAdapter.fallback.test.ts +// drives the real adapter's retry, cancellation, and conversion deadlines. +vi.mock('./dictationAdapter', () => ({ + createOpenHumanDictationAdapter: mocks.createAdapter, + isDictationCaptureSupported: mocks.captureSupported, +})); + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +function makeAdapter() { + let snapshot: DictationSnapshot = { phase: 'idle', error: null }; + const listeners = new Set<() => void>(); + return { + getSnapshot: () => snapshot, + subscribe: vi.fn((listener: () => void) => { + listeners.add(listener); + return () => listeners.delete(listener); + }), + cancel: vi.fn(), + dispose: vi.fn(), + publish(next: DictationSnapshot) { + snapshot = next; + for (const listener of listeners) listener(); + }, + get subscriberCount() { + return listeners.size; + }, + }; +} + +type FakeAdapter = ReturnType; +let adapters: FakeAdapter[]; + +beforeEach(() => { + mocks.callCoreRpc.mockReset().mockResolvedValue({ stt_available: true }); + mocks.captureSupported.mockReset().mockReturnValue(true); + mocks.createAdapter.mockReset().mockImplementation(() => { + const adapter = makeAdapter(); + adapters.push(adapter); + return adapter; + }); + adapters = []; +}); + +afterEach(() => { + cleanup(); + mocks.invalidationListeners.clear(); + vi.useRealTimers(); +}); + +describe('useComposerDictation', () => { + it('retries transient probe failures with bounded exponential backoff', async () => { + vi.useFakeTimers(); + mocks.callCoreRpc.mockRejectedValue(new Error('temporary failure')); + const { result } = renderHook(() => useComposerDictation('thread-a')); + await act(async () => {}); + expect(result.current.error).toBe('voice-status-failed'); + for (const delay of [1_000, 2_000, 4_000]) { + const calls = mocks.callCoreRpc.mock.calls.length; + await act(async () => vi.advanceTimersByTimeAsync(delay - 1)); + expect(mocks.callCoreRpc).toHaveBeenCalledTimes(calls); + await act(async () => vi.advanceTimersByTimeAsync(1)); + expect(mocks.callCoreRpc).toHaveBeenCalledTimes(calls + 1); + } + await act(async () => vi.advanceTimersByTimeAsync(60_000)); + expect(mocks.callCoreRpc).toHaveBeenCalledTimes(4); + expect(result.current.adapter).toBeUndefined(); + + mocks.callCoreRpc.mockResolvedValue({ stt_available: true }); + await act(async () => window.dispatchEvent(new Event('focus'))); + expect(result.current.adapter).toBe(adapters[0]); + expect(result.current.error).toBeNull(); + }); + + it('recovers on a scheduled retry and stops retrying once available', async () => { + vi.useFakeTimers(); + mocks.callCoreRpc.mockRejectedValueOnce(new Error('core starting')); + const { result } = renderHook(() => useComposerDictation('thread-a')); + await act(async () => {}); + await act(async () => vi.advanceTimersByTimeAsync(1_000)); + expect(result.current.adapter).toBe(adapters[0]); + await act(async () => vi.advanceTimersByTimeAsync(60_000)); + expect(mocks.callCoreRpc).toHaveBeenCalledTimes(2); + }); + + it.each(['unconfigured', 'missing-method'])( + 'does not automatically retry %s but rechecks on focus', + async condition => { + vi.useFakeTimers(); + if (condition === 'missing-method') { + mocks.callCoreRpc.mockRejectedValue(new CoreRpcError('missing', 'method_not_found')); + } else { + mocks.callCoreRpc.mockResolvedValue({ stt_available: false }); + } + const { result } = renderHook(() => useComposerDictation('thread-a')); + await act(async () => {}); + await act(async () => vi.advanceTimersByTimeAsync(60_000)); + expect(result.current.error).toBe('stt-unavailable'); + expect(mocks.callCoreRpc).toHaveBeenCalledOnce(); + + mocks.callCoreRpc.mockResolvedValue({ stt_available: true }); + await act(async () => window.dispatchEvent(new Event('focus'))); + expect(result.current.adapter).toBe(adapters[0]); + expect(result.current.error).toBeNull(); + } + ); + + it('coalesces focus probes and preserves an existing capture session', async () => { + const probe = deferred<{ stt_available: boolean }>(); + mocks.callCoreRpc.mockReturnValue(probe.promise); + const { result, rerender } = renderHook(() => useComposerDictation('thread-a')); + act(() => window.dispatchEvent(new Event('focus'))); + expect(mocks.callCoreRpc).toHaveBeenCalledOnce(); + await act(async () => probe.resolve({ stt_available: true })); + act(() => adapters[0]!.publish({ phase: 'recording', error: null })); + const state = result.current; + rerender(); + expect(result.current).toBe(state); + act(() => window.dispatchEvent(new Event('focus'))); + expect(mocks.callCoreRpc).toHaveBeenCalledOnce(); + expect(adapters[0]!.cancel).not.toHaveBeenCalled(); + expect(adapters[0]!.dispose).not.toHaveBeenCalled(); + }); + + it('cancels retry timers on scope changes and all listeners on unmount', async () => { + vi.useFakeTimers(); + mocks.callCoreRpc.mockRejectedValueOnce(new Error('temporary failure')); + const { result, rerender, unmount } = renderHook(({ thread }) => useComposerDictation(thread), { + initialProps: { thread: 'thread-a' }, + }); + await act(async () => {}); + rerender({ thread: 'thread-b' }); + await act(async () => {}); + expect(result.current.adapter).toBe(adapters[0]); + await act(async () => vi.advanceTimersByTimeAsync(60_000)); + expect(mocks.callCoreRpc).toHaveBeenCalledTimes(2); + + mocks.callCoreRpc.mockRejectedValue(new Error('temporary failure')); + rerender({ thread: 'thread-c' }); + await act(async () => {}); + unmount(); + await act(async () => { + window.dispatchEvent(new Event('focus')); + await vi.advanceTimersByTimeAsync(60_000); + }); + expect(mocks.callCoreRpc).toHaveBeenCalledTimes(3); + }); + + it('fails closed until the active core confirms a usable STT provider', async () => { + const probe = deferred<{ stt_available: boolean }>(); + mocks.callCoreRpc.mockReturnValue(probe.promise); + const { result } = renderHook(() => useComposerDictation('thread-a')); + + expect(result.current.adapter).toBeUndefined(); + expect(result.current.status).toBe('checking'); + expect(mocks.createAdapter).not.toHaveBeenCalled(); + expect(mocks.callCoreRpc).toHaveBeenCalledWith({ + method: 'openhuman.voice_status', + params: {}, + suppressAuthExpiredEvent: true, + }); + await act(async () => probe.resolve({ stt_available: true })); + + expect(result.current.adapter).toBe(adapters[0]); + expect(result.current.status).toBe('idle'); + expect(result.current.error).toBeNull(); + }); + + it('does not probe or offer dictation when browser capture is unsupported', () => { + mocks.captureSupported.mockReturnValue(false); + const { result } = renderHook(() => useComposerDictation('thread-a')); + + expect(result.current.adapter).toBeUndefined(); + expect(result.current.status).toBe('unavailable'); + expect(result.current.error).toBeNull(); + expect(mocks.callCoreRpc).not.toHaveBeenCalled(); + }); + + it('does not offer capture before a thread exists', () => { + const { result } = renderHook(() => useComposerDictation(null)); + + expect(result.current.adapter).toBeUndefined(); + expect(result.current.status).toBe('unavailable'); + expect(result.current.error).toBeNull(); + expect(mocks.callCoreRpc).not.toHaveBeenCalled(); + }); + + it('withholds dictation when the configured engine is unavailable', async () => { + mocks.callCoreRpc.mockResolvedValue({ stt_available: false, stt_error: 'private detail' }); + const { result } = renderHook(() => useComposerDictation('thread-a')); + + await waitFor(() => expect(result.current.status).toBe('unavailable')); + expect(result.current.adapter).toBeUndefined(); + expect(result.current.error).toBe('stt-unavailable'); + expect(mocks.createAdapter).not.toHaveBeenCalled(); + }); + + it('fails closed on a status probe failure without exposing RPC message text', async () => { + mocks.callCoreRpc.mockRejectedValue(new Error('private provider detail')); + const { result } = renderHook(() => useComposerDictation('thread-a')); + + await waitFor(() => expect(result.current.status).toBe('unavailable')); + expect(result.current.adapter).toBeUndefined(); + expect(result.current.error).toBe('voice-status-failed'); + }); + + it('quietly withholds capability for a core built without the voice status method', async () => { + mocks.callCoreRpc.mockRejectedValue( + new CoreRpcError('unknown method: openhuman.voice_status', 'method_not_found') + ); + const { result } = renderHook(() => useComposerDictation('thread-a')); + + await waitFor(() => expect(result.current.status).toBe('unavailable')); + expect(result.current.adapter).toBeUndefined(); + expect(result.current.error).toBe('stt-unavailable'); + }); + + it('publishes capture and transcription phases', async () => { + const { result } = renderHook(() => useComposerDictation('thread-a')); + await waitFor(() => expect(result.current.adapter).toBeDefined()); + const adapter = adapters[0]!; + + for (const phase of ['starting', 'recording', 'transcribing'] as const) { + act(() => adapter.publish({ phase, error: null })); + expect(result.current.status).toBe(phase); + } + }); + + // Keep this exhaustive as the adapter adds recoverable errors. A missing voice + // domain withdraws the capability instead and is covered separately below. + const recoverableErrors = { + 'microphone-unavailable': true, + 'permission-denied': true, + 'device-unavailable': true, + 'device-in-use': true, + 'recorder-failed': true, + 'no-audio': true, + 'no-speech': true, + 'transcription-failed': true, + 'timed-out': true, + } satisfies Record, true>; + + it.each(Object.keys(recoverableErrors) as (keyof typeof recoverableErrors)[])( + 'publishes and clears %s while retaining the adapter for retry', + async error => { + const { result } = renderHook(() => useComposerDictation('thread-a')); + await waitFor(() => expect(result.current.adapter).toBeDefined()); + const adapter = adapters[0]!; + + act(() => adapter.publish({ phase: 'idle', error })); + expect(result.current.status).toBe('idle'); + expect(result.current.error).toBe(error); + expect(result.current.adapter).toBe(adapter); + expect(adapter.cancel).not.toHaveBeenCalled(); + expect(adapter.dispose).not.toHaveBeenCalled(); + + act(() => adapter.publish({ phase: 'recording', error: null })); + expect(result.current.status).toBe('recording'); + expect(result.current.error).toBeNull(); + expect(result.current.adapter).toBe(adapter); + } + ); + + it('withdraws and cancels dictation when STT discovers the voice domain is absent', async () => { + const { result } = renderHook(() => useComposerDictation('thread-a')); + await waitFor(() => expect(result.current.adapter).toBeDefined()); + const adapter = adapters[0]!; + + act(() => adapter.publish({ phase: 'idle', error: 'voice-unavailable' })); + expect(result.current.adapter).toBeUndefined(); + expect(result.current.status).toBe('unavailable'); + expect(result.current.error).toBe('voice-unavailable'); + expect(adapter.cancel).toHaveBeenCalledOnce(); + expect(adapter.dispose).toHaveBeenCalledOnce(); + expect(adapter.subscriberCount).toBe(0); + act(() => adapter.publish({ phase: 'idle', error: null })); + expect(result.current.adapter).toBeUndefined(); + }); + + it('cancels the active session on request without withdrawing the capability', async () => { + const { result } = renderHook(() => useComposerDictation('thread-a')); + await waitFor(() => expect(result.current.adapter).toBeDefined()); + + act(() => result.current.cancel()); + expect(adapters[0]!.cancel).toHaveBeenCalledOnce(); + expect(result.current.adapter).toBe(adapters[0]); + }); + + it('disposes the previous thread before probing and exposing a new adapter', async () => { + const { result, rerender } = renderHook(({ threadId }) => useComposerDictation(threadId), { + initialProps: { threadId: 'thread-a' }, + }); + await waitFor(() => expect(result.current.adapter).toBeDefined()); + const old = adapters[0]!; + const nextProbe = deferred<{ stt_available: boolean }>(); + mocks.callCoreRpc.mockReturnValueOnce(nextProbe.promise); + + rerender({ threadId: 'thread-b' }); + expect(old.dispose).toHaveBeenCalledOnce(); + expect(old.subscriberCount).toBe(0); + expect(result.current.adapter).toBeUndefined(); + expect(result.current.status).toBe('checking'); + act(() => old.publish({ phase: 'idle', error: 'voice-unavailable' })); + expect(result.current.error).toBeNull(); + await act(async () => nextProbe.resolve({ stt_available: true })); + expect(result.current.adapter).toBe(adapters[1]); + }); + + it('ignores a capability probe arriving after a thread switch', async () => { + const oldProbe = deferred<{ stt_available: boolean }>(); + const nextProbe = deferred<{ stt_available: boolean }>(); + mocks.callCoreRpc.mockReturnValueOnce(oldProbe.promise).mockReturnValueOnce(nextProbe.promise); + const { result, rerender } = renderHook(({ threadId }) => useComposerDictation(threadId), { + initialProps: { threadId: 'thread-a' }, + }); + + rerender({ threadId: 'thread-b' }); + await act(async () => oldProbe.resolve({ stt_available: true })); + expect(mocks.createAdapter).not.toHaveBeenCalled(); + expect(result.current.adapter).toBeUndefined(); + await act(async () => nextProbe.resolve({ stt_available: false })); + expect(result.current.error).toBe('stt-unavailable'); + }); + + it('reprobes after bearer invalidation even if the thread and URL are unchanged', async () => { + const { result } = renderHook(() => useComposerDictation('thread-a')); + await waitFor(() => expect(result.current.adapter).toBeDefined()); + const old = adapters[0]!; + const nextProbe = deferred<{ stt_available: boolean }>(); + mocks.callCoreRpc.mockReturnValueOnce(nextProbe.promise); + + act(() => { + for (const invalidate of mocks.invalidationListeners) invalidate(); + }); + expect(old.cancel).toHaveBeenCalledOnce(); + expect(old.dispose).toHaveBeenCalledOnce(); + expect(result.current.adapter).toBeUndefined(); + await act(async () => nextProbe.resolve({ stt_available: false })); + expect(result.current.status).toBe('unavailable'); + expect(result.current.adapter).toBeUndefined(); + expect(mocks.callCoreRpc).toHaveBeenCalledTimes(2); + }); + + it('does not reuse capability or capture across credential and profile changes', async () => { + let context = { + snapshot: { auth: { userId: 'user-a', profileId: 'profile-a' }, sessionToken: 'session-a' }, + } as unknown as CoreStateContextValue; + const wrapper = ({ children }: { children: ReactNode }) => ( + {children} + ); + const { result, rerender } = renderHook(() => useComposerDictation('thread-a'), { wrapper }); + await waitFor(() => expect(result.current.adapter).toBeDefined()); + const old = adapters[0]!; + mocks.callCoreRpc.mockResolvedValueOnce({ stt_available: false }); + context = { + snapshot: { auth: { userId: 'user-b', profileId: 'profile-b' }, sessionToken: 'session-b' }, + } as unknown as CoreStateContextValue; + + rerender(); + expect(old.dispose).toHaveBeenCalledOnce(); + expect(result.current.adapter).toBeUndefined(); + await waitFor(() => expect(result.current.error).toBe('stt-unavailable')); + }); + + it('disposes active capture if browser support becomes unavailable', async () => { + const { result, rerender } = renderHook(() => useComposerDictation('thread-a')); + await waitFor(() => expect(result.current.adapter).toBeDefined()); + mocks.captureSupported.mockReturnValue(false); + + rerender(); + expect(adapters[0]!.dispose).toHaveBeenCalledOnce(); + expect(result.current.adapter).toBeUndefined(); + expect(result.current.error).toBeNull(); + }); + + it('releases subscriptions and capture on unmount', async () => { + const { result, unmount } = renderHook(() => useComposerDictation('thread-a')); + await waitFor(() => expect(result.current.adapter).toBeDefined()); + unmount(); + + expect(adapters[0]!.dispose).toHaveBeenCalledOnce(); + expect(adapters[0]!.subscriberCount).toBe(0); + expect(mocks.invalidationListeners.size).toBe(0); + }); + + it('ignores a status probe that resolves after unmount', async () => { + const probe = deferred<{ stt_available: boolean }>(); + mocks.callCoreRpc.mockReturnValueOnce(probe.promise); + const { unmount } = renderHook(() => useComposerDictation('thread-a')); + unmount(); + + await act(async () => probe.resolve({ stt_available: true })); + expect(mocks.createAdapter).not.toHaveBeenCalled(); + }); +}); diff --git a/app/src/providers/useComposerDictation.ts b/app/src/providers/useComposerDictation.ts new file mode 100644 index 00000000000..fde6b73a000 --- /dev/null +++ b/app/src/providers/useComposerDictation.ts @@ -0,0 +1,170 @@ +import type { DictationAdapter } from '@assistant-ui/react'; +import { useCallback, useContext, useLayoutEffect, useMemo, useRef, useState } from 'react'; + +import { + callCoreRpc, + CoreRpcError, + subscribeCoreRpcTokenInvalidated, +} from '../services/coreRpcClient'; +import type { VoiceStatus } from '../utils/tauriCommands/voice'; +import { CoreStateContext } from './coreStateContext'; +import { + createOpenHumanDictationAdapter, + type DictationErrorCode, + type DictationSnapshot, + isDictationCaptureSupported, + type OpenHumanDictationAdapter, +} from './dictationAdapter'; + +export type ComposerDictationStatus = DictationSnapshot['phase'] | 'checking' | 'unavailable'; +export type ComposerDictationError = DictationErrorCode | 'stt-unavailable' | 'voice-status-failed'; + +interface ScopedDictationState { + scope: object; + adapter?: OpenHumanDictationAdapter; + status: ComposerDictationStatus; + error: ComposerDictationError | null; +} + +const PROBE_RETRY_DELAYS_MS = [1_000, 2_000, 4_000]; + +/** + * Offer dictation only after the active core confirms a usable STT provider. + * `voice_status` is a config-only read: it does not record or contact a provider, + * and its absence also identifies cores whose `voice` gate is disabled. + * + * Neither capability nor capture is shared across thread/profile/core changes. + * The existing bearer invalidation subscription covers connection preference + * changes and an embedded-core restart, even when its URL stays the same. + */ +export function useComposerDictation(threadId: string | null): { + adapter: DictationAdapter | undefined; + status: ComposerDictationStatus; + error: ComposerDictationError | null; + cancel: () => void; +} { + const core = useContext(CoreStateContext); + const [coreRevision, setCoreRevision] = useState(0); + const captureSupported = isDictationCaptureSupported(); + const userId = core?.snapshot.auth.userId; + const profileId = core?.snapshot.auth.profileId; + const sessionToken = core?.snapshot.sessionToken; + const scope = useMemo( + () => ({ threadId, userId, profileId, sessionToken, coreRevision, captureSupported }), + [threadId, userId, profileId, sessionToken, coreRevision, captureSupported] + ); + const [state, setState] = useState(null); + const activeAdapter = useRef(null); + + useLayoutEffect( + () => + subscribeCoreRpcTokenInvalidated(() => { + // Stop a hot mic before waiting for React to apply the new scope. + activeAdapter.current?.cancel(); + setCoreRevision(previous => previous + 1); + }), + [] + ); + + useLayoutEffect(() => { + if (!threadId || !captureSupported) { + setState({ scope, status: 'unavailable', error: null }); + return; + } + + let disposed = false; + let revoked = false; + let adapter: OpenHumanDictationAdapter | undefined; + let unsubscribe: (() => void) | undefined; + let retryTimer: ReturnType | undefined; + let retryCount = 0; + let probing = false; + setState({ scope, status: 'checking', error: null }); + + /** Coalesce availability reads and retry transient failures within this scope. */ + const probe = () => { + if (disposed || probing || adapter || revoked) return; + probing = true; + clearTimeout(retryTimer); + void callCoreRpc({ + method: 'openhuman.voice_status', + params: {}, + // Availability is a narrow read, not evidence that the login expired. + suppressAuthExpiredEvent: true, + }) + .then(voice => { + if (disposed) return; + if (voice?.stt_available !== true) { + setState({ scope, status: 'unavailable', error: 'stt-unavailable' }); + return; + } + adapter = createOpenHumanDictationAdapter(); + activeAdapter.current = adapter; + /** Project adapter state, withdrawing it if dispatch is unsupported. */ + const publish = () => { + if (disposed || revoked || !adapter) return; + const snapshot = adapter.getSnapshot(); + if (snapshot.error === 'voice-unavailable') { + // The STT call can discover a missing dispatch method after the + // status probe succeeded (for example with a mismatched core). + revoked = true; + adapter.cancel(); + unsubscribe?.(); + adapter.dispose(); + if (activeAdapter.current === adapter) activeAdapter.current = null; + setState({ scope, status: 'unavailable', error: 'voice-unavailable' }); + return; + } + setState({ scope, adapter, status: snapshot.phase, error: snapshot.error }); + }; + unsubscribe = adapter.subscribe(publish); + publish(); + }) + .catch(error => { + if (disposed) return; + const missingVoice = error instanceof CoreRpcError && error.kind === 'method_not_found'; + console.debug('[composer-dictation] voice availability probe failed', { + kind: missingVoice ? 'method_not_found' : 'unavailable', + }); + setState({ + scope, + status: 'unavailable', + error: missingVoice ? 'stt-unavailable' : 'voice-status-failed', + }); + if (!missingVoice && retryCount < PROBE_RETRY_DELAYS_MS.length) { + retryTimer = setTimeout(probe, PROBE_RETRY_DELAYS_MS[retryCount++]); + } + }) + .finally(() => { + probing = false; + }); + }; + + /** Recheck on focus after a failed probe or speech setup in another window. */ + const onFocus = () => { + if (probing || adapter || revoked) return; + retryCount = 0; + probe(); + }; + window.addEventListener('focus', onFocus); + probe(); + + return () => { + disposed = true; + clearTimeout(retryTimer); + window.removeEventListener('focus', onFocus); + unsubscribe?.(); + adapter?.dispose(); + if (activeAdapter.current === adapter) activeAdapter.current = null; + }; + }, [scope, threadId, captureSupported]); + + const cancel = useCallback(() => activeAdapter.current?.cancel(), []); + // A render with a different scope must withdraw the old adapter immediately, + // before effects run or a slow probe returns for the new connection/thread. + const current = state?.scope === scope ? state : null; + const adapter = current?.adapter; + const status = current?.status ?? (threadId && captureSupported ? 'checking' : 'unavailable'); + const error = current?.error ?? null; + return useMemo(() => ({ adapter, status, error, cancel }), [adapter, status, error, cancel]); +} diff --git a/app/src/providers/useOpenHumanExternalStore.ts b/app/src/providers/useOpenHumanExternalStore.ts index 24926db2519..c1e03cabcc9 100644 --- a/app/src/providers/useOpenHumanExternalStore.ts +++ b/app/src/providers/useOpenHumanExternalStore.ts @@ -2,6 +2,7 @@ import type { AddToolResultOptions, AppendMessage, ThreadMessage as AuiThreadMessage, + DictationAdapter, RespondToToolApprovalOptions, ThreadSuggestion, } from '@assistant-ui/react'; @@ -347,6 +348,7 @@ export function useOpenHumanExternalStore( threadId: string | null, { welcomeSuggestions = true, + dictationAdapter, }: { /** * Offer the home chat's starter prompts on an empty thread. Off for a @@ -354,6 +356,8 @@ export function useOpenHumanExternalStore( * a click SENDS the prompt, and those prompts are not builder requests. */ welcomeSuggestions?: boolean; + /** Configured core STT with a capture lifecycle owned by this runtime. */ + dictationAdapter?: DictationAdapter; } = {} ) { const dispatch = useAppDispatch(); @@ -686,43 +690,8 @@ export function useOpenHumanExternalStore( [threadId] ); - // DO NOT add `dictation: new WebSpeechDictationAdapter()` to the `adapters` - // key below. - // - // It is exported by `@assistant-ui/react` at our pinned 0.15.16 and looks like - // a one-line win: the transcript already renders Dictate / StopDictation - // behind `s.thread.capabilities.dictation` (`thread.tsx`), and the runtime - // derives that capability from nothing but the key's presence — - // `dictation: this._store.adapters?.dictation !== void 0`. Supplying the - // adapter would light the button up immediately. It would also trap the user. - // - // Measured in a WKWebView harness, which is the engine Wry gives us on macOS: - // - // no Info.plist `webkitSpeechRecognition` present, start() → - // onerror "service-not-allowed" - // + NSMicrophoneUsage… `webkitSpeechRecognition` present, start() → - // + NSSpeechRecognition… NO EVENT AT ALL within 6s - // - // The second row is the dangerous one. A detectable error could be caught and - // the capability withdrawn; silence cannot. The composer would enter - // `dictation != null`, never leave it, and `StopDictation` would be the only - // way out — an affordance that looks like it works, unlike the merely inert - // Edit / BranchPicker / Reload controls gated off in #5897 and #6467. - // - // This is NOT "speech is impossible here". The app already ships working - // speech-to-text by a different route: the `mic-cloud` composer captures with - // `MediaRecorder` and transcribes through the core's `voice_*` RPC - // (`features/human/voice/sttClient.ts`), with `voice` in - // `scripts/ci/product-features.txt`. Its entry point is the "Voice mode" mic - // button rendered a few pixels from the dictation gate it would duplicate. - // The desktop `Info.plist` even describes that path — it declares - // `NSMicrophoneUsageDescription` ("voice dictation") and, tellingly, no - // `NSSpeechRecognitionUsageDescription`, which only the mobile plist carries. - // - // Inline dictation into the composer is still worth having; it should reuse - // that shipped capture + transcription rather than Web Speech. Tracked - // separately — it needs a capture lifecycle, interim results, cancellation - // and error surfacing, none of which this key would provide. + // Inline dictation uses MediaRecorder and the configured core STT provider. + // The runtime appends its final transcript to the editable draft. return useMemo( () => ({ messages: runtimeMessages, @@ -752,8 +721,11 @@ export function useOpenHumanExternalStore( // without an adapter is enabled, clickable, and throws "Runtime does not // support speech." That is the #5897 defect shape, and Reload already // sits in the same trap today. - // No `dictation` key here — see the Web Speech note above this object. - adapters: { feedback: feedbackAdapter, speech: openHumanSpeechAdapter }, + adapters: { + feedback: feedbackAdapter, + speech: openHumanSpeechAdapter, + dictation: dictationAdapter, + }, }), [ runtimeMessages, @@ -762,6 +734,7 @@ export function useOpenHumanExternalStore( extras, suggestions, feedbackAdapter, + dictationAdapter, onNew, onCancel, queue, diff --git a/app/test/e2e/helpers/element-helpers.ts b/app/test/e2e/helpers/element-helpers.ts index 600446c722a..b6591bbe988 100644 --- a/app/test/e2e/helpers/element-helpers.ts +++ b/app/test/e2e/helpers/element-helpers.ts @@ -21,6 +21,8 @@ import type { ChainablePromiseElement } from 'webdriverio'; import { isTauriDriver } from './platform'; +export { webElements, type WebTestElement } from './web-elements'; + // --------------------------------------------------------------------------- // XPath helpers (macOS / Appium Mac2 path) // --------------------------------------------------------------------------- diff --git a/app/test/e2e/helpers/web-elements.ts b/app/test/e2e/helpers/web-elements.ts new file mode 100644 index 00000000000..e72025151b5 --- /dev/null +++ b/app/test/e2e/helpers/web-elements.ts @@ -0,0 +1,35 @@ +import type { Locator, Page } from '@playwright/test'; + +import { + clearChatComposer, + composerText, + replaceChatComposerText, +} from '../../playwright/helpers/chat-composer'; + +/** Keep browser selectors and platform element operations inside the helper layer. */ +function webElement(locator: Locator) { + return { + click: () => locator.click(), + press: (key: string) => locator.press(key), + type: (text: string) => locator.pressSequentially(text), + isVisible: () => locator.isVisible(), + isEnabled: () => locator.isEnabled(), + isEditable: () => locator.isEditable(), + count: () => locator.count(), + text: () => locator.textContent(), + composerText: () => composerText(locator), + clearComposer: () => clearChatComposer(locator), + replaceComposerText: (text: string) => replaceChatComposerText(locator, text), + }; +} + +export type WebTestElement = ReturnType; + +/** Playwright counterpart to the native WebView element helpers. */ +export function webElements(page: Page) { + return { + byTestId: (id: string) => webElement(page.getByTestId(id)), + button: (name: string) => webElement(page.getByRole('button', { name, exact: true })), + alert: (text: string) => webElement(page.getByRole('alert').filter({ hasText: text })), + }; +} diff --git a/app/test/e2e/specs/browser/chat-composer-dictation.spec.ts b/app/test/e2e/specs/browser/chat-composer-dictation.spec.ts new file mode 100644 index 00000000000..06d0cd27fb8 --- /dev/null +++ b/app/test/e2e/specs/browser/chat-composer-dictation.spec.ts @@ -0,0 +1,397 @@ +import { expect, test } from '@playwright/test'; + +import { + type CaptureFailure, + captureState, + createNewThread, + installFakeCapture, + mockDictationRpc, + openChat, + setCaptureFailure, + waitForSocketConnected, +} from '../../../playwright/helpers/dictation'; +import { webElements } from '../../helpers/element-helpers'; + +test.describe('Chat composer inline dictation', () => { + test('keeps the draft editable, inserts one final transcript, and waits for Send', async ({ + page, + }) => { + await installFakeCapture(page); + const { sttCalls, sentMessages } = await mockDictationRpc(page); + const input = await openChat(page, 'pw-composer-dictation-edit'); + await input.replaceComposerText('Typed first'); + + await webElements(page).button('Dictate').click(); + await expect.poll(() => webElements(page).button('Finish dictation').isVisible()).toBe(true); + await expect.poll(() => input.isEditable()).toBe(true); + await input.click(); + await input.press('End'); + await input.type(' and edited while speaking'); + await expect.poll(() => input.composerText()).toBe('Typed first and edited while speaking'); + expect(sentMessages).toEqual([]); + + await webElements(page).button('Finish dictation').click(); + const draft = 'Typed first and edited while speaking dictated final words'; + await expect.poll(() => input.composerText()).toBe(draft); + expect(sttCalls).toHaveLength(1); + expect(sttCalls[0].params.audio_base64).toBeTruthy(); + expect(sttCalls[0].params.mime_type).toBe('audio/webm'); + expect(sttCalls[0].params).not.toHaveProperty('provider'); + expect(sentMessages).toEqual([]); + await expect.poll(async () => (await captureState(page)).tracksStopped).toBe(1); + + await input.click(); + await input.press('End'); + await input.type(' — reviewed'); + await waitForSocketConnected(page); + await expect + .poll(() => webElements(page).byTestId('send-message-button').isEnabled()) + .toBe(true); + await webElements(page).byTestId('send-message-button').click(); + await expect.poll(() => sentMessages).toEqual([`${draft} — reviewed`]); + }); + + for (const sttResult of ['success', 'wav-only'] as const) { + const expectedAttempts = sttResult === 'wav-only' ? 2 : 1; + for (const cancellation of ['Discard dictation', 'Escape'] as const) { + test(`${cancellation} preserves typed text and ignores a pending ${sttResult} STT result`, async ({ + page, + }) => { + await installFakeCapture(page); + const rpc = await mockDictationRpc(page, { holdTranscript: true, sttResult }); + const input = await openChat( + page, + `pw-composer-dictation-${cancellation.split(' ')[0]}-${sttResult}` + ); + await input.replaceComposerText('Keep this draft'); + await webElements(page).button('Dictate').click(); + await expect + .poll(() => webElements(page).button('Finish dictation').isVisible()) + .toBe(true); + await webElements(page).button('Finish dictation').click(); + await expect.poll(() => rpc.sttCalls.length).toBe(expectedAttempts); + + if (cancellation === 'Escape') { + await input.click(); + await input.press('Escape'); + } else { + await webElements(page).button(cancellation).click(); + } + await expect.poll(() => webElements(page).button('Dictate').isVisible()).toBe(true); + await input.click(); + await input.press('End'); + await input.type(' and keep editing'); + await rpc.releaseTranscript('late transcript that must be ignored', expectedAttempts); + expect((await captureState(page)).sttSettled).toBe(expectedAttempts); + await input.type(' after cancellation'); + + await expect + .poll(() => input.composerText()) + .toBe('Keep this draft and keep editing after cancellation'); + await expect.poll(() => webElements(page).button('Finish dictation').count()).toBe(0); + expect(rpc.sentMessages).toEqual([]); + await expect.poll(async () => (await captureState(page)).tracksStopped).toBe(1); + }); + } + + test(`switching threads ignores the previous composer’s pending ${sttResult} transcript`, async ({ + page, + }) => { + await installFakeCapture(page); + const rpc = await mockDictationRpc(page, { holdTranscript: true, sttResult }); + await openChat(page, `pw-composer-dictation-thread-switch-${sttResult}`); + const input = webElements(page).byTestId('chat-message-input'); + await input.replaceComposerText('First thread draft'); + await webElements(page).button('Dictate').click(); + await expect.poll(() => webElements(page).button('Finish dictation').isVisible()).toBe(true); + await webElements(page).button('Finish dictation').click(); + await expect.poll(() => rpc.sttCalls.length).toBe(expectedAttempts); + + await createNewThread(page); + await expect.poll(() => webElements(page).button('Dictate').isVisible()).toBe(true); + await input.replaceComposerText('Second thread draft'); + await rpc.releaseTranscript('late words from the first thread', expectedAttempts); + expect((await captureState(page)).sttSettled).toBe(expectedAttempts); + await input.type(' checked'); + + await expect.poll(() => input.composerText()).toBe('Second thread draft checked'); + expect(rpc.sentMessages).toEqual([]); + await expect.poll(async () => (await captureState(page)).tracksStopped).toBe(1); + }); + } + + for (const capability of ['missing', 'unavailable'] as const) { + test(`hides Dictate when the core voice capability is ${capability}`, async ({ page }) => { + await installFakeCapture(page); + const rpc = await mockDictationRpc(page, { capability }); + const input = await openChat(page, `pw-composer-dictation-${capability}`); + await expect.poll(() => rpc.voiceStatusCalls.length).toBeGreaterThan(0); + await input.replaceComposerText('Typing remains available'); + + await expect.poll(() => webElements(page).button('Dictate').count()).toBe(0); + await expect.poll(() => input.isEditable()).toBe(true); + expect((await captureState(page)).permissionRequests).toBe(0); + }); + } + + test('hides Dictate when browser recording is unsupported', async ({ page }) => { + await installFakeCapture(page, { unsupported: true }); + await mockDictationRpc(page); + const input = await openChat(page, 'pw-composer-dictation-unsupported'); + + await expect.poll(() => webElements(page).button('Dictate').count()).toBe(0); + await expect.poll(() => input.isEditable()).toBe(true); + expect((await captureState(page)).permissionRequests).toBe(0); + }); + + test('explains unavailable speech and recovers on focus without switching threads', async ({ + page, + }) => { + await installFakeCapture(page); + const rpc = await mockDictationRpc(page, { capability: 'unavailable' }); + const input = await openChat(page, 'pw-composer-dictation-recovery'); + await input.replaceComposerText('Keep my draft'); + const error = webElements(page).alert('Dictation is unavailable'); + await expect.poll(() => error.isVisible()).toBe(true); + await expect.poll(() => webElements(page).button('Dictate').count()).toBe(0); + const threadUrl = page.url(); + + rpc.setCapability('available'); + await page.evaluate(() => window.dispatchEvent(new Event('focus'))); + await expect.poll(() => webElements(page).button('Dictate').isVisible()).toBe(true); + await expect.poll(() => error.count()).toBe(0); + expect(page.url()).toBe(threadUrl); + expect(await input.composerText()).toBe('Keep my draft'); + expect((await captureState(page)).permissionRequests).toBe(0); + }); + + test('permission denial shows an actionable error and keeps the draft', async ({ page }) => { + await installFakeCapture(page, { permissionDenied: true }); + const { sttCalls, sentMessages } = await mockDictationRpc(page); + const input = await openChat(page, 'pw-composer-dictation-permission'); + await input.replaceComposerText('My draft stays'); + await webElements(page).button('Dictate').click(); + + const error = webElements(page).alert('Microphone permission denied'); + await expect.poll(() => error.isVisible()).toBe(true); + await expect.poll(() => error.text()).toMatch(/permission|denied|microphone/i); + await expect.poll(() => webElements(page).button('Dictate').isEnabled()).toBe(true); + await expect.poll(() => input.isEditable()).toBe(true); + expect(await input.composerText()).toBe('My draft stays'); + expect(sttCalls).toEqual([]); + expect(sentMessages).toEqual([]); + expect((await captureState(page)).recordingsStarted).toBe(0); + }); +}); + +type FailureCase = { + capture?: CaptureFailure; + stt?: 'error' | 'empty'; + alert: string; + finish?: boolean; + timeout?: boolean; + releasedTracks: number; + sttCalls: number; +}; + +const recoverableErrors = { + 'microphone-unavailable': { + capture: 'AbortError', + alert: 'Microphone is not available', + releasedTracks: 0, + sttCalls: 0, + }, + 'permission-denied': { + capture: 'NotAllowedError', + alert: 'Microphone permission denied', + releasedTracks: 0, + sttCalls: 0, + }, + 'device-unavailable': { + capture: 'NotFoundError', + alert: 'Selected microphone is unavailable', + releasedTracks: 0, + sttCalls: 0, + }, + 'device-in-use': { + capture: 'NotReadableError', + alert: 'Microphone is in use', + releasedTracks: 0, + sttCalls: 0, + }, + 'recorder-failed': { + capture: 'recorder', + alert: 'Failed to start recorder', + releasedTracks: 1, + sttCalls: 0, + }, + 'no-audio': { + capture: 'empty', + alert: 'No audio captured', + finish: true, + releasedTracks: 1, + sttCalls: 0, + }, + 'no-speech': { + stt: 'empty', + alert: 'No speech detected', + finish: true, + releasedTracks: 1, + sttCalls: 1, + }, + 'transcription-failed': { + stt: 'error', + alert: 'Transcription failed', + finish: true, + releasedTracks: 1, + sttCalls: 2, + }, + 'timed-out': { alert: 'Dictation timed out', timeout: true, releasedTracks: 1, sttCalls: 0 }, +} satisfies Record; + +test.describe('Chat composer dictation errors and fallback', () => { + for (const [code, failure] of Object.entries(recoverableErrors)) { + test(`${code} releases capture, preserves the draft, and allows a successful retry`, async ({ + page, + }) => { + await installFakeCapture(page, { failure: failure.capture }); + const rpc = await mockDictationRpc(page, { sttResult: failure.stt }); + const input = await openChat(page, `pw-dictation-error-${code}`); + await input.replaceComposerText('Keep this draft'); + if (failure.timeout) await page.clock.install(); + await webElements(page).button('Dictate').click(); + if (failure.finish || failure.timeout) { + await expect + .poll(() => webElements(page).button('Finish dictation').isVisible()) + .toBe(true); + } + if (failure.finish) await webElements(page).button('Finish dictation').click(); + if (failure.timeout) await page.clock.fastForward(60_001); + + const alert = webElements(page).alert(failure.alert); + await expect.poll(() => alert.isVisible()).toBe(true); + expect(await alert.text()).not.toContain('Private'); + expect(await input.composerText()).toBe('Keep this draft'); + expect(await input.isEditable()).toBe(true); + await expect.poll(() => webElements(page).button('Dictate').isEnabled()).toBe(true); + expect(await webElements(page).button('Discard dictation').count()).toBe(0); + expect((await captureState(page)).tracksStopped).toBe(failure.releasedTracks); + expect(rpc.sttCalls).toHaveLength(failure.sttCalls); + expect(rpc.sentMessages).toEqual([]); + + await setCaptureFailure(page); + rpc.setSttResult('success'); + await input.click(); + await input.press('End'); + await input.type(' and edited'); + await webElements(page).button('Dictate').click(); + await expect.poll(() => webElements(page).button('Finish dictation').isVisible()).toBe(true); + await expect.poll(() => alert.count()).toBe(0); + await webElements(page).button('Finish dictation').click(); + await expect + .poll(() => input.composerText()) + .toBe('Keep this draft and edited dictated final words'); + await expect.poll(() => webElements(page).button('Dictate').isEnabled()).toBe(true); + expect((await captureState(page)).tracksStopped).toBe(failure.releasedTracks + 1); + expect(rpc.sttCalls).toHaveLength(failure.sttCalls + 1); + expect(rpc.sentMessages).toEqual([]); + }); + } + + test('a failed voice-status RPC explains availability and recovers in the same composer', async ({ + page, + }) => { + await installFakeCapture(page); + const rpc = await mockDictationRpc(page, { capability: 'error' }); + const input = await openChat(page, 'pw-dictation-status-error'); + await input.replaceComposerText('Keep this draft'); + const alert = webElements(page).alert('Could not check dictation availability'); + await expect.poll(() => alert.isVisible()).toBe(true); + expect(await alert.text()).not.toContain('Private'); + expect(await webElements(page).button('Dictate').count()).toBe(0); + expect(await input.isEditable()).toBe(true); + expect((await captureState(page)).permissionRequests).toBe(0); + expect(rpc.sttCalls).toEqual([]); + const threadUrl = page.url(); + + rpc.setCapability('available'); + await page.evaluate(() => window.dispatchEvent(new Event('focus'))); + await expect.poll(() => webElements(page).button('Dictate').isEnabled()).toBe(true); + await expect.poll(() => alert.count()).toBe(0); + expect(page.url()).toBe(threadUrl); + expect(await input.composerText()).toBe('Keep this draft'); + await webElements(page).button('Dictate').click(); + await webElements(page).button('Finish dictation').click(); + await expect.poll(() => input.composerText()).toBe('Keep this draft dictated final words'); + expect((await captureState(page)).tracksStopped).toBe(1); + expect(rpc.sentMessages).toEqual([]); + }); + + test('a missing STT method withdraws dictation, preserves the draft, and rechecks for the next thread', async ({ + page, + }) => { + await installFakeCapture(page); + const rpc = await mockDictationRpc(page, { sttResult: 'missing' }); + const input = await openChat(page, 'pw-dictation-missing-stt'); + await input.replaceComposerText('Keep this draft'); + await webElements(page).button('Dictate').click(); + await webElements(page).button('Finish dictation').click(); + const alert = webElements(page).alert('Voice transcription is not included'); + await expect.poll(() => alert.isVisible()).toBe(true); + expect(await input.composerText()).toBe('Keep this draft'); + expect(await input.isEditable()).toBe(true); + expect(await webElements(page).button('Dictate').count()).toBe(0); + expect(await webElements(page).button('Discard dictation').count()).toBe(0); + expect((await captureState(page)).tracksStopped).toBe(1); + expect(rpc.sttCalls).toHaveLength(1); + expect(rpc.sentMessages).toEqual([]); + + rpc.setSttResult('success'); + await createNewThread(page); + await input.replaceComposerText('Next thread'); + await webElements(page).button('Dictate').click(); + await expect.poll(() => alert.count()).toBe(0); + await webElements(page).button('Finish dictation').click(); + await expect.poll(() => input.composerText()).toBe('Next thread dictated final words'); + expect((await captureState(page)).tracksStopped).toBe(2); + expect(rpc.sentMessages).toEqual([]); + }); + + test('retries rejected native audio as real PCM WAV and appends the final transcript once', async ({ + page, + }) => { + await installFakeCapture(page); + const rpc = await mockDictationRpc(page, { sttResult: 'wav-only', holdTranscript: true }); + const input = await openChat(page, 'pw-dictation-wav-fallback'); + await input.replaceComposerText('Typed first'); + await webElements(page).button('Dictate').click(); + await webElements(page).button('Finish dictation').click(); + await expect.poll(() => rpc.sttCalls.length).toBe(2); + expect(await input.composerText()).toBe('Typed first'); + await rpc.releaseTranscript('dictated final words', 2); + expect((await captureState(page)).sttSettled).toBe(2); + await expect.poll(() => input.composerText()).toBe('Typed first dictated final words'); + await expect.poll(() => webElements(page).button('Dictate').isEnabled()).toBe(true); + expect(rpc.sttCalls.map(call => call.params.mime_type)).toEqual(['audio/webm', 'audio/wav']); + const native = Buffer.from(String(rpc.sttCalls[0].params.audio_base64), 'base64'); + expect(native.subarray(0, 4).toString('hex')).toBe('1a45dfa3'); // EBML/WebM header + const wav = Buffer.from(String(rpc.sttCalls[1].params.audio_base64), 'base64'); + expect(wav.toString('ascii', 0, 4)).toBe('RIFF'); + expect(wav.toString('ascii', 8, 12)).toBe('WAVE'); + expect(wav.readUInt16LE(20)).toBe(1); // PCM + expect(wav.readUInt16LE(22)).toBe(1); // mono + expect(wav.readUInt32LE(24)).toBe(16_000); + expect(wav.readUInt16LE(34)).toBe(16); + expect(wav.readUInt32LE(40)).toBe(wav.length - 44); + expect(wav.subarray(44).some(byte => byte !== 0)).toBe(true); + expect(rpc.sttCalls[1].params.file_name).toBe('audio.wav'); + expect((await captureState(page)).tracksStopped).toBe(1); + expect(rpc.sentMessages).toEqual([]); + // Editing after completion proves the composer is idle and has one result. + await input.click(); + await input.press('End'); + await input.type(' reviewed'); + expect(await input.composerText()).toBe('Typed first dictated final words reviewed'); + expect(rpc.sttCalls).toHaveLength(2); + }); +}); diff --git a/app/test/playwright/fixtures/dictation-tone.md b/app/test/playwright/fixtures/dictation-tone.md new file mode 100644 index 00000000000..3affd1638e4 --- /dev/null +++ b/app/test/playwright/fixtures/dictation-tone.md @@ -0,0 +1,12 @@ +# Dictation audio fixture + +`dictation-tone.webm` is a short synthetic 440 Hz tone recorded as WebM/Opus +using Chrome's `MediaRecorder`. A Web Audio oscillator at gain 0.1 was connected +to an `AudioContext.createMediaStreamDestination()` and stopped after 0.25 seconds. +No microphone or personal audio was used. + +The fake recorder emits these valid compressed bytes so browser tests exercise +the application's real `encodeBlobToWav` path, including browser decoding and +resampling. Tests check that a rejected native request is retried with a valid, +non-silent 16 kHz mono PCM WAV and inserts only one final transcript. Both STT +responses are mocked, and external HTTP requests are blocked. diff --git a/app/test/playwright/fixtures/dictation-tone.webm b/app/test/playwright/fixtures/dictation-tone.webm new file mode 100644 index 00000000000..768a8073190 Binary files /dev/null and b/app/test/playwright/fixtures/dictation-tone.webm differ diff --git a/app/test/playwright/helpers/dictation.ts b/app/test/playwright/helpers/dictation.ts new file mode 100644 index 00000000000..fbc5862c1c1 --- /dev/null +++ b/app/test/playwright/helpers/dictation.ts @@ -0,0 +1,330 @@ +import { expect, type Page } from '@playwright/test'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; + +// The shared E2E entry point re-exports these from web-elements.ts for browser tests. +import { webElements, type WebTestElement } from '../../e2e/helpers/element-helpers'; +import { bootAuthenticatedPage, dismissWalkthroughIfPresent } from './core-rpc'; + +interface CaptureState { + permissionRequests: number; + recordingsStarted: number; + recordingsStopped: number; + tracksStopped: number; + sttSettled: number; +} + +interface RpcCall { + method: string; + params: Record; +} + +type VoiceCapability = 'available' | 'unavailable' | 'missing' | 'error'; +export type CaptureFailure = + | 'NotAllowedError' + | 'NotFoundError' + | 'NotReadableError' + | 'AbortError' + | 'recorder' + | 'empty'; +type SttResult = 'success' | 'error' | 'missing' | 'empty' | 'wav-only'; +type CaptureWindow = Window & { + __dictationCapture: CaptureState; + __dictationFailure?: CaptureFailure; +}; + +// Real synthetic WebM/Opus data lets the production WAV encoder decode audio. +// Capture is synthetic; the AudioContext and encoder implementation remain real. +const audioBase64 = readFileSync( + fileURLToPath(new URL('../fixtures/dictation-tone.webm', import.meta.url)) +).toString('base64'); + +/** Install before bootstrap: the suite never opens an actual microphone. */ +export async function installFakeCapture( + page: Page, + options: { permissionDenied?: boolean; unsupported?: boolean; failure?: CaptureFailure } = {} +): Promise { + await page.route( + url => + ['http:', 'https:'].includes(url.protocol) && + !['127.0.0.1', 'localhost', '[::1]'].includes(url.hostname), + route => route.abort() + ); + await page.addInitScript( + ({ permissionDenied, unsupported, failure, audioBase64 }) => { + const state: CaptureState = { + permissionRequests: 0, + recordingsStarted: 0, + recordingsStopped: 0, + tracksStopped: 0, + sttSettled: 0, + }; + const captureWindow = window as unknown as CaptureWindow; + captureWindow.__dictationCapture = state; + captureWindow.__dictationFailure = permissionDenied ? 'NotAllowedError' : failure; + const audioBytes = Uint8Array.from(atob(audioBase64), char => char.charCodeAt(0)); + + // Observe STT settlement so cancellation assertions wait for the late + // response (or its abort), rather than passing before it reaches the app. + const originalFetch = window.fetch.bind(window); + window.fetch = (input, init) => { + let isStt = false; + if (typeof init?.body === 'string') { + try { + isStt = JSON.parse(init.body).method === 'openhuman.voice_stt_dispatch'; + } catch {} + } + const response = originalFetch(input, init); + if (!isStt) return response; + return response.then( + reply => { + const originalJson = reply.json.bind(reply); + reply.json = () => originalJson().finally(() => (state.sttSettled += 1)); + return reply; + }, + error => { + state.sttSettled += 1; + throw error; + } + ); + }; + + const mediaDevices = new EventTarget(); + Object.assign(mediaDevices, { + enumerateDevices: async () => [], + getUserMedia: async () => { + state.permissionRequests += 1; + const failure = captureWindow.__dictationFailure; + if (failure?.endsWith('Error')) { + throw new DOMException('Private capture failure detail', failure); + } + const track = Object.assign(new EventTarget(), { + kind: 'audio', + enabled: true, + readyState: 'live', + stop() { + if (this.readyState === 'ended') return; + this.readyState = 'ended'; + state.tracksStopped += 1; + }, + }); + return { active: true, getTracks: () => [track], getAudioTracks: () => [track] }; + }, + }); + Object.defineProperty(navigator, 'mediaDevices', { configurable: true, value: mediaDevices }); + + class FakeMediaRecorder extends EventTarget { + static isTypeSupported(mime: string): boolean { + return mime.startsWith('audio/webm'); + } + + state = 'inactive'; + mimeType: string; + ondataavailable: ((event: BlobEvent) => void) | null = null; + onstop: ((event: Event) => void) | null = null; + onerror: ((event: Event) => void) | null = null; + + constructor(_stream: MediaStream, options?: MediaRecorderOptions) { + super(); + this.mimeType = options?.mimeType ?? 'audio/webm'; + } + + start(): void { + if (captureWindow.__dictationFailure === 'recorder') { + throw new Error('Private recorder failure detail'); + } + this.state = 'recording'; + state.recordingsStarted += 1; + } + + stop(): void { + if (this.state === 'inactive') return; + this.state = 'inactive'; + state.recordingsStopped += 1; + queueMicrotask(() => { + const data = new BlobEvent('dataavailable', { + data: new Blob(captureWindow.__dictationFailure === 'empty' ? [] : [audioBytes], { + type: this.mimeType, + }), + }); + this.dispatchEvent(data); + this.ondataavailable?.(data); + const stopped = new Event('stop'); + this.dispatchEvent(stopped); + this.onstop?.(stopped); + }); + } + } + + Object.defineProperty(window, 'MediaRecorder', { + configurable: true, + value: unsupported ? undefined : FakeMediaRecorder, + }); + }, + { ...options, audioBase64 } + ); +} + +/** Repair the fake device without remounting the composer or replacing its draft. */ +export async function setCaptureFailure(page: Page, failure?: CaptureFailure): Promise { + await page.evaluate(failure => { + (window as unknown as CaptureWindow).__dictationFailure = failure; + }, failure); +} + +/** Read capture cleanup and RPC settlement counters from the browser fixture. */ +export async function captureState(page: Page): Promise { + return page.evaluate( + () => (window as unknown as { __dictationCapture: CaptureState }).__dictationCapture + ); +} + +/** Control speech replies while keeping authentication and other RPCs on the local core. */ +export async function mockDictationRpc( + page: Page, + options: { capability?: VoiceCapability; holdTranscript?: boolean; sttResult?: SttResult } = {} +): Promise<{ + voiceStatusCalls: RpcCall[]; + sttCalls: RpcCall[]; + sentMessages: string[]; + setCapability: (capability: VoiceCapability) => void; + setSttResult: (result: SttResult) => void; + releaseTranscript: (text: string, expectedAttempts: number) => Promise; +}> { + let capability = options.capability ?? 'available'; + let sttResult = options.sttResult ?? 'success'; + const voiceStatusCalls: RpcCall[] = []; + const sttCalls: RpcCall[] = []; + const sentMessages: string[] = []; + let resolveTranscript: ((text: string) => void) | undefined; + const transcript = options.holdTranscript + ? new Promise(resolve => (resolveTranscript = resolve)) + : Promise.resolve('dictated final words'); + + await page.route('**/rpc', async (route, request) => { + const body = JSON.parse(request.postData() || '{}'); + const fulfill = (result: unknown) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ jsonrpc: '2.0', id: body.id, result }), + }); + const reject = (code: number, message: string) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ jsonrpc: '2.0', id: body.id, error: { code, message } }), + }); + + if (body.method === 'openhuman.voice_status') { + voiceStatusCalls.push({ method: body.method, params: body.params }); + if (capability === 'missing' || capability === 'error') { + await reject( + capability === 'missing' ? -32601 : -32603, + capability === 'missing' + ? 'unknown method: openhuman.voice_status' + : 'Private status detail' + ); + } else { + await fulfill({ + stt_available: capability !== 'unavailable', + tts_available: true, + stt_engine: 'hosted', + stt_error: null, + }); + } + return; + } + if (body.method === 'openhuman.voice_stt_dispatch') { + sttCalls.push({ method: body.method, params: body.params }); + if (sttResult === 'missing') { + await reject(-32601, 'unknown method: openhuman.voice_stt_dispatch'); + return; + } + if ( + sttResult === 'error' || + (sttResult === 'wav-only' && body.params.mime_type !== 'audio/wav') + ) { + await reject(-32603, 'Private transcription failure detail'); + return; + } + const text = await transcript; + // A canceled fetch may have closed its route while the result was held. + await fulfill({ + text: sttResult === 'empty' ? ' ' : text, + provider: 'configured-stt', + }).catch(() => {}); + return; + } + if (body.method === 'openhuman.channel_web_chat') { + sentMessages.push(String(body.params.message)); + await fulfill({ accepted: true }); + return; + } + await route.fallback(); + }); + + return { + voiceStatusCalls, + sttCalls, + sentMessages, + setCapability: next => { + capability = next; + }, + setSttResult: next => { + sttResult = next; + }, + releaseTranscript: async (text, expectedAttempts) => { + // Wait for the held request to exist before releasing it. A rejected native + // request can already be settled while its WAV retry is still in flight. + await expect.poll(() => sttCalls.length).toBe(expectedAttempts); + resolveTranscript?.(text); + await expect.poll(async () => (await captureState(page)).sttSettled).toBe(expectedAttempts); + }, + }; +} + +/** Bootstrap a mock user and open an empty thread with an editable composer. */ +export async function openChat(page: Page, userId: string): Promise { + await bootAuthenticatedPage(page, userId, '/chat'); + await dismissWalkthroughIfPresent(page); + const input = webElements(page).byTestId('chat-message-input'); + await expect.poll(() => input.isVisible()).toBe(true); + await createNewThread(page); + await expect.poll(() => input.isVisible()).toBe(true); + await input.clearComposer(); + return input; +} + +/** Wait for the local core socket before asserting that Send is enabled. */ +export async function waitForSocketConnected(page: Page): Promise { + await expect + .poll(() => + page.evaluate(() => { + const store = ( + window as unknown as { + __OPENHUMAN_STORE__?: { + getState: () => { socket?: { byUser?: Record } }; + }; + } + ).__OPENHUMAN_STORE__; + return Object.values(store?.getState().socket?.byUser ?? {}).some( + entry => entry.status === 'connected' + ); + }) + ) + .toBe(true); +} + +/** Start a thread through the shared UI helpers and wait for its route and composer. */ +export async function createNewThread(page: Page): Promise { + const before = page.url(); + const sidebarButton = webElements(page).byTestId('new-thread-sidebar-button'); + if (await sidebarButton.isVisible().catch(() => false)) { + await sidebarButton.click(); + } else { + await webElements(page).byTestId('new-thread-button').click(); + } + await expect.poll(() => page.url()).not.toBe(before); + await expect.poll(() => webElements(page).byTestId('chat-message-input').isVisible()).toBe(true); +} diff --git a/app/test/wdio.conf.ts b/app/test/wdio.conf.ts index 62be0626dae..cec9c0e4e9a 100644 --- a/app/test/wdio.conf.ts +++ b/app/test/wdio.conf.ts @@ -84,6 +84,8 @@ export const config: Options.Testrunner & Record = { port: parseInt(process.env.TAURI_DRIVER_PORT || '4444', 10), path: '/', specs: [testSpecsPath], + // Browser-only mocked flows run through playwright.config.ts, not the native driver. + exclude: [path.join(projectRoot, 'test', 'e2e', 'specs', 'browser', '**', '*.spec.ts')], rootDir: projectRoot, // `tauri-driver` owns the native WebKit display lifecycle. WDIO's automatic // `xvfb-run` wrapper drops the local-runner IPC descriptor, so workers exit diff --git a/crates/openhuman-core/src/platform/about_app/catalog_conversation_intelligence.rs b/crates/openhuman-core/src/platform/about_app/catalog_conversation_intelligence.rs index 2f9c9c8c3b3..49263f95e0a 100644 --- a/crates/openhuman-core/src/platform/about_app/catalog_conversation_intelligence.rs +++ b/crates/openhuman-core/src/platform/about_app/catalog_conversation_intelligence.rs @@ -48,8 +48,8 @@ Capability { name: "Send Voice Messages", domain: "conversation", category: CapabilityCategory::Conversation, - description: "Record or attach voice input and send it as a message.", - how_to: "Conversations > Voice input", + description: "Dictate into the message composer, edit the transcript, and send when ready. Voice mode records and sends a voice message.", + how_to: "Conversations > Dictate or Voice mode", status: CapabilityStatus::Beta, privacy: DERIVED_TO_BACKEND, }, diff --git a/docs/RELEASE-MANUAL-SMOKE.md b/docs/RELEASE-MANUAL-SMOKE.md index 023788306fe..14e8c89c371 100644 --- a/docs/RELEASE-MANUAL-SMOKE.md +++ b/docs/RELEASE-MANUAL-SMOKE.md @@ -27,6 +27,14 @@ Applies to every release, all platforms. - [ ] **A failed chat turn does not offer an invalid regenerate action** — Trigger a provider failure in a test profile and inspect its error card. Expected: the diagnostic text remains visible, with no Retry or Refresh button on that failed message. A completed assistant reply still offers Refresh (#6613). - [ ] **A failed turn leaves its thread usable** — In a test profile, get one successful reply, trigger a streamed provider failure on the next turn, then send another message in the same thread. Expected: the error card appears, the composer re-enables, the next reply streams normally, and the agent still has the first turn's context. If a queued follow-up starts as the failed turn ends, its stream and composer state stay active. +### Editable composer dictation + +- [ ] **The recording limit does not upload audio**: Leave dictation recording for one minute without selecting Finish. Confirm capture stops, a timeout appears, and the draft remains unchanged without a transcription request. + +- [ ] **Dictation remains an editable draft on Windows, macOS, and Linux**: On Chat, select Dictate and grant microphone permission. Edit the draft while speaking, select Finish dictation, and correct the appended transcript before selecting Send. Confirm the microphone indicator clears when recording finishes. +- [ ] **Discard and navigation release microphone capture**: Start dictation, then discard it or press Escape. Repeat while transcription is pending and while switching threads or opening Voice mode. Confirm the draft stays intact and the microphone indicator clears. +- [ ] **Permission denial and missing voice support stay usable**: Deny microphone access and confirm a localized error with an editable draft. In a build with the voice feature disabled, confirm the Dictate control stays hidden. + ### Refreshed application UI - [ ] **Settings, Connections, theme, and workflow surfaces remain usable** — On desktop, navigate between Settings sections and Connections tabs, switch between light/dark and at least one named theme family, export and re-import a custom theme, then open a workflow canvas and its run history. Expected: each route renders its own content without stale panels or horizontal overflow; imported themes can be selected and theme selection and run details survive a reload where persistence is expected. diff --git a/docs/TEST-COVERAGE-MATRIX.md b/docs/TEST-COVERAGE-MATRIX.md index bde4641cdb2..5edb749b52b 100644 --- a/docs/TEST-COVERAGE-MATRIX.md +++ b/docs/TEST-COVERAGE-MATRIX.md @@ -246,6 +246,7 @@ End-to-end coverage of the agent harness via the web-chat RPC surface against an | 5.1.2 | Speech-to-Text Processing | WD | `voice-mode.spec.ts` | ✅ | | | 5.1.3 | Voice Command Execution | WD | `voice-mode.spec.ts` | ✅ | | | 5.1.4 | Mascot Voice Selection | VU | `app/src/store/__tests__/mascotSlice.test.ts`, `app/src/components/settings/panels/__tests__/VoicePanel.test.tsx`, `app/src/features/human/useHumanMascot.test.ts` | ✅ | Slice validation + persist REHYDRATE, Settings picker UI (#1762), `synthesizeSpeech` voiceId override propagation | +| 5.1.5 | Editable Composer Dictation | VU+PW+MS | `app/src/providers/dictationAdapter.test.ts`, `app/src/providers/dictationAdapter.fallback.test.ts`, `app/src/providers/useComposerDictation.test.tsx`, `app/src/components/assistant-ui/composer-dictation.test.tsx`, `app/test/e2e/specs/browser/chat-composer-dictation.spec.ts` | ✅ | Final transcript appends to an editable draft; fake capture covers all error alerts, retry recovery, cancellation and late native/WAV replies. Real WebM decoding and PCM WAV retry run through the browser. Native microphone grants are in release smoke. | ### 5.2 Persona diff --git a/gitbooks/features/native-tools/voice.md b/gitbooks/features/native-tools/voice.md index 00e05bf17c5..054e282b9f5 100644 --- a/gitbooks/features/native-tools/voice.md +++ b/gitbooks/features/native-tools/voice.md @@ -19,6 +19,16 @@ OpenHuman is voice-first when you want it to be. STT, TTS, and the live Google M Dictation can replace the active text input on your desktop, or be sent straight into a chat with the agent. +## Dictate into a chat draft + +On Chat, select **Dictate** to record into the text composer. You can keep editing while recording. Select **Finish dictation** to transcribe the clip through your configured speech provider and append one final transcript to the draft. Review or edit it, then select **Send**. + +Select Finish within one minute. At that limit, capture stops and the clip is discarded with a timeout message; it is not uploaded or transcribed. + +**Discard dictation** or Escape discards the recording or pending transcript. Switching threads, switching to Voice mode, or leaving the composer cancels dictation. The microphone stops when recording ends. Dictation appears when microphone capture and the core's speech provider are available. + +**Voice mode**, shown with a waveform icon, opens the microphone composer and sends its transcript as a conversation message. **Dictate**, shown with a microphone icon, adds text to your editable draft. + ## Text-to-speech Reply speech routes through a hosted TTS model. The agent's responses can be spoken back in a voice you pick, with natural timing and prosody. Voice selection is configurable per user, and the mascot avatar lip-syncs to the audio stream via a viseme map.