Skip to content

Commit d4d3387

Browse files
committed
## Speech-to-text dictation
Adds voice dictation to Zero. Run `/voice` to enter hold-to-record mode, then hold **Space** to dictate and release to transcribe (run `/voice` again to turn it off so Space types normally). Pick a backend with `/stt-model`. ### Backends - **Local (sherpa-onnx)** — offline, no API key. Auto-downloads a model via `/stt-model`; the engine binary is fetched once and shared across models. Family auto-detection covers the mainstream offline families (Whisper, transducer/Zipformer, Moonshine, Paraformer, SenseVoice, and the CTC families FireRedASR, NeMo, TeleSpeech, WeNet, Zipformer-CTC, Dolphin, Omnilingual — plus the encoder/decoder families Canary and FireRedASR-AED). - **Cloud batch** — Groq and OpenAI Whisper. Keys resolve from a configured provider, the encrypted credential store, or the env var; a missing/invalid key is entered inline. - **Streaming** — a warm local sherpa-onnx websocket server (live transcript as you speak) on desktop, or Deepgram / OpenAI Realtime. ### Highlights - Live audio-reactive waveform while recording (real mic levels when streaming; a synthesized wave for batch). - `/stt-model` download chooser: recommended models first, Live/Batch tag per row, size + download state, and a search filter. - Streaming transcript is inserted live into the composer and committed on stop; batch inserts on stop. Insert-for-review by default (`stt.autoSubmit` to fire straight at the agent). - The warm streaming server is released when `/voice` is turned off and on exit. ### Implementation notes - No CGO and no bundled runtimes — every OS integration shells out via `exec.Command` with discrete argv (never a shell string). - The streaming server manager mirrors the LSP manager's warm-subprocess lifecycle. - OS-boundary code (recorders, transcribers, server) is unit-tested via injected fakes; real microphone / websocket behavior is covered by env-gated smoke tests. ### Testing - `go build ./...`, `go vet ./...`, and the config / dictation / cli / tui suites pass. - Verified end-to-end against real downloaded binaries + models (offline batch, streaming websocket server, and the model auto-download), including the FireRedASR-CTC and NeMo streaming transducer families. <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit * **New Features** * Added speech-to-text dictation with both batch and streaming modes, supporting local (sherpa-onnx) and cloud providers, including WAV/M4A detection. * Introduced voice dictation mode with STT model selection, local engine/model auto-download, and an inline prompt to save cloud API keys. * Added new TUI commands for voice/STT model selection and enabled F1–F12 key bindings. * **Bug Fixes** * Improved streaming dictation to preserve accumulated transcript partials and keep the editor region consistent. * Refined TUI status/picker rendering, including clearer loading/search behavior and more reliable selection alignment. <!-- end of auto-generated comment: release notes by coderabbit.ai -->
1 parent 008bc9b commit d4d3387

55 files changed

Lines changed: 8283 additions & 18 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ require (
1414
github.com/charmbracelet/colorprofile v0.4.3
1515
github.com/charmbracelet/x/ansi v0.11.7
1616
github.com/charmbracelet/x/term v0.2.2
17+
github.com/coder/websocket v1.8.15
1718
github.com/ledongthuc/pdf v0.0.0-20250511090121-5959a4027728
1819
golang.org/x/sys v0.46.0
1920
mvdan.cc/sh/v3 v3.13.1

‎go.sum‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSE
3232
github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0=
3333
github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk=
3434
github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
35+
github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA=
36+
github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg=
3537
github.com/dlclark/regexp2/v2 v2.2.1 h1:mf4KkFUj0gJuarK8P+LgiS+Lit7m9N1yAwEfPbee7R0=
3638
github.com/dlclark/regexp2/v2 v2.2.1/go.mod h1:avUrQvPaLz2DrFNHJF0taWAFFX2C1GMSSoeiqFjcBmU=
3739
github.com/go-quicktest/qt v1.102.0 h1:HSQxCeh5YZH3EL3W39ixjtyaEhcWSXQHtHnMBzSs474=

‎internal/cli/app.go‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -729,6 +729,16 @@ func runInteractiveTUIWithSetup(stderr io.Writer, deps appDeps, permissionMode a
729729
Scope: scope,
730730
})
731731
lastKnownMCPConfig := mcpConfig
732+
sttServerManager := newDictationServerManager(resolved.STT)
733+
// Keep STT downloads in the SAME config tree the rest of the TUI uses. Deriving
734+
// from userConfigPath (rather than config.UserConfigDir()) matters when the config
735+
// root is overridden — e.g. in tests or a custom ZERO config dir — so the two
736+
// don't diverge. userConfigPath points at .../zero/config.json, so its dir is the
737+
// zero config dir.
738+
sttDownloadRoot := ""
739+
if userConfigPath != "" {
740+
sttDownloadRoot = filepath.Join(filepath.Dir(userConfigPath), "stt")
741+
}
732742
return deps.runTUI(context.Background(), tui.Options{
733743
Cwd: workspaceRoot,
734744
Version: version,
@@ -792,9 +802,15 @@ func runInteractiveTUIWithSetup(stderr io.Writer, deps appDeps, permissionMode a
792802
merged, _ := plugins.MergedSkillsLoaded(deps.skillsDir(), pluginActivation.skillRoots)
793803
return merged
794804
}),
795-
PermissionMode: permissionMode,
796-
Notify: resolved.Notify,
797-
KeyBindings: resolved.KeyBindings,
805+
PermissionMode: permissionMode,
806+
Notify: resolved.Notify,
807+
KeyBindings: resolved.KeyBindings,
808+
STT: resolved.STT,
809+
BuildDictationTranscriber: newDictationTranscriberFactory(resolved, userConfigPath, sttServerManager),
810+
ShutdownDictationServer: sttServerManager.Shutdown,
811+
STTDownloadRoot: sttDownloadRoot,
812+
STTKeyStatus: newSTTKeyStatus(resolved, userConfigPath),
813+
SaveSTTKey: newSaveSTTKey(userConfigPath),
798814
Setup: tui.SetupOptions{
799815
Visible: setupVisible,
800816
Required: needsSetup,

‎internal/cli/dictation.go‎

Lines changed: 248 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,248 @@
1+
package cli
2+
3+
import (
4+
"fmt"
5+
neturl "net/url"
6+
"os"
7+
"path/filepath"
8+
"strings"
9+
10+
"github.com/Gitlawb/zero/internal/config"
11+
"github.com/Gitlawb/zero/internal/dictation"
12+
"github.com/Gitlawb/zero/internal/tui"
13+
)
14+
15+
// Default batch cloud models (§8). Overridable via stt.model.
16+
const (
17+
defaultGroqSTTModel = "whisper-large-v3-turbo"
18+
defaultOpenAISTTModel = "whisper-1"
19+
groqSTTBaseURL = "https://api.groq.com/openai/v1"
20+
)
21+
22+
// newDictationTranscriberFactory returns the closure the TUI calls to build a
23+
// transcriber when a recording starts. It lives here, not in the TUI, because it
24+
// resolves provider API keys (credstore + env + configured providers) — logic
25+
// that belongs with the rest of the CLI's credential handling.
26+
//
27+
// serverMgr is the shared, session-long sherpa-onnx streaming server; it is
28+
// spawned lazily on the first local-streaming recording and torn down on exit.
29+
//
30+
// preferStreaming asks for the streaming backend; the returned bool reports
31+
// whether streaming was actually built (false → the caller uses the batch
32+
// pipeline). A misconfigured streaming backend surfaces its own SetupError.
33+
func newDictationTranscriberFactory(resolved config.ResolvedConfig, userConfigPath string, serverMgr *dictation.ServerManager) func(config.STTConfig, bool) (tui.Transcriber, bool, error) {
34+
return func(stt config.STTConfig, preferStreaming bool) (tui.Transcriber, bool, error) {
35+
// stt is passed in (not captured) so a mid-session change — e.g. the F9
36+
// auto-download writing localModelPath — takes effect on the next call.
37+
if serverMgr != nil {
38+
serverMgr.SetModelPath(stt.LocalModelPath)
39+
}
40+
if preferStreaming {
41+
t, err := buildStreamingTranscriber(stt, resolved, userConfigPath, serverMgr)
42+
switch {
43+
case err == nil && t != nil:
44+
return t, true, nil
45+
case err != nil && stt.STTStreamProvider() != config.STTProviderLocal:
46+
// An explicitly-chosen cloud streaming backend that failed to build
47+
// (e.g. a missing key) should surface its own error, not silently
48+
// fall back to batch — the user asked for that backend specifically.
49+
return nil, true, err
50+
}
51+
// The default local streaming backend isn't set up: fall through to the
52+
// batch pipeline so a cloud-batch provider (or a batch-only setup) still
53+
// works instead of erroring on a streaming server the user never asked for.
54+
}
55+
t, err := buildBatchTranscriber(stt, resolved, userConfigPath)
56+
return t, false, err
57+
}
58+
}
59+
60+
// buildStreamingTranscriber constructs the configured streaming backend, or
61+
// returns (nil, nil) when streaming is not applicable so the caller falls back
62+
// to batch.
63+
func buildStreamingTranscriber(stt config.STTConfig, resolved config.ResolvedConfig, userConfigPath string, serverMgr *dictation.ServerManager) (dictation.Transcriber, error) {
64+
switch stt.STTStreamProvider() {
65+
case config.STTProviderDeepgram:
66+
key := resolveSTTKey(resolved, userConfigPath, dictation.StreamProviderDeepgram, "DEEPGRAM_API_KEY")
67+
return dictation.NewDeepgramTranscriber(dictation.DeepgramConfig{
68+
APIKey: key,
69+
Model: stt.StreamModel,
70+
Language: stt.Language,
71+
})
72+
case config.STTProviderOpenAI:
73+
key := resolveSTTKey(resolved, userConfigPath, dictation.ProviderOpenAI, "OPENAI_API_KEY")
74+
return dictation.NewOpenAIRealtimeTranscriber(dictation.OpenAIRealtimeConfig{
75+
APIKey: key,
76+
Model: stt.StreamModel,
77+
})
78+
default: // local streaming via the warm sherpa-onnx server
79+
// STTStreamProvider() defaults to "local" whenever streamProvider is unset —
80+
// including when the user picked a CLOUD batch provider (Groq/OpenAI) and
81+
// never touched streaming. Don't silently stream through the local sherpa
82+
// server in that case: local streaming is only appropriate when it was
83+
// explicitly requested (streamProvider: "local") OR the batch provider is
84+
// itself local. Otherwise report "not applicable" so the factory falls back
85+
// to the cloud batch pipeline the user actually chose.
86+
localWanted := stt.StreamProvider == config.STTProviderLocal || stt.STTProvider() == config.STTProviderLocal
87+
if !localWanted {
88+
return nil, nil
89+
}
90+
// The local streaming transcriber validates the model only lazily (at
91+
// connect time), so gate on the model path here — with none set, report
92+
// "not applicable" (nil, nil) so the factory falls back to the batch
93+
// pipeline instead of committing to a server that can't start.
94+
if serverMgr == nil || strings.TrimSpace(stt.LocalModelPath) == "" {
95+
return nil, nil
96+
}
97+
return dictation.NewLocalStreamingTranscriber(serverMgr), nil
98+
}
99+
}
100+
101+
// newDictationServerManager builds the warm sherpa-onnx streaming server manager
102+
// from config. Construction is side-effect free — the server spawns lazily on
103+
// the first local-streaming recording — so it is always safe to create.
104+
func newDictationServerManager(stt config.STTConfig) *dictation.ServerManager {
105+
return dictation.NewServerManager(dictation.ServerConfig{
106+
Binary: stt.LocalServerBinary,
107+
ModelPath: stt.LocalModelPath,
108+
Port: stt.LocalServerPort,
109+
NumThreads: stt.NumThreads,
110+
})
111+
}
112+
113+
// buildBatchTranscriber constructs the configured batch transcriber, resolving
114+
// credentials for the cloud providers.
115+
func buildBatchTranscriber(stt config.STTConfig, resolved config.ResolvedConfig, userConfigPath string) (dictation.Transcriber, error) {
116+
switch stt.STTProvider() {
117+
case config.STTProviderGroq:
118+
key := resolveSTTKey(resolved, userConfigPath, dictation.ProviderGroq, "GROQ_API_KEY")
119+
model := stt.Model
120+
if model == "" {
121+
model = defaultGroqSTTModel
122+
}
123+
return dictation.NewCloudTranscriber(dictation.CloudConfig{
124+
Provider: dictation.ProviderGroq,
125+
BaseURL: groqSTTBaseURL,
126+
APIKey: key,
127+
Model: model,
128+
Language: stt.Language,
129+
})
130+
case config.STTProviderOpenAI:
131+
key := resolveSTTKey(resolved, userConfigPath, dictation.ProviderOpenAI, "OPENAI_API_KEY")
132+
model := stt.Model
133+
if model == "" {
134+
model = defaultOpenAISTTModel
135+
}
136+
return dictation.NewCloudTranscriber(dictation.CloudConfig{
137+
Provider: dictation.ProviderOpenAI,
138+
BaseURL: config.OpenAIBaseURL,
139+
APIKey: key,
140+
Model: model,
141+
Language: stt.Language,
142+
})
143+
default: // local
144+
return dictation.NewLocalTranscriber(dictation.LocalConfig{
145+
Binary: stt.LocalBinary,
146+
ModelPath: stt.LocalModelPath,
147+
NumThreads: stt.NumThreads,
148+
})
149+
}
150+
}
151+
152+
// sttEnvVar returns the conventional environment variable a cloud STT provider's
153+
// key is read from.
154+
func sttEnvVar(provider string) string {
155+
switch provider {
156+
case dictation.ProviderGroq:
157+
return "GROQ_API_KEY"
158+
case dictation.ProviderOpenAI: // == StreamProviderOpenAI ("openai")
159+
return "OPENAI_API_KEY"
160+
case dictation.StreamProviderDeepgram:
161+
return "DEEPGRAM_API_KEY"
162+
}
163+
return ""
164+
}
165+
166+
// newSTTKeyStatus returns a predicate reporting whether a cloud STT provider
167+
// already has a resolvable API key (configured provider, credstore, or env).
168+
func newSTTKeyStatus(resolved config.ResolvedConfig, userConfigPath string) func(string) bool {
169+
return func(provider string) bool {
170+
return resolveSTTKey(resolved, userConfigPath, provider, sttEnvVar(provider)) != ""
171+
}
172+
}
173+
174+
// newSaveSTTKey returns a function that stores a cloud STT provider's API key in
175+
// the encrypted credential store, so the inline prompt can persist it.
176+
func newSaveSTTKey(userConfigPath string) func(string, string) error {
177+
return func(provider, key string) error {
178+
if strings.TrimSpace(userConfigPath) == "" {
179+
return fmt.Errorf("no config path to store the key")
180+
}
181+
store, err := config.ProviderKeyStoreAt(filepath.Dir(userConfigPath))
182+
if err != nil {
183+
return err
184+
}
185+
return store.Set(provider, strings.TrimSpace(key))
186+
}
187+
}
188+
189+
// resolveSTTKey finds an API key for an STT cloud provider, reusing Zero's
190+
// existing credentials (§2 — "zero new credential UI"). Precedence: a matching
191+
// configured provider's resolved key, then the encrypted credential store, then
192+
// the provider's conventional environment variable.
193+
func resolveSTTKey(resolved config.ResolvedConfig, userConfigPath, provider, envVar string) string {
194+
if key := keyFromConfiguredProviders(resolved, provider); key != "" {
195+
return key
196+
}
197+
if userConfigPath != "" {
198+
if store, err := config.ProviderKeyStoreAt(filepath.Dir(userConfigPath)); err == nil {
199+
if key, ok, _ := store.Get(provider); ok && strings.TrimSpace(key) != "" {
200+
return strings.TrimSpace(key)
201+
}
202+
}
203+
}
204+
return strings.TrimSpace(os.Getenv(envVar))
205+
}
206+
207+
// keyFromConfiguredProviders returns the resolved API key of a configured
208+
// provider that belongs to the given cloud provider (matched by catalog id,
209+
// provider name/kind, or base-URL host), or "" when none is configured.
210+
func keyFromConfiguredProviders(resolved config.ResolvedConfig, provider string) string {
211+
provider = strings.ToLower(provider)
212+
profiles := append([]config.ProviderProfile{resolved.Provider}, resolved.Providers...)
213+
for _, p := range profiles {
214+
if strings.TrimSpace(p.APIKey) == "" {
215+
continue
216+
}
217+
if providerMatches(p, provider) {
218+
return strings.TrimSpace(p.APIKey)
219+
}
220+
}
221+
return ""
222+
}
223+
224+
func providerMatches(p config.ProviderProfile, provider string) bool {
225+
provider = strings.ToLower(provider)
226+
// Match on stable identifiers (catalog id, provider name/kind, profile
227+
// name) rather than raw BaseURL substring matching: a Groq profile's
228+
// BaseURL (api.groq.com/openai/v1) contains "openai", so a substring
229+
// check can hand the wrong key to an OpenAI STT lookup and fail auth.
230+
for _, f := range []string{
231+
strings.ToLower(p.CatalogID),
232+
strings.ToLower(p.Provider),
233+
strings.ToLower(string(p.ProviderKind)),
234+
strings.ToLower(p.Name),
235+
} {
236+
if f != "" && strings.Contains(f, provider) {
237+
return true
238+
}
239+
}
240+
// BaseURL: compare the parsed host only, not a raw substring on the
241+
// full URL (which lets "openai" match api.groq.com/openai/v1).
242+
if u, err := neturl.Parse(p.BaseURL); err == nil && u.Host != "" {
243+
if host := strings.ToLower(u.Host); strings.Contains(host, provider) {
244+
return true
245+
}
246+
}
247+
return false
248+
}

‎internal/cli/dictation_test.go‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
package cli
2+
3+
import (
4+
"errors"
5+
"testing"
6+
7+
"github.com/Gitlawb/zero/internal/config"
8+
"github.com/Gitlawb/zero/internal/dictation"
9+
)
10+
11+
func TestFactoryFallsBackToBatchWhenLocalStreamingNotConfigured(t *testing.T) {
12+
t.Setenv("GROQ_API_KEY", "gsk-test")
13+
resolved := config.ResolvedConfig{STT: config.STTConfig{
14+
Provider: config.STTProviderGroq, // cloud batch
15+
// StreamProvider unset → defaults to local, which has no model configured.
16+
}}
17+
factory := newDictationTranscriberFactory(resolved, "", newDictationServerManager(resolved.STT))
18+
19+
tr, streaming, err := factory(resolved.STT, true) // prefer streaming
20+
if err != nil {
21+
t.Fatalf("expected batch fallback, got error: %v", err)
22+
}
23+
if streaming {
24+
t.Error("should have fallen back to batch (streaming=false)")
25+
}
26+
if tr == nil {
27+
t.Error("expected a Groq batch transcriber")
28+
}
29+
}
30+
31+
func TestFactoryCloudProviderIgnoresLeftoverLocalModel(t *testing.T) {
32+
// Regression: pick local once (LocalModelPath gets set), then switch to a
33+
// cloud batch provider. streamProvider stays unset (defaults to "local"), so
34+
// the streaming factory must NOT fire up the sherpa server on the leftover
35+
// path — it should fall back to the cloud batch pipeline the user chose.
36+
t.Setenv("GROQ_API_KEY", "gsk-test")
37+
stt := config.STTConfig{
38+
Provider: config.STTProviderGroq, // cloud batch, chosen now
39+
LocalModelPath: "/leftover/model/dir", // set by an earlier local selection
40+
}
41+
resolved := config.ResolvedConfig{STT: stt}
42+
factory := newDictationTranscriberFactory(resolved, "", newDictationServerManager(stt))
43+
44+
tr, streaming, err := factory(stt, true) // prefer streaming
45+
if err != nil {
46+
t.Fatalf("expected cloud batch fallback, got error: %v", err)
47+
}
48+
if streaming {
49+
t.Error("a cloud provider must not stream through the local sherpa server")
50+
}
51+
if tr == nil {
52+
t.Error("expected a Groq batch transcriber")
53+
}
54+
}
55+
56+
func TestFactorySurfacesExplicitCloudStreamingError(t *testing.T) {
57+
// Deepgram chosen explicitly but no key resolvable → surface the setup error.
58+
resolved := config.ResolvedConfig{STT: config.STTConfig{
59+
StreamProvider: config.STTProviderDeepgram,
60+
}}
61+
factory := newDictationTranscriberFactory(resolved, "", nil)
62+
63+
_, streaming, err := factory(resolved.STT, true)
64+
if err == nil {
65+
t.Fatal("expected a setup error for missing Deepgram key")
66+
}
67+
if !streaming {
68+
t.Error("an explicit cloud streaming choice should report streaming=true even on error")
69+
}
70+
var setupErr *dictation.SetupError
71+
if !errors.As(err, &setupErr) {
72+
t.Errorf("want *SetupError, got %v", err)
73+
}
74+
}
75+
76+
func TestFactoryLocalBatchSetupErrorWhenNoModel(t *testing.T) {
77+
resolved := config.ResolvedConfig{STT: config.STTConfig{Provider: config.STTProviderLocal}}
78+
factory := newDictationTranscriberFactory(resolved, "", newDictationServerManager(resolved.STT))
79+
80+
_, _, err := factory(resolved.STT, true)
81+
var setupErr *dictation.SetupError
82+
if !errors.As(err, &setupErr) {
83+
t.Fatalf("want *SetupError guiding local model setup, got %v", err)
84+
}
85+
}
86+
87+
func TestResolveSTTKeyPrefersConfiguredProvider(t *testing.T) {
88+
resolved := config.ResolvedConfig{
89+
Providers: []config.ProviderProfile{
90+
{Name: "groq", CatalogID: "groq", BaseURL: "https://api.groq.com/openai/v1", APIKey: "gsk-configured"},
91+
},
92+
}
93+
if got := resolveSTTKey(resolved, "", dictation.ProviderGroq, "GROQ_API_KEY"); got != "gsk-configured" {
94+
t.Errorf("key = %q, want the configured provider's key", got)
95+
}
96+
}

0 commit comments

Comments
 (0)