diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..b68324d --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,450 @@ +# Recorder Roadmap + +## Speaker Attribution Improvements + +### Goals + +- Keep long Whisper chunks for transcription quality +- Attribute speech at Whisper segment level, not audio chunk level +- Preserve useful ambiguity instead of discarding speaker evidence +- Keep complexity isolated behind testable interfaces + +### Invariants + +- Do not flush audio chunks on speaker changes +- Do not rely on word timestamps + - `whisper.cpp --vad` preserves segment offsets + - word offsets were inconsistent in validation +- Use Whisper `verbose_json` segment timestamps +- Treat speaker percentages as active-indicator coverage, not word ownership + +### Validated Whisper Capability + +- Server: `whisper.cpp` on `http://odsod-desktop:8178` +- Endpoint: `/v1/audio/transcriptions` +- Request format: + +```text +response_format=verbose_json +``` + +- Response includes: + - `text` + - `segments[].start` + - `segments[].end` + - `segments[].text` + - `segments[].words` +- Use `segments[]` +- Ignore `words[]` for attribution + +### Transcript Target + +```md +[11:28:14] πŸ”Š **sys** [Andreas BΓ€ckevik 85%] clear segment... +[11:28:19] πŸ”Š **sys** [Andreas BΓ€ckevik 55% / Sofia ThorΓ©n 35%] mixed segment... +[11:28:22] πŸ”Š **sys** [Andreas BΓ€ckevik 35% / Sofia ThorΓ©n 28% / Oscar SΓΆderlund 9%] group segment... +``` + +### Attribution Rules + +- For each Whisper segment: + - `absoluteStart = chunk.StartTime + segment.StartSec` + - `absoluteEnd = chunk.StartTime + segment.EndSec` + - compute active speaker coverage over that interval +- Include every active speaker above threshold: + - `minCandidatePct = 5-10%` + - `minCandidateDuration = 250ms` +- Sort speakers by coverage descending +- Render all included speakers in the prefix +- Percentages are: + +```text +speaker active duration / segment duration +``` + +- Percentages may sum over 100% when Meet shows overlapping active indicators + +### Implementation Steps + +1. **Whisper Client** + +- Send `response_format=verbose_json` +- Extend response model: + +```go +type Segment struct { + StartSec float64 + EndSec float64 + Text string +} + +type TranscribeResponse struct { + Text string + Segments []Segment +} +``` + +- Preserve `Text` fallback when `Segments` is empty +- Add protocol tests for request format and segment parsing + +2. **Speaker Tracker** + +- Add isolated debounced tracker +- Poll CDP at higher frequency, e.g. `250ms` +- Tracker input: + +```go +[]signals.ParticipantState +``` + +- Tracker output: + - stable speaker start/stop transitions +- Rules: + - accept repeated short Meet indicator flashes + - ignore isolated blips + - hold through short UI dropouts + - emit stop after grace period + +3. **Timeline Coverage API** + +- Add ranked speaker coverage lookup: + +```go +type SpeakerCandidate struct { + Name string + CoverageSec float64 + CoveragePct float64 +} + +type SpeakerAttribution struct { + Candidates []SpeakerCandidate +} +``` + +- API should return all candidates above threshold, sorted descending +- Keep percentage math inside `internal/timeline` +- Add tests for: + - overlapping speakers + - low-threshold inclusion + - min-duration filtering + - no active speakers + - percentages summing over 100% + +4. **Transcription Worker** + +- Emit one transcript event per Whisper segment +- Event time is segment absolute start +- Render speaker prefix with all candidates and percentages +- Keep long audio chunking unchanged +- Keep existing fallback path for text-only responses + +5. **Cleanup** + +- First implementation: cleanup each emitted segment independently +- Preserve existing chunk-level behavior as fallback for text-only responses +- Revisit segment-preserving cleanup only if per-segment cleanup degrades quality + +6. **Mic/System Dedup** + +- Target behavior: + - compare mic segments against nearby system segments + - dedup per segment instead of whole chunk +- First implementation may keep chunk-level dedup if needed +- Avoid blocking segment-level system attribution on dedup complexity + +7. **Diagnostics** + +- Log debounced speaker transitions +- Log segment attribution decisions: + - chunk number + - segment start/end + - candidates + - thresholds applied +- Do not log every raw 250ms sample by default + +### Verification + +```bash +mise run test +mise run build +``` + +### Live Validation + +- Run recorder in a Google Meet +- Inspect: + - `~/.local/share/recorder/recorder.jsonl` + - configured transcript file for the current day +- Confirm: + - long audio chunks produce multiple transcript events + - back-and-forth segments show multiple active speakers + - speaker percentages are visible + - unattributed lines only happen when no speaker crosses threshold + +## Follow-Up Modularization + +### Goals + +- Raise confidence in the new segment-level attribution path +- Keep recorder orchestration small +- Move policy into focused, testable units +- Avoid leaking attribution complexity into capture, transcript writing, or segmenting + +### Target Package Structure + +```text +internal/ +β”œβ”€β”€ protocol/ +β”‚ β”œβ”€β”€ whisper/ # wire client; verbose_json structs +β”‚ β”œβ”€β”€ llm/ +β”‚ β”œβ”€β”€ cdp/ +β”‚ └── parec/ +β”œβ”€β”€ signals/ +β”‚ β”œβ”€β”€ speaker.go # collector ticker wiring +β”‚ β”œβ”€β”€ speaker_collector.go # SpeakerCollector.PollOnce +β”‚ β”œβ”€β”€ speaker_tracker.go # debounce policy +β”‚ └── silence.go +β”œβ”€β”€ timeline/ +β”‚ β”œβ”€β”€ speaker.go # speaker intervals + coverage lookup +β”‚ └── meeting.go +β”œβ”€β”€ speech/ +β”‚ β”œβ”€β”€ segment.go # canonical SpeechSegment type + normalizer +β”‚ β”œβ”€β”€ attribution.go # speaker percentage formatting +β”‚ β”œβ”€β”€ dedup.go # segment-level mic/sys dedup +β”‚ └── emitter.go # segments -> transcript.Event +β”œβ”€β”€ recorder/ +β”‚ β”œβ”€β”€ recorder.go # lifecycle, goroutines +β”‚ β”œβ”€β”€ capture.go # audio chunk production +β”‚ β”œβ”€β”€ transcribe.go # chunk orchestration only +β”‚ β”œβ”€β”€ services.go +β”‚ β”œβ”€β”€ types.go +β”‚ └── writer.go +β”œβ”€β”€ transcript/ +β”œβ”€β”€ segment/ +β”œβ”€β”€ summarize/ +└── transcribe/ # LLM cleanup + text overlap helpers +``` + +### Package Responsibilities + +- `protocol/whisper` + - HTTP multipart wire protocol + - OpenAI-compatible response parsing + - `verbose_json` segment structs +- `timeline` + - time-indexed meeting/speaker state + - active speaker interval storage + - coverage math + - no transcript rendering +- `signals` + - CDP polling + - participant and meeting signal collection + - speaker debounce state + - ticker loop and `PollOnce` +- `speech` + - turn Whisper responses into transcript speech events + - normalize Whisper segments into absolute-time speech segments + - apply cleanup + - apply segment-level dedup + - format speaker attribution percentages +- `recorder` + - lifecycle + - goroutines + - capture loop + - chunk-level transcription orchestration + - transcript append and segmenter feed +- `transcript` + - event format + - event parsing +- `segment` + - transcript segmentation + - summarization input formatting + +### `internal/speech` API Sketch + +```go +package speech + +type Segment struct { + Start time.Time + End time.Time + Text string +} + +func FromWhisper(resp whisper.TranscribeResponse, chunkStart, chunkEnd time.Time) []Segment +``` + +```go +type SpeakerLookup interface { + Coverage(start, end time.Time, opts timeline.SpeakerLookupOptions) timeline.SpeakerAttribution +} + +type Cleaner interface { + Cleanup(ctx context.Context, text string) (string, error) +} + +type Deduper interface { + IsDuplicate(segment Segment, refs []transcript.Event) bool +} + +type Emitter struct { + Cleaner Cleaner + SpeakerLookup SpeakerLookup + Deduper Deduper +} + +func (e *Emitter) Emit(ctx context.Context, source string, segments []Segment, refs []transcript.Event) ([]transcript.Event, error) +``` + +```go +func FormatAttribution(a timeline.SpeakerAttribution) string +``` + +### Recorder Shape After Refactor + +```go +sysSegments := speech.FromWhisper(sysResp, chunk.StartTime, chunk.EndTime) +sysEvents := r.speechEmitter.Emit(ctx, "sys", sysSegments, nil) + +micSegments := speech.FromWhisper(micResp, chunk.StartTime, chunk.EndTime) +micEvents := r.speechEmitter.Emit(ctx, "mic", micSegments, sysEvents) +``` + +### Package Boundaries To Avoid + +- Do not create package-per-helper boundaries: + - `internal/attribution` + - `internal/dedup` + - `internal/whispersegments` +- Keep related speech-event policy together in `internal/speech` +- Keep rendering out of `timeline` +- Keep lifecycle/goroutine concerns out of `speech` + +### Recommended Order + +1. **Speech Segment Emitter** + +- Extract from recorder transcription flow +- Input: + - source (`sys` / `mic`) + - audio chunk timing + - Whisper segments + - nearby opposite-channel transcript events +- Output: + - `[]transcript.Event` +- Owns: + - segment cleanup + - speaker attribution lookup + - transcript event construction + - segment-level dedup decision +- Rationale: + - highest-risk new behavior + - currently coupled to `Recorder` + +2. **Speaker Collector `PollOnce`** + +- Keep ticker loop thin +- Add testable collector unit: + +```go +type SpeakerCollector struct { + Detector SpeakerPoller + Tracker *SpeakerTracker + Timeline SpeakerTimelineWriter + People ParticipantWriter + Meetings MeetingWriter +} + +func (c *SpeakerCollector) PollOnce(ctx context.Context, now time.Time) error +``` + +- Test: + - participant updates + - meeting reset + - tracker reset on meeting change + - active/inactive timeline writes + - CDP error handling + +3. **Whisper Segment Normalizer** + +- Move segment normalization out of `internal/recorder/transcribe.go` +- Input: + - `whisper.TranscribeResponse` + - chunk start/end +- Output: + - `[]SpeechSegment` +- Test: + - verbose segments + - empty segments + - text fallback + - zero/invalid segment duration fallback + +4. **Speaker Attribution Formatter** + +- Extract prefix rendering policy +- Input: + - `timeline.SpeakerAttribution` +- Output: + - speaker prefix string +- Test: + - rounding + - ordering + - empty attribution + - multiple speakers + +5. **Segment Deduper** + +- Extract mic/system dedup policy +- Interface: + +```go +type SegmentDeduper interface { + IsDuplicate(segment SpeechSegment, refs []transcript.Event) bool +} +``` + +- Test: + - nearby duplicate + - distant non-duplicate + - threshold behavior + - empty references + +6. **Transcription Orchestrator** + +- Longer-term extraction: + +```go +type ChunkTranscriber struct { + Transcriber Transcriber + Emitter SegmentEmitter +} + +func (t *ChunkTranscriber) Transcribe(ctx context.Context, chunk AudioChunk) ([]transcript.Event, error) +``` + +- Recorder stays responsible for: + - appending events + - feeding segmenter + - lifecycle and shutdown + +### Interfaces To Keep Narrow + +```go +type SpeakerLookup interface { + Coverage(start, end time.Time, opts timeline.SpeakerLookupOptions) timeline.SpeakerAttribution +} + +type SpeakerTimelineWriter interface { + SetSpeakerActive(ts time.Time, name string, active bool) + Append(ts time.Time, name string) +} +``` + +### Risk Reduction + +- Add direct tests for: + - one audio chunk producing multiple transcript events + - percentage attribution attached to each emitted event + - mixed speaker segment preserving all active candidates + - collector reset on meeting/tab change + - no stale speaker carry-over between meetings diff --git a/internal/protocol/whisper/client.go b/internal/protocol/whisper/client.go index 9d688c1..c51a1a4 100644 --- a/internal/protocol/whisper/client.go +++ b/internal/protocol/whisper/client.go @@ -50,6 +50,15 @@ type TranscribeRequest struct { type TranscribeResponse struct { // Text is the transcribed speech with normalized whitespace. Text string + // Segments are timestamped transcription chunks from verbose_json responses. + Segments []Segment +} + +// Segment is a timestamped transcription segment relative to the uploaded audio. +type Segment struct { + StartSec float64 + EndSec float64 + Text string } // Client communicates with an OpenAI-compatible audio transcription endpoint. @@ -80,7 +89,7 @@ func (c *Client) Transcribe(ctx context.Context, req TranscribeRequest) (Transcr if err := writer.WriteField("model", "whisper-1"); err != nil { return TranscribeResponse{}, err } - if err := writer.WriteField("response_format", "json"); err != nil { + if err := writer.WriteField("response_format", "verbose_json"); err != nil { return TranscribeResponse{}, err } if err := writer.Close(); err != nil { @@ -111,14 +120,43 @@ func (c *Client) Transcribe(ctx context.Context, req TranscribeRequest) (Transcr return TranscribeResponse{}, &TranscriptionError{StatusCode: resp.StatusCode, Body: respBody} } - var result struct { - Text string `json:"text"` - } + var result wireResponse if err := json.Unmarshal(respBody, &result); err != nil { return TranscribeResponse{}, err } - text := strings.TrimSpace(result.Text) - text = spaceRe.ReplaceAllString(text, " ") - return TranscribeResponse{Text: text}, nil + return normalizeResponse(result), nil +} + +type wireResponse struct { + Text string `json:"text"` + Segments []wireSegment `json:"segments"` +} + +type wireSegment struct { + StartSec float64 `json:"start"` + EndSec float64 `json:"end"` + Text string `json:"text"` +} + +func normalizeResponse(result wireResponse) TranscribeResponse { + text := normalizeText(result.Text) + segments := make([]Segment, 0, len(result.Segments)) + for _, s := range result.Segments { + segmentText := normalizeText(s.Text) + if segmentText == "" { + continue + } + segments = append(segments, Segment{ + StartSec: s.StartSec, + EndSec: s.EndSec, + Text: segmentText, + }) + } + return TranscribeResponse{Text: text, Segments: segments} +} + +func normalizeText(text string) string { + text = strings.TrimSpace(text) + return spaceRe.ReplaceAllString(text, " ") } diff --git a/internal/protocol/whisper/client_test.go b/internal/protocol/whisper/client_test.go index 7fd5086..8bc65ce 100644 --- a/internal/protocol/whisper/client_test.go +++ b/internal/protocol/whisper/client_test.go @@ -26,8 +26,8 @@ func TestTranscribe_Success(t *testing.T) { if model := r.FormValue("model"); model != "whisper-1" { t.Errorf("expected model whisper-1, got %s", model) } - if rf := r.FormValue("response_format"); rf != "json" { - t.Errorf("expected response_format json, got %s", rf) + if rf := r.FormValue("response_format"); rf != "verbose_json" { + t.Errorf("expected response_format verbose_json, got %s", rf) } file, header, err := r.FormFile("file") @@ -44,7 +44,13 @@ func TestTranscribe_Success(t *testing.T) { t.Errorf("unexpected file content: %q", string(data)) } - resp := map[string]string{"text": " Hello world "} + resp := map[string]any{ + "text": " Hello world ", + "segments": []map[string]any{ + {"start": 0.5, "end": 1.25, "text": " Hello segment "}, + {"start": 1.25, "end": 2.0, "text": " "}, + }, + } _ = json.NewEncoder(w).Encode(resp) })) defer srv.Close() @@ -63,6 +69,13 @@ func TestTranscribe_Success(t *testing.T) { if resp.Text != "Hello world" { t.Errorf("expected 'Hello world', got %q", resp.Text) } + if len(resp.Segments) != 1 { + t.Fatalf("expected 1 segment, got %d", len(resp.Segments)) + } + seg := resp.Segments[0] + if seg.StartSec != 0.5 || seg.EndSec != 1.25 || seg.Text != "Hello segment" { + t.Errorf("unexpected segment: %+v", seg) + } } func TestTranscribe_NonOKStatus(t *testing.T) { diff --git a/internal/recorder/recorder.go b/internal/recorder/recorder.go index 52a2b3e..46216f5 100644 --- a/internal/recorder/recorder.go +++ b/internal/recorder/recorder.go @@ -10,6 +10,7 @@ import ( "github.com/odsod/recorder/internal/lock" "github.com/odsod/recorder/internal/segment" "github.com/odsod/recorder/internal/signals" + "github.com/odsod/recorder/internal/speech" "github.com/odsod/recorder/internal/timeline" "github.com/odsod/recorder/internal/transcript" ) @@ -30,6 +31,7 @@ type Recorder struct { meetingState *timeline.MeetingState silenceMonitor *signals.SilenceMonitor segmenter *segment.IncrementalSegmenter + speechEmitter *speech.Emitter lastSystemText string chunkNum int lastFlushedTime time.Time @@ -51,6 +53,19 @@ func New(ctx context.Context, cfg config.Config, svc Services) (*Recorder, error silenceMonitor: signals.NewSilenceMonitor(cfg.Signals.SilenceThresholdS), lastPplSet: make(map[string]struct{}), } + r.speechEmitter = &speech.Emitter{ + Cleaner: svc.Cleaner, + SpeakerLookup: r.speakerTimeline, + Participants: r.currentParticipants, + Deduper: speech.NearbyDeduper{ + Threshold: cfg.Dedup.Threshold, + Tolerance: 5 * time.Second, + }, + LookupOptions: timeline.SpeakerLookupOptions{ + MinCandidatePct: minSpeakerCandidatePct, + MinCandidateDuration: minSpeakerCandidateDuration, + }, + } r.segmenter = segment.NewSegmenter(ctx, svc.SegmentHandler, func(e transcript.Event) { t.AppendEvent(e) diff --git a/internal/recorder/transcribe.go b/internal/recorder/transcribe.go index b51e48c..c751424 100644 --- a/internal/recorder/transcribe.go +++ b/internal/recorder/transcribe.go @@ -2,18 +2,22 @@ package recorder import ( "context" - "fmt" "log/slog" "maps" "slices" + "strings" "time" "github.com/odsod/recorder/internal/protocol/whisper" - "github.com/odsod/recorder/internal/timeline" - "github.com/odsod/recorder/internal/transcribe" + "github.com/odsod/recorder/internal/speech" "github.com/odsod/recorder/internal/transcript" ) +const ( + minSpeakerCandidatePct = 0.05 + minSpeakerCandidateDuration = 250 * time.Millisecond +) + func (r *Recorder) transcriptionWorker(ctx context.Context, chunkCh <-chan AudioChunk) { for chunk := range chunkCh { slog.InfoContext(ctx, "transcribing") @@ -43,109 +47,35 @@ func (r *Recorder) transcribeChunk(ctx context.Context, chunk AudioChunk) { ) } - sysText := sysResp.Text - micText := micResp.Text - r.flushSignalEvents(ctx, chunk.StartTime, chunk.EndTime) - speakers := r.speakerTimeline.SpeakersInWithDurations(chunk.StartTime, chunk.EndTime) - speaker := attributeSpeaker(speakers, r.cfg.Speaker.AmbiguityRatio) - participants := r.currentParticipants() - - switch { - case sysText != "": - cleaned, err := r.svc.Cleaner.Cleanup(ctx, sysText, participants) - if err != nil { - slog.ErrorContext(ctx, "cleanup sys failed", - "err", err, - ) - } - if cleaned == "" { - cleaned = sysText - } - if cleaned != "" { - e := transcript.Event{ - Time: chunk.StartTime, - Type: transcript.Speech, - Source: "sys", - Text: cleaned, - Speaker: speaker, - } - r.appendEvent(ctx, e) - r.lastSystemText = cleaned - r.segmenter.OnSpeech(e) - - if micText != "" && !transcribe.TextsOverlap(cleaned, micText, r.cfg.Dedup.Threshold) { - micCleaned, err := r.svc.Cleaner.Cleanup(ctx, micText, participants) - if err != nil { - slog.ErrorContext(ctx, "cleanup mic failed", - "err", err, - ) - } - if micCleaned == "" { - micCleaned = micText - } - if micCleaned != "" { - me := transcript.Event{ - Time: chunk.StartTime, - Type: transcript.Speech, - Source: "mic", - Text: micCleaned, - Speaker: speaker, - } - r.appendEvent(ctx, me) - r.segmenter.OnSpeech(me) - } - } - } - case micText != "": - if r.lastSystemText != "" && transcribe.TextsOverlap(r.lastSystemText, micText, r.cfg.Dedup.Threshold) { - slog.InfoContext(ctx, "mic deduped", - "text", truncate(micText, 60), - ) - } else { - cleaned, err := r.svc.Cleaner.Cleanup(ctx, micText, participants) - if err != nil { - slog.ErrorContext(ctx, "cleanup mic failed", - "err", err, - ) - } - if cleaned == "" { - cleaned = micText - } - if cleaned != "" { - e := transcript.Event{ - Time: chunk.StartTime, - Type: transcript.Speech, - Source: "mic", - Text: cleaned, - Speaker: speaker, - } - r.appendEvent(ctx, e) - r.segmenter.OnSpeech(e) - } - } - default: - slog.InfoContext(ctx, "no speech detected") + sysSegments := speech.FromWhisper(sysResp, chunk.StartTime, chunk.EndTime) + micSegments := speech.FromWhisper(micResp, chunk.StartTime, chunk.EndTime) + + priorSystemText := r.lastSystemText + sysEvents, err := r.speechEmitter.Emit(ctx, "sys", sysSegments, nil) + if err != nil { + slog.ErrorContext(ctx, "emit sys speech failed", "err", err) + } + r.appendSpeechEvents(ctx, sysEvents) + if len(sysEvents) > 0 { + r.lastSystemText = joinEventText(sysEvents) } - slog.InfoContext(ctx, "listening") -} -func attributeSpeaker(speakers []timeline.SpeakerDuration, ambiguityRatio float64) string { - switch { - case len(speakers) == 0: - return "" - case len(speakers) == 1: - return speakers[0].Name - default: - if float64(speakers[1].Duration) >= float64(speakers[0].Duration)*ambiguityRatio { - total := speakers[0].Duration + speakers[1].Duration - pct0 := int(float64(speakers[0].Duration) * 100 / float64(total)) - pct1 := 100 - pct0 - return fmt.Sprintf("%s(%d%%),%s(%d%%)", speakers[0].Name, pct0, speakers[1].Name, pct1) - } - return speakers[0].Name + micDedupEvents := sysEvents + if len(micDedupEvents) == 0 && priorSystemText != "" { + micDedupEvents = []transcript.Event{{Time: chunk.StartTime, Text: priorSystemText}} } + micEvents, err := r.speechEmitter.Emit(ctx, "mic", micSegments, micDedupEvents) + if err != nil { + slog.ErrorContext(ctx, "emit mic speech failed", "err", err) + } + r.appendSpeechEvents(ctx, micEvents) + + if len(sysSegments) == 0 && len(micSegments) == 0 { + slog.InfoContext(ctx, "no speech detected") + } + slog.InfoContext(ctx, "listening") } func (r *Recorder) currentParticipants() []string { @@ -156,6 +86,23 @@ func (r *Recorder) currentParticipants() []string { return slices.Sorted(maps.Keys(all)) } +func joinEventText(events []transcript.Event) string { + parts := make([]string, 0, len(events)) + for _, e := range events { + if e.Text != "" { + parts = append(parts, e.Text) + } + } + return strings.Join(parts, " ") +} + +func (r *Recorder) appendSpeechEvents(ctx context.Context, events []transcript.Event) { + for _, e := range events { + r.appendEvent(ctx, e) + r.segmenter.OnSpeech(e) + } +} + func (r *Recorder) flushSignalEvents(ctx context.Context, start, end time.Time) { r.lastFlushedTime = end diff --git a/internal/recorder/transcribe_test.go b/internal/recorder/transcribe_test.go deleted file mode 100644 index 663abc9..0000000 --- a/internal/recorder/transcribe_test.go +++ /dev/null @@ -1,79 +0,0 @@ -package recorder - -import ( - "testing" - "time" - - "github.com/odsod/recorder/internal/timeline" -) - -func TestAttributeSpeaker(t *testing.T) { - const ratio = 0.05 - - tests := []struct { - name string - speakers []timeline.SpeakerDuration - want string - }{ - { - name: "empty", - speakers: nil, - want: "", - }, - { - name: "single speaker", - speakers: []timeline.SpeakerDuration{{Name: "Alice", Duration: 10 * time.Second}}, - want: "Alice", - }, - { - name: "unambiguous - second speaker below 5% threshold", - speakers: []timeline.SpeakerDuration{ - {Name: "Alice", Duration: 20 * time.Second}, - {Name: "Bob", Duration: 500 * time.Millisecond}, - }, - want: "Alice", - }, - { - name: "ambiguous - short interjection above threshold", - speakers: []timeline.SpeakerDuration{ - {Name: "Alice", Duration: 15 * time.Second}, - {Name: "Bob", Duration: 1500 * time.Millisecond}, - }, - want: "Alice(90%),Bob(10%)", - }, - { - name: "ambiguous - roughly equal", - speakers: []timeline.SpeakerDuration{ - {Name: "Alice", Duration: 8 * time.Second}, - {Name: "Bob", Duration: 7 * time.Second}, - }, - want: "Alice(53%),Bob(47%)", - }, - { - name: "equal time", - speakers: []timeline.SpeakerDuration{ - {Name: "Alice", Duration: 10 * time.Second}, - {Name: "Bob", Duration: 10 * time.Second}, - }, - want: "Alice(50%),Bob(50%)", - }, - { - name: "three speakers - only top two considered", - speakers: []timeline.SpeakerDuration{ - {Name: "Alice", Duration: 10 * time.Second}, - {Name: "Bob", Duration: 8 * time.Second}, - {Name: "Carol", Duration: 2 * time.Second}, - }, - want: "Alice(55%),Bob(45%)", - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - got := attributeSpeaker(tc.speakers, ratio) - if got != tc.want { - t.Errorf("attributeSpeaker(%v) = %q, want %q", tc.speakers, got, tc.want) - } - }) - } -} diff --git a/internal/recorder/util.go b/internal/recorder/util.go index 5c549e7..b841bba 100644 --- a/internal/recorder/util.go +++ b/internal/recorder/util.go @@ -14,13 +14,6 @@ func (r *Recorder) appendEvent(ctx context.Context, e transcript.Event) { ) } -func truncate(s string, n int) string { - if len(s) <= n { - return s - } - return s[:n] -} - func setsEqual(a, b map[string]struct{}) bool { if len(a) != len(b) { return false diff --git a/internal/signals/speaker.go b/internal/signals/speaker.go index 6750c89..6a9592b 100644 --- a/internal/signals/speaker.go +++ b/internal/signals/speaker.go @@ -30,10 +30,6 @@ type SpeakerPoller interface { Poll(ctx context.Context) (PollResult, error) } -// flickerFilterTicks requires a speaker to be seen speaking for this many -// consecutive polls before being recorded in the timeline. -const flickerFilterTicks = 2 - // RunSpeakerCollector polls CDP and updates speaker and meeting timelines. func RunSpeakerCollector( ctx context.Context, @@ -42,11 +38,9 @@ func RunSpeakerCollector( participantSet *timeline.ParticipantSet, meetingState *timeline.MeetingState, ) { - activeSpeakers := make(map[string]struct{}) - // Track consecutive speaking ticks per participant for flicker filtering. - speakingTicks := make(map[string]int) + tracker := NewSpeakerTracker(DefaultSpeakerTrackerConfig()) - ticker := time.NewTicker(1 * time.Second) + ticker := time.NewTicker(250 * time.Millisecond) defer ticker.Stop() for { @@ -72,8 +66,8 @@ func RunSpeakerCollector( slog.InfoContext(ctx, "meeting ended") } participantSet.Reset() - activeSpeakers = make(map[string]struct{}) - speakingTicks = make(map[string]int) + tracker = NewSpeakerTracker(DefaultSpeakerTrackerConfig()) + speakerTimeline.Append(time.Now(), "") } if result.Participants == nil { @@ -87,35 +81,18 @@ func RunSpeakerCollector( } participantSet.Update(names) - currentSpeaking := make(map[string]struct{}) - for _, s := range result.Participants { - if s.Speaking { - speakingTicks[s.Name]++ - if speakingTicks[s.Name] >= flickerFilterTicks { - currentSpeaking[s.Name] = struct{}{} - } - } else { - speakingTicks[s.Name] = 0 - } - } - - for name := range currentSpeaking { - if _, was := activeSpeakers[name]; !was { + for _, transition := range tracker.Observe(now, result.Participants) { + speakerTimeline.SetSpeakerActive(transition.Time, transition.Name, transition.Active) + if transition.Active { slog.InfoContext(ctx, "speaker started", - "name", name, + "name", transition.Name, ) - speakerTimeline.Append(now, name) - } - } - for name := range activeSpeakers { - if _, is := currentSpeaking[name]; !is { + } else { slog.InfoContext(ctx, "speaker stopped", - "name", name, + "name", transition.Name, ) - speakerTimeline.Append(now, "") } } - activeSpeakers = currentSpeaking } } } diff --git a/internal/signals/speaker_tracker.go b/internal/signals/speaker_tracker.go new file mode 100644 index 0000000..ba4936a --- /dev/null +++ b/internal/signals/speaker_tracker.go @@ -0,0 +1,132 @@ +package signals + +import ( + "slices" + "sort" + "time" +) + +// SpeakerTransition is a debounced active/inactive transition for one speaker. +type SpeakerTransition struct { + Time time.Time + Name string + Active bool +} + +// SpeakerTracker converts noisy raw speaker samples into stable transitions. +type SpeakerTracker struct { + cfg SpeakerTrackerConfig + states map[string]*speakerState +} + +// SpeakerTrackerConfig controls debounce behavior. +type SpeakerTrackerConfig struct { + StartWindow time.Duration + StartSamples int + StopGrace time.Duration +} + +type speakerState struct { + samples []time.Time + lastSeen time.Time + active bool +} + +// DefaultSpeakerTrackerConfig returns conservative settings for Meet's short indicator flashes. +func DefaultSpeakerTrackerConfig() SpeakerTrackerConfig { + return SpeakerTrackerConfig{ + StartWindow: 1500 * time.Millisecond, + StartSamples: 2, + StopGrace: 2 * time.Second, + } +} + +// NewSpeakerTracker creates a tracker with the given config. +func NewSpeakerTracker(cfg SpeakerTrackerConfig) *SpeakerTracker { + if cfg.StartWindow <= 0 { + cfg.StartWindow = 1500 * time.Millisecond + } + if cfg.StartSamples <= 0 { + cfg.StartSamples = 2 + } + if cfg.StopGrace <= 0 { + cfg.StopGrace = 2 * time.Second + } + return &SpeakerTracker{ + cfg: cfg, + states: make(map[string]*speakerState), + } +} + +// Observe ingests one raw poll sample and returns stable transitions. +func (t *SpeakerTracker) Observe(at time.Time, participants []ParticipantState) []SpeakerTransition { + seenNames := make(map[string]struct{}) + for _, p := range participants { + if p.Name == "" { + continue + } + seenNames[p.Name] = struct{}{} + state := t.state(p.Name) + state.trimSamples(at, t.cfg.StartWindow) + if p.Speaking { + state.samples = append(state.samples, at) + state.lastSeen = at + } + } + + for name, state := range t.states { + if _, ok := seenNames[name]; !ok { + state.trimSamples(at, t.cfg.StartWindow) + } + } + + var transitions []SpeakerTransition + names := make([]string, 0, len(t.states)) + for name := range t.states { + names = append(names, name) + } + sort.Strings(names) + + for _, name := range names { + state := t.states[name] + if !state.active && len(state.samples) >= t.cfg.StartSamples { + state.active = true + transitions = append(transitions, SpeakerTransition{Time: at, Name: name, Active: true}) + continue + } + if state.active && !state.lastSeen.IsZero() && at.Sub(state.lastSeen) > t.cfg.StopGrace { + state.active = false + state.samples = nil + transitions = append(transitions, SpeakerTransition{ + Time: state.lastSeen.Add(t.cfg.StopGrace), + Name: name, + Active: false, + }) + } + } + + return transitions +} + +func (t *SpeakerTracker) state(name string) *speakerState { + state := t.states[name] + if state == nil { + state = &speakerState{} + t.states[name] = state + } + return state +} + +func (s *speakerState) trimSamples(now time.Time, window time.Duration) { + cutoff := now.Add(-window) + idx := slices.IndexFunc(s.samples, func(sample time.Time) bool { + return !sample.Before(cutoff) + }) + if idx == -1 { + s.samples = nil + return + } + if idx > 0 { + s.samples = s.samples[idx:] + } +} diff --git a/internal/signals/speaker_tracker_test.go b/internal/signals/speaker_tracker_test.go new file mode 100644 index 0000000..c22372c --- /dev/null +++ b/internal/signals/speaker_tracker_test.go @@ -0,0 +1,96 @@ +package signals + +import ( + "reflect" + "testing" + "time" +) + +func TestSpeakerTracker_RepeatedFlashesStartSpeaker(t *testing.T) { + tracker := testTracker() + start := time.Unix(0, 0) + + got := tracker.Observe(start, participants(speaking("Alice"))) + if len(got) != 0 { + t.Fatalf("got %v, want no transition after one flash", got) + } + + got = tracker.Observe(start.Add(750*time.Millisecond), participants(speaking("Alice"))) + assertTransitions(t, got, []SpeakerTransition{ + {Time: start.Add(750 * time.Millisecond), Name: "Alice", Active: true}, + }) +} + +func TestSpeakerTracker_IgnoresIsolatedBlip(t *testing.T) { + tracker := testTracker() + start := time.Unix(0, 0) + + _ = tracker.Observe(start, participants(speaking("Alice"))) + got := tracker.Observe(start.Add(2*time.Second), participants(silent("Alice"))) + if len(got) != 0 { + t.Fatalf("got %v, want no transition", got) + } +} + +func TestSpeakerTracker_HoldsThroughDropout(t *testing.T) { + tracker := testTracker() + start := time.Unix(0, 0) + + _ = tracker.Observe(start, participants(speaking("Alice"))) + _ = tracker.Observe(start.Add(500*time.Millisecond), participants(speaking("Alice"))) + got := tracker.Observe(start.Add(1500*time.Millisecond), participants(silent("Alice"))) + if len(got) != 0 { + t.Fatalf("got %v, want no stop during grace period", got) + } +} + +func TestSpeakerTracker_StopsAfterGrace(t *testing.T) { + tracker := testTracker() + start := time.Unix(0, 0) + + _ = tracker.Observe(start, participants(speaking("Alice"))) + _ = tracker.Observe(start.Add(500*time.Millisecond), participants(speaking("Alice"))) + got := tracker.Observe(start.Add(3*time.Second), participants(silent("Alice"))) + assertTransitions(t, got, []SpeakerTransition{ + {Time: start.Add(2500 * time.Millisecond), Name: "Alice", Active: false}, + }) +} + +func TestSpeakerTracker_AllowsOverlappingSpeakers(t *testing.T) { + tracker := testTracker() + start := time.Unix(0, 0) + + _ = tracker.Observe(start, participants(speaking("Alice"), speaking("Bob"))) + got := tracker.Observe(start.Add(500*time.Millisecond), participants(speaking("Alice"), speaking("Bob"))) + assertTransitions(t, got, []SpeakerTransition{ + {Time: start.Add(500 * time.Millisecond), Name: "Alice", Active: true}, + {Time: start.Add(500 * time.Millisecond), Name: "Bob", Active: true}, + }) +} + +func testTracker() *SpeakerTracker { + return NewSpeakerTracker(SpeakerTrackerConfig{ + StartWindow: 1500 * time.Millisecond, + StartSamples: 2, + StopGrace: 2 * time.Second, + }) +} + +func speaking(name string) ParticipantState { + return ParticipantState{Name: name, Speaking: true} +} + +func silent(name string) ParticipantState { + return ParticipantState{Name: name} +} + +func participants(states ...ParticipantState) []ParticipantState { + return states +} + +func assertTransitions(t *testing.T, got, want []SpeakerTransition) { + t.Helper() + if !reflect.DeepEqual(got, want) { + t.Fatalf("got %v, want %v", got, want) + } +} diff --git a/internal/speech/attribution.go b/internal/speech/attribution.go new file mode 100644 index 0000000..0b59712 --- /dev/null +++ b/internal/speech/attribution.go @@ -0,0 +1,21 @@ +package speech + +import ( + "fmt" + "math" + "strings" + + "github.com/odsod/recorder/internal/timeline" +) + +// FormatAttribution renders speaker coverage for transcript prefixes. +func FormatAttribution(attribution timeline.SpeakerAttribution) string { + if len(attribution.Candidates) == 0 { + return "" + } + parts := make([]string, 0, len(attribution.Candidates)) + for _, candidate := range attribution.Candidates { + parts = append(parts, fmt.Sprintf("%s %.0f%%", candidate.Name, math.Round(candidate.CoveragePct*100))) + } + return strings.Join(parts, " / ") +} diff --git a/internal/speech/attribution_test.go b/internal/speech/attribution_test.go new file mode 100644 index 0000000..0972a3c --- /dev/null +++ b/internal/speech/attribution_test.go @@ -0,0 +1,50 @@ +package speech + +import ( + "testing" + + "github.com/odsod/recorder/internal/timeline" +) + +func TestFormatAttribution_Empty(t *testing.T) { + if got := FormatAttribution(timeline.SpeakerAttribution{}); got != "" { + t.Fatalf("got %q, want empty", got) + } +} + +func TestFormatAttribution_OneSpeaker(t *testing.T) { + got := FormatAttribution(timeline.SpeakerAttribution{ + Candidates: []timeline.SpeakerCandidate{ + {Name: "Alice", CoveragePct: 0.55}, + }, + }) + want := "Alice 55%" + if got != want { + t.Fatalf("got %q, want %q", got, want) + } +} + +func TestFormatAttribution_MultipleSpeakers(t *testing.T) { + got := FormatAttribution(timeline.SpeakerAttribution{ + Candidates: []timeline.SpeakerCandidate{ + {Name: "Alice", CoveragePct: 0.553}, + {Name: "Bob", CoveragePct: 0.094}, + }, + }) + want := "Alice 55% / Bob 9%" + if got != want { + t.Fatalf("got %q, want %q", got, want) + } +} + +func TestFormatAttribution_RoundsPercentages(t *testing.T) { + got := FormatAttribution(timeline.SpeakerAttribution{ + Candidates: []timeline.SpeakerCandidate{ + {Name: "Alice", CoveragePct: 0.556}, + }, + }) + want := "Alice 56%" + if got != want { + t.Fatalf("got %q, want %q", got, want) + } +} diff --git a/internal/speech/dedup.go b/internal/speech/dedup.go new file mode 100644 index 0000000..46d4ee3 --- /dev/null +++ b/internal/speech/dedup.go @@ -0,0 +1,40 @@ +package speech + +import ( + "time" + + "github.com/odsod/recorder/internal/transcribe" + "github.com/odsod/recorder/internal/transcript" +) + +// Deduper decides whether a speech segment duplicates nearby reference events. +type Deduper interface { + IsDuplicate(segment Segment, refs []transcript.Event) bool +} + +// NearbyDeduper compares text only against references near the segment start. +type NearbyDeduper struct { + Threshold float64 + Tolerance time.Duration +} + +// IsDuplicate reports whether segment text overlaps a nearby reference event. +func (d NearbyDeduper) IsDuplicate(segment Segment, refs []transcript.Event) bool { + for _, e := range refs { + if !nearby(segment.Start, e.Time, d.Tolerance) { + continue + } + if transcribe.TextsOverlap(e.Text, segment.Text, d.Threshold) { + return true + } + } + return false +} + +func nearby(a, b time.Time, tolerance time.Duration) bool { + diff := a.Sub(b) + if diff < 0 { + diff = -diff + } + return diff <= tolerance +} diff --git a/internal/speech/dedup_test.go b/internal/speech/dedup_test.go new file mode 100644 index 0000000..aefb682 --- /dev/null +++ b/internal/speech/dedup_test.go @@ -0,0 +1,58 @@ +package speech + +import ( + "testing" + "time" + + "github.com/odsod/recorder/internal/transcript" +) + +func TestNearbyDeduper_NearbyDuplicate(t *testing.T) { + start := time.Date(2026, 6, 3, 9, 0, 0, 0, time.UTC) + deduper := NearbyDeduper{Threshold: 0.6, Tolerance: 5 * time.Second} + + got := deduper.IsDuplicate( + Segment{Start: start, Text: "hello from the meeting room"}, + []transcript.Event{{Time: start.Add(2 * time.Second), Text: "Hello from the meeting room."}}, + ) + + if !got { + t.Fatal("got false, want true") + } +} + +func TestNearbyDeduper_DistantDuplicateFalse(t *testing.T) { + start := time.Date(2026, 6, 3, 9, 0, 0, 0, time.UTC) + deduper := NearbyDeduper{Threshold: 0.6, Tolerance: 5 * time.Second} + + got := deduper.IsDuplicate( + Segment{Start: start, Text: "hello from the meeting room"}, + []transcript.Event{{Time: start.Add(6 * time.Second), Text: "hello from the meeting room"}}, + ) + + if got { + t.Fatal("got true, want false") + } +} + +func TestNearbyDeduper_TextNonOverlapFalse(t *testing.T) { + start := time.Date(2026, 6, 3, 9, 0, 0, 0, time.UTC) + deduper := NearbyDeduper{Threshold: 0.6, Tolerance: 5 * time.Second} + + got := deduper.IsDuplicate( + Segment{Start: start, Text: "hello from the meeting room"}, + []transcript.Event{{Time: start, Text: "completely unrelated words here"}}, + ) + + if got { + t.Fatal("got true, want false") + } +} + +func TestNearbyDeduper_EmptyReferencesFalse(t *testing.T) { + deduper := NearbyDeduper{Threshold: 0.6, Tolerance: 5 * time.Second} + + if deduper.IsDuplicate(Segment{Start: time.Now(), Text: "hello"}, nil) { + t.Fatal("got true, want false") + } +} diff --git a/internal/speech/emitter.go b/internal/speech/emitter.go new file mode 100644 index 0000000..729d3a3 --- /dev/null +++ b/internal/speech/emitter.go @@ -0,0 +1,87 @@ +package speech + +import ( + "context" + "errors" + "time" + + "github.com/odsod/recorder/internal/timeline" + "github.com/odsod/recorder/internal/transcript" +) + +// Cleaner cleans raw transcription text. +type Cleaner interface { + Cleanup(ctx context.Context, text string, participants []string) (string, error) +} + +// SpeakerLookup returns speaker coverage for a time window. +type SpeakerLookup interface { + Coverage(start, end time.Time, opts timeline.SpeakerLookupOptions) timeline.SpeakerAttribution +} + +// ParticipantProvider returns the current known meeting participants. +type ParticipantProvider func() []string + +// Emitter converts speech segments into transcript events. +type Emitter struct { + Cleaner Cleaner + SpeakerLookup SpeakerLookup + Deduper Deduper + Participants ParticipantProvider + LookupOptions timeline.SpeakerLookupOptions +} + +// Emit cleans, attributes, and deduplicates segments without writing them. +func (e *Emitter) Emit( + ctx context.Context, + source string, + segments []Segment, + refs []transcript.Event, +) ([]transcript.Event, error) { + var emitted []transcript.Event + var errs []error + for _, segment := range segments { + if source == "mic" && e.Deduper != nil && e.Deduper.IsDuplicate(segment, refs) { + continue + } + + cleaned := segment.Text + if e.Cleaner != nil { + cleanupText, err := e.Cleaner.Cleanup(ctx, segment.Text, e.participants()) + if err != nil { + errs = append(errs, err) + } + if cleanupText != "" { + cleaned = cleanupText + } + } + if cleaned == "" { + continue + } + + speaker := "" + if e.SpeakerLookup != nil { + speaker = FormatAttribution(e.SpeakerLookup.Coverage( + segment.Start, + segment.End, + e.LookupOptions, + )) + } + + emitted = append(emitted, transcript.Event{ + Time: segment.Start, + Type: transcript.Speech, + Source: source, + Text: cleaned, + Speaker: speaker, + }) + } + return emitted, errors.Join(errs...) +} + +func (e *Emitter) participants() []string { + if e.Participants == nil { + return nil + } + return e.Participants() +} diff --git a/internal/speech/emitter_test.go b/internal/speech/emitter_test.go new file mode 100644 index 0000000..e6cece3 --- /dev/null +++ b/internal/speech/emitter_test.go @@ -0,0 +1,146 @@ +package speech + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + "time" + + "github.com/odsod/recorder/internal/timeline" + "github.com/odsod/recorder/internal/transcript" +) + +func TestEmitter_OneSegmentEmitsOneEvent(t *testing.T) { + start := time.Date(2026, 6, 3, 9, 0, 0, 0, time.UTC) + emitter := Emitter{Cleaner: staticCleaner{text: "cleaned"}} + + got, err := emitter.Emit(context.Background(), "sys", []Segment{ + {Start: start, End: start.Add(time.Second), Text: "raw"}, + }, nil) + if err != nil { + t.Fatalf("err = %v", err) + } + if len(got) != 1 { + t.Fatalf("got %d events, want 1", len(got)) + } + want := transcript.Event{Time: start, Type: transcript.Speech, Source: "sys", Text: "cleaned"} + if !reflect.DeepEqual(got[0], want) { + t.Fatalf("event = %+v, want %+v", got[0], want) + } +} + +func TestEmitter_MultipleSegmentsEmitMultipleEvents(t *testing.T) { + start := time.Date(2026, 6, 3, 9, 0, 0, 0, time.UTC) + emitter := Emitter{Cleaner: identityCleaner{}} + + got, err := emitter.Emit(context.Background(), "sys", []Segment{ + {Start: start, End: start.Add(time.Second), Text: "first"}, + {Start: start.Add(2 * time.Second), End: start.Add(3 * time.Second), Text: "second"}, + }, nil) + if err != nil { + t.Fatalf("err = %v", err) + } + if len(got) != 2 || got[0].Text != "first" || got[1].Text != "second" { + t.Fatalf("events = %+v", got) + } +} + +func TestEmitter_AttachesSpeakerPercentages(t *testing.T) { + start := time.Date(2026, 6, 3, 9, 0, 0, 0, time.UTC) + lookup := staticSpeakerLookup{attribution: timeline.SpeakerAttribution{ + Candidates: []timeline.SpeakerCandidate{ + {Name: "Alice", CoveragePct: 0.55}, + {Name: "Bob", CoveragePct: 0.09}, + }, + }} + emitter := Emitter{Cleaner: identityCleaner{}, SpeakerLookup: lookup} + + got, err := emitter.Emit(context.Background(), "sys", []Segment{ + {Start: start, End: start.Add(time.Second), Text: "raw"}, + }, nil) + if err != nil { + t.Fatalf("err = %v", err) + } + if got[0].Speaker != "Alice 55% / Bob 9%" { + t.Fatalf("speaker = %q", got[0].Speaker) + } +} + +func TestEmitter_MicDuplicateSkipped(t *testing.T) { + start := time.Date(2026, 6, 3, 9, 0, 0, 0, time.UTC) + emitter := Emitter{ + Cleaner: identityCleaner{}, + Deduper: NearbyDeduper{Threshold: 0.6, Tolerance: 5 * time.Second}, + } + + got, err := emitter.Emit(context.Background(), "mic", []Segment{ + {Start: start, End: start.Add(time.Second), Text: "hello from the room"}, + }, []transcript.Event{{Time: start, Text: "hello from the room"}}) + if err != nil { + t.Fatalf("err = %v", err) + } + if len(got) != 0 { + t.Fatalf("got %d events, want 0", len(got)) + } +} + +func TestEmitter_CleanupEmptyFallback(t *testing.T) { + start := time.Date(2026, 6, 3, 9, 0, 0, 0, time.UTC) + emitter := Emitter{Cleaner: staticCleaner{text: ""}} + + got, err := emitter.Emit(context.Background(), "sys", []Segment{ + {Start: start, End: start.Add(time.Second), Text: "raw"}, + }, nil) + if err != nil { + t.Fatalf("err = %v", err) + } + if got[0].Text != "raw" { + t.Fatalf("text = %q, want raw", got[0].Text) + } +} + +func TestEmitter_CleanupErrorReturnsFallbackEventAndError(t *testing.T) { + start := time.Date(2026, 6, 3, 9, 0, 0, 0, time.UTC) + cleanupErr := errors.New("cleanup failed") + emitter := Emitter{Cleaner: staticCleaner{err: cleanupErr}} + + got, err := emitter.Emit(context.Background(), "sys", []Segment{ + {Start: start, End: start.Add(time.Second), Text: "raw"}, + }, nil) + + if len(got) != 1 || got[0].Text != "raw" { + t.Fatalf("events = %+v", got) + } + if err == nil || !strings.Contains(err.Error(), cleanupErr.Error()) { + t.Fatalf("err = %v, want cleanup error", err) + } +} + +type staticCleaner struct { + text string + err error +} + +func (c staticCleaner) Cleanup(context.Context, string, []string) (string, error) { + return c.text, c.err +} + +type identityCleaner struct{} + +func (identityCleaner) Cleanup(_ context.Context, text string, _ []string) (string, error) { + return text, nil +} + +type staticSpeakerLookup struct { + attribution timeline.SpeakerAttribution +} + +func (l staticSpeakerLookup) Coverage( + time.Time, + time.Time, + timeline.SpeakerLookupOptions, +) timeline.SpeakerAttribution { + return l.attribution +} diff --git a/internal/speech/segment.go b/internal/speech/segment.go new file mode 100644 index 0000000..49e96ed --- /dev/null +++ b/internal/speech/segment.go @@ -0,0 +1,45 @@ +package speech + +import ( + "strings" + "time" + + "github.com/odsod/recorder/internal/protocol/whisper" +) + +// Segment is one timestamped speech span ready for cleanup and emission. +type Segment struct { + Start time.Time + End time.Time + Text string +} + +// FromWhisper converts a Whisper response into wall-clock speech segments. +func FromWhisper(resp whisper.TranscribeResponse, chunkStart, chunkEnd time.Time) []Segment { + if len(resp.Segments) > 0 { + segments := make([]Segment, 0, len(resp.Segments)) + for _, segment := range resp.Segments { + text := strings.TrimSpace(segment.Text) + if text == "" { + continue + } + start := chunkStart.Add(durationFromSeconds(segment.StartSec)) + end := chunkStart.Add(durationFromSeconds(segment.EndSec)) + if !end.After(start) { + end = start.Add(time.Second) + } + segments = append(segments, Segment{Start: start, End: end, Text: text}) + } + return segments + } + + text := strings.TrimSpace(resp.Text) + if text == "" { + return nil + } + return []Segment{{Start: chunkStart, End: chunkEnd, Text: text}} +} + +func durationFromSeconds(seconds float64) time.Duration { + return time.Duration(seconds * float64(time.Second)) +} diff --git a/internal/speech/segment_test.go b/internal/speech/segment_test.go new file mode 100644 index 0000000..e54843f --- /dev/null +++ b/internal/speech/segment_test.go @@ -0,0 +1,78 @@ +package speech + +import ( + "testing" + "time" + + "github.com/odsod/recorder/internal/protocol/whisper" +) + +func TestFromWhisper_UsesVerboseSegments(t *testing.T) { + start := time.Date(2026, 6, 3, 9, 0, 0, 0, time.UTC) + end := start.Add(30 * time.Second) + + got := FromWhisper(whisper.TranscribeResponse{ + Text: "full text", + Segments: []whisper.Segment{ + {StartSec: 1.5, EndSec: 3.25, Text: " first segment "}, + {StartSec: 4, EndSec: 4, Text: "second segment"}, + }, + }, start, end) + + if len(got) != 2 { + t.Fatalf("got %d segments, want 2", len(got)) + } + if !got[0].Start.Equal(start.Add(1500*time.Millisecond)) || !got[0].End.Equal(start.Add(3250*time.Millisecond)) { + t.Fatalf("first segment times = %s-%s", got[0].Start, got[0].End) + } + if got[0].Text != "first segment" { + t.Fatalf("first segment text = %q", got[0].Text) + } +} + +func TestFromWhisper_TextFallback(t *testing.T) { + start := time.Date(2026, 6, 3, 9, 0, 0, 0, time.UTC) + end := start.Add(30 * time.Second) + + got := FromWhisper(whisper.TranscribeResponse{Text: " full text "}, start, end) + + if len(got) != 1 { + t.Fatalf("got %d segments, want 1", len(got)) + } + if !got[0].Start.Equal(start) || !got[0].End.Equal(end) || got[0].Text != "full text" { + t.Fatalf("segment = %+v", got[0]) + } +} + +func TestFromWhisper_SkipsEmptySegmentText(t *testing.T) { + start := time.Date(2026, 6, 3, 9, 0, 0, 0, time.UTC) + + got := FromWhisper(whisper.TranscribeResponse{ + Segments: []whisper.Segment{ + {StartSec: 1, EndSec: 2, Text: " "}, + {StartSec: 3, EndSec: 4, Text: "kept"}, + }, + }, start, start.Add(10*time.Second)) + + if len(got) != 1 || got[0].Text != "kept" { + t.Fatalf("segments = %+v", got) + } +} + +func TestFromWhisper_InvalidDurationFallback(t *testing.T) { + start := time.Date(2026, 6, 3, 9, 0, 0, 0, time.UTC) + + got := FromWhisper(whisper.TranscribeResponse{ + Segments: []whisper.Segment{ + {StartSec: 4, EndSec: 4, Text: "segment"}, + }, + }, start, start.Add(10*time.Second)) + + if len(got) != 1 { + t.Fatalf("got %d segments, want 1", len(got)) + } + wantEnd := start.Add(5 * time.Second) + if !got[0].End.Equal(wantEnd) { + t.Fatalf("end = %s, want %s", got[0].End, wantEnd) + } +} diff --git a/internal/timeline/participant.go b/internal/timeline/participant.go new file mode 100644 index 0000000..20f0d06 --- /dev/null +++ b/internal/timeline/participant.go @@ -0,0 +1,55 @@ +package timeline + +import "sync" + +// ParticipantSet tracks known participants for the current meeting. +type ParticipantSet struct { + mu sync.Mutex + names map[string]struct{} +} + +// NewParticipantSet creates an empty participant set. +func NewParticipantSet() *ParticipantSet { + return &ParticipantSet{names: make(map[string]struct{})} +} + +// Update merges names into the set and returns only newly seen names. +func (p *ParticipantSet) Update(names map[string]struct{}) map[string]struct{} { + p.mu.Lock() + defer p.mu.Unlock() + + newNames := make(map[string]struct{}) + for name := range names { + if name == "" { + continue + } + if _, ok := p.names[name]; ok { + continue + } + p.names[name] = struct{}{} + newNames[name] = struct{}{} + } + if len(newNames) == 0 { + return nil + } + return newNames +} + +// GetAll returns a snapshot of all known participants. +func (p *ParticipantSet) GetAll() map[string]struct{} { + p.mu.Lock() + defer p.mu.Unlock() + + all := make(map[string]struct{}, len(p.names)) + for name := range p.names { + all[name] = struct{}{} + } + return all +} + +// Reset clears all known participants. +func (p *ParticipantSet) Reset() { + p.mu.Lock() + defer p.mu.Unlock() + p.names = make(map[string]struct{}) +} diff --git a/internal/timeline/speaker.go b/internal/timeline/speaker.go index 94ec51b..59433d8 100644 --- a/internal/timeline/speaker.go +++ b/internal/timeline/speaker.go @@ -1,6 +1,8 @@ package timeline import ( + "slices" + "sort" "sync" "time" ) @@ -8,7 +10,10 @@ import ( // SpeakerChange records a speaker transition at a point in time. type SpeakerChange struct { Time time.Time - Name string // empty string means a speaker stopped + Name string + // Active reports whether Name became active or inactive at Time. + // Empty Name with Active=false means all speakers became inactive. + Active bool } // SpeakerTimeline is a time-indexed log of speaker start/stop events with LRU eviction. @@ -23,11 +28,49 @@ func NewSpeakerTimeline(maxAgeSecs int) *SpeakerTimeline { return &SpeakerTimeline{maxAgeSec: float64(maxAgeSecs)} } -// Append records a speaker change at the given timestamp. +// Append records a full active-speaker-set change at the given timestamp. func (t *SpeakerTimeline) Append(ts time.Time, name string) { + active := make(map[string]struct{}) + if name != "" { + active[name] = struct{}{} + } + t.SetActive(ts, active) +} + +// SetSpeakerActive records an independent active/inactive transition for one speaker. +func (t *SpeakerTimeline) SetSpeakerActive(ts time.Time, name string, active bool) { + if name == "" { + return + } + t.mu.Lock() + defer t.mu.Unlock() + t.changes = append(t.changes, SpeakerChange{Time: ts, Name: name, Active: active}) + t.evict() +} + +// SetActive records the full active speaker set at the given timestamp. +func (t *SpeakerTimeline) SetActive(ts time.Time, active map[string]struct{}) { t.mu.Lock() defer t.mu.Unlock() - t.changes = append(t.changes, SpeakerChange{Time: ts, Name: name}) + + current := t.activeAtLocked(ts) + for name := range current { + if _, ok := active[name]; !ok { + t.changes = append(t.changes, SpeakerChange{Time: ts, Name: name, Active: false}) + } + } + + names := make([]string, 0, len(active)) + for name := range active { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + if _, ok := current[name]; !ok { + t.changes = append(t.changes, SpeakerChange{Time: ts, Name: name, Active: true}) + } + } + t.evict() } @@ -40,73 +83,99 @@ type SpeakerDuration struct { // SpeakersInWithDurations returns speakers active during [start, end], ordered // by total speaking time (dominant speaker first), with durations included. func (t *SpeakerTimeline) SpeakersInWithDurations(start, end time.Time) []SpeakerDuration { + attribution := t.Coverage(start, end, SpeakerLookupOptions{}) + entries := make([]SpeakerDuration, 0, len(attribution.Candidates)) + for _, candidate := range attribution.Candidates { + entries = append(entries, SpeakerDuration{ + Name: candidate.Name, + Duration: candidate.Coverage, + }) + } + return entries +} + +// SpeakersIn returns speakers active during [start, end], ordered by total +// speaking time (dominant speaker first). +func (t *SpeakerTimeline) SpeakersIn(start, end time.Time) []string { + entries := t.SpeakersInWithDurations(start, end) + result := make([]string, len(entries)) + for i, e := range entries { + result[i] = e.Name + } + return result +} + +// SpeakerLookupOptions controls percentage-based speaker attribution. +type SpeakerLookupOptions struct { + MinCandidatePct float64 + MinCandidateDuration time.Duration +} + +// SpeakerCandidate is one active speaker observed in a lookup window. +type SpeakerCandidate struct { + Name string + Coverage time.Duration + CoveragePct float64 +} + +// SpeakerAttribution is the ranked speaker coverage for a lookup window. +type SpeakerAttribution struct { + Candidates []SpeakerCandidate +} + +// Coverage returns active speakers in [start, end], ranked by coverage percentage. +func (t *SpeakerTimeline) Coverage(start, end time.Time, opts SpeakerLookupOptions) SpeakerAttribution { t.mu.Lock() defer t.mu.Unlock() - type span struct { - name string - spanStart time.Time - spanEnd time.Time + if !end.After(start) { + return SpeakerAttribution{} } - activeSet := make(map[string]time.Time) - var spans []span + window := end.Sub(start) + active := make(map[string]struct{}) + coverage := make(map[string]time.Duration) + cursor := start for _, c := range t.changes { + if !c.Time.After(start) { + applyChange(active, c) + continue + } if c.Time.After(end) { break } - if !c.Time.After(start) { - if c.Name != "" { - activeSet[c.Name] = start - } else { - activeSet = make(map[string]time.Time) - } - } else { - if c.Name != "" { - activeSet[c.Name] = c.Time - } else { - for name, spanStart := range activeSet { - spans = append(spans, span{name: name, spanStart: spanStart, spanEnd: c.Time}) - } - activeSet = make(map[string]time.Time) - } - } - } - - for name, spanStart := range activeSet { - spans = append(spans, span{name: name, spanStart: spanStart, spanEnd: end}) - } - - durations := make(map[string]time.Duration) - for _, s := range spans { - durations[s.name] += s.spanEnd.Sub(s.spanStart) + addCoverage(coverage, active, c.Time.Sub(cursor)) + applyChange(active, c) + cursor = c.Time } + addCoverage(coverage, active, end.Sub(cursor)) - entries := make([]SpeakerDuration, 0, len(durations)) - for name, dur := range durations { - entries = append(entries, SpeakerDuration{name, dur}) + candidates := make([]SpeakerCandidate, 0, len(coverage)) + for name, duration := range coverage { + pct := duration.Seconds() / window.Seconds() + if duration < opts.MinCandidateDuration { + continue + } + if pct+floatEpsilon < opts.MinCandidatePct { + continue + } + candidates = append(candidates, SpeakerCandidate{ + Name: name, + Coverage: duration, + CoveragePct: pct, + }) } - for i := range entries { - for j := i + 1; j < len(entries); j++ { - if entries[j].Duration > entries[i].Duration { - entries[i], entries[j] = entries[j], entries[i] + slices.SortFunc(candidates, func(a, b SpeakerCandidate) int { + if diff := b.Coverage - a.Coverage; diff != 0 { + if diff > 0 { + return 1 } + return -1 } - } - - return entries -} - -// SpeakersIn returns speakers active during [start, end], ordered by total -// speaking time (dominant speaker first). -func (t *SpeakerTimeline) SpeakersIn(start, end time.Time) []string { - entries := t.SpeakersInWithDurations(start, end) - result := make([]string, len(entries)) - for i, e := range entries { - result[i] = e.Name - } - return result + return stringsCompare(a.Name, b.Name) + }) + return SpeakerAttribution{Candidates: candidates} } func (t *SpeakerTimeline) evict() { @@ -128,50 +197,47 @@ func (t *SpeakerTimeline) evict() { } } -// ParticipantSet tracks unique participant names with change detection. -type ParticipantSet struct { - mu sync.Mutex - names map[string]struct{} -} +const floatEpsilon = 1e-9 -// NewParticipantSet creates an empty participant set. -func NewParticipantSet() *ParticipantSet { - return &ParticipantSet{names: make(map[string]struct{})} +func (t *SpeakerTimeline) activeAtLocked(ts time.Time) map[string]struct{} { + active := make(map[string]struct{}) + for _, c := range t.changes { + if c.Time.After(ts) { + break + } + applyChange(active, c) + } + return active } -// Update adds names to the set and returns only newly seen names. -func (p *ParticipantSet) Update(names map[string]struct{}) map[string]struct{} { - p.mu.Lock() - defer p.mu.Unlock() - - newNames := make(map[string]struct{}) - for name := range names { - if _, exists := p.names[name]; !exists { - newNames[name] = struct{}{} - p.names[name] = struct{}{} - } +func applyChange(active map[string]struct{}, c SpeakerChange) { + if c.Name == "" && !c.Active { + clear(active) + return } - if len(newNames) == 0 { - return nil + if c.Active { + active[c.Name] = struct{}{} + } else { + delete(active, c.Name) } - return newNames } -// GetAll returns a copy of all known participant names. -func (p *ParticipantSet) GetAll() map[string]struct{} { - p.mu.Lock() - defer p.mu.Unlock() - - result := make(map[string]struct{}, len(p.names)) - for name := range p.names { - result[name] = struct{}{} +func addCoverage(coverage map[string]time.Duration, active map[string]struct{}, duration time.Duration) { + if duration <= 0 { + return + } + for name := range active { + coverage[name] += duration } - return result } -// Reset clears all tracked participants. -func (p *ParticipantSet) Reset() { - p.mu.Lock() - defer p.mu.Unlock() - p.names = make(map[string]struct{}) +func stringsCompare(a, b string) int { + switch { + case a < b: + return -1 + case a > b: + return 1 + default: + return 0 + } } diff --git a/internal/timeline/timeline_test.go b/internal/timeline/timeline_test.go index 39b8514..cdf190f 100644 --- a/internal/timeline/timeline_test.go +++ b/internal/timeline/timeline_test.go @@ -97,7 +97,7 @@ func TestSpeakerTimeline_Eviction(tt *testing.T) { result := tl.SpeakersIn(ts("09:00:00"), ts("09:00:30")) assertStrings(tt, result, nil) - result = tl.SpeakersIn(ts("09:04:30"), ts("09:05:00")) + result = tl.SpeakersIn(ts("09:04:30"), ts("09:05:01")) assertStrings(tt, result, []string{"Bob"}) } @@ -108,33 +108,20 @@ func TestSpeakerTimeline_EmptyTimeline(tt *testing.T) { } func TestSpeakerTimeline_ConcurrentSpeakers(tt *testing.T) { - // Simulates multi-speaker timeline: both Alice and Bob start speaking, - // their events interleave. tl := NewSpeakerTimeline(600) - tl.Append(ts("09:00:00"), "Alice") - tl.Append(ts("09:00:02"), "Bob") - // Alice stops - tl.Append(ts("09:00:10"), "") - // Bob continues (re-appears after the stop-all) - tl.Append(ts("09:00:10"), "Bob") - tl.Append(ts("09:00:15"), "") + tl.SetSpeakerActive(ts("09:00:00"), "Alice", true) + tl.SetSpeakerActive(ts("09:00:02"), "Bob", true) + tl.SetSpeakerActive(ts("09:00:10"), "Alice", false) + tl.SetSpeakerActive(ts("09:00:15"), "Bob", false) result := tl.SpeakersIn(ts("09:00:00"), ts("09:00:15")) - // Bob: 2s + 5s = 7s (two spans: 09:00:02-09:00:10 via first clear, 09:00:10-09:00:15) - // Wait β€” with "" clearing all, Alice: 09:00:00-09:00:10 = 10s, Bob: 09:00:02-09:00:10 + 09:00:10-09:00:15 = 8+5=13s - // Actually: first "" at 09:00:10 closes all active (Alice started at 00, Bob at 02). - // Alice span: 00-10 = 10s. Bob first span: 02-10 = 8s. - // Then Bob starts again at 10, stops at 15: 5s. Bob total = 13s. - // Bob > Alice, so Bob first. assertStrings(tt, result, []string{"Bob", "Alice"}) } func TestSpeakerTimeline_WithDurations(tt *testing.T) { tl := NewSpeakerTimeline(600) - // Alice speaks 09:00:00-09:00:05 (5s) tl.Append(ts("09:00:00"), "Alice") tl.Append(ts("09:00:05"), "") - // Bob speaks 09:00:05-09:00:20 (15s) tl.Append(ts("09:00:05"), "Bob") tl.Append(ts("09:00:20"), "") @@ -150,6 +137,70 @@ func TestSpeakerTimeline_WithDurations(tt *testing.T) { } } +func TestSpeakerTimeline_CoverageOverlappingSpeakers(tt *testing.T) { + tl := NewSpeakerTimeline(600) + tl.SetSpeakerActive(ts("09:00:00"), "Alice", true) + tl.SetSpeakerActive(ts("09:00:02"), "Bob", true) + tl.SetSpeakerActive(ts("09:00:08"), "Alice", false) + tl.SetSpeakerActive(ts("09:00:10"), "Bob", false) + + got := tl.Coverage(ts("09:00:00"), ts("09:00:10"), SpeakerLookupOptions{ + MinCandidatePct: 0.05, + MinCandidateDuration: 250 * time.Millisecond, + }) + + assertCandidates(tt, got.Candidates, []wantCandidate{ + {Name: "Alice", Coverage: 8 * time.Second, Pct: 0.8}, + {Name: "Bob", Coverage: 8 * time.Second, Pct: 0.8}, + }) +} + +func TestSpeakerTimeline_CoverageFiltersByLowThreshold(tt *testing.T) { + tl := NewSpeakerTimeline(600) + tl.SetSpeakerActive(ts("09:00:00"), "Alice", true) + tl.SetSpeakerActive(ts("09:00:10"), "Alice", false) + tl.SetSpeakerActive(ts("09:00:09"), "Bob", true) + tl.SetSpeakerActive(ts("09:00:10"), "Bob", false) + + got := tl.Coverage(ts("09:00:00"), ts("09:00:10"), SpeakerLookupOptions{ + MinCandidatePct: 0.05, + MinCandidateDuration: 250 * time.Millisecond, + }) + + assertCandidates(tt, got.Candidates, []wantCandidate{ + {Name: "Alice", Coverage: 10 * time.Second, Pct: 1.0}, + {Name: "Bob", Coverage: 1 * time.Second, Pct: 0.1}, + }) +} + +func TestSpeakerTimeline_CoverageFiltersByDuration(tt *testing.T) { + tl := NewSpeakerTimeline(600) + tl.SetSpeakerActive(ts("09:00:00"), "Alice", true) + tl.SetSpeakerActive(ts("09:00:00").Add(100*time.Millisecond), "Alice", false) + + got := tl.Coverage(ts("09:00:00"), ts("09:00:01"), SpeakerLookupOptions{ + MinCandidatePct: 0.05, + MinCandidateDuration: 250 * time.Millisecond, + }) + + if len(got.Candidates) != 0 { + tt.Fatalf("got %v, want no candidates", got.Candidates) + } +} + +func TestSpeakerTimeline_CoverageNoActiveSpeakers(tt *testing.T) { + tl := NewSpeakerTimeline(600) + + got := tl.Coverage(ts("09:00:00"), ts("09:00:10"), SpeakerLookupOptions{ + MinCandidatePct: 0.05, + MinCandidateDuration: 250 * time.Millisecond, + }) + + if len(got.Candidates) != 0 { + tt.Fatalf("got %v, want no candidates", got.Candidates) + } +} + func TestParticipantSet_InitialUpdate(tt *testing.T) { ps := NewParticipantSet() newNames := ps.Update(setOf("Alice", "Bob")) @@ -188,6 +239,31 @@ func TestParticipantSet_Reset(tt *testing.T) { assertSet(tt, newNames, setOf("Alice")) } +type wantCandidate struct { + Name string + Coverage time.Duration + Pct float64 +} + +func assertCandidates(tt *testing.T, got []SpeakerCandidate, want []wantCandidate) { + tt.Helper() + if len(got) != len(want) { + tt.Fatalf("got %v, want %v", got, want) + } + for i, candidate := range got { + w := want[i] + if candidate.Name != w.Name { + tt.Fatalf("candidate[%d].Name = %q, want %q", i, candidate.Name, w.Name) + } + if candidate.Coverage != w.Coverage { + tt.Fatalf("candidate[%d].Coverage = %s, want %s", i, candidate.Coverage, w.Coverage) + } + if diff := candidate.CoveragePct - w.Pct; diff < -0.0001 || diff > 0.0001 { + tt.Fatalf("candidate[%d].CoveragePct = %.4f, want %.4f", i, candidate.CoveragePct, w.Pct) + } + } +} + func assertStrings(tt *testing.T, got, want []string) { tt.Helper() if len(got) != len(want) { diff --git a/internal/transcript/event_test.go b/internal/transcript/event_test.go index 1373cee..e808692 100644 --- a/internal/transcript/event_test.go +++ b/internal/transcript/event_test.go @@ -19,6 +19,21 @@ func TestEvent_String_SysWithSpeaker(t *testing.T) { } } +func TestEvent_String_SysWithSpeakerPercentages(t *testing.T) { + e := Event{ + Time: ts("15:04:32"), + Type: Speech, + Source: "sys", + Speaker: "Alice 55% / Bob 35%", + Text: "Hello world", + } + got := e.String() + want := "[15:04:32] πŸ”Š **sys** [Alice 55% / Bob 35%] Hello world" + if got != want { + t.Errorf("got %q, want %q", got, want) + } +} + func TestEvent_String_MicNoSpeaker(t *testing.T) { e := Event{Time: ts("09:00:00"), Type: Speech, Source: "mic", Text: "Some text"} got := e.String() diff --git a/internal/transcript/parse_test.go b/internal/transcript/parse_test.go index e76dc32..4f99802 100644 --- a/internal/transcript/parse_test.go +++ b/internal/transcript/parse_test.go @@ -84,3 +84,21 @@ func TestParse_SkipsInvalidLines(t *testing.T) { t.Fatalf("parsed %d events, want 1", len(got.Events)) } } + +func TestParse_SpeakerPercentages(t *testing.T) { + got, err := Parse([]byte("[09:00:00] πŸ”Š **sys** [Alice 55% / Bob 35%] hello\n")) + if err != nil { + t.Fatal(err) + } + + if len(got.Events) != 1 { + t.Fatalf("parsed %d events, want 1", len(got.Events)) + } + e := got.Events[0] + if e.Speaker != "Alice 55% / Bob 35%" { + t.Fatalf("Speaker = %q", e.Speaker) + } + if e.Text != "hello" { + t.Fatalf("Text = %q", e.Text) + } +}