refactor: establish refactoring roadmap

internal/protocol/whisper/client.go

@@ -50,6 +50,15 @@ type TranscribeRequest struct {
 type TranscribeResponse struct {
 	// Text is the transcribed speech with normalized whitespace.
 	Text string
+	// Segments are timestamped transcription chunks from verbose_json responses.
+	Segments []Segment
+}
+
+// Segment is a timestamped transcription segment relative to the uploaded audio.
+type Segment struct {
+	StartSec float64
+	EndSec   float64
+	Text     string
 }
 
 // Client communicates with an OpenAI-compatible audio transcription endpoint.
@@ -80,7 +89,7 @@ func (c *Client) Transcribe(ctx context.Context, req TranscribeRequest) (Transcr
 	if err := writer.WriteField("model", "whisper-1"); err != nil {
 		return TranscribeResponse{}, err
 	}
-	if err := writer.WriteField("response_format", "json"); err != nil {
+	if err := writer.WriteField("response_format", "verbose_json"); err != nil {
 		return TranscribeResponse{}, err
 	}
 	if err := writer.Close(); err != nil {
@@ -111,14 +120,43 @@ func (c *Client) Transcribe(ctx context.Context, req TranscribeRequest) (Transcr
 		return TranscribeResponse{}, &TranscriptionError{StatusCode: resp.StatusCode, Body: respBody}
 	}
 
-	var result struct {
-		Text string `json:"text"`
-	}
+	var result wireResponse
 	if err := json.Unmarshal(respBody, &result); err != nil {
 		return TranscribeResponse{}, err
 	}
 
-	text := strings.TrimSpace(result.Text)
-	text = spaceRe.ReplaceAllString(text, " ")
-	return TranscribeResponse{Text: text}, nil
+	return normalizeResponse(result), nil
+}
+
+type wireResponse struct {
+	Text     string        `json:"text"`
+	Segments []wireSegment `json:"segments"`
+}
+
+type wireSegment struct {
+	StartSec float64 `json:"start"`
+	EndSec   float64 `json:"end"`
+	Text     string  `json:"text"`
+}
+
+func normalizeResponse(result wireResponse) TranscribeResponse {
+	text := normalizeText(result.Text)
+	segments := make([]Segment, 0, len(result.Segments))
+	for _, s := range result.Segments {
+		segmentText := normalizeText(s.Text)
+		if segmentText == "" {
+			continue
+		}
+		segments = append(segments, Segment{
+			StartSec: s.StartSec,
+			EndSec:   s.EndSec,
+			Text:     segmentText,
+		})
+	}
+	return TranscribeResponse{Text: text, Segments: segments}
+}
+
+func normalizeText(text string) string {
+	text = strings.TrimSpace(text)
+	return spaceRe.ReplaceAllString(text, " ")
 }

internal/protocol/whisper/client_test.go

@@ -26,8 +26,8 @@ func TestTranscribe_Success(t *testing.T) {
 		if model := r.FormValue("model"); model != "whisper-1" {
 			t.Errorf("expected model whisper-1, got %s", model)
 		}
-		if rf := r.FormValue("response_format"); rf != "json" {
-			t.Errorf("expected response_format json, got %s", rf)
+		if rf := r.FormValue("response_format"); rf != "verbose_json" {
+			t.Errorf("expected response_format verbose_json, got %s", rf)
 		}
 
 		file, header, err := r.FormFile("file")
@@ -44,7 +44,13 @@ func TestTranscribe_Success(t *testing.T) {
 			t.Errorf("unexpected file content: %q", string(data))
 		}
 
-		resp := map[string]string{"text": " Hello   world "}
+		resp := map[string]any{
+			"text": " Hello   world ",
+			"segments": []map[string]any{
+				{"start": 0.5, "end": 1.25, "text": " Hello   segment "},
+				{"start": 1.25, "end": 2.0, "text": "   "},
+			},
+		}
 		_ = json.NewEncoder(w).Encode(resp)
 	}))
 	defer srv.Close()
@@ -63,6 +69,13 @@ func TestTranscribe_Success(t *testing.T) {
 	if resp.Text != "Hello world" {
 		t.Errorf("expected 'Hello world', got %q", resp.Text)
 	}
+	if len(resp.Segments) != 1 {
+		t.Fatalf("expected 1 segment, got %d", len(resp.Segments))
+	}
+	seg := resp.Segments[0]
+	if seg.StartSec != 0.5 || seg.EndSec != 1.25 || seg.Text != "Hello segment" {
+		t.Errorf("unexpected segment: %+v", seg)
+	}
 }
 
 func TestTranscribe_NonOKStatus(t *testing.T) {

internal/recorder/recorder.go

@@ -10,6 +10,7 @@ import (
 	"github.com/odsod/recorder/internal/lock"
 	"github.com/odsod/recorder/internal/segment"
 	"github.com/odsod/recorder/internal/signals"
+	"github.com/odsod/recorder/internal/speech"
 	"github.com/odsod/recorder/internal/timeline"
 	"github.com/odsod/recorder/internal/transcript"
 )
@@ -30,6 +31,7 @@ type Recorder struct {
 	meetingState    *timeline.MeetingState
 	silenceMonitor  *signals.SilenceMonitor
 	segmenter       *segment.IncrementalSegmenter
+	speechEmitter   *speech.Emitter
 	lastSystemText  string
 	chunkNum        int
 	lastFlushedTime time.Time
@@ -51,6 +53,19 @@ func New(ctx context.Context, cfg config.Config, svc Services) (*Recorder, error
 		silenceMonitor:  signals.NewSilenceMonitor(cfg.Signals.SilenceThresholdS),
 		lastPplSet:      make(map[string]struct{}),
 	}
+	r.speechEmitter = &speech.Emitter{
+		Cleaner:       svc.Cleaner,
+		SpeakerLookup: r.speakerTimeline,
+		Participants:  r.currentParticipants,
+		Deduper: speech.NearbyDeduper{
+			Threshold: cfg.Dedup.Threshold,
+			Tolerance: 5 * time.Second,
+		},
+		LookupOptions: timeline.SpeakerLookupOptions{
+			MinCandidatePct:      minSpeakerCandidatePct,
+			MinCandidateDuration: minSpeakerCandidateDuration,
+		},
+	}
 
 	r.segmenter = segment.NewSegmenter(ctx, svc.SegmentHandler, func(e transcript.Event) {
 		t.AppendEvent(e)

internal/recorder/transcribe.go

@@ -2,18 +2,22 @@ package recorder
 
 import (
 	"context"
-	"fmt"
 	"log/slog"
 	"maps"
 	"slices"
+	"strings"
 	"time"
 
 	"github.com/odsod/recorder/internal/protocol/whisper"
-	"github.com/odsod/recorder/internal/timeline"
-	"github.com/odsod/recorder/internal/transcribe"
+	"github.com/odsod/recorder/internal/speech"
 	"github.com/odsod/recorder/internal/transcript"
 )
 
+const (
+	minSpeakerCandidatePct      = 0.05
+	minSpeakerCandidateDuration = 250 * time.Millisecond
+)
+
 func (r *Recorder) transcriptionWorker(ctx context.Context, chunkCh <-chan AudioChunk) {
 	for chunk := range chunkCh {
 		slog.InfoContext(ctx, "transcribing")
@@ -43,109 +47,35 @@ func (r *Recorder) transcribeChunk(ctx context.Context, chunk AudioChunk) {
 		)
 	}
 
-	sysText := sysResp.Text
-	micText := micResp.Text
-
 	r.flushSignalEvents(ctx, chunk.StartTime, chunk.EndTime)
 
-	speakers := r.speakerTimeline.SpeakersInWithDurations(chunk.StartTime, chunk.EndTime)
-	speaker := attributeSpeaker(speakers, r.cfg.Speaker.AmbiguityRatio)
-	participants := r.currentParticipants()
-
-	switch {
-	case sysText != "":
-		cleaned, err := r.svc.Cleaner.Cleanup(ctx, sysText, participants)
-		if err != nil {
-			slog.ErrorContext(ctx, "cleanup sys failed",
-				"err", err,
-			)
-		}
-		if cleaned == "" {
-			cleaned = sysText
-		}
-		if cleaned != "" {
-			e := transcript.Event{
-				Time:    chunk.StartTime,
-				Type:    transcript.Speech,
-				Source:  "sys",
-				Text:    cleaned,
-				Speaker: speaker,
-			}
-			r.appendEvent(ctx, e)
-			r.lastSystemText = cleaned
-			r.segmenter.OnSpeech(e)
-
-			if micText != "" && !transcribe.TextsOverlap(cleaned, micText, r.cfg.Dedup.Threshold) {
-				micCleaned, err := r.svc.Cleaner.Cleanup(ctx, micText, participants)
-				if err != nil {
-					slog.ErrorContext(ctx, "cleanup mic failed",
-						"err", err,
-					)
-				}
-				if micCleaned == "" {
-					micCleaned = micText
-				}
-				if micCleaned != "" {
-					me := transcript.Event{
-						Time:    chunk.StartTime,
-						Type:    transcript.Speech,
-						Source:  "mic",
-						Text:    micCleaned,
-						Speaker: speaker,
-					}
-					r.appendEvent(ctx, me)
-					r.segmenter.OnSpeech(me)
-				}
-			}
-		}
-	case micText != "":
-		if r.lastSystemText != "" && transcribe.TextsOverlap(r.lastSystemText, micText, r.cfg.Dedup.Threshold) {
-			slog.InfoContext(ctx, "mic deduped",
-				"text", truncate(micText, 60),
-			)
-		} else {
-			cleaned, err := r.svc.Cleaner.Cleanup(ctx, micText, participants)
-			if err != nil {
-				slog.ErrorContext(ctx, "cleanup mic failed",
-					"err", err,
-				)
-			}
-			if cleaned == "" {
-				cleaned = micText
-			}
-			if cleaned != "" {
-				e := transcript.Event{
-					Time:    chunk.StartTime,
-					Type:    transcript.Speech,
-					Source:  "mic",
-					Text:    cleaned,
-					Speaker: speaker,
-				}
-				r.appendEvent(ctx, e)
-				r.segmenter.OnSpeech(e)
-			}
-		}
-	default:
-		slog.InfoContext(ctx, "no speech detected")
+	sysSegments := speech.FromWhisper(sysResp, chunk.StartTime, chunk.EndTime)
+	micSegments := speech.FromWhisper(micResp, chunk.StartTime, chunk.EndTime)
+
+	priorSystemText := r.lastSystemText
+	sysEvents, err := r.speechEmitter.Emit(ctx, "sys", sysSegments, nil)
+	if err != nil {
+		slog.ErrorContext(ctx, "emit sys speech failed", "err", err)
+	}
+	r.appendSpeechEvents(ctx, sysEvents)
+	if len(sysEvents) > 0 {
+		r.lastSystemText = joinEventText(sysEvents)
 	}
-	slog.InfoContext(ctx, "listening")
-}
 
-func attributeSpeaker(speakers []timeline.SpeakerDuration, ambiguityRatio float64) string {
-	switch {
-	case len(speakers) == 0:
-		return ""
-	case len(speakers) == 1:
-		return speakers[0].Name
-	default:
-		if float64(speakers[1].Duration) >= float64(speakers[0].Duration)*ambiguityRatio {
-			total := speakers[0].Duration + speakers[1].Duration
-			pct0 := int(float64(speakers[0].Duration) * 100 / float64(total))
-			pct1 := 100 - pct0
-			return fmt.Sprintf("%s(%d%%),%s(%d%%)", speakers[0].Name, pct0, speakers[1].Name, pct1)
-		}
-		return speakers[0].Name
+	micDedupEvents := sysEvents
+	if len(micDedupEvents) == 0 && priorSystemText != "" {
+		micDedupEvents = []transcript.Event{{Time: chunk.StartTime, Text: priorSystemText}}
 	}
+	micEvents, err := r.speechEmitter.Emit(ctx, "mic", micSegments, micDedupEvents)
+	if err != nil {
+		slog.ErrorContext(ctx, "emit mic speech failed", "err", err)
+	}
+	r.appendSpeechEvents(ctx, micEvents)
+
+	if len(sysSegments) == 0 && len(micSegments) == 0 {
+		slog.InfoContext(ctx, "no speech detected")
+	}
+	slog.InfoContext(ctx, "listening")
 }
 
 func (r *Recorder) currentParticipants() []string {
@@ -156,6 +86,23 @@ func (r *Recorder) currentParticipants() []string {
 	return slices.Sorted(maps.Keys(all))
 }
 
+func joinEventText(events []transcript.Event) string {
+	parts := make([]string, 0, len(events))
+	for _, e := range events {
+		if e.Text != "" {
+			parts = append(parts, e.Text)
+		}
+	}
+	return strings.Join(parts, " ")
+}
+
+func (r *Recorder) appendSpeechEvents(ctx context.Context, events []transcript.Event) {
+	for _, e := range events {
+		r.appendEvent(ctx, e)
+		r.segmenter.OnSpeech(e)
+	}
+}
+
 func (r *Recorder) flushSignalEvents(ctx context.Context, start, end time.Time) {
 	r.lastFlushedTime = end