From 6e1f8727a268c3bf767c478b026c7844946170c2 Mon Sep 17 00:00:00 2001 From: joao Date: Thu, 11 Jun 2026 16:54:05 -0300 Subject: [PATCH 1/3] feat(captions): generate auto-captions as styled subtitles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Auto-captions previously always became plain text annotations (fixed white Inter style). The editor already has a styled-subtitle system (SubtitleOverlay with classic/minimal/bold/boxed/cinematic templates) that renders in preview and export — wire the caption pipeline to it. - captionSegmentsToSubtitleItems: same grouping/dedupe/finalize chain as the annotation converter, emitting SubtitleItem[]; timeline-gap reconciliation skipped (subtitles render serially in a fixed slot). - 'Caption style' selector in the auto-captions dialog: styled subtitles (default) or the original text annotations. - Subtitle generation replaces existing subtitleRegions (a generation describes the whole video), matching the sidebar generator. With this, the in-app pipeline covers the Electron subtitle script's remaining advantage; the script and generate-subtitles IPC are now candidates for retirement. Co-Authored-By: Claude Fable 5 --- src/components/video-editor/VideoEditor.tsx | 93 ++++++++++++++----- src/i18n/locales/ar/editor.json | 5 +- src/i18n/locales/en/editor.json | 5 +- src/i18n/locales/es/editor.json | 5 +- src/i18n/locales/fr/editor.json | 5 +- src/i18n/locales/it/editor.json | 5 +- src/i18n/locales/ja-JP/editor.json | 5 +- src/i18n/locales/ko-KR/editor.json | 5 +- src/i18n/locales/pt-BR/editor.json | 5 +- src/i18n/locales/ru/editor.json | 5 +- src/i18n/locales/tr/editor.json | 5 +- src/i18n/locales/vi/editor.json | 5 +- src/i18n/locales/zh-CN/editor.json | 5 +- src/i18n/locales/zh-TW/editor.json | 5 +- src/lib/captioning/annotationsFromCaptions.ts | 39 +++++++- src/lib/captioning/index.ts | 1 + 16 files changed, 159 insertions(+), 39 deletions(-) diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index 984d5cdb55..6660fe1106 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -28,6 +28,7 @@ import { type Locale } from "@/i18n/config"; import { getAvailableLocales, getLocaleName } from "@/i18n/loader"; import { captionSegmentsToAnnotationRegions, + captionSegmentsToSubtitleItems, extractMono16kFromVideoUrl, MAX_CAPTION_AUDIO_SEC, reconcileAutoCaptionTimelineGaps, @@ -393,6 +394,7 @@ export default function VideoEditor() { const showCursorSettings = hasEditableCursorRecording; const { locale, setLocale, t: rawT } = useI18n(); const [captionLanguage, setCaptionLanguage] = useState(() => captionLanguageForLocale(locale)); + const [captionOutput, setCaptionOutput] = useState<"subtitles" | "annotations">("subtitles"); const t = useScopedT("editor"); const ts = useScopedT("settings"); const availableLocales = getAvailableLocales(); @@ -2688,48 +2690,74 @@ export default function VideoEditor() { })) : segmentsRaw; - let { regions, nextNumericId, nextZIndex } = captionSegmentsToAnnotationRegions( - segments, - nextAnnotationIdRef.current, - nextAnnotationZIndexRef.current, - { + let createdCount = 0; + if (captionOutput === "subtitles") { + let items = captionSegmentsToSubtitleItems(segments, { minWordsPerCaption: minW, maxWordsPerCaption: maxW, timestampGranularity: granularity, - }, - ); - - if (regions.length === 0 && segments.length > 0) { - ({ regions, nextNumericId, nextZIndex } = captionSegmentsToAnnotationRegions( + }); + if (items.length === 0 && segments.length > 0) { + items = captionSegmentsToSubtitleItems(segments, { + minWordsPerCaption: 1, + maxWordsPerCaption: Number.MAX_SAFE_INTEGER, + timestampGranularity: granularity, + }); + } + if (items.length === 0) { + toast.dismiss(AUTO_CAPTION_PROGRESS_TOAST_ID); + toast.info(t("autoCaptions.noneHeard")); + return; + } + // A generation describes the whole video: replace, like the sidebar + // subtitle generator does, instead of stacking duplicate lines. + pushState({ subtitleRegions: items, showSubtitles: true }); + createdCount = items.length; + } else { + let { regions, nextNumericId, nextZIndex } = captionSegmentsToAnnotationRegions( segments, nextAnnotationIdRef.current, nextAnnotationZIndexRef.current, { - minWordsPerCaption: 1, - maxWordsPerCaption: Number.MAX_SAFE_INTEGER, + minWordsPerCaption: minW, + maxWordsPerCaption: maxW, timestampGranularity: granularity, }, - )); - } + ); - if (regions.length === 0) { - toast.dismiss(AUTO_CAPTION_PROGRESS_TOAST_ID); - toast.info(t("autoCaptions.noneHeard")); - return; - } + if (regions.length === 0 && segments.length > 0) { + ({ regions, nextNumericId, nextZIndex } = captionSegmentsToAnnotationRegions( + segments, + nextAnnotationIdRef.current, + nextAnnotationZIndexRef.current, + { + minWordsPerCaption: 1, + maxWordsPerCaption: Number.MAX_SAFE_INTEGER, + timestampGranularity: granularity, + }, + )); + } - pushState((prev) => ({ annotationRegions: [...prev.annotationRegions, ...regions] })); - nextAnnotationIdRef.current = nextNumericId; - nextAnnotationZIndexRef.current = nextZIndex; + if (regions.length === 0) { + toast.dismiss(AUTO_CAPTION_PROGRESS_TOAST_ID); + toast.info(t("autoCaptions.noneHeard")); + return; + } + + pushState((prev) => ({ annotationRegions: [...prev.annotationRegions, ...regions] })); + nextAnnotationIdRef.current = nextNumericId; + nextAnnotationZIndexRef.current = nextZIndex; + createdCount = regions.length; + } toast.dismiss(AUTO_CAPTION_PROGRESS_TOAST_ID); const minutesTrunc = String(Math.round(MAX_CAPTION_AUDIO_SEC / 60)); if (truncated) { - toast.success(t("autoCaptions.done", { count: String(regions.length) }), { + toast.success(t("autoCaptions.done", { count: String(createdCount) }), { description: t("autoCaptions.truncated", { minutes: minutesTrunc }), }); } else { - toast.success(t("autoCaptions.done", { count: String(regions.length) })); + toast.success(t("autoCaptions.done", { count: String(createdCount) })); } } catch (e) { console.error(e); @@ -2741,7 +2769,7 @@ export default function VideoEditor() { setIsAutoCaptioning(false); } }, - [videoPath, trimRegions, pushState, t, captionLanguage], + [videoPath, trimRegions, pushState, t, captionLanguage, captionOutput], ); const handleSaveDiagnostic = useCallback(async () => { @@ -2821,6 +2849,21 @@ export default function VideoEditor() { {t("autoCaptions.dialogDescription")}
+
+ + +