diff --git a/Package.swift b/Package.swift index 2733673..62d3e11 100644 --- a/Package.swift +++ b/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version: 5.10 +// swift-tools-version: 5.9 import PackageDescription @@ -22,6 +22,9 @@ let package = Package( resources: [ .copy("Resources/AppIcon/AppIcon-512.png"), .copy("Resources/silero_vad.onnx"), + ], + swiftSettings: [ + .define("SUPPORTS_SPEECH_TRANSCRIBER"), ] ), .testTarget( diff --git a/README.md b/README.md index b75acd2..4163f72 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,8 @@ v2s will ask for permissions on first use: ## Requirements -- Speech transcription and translation require macOS 26 or newer +- **Basic speech transcription**: macOS 15 (Sequoia) or newer +- **Full features (SpeechTranscriber / AI Summarization)**: macOS 26 or newer ## Building from Source diff --git a/README.zh-CN.md b/README.zh-CN.md index 1aa601e..4e74fc0 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -66,7 +66,8 @@ v2s 的输入语言列表仅包含 Apple SpeechAnalyzer/SpeechTranscriber 路径 ## 环境要求 -- 语音转写和翻译功能需要 macOS 26 或更高版本 +- **基础功能(语音转写)**:macOS 15 (Sequoia) 或更高版本 +- **完整功能(SpeechTranscriber / AI 摘要)**:macOS 26 或更高版本 ## 从源码构建 diff --git a/Sources/V2SApp/App/AppModel.swift b/Sources/V2SApp/App/AppModel.swift index 772df66..dc3d65a 100644 --- a/Sources/V2SApp/App/AppModel.swift +++ b/Sources/V2SApp/App/AppModel.swift @@ -899,7 +899,10 @@ final class AppModel: ObservableObject { private func prepareSpeechRecognitionResourceIfNeeded( for languageID: String ) async -> LanguageResourceSystemSettingsDestination? { - guard #available(macOS 26.0, *) else { +#if !SUPPORTS_SPEECH_TRANSCRIBER + return nil +#else + guard #available(macOS 26.0, *), SpeechTranscriber.isAvailable else { return nil } @@ -946,8 +949,10 @@ final class AppModel: ObservableObject { } return nil +#endif } +#if SUPPORTS_SPEECH_TRANSCRIBER @available(macOS 26.0, *) private func ensureSpeechAssetsReady( for modules: [any SpeechModule], @@ -1065,6 +1070,7 @@ final class AppModel: ObservableObject { try await request.downloadAndInstall() } +#endif private func prepareTranslationResourceIfNeeded( from sourceLanguageID: String, diff --git a/Sources/V2SApp/Models/AppSettings.swift b/Sources/V2SApp/Models/AppSettings.swift index bc4b7b4..36034a3 100644 --- a/Sources/V2SApp/Models/AppSettings.swift +++ b/Sources/V2SApp/Models/AppSettings.swift @@ -22,7 +22,7 @@ struct AppSettings: Codable { outputLanguageID: "zh-Hans", interfaceLanguageID: nil, overlayStyle: .default, - subtitleMode: .balanced, + subtitleMode: .follow, subtitleDisplayMode: .both, glossary: [:] ) diff --git a/Sources/V2SApp/Models/InputSource.swift b/Sources/V2SApp/Models/InputSource.swift index 90dc1d8..9873100 100644 --- a/Sources/V2SApp/Models/InputSource.swift +++ b/Sources/V2SApp/Models/InputSource.swift @@ -3,6 +3,7 @@ import Foundation enum InputSourceCategory: String, CaseIterable, Codable { case application case microphone + case systemAudio func displayName(in languageID: String) -> String { switch self { @@ -10,6 +11,8 @@ enum InputSourceCategory: String, CaseIterable, Codable { return AppLocalization.string(.application, languageID: languageID) case .microphone: return AppLocalization.string(.microphone, languageID: languageID) + case .systemAudio: + return "System Audio" } } } @@ -20,6 +23,13 @@ struct InputSource: Identifiable, Hashable, Codable { let detail: String let category: InputSourceCategory + static let systemAudio = InputSource( + id: "system:audio", + name: "System Audio", + detail: "system-audio", + category: .systemAudio + ) + static let preview = InputSource( id: "preview", name: AppLocalization.string(.previewSource, languageID: "en"), diff --git a/Sources/V2SApp/Models/ModeConfig.swift b/Sources/V2SApp/Models/ModeConfig.swift index 8b4b647..2b45abf 100644 --- a/Sources/V2SApp/Models/ModeConfig.swift +++ b/Sources/V2SApp/Models/ModeConfig.swift @@ -75,19 +75,19 @@ struct ModeConfig: Sendable { let minSilenceCommitMs: Int static let balanced = ModeConfig( - firstTokenTargetMs: 300, - commitSourceTargetMs: 600, - commitTranslationTargetMs: 900, + firstTokenTargetMs: 250, + commitSourceTargetMs: 500, + commitTranslationTargetMs: 750, maxChunkAudioSec: 3.0, - minSilenceCommitMs: 280 + minSilenceCommitMs: 200 ) static let follow = ModeConfig( - firstTokenTargetMs: 200, - commitSourceTargetMs: 450, - commitTranslationTargetMs: 700, - maxChunkAudioSec: 2.2, - minSilenceCommitMs: 220 + firstTokenTargetMs: 150, + commitSourceTargetMs: 350, + commitTranslationTargetMs: 500, + maxChunkAudioSec: 1.8, + minSilenceCommitMs: 150 ) static let reading = ModeConfig( diff --git a/Sources/V2SApp/Services/LiveTranscriptionSession.swift b/Sources/V2SApp/Services/LiveTranscriptionSession.swift index 2d481bb..5bc3f46 100644 --- a/Sources/V2SApp/Services/LiveTranscriptionSession.swift +++ b/Sources/V2SApp/Services/LiveTranscriptionSession.swift @@ -164,7 +164,7 @@ final class LiveTranscriptionSession: NSObject, @unchecked Sendable { private var modernCommittedPrefixText = "" private var microphoneCaptureSession: AVCaptureSession? - private var applicationAudioCapture: ApplicationAudioCapture? + private var audioCapture: AnyAudioCapture? private var transcriptHandler: (@MainActor (RecognizedSentence) -> Void)? private var partialHandler: (@MainActor (DraftSegment?) -> Void)? @@ -245,7 +245,17 @@ final class LiveTranscriptionSession: NSObject, @unchecked Sendable { } try await requestRequiredPermissions(for: source) - if try await configureModernSpeechRecognizer(localeIdentifier: localeIdentifier) == false { + var useLegacy = false +#if SUPPORTS_SPEECH_TRANSCRIBER + if #available(macOS 26.0, *) { + useLegacy = try await configureModernSpeechRecognizer(localeIdentifier: localeIdentifier) == false + } else { + useLegacy = true + } +#else + useLegacy = true +#endif + if useLegacy { try await runOnCaptureQueue { try self.configureSpeechRecognizer(localeIdentifier: localeIdentifier) } @@ -263,6 +273,10 @@ final class LiveTranscriptionSession: NSObject, @unchecked Sendable { try await runOnCaptureQueue { try self.startApplicationAudioCapture(descriptor: captureDescriptor) } + case .systemAudio: + try await runOnCaptureQueue { + try self.startSystemAudioCapture() + } } } @@ -279,8 +293,8 @@ final class LiveTranscriptionSession: NSObject, @unchecked Sendable { microphoneCaptureSession?.stopRunning() microphoneCaptureSession = nil - applicationAudioCapture?.stop() - applicationAudioCapture = nil + audioCapture?.stop() + audioCapture = nil stopModernSpeechRecognizer() recognitionRequest?.endAudio() @@ -339,6 +353,8 @@ final class LiveTranscriptionSession: NSObject, @unchecked Sendable { } case .application: break + case .systemAudio: + break } } @@ -384,8 +400,10 @@ final class LiveTranscriptionSession: NSObject, @unchecked Sendable { } } +#if SUPPORTS_SPEECH_TRANSCRIBER + @available(macOS 26.0, *) private func configureModernSpeechRecognizer(localeIdentifier: String) async throws -> Bool { - guard #available(macOS 26.0, *), SpeechTranscriber.isAvailable else { + guard SpeechTranscriber.isAvailable else { return false } @@ -497,6 +515,17 @@ final class LiveTranscriptionSession: NSObject, @unchecked Sendable { try await installer.downloadAndInstall() } } +#else + private func configureModernSpeechRecognizer(localeIdentifier: String) async throws -> Bool { + return false + } + + private func ensureSpeechAnalyzerAssetsIfNeeded( + for transcriber: AnyObject, + locale: Locale + ) async throws { + } +#endif private func stopModernSpeechRecognizer() { modernAnalyzerTask?.cancel() @@ -509,21 +538,30 @@ final class LiveTranscriptionSession: NSObject, @unchecked Sendable { modernAudioConverterInputSignature = nil resetModernTranscriptionState() +#if SUPPORTS_SPEECH_TRANSCRIBER if #available(macOS 26.0, *) { - (analyzerInputContinuationState as? AsyncStream.Continuation)?.finish() - analyzerInputContinuationState = nil - let analyzer = speechAnalyzerState as? SpeechAnalyzer - speechAnalyzerState = nil - speechTranscriberState = nil - analyzerInputFormat = nil - - if let analyzer { - Task { - await analyzer.cancelAndFinishNow() - } + stopModernSpeechAnalyzerInternal() + } +#endif + } + +#if SUPPORTS_SPEECH_TRANSCRIBER + @available(macOS 26.0, *) + private func stopModernSpeechAnalyzerInternal() { + (analyzerInputContinuationState as? AsyncStream.Continuation)?.finish() + analyzerInputContinuationState = nil + let analyzer = speechAnalyzerState as? SpeechAnalyzer + speechAnalyzerState = nil + speechTranscriberState = nil + analyzerInputFormat = nil + + if let analyzer { + Task { + await analyzer.cancelAndFinishNow() } } } +#endif private func fallbackFromSpeechAnalyzer(_ error: Error) { captureQueue.async { [weak self] in @@ -666,7 +704,7 @@ final class LiveTranscriptionSession: NSObject, @unchecked Sendable { do { try capture.start() - applicationAudioCapture = capture + audioCapture = capture } catch let error as ApplicationAudioCapture.CaptureError { throw mapApplicationCaptureError(error) } catch { @@ -680,6 +718,50 @@ final class LiveTranscriptionSession: NSObject, @unchecked Sendable { } } + private func startSystemAudioCapture() throws { + let capture = SystemAudioCapture( + queue: captureQueue, + audioHandler: { [weak self] buffer in + self?.append(audioBuffer: buffer) + }, + errorHandler: { [weak self] message in + Task { + await self?.emitError(message) + } + } + ) + + do { + try capture.start() + audioCapture = capture + } catch let error as SystemAudioCapture.CaptureError { + switch error { + case .missingOutputDevice: + throw SessionError.failedToStartCapture( + localized(.failedToStageWithReasonFormat, "start system audio capture", "No output audio device found") + ) + case .failed(stage: let stage, status: let status): + throw SessionError.failedToStartCapture( + localized(.failedToStageWithReasonFormat, stage, String(status)) + ) + case .permissionDenied: + throw SessionError.audioCapturePermissionDenied + case .tapFormatUnavailable: + throw SessionError.failedToStartCapture( + localized(.failedToStageWithReasonFormat, "get system audio format", "unavailable") + ) + } + } catch { + throw SessionError.failedToStartCapture( + localized( + .failedToStageWithReasonFormat, + "start system audio capture", + localizedErrorDescription(error) + ) + ) + } + } + private func resolveApplicationProcessObjectIDs(for source: InputSource) throws -> [AudioObjectID] { let runningApp = try resolveRunningApplication(for: source) let system = AudioHardwareSystem.shared @@ -823,8 +905,19 @@ final class LiveTranscriptionSession: NSObject, @unchecked Sendable { } private func appendToSpeechAnalyzer(_ processingBuffer: AVAudioPCMBuffer) { - guard #available(macOS 26.0, *), - recognitionBackend == .speechAnalyzer, +#if SUPPORTS_SPEECH_TRANSCRIBER + if #available(macOS 26.0, *) { + appendToSpeechAnalyzerInternal(processingBuffer) + } +#else + _ = processingBuffer +#endif + } + +#if SUPPORTS_SPEECH_TRANSCRIBER + @available(macOS 26.0, *) + private func appendToSpeechAnalyzerInternal(_ processingBuffer: AVAudioPCMBuffer) { + guard recognitionBackend == .speechAnalyzer, let continuation = analyzerInputContinuationState as? AsyncStream.Continuation else { return } @@ -835,6 +928,7 @@ final class LiveTranscriptionSession: NSObject, @unchecked Sendable { continuation.yield(AnalyzerInput(buffer: analyzerBuffer)) } +#endif private func prepareProcessingBuffer(from audioBuffer: AVAudioPCMBuffer) -> AVAudioPCMBuffer? { if audioBuffer.format.matches(processingFormat) { @@ -900,8 +994,20 @@ final class LiveTranscriptionSession: NSObject, @unchecked Sendable { } private func makeSpeechAnalyzerBuffer(from processingBuffer: AVAudioPCMBuffer) -> AVAudioPCMBuffer? { - guard #available(macOS 26.0, *), - let analyzerInputFormat else { +#if SUPPORTS_SPEECH_TRANSCRIBER + if #available(macOS 26.0, *) { + return makeSpeechAnalyzerBufferInternal(from: processingBuffer) + } + return processingBuffer +#else + return processingBuffer +#endif + } + +#if SUPPORTS_SPEECH_TRANSCRIBER + @available(macOS 26.0, *) + private func makeSpeechAnalyzerBufferInternal(from processingBuffer: AVAudioPCMBuffer) -> AVAudioPCMBuffer? { + guard let analyzerInputFormat else { return processingBuffer } @@ -927,6 +1033,7 @@ final class LiveTranscriptionSession: NSObject, @unchecked Sendable { failurePrefix: localized(.failedToConvertCapturedAudioForSpeechAnalyzer) ) } +#endif private func convertBuffer( _ inputBuffer: AVAudioPCMBuffer, @@ -1702,6 +1809,7 @@ final class LiveTranscriptionSession: NSObject, @unchecked Sendable { } } +#if SUPPORTS_SPEECH_TRANSCRIBER @available(macOS 26.0, *) private func processModernRecognitionResult(_ result: SpeechTranscriber.Result) { let now = Date() @@ -1783,6 +1891,11 @@ final class LiveTranscriptionSession: NSObject, @unchecked Sendable { emitDraftUpdate(from: result, text: text) scheduleSilenceCommit() } +#else + private func processModernRecognitionResult(_ result: Any) { + // Not supported without SpeechTranscriber API + } +#endif private func observeDraftText(_ text: String, at now: Date) { if text != lastDraftText { @@ -1873,6 +1986,7 @@ final class LiveTranscriptionSession: NSObject, @unchecked Sendable { .lowercased() } +#if SUPPORTS_SPEECH_TRANSCRIBER @available(macOS 26.0, *) private func emitDraftUpdate(from result: SpeechTranscriber.Result, text: String) { let now = Date() @@ -1957,6 +2071,11 @@ final class LiveTranscriptionSession: NSObject, @unchecked Sendable { let durationMs = cmTimeMilliseconds(result.range.duration) return "\(startMs):\(durationMs):\(normalizedTranscriberText(result.text))" } +#else + private func emitDraftUpdate(from result: Any, text: String) { + // Not supported without SpeechTranscriber API + } +#endif private func draftLengthFitScore(for text: String) -> Float { let charCount = text.count @@ -1997,19 +2116,19 @@ final class LiveTranscriptionSession: NSObject, @unchecked Sendable { /// • natural within-sentence pauses in Mandarin/Japanese (200–450 ms) /// and below clear sentence-ending silences (≥ 600 ms for most speakers). /// - /// Follow ≈ 600 ms · Balanced ≈ 630 ms · Reading ≈ 690 ms. + /// Follow ≈ 450 ms · Balanced ≈ 530 ms · Reading ≈ 590 ms. private var silenceCommitDeadlineMs: Int { - max(600, modeConfig.minSilenceCommitMs + 350) + max(400, modeConfig.minSilenceCommitMs + 250) } /// Require a short stable window before promoting a punctuation-ended partial. /// This keeps the fast path responsive without freezing a still-revisable boundary. private var modernBoundaryCommitStabilityDelayMs: Int { - max(160, min(modeConfig.minSilenceCommitMs, 240)) + max(120, min(modeConfig.minSilenceCommitMs, 200)) } private var vadSilenceCommitDeadlineMs: Int { - max(280, modeConfig.minSilenceCommitMs) + max(150, modeConfig.minSilenceCommitMs - 80) } private func scheduleSilenceCommit() { @@ -2354,7 +2473,14 @@ extension LiveTranscriptionSession: AVCaptureAudioDataOutputSampleBufferDelegate } } -private final class ApplicationAudioCapture { +// MARK: - Audio Capture Protocol + +protocol AnyAudioCapture: AnyObject { + func start() throws + func stop() +} + +private final class ApplicationAudioCapture: AnyAudioCapture { enum CaptureError: Error { case permissionDenied case missingOutputDevice @@ -2466,16 +2592,13 @@ private final class ApplicationAudioCapture { guard startStatus == noErr else { throw CaptureError.failed(stage: "start app audio capture", status: startStatus) } - } catch let error as AudioHardwareError { + } catch { stop() - if error.error == permErr { + let nsError = error as NSError + if nsError.code == permErr { throw CaptureError.permissionDenied } - - throw CaptureError.failed(stage: "configure app audio capture", status: error.error) - } catch { - stop() throw error } } @@ -2765,3 +2888,121 @@ private extension URL { return nil } } + +/// Captures audio from the system output (all apps combined). +private final class SystemAudioCapture: AnyAudioCapture { + enum CaptureError: LocalizedError { + case missingOutputDevice + case failed(stage: String, status: OSStatus) + case permissionDenied + case tapFormatUnavailable + } + + private let system = AudioHardwareSystem.shared + private var aggregateDevice: AudioHardwareAggregateDevice? + private var deviceIOProcID: AudioDeviceIOProcID? + private var tapFormat: AVAudioFormat? + + private let queue: DispatchQueue + private let audioHandler: (AVAudioPCMBuffer) -> Void + private let errorHandler: (String) -> Void + + init( + queue: DispatchQueue, + audioHandler: @escaping (AVAudioPCMBuffer) -> Void, + errorHandler: @escaping (String) -> Void + ) { + self.queue = queue + self.audioHandler = audioHandler + self.errorHandler = errorHandler + } + + func start() throws { + do { + guard let outputDevice = try system.defaultOutputDevice else { + throw CaptureError.missingOutputDevice + } + + let outputUID = try outputDevice.uid + let aggregateDescription: [String: Any] = [ + kAudioAggregateDeviceNameKey: "v2s-system-audio", + kAudioAggregateDeviceUIDKey: UUID().uuidString, + kAudioAggregateDeviceMainSubDeviceKey: outputUID, + kAudioAggregateDeviceIsPrivateKey: true, + kAudioAggregateDeviceIsStackedKey: false, + kAudioAggregateDeviceSubDeviceListKey: [ + [kAudioSubDeviceUIDKey: outputUID] + ] + ] + + guard let aggregateDevice = try system.makeAggregateDevice(description: aggregateDescription) else { + throw CaptureError.failed(stage: "create aggregate device", status: kAudioHardwareIllegalOperationError) + } + self.aggregateDevice = aggregateDevice + + // Use a 16-bit stereo format as fallback (44.1kHz, 2ch) + guard let tapFormat = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: 44100, channels: 2, interleaved: false) else { + throw CaptureError.tapFormatUnavailable + } + self.tapFormat = tapFormat + + // Set up IO proc using the block-based API + var deviceIOProcID: AudioDeviceIOProcID? + let createIOProcStatus = AudioDeviceCreateIOProcIDWithBlock( + &deviceIOProcID, + aggregateDevice.id, + queue + ) { [weak self] _, inputData, _, _, _ in + guard let self else { return } + self.handleCapturedAudio(UnsafeMutableAudioBufferListPointer(UnsafeMutablePointer(mutating: inputData))) + } + + guard createIOProcStatus == noErr, let deviceIOProcID else { + throw CaptureError.failed(stage: "create the capture callback", status: createIOProcStatus) + } + self.deviceIOProcID = deviceIOProcID + + let startStatus = AudioDeviceStart(aggregateDevice.id, deviceIOProcID) + guard startStatus == noErr else { + throw CaptureError.failed(stage: "start system audio capture", status: startStatus) + } + } catch { + stop() + throw error + } + } + + func stop() { + if let aggregateDevice, let deviceIOProcID { + AudioDeviceStop(aggregateDevice.id, deviceIOProcID) + AudioDeviceDestroyIOProcID(aggregateDevice.id, deviceIOProcID) + } + deviceIOProcID = nil + if let aggregateDevice { + try? system.destroyAggregateDevice(aggregateDevice) + } + aggregateDevice = nil + tapFormat = nil + } + + private func handleCapturedAudio(_ ioData: UnsafeMutableAudioBufferListPointer) { + guard let tapFormat else { return } + + guard let firstBuffer = ioData.first, + let data = firstBuffer.mData else { return } + + let frameCount = firstBuffer.mDataByteSize / UInt32(MemoryLayout.size) + + guard let buffer = AVAudioPCMBuffer(pcmFormat: tapFormat, frameCapacity: AVAudioFrameCount(frameCount)) else { + return + } + buffer.frameLength = AVAudioFrameCount(frameCount) + + let source = data.assumingMemoryBound(to: Float.self) + if let dest = buffer.floatChannelData?[0] { + dest.update(from: source, count: Int(frameCount)) + } + + audioHandler(buffer) + } +} diff --git a/Sources/V2SApp/Services/SourceCatalogService.swift b/Sources/V2SApp/Services/SourceCatalogService.swift index d88480f..6451159 100644 --- a/Sources/V2SApp/Services/SourceCatalogService.swift +++ b/Sources/V2SApp/Services/SourceCatalogService.swift @@ -18,7 +18,7 @@ final class SourceCatalogService { func loadSnapshot() -> SourceCatalogSnapshot { SourceCatalogSnapshot( applications: loadApplications(), - microphones: loadMicrophones() + microphones: [InputSource.systemAudio] + loadMicrophones() ) }