From e1ded277d4cbbbcd328269d183f24714fe5d959a Mon Sep 17 00:00:00 2001 From: Ker102 Date: Sun, 25 Oct 2026 21:37:04 +0200 Subject: [PATCH 1/2] refactor: move caption word utilities into cap-project (#3274) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit caption_text_from_words, the attach-to-previous helpers, normalize_caption_words, caption_word_chunks and the segment size constants were copy-pasted across captions.rs (tauri), transcription.rs (gpui), editor_window.rs (gpui) and caption_timing.rs, and the copies had drifted. They now all consume the single set in cap_project::caption_words. One behavior change: the weak-word boundary check on the Tauri side compared byte length (normalized.len() <= 1), so non-ASCII one letter words like "à" counted as two letters there. The shared version compares chars().count(), matching what #2685 landed in the gpui backend. --- apps/desktop-gpui/src/editor_window.rs | 39 +--- apps/desktop-gpui/src/transcription.rs | 165 +------------- apps/desktop/src-tauri/src/captions.rs | 162 +------------- crates/project/src/caption_timing.rs | 52 +---- crates/project/src/caption_words.rs | 284 ++++++++++++++++++++++++ crates/project/src/lib.rs | 6 + crates/rendering/src/layers/captions.rs | 6 +- 7 files changed, 304 insertions(+), 410 deletions(-) create mode 100644 crates/project/src/caption_words.rs diff --git a/apps/desktop-gpui/src/editor_window.rs b/apps/desktop-gpui/src/editor_window.rs index e89dee55090..046a18b1f47 100644 --- a/apps/desktop-gpui/src/editor_window.rs +++ b/apps/desktop-gpui/src/editor_window.rs @@ -49,6 +49,7 @@ use std::{ use cap_cursor_info::CursorFamily; use cap_editor::{EditorFrameOutput, EditorInstance, EditorState}; +use cap_project::caption_words::caption_text_from_words; use cap_project::{ ClipSpeedAudioMode, Cursors, ProjectConfiguration, RecordingMeta, RecordingMetaInner, StudioRecordingMeta, TimelineConfiguration, XY, @@ -5346,44 +5347,6 @@ pub(crate) struct EditorNotice { /// How long a notice stays up. const EDITOR_NOTICE_DURATION: Duration = Duration::from_millis(2500); -fn caption_text_from_words(words: &[cap_project::CaptionWord]) -> String { - let mut text = String::new(); - for word in words { - let value = word.text.trim(); - if value.is_empty() { - continue; - } - let attaches = value.chars().next().is_some_and(|value| { - matches!( - value, - ',' | '.' - | '!' - | '?' - | ';' - | ':' - | '%' - | ')' - | ']' - | '}' - | '\'' - | '’' - | '、' - | '。' - | '！' - | '？' - | '；' - | '：' - | '，' - ) - }); - if !text.is_empty() && !attaches { - text.push(' '); - } - text.push_str(value); - } - text -} - fn fresh_caption_source_id(project: &ProjectConfiguration) -> String { use std::hash::{BuildHasher, Hasher}; diff --git a/apps/desktop-gpui/src/transcription.rs b/apps/desktop-gpui/src/transcription.rs index 5ea9ccdaa1d..92fa942a9cc 100644 --- a/apps/desktop-gpui/src/transcription.rs +++ b/apps/desktop-gpui/src/transcription.rs @@ -1856,167 +1856,10 @@ fn extract_audio_from_video(video_path: &Path, output_path: &Path) -> Result<(), // Caption words & chunking // --------------------------------------------------------------------------- -/// `MAX_CAPTION_WORD_DURATION` (`captions.rs:75`): Whisper/Parakeet sometimes -/// stretch a trailing word's end across a following silence, which leaves the -/// rendered caption stuck on screen and duplicates the word across timeline -/// cuts once projected. -pub const MAX_CAPTION_WORD_DURATION: f32 = 2.5; -/// `captions.rs:67-69`. -const TARGET_CAPTION_WORDS_PER_SEGMENT: usize = 6; -const MAX_CAPTION_WORDS_PER_SEGMENT: usize = 8; -const MIN_FINAL_CAPTION_WORDS: usize = 3; - -/// `caption_char_attaches_to_previous` (`captions.rs:745-767`). -fn caption_char_attaches_to_previous(value: char) -> bool { - matches!( - value, - ',' | '.' - | '!' - | '?' - | ';' - | ':' - | '%' - | ')' - | ']' - | '}' - | '\'' - | '’' - | '、' - | '。' - | '！' - | '？' - | '；' - | '：' - | '，' - ) -} - -fn caption_token_attaches_to_previous(text: &str) -> bool { - text.trim() - .chars() - .next() - .is_some_and(caption_char_attaches_to_previous) -} - -/// `caption_boundary_word_is_weak` (`captions.rs:769-801`). -fn caption_boundary_word_is_weak(word: &CaptionWord) -> bool { - let normalized = word - .text - .trim() - .chars() - .filter(|c| c.is_alphanumeric()) - .collect::() - .to_lowercase(); - - normalized.chars().count() <= 1 - || matches!( - normalized.as_str(), - "an" | "as" - | "at" - | "be" - | "by" - | "do" - | "he" - | "if" - | "in" - | "is" - | "it" - | "me" - | "my" - | "of" - | "on" - | "or" - | "so" - | "to" - | "up" - | "we" - ) -} - -/// `normalize_caption_words` (`captions.rs:803-831`). -fn normalize_caption_words(words: Vec) -> Vec { - let mut normalized: Vec = Vec::with_capacity(words.len()); - - for word in words { - let text = word.text.trim(); - if text.is_empty() { - continue; - } - - if caption_token_attaches_to_previous(text) - && let Some(previous) = normalized.last_mut() - { - previous.text.push_str(text); - previous.end = word.end; - } else { - normalized.push(CaptionWord { - text: text.to_string(), - start: word.start, - end: word.end, - }); - } - } - - for word in &mut normalized { - word.end = word.end.min(word.start + MAX_CAPTION_WORD_DURATION); - } - - normalized -} - -/// `caption_text_from_words` (`captions.rs:833-849`). -fn caption_text_from_words<'a>(words: impl IntoIterator) -> String { - let mut text = String::new(); - - for word in words { - let word_text = word.text.trim(); - if word_text.is_empty() { - continue; - } - - if !text.is_empty() && !caption_token_attaches_to_previous(word_text) { - text.push(' '); - } - text.push_str(word_text); - } - - text -} - -/// `caption_word_chunks` (`captions.rs:851-883`). -fn caption_word_chunks(words: &[CaptionWord]) -> Vec<&[CaptionWord]> { - let mut chunks = Vec::new(); - let mut start = 0; - - while start < words.len() { - let remaining = words.len() - start; - if remaining <= TARGET_CAPTION_WORDS_PER_SEGMENT { - chunks.push(&words[start..]); - break; - } - - let mut end = (start + TARGET_CAPTION_WORDS_PER_SEGMENT).min(words.len()); - while end < words.len() - && caption_boundary_word_is_weak(&words[end - 1]) - && end - start < MAX_CAPTION_WORDS_PER_SEGMENT - { - end += 1; - } - - let remaining_after = words.len() - end; - if remaining_after > 0 - && remaining_after < MIN_FINAL_CAPTION_WORDS - && caption_boundary_word_is_weak(&words[end]) - { - end = words.len(); - } - - chunks.push(&words[start..end]); - start = end; - } - - chunks -} +use cap_project::caption_words::{ + caption_text_from_words, caption_word_chunks, normalize_caption_words, + MAX_CAPTION_WORD_DURATION, +}; // --------------------------------------------------------------------------- // Track derivation -- deriveCaptionTrackSegments, in Rust diff --git a/apps/desktop/src-tauri/src/captions.rs b/apps/desktop/src-tauri/src/captions.rs index b64305aa478..90d6712ab13 100644 --- a/apps/desktop/src-tauri/src/captions.rs +++ b/apps/desktop/src-tauri/src/captions.rs @@ -74,15 +74,11 @@ lazy_static::lazy_static! { } const WHISPER_SAMPLE_RATE: u32 = 16000; -const TARGET_CAPTION_WORDS_PER_SEGMENT: usize = 6; -const MAX_CAPTION_WORDS_PER_SEGMENT: usize = 8; -const MIN_FINAL_CAPTION_WORDS: usize = 3; -// Whisper/Parakeet sometimes stretch a trailing word's end across a following -// silence (e.g. a 16s "seconds."), which leaves the rendered caption stuck on -// screen and duplicates the word across timeline cuts once projected. Real -// spoken words never approach this, so cap each word's duration to keep timing -// tied to speech rather than silence. -const MAX_CAPTION_WORD_DURATION: f32 = 2.5; + +use cap_project::caption_words::{ + caption_text_from_words, caption_word_chunks, normalize_caption_words, + MAX_CAPTION_WORD_DURATION, +}; #[cfg(not(all(target_os = "macos", target_arch = "x86_64")))] struct CachedParakeetContext { @@ -769,154 +765,6 @@ fn is_special_token(token_text: &str) -> bool { is_special } -fn caption_token_attaches_to_previous(text: &str) -> bool { - let Some(first_char) = text.trim().chars().next() else { - return false; - }; - - caption_char_attaches_to_previous(first_char) -} - -fn caption_char_attaches_to_previous(value: char) -> bool { - matches!( - value, - ',' | '.' - | '!' - | '?' - | ';' - | ':' - | '%' - | ')' - | ']' - | '}' - | '\'' - | '’' - | '、' - | '。' - | '！' - | '？' - | '；' - | '：' - | '，' - ) -} - -fn caption_boundary_word_is_weak(word: &CaptionWord) -> bool { - let normalized = word - .text - .trim() - .chars() - .filter(|c| c.is_alphanumeric()) - .collect::() - .to_lowercase(); - - normalized.len() <= 1 - || matches!( - normalized.as_str(), - "an" | "as" - | "at" - | "be" - | "by" - | "do" - | "he" - | "if" - | "in" - | "is" - | "it" - | "me" - | "my" - | "of" - | "on" - | "or" - | "so" - | "to" - | "up" - | "we" - ) -} - -fn normalize_caption_words(words: Vec) -> Vec { - let mut normalized: Vec = Vec::with_capacity(words.len()); - - for word in words { - let text = word.text.trim(); - if text.is_empty() { - continue; - } - - if caption_token_attaches_to_previous(text) - && let Some(previous) = normalized.last_mut() - { - previous.text.push_str(text); - previous.end = word.end; - } else { - normalized.push(CaptionWord { - text: text.to_string(), - start: word.start, - end: word.end, - }); - } - } - - for word in &mut normalized { - word.end = word.end.min(word.start + MAX_CAPTION_WORD_DURATION); - } - - normalized -} - -fn caption_text_from_words<'a>(words: impl IntoIterator) -> String { - let mut text = String::new(); - - for word in words { - let word_text = word.text.trim(); - if word_text.is_empty() { - continue; - } - - if !text.is_empty() && !caption_token_attaches_to_previous(word_text) { - text.push(' '); - } - text.push_str(word_text); - } - - text -} - -fn caption_word_chunks(words: &[CaptionWord]) -> Vec<&[CaptionWord]> { - let mut chunks = Vec::new(); - let mut start = 0; - - while start < words.len() { - let remaining = words.len() - start; - if remaining <= TARGET_CAPTION_WORDS_PER_SEGMENT { - chunks.push(&words[start..]); - break; - } - - let mut end = (start + TARGET_CAPTION_WORDS_PER_SEGMENT).min(words.len()); - while end < words.len() - && caption_boundary_word_is_weak(&words[end - 1]) - && end - start < MAX_CAPTION_WORDS_PER_SEGMENT - { - end += 1; - } - - let remaining_after = words.len() - end; - if remaining_after > 0 - && remaining_after < MIN_FINAL_CAPTION_WORDS - && caption_boundary_word_is_weak(&words[end]) - { - end = words.len(); - } - - chunks.push(&words[start..end]); - start = end; - } - - chunks -} - fn normalize_audio_for_transcription(samples: &mut [f32]) -> f32 { if samples.is_empty() { return 1.0; diff --git a/crates/project/src/caption_timing.rs b/crates/project/src/caption_timing.rs index b938177af4d..5f9fa655b1f 100644 --- a/crates/project/src/caption_timing.rs +++ b/crates/project/src/caption_timing.rs @@ -1,10 +1,9 @@ +use crate::caption_words::{MAX_CAPTION_WORD_DURATION, caption_text_from_words}; use crate::{ CaptionSegment, CaptionTrackSegment, CaptionWord, ProjectConfiguration, TimelineConfiguration, }; use std::collections::HashMap; -const MAX_CAPTION_WORD_DURATION: f32 = 2.5; - pub(crate) fn clip_timeline_offsets(timeline: &TimelineConfiguration) -> Vec { let mut offset = 0.0; timeline @@ -22,55 +21,6 @@ pub(crate) fn clip_timeline_offsets(timeline: &TimelineConfiguration) -> Vec bool { - matches!( - value, - ',' | '.' - | '!' - | '?' - | ';' - | ':' - | '%' - | ')' - | ']' - | '}' - | '\'' - | '’' - | '、' - | '。' - | '！' - | '？' - | '；' - | '：' - | '，' - ) -} - -fn caption_token_attaches_to_previous(text: &str) -> bool { - text.trim() - .chars() - .next() - .is_some_and(caption_char_attaches_to_previous) -} - -fn caption_text_from_words<'a>(words: impl IntoIterator) -> String { - let mut text = String::new(); - - for word in words { - let word_text = word.text.trim(); - if word_text.is_empty() { - continue; - } - - if !text.is_empty() && !caption_token_attaches_to_previous(word_text) { - text.push(' '); - } - text.push_str(word_text); - } - - text -} - const CAPTION_EDL_SEPARATOR: &str = "::edl"; pub fn source_caption_id(track_id: &str) -> &str { diff --git a/crates/project/src/caption_words.rs b/crates/project/src/caption_words.rs new file mode 100644 index 00000000000..767ed15ff9e --- /dev/null +++ b/crates/project/src/caption_words.rs @@ -0,0 +1,284 @@ +//! Shared caption word normalization and segmentation. +//! +//! Single source of truth for how transcribed words are cleaned up and grouped +//! into on-screen caption segments. This used to be copy-pasted across the +//! Tauri captions pipeline (`apps/desktop/src-tauri/src/captions.rs`), the gpui +//! transcription pipeline (`apps/desktop-gpui/src/transcription.rs`), +//! `caption_timing`, and `editor_window`, and the copies drifted apart. + +use crate::CaptionWord; + +/// Whisper/Parakeet sometimes stretch a trailing word's end across a following +/// silence (e.g. a 16s "seconds."), which leaves the rendered caption stuck on +/// screen and duplicates the word across timeline cuts once projected. Real +/// spoken words never approach this, so cap each word's duration to keep timing +/// tied to speech rather than silence. +pub const MAX_CAPTION_WORD_DURATION: f32 = 2.5; +pub const TARGET_CAPTION_WORDS_PER_SEGMENT: usize = 6; +pub const MAX_CAPTION_WORDS_PER_SEGMENT: usize = 8; +pub const MIN_FINAL_CAPTION_WORDS: usize = 3; + +fn caption_char_attaches_to_previous(value: char) -> bool { + matches!( + value, + ',' | '.' + | '!' + | '?' + | ';' + | ':' + | '%' + | ')' + | ']' + | '}' + | '\'' + | '’' + | '、' + | '。' + | '！' + | '？' + | '；' + | '：' + | '，' + ) +} + +fn caption_token_attaches_to_previous(text: &str) -> bool { + text.trim() + .chars() + .next() + .is_some_and(caption_char_attaches_to_previous) +} + +/// A caption segment should not end on a one letter word or a short connector +/// when more words follow, since that reads badly on screen. Comparing +/// `chars().count()` matters: a byte length check counts non-ASCII letters like +/// "à" as two and treats them as strong boundary words. +pub fn caption_boundary_word_is_weak(word: &CaptionWord) -> bool { + let normalized = word + .text + .trim() + .chars() + .filter(|c| c.is_alphanumeric()) + .collect::() + .to_lowercase(); + + normalized.chars().count() <= 1 + || matches!( + normalized.as_str(), + "an" | "as" + | "at" + | "be" + | "by" + | "do" + | "he" + | "if" + | "in" + | "is" + | "it" + | "me" + | "my" + | "of" + | "on" + | "or" + | "so" + | "to" + | "up" + | "we" + ) +} + +/// Drops empty tokens, attaches punctuation-only tokens to the previous word, +/// and clamps inflated trailing word durations. +pub fn normalize_caption_words(words: Vec) -> Vec { + let mut normalized: Vec = Vec::with_capacity(words.len()); + + for word in words { + let text = word.text.trim(); + if text.is_empty() { + continue; + } + + if caption_token_attaches_to_previous(text) + && let Some(previous) = normalized.last_mut() + { + previous.text.push_str(text); + previous.end = word.end; + } else { + normalized.push(CaptionWord { + text: text.to_string(), + start: word.start, + end: word.end, + }); + } + } + + for word in &mut normalized { + word.end = word.end.min(word.start + MAX_CAPTION_WORD_DURATION); + } + + normalized +} + +pub fn caption_text_from_words<'a>(words: impl IntoIterator) -> String { + let mut text = String::new(); + + for word in words { + let word_text = word.text.trim(); + if word_text.is_empty() { + continue; + } + + if !text.is_empty() && !caption_token_attaches_to_previous(word_text) { + text.push(' '); + } + text.push_str(word_text); + } + + text +} + +/// Groups normalized words into caption segments of at most +/// [`MAX_CAPTION_WORDS_PER_SEGMENT`] words, aiming for +/// [`TARGET_CAPTION_WORDS_PER_SEGMENT`]. The boundary word may move past the +/// target to avoid ending on a weak word, and a trailing fragment shorter than +/// [`MIN_FINAL_CAPTION_WORDS`] merges into the previous segment when it starts +/// with a weak word. +pub fn caption_word_chunks(words: &[CaptionWord]) -> Vec<&[CaptionWord]> { + let mut chunks = Vec::new(); + let mut start = 0; + + while start < words.len() { + let remaining = words.len() - start; + if remaining <= TARGET_CAPTION_WORDS_PER_SEGMENT { + chunks.push(&words[start..]); + break; + } + + let mut end = (start + TARGET_CAPTION_WORDS_PER_SEGMENT).min(words.len()); + while end < words.len() + && caption_boundary_word_is_weak(&words[end - 1]) + && end - start < MAX_CAPTION_WORDS_PER_SEGMENT + { + end += 1; + } + + let remaining_after = words.len() - end; + if remaining_after > 0 + && remaining_after < MIN_FINAL_CAPTION_WORDS + && caption_boundary_word_is_weak(&words[end]) + { + end = words.len(); + } + + chunks.push(&words[start..end]); + start = end; + } + + chunks +} + +#[cfg(test)] +mod tests { + use super::*; + + fn word(text: &str, index: usize) -> CaptionWord { + CaptionWord { + text: text.to_string(), + start: index as f32, + end: index as f32 + 0.5, + } + } + + #[test] + fn normalize_caption_words_attaches_punctuation() { + let words = normalize_caption_words(vec![ + word("test", 0), + word(",", 1), + word("test", 2), + word(".", 3), + ]); + + assert_eq!(caption_text_from_words(&words), "test, test."); + assert_eq!(words.len(), 2); + } + + #[test] + fn normalize_caption_words_clamps_inflated_trailing_word() { + let words = normalize_caption_words(vec![CaptionWord { + text: "seconds.".to_string(), + start: 53.92, + end: 70.16, + }]); + + assert_eq!(words.len(), 1); + assert!((words[0].end - (53.92 + MAX_CAPTION_WORD_DURATION)).abs() < 1e-4); + } + + #[test] + fn normalize_caption_words_keeps_normal_word_durations() { + let words = normalize_caption_words(vec![CaptionWord { + text: "hello".to_string(), + start: 1.0, + end: 1.4, + }]); + + assert_eq!(words.len(), 1); + assert!((words[0].end - 1.4).abs() < 1e-4); + } + + #[test] + fn caption_word_chunks_do_not_end_on_short_connector_when_more_words_follow() { + for connector in ["I", "à", "à,", "a\u{300}", "日"] { + let words = [ + "This", "is", "where", "we", "record", connector, "want", "clean", "captions", + ] + .iter() + .enumerate() + .map(|(index, text)| word(text, index)) + .collect::>(); + + let chunks = caption_word_chunks(&words); + + assert_eq!( + caption_text_from_words(chunks[0]), + format!("This is where we record {connector} want"), + "connector {connector}" + ); + assert_eq!(caption_text_from_words(chunks[1]), "clean captions"); + } + } + + #[test] + fn caption_word_chunks_merge_short_tail_starting_with_connector() { + for connector in ["I", "à", "à,", "a\u{300}", "日"] { + let words = [ + "These", "captions", "should", "stay", "grouped", "together", connector, "Paris", + ] + .iter() + .enumerate() + .map(|(index, text)| word(text, index)) + .collect::>(); + + let chunks = caption_word_chunks(&words); + + assert_eq!(chunks.len(), 1, "connector {connector}"); + assert_eq!( + caption_text_from_words(chunks[0]), + format!("These captions should stay grouped together {connector} Paris") + ); + } + } + + #[test] + fn caption_word_chunks_target_size_groups_strong_boundaries() { + let words = (0..12) + .map(|index| word(&format!("w{index}"), index)) + .collect::>(); + + let chunks = caption_word_chunks(&words); + + assert_eq!(chunks.len(), 2); + assert_eq!(chunks[0].len(), 6); + assert_eq!(chunks[1].len(), 6); + } +} diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index 080355b0578..f4bdcf5869d 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -1,5 +1,6 @@ mod animated_gradient; mod caption_timing; +pub mod caption_words; #[cfg(test)] mod config_write_tests; mod configuration; @@ -9,6 +10,11 @@ mod meta; pub use animated_gradient::*; pub use caption_timing::{derive_caption_track_segments, source_caption_id, synchronize_captions}; +pub use caption_words::{ + MAX_CAPTION_WORD_DURATION, MAX_CAPTION_WORDS_PER_SEGMENT, MIN_FINAL_CAPTION_WORDS, + TARGET_CAPTION_WORDS_PER_SEGMENT, caption_boundary_word_is_weak, caption_text_from_words, + caption_word_chunks, normalize_caption_words, +}; pub use configuration::*; pub use cursor::*; pub use keyboard::*; diff --git a/crates/rendering/src/layers/captions.rs b/crates/rendering/src/layers/captions.rs index 5a6d10bbef7..c26e715b3e6 100644 --- a/crates/rendering/src/layers/captions.rs +++ b/crates/rendering/src/layers/captions.rs @@ -139,9 +139,9 @@ const BASE_TEXT_OPACITY: f32 = 0.8; const BOUNCE_OFFSET_PIXELS: f32 = 8.0; // Safety net for caption segments whose trailing word end was stretched across a // silence by transcription (e.g. a 16s "seconds."). Without this, such a segment -// stays on screen for the whole inflated duration. Kept in sync with -// MAX_CAPTION_WORD_DURATION in the desktop transcription/projection layers. -const MAX_CAPTION_WORD_DURATION: f64 = 2.5; +// stays on screen for the whole inflated duration. Value comes from the shared +// caption word utilities so it stays in sync with the transcription layers. +const MAX_CAPTION_WORD_DURATION: f64 = cap_project::caption_words::MAX_CAPTION_WORD_DURATION as f64; fn ease_out_cubic(t: f32) -> f32 { let t = t.clamp(0.0, 1.0); From 77b290923d7710588a999de94180c55780ce66d6 Mon Sep 17 00:00:00 2001 From: Ker102 Date: Sun, 25 Oct 2026 21:55:08 +0200 Subject: [PATCH 2/2] feat: configurable words per caption segment (#3275) Adds a segmentWords caption setting (1-8, default 6) that controls how many words a generated caption segment holds, threaded from the captions panels through both transcription pipelines into the shared chunker. At 1 the chunker switches to the one word on screen at a time style: every word becomes its own segment and the multi-word rules (weak boundary extension, short tail merge) are bypassed. Above 1 the existing rules apply relative to the chosen size, so the default of 6 reproduces current behavior exactly. Settings surface: cap_project::CaptionSettings.segmentWords (JSON segmentWords), a Words Per Caption slider in the Tauri editor's captions tab and in the gpui captions sidebar, and the tauri transcribe_audio command gains an optional segmentWords argument. --- apps/desktop-gpui/src/editor_tabs.rs | 17 ++++- apps/desktop-gpui/src/transcription.rs | 18 ++++-- apps/desktop/src-tauri/src/captions.rs | 33 ++++++++-- .../routes/editor/CaptionsRegenerateBadge.tsx | 1 + .../desktop/src/routes/editor/CaptionsTab.tsx | 13 ++++ apps/desktop/src/routes/editor/captions.ts | 9 ++- apps/desktop/src/store/captions.ts | 1 + apps/desktop/src/utils/tauri.ts | 6 +- crates/project/src/caption_words.rs | 64 ++++++++++++++++++- crates/project/src/configuration.rs | 7 ++ crates/project/src/lib.rs | 2 +- crates/verify/tests/captions.rs | 1 + 12 files changed, 151 insertions(+), 21 deletions(-) diff --git a/apps/desktop-gpui/src/editor_tabs.rs b/apps/desktop-gpui/src/editor_tabs.rs index 7801a8d0f4a..a4a7d635e83 100644 --- a/apps/desktop-gpui/src/editor_tabs.rs +++ b/apps/desktop-gpui/src/editor_tabs.rs @@ -497,6 +497,7 @@ pub enum CaptionSlider { Size, BackgroundOpacity, FadeDuration, + SegmentWords, ManualX, ManualY, } @@ -563,6 +564,8 @@ impl CaptionSlider { pub(crate) fn changes_timeline_model(self) -> bool { match self { Self::Size | Self::BackgroundOpacity | Self::FadeDuration => false, + // Only feeds the next caption generation, existing segments are untouched. + Self::SegmentWords => false, Self::ManualX | Self::ManualY => false, } } @@ -573,6 +576,7 @@ impl CaptionSlider { Self::BackgroundOpacity => (0., 100., 1.), // `value={[getSetting("fadeDuration") * 100]}` over 0..50. Self::FadeDuration => (0., 50., 1.), + Self::SegmentWords => (1., 8., 1.), Self::ManualX | Self::ManualY => (0., 100., 1.), } } @@ -583,6 +587,7 @@ impl CaptionSlider { Self::Size => settings.size as f32, Self::BackgroundOpacity => settings.background_opacity as f32, Self::FadeDuration => settings.fade_duration * 100., + Self::SegmentWords => settings.segment_words as f32, Self::ManualX => caption_manual_center(&settings).0 * 100., Self::ManualY => caption_manual_center(&settings).1 * 100., } @@ -1269,6 +1274,9 @@ impl EditorWindow { settings.background_opacity = value.round() as u32 } CaptionSlider::FadeDuration => settings.fade_duration = value / 100., + CaptionSlider::SegmentWords => { + settings.segment_words = (value.round() as u32).max(1) + } CaptionSlider::ManualX => set_caption_manual_center(settings, Some(value), None), CaptionSlider::ManualY => set_caption_manual_center(settings, None, Some(value)), }, @@ -3085,6 +3093,12 @@ impl EditorWindow { SliderKey::Caption(CaptionSlider::FadeDuration), format!("{:.0}ms", settings.fade_duration * 1000.), cx, + )) + .child(self.labelled_readout( + "Words Per Caption", + SliderKey::Caption(CaptionSlider::SegmentWords), + format!("{:.0}", settings.segment_words), + cx, )), ), ) @@ -3396,6 +3410,7 @@ impl EditorWindow { let project_path = self.project_path.clone(); let model = self.selected_caption_model().name; let language = self.sidebar.caption_language; + let segment_words = caption_settings(&self.project).segment_words.max(1) as usize; if !transcription::begin_generation(&project_path) { return; @@ -3406,7 +3421,7 @@ impl EditorWindow { let task_path = project_path.clone(); let task = gpui_tokio::Tokio::spawn(cx, async move { tokio::task::spawn_blocking(move || { - transcription::transcribe_blocking(&task_path, model, language) + transcription::transcribe_blocking(&task_path, model, language, segment_words) }) .await .map_err(|error| format!("Transcription task panicked: {error}"))? diff --git a/apps/desktop-gpui/src/transcription.rs b/apps/desktop-gpui/src/transcription.rs index 92fa942a9cc..b7a82457354 100644 --- a/apps/desktop-gpui/src/transcription.rs +++ b/apps/desktop-gpui/src/transcription.rs @@ -1430,6 +1430,7 @@ fn process_with_whisper( context: Arc, language: &str, transcription_hints: &[String], + segment_words: usize, ) -> Result, String> { let mut params = FullParams::new(SamplingStrategy::BeamSearch { beam_size: 5, @@ -1555,7 +1556,7 @@ fn process_with_whisper( continue; } - for (chunk_idx, chunk_words) in caption_word_chunks(&words).into_iter().enumerate() { + for (chunk_idx, chunk_words) in caption_word_chunks_with(&words, segment_words).into_iter().enumerate() { let segment_text = caption_text_from_words(chunk_words); let segment_start = chunk_words .first() @@ -1623,6 +1624,7 @@ fn parakeet_chunk_ranges(samples: &[f32], sample_rate: usize) -> Vec Result, String> { let cached_model = { let guard = PARAKEET_CONTEXT @@ -1707,7 +1709,7 @@ fn process_with_parakeet( tracing::info!("Parakeet produced {} timed tokens", tokens.len()); - caption_segments_from_tokens(tokens) + caption_segments_from_tokens(tokens, segment_words) } /// Parakeet's timed word tokens -> caption segments: drop empty tokens, @@ -1715,6 +1717,7 @@ fn process_with_parakeet( #[cfg(any(test, not(all(target_os = "macos", target_arch = "x86_64"))))] fn caption_segments_from_tokens( tokens: impl IntoIterator, + segment_words: usize, ) -> Result, String> { let words = normalize_caption_words( tokens @@ -1733,7 +1736,7 @@ fn caption_segments_from_tokens( } let mut segments = Vec::new(); - for (chunk_idx, chunk) in caption_word_chunks(&words).into_iter().enumerate() { + for (chunk_idx, chunk) in caption_word_chunks_with(&words, segment_words).into_iter().enumerate() { segments.push(CaptionSegment { id: format!("segment-{chunk_idx}"), start: chunk.first().map_or(0.0, |word| word.start), @@ -1750,6 +1753,7 @@ fn caption_segments_from_tokens( fn process_with_parakeet( _audio_path: &Path, _model_dir: &str, + _segment_words: usize, ) -> Result, String> { Err(PARAKEET_UNSUPPORTED_MESSAGE.to_string()) } @@ -1766,6 +1770,7 @@ pub fn transcribe_blocking( project_path: &Path, model: &str, language: &str, + segment_words: usize, ) -> Result, String> { let requested = model; // A download writes straight to the model's final path, so a model still @@ -1794,14 +1799,15 @@ pub fn transcribe_blocking( let _guard = lock_transcription_worker_slot(); if is_parakeet_model(model) { tracing::info!("Using Parakeet TDT engine"); - let result = process_with_parakeet(audio_path, model_file.to_string_lossy().as_ref()); + let result = + process_with_parakeet(audio_path, model_file.to_string_lossy().as_ref(), segment_words); schedule_parakeet_release(); result } else { tracing::info!("Using Whisper engine"); let hints = transcription_hints(); let context = get_whisper_context(model_file.to_string_lossy().as_ref())?; - let result = process_with_whisper(audio_path, context, language, &hints); + let result = process_with_whisper(audio_path, context, language, &hints, segment_words); release_whisper_context_after_transcription(); result } @@ -1857,7 +1863,7 @@ fn extract_audio_from_video(video_path: &Path, output_path: &Path) -> Result<(), // --------------------------------------------------------------------------- use cap_project::caption_words::{ - caption_text_from_words, caption_word_chunks, normalize_caption_words, + caption_text_from_words, caption_word_chunks, caption_word_chunks_with, normalize_caption_words, MAX_CAPTION_WORD_DURATION, }; diff --git a/apps/desktop/src-tauri/src/captions.rs b/apps/desktop/src-tauri/src/captions.rs index 90d6712ab13..92e1476bac0 100644 --- a/apps/desktop/src-tauri/src/captions.rs +++ b/apps/desktop/src-tauri/src/captions.rs @@ -76,7 +76,7 @@ lazy_static::lazy_static! { const WHISPER_SAMPLE_RATE: u32 = 16000; use cap_project::caption_words::{ - caption_text_from_words, caption_word_chunks, normalize_caption_words, + caption_text_from_words, caption_word_chunks, caption_word_chunks_with, normalize_caption_words, MAX_CAPTION_WORD_DURATION, }; @@ -802,6 +802,7 @@ fn process_with_whisper( context: Arc, language: &str, transcription_hints: &[String], + segment_words: usize, ) -> Result { log::info!("=== WHISPER TRANSCRIPTION START ==="); log::info!("Processing audio file: {audio_path:?}"); @@ -1023,7 +1024,7 @@ fn process_with_whisper( continue; } - for (chunk_idx, chunk_words) in caption_word_chunks(&words).into_iter().enumerate() { + for (chunk_idx, chunk_words) in caption_word_chunks_with(&words, segment_words).into_iter().enumerate() { let segment_text = caption_text_from_words(chunk_words); let segment_start = chunk_words @@ -1091,6 +1092,7 @@ fn build_initial_prompt(transcription_hints: &[String]) -> Option { fn process_with_parakeet( audio_path: &std::path::Path, model_dir: &str, + segment_words: usize, ) -> Result { tracing::info!("Processing audio file: {audio_path:?}"); tracing::info!("Model directory: {model_dir}"); @@ -1164,7 +1166,7 @@ fn process_with_parakeet( let mut segments = Vec::new(); - for (chunk_idx, chunk) in caption_word_chunks(&words).into_iter().enumerate() { + for (chunk_idx, chunk) in caption_word_chunks_with(&words, segment_words).into_iter().enumerate() { let segment_text = caption_text_from_words(chunk); let segment_start = chunk.first().map(|w| w.start).unwrap_or(0.0); let segment_end = chunk.last().map(|w| w.end).unwrap_or(0.0); @@ -1194,6 +1196,7 @@ fn process_with_parakeet( fn process_with_parakeet( _audio_path: &std::path::Path, _model_dir: &str, + _segment_words: usize, ) -> Result { Err(PARAKEET_UNSUPPORTED_MESSAGE.to_string()) } @@ -1207,11 +1210,13 @@ pub async fn transcribe_audio( model_path: String, language: String, engine: TranscriptionEngine, + segment_words: Option, ) -> Result { log::info!("=== TRANSCRIBE AUDIO COMMAND START ==="); log::info!("Video path: {}", video_path); log::info!("Model path: {}", model_path); log::info!("Language: {}", language); + let segment_words = segment_words.unwrap_or(6).max(1) as usize; let validated_model_path = validate_model_path(&app, &model_path)?; @@ -1259,7 +1264,7 @@ pub async fn transcribe_audio( let model_dir = model_path.clone(); tokio::task::spawn_blocking(move || { let _guard = lock_transcription_worker_slot(); - process_with_parakeet(&audio_path, &model_dir) + process_with_parakeet(&audio_path, &model_dir, segment_words) }) .await .map_err(|e| format!("Parakeet task panicked: {e}"))? @@ -1284,8 +1289,13 @@ pub async fn transcribe_audio( return Err(format!("Failed to initialize transcription model: {e}")); } }; - let result = - process_with_whisper(&audio_path, context, &language, &transcription_hints); + let result = process_with_whisper( + &audio_path, + context, + &language, + &transcription_hints, + segment_words, + ); release_whisper_context_after_transcription(); result }) @@ -1499,6 +1509,10 @@ pub async fn save_captions( "activeWordHighlight".to_string(), serde_json::Value::Bool(settings.active_word_highlight), ); + settings_obj.insert( + "segmentWords".to_string(), + serde_json::Value::Number(settings.segment_words.into()), + ); json_obj.insert( "settings".to_string(), @@ -1655,6 +1669,12 @@ pub fn parse_captions_json(json: &str) -> Result Result 0) { @@ -1138,6 +1139,18 @@ export function CaptionsTab(props: { /> + + + updateCaptionSetting("segmentWords", Math.max(1, v[0])) + } + minValue={1} + maxValue={8} + step={1} + /> + + { const resolvedModelName = resolveCaptionModel(modelName); const modelPath = await getModelPath(resolvedModelName); const engine = PARAKEET_DIR_MODELS.has(resolvedModelName) ? "Parakeet" : "Whisper"; - return await commands.transcribeAudio(videoPath, modelPath, language, engine); + return await commands.transcribeAudio( + videoPath, + modelPath, + language, + engine, + segmentWords, + ); } const CAPTION_ATTACHING_PUNCTUATION = new Set([ diff --git a/apps/desktop/src/store/captions.ts b/apps/desktop/src/store/captions.ts index 27eab47ca90..3ee38875fac 100644 --- a/apps/desktop/src/store/captions.ts +++ b/apps/desktop/src/store/captions.ts @@ -180,6 +180,7 @@ export const defaultCaptionSettings: EditorCaptionSettings = { exportWithSubtitles: false, lingerDuration: 0.4, wordTransitionDuration: 0.25, + segmentWords: 6, manualPosition: null, preset: classicPreset.id, ...classicPreset.style, diff --git a/apps/desktop/src/utils/tauri.ts b/apps/desktop/src/utils/tauri.ts index 65e5b101a40..cb4d7945502 100644 --- a/apps/desktop/src/utils/tauri.ts +++ b/apps/desktop/src/utils/tauri.ts @@ -489,8 +489,8 @@ async createDir(path: string, recursive: boolean) : Promise { async saveModelFile(path: string, data: number[]) : Promise { return await TAURI_INVOKE("save_model_file", { path, data }); }, -async transcribeAudio(videoPath: string, modelPath: string, language: string, engine: TranscriptionEngine) : Promise { - return await TAURI_INVOKE("transcribe_audio", { videoPath, modelPath, language, engine }); +async transcribeAudio(videoPath: string, modelPath: string, language: string, engine: TranscriptionEngine, segmentWords: number | null) : Promise { + return await TAURI_INVOKE("transcribe_audio", { videoPath, modelPath, language, engine, segmentWords }); }, async saveCaptions(videoId: string, captions: CaptionData) : Promise { return await TAURI_INVOKE("save_captions", { videoId, captions }); @@ -892,7 +892,7 @@ export type CameraXPosition = "left" | "center" | "right" export type CameraYPosition = "top" | "bottom" export type CaptionData = { segments: CaptionSegment[]; settings: CaptionSettings | null } export type CaptionSegment = { id: string; start: number; end: number; text: string; words?: CaptionWord[] } -export type CaptionSettings = { enabled: boolean; font: string; size: number; color: string; backgroundColor: string; backgroundOpacity: number; position: string; italic: boolean; fontWeight: number; outline: boolean; outlineColor: string; exportWithSubtitles: boolean; highlightColor: string; fadeDuration: number; lingerDuration: number; wordTransitionDuration: number; activeWordHighlight: boolean; manualPosition: XY | null; preset: string; animation: string; highlightStyle: string; uppercase: boolean } +export type CaptionSettings = { enabled: boolean; font: string; size: number; color: string; backgroundColor: string; backgroundOpacity: number; position: string; italic: boolean; fontWeight: number; outline: boolean; outlineColor: string; exportWithSubtitles: boolean; highlightColor: string; fadeDuration: number; lingerDuration: number; wordTransitionDuration: number; activeWordHighlight: boolean; segmentWords: number; manualPosition: XY | null; preset: string; animation: string; highlightStyle: string; uppercase: boolean } export type CaptionTrackSegment = { id: string; start: number; end: number; text: string; words?: CaptionWord[]; fadeDurationOverride?: number | null; lingerDurationOverride?: number | null; positionOverride?: string | null; colorOverride?: string | null; backgroundColorOverride?: string | null; fontSizeOverride?: number | null } export type CaptionWord = { text: string; start: number; end: number } export type CaptionsData = { segments: CaptionSegment[]; settings: CaptionSettings; diff --git a/crates/project/src/caption_words.rs b/crates/project/src/caption_words.rs index 767ed15ff9e..cb9ce4d4c27 100644 --- a/crates/project/src/caption_words.rs +++ b/crates/project/src/caption_words.rs @@ -144,20 +144,40 @@ pub fn caption_text_from_words<'a>(words: impl IntoIterator Vec<&[CaptionWord]> { + caption_word_chunks_with(words, TARGET_CAPTION_WORDS_PER_SEGMENT) +} + +/// [`caption_word_chunks`] with a caller-chosen segment size. `words_per_segment` +/// of 1 is the one word on screen at a time style: the multi-word rules (weak +/// boundary extension and the short tail merge) do not apply there and every +/// word becomes its own segment. Above 1 the chunker aims for `words_per_segment` +/// words, extends up to two past it to avoid ending on a weak word, and merges a +/// trailing fragment shorter than [`MIN_FINAL_CAPTION_WORDS`]. +pub fn caption_word_chunks_with( + words: &[CaptionWord], + words_per_segment: usize, +) -> Vec<&[CaptionWord]> { + if words_per_segment <= 1 { + return words.chunks(1).collect(); + } + + let target = words_per_segment; + let max = words_per_segment + 2; + let mut chunks = Vec::new(); let mut start = 0; while start < words.len() { let remaining = words.len() - start; - if remaining <= TARGET_CAPTION_WORDS_PER_SEGMENT { + if remaining <= target { chunks.push(&words[start..]); break; } - let mut end = (start + TARGET_CAPTION_WORDS_PER_SEGMENT).min(words.len()); + let mut end = (start + target).min(words.len()); while end < words.len() && caption_boundary_word_is_weak(&words[end - 1]) - && end - start < MAX_CAPTION_WORDS_PER_SEGMENT + && end - start < max { end += 1; } @@ -281,4 +301,42 @@ mod tests { assert_eq!(chunks[0].len(), 6); assert_eq!(chunks[1].len(), 6); } + + #[test] + fn caption_word_chunks_with_one_word_per_segment_bypasses_multi_word_rules() { + let words = [ + "This", "is", "where", "we", "record", "I", "want", "clean", "captions", + ] + .iter() + .enumerate() + .map(|(index, text)| word(text, index)) + .collect::>(); + + let chunks = caption_word_chunks_with(&words, 1); + + assert_eq!(chunks.len(), words.len()); + for (chunk, word) in chunks.iter().zip(&words) { + assert_eq!(chunk.len(), 1); + assert_eq!(chunk[0].text, word.text); + } + } + + #[test] + fn caption_word_chunks_with_custom_size_keeps_the_multi_word_rules() { + let words = [ + "one", "two", "of", "four", "five", "six", "seven", "eight", "nine", + ] + .iter() + .enumerate() + .map(|(index, text)| word(text, index)) + .collect::>(); + + let chunks = caption_word_chunks_with(&words, 3); + + // The first boundary lands on "of", a weak word, so it extends one past + // the target of 3 instead of ending there. + assert_eq!(caption_text_from_words(chunks[0]), "one two of four"); + assert_eq!(caption_text_from_words(chunks[1]), "five six seven"); + assert_eq!(caption_text_from_words(chunks[2]), "eight nine"); + } } diff --git a/crates/project/src/configuration.rs b/crates/project/src/configuration.rs index edda76a60be..7e968da07df 100644 --- a/crates/project/src/configuration.rs +++ b/crates/project/src/configuration.rs @@ -2355,6 +2355,8 @@ pub struct CaptionSettings { pub word_transition_duration: f32, #[serde(alias = "activeWordHighlight")] pub active_word_highlight: bool, + #[serde(alias = "segmentWords")] + pub segment_words: u32, #[serde(alias = "manualPosition")] pub manual_position: Option>, pub preset: String, @@ -2389,6 +2391,10 @@ impl CaptionSettings { false } + fn default_segment_words() -> u32 { + 6 + } + fn default_preset() -> String { "classic".to_string() } @@ -2422,6 +2428,7 @@ impl Default for CaptionSettings { linger_duration: Self::default_linger_duration(), word_transition_duration: Self::default_word_transition_duration(), active_word_highlight: Self::default_active_word_highlight(), + segment_words: Self::default_segment_words(), manual_position: None, preset: Self::default_preset(), animation: Self::default_animation(), diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index f4bdcf5869d..9b5e2bc3c8f 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -13,7 +13,7 @@ pub use caption_timing::{derive_caption_track_segments, source_caption_id, synch pub use caption_words::{ MAX_CAPTION_WORD_DURATION, MAX_CAPTION_WORDS_PER_SEGMENT, MIN_FINAL_CAPTION_WORDS, TARGET_CAPTION_WORDS_PER_SEGMENT, caption_boundary_word_is_weak, caption_text_from_words, - caption_word_chunks, normalize_caption_words, + caption_word_chunks, caption_word_chunks_with, normalize_caption_words, }; pub use configuration::*; pub use cursor::*; diff --git a/crates/verify/tests/captions.rs b/crates/verify/tests/captions.rs index d1f3d72f868..770fc7dc8b6 100644 --- a/crates/verify/tests/captions.rs +++ b/crates/verify/tests/captions.rs @@ -899,6 +899,7 @@ fn styled_settings(rng: &mut Rng) -> CaptionSettings { linger_duration: 0.7, word_transition_duration: 0.1, active_word_highlight: true, + segment_words: rng.range(1, 9) as u32, manual_position: Some(cap_project::XY::new(0.25, 0.75)), preset: "custom".into(), animation: "pop".into(),