Skip to content

feat: configurable words per caption segment #3284

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Openroot-MI wants to merge 2 commits into from
Closed

feat: configurable words per caption segment #3284

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e1ded27
refactor: move caption word utilities into cap-project (#3274)
Ker102 Oct 25, 2026
77b2909
feat: configurable words per caption segment (#3275)
Ker102 Oct 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 16 additions & 1 deletion apps/desktop-gpui/src/editor_tabs.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -497,6 +497,7 @@ pub enum CaptionSlider {
Size,
BackgroundOpacity,
FadeDuration,
SegmentWords,
ManualX,
ManualY,
}
Expand Down Expand Up @@ -563,6 +564,8 @@ impl CaptionSlider {
pub(crate) fn changes_timeline_model(self) -> bool {
match self {
Self::Size | Self::BackgroundOpacity | Self::FadeDuration => false,
// Only feeds the next caption generation, existing segments are untouched.
Self::SegmentWords => false,
Self::ManualX | Self::ManualY => false,
}
}
Expand All @@ -573,6 +576,7 @@ impl CaptionSlider {
Self::BackgroundOpacity => (0., 100., 1.),
// `value={[getSetting("fadeDuration") * 100]}` over 0..50.
Self::FadeDuration => (0., 50., 1.),
Self::SegmentWords => (1., 8., 1.),
Self::ManualX | Self::ManualY => (0., 100., 1.),
}
}
Expand All @@ -583,6 +587,7 @@ impl CaptionSlider {
Self::Size => settings.size as f32,
Self::BackgroundOpacity => settings.background_opacity as f32,
Self::FadeDuration => settings.fade_duration * 100.,
Self::SegmentWords => settings.segment_words as f32,
Self::ManualX => caption_manual_center(&settings).0 * 100.,
Self::ManualY => caption_manual_center(&settings).1 * 100.,
}
Expand Down Expand Up @@ -1269,6 +1274,9 @@ impl EditorWindow {
settings.background_opacity = value.round() as u32
}
CaptionSlider::FadeDuration => settings.fade_duration = value / 100.,
CaptionSlider::SegmentWords => {
settings.segment_words = (value.round() as u32).max(1)
}
CaptionSlider::ManualX => set_caption_manual_center(settings, Some(value), None),
CaptionSlider::ManualY => set_caption_manual_center(settings, None, Some(value)),
},
Expand Down Expand Up @@ -3085,6 +3093,12 @@ impl EditorWindow {
SliderKey::Caption(CaptionSlider::FadeDuration),
format!("{:.0}ms", settings.fade_duration * 1000.),
cx,
))
.child(self.labelled_readout(
"Words Per Caption",
SliderKey::Caption(CaptionSlider::SegmentWords),
format!("{:.0}", settings.segment_words),
cx,
)),
),
)
Expand Down Expand Up @@ -3396,6 +3410,7 @@ impl EditorWindow {
let project_path = self.project_path.clone();
let model = self.selected_caption_model().name;
let language = self.sidebar.caption_language;
let segment_words = caption_settings(&self.project).segment_words.max(1) as usize;

if !transcription::begin_generation(&project_path) {
return;
Expand All @@ -3406,7 +3421,7 @@ impl EditorWindow {
let task_path = project_path.clone();
let task = gpui_tokio::Tokio::spawn(cx, async move {
tokio::task::spawn_blocking(move || {
transcription::transcribe_blocking(&task_path, model, language)
transcription::transcribe_blocking(&task_path, model, language, segment_words)
})
.await
.map_err(|error| format!("Transcription task panicked: {error}"))?
Expand Down
39 changes: 1 addition & 38 deletions apps/desktop-gpui/src/editor_window.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ use std::{

use cap_cursor_info::CursorFamily;
use cap_editor::{EditorFrameOutput, EditorInstance, EditorState};
use cap_project::caption_words::caption_text_from_words;
use cap_project::{
ClipSpeedAudioMode, Cursors, ProjectConfiguration, RecordingMeta, RecordingMetaInner,
StudioRecordingMeta, TimelineConfiguration, XY,
Expand Down Expand Up @@ -5346,44 +5347,6 @@ pub(crate) struct EditorNotice {
/// How long a notice stays up.
const EDITOR_NOTICE_DURATION: Duration = Duration::from_millis(2500);

fn caption_text_from_words(words: &[cap_project::CaptionWord]) -> String {
let mut text = String::new();
for word in words {
let value = word.text.trim();
if value.is_empty() {
continue;
}
let attaches = value.chars().next().is_some_and(|value| {
matches!(
value,
',' | '.'
| '!'
| '?'
| ';'
| ':'
| '%'
| ')'
| ']'
| '}'
| '\''
| '’'
| '、'
| '。'
| '！'
| '？'
| '；'
| '：'
| '，'
)
});
if !text.is_empty() && !attaches {
text.push(' ');
}
text.push_str(value);
}
text
}

fn fresh_caption_source_id(project: &ProjectConfiguration) -> String {
use std::hash::{BuildHasher, Hasher};

Expand Down
181 changes: 15 additions & 166 deletions apps/desktop-gpui/src/transcription.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1430,6 +1430,7 @@ fn process_with_whisper(
context: Arc<WhisperContext>,
language: &str,
transcription_hints: &[String],
segment_words: usize,
) -> Result<Vec<CaptionSegment>, String> {
let mut params = FullParams::new(SamplingStrategy::BeamSearch {
beam_size: 5,
Expand Down Expand Up @@ -1555,7 +1556,7 @@ fn process_with_whisper(
continue;
}

for (chunk_idx, chunk_words) in caption_word_chunks(&words).into_iter().enumerate() {
for (chunk_idx, chunk_words) in caption_word_chunks_with(&words, segment_words).into_iter().enumerate() {
let segment_text = caption_text_from_words(chunk_words);
let segment_start = chunk_words
.first()
Expand Down Expand Up @@ -1623,6 +1624,7 @@ fn parakeet_chunk_ranges(samples: &[f32], sample_rate: usize) -> Vec<std::ops::R
fn process_with_parakeet(
audio_path: &Path,
model_dir: &str,
segment_words: usize,
) -> Result<Vec<CaptionSegment>, String> {
let cached_model = {
let guard = PARAKEET_CONTEXT
Expand Down Expand Up @@ -1707,14 +1709,15 @@ fn process_with_parakeet(

tracing::info!("Parakeet produced {} timed tokens", tokens.len());

caption_segments_from_tokens(tokens)
caption_segments_from_tokens(tokens, segment_words)
}

/// Parakeet's timed word tokens -> caption segments: drop empty tokens,
/// normalize the words and chunk them.
#[cfg(any(test, not(all(target_os = "macos", target_arch = "x86_64"))))]
fn caption_segments_from_tokens(
tokens: impl IntoIterator<Item = CaptionWord>,
segment_words: usize,

@greptile-apps greptile-apps Bot Oct 25, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 GPUI tests cannot compile

Adding segment_words leaves existing tests calling the old signatures. verify_tests.rs calls caption_segments_from_tokens with one argument twice, and benchmark_parakeet_transcription calls process_with_parakeet with two. These missing arguments prevent the GPUI test target from compiling; #[ignore] does not skip compilation. Pass the default size in these existing tests.

Prompt To Fix With AI 
This is a comment left during a code review.
Path: apps/desktop-gpui/src/transcription.rs
Line: 1720

Comment:
**GPUI tests cannot compile**

Adding `segment_words` leaves existing tests calling the old signatures. `verify_tests.rs` calls `caption_segments_from_tokens` with one argument twice, and `benchmark_parakeet_transcription` calls `process_with_parakeet` with two. These missing arguments prevent the GPUI test target from compiling; `#[ignore]` does not skip compilation. Pass the default size in these existing tests.

---

For each issue above, determine whether it is valid and should be fixed. If so, fix it directly.

) -> Result<Vec<CaptionSegment>, String> {
let words = normalize_caption_words(
tokens
Expand All @@ -1733,7 +1736,7 @@ fn caption_segments_from_tokens(
}

let mut segments = Vec::new();
for (chunk_idx, chunk) in caption_word_chunks(&words).into_iter().enumerate() {
for (chunk_idx, chunk) in caption_word_chunks_with(&words, segment_words).into_iter().enumerate() {
segments.push(CaptionSegment {
id: format!("segment-{chunk_idx}"),
start: chunk.first().map_or(0.0, |word| word.start),
Expand All @@ -1750,6 +1753,7 @@ fn caption_segments_from_tokens(
fn process_with_parakeet(
_audio_path: &Path,
_model_dir: &str,
_segment_words: usize,
) -> Result<Vec<CaptionSegment>, String> {
Err(PARAKEET_UNSUPPORTED_MESSAGE.to_string())
}
Expand All @@ -1766,6 +1770,7 @@ pub fn transcribe_blocking(
project_path: &Path,
model: &str,
language: &str,
segment_words: usize,
) -> Result<Vec<CaptionSegment>, String> {
let requested = model;
// A download writes straight to the model's final path, so a model still
Expand Down Expand Up @@ -1794,14 +1799,15 @@ pub fn transcribe_blocking(
let _guard = lock_transcription_worker_slot();
if is_parakeet_model(model) {
tracing::info!("Using Parakeet TDT engine");
let result = process_with_parakeet(audio_path, model_file.to_string_lossy().as_ref());
let result =
process_with_parakeet(audio_path, model_file.to_string_lossy().as_ref(), segment_words);
schedule_parakeet_release();
result
} else {
tracing::info!("Using Whisper engine");
let hints = transcription_hints();
let context = get_whisper_context(model_file.to_string_lossy().as_ref())?;
let result = process_with_whisper(audio_path, context, language, &hints);
let result = process_with_whisper(audio_path, context, language, &hints, segment_words);
release_whisper_context_after_transcription();
result
}
Expand Down Expand Up @@ -1856,167 +1862,10 @@ fn extract_audio_from_video(video_path: &Path, output_path: &Path) -> Result<(),
// Caption words & chunking
// ---------------------------------------------------------------------------

/// `MAX_CAPTION_WORD_DURATION` (`captions.rs:75`): Whisper/Parakeet sometimes
/// stretch a trailing word's end across a following silence, which leaves the
/// rendered caption stuck on screen and duplicates the word across timeline
/// cuts once projected.
pub const MAX_CAPTION_WORD_DURATION: f32 = 2.5;
/// `captions.rs:67-69`.
const TARGET_CAPTION_WORDS_PER_SEGMENT: usize = 6;
const MAX_CAPTION_WORDS_PER_SEGMENT: usize = 8;
const MIN_FINAL_CAPTION_WORDS: usize = 3;

/// `caption_char_attaches_to_previous` (`captions.rs:745-767`).
fn caption_char_attaches_to_previous(value: char) -> bool {
matches!(
value,
',' | '.'
| '!'
| '?'
| ';'
| ':'
| '%'
| ')'
| ']'
| '}'
| '\''
| '’'
| '、'
| '。'
| '！'
| '？'
| '；'
| '：'
| '，'
)
}

fn caption_token_attaches_to_previous(text: &str) -> bool {
text.trim()
.chars()
.next()
.is_some_and(caption_char_attaches_to_previous)
}

/// `caption_boundary_word_is_weak` (`captions.rs:769-801`).
fn caption_boundary_word_is_weak(word: &CaptionWord) -> bool {
let normalized = word
.text
.trim()
.chars()
.filter(|c| c.is_alphanumeric())
.collect::<String>()
.to_lowercase();

normalized.chars().count() <= 1
|| matches!(
normalized.as_str(),
"an" | "as"
| "at"
| "be"
| "by"
| "do"
| "he"
| "if"
| "in"
| "is"
| "it"
| "me"
| "my"
| "of"
| "on"
| "or"
| "so"
| "to"
| "up"
| "we"
)
}

/// `normalize_caption_words` (`captions.rs:803-831`).
fn normalize_caption_words(words: Vec<CaptionWord>) -> Vec<CaptionWord> {
let mut normalized: Vec<CaptionWord> = Vec::with_capacity(words.len());

for word in words {
let text = word.text.trim();
if text.is_empty() {
continue;
}

if caption_token_attaches_to_previous(text)
&& let Some(previous) = normalized.last_mut()
{
previous.text.push_str(text);
previous.end = word.end;
} else {
normalized.push(CaptionWord {
text: text.to_string(),
start: word.start,
end: word.end,
});
}
}

for word in &mut normalized {
word.end = word.end.min(word.start + MAX_CAPTION_WORD_DURATION);
}

normalized
}

/// `caption_text_from_words` (`captions.rs:833-849`).
fn caption_text_from_words<'a>(words: impl IntoIterator<Item = &'a CaptionWord>) -> String {
let mut text = String::new();

for word in words {
let word_text = word.text.trim();
if word_text.is_empty() {
continue;
}

if !text.is_empty() && !caption_token_attaches_to_previous(word_text) {
text.push(' ');
}
text.push_str(word_text);
}

text
}

/// `caption_word_chunks` (`captions.rs:851-883`).
fn caption_word_chunks(words: &[CaptionWord]) -> Vec<&[CaptionWord]> {
let mut chunks = Vec::new();
let mut start = 0;

while start < words.len() {
let remaining = words.len() - start;
if remaining <= TARGET_CAPTION_WORDS_PER_SEGMENT {
chunks.push(&words[start..]);
break;
}

let mut end = (start + TARGET_CAPTION_WORDS_PER_SEGMENT).min(words.len());
while end < words.len()
&& caption_boundary_word_is_weak(&words[end - 1])
&& end - start < MAX_CAPTION_WORDS_PER_SEGMENT
{
end += 1;
}

let remaining_after = words.len() - end;
if remaining_after > 0
&& remaining_after < MIN_FINAL_CAPTION_WORDS
&& caption_boundary_word_is_weak(&words[end])
{
end = words.len();
}

chunks.push(&words[start..end]);
start = end;
}

chunks
}
use cap_project::caption_words::{
caption_text_from_words, caption_word_chunks, caption_word_chunks_with, normalize_caption_words,

@greptile-apps greptile-apps Bot Oct 25, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Normal builds gain unused imports

caption_word_chunks is now used only by tests, but both transcription modules import it in normal builds. This adds an unused-import warning. If a check treats warnings as errors, it fails. Move this import into the test modules or gate it with #[cfg(test)].

Prompt To Fix With AI 
This is a comment left during a code review.
Path: apps/desktop-gpui/src/transcription.rs
Line: 1866

Comment:
**Normal builds gain unused imports**

`caption_word_chunks` is now used only by tests, but both transcription modules import it in normal builds. This adds an unused-import warning. If a check treats warnings as errors, it fails. Move this import into the test modules or gate it with `#[cfg(test)]`.

---

For each issue above, determine whether it is valid and should be fixed. If so, fix it directly.

MAX_CAPTION_WORD_DURATION,
};

// ---------------------------------------------------------------------------
// Track derivation -- deriveCaptionTrackSegments, in Rust
Expand Down
Loading
Loading