Skip to main content

rightkit_transcript/
segment.rs

1//! Cue segmentation: phrase grouping, splitting, merging, line breaking.
2//!
3//! Donor: CutRight `cutright-captions` (itself ported from `video-media`'s
4//! caption model). Deterministic: identical words + config give identical cues.
5
6use serde::{Deserialize, Serialize};
7
8use crate::align::ALIGNMENT_TOLERANCE_MS;
9use crate::words::TimedWord;
10
11/// Named segmentation profile (replaces CutRight's timeline `Preset`).
12#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
13#[serde(rename_all = "snake_case")]
14pub enum Profile {
15    /// Landscape video (YouTube): 42 chars x 2 lines.
16    Standard,
17    /// Vertical feed (TikTok): 20 chars x 2 lines, short cues.
18    Vertical,
19    /// Reels / square feed: 24 chars x 2 lines.
20    Compact,
21}
22
23/// Per-profile segmentation limits.
24#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
25pub struct CaptionConfig {
26    pub max_chars_per_line: usize,
27    pub max_lines: usize,
28    /// Reading-speed ceiling: characters per second of cue time.
29    pub max_chars_per_second: f64,
30    /// Cues shorter than this are merged with a neighbour when possible, else
31    /// held on screen up to [`ALIGNMENT_TOLERANCE_MS`] into the following gap.
32    pub min_cue_ms: u64,
33    /// A cue never spans longer than this (it splits instead).
34    pub max_cue_ms: u64,
35    /// A gap between words larger than this always starts a new cue.
36    pub max_gap_ms: u64,
37}
38
39impl CaptionConfig {
40    pub fn for_profile(profile: Profile) -> Self {
41        match profile {
42            Profile::Standard => Self {
43                max_chars_per_line: 42,
44                max_lines: 2,
45                max_chars_per_second: 21.0,
46                min_cue_ms: 600,
47                max_cue_ms: 7000,
48                max_gap_ms: 700,
49            },
50            Profile::Vertical => Self {
51                max_chars_per_line: 20,
52                max_lines: 2,
53                max_chars_per_second: 18.0,
54                min_cue_ms: 600,
55                max_cue_ms: 4500,
56                max_gap_ms: 700,
57            },
58            Profile::Compact => Self {
59                max_chars_per_line: 24,
60                max_lines: 2,
61                max_chars_per_second: 18.0,
62                min_cue_ms: 600,
63                max_cue_ms: 4500,
64                max_gap_ms: 700,
65            },
66        }
67    }
68}
69
70impl Default for CaptionConfig {
71    fn default() -> Self {
72        Self::for_profile(Profile::Standard)
73    }
74}
75
76/// One caption cue on the output timeline. Cues never overlap.
77#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
78pub struct Cue {
79    pub start_ms: u64,
80    pub end_ms: u64,
81    /// Display lines (already broken; at most `max_lines` unless a single word
82    /// is longer than a line).
83    pub lines: Vec<String>,
84    /// Index range `[first_word, last_word]` into the normalized word list.
85    pub first_word: usize,
86    pub last_word: usize,
87}
88
89impl Cue {
90    pub fn text(&self) -> String {
91        self.lines.join("\n")
92    }
93}
94
95fn sentence_end(t: &str) -> bool {
96    t.ends_with(['.', '?', '!', '\u{2026}'])
97}
98
99fn clause_end(t: &str) -> bool {
100    t.ends_with([',', ';', ':', '\u{2014}', '\u{2013}'])
101}
102
103/// Trims, drops empty/zero-length words, collapses inner whitespace and clamps
104/// overlaps so each word ends no later than the next starts.
105pub fn normalize_words(words: &[TimedWord]) -> Vec<TimedWord> {
106    let mut out: Vec<TimedWord> = Vec::with_capacity(words.len());
107    for w in words {
108        let text = w.text.split_whitespace().collect::<Vec<_>>().join(" ");
109        if text.is_empty() || w.end_ms <= w.start_ms {
110            continue;
111        }
112        if let Some(prev) = out.last_mut() {
113            if prev.end_ms > w.start_ms {
114                prev.end_ms = w.start_ms;
115                if prev.end_ms <= prev.start_ms {
116                    out.pop();
117                }
118            }
119        }
120        out.push(TimedWord::new(text, w.start_ms, w.end_ms));
121    }
122    out
123}
124
125fn chars(s: &str) -> usize {
126    s.chars().count()
127}
128
129fn greedy_wrap(words: &[&str], max: usize) -> Vec<Vec<usize>> {
130    // Returns word-index ranges per line as index lists (start..end pairs flattened).
131    let mut lines: Vec<Vec<usize>> = Vec::new();
132    let mut cur: Vec<usize> = Vec::new();
133    let mut len = 0usize;
134    for (i, w) in words.iter().enumerate() {
135        let wl = chars(w);
136        if cur.is_empty() {
137            cur.push(i);
138            len = wl;
139        } else if len + 1 + wl <= max {
140            cur.push(i);
141            len += 1 + wl;
142        } else {
143            lines.push(std::mem::take(&mut cur));
144            cur.push(i);
145            len = wl;
146        }
147    }
148    if !cur.is_empty() {
149        lines.push(cur);
150    }
151    lines
152}
153
154fn line_count(group: &[TimedWord], max: usize) -> usize {
155    let w: Vec<&str> = group.iter().map(|w| w.text.as_str()).collect();
156    greedy_wrap(&w, max).len()
157}
158
159fn text_chars(group: &[TimedWord]) -> usize {
160    group
161        .iter()
162        .map(|w| chars(&w.text) + 1)
163        .sum::<usize>()
164        .saturating_sub(1)
165}
166
167fn span(group: &[TimedWord]) -> u64 {
168    group.last().unwrap().end_ms - group.first().unwrap().start_ms
169}
170
171fn needs_split(group: &[TimedWord], cfg: &CaptionConfig) -> bool {
172    if group.len() <= 1 {
173        return false;
174    }
175    let dur = span(group);
176    if dur > cfg.max_cue_ms {
177        return true;
178    }
179    if line_count(group, cfg.max_chars_per_line) > cfg.max_lines {
180        return true;
181    }
182    // Judge reading speed against the time the cue will actually be on screen.
183    let shown = dur.max(cfg.min_cue_ms) as f64 / 1000.0;
184    text_chars(group) as f64 > cfg.max_chars_per_second * shown
185}
186
187/// Split index (start of the right half): prefers sentence then clause ends
188/// near the middle, then the largest silence, falling back to the character
189/// midpoint.
190fn split_index(group: &[TimedWord]) -> usize {
191    let total = text_chars(group) as i64;
192    let mut best: Option<(i64, i64, usize)> = None; // (punct rank, distance, idx)
193    let mut run = 0i64;
194    for (i, w) in group.iter().enumerate().take(group.len() - 1) {
195        run += chars(&w.text) as i64 + 1;
196        let dist = (run - total / 2).abs();
197        // Punctuation only counts when it leaves a reasonable remainder.
198        let balanced = run * 4 >= total && (total - run) * 4 >= total;
199        let rank = if balanced && sentence_end(&w.text) {
200            0
201        } else if balanced && clause_end(&w.text) {
202            1
203        } else {
204            2
205        };
206        if best.is_none_or(|(r, d, _)| (rank, dist) < (r, d)) {
207            best = Some((rank, dist, i + 1));
208        }
209    }
210    best.map(|b| b.2).unwrap_or(1).clamp(1, group.len() - 1)
211}
212
213fn split_rec(group: Vec<TimedWord>, cfg: &CaptionConfig, out: &mut Vec<Vec<TimedWord>>) {
214    if group.is_empty() {
215        return;
216    }
217    if !needs_split(&group, cfg) {
218        out.push(group);
219        return;
220    }
221    let at = split_index(&group);
222    let mut left = group;
223    let right = left.split_off(at);
224    split_rec(left, cfg, out);
225    split_rec(right, cfg, out);
226}
227
228fn phrase_groups(words: &[TimedWord], cfg: &CaptionConfig) -> Vec<Vec<TimedWord>> {
229    let mut groups: Vec<Vec<TimedWord>> = Vec::new();
230    for w in words {
231        let new = match groups.last().and_then(|g| g.last()) {
232            None => true,
233            Some(last) => w.start_ms - last.end_ms > cfg.max_gap_ms || sentence_end(&last.text),
234        };
235        if new {
236            groups.push(vec![w.clone()]);
237        } else {
238            groups.last_mut().unwrap().push(w.clone());
239        }
240    }
241    groups
242}
243
244/// Merges a too-short cue into its neighbour when the pair still fits the
245/// limits, the gap is small and no sentence ends between them.
246fn merge_short(groups: Vec<Vec<TimedWord>>, cfg: &CaptionConfig) -> Vec<Vec<TimedWord>> {
247    let mut out: Vec<Vec<TimedWord>> = Vec::new();
248    let mut iter = groups.into_iter().peekable();
249    while let Some(mut cur) = iter.next() {
250        while span(&cur) < cfg.min_cue_ms {
251            let Some(next) = iter.peek() else { break };
252            let gap = next[0].start_ms - cur.last().unwrap().end_ms;
253            if gap > cfg.max_gap_ms || sentence_end(&cur.last().unwrap().text) {
254                break;
255            }
256            let mut joined = cur.clone();
257            joined.extend(next.iter().cloned());
258            if needs_split(&joined, cfg) {
259                break;
260            }
261            cur = joined;
262            iter.next();
263        }
264        // Also fold a short leftover into the previous cue when it fits.
265        if span(&cur) < cfg.min_cue_ms {
266            if let Some(prev) = out.last_mut() {
267                let gap = cur[0].start_ms - prev.last().unwrap().end_ms;
268                if gap <= cfg.max_gap_ms && !sentence_end(&prev.last().unwrap().text) {
269                    let mut joined = prev.clone();
270                    joined.extend(cur.iter().cloned());
271                    if !needs_split(&joined, cfg) {
272                        *prev = joined;
273                        continue;
274                    }
275                }
276            }
277        }
278        out.push(cur);
279    }
280    out
281}
282
283/// Breaks a cue's words into display lines: greedy wrap, then for two lines the
284/// balanced break that prefers a clause boundary.
285fn break_lines(group: &[TimedWord], cfg: &CaptionConfig) -> Vec<String> {
286    let w: Vec<&str> = group.iter().map(|w| w.text.as_str()).collect();
287    let greedy = greedy_wrap(&w, cfg.max_chars_per_line);
288    let join = |r: &[usize]| r.iter().map(|&i| w[i]).collect::<Vec<_>>().join(" ");
289    if greedy.len() != 2 || cfg.max_lines < 2 {
290        return greedy.iter().map(|l| join(l)).collect();
291    }
292    let all: Vec<usize> = (0..w.len()).collect();
293    let total: usize = all.iter().map(|&i| chars(w[i]) + 1).sum::<usize>() - 1;
294    let mut best: Option<((u8, usize), usize)> = None;
295    for k in 1..w.len() {
296        let l = join(&all[..k]);
297        let r = join(&all[k..]);
298        if chars(&l) > cfg.max_chars_per_line || chars(&r) > cfg.max_chars_per_line {
299            continue;
300        }
301        let diff = chars(&l).abs_diff(chars(&r));
302        let short = chars(&l).min(chars(&r));
303        let punct = (sentence_end(w[k - 1]) || clause_end(w[k - 1])) && short * 4 >= total;
304        let key = (u8::from(!punct), diff);
305        if best.is_none_or(|(b, _)| key < b) {
306            best = Some((key, k));
307        }
308    }
309    match best {
310        Some((_, k)) => vec![join(&all[..k]), join(&all[k..])],
311        None => greedy.iter().map(|l| join(l)).collect(),
312    }
313}
314
315/// Rules for [`group_words`], the lightweight cue grouper for engines whose
316/// words are already clean (no line breaking, no reading-speed limits). Use
317/// [`build_cues`] for caption-grade segmentation.
318#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
319pub struct GroupRules {
320    /// A cue closes once it holds this many words.
321    pub max_words: usize,
322    /// A cue closes once its span reaches this length.
323    pub max_span_ms: u64,
324    /// A gap larger than this between words starts a new cue.
325    pub max_gap_ms: u64,
326    /// Sentence-final punctuation (`.`, `?`, `!`) closes a cue holding at
327    /// least this many words.
328    pub sentence_min_words: usize,
329    /// Cues shorter than this are widened to it (zero-width cues never export).
330    pub min_cue_ms: u64,
331}
332
333impl Default for GroupRules {
334    /// The engine-side defaults `rightkit-asr` has always used: 12 words, 7 s,
335    /// 1 s gap, sentence close from 3 words, 80 ms floor (one encoder frame).
336    fn default() -> Self {
337        Self {
338            max_words: 12,
339            max_span_ms: 7_000,
340            max_gap_ms: 1_000,
341            sentence_min_words: 3,
342            min_cue_ms: 80,
343        }
344    }
345}
346
347fn single_line_cue(
348    text: &str,
349    start_ms: u64,
350    end_ms: u64,
351    first: usize,
352    last: usize,
353    min_cue_ms: u64,
354) -> Cue {
355    Cue {
356        start_ms,
357        end_ms: end_ms.max(start_ms + min_cue_ms),
358        lines: vec![text.trim().to_string()],
359        first_word: first,
360        last_word: last,
361    }
362}
363
364/// Groups words (in order, not re-normalised) into single-line cues by
365/// [`GroupRules`]. `first_word`/`last_word` index the input slice. A cue closes
366/// on sentence-final punctuation (after `sentence_min_words` words), at
367/// `max_words`, at `max_span_ms`, or on a gap over `max_gap_ms`.
368pub fn group_words(words: &[TimedWord], rules: &GroupRules) -> Vec<Cue> {
369    struct Open {
370        text: String,
371        start: u64,
372        end: u64,
373        first: usize,
374        last: usize,
375        count: usize,
376    }
377    let close =
378        |o: &Open| single_line_cue(&o.text, o.start, o.end, o.first, o.last, rules.min_cue_ms);
379    let open = |i: usize, w: &TimedWord| Open {
380        text: w.text.clone(),
381        start: w.start_ms,
382        end: w.end_ms,
383        first: i,
384        last: i,
385        count: 1,
386    };
387    let mut cues = Vec::new();
388    let mut current: Option<Open> = None;
389    for (i, word) in words.iter().enumerate() {
390        match current.as_mut() {
391            Some(c) => {
392                let gap_over = word.start_ms > c.end && word.start_ms - c.end > rules.max_gap_ms;
393                if gap_over || c.count >= rules.max_words || c.end - c.start >= rules.max_span_ms {
394                    cues.push(close(c));
395                    current = Some(open(i, word));
396                } else {
397                    c.text.push(' ');
398                    c.text.push_str(&word.text);
399                    c.end = word.end_ms;
400                    c.last = i;
401                    c.count += 1;
402                }
403            }
404            None => current = Some(open(i, word)),
405        }
406        if current.as_ref().is_some_and(|c| {
407            c.text.ends_with(['.', '?', '!']) && c.count >= rules.sentence_min_words
408        }) {
409            cues.push(close(&current.take().expect("cue present")));
410        }
411    }
412    if let Some(c) = current {
413        cues.push(close(&c));
414    }
415    cues
416}
417
418/// One single-line cue per span, in order (`first_word == last_word ==` the
419/// span's index), widened to `min_cue_ms`. For callers that already hold cues
420/// as [`TimedWord`] spans (text = cue text) and only need export.
421pub fn cues_from_spans(spans: &[TimedWord], min_cue_ms: u64) -> Vec<Cue> {
422    spans
423        .iter()
424        .enumerate()
425        .map(|(i, s)| single_line_cue(&s.text, s.start_ms, s.end_ms, i, i, min_cue_ms))
426        .collect()
427}
428
429/// Builds non-overlapping cues from output-time words.
430///
431/// Every cue starts at a word start and ends at a word end, except that a cue
432/// shorter than `min_cue_ms` is held on screen up to [`ALIGNMENT_TOLERANCE_MS`]
433/// past its last word (never into the next cue).
434pub fn build_cues(words: &[TimedWord], cfg: &CaptionConfig) -> Vec<Cue> {
435    let words = normalize_words(words);
436    let mut groups = Vec::new();
437    for phrase in phrase_groups(&words, cfg) {
438        split_rec(phrase, cfg, &mut groups);
439    }
440    let groups = merge_short(groups, cfg);
441
442    // Word index of the start of each group.
443    let mut cues: Vec<Cue> = Vec::with_capacity(groups.len());
444    let mut idx = 0usize;
445    for g in &groups {
446        let first = idx;
447        idx += g.len();
448        cues.push(Cue {
449            start_ms: g[0].start_ms,
450            end_ms: g.last().unwrap().end_ms,
451            lines: break_lines(g, cfg),
452            first_word: first,
453            last_word: idx - 1,
454        });
455    }
456    for i in 0..cues.len() {
457        let next_start = cues.get(i + 1).map(|c| c.start_ms);
458        let c = &mut cues[i];
459        if c.end_ms - c.start_ms < cfg.min_cue_ms {
460            let mut end = (c.start_ms + cfg.min_cue_ms).min(c.end_ms + ALIGNMENT_TOLERANCE_MS);
461            if let Some(n) = next_start {
462                end = end.min(n);
463            }
464            c.end_ms = end.max(c.end_ms);
465        }
466    }
467    cues
468}