Skip to main content

alpha2_model/
caption.rs

1//! Caption tracks: timed text drawn in a track's style, each cue in a named style or its
2//! speaker's.
3
4use std::borrow::Cow;
5use std::collections::BTreeMap;
6
7use crate::{
8    FontSizeFit, LineClamp, MAX_TEXT_CONTENT_BYTES, Paint, TextAlign, TextAnimator, TextBackground,
9    TextShadow, TextStroke, TextStyle, Vec2,
10};
11use alpha2_time::TimeRange;
12use serde::{Deserialize, Serialize};
13
14/// Most named styles one caption track holds.
15pub const MAX_CAPTION_STYLES: usize = 32;
16
17/// Most bytes of a caption style's name or a cue's speaker.
18pub const MAX_CAPTION_NAME_BYTES: usize = 64;
19
20/// What a caption track shows: its cues, in its style.
21#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
22#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
23#[serde(deny_unknown_fields)]
24pub struct Captions {
25    /// The style every cue starts from.
26    #[serde(default)]
27    pub style: TextStyle,
28    /// Named changes to `style` for cues that name one or whose speaker has one. At most
29    /// [`MAX_CAPTION_STYLES`], names 1 to 64 bytes.
30    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
31    pub styles: BTreeMap<String, TextStylePatch>,
32    /// Cues in timeline order, none overlapping; a track shows one cue at a time. Empty when
33    /// `from_words` gives the cues.
34    #[serde(default)]
35    #[serde(deserialize_with = "crate::bounded::seq::<_, _, { crate::MAX_CLIPS_PER_TRACK }>")]
36    #[cfg_attr(feature = "schema", schemars(length(max = crate::MAX_CLIPS_PER_TRACK)))]
37    pub cues: Vec<CaptionCue>,
38    /// Cues built at evaluation from a `words` data set, instead of `cues`.
39    #[serde(default, skip_serializing_if = "Option::is_none")]
40    pub from_words: Option<crate::CaptionsFromWords>,
41}
42
43/// Text shown over one timeline interval.
44#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
45#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
46#[serde(deny_unknown_fields)]
47pub struct CaptionCue {
48    /// When the cue shows. Non-empty and not before zero.
49    pub timeline_range: TimeRange,
50    /// The text, a string or runs; `\n` starts a line. Non-empty, at most
51    /// [`MAX_TEXT_CONTENT_BYTES`] bytes. Run times are from the cue's start and lie within it.
52    pub text: crate::TextContent,
53    /// Who says it, as the source names them (a WebVTT voice), at most 64 bytes. A track style
54    /// of that name applies when the cue names no style.
55    #[serde(default, skip_serializing_if = "Option::is_none")]
56    pub speaker: Option<String>,
57    /// The name of one of the track's `styles`, applied over its style.
58    #[serde(default, skip_serializing_if = "Option::is_none")]
59    pub style: Option<String>,
60}
61
62impl CaptionCue {
63    /// A cue showing `text` over `timeline_range`, with no speaker or style.
64    #[must_use]
65    pub fn new(timeline_range: TimeRange, text: impl Into<crate::TextContent>) -> Self {
66        Self {
67            timeline_range,
68            text: text.into(),
69            speaker: None,
70            style: None,
71        }
72    }
73}
74
75/// Changes to a [`TextStyle`]. Each field set replaces the base's whole value (lists are not
76/// merged); a field left out keeps the base's.
77#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
78#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
79#[serde(deny_unknown_fields)]
80pub struct TextStylePatch {
81    /// The family; `null` draws from the fallback order alone.
82    #[serde(
83        default,
84        skip_serializing_if = "Option::is_none",
85        with = "double_option"
86    )]
87    #[cfg_attr(feature = "schema", schemars(with = "Option<String>"))]
88    pub font_family: Option<Option<String>>,
89    /// See [`TextStyle::font_size`].
90    #[serde(default, skip_serializing_if = "Option::is_none")]
91    pub font_size: Option<f32>,
92    /// See [`TextStyle::fill`].
93    #[serde(default, skip_serializing_if = "Option::is_none")]
94    pub fill: Option<Paint>,
95    /// See [`TextStyle::line_height`].
96    #[serde(default, skip_serializing_if = "Option::is_none")]
97    pub line_height: Option<f32>,
98    /// See [`TextStyle::letter_spacing`].
99    #[serde(default, skip_serializing_if = "Option::is_none")]
100    pub letter_spacing: Option<f32>,
101    /// See [`TextStyle::font_weight`].
102    #[serde(default, skip_serializing_if = "Option::is_none")]
103    pub font_weight: Option<u16>,
104    /// See [`TextStyle::align`].
105    #[serde(default, skip_serializing_if = "Option::is_none")]
106    pub align: Option<TextAlign>,
107    /// See [`TextStyle::max_width`].
108    #[serde(default, skip_serializing_if = "Option::is_none")]
109    pub max_width: Option<f32>,
110    /// See [`TextStyle::max_height`]; `null` removes the base's.
111    #[serde(
112        default,
113        skip_serializing_if = "Option::is_none",
114        with = "double_option"
115    )]
116    #[cfg_attr(feature = "schema", schemars(with = "Option<f32>"))]
117    pub max_height: Option<Option<f32>>,
118    /// See [`TextStyle::font_size_fit`]; `null` removes the base's.
119    #[serde(
120        default,
121        skip_serializing_if = "Option::is_none",
122        with = "double_option"
123    )]
124    #[cfg_attr(feature = "schema", schemars(with = "Option<FontSizeFit>"))]
125    pub font_size_fit: Option<Option<FontSizeFit>>,
126    /// See [`TextStyle::line_clamp`]; `null` removes the base's.
127    #[serde(
128        default,
129        skip_serializing_if = "Option::is_none",
130        with = "double_option"
131    )]
132    #[cfg_attr(feature = "schema", schemars(with = "Option<LineClamp>"))]
133    pub line_clamp: Option<Option<LineClamp>>,
134    /// See [`TextStyle::position`].
135    #[serde(default, skip_serializing_if = "Option::is_none")]
136    pub position: Option<Vec2>,
137    /// See [`TextStyle::anchor`].
138    #[serde(default, skip_serializing_if = "Option::is_none")]
139    pub anchor: Option<Vec2>,
140    /// See [`TextStyle::strokes`].
141    #[serde(default, skip_serializing_if = "Option::is_none")]
142    #[serde(deserialize_with = "crate::bounded::option_seq::<_, _, { crate::MAX_TEXT_STROKES }>")]
143    #[cfg_attr(feature = "schema", schemars(length(max = crate::MAX_TEXT_STROKES)))]
144    pub strokes: Option<Vec<TextStroke>>,
145    /// See [`TextStyle::shadows`].
146    #[serde(default, skip_serializing_if = "Option::is_none")]
147    #[serde(deserialize_with = "crate::bounded::option_seq::<_, _, { crate::MAX_TEXT_SHADOWS }>")]
148    #[cfg_attr(feature = "schema", schemars(length(max = crate::MAX_TEXT_SHADOWS)))]
149    pub shadows: Option<Vec<TextShadow>>,
150    /// The box behind each line; `null` removes the base's.
151    #[serde(
152        default,
153        skip_serializing_if = "Option::is_none",
154        with = "double_option"
155    )]
156    #[cfg_attr(feature = "schema", schemars(with = "Option<TextBackground>"))]
157    pub background: Option<Option<TextBackground>>,
158    /// See [`TextStyle::animators`].
159    #[serde(default, skip_serializing_if = "Option::is_none")]
160    #[serde(
161        deserialize_with = "crate::bounded::option_seq::<_, _, { crate::MAX_TEXT_ANIMATORS }>"
162    )]
163    #[cfg_attr(feature = "schema", schemars(length(max = crate::MAX_TEXT_ANIMATORS)))]
164    pub animators: Option<Vec<TextAnimator>>,
165}
166
167impl TextStylePatch {
168    /// `base` with every field this patch sets replaced.
169    #[must_use]
170    pub fn apply(&self, base: &TextStyle) -> TextStyle {
171        let mut style = base.clone();
172        if let Some(family) = &self.font_family {
173            style.font_family.clone_from(family);
174        }
175        if let Some(size) = self.font_size {
176            style.font_size = size;
177        }
178        if let Some(fill) = &self.fill {
179            style.fill = fill.clone();
180        }
181        if let Some(height) = self.line_height {
182            style.line_height = height;
183        }
184        if let Some(spacing) = self.letter_spacing {
185            style.letter_spacing = spacing;
186        }
187        if let Some(weight) = self.font_weight {
188            style.font_weight = weight;
189        }
190        if let Some(align) = self.align {
191            style.align = align;
192        }
193        if let Some(width) = self.max_width {
194            style.max_width = width;
195        }
196        if let Some(height) = self.max_height {
197            style.max_height = height;
198        }
199        if let Some(fit) = self.font_size_fit {
200            style.font_size_fit = fit;
201        }
202        if let Some(clamp) = &self.line_clamp {
203            style.line_clamp.clone_from(clamp);
204        }
205        if let Some(position) = self.position {
206            style.position = position;
207        }
208        if let Some(anchor) = self.anchor {
209            style.anchor = anchor;
210        }
211        if let Some(strokes) = &self.strokes {
212            style.strokes.clone_from(strokes);
213        }
214        if let Some(shadows) = &self.shadows {
215            style.shadows.clone_from(shadows);
216        }
217        if let Some(background) = self.background {
218            style.background = background;
219        }
220        if let Some(animators) = &self.animators {
221            style.animators.clone_from(animators);
222        }
223        style
224    }
225}
226
227/// Largest caption font size, as a fraction of the frame's shorter side.
228pub const MAX_CAPTION_FONT_SIZE: f32 = 0.25;
229
230/// Why a caption track's cue or style is refused.
231#[derive(Clone, Debug, PartialEq, Eq)]
232pub(crate) enum CaptionIssue {
233    /// The track's style, a style name, a speaker, a cue's range, text or run times, or the
234    /// cues' order.
235    Invalid,
236    /// Cue `cue` names a style the track does not have.
237    UnknownStyle { cue: usize, name: String },
238    /// Cue `cue`'s style, its patch applied, is out of range.
239    CueStyle { cue: usize },
240}
241
242impl Captions {
243    /// The patch cue `cue` takes: the style it names, else its speaker's if the track has one,
244    /// else none.
245    #[must_use]
246    pub fn cue_patch(&self, cue: &CaptionCue) -> Option<&TextStylePatch> {
247        match &cue.style {
248            Some(name) => self.styles.get(name),
249            None => cue
250                .speaker
251                .as_ref()
252                .and_then(|speaker| self.styles.get(speaker)),
253        }
254    }
255
256    /// The style cue `cue` draws in: the track's, with its patch applied.
257    #[must_use]
258    pub fn cue_style(&self, cue: &CaptionCue) -> Cow<'_, TextStyle> {
259        match self.cue_patch(cue) {
260            Some(patch) => Cow::Owned(patch.apply(&self.style)),
261            None => Cow::Borrowed(&self.style),
262        }
263    }
264
265    pub(crate) fn validate(&self) -> Result<(), CaptionIssue> {
266        let name = |name: &str| !name.is_empty() && name.len() <= MAX_CAPTION_NAME_BYTES;
267        let valid = (self.from_words.is_none() || self.cues.is_empty())
268            && self.style.is_valid(MAX_CAPTION_FONT_SIZE)
269            && self.styles.len() <= MAX_CAPTION_STYLES
270            && self.styles.keys().all(|key| name(key))
271            && self.cues.iter().all(|cue| {
272                !cue.timeline_range.is_empty()
273                    && cue.timeline_range.start().ticks() >= 0
274                    && cue.text.is_valid(MAX_TEXT_CONTENT_BYTES)
275                    && cue.speaker.as_deref().is_none_or(name)
276                    && run_times_lie_in(cue)
277            })
278            && self
279                .cues
280                .windows(2)
281                .all(|pair| pair[0].timeline_range.end() <= pair[1].timeline_range.start());
282        if !valid {
283            return Err(CaptionIssue::Invalid);
284        }
285        for (index, cue) in self.cues.iter().enumerate() {
286            if let Some(style) = &cue.style
287                && !self.styles.contains_key(style)
288            {
289                return Err(CaptionIssue::UnknownStyle {
290                    cue: index,
291                    name: style.clone(),
292                });
293            }
294            if self.cue_patch(cue).is_some() && !self.cue_style(cue).is_valid(MAX_CAPTION_FONT_SIZE)
295            {
296                return Err(CaptionIssue::CueStyle { cue: index });
297            }
298        }
299        Ok(())
300    }
301}
302
303/// Whether every run time of `cue` lies within the cue's duration.
304fn run_times_lie_in(cue: &CaptionCue) -> bool {
305    let duration = cue
306        .timeline_range
307        .duration()
308        .map_or(0, alpha2_time::MediaTime::ticks);
309    cue.text.runs().iter().all(|run| {
310        run.time
311            .is_none_or(|time| time.start.ticks() >= 0 && time.end.ticks() <= duration)
312    })
313}
314
315/// A field that is absent (no change), `null` (clear) or a value, read as `Option<Option<T>>`
316/// with `null` kept apart from absent.
317#[allow(clippy::option_option, clippy::ref_option)]
318mod double_option {
319    use serde::{Deserialize, Deserializer, Serialize, Serializer};
320
321    pub(super) fn serialize<T: Serialize, S: Serializer>(
322        value: &Option<Option<T>>,
323        serializer: S,
324    ) -> Result<S::Ok, S::Error> {
325        match value {
326            Some(inner) => inner.serialize(serializer),
327            None => serializer.serialize_none(),
328        }
329    }
330
331    pub(super) fn deserialize<'de, T: Deserialize<'de>, D: Deserializer<'de>>(
332        deserializer: D,
333    ) -> Result<Option<Option<T>>, D::Error> {
334        Option::<T>::deserialize(deserializer).map(Some)
335    }
336}
337
338/// What joins word `left` to word `right` in a cue: nothing between two characters of scripts
339/// written without word spaces, else one space.
340fn word_separator(left: &str, right: &str) -> &'static str {
341    match (left.chars().next_back(), right.chars().next()) {
342        (Some(last), Some(first)) if unspaced(last) && unspaced(first) => "",
343        _ => " ",
344    }
345}
346
347/// Whether `character` is of line breaking class ID, CJ or SA (UAX #14), as far as scripts
348/// written without spaces between words go. Hangul is spaced and not included.
349fn unspaced(character: char) -> bool {
350    matches!(
351        u32::from(character),
352        0x0E00..=0x0EFF // Thai, Lao
353            | 0x1000..=0x109F // Myanmar
354            | 0x1780..=0x17FF // Khmer
355            | 0x19E0..=0x19FF // Khmer symbols
356            | 0x2E80..=0x2FDF // CJK and Kangxi radicals
357            | 0x3000..=0x303F // CJK symbols and punctuation
358            | 0x3040..=0x30FF // Hiragana, Katakana
359            | 0x3100..=0x312F // Bopomofo
360            | 0x31F0..=0x31FF // Katakana phonetic extensions
361            | 0x3400..=0x4DBF // CJK extension A
362            | 0x4E00..=0x9FFF // CJK unified ideographs
363            | 0xA9E0..=0xA9FF // Myanmar extended B
364            | 0xAA60..=0xAA7F // Myanmar extended A
365            | 0xF900..=0xFAFF // CJK compatibility ideographs
366            | 0xFF00..=0xFFEF // Halfwidth and fullwidth forms
367            | 0x20000..=0x3FFFF // CJK extensions B and later
368    )
369}
370
371/// The longest gap between two words of one cue, in ticks (0.6 s).
372pub const WORD_PAUSE_TICKS: i64 = 72_000;
373
374/// Most cues a caption track derives from words, as many as one subtitle file holds.
375pub const MAX_DERIVED_CUES: usize = crate::MAX_CAPTION_FILE_CUES;
376
377/// Words grouped into cues that end at `max_chars` characters, a sentence's end, a pause longer
378/// than [`WORD_PAUSE_TICKS`], a change of speaker, or before the text would pass
379/// [`MAX_TEXT_CONTENT_BYTES`] or [`crate::MAX_TEXT_RUNS`] runs. Each cue has one run per word
380/// (with the separator that follows it), timed from the cue's start, so Timing selectors follow
381/// the speech. A word whose text trims to nothing is skipped.
382///
383/// Words are joined by a space, except between two characters of scripts written without word
384/// spaces (line breaking classes ID, CJ and SA of UAX #14).
385///
386/// `words` are in start order, each ending after it starts. Every cue is a valid explicit cue,
387/// and the cues are ordered and disjoint: where words overlap, each cue ends where the next
388/// starts, and of two cues with one start only the later is kept.
389#[must_use]
390pub fn group_words(words: &[crate::DataWord], max_chars: u32) -> Vec<CaptionCue> {
391    // Each cue being built: its start, end, speaker, words with their times, and its length in
392    // characters and in bytes.
393    type Words = (
394        i64,
395        i64,
396        Option<String>,
397        Vec<(String, i64, i64)>,
398        usize,
399        usize,
400    );
401    let mut groups: Vec<Words> = Vec::new();
402    let mut current: Option<Words> = None;
403    for word in words {
404        let (start, end) = (word.start.ticks(), word.end.ticks());
405        let text = word.text.trim().to_owned();
406        if text.is_empty() {
407            continue;
408        }
409        let characters = text.chars().count();
410        if let Some((_, cue_end, speaker, held, held_length, held_bytes)) = &mut current {
411            let last = held.last().map_or("", |(word, ..)| word.as_str());
412            let separator = word_separator(last, &text).len();
413            let length = *held_length + separator + characters;
414            let bytes = *held_bytes + separator + text.len();
415            let sentence_end = last.ends_with([
416                '.', '?', '!', '\u{3002}', '\u{ff01}', '\u{ff0e}', '\u{ff1f}',
417            ]);
418            if length <= max_chars as usize
419                && bytes <= MAX_TEXT_CONTENT_BYTES
420                && held.len() < crate::MAX_TEXT_RUNS
421                && start - *cue_end <= WORD_PAUSE_TICKS
422                && !sentence_end
423                && *speaker == word.speaker
424            {
425                held.push((text, start, end));
426                *cue_end = (*cue_end).max(end);
427                *held_length = length;
428                *held_bytes = bytes;
429                continue;
430            }
431            groups.extend(current.take());
432        }
433        let bytes = text.len();
434        current = Some((
435            start,
436            end,
437            word.speaker.clone(),
438            vec![(text, start, end)],
439            characters,
440            bytes,
441        ));
442    }
443    groups.extend(current);
444    let mut cues = groups
445        .into_iter()
446        .filter_map(|(start, end, speaker, held, _, _)| {
447            let separators = held
448                .windows(2)
449                .map(|pair| word_separator(&pair[0].0, &pair[1].0))
450                .chain([""])
451                .collect::<Vec<_>>();
452            let runs = held
453                .into_iter()
454                .zip(separators)
455                .map(|((mut text, from, to), separator)| {
456                    text.push_str(separator);
457                    crate::TextRun {
458                        text,
459                        class: Vec::new(),
460                        time: Some(crate::RunTime {
461                            start: alpha2_time::MediaTime(from - start),
462                            end: alpha2_time::MediaTime(to - start),
463                        }),
464                    }
465                })
466                .collect();
467            let range =
468                TimeRange::new(alpha2_time::MediaTime(start), alpha2_time::MediaTime(end)).ok()?;
469            Some(CaptionCue {
470                speaker,
471                ..CaptionCue::new(range, crate::TextContent::Runs(runs))
472            })
473        })
474        .collect::<Vec<_>>();
475    end_cues_where_the_next_starts(&mut cues);
476    cues
477}
478
479/// Ends each cue of `cues`, sorted by start, where the next starts, and drops a cue left empty.
480pub fn end_cues_where_the_next_starts(cues: &mut Vec<CaptionCue>) {
481    for index in 1..cues.len() {
482        let next = cues[index].timeline_range.start();
483        let previous = &mut cues[index - 1];
484        if previous.timeline_range.end() > next {
485            let start = previous.timeline_range.start().min(next);
486            previous.timeline_range =
487                TimeRange::new(start, next).unwrap_or(previous.timeline_range);
488            clip_runs(previous);
489        }
490    }
491    cues.retain(|cue| !cue.timeline_range.is_empty());
492}
493
494/// Clamps a cue's run times to its range after the range was shortened. A run that would start
495/// at or after the end gets no time.
496pub fn clip_runs(cue: &mut CaptionCue) {
497    let duration = cue
498        .timeline_range
499        .duration()
500        .map_or(0, alpha2_time::MediaTime::ticks);
501    if let crate::TextContent::Runs(runs) = &mut cue.text {
502        for run in runs {
503            run.time = run.time.and_then(|time| {
504                let end = time.end.ticks().min(duration);
505                (time.start.ticks() < end).then_some(crate::RunTime {
506                    start: time.start,
507                    end: alpha2_time::MediaTime(end),
508                })
509            });
510        }
511    }
512}