typecast_rust/
timestamps.rs

1//! Timestamp-aware TTS types and captioning helpers.
2//!
3//! This module exposes [`TTSRequestWithTimestamps`], [`TTSWithTimestampsResponse`],
4//! and alignment segment types. The response type provides [`TTSWithTimestampsResponse::to_srt`]
5//! and [`TTSWithTimestampsResponse::to_vtt`] for generating subtitle files from the
6//! word- or character-level alignment data returned by the API.
7
8use crate::errors::{Result, TypecastError};
9use base64::{engine::general_purpose::STANDARD as B64, Engine};
10use serde::{Deserialize, Serialize};
11use std::fs;
12use std::path::Path;
13
14// ---------------------------------------------------------------------------
15// Public types
16// ---------------------------------------------------------------------------
17
18/// A word-level alignment segment returned by the with-timestamps endpoint.
19#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
20pub struct AlignmentSegmentWord {
21    /// The word text.
22    pub text: String,
23    /// Start time in seconds.
24    pub start: f64,
25    /// End time in seconds.
26    pub end: f64,
27}
28
29/// A character-level alignment segment returned by the with-timestamps endpoint.
30#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
31pub struct AlignmentSegmentCharacter {
32    /// The character text.
33    pub text: String,
34    /// Start time in seconds.
35    pub start: f64,
36    /// End time in seconds.
37    pub end: f64,
38}
39
40/// Request body for `POST /v1/text-to-speech/with-timestamps`.
41#[derive(Debug, Clone, Serialize)]
42pub struct TTSRequestWithTimestamps {
43    /// Voice ID (e.g. `tc_60e5426de8b95f1d3000d7b5`).
44    /// Browse available API voices at <https://typecast.ai/developers/api/voices>.
45    pub voice_id: String,
46    /// Text to synthesize (max 2000 characters).
47    pub text: String,
48    /// TTS model to use (e.g. `"ssfm-v30"`).
49    pub model: crate::models::TTSModel,
50    /// Language code (ISO 639-3). Auto-detected when omitted.
51    #[serde(skip_serializing_if = "Option::is_none")]
52    pub language: Option<String>,
53    /// Emotion/style settings (accepts any serializable value that matches the
54    /// API's `prompt` field — use [`crate::models::TTSPrompt`] or raw JSON).
55    #[serde(skip_serializing_if = "Option::is_none")]
56    pub prompt: Option<serde_json::Value>,
57    /// Audio output settings (accepts any serializable value that matches the
58    /// API's `output` field — use [`crate::models::Output`] serialized to JSON).
59    #[serde(skip_serializing_if = "Option::is_none")]
60    pub output: Option<serde_json::Value>,
61    /// Random seed for reproducible results.
62    #[serde(skip_serializing_if = "Option::is_none")]
63    pub seed: Option<u32>,
64}
65
66impl TTSRequestWithTimestamps {
67    /// Create a new request with the required fields.
68    pub fn new(
69        voice_id: impl Into<String>,
70        text: impl Into<String>,
71        model: crate::models::TTSModel,
72    ) -> Self {
73        Self {
74            voice_id: voice_id.into(),
75            text: text.into(),
76            model,
77            language: None,
78            prompt: None,
79            output: None,
80            seed: None,
81        }
82    }
83
84    /// Set the language code (ISO 639-3).
85    pub fn language(mut self, language: impl Into<String>) -> Self {
86        self.language = Some(language.into());
87        self
88    }
89
90    /// Set the prompt field as a raw JSON value.
91    pub fn prompt(mut self, prompt: serde_json::Value) -> Self {
92        self.prompt = Some(prompt);
93        self
94    }
95
96    /// Set the output field as a raw JSON value.
97    pub fn output(mut self, output: serde_json::Value) -> Self {
98        self.output = Some(output);
99        self
100    }
101
102    /// Set the random seed.
103    pub fn seed(mut self, seed: u32) -> Self {
104        self.seed = Some(seed);
105        self
106    }
107}
108
109/// Response from `POST /v1/text-to-speech/with-timestamps`.
110///
111/// The `audio` field contains Base64-encoded audio data. Use
112/// [`audio_bytes`][TTSWithTimestampsResponse::audio_bytes] to decode it, or
113/// [`save_audio`][TTSWithTimestampsResponse::save_audio] to write directly to a file.
114///
115/// Call [`to_srt`][TTSWithTimestampsResponse::to_srt] or
116/// [`to_vtt`][TTSWithTimestampsResponse::to_vtt] to generate subtitle output
117/// from the alignment data.
118#[derive(Debug, Clone, Deserialize, Serialize)]
119pub struct TTSWithTimestampsResponse {
120    /// Base64-encoded audio bytes.
121    pub audio: String,
122    /// Audio container format (e.g. `"wav"` or `"mp3"`).
123    pub audio_format: String,
124    /// Total audio duration in seconds.
125    pub audio_duration: f64,
126    /// Word-level alignment segments (present when `granularity` is `"word"` or
127    /// when both granularities are returned).
128    pub words: Option<Vec<AlignmentSegmentWord>>,
129    /// Character-level alignment segments (present when `granularity` is
130    /// `"char"` or when both granularities are returned).
131    pub characters: Option<Vec<AlignmentSegmentCharacter>>,
132}
133
134impl TTSWithTimestampsResponse {
135    /// Decode the Base64-encoded audio field into raw bytes.
136    pub fn audio_bytes(&self) -> Result<Vec<u8>> {
137        B64.decode(&self.audio)
138            .map_err(|e| TypecastError::DecodeError(e.to_string()))
139    }
140
141    /// Decode the audio and write it to `path`.
142    pub fn save_audio<P: AsRef<Path>>(&self, path: P) -> Result<()> {
143        let bytes = self.audio_bytes()?;
144        fs::write(path, bytes).map_err(|e| TypecastError::IoError(e.to_string()))
145    }
146
147    /// Generate an SRT subtitle string from the alignment data.
148    ///
149    /// Word segments are preferred when there are at least two words; otherwise
150    /// character segments are used.
151    pub fn to_srt(&self) -> Result<String> {
152        format_captions(self, true)
153    }
154
155    /// Generate a WebVTT subtitle string from the alignment data.
156    ///
157    /// Word segments are preferred when there are at least two words; otherwise
158    /// character segments are used.
159    pub fn to_vtt(&self) -> Result<String> {
160        format_captions(self, false)
161    }
162}
163
164// ---------------------------------------------------------------------------
165// Internal captioning helpers
166// ---------------------------------------------------------------------------
167
168const MAX_CAPTION_SECONDS: f64 = 7.0;
169const MAX_CAPTION_CHARS: usize = 42;
170const SENTENCE_TERMINATORS: &[&str] = &[".", "?", "!", "\u{3002}", "\u{ff1f}", "\u{ff01}"];
171
172struct Segment {
173    text: String,
174    start: f64,
175    end: f64,
176}
177
178struct Cue {
179    text: String,
180    start: f64,
181    end: f64,
182}
183
184/// Choose which set of segments to use for captioning, and whether word-joining
185/// (space-separated) mode applies.
186///
187/// Priority: words (≥ 2) → characters (non-empty) → words (exactly 1) → error.
188fn pick_segments(resp: &TTSWithTimestampsResponse) -> Result<(Vec<Segment>, bool)> {
189    let word_segs = |words: &[crate::timestamps::AlignmentSegmentWord]| -> Vec<Segment> {
190        words
191            .iter()
192            .map(|w| Segment {
193                text: w.text.clone(),
194                start: w.start,
195                end: w.end,
196            })
197            .collect()
198    };
199    let char_segs = |chars: &[crate::timestamps::AlignmentSegmentCharacter]| -> Vec<Segment> {
200        chars
201            .iter()
202            .map(|c| Segment {
203                text: c.text.clone(),
204                start: c.start,
205                end: c.end,
206            })
207            .collect()
208    };
209
210    // Prefer words when there are at least 2 (single-word edge case falls through).
211    let multi_words = resp.words.as_deref().filter(|w| w.len() >= 2);
212    // Fall back to characters.
213    let chars = resp.characters.as_deref().filter(|c| !c.is_empty());
214    // Single-word fallback.
215    let single_word = resp.words.as_deref().filter(|w| w.len() == 1);
216
217    if let Some(words) = multi_words {
218        Ok((word_segs(words), true))
219    } else if let Some(c) = chars {
220        Ok((char_segs(c), false))
221    } else if let Some(words) = single_word {
222        Ok((word_segs(words), true))
223    } else {
224        Err(TypecastError::CaptioningError(
225            "no alignment segments to caption from".into(),
226        ))
227    }
228}
229
230/// Concatenate parts into a cue text.  In word mode parts are joined with a
231/// space; in character mode they are concatenated directly.
232fn join_parts(parts: &[String], word_mode: bool) -> String {
233    let sep = if word_mode { " " } else { "" };
234    parts.join(sep).trim().to_string()
235}
236
237/// Return `true` if `text` ends with a sentence-terminating punctuation mark.
238fn ends_in_sentence(text: &str) -> bool {
239    let trimmed = text.trim_end();
240    SENTENCE_TERMINATORS.iter().any(|t| trimmed.ends_with(t))
241}
242
243/// Group flat segments into captioning cues obeying the max-duration and
244/// max-char-count constraints and breaking on sentence-terminating punctuation.
245///
246/// TODO(TASK-12430-followup): expose max_seconds / max_chars override to match Python/JS API surface. Default 7.0s / 42 chars (BBC/Netflix guideline).
247/// TODO(TASK-12430-followup): warn or error when alignment array contains majority-empty text segments — server contract should never produce these but defense-in-depth is desirable.
248fn group_into_cues(segs: &[Segment], word_mode: bool) -> Vec<Cue> {
249    let mut cues: Vec<Cue> = Vec::new();
250    let mut parts: Vec<String> = Vec::new();
251    // Invariant: cur_start and last_end are always set whenever parts is non-empty.
252    // Using 0.0 as default sentinels; they are only read when parts is non-empty.
253    let mut cur_start: f64 = 0.0;
254    let mut last_end: f64 = 0.0;
255
256    /// Appends a cue only when its text is non-empty.
257    fn emit(cues: &mut Vec<Cue>, text: String, start: f64, end: f64) {
258        if !text.is_empty() {
259            cues.push(Cue { text, start, end });
260        }
261    }
262
263    for seg in segs {
264        // If we already have content, check whether adding this segment would
265        // violate a hard limit.
266        if !parts.is_empty() {
267            let mut tentative = parts.clone();
268            tentative.push(seg.text.clone());
269            let would_be = join_parts(&tentative, word_mode);
270            let too_long_secs = (seg.end - cur_start) > MAX_CAPTION_SECONDS;
271            let too_long_chars = would_be.chars().count() > MAX_CAPTION_CHARS;
272            if too_long_secs || too_long_chars {
273                // Flush the current cue before starting a new one.
274                emit(
275                    &mut cues,
276                    join_parts(&parts, word_mode),
277                    cur_start,
278                    last_end,
279                );
280                parts.clear();
281            }
282        }
283
284        // Record the start of a new cue.
285        if parts.is_empty() {
286            cur_start = seg.start;
287        }
288        parts.push(seg.text.clone());
289        last_end = seg.end;
290
291        // Break on sentence-terminating punctuation.
292        if ends_in_sentence(&seg.text) {
293            emit(&mut cues, join_parts(&parts, word_mode), cur_start, seg.end);
294            parts.clear();
295        }
296    }
297
298    // Flush any remaining parts.
299    if !parts.is_empty() {
300        emit(
301            &mut cues,
302            join_parts(&parts, word_mode),
303            cur_start,
304            last_end,
305        );
306    }
307
308    cues
309}
310
311/// Format `seconds` as `HH:MM:SS,mmm` (SRT comma separator).
312fn format_srt_time(seconds: f64) -> String {
313    let total_ms = (seconds * 1000.0).round() as i64;
314    let ms = total_ms % 1000;
315    let total_sec = total_ms / 1000;
316    let ss = total_sec % 60;
317    let total_min = total_sec / 60;
318    let mm = total_min % 60;
319    let hh = total_min / 60;
320    format!("{:02}:{:02}:{:02},{:03}", hh, mm, ss, ms)
321}
322
323/// Format `seconds` as `HH:MM:SS.mmm` (VTT dot separator).
324fn format_vtt_time(seconds: f64) -> String {
325    format_srt_time(seconds).replace(',', ".")
326}
327
328/// Core captioning formatter.  When `srt` is `true` emits SRT; otherwise VTT.
329fn format_captions(resp: &TTSWithTimestampsResponse, srt: bool) -> Result<String> {
330    let (segs, word_mode) = pick_segments(resp)?;
331    let cues = group_into_cues(&segs, word_mode);
332    if cues.is_empty() {
333        return Err(TypecastError::CaptioningError(
334            "no alignment segments to caption from".into(),
335        ));
336    }
337
338    let mut out = String::new();
339    if !srt {
340        out.push_str("WEBVTT\n\n");
341    }
342    for (i, cue) in cues.iter().enumerate() {
343        if srt {
344            out.push_str(&format!("{}\n", i + 1));
345        }
346        let (s, e) = if srt {
347            (format_srt_time(cue.start), format_srt_time(cue.end))
348        } else {
349            (format_vtt_time(cue.start), format_vtt_time(cue.end))
350        };
351        out.push_str(&format!("{} --> {}\n", s, e));
352        out.push_str(&cue.text);
353        out.push_str("\n\n");
354    }
355    Ok(out)
356}
typecast_rust/timestamps.rs

typecast_rust/
timestamps.rs
