Skip to main content

radio_cli/podcast/
transcript.rs

1//! Finite episode captions use media time, never the live capture clock.
2use super::{Episode, EpisodeMedia, error};
3use crate::{
4    config::Config,
5    error::{DomainError, ErrorCode},
6    speech::Segment,
7    store::Store,
8};
9use anyhow::{Result, ensure};
10use fs2::FileExt;
11use serde::{Deserialize, Serialize};
12use std::{
13    fs,
14    io::{Read, Write},
15    path::{Path, PathBuf},
16    sync::{
17        Arc,
18        atomic::{AtomicBool, Ordering},
19    },
20};
21
22pub(crate) const DOWNLOADING: u64 = u64::MAX;
23
24const LIMIT: u64 = 10 * 1024 * 1024;
25
26#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
27pub struct Transcript {
28    pub url: String,
29    pub mime: String,
30    pub language: Option<String>,
31}
32
33#[derive(Debug, Serialize, Deserialize)]
34struct Cache {
35    version: u8,
36    media: EpisodeMedia,
37    source: String,
38    segments: Vec<Segment>,
39}
40
41fn invalid() -> anyhow::Error {
42    error(crate::t!("podcast_subtitle_invalid"))
43}
44pub(super) fn check(stop: &AtomicBool) -> Result<()> {
45    ensure!(
46        !stop.load(Ordering::Relaxed),
47        DomainError::new(ErrorCode::Cancelled)
48    );
49    Ok(())
50}
51pub(super) fn path(config: &Config, episode: &Episode) -> PathBuf {
52    // IDs normally are UUIDs; hash anyway so imported or corrupt IDs cannot escape.
53    use sha2::{Digest, Sha256};
54    let key = format!("{:x}", Sha256::digest(episode.id.as_bytes()));
55    config
56        .data_dir
57        .join("podcasts/transcripts")
58        .join(format!("{key}.json"))
59}
60pub(super) fn read(path: &Path) -> Result<Vec<u8>> {
61    let mut data = vec![];
62    fs::File::open(path)?
63        .take(LIMIT + 1)
64        .read_to_end(&mut data)?;
65    ensure!(data.len() as u64 <= LIMIT, invalid());
66    Ok(data)
67}
68pub(super) fn validate(segments: &mut [Segment]) -> Result<()> {
69    ensure!(segments.len() <= 100_000, invalid());
70    for s in segments.iter_mut() {
71        ensure!(s.end_ms > s.start_ms, invalid());
72        s.epoch = 0;
73        s.text = crate::clean_text(&s.text);
74        ensure!(!s.text.trim().is_empty(), invalid());
75    }
76    segments.sort_by_key(|s| (s.start_ms, s.end_ms));
77    Ok(())
78}
79fn cached(config: &Config, episode: &Episode) -> Result<Option<Vec<Segment>>> {
80    let p = path(config, episode);
81    if !p.exists() {
82        return Ok(None);
83    }
84    let mut c: Cache = serde_json::from_slice(&read(&p)?)?;
85    if c.version != 1 || c.media != episode.media() {
86        return Ok(None);
87    }
88    // Explicit imports and generated captions stay usable offline. Feed captions
89    // are invalidated when their source disappears or changes.
90    if c.source.starts_with("http") && !episode.transcripts.iter().any(|t| t.url == c.source) {
91        return Ok(None);
92    }
93    validate(&mut c.segments)?;
94    Ok(Some(c.segments))
95}
96pub(super) fn save(
97    config: &Config,
98    episode: &Episode,
99    source: String,
100    mut segments: Vec<Segment>,
101    stop: &AtomicBool,
102) -> Result<Vec<Segment>> {
103    check(stop)?;
104    validate(&mut segments)?;
105    let p = path(config, episode);
106    fs::create_dir_all(p.parent().unwrap())?;
107    let lock = fs::OpenOptions::new()
108        .create(true)
109        .truncate(false)
110        .read(true)
111        .write(true)
112        .open(p.with_extension("lock"))?;
113    lock.lock_exclusive()?;
114    check(stop)?;
115    // A slow automatic fetch must never overwrite an explicit import or ASR
116    // result that finished while its request was in flight.
117    if source.starts_with("http")
118        && let Ok(Some(existing)) = cached(config, episode)
119    {
120        return Ok(existing);
121    }
122    let tmp = p.with_extension(format!("{}.tmp", uuid::Uuid::new_v4()));
123    let result = (|| {
124        let data = serde_json::to_vec(&Cache {
125            version: 1,
126            media: episode.media(),
127            source,
128            segments: segments.clone(),
129        })?;
130        ensure!(data.len() as u64 <= LIMIT, invalid());
131        let mut file = fs::OpenOptions::new()
132            .write(true)
133            .create_new(true)
134            .open(&tmp)?;
135        file.write_all(&data)?;
136        file.sync_all()?;
137        check(stop)?;
138        fs::rename(&tmp, &p)?;
139        Ok(segments)
140    })();
141    let _ = fs::remove_file(tmp);
142    result
143}
144
145/// SRT and WebVTT cues. Layout/style metadata is ignored; text is always plain.
146pub fn parse(data: &[u8]) -> Result<Vec<Segment>> {
147    let text = std::str::from_utf8(data)
148        .map_err(|_| invalid())?
149        .trim_start_matches('\u{feff}')
150        .replace("\r\n", "\n");
151    ensure!(data.len() as u64 <= LIMIT, invalid());
152    let mut segments = vec![];
153    for block in text.split("\n\n") {
154        let lines: Vec<_> = block.lines().collect();
155        if lines.first().is_none_or(|l| {
156            l.starts_with("NOTE") || l.starts_with("STYLE") || l.starts_with("REGION")
157        }) {
158            continue;
159        }
160        let Some(index) = lines.iter().position(|l| l.contains("-->")) else {
161            continue;
162        };
163        let (start, end) = lines[index].split_once("-->").ok_or_else(invalid)?;
164        let start_ms = timestamp(start.trim())?;
165        let end_ms = timestamp(end.split_whitespace().next().ok_or_else(invalid)?)?;
166        let body = lines[index + 1..].join("\n");
167        let body = scraper::Html::parse_fragment(&body)
168            .root_element()
169            .text()
170            .collect::<Vec<_>>()
171            .join("");
172        let body = body.split_whitespace().collect::<Vec<_>>().join(" ");
173        segments.push(Segment {
174            epoch: 0,
175            start_ms,
176            end_ms,
177            text: body,
178        });
179    }
180    ensure!(!segments.is_empty(), invalid());
181    validate(&mut segments)?;
182    Ok(segments)
183}
184fn timestamp(s: &str) -> Result<u64> {
185    let normalized = s.replace(',', ".");
186    let fields: Vec<_> = normalized.split(':').collect();
187    ensure!((2..=3).contains(&fields.len()), invalid());
188    let mut total = 0u64;
189    for (i, part) in fields.iter().enumerate() {
190        let value = if i + 1 == fields.len() {
191            let (whole, fraction) = part.split_once('.').ok_or_else(invalid)?;
192            ensure!(
193                fraction.len() == 3 && fraction.bytes().all(|c| c.is_ascii_digit()),
194                invalid()
195            );
196            let seconds = whole.parse::<u64>().map_err(|_| invalid())?;
197            ensure!(seconds < 60, invalid());
198            seconds * 1000 + fraction.parse::<u64>().map_err(|_| invalid())?
199        } else {
200            let n = part.parse::<u64>().map_err(|_| invalid())?;
201            ensure!(i == 0 && fields.len() == 3 || n < 60, invalid());
202            n.checked_mul(1000).ok_or_else(invalid)?
203        };
204        total = total
205            .checked_mul(60)
206            .and_then(|t| t.checked_add(value))
207            .ok_or_else(invalid)?;
208    }
209    Ok(total)
210}
211
212pub(crate) fn load(
213    config: &Config,
214    episode: &Episode,
215    stop: &Arc<AtomicBool>,
216) -> Result<Option<Vec<Segment>>> {
217    check(stop)?;
218    if let Ok(Some(segments)) = cached(config, episode) {
219        return Ok(Some(segments));
220    }
221    if let Ok(Some(segments)) = super::progressive::load_partial(config, episode) {
222        return Ok(Some(segments));
223    }
224    let mut last_error = None;
225    for t in &episode.transcripts {
226        if !matches!(
227            t.mime.as_str(),
228            "text/vtt" | "application/x-subrip" | "text/srt"
229        ) {
230            continue;
231        }
232        let result = (|| {
233            check(stop)?;
234            let url = super::feed::url(&t.url, None)?;
235            let response = super::feed::client(config)?
236                .get(url)
237                .send()
238                .map_err(super::feed::network_error)?;
239            let (_, data) = super::feed::body(response, stop)?;
240            save(config, episode, t.url.clone(), parse(&data)?, stop)
241        })();
242        match result {
243            Ok(s) => return Ok(Some(s)),
244            Err(e) => {
245                check(stop)?;
246                last_error = Some(e);
247            }
248        }
249    }
250    if let Some(e) = last_error {
251        return Err(e);
252    }
253    Ok(None)
254}
255
256fn prepare_audio(
257    config: &Config,
258    store: &Store,
259    episode: &Episode,
260    stop: &Arc<AtomicBool>,
261    progress: Option<&std::sync::atomic::AtomicU64>,
262) -> Result<super::Download> {
263    let download = store.podcast_download(&episode.id)?.filter(|d| {
264        d.state == "complete" && d.matches_media(episode) && Path::new(&d.path).is_file()
265    });
266    let download = if let Some(download) = download {
267        download
268    } else {
269        if let Some(progress) = progress {
270            progress.store(DOWNLOADING, Ordering::Relaxed);
271        } else {
272            eprintln!("{}", crate::t!("podcast_subtitle_downloading"));
273        }
274        super::download(config, &episode.id, stop)?
275    };
276    ensure!(
277        download.matches_media(episode),
278        error(crate::t!("podcast_subtitle_changed"))
279    );
280    if let Some(progress) = progress {
281        progress.store(0, Ordering::Relaxed);
282    }
283    Ok(download)
284}
285
286pub(crate) fn manage(
287    config: &Config,
288    store: &Store,
289    id: &str,
290    file: Option<&Path>,
291    transcribe: bool,
292    stop: &Arc<AtomicBool>,
293    progress: Option<&std::sync::atomic::AtomicU64>,
294) -> Result<serde_json::Value> {
295    let episode = store.podcast_episode(id)?;
296    let segments = if let Some(file) = file {
297        save(
298            config,
299            &episode,
300            "import".into(),
301            parse(&read(file)?)?,
302            stop,
303        )?
304    } else if transcribe {
305        crate::speech::check_transcription(config, stop)?;
306        let download = prepare_audio(config, store, &episode, stop, progress)?;
307        let segments =
308            crate::speech::transcribe(config, Path::new(&download.path), stop, progress)?;
309        // A refresh may have replaced the enclosure while inference was running.
310        ensure!(
311            store.podcast_episode(id)?.media() == episode.media(),
312            error(crate::t!("podcast_subtitle_changed"))
313        );
314        save(config, &episode, "local-asr".into(), segments, stop)?
315    } else {
316        load(config, &episode, stop)?.ok_or_else(|| error(crate::t!("podcast_subtitle_missing")))?
317    };
318    Ok(serde_json::json!({"id":id,"segments":segments.len()}))
319}
320
321#[cfg(test)]
322mod tests {
323    use super::*;
324    #[test]
325    fn transcription_prepares_audio_once_without_requiring_a_separate_download() {
326        use super::super::{Feed, Preview};
327        use crate::test_support::{Server, response};
328        let audio = crate::test_support::fixture("libmp3lame", "mp3");
329        let server = Server::new(move |_, s| response(s, 200, "audio/mpeg", &audio));
330        let dir = tempfile::tempdir().unwrap();
331        let config = crate::test_support::config(dir.path());
332        let store = Store::open(&config.data_dir.join("radio.db")).unwrap();
333        let p = Preview {
334            feed: Feed {
335                url: format!("{}/rss", server.url),
336                ..Default::default()
337            },
338            episodes: vec![Episode {
339                title: "Local episode".into(),
340                url: format!("{}/audio", server.url),
341                ..Default::default()
342            }],
343        };
344        let f = store.podcast_subscribe(&p, &p.feed.url).unwrap();
345        let episode = store.podcast_episodes(&f.id).unwrap().remove(0);
346        let stop = Arc::new(AtomicBool::new(false));
347        let progress = std::sync::atomic::AtomicU64::new(99);
348        let d = prepare_audio(&config, &store, &episode, &stop, Some(&progress)).unwrap();
349        assert_eq!(d.state, "complete");
350        assert!(d.matches_media(&episode));
351        assert_eq!(progress.load(Ordering::Relaxed), 0);
352        drop(server);
353        let cached = prepare_audio(&config, &store, &episode, &stop, None).unwrap();
354        assert_eq!(d.path, cached.path, "reuse current downloads offline");
355    }
356
357    #[test]
358    fn imports_are_atomic_offline_and_bound_to_media() {
359        let dir = tempfile::tempdir().unwrap();
360        let config = crate::test_support::config(dir.path());
361        let mut episode = Episode {
362            id: "../unsafe-id".into(),
363            url: "https://example.test/audio".into(),
364            ..Default::default()
365        };
366        let stop = AtomicBool::new(false);
367        let captions = parse(b"00:00.000 --> 00:01.000\nHello").unwrap();
368        save(&config, &episode, "import".into(), captions.clone(), &stop).unwrap();
369        assert_eq!(cached(&config, &episode).unwrap(), Some(captions.clone()));
370        assert!(path(&config, &episode).starts_with(config.data_dir.join("podcasts/transcripts")));
371        stop.store(true, Ordering::Relaxed);
372        assert!(save(&config, &episode, "import".into(), captions, &stop).is_err());
373        assert_eq!(cached(&config, &episode).unwrap().unwrap()[0].text, "Hello");
374        episode.bytes = Some(123);
375        assert!(cached(&config, &episode).unwrap().is_none());
376    }
377
378    #[test]
379    fn feed_subtitles_fall_back_then_cache_without_another_request() {
380        use crate::test_support::{Server, response};
381        let server = Server::new(|request, stream| {
382            if request.contains("/bad") {
383                response(stream, 200, "text/vtt", b"broken");
384            } else {
385                response(
386                    stream,
387                    200,
388                    "text/vtt",
389                    b"WEBVTT\n\n00:00.000 --> 00:02.000\nHello",
390                );
391            }
392        });
393        let dir = tempfile::tempdir().unwrap();
394        let config = crate::test_support::config(dir.path());
395        let mut episode = Episode {
396            id: "one".into(),
397            url: "https://example.test/audio".into(),
398            transcripts: ["bad", "good"]
399                .iter()
400                .map(|p| Transcript {
401                    url: format!("{}/{p}", server.url),
402                    mime: "text/vtt".into(),
403                    language: None,
404                })
405                .collect(),
406            ..Default::default()
407        };
408        let stop = Arc::new(AtomicBool::new(false));
409        assert_eq!(
410            load(&config, &episode, &stop).unwrap().unwrap()[0].text,
411            "Hello"
412        );
413        drop(server);
414        assert_eq!(load(&config, &episode, &stop).unwrap().unwrap().len(), 1);
415        episode.transcripts.clear();
416        assert!(cached(&config, &episode).unwrap().is_none());
417    }
418
419    #[test]
420    fn parses_vtt_and_srt_with_identifiers_settings_markup_and_unicode() {
421        let s = parse(b"\xef\xbb\xbfWEBVTT\r\n\r\ncue\r\n00:01.000 --> 00:03.500 align:start\r\n<v Alice>Hello &amp; welcome</v>\r\nagain\r\n\r\nNOTE ignored\r\n00:00.000 --> 00:01.000\r\nignore").unwrap();
422        assert_eq!((s[0].start_ms, s[0].end_ms), (1000, 3500));
423        assert_eq!(s[0].text, "Hello & welcome again");
424        let s = parse("1\n01:02:03,004 --> 01:02:04,005\n中文字幕".as_bytes()).unwrap();
425        assert_eq!(s[0].start_ms, 3723004);
426        assert_eq!(s[0].text, "中文字幕");
427        for bad in [
428            "text only",
429            "00:61.000 --> 00:62.000\nx",
430            "00:02.000 --> 00:01.000\nx",
431            "00:01.000 --> 00:02.000\n",
432        ] {
433            assert!(parse(bad.as_bytes()).is_err(), "{bad}");
434        }
435    }
436}