1use super::{Episode, EpisodeMedia, error};
3use crate::{
4 config::Config,
5 error::{DomainError, ErrorCode},
6 speech::Segment,
7 store::Store,
8};
9use anyhow::{Result, ensure};
10use fs2::FileExt;
11use serde::{Deserialize, Serialize};
12use std::{
13 fs,
14 io::{Read, Write},
15 path::{Path, PathBuf},
16 sync::{
17 Arc,
18 atomic::{AtomicBool, Ordering},
19 },
20};
21
22pub(crate) const DOWNLOADING: u64 = u64::MAX;
23
24const LIMIT: u64 = 10 * 1024 * 1024;
25
26#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
27pub struct Transcript {
28 pub url: String,
29 pub mime: String,
30 pub language: Option<String>,
31}
32
33#[derive(Debug, Serialize, Deserialize)]
34struct Cache {
35 version: u8,
36 media: EpisodeMedia,
37 source: String,
38 segments: Vec<Segment>,
39}
40
41fn invalid() -> anyhow::Error {
42 error(crate::t!("podcast_subtitle_invalid"))
43}
44pub(super) fn check(stop: &AtomicBool) -> Result<()> {
45 ensure!(
46 !stop.load(Ordering::Relaxed),
47 DomainError::new(ErrorCode::Cancelled)
48 );
49 Ok(())
50}
51pub(super) fn path(config: &Config, episode: &Episode) -> PathBuf {
52 use sha2::{Digest, Sha256};
54 let key = format!("{:x}", Sha256::digest(episode.id.as_bytes()));
55 config
56 .data_dir
57 .join("podcasts/transcripts")
58 .join(format!("{key}.json"))
59}
60pub(super) fn read(path: &Path) -> Result<Vec<u8>> {
61 let mut data = vec![];
62 fs::File::open(path)?
63 .take(LIMIT + 1)
64 .read_to_end(&mut data)?;
65 ensure!(data.len() as u64 <= LIMIT, invalid());
66 Ok(data)
67}
68pub(super) fn validate(segments: &mut [Segment]) -> Result<()> {
69 ensure!(segments.len() <= 100_000, invalid());
70 for s in segments.iter_mut() {
71 ensure!(s.end_ms > s.start_ms, invalid());
72 s.epoch = 0;
73 s.text = crate::clean_text(&s.text);
74 ensure!(!s.text.trim().is_empty(), invalid());
75 }
76 segments.sort_by_key(|s| (s.start_ms, s.end_ms));
77 Ok(())
78}
79fn cached(config: &Config, episode: &Episode) -> Result<Option<Vec<Segment>>> {
80 let p = path(config, episode);
81 if !p.exists() {
82 return Ok(None);
83 }
84 let mut c: Cache = serde_json::from_slice(&read(&p)?)?;
85 if c.version != 1 || c.media != episode.media() {
86 return Ok(None);
87 }
88 if c.source.starts_with("http") && !episode.transcripts.iter().any(|t| t.url == c.source) {
91 return Ok(None);
92 }
93 validate(&mut c.segments)?;
94 Ok(Some(c.segments))
95}
96pub(super) fn save(
97 config: &Config,
98 episode: &Episode,
99 source: String,
100 mut segments: Vec<Segment>,
101 stop: &AtomicBool,
102) -> Result<Vec<Segment>> {
103 check(stop)?;
104 validate(&mut segments)?;
105 let p = path(config, episode);
106 fs::create_dir_all(p.parent().unwrap())?;
107 let lock = fs::OpenOptions::new()
108 .create(true)
109 .truncate(false)
110 .read(true)
111 .write(true)
112 .open(p.with_extension("lock"))?;
113 lock.lock_exclusive()?;
114 check(stop)?;
115 if source.starts_with("http")
118 && let Ok(Some(existing)) = cached(config, episode)
119 {
120 return Ok(existing);
121 }
122 let tmp = p.with_extension(format!("{}.tmp", uuid::Uuid::new_v4()));
123 let result = (|| {
124 let data = serde_json::to_vec(&Cache {
125 version: 1,
126 media: episode.media(),
127 source,
128 segments: segments.clone(),
129 })?;
130 ensure!(data.len() as u64 <= LIMIT, invalid());
131 let mut file = fs::OpenOptions::new()
132 .write(true)
133 .create_new(true)
134 .open(&tmp)?;
135 file.write_all(&data)?;
136 file.sync_all()?;
137 check(stop)?;
138 fs::rename(&tmp, &p)?;
139 Ok(segments)
140 })();
141 let _ = fs::remove_file(tmp);
142 result
143}
144
145pub fn parse(data: &[u8]) -> Result<Vec<Segment>> {
147 let text = std::str::from_utf8(data)
148 .map_err(|_| invalid())?
149 .trim_start_matches('\u{feff}')
150 .replace("\r\n", "\n");
151 ensure!(data.len() as u64 <= LIMIT, invalid());
152 let mut segments = vec![];
153 for block in text.split("\n\n") {
154 let lines: Vec<_> = block.lines().collect();
155 if lines.first().is_none_or(|l| {
156 l.starts_with("NOTE") || l.starts_with("STYLE") || l.starts_with("REGION")
157 }) {
158 continue;
159 }
160 let Some(index) = lines.iter().position(|l| l.contains("-->")) else {
161 continue;
162 };
163 let (start, end) = lines[index].split_once("-->").ok_or_else(invalid)?;
164 let start_ms = timestamp(start.trim())?;
165 let end_ms = timestamp(end.split_whitespace().next().ok_or_else(invalid)?)?;
166 let body = lines[index + 1..].join("\n");
167 let body = scraper::Html::parse_fragment(&body)
168 .root_element()
169 .text()
170 .collect::<Vec<_>>()
171 .join("");
172 let body = body.split_whitespace().collect::<Vec<_>>().join(" ");
173 segments.push(Segment {
174 epoch: 0,
175 start_ms,
176 end_ms,
177 text: body,
178 });
179 }
180 ensure!(!segments.is_empty(), invalid());
181 validate(&mut segments)?;
182 Ok(segments)
183}
184fn timestamp(s: &str) -> Result<u64> {
185 let normalized = s.replace(',', ".");
186 let fields: Vec<_> = normalized.split(':').collect();
187 ensure!((2..=3).contains(&fields.len()), invalid());
188 let mut total = 0u64;
189 for (i, part) in fields.iter().enumerate() {
190 let value = if i + 1 == fields.len() {
191 let (whole, fraction) = part.split_once('.').ok_or_else(invalid)?;
192 ensure!(
193 fraction.len() == 3 && fraction.bytes().all(|c| c.is_ascii_digit()),
194 invalid()
195 );
196 let seconds = whole.parse::<u64>().map_err(|_| invalid())?;
197 ensure!(seconds < 60, invalid());
198 seconds * 1000 + fraction.parse::<u64>().map_err(|_| invalid())?
199 } else {
200 let n = part.parse::<u64>().map_err(|_| invalid())?;
201 ensure!(i == 0 && fields.len() == 3 || n < 60, invalid());
202 n.checked_mul(1000).ok_or_else(invalid)?
203 };
204 total = total
205 .checked_mul(60)
206 .and_then(|t| t.checked_add(value))
207 .ok_or_else(invalid)?;
208 }
209 Ok(total)
210}
211
212pub(crate) fn load(
213 config: &Config,
214 episode: &Episode,
215 stop: &Arc<AtomicBool>,
216) -> Result<Option<Vec<Segment>>> {
217 check(stop)?;
218 if let Ok(Some(segments)) = cached(config, episode) {
219 return Ok(Some(segments));
220 }
221 if let Ok(Some(segments)) = super::progressive::load_partial(config, episode) {
222 return Ok(Some(segments));
223 }
224 let mut last_error = None;
225 for t in &episode.transcripts {
226 if !matches!(
227 t.mime.as_str(),
228 "text/vtt" | "application/x-subrip" | "text/srt"
229 ) {
230 continue;
231 }
232 let result = (|| {
233 check(stop)?;
234 let url = super::feed::url(&t.url, None)?;
235 let response = super::feed::client(config)?
236 .get(url)
237 .send()
238 .map_err(super::feed::network_error)?;
239 let (_, data) = super::feed::body(response, stop)?;
240 save(config, episode, t.url.clone(), parse(&data)?, stop)
241 })();
242 match result {
243 Ok(s) => return Ok(Some(s)),
244 Err(e) => {
245 check(stop)?;
246 last_error = Some(e);
247 }
248 }
249 }
250 if let Some(e) = last_error {
251 return Err(e);
252 }
253 Ok(None)
254}
255
256fn prepare_audio(
257 config: &Config,
258 store: &Store,
259 episode: &Episode,
260 stop: &Arc<AtomicBool>,
261 progress: Option<&std::sync::atomic::AtomicU64>,
262) -> Result<super::Download> {
263 let download = store.podcast_download(&episode.id)?.filter(|d| {
264 d.state == "complete" && d.matches_media(episode) && Path::new(&d.path).is_file()
265 });
266 let download = if let Some(download) = download {
267 download
268 } else {
269 if let Some(progress) = progress {
270 progress.store(DOWNLOADING, Ordering::Relaxed);
271 } else {
272 eprintln!("{}", crate::t!("podcast_subtitle_downloading"));
273 }
274 super::download(config, &episode.id, stop)?
275 };
276 ensure!(
277 download.matches_media(episode),
278 error(crate::t!("podcast_subtitle_changed"))
279 );
280 if let Some(progress) = progress {
281 progress.store(0, Ordering::Relaxed);
282 }
283 Ok(download)
284}
285
286pub(crate) fn manage(
287 config: &Config,
288 store: &Store,
289 id: &str,
290 file: Option<&Path>,
291 transcribe: bool,
292 stop: &Arc<AtomicBool>,
293 progress: Option<&std::sync::atomic::AtomicU64>,
294) -> Result<serde_json::Value> {
295 let episode = store.podcast_episode(id)?;
296 let segments = if let Some(file) = file {
297 save(
298 config,
299 &episode,
300 "import".into(),
301 parse(&read(file)?)?,
302 stop,
303 )?
304 } else if transcribe {
305 crate::speech::check_transcription(config, stop)?;
306 let download = prepare_audio(config, store, &episode, stop, progress)?;
307 let segments =
308 crate::speech::transcribe(config, Path::new(&download.path), stop, progress)?;
309 ensure!(
311 store.podcast_episode(id)?.media() == episode.media(),
312 error(crate::t!("podcast_subtitle_changed"))
313 );
314 save(config, &episode, "local-asr".into(), segments, stop)?
315 } else {
316 load(config, &episode, stop)?.ok_or_else(|| error(crate::t!("podcast_subtitle_missing")))?
317 };
318 Ok(serde_json::json!({"id":id,"segments":segments.len()}))
319}
320
321#[cfg(test)]
322mod tests {
323 use super::*;
324 #[test]
325 fn transcription_prepares_audio_once_without_requiring_a_separate_download() {
326 use super::super::{Feed, Preview};
327 use crate::test_support::{Server, response};
328 let audio = crate::test_support::fixture("libmp3lame", "mp3");
329 let server = Server::new(move |_, s| response(s, 200, "audio/mpeg", &audio));
330 let dir = tempfile::tempdir().unwrap();
331 let config = crate::test_support::config(dir.path());
332 let store = Store::open(&config.data_dir.join("radio.db")).unwrap();
333 let p = Preview {
334 feed: Feed {
335 url: format!("{}/rss", server.url),
336 ..Default::default()
337 },
338 episodes: vec![Episode {
339 title: "Local episode".into(),
340 url: format!("{}/audio", server.url),
341 ..Default::default()
342 }],
343 };
344 let f = store.podcast_subscribe(&p, &p.feed.url).unwrap();
345 let episode = store.podcast_episodes(&f.id).unwrap().remove(0);
346 let stop = Arc::new(AtomicBool::new(false));
347 let progress = std::sync::atomic::AtomicU64::new(99);
348 let d = prepare_audio(&config, &store, &episode, &stop, Some(&progress)).unwrap();
349 assert_eq!(d.state, "complete");
350 assert!(d.matches_media(&episode));
351 assert_eq!(progress.load(Ordering::Relaxed), 0);
352 drop(server);
353 let cached = prepare_audio(&config, &store, &episode, &stop, None).unwrap();
354 assert_eq!(d.path, cached.path, "reuse current downloads offline");
355 }
356
357 #[test]
358 fn imports_are_atomic_offline_and_bound_to_media() {
359 let dir = tempfile::tempdir().unwrap();
360 let config = crate::test_support::config(dir.path());
361 let mut episode = Episode {
362 id: "../unsafe-id".into(),
363 url: "https://example.test/audio".into(),
364 ..Default::default()
365 };
366 let stop = AtomicBool::new(false);
367 let captions = parse(b"00:00.000 --> 00:01.000\nHello").unwrap();
368 save(&config, &episode, "import".into(), captions.clone(), &stop).unwrap();
369 assert_eq!(cached(&config, &episode).unwrap(), Some(captions.clone()));
370 assert!(path(&config, &episode).starts_with(config.data_dir.join("podcasts/transcripts")));
371 stop.store(true, Ordering::Relaxed);
372 assert!(save(&config, &episode, "import".into(), captions, &stop).is_err());
373 assert_eq!(cached(&config, &episode).unwrap().unwrap()[0].text, "Hello");
374 episode.bytes = Some(123);
375 assert!(cached(&config, &episode).unwrap().is_none());
376 }
377
378 #[test]
379 fn feed_subtitles_fall_back_then_cache_without_another_request() {
380 use crate::test_support::{Server, response};
381 let server = Server::new(|request, stream| {
382 if request.contains("/bad") {
383 response(stream, 200, "text/vtt", b"broken");
384 } else {
385 response(
386 stream,
387 200,
388 "text/vtt",
389 b"WEBVTT\n\n00:00.000 --> 00:02.000\nHello",
390 );
391 }
392 });
393 let dir = tempfile::tempdir().unwrap();
394 let config = crate::test_support::config(dir.path());
395 let mut episode = Episode {
396 id: "one".into(),
397 url: "https://example.test/audio".into(),
398 transcripts: ["bad", "good"]
399 .iter()
400 .map(|p| Transcript {
401 url: format!("{}/{p}", server.url),
402 mime: "text/vtt".into(),
403 language: None,
404 })
405 .collect(),
406 ..Default::default()
407 };
408 let stop = Arc::new(AtomicBool::new(false));
409 assert_eq!(
410 load(&config, &episode, &stop).unwrap().unwrap()[0].text,
411 "Hello"
412 );
413 drop(server);
414 assert_eq!(load(&config, &episode, &stop).unwrap().unwrap().len(), 1);
415 episode.transcripts.clear();
416 assert!(cached(&config, &episode).unwrap().is_none());
417 }
418
419 #[test]
420 fn parses_vtt_and_srt_with_identifiers_settings_markup_and_unicode() {
421 let s = parse(b"\xef\xbb\xbfWEBVTT\r\n\r\ncue\r\n00:01.000 --> 00:03.500 align:start\r\n<v Alice>Hello & welcome</v>\r\nagain\r\n\r\nNOTE ignored\r\n00:00.000 --> 00:01.000\r\nignore").unwrap();
422 assert_eq!((s[0].start_ms, s[0].end_ms), (1000, 3500));
423 assert_eq!(s[0].text, "Hello & welcome again");
424 let s = parse("1\n01:02:03,004 --> 01:02:04,005\n中文字幕".as_bytes()).unwrap();
425 assert_eq!(s[0].start_ms, 3723004);
426 assert_eq!(s[0].text, "中文字幕");
427 for bad in [
428 "text only",
429 "00:61.000 --> 00:62.000\nx",
430 "00:02.000 --> 00:01.000\nx",
431 "00:01.000 --> 00:02.000\n",
432 ] {
433 assert!(parse(bad.as_bytes()).is_err(), "{bad}");
434 }
435 }
436}