1use super::models;
2use crate::{
3 config::Config,
4 error::{DomainError, ErrorCode},
5 process::Background,
6};
7use anyhow::{Result, bail, ensure};
8use serde::Deserialize;
9use std::{
10 borrow::Cow,
11 collections::VecDeque,
12 io::{BufRead, BufReader, Read},
13 process::{Child, Command, Stdio},
14 sync::{
15 Arc, Mutex,
16 atomic::{AtomicBool, AtomicU64, Ordering},
17 mpsc::{self, Receiver, SyncSender},
18 },
19 thread,
20 time::{Duration, Instant},
21};
22
23const STABLE_SUBTITLE_PERIOD: Duration = Duration::from_secs(60);
24
25#[derive(Clone, Debug, PartialEq, Eq)]
26pub struct Segment {
27 pub epoch: u64,
28 pub start_ms: u64,
29 pub end_ms: u64,
30 pub text: String,
31}
32
33#[derive(Clone, Debug, PartialEq, Eq)]
34pub struct Snapshot {
35 pub enabled: bool,
36 pub failed: bool,
37 pub revision: u64,
38 pub message: String,
39 pub history: Arc<VecDeque<Segment>>,
40}
41
42impl Default for Snapshot {
43 fn default() -> Self {
44 Self {
45 enabled: false,
46 failed: false,
47 revision: 0,
48 message: crate::t!("subtitle_off").into(),
49 history: Arc::default(),
50 }
51 }
52}
53
54enum Notice {
55 Message(String),
56 Segment(Segment),
57}
58
59struct Task {
60 job: Background<bool>,
61 events: Receiver<Notice>,
62 first_segment_at: Option<Instant>,
63}
64
65#[derive(Clone, Copy, PartialEq, Eq)]
66enum StatusMessage {
67 Off,
68 Local,
69 Paused,
70 Waiting,
71 Loading,
72}
73
74impl StatusMessage {
75 fn text(self, locale: crate::messages::Locale) -> &'static str {
76 match self {
77 Self::Off => crate::messages::text_in(locale, "subtitle_off"),
78 Self::Local => crate::messages::text_in(locale, "subtitle_local"),
79 Self::Paused => crate::messages::text_in(locale, "subtitle_paused"),
80 Self::Waiting => crate::messages::text_in(locale, "subtitle_waiting"),
81 Self::Loading => crate::messages::text_in(locale, "subtitle_loading"),
82 }
83 }
84}
85
86#[derive(Default)]
87pub struct Controller {
88 pub snapshot: Snapshot,
89 task: Option<Task>,
90 retiring: Vec<Background<bool>>,
91 key: Option<(u64, String)>,
92 epoch: u64,
93 attempts: usize,
94 due: Option<Instant>,
95 status_key: Option<(StatusMessage, crate::messages::Locale)>,
96 verification: Option<Arc<Mutex<models::VerificationCache>>>,
97}
98
99impl Controller {
100 fn message(&mut self, message: impl Into<Cow<'static, str>>) {
101 self.status_key = None;
102 let message = message.into();
103 if self.snapshot.message != message {
104 self.snapshot.message = message.into_owned();
105 self.snapshot.revision += 1;
106 }
107 }
108 fn status(&mut self, key: StatusMessage) {
109 let locale = crate::messages::locale();
110 if self.status_key != Some((key, locale)) {
111 self.message(key.text(locale));
112 self.status_key = Some((key, locale));
113 }
114 }
115 pub fn clear(&mut self) {
116 if !self.snapshot.history.is_empty() {
117 self.snapshot.history = Arc::default();
118 self.snapshot.revision += 1;
119 }
120 }
121 pub fn retry(&mut self) {
122 self.cancel();
123 if self.snapshot.failed {
124 self.snapshot.failed = false;
125 self.snapshot.revision += 1;
126 }
127 self.attempts = 0;
128 self.due = None;
129 }
130 pub fn cancel(&mut self) {
131 self.epoch = self.epoch.wrapping_add(1);
132 if let Some(task) = self.task.take() {
133 self.retiring.push(task.job);
134 }
135 for job in &self.retiring {
138 job.cancel();
139 }
140 self.due = None;
141 }
142 pub fn quiescent(&self) -> bool {
143 self.task.is_none() && self.retiring.is_empty()
144 }
145 pub fn poll_retiring(&mut self) {
146 self.retiring.retain(|task| task.poll().is_none());
147 }
148 pub fn active(&self) -> bool {
149 self.task.is_some()
150 }
151 pub fn enabled(&mut self, enabled: bool) {
152 if self.snapshot.enabled != enabled {
153 self.snapshot.enabled = enabled;
154 self.snapshot.revision += 1;
155 self.retry();
156 }
157 }
158 pub fn update(
159 &mut self,
160 config: &Config,
161 key: (u64, &str),
162 playing: bool,
163 paused: bool,
164 local: bool,
165 stop: &Arc<AtomicBool>,
166 ) {
167 self.poll_retiring();
168 if !self.snapshot.enabled {
169 if playing && self.key.as_ref().is_some_and(|old| old.0 != key.0) {
170 self.clear();
171 self.key = None;
172 }
173 self.status(StatusMessage::Off);
174 return;
175 }
176 if playing
177 && self
178 .key
179 .as_ref()
180 .map(|(generation, url)| (*generation, url.as_str()))
181 != Some(key)
182 {
183 if self.key.as_ref().is_some_and(|old| old.0 != key.0) {
186 self.clear();
187 }
188 self.retry();
189 self.key = Some((key.0, key.1.to_owned()));
190 }
191 if !playing || paused || local || stop.load(Ordering::Relaxed) {
192 self.cancel();
193 self.status(if local {
194 StatusMessage::Local
195 } else if paused {
196 StatusMessage::Paused
197 } else {
198 StatusMessage::Waiting
199 });
200 return;
201 }
202 let finished = self.task.as_ref().and_then(|t| t.job.poll());
205 if let Some(task) = &self.task {
206 let notices: Vec<_> = task.events.try_iter().collect();
207 for notice in notices {
208 match notice {
209 Notice::Message(message) => self.message(message),
210 Notice::Segment(seg) if seg.epoch == self.epoch => {
211 let first = self
212 .task
213 .as_mut()
214 .unwrap()
215 .first_segment_at
216 .get_or_insert_with(Instant::now);
217 if first.elapsed() >= STABLE_SUBTITLE_PERIOD {
218 self.attempts = 0;
219 }
220 let history = Arc::make_mut(&mut self.snapshot.history);
221 history.push_back(seg);
222 while history.len() > 500 {
223 history.pop_front();
224 }
225 self.snapshot.revision += 1;
226 }
227 _ => {}
228 }
229 }
230 }
231 if let Some(result) = finished {
232 self.task = None;
233 let retry = match result {
234 Ok(retry) => retry,
235 Err(error) => {
236 self.message(crate::t!("subtitle_failed", "detail" => crate::redact::urls(&format!("{error:#}"))));
237 false
238 }
239 };
240 if retry && self.attempts < 3 {
241 let seconds = [1, 3, 7][self.attempts];
242 self.attempts += 1;
243 self.due = Some(Instant::now() + Duration::from_secs(seconds));
244 self.message(crate::t!("subtitle_retrying", "attempt" => self.attempts));
245 } else {
246 self.snapshot.failed = true;
247 self.snapshot.revision += 1;
248 if retry {
249 self.message(
250 crate::t!("subtitle_failed", "detail" => crate::t!("subtitle_connection")),
251 );
252 }
253 }
254 }
255 if self.task.is_none()
256 && self.retiring.is_empty()
257 && !self.snapshot.failed
258 && self.due.is_none_or(|d| Instant::now() >= d)
259 {
260 self.due = None;
261 self.epoch = self.epoch.wrapping_add(1);
262 self.status(StatusMessage::Loading);
263 let (sender, events) = mpsc::sync_channel(64);
264 let config = config.clone();
265 let url = self.key.as_ref().unwrap().1.clone();
266 let epoch = self.epoch;
267 let verification = self
268 .verification
269 .get_or_insert_with(Default::default)
270 .clone();
271 match Background::start("radio-speech", move |cancel| {
272 run(&config, &url, epoch, &cancel, sender, &verification)
273 }) {
274 Ok(job) => {
275 self.task = Some(Task {
276 job,
277 events,
278 first_segment_at: None,
279 })
280 }
281 Err(error) => {
282 self.snapshot.failed = true;
283 self.snapshot.revision += 1;
284 self.message(format!("{error:#}"));
285 }
286 }
287 }
288 }
289}
290
291struct OwnedChildren {
292 worker: Child,
293 ffmpeg: Option<Child>,
294 _lock: models::ResourceLock,
295}
296impl Drop for OwnedChildren {
297 fn drop(&mut self) {
298 if let Some(child) = &mut self.ffmpeg {
299 let _ = child.kill();
300 let _ = child.wait();
301 }
302 let _ = self.worker.kill();
303 let _ = self.worker.wait();
304 }
305}
306
307#[derive(Deserialize)]
308struct Event {
309 v: u8,
310 epoch: u64,
311 #[serde(rename = "type")]
312 kind: String,
313 #[serde(default)]
314 start_ms: u64,
315 #[serde(default)]
316 end_ms: u64,
317 #[serde(default)]
318 lag_ms: u64,
319 #[serde(default)]
320 text: String,
321}
322
323fn send(sender: &SyncSender<Notice>, notice: Notice) {
324 let _ = sender.try_send(notice);
325}
326
327fn run(
328 config: &Config,
329 url: &str,
330 epoch: u64,
331 cancel: &Arc<AtomicBool>,
332 sender: SyncSender<Notice>,
333 verification: &Mutex<models::VerificationCache>,
334) -> Result<bool> {
335 let lock = models::lock(config, false)?;
336 models::runtime_check(config, cancel)?;
337 verification
338 .lock()
339 .unwrap()
340 .verify(config, &config.settings.subtitles.model, cancel)?;
341 ensure!(
342 !cancel.load(Ordering::Relaxed),
343 DomainError::new(ErrorCode::Cancelled)
344 );
345 let settings = &config.settings.subtitles;
346 let args = serde_json::json!({"epoch":epoch,"model":settings.model,"model_dir":models::model_dir(config, &settings.model),"vad":models::model_dir(config,"silero-vad").join("silero_vad.onnx"),"language":settings.language,"threads":settings.effective_threads(),"max_segment_secs":settings.max_segment_secs});
347 let mut command = Command::new(models::python(config));
348 command
349 .args([
350 "-I",
351 "-u",
352 "-c",
353 include_str!("worker.py"),
354 &args.to_string(),
355 ])
356 .stdin(Stdio::piped())
357 .stdout(Stdio::piped())
358 .stderr(Stdio::piped());
359 run_processes(config, url, epoch, cancel, sender, command, lock)
360}
361
362fn run_processes(
363 config: &Config,
364 url: &str,
365 epoch: u64,
366 cancel: &Arc<AtomicBool>,
367 sender: SyncSender<Notice>,
368 command: Command,
369 lock: models::ResourceLock,
370) -> Result<bool> {
371 let worker = crate::process::spawn_owned(command)?;
372 let mut children = OwnedChildren {
373 worker,
374 ffmpeg: None,
375 _lock: lock,
376 };
377 let stderr = crate::process::drain(children.worker.stderr.take().unwrap(), 16384);
378 let stdout = children.worker.stdout.take().unwrap();
379 let (events, receiver) = mpsc::sync_channel(64);
380 let reader = thread::spawn(move || {
381 let mut reader = BufReader::new(stdout);
382 loop {
383 let mut line = Vec::new();
384 let n = reader.by_ref().take(65537).read_until(b'\n', &mut line);
385 match n {
386 Ok(0) => break,
387 Ok(n) if n <= 65536 => {
388 let parsed = serde_json::from_slice::<Event>(&line).map_err(|e| e.to_string());
389 if events.send(parsed).is_err() {
390 break;
391 }
392 }
393 _ => {
394 let _ = events.send(Err("invalid speech protocol line".into()));
395 break;
396 }
397 }
398 }
399 });
400 let started = Instant::now();
401 let mut ready = false;
402 let mut decoding = None::<Instant>;
403 let mut pump = None;
404 let mut ffmpeg_stderr = None;
405 let bytes = Arc::new(AtomicU64::new(0));
406 let mut last_bytes = 0;
407 let mut last_audio = Instant::now();
408 let result = (|| -> Result<bool> {
409 loop {
410 if cancel.load(Ordering::Relaxed) {
411 return Ok(false);
412 }
413 ensure!(
414 ready || started.elapsed() < Duration::from_secs(90),
415 "speech model loading timed out"
416 );
417 ensure!(
418 decoding.is_none_or(|t| t.elapsed() < Duration::from_secs(60)),
419 "speech inference timed out"
420 );
421 let incoming = match receiver.recv_timeout(Duration::from_millis(25)) {
422 Ok(event) => Some(event),
423 Err(mpsc::RecvTimeoutError::Timeout) => None,
424 Err(mpsc::RecvTimeoutError::Disconnected) => {
425 bail!("speech worker closed its output; run radio subtitles doctor")
426 }
427 };
428 if let Some(event) = incoming {
429 let event = event.map_err(anyhow::Error::msg)?;
430 ensure!(
431 event.v == 1 && event.epoch == epoch,
432 "speech protocol version or epoch mismatch"
433 );
434 match event.kind.as_str() {
435 "ready" if !ready => {
436 ready = true;
437 send(
438 &sender,
439 Notice::Message(crate::t!("subtitle_listening").into()),
440 );
441 let mut command = Command::new(&config.settings.ffmpeg);
442 command
443 .args([
444 "-hide_banner",
445 "-nostdin",
446 "-loglevel",
447 "error",
448 "-rw_timeout",
449 &(config.settings.request_timeout_secs * 1_000_000).to_string(),
450 "-user_agent",
451 crate::USER_AGENT,
452 "-readrate",
453 "1",
454 "-i",
455 url,
456 "-map",
457 "0:a:0",
458 "-vn",
459 "-ac",
460 "1",
461 "-ar",
462 "16000",
463 "-f",
464 "f32le",
465 "pipe:1",
466 ])
467 .stdin(Stdio::null())
468 .stdout(Stdio::piped())
469 .stderr(Stdio::piped());
470 let child = crate::process::spawn_owned(command)?;
471 children.ffmpeg = Some(child);
472 let ffmpeg = children.ffmpeg.as_mut().unwrap();
473 ffmpeg_stderr =
474 Some(crate::process::drain(ffmpeg.stderr.take().unwrap(), 16384));
475 let mut input = ffmpeg.stdout.take().unwrap();
476 let mut output = children.worker.stdin.take().unwrap();
477 let bytes = bytes.clone();
478 pump = Some(thread::spawn(move || {
479 use std::io::Write;
480 let mut buf = [0; 8192];
481 while let Ok(n) = input.read(&mut buf) {
482 if n == 0 || output.write_all(&buf[..n]).is_err() {
483 break;
484 }
485 bytes.fetch_add(n as u64, Ordering::Relaxed);
486 }
487 }));
488 last_audio = Instant::now();
489 }
490 "decoding" => {
491 decoding = Some(Instant::now());
492 send(
493 &sender,
494 Notice::Message(crate::t!("subtitle_decoding").into()),
495 );
496 }
497 "listening" => {
498 decoding = None;
499 send(
500 &sender,
501 Notice::Message(crate::t!("subtitle_listening").into()),
502 );
503 }
504 "segment" => {
505 ensure!(
506 event.end_ms >= event.start_ms
507 && event.end_ms - event.start_ms <= 20000,
508 "invalid speech segment time range"
509 );
510 if event.lag_ms <= 30000 {
511 let text = crate::clean_text(&event.text);
512 if !text.is_empty() {
513 send(
514 &sender,
515 Notice::Segment(Segment {
516 epoch,
517 start_ms: event.start_ms,
518 end_ms: event.end_ms,
519 text,
520 }),
521 );
522 }
523 }
524 }
525 "gap" => send(&sender, Notice::Message(crate::t!("subtitle_gap").into())),
526 "done" => return Ok(true),
527 _ => bail!("unexpected speech protocol event"),
528 }
529 }
530 let now_bytes = bytes.load(Ordering::Relaxed);
531 if now_bytes != last_bytes {
532 last_bytes = now_bytes;
533 last_audio = Instant::now();
534 }
535 if ready
536 && last_audio.elapsed() >= Duration::from_secs(config.settings.request_timeout_secs)
537 {
538 return Ok(true);
539 }
540 }
541 })();
542 drop(children); drop(receiver);
544 let _ = reader.join();
545 if let Some(pump) = pump {
546 let _ = pump.join();
547 }
548 let diagnostics = stderr.join().unwrap_or_default();
549 if let Some(stderr) = ffmpeg_stderr {
550 let _ = stderr.join();
551 }
552 if !diagnostics.is_empty() {
553 return result.map_err(|e| {
554 e.context(crate::clean_text(&crate::redact::urls(
555 &String::from_utf8_lossy(&diagnostics),
556 )))
557 });
558 }
559 result
560}
561
562#[cfg(test)]
563mod tests {
564 use super::*;
565 use crate::test_support::{Server, response};
566
567 #[test]
568 #[ignore = "requires explicitly installed models and RADIO_TEST_SPEECH_DATA/WAV"]
569 fn real_models_transcribe_a_local_http_fixture() {
570 let data =
571 std::env::var_os("RADIO_TEST_SPEECH_DATA").expect("isolated speech data directory");
572 let wav = std::fs::read(
573 std::env::var_os("RADIO_TEST_SPEECH_WAV").expect("local speech WAV fixture"),
574 )
575 .unwrap();
576 let server = Server::new(move |_, stream| response(stream, 200, "audio/wav", &wav));
577 let dir = tempfile::tempdir().unwrap();
578 let mut config = crate::test_support::config(dir.path());
579 config.data_dir = data.into();
580 for model in ["sensevoice-small-yue-int8", "qwen3-asr-0.6b-int8"] {
581 config.settings.subtitles.model = model.into();
582 let (sender, events) = mpsc::sync_channel(64);
583 let start = Instant::now();
584 assert!(
585 run(
586 &config,
587 &server.url,
588 1,
589 &Arc::new(AtomicBool::new(false)),
590 sender,
591 &Mutex::default()
592 )
593 .unwrap()
594 );
595 let mut count = 0;
596 for notice in events.try_iter() {
597 if let Notice::Segment(segment) = notice {
598 eprintln!(
599 "{model} {}-{} ms: {}",
600 segment.start_ms, segment.end_ms, segment.text
601 );
602 count += 1;
603 }
604 }
605 assert!(count > 0, "{model} produced no transcription");
606 eprintln!(
607 "{model}: {count} segments, {:.2}s including realtime audio",
608 start.elapsed().as_secs_f32()
609 );
610 }
611 }
612
613 fn fake_command(script: &str) -> Command {
614 let mut command = Command::new("sh");
615 command
616 .args(["-c", script])
617 .stdin(Stdio::piped())
618 .stdout(Stdio::piped())
619 .stderr(Stdio::piped());
620 command
621 }
622
623 #[test]
624 fn pcm_reaches_worker_and_protocol_segments_are_sanitized() {
625 let audio = crate::test_support::fixture("pcm_s16le", "wav");
626 let server = Server::new(move |_, stream| response(stream, 200, "audio/wav", &audio));
627 let dir = tempfile::tempdir().unwrap();
628 let config = crate::test_support::config(dir.path());
629 let (sender, events) = mpsc::sync_channel(64);
630 let command = fake_command(
631 "printf '%s\n' '{\"v\":1,\"epoch\":7,\"type\":\"ready\"}'; head -c 4096 >/dev/null; printf '%s\n' '{\"v\":1,\"epoch\":7,\"type\":\"segment\",\"start_ms\":0,\"end_ms\":64,\"text\":\"hello\\u001b world\"}' '{\"v\":1,\"epoch\":7,\"type\":\"done\"}'; cat >/dev/null",
632 );
633 assert!(
634 run_processes(
635 &config,
636 &server.url,
637 7,
638 &Arc::new(AtomicBool::new(false)),
639 sender,
640 command,
641 models::lock(&config, false).unwrap()
642 )
643 .unwrap()
644 );
645 let segments: Vec<_> = events
646 .try_iter()
647 .filter_map(|n| {
648 if let Notice::Segment(s) = n {
649 Some(s)
650 } else {
651 None
652 }
653 })
654 .collect();
655 assert_eq!(segments.len(), 1);
656 assert_eq!(segments[0].text, "hello world");
657 assert!(models::lock(&config, true).is_ok());
658 }
659
660 #[test]
661 fn hls_and_icy_sources_feed_the_same_pcm_worker_protocol() {
662 use std::io::Write;
663 let source = tempfile::tempdir().unwrap();
664 assert!(
665 Command::new("ffmpeg")
666 .args([
667 "-v",
668 "error",
669 "-f",
670 "lavfi",
671 "-i",
672 "sine=frequency=440:duration=4",
673 "-c:a",
674 "aac",
675 "-f",
676 "hls",
677 "-hls_time",
678 "1",
679 "-hls_list_size",
680 "0"
681 ])
682 .arg(source.path().join("index.m3u8"))
683 .status()
684 .unwrap()
685 .success()
686 );
687 let root = source.path().to_path_buf();
688 let hls = Server::new(move |request, stream| {
689 let path = request
690 .split_whitespace()
691 .nth(1)
692 .unwrap()
693 .trim_start_matches('/');
694 if path.contains("..") {
695 response(stream, 404, "text/plain", b"");
696 return;
697 }
698 let data = std::fs::read(root.join(path)).unwrap();
699 response(
700 stream,
701 200,
702 if path.ends_with("m3u8") {
703 "application/vnd.apple.mpegurl"
704 } else {
705 "video/mp2t"
706 },
707 &data,
708 );
709 });
710 let mp3 = crate::test_support::fixture("libmp3lame", "mp3");
711 let icy = Server::new(move |request, stream| {
712 assert!(request.to_ascii_lowercase().contains("icy-metadata: 1"));
713 if stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Type: audio/mpeg\r\nicy-metaint: 1024\r\nConnection: close\r\n\r\n").is_err() { return; }
714 for chunk in mp3.chunks(1024) {
715 if stream.write_all(chunk).is_err() {
716 break;
717 }
718 if chunk.len() == 1024 && stream.write_all(&[0]).is_err() {
719 break;
720 }
721 }
722 });
723 for url in [format!("{}/index.m3u8", hls.url), icy.url.clone()] {
724 let dir = tempfile::tempdir().unwrap();
725 let config = crate::test_support::config(dir.path());
726 let (sender, events) = mpsc::sync_channel(64);
727 let command = fake_command(
728 "printf '%s\n' '{\"v\":1,\"epoch\":7,\"type\":\"ready\"}'; head -c 4096 >/dev/null; printf '%s\n' '{\"v\":1,\"epoch\":7,\"type\":\"segment\",\"start_ms\":0,\"end_ms\":64,\"text\":\"audio received\"}' '{\"v\":1,\"epoch\":7,\"type\":\"done\"}'; cat >/dev/null",
729 );
730 run_processes(
731 &config,
732 &url,
733 7,
734 &Arc::new(AtomicBool::new(false)),
735 sender,
736 command,
737 models::lock(&config, false).unwrap(),
738 )
739 .unwrap();
740 assert!(
741 events.try_iter().any(|n| matches!(n, Notice::Segment(_))),
742 "{url}"
743 );
744 }
745 }
746
747 #[test]
748 fn wrong_epoch_and_oversized_protocol_are_rejected_and_reaped() {
749 for script in [
750 "printf '%s\n' '{\"v\":1,\"epoch\":8,\"type\":\"ready\"}'",
751 "head -c 70000 /dev/zero | tr '\\000' x",
752 ] {
753 let dir = tempfile::tempdir().unwrap();
754 let config = crate::test_support::config(dir.path());
755 let (sender, _) = mpsc::sync_channel(64);
756 assert!(
757 run_processes(
758 &config,
759 "http://127.0.0.1:1",
760 7,
761 &Arc::new(AtomicBool::new(false)),
762 sender,
763 fake_command(script),
764 models::lock(&config, false).unwrap()
765 )
766 .is_err()
767 );
768 assert!(models::lock(&config, true).is_ok());
769 }
770 }
771
772 #[test]
773 fn cancellation_reaps_worker_while_it_waits_for_audio() {
774 let dir = tempfile::tempdir().unwrap();
775 let config = crate::test_support::config(dir.path());
776 let cancel = Arc::new(AtomicBool::new(false));
777 let flag = cancel.clone();
778 let (sender, events) = mpsc::sync_channel(64);
779 let (done_tx, done) = mpsc::sync_channel(1);
780 let command = fake_command(
781 "printf '%s\n' '{\"v\":1,\"epoch\":7,\"type\":\"ready\"}'; exec cat >/dev/null",
782 );
783 thread::spawn(move || {
784 let lock = models::lock(&config, false).unwrap();
785 done_tx
786 .send(run_processes(
787 &config,
788 "http://127.0.0.1:1",
789 7,
790 &flag,
791 sender,
792 command,
793 lock,
794 ))
795 .unwrap();
796 });
797 events.recv_timeout(Duration::from_secs(5)).unwrap();
798 cancel.store(true, Ordering::Relaxed);
799 assert!(!done.recv_timeout(Duration::from_secs(2)).unwrap().unwrap());
800 }
801
802 #[test]
803 fn cancellation_kills_and_reaps_a_busy_decoder_without_waiting_for_timeout() {
804 let dir = tempfile::tempdir().unwrap();
805 let config = crate::test_support::config(dir.path());
806 let cancel = Arc::new(AtomicBool::new(false));
807 let flag = cancel.clone();
808 let (sender, events) = mpsc::sync_channel(64);
809 let (done_tx, done) = mpsc::sync_channel(1);
810 let pid_path = dir.path().join("decoder.pid");
811 let mut command = fake_command(
814 "printf '%s' $$ > \"$1\"; trap '' TERM; printf '%s\n' '{\"v\":1,\"epoch\":7,\"type\":\"decoding\"}'; while :; do :; done",
815 );
816 command.arg("decoder").arg(&pid_path);
817 let handle = thread::spawn(move || {
818 let lock = models::lock(&config, false).unwrap();
819 done_tx
820 .send(run_processes(
821 &config,
822 "http://127.0.0.1:1",
823 7,
824 &flag,
825 sender,
826 command,
827 lock,
828 ))
829 .unwrap();
830 });
831 assert!(
833 matches!(events.recv_timeout(Duration::from_secs(5)).unwrap(), Notice::Message(message) if message == crate::t!("subtitle_decoding"))
834 );
835 let pid: libc::pid_t = std::fs::read_to_string(pid_path).unwrap().parse().unwrap();
836 let started = Instant::now();
837 cancel.store(true, Ordering::Relaxed);
838 assert!(!done.recv_timeout(Duration::from_secs(2)).unwrap().unwrap());
839 assert!(started.elapsed() < Duration::from_secs(2));
840 handle.join().unwrap();
841 assert_eq!(unsafe { libc::kill(pid, 0) }, -1);
842 assert_eq!(
843 std::io::Error::last_os_error().raw_os_error(),
844 Some(libc::ESRCH)
845 );
846 }
847
848 #[test]
849 fn pause_and_same_url_restart_discard_old_epoch_results() {
850 let dir = tempfile::tempdir().unwrap();
851 let config = crate::test_support::config(dir.path());
852 let stop = Arc::new(AtomicBool::new(false));
853 let mut controller = Controller::default();
854 controller.enabled(true);
855 controller.key = Some((1, "http://example.com/live".into()));
856 let (sender, events) = mpsc::sync_channel(64);
857 let job = Background::start("fake-speech", |cancel| {
858 while !cancel.load(Ordering::Relaxed) {
859 thread::sleep(Duration::from_millis(5));
860 }
861 Ok(false)
862 })
863 .unwrap();
864 controller.task = Some(Task {
865 job,
866 events,
867 first_segment_at: None,
868 });
869 sender
870 .send(Notice::Segment(Segment {
871 epoch: controller.epoch,
872 start_ms: 0,
873 end_ms: 500,
874 text: "current".into(),
875 }))
876 .ok();
877 sender
878 .send(Notice::Segment(Segment {
879 epoch: controller.epoch.wrapping_sub(1),
880 start_ms: 0,
881 end_ms: 500,
882 text: "stale".into(),
883 }))
884 .ok();
885 let key = controller.key.clone().unwrap();
886 controller.update(&config, (key.0, &key.1), true, false, false, &stop);
887 assert_eq!(controller.snapshot.history.len(), 1);
888 let epoch = controller.epoch;
889 controller.update(&config, (key.0, &key.1), false, true, false, &stop);
890 assert!(controller.epoch != epoch && controller.task.is_none());
891 assert_eq!(controller.snapshot.history.len(), 1);
892 controller.update(&config, (2, &key.1), true, false, false, &stop);
893 assert!(controller.snapshot.history.is_empty());
894 controller.enabled(false);
895 }
896
897 #[test]
898 fn short_connections_exhaust_retries_even_when_they_produce_segments() {
899 let dir = tempfile::tempdir().unwrap();
900 let config = crate::test_support::config(dir.path());
901 let stop = Arc::new(AtomicBool::new(false));
902 for productive in [false, true] {
903 let mut controller = Controller::default();
904 controller.enabled(true);
905 controller.key = Some((1, "http://localhost/live".into()));
906 for attempt in 0..6 {
907 let (sender, events) = mpsc::sync_channel(64);
908 if productive {
909 sender
910 .send(Notice::Segment(Segment {
911 epoch: controller.epoch,
912 start_ms: 0,
913 end_ms: 500,
914 text: "speech".into(),
915 }))
916 .unwrap();
917 }
918 let job = Background::start("fake-speech", |_| Ok(true)).unwrap();
919 controller.task = Some(Task {
920 job,
921 events,
922 first_segment_at: None,
923 });
924 controller.due = None;
925 let deadline = Instant::now() + Duration::from_secs(2);
926 while controller.task.is_some() {
927 controller.update(
928 &config,
929 (1, "http://localhost/live"),
930 true,
931 false,
932 false,
933 &stop,
934 );
935 assert!(Instant::now() < deadline);
936 thread::yield_now();
937 }
938 if attempt == 3 {
939 assert!(controller.snapshot.failed);
940 assert!(controller.due.is_none());
941 break;
942 } else {
943 assert_eq!(controller.attempts, attempt + 1);
944 assert!(!controller.snapshot.failed);
945 }
946 }
947 }
948 }
949
950 #[test]
951 fn only_sustained_speech_on_the_same_connection_resets_retries() {
952 let dir = tempfile::tempdir().unwrap();
953 let config = crate::test_support::config(dir.path());
954 let stop = Arc::new(AtomicBool::new(false));
955 for elapsed in [
958 None,
959 Some(Duration::from_secs(30)),
960 Some(STABLE_SUBTITLE_PERIOD),
961 ] {
962 let mut controller = Controller::default();
963 controller.enabled(true);
964 controller.key = Some((1, "http://localhost/live".into()));
965 controller.attempts = 3;
966 let (sender, events) = mpsc::sync_channel(64);
967 sender
968 .send(Notice::Segment(Segment {
969 epoch: controller.epoch,
970 start_ms: 0,
971 end_ms: 500,
972 text: "speech".into(),
973 }))
974 .unwrap();
975 let job = Background::start("fake-speech", |_| Ok(true)).unwrap();
976 controller.task = Some(Task {
977 job,
978 events,
979 first_segment_at: elapsed.map(|elapsed| Instant::now() - elapsed),
980 });
981 let deadline = Instant::now() + Duration::from_secs(2);
982 while controller.task.is_some() {
983 controller.update(
984 &config,
985 (1, "http://localhost/live"),
986 true,
987 false,
988 false,
989 &stop,
990 );
991 assert!(Instant::now() < deadline);
992 thread::yield_now();
993 }
994 if elapsed == Some(STABLE_SUBTITLE_PERIOD) {
995 assert_eq!(controller.attempts, 1);
996 assert!(!controller.snapshot.failed);
997 assert!(controller.due.is_some());
998 } else {
999 assert_eq!(controller.attempts, 3);
1000 assert!(controller.snapshot.failed);
1001 assert!(controller.due.is_none());
1002 }
1003 }
1004 }
1005
1006 #[test]
1007 fn history_is_bounded_and_empty_clear_preserves_revision() {
1008 let dir = tempfile::tempdir().unwrap();
1009 let config = crate::test_support::config(dir.path());
1010 let stop = Arc::new(AtomicBool::new(false));
1011 let mut controller = Controller::default();
1012 controller.clear();
1013 assert_eq!(controller.snapshot.revision, 0);
1014 controller.enabled(true);
1015 controller.key = Some((1, "http://localhost/live".into()));
1016 let (sender, events) = mpsc::sync_channel(501);
1017 for i in 0..501 {
1018 sender
1019 .send(Notice::Segment(Segment {
1020 epoch: controller.epoch,
1021 start_ms: i,
1022 end_ms: i + 1,
1023 text: i.to_string(),
1024 }))
1025 .unwrap();
1026 }
1027 let job = Background::start("fake-speech", |_| Ok(true)).unwrap();
1028 controller.task = Some(Task {
1029 job,
1030 events,
1031 first_segment_at: None,
1032 });
1033 controller.update(
1034 &config,
1035 (1, "http://localhost/live"),
1036 true,
1037 false,
1038 false,
1039 &stop,
1040 );
1041 assert_eq!(controller.snapshot.history.len(), 500);
1042 assert_eq!(controller.snapshot.history.front().unwrap().start_ms, 1);
1043 controller.enabled(false);
1044 controller.update(
1045 &config,
1046 (2, "http://localhost/other"),
1047 true,
1048 false,
1049 false,
1050 &stop,
1051 );
1052 assert!(controller.snapshot.history.is_empty());
1053 let revision = controller.snapshot.revision;
1054 controller.clear();
1055 assert_eq!(controller.snapshot.revision, revision);
1056 }
1057}