1use super::models;
2use crate::{
3 config::Config,
4 error::{DomainError, ErrorCode},
5 process::Background,
6};
7use anyhow::{Result, bail, ensure};
8use serde::Deserialize;
9use std::{
10 borrow::Cow,
11 collections::VecDeque,
12 io::{BufRead, BufReader, Read},
13 process::{Child, Command, Stdio},
14 sync::{
15 Arc, Mutex,
16 atomic::{AtomicBool, AtomicU64, Ordering},
17 mpsc::{self, Receiver, SyncSender},
18 },
19 thread,
20 time::{Duration, Instant},
21};
22
23const STABLE_SUBTITLE_PERIOD: Duration = Duration::from_secs(60);
24
25#[derive(Clone, Debug, PartialEq, Eq)]
26pub struct Segment {
27 pub epoch: u64,
28 pub start_ms: u64,
29 pub end_ms: u64,
30 pub text: String,
31}
32
33#[derive(Clone, Debug, PartialEq, Eq)]
34pub struct Snapshot {
35 pub enabled: bool,
36 pub failed: bool,
37 pub revision: u64,
38 pub message: String,
39 pub history: Arc<VecDeque<Segment>>,
40}
41
42impl Default for Snapshot {
43 fn default() -> Self {
44 Self {
45 enabled: false,
46 failed: false,
47 revision: 0,
48 message: crate::t!("subtitle_off").into(),
49 history: Arc::default(),
50 }
51 }
52}
53
54enum Notice {
55 Message(String),
56 Segment(Segment),
57}
58
59struct Task {
60 job: Background<bool>,
61 events: Receiver<Notice>,
62 first_segment_at: Option<Instant>,
63}
64
65#[derive(Clone, Copy, PartialEq, Eq)]
66enum StatusMessage {
67 Off,
68 Local,
69 Paused,
70 Waiting,
71 Loading,
72}
73
74impl StatusMessage {
75 fn text(self, locale: crate::messages::Locale) -> &'static str {
76 match self {
77 Self::Off => crate::messages::text_in(locale, "subtitle_off"),
78 Self::Local => crate::messages::text_in(locale, "subtitle_local"),
79 Self::Paused => crate::messages::text_in(locale, "subtitle_paused"),
80 Self::Waiting => crate::messages::text_in(locale, "subtitle_waiting"),
81 Self::Loading => crate::messages::text_in(locale, "subtitle_loading"),
82 }
83 }
84}
85
86#[derive(Default)]
87pub struct Controller {
88 pub snapshot: Snapshot,
89 task: Option<Task>,
90 retiring: Vec<Background<bool>>,
91 key: Option<(u64, String)>,
92 epoch: u64,
93 attempts: usize,
94 due: Option<Instant>,
95 status_key: Option<(StatusMessage, crate::messages::Locale)>,
96 verification: Option<Arc<Mutex<models::VerificationCache>>>,
97}
98
99impl Controller {
100 fn message(&mut self, message: impl Into<Cow<'static, str>>) {
101 self.status_key = None;
102 let message = message.into();
103 if self.snapshot.message != message {
104 self.snapshot.message = message.into_owned();
105 self.snapshot.revision += 1;
106 }
107 }
108 fn status(&mut self, key: StatusMessage) {
109 let locale = crate::messages::locale();
110 if self.status_key != Some((key, locale)) {
111 self.message(key.text(locale));
112 self.status_key = Some((key, locale));
113 }
114 }
115 pub fn clear(&mut self) {
116 if !self.snapshot.history.is_empty() {
117 self.snapshot.history = Arc::default();
118 self.snapshot.revision += 1;
119 }
120 }
121 pub fn retry(&mut self) {
122 self.cancel();
123 if self.snapshot.failed {
124 self.snapshot.failed = false;
125 self.snapshot.revision += 1;
126 }
127 self.attempts = 0;
128 self.due = None;
129 }
130 pub fn cancel(&mut self) {
131 self.epoch = self.epoch.wrapping_add(1);
132 if let Some(task) = self.task.take() {
133 self.retiring.push(task.job);
134 }
135 for job in &self.retiring {
138 job.cancel();
139 }
140 self.due = None;
141 }
142 pub fn quiescent(&self) -> bool {
143 self.task.is_none() && self.retiring.is_empty()
144 }
145 pub fn poll_retiring(&mut self) {
146 self.retiring.retain(|task| task.poll().is_none());
147 }
148 pub fn active(&self) -> bool {
149 self.task.is_some()
150 }
151 pub fn enabled(&mut self, enabled: bool) {
152 if self.snapshot.enabled != enabled {
153 self.snapshot.enabled = enabled;
154 self.snapshot.revision += 1;
155 self.retry();
156 }
157 }
158 pub fn update(
159 &mut self,
160 config: &Config,
161 key: (u64, &str),
162 playing: bool,
163 paused: bool,
164 local: bool,
165 stop: &Arc<AtomicBool>,
166 ) {
167 self.poll_retiring();
168 if !self.snapshot.enabled {
169 if playing && self.key.as_ref().is_some_and(|old| old.0 != key.0) {
170 self.clear();
171 self.key = None;
172 }
173 self.status(StatusMessage::Off);
174 return;
175 }
176 if playing
177 && self
178 .key
179 .as_ref()
180 .map(|(generation, url)| (*generation, url.as_str()))
181 != Some(key)
182 {
183 if self.key.as_ref().is_some_and(|old| old.0 != key.0) {
186 self.clear();
187 }
188 self.retry();
189 self.key = Some((key.0, key.1.to_owned()));
190 }
191 if !playing || paused || local || stop.load(Ordering::Relaxed) {
192 self.cancel();
193 self.status(if local {
194 StatusMessage::Local
195 } else if paused {
196 StatusMessage::Paused
197 } else {
198 StatusMessage::Waiting
199 });
200 return;
201 }
202 let finished = self.task.as_ref().and_then(|t| t.job.poll());
205 if let Some(task) = &self.task {
206 let notices: Vec<_> = task.events.try_iter().collect();
207 for notice in notices {
208 match notice {
209 Notice::Message(message) => self.message(message),
210 Notice::Segment(seg) if seg.epoch == self.epoch => {
211 let first = self
212 .task
213 .as_mut()
214 .unwrap()
215 .first_segment_at
216 .get_or_insert_with(Instant::now);
217 if first.elapsed() >= STABLE_SUBTITLE_PERIOD {
218 self.attempts = 0;
219 }
220 let history = Arc::make_mut(&mut self.snapshot.history);
221 history.push_back(seg);
222 while history.len() > 500 {
223 history.pop_front();
224 }
225 self.snapshot.revision += 1;
226 }
227 _ => {}
228 }
229 }
230 }
231 if let Some(result) = finished {
232 self.task = None;
233 let retry = match result {
234 Ok(retry) => retry,
235 Err(error) => {
236 self.message(crate::t!("subtitle_failed", "detail" => crate::redact::urls(&format!("{error:#}"))));
237 false
238 }
239 };
240 if retry && self.attempts < 3 {
241 let seconds = [1, 3, 7][self.attempts];
242 self.attempts += 1;
243 self.due = Some(Instant::now() + Duration::from_secs(seconds));
244 self.message(crate::t!("subtitle_retrying", "attempt" => self.attempts));
245 } else {
246 self.snapshot.failed = true;
247 self.snapshot.revision += 1;
248 if retry {
249 self.message(
250 crate::t!("subtitle_failed", "detail" => crate::t!("subtitle_connection")),
251 );
252 }
253 }
254 }
255 if self.task.is_none()
256 && self.retiring.is_empty()
257 && !self.snapshot.failed
258 && self.due.is_none_or(|d| Instant::now() >= d)
259 {
260 self.due = None;
261 self.epoch = self.epoch.wrapping_add(1);
262 self.status(StatusMessage::Loading);
263 let (sender, events) = mpsc::sync_channel(64);
264 let config = config.clone();
265 let url = self.key.as_ref().unwrap().1.clone();
266 let epoch = self.epoch;
267 let verification = self
268 .verification
269 .get_or_insert_with(Default::default)
270 .clone();
271 match Background::start("radio-speech", move |cancel| {
272 run(&config, &url, epoch, &cancel, sender, &verification)
273 }) {
274 Ok(job) => {
275 self.task = Some(Task {
276 job,
277 events,
278 first_segment_at: None,
279 })
280 }
281 Err(error) => {
282 self.snapshot.failed = true;
283 self.snapshot.revision += 1;
284 self.message(format!("{error:#}"));
285 }
286 }
287 }
288 }
289}
290
291struct OwnedChildren {
292 worker: Child,
293 ffmpeg: Option<Child>,
294 _lock: models::ResourceLock,
295}
296impl Drop for OwnedChildren {
297 fn drop(&mut self) {
298 if let Some(child) = &mut self.ffmpeg {
299 let _ = child.kill();
300 let _ = child.wait();
301 }
302 let _ = self.worker.kill();
303 let _ = self.worker.wait();
304 }
305}
306
307#[derive(Deserialize)]
308struct Event {
309 v: u8,
310 epoch: u64,
311 #[serde(rename = "type")]
312 kind: String,
313 #[serde(default)]
314 start_ms: u64,
315 #[serde(default)]
316 end_ms: u64,
317 #[serde(default)]
318 lag_ms: u64,
319 #[serde(default)]
320 text: String,
321}
322
323fn send(sender: &SyncSender<Notice>, notice: Notice) {
324 let _ = sender.try_send(notice);
325}
326
327fn run(
328 config: &Config,
329 url: &str,
330 epoch: u64,
331 cancel: &Arc<AtomicBool>,
332 sender: SyncSender<Notice>,
333 verification: &Mutex<models::VerificationCache>,
334) -> Result<bool> {
335 let lock = models::lock(config, false)?;
336 models::runtime_check(config, cancel)?;
337 verification
338 .lock()
339 .unwrap()
340 .verify(config, &config.settings.subtitles.model, cancel)?;
341 ensure!(
342 !cancel.load(Ordering::Relaxed),
343 DomainError::new(ErrorCode::Cancelled)
344 );
345 let settings = &config.settings.subtitles;
346 let args = serde_json::json!({"epoch":epoch,"model":settings.model,"model_dir":models::model_dir(config, &settings.model),"vad":models::model_dir(config,"silero-vad").join("silero_vad.onnx"),"language":settings.language,"threads":settings.effective_threads(),"max_segment_secs":settings.max_segment_secs});
347 let mut command = Command::new(models::python(config));
348 command
349 .args([
350 "-I",
351 "-u",
352 "-c",
353 include_str!("worker.py"),
354 &args.to_string(),
355 ])
356 .stdin(Stdio::piped())
357 .stdout(Stdio::piped())
358 .stderr(Stdio::piped());
359 run_processes(config, url, epoch, cancel, sender, command, lock)
360}
361
362fn run_processes(
363 config: &Config,
364 url: &str,
365 epoch: u64,
366 cancel: &Arc<AtomicBool>,
367 sender: SyncSender<Notice>,
368 command: Command,
369 lock: models::ResourceLock,
370) -> Result<bool> {
371 let worker = crate::process::spawn_owned(command)?;
372 let mut children = OwnedChildren {
373 worker,
374 ffmpeg: None,
375 _lock: lock,
376 };
377 let stderr = crate::process::drain(children.worker.stderr.take().unwrap(), 16384);
378 let stdout = children.worker.stdout.take().unwrap();
379 let (events, receiver) = mpsc::sync_channel(64);
380 let reader = thread::spawn(move || {
381 let mut reader = BufReader::new(stdout);
382 loop {
383 let mut line = Vec::new();
384 let n = reader.by_ref().take(65537).read_until(b'\n', &mut line);
385 match n {
386 Ok(0) => break,
387 Ok(n) if n <= 65536 => {
388 let parsed = serde_json::from_slice::<Event>(&line).map_err(|e| e.to_string());
389 if events.send(parsed).is_err() {
390 break;
391 }
392 }
393 _ => {
394 let _ = events.send(Err("invalid speech protocol line".into()));
395 break;
396 }
397 }
398 }
399 });
400 let started = Instant::now();
401 let mut ready = false;
402 let mut decoding = None::<Instant>;
403 let mut pump = None;
404 let mut ffmpeg_stderr = None;
405 let bytes = Arc::new(AtomicU64::new(0));
406 let mut last_bytes = 0;
407 let mut last_audio = Instant::now();
408 let result = (|| -> Result<bool> {
409 loop {
410 if cancel.load(Ordering::Relaxed) {
411 return Ok(false);
412 }
413 ensure!(
414 ready || started.elapsed() < Duration::from_secs(90),
415 "speech model loading timed out"
416 );
417 ensure!(
418 decoding.is_none_or(|t| t.elapsed() < Duration::from_secs(60)),
419 "speech inference timed out"
420 );
421 let incoming = match receiver.recv_timeout(Duration::from_millis(25)) {
422 Ok(event) => Some(event),
423 Err(mpsc::RecvTimeoutError::Timeout) => None,
424 Err(mpsc::RecvTimeoutError::Disconnected) => {
425 bail!("speech worker closed its output; run radio subtitles doctor")
426 }
427 };
428 if let Some(event) = incoming {
429 let event = event.map_err(anyhow::Error::msg)?;
430 ensure!(
431 event.v == 1 && event.epoch == epoch,
432 "speech protocol version or epoch mismatch"
433 );
434 match event.kind.as_str() {
435 "ready" if !ready => {
436 ready = true;
437 send(
438 &sender,
439 Notice::Message(crate::t!("subtitle_listening").into()),
440 );
441 let mut command = Command::new(&config.settings.ffmpeg);
442 command
443 .args([
444 "-hide_banner",
445 "-nostdin",
446 "-loglevel",
447 "error",
448 "-rw_timeout",
449 &(config.settings.request_timeout_secs * 1_000_000).to_string(),
450 "-user_agent",
451 crate::USER_AGENT,
452 "-readrate",
453 "1",
454 "-i",
455 url,
456 "-map",
457 "0:a:0",
458 "-vn",
459 "-ac",
460 "1",
461 "-ar",
462 "16000",
463 "-f",
464 "f32le",
465 "pipe:1",
466 ])
467 .stdin(Stdio::null())
468 .stdout(Stdio::piped())
469 .stderr(Stdio::piped());
470 let child = crate::process::spawn_owned(command)?;
471 children.ffmpeg = Some(child);
472 let ffmpeg = children.ffmpeg.as_mut().unwrap();
473 ffmpeg_stderr =
474 Some(crate::process::drain(ffmpeg.stderr.take().unwrap(), 16384));
475 let mut input = ffmpeg.stdout.take().unwrap();
476 let mut output = children.worker.stdin.take().unwrap();
477 let bytes = bytes.clone();
478 pump = Some(thread::spawn(move || {
479 use std::io::Write;
480 let mut buf = [0; 8192];
481 while let Ok(n) = input.read(&mut buf) {
482 if n == 0 || output.write_all(&buf[..n]).is_err() {
483 break;
484 }
485 bytes.fetch_add(n as u64, Ordering::Relaxed);
486 }
487 }));
488 last_audio = Instant::now();
489 }
490 "decoding" => {
491 decoding = Some(Instant::now());
492 send(
493 &sender,
494 Notice::Message(crate::t!("subtitle_decoding").into()),
495 );
496 }
497 "listening" => {
498 decoding = None;
499 send(
500 &sender,
501 Notice::Message(crate::t!("subtitle_listening").into()),
502 );
503 }
504 "segment" => {
505 ensure!(
506 event.end_ms >= event.start_ms
507 && event.end_ms - event.start_ms <= 20000,
508 "invalid speech segment time range"
509 );
510 if event.lag_ms <= 30000 {
511 let text = crate::clean_text(&event.text);
512 if !text.is_empty() {
513 send(
514 &sender,
515 Notice::Segment(Segment {
516 epoch,
517 start_ms: event.start_ms,
518 end_ms: event.end_ms,
519 text,
520 }),
521 );
522 }
523 }
524 }
525 "gap" => send(&sender, Notice::Message(crate::t!("subtitle_gap").into())),
526 "done" => return Ok(true),
527 _ => bail!("unexpected speech protocol event"),
528 }
529 }
530 let now_bytes = bytes.load(Ordering::Relaxed);
531 if now_bytes != last_bytes {
532 last_bytes = now_bytes;
533 last_audio = Instant::now();
534 }
535 if ready
536 && last_audio.elapsed() >= Duration::from_secs(config.settings.request_timeout_secs)
537 {
538 return Ok(true);
539 }
540 }
541 })();
542 drop(children); drop(receiver);
544 let _ = reader.join();
545 if let Some(pump) = pump {
546 let _ = pump.join();
547 }
548 let diagnostics = stderr.join().unwrap_or_default();
549 if let Some(stderr) = ffmpeg_stderr {
550 let _ = stderr.join();
551 }
552 if !diagnostics.is_empty() {
553 return result.map_err(|e| {
554 e.context(crate::clean_text(&crate::redact::urls(
555 &String::from_utf8_lossy(&diagnostics),
556 )))
557 });
558 }
559 result
560}
561
562#[cfg(test)]
563mod tests {
564 use super::*;
565 use crate::test_support::{Server, response};
566
567 #[test]
568 #[ignore = "requires explicitly installed models and RADIO_TEST_SPEECH_DATA/WAV"]
569 fn real_models_transcribe_a_local_http_fixture() {
570 let data =
571 std::env::var_os("RADIO_TEST_SPEECH_DATA").expect("isolated speech data directory");
572 let wav = std::fs::read(
573 std::env::var_os("RADIO_TEST_SPEECH_WAV").expect("local speech WAV fixture"),
574 )
575 .unwrap();
576 let server = Server::new(move |_, stream| response(stream, 200, "audio/wav", &wav));
577 let dir = tempfile::tempdir().unwrap();
578 let mut config = crate::test_support::config(dir.path());
579 config.data_dir = data.into();
580 for model in [
581 "sensevoice-small-int8",
582 "sensevoice-small-yue-int8",
583 "qwen3-asr-0.6b-int8",
584 ] {
585 config.settings.subtitles.model = model.into();
586 let (sender, events) = mpsc::sync_channel(64);
587 let start = Instant::now();
588 assert!(
589 run(
590 &config,
591 &server.url,
592 1,
593 &Arc::new(AtomicBool::new(false)),
594 sender,
595 &Mutex::default()
596 )
597 .unwrap()
598 );
599 let mut count = 0;
600 for notice in events.try_iter() {
601 if let Notice::Segment(segment) = notice {
602 eprintln!(
603 "{model} {}-{} ms: {}",
604 segment.start_ms, segment.end_ms, segment.text
605 );
606 count += 1;
607 }
608 }
609 assert!(count > 0, "{model} produced no transcription");
610 eprintln!(
611 "{model}: {count} segments, {:.2}s including realtime audio",
612 start.elapsed().as_secs_f32()
613 );
614 }
615 }
616
617 fn fake_command(script: &str) -> Command {
618 let mut command = Command::new("sh");
619 command
620 .args(["-c", script])
621 .stdin(Stdio::piped())
622 .stdout(Stdio::piped())
623 .stderr(Stdio::piped());
624 command
625 }
626
627 #[test]
628 fn pcm_reaches_worker_and_protocol_segments_are_sanitized() {
629 let audio = crate::test_support::fixture("pcm_s16le", "wav");
630 let server = Server::new(move |_, stream| response(stream, 200, "audio/wav", &audio));
631 let dir = tempfile::tempdir().unwrap();
632 let config = crate::test_support::config(dir.path());
633 let (sender, events) = mpsc::sync_channel(64);
634 let command = fake_command(
635 "printf '%s\n' '{\"v\":1,\"epoch\":7,\"type\":\"ready\"}'; head -c 4096 >/dev/null; printf '%s\n' '{\"v\":1,\"epoch\":7,\"type\":\"segment\",\"start_ms\":0,\"end_ms\":64,\"text\":\"hello\\u001b world\"}' '{\"v\":1,\"epoch\":7,\"type\":\"done\"}'; cat >/dev/null",
636 );
637 assert!(
638 run_processes(
639 &config,
640 &server.url,
641 7,
642 &Arc::new(AtomicBool::new(false)),
643 sender,
644 command,
645 models::lock(&config, false).unwrap()
646 )
647 .unwrap()
648 );
649 let segments: Vec<_> = events
650 .try_iter()
651 .filter_map(|n| {
652 if let Notice::Segment(s) = n {
653 Some(s)
654 } else {
655 None
656 }
657 })
658 .collect();
659 assert_eq!(segments.len(), 1);
660 assert_eq!(segments[0].text, "hello world");
661 assert!(models::lock(&config, true).is_ok());
662 }
663
664 #[test]
665 fn hls_and_icy_sources_feed_the_same_pcm_worker_protocol() {
666 use std::io::Write;
667 let source = tempfile::tempdir().unwrap();
668 assert!(
669 Command::new("ffmpeg")
670 .args([
671 "-v",
672 "error",
673 "-f",
674 "lavfi",
675 "-i",
676 "sine=frequency=440:duration=4",
677 "-c:a",
678 "aac",
679 "-f",
680 "hls",
681 "-hls_time",
682 "1",
683 "-hls_list_size",
684 "0"
685 ])
686 .arg(source.path().join("index.m3u8"))
687 .status()
688 .unwrap()
689 .success()
690 );
691 let root = source.path().to_path_buf();
692 let hls = Server::new(move |request, stream| {
693 let path = request
694 .split_whitespace()
695 .nth(1)
696 .unwrap()
697 .trim_start_matches('/');
698 if path.contains("..") {
699 response(stream, 404, "text/plain", b"");
700 return;
701 }
702 let data = std::fs::read(root.join(path)).unwrap();
703 response(
704 stream,
705 200,
706 if path.ends_with("m3u8") {
707 "application/vnd.apple.mpegurl"
708 } else {
709 "video/mp2t"
710 },
711 &data,
712 );
713 });
714 let mp3 = crate::test_support::fixture("libmp3lame", "mp3");
715 let icy = Server::new(move |request, stream| {
716 assert!(request.to_ascii_lowercase().contains("icy-metadata: 1"));
717 if stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Type: audio/mpeg\r\nicy-metaint: 1024\r\nConnection: close\r\n\r\n").is_err() { return; }
718 for chunk in mp3.chunks(1024) {
719 if stream.write_all(chunk).is_err() {
720 break;
721 }
722 if chunk.len() == 1024 && stream.write_all(&[0]).is_err() {
723 break;
724 }
725 }
726 });
727 for url in [format!("{}/index.m3u8", hls.url), icy.url.clone()] {
728 let dir = tempfile::tempdir().unwrap();
729 let config = crate::test_support::config(dir.path());
730 let (sender, events) = mpsc::sync_channel(64);
731 let command = fake_command(
732 "printf '%s\n' '{\"v\":1,\"epoch\":7,\"type\":\"ready\"}'; head -c 4096 >/dev/null; printf '%s\n' '{\"v\":1,\"epoch\":7,\"type\":\"segment\",\"start_ms\":0,\"end_ms\":64,\"text\":\"audio received\"}' '{\"v\":1,\"epoch\":7,\"type\":\"done\"}'; cat >/dev/null",
733 );
734 run_processes(
735 &config,
736 &url,
737 7,
738 &Arc::new(AtomicBool::new(false)),
739 sender,
740 command,
741 models::lock(&config, false).unwrap(),
742 )
743 .unwrap();
744 assert!(
745 events.try_iter().any(|n| matches!(n, Notice::Segment(_))),
746 "{url}"
747 );
748 }
749 }
750
751 #[test]
752 fn wrong_epoch_and_oversized_protocol_are_rejected_and_reaped() {
753 for script in [
754 "printf '%s\n' '{\"v\":1,\"epoch\":8,\"type\":\"ready\"}'",
755 "head -c 70000 /dev/zero | tr '\\000' x",
756 ] {
757 let dir = tempfile::tempdir().unwrap();
758 let config = crate::test_support::config(dir.path());
759 let (sender, _) = mpsc::sync_channel(64);
760 assert!(
761 run_processes(
762 &config,
763 "http://127.0.0.1:1",
764 7,
765 &Arc::new(AtomicBool::new(false)),
766 sender,
767 fake_command(script),
768 models::lock(&config, false).unwrap()
769 )
770 .is_err()
771 );
772 assert!(models::lock(&config, true).is_ok());
773 }
774 }
775
776 #[test]
777 fn cancellation_reaps_worker_while_it_waits_for_audio() {
778 let dir = tempfile::tempdir().unwrap();
779 let config = crate::test_support::config(dir.path());
780 let cancel = Arc::new(AtomicBool::new(false));
781 let flag = cancel.clone();
782 let (sender, events) = mpsc::sync_channel(64);
783 let (done_tx, done) = mpsc::sync_channel(1);
784 let command = fake_command(
785 "printf '%s\n' '{\"v\":1,\"epoch\":7,\"type\":\"ready\"}'; exec cat >/dev/null",
786 );
787 thread::spawn(move || {
788 let lock = models::lock(&config, false).unwrap();
789 done_tx
790 .send(run_processes(
791 &config,
792 "http://127.0.0.1:1",
793 7,
794 &flag,
795 sender,
796 command,
797 lock,
798 ))
799 .unwrap();
800 });
801 events.recv_timeout(Duration::from_secs(5)).unwrap();
802 cancel.store(true, Ordering::Relaxed);
803 assert!(!done.recv_timeout(Duration::from_secs(2)).unwrap().unwrap());
804 }
805
806 #[test]
807 fn cancellation_kills_and_reaps_a_busy_decoder_without_waiting_for_timeout() {
808 let dir = tempfile::tempdir().unwrap();
809 let config = crate::test_support::config(dir.path());
810 let cancel = Arc::new(AtomicBool::new(false));
811 let flag = cancel.clone();
812 let (sender, events) = mpsc::sync_channel(64);
813 let (done_tx, done) = mpsc::sync_channel(1);
814 let pid_path = dir.path().join("decoder.pid");
815 let mut command = fake_command(
818 "printf '%s' $$ > \"$1\"; trap '' TERM; printf '%s\n' '{\"v\":1,\"epoch\":7,\"type\":\"decoding\"}'; while :; do :; done",
819 );
820 command.arg("decoder").arg(&pid_path);
821 let handle = thread::spawn(move || {
822 let lock = models::lock(&config, false).unwrap();
823 done_tx
824 .send(run_processes(
825 &config,
826 "http://127.0.0.1:1",
827 7,
828 &flag,
829 sender,
830 command,
831 lock,
832 ))
833 .unwrap();
834 });
835 assert!(
837 matches!(events.recv_timeout(Duration::from_secs(5)).unwrap(), Notice::Message(message) if message == crate::t!("subtitle_decoding"))
838 );
839 let pid: libc::pid_t = std::fs::read_to_string(pid_path).unwrap().parse().unwrap();
840 let started = Instant::now();
841 cancel.store(true, Ordering::Relaxed);
842 assert!(!done.recv_timeout(Duration::from_secs(2)).unwrap().unwrap());
843 assert!(started.elapsed() < Duration::from_secs(2));
844 handle.join().unwrap();
845 assert_eq!(unsafe { libc::kill(pid, 0) }, -1);
846 assert_eq!(
847 std::io::Error::last_os_error().raw_os_error(),
848 Some(libc::ESRCH)
849 );
850 }
851
852 #[test]
853 fn pause_and_same_url_restart_discard_old_epoch_results() {
854 let dir = tempfile::tempdir().unwrap();
855 let config = crate::test_support::config(dir.path());
856 let stop = Arc::new(AtomicBool::new(false));
857 let mut controller = Controller::default();
858 controller.enabled(true);
859 controller.key = Some((1, "http://example.com/live".into()));
860 let (sender, events) = mpsc::sync_channel(64);
861 let job = Background::start("fake-speech", |cancel| {
862 while !cancel.load(Ordering::Relaxed) {
863 thread::sleep(Duration::from_millis(5));
864 }
865 Ok(false)
866 })
867 .unwrap();
868 controller.task = Some(Task {
869 job,
870 events,
871 first_segment_at: None,
872 });
873 sender
874 .send(Notice::Segment(Segment {
875 epoch: controller.epoch,
876 start_ms: 0,
877 end_ms: 500,
878 text: "current".into(),
879 }))
880 .ok();
881 sender
882 .send(Notice::Segment(Segment {
883 epoch: controller.epoch.wrapping_sub(1),
884 start_ms: 0,
885 end_ms: 500,
886 text: "stale".into(),
887 }))
888 .ok();
889 let key = controller.key.clone().unwrap();
890 controller.update(&config, (key.0, &key.1), true, false, false, &stop);
891 assert_eq!(controller.snapshot.history.len(), 1);
892 let epoch = controller.epoch;
893 controller.update(&config, (key.0, &key.1), false, true, false, &stop);
894 assert!(controller.epoch != epoch && controller.task.is_none());
895 assert_eq!(controller.snapshot.history.len(), 1);
896 controller.update(&config, (2, &key.1), true, false, false, &stop);
897 assert!(controller.snapshot.history.is_empty());
898 controller.enabled(false);
899 }
900
901 #[test]
902 fn short_connections_exhaust_retries_even_when_they_produce_segments() {
903 let dir = tempfile::tempdir().unwrap();
904 let config = crate::test_support::config(dir.path());
905 let stop = Arc::new(AtomicBool::new(false));
906 for productive in [false, true] {
907 let mut controller = Controller::default();
908 controller.enabled(true);
909 controller.key = Some((1, "http://localhost/live".into()));
910 for attempt in 0..6 {
911 let (sender, events) = mpsc::sync_channel(64);
912 if productive {
913 sender
914 .send(Notice::Segment(Segment {
915 epoch: controller.epoch,
916 start_ms: 0,
917 end_ms: 500,
918 text: "speech".into(),
919 }))
920 .unwrap();
921 }
922 let job = Background::start("fake-speech", |_| Ok(true)).unwrap();
923 controller.task = Some(Task {
924 job,
925 events,
926 first_segment_at: None,
927 });
928 controller.due = None;
929 let deadline = Instant::now() + Duration::from_secs(2);
930 while controller.task.is_some() {
931 controller.update(
932 &config,
933 (1, "http://localhost/live"),
934 true,
935 false,
936 false,
937 &stop,
938 );
939 assert!(Instant::now() < deadline);
940 thread::yield_now();
941 }
942 if attempt == 3 {
943 assert!(controller.snapshot.failed);
944 assert!(controller.due.is_none());
945 break;
946 } else {
947 assert_eq!(controller.attempts, attempt + 1);
948 assert!(!controller.snapshot.failed);
949 }
950 }
951 }
952 }
953
954 #[test]
955 fn only_sustained_speech_on_the_same_connection_resets_retries() {
956 let dir = tempfile::tempdir().unwrap();
957 let config = crate::test_support::config(dir.path());
958 let stop = Arc::new(AtomicBool::new(false));
959 for elapsed in [
962 None,
963 Some(Duration::from_secs(30)),
964 Some(STABLE_SUBTITLE_PERIOD),
965 ] {
966 let mut controller = Controller::default();
967 controller.enabled(true);
968 controller.key = Some((1, "http://localhost/live".into()));
969 controller.attempts = 3;
970 let (sender, events) = mpsc::sync_channel(64);
971 sender
972 .send(Notice::Segment(Segment {
973 epoch: controller.epoch,
974 start_ms: 0,
975 end_ms: 500,
976 text: "speech".into(),
977 }))
978 .unwrap();
979 let job = Background::start("fake-speech", |_| Ok(true)).unwrap();
980 controller.task = Some(Task {
981 job,
982 events,
983 first_segment_at: elapsed.map(|elapsed| Instant::now() - elapsed),
984 });
985 let deadline = Instant::now() + Duration::from_secs(2);
986 while controller.task.is_some() {
987 controller.update(
988 &config,
989 (1, "http://localhost/live"),
990 true,
991 false,
992 false,
993 &stop,
994 );
995 assert!(Instant::now() < deadline);
996 thread::yield_now();
997 }
998 if elapsed == Some(STABLE_SUBTITLE_PERIOD) {
999 assert_eq!(controller.attempts, 1);
1000 assert!(!controller.snapshot.failed);
1001 assert!(controller.due.is_some());
1002 } else {
1003 assert_eq!(controller.attempts, 3);
1004 assert!(controller.snapshot.failed);
1005 assert!(controller.due.is_none());
1006 }
1007 }
1008 }
1009
1010 #[test]
1011 fn history_is_bounded_and_empty_clear_preserves_revision() {
1012 let dir = tempfile::tempdir().unwrap();
1013 let config = crate::test_support::config(dir.path());
1014 let stop = Arc::new(AtomicBool::new(false));
1015 let mut controller = Controller::default();
1016 controller.clear();
1017 assert_eq!(controller.snapshot.revision, 0);
1018 controller.enabled(true);
1019 controller.key = Some((1, "http://localhost/live".into()));
1020 let (sender, events) = mpsc::sync_channel(501);
1021 for i in 0..501 {
1022 sender
1023 .send(Notice::Segment(Segment {
1024 epoch: controller.epoch,
1025 start_ms: i,
1026 end_ms: i + 1,
1027 text: i.to_string(),
1028 }))
1029 .unwrap();
1030 }
1031 let job = Background::start("fake-speech", |_| Ok(true)).unwrap();
1032 controller.task = Some(Task {
1033 job,
1034 events,
1035 first_segment_at: None,
1036 });
1037 controller.update(
1038 &config,
1039 (1, "http://localhost/live"),
1040 true,
1041 false,
1042 false,
1043 &stop,
1044 );
1045 assert_eq!(controller.snapshot.history.len(), 500);
1046 assert_eq!(controller.snapshot.history.front().unwrap().start_ms, 1);
1047 controller.enabled(false);
1048 controller.update(
1049 &config,
1050 (2, "http://localhost/other"),
1051 true,
1052 false,
1053 false,
1054 &stop,
1055 );
1056 assert!(controller.snapshot.history.is_empty());
1057 let revision = controller.snapshot.revision;
1058 controller.clear();
1059 assert_eq!(controller.snapshot.revision, revision);
1060 }
1061}