Skip to main content

radio_cli/speech/
live.rs

1use super::models;
2use crate::{
3    config::Config,
4    error::{DomainError, ErrorCode},
5    process::Background,
6};
7use anyhow::{Result, bail, ensure};
8use serde::Deserialize;
9use std::{
10    borrow::Cow,
11    collections::VecDeque,
12    io::{BufRead, BufReader, Read},
13    process::{Child, Command, Stdio},
14    sync::{
15        Arc, Mutex,
16        atomic::{AtomicBool, AtomicU64, Ordering},
17        mpsc::{self, Receiver, SyncSender},
18    },
19    thread,
20    time::{Duration, Instant},
21};
22
23const STABLE_SUBTITLE_PERIOD: Duration = Duration::from_secs(60);
24
25#[derive(Clone, Debug, PartialEq, Eq)]
26pub struct Segment {
27    pub epoch: u64,
28    pub start_ms: u64,
29    pub end_ms: u64,
30    pub text: String,
31}
32
33#[derive(Clone, Debug, PartialEq, Eq)]
34pub struct Snapshot {
35    pub enabled: bool,
36    pub failed: bool,
37    pub revision: u64,
38    pub message: String,
39    pub history: Arc<VecDeque<Segment>>,
40}
41
42impl Default for Snapshot {
43    fn default() -> Self {
44        Self {
45            enabled: false,
46            failed: false,
47            revision: 0,
48            message: crate::t!("subtitle_off").into(),
49            history: Arc::default(),
50        }
51    }
52}
53
54enum Notice {
55    Message(String),
56    Segment(Segment),
57}
58
59struct Task {
60    job: Background<bool>,
61    events: Receiver<Notice>,
62    first_segment_at: Option<Instant>,
63}
64
65#[derive(Clone, Copy, PartialEq, Eq)]
66enum StatusMessage {
67    Off,
68    Local,
69    Paused,
70    Waiting,
71    Loading,
72}
73
74impl StatusMessage {
75    fn text(self, locale: crate::messages::Locale) -> &'static str {
76        match self {
77            Self::Off => crate::messages::text_in(locale, "subtitle_off"),
78            Self::Local => crate::messages::text_in(locale, "subtitle_local"),
79            Self::Paused => crate::messages::text_in(locale, "subtitle_paused"),
80            Self::Waiting => crate::messages::text_in(locale, "subtitle_waiting"),
81            Self::Loading => crate::messages::text_in(locale, "subtitle_loading"),
82        }
83    }
84}
85
86#[derive(Default)]
87pub struct Controller {
88    pub snapshot: Snapshot,
89    task: Option<Task>,
90    retiring: Vec<Background<bool>>,
91    key: Option<(u64, String)>,
92    epoch: u64,
93    attempts: usize,
94    due: Option<Instant>,
95    status_key: Option<(StatusMessage, crate::messages::Locale)>,
96    verification: Option<Arc<Mutex<models::VerificationCache>>>,
97}
98
99impl Controller {
100    fn message(&mut self, message: impl Into<Cow<'static, str>>) {
101        self.status_key = None;
102        let message = message.into();
103        if self.snapshot.message != message {
104            self.snapshot.message = message.into_owned();
105            self.snapshot.revision += 1;
106        }
107    }
108    fn status(&mut self, key: StatusMessage) {
109        let locale = crate::messages::locale();
110        if self.status_key != Some((key, locale)) {
111            self.message(key.text(locale));
112            self.status_key = Some((key, locale));
113        }
114    }
115    pub fn clear(&mut self) {
116        if !self.snapshot.history.is_empty() {
117            self.snapshot.history = Arc::default();
118            self.snapshot.revision += 1;
119        }
120    }
121    pub fn retry(&mut self) {
122        self.cancel();
123        if self.snapshot.failed {
124            self.snapshot.failed = false;
125            self.snapshot.revision += 1;
126        }
127        self.attempts = 0;
128        self.due = None;
129    }
130    pub fn cancel(&mut self) {
131        self.epoch = self.epoch.wrapping_add(1);
132        if let Some(task) = self.task.take() {
133            self.retiring.push(task.job);
134        }
135        // Dropping Background requests cancellation, but retain retiring work
136        // until its resources are released so a new worker cannot overlap it.
137        for job in &self.retiring {
138            job.cancel();
139        }
140        self.due = None;
141    }
142    pub fn quiescent(&self) -> bool {
143        self.task.is_none() && self.retiring.is_empty()
144    }
145    pub fn poll_retiring(&mut self) {
146        self.retiring.retain(|task| task.poll().is_none());
147    }
148    pub fn active(&self) -> bool {
149        self.task.is_some()
150    }
151    pub fn enabled(&mut self, enabled: bool) {
152        if self.snapshot.enabled != enabled {
153            self.snapshot.enabled = enabled;
154            self.snapshot.revision += 1;
155            self.retry();
156        }
157    }
158    pub fn update(
159        &mut self,
160        config: &Config,
161        key: (u64, &str),
162        playing: bool,
163        paused: bool,
164        local: bool,
165        stop: &Arc<AtomicBool>,
166    ) {
167        self.poll_retiring();
168        if !self.snapshot.enabled {
169            if playing && self.key.as_ref().is_some_and(|old| old.0 != key.0) {
170                self.clear();
171                self.key = None;
172            }
173            self.status(StatusMessage::Off);
174            return;
175        }
176        if playing
177            && self
178                .key
179                .as_ref()
180                .map(|(generation, url)| (*generation, url.as_str()))
181                != Some(key)
182        {
183            // Station generation changes clear history; reconnects have their
184            // own epoch even when generation and URL remain the same.
185            if self.key.as_ref().is_some_and(|old| old.0 != key.0) {
186                self.clear();
187            }
188            self.retry();
189            self.key = Some((key.0, key.1.to_owned()));
190        }
191        if !playing || paused || local || stop.load(Ordering::Relaxed) {
192            self.cancel();
193            self.status(if local {
194                StatusMessage::Local
195            } else if paused {
196                StatusMessage::Paused
197            } else {
198                StatusMessage::Waiting
199            });
200            return;
201        }
202        // Poll completion before draining events: once a finished job is removed,
203        // no final segment may remain stranded in its event receiver.
204        let finished = self.task.as_ref().and_then(|t| t.job.poll());
205        if let Some(task) = &self.task {
206            let notices: Vec<_> = task.events.try_iter().collect();
207            for notice in notices {
208                match notice {
209                    Notice::Message(message) => self.message(message),
210                    Notice::Segment(seg) if seg.epoch == self.epoch => {
211                        let first = self
212                            .task
213                            .as_mut()
214                            .unwrap()
215                            .first_segment_at
216                            .get_or_insert_with(Instant::now);
217                        if first.elapsed() >= STABLE_SUBTITLE_PERIOD {
218                            self.attempts = 0;
219                        }
220                        let history = Arc::make_mut(&mut self.snapshot.history);
221                        history.push_back(seg);
222                        while history.len() > 500 {
223                            history.pop_front();
224                        }
225                        self.snapshot.revision += 1;
226                    }
227                    _ => {}
228                }
229            }
230        }
231        if let Some(result) = finished {
232            self.task = None;
233            let retry = match result {
234                Ok(retry) => retry,
235                Err(error) => {
236                    self.message(crate::t!("subtitle_failed", "detail" => crate::redact::urls(&format!("{error:#}"))));
237                    false
238                }
239            };
240            if retry && self.attempts < 3 {
241                let seconds = [1, 3, 7][self.attempts];
242                self.attempts += 1;
243                self.due = Some(Instant::now() + Duration::from_secs(seconds));
244                self.message(crate::t!("subtitle_retrying", "attempt" => self.attempts));
245            } else {
246                self.snapshot.failed = true;
247                self.snapshot.revision += 1;
248                if retry {
249                    self.message(
250                        crate::t!("subtitle_failed", "detail" => crate::t!("subtitle_connection")),
251                    );
252                }
253            }
254        }
255        if self.task.is_none()
256            && self.retiring.is_empty()
257            && !self.snapshot.failed
258            && self.due.is_none_or(|d| Instant::now() >= d)
259        {
260            self.due = None;
261            self.epoch = self.epoch.wrapping_add(1);
262            self.status(StatusMessage::Loading);
263            let (sender, events) = mpsc::sync_channel(64);
264            let config = config.clone();
265            let url = self.key.as_ref().unwrap().1.clone();
266            let epoch = self.epoch;
267            let verification = self
268                .verification
269                .get_or_insert_with(Default::default)
270                .clone();
271            match Background::start("radio-speech", move |cancel| {
272                run(&config, &url, epoch, &cancel, sender, &verification)
273            }) {
274                Ok(job) => {
275                    self.task = Some(Task {
276                        job,
277                        events,
278                        first_segment_at: None,
279                    })
280                }
281                Err(error) => {
282                    self.snapshot.failed = true;
283                    self.snapshot.revision += 1;
284                    self.message(format!("{error:#}"));
285                }
286            }
287        }
288    }
289}
290
291struct OwnedChildren {
292    worker: Child,
293    ffmpeg: Option<Child>,
294    _lock: models::ResourceLock,
295}
296impl Drop for OwnedChildren {
297    fn drop(&mut self) {
298        if let Some(child) = &mut self.ffmpeg {
299            let _ = child.kill();
300            let _ = child.wait();
301        }
302        let _ = self.worker.kill();
303        let _ = self.worker.wait();
304    }
305}
306
307#[derive(Deserialize)]
308struct Event {
309    v: u8,
310    epoch: u64,
311    #[serde(rename = "type")]
312    kind: String,
313    #[serde(default)]
314    start_ms: u64,
315    #[serde(default)]
316    end_ms: u64,
317    #[serde(default)]
318    lag_ms: u64,
319    #[serde(default)]
320    text: String,
321}
322
323fn send(sender: &SyncSender<Notice>, notice: Notice) {
324    let _ = sender.try_send(notice);
325}
326
327fn run(
328    config: &Config,
329    url: &str,
330    epoch: u64,
331    cancel: &Arc<AtomicBool>,
332    sender: SyncSender<Notice>,
333    verification: &Mutex<models::VerificationCache>,
334) -> Result<bool> {
335    let lock = models::lock(config, false)?;
336    models::runtime_check(config, cancel)?;
337    verification
338        .lock()
339        .unwrap()
340        .verify(config, &config.settings.subtitles.model, cancel)?;
341    ensure!(
342        !cancel.load(Ordering::Relaxed),
343        DomainError::new(ErrorCode::Cancelled)
344    );
345    let settings = &config.settings.subtitles;
346    let args = serde_json::json!({"epoch":epoch,"model":settings.model,"model_dir":models::model_dir(config, &settings.model),"vad":models::model_dir(config,"silero-vad").join("silero_vad.onnx"),"language":settings.language,"threads":settings.effective_threads(),"max_segment_secs":settings.max_segment_secs});
347    let mut command = Command::new(models::python(config));
348    command
349        .args([
350            "-I",
351            "-u",
352            "-c",
353            include_str!("worker.py"),
354            &args.to_string(),
355        ])
356        .stdin(Stdio::piped())
357        .stdout(Stdio::piped())
358        .stderr(Stdio::piped());
359    run_processes(config, url, epoch, cancel, sender, command, lock)
360}
361
362fn run_processes(
363    config: &Config,
364    url: &str,
365    epoch: u64,
366    cancel: &Arc<AtomicBool>,
367    sender: SyncSender<Notice>,
368    command: Command,
369    lock: models::ResourceLock,
370) -> Result<bool> {
371    let worker = crate::process::spawn_owned(command)?;
372    let mut children = OwnedChildren {
373        worker,
374        ffmpeg: None,
375        _lock: lock,
376    };
377    let stderr = crate::process::drain(children.worker.stderr.take().unwrap(), 16384);
378    let stdout = children.worker.stdout.take().unwrap();
379    let (events, receiver) = mpsc::sync_channel(64);
380    let reader = thread::spawn(move || {
381        let mut reader = BufReader::new(stdout);
382        loop {
383            let mut line = Vec::new();
384            let n = reader.by_ref().take(65537).read_until(b'\n', &mut line);
385            match n {
386                Ok(0) => break,
387                Ok(n) if n <= 65536 => {
388                    let parsed = serde_json::from_slice::<Event>(&line).map_err(|e| e.to_string());
389                    if events.send(parsed).is_err() {
390                        break;
391                    }
392                }
393                _ => {
394                    let _ = events.send(Err("invalid speech protocol line".into()));
395                    break;
396                }
397            }
398        }
399    });
400    let started = Instant::now();
401    let mut ready = false;
402    let mut decoding = None::<Instant>;
403    let mut pump = None;
404    let mut ffmpeg_stderr = None;
405    let bytes = Arc::new(AtomicU64::new(0));
406    let mut last_bytes = 0;
407    let mut last_audio = Instant::now();
408    let result = (|| -> Result<bool> {
409        loop {
410            if cancel.load(Ordering::Relaxed) {
411                return Ok(false);
412            }
413            ensure!(
414                ready || started.elapsed() < Duration::from_secs(90),
415                "speech model loading timed out"
416            );
417            ensure!(
418                decoding.is_none_or(|t| t.elapsed() < Duration::from_secs(60)),
419                "speech inference timed out"
420            );
421            let incoming = match receiver.recv_timeout(Duration::from_millis(25)) {
422                Ok(event) => Some(event),
423                Err(mpsc::RecvTimeoutError::Timeout) => None,
424                Err(mpsc::RecvTimeoutError::Disconnected) => {
425                    bail!("speech worker closed its output; run radio subtitles doctor")
426                }
427            };
428            if let Some(event) = incoming {
429                let event = event.map_err(anyhow::Error::msg)?;
430                ensure!(
431                    event.v == 1 && event.epoch == epoch,
432                    "speech protocol version or epoch mismatch"
433                );
434                match event.kind.as_str() {
435                    "ready" if !ready => {
436                        ready = true;
437                        send(
438                            &sender,
439                            Notice::Message(crate::t!("subtitle_listening").into()),
440                        );
441                        let mut command = Command::new(&config.settings.ffmpeg);
442                        command
443                            .args([
444                                "-hide_banner",
445                                "-nostdin",
446                                "-loglevel",
447                                "error",
448                                "-rw_timeout",
449                                &(config.settings.request_timeout_secs * 1_000_000).to_string(),
450                                "-user_agent",
451                                crate::USER_AGENT,
452                                "-readrate",
453                                "1",
454                                "-i",
455                                url,
456                                "-map",
457                                "0:a:0",
458                                "-vn",
459                                "-ac",
460                                "1",
461                                "-ar",
462                                "16000",
463                                "-f",
464                                "f32le",
465                                "pipe:1",
466                            ])
467                            .stdin(Stdio::null())
468                            .stdout(Stdio::piped())
469                            .stderr(Stdio::piped());
470                        let child = crate::process::spawn_owned(command)?;
471                        children.ffmpeg = Some(child);
472                        let ffmpeg = children.ffmpeg.as_mut().unwrap();
473                        ffmpeg_stderr =
474                            Some(crate::process::drain(ffmpeg.stderr.take().unwrap(), 16384));
475                        let mut input = ffmpeg.stdout.take().unwrap();
476                        let mut output = children.worker.stdin.take().unwrap();
477                        let bytes = bytes.clone();
478                        pump = Some(thread::spawn(move || {
479                            use std::io::Write;
480                            let mut buf = [0; 8192];
481                            while let Ok(n) = input.read(&mut buf) {
482                                if n == 0 || output.write_all(&buf[..n]).is_err() {
483                                    break;
484                                }
485                                bytes.fetch_add(n as u64, Ordering::Relaxed);
486                            }
487                        }));
488                        last_audio = Instant::now();
489                    }
490                    "decoding" => {
491                        decoding = Some(Instant::now());
492                        send(
493                            &sender,
494                            Notice::Message(crate::t!("subtitle_decoding").into()),
495                        );
496                    }
497                    "listening" => {
498                        decoding = None;
499                        send(
500                            &sender,
501                            Notice::Message(crate::t!("subtitle_listening").into()),
502                        );
503                    }
504                    "segment" => {
505                        ensure!(
506                            event.end_ms >= event.start_ms
507                                && event.end_ms - event.start_ms <= 20000,
508                            "invalid speech segment time range"
509                        );
510                        if event.lag_ms <= 30000 {
511                            let text = crate::clean_text(&event.text);
512                            if !text.is_empty() {
513                                send(
514                                    &sender,
515                                    Notice::Segment(Segment {
516                                        epoch,
517                                        start_ms: event.start_ms,
518                                        end_ms: event.end_ms,
519                                        text,
520                                    }),
521                                );
522                            }
523                        }
524                    }
525                    "gap" => send(&sender, Notice::Message(crate::t!("subtitle_gap").into())),
526                    "done" => return Ok(true),
527                    _ => bail!("unexpected speech protocol event"),
528                }
529            }
530            let now_bytes = bytes.load(Ordering::Relaxed);
531            if now_bytes != last_bytes {
532                last_bytes = now_bytes;
533                last_audio = Instant::now();
534            }
535            if ready
536                && last_audio.elapsed() >= Duration::from_secs(config.settings.request_timeout_secs)
537            {
538                return Ok(true);
539            }
540        }
541    })();
542    drop(children); // Close pipes before joining their readers/writers.
543    drop(receiver);
544    let _ = reader.join();
545    if let Some(pump) = pump {
546        let _ = pump.join();
547    }
548    let diagnostics = stderr.join().unwrap_or_default();
549    if let Some(stderr) = ffmpeg_stderr {
550        let _ = stderr.join();
551    }
552    if !diagnostics.is_empty() {
553        return result.map_err(|e| {
554            e.context(crate::clean_text(&crate::redact::urls(
555                &String::from_utf8_lossy(&diagnostics),
556            )))
557        });
558    }
559    result
560}
561
562#[cfg(test)]
563mod tests {
564    use super::*;
565    use crate::test_support::{Server, response};
566
567    #[test]
568    #[ignore = "requires explicitly installed models and RADIO_TEST_SPEECH_DATA/WAV"]
569    fn real_models_transcribe_a_local_http_fixture() {
570        let data =
571            std::env::var_os("RADIO_TEST_SPEECH_DATA").expect("isolated speech data directory");
572        let wav = std::fs::read(
573            std::env::var_os("RADIO_TEST_SPEECH_WAV").expect("local speech WAV fixture"),
574        )
575        .unwrap();
576        let server = Server::new(move |_, stream| response(stream, 200, "audio/wav", &wav));
577        let dir = tempfile::tempdir().unwrap();
578        let mut config = crate::test_support::config(dir.path());
579        config.data_dir = data.into();
580        for model in ["sensevoice-small-yue-int8", "qwen3-asr-0.6b-int8"] {
581            config.settings.subtitles.model = model.into();
582            let (sender, events) = mpsc::sync_channel(64);
583            let start = Instant::now();
584            assert!(
585                run(
586                    &config,
587                    &server.url,
588                    1,
589                    &Arc::new(AtomicBool::new(false)),
590                    sender,
591                    &Mutex::default()
592                )
593                .unwrap()
594            );
595            let mut count = 0;
596            for notice in events.try_iter() {
597                if let Notice::Segment(segment) = notice {
598                    eprintln!(
599                        "{model} {}-{} ms: {}",
600                        segment.start_ms, segment.end_ms, segment.text
601                    );
602                    count += 1;
603                }
604            }
605            assert!(count > 0, "{model} produced no transcription");
606            eprintln!(
607                "{model}: {count} segments, {:.2}s including realtime audio",
608                start.elapsed().as_secs_f32()
609            );
610        }
611    }
612
613    fn fake_command(script: &str) -> Command {
614        let mut command = Command::new("sh");
615        command
616            .args(["-c", script])
617            .stdin(Stdio::piped())
618            .stdout(Stdio::piped())
619            .stderr(Stdio::piped());
620        command
621    }
622
623    #[test]
624    fn pcm_reaches_worker_and_protocol_segments_are_sanitized() {
625        let audio = crate::test_support::fixture("pcm_s16le", "wav");
626        let server = Server::new(move |_, stream| response(stream, 200, "audio/wav", &audio));
627        let dir = tempfile::tempdir().unwrap();
628        let config = crate::test_support::config(dir.path());
629        let (sender, events) = mpsc::sync_channel(64);
630        let command = fake_command(
631            "printf '%s\n' '{\"v\":1,\"epoch\":7,\"type\":\"ready\"}'; head -c 4096 >/dev/null; printf '%s\n' '{\"v\":1,\"epoch\":7,\"type\":\"segment\",\"start_ms\":0,\"end_ms\":64,\"text\":\"hello\\u001b world\"}' '{\"v\":1,\"epoch\":7,\"type\":\"done\"}'; cat >/dev/null",
632        );
633        assert!(
634            run_processes(
635                &config,
636                &server.url,
637                7,
638                &Arc::new(AtomicBool::new(false)),
639                sender,
640                command,
641                models::lock(&config, false).unwrap()
642            )
643            .unwrap()
644        );
645        let segments: Vec<_> = events
646            .try_iter()
647            .filter_map(|n| {
648                if let Notice::Segment(s) = n {
649                    Some(s)
650                } else {
651                    None
652                }
653            })
654            .collect();
655        assert_eq!(segments.len(), 1);
656        assert_eq!(segments[0].text, "hello world");
657        assert!(models::lock(&config, true).is_ok());
658    }
659
660    #[test]
661    fn hls_and_icy_sources_feed_the_same_pcm_worker_protocol() {
662        use std::io::Write;
663        let source = tempfile::tempdir().unwrap();
664        assert!(
665            Command::new("ffmpeg")
666                .args([
667                    "-v",
668                    "error",
669                    "-f",
670                    "lavfi",
671                    "-i",
672                    "sine=frequency=440:duration=4",
673                    "-c:a",
674                    "aac",
675                    "-f",
676                    "hls",
677                    "-hls_time",
678                    "1",
679                    "-hls_list_size",
680                    "0"
681                ])
682                .arg(source.path().join("index.m3u8"))
683                .status()
684                .unwrap()
685                .success()
686        );
687        let root = source.path().to_path_buf();
688        let hls = Server::new(move |request, stream| {
689            let path = request
690                .split_whitespace()
691                .nth(1)
692                .unwrap()
693                .trim_start_matches('/');
694            if path.contains("..") {
695                response(stream, 404, "text/plain", b"");
696                return;
697            }
698            let data = std::fs::read(root.join(path)).unwrap();
699            response(
700                stream,
701                200,
702                if path.ends_with("m3u8") {
703                    "application/vnd.apple.mpegurl"
704                } else {
705                    "video/mp2t"
706                },
707                &data,
708            );
709        });
710        let mp3 = crate::test_support::fixture("libmp3lame", "mp3");
711        let icy = Server::new(move |request, stream| {
712            assert!(request.to_ascii_lowercase().contains("icy-metadata: 1"));
713            if stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Type: audio/mpeg\r\nicy-metaint: 1024\r\nConnection: close\r\n\r\n").is_err() { return; }
714            for chunk in mp3.chunks(1024) {
715                if stream.write_all(chunk).is_err() {
716                    break;
717                }
718                if chunk.len() == 1024 && stream.write_all(&[0]).is_err() {
719                    break;
720                }
721            }
722        });
723        for url in [format!("{}/index.m3u8", hls.url), icy.url.clone()] {
724            let dir = tempfile::tempdir().unwrap();
725            let config = crate::test_support::config(dir.path());
726            let (sender, events) = mpsc::sync_channel(64);
727            let command = fake_command(
728                "printf '%s\n' '{\"v\":1,\"epoch\":7,\"type\":\"ready\"}'; head -c 4096 >/dev/null; printf '%s\n' '{\"v\":1,\"epoch\":7,\"type\":\"segment\",\"start_ms\":0,\"end_ms\":64,\"text\":\"audio received\"}' '{\"v\":1,\"epoch\":7,\"type\":\"done\"}'; cat >/dev/null",
729            );
730            run_processes(
731                &config,
732                &url,
733                7,
734                &Arc::new(AtomicBool::new(false)),
735                sender,
736                command,
737                models::lock(&config, false).unwrap(),
738            )
739            .unwrap();
740            assert!(
741                events.try_iter().any(|n| matches!(n, Notice::Segment(_))),
742                "{url}"
743            );
744        }
745    }
746
747    #[test]
748    fn wrong_epoch_and_oversized_protocol_are_rejected_and_reaped() {
749        for script in [
750            "printf '%s\n' '{\"v\":1,\"epoch\":8,\"type\":\"ready\"}'",
751            "head -c 70000 /dev/zero | tr '\\000' x",
752        ] {
753            let dir = tempfile::tempdir().unwrap();
754            let config = crate::test_support::config(dir.path());
755            let (sender, _) = mpsc::sync_channel(64);
756            assert!(
757                run_processes(
758                    &config,
759                    "http://127.0.0.1:1",
760                    7,
761                    &Arc::new(AtomicBool::new(false)),
762                    sender,
763                    fake_command(script),
764                    models::lock(&config, false).unwrap()
765                )
766                .is_err()
767            );
768            assert!(models::lock(&config, true).is_ok());
769        }
770    }
771
772    #[test]
773    fn cancellation_reaps_worker_while_it_waits_for_audio() {
774        let dir = tempfile::tempdir().unwrap();
775        let config = crate::test_support::config(dir.path());
776        let cancel = Arc::new(AtomicBool::new(false));
777        let flag = cancel.clone();
778        let (sender, events) = mpsc::sync_channel(64);
779        let (done_tx, done) = mpsc::sync_channel(1);
780        let command = fake_command(
781            "printf '%s\n' '{\"v\":1,\"epoch\":7,\"type\":\"ready\"}'; exec cat >/dev/null",
782        );
783        thread::spawn(move || {
784            let lock = models::lock(&config, false).unwrap();
785            done_tx
786                .send(run_processes(
787                    &config,
788                    "http://127.0.0.1:1",
789                    7,
790                    &flag,
791                    sender,
792                    command,
793                    lock,
794                ))
795                .unwrap();
796        });
797        events.recv_timeout(Duration::from_secs(5)).unwrap();
798        cancel.store(true, Ordering::Relaxed);
799        assert!(!done.recv_timeout(Duration::from_secs(2)).unwrap().unwrap());
800    }
801
802    #[test]
803    fn cancellation_kills_and_reaps_a_busy_decoder_without_waiting_for_timeout() {
804        let dir = tempfile::tempdir().unwrap();
805        let config = crate::test_support::config(dir.path());
806        let cancel = Arc::new(AtomicBool::new(false));
807        let flag = cancel.clone();
808        let (sender, events) = mpsc::sync_channel(64);
809        let (done_tx, done) = mpsc::sync_channel(1);
810        let pid_path = dir.path().join("decoder.pid");
811        // This worker stays CPU busy and does not read stdin. SIGTERM is ignored;
812        // the parent must explicitly kill and reap it, on Linux and macOS alike.
813        let mut command = fake_command(
814            "printf '%s' $$ > \"$1\"; trap '' TERM; printf '%s\n' '{\"v\":1,\"epoch\":7,\"type\":\"decoding\"}'; while :; do :; done",
815        );
816        command.arg("decoder").arg(&pid_path);
817        let handle = thread::spawn(move || {
818            let lock = models::lock(&config, false).unwrap();
819            done_tx
820                .send(run_processes(
821                    &config,
822                    "http://127.0.0.1:1",
823                    7,
824                    &flag,
825                    sender,
826                    command,
827                    lock,
828                ))
829                .unwrap();
830        });
831        // Wait for inference to begin, rather than cancelling during startup.
832        assert!(
833            matches!(events.recv_timeout(Duration::from_secs(5)).unwrap(), Notice::Message(message) if message == crate::t!("subtitle_decoding"))
834        );
835        let pid: libc::pid_t = std::fs::read_to_string(pid_path).unwrap().parse().unwrap();
836        let started = Instant::now();
837        cancel.store(true, Ordering::Relaxed);
838        assert!(!done.recv_timeout(Duration::from_secs(2)).unwrap().unwrap());
839        assert!(started.elapsed() < Duration::from_secs(2));
840        handle.join().unwrap();
841        assert_eq!(unsafe { libc::kill(pid, 0) }, -1);
842        assert_eq!(
843            std::io::Error::last_os_error().raw_os_error(),
844            Some(libc::ESRCH)
845        );
846    }
847
848    #[test]
849    fn pause_and_same_url_restart_discard_old_epoch_results() {
850        let dir = tempfile::tempdir().unwrap();
851        let config = crate::test_support::config(dir.path());
852        let stop = Arc::new(AtomicBool::new(false));
853        let mut controller = Controller::default();
854        controller.enabled(true);
855        controller.key = Some((1, "http://example.com/live".into()));
856        let (sender, events) = mpsc::sync_channel(64);
857        let job = Background::start("fake-speech", |cancel| {
858            while !cancel.load(Ordering::Relaxed) {
859                thread::sleep(Duration::from_millis(5));
860            }
861            Ok(false)
862        })
863        .unwrap();
864        controller.task = Some(Task {
865            job,
866            events,
867            first_segment_at: None,
868        });
869        sender
870            .send(Notice::Segment(Segment {
871                epoch: controller.epoch,
872                start_ms: 0,
873                end_ms: 500,
874                text: "current".into(),
875            }))
876            .ok();
877        sender
878            .send(Notice::Segment(Segment {
879                epoch: controller.epoch.wrapping_sub(1),
880                start_ms: 0,
881                end_ms: 500,
882                text: "stale".into(),
883            }))
884            .ok();
885        let key = controller.key.clone().unwrap();
886        controller.update(&config, (key.0, &key.1), true, false, false, &stop);
887        assert_eq!(controller.snapshot.history.len(), 1);
888        let epoch = controller.epoch;
889        controller.update(&config, (key.0, &key.1), false, true, false, &stop);
890        assert!(controller.epoch != epoch && controller.task.is_none());
891        assert_eq!(controller.snapshot.history.len(), 1);
892        controller.update(&config, (2, &key.1), true, false, false, &stop);
893        assert!(controller.snapshot.history.is_empty());
894        controller.enabled(false);
895    }
896
897    #[test]
898    fn short_connections_exhaust_retries_even_when_they_produce_segments() {
899        let dir = tempfile::tempdir().unwrap();
900        let config = crate::test_support::config(dir.path());
901        let stop = Arc::new(AtomicBool::new(false));
902        for productive in [false, true] {
903            let mut controller = Controller::default();
904            controller.enabled(true);
905            controller.key = Some((1, "http://localhost/live".into()));
906            for attempt in 0..6 {
907                let (sender, events) = mpsc::sync_channel(64);
908                if productive {
909                    sender
910                        .send(Notice::Segment(Segment {
911                            epoch: controller.epoch,
912                            start_ms: 0,
913                            end_ms: 500,
914                            text: "speech".into(),
915                        }))
916                        .unwrap();
917                }
918                let job = Background::start("fake-speech", |_| Ok(true)).unwrap();
919                controller.task = Some(Task {
920                    job,
921                    events,
922                    first_segment_at: None,
923                });
924                controller.due = None;
925                let deadline = Instant::now() + Duration::from_secs(2);
926                while controller.task.is_some() {
927                    controller.update(
928                        &config,
929                        (1, "http://localhost/live"),
930                        true,
931                        false,
932                        false,
933                        &stop,
934                    );
935                    assert!(Instant::now() < deadline);
936                    thread::yield_now();
937                }
938                if attempt == 3 {
939                    assert!(controller.snapshot.failed);
940                    assert!(controller.due.is_none());
941                    break;
942                } else {
943                    assert_eq!(controller.attempts, attempt + 1);
944                    assert!(!controller.snapshot.failed);
945                }
946            }
947        }
948    }
949
950    #[test]
951    fn only_sustained_speech_on_the_same_connection_resets_retries() {
952        let dir = tempfile::tempdir().unwrap();
953        let config = crate::test_support::config(dir.path());
954        let stop = Arc::new(AtomicBool::new(false));
955        // Startup, model verification and silence before the first segment do
956        // not count as productive time. Only later segments on this task do.
957        for elapsed in [
958            None,
959            Some(Duration::from_secs(30)),
960            Some(STABLE_SUBTITLE_PERIOD),
961        ] {
962            let mut controller = Controller::default();
963            controller.enabled(true);
964            controller.key = Some((1, "http://localhost/live".into()));
965            controller.attempts = 3;
966            let (sender, events) = mpsc::sync_channel(64);
967            sender
968                .send(Notice::Segment(Segment {
969                    epoch: controller.epoch,
970                    start_ms: 0,
971                    end_ms: 500,
972                    text: "speech".into(),
973                }))
974                .unwrap();
975            let job = Background::start("fake-speech", |_| Ok(true)).unwrap();
976            controller.task = Some(Task {
977                job,
978                events,
979                first_segment_at: elapsed.map(|elapsed| Instant::now() - elapsed),
980            });
981            let deadline = Instant::now() + Duration::from_secs(2);
982            while controller.task.is_some() {
983                controller.update(
984                    &config,
985                    (1, "http://localhost/live"),
986                    true,
987                    false,
988                    false,
989                    &stop,
990                );
991                assert!(Instant::now() < deadline);
992                thread::yield_now();
993            }
994            if elapsed == Some(STABLE_SUBTITLE_PERIOD) {
995                assert_eq!(controller.attempts, 1);
996                assert!(!controller.snapshot.failed);
997                assert!(controller.due.is_some());
998            } else {
999                assert_eq!(controller.attempts, 3);
1000                assert!(controller.snapshot.failed);
1001                assert!(controller.due.is_none());
1002            }
1003        }
1004    }
1005
1006    #[test]
1007    fn history_is_bounded_and_empty_clear_preserves_revision() {
1008        let dir = tempfile::tempdir().unwrap();
1009        let config = crate::test_support::config(dir.path());
1010        let stop = Arc::new(AtomicBool::new(false));
1011        let mut controller = Controller::default();
1012        controller.clear();
1013        assert_eq!(controller.snapshot.revision, 0);
1014        controller.enabled(true);
1015        controller.key = Some((1, "http://localhost/live".into()));
1016        let (sender, events) = mpsc::sync_channel(501);
1017        for i in 0..501 {
1018            sender
1019                .send(Notice::Segment(Segment {
1020                    epoch: controller.epoch,
1021                    start_ms: i,
1022                    end_ms: i + 1,
1023                    text: i.to_string(),
1024                }))
1025                .unwrap();
1026        }
1027        let job = Background::start("fake-speech", |_| Ok(true)).unwrap();
1028        controller.task = Some(Task {
1029            job,
1030            events,
1031            first_segment_at: None,
1032        });
1033        controller.update(
1034            &config,
1035            (1, "http://localhost/live"),
1036            true,
1037            false,
1038            false,
1039            &stop,
1040        );
1041        assert_eq!(controller.snapshot.history.len(), 500);
1042        assert_eq!(controller.snapshot.history.front().unwrap().start_ms, 1);
1043        controller.enabled(false);
1044        controller.update(
1045            &config,
1046            (2, "http://localhost/other"),
1047            true,
1048            false,
1049            false,
1050            &stop,
1051        );
1052        assert!(controller.snapshot.history.is_empty());
1053        let revision = controller.snapshot.revision;
1054        controller.clear();
1055        assert_eq!(controller.snapshot.revision, revision);
1056    }
1057}