Skip to main content

radio_cli/speech/
live.rs

1use super::models;
2use crate::{
3    config::Config,
4    error::{DomainError, ErrorCode},
5    process::Background,
6};
7use anyhow::{Result, bail, ensure};
8use serde::Deserialize;
9use std::{
10    borrow::Cow,
11    collections::VecDeque,
12    io::{BufRead, BufReader, Read},
13    process::{Child, Command, Stdio},
14    sync::{
15        Arc, Mutex,
16        atomic::{AtomicBool, AtomicU64, Ordering},
17        mpsc::{self, Receiver, SyncSender},
18    },
19    thread,
20    time::{Duration, Instant},
21};
22
23const STABLE_SUBTITLE_PERIOD: Duration = Duration::from_secs(60);
24
25#[derive(Clone, Debug, PartialEq, Eq)]
26pub struct Segment {
27    pub epoch: u64,
28    pub start_ms: u64,
29    pub end_ms: u64,
30    pub text: String,
31}
32
33#[derive(Clone, Debug, PartialEq, Eq)]
34pub struct Snapshot {
35    pub enabled: bool,
36    pub failed: bool,
37    pub revision: u64,
38    pub message: String,
39    pub history: Arc<VecDeque<Segment>>,
40}
41
42impl Default for Snapshot {
43    fn default() -> Self {
44        Self {
45            enabled: false,
46            failed: false,
47            revision: 0,
48            message: crate::t!("subtitle_off").into(),
49            history: Arc::default(),
50        }
51    }
52}
53
54enum Notice {
55    Message(String),
56    Segment(Segment),
57}
58
59struct Task {
60    job: Background<bool>,
61    events: Receiver<Notice>,
62    first_segment_at: Option<Instant>,
63}
64
65#[derive(Clone, Copy, PartialEq, Eq)]
66enum StatusMessage {
67    Off,
68    Local,
69    Paused,
70    Waiting,
71    Loading,
72}
73
74impl StatusMessage {
75    fn text(self, locale: crate::messages::Locale) -> &'static str {
76        match self {
77            Self::Off => crate::messages::text_in(locale, "subtitle_off"),
78            Self::Local => crate::messages::text_in(locale, "subtitle_local"),
79            Self::Paused => crate::messages::text_in(locale, "subtitle_paused"),
80            Self::Waiting => crate::messages::text_in(locale, "subtitle_waiting"),
81            Self::Loading => crate::messages::text_in(locale, "subtitle_loading"),
82        }
83    }
84}
85
86#[derive(Default)]
87pub struct Controller {
88    pub snapshot: Snapshot,
89    task: Option<Task>,
90    retiring: Vec<Background<bool>>,
91    key: Option<(u64, String)>,
92    epoch: u64,
93    attempts: usize,
94    due: Option<Instant>,
95    status_key: Option<(StatusMessage, crate::messages::Locale)>,
96    verification: Option<Arc<Mutex<models::VerificationCache>>>,
97}
98
99impl Controller {
100    fn message(&mut self, message: impl Into<Cow<'static, str>>) {
101        self.status_key = None;
102        let message = message.into();
103        if self.snapshot.message != message {
104            self.snapshot.message = message.into_owned();
105            self.snapshot.revision += 1;
106        }
107    }
108    fn status(&mut self, key: StatusMessage) {
109        let locale = crate::messages::locale();
110        if self.status_key != Some((key, locale)) {
111            self.message(key.text(locale));
112            self.status_key = Some((key, locale));
113        }
114    }
115    pub fn clear(&mut self) {
116        if !self.snapshot.history.is_empty() {
117            self.snapshot.history = Arc::default();
118            self.snapshot.revision += 1;
119        }
120    }
121    pub fn retry(&mut self) {
122        self.cancel();
123        if self.snapshot.failed {
124            self.snapshot.failed = false;
125            self.snapshot.revision += 1;
126        }
127        self.attempts = 0;
128        self.due = None;
129    }
130    pub fn cancel(&mut self) {
131        self.epoch = self.epoch.wrapping_add(1);
132        if let Some(task) = self.task.take() {
133            self.retiring.push(task.job);
134        }
135        // Dropping Background requests cancellation, but retain retiring work
136        // until its resources are released so a new worker cannot overlap it.
137        for job in &self.retiring {
138            job.cancel();
139        }
140        self.due = None;
141    }
142    pub fn quiescent(&self) -> bool {
143        self.task.is_none() && self.retiring.is_empty()
144    }
145    pub fn poll_retiring(&mut self) {
146        self.retiring.retain(|task| task.poll().is_none());
147    }
148    pub fn active(&self) -> bool {
149        self.task.is_some()
150    }
151    pub fn enabled(&mut self, enabled: bool) {
152        if self.snapshot.enabled != enabled {
153            self.snapshot.enabled = enabled;
154            self.snapshot.revision += 1;
155            self.retry();
156        }
157    }
158    pub fn update(
159        &mut self,
160        config: &Config,
161        key: (u64, &str),
162        playing: bool,
163        paused: bool,
164        local: bool,
165        stop: &Arc<AtomicBool>,
166    ) {
167        self.poll_retiring();
168        if !self.snapshot.enabled {
169            if playing && self.key.as_ref().is_some_and(|old| old.0 != key.0) {
170                self.clear();
171                self.key = None;
172            }
173            self.status(StatusMessage::Off);
174            return;
175        }
176        if playing
177            && self
178                .key
179                .as_ref()
180                .map(|(generation, url)| (*generation, url.as_str()))
181                != Some(key)
182        {
183            // Station generation changes clear history; reconnects have their
184            // own epoch even when generation and URL remain the same.
185            if self.key.as_ref().is_some_and(|old| old.0 != key.0) {
186                self.clear();
187            }
188            self.retry();
189            self.key = Some((key.0, key.1.to_owned()));
190        }
191        if !playing || paused || local || stop.load(Ordering::Relaxed) {
192            self.cancel();
193            self.status(if local {
194                StatusMessage::Local
195            } else if paused {
196                StatusMessage::Paused
197            } else {
198                StatusMessage::Waiting
199            });
200            return;
201        }
202        // Poll completion before draining events: once a finished job is removed,
203        // no final segment may remain stranded in its event receiver.
204        let finished = self.task.as_ref().and_then(|t| t.job.poll());
205        if let Some(task) = &self.task {
206            let notices: Vec<_> = task.events.try_iter().collect();
207            for notice in notices {
208                match notice {
209                    Notice::Message(message) => self.message(message),
210                    Notice::Segment(seg) if seg.epoch == self.epoch => {
211                        let first = self
212                            .task
213                            .as_mut()
214                            .unwrap()
215                            .first_segment_at
216                            .get_or_insert_with(Instant::now);
217                        if first.elapsed() >= STABLE_SUBTITLE_PERIOD {
218                            self.attempts = 0;
219                        }
220                        let history = Arc::make_mut(&mut self.snapshot.history);
221                        history.push_back(seg);
222                        while history.len() > 500 {
223                            history.pop_front();
224                        }
225                        self.snapshot.revision += 1;
226                    }
227                    _ => {}
228                }
229            }
230        }
231        if let Some(result) = finished {
232            self.task = None;
233            let retry = match result {
234                Ok(retry) => retry,
235                Err(error) => {
236                    self.message(crate::t!("subtitle_failed", "detail" => crate::redact::urls(&format!("{error:#}"))));
237                    false
238                }
239            };
240            if retry && self.attempts < 3 {
241                let seconds = [1, 3, 7][self.attempts];
242                self.attempts += 1;
243                self.due = Some(Instant::now() + Duration::from_secs(seconds));
244                self.message(crate::t!("subtitle_retrying", "attempt" => self.attempts));
245            } else {
246                self.snapshot.failed = true;
247                self.snapshot.revision += 1;
248                if retry {
249                    self.message(
250                        crate::t!("subtitle_failed", "detail" => crate::t!("subtitle_connection")),
251                    );
252                }
253            }
254        }
255        if self.task.is_none()
256            && self.retiring.is_empty()
257            && !self.snapshot.failed
258            && self.due.is_none_or(|d| Instant::now() >= d)
259        {
260            self.due = None;
261            self.epoch = self.epoch.wrapping_add(1);
262            self.status(StatusMessage::Loading);
263            let (sender, events) = mpsc::sync_channel(64);
264            let config = config.clone();
265            let url = self.key.as_ref().unwrap().1.clone();
266            let epoch = self.epoch;
267            let verification = self
268                .verification
269                .get_or_insert_with(Default::default)
270                .clone();
271            match Background::start("radio-speech", move |cancel| {
272                run(&config, &url, epoch, &cancel, sender, &verification)
273            }) {
274                Ok(job) => {
275                    self.task = Some(Task {
276                        job,
277                        events,
278                        first_segment_at: None,
279                    })
280                }
281                Err(error) => {
282                    self.snapshot.failed = true;
283                    self.snapshot.revision += 1;
284                    self.message(format!("{error:#}"));
285                }
286            }
287        }
288    }
289}
290
291struct OwnedChildren {
292    worker: Child,
293    ffmpeg: Option<Child>,
294    _lock: models::ResourceLock,
295}
296impl Drop for OwnedChildren {
297    fn drop(&mut self) {
298        if let Some(child) = &mut self.ffmpeg {
299            let _ = child.kill();
300            let _ = child.wait();
301        }
302        let _ = self.worker.kill();
303        let _ = self.worker.wait();
304    }
305}
306
307#[derive(Deserialize)]
308struct Event {
309    v: u8,
310    epoch: u64,
311    #[serde(rename = "type")]
312    kind: String,
313    #[serde(default)]
314    start_ms: u64,
315    #[serde(default)]
316    end_ms: u64,
317    #[serde(default)]
318    lag_ms: u64,
319    #[serde(default)]
320    text: String,
321}
322
323fn send(sender: &SyncSender<Notice>, notice: Notice) {
324    let _ = sender.try_send(notice);
325}
326
327fn run(
328    config: &Config,
329    url: &str,
330    epoch: u64,
331    cancel: &Arc<AtomicBool>,
332    sender: SyncSender<Notice>,
333    verification: &Mutex<models::VerificationCache>,
334) -> Result<bool> {
335    let lock = models::lock(config, false)?;
336    models::runtime_check(config, cancel)?;
337    verification
338        .lock()
339        .unwrap()
340        .verify(config, &config.settings.subtitles.model, cancel)?;
341    ensure!(
342        !cancel.load(Ordering::Relaxed),
343        DomainError::new(ErrorCode::Cancelled)
344    );
345    let settings = &config.settings.subtitles;
346    let args = serde_json::json!({"epoch":epoch,"model":settings.model,"model_dir":models::model_dir(config, &settings.model),"vad":models::model_dir(config,"silero-vad").join("silero_vad.onnx"),"language":settings.language,"threads":settings.effective_threads(),"max_segment_secs":settings.max_segment_secs});
347    let mut command = Command::new(models::python(config));
348    command
349        .args([
350            "-I",
351            "-u",
352            "-c",
353            include_str!("worker.py"),
354            &args.to_string(),
355        ])
356        .stdin(Stdio::piped())
357        .stdout(Stdio::piped())
358        .stderr(Stdio::piped());
359    run_processes(config, url, epoch, cancel, sender, command, lock)
360}
361
362fn run_processes(
363    config: &Config,
364    url: &str,
365    epoch: u64,
366    cancel: &Arc<AtomicBool>,
367    sender: SyncSender<Notice>,
368    command: Command,
369    lock: models::ResourceLock,
370) -> Result<bool> {
371    let worker = crate::process::spawn_owned(command)?;
372    let mut children = OwnedChildren {
373        worker,
374        ffmpeg: None,
375        _lock: lock,
376    };
377    let stderr = crate::process::drain(children.worker.stderr.take().unwrap(), 16384);
378    let stdout = children.worker.stdout.take().unwrap();
379    let (events, receiver) = mpsc::sync_channel(64);
380    let reader = thread::spawn(move || {
381        let mut reader = BufReader::new(stdout);
382        loop {
383            let mut line = Vec::new();
384            let n = reader.by_ref().take(65537).read_until(b'\n', &mut line);
385            match n {
386                Ok(0) => break,
387                Ok(n) if n <= 65536 => {
388                    let parsed = serde_json::from_slice::<Event>(&line).map_err(|e| e.to_string());
389                    if events.send(parsed).is_err() {
390                        break;
391                    }
392                }
393                _ => {
394                    let _ = events.send(Err("invalid speech protocol line".into()));
395                    break;
396                }
397            }
398        }
399    });
400    let started = Instant::now();
401    let mut ready = false;
402    let mut decoding = None::<Instant>;
403    let mut pump = None;
404    let mut ffmpeg_stderr = None;
405    let bytes = Arc::new(AtomicU64::new(0));
406    let mut last_bytes = 0;
407    let mut last_audio = Instant::now();
408    let result = (|| -> Result<bool> {
409        loop {
410            if cancel.load(Ordering::Relaxed) {
411                return Ok(false);
412            }
413            ensure!(
414                ready || started.elapsed() < Duration::from_secs(90),
415                "speech model loading timed out"
416            );
417            ensure!(
418                decoding.is_none_or(|t| t.elapsed() < Duration::from_secs(60)),
419                "speech inference timed out"
420            );
421            let incoming = match receiver.recv_timeout(Duration::from_millis(25)) {
422                Ok(event) => Some(event),
423                Err(mpsc::RecvTimeoutError::Timeout) => None,
424                Err(mpsc::RecvTimeoutError::Disconnected) => {
425                    bail!("speech worker closed its output; run radio subtitles doctor")
426                }
427            };
428            if let Some(event) = incoming {
429                let event = event.map_err(anyhow::Error::msg)?;
430                ensure!(
431                    event.v == 1 && event.epoch == epoch,
432                    "speech protocol version or epoch mismatch"
433                );
434                match event.kind.as_str() {
435                    "ready" if !ready => {
436                        ready = true;
437                        send(
438                            &sender,
439                            Notice::Message(crate::t!("subtitle_listening").into()),
440                        );
441                        let mut command = Command::new(&config.settings.ffmpeg);
442                        command
443                            .args([
444                                "-hide_banner",
445                                "-nostdin",
446                                "-loglevel",
447                                "error",
448                                "-rw_timeout",
449                                &(config.settings.request_timeout_secs * 1_000_000).to_string(),
450                                "-user_agent",
451                                crate::USER_AGENT,
452                                "-readrate",
453                                "1",
454                                "-i",
455                                url,
456                                "-map",
457                                "0:a:0",
458                                "-vn",
459                                "-ac",
460                                "1",
461                                "-ar",
462                                "16000",
463                                "-f",
464                                "f32le",
465                                "pipe:1",
466                            ])
467                            .stdin(Stdio::null())
468                            .stdout(Stdio::piped())
469                            .stderr(Stdio::piped());
470                        let child = crate::process::spawn_owned(command)?;
471                        children.ffmpeg = Some(child);
472                        let ffmpeg = children.ffmpeg.as_mut().unwrap();
473                        ffmpeg_stderr =
474                            Some(crate::process::drain(ffmpeg.stderr.take().unwrap(), 16384));
475                        let mut input = ffmpeg.stdout.take().unwrap();
476                        let mut output = children.worker.stdin.take().unwrap();
477                        let bytes = bytes.clone();
478                        pump = Some(thread::spawn(move || {
479                            use std::io::Write;
480                            let mut buf = [0; 8192];
481                            while let Ok(n) = input.read(&mut buf) {
482                                if n == 0 || output.write_all(&buf[..n]).is_err() {
483                                    break;
484                                }
485                                bytes.fetch_add(n as u64, Ordering::Relaxed);
486                            }
487                        }));
488                        last_audio = Instant::now();
489                    }
490                    "decoding" => {
491                        decoding = Some(Instant::now());
492                        send(
493                            &sender,
494                            Notice::Message(crate::t!("subtitle_decoding").into()),
495                        );
496                    }
497                    "listening" => {
498                        decoding = None;
499                        send(
500                            &sender,
501                            Notice::Message(crate::t!("subtitle_listening").into()),
502                        );
503                    }
504                    "segment" => {
505                        ensure!(
506                            event.end_ms >= event.start_ms
507                                && event.end_ms - event.start_ms <= 20000,
508                            "invalid speech segment time range"
509                        );
510                        if event.lag_ms <= 30000 {
511                            let text = crate::clean_text(&event.text);
512                            if !text.is_empty() {
513                                send(
514                                    &sender,
515                                    Notice::Segment(Segment {
516                                        epoch,
517                                        start_ms: event.start_ms,
518                                        end_ms: event.end_ms,
519                                        text,
520                                    }),
521                                );
522                            }
523                        }
524                    }
525                    "gap" => send(&sender, Notice::Message(crate::t!("subtitle_gap").into())),
526                    "done" => return Ok(true),
527                    _ => bail!("unexpected speech protocol event"),
528                }
529            }
530            let now_bytes = bytes.load(Ordering::Relaxed);
531            if now_bytes != last_bytes {
532                last_bytes = now_bytes;
533                last_audio = Instant::now();
534            }
535            if ready
536                && last_audio.elapsed() >= Duration::from_secs(config.settings.request_timeout_secs)
537            {
538                return Ok(true);
539            }
540        }
541    })();
542    drop(children); // Close pipes before joining their readers/writers.
543    drop(receiver);
544    let _ = reader.join();
545    if let Some(pump) = pump {
546        let _ = pump.join();
547    }
548    let diagnostics = stderr.join().unwrap_or_default();
549    if let Some(stderr) = ffmpeg_stderr {
550        let _ = stderr.join();
551    }
552    if !diagnostics.is_empty() {
553        return result.map_err(|e| {
554            e.context(crate::clean_text(&crate::redact::urls(
555                &String::from_utf8_lossy(&diagnostics),
556            )))
557        });
558    }
559    result
560}
561
562#[cfg(test)]
563mod tests {
564    use super::*;
565    use crate::test_support::{Server, response};
566
567    #[test]
568    #[ignore = "requires explicitly installed models and RADIO_TEST_SPEECH_DATA/WAV"]
569    fn real_models_transcribe_a_local_http_fixture() {
570        let data =
571            std::env::var_os("RADIO_TEST_SPEECH_DATA").expect("isolated speech data directory");
572        let wav = std::fs::read(
573            std::env::var_os("RADIO_TEST_SPEECH_WAV").expect("local speech WAV fixture"),
574        )
575        .unwrap();
576        let server = Server::new(move |_, stream| response(stream, 200, "audio/wav", &wav));
577        let dir = tempfile::tempdir().unwrap();
578        let mut config = crate::test_support::config(dir.path());
579        config.data_dir = data.into();
580        for model in [
581            "sensevoice-small-int8",
582            "sensevoice-small-yue-int8",
583            "qwen3-asr-0.6b-int8",
584        ] {
585            config.settings.subtitles.model = model.into();
586            let (sender, events) = mpsc::sync_channel(64);
587            let start = Instant::now();
588            assert!(
589                run(
590                    &config,
591                    &server.url,
592                    1,
593                    &Arc::new(AtomicBool::new(false)),
594                    sender,
595                    &Mutex::default()
596                )
597                .unwrap()
598            );
599            let mut count = 0;
600            for notice in events.try_iter() {
601                if let Notice::Segment(segment) = notice {
602                    eprintln!(
603                        "{model} {}-{} ms: {}",
604                        segment.start_ms, segment.end_ms, segment.text
605                    );
606                    count += 1;
607                }
608            }
609            assert!(count > 0, "{model} produced no transcription");
610            eprintln!(
611                "{model}: {count} segments, {:.2}s including realtime audio",
612                start.elapsed().as_secs_f32()
613            );
614        }
615    }
616
617    fn fake_command(script: &str) -> Command {
618        let mut command = Command::new("sh");
619        command
620            .args(["-c", script])
621            .stdin(Stdio::piped())
622            .stdout(Stdio::piped())
623            .stderr(Stdio::piped());
624        command
625    }
626
627    #[test]
628    fn pcm_reaches_worker_and_protocol_segments_are_sanitized() {
629        let audio = crate::test_support::fixture("pcm_s16le", "wav");
630        let server = Server::new(move |_, stream| response(stream, 200, "audio/wav", &audio));
631        let dir = tempfile::tempdir().unwrap();
632        let config = crate::test_support::config(dir.path());
633        let (sender, events) = mpsc::sync_channel(64);
634        let command = fake_command(
635            "printf '%s\n' '{\"v\":1,\"epoch\":7,\"type\":\"ready\"}'; head -c 4096 >/dev/null; printf '%s\n' '{\"v\":1,\"epoch\":7,\"type\":\"segment\",\"start_ms\":0,\"end_ms\":64,\"text\":\"hello\\u001b world\"}' '{\"v\":1,\"epoch\":7,\"type\":\"done\"}'; cat >/dev/null",
636        );
637        assert!(
638            run_processes(
639                &config,
640                &server.url,
641                7,
642                &Arc::new(AtomicBool::new(false)),
643                sender,
644                command,
645                models::lock(&config, false).unwrap()
646            )
647            .unwrap()
648        );
649        let segments: Vec<_> = events
650            .try_iter()
651            .filter_map(|n| {
652                if let Notice::Segment(s) = n {
653                    Some(s)
654                } else {
655                    None
656                }
657            })
658            .collect();
659        assert_eq!(segments.len(), 1);
660        assert_eq!(segments[0].text, "hello world");
661        assert!(models::lock(&config, true).is_ok());
662    }
663
664    #[test]
665    fn hls_and_icy_sources_feed_the_same_pcm_worker_protocol() {
666        use std::io::Write;
667        let source = tempfile::tempdir().unwrap();
668        assert!(
669            Command::new("ffmpeg")
670                .args([
671                    "-v",
672                    "error",
673                    "-f",
674                    "lavfi",
675                    "-i",
676                    "sine=frequency=440:duration=4",
677                    "-c:a",
678                    "aac",
679                    "-f",
680                    "hls",
681                    "-hls_time",
682                    "1",
683                    "-hls_list_size",
684                    "0"
685                ])
686                .arg(source.path().join("index.m3u8"))
687                .status()
688                .unwrap()
689                .success()
690        );
691        let root = source.path().to_path_buf();
692        let hls = Server::new(move |request, stream| {
693            let path = request
694                .split_whitespace()
695                .nth(1)
696                .unwrap()
697                .trim_start_matches('/');
698            if path.contains("..") {
699                response(stream, 404, "text/plain", b"");
700                return;
701            }
702            let data = std::fs::read(root.join(path)).unwrap();
703            response(
704                stream,
705                200,
706                if path.ends_with("m3u8") {
707                    "application/vnd.apple.mpegurl"
708                } else {
709                    "video/mp2t"
710                },
711                &data,
712            );
713        });
714        let mp3 = crate::test_support::fixture("libmp3lame", "mp3");
715        let icy = Server::new(move |request, stream| {
716            assert!(request.to_ascii_lowercase().contains("icy-metadata: 1"));
717            if stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Type: audio/mpeg\r\nicy-metaint: 1024\r\nConnection: close\r\n\r\n").is_err() { return; }
718            for chunk in mp3.chunks(1024) {
719                if stream.write_all(chunk).is_err() {
720                    break;
721                }
722                if chunk.len() == 1024 && stream.write_all(&[0]).is_err() {
723                    break;
724                }
725            }
726        });
727        for url in [format!("{}/index.m3u8", hls.url), icy.url.clone()] {
728            let dir = tempfile::tempdir().unwrap();
729            let config = crate::test_support::config(dir.path());
730            let (sender, events) = mpsc::sync_channel(64);
731            let command = fake_command(
732                "printf '%s\n' '{\"v\":1,\"epoch\":7,\"type\":\"ready\"}'; head -c 4096 >/dev/null; printf '%s\n' '{\"v\":1,\"epoch\":7,\"type\":\"segment\",\"start_ms\":0,\"end_ms\":64,\"text\":\"audio received\"}' '{\"v\":1,\"epoch\":7,\"type\":\"done\"}'; cat >/dev/null",
733            );
734            run_processes(
735                &config,
736                &url,
737                7,
738                &Arc::new(AtomicBool::new(false)),
739                sender,
740                command,
741                models::lock(&config, false).unwrap(),
742            )
743            .unwrap();
744            assert!(
745                events.try_iter().any(|n| matches!(n, Notice::Segment(_))),
746                "{url}"
747            );
748        }
749    }
750
751    #[test]
752    fn wrong_epoch_and_oversized_protocol_are_rejected_and_reaped() {
753        for script in [
754            "printf '%s\n' '{\"v\":1,\"epoch\":8,\"type\":\"ready\"}'",
755            "head -c 70000 /dev/zero | tr '\\000' x",
756        ] {
757            let dir = tempfile::tempdir().unwrap();
758            let config = crate::test_support::config(dir.path());
759            let (sender, _) = mpsc::sync_channel(64);
760            assert!(
761                run_processes(
762                    &config,
763                    "http://127.0.0.1:1",
764                    7,
765                    &Arc::new(AtomicBool::new(false)),
766                    sender,
767                    fake_command(script),
768                    models::lock(&config, false).unwrap()
769                )
770                .is_err()
771            );
772            assert!(models::lock(&config, true).is_ok());
773        }
774    }
775
776    #[test]
777    fn cancellation_reaps_worker_while_it_waits_for_audio() {
778        let dir = tempfile::tempdir().unwrap();
779        let config = crate::test_support::config(dir.path());
780        let cancel = Arc::new(AtomicBool::new(false));
781        let flag = cancel.clone();
782        let (sender, events) = mpsc::sync_channel(64);
783        let (done_tx, done) = mpsc::sync_channel(1);
784        let command = fake_command(
785            "printf '%s\n' '{\"v\":1,\"epoch\":7,\"type\":\"ready\"}'; exec cat >/dev/null",
786        );
787        thread::spawn(move || {
788            let lock = models::lock(&config, false).unwrap();
789            done_tx
790                .send(run_processes(
791                    &config,
792                    "http://127.0.0.1:1",
793                    7,
794                    &flag,
795                    sender,
796                    command,
797                    lock,
798                ))
799                .unwrap();
800        });
801        events.recv_timeout(Duration::from_secs(5)).unwrap();
802        cancel.store(true, Ordering::Relaxed);
803        assert!(!done.recv_timeout(Duration::from_secs(2)).unwrap().unwrap());
804    }
805
806    #[test]
807    fn cancellation_kills_and_reaps_a_busy_decoder_without_waiting_for_timeout() {
808        let dir = tempfile::tempdir().unwrap();
809        let config = crate::test_support::config(dir.path());
810        let cancel = Arc::new(AtomicBool::new(false));
811        let flag = cancel.clone();
812        let (sender, events) = mpsc::sync_channel(64);
813        let (done_tx, done) = mpsc::sync_channel(1);
814        let pid_path = dir.path().join("decoder.pid");
815        // This worker stays CPU busy and does not read stdin. SIGTERM is ignored;
816        // the parent must explicitly kill and reap it, on Linux and macOS alike.
817        let mut command = fake_command(
818            "printf '%s' $$ > \"$1\"; trap '' TERM; printf '%s\n' '{\"v\":1,\"epoch\":7,\"type\":\"decoding\"}'; while :; do :; done",
819        );
820        command.arg("decoder").arg(&pid_path);
821        let handle = thread::spawn(move || {
822            let lock = models::lock(&config, false).unwrap();
823            done_tx
824                .send(run_processes(
825                    &config,
826                    "http://127.0.0.1:1",
827                    7,
828                    &flag,
829                    sender,
830                    command,
831                    lock,
832                ))
833                .unwrap();
834        });
835        // Wait for inference to begin, rather than cancelling during startup.
836        assert!(
837            matches!(events.recv_timeout(Duration::from_secs(5)).unwrap(), Notice::Message(message) if message == crate::t!("subtitle_decoding"))
838        );
839        let pid: libc::pid_t = std::fs::read_to_string(pid_path).unwrap().parse().unwrap();
840        let started = Instant::now();
841        cancel.store(true, Ordering::Relaxed);
842        assert!(!done.recv_timeout(Duration::from_secs(2)).unwrap().unwrap());
843        assert!(started.elapsed() < Duration::from_secs(2));
844        handle.join().unwrap();
845        assert_eq!(unsafe { libc::kill(pid, 0) }, -1);
846        assert_eq!(
847            std::io::Error::last_os_error().raw_os_error(),
848            Some(libc::ESRCH)
849        );
850    }
851
852    #[test]
853    fn pause_and_same_url_restart_discard_old_epoch_results() {
854        let dir = tempfile::tempdir().unwrap();
855        let config = crate::test_support::config(dir.path());
856        let stop = Arc::new(AtomicBool::new(false));
857        let mut controller = Controller::default();
858        controller.enabled(true);
859        controller.key = Some((1, "http://example.com/live".into()));
860        let (sender, events) = mpsc::sync_channel(64);
861        let job = Background::start("fake-speech", |cancel| {
862            while !cancel.load(Ordering::Relaxed) {
863                thread::sleep(Duration::from_millis(5));
864            }
865            Ok(false)
866        })
867        .unwrap();
868        controller.task = Some(Task {
869            job,
870            events,
871            first_segment_at: None,
872        });
873        sender
874            .send(Notice::Segment(Segment {
875                epoch: controller.epoch,
876                start_ms: 0,
877                end_ms: 500,
878                text: "current".into(),
879            }))
880            .ok();
881        sender
882            .send(Notice::Segment(Segment {
883                epoch: controller.epoch.wrapping_sub(1),
884                start_ms: 0,
885                end_ms: 500,
886                text: "stale".into(),
887            }))
888            .ok();
889        let key = controller.key.clone().unwrap();
890        controller.update(&config, (key.0, &key.1), true, false, false, &stop);
891        assert_eq!(controller.snapshot.history.len(), 1);
892        let epoch = controller.epoch;
893        controller.update(&config, (key.0, &key.1), false, true, false, &stop);
894        assert!(controller.epoch != epoch && controller.task.is_none());
895        assert_eq!(controller.snapshot.history.len(), 1);
896        controller.update(&config, (2, &key.1), true, false, false, &stop);
897        assert!(controller.snapshot.history.is_empty());
898        controller.enabled(false);
899    }
900
901    #[test]
902    fn short_connections_exhaust_retries_even_when_they_produce_segments() {
903        let dir = tempfile::tempdir().unwrap();
904        let config = crate::test_support::config(dir.path());
905        let stop = Arc::new(AtomicBool::new(false));
906        for productive in [false, true] {
907            let mut controller = Controller::default();
908            controller.enabled(true);
909            controller.key = Some((1, "http://localhost/live".into()));
910            for attempt in 0..6 {
911                let (sender, events) = mpsc::sync_channel(64);
912                if productive {
913                    sender
914                        .send(Notice::Segment(Segment {
915                            epoch: controller.epoch,
916                            start_ms: 0,
917                            end_ms: 500,
918                            text: "speech".into(),
919                        }))
920                        .unwrap();
921                }
922                let job = Background::start("fake-speech", |_| Ok(true)).unwrap();
923                controller.task = Some(Task {
924                    job,
925                    events,
926                    first_segment_at: None,
927                });
928                controller.due = None;
929                let deadline = Instant::now() + Duration::from_secs(2);
930                while controller.task.is_some() {
931                    controller.update(
932                        &config,
933                        (1, "http://localhost/live"),
934                        true,
935                        false,
936                        false,
937                        &stop,
938                    );
939                    assert!(Instant::now() < deadline);
940                    thread::yield_now();
941                }
942                if attempt == 3 {
943                    assert!(controller.snapshot.failed);
944                    assert!(controller.due.is_none());
945                    break;
946                } else {
947                    assert_eq!(controller.attempts, attempt + 1);
948                    assert!(!controller.snapshot.failed);
949                }
950            }
951        }
952    }
953
954    #[test]
955    fn only_sustained_speech_on_the_same_connection_resets_retries() {
956        let dir = tempfile::tempdir().unwrap();
957        let config = crate::test_support::config(dir.path());
958        let stop = Arc::new(AtomicBool::new(false));
959        // Startup, model verification and silence before the first segment do
960        // not count as productive time. Only later segments on this task do.
961        for elapsed in [
962            None,
963            Some(Duration::from_secs(30)),
964            Some(STABLE_SUBTITLE_PERIOD),
965        ] {
966            let mut controller = Controller::default();
967            controller.enabled(true);
968            controller.key = Some((1, "http://localhost/live".into()));
969            controller.attempts = 3;
970            let (sender, events) = mpsc::sync_channel(64);
971            sender
972                .send(Notice::Segment(Segment {
973                    epoch: controller.epoch,
974                    start_ms: 0,
975                    end_ms: 500,
976                    text: "speech".into(),
977                }))
978                .unwrap();
979            let job = Background::start("fake-speech", |_| Ok(true)).unwrap();
980            controller.task = Some(Task {
981                job,
982                events,
983                first_segment_at: elapsed.map(|elapsed| Instant::now() - elapsed),
984            });
985            let deadline = Instant::now() + Duration::from_secs(2);
986            while controller.task.is_some() {
987                controller.update(
988                    &config,
989                    (1, "http://localhost/live"),
990                    true,
991                    false,
992                    false,
993                    &stop,
994                );
995                assert!(Instant::now() < deadline);
996                thread::yield_now();
997            }
998            if elapsed == Some(STABLE_SUBTITLE_PERIOD) {
999                assert_eq!(controller.attempts, 1);
1000                assert!(!controller.snapshot.failed);
1001                assert!(controller.due.is_some());
1002            } else {
1003                assert_eq!(controller.attempts, 3);
1004                assert!(controller.snapshot.failed);
1005                assert!(controller.due.is_none());
1006            }
1007        }
1008    }
1009
1010    #[test]
1011    fn history_is_bounded_and_empty_clear_preserves_revision() {
1012        let dir = tempfile::tempdir().unwrap();
1013        let config = crate::test_support::config(dir.path());
1014        let stop = Arc::new(AtomicBool::new(false));
1015        let mut controller = Controller::default();
1016        controller.clear();
1017        assert_eq!(controller.snapshot.revision, 0);
1018        controller.enabled(true);
1019        controller.key = Some((1, "http://localhost/live".into()));
1020        let (sender, events) = mpsc::sync_channel(501);
1021        for i in 0..501 {
1022            sender
1023                .send(Notice::Segment(Segment {
1024                    epoch: controller.epoch,
1025                    start_ms: i,
1026                    end_ms: i + 1,
1027                    text: i.to_string(),
1028                }))
1029                .unwrap();
1030        }
1031        let job = Background::start("fake-speech", |_| Ok(true)).unwrap();
1032        controller.task = Some(Task {
1033            job,
1034            events,
1035            first_segment_at: None,
1036        });
1037        controller.update(
1038            &config,
1039            (1, "http://localhost/live"),
1040            true,
1041            false,
1042            false,
1043            &stop,
1044        );
1045        assert_eq!(controller.snapshot.history.len(), 500);
1046        assert_eq!(controller.snapshot.history.front().unwrap().start_ms, 1);
1047        controller.enabled(false);
1048        controller.update(
1049            &config,
1050            (2, "http://localhost/other"),
1051            true,
1052            false,
1053            false,
1054            &stop,
1055        );
1056        assert!(controller.snapshot.history.is_empty());
1057        let revision = controller.snapshot.revision;
1058        controller.clear();
1059        assert_eq!(controller.snapshot.revision, revision);
1060    }
1061}