Skip to main content

radio_cli/speech/
live.rs

1use super::models;
2use crate::{
3    config::Config,
4    error::{DomainError, ErrorCode},
5    process::Background,
6};
7use anyhow::{Result, bail, ensure};
8use serde::{Deserialize, Serialize};
9use std::{
10    borrow::Cow,
11    collections::VecDeque,
12    io::{BufRead, BufReader, Read},
13    process::{Child, Command, Stdio},
14    sync::{
15        Arc, Mutex,
16        atomic::{AtomicBool, AtomicU64, Ordering},
17        mpsc::{self, Receiver, SyncSender},
18    },
19    thread,
20    time::{Duration, Instant},
21};
22
23const STABLE_SUBTITLE_PERIOD: Duration = Duration::from_secs(60);
24
25#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
26pub struct Segment {
27    pub epoch: u64,
28    pub start_ms: u64,
29    pub end_ms: u64,
30    pub text: String,
31}
32
33#[derive(Clone, Debug, PartialEq, Eq)]
34pub struct Snapshot {
35    pub enabled: bool,
36    pub failed: bool,
37    pub revision: u64,
38    pub message: String,
39    pub history: Arc<VecDeque<Segment>>,
40    pub timed: bool,
41    pub timeline: Option<(u64, String)>,
42    pub current: Vec<usize>,
43    pub anchor: Option<usize>,
44}
45
46impl Default for Snapshot {
47    fn default() -> Self {
48        Self {
49            enabled: false,
50            failed: false,
51            revision: 0,
52            message: crate::t!("subtitle_off").into(),
53            history: Arc::default(),
54            timed: false,
55            timeline: None,
56            current: vec![],
57            anchor: None,
58        }
59    }
60}
61
62impl Snapshot {
63    fn timeline(&mut self, generation: u64, source: &str) {
64        if self
65            .timeline
66            .as_ref()
67            .is_none_or(|(old, id)| *old != generation || id != source)
68        {
69            self.timeline = Some((generation, source.into()));
70            self.revision += 1;
71        }
72    }
73    fn seek(&mut self, position: u64, index: &mut IntervalIndex) {
74        let end = self.history.partition_point(|s| s.start_ms <= position);
75        let current = index.query(&self.history, end, position);
76        let anchor = end.checked_sub(1);
77        if self.current != current || self.anchor != anchor {
78            self.current = current;
79            self.anchor = anchor;
80            self.revision += 1;
81        }
82    }
83}
84
85/// Max-end interval tree over cues sorted by start time. Long overlapping cues
86/// stay discoverable without scanning every earlier cue on each player tick.
87#[derive(Default)]
88struct IntervalIndex {
89    history: std::sync::Weak<VecDeque<Segment>>,
90    max_end: Vec<u64>,
91    leaves: usize,
92    len: usize,
93    #[cfg(test)]
94    visits: usize,
95    #[cfg(test)]
96    rebuilds: usize,
97}
98impl IntervalIndex {
99    fn query(
100        &mut self,
101        history: &Arc<VecDeque<Segment>>,
102        upper: usize,
103        position: u64,
104    ) -> Vec<usize> {
105        let key = Arc::downgrade(history);
106        if !std::sync::Weak::ptr_eq(&self.history, &key) || self.len != history.len() {
107            self.history = key;
108            self.len = history.len();
109            self.leaves = self.len.next_power_of_two();
110            self.max_end = vec![0; self.leaves * 2];
111            for (i, cue) in history.iter().enumerate() {
112                self.max_end[self.leaves + i] = cue.end_ms;
113            }
114            for i in (1..self.leaves).rev() {
115                self.max_end[i] = self.max_end[i * 2].max(self.max_end[i * 2 + 1]);
116            }
117            #[cfg(test)]
118            {
119                self.rebuilds += 1;
120            }
121        }
122        #[cfg(test)]
123        {
124            self.visits = 0;
125        }
126        let mut found = vec![];
127        if !history.is_empty() {
128            self.collect(1, 0, self.leaves, upper, position, &mut found);
129        }
130        found
131    }
132    fn collect(
133        &mut self,
134        node: usize,
135        from: usize,
136        to: usize,
137        upper: usize,
138        position: u64,
139        found: &mut Vec<usize>,
140    ) {
141        #[cfg(test)]
142        {
143            self.visits += 1;
144        }
145        if from >= upper || self.max_end[node] <= position {
146            return;
147        }
148        if to - from == 1 {
149            found.push(from);
150            return;
151        }
152        let mid = from + (to - from) / 2;
153        self.collect(node * 2, from, mid, upper, position, found);
154        self.collect(node * 2 + 1, mid, to, upper, position, found);
155    }
156}
157
158enum Notice {
159    Message(String),
160    Segment(Segment),
161}
162
163struct Task {
164    job: Background<bool>,
165    events: Receiver<Notice>,
166    first_segment_at: Option<Instant>,
167}
168
169#[derive(Clone, Copy, PartialEq, Eq)]
170enum StatusMessage {
171    Off,
172    Local,
173    Paused,
174    Waiting,
175    Loading,
176}
177
178impl StatusMessage {
179    fn text(self, locale: crate::messages::Locale) -> &'static str {
180        match self {
181            Self::Off => crate::messages::text_in(locale, "subtitle_off"),
182            Self::Local => crate::messages::text_in(locale, "subtitle_local"),
183            Self::Paused => crate::messages::text_in(locale, "subtitle_paused"),
184            Self::Waiting => crate::messages::text_in(locale, "subtitle_waiting"),
185            Self::Loading => crate::messages::text_in(locale, "subtitle_loading"),
186        }
187    }
188}
189
190struct EpisodeTask {
191    key: (u64, crate::podcast::EpisodeMedia),
192    job: Option<Background<Option<Vec<Segment>>>>,
193    incremental: bool,
194}
195
196#[derive(Default)]
197pub struct Controller {
198    pub snapshot: Snapshot,
199    task: Option<Task>,
200    episode: Option<EpisodeTask>,
201    retiring: Vec<Background<bool>>,
202    key: Option<(u64, String)>,
203    epoch: u64,
204    attempts: usize,
205    due: Option<Instant>,
206    status_key: Option<(StatusMessage, crate::messages::Locale)>,
207    verification: Option<Arc<Mutex<models::VerificationCache>>>,
208    intervals: IntervalIndex,
209}
210
211impl Controller {
212    fn message(&mut self, message: impl Into<Cow<'static, str>>) {
213        self.status_key = None;
214        let message = message.into();
215        if self.snapshot.message != message {
216            self.snapshot.message = message.into_owned();
217            self.snapshot.revision += 1;
218        }
219    }
220    fn status(&mut self, key: StatusMessage) {
221        let locale = crate::messages::locale();
222        if self.status_key != Some((key, locale)) {
223            self.message(key.text(locale));
224            self.status_key = Some((key, locale));
225        }
226    }
227    pub fn clear(&mut self) {
228        if let Some(episode) = &mut self.episode {
229            episode.incremental = false;
230        }
231        if !self.snapshot.history.is_empty() {
232            self.snapshot.history = Arc::default();
233            self.snapshot.current.clear();
234            self.snapshot.anchor = None;
235            self.snapshot.revision += 1;
236        }
237    }
238    pub fn retry(&mut self) {
239        self.cancel();
240        if self.snapshot.failed {
241            self.snapshot.failed = false;
242            self.snapshot.revision += 1;
243        }
244        self.attempts = 0;
245        self.due = None;
246    }
247    pub fn cancel(&mut self) {
248        self.episode = None;
249        self.epoch = self.epoch.wrapping_add(1);
250        if let Some(task) = self.task.take() {
251            self.retiring.push(task.job);
252        }
253        // Dropping Background requests cancellation, but retain retiring work
254        // until its resources are released so a new worker cannot overlap it.
255        for job in &self.retiring {
256            job.cancel();
257        }
258        self.due = None;
259    }
260    pub fn quiescent(&self) -> bool {
261        self.task.is_none() && self.retiring.is_empty()
262    }
263    pub fn poll_retiring(&mut self) {
264        self.retiring.retain(|task| task.poll().is_none());
265    }
266    pub fn active(&self) -> bool {
267        self.task.is_some()
268    }
269    pub fn enabled(&mut self, enabled: bool) {
270        if self.snapshot.enabled != enabled {
271            self.snapshot.enabled = enabled;
272            self.snapshot.revision += 1;
273            self.retry();
274        }
275    }
276    pub fn update_episode(
277        &mut self,
278        config: &Config,
279        generation: u64,
280        episode: &crate::podcast::Episode,
281        playing: bool,
282    ) {
283        self.poll_retiring();
284        if !self.snapshot.enabled {
285            self.status(StatusMessage::Off);
286            return;
287        }
288        self.snapshot.timeline(generation, &episode.id);
289        if !self.snapshot.timed {
290            self.cancel();
291            self.clear();
292            self.snapshot.timed = true;
293            self.snapshot.revision += 1;
294        }
295        let key = (generation, episode.media());
296        if self.episode.as_ref().is_none_or(|e| e.key != key) && playing {
297            self.clear();
298            self.snapshot.failed = false;
299            self.message(crate::t!("podcast_subtitle_loading"));
300            let config = config.clone();
301            let episode = episode.clone();
302            let job = Background::start("radio-episode-subtitles", move |stop| {
303                crate::podcast::transcript::load(&config, &episode, &stop)
304            });
305            match job {
306                Ok(job) => {
307                    self.episode = Some(EpisodeTask {
308                        key,
309                        job: Some(job),
310                        incremental: false,
311                    })
312                }
313                Err(e) => {
314                    self.snapshot.failed = true;
315                    self.message(format!("{e:#}"));
316                }
317            }
318        }
319        if let Some(result) = self
320            .episode
321            .as_ref()
322            .and_then(|e| e.job.as_ref())
323            .and_then(|j| j.poll())
324        {
325            self.episode.as_mut().unwrap().job = None;
326            match result {
327                Ok(Some(segments)) => {
328                    self.snapshot.history = Arc::new(segments.into());
329                    self.message(crate::t!("podcast_subtitle_synced"));
330                }
331                Ok(None) => self.message(crate::t!("podcast_subtitle_missing")),
332                Err(e) => {
333                    self.snapshot.failed = true;
334                    self.message(crate::t!("podcast_subtitle_failed", "detail" => crate::redact::urls(&format!("{e:#}"))));
335                }
336            }
337            self.snapshot.revision += 1;
338        }
339        self.snapshot.seek(episode.position_ms, &mut self.intervals);
340    }
341
342    pub(crate) fn publish_episode(
343        &mut self,
344        generation: u64,
345        episode: &crate::podcast::Episode,
346        update: crate::podcast::progressive::CaptionUpdate,
347    ) {
348        if !self.snapshot.enabled {
349            return;
350        }
351        self.snapshot.timeline(generation, &episode.id);
352        // Supersede a slow feed fetch so it cannot replace an incremental result.
353        self.episode = Some(EpisodeTask {
354            key: (generation, episode.media()),
355            job: None,
356            incremental: true,
357        });
358        self.snapshot.timed = true;
359        self.snapshot.failed = false;
360        if update.reset {
361            let mut segments = update.segments;
362            segments.sort_by_key(|s| (s.start_ms, s.end_ms));
363            self.snapshot.history = Arc::new(segments.into());
364        } else if !update.segments.is_empty() {
365            let history = Arc::make_mut(&mut self.snapshot.history);
366            for segment in update.segments {
367                let key = (segment.start_ms, segment.end_ms);
368                let at = history.partition_point(|s| (s.start_ms, s.end_ms) <= key);
369                history.insert(at, segment);
370            }
371        }
372        self.snapshot.revision += 1;
373        self.message(crate::t!("podcast_subtitle_generating"));
374        self.snapshot.seek(episode.position_ms, &mut self.intervals);
375    }
376
377    pub(crate) fn incremental_episode(
378        &self,
379        generation: u64,
380        episode: &crate::podcast::Episode,
381    ) -> bool {
382        self.episode
383            .as_ref()
384            .is_some_and(|task| task.key == (generation, episode.media()) && task.incremental)
385    }
386
387    pub fn update(
388        &mut self,
389        config: &Config,
390        key: (u64, &str),
391        playing: bool,
392        paused: bool,
393        local: bool,
394        stop: &Arc<AtomicBool>,
395    ) {
396        self.poll_retiring();
397        if playing {
398            self.snapshot.timeline(key.0, key.1);
399        }
400        if self.snapshot.timed {
401            self.cancel();
402            self.clear();
403            self.snapshot.timed = false;
404            self.snapshot.revision += 1;
405        }
406        if !self.snapshot.enabled {
407            if playing && self.key.as_ref().is_some_and(|old| old.0 != key.0) {
408                self.clear();
409                self.key = None;
410            }
411            self.status(StatusMessage::Off);
412            return;
413        }
414        if playing
415            && self
416                .key
417                .as_ref()
418                .map(|(generation, url)| (*generation, url.as_str()))
419                != Some(key)
420        {
421            // Station generation changes clear history; reconnects have their
422            // own epoch even when generation and URL remain the same.
423            if self.key.as_ref().is_some_and(|old| old.0 != key.0) {
424                self.clear();
425            }
426            self.retry();
427            self.key = Some((key.0, key.1.to_owned()));
428        }
429        if !playing || paused || local || stop.load(Ordering::Relaxed) {
430            self.cancel();
431            self.status(if local {
432                StatusMessage::Local
433            } else if paused {
434                StatusMessage::Paused
435            } else {
436                StatusMessage::Waiting
437            });
438            return;
439        }
440        // Poll completion before draining events: once a finished job is removed,
441        // no final segment may remain stranded in its event receiver.
442        let finished = self.task.as_ref().and_then(|t| t.job.poll());
443        if let Some(task) = &self.task {
444            let notices: Vec<_> = task.events.try_iter().collect();
445            for notice in notices {
446                match notice {
447                    Notice::Message(message) => self.message(message),
448                    Notice::Segment(seg) if seg.epoch == self.epoch => {
449                        let first = self
450                            .task
451                            .as_mut()
452                            .unwrap()
453                            .first_segment_at
454                            .get_or_insert_with(Instant::now);
455                        if first.elapsed() >= STABLE_SUBTITLE_PERIOD {
456                            self.attempts = 0;
457                        }
458                        let history = Arc::make_mut(&mut self.snapshot.history);
459                        history.push_back(seg);
460                        while history.len() > 500 {
461                            history.pop_front();
462                        }
463                        self.snapshot.revision += 1;
464                    }
465                    _ => {}
466                }
467            }
468        }
469        if let Some(result) = finished {
470            self.task = None;
471            let retry = match result {
472                Ok(retry) => retry,
473                Err(error) => {
474                    self.message(crate::t!("subtitle_failed", "detail" => crate::redact::urls(&format!("{error:#}"))));
475                    false
476                }
477            };
478            if retry && self.attempts < 3 {
479                let seconds = [1, 3, 7][self.attempts];
480                self.attempts += 1;
481                self.due = Some(Instant::now() + Duration::from_secs(seconds));
482                self.message(crate::t!("subtitle_retrying", "attempt" => self.attempts));
483            } else {
484                self.snapshot.failed = true;
485                self.snapshot.revision += 1;
486                if retry {
487                    self.message(
488                        crate::t!("subtitle_failed", "detail" => crate::t!("subtitle_connection")),
489                    );
490                }
491            }
492        }
493        if self.task.is_none()
494            && self.retiring.is_empty()
495            && !self.snapshot.failed
496            && self.due.is_none_or(|d| Instant::now() >= d)
497        {
498            self.due = None;
499            self.epoch = self.epoch.wrapping_add(1);
500            self.status(StatusMessage::Loading);
501            let (sender, events) = mpsc::sync_channel(64);
502            let config = config.clone();
503            let url = self.key.as_ref().unwrap().1.clone();
504            let epoch = self.epoch;
505            let verification = self
506                .verification
507                .get_or_insert_with(Default::default)
508                .clone();
509            match Background::start("radio-speech", move |cancel| {
510                run(&config, &url, epoch, &cancel, sender, &verification)
511            }) {
512                Ok(job) => {
513                    self.task = Some(Task {
514                        job,
515                        events,
516                        first_segment_at: None,
517                    })
518                }
519                Err(error) => {
520                    self.snapshot.failed = true;
521                    self.snapshot.revision += 1;
522                    self.message(format!("{error:#}"));
523                }
524            }
525        }
526    }
527}
528
529struct OwnedChildren {
530    worker: Child,
531    ffmpeg: Option<Child>,
532    _lock: models::ResourceLock,
533}
534impl Drop for OwnedChildren {
535    fn drop(&mut self) {
536        if let Some(child) = &mut self.ffmpeg {
537            let _ = child.kill();
538            let _ = child.wait();
539        }
540        let _ = self.worker.kill();
541        let _ = self.worker.wait();
542    }
543}
544
545#[derive(Deserialize)]
546struct Event {
547    v: u8,
548    epoch: u64,
549    #[serde(rename = "type")]
550    kind: String,
551    #[serde(default)]
552    start_ms: u64,
553    #[serde(default)]
554    end_ms: u64,
555    #[serde(default)]
556    lag_ms: u64,
557    #[serde(default)]
558    text: String,
559}
560
561fn send(sender: &SyncSender<Notice>, notice: Notice) {
562    let _ = sender.try_send(notice);
563}
564
565fn run(
566    config: &Config,
567    url: &str,
568    epoch: u64,
569    cancel: &Arc<AtomicBool>,
570    sender: SyncSender<Notice>,
571    verification: &Mutex<models::VerificationCache>,
572) -> Result<bool> {
573    let lock = models::lock(config, false)?;
574    models::runtime_check(config, cancel)?;
575    verification
576        .lock()
577        .unwrap()
578        .verify(config, &config.settings.subtitles.model, cancel)?;
579    ensure!(
580        !cancel.load(Ordering::Relaxed),
581        DomainError::new(ErrorCode::Cancelled)
582    );
583    let command = worker_command(config, epoch, false);
584    run_processes(config, url, epoch, cancel, sender, command, lock)
585}
586
587fn worker_command(config: &Config, epoch: u64, offline: bool) -> Command {
588    worker_command_mode(config, epoch, offline, false)
589}
590
591pub(super) fn worker_command_mode(
592    config: &Config,
593    epoch: u64,
594    offline: bool,
595    chunks: bool,
596) -> Command {
597    let settings = &config.settings.subtitles;
598    let args = serde_json::json!({"epoch":epoch,"offline":offline,"chunks":chunks,"model":settings.model,"model_dir":models::model_dir(config, &settings.model),"vad":models::model_dir(config,"silero-vad").join("silero_vad.onnx"),"language":settings.language,"threads":settings.effective_threads(),"max_segment_secs":settings.max_segment_secs.min(if chunks { 8 } else { 20 })});
599    let mut command = Command::new(models::python(config));
600    command
601        .args([
602            "-I",
603            "-u",
604            "-c",
605            include_str!("worker.py"),
606            &args.to_string(),
607        ])
608        .stdin(Stdio::piped())
609        .stdout(Stdio::piped())
610        .stderr(Stdio::piped());
611    command
612}
613
614pub(crate) fn transcribe(
615    config: &Config,
616    file: &std::path::Path,
617    stop: &Arc<AtomicBool>,
618    progress: Option<&AtomicU64>,
619) -> Result<Vec<Segment>> {
620    let lock = models::lock(config, false)?;
621    models::runtime_check(config, stop)?;
622    models::VerificationCache::default().verify(config, &config.settings.subtitles.model, stop)?;
623    let command = worker_command(config, 0, true);
624    let (sender, _events) = mpsc::sync_channel(1);
625    let mut segments = Vec::new();
626    run_processes_inner(
627        config,
628        (
629            file.to_str()
630                .ok_or_else(|| anyhow::anyhow!("invalid audio path"))?,
631            0,
632        ),
633        stop,
634        sender,
635        command,
636        lock,
637        Some(FileOutput {
638            segments: &mut segments,
639            progress,
640        }),
641    )?;
642    ensure!(
643        !stop.load(Ordering::Relaxed),
644        DomainError::new(ErrorCode::Cancelled)
645    );
646    Ok(segments)
647}
648
649fn run_processes(
650    config: &Config,
651    url: &str,
652    epoch: u64,
653    cancel: &Arc<AtomicBool>,
654    sender: SyncSender<Notice>,
655    command: Command,
656    lock: models::ResourceLock,
657) -> Result<bool> {
658    run_processes_inner(config, (url, epoch), cancel, sender, command, lock, None)
659}
660
661struct FileOutput<'a> {
662    segments: &'a mut Vec<Segment>,
663    progress: Option<&'a AtomicU64>,
664}
665
666fn run_processes_inner(
667    config: &Config,
668    input: (&str, u64),
669    cancel: &Arc<AtomicBool>,
670    sender: SyncSender<Notice>,
671    command: Command,
672    lock: models::ResourceLock,
673    mut output: Option<FileOutput<'_>>,
674) -> Result<bool> {
675    let (url, epoch) = input;
676    let offline = output.is_some();
677    let worker = crate::process::spawn_owned(command)?;
678    let mut children = OwnedChildren {
679        worker,
680        ffmpeg: None,
681        _lock: lock,
682    };
683    let stderr = crate::process::drain(children.worker.stderr.take().unwrap(), 16384);
684    let stdout = children.worker.stdout.take().unwrap();
685    let (events, receiver) = mpsc::sync_channel(64);
686    let reader = thread::spawn(move || {
687        let mut reader = BufReader::new(stdout);
688        loop {
689            let mut line = Vec::new();
690            let n = reader.by_ref().take(65537).read_until(b'\n', &mut line);
691            match n {
692                Ok(0) => break,
693                Ok(n) if n <= 65536 => {
694                    let parsed = serde_json::from_slice::<Event>(&line).map_err(|e| e.to_string());
695                    if events.send(parsed).is_err() {
696                        break;
697                    }
698                }
699                _ => {
700                    let _ = events.send(Err("invalid speech protocol line".into()));
701                    break;
702                }
703            }
704        }
705    });
706    let started = Instant::now();
707    let mut ready = false;
708    let mut decoding = None::<Instant>;
709    let mut pump = None;
710    let mut ffmpeg_stderr = None;
711    let bytes = Arc::new(AtomicU64::new(0));
712    let mut last_bytes = 0;
713    let mut last_audio = Instant::now();
714    let mut last_event = Instant::now();
715    let mut reported_ms = 0;
716    let result = (|| -> Result<bool> {
717        loop {
718            if cancel.load(Ordering::Relaxed) {
719                return Ok(false);
720            }
721            ensure!(
722                ready || started.elapsed() < Duration::from_secs(90),
723                "speech model loading timed out"
724            );
725            ensure!(
726                decoding.is_none_or(|t| t.elapsed() < Duration::from_secs(60)),
727                "speech inference timed out"
728            );
729            let incoming = match receiver.recv_timeout(Duration::from_millis(25)) {
730                Ok(event) => Some(event),
731                Err(mpsc::RecvTimeoutError::Timeout) => None,
732                Err(mpsc::RecvTimeoutError::Disconnected) => {
733                    bail!("speech worker closed its output; run radio subtitles doctor")
734                }
735            };
736            if let Some(event) = incoming {
737                last_event = Instant::now();
738                let event = event.map_err(anyhow::Error::msg)?;
739                ensure!(
740                    event.v == 1 && event.epoch == epoch,
741                    "speech protocol version or epoch mismatch"
742                );
743                match event.kind.as_str() {
744                    "ready" if !ready => {
745                        ready = true;
746                        send(
747                            &sender,
748                            Notice::Message(crate::t!("subtitle_listening").into()),
749                        );
750                        let mut command = Command::new(&config.settings.ffmpeg);
751                        command.args(["-hide_banner", "-nostdin", "-loglevel", "error"]);
752                        if !offline {
753                            command.args([
754                                "-rw_timeout",
755                                &(config.settings.request_timeout_secs * 1_000_000).to_string(),
756                                "-user_agent",
757                                crate::USER_AGENT,
758                                "-readrate",
759                                "1",
760                            ]);
761                        }
762                        command
763                            .args([
764                                "-i", url, "-map", "0:a:0", "-vn", "-ac", "1", "-ar", "16000",
765                                "-f", "f32le", "pipe:1",
766                            ])
767                            .stdin(Stdio::null())
768                            .stdout(Stdio::piped())
769                            .stderr(Stdio::piped());
770                        let child = crate::process::spawn_owned(command)?;
771                        children.ffmpeg = Some(child);
772                        let ffmpeg = children.ffmpeg.as_mut().unwrap();
773                        ffmpeg_stderr =
774                            Some(crate::process::drain(ffmpeg.stderr.take().unwrap(), 16384));
775                        let mut input = ffmpeg.stdout.take().unwrap();
776                        let mut output = children.worker.stdin.take().unwrap();
777                        let bytes = bytes.clone();
778                        pump = Some(thread::spawn(move || {
779                            use std::io::Write;
780                            let mut buf = [0; 8192];
781                            while let Ok(n) = input.read(&mut buf) {
782                                if n == 0 || output.write_all(&buf[..n]).is_err() {
783                                    break;
784                                }
785                                bytes.fetch_add(n as u64, Ordering::Relaxed);
786                            }
787                        }));
788                        last_audio = Instant::now();
789                    }
790                    "decoding" => {
791                        decoding = Some(Instant::now());
792                        send(
793                            &sender,
794                            Notice::Message(crate::t!("subtitle_decoding").into()),
795                        );
796                    }
797                    "listening" => {
798                        decoding = None;
799                        send(
800                            &sender,
801                            Notice::Message(crate::t!("subtitle_listening").into()),
802                        );
803                    }
804                    "segment" => {
805                        ensure!(
806                            event.end_ms >= event.start_ms
807                                && event.end_ms - event.start_ms <= 20000,
808                            "invalid speech segment time range"
809                        );
810                        if event.lag_ms <= 30000 {
811                            let text = crate::clean_text(&event.text);
812                            if !text.is_empty() {
813                                let segment = Segment {
814                                    epoch,
815                                    start_ms: event.start_ms,
816                                    end_ms: event.end_ms,
817                                    text,
818                                };
819                                if let Some(file) = &mut output {
820                                    ensure!(
821                                        file.segments.len() < 100_000,
822                                        "too many speech segments"
823                                    );
824                                    if let Some(progress) = file.progress {
825                                        progress.store(segment.end_ms, Ordering::Relaxed);
826                                    } else if segment.end_ms.saturating_sub(reported_ms) >= 30000 {
827                                        reported_ms = segment.end_ms;
828                                        eprintln!(
829                                            "{}",
830                                            crate::t!("podcast_subtitle_progress", "seconds" => reported_ms / 1000)
831                                        );
832                                    }
833                                    file.segments.push(segment);
834                                } else {
835                                    send(&sender, Notice::Segment(segment));
836                                }
837                            }
838                        }
839                    }
840                    "gap" => send(&sender, Notice::Message(crate::t!("subtitle_gap").into())),
841                    "done" => {
842                        ensure!(ready, "speech worker finished before ready");
843                        if offline {
844                            let timeout = Duration::from_secs(5);
845                            ensure!(
846                                crate::process::wait_until(&mut children.worker, timeout, cancel)?
847                                    .success(),
848                                "speech worker failed"
849                            );
850                            ensure!(
851                                crate::process::wait_until(
852                                    children.ffmpeg.as_mut().unwrap(),
853                                    timeout,
854                                    cancel
855                                )?
856                                .success(),
857                                "audio decoding failed"
858                            );
859                        }
860                        return Ok(true);
861                    }
862                    _ => bail!("unexpected speech protocol event"),
863                }
864            }
865            let now_bytes = bytes.load(Ordering::Relaxed);
866            if now_bytes != last_bytes {
867                last_bytes = now_bytes;
868                last_audio = Instant::now();
869            }
870            if ready
871                && !(offline && decoding.is_some())
872                && if offline {
873                    last_audio.max(last_event)
874                } else {
875                    last_audio
876                }
877                .elapsed()
878                    >= Duration::from_secs(config.settings.request_timeout_secs)
879            {
880                ensure!(!offline, "audio transcription stalled before completion");
881                return Ok(true);
882            }
883        }
884    })();
885    drop(children); // Close pipes before joining their readers/writers.
886    drop(receiver);
887    let _ = reader.join();
888    if let Some(pump) = pump {
889        let _ = pump.join();
890    }
891    let diagnostics = stderr.join().unwrap_or_default();
892    let mut diagnostics = diagnostics;
893    if let Some(stderr) = ffmpeg_stderr {
894        diagnostics.extend(stderr.join().unwrap_or_default());
895    }
896    if !diagnostics.is_empty() {
897        return result.map_err(|e| {
898            e.context(crate::clean_text(&crate::redact::urls(
899                &String::from_utf8_lossy(&diagnostics),
900            )))
901        });
902    }
903    result
904}
905
906#[cfg(test)]
907mod tests {
908    use super::*;
909    use crate::test_support::{Server, response};
910
911    #[test]
912    fn incremental_captions_supersede_a_pending_fetch_and_stay_on_media_time() {
913        let dir = tempfile::tempdir().unwrap();
914        let config = crate::test_support::config(dir.path());
915        let episode = crate::podcast::Episode {
916            id: "episode A".into(),
917            position_ms: 61_500,
918            ..Default::default()
919        };
920        let mut controller = Controller::default();
921        controller.enabled(true);
922        controller.episode = Some(EpisodeTask {
923            key: (7, episode.media()),
924            incremental: false,
925            job: Some(
926                Background::start("old-caption-fetch", |_| {
927                    Ok(Some(vec![Segment {
928                        epoch: 0,
929                        start_ms: 0,
930                        end_ms: 1000,
931                        text: "old fetch".into(),
932                    }]))
933                })
934                .unwrap(),
935            ),
936        });
937        controller.publish_episode(
938            7,
939            &episode,
940            crate::podcast::progressive::CaptionUpdate {
941                reset: true,
942                segments: vec![Segment {
943                    epoch: 0,
944                    start_ms: 61_000,
945                    end_ms: 62_000,
946                    text: "incremental".into(),
947                }],
948            },
949        );
950        controller.update_episode(&config, 7, &episode, true);
951        assert_eq!(controller.snapshot.history[0].text, "incremental");
952        assert_eq!(controller.snapshot.current, [0]);
953        assert!(controller.incremental_episode(7, &episode));
954        let previous = controller.snapshot.history.clone();
955        controller.publish_episode(
956            7,
957            &episode,
958            crate::podcast::progressive::CaptionUpdate {
959                reset: false,
960                segments: vec![Segment {
961                    epoch: 0,
962                    start_ms: 60_000,
963                    end_ms: 62_000,
964                    text: "overlapping earlier cue".into(),
965                }],
966            },
967        );
968        assert_eq!(previous.len(), 1, "published snapshots remain immutable");
969        assert_eq!(
970            controller.snapshot.history[0].text,
971            "overlapping earlier cue"
972        );
973        assert_eq!(controller.snapshot.current, [0, 1]);
974        let mut rewind = episode.clone();
975        rewind.position_ms = 60_000;
976        controller.update_episode(&config, 7, &rewind, true);
977        assert_eq!(controller.snapshot.current, [0]);
978        assert_eq!(controller.snapshot.timeline, Some((7, episode.id.clone())));
979        controller.enabled(false);
980        assert!(!controller.incremental_episode(7, &episode));
981        controller.publish_episode(
982            7,
983            &episode,
984            crate::podcast::progressive::CaptionUpdate {
985                reset: false,
986                segments: vec![],
987            },
988        );
989        assert!(!controller.snapshot.enabled);
990        assert_eq!(
991            controller.snapshot.history[0].text,
992            "overlapping earlier cue"
993        );
994        controller.enabled(true);
995        controller.update_episode(&config, 8, &episode, false);
996        assert_eq!(controller.snapshot.timeline, Some((8, episode.id.clone())));
997        let mut next = episode;
998        next.id = "episode B".into();
999        controller.update_episode(&config, 8, &next, false);
1000        assert_eq!(controller.snapshot.timeline, Some((8, next.id)));
1001    }
1002
1003    #[test]
1004    fn interval_queries_skip_expired_history_and_keep_long_overlapping_cues() {
1005        let mut snapshot = Snapshot {
1006            history: Arc::new(
1007                (0..100_000)
1008                    .map(|i| Segment {
1009                        epoch: 0,
1010                        start_ms: i * 1000,
1011                        end_ms: if i == 0 { 100_000_000 } else { i * 1000 + 500 },
1012                        text: String::new(),
1013                    })
1014                    .collect(),
1015            ),
1016            ..Default::default()
1017        };
1018        let mut index = IntervalIndex::default();
1019        snapshot.seek(99_999_250, &mut index);
1020        assert_eq!(snapshot.current, [0, 99_999]);
1021        assert!(index.visits < 100, "visited {} nodes", index.visits);
1022        for position in [0, 999, 1500, 99_999_250, 100_000_000, 42_250] {
1023            snapshot.seek(position, &mut index);
1024            let expected: Vec<_> = snapshot
1025                .history
1026                .iter()
1027                .enumerate()
1028                .filter_map(|(i, s)| (s.start_ms <= position && position < s.end_ms).then_some(i))
1029                .collect();
1030            assert_eq!(snapshot.current, expected);
1031        }
1032        assert_eq!(
1033            index.rebuilds, 1,
1034            "reuse the index across pause, seek and speed ticks"
1035        );
1036        let last = Arc::make_mut(&mut snapshot.history).back_mut().unwrap();
1037        last.end_ms = 100_001_000;
1038        snapshot.seek(100_000_500, &mut index);
1039        assert_eq!(snapshot.current, [99_999]);
1040        assert_eq!(index.rebuilds, 2);
1041    }
1042
1043    #[test]
1044    fn timed_cues_follow_media_position_through_gaps_overlaps_and_rewind() {
1045        let mut index = IntervalIndex::default();
1046        let mut snapshot = Snapshot {
1047            timed: true,
1048            history: Arc::new(
1049                [
1050                    Segment {
1051                        epoch: 0,
1052                        start_ms: 1000,
1053                        end_ms: 3000,
1054                        text: "one".into(),
1055                    },
1056                    Segment {
1057                        epoch: 0,
1058                        start_ms: 2000,
1059                        end_ms: 2500,
1060                        text: "two".into(),
1061                    },
1062                    Segment {
1063                        epoch: 0,
1064                        start_ms: 4000,
1065                        end_ms: 5000,
1066                        text: "three".into(),
1067                    },
1068                ]
1069                .into(),
1070            ),
1071            ..Default::default()
1072        };
1073        for (position, expected) in [
1074            (0, vec![]),
1075            (1000, vec![0]),
1076            (2000, vec![0, 1]),
1077            (2500, vec![0]),
1078            (3000, vec![]),
1079            (4900, vec![2]),
1080            (6000, vec![]),
1081            (1100, vec![0]),
1082        ] {
1083            snapshot.seek(position, &mut index);
1084            assert_eq!(snapshot.current, expected, "{position}");
1085        }
1086        let revision = snapshot.revision;
1087        snapshot.seek(1200, &mut index);
1088        assert_eq!(
1089            snapshot.revision, revision,
1090            "no redraw until a cue boundary"
1091        );
1092    }
1093
1094    #[test]
1095    fn file_transcription_waits_for_done_after_slow_inference() {
1096        let dir = tempfile::tempdir().unwrap();
1097        let mut config = crate::test_support::config(dir.path());
1098        config.settings.request_timeout_secs = 1;
1099        let path = dir.path().join("audio.wav");
1100        std::fs::write(&path, crate::test_support::fixture("pcm_s16le", "wav")).unwrap();
1101        let command = fake_command(
1102            "printf '%s\\n' '{\"v\":1,\"epoch\":0,\"type\":\"ready\"}'; cat >/dev/null; printf '%s\\n' '{\"v\":1,\"epoch\":0,\"type\":\"decoding\"}'; sleep 2; printf '%s\\n' '{\"v\":1,\"epoch\":0,\"type\":\"listening\"}' '{\"v\":1,\"epoch\":0,\"type\":\"segment\",\"start_ms\":0,\"end_ms\":1000,\"text\":\"final caption\"}' '{\"v\":1,\"epoch\":0,\"type\":\"done\"}'",
1103        );
1104        let (sender, _events) = mpsc::sync_channel(1);
1105        let mut segments = vec![];
1106        let result = run_processes_inner(
1107            &config,
1108            (path.to_str().unwrap(), 0),
1109            &Arc::new(AtomicBool::new(false)),
1110            sender,
1111            command,
1112            models::lock(&config, false).unwrap(),
1113            Some(FileOutput {
1114                segments: &mut segments,
1115                progress: None,
1116            }),
1117        );
1118        assert!(result.unwrap());
1119        assert_eq!(
1120            segments.len(),
1121            1,
1122            "do not succeed before the final segment and done"
1123        );
1124        assert_eq!(segments[0].text, "final caption");
1125    }
1126
1127    #[test]
1128    fn file_transcription_rejects_stalls_missing_done_and_failed_workers() {
1129        let dir = tempfile::tempdir().unwrap();
1130        let mut config = crate::test_support::config(dir.path());
1131        config.settings.request_timeout_secs = 1;
1132        let path = dir.path().join("audio.wav");
1133        std::fs::write(&path, crate::test_support::fixture("pcm_s16le", "wav")).unwrap();
1134        for (tail, expected) in [
1135            ("sleep 2", "stalled before completion"),
1136            ("exit 0", "closed its output"),
1137            (
1138                "printf '%s\\n' '{\"v\":1,\"epoch\":0,\"type\":\"done\"}'; exit 1",
1139                "speech worker failed",
1140            ),
1141        ] {
1142            let script = format!(
1143                "printf '%s\\n' '{{\"v\":1,\"epoch\":0,\"type\":\"ready\"}}'; cat >/dev/null; printf '%s\\n' '{{\"v\":1,\"epoch\":0,\"type\":\"segment\",\"start_ms\":0,\"end_ms\":1000,\"text\":\"partial\"}}'; {tail}"
1144            );
1145            let (sender, _events) = mpsc::sync_channel(1);
1146            let mut segments = vec![];
1147            let result = run_processes_inner(
1148                &config,
1149                (path.to_str().unwrap(), 0),
1150                &Arc::new(AtomicBool::new(false)),
1151                sender,
1152                fake_command(&script),
1153                models::lock(&config, false).unwrap(),
1154                Some(FileOutput {
1155                    segments: &mut segments,
1156                    progress: None,
1157                }),
1158            );
1159            assert_eq!(segments.len(), 1);
1160            assert!(
1161                format!("{:#}", result.unwrap_err()).contains(expected),
1162                "{tail}"
1163            );
1164            assert!(models::lock(&config, true).is_ok());
1165        }
1166    }
1167
1168    #[test]
1169    fn file_transcription_collects_every_segment_and_checks_decoder_exit() {
1170        let dir = tempfile::tempdir().unwrap();
1171        let config = crate::test_support::config(dir.path());
1172        let path = dir.path().join("audio.wav");
1173        std::fs::write(&path, crate::test_support::fixture("pcm_s16le", "wav")).unwrap();
1174        for valid in [true, false] {
1175            let (sender, _events) = mpsc::sync_channel(1);
1176            let command = fake_command(
1177                "printf '%s\\n' '{\"v\":1,\"epoch\":0,\"type\":\"ready\"}'; cat >/dev/null; i=0; while [ $i -lt 80 ]; do printf '%s\\n' '{\"v\":1,\"epoch\":0,\"type\":\"segment\",\"start_ms\":0,\"end_ms\":1000,\"text\":\"caption\"}'; i=$((i+1)); done; printf '%s\\n' '{\"v\":1,\"epoch\":0,\"type\":\"done\"}'",
1178            );
1179            let mut segments = vec![];
1180            let input = if valid {
1181                path.to_str().unwrap()
1182            } else {
1183                "/does/not/exist.wav"
1184            };
1185            let result = run_processes_inner(
1186                &config,
1187                (input, 0),
1188                &Arc::new(AtomicBool::new(false)),
1189                sender,
1190                command,
1191                models::lock(&config, false).unwrap(),
1192                Some(FileOutput {
1193                    segments: &mut segments,
1194                    progress: None,
1195                }),
1196            );
1197            assert_eq!(result.is_ok(), valid, "{result:?}");
1198            if valid {
1199                assert_eq!(segments.len(), 80);
1200            }
1201            assert!(models::lock(&config, true).is_ok());
1202        }
1203    }
1204
1205    #[test]
1206    #[ignore = "requires explicitly installed models and RADIO_TEST_SPEECH_DATA/WAV"]
1207    fn real_models_transcribe_a_local_http_fixture() {
1208        let data =
1209            std::env::var_os("RADIO_TEST_SPEECH_DATA").expect("isolated speech data directory");
1210        let wav = std::fs::read(
1211            std::env::var_os("RADIO_TEST_SPEECH_WAV").expect("local speech WAV fixture"),
1212        )
1213        .unwrap();
1214        let server = Server::new(move |_, stream| response(stream, 200, "audio/wav", &wav));
1215        let dir = tempfile::tempdir().unwrap();
1216        let mut config = crate::test_support::config(dir.path());
1217        config.data_dir = data.into();
1218        for model in [
1219            "sensevoice-small-int8",
1220            "sensevoice-small-yue-int8",
1221            "qwen3-asr-0.6b-int8",
1222        ] {
1223            config.settings.subtitles.model = model.into();
1224            let (sender, events) = mpsc::sync_channel(64);
1225            let start = Instant::now();
1226            assert!(
1227                run(
1228                    &config,
1229                    &server.url,
1230                    1,
1231                    &Arc::new(AtomicBool::new(false)),
1232                    sender,
1233                    &Mutex::default()
1234                )
1235                .unwrap()
1236            );
1237            let mut count = 0;
1238            for notice in events.try_iter() {
1239                if let Notice::Segment(segment) = notice {
1240                    eprintln!(
1241                        "{model} {}-{} ms: {}",
1242                        segment.start_ms, segment.end_ms, segment.text
1243                    );
1244                    count += 1;
1245                }
1246            }
1247            assert!(count > 0, "{model} produced no transcription");
1248            eprintln!(
1249                "{model}: {count} segments, {:.2}s including realtime audio",
1250                start.elapsed().as_secs_f32()
1251            );
1252        }
1253    }
1254
1255    fn fake_command(script: &str) -> Command {
1256        let mut command = Command::new("sh");
1257        command
1258            .args(["-c", script])
1259            .stdin(Stdio::piped())
1260            .stdout(Stdio::piped())
1261            .stderr(Stdio::piped());
1262        command
1263    }
1264
1265    #[test]
1266    fn pcm_reaches_worker_and_protocol_segments_are_sanitized() {
1267        let audio = crate::test_support::fixture("pcm_s16le", "wav");
1268        let server = Server::new(move |_, stream| response(stream, 200, "audio/wav", &audio));
1269        let dir = tempfile::tempdir().unwrap();
1270        let config = crate::test_support::config(dir.path());
1271        let (sender, events) = mpsc::sync_channel(64);
1272        let command = fake_command(
1273            "printf '%s\n' '{\"v\":1,\"epoch\":7,\"type\":\"ready\"}'; head -c 4096 >/dev/null; printf '%s\n' '{\"v\":1,\"epoch\":7,\"type\":\"segment\",\"start_ms\":0,\"end_ms\":64,\"text\":\"hello\\u001b world\"}' '{\"v\":1,\"epoch\":7,\"type\":\"done\"}'; cat >/dev/null",
1274        );
1275        assert!(
1276            run_processes(
1277                &config,
1278                &server.url,
1279                7,
1280                &Arc::new(AtomicBool::new(false)),
1281                sender,
1282                command,
1283                models::lock(&config, false).unwrap()
1284            )
1285            .unwrap()
1286        );
1287        let segments: Vec<_> = events
1288            .try_iter()
1289            .filter_map(|n| {
1290                if let Notice::Segment(s) = n {
1291                    Some(s)
1292                } else {
1293                    None
1294                }
1295            })
1296            .collect();
1297        assert_eq!(segments.len(), 1);
1298        assert_eq!(segments[0].text, "hello world");
1299        assert!(models::lock(&config, true).is_ok());
1300    }
1301
1302    #[test]
1303    fn hls_and_icy_sources_feed_the_same_pcm_worker_protocol() {
1304        use std::io::Write;
1305        let source = tempfile::tempdir().unwrap();
1306        assert!(
1307            Command::new("ffmpeg")
1308                .args([
1309                    "-v",
1310                    "error",
1311                    "-f",
1312                    "lavfi",
1313                    "-i",
1314                    "sine=frequency=440:duration=4",
1315                    "-c:a",
1316                    "aac",
1317                    "-f",
1318                    "hls",
1319                    "-hls_time",
1320                    "1",
1321                    "-hls_list_size",
1322                    "0"
1323                ])
1324                .arg(source.path().join("index.m3u8"))
1325                .status()
1326                .unwrap()
1327                .success()
1328        );
1329        let root = source.path().to_path_buf();
1330        let hls = Server::new(move |request, stream| {
1331            let path = request
1332                .split_whitespace()
1333                .nth(1)
1334                .unwrap()
1335                .trim_start_matches('/');
1336            if path.contains("..") {
1337                response(stream, 404, "text/plain", b"");
1338                return;
1339            }
1340            let data = std::fs::read(root.join(path)).unwrap();
1341            response(
1342                stream,
1343                200,
1344                if path.ends_with("m3u8") {
1345                    "application/vnd.apple.mpegurl"
1346                } else {
1347                    "video/mp2t"
1348                },
1349                &data,
1350            );
1351        });
1352        let mp3 = crate::test_support::fixture("libmp3lame", "mp3");
1353        let icy = Server::new(move |request, stream| {
1354            assert!(request.to_ascii_lowercase().contains("icy-metadata: 1"));
1355            if stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Type: audio/mpeg\r\nicy-metaint: 1024\r\nConnection: close\r\n\r\n").is_err() { return; }
1356            for chunk in mp3.chunks(1024) {
1357                if stream.write_all(chunk).is_err() {
1358                    break;
1359                }
1360                if chunk.len() == 1024 && stream.write_all(&[0]).is_err() {
1361                    break;
1362                }
1363            }
1364        });
1365        for url in [format!("{}/index.m3u8", hls.url), icy.url.clone()] {
1366            let dir = tempfile::tempdir().unwrap();
1367            let config = crate::test_support::config(dir.path());
1368            let (sender, events) = mpsc::sync_channel(64);
1369            let command = fake_command(
1370                "printf '%s\n' '{\"v\":1,\"epoch\":7,\"type\":\"ready\"}'; head -c 4096 >/dev/null; printf '%s\n' '{\"v\":1,\"epoch\":7,\"type\":\"segment\",\"start_ms\":0,\"end_ms\":64,\"text\":\"audio received\"}' '{\"v\":1,\"epoch\":7,\"type\":\"done\"}'; cat >/dev/null",
1371            );
1372            run_processes(
1373                &config,
1374                &url,
1375                7,
1376                &Arc::new(AtomicBool::new(false)),
1377                sender,
1378                command,
1379                models::lock(&config, false).unwrap(),
1380            )
1381            .unwrap();
1382            assert!(
1383                events.try_iter().any(|n| matches!(n, Notice::Segment(_))),
1384                "{url}"
1385            );
1386        }
1387    }
1388
1389    #[test]
1390    fn wrong_epoch_and_oversized_protocol_are_rejected_and_reaped() {
1391        for script in [
1392            "printf '%s\n' '{\"v\":1,\"epoch\":8,\"type\":\"ready\"}'",
1393            "head -c 70000 /dev/zero | tr '\\000' x",
1394        ] {
1395            let dir = tempfile::tempdir().unwrap();
1396            let config = crate::test_support::config(dir.path());
1397            let (sender, _) = mpsc::sync_channel(64);
1398            assert!(
1399                run_processes(
1400                    &config,
1401                    "http://127.0.0.1:1",
1402                    7,
1403                    &Arc::new(AtomicBool::new(false)),
1404                    sender,
1405                    fake_command(script),
1406                    models::lock(&config, false).unwrap()
1407                )
1408                .is_err()
1409            );
1410            assert!(models::lock(&config, true).is_ok());
1411        }
1412    }
1413
1414    #[test]
1415    fn cancellation_reaps_worker_while_it_waits_for_audio() {
1416        let dir = tempfile::tempdir().unwrap();
1417        let config = crate::test_support::config(dir.path());
1418        let cancel = Arc::new(AtomicBool::new(false));
1419        let flag = cancel.clone();
1420        let (sender, events) = mpsc::sync_channel(64);
1421        let (done_tx, done) = mpsc::sync_channel(1);
1422        let command = fake_command(
1423            "printf '%s\n' '{\"v\":1,\"epoch\":7,\"type\":\"ready\"}'; exec cat >/dev/null",
1424        );
1425        thread::spawn(move || {
1426            let lock = models::lock(&config, false).unwrap();
1427            done_tx
1428                .send(run_processes(
1429                    &config,
1430                    "http://127.0.0.1:1",
1431                    7,
1432                    &flag,
1433                    sender,
1434                    command,
1435                    lock,
1436                ))
1437                .unwrap();
1438        });
1439        events.recv_timeout(Duration::from_secs(5)).unwrap();
1440        cancel.store(true, Ordering::Relaxed);
1441        assert!(!done.recv_timeout(Duration::from_secs(2)).unwrap().unwrap());
1442    }
1443
1444    #[test]
1445    fn cancellation_kills_and_reaps_a_busy_decoder_without_waiting_for_timeout() {
1446        let dir = tempfile::tempdir().unwrap();
1447        let config = crate::test_support::config(dir.path());
1448        let cancel = Arc::new(AtomicBool::new(false));
1449        let flag = cancel.clone();
1450        let (sender, events) = mpsc::sync_channel(64);
1451        let (done_tx, done) = mpsc::sync_channel(1);
1452        let pid_path = dir.path().join("decoder.pid");
1453        // This worker stays CPU busy and does not read stdin. SIGTERM is ignored;
1454        // the parent must explicitly kill and reap it, on Linux and macOS alike.
1455        let mut command = fake_command(
1456            "printf '%s' $$ > \"$1\"; trap '' TERM; printf '%s\n' '{\"v\":1,\"epoch\":7,\"type\":\"decoding\"}'; while :; do :; done",
1457        );
1458        command.arg("decoder").arg(&pid_path);
1459        let handle = thread::spawn(move || {
1460            let lock = models::lock(&config, false).unwrap();
1461            done_tx
1462                .send(run_processes(
1463                    &config,
1464                    "http://127.0.0.1:1",
1465                    7,
1466                    &flag,
1467                    sender,
1468                    command,
1469                    lock,
1470                ))
1471                .unwrap();
1472        });
1473        // Wait for inference to begin, rather than cancelling during startup.
1474        assert!(
1475            matches!(events.recv_timeout(Duration::from_secs(5)).unwrap(), Notice::Message(message) if message == crate::t!("subtitle_decoding"))
1476        );
1477        let pid: libc::pid_t = std::fs::read_to_string(pid_path).unwrap().parse().unwrap();
1478        let started = Instant::now();
1479        cancel.store(true, Ordering::Relaxed);
1480        assert!(!done.recv_timeout(Duration::from_secs(2)).unwrap().unwrap());
1481        assert!(started.elapsed() < Duration::from_secs(2));
1482        handle.join().unwrap();
1483        assert_eq!(unsafe { libc::kill(pid, 0) }, -1);
1484        assert_eq!(
1485            std::io::Error::last_os_error().raw_os_error(),
1486            Some(libc::ESRCH)
1487        );
1488    }
1489
1490    #[test]
1491    fn pause_and_same_url_restart_discard_old_epoch_results() {
1492        let dir = tempfile::tempdir().unwrap();
1493        let config = crate::test_support::config(dir.path());
1494        let stop = Arc::new(AtomicBool::new(false));
1495        let mut controller = Controller::default();
1496        controller.enabled(true);
1497        controller.key = Some((1, "http://example.com/live".into()));
1498        let (sender, events) = mpsc::sync_channel(64);
1499        let job = Background::start("fake-speech", |cancel| {
1500            while !cancel.load(Ordering::Relaxed) {
1501                thread::sleep(Duration::from_millis(5));
1502            }
1503            Ok(false)
1504        })
1505        .unwrap();
1506        controller.task = Some(Task {
1507            job,
1508            events,
1509            first_segment_at: None,
1510        });
1511        sender
1512            .send(Notice::Segment(Segment {
1513                epoch: controller.epoch,
1514                start_ms: 0,
1515                end_ms: 500,
1516                text: "current".into(),
1517            }))
1518            .ok();
1519        sender
1520            .send(Notice::Segment(Segment {
1521                epoch: controller.epoch.wrapping_sub(1),
1522                start_ms: 0,
1523                end_ms: 500,
1524                text: "stale".into(),
1525            }))
1526            .ok();
1527        let key = controller.key.clone().unwrap();
1528        controller.update(&config, (key.0, &key.1), true, false, false, &stop);
1529        assert_eq!(controller.snapshot.history.len(), 1);
1530        let epoch = controller.epoch;
1531        controller.update(&config, (key.0, &key.1), false, true, false, &stop);
1532        assert!(controller.epoch != epoch && controller.task.is_none());
1533        assert_eq!(controller.snapshot.history.len(), 1);
1534        controller.update(&config, (2, &key.1), true, false, false, &stop);
1535        assert!(controller.snapshot.history.is_empty());
1536        controller.enabled(false);
1537    }
1538
1539    #[test]
1540    fn short_connections_exhaust_retries_even_when_they_produce_segments() {
1541        let dir = tempfile::tempdir().unwrap();
1542        let config = crate::test_support::config(dir.path());
1543        let stop = Arc::new(AtomicBool::new(false));
1544        for productive in [false, true] {
1545            let mut controller = Controller::default();
1546            controller.enabled(true);
1547            controller.key = Some((1, "http://localhost/live".into()));
1548            for attempt in 0..6 {
1549                let (sender, events) = mpsc::sync_channel(64);
1550                if productive {
1551                    sender
1552                        .send(Notice::Segment(Segment {
1553                            epoch: controller.epoch,
1554                            start_ms: 0,
1555                            end_ms: 500,
1556                            text: "speech".into(),
1557                        }))
1558                        .unwrap();
1559                }
1560                let job = Background::start("fake-speech", |_| Ok(true)).unwrap();
1561                controller.task = Some(Task {
1562                    job,
1563                    events,
1564                    first_segment_at: None,
1565                });
1566                controller.due = None;
1567                let deadline = Instant::now() + Duration::from_secs(2);
1568                while controller.task.is_some() {
1569                    controller.update(
1570                        &config,
1571                        (1, "http://localhost/live"),
1572                        true,
1573                        false,
1574                        false,
1575                        &stop,
1576                    );
1577                    assert!(Instant::now() < deadline);
1578                    thread::yield_now();
1579                }
1580                if attempt == 3 {
1581                    assert!(controller.snapshot.failed);
1582                    assert!(controller.due.is_none());
1583                    break;
1584                } else {
1585                    assert_eq!(controller.attempts, attempt + 1);
1586                    assert!(!controller.snapshot.failed);
1587                }
1588            }
1589        }
1590    }
1591
1592    #[test]
1593    fn only_sustained_speech_on_the_same_connection_resets_retries() {
1594        let dir = tempfile::tempdir().unwrap();
1595        let config = crate::test_support::config(dir.path());
1596        let stop = Arc::new(AtomicBool::new(false));
1597        // Startup, model verification and silence before the first segment do
1598        // not count as productive time. Only later segments on this task do.
1599        for elapsed in [
1600            None,
1601            Some(Duration::from_secs(30)),
1602            Some(STABLE_SUBTITLE_PERIOD),
1603        ] {
1604            let mut controller = Controller::default();
1605            controller.enabled(true);
1606            controller.key = Some((1, "http://localhost/live".into()));
1607            controller.attempts = 3;
1608            let (sender, events) = mpsc::sync_channel(64);
1609            sender
1610                .send(Notice::Segment(Segment {
1611                    epoch: controller.epoch,
1612                    start_ms: 0,
1613                    end_ms: 500,
1614                    text: "speech".into(),
1615                }))
1616                .unwrap();
1617            let job = Background::start("fake-speech", |_| Ok(true)).unwrap();
1618            controller.task = Some(Task {
1619                job,
1620                events,
1621                first_segment_at: elapsed.map(|elapsed| Instant::now() - elapsed),
1622            });
1623            let deadline = Instant::now() + Duration::from_secs(2);
1624            while controller.task.is_some() {
1625                controller.update(
1626                    &config,
1627                    (1, "http://localhost/live"),
1628                    true,
1629                    false,
1630                    false,
1631                    &stop,
1632                );
1633                assert!(Instant::now() < deadline);
1634                thread::yield_now();
1635            }
1636            if elapsed == Some(STABLE_SUBTITLE_PERIOD) {
1637                assert_eq!(controller.attempts, 1);
1638                assert!(!controller.snapshot.failed);
1639                assert!(controller.due.is_some());
1640            } else {
1641                assert_eq!(controller.attempts, 3);
1642                assert!(controller.snapshot.failed);
1643                assert!(controller.due.is_none());
1644            }
1645        }
1646    }
1647
1648    #[test]
1649    fn history_is_bounded_and_empty_clear_preserves_revision() {
1650        let dir = tempfile::tempdir().unwrap();
1651        let config = crate::test_support::config(dir.path());
1652        let stop = Arc::new(AtomicBool::new(false));
1653        let mut controller = Controller::default();
1654        controller.clear();
1655        assert_eq!(controller.snapshot.revision, 0);
1656        controller.enabled(true);
1657        controller.key = Some((1, "http://localhost/live".into()));
1658        let (sender, events) = mpsc::sync_channel(501);
1659        for i in 0..501 {
1660            sender
1661                .send(Notice::Segment(Segment {
1662                    epoch: controller.epoch,
1663                    start_ms: i,
1664                    end_ms: i + 1,
1665                    text: i.to_string(),
1666                }))
1667                .unwrap();
1668        }
1669        let job = Background::start("fake-speech", |_| Ok(true)).unwrap();
1670        controller.task = Some(Task {
1671            job,
1672            events,
1673            first_segment_at: None,
1674        });
1675        controller.update(
1676            &config,
1677            (1, "http://localhost/live"),
1678            true,
1679            false,
1680            false,
1681            &stop,
1682        );
1683        assert_eq!(controller.snapshot.history.len(), 500);
1684        assert_eq!(controller.snapshot.history.front().unwrap().start_ms, 1);
1685        controller.enabled(false);
1686        controller.update(
1687            &config,
1688            (2, "http://localhost/other"),
1689            true,
1690            false,
1691            false,
1692            &stop,
1693        );
1694        assert!(controller.snapshot.history.is_empty());
1695        let revision = controller.snapshot.revision;
1696        controller.clear();
1697        assert_eq!(controller.snapshot.revision, revision);
1698    }
1699}