Skip to main content

silence_split/
analyze.rs

1//! Frame analysis: PCM levels or external scores on the original sample timeline.
2
3use alloc::collections::VecDeque;
4use alloc::vec::Vec;
5use core::time::Duration;
6
7use crate::errors::{ConfigError, PLANAR_CHANNELS, PLANAR_LENGTHS, PLANAR_PENDING};
8use crate::filter::{CUTOFF_HZ, HighPass};
9use crate::math;
10use crate::sample::{Layout, Sample};
11
12const NANOS_PER_SEC: u128 = 1_000_000_000;
13
14/// `d` in whole samples at `rate`, rounded to nearest, in exact integer arithmetic.
15fn round_samples(d: Duration, rate: u32) -> u128 {
16    (d.as_nanos() * u128::from(rate) + NANOS_PER_SEC / 2) / NANOS_PER_SEC
17}
18
19fn saturating_usize(v: u128) -> usize {
20    usize::try_from(v).unwrap_or(usize::MAX)
21}
22
23/// Window and hop of the PCM analyzer.
24///
25/// Both are given in time and rounded once to whole samples when an [`Analyzer`] is created, so
26/// frame positions never drift at sample rates where the hop is not a whole number of samples.
27#[derive(Clone, Debug, PartialEq)]
28#[non_exhaustive]
29pub struct AnalyzeConfig {
30    window: Duration,
31    hop: Duration,
32}
33
34impl Default for AnalyzeConfig {
35    /// A 30 ms window every 10 ms.
36    fn default() -> Self {
37        Self {
38            window: Duration::from_millis(30),
39            hop: Duration::from_millis(10),
40        }
41    }
42}
43
44impl AnalyzeConfig {
45    /// Sets the level window and the hop between frames. The window is centered on its hop.
46    ///
47    /// # Errors
48    ///
49    /// [`ConfigError::InvalidFrames`] if the hop is zero or longer than the window.
50    pub fn frames(mut self, window: Duration, hop: Duration) -> Result<Self, ConfigError> {
51        if hop.is_zero() || window < hop {
52            return Err(ConfigError::InvalidFrames);
53        }
54        self.window = window;
55        self.hop = hop;
56        Ok(self)
57    }
58}
59
60/// The detector's sample rate and hop for [`Analysis::from_scores`].
61///
62/// Construct with named fields; both are validated when the analysis is created.
63#[derive(Clone, Copy, Debug, PartialEq, Eq)]
64pub struct ScoreFrames {
65    /// Detector sample rate in Hz, which may differ from the original audio's rate.
66    pub sample_rate: u32,
67    /// Samples per score frame at the detector's sample rate.
68    pub hop: u32,
69}
70
71/// What the frame values of an [`Analysis`] measure.
72#[derive(Clone, Copy, Debug, PartialEq, Eq)]
73#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
74pub(crate) enum Domain {
75    /// Frame level in dBFS of the high-passed signal.
76    Decibels,
77    /// External speech probability in `0.0..=1.0`.
78    Scores,
79}
80
81/// Incremental PCM analyzer. Push decoded audio in any block size, then [`finish`](Self::finish).
82///
83/// Each channel is high-passed at 80 Hz for detection only, the channel powers are averaged, and
84/// every hop produces one frame level in dBFS over a window centered on that hop. Only the frame
85/// levels and signal-free flags are kept, about 1.8 MB per hour at the default 10 ms hop. Samples
86/// that are not finite (NaN or infinite) are read as zero. A frame is signal-free when all raw
87/// samples are zero or its filtered mean power is at most `1e-20` (-200 dBFS).
88#[derive(Clone, Debug)]
89pub struct Analyzer {
90    layout: Layout,
91    hop: u64,
92    window: u64,
93    filters: Vec<HighPass>,
94    /// Channel samples of an interleaved frame split across pushes.
95    pending: Vec<f32>,
96    /// Per-sample channel-mean filtered power and whether any raw channel sample was non-zero.
97    buffer: VecDeque<(f64, bool)>,
98    /// Sample index of `buffer[0]`.
99    buffer_start: u64,
100    received: u64,
101    values: Vec<f32>,
102    no_signal: Vec<bool>,
103}
104
105impl Analyzer {
106    /// An analyzer for audio with `layout`.
107    ///
108    /// # Errors
109    ///
110    /// [`ConfigError::ZeroSampleRate`], [`ConfigError::ZeroChannels`],
111    /// [`ConfigError::SampleRateTooLow`] if the rate is at or below 160 Hz,
112    /// [`ConfigError::HopTooShort`] if the hop rounds to zero samples, or
113    /// [`ConfigError::InvalidFrames`] if the hop exceeds `u32::MAX` samples or the window exceeds
114    /// `u64::MAX` samples.
115    pub fn new(layout: Layout, config: &AnalyzeConfig) -> Result<Self, ConfigError> {
116        validate_layout(layout)?;
117        if f64::from(layout.sample_rate) <= 2.0 * CUTOFF_HZ {
118            return Err(ConfigError::SampleRateTooLow);
119        }
120        let hop = round_samples(config.hop, layout.sample_rate);
121        if hop == 0 {
122            return Err(ConfigError::HopTooShort);
123        }
124        let hop = u64::from(u32::try_from(hop).map_err(|_| ConfigError::InvalidFrames)?);
125        // The window is at least the hop, so its rounding is at least the hop's.
126        let window = u64::try_from(round_samples(config.window, layout.sample_rate))
127            .map_err(|_| ConfigError::InvalidFrames)?;
128        Ok(Self {
129            layout,
130            hop,
131            window,
132            filters: (0..layout.channels)
133                .map(|_| HighPass::new(layout.sample_rate))
134                .collect(),
135            pending: Vec::new(),
136            buffer: VecDeque::new(),
137            buffer_start: 0,
138            received: 0,
139            values: Vec::new(),
140            no_signal: Vec::new(),
141        })
142    }
143
144    /// Pushes interleaved samples. A trailing partial frame is kept until the next push.
145    pub fn push_interleaved<S: Sample>(&mut self, pcm: &[S]) {
146        let channels = usize::from(self.layout.channels);
147        let mut rest = pcm;
148        if !self.pending.is_empty() {
149            let take = (channels - self.pending.len()).min(rest.len());
150            self.pending.extend(rest[..take].iter().map(|s| s.to_f32()));
151            rest = &rest[take..];
152            if self.pending.len() < channels {
153                return;
154            }
155            let frame = core::mem::take(&mut self.pending);
156            self.process(frame.iter().copied());
157        }
158        let mut chunks = rest.chunks_exact(channels);
159        for frame in &mut chunks {
160            self.process(frame.iter().map(|s| s.to_f32()));
161        }
162        self.pending
163            .extend(chunks.remainder().iter().map(|s| s.to_f32()));
164    }
165
166    /// Pushes one slice per channel.
167    ///
168    /// # Panics
169    ///
170    /// If the number of slices differs from the channel count, the slices differ in length, or a
171    /// partial interleaved frame from [`push_interleaved`](Self::push_interleaved) is pending.
172    pub fn push_planar<S: Sample>(&mut self, pcm: &[&[S]]) {
173        assert!(self.pending.is_empty(), "{PLANAR_PENDING}");
174        assert_eq!(
175            pcm.len(),
176            usize::from(self.layout.channels),
177            "{PLANAR_CHANNELS}"
178        );
179        let len = pcm.first().map_or(0, |c| c.len());
180        assert!(pcm.iter().all(|c| c.len() == len), "{PLANAR_LENGTHS}");
181        for i in 0..len {
182            self.process(pcm.iter().map(|c| c[i].to_f32()));
183        }
184    }
185
186    /// Finishes the stream and returns the frame levels. A partial interleaved frame is dropped.
187    #[must_use]
188    pub fn finish(mut self) -> Analysis {
189        let len = self.received;
190        let frames = len.div_ceil(self.hop);
191        while (self.values.len() as u64) < frames {
192            self.emit(len);
193        }
194        Analysis {
195            layout: self.layout,
196            len,
197            frame_rate: self.layout.sample_rate,
198            hop: self.hop as u32,
199            domain: Domain::Decibels,
200            values: self.values,
201            no_signal: self.no_signal,
202        }
203    }
204
205    fn process(&mut self, frame: impl Iterator<Item = f32>) {
206        let mut power = 0.0;
207        let mut nonzero = false;
208        for (filter, x) in self.filters.iter_mut().zip(frame) {
209            // One NaN or infinity would stay in the filter state and silence the rest of the file.
210            let x = if x.is_finite() { x } else { 0.0 };
211            nonzero |= x != 0.0;
212            let y = filter.process(f64::from(x));
213            power += y * y;
214        }
215        self.buffer
216            .push_back((power / f64::from(self.layout.channels), nonzero));
217        self.received += 1;
218        while self.window_end(self.values.len() as u64) <= self.received {
219            self.emit(self.received);
220        }
221    }
222
223    /// Sample range `[start, end)` of frame `i`'s window, before clipping to the stream.
224    fn window_start(&self, i: u64) -> u64 {
225        (i * self.hop).saturating_sub((self.window - self.hop) / 2)
226    }
227
228    fn window_end(&self, i: u64) -> u64 {
229        let ahead = self.window - (self.window - self.hop) / 2;
230        (i * self.hop).saturating_add(ahead)
231    }
232
233    /// Emits the next frame with its window clipped to `[0, available)`.
234    fn emit(&mut self, available: u64) {
235        let i = self.values.len() as u64;
236        let start = self.window_start(i);
237        let end = self.window_end(i).min(available);
238        let from = (start - self.buffer_start) as usize;
239        let to = (end - self.buffer_start) as usize;
240        let mut power = 0.0;
241        let mut nonzero = false;
242        for &(p, nz) in self.buffer.range(from..to) {
243            power += p;
244            nonzero |= nz;
245        }
246        // The filter rings on after the input stops, so a signal-free window gets the floor level.
247        let power = if nonzero {
248            power / (to - from) as f64
249        } else {
250            0.0
251        };
252        self.values.push(math::power_to_db(power) as f32);
253        self.no_signal.push(!nonzero || power <= math::MIN_POWER);
254        let next = self.window_start(i + 1).min(self.received);
255        while self.buffer_start < next {
256            self.buffer.pop_front();
257            self.buffer_start += 1;
258        }
259    }
260}
261
262fn validate_layout(layout: Layout) -> Result<(), ConfigError> {
263    if layout.sample_rate == 0 {
264        return Err(ConfigError::ZeroSampleRate);
265    }
266    if layout.channels == 0 {
267        return Err(ConfigError::ZeroChannels);
268    }
269    Ok(())
270}
271
272/// Frame values on the original sample timeline, from an [`Analyzer`] or external scores.
273///
274/// Frame `i` stands for original samples `[start(i), start(i + 1))`, where
275/// `start(i) = floor(i · hop · sample_rate / frame_rate)` in exact integer arithmetic and the last
276/// frame ends at the audio length. Thresholds are resolved when segmenting or planning, not here,
277/// so one analysis serves any threshold.
278#[derive(Clone, Debug, PartialEq)]
279#[cfg_attr(
280    feature = "serde",
281    derive(serde::Serialize, serde::Deserialize),
282    serde(try_from = "AnalysisData")
283)]
284pub struct Analysis {
285    layout: Layout,
286    len: u64,
287    frame_rate: u32,
288    hop: u32,
289    domain: Domain,
290    values: Vec<f32>,
291    no_signal: Vec<bool>,
292}
293
294impl Analysis {
295    /// Wraps per-frame speech probabilities from an external voice activity detector.
296    ///
297    /// `len` is the length of the original audio in samples per channel. `frames.hop` is the
298    /// detector's hop in samples at `frames.sample_rate`, which may differ from the original
299    /// sample rate: a 16 kHz detector with a 512-sample hop maps frame `i` to original sample
300    /// `floor(i · 512 · sample_rate / 16000)`. Each frame must span at least one original sample.
301    /// The scores must cover the audio: the last score starts before `len`, and less than one hop
302    /// may be missing at the end.
303    ///
304    /// # Errors
305    ///
306    /// [`ConfigError::ZeroSampleRate`], [`ConfigError::ZeroChannels`],
307    /// [`ConfigError::InvalidScoreFrames`] for a zero rate or hop or a frame shorter than one
308    /// original sample, [`ConfigError::InvalidValue`]
309    /// for a score outside `0.0..=1.0`, or [`ConfigError::FrameCountMismatch`].
310    pub fn from_scores(
311        layout: Layout,
312        len: u64,
313        frames: ScoreFrames,
314        scores: Vec<f32>,
315    ) -> Result<Self, ConfigError> {
316        let no_signal = alloc::vec![false; scores.len()];
317        let analysis = Self {
318            layout,
319            len,
320            frame_rate: frames.sample_rate,
321            hop: frames.hop,
322            domain: Domain::Scores,
323            values: scores,
324            no_signal,
325        };
326        analysis.validate()?;
327        Ok(analysis)
328    }
329
330    /// Layout of the original audio.
331    #[must_use]
332    pub fn layout(&self) -> Layout {
333        self.layout
334    }
335
336    /// Length of the original audio in samples per channel.
337    #[must_use]
338    pub fn sample_count(&self) -> u64 {
339        self.len
340    }
341
342    fn validate(&self) -> Result<(), ConfigError> {
343        validate_layout(self.layout)?;
344        let frame_samples = u64::from(self.hop) * u64::from(self.layout.sample_rate);
345        if self.frame_rate == 0 || self.hop == 0 || frame_samples < u64::from(self.frame_rate) {
346            return Err(ConfigError::InvalidScoreFrames);
347        }
348        let valid = |v: &f32| match self.domain {
349            Domain::Decibels => v.is_finite(),
350            Domain::Scores => (0.0..=1.0).contains(v),
351        };
352        let marked_scores =
353            self.domain == Domain::Scores && self.no_signal.iter().any(|no_signal| *no_signal);
354        if !self.values.iter().all(valid) || marked_scores {
355            return Err(ConfigError::InvalidValue);
356        }
357        let n = self.values.len();
358        let len = u128::from(self.len);
359        let covered = if self.len == 0 {
360            n == 0
361        } else {
362            n > 0 && self.start(n - 1) < len && len < self.start(n + 1)
363        };
364        if !covered || self.no_signal.len() != n {
365            return Err(ConfigError::FrameCountMismatch);
366        }
367        Ok(())
368    }
369
370    /// Number of frames.
371    pub(crate) fn frames(&self) -> usize {
372        self.values.len()
373    }
374
375    pub(crate) fn domain(&self) -> Domain {
376        self.domain
377    }
378
379    pub(crate) fn values(&self) -> &[f32] {
380        &self.values
381    }
382
383    /// Whether the raw window is all zero or its filtered mean power is at the floor.
384    pub(crate) fn no_signal(&self) -> &[bool] {
385        &self.no_signal
386    }
387
388    /// Unclipped original start sample of frame `i`.
389    fn start(&self, i: usize) -> u128 {
390        let num = i as u128 * u128::from(self.hop) * u128::from(self.layout.sample_rate);
391        num / u128::from(self.frame_rate)
392    }
393
394    /// Original sample of frame boundary `k` in `0..=frames`; the last boundary is the length.
395    pub(crate) fn position(&self, k: usize) -> u64 {
396        if k >= self.frames() {
397            self.len
398        } else {
399            // Validation keeps every frame start below the length.
400            self.start(k) as u64
401        }
402    }
403
404    /// Duration of one frame in seconds.
405    pub(crate) fn frame_secs(&self) -> f64 {
406        f64::from(self.hop) / f64::from(self.frame_rate)
407    }
408
409    /// `d` in frames as the exact fraction `(numerator, denominator)`.
410    fn frame_fraction(&self, d: Duration) -> (u128, u128) {
411        (
412            d.as_nanos() * u128::from(self.frame_rate),
413            u128::from(self.hop) * NANOS_PER_SEC,
414        )
415    }
416
417    /// `d` in whole frames, rounded to nearest.
418    pub(crate) fn frames_in(&self, d: Duration) -> usize {
419        let (num, den) = self.frame_fraction(d);
420        saturating_usize((num + den / 2) / den)
421    }
422
423    /// `d` in whole frames, rounded up.
424    pub(crate) fn frames_ceil(&self, d: Duration) -> usize {
425        let (num, den) = self.frame_fraction(d);
426        saturating_usize(num.div_ceil(den))
427    }
428
429    /// `d` in whole frames, rounded down.
430    pub(crate) fn frames_floor(&self, d: Duration) -> usize {
431        let (num, den) = self.frame_fraction(d);
432        saturating_usize(num / den)
433    }
434
435    /// `d` in whole original samples, rounded down.
436    pub(crate) fn samples_floor(&self, d: Duration) -> u64 {
437        let samples = d.as_nanos() * u128::from(self.layout.sample_rate) / NANOS_PER_SEC;
438        u64::try_from(samples).unwrap_or(u64::MAX)
439    }
440
441    /// `d` in original samples as a real number.
442    pub(crate) fn samples_in(&self, d: Duration) -> f64 {
443        (d.as_nanos() * u128::from(self.layout.sample_rate)) as f64 / NANOS_PER_SEC as f64
444    }
445}
446
447/// Unvalidated serialized form of [`Analysis`].
448#[cfg(feature = "serde")]
449#[derive(serde::Deserialize)]
450struct AnalysisData {
451    layout: Layout,
452    len: u64,
453    frame_rate: u32,
454    hop: u32,
455    domain: Domain,
456    values: Vec<f32>,
457    no_signal: Vec<bool>,
458}
459
460#[cfg(feature = "serde")]
461impl TryFrom<AnalysisData> for Analysis {
462    type Error = ConfigError;
463
464    fn try_from(d: AnalysisData) -> Result<Self, ConfigError> {
465        let analysis = Self {
466            layout: d.layout,
467            len: d.len,
468            frame_rate: d.frame_rate,
469            hop: d.hop,
470            domain: d.domain,
471            values: d.values,
472            no_signal: d.no_signal,
473        };
474        analysis.validate()?;
475        Ok(analysis)
476    }
477}