mod table;
mod table16;
use crate::types::{FrameRange, RATE_8K, Rate, SignalBuffer, Utterance};
use rustfft::Fft;
use rustfft::num_complex::Complex;
pub const FRAME_LEN: usize = RATE_8K.frame_len();
pub const FRAME_HOP: usize = RATE_8K.frame_hop();
pub const NUM_BANDS: usize = RATE_8K.num_bands();
pub const NUM_POWER_BINS: usize = RATE_8K.num_power_bins();
const SILENCE_SKIP_THRESHOLD: f64 = 500.0;
const SILENCE_FLAG_FACTOR: f64 = 100.0;
const SILENCE_FLAG_POWER: f64 = 1e7;
const COMPENSATION_FACTOR: f64 = 100.0;
const COMPENSATION_OFFSET: f64 = 1000.0;
const COMPENSATION_MIN: f64 = 0.01;
const COMPENSATION_MAX: f64 = 100.0;
const SCALE_OFFSET: f64 = 5000.0;
const SCALE_SMOOTH_PREVIOUS: f64 = 0.2;
const SCALE_SMOOTH_CURRENT: f64 = 0.8;
const SCALE_MIN: f64 = 3e-4;
const SCALE_MAX: f64 = 5.0;
pub fn bark_width(band: usize, rate: Rate) -> f32 {
table::bark_table(rate)[band].bark_width
}
struct Spectra {
fft: std::sync::Arc<dyn Fft<f32>>,
window: Vec<f32>,
scratch: Vec<Complex<f32>>,
power_bins: usize,
}
impl Spectra {
fn new(rate: Rate) -> Self {
Self {
fft: crate::dsp_fft::forward_plan(rate.frame_len()),
window: crate::dsp::hann_window(rate.frame_len()),
scratch: Vec::with_capacity(rate.frame_len()),
power_bins: rate.num_power_bins(),
}
}
fn power(&mut self, samples: &[f32], start: usize, out: &mut [f64]) {
debug_assert_eq!(out.len(), self.power_bins);
self.scratch.clear();
self.scratch.extend(
self.window
.iter()
.enumerate()
.map(|(i, &weight)| Complex::new(samples[start + i] * weight, 0.0)),
);
self.fft.process(&mut self.scratch);
for (bin, value) in self.scratch.iter().take(self.power_bins).enumerate() {
out[bin] = f64::from(value.re * value.re + value.im * value.im);
}
out[0] = 0.0;
}
}
fn silence_skip_start(reference: &SignalBuffer, n_max: usize) -> usize {
let margin = reference.rate.margin_samples();
let mut skip = 0;
while skip < n_max / 2
&& (0..5)
.map(|i| f64::from(reference.samples[margin + skip + i].abs()))
.sum::<f64>()
< SILENCE_SKIP_THRESHOLD
{
skip += 1;
}
skip
}
fn silence_skip_end(reference: &SignalBuffer, n_max: usize) -> usize {
let margin = reference.rate.margin_samples();
let last = n_max - margin + reference.rate.padding_samples() - 1;
let mut skip = 0;
while skip < n_max / 2
&& (0..5)
.map(|i| f64::from(reference.samples[last - skip - i].abs()))
.sum::<f64>()
< SILENCE_SKIP_THRESHOLD
{
skip += 1;
}
skip
}
pub fn frame_range(reference: &SignalBuffer, n_max: usize) -> FrameRange {
let rate = reference.rate;
let skip_start = silence_skip_start(reference, n_max);
let skip_end = silence_skip_end(reference, n_max);
let start = skip_start / rate.frame_hop();
let stop = ((n_max - 2 * rate.margin_samples() + rate.padding_samples() - skip_end)
/ rate.frame_hop())
.saturating_sub(1);
FrameRange {
start,
stop,
skip_start,
skip_end,
}
}
pub(crate) fn governing_delay(r0: usize, utterances: &[Utterance], rate: Rate) -> i32 {
utterances
.iter()
.rev()
.find(|u| u.start_window * rate.window_samples() <= r0)
.or_else(|| utterances.first())
.map_or(0, |u| u.fine_delay)
}
fn degraded_start(r0: usize, n_max: usize, utterances: &[Utterance], rate: Rate) -> Option<usize> {
let d0 = r0 as i64 + i64::from(governing_delay(r0, utterances, rate));
if d0 <= 0 || d0 + rate.frame_len() as i64 >= (n_max + rate.padding_samples()) as i64 {
return None;
}
Some(d0 as usize)
}
pub fn warp_to_bark(power: &[f64], rate: Rate) -> Vec<f32> {
let mut density = vec![0.0f32; rate.num_bands()];
let mut bin = 0;
for (band, row) in table::bark_table(rate).iter().enumerate() {
let sum: f64 = power[bin..bin + row.bins].iter().sum();
bin += row.bins;
density[band] = (sum * f64::from(row.correction) * rate.pitch_power_scale()) as f32;
}
debug_assert_eq!(bin, rate.num_power_bins());
density
}
pub fn audible_power(density: &[f32], factor: f64, rate: Rate) -> f64 {
let bands = table::bark_table(rate);
let mut total = 0.0f64;
for (band, &p) in density.iter().enumerate().skip(1) {
if f64::from(p) > factor * f64::from(bands[band].threshold) {
total += f64::from(p);
}
}
total
}
pub fn zwicker_loudness(pitch_power: f32, band: usize, rate: Rate) -> f32 {
let row = &table::bark_table(rate)[band];
let threshold = f64::from(row.threshold);
let p = f64::from(pitch_power);
if p <= threshold {
return 0.0;
}
let h = (if row.bark_centre < 4.0 {
(6.0 / (f64::from(row.bark_centre) + 2.0)).min(2.0)
} else {
1.0
})
.powf(0.15);
let z = 0.23 * h;
let loudness = (threshold / 0.5).powf(z) * ((0.5 + 0.5 * p / threshold).powf(z) - 1.0);
(loudness * rate.loudness_scale()) as f32
}
pub fn compensation_factor(avg_ref: f64, avg_deg: f64) -> f64 {
((avg_deg + COMPENSATION_OFFSET) / (avg_ref + COMPENSATION_OFFSET))
.clamp(COMPENSATION_MIN, COMPENSATION_MAX)
}
pub fn local_scale(a_ref: f64, a_deg: f64, previous: f64, frame: usize) -> (f64, f64) {
let raw = (a_ref + SCALE_OFFSET) / (a_deg + SCALE_OFFSET);
let unclamped = if frame > 0 {
SCALE_SMOOTH_PREVIOUS * previous + SCALE_SMOOTH_CURRENT * raw
} else {
raw
};
(unclamped, unclamped.clamp(SCALE_MIN, SCALE_MAX))
}
#[derive(Debug, Clone)]
pub struct PerceptualModel {
pub rate: Rate,
pub frame_range: FrameRange,
pub pitch_ref: Vec<f32>,
pub pitch_deg: Vec<f32>,
pub loudness_ref: Vec<f32>,
pub loudness_deg: Vec<f32>,
pub silence_flags: Vec<bool>,
pub audible_ref: Vec<f32>,
}
impl PerceptualModel {
pub fn frame_count(&self) -> usize {
self.silence_flags.len()
}
pub fn pitch_ref_at(&self, frame: usize, band: usize) -> f32 {
self.pitch_ref[frame * self.rate.num_bands() + band]
}
pub fn pitch_deg_at(&self, frame: usize, band: usize) -> f32 {
self.pitch_deg[frame * self.rate.num_bands() + band]
}
pub fn loudness_ref_at(&self, frame: usize, band: usize) -> f32 {
self.loudness_ref[frame * self.rate.num_bands() + band]
}
pub fn loudness_deg_at(&self, frame: usize, band: usize) -> f32 {
self.loudness_deg[frame * self.rate.num_bands() + band]
}
}
pub fn run_frame_loop(
reference: &SignalBuffer,
degraded: &SignalBuffer,
utterances: &[Utterance],
) -> PerceptualModel {
let rate = reference.rate;
let bands = rate.num_bands();
let n_max = reference.nominal_len.max(degraded.nominal_len);
let frame_range = frame_range(reference, n_max);
let frame_count = frame_range.stop + 1;
let mut spectra = Spectra::new(rate);
let mut pitch_ref = vec![0.0f32; frame_count * bands];
let mut pitch_deg = vec![0.0f32; frame_count * bands];
let mut silence_flags = vec![false; frame_count];
let mut ref_sums = vec![0.0f64; bands];
let mut deg_sums = vec![0.0f64; bands];
let mut ref_power = vec![0.0f64; rate.num_power_bins()];
let mut deg_power = vec![0.0f64; rate.num_power_bins()];
for frame in 0..=frame_range.stop {
let r0 = rate.margin_samples() + frame * rate.frame_hop();
spectra.power(&reference.samples, r0, &mut ref_power);
let ref_density = warp_to_bark(&ref_power, rate);
let deg_density = match degraded_start(r0, n_max, utterances, rate) {
Some(d0) => {
spectra.power(°raded.samples, d0, &mut deg_power);
warp_to_bark(°_power, rate)
}
None => vec![0.0f32; bands],
};
silence_flags[frame] =
audible_power(&ref_density, SILENCE_FLAG_FACTOR, rate) < SILENCE_FLAG_POWER;
for band in 0..bands {
pitch_ref[frame * bands + band] = ref_density[band];
pitch_deg[frame * bands + band] = deg_density[band];
if silence_flags[frame] {
continue;
}
let limit = COMPENSATION_FACTOR * f64::from(table::bark_table(rate)[band].threshold);
if f64::from(ref_density[band]) > limit {
ref_sums[band] += f64::from(ref_density[band]);
}
if f64::from(deg_density[band]) > limit {
deg_sums[band] += f64::from(deg_density[band]);
}
}
}
let divisor = (((n_max - 2 * rate.margin_samples() + rate.padding_samples())
/ rate.frame_hop())
- 1) as f64;
for band in 0..bands {
let factor = compensation_factor(ref_sums[band] / divisor, deg_sums[band] / divisor);
for frame in 0..=frame_range.stop {
pitch_ref[frame * bands + band] =
(f64::from(pitch_ref[frame * bands + band]) * factor) as f32;
}
}
let mut loudness_ref = vec![0.0f32; frame_count * bands];
let mut loudness_deg = vec![0.0f32; frame_count * bands];
let mut audible_ref = vec![0.0f32; frame_count];
let mut previous_scale = 1.0;
for (frame, audible) in
(frame_range.start..=frame_range.stop).zip(audible_ref.iter_mut().skip(frame_range.start))
{
let base = frame * bands;
let a_ref = audible_power(&pitch_ref[base..base + bands], 1.0, rate);
let a_deg = audible_power(&pitch_deg[base..base + bands], 1.0, rate);
let (unclamped, clamped) = local_scale(a_ref, a_deg, previous_scale, frame);
previous_scale = unclamped;
*audible = a_ref as f32;
for band in 0..bands {
pitch_deg[base + band] *= clamped as f32;
}
for band in 0..bands {
loudness_ref[base + band] = zwicker_loudness(pitch_ref[base + band], band, rate);
loudness_deg[base + band] = zwicker_loudness(pitch_deg[base + band], band, rate);
}
}
PerceptualModel {
rate,
frame_range,
pitch_ref,
pitch_deg,
loudness_ref,
loudness_deg,
silence_flags,
audible_ref,
}
}
#[cfg(test)]
mod tests;