use ::std::sync::Arc;
use ::std::sync::atomic::{AtomicU64, Ordering};
use ::anyhow::{Result, ensure};
use ::deep_filter_rt::tract::{DfParams, DfTract, RuntimeParams};
use ::ndarray::Array2;
use crate::atomic::AtomicF32;
use crate::dsp::{db_to_gain, ramp};
use crate::echo::Sidechain;
use crate::level::{self, SpeechLevel};
use crate::param::Param;
use crate::stage::{Frame, Stage};
use crate::{FRAME_LEN, FRAME_S, SAMPLE_RATE};
pub struct Denoise {
df: DfTract,
input_gain: InputGain,
noisy: Array2<f32>,
enhanced: Array2<f32>,
apply: Array2<f32>,
atten_lim: Param,
applied_atten_lim: f32,
speech: Arc<Speech>,
sidechain: Option<Arc<Sidechain>>,
sidechain_frames: u64,
mix_before: f32,
voice_hold: usize,
}
#[derive(Debug)]
pub struct Speech {
snr_db: AtomicF32,
frames: AtomicU64,
}
impl Speech {
pub fn new() -> Self {
Self {
snr_db: AtomicF32::new(-15.),
frames: AtomicU64::new(0),
}
}
pub fn snr_db(&self) -> f32 {
self.snr_db.load()
}
pub fn frames(&self) -> u64 {
self.frames.load(Ordering::Relaxed)
}
pub fn update(&self, snr_db: f32) {
self.snr_db.store(snr_db);
self.frames.fetch_add(1, Ordering::Relaxed);
}
}
impl Default for Speech {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug)]
pub struct SpeechReader {
speech: Arc<Speech>,
frames: u64,
}
impl SpeechReader {
pub fn new(speech: Arc<Speech>) -> Self {
let frames = speech.frames();
Self { speech, frames }
}
pub fn read(&mut self) -> Option<f32> {
let frames = self.speech.frames();
let running = frames != self.frames;
self.frames = frames;
running.then(|| self.speech.snr_db())
}
}
pub const UNLIMITED: f32 = 100.;
pub const INPUT_LEVEL_DB: f32 = -35.;
pub const MAX_INPUT_GAIN_DB: f32 = 20.;
const INPUT_GATE_DB: f32 = -70.;
const INPUT_RISE_DB_PER_S: f32 = 20.;
const INPUT_FALL_DB_PER_S: f32 = 30.;
const MAX_DELAY_FRAMES: usize = 8;
struct InputGain {
speech: SpeechLevel,
gain_db: f32,
level: Param,
max_gain: Param,
gains: [(f32, f32); MAX_DELAY_FRAMES + 1],
next: usize,
delay: usize,
}
impl InputGain {
fn new(delay: usize) -> Result<Self> {
ensure!(
delay <= MAX_DELAY_FRAMES,
"the model's latency is {delay} frames, at most {MAX_DELAY_FRAMES} are supported"
);
Ok(Self {
speech: SpeechLevel::new(INPUT_GATE_DB, INPUT_LEVEL_DB),
gain_db: 0.,
level: Param::new(
"level",
"Input Level",
"dBFS",
(-45., -20., 1.),
INPUT_LEVEL_DB,
),
max_gain: Param::new(
"max-gain",
"Maximum Input Gain",
"dB",
(0., 30., 1.),
MAX_INPUT_GAIN_DB,
),
gains: [(1., 1.); MAX_DELAY_FRAMES + 1],
next: 0,
delay,
})
}
fn next(&mut self, frame: &Frame) -> ((f32, f32), (f32, f32)) {
let level_db = self.speech.push(level::frame_db(frame), true);
let wanted = (self.level.get() - level_db).clamp(0., self.max_gain.get());
let previous = self.gain_db;
self.gain_db = if wanted > previous {
wanted.min(previous + INPUT_RISE_DB_PER_S * FRAME_S)
} else {
wanted.max(previous - INPUT_FALL_DB_PER_S * FRAME_S)
};
let gain = (db_to_gain(previous), db_to_gain(self.gain_db));
let len = self.gains.len();
self.gains[self.next] = gain;
let output = self.gains[(self.next + len - self.delay) % len];
self.next = (self.next + 1) % len;
(gain, output)
}
}
impl Denoise {
pub fn new() -> Result<Self> {
let params = RuntimeParams::default();
let zeros_below_db = params.min_db_thresh;
let params = params.with_thresholds(zeros_below_db, f32::INFINITY, f32::INFINITY);
let df = DfTract::new(DfParams::default(), ¶ms)?;
ensure!(
df.sr == SAMPLE_RATE as usize && df.hop_size == FRAME_LEN && df.ch == 1,
"the model expects {} Hz, {} samples per frame and {} channels",
df.sr,
df.hop_size,
df.ch,
);
ensure!(
df.delay() % FRAME_LEN == 0,
"the model's latency of {} samples isn't whole frames",
df.delay()
);
let input_gain = InputGain::new(df.delay() / FRAME_LEN)?;
Ok(Self {
df,
input_gain,
noisy: Array2::zeros((1, FRAME_LEN)),
enhanced: Array2::zeros((1, FRAME_LEN)),
apply: Array2::zeros((1, FRAME_LEN)),
atten_lim: Param::new("limit", "Limit", "dB", (0., UNLIMITED, 1.), UNLIMITED),
applied_atten_lim: UNLIMITED,
speech: Arc::new(Speech::new()),
sidechain: None,
sidechain_frames: 0,
mix_before: 1.,
voice_hold: 0,
})
}
pub fn with_atten_lim(self, db: f32) -> Self {
self.atten_lim
.set(db.clamp(0., UNLIMITED))
.expect("clamped to the range");
self
}
pub fn with_sidechain(mut self, sidechain: Arc<Sidechain>) -> Self {
self.sidechain = Some(sidechain);
self
}
pub fn with_thresholds(
mut self,
zeros_below_db: f32,
unchanged_above_db: f32,
no_df_above_db: f32,
) -> Self {
self.df.min_db_thresh = zeros_below_db;
self.df.max_db_erb_thresh = unchanged_above_db;
self.df.max_db_df_thresh = no_df_above_db;
self
}
pub fn speech(&self) -> Arc<Speech> {
self.speech.clone()
}
pub fn atten_lim(&self) -> Param {
self.atten_lim.clone()
}
pub fn input_gain_db(&self) -> f32 {
self.input_gain.gain_db
}
}
impl Stage for Denoise {
fn name(&self) -> &'static str {
"denoise"
}
fn latency(&self) -> usize {
self.df.delay()
}
fn resume_frames(&self) -> usize {
self.df.delay().div_ceil(FRAME_LEN) + self.df.df_order - self.df.df_lookahead
}
fn params(&self) -> Vec<Param> {
vec![
self.atten_lim.clone(),
self.input_gain.level.clone(),
self.input_gain.max_gain.clone(),
]
}
fn trace(&self) -> Option<(&'static str, f32)> {
Some(("snr", self.speech.snr_db()))
}
fn process(&mut self, frame: &mut Frame) -> Result<()> {
let Some(sidechain) = self.sidechain.clone() else {
return self.process_with(frame, None, true);
};
let frames = sidechain.frames();
let fresh = frames != self.sidechain_frames;
self.sidechain_frames = frames;
let protect = sidechain.protect();
let active = fresh && protect > 0. && sidechain.playing() && self.voice_hold > 0;
let mix = if active { 1. - protect } else { 1. };
if mix >= 1. && self.mix_before >= 1. {
self.process_with(frame, None, true)?;
} else {
let mut raw = *frame;
if fresh {
sidechain.raw(&mut raw);
}
let mut apply = [0.; FRAME_LEN];
for (((a, e), r), m) in apply
.iter_mut()
.zip(frame.iter())
.zip(&raw)
.zip(ramp(self.mix_before, mix))
{
*a = m * e + (1. - m) * r;
}
self.process_with(frame, Some(&apply), !active)?;
}
self.mix_before = mix;
self.voice_hold = if self.speech.snr_db() > VOICE_DB {
VOICE_HOLD_FRAMES
} else {
self.voice_hold.saturating_sub(1)
};
Ok(())
}
}
pub const VOICE_DB: f32 = 10.;
pub const VOICE_HOLD_FRAMES: usize = 20;
impl Denoise {
pub fn process_with(
&mut self,
frame: &mut Frame,
apply: Option<&Frame>,
df: bool,
) -> Result<()> {
let atten_lim = self.atten_lim.get();
if atten_lim != self.applied_atten_lim {
self.df.set_atten_lim(atten_lim);
self.applied_atten_lim = atten_lim;
}
let (gain, output_gain) = self.input_gain.next(frame);
let noisy = self.noisy.row_mut(0).into_slice().unwrap();
for ((x, &y), g) in noisy.iter_mut().zip(frame.iter()).zip(ramp(gain.0, gain.1)) {
*x = y * g;
}
if let Some(apply) = apply {
let to = self.apply.row_mut(0).into_slice().unwrap();
for ((x, &y), g) in to.iter_mut().zip(apply.iter()).zip(ramp(gain.0, gain.1)) {
*x = y * g;
}
}
let snr_db = self.df.process_with(
self.noisy.view(),
apply.map(|_| self.apply.view()),
df,
self.enhanced.view_mut(),
)?;
if self.df.attenuates() {
self.speech.update(snr_db);
}
let enhanced = self.enhanced.row(0).to_slice().unwrap();
for ((y, &x), g) in frame
.iter_mut()
.zip(enhanced)
.zip(ramp(output_gain.0, output_gain.1))
{
*y = x / g;
}
Ok(())
}
}