use crate::block::Block;
use crate::common::{BLOCK_SIZE, FFT_LENGTH_BY_2, FFT_LENGTH_BY_2_PLUS_1, NUM_BLOCKS_PER_SECOND};
use crate::config::EchoCanceller3Config;
use crate::delay_estimate::DelayEstimate;
use crate::echo_audibility::EchoAudibility;
use crate::echo_path_variability::{DelayAdjustment, EchoPathVariability};
use crate::erl_estimator::ErlEstimator;
use crate::erle_estimator::ErleEstimator;
use crate::filter_analyzer::FilterAnalyzer;
use crate::render_buffer::RenderBuffer;
use crate::reverb_model::ReverbModel;
use crate::reverb_model_estimator::ReverbModelEstimator;
use crate::spectrum_buffer::SpectrumBuffer;
use crate::subtractor_output::SubtractorOutput;
use crate::subtractor_output_analyzer::SubtractorOutputAnalyzer;
use crate::transparent_mode::{TransparentMode, TransparentModeState};
pub(crate) struct AecStateUpdate<'a> {
pub external_delay: &'a Option<DelayEstimate>,
pub adaptive_filter_frequency_responses: &'a [Vec<[f32; FFT_LENGTH_BY_2_PLUS_1]>],
pub adaptive_filter_impulse_responses: &'a [Vec<f32>],
pub render_buffer: &'a RenderBuffer<'a>,
pub e2_refined: &'a [[f32; FFT_LENGTH_BY_2_PLUS_1]],
pub y2: &'a [[f32; FFT_LENGTH_BY_2_PLUS_1]],
pub subtractor_output: &'a [SubtractorOutput],
}
fn compute_avg_render_reverb(
spectrum_buffer: &SpectrumBuffer,
delay_blocks: i32,
reverb_decay: f32,
reverb_model: &mut ReverbModel,
reverb_power_spectrum: &mut [f32; FFT_LENGTH_BY_2_PLUS_1],
) {
let num_render_channels = spectrum_buffer.buffer[0].len();
let idx_at_delay = spectrum_buffer
.index
.offset_index(spectrum_buffer.index.read, delay_blocks);
let idx_past = spectrum_buffer.index.inc_index(idx_at_delay);
if num_render_channels > 1 {
let mut x2_past = [0.0f32; FFT_LENGTH_BY_2_PLUS_1];
let normalizer = 1.0 / num_render_channels as f32;
for ch_buf in &spectrum_buffer.buffer[idx_past][..num_render_channels] {
for (x2_val, &buf_val) in x2_past.iter_mut().zip(ch_buf.iter()) {
*x2_val += buf_val;
}
}
for x2_val in &mut x2_past {
*x2_val *= normalizer;
}
reverb_model.update_reverb_no_freq_shaping(&x2_past, 1.0, reverb_decay);
let mut x2_at_delay = [0.0f32; FFT_LENGTH_BY_2_PLUS_1];
for ch_buf in &spectrum_buffer.buffer[idx_at_delay][..num_render_channels] {
for (x2_val, &buf_val) in x2_at_delay.iter_mut().zip(ch_buf.iter()) {
*x2_val += buf_val;
}
}
for x2_val in &mut x2_at_delay {
*x2_val *= normalizer;
}
let reverb_power = reverb_model.reverb();
for (rps, (x2_val, &rev_val)) in reverb_power_spectrum
.iter_mut()
.zip(x2_at_delay.iter().zip(reverb_power.iter()))
{
*rps = *x2_val + rev_val;
}
} else {
reverb_model.update_reverb_no_freq_shaping(
&spectrum_buffer.buffer[idx_past][0],
1.0,
reverb_decay,
);
let reverb_power = reverb_model.reverb();
for (rps, (&spec_val, &rev_val)) in reverb_power_spectrum.iter_mut().zip(
spectrum_buffer.buffer[idx_at_delay][0]
.iter()
.zip(reverb_power.iter()),
) {
*rps = spec_val + rev_val;
}
}
}
#[derive(Debug)]
struct InitialState {
conservative_initial_phase: bool,
initial_state_seconds: f32,
transition_triggered: bool,
initial_state: bool,
strong_not_saturated_render_blocks: usize,
}
impl InitialState {
fn new(config: &EchoCanceller3Config) -> Self {
let mut s = Self {
conservative_initial_phase: config.filter.conservative_initial_phase,
initial_state_seconds: config.filter.initial_state_seconds,
transition_triggered: false,
initial_state: true,
strong_not_saturated_render_blocks: 0,
};
s.reset();
s
}
fn reset(&mut self) {
self.initial_state = true;
self.strong_not_saturated_render_blocks = 0;
}
fn update(&mut self, active_render: bool, saturated_capture: bool) {
self.strong_not_saturated_render_blocks += if active_render && !saturated_capture {
1
} else {
0
};
let prev_initial_state = self.initial_state;
if self.conservative_initial_phase {
self.initial_state =
self.strong_not_saturated_render_blocks < 5 * NUM_BLOCKS_PER_SECOND;
} else {
self.initial_state = (self.strong_not_saturated_render_blocks as f32)
< self.initial_state_seconds * NUM_BLOCKS_PER_SECOND as f32;
}
self.transition_triggered = !self.initial_state && prev_initial_state;
}
fn transition_triggered(&self) -> bool {
self.transition_triggered
}
}
#[derive(Debug)]
struct FilterDelay {
delay_headroom_blocks: i32,
filter_delays_blocks: Vec<i32>,
min_filter_delay: i32,
external_delay: Option<DelayEstimate>,
}
impl FilterDelay {
fn new(config: &EchoCanceller3Config, num_capture_channels: usize) -> Self {
let delay_headroom_blocks = (config.delay.delay_headroom_samples / BLOCK_SIZE) as i32;
Self {
delay_headroom_blocks,
filter_delays_blocks: vec![delay_headroom_blocks; num_capture_channels],
min_filter_delay: delay_headroom_blocks,
external_delay: None,
}
}
fn external_delay_reported(&self) -> bool {
self.external_delay.is_some()
}
fn direct_path_filter_delays(&self) -> &[i32] {
&self.filter_delays_blocks
}
fn min_direct_path_filter_delay(&self) -> i32 {
self.min_filter_delay
}
fn update(
&mut self,
analyzer_filter_delay_estimates_blocks: &[i32],
external_delay: &Option<DelayEstimate>,
blocks_with_proper_filter_adaptation: usize,
) {
if let Some(ext) = external_delay
&& (self.external_delay.is_none() || self.external_delay.unwrap().delay != ext.delay)
{
self.external_delay = Some(*ext);
}
let delay_estimator_may_not_have_converged =
blocks_with_proper_filter_adaptation < 2 * NUM_BLOCKS_PER_SECOND;
if delay_estimator_may_not_have_converged && self.external_delay.is_some() {
let delay_guess = self.delay_headroom_blocks;
self.filter_delays_blocks.fill(delay_guess);
} else {
debug_assert_eq!(
self.filter_delays_blocks.len(),
analyzer_filter_delay_estimates_blocks.len()
);
self.filter_delays_blocks
.copy_from_slice(analyzer_filter_delay_estimates_blocks);
}
self.min_filter_delay = *self
.filter_delays_blocks
.iter()
.min()
.unwrap_or(&self.delay_headroom_blocks);
}
}
#[derive(Debug)]
struct FilteringQualityAnalyzer {
use_linear_filter: bool,
overall_usable_linear_estimates: bool,
filter_update_blocks_since_reset: usize,
filter_update_blocks_since_start: usize,
convergence_seen: bool,
usable_linear_filter_estimates: Vec<bool>,
}
impl FilteringQualityAnalyzer {
fn new(config: &EchoCanceller3Config, num_capture_channels: usize) -> Self {
Self {
use_linear_filter: config.filter.use_linear_filter,
overall_usable_linear_estimates: false,
filter_update_blocks_since_reset: 0,
filter_update_blocks_since_start: 0,
convergence_seen: false,
usable_linear_filter_estimates: vec![false; num_capture_channels],
}
}
fn linear_filter_usable(&self) -> bool {
self.overall_usable_linear_estimates
}
fn usable_linear_filter_outputs(&self) -> &[bool] {
&self.usable_linear_filter_estimates
}
fn reset(&mut self) {
self.usable_linear_filter_estimates.fill(false);
self.overall_usable_linear_estimates = false;
self.filter_update_blocks_since_reset = 0;
}
fn update(
&mut self,
active_render: bool,
transparent_mode: bool,
saturated_capture: bool,
external_delay: &Option<DelayEstimate>,
any_filter_converged: bool,
) {
let filter_update = active_render && !saturated_capture;
self.filter_update_blocks_since_reset += if filter_update { 1 } else { 0 };
self.filter_update_blocks_since_start += if filter_update { 1 } else { 0 };
self.convergence_seen = self.convergence_seen || any_filter_converged;
let sufficient_data_to_converge_at_startup =
self.filter_update_blocks_since_start as f32 > NUM_BLOCKS_PER_SECOND as f32 * 0.4;
let sufficient_data_to_converge_at_reset = sufficient_data_to_converge_at_startup
&& self.filter_update_blocks_since_reset as f32 > NUM_BLOCKS_PER_SECOND as f32 * 0.2;
self.overall_usable_linear_estimates =
sufficient_data_to_converge_at_startup && sufficient_data_to_converge_at_reset;
self.overall_usable_linear_estimates = self.overall_usable_linear_estimates
&& (external_delay.is_some() || self.convergence_seen);
self.overall_usable_linear_estimates =
self.overall_usable_linear_estimates && !transparent_mode;
if self.use_linear_filter {
self.usable_linear_filter_estimates
.fill(self.overall_usable_linear_estimates);
}
}
}
#[derive(Debug)]
struct SaturationDetector {
saturated_echo: bool,
}
impl SaturationDetector {
fn new() -> Self {
Self {
saturated_echo: false,
}
}
fn saturated_echo(&self) -> bool {
self.saturated_echo
}
fn update(
&mut self,
x: &Block,
saturated_capture: bool,
usable_linear_estimate: bool,
subtractor_output: &[SubtractorOutput],
echo_path_gain: f32,
) {
self.saturated_echo = false;
if !saturated_capture {
return;
}
if usable_linear_estimate {
const SATURATION_THRESHOLD: f32 = 20000.0;
for output in subtractor_output {
self.saturated_echo = self.saturated_echo
|| output.s_refined_max_abs > SATURATION_THRESHOLD
|| output.s_coarse_max_abs > SATURATION_THRESHOLD;
}
} else {
let mut max_sample = 0.0f32;
for ch in 0..x.num_channels() {
let x_ch = x.view(0, ch);
for &sample in x_ch {
max_sample = max_sample.max(sample.abs());
}
}
const MARGIN: f32 = 10.0;
let peak_echo_amplitude = max_sample * echo_path_gain * MARGIN;
self.saturated_echo = self.saturated_echo || peak_echo_amplitude > 32000.0;
}
}
}
#[derive(Debug)]
pub(crate) struct AecState {
config: EchoCanceller3Config,
num_capture_channels: usize,
deactivate_initial_state_reset_at_echo_path_change: bool,
full_reset_at_echo_path_change: bool,
subtractor_analyzer_reset_at_echo_path_change: bool,
initial_state: InitialState,
delay_state: FilterDelay,
transparent_state: Option<TransparentMode>,
filter_quality_state: FilteringQualityAnalyzer,
saturation_detector: SaturationDetector,
erl_estimator: ErlEstimator,
erle_estimator: ErleEstimator,
strong_not_saturated_render_blocks: usize,
blocks_with_active_render: usize,
capture_signal_saturation: bool,
filter_analyzer: FilterAnalyzer,
echo_audibility: EchoAudibility,
reverb_model_estimator: ReverbModelEstimator,
avg_render_reverb: ReverbModel,
subtractor_output_analyzer: SubtractorOutputAnalyzer,
}
impl AecState {
pub(crate) fn new(config: &EchoCanceller3Config, num_capture_channels: usize) -> Self {
Self {
config: config.clone(),
num_capture_channels,
deactivate_initial_state_reset_at_echo_path_change: false,
full_reset_at_echo_path_change: true,
subtractor_analyzer_reset_at_echo_path_change: true,
initial_state: InitialState::new(config),
delay_state: FilterDelay::new(config, num_capture_channels),
transparent_state: TransparentMode::create(config),
filter_quality_state: FilteringQualityAnalyzer::new(config, num_capture_channels),
saturation_detector: SaturationDetector::new(),
erl_estimator: ErlEstimator::new(2 * NUM_BLOCKS_PER_SECOND),
erle_estimator: ErleEstimator::new(
2 * NUM_BLOCKS_PER_SECOND,
config,
num_capture_channels,
),
strong_not_saturated_render_blocks: 0,
blocks_with_active_render: 0,
capture_signal_saturation: false,
filter_analyzer: FilterAnalyzer::new(config, num_capture_channels),
echo_audibility: EchoAudibility::new(
config.echo_audibility.use_stationarity_properties_at_init,
),
reverb_model_estimator: ReverbModelEstimator::new(config, num_capture_channels),
avg_render_reverb: ReverbModel::new(),
subtractor_output_analyzer: SubtractorOutputAnalyzer::new(num_capture_channels),
}
}
pub(crate) fn usable_linear_estimate(&self) -> bool {
self.filter_quality_state.linear_filter_usable() && self.config.filter.use_linear_filter
}
pub(crate) fn use_linear_filter_output(&self) -> bool {
self.filter_quality_state.linear_filter_usable() && self.config.filter.use_linear_filter
}
pub(crate) fn get_residual_echo_scaling(
&self,
residual_scaling: &mut [f32; FFT_LENGTH_BY_2_PLUS_1],
) {
let filter_has_had_time_to_converge = if self.config.filter.conservative_initial_phase {
self.strong_not_saturated_render_blocks as f32 >= 1.5 * NUM_BLOCKS_PER_SECOND as f32
} else {
self.strong_not_saturated_render_blocks as f32 >= 0.8 * NUM_BLOCKS_PER_SECOND as f32
};
self.echo_audibility
.get_residual_echo_scaling(filter_has_had_time_to_converge, residual_scaling);
}
pub(crate) fn use_stationarity_properties(&self) -> bool {
self.config.echo_audibility.use_stationarity_properties
}
pub(crate) fn erle(&self, onset_compensated: bool) -> &[[f32; FFT_LENGTH_BY_2_PLUS_1]] {
self.erle_estimator.erle(onset_compensated)
}
pub(crate) fn erle_unbounded(&self) -> &[[f32; FFT_LENGTH_BY_2_PLUS_1]] {
self.erle_estimator.erle_unbounded()
}
pub(crate) fn fullband_erle_log2(&self) -> f32 {
self.erle_estimator.fullband_erle_log2()
}
pub(crate) fn erl_time_domain(&self) -> f32 {
self.erl_estimator.erl_time_domain()
}
pub(crate) fn min_direct_path_filter_delay(&self) -> i32 {
self.delay_state.min_direct_path_filter_delay()
}
pub(crate) fn saturated_capture(&self) -> bool {
self.capture_signal_saturation
}
pub(crate) fn saturated_echo(&self) -> bool {
self.saturation_detector.saturated_echo()
}
pub(crate) fn update_capture_saturation(&mut self, capture_signal_saturation: bool) {
self.capture_signal_saturation = capture_signal_saturation;
}
pub(crate) fn transparent_mode_active(&self) -> bool {
self.transparent_state.as_ref().is_some_and(|t| t.active())
}
pub(crate) fn handle_echo_path_change(&mut self, echo_path_variability: &EchoPathVariability) {
if self.full_reset_at_echo_path_change
&& echo_path_variability.delay_change != DelayAdjustment::None
{
self.filter_analyzer.reset();
self.capture_signal_saturation = false;
self.strong_not_saturated_render_blocks = 0;
self.blocks_with_active_render = 0;
if !self.deactivate_initial_state_reset_at_echo_path_change {
self.initial_state.reset();
}
if let Some(ref mut ts) = self.transparent_state {
ts.reset();
}
self.erle_estimator.reset(true);
self.erl_estimator.reset();
self.filter_quality_state.reset();
} else if echo_path_variability.gain_change {
self.erle_estimator.reset(false);
}
if self.subtractor_analyzer_reset_at_echo_path_change {
self.subtractor_output_analyzer.handle_echo_path_change();
}
}
pub(crate) fn reverb_decay(&self, mild: bool) -> f32 {
self.reverb_model_estimator.reverb_decay(mild)
}
pub(crate) fn get_reverb_frequency_response(&self) -> &[f32; FFT_LENGTH_BY_2_PLUS_1] {
self.reverb_model_estimator.get_reverb_frequency_response()
}
pub(crate) fn transition_triggered(&self) -> bool {
self.initial_state.transition_triggered()
}
pub(crate) fn filter_length_blocks(&self) -> usize {
self.filter_analyzer.filter_length_blocks()
}
pub(crate) fn update(&mut self, ctx: &AecStateUpdate<'_>) {
debug_assert_eq!(self.num_capture_channels, ctx.y2.len());
debug_assert_eq!(self.num_capture_channels, ctx.subtractor_output.len());
debug_assert_eq!(
self.num_capture_channels,
ctx.adaptive_filter_frequency_responses.len()
);
debug_assert_eq!(
self.num_capture_channels,
ctx.adaptive_filter_impulse_responses.len()
);
let (any_filter_converged, any_coarse_filter_converged, all_filters_diverged) = self
.subtractor_output_analyzer
.update(ctx.subtractor_output);
let mut any_filter_consistent = false;
let mut max_echo_path_gain = 0.0f32;
self.filter_analyzer.update(
ctx.adaptive_filter_impulse_responses,
ctx.render_buffer,
&mut any_filter_consistent,
&mut max_echo_path_gain,
);
if self.config.filter.use_linear_filter {
self.delay_state.update(
self.filter_analyzer.filter_delays_blocks(),
ctx.external_delay,
self.strong_not_saturated_render_blocks,
);
}
let aligned_render_block = ctx
.render_buffer
.get_block(-self.delay_state.min_direct_path_filter_delay());
let mut active_render = false;
for ch in 0..aligned_render_block.num_channels() {
let block = aligned_render_block.view(0, ch);
let render_energy: f32 = block.iter().map(|&v| v * v).sum();
if render_energy
> (self.config.render_levels.active_render_limit
* self.config.render_levels.active_render_limit)
* FFT_LENGTH_BY_2 as f32
{
active_render = true;
break;
}
}
self.blocks_with_active_render += if active_render { 1 } else { 0 };
self.strong_not_saturated_render_blocks += if active_render && !self.saturated_capture() {
1
} else {
0
};
let mut avg_render_spectrum_with_reverb = [0.0f32; FFT_LENGTH_BY_2_PLUS_1];
compute_avg_render_reverb(
ctx.render_buffer.get_spectrum_buffer(),
self.delay_state.min_direct_path_filter_delay(),
self.reverb_decay(false),
&mut self.avg_render_reverb,
&mut avg_render_spectrum_with_reverb,
);
if self.config.echo_audibility.use_stationarity_properties {
self.echo_audibility.update(
ctx.render_buffer,
self.avg_render_reverb.reverb(),
self.delay_state.min_direct_path_filter_delay(),
self.delay_state.external_delay_reported(),
);
}
if self.initial_state.transition_triggered() {
self.erle_estimator.reset(false);
}
self.erle_estimator.update(
ctx.render_buffer,
ctx.adaptive_filter_frequency_responses,
&avg_render_spectrum_with_reverb,
ctx.y2,
ctx.e2_refined,
self.subtractor_output_analyzer.converged_filters(),
);
self.erl_estimator.update(
self.subtractor_output_analyzer.converged_filters(),
ctx.render_buffer
.spectrum(self.delay_state.min_direct_path_filter_delay()),
ctx.y2,
);
if self.config.ep_strength.echo_can_saturate {
self.saturation_detector.update(
aligned_render_block,
self.saturated_capture(),
self.usable_linear_estimate(),
ctx.subtractor_output,
max_echo_path_gain,
);
} else {
debug_assert!(!self.saturation_detector.saturated_echo());
}
self.initial_state
.update(active_render, self.saturated_capture());
let saturated_capture = self.saturated_capture();
if let Some(ref mut ts) = self.transparent_state {
ts.update(&TransparentModeState {
filter_delay_blocks: self.delay_state.min_direct_path_filter_delay(),
any_filter_consistent,
any_filter_converged,
any_coarse_filter_converged,
all_filters_diverged,
active_render,
saturated_capture,
});
}
self.filter_quality_state.update(
active_render,
self.transparent_mode_active(),
self.saturated_capture(),
ctx.external_delay,
any_filter_converged,
);
let stationary_block = self.config.echo_audibility.use_stationarity_properties
&& self.echo_audibility.is_block_stationary();
self.reverb_model_estimator.update(
self.filter_analyzer.get_adjusted_filters(),
ctx.adaptive_filter_frequency_responses,
self.erle_estimator.get_inst_linear_quality_estimates(),
self.delay_state.direct_path_filter_delays(),
self.filter_quality_state.usable_linear_filter_outputs(),
stationary_block,
);
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn initial_state_is_not_usable() {
let config = EchoCanceller3Config::default();
let state = AecState::new(&config, 1);
assert!(!state.usable_linear_estimate());
assert!(!state.saturated_capture());
assert!(!state.saturated_echo());
assert!(!state.transparent_mode_active());
}
#[test]
fn capture_saturation_flag() {
let config = EchoCanceller3Config::default();
let mut state = AecState::new(&config, 1);
assert!(!state.saturated_capture());
state.update_capture_saturation(true);
assert!(state.saturated_capture());
state.update_capture_saturation(false);
assert!(!state.saturated_capture());
}
}