use crate::error::check_range;
use crate::filter::{Biquad, Design};
use crate::lpc;
use crate::numeric::fixed::{Q14, Q6};
use crate::numeric::{db_to_gain, fast_sinc, mul14, round_q14, NumericContext};
use crate::{Error, SampleRate};
#[derive(Clone, Copy, Debug, PartialEq)]
#[non_exhaustive]
#[cfg_attr(
all(feature = "wasm", target_arch = "wasm32"),
derive(serde::Serialize, serde::Deserialize, tsify::Tsify),
serde(rename_all = "camelCase")
)]
pub struct FormantConfig {
pub enabled: bool,
pub width: f32,
pub stretch: f32,
pub aspiration_mix: f32,
pub aspiration_cutoff_hz: f32,
}
impl Default for FormantConfig {
fn default() -> Self {
Self {
enabled: false,
width: 1.0,
stretch: 1.0,
aspiration_mix: 0.0,
aspiration_cutoff_hz: 4000.0,
}
}
}
impl FormantConfig {
pub fn validate(self) -> Result<(), Error> {
check_range("resynthesis.formant.width", self.width, 0.5, 2.0)?;
check_range("resynthesis.formant.stretch", self.stretch, 0.5, 2.0)?;
let field = "resynthesis.formant.aspiration_mix";
check_range(field, self.aspiration_mix, 0.0, 1.0)?;
let field = "resynthesis.formant.aspiration_cutoff_hz";
check_range(field, self.aspiration_cutoff_hz, 500.0, 10000.0)
}
}
pub(crate) const HISTORY_FRAMES: usize = 11568;
pub(crate) const ANALYSIS_FRAMES: usize = 441;
const ORDER: usize = 50;
const MAX_ORDER: usize = 110;
const BANKS: usize = 30;
#[derive(Clone, Copy, Debug)]
struct Stamp {
current: usize,
previous: usize,
blend: i32,
}
impl Default for Stamp {
fn default() -> Self {
Self {
current: 0,
previous: 0,
blend: 16384,
}
}
}
#[derive(Debug)]
pub(crate) struct Formant {
config: FormantConfig,
rate: SampleRate,
input: Vec<i32>,
temporary: Vec<i32>,
inverse: Vec<[i32; MAX_ORDER]>,
models: Vec<[i32; MAX_ORDER]>,
orders: [usize; BANKS],
stamps: Vec<Stamp>,
current: usize,
previous: usize,
blend: i32,
design_counter: i32,
history: [[i32; MAX_ORDER]; 2],
history_index: usize,
correlation_history: [[i32; 111]; 2],
correlation_sum: [i32; 111],
smooth_index: usize,
smooth_filled: usize,
smooth_reset: bool,
glottal: i32,
width: i32,
width_squared: i32,
stretch: i32,
stretch_count: usize,
stretch_map: [i32; MAX_ORDER],
areas: [i32; MAX_ORDER],
stretched_areas: [i32; MAX_ORDER],
noise_filter: Biquad,
noise_gain: i32,
noise_amplitude: i32,
}
impl Formant {
pub(crate) fn new(rate: SampleRate, config: FormantConfig) -> Self {
let d1 = -std::f64::consts::LN_2 / 0.9_f64.ln();
let d2 = rate.hz() as f64 * d1 / 44100.0;
let mut result = Self {
config,
rate,
input: vec![0; HISTORY_FRAMES],
temporary: vec![0_i32; (2 * (rate.as_u32() as usize / 40) + MAX_ORDER + 256).max(2865)],
inverse: vec![[0; MAX_ORDER]; BANKS],
models: vec![[0; MAX_ORDER]; BANKS],
orders: [0; BANKS],
stamps: vec![Stamp::default(); HISTORY_FRAMES],
current: BANKS - 1,
previous: 0,
blend: 0,
design_counter: 0,
history: [[0; MAX_ORDER]; 2],
history_index: 0,
correlation_history: [[0; 111]; 2],
correlation_sum: [0; 111],
smooth_index: 0,
smooth_filled: 0,
smooth_reset: true,
glottal: (0.5_f64.powf(1.0 / d2) * 16384.0) as i32,
width: 16384,
width_squared: 16384,
stretch: 16384,
stretch_count: ORDER,
stretch_map: [0; MAX_ORDER],
areas: [0; MAX_ORDER],
stretched_areas: [0; MAX_ORDER],
noise_filter: Biquad::new(),
noise_gain: 0_i32,
noise_amplitude: 0_i32,
};
result.set_config(config);
result.reset();
result
}
pub(crate) fn set_config(&mut self, config: FormantConfig) {
self.config = config;
self.width = (config.width as f64 * 16384.0) as i32;
self.width_squared = (config.width as f64 * config.width as f64 * 16384.0) as i32;
self.stretch = (config.stretch as f64 * 16384.0) as i32;
self.stretch_count =
(((ORDER - 1) as f64 * config.stretch as f64 + 1.001) as usize).min(MAX_ORDER);
for i in 0..self.stretch_count - 1 {
let fraction = i as f64 / (self.stretch_count - 1) as f64;
self.stretch_map[i] = ((ORDER - 1) as f64 * fraction * 128.0) as i32;
}
self.stretch_map[self.stretch_count - 1] = ((ORDER - 1) * 128) as i32;
self.noise_gain = (config.aspiration_mix as f64 * 16384.0) as i32;
let frequency = (config.aspiration_cutoff_hz as i32)
.min((self.rate.hz() as f64 * 0.48) as i32)
.max(500);
self.noise_amplitude = (frequency as f64 * 0.8 / self.rate.hz() as f64 * 16384.0) as i32;
self.noise_filter
.set_design(Design::high_pass(self.rate.hz(), frequency as f32, 0.707));
}
pub(crate) fn reset(&mut self) {
self.input.fill(0_i32);
self.stamps.fill(Stamp::default());
self.history = [[0; MAX_ORDER]; 2];
self.history_index = 0;
self.current = BANKS - 1;
self.next_filter();
self.next_filter();
self.design_counter = 0;
self.noise_filter.reset();
self.smooth_reset = true;
}
pub(crate) fn detection_mode_off(&mut self) {
self.smooth_reset = true;
self.design_counter = 0;
}
#[inline]
pub(crate) fn remove_glottal(&self, sample: i32, previous: i32) -> i32 {
sample.wrapping_sub(round_q14(previous.wrapping_mul(self.glottal)))
}
#[inline]
pub(crate) fn model_glottal(&self, sample: i32, previous: i32) -> i32 {
sample.wrapping_add(round_q14(previous.wrapping_mul(self.glottal)))
}
pub(crate) fn process_analysis(
&mut self,
sample: i32,
previous: i32,
periods: &[Q6],
write: usize,
output_period_q6: i32,
) -> i32 {
self.input[write] = self.remove_glottal(sample, previous);
self.design(periods, write, output_period_q6);
self.inverse(write)
}
fn next_filter(&mut self) {
self.previous = self.current;
self.current += 1;
if self.current == BANKS {
self.current = 0;
}
self.blend = 0;
self.inverse[self.current].fill(0_i32);
self.models[self.current].fill(0_i32);
self.orders[self.current] = 0;
}
fn smooth(&mut self, values: &mut [i32]) {
if self.smooth_reset {
self.smooth_index = 0;
self.smooth_filled = 0;
self.smooth_reset = false;
self.correlation_sum[..values.len()].fill(0_i32);
}
for (i, value) in values.iter_mut().enumerate() {
if self.smooth_filled == 2 {
self.correlation_sum[i] = self.correlation_sum[i]
.wrapping_sub(self.correlation_history[self.smooth_index][i]);
}
*value = value.wrapping_div(2);
self.correlation_history[self.smooth_index][i] = *value;
self.correlation_sum[i] = self.correlation_sum[i].wrapping_add(*value);
}
self.smooth_filled = (self.smooth_filled + 1).min(2);
for (i, value) in values.iter_mut().enumerate() {
*value = if self.smooth_filled < 2 {
self.correlation_sum[i]
.wrapping_div(self.smooth_filled as i32)
.wrapping_mul(2)
} else {
self.correlation_sum[i]
};
}
self.smooth_index = (self.smooth_index + 1) % 2;
}
fn design(&mut self, periods: &[Q6], position: usize, output_period_q6: i32) {
let period = output_period_q6.wrapping_shr(6).max(0) as usize;
let design_now = self.design_counter == 0;
self.design_counter += 1;
let one = 16384;
if design_now {
self.design_counter -= ANALYSIS_FRAMES as i32;
let count;
if period == 0 {
count = (self.rate.as_u32() as usize + 50) / 100;
for i in 0..count {
self.temporary[i] =
self.input[(position + HISTORY_FRAMES - i) % HISTORY_FRAMES];
}
self.temporary[count..count + ORDER].fill(0_i32);
} else {
count = if periods[position] == Q6::ZERO {
period * 2
} else {
period
};
let extra = (period / 10).max(1);
if periods[position] == Q6::ZERO {
for i in 0..count + extra {
self.temporary[i] =
self.input[(position + HISTORY_FRAMES - i) % HISTORY_FRAMES];
}
} else {
for i in 0..count + extra {
let index = (position + HISTORY_FRAMES - i) % HISTORY_FRAMES;
let mut accumulator = (index as i32) << 6;
let mut sum = 0_i32;
let mut reads = 0;
for _ in 0..2 {
let index = accumulator
.wrapping_add(32)
.wrapping_shr(6)
.rem_euclid(HISTORY_FRAMES as i32)
as usize;
sum = sum.wrapping_add(self.input[index]);
reads += 1;
if periods[index] == Q6::ZERO {
break;
}
accumulator = accumulator.wrapping_sub(periods[index].to_bits());
}
self.temporary[i] = sum.wrapping_div(reads);
}
}
let mut blend = 16384;
let step = 16384 / extra as i32;
for i in 0..extra {
self.temporary[i] = mul14(16384 - blend, self.temporary[i])
.wrapping_add(mul14(blend, self.temporary[i + count]));
blend -= step;
}
for i in count..count + ORDER {
self.temporary[i] = self.temporary[i - count];
}
}
let mut previous = self.temporary[count - 1];
for sample in &mut self.temporary[..count + ORDER] {
let current = *sample;
*sample = current.wrapping_sub(round_q14(previous.wrapping_mul(self.glottal)));
previous = current;
}
let mut correlations = [0_i32; 111];
let mut shift = 0;
for lag in 0..=ORDER {
let mut correlation = 0_i32;
let (left_chunks, left_tail) = self.temporary[..count].as_chunks::<4>();
let (right_chunks, right_tail) = self.temporary[lag..lag + count].as_chunks::<4>();
let mut accumulate = |product: i32| {
correlation = correlation.wrapping_add(product.wrapping_shr(shift));
if correlation.wrapping_abs() > 0x4000000 {
shift += 1;
for prior in &mut correlations[..lag] {
*prior = prior.wrapping_shr(1);
}
correlation = correlation.wrapping_shr(1);
}
};
for (left, right) in left_chunks.iter().zip(right_chunks) {
let products = [
left[0].wrapping_mul(right[0]),
left[1].wrapping_mul(right[1]),
left[2].wrapping_mul(right[2]),
left[3].wrapping_mul(right[3]),
];
for product in products {
accumulate(product);
}
}
for (&left, &right) in left_tail.iter().zip(right_tail) {
accumulate(left.wrapping_mul(right));
}
correlations[lag] = correlation;
}
let mut normalization = 0;
while correlations[0] > one {
correlations[0] = correlations[0].wrapping_shr(1);
normalization += 1;
}
for value in &mut correlations[1..=ORDER] {
*value = value.wrapping_shr(normalization);
}
let scale = (8192 / if period != 0 { period } else { count })
.min((2.0 / ORDER as f64 * 16384.0) as usize) as i32;
let mut accumulator = 0;
for value in &mut correlations[1..=ORDER] {
accumulator += scale;
let sinc = fast_sinc(accumulator);
*value = mul14(*value, mul14(sinc, sinc));
}
self.smooth(&mut correlations[..=ORDER]);
self.next_filter();
let design_gain = ((db_to_gain(-60.0) + 1.0) * 16384.0) as i32;
correlations[0] = mul14(correlations[0], design_gain).wrapping_add(200);
let mut parcor = [0; MAX_ORDER];
let mut scratch = [0; 112];
let valid = lpc::durbin(
&correlations,
&mut self.inverse[self.current][..ORDER],
&mut parcor,
&mut scratch,
);
if valid {
if self.width != one || self.stretch != one {
if let Some(scale) = lpc::parcor_to_areas(&parcor[..ORDER], &mut self.areas) {
for i in 0..self.stretch_count {
let map = self.stretch_map[i];
let index = map.wrapping_shr(7).min((ORDER - 2) as i32) as usize;
let fraction = map.wrapping_sub((index as i32) << 7).wrapping_shl(7);
self.stretched_areas[i] = mul14(
mul14(one.wrapping_sub(fraction), self.areas[index])
.wrapping_add(mul14(fraction, self.areas[index + 1])),
self.width_squared,
);
if self.stretched_areas[i] == 0_i32 {
self.stretched_areas[i] = 1;
}
}
if lpc::areas_to_parcor(
&self.stretched_areas[..self.stretch_count],
&mut parcor,
scale,
) {
lpc::parcor_to_pef(
&parcor[..self.stretch_count],
&mut self.models[self.current],
&mut scratch,
14,
);
self.orders[self.current] = self.stretch_count;
}
}
} else {
self.models[self.current][..ORDER]
.copy_from_slice(&self.inverse[self.current][..ORDER]);
self.orders[self.current] = ORDER;
}
} else {
self.smooth_reset = true;
}
}
if self.blend < 16384 {
self.blend = (self.blend + 1638).min(16384);
}
self.stamps[position] = Stamp {
current: self.current,
previous: self.previous,
blend: self.blend,
};
}
fn inverse(&self, position: usize) -> i32 {
let read = if position < ANALYSIS_FRAMES {
position + HISTORY_FRAMES - ANALYSIS_FRAMES
} else {
position - ANALYSIS_FRAMES
};
let mut cursor = read;
let mut sum = 0_i32;
for i in 0..ORDER {
let coefficient = if self.blend < 16384 {
mul14(16384 - self.blend, self.inverse[self.previous][i])
.wrapping_add(mul14(self.blend, self.inverse[self.current][i]))
} else {
self.inverse[self.current][i]
};
cursor = if cursor == 0 {
HISTORY_FRAMES - 1
} else {
cursor - 1
};
sum = sum.wrapping_add(self.input[cursor].wrapping_mul(coefficient));
}
self.input[read].wrapping_sub(round_q14(sum))
}
fn add_noise(&mut self, sample: i32, context: &mut NumericContext) -> i32 {
if self.config.aspiration_mix <= 0.01 {
return sample;
}
let random = context.random.next_u31() >> 17;
let noise = mul14(random - 8192, sample);
let filtered = self
.noise_filter
.process_sample(Q14::from_bits(noise))
.to_bits();
let noise = mul14(self.noise_amplitude, filtered).wrapping_shl(6);
mul14(sample, 16384_i32.wrapping_sub(self.noise_gain))
.wrapping_add(mul14(noise, self.noise_gain))
}
fn coefficients(&self, position: i32, output: &mut [i32; MAX_ORDER]) -> usize {
let stamp = self.stamps[position.rem_euclid(HISTORY_FRAMES as i32) as usize];
let mut order = self.orders[stamp.current];
if stamp.blend < 16384 && self.orders[stamp.previous] != 0 {
order = order.max(self.orders[stamp.previous]);
for (i, value) in output[..order].iter_mut().enumerate() {
*value = mul14(16384 - stamp.blend, self.models[stamp.previous][i])
.wrapping_add(mul14(stamp.blend, self.models[stamp.current][i]));
}
} else {
output[..order].copy_from_slice(&self.models[stamp.current][..order]);
}
order
}
pub(crate) fn forward(
&mut self,
sample: i32,
position: i32,
context: &mut NumericContext,
) -> i32 {
let sample = self.add_noise(sample, context);
let mut coefficients = [0; MAX_ORDER];
let order = self.coefficients(position, &mut coefficients);
let mut sum = 0_i32;
let mut cursor = self.history_index;
for &coefficient in &coefficients[..order] {
cursor = if cursor == 0 {
MAX_ORDER - 1
} else {
cursor - 1
};
sum = sum.wrapping_add(self.history[0][cursor].wrapping_mul(coefficient));
}
let sample = sample.wrapping_add(round_q14(sum));
self.history[0][self.history_index] = sample;
self.history_index += 1;
if self.history_index == MAX_ORDER {
self.history_index = 0;
}
round_q14(sample.wrapping_mul(self.width))
}
pub(crate) fn forward_stereo(
&mut self,
mut samples: [i32; 2],
position: i32,
context: &mut NumericContext,
) -> [i32; 2] {
let mid = samples[0].wrapping_add(samples[1]).wrapping_shr(1);
let noise = self.add_noise(mid, context).wrapping_sub(mid);
for sample in &mut samples {
*sample = sample.wrapping_add(noise);
}
let mut coefficients = [0; MAX_ORDER];
let order = self.coefficients(position, &mut coefficients);
for (channel, sample) in samples.iter_mut().enumerate() {
let mut sum = 0_i32;
for (i, &coefficient) in coefficients[..order].iter().enumerate() {
sum = sum.wrapping_add(
self.history[channel][(self.history_index + MAX_ORDER - i - 1) % MAX_ORDER]
.wrapping_mul(coefficient),
);
}
*sample = sample.wrapping_add(round_q14(sum));
self.history[channel][self.history_index] = *sample;
*sample = round_q14(sample.wrapping_mul(self.width));
}
self.history_index = (self.history_index + 1) % MAX_ORDER;
samples
}
}