#[cfg(not(feature = "std"))]
use crate::compat::Math;
use crate::dnn::Activation;
use crate::dnn::arch::DnnArch;
use crate::in_place::{write_fields, write_zero};
use crate::dnn::freq::{self, FRAME_SIZE, NB_BANDS, fmaf};
use crate::dnn::models::DeepPlcModels;
use crate::dnn::models::plc::{PLC_DENSE_IN_OUT_SIZE, PLC_GRU1_STATE_SIZE, PLC_GRU2_STATE_SIZE, PlcModel};
use crate::dnn::nnet::{compute_generic_dense, compute_generic_gru};
use super::fargan::{FARGAN_CONT_SAMPLES, FarganState, fargan_cont, fargan_synthesize_int};
use super::features::{LpcnetEncState, compute_single_frame_features};
use super::{
CONT_VECTORS, LPCNET_FRAME_SIZE, NB_FEATURES, NB_TOTAL_FEATURES, PLC_BUF_SIZE,
PLC_INPUT_SIZE, PLC_MAX_FEC, max_c, min_c,
};
#[derive(Clone, Copy, PartialEq, Debug)]
pub struct PlcNetState {
pub gru1_state: [f32; PLC_GRU1_STATE_SIZE],
pub gru2_state: [f32; PLC_GRU2_STATE_SIZE],
}
impl PlcNetState {
pub const fn new() -> Self {
PlcNetState { gru1_state: [0.0; PLC_GRU1_STATE_SIZE], gru2_state: [0.0; PLC_GRU2_STATE_SIZE] }
}
}
impl Default for PlcNetState {
fn default() -> Self {
Self::new()
}
}
pub fn compute_plc_pred(
net: &mut PlcNetState,
out: &mut [f32],
input: &[f32; PLC_INPUT_SIZE],
model: &PlcModel,
arch: DnnArch,
) {
let mut tmp = [0.0f32; PLC_DENSE_IN_OUT_SIZE];
compute_generic_dense(&model.plc_dense_in, &mut tmp, input, Activation::Tanh, arch);
compute_generic_gru(&model.plc_gru1_input, &model.plc_gru1_recurrent, &mut net.gru1_state, &tmp, arch);
compute_generic_gru(
&model.plc_gru2_input,
&model.plc_gru2_recurrent,
&mut net.gru2_state,
&net.gru1_state,
arch,
);
compute_generic_dense(&model.plc_dense_out, &mut out[..NB_FEATURES], &net.gru2_state, Activation::Linear, arch);
}
const ATT_TABLE: [f32; 10] = [0.0, 0.0, -0.2, -0.2, -0.4, -0.4, -0.8, -0.8, -1.6, -1.6];
pub const CELT_DECODE_BUFFER_SIZE: usize = 2048;
pub const SINC_ORDER: usize = 48;
const PLC_UPDATE_FRAMES: usize = 4;
const PLC_UPDATE_SAMPLES: usize = PLC_UPDATE_FRAMES * FRAME_SIZE;
const CELT_PREEMPHASIS: f32 = 0.85;
#[allow(clippy::excessive_precision)]
pub const SINC_FILTER: [f32; SINC_ORDER + 1] = [
4.2931e-05, -0.000190293, -0.000816132, -0.000637162, 0.00141662, 0.00354764, 0.00184368,
-0.00428274, -0.00856105, -0.0034003, 0.00930201, 0.0159616, 0.00489785, -0.0169649,
-0.0259484, -0.00596856, 0.0286551, 0.0405872, 0.00649994, -0.0509284, -0.0716655,
-0.00665212, 0.134336, 0.278927, 0.339995, 0.278927, 0.134336, -0.00665212, -0.0716655,
-0.0509284, 0.00649994, 0.0405872, 0.0286551, -0.00596856, -0.0259484, -0.0169649, 0.00489785,
0.0159616, 0.00930201, -0.0034003, -0.00856105, -0.00428274, 0.00184368, 0.00354764,
0.00141662, -0.000637162, -0.000816132, -0.000190293, 4.2931e-05,
];
#[derive(Clone)]
pub struct DeepPlc {
fargan: FarganState,
enc: LpcnetEncState,
fec: [[f32; NB_FEATURES]; PLC_MAX_FEC],
analysis_gap: bool,
fec_read_pos: i32,
fec_fill_pos: i32,
fec_skip: i32,
analysis_pos: i32,
predict_pos: i32,
pcm: [f32; PLC_BUF_SIZE],
blend: bool,
features: [f32; NB_TOTAL_FEATURES],
cont_features: [f32; CONT_VECTORS * NB_FEATURES],
loss_count: i32,
plc_net: PlcNetState,
plc_bak: [PlcNetState; 2],
}
crate::in_place::zeroable_new!(PlcNetState);
impl Default for DeepPlc {
fn default() -> Self {
Self::new()
}
}
impl DeepPlc {
pub fn new() -> Self {
let mut st = core::mem::MaybeUninit::<Self>::uninit();
unsafe {
Self::write_new(st.as_mut_ptr());
st.assume_init()
}
}
pub(crate) unsafe fn write_new(this: *mut Self) {
unsafe {
write_fields!(this, {
fargan: FarganState::new(),
enc = write_zero,
fec = write_zero,
analysis_gap: true,
fec_read_pos: 0,
fec_fill_pos: 0,
fec_skip: 0,
analysis_pos: PLC_BUF_SIZE as i32,
predict_pos: PLC_BUF_SIZE as i32,
pcm = write_zero,
blend: false,
features = write_zero,
cont_features = write_zero,
loss_count: 0,
plc_net = write_zero,
plc_bak = write_zero,
});
}
}
pub fn reset(&mut self) {
self.enc.reset();
for v in self.fec.iter_mut() {
v.fill(0.0);
}
self.analysis_gap = true;
self.fec_read_pos = 0;
self.fec_fill_pos = 0;
self.fec_skip = 0;
self.analysis_pos = PLC_BUF_SIZE as i32;
self.predict_pos = PLC_BUF_SIZE as i32;
self.pcm.fill(0.0);
self.blend = false;
self.features.fill(0.0);
self.cont_features.fill(0.0);
self.loss_count = 0;
self.plc_net = PlcNetState::new();
self.plc_bak = [PlcNetState::new(); 2];
}
pub fn update(&mut self, pcm: &[i16; LPCNET_FRAME_SIZE]) {
let n = FRAME_SIZE as i32;
if self.analysis_pos - n >= 0 {
self.analysis_pos -= n;
} else {
self.analysis_gap = true;
}
if self.predict_pos - n >= 0 {
self.predict_pos -= n;
}
self.append(pcm);
self.loss_count = 0;
self.blend = false;
}
fn append(&mut self, pcm: &[i16; LPCNET_FRAME_SIZE]) {
self.pcm.copy_within(FRAME_SIZE.., 0);
for (h, &s) in self.pcm[PLC_BUF_SIZE - FRAME_SIZE..].iter_mut().zip(pcm.iter()) {
*h = (1.0 / 32768.0) * s as f32;
}
}
pub fn conceal(
&mut self,
models: &DeepPlcModels,
arch: DnnArch,
pcm: &mut [i16; LPCNET_FRAME_SIZE],
) {
let buf_size = PLC_BUF_SIZE as i32;
let n = FRAME_SIZE as i32;
if !self.blend {
let mut count = 0;
self.plc_net = self.plc_bak[0];
while self.analysis_pos + n <= buf_size {
debug_assert!(self.analysis_pos >= 0);
let start = self.analysis_pos as usize;
let mut x = [0.0f32; FRAME_SIZE];
for (v, &h) in x.iter_mut().zip(&self.pcm[start..start + FRAME_SIZE]) {
*v = 32768.0 * h;
}
let mut plc_features = [0.0f32; PLC_INPUT_SIZE];
let mut burg = [0.0f32; 2 * NB_BANDS];
freq::burg_cepstral_analysis(&mut burg, &x);
plc_features[..2 * NB_BANDS].copy_from_slice(&burg);
compute_single_frame_features(
&mut self.enc,
&x,
&mut self.features,
&models.pitchdnn,
arch,
);
if (!self.analysis_gap || count > 0) && self.analysis_pos >= self.predict_pos {
let mut f = [0.0f32; NB_FEATURES];
f.copy_from_slice(&self.features[..NB_FEATURES]);
self.queue_features(&f);
plc_features[2 * NB_BANDS..2 * NB_BANDS + NB_FEATURES].copy_from_slice(&f);
plc_features[2 * NB_BANDS + NB_FEATURES] = 1.0;
self.rotate_backups();
compute_plc_pred(
&mut self.plc_net,
&mut self.features,
&plc_features,
&models.plc,
arch,
);
}
self.analysis_pos += n;
count += 1;
}
for _ in 0..2 {
self.rotate_backups();
self.get_fec_or_pred(&models.plc, arch);
let f = self.first_features();
self.queue_features(&f);
}
let pcm0: &[f32; FARGAN_CONT_SAMPLES] =
match self.pcm[PLC_BUF_SIZE - FARGAN_CONT_SAMPLES..].try_into() {
Ok(p) => p,
Err(_) => unreachable!("the last FARGAN_CONT_SAMPLES of the history"),
};
fargan_cont(&mut self.fargan, pcm0, &self.cont_features, &models.fargan, arch);
self.analysis_gap = false;
}
self.rotate_backups();
if self.get_fec_or_pred(&models.plc, arch) {
self.loss_count = 0;
} else {
self.loss_count = self.loss_count.saturating_add(1);
}
let f0 = if self.loss_count >= 10 {
let ramp = (2 * (self.loss_count as i64 - 9)) as f32;
(self.features[0] + ATT_TABLE[9]) - ramp
} else {
self.features[0] + ATT_TABLE[self.loss_count as usize]
};
self.features[0] = max_c(-15.0, f0);
let f = self.first_features();
fargan_synthesize_int(&mut self.fargan, pcm, &f, &models.fargan, arch);
self.queue_features(&f);
if self.analysis_pos - n >= 0 {
self.analysis_pos -= n;
} else {
self.analysis_gap = true;
}
self.predict_pos = buf_size;
self.append(pcm);
self.blend = true;
}
pub fn update_from_celt(&mut self, decode_mem: &[&[f32]], preemph_mem: &mut f32) {
assert!(matches!(decode_mem.len(), 1 | 2), "one or two channels");
let mut buf48k = [0.0f32; CELT_DECODE_BUFFER_SIZE];
if decode_mem.len() == 1 {
buf48k.copy_from_slice(&decode_mem[0][..CELT_DECODE_BUFFER_SIZE]);
} else {
let (c0, c1) =
(&decode_mem[0][..CELT_DECODE_BUFFER_SIZE], &decode_mem[1][..CELT_DECODE_BUFFER_SIZE]);
for ((b, &a0), &a1) in buf48k.iter_mut().zip(c0).zip(c1) {
*b = 0.5 * (a0 + a1);
}
}
for i in 1..CELT_DECODE_BUFFER_SIZE {
buf48k[i] = fmaf(CELT_PREEMPHASIS, buf48k[i - 1], buf48k[i]);
}
*preemph_mem = buf48k[CELT_DECODE_BUFFER_SIZE - 1];
let offset = CELT_DECODE_BUFFER_SIZE - SINC_ORDER - 1 - 3 * (PLC_UPDATE_SAMPLES - 1);
let mut buf16k = [0i16; PLC_UPDATE_SAMPLES];
for (i, out) in buf16k.iter_mut().enumerate() {
let taps = &buf48k[3 * i + offset..3 * i + offset + SINC_ORDER + 1];
let mut sum = 0.0f32;
for (&x, &h) in taps.iter().zip(SINC_FILTER.iter()) {
sum = fmaf(x, h, sum);
}
*out = min_c(32767.0, max_c(-32767.0, sum)).round_ties_even() as i32 as i16;
}
for frame in buf16k.as_chunks::<LPCNET_FRAME_SIZE>().0 {
self.update(frame);
}
}
pub fn fec_add(&mut self, features: Option<&[f32; NB_FEATURES]>) {
match features {
None => self.fec_skip = self.fec_skip.saturating_add(1),
Some(f) => {
if let Some(slot) = self.fec.get_mut(self.fec_fill_pos as usize) {
*slot = *f;
self.fec_fill_pos += 1;
}
}
}
}
pub fn fec_clear(&mut self) {
self.fec_read_pos = 0;
self.fec_fill_pos = 0;
self.fec_skip = 0;
}
pub fn fec_fill_pos(&self) -> i32 {
self.fec_fill_pos
}
pub fn fec_cursor(&self) -> (i32, i32) {
(self.fec_read_pos, self.fec_skip)
}
pub fn blend(&self) -> bool {
self.blend
}
pub fn clear_blend(&mut self) {
self.blend = false;
}
pub fn features(&self) -> &[f32; NB_TOTAL_FEATURES] {
&self.features
}
pub fn cont_features(&self) -> &[f32; CONT_VECTORS * NB_FEATURES] {
&self.cont_features
}
pub fn history(&self) -> &[f32; PLC_BUF_SIZE] {
&self.pcm
}
pub fn positions(&self) -> (i32, i32, bool, i32) {
(self.analysis_pos, self.predict_pos, self.analysis_gap, self.loss_count)
}
pub fn plc_states(&self) -> (&PlcNetState, &[PlcNetState; 2]) {
(&self.plc_net, &self.plc_bak)
}
pub fn fargan(&self) -> &FarganState {
&self.fargan
}
pub fn analysis(&self) -> &LpcnetEncState {
&self.enc
}
#[inline]
fn rotate_backups(&mut self) {
self.plc_bak[0] = self.plc_bak[1];
self.plc_bak[1] = self.plc_net;
}
#[inline]
fn first_features(&self) -> [f32; NB_FEATURES] {
let mut f = [0.0f32; NB_FEATURES];
f.copy_from_slice(&self.features[..NB_FEATURES]);
f
}
fn queue_features(&mut self, features: &[f32; NB_FEATURES]) {
self.cont_features.copy_within(NB_FEATURES.., 0);
self.cont_features[(CONT_VECTORS - 1) * NB_FEATURES..].copy_from_slice(features);
}
fn get_fec_or_pred(&mut self, model: &PlcModel, arch: DnnArch) -> bool {
if self.fec_read_pos != self.fec_fill_pos && self.fec_skip == 0 {
let f = self.fec[self.fec_read_pos as usize];
self.features[..NB_FEATURES].copy_from_slice(&f);
self.fec_read_pos += 1;
let mut plc_features = [0.0f32; PLC_INPUT_SIZE];
plc_features[2 * NB_BANDS..2 * NB_BANDS + NB_FEATURES].copy_from_slice(&f);
plc_features[2 * NB_BANDS + NB_FEATURES] = -1.0;
let mut discard = [0.0f32; NB_FEATURES];
compute_plc_pred(&mut self.plc_net, &mut discard, &plc_features, model, arch);
true
} else {
let zeros = [0.0f32; PLC_INPUT_SIZE];
compute_plc_pred(&mut self.plc_net, &mut self.features, &zeros, model, arch);
if self.fec_skip > 0 {
self.fec_skip -= 1;
}
false
}
}
}