1use crate::analysis_data::*;
11use crate::kiss_fft::{KissCpx, KissFftState, opus_fft_impl};
12
13pub const NB_FRAMES: usize = 8;
14pub const NB_TBANDS: usize = 18;
15pub const ANALYSIS_BUF_SIZE: usize = 720; pub const DETECT_SIZE: usize = 100;
17pub const ANALYSIS_COUNT_MAX: i32 = 10000;
18pub const LEAK_BANDS: usize = 19;
19const NB_TONAL_SKIP_BANDS: usize = 9;
20const TRANSITION_PENALTY: f32 = 10.0;
21const LEAKAGE_OFFSET: f32 = 2.5;
22const LEAKAGE_SLOPE: f32 = 2.0;
23
24#[derive(Clone, Copy, Debug)]
26pub struct AnalysisInfo {
27 pub valid: bool,
28 pub tonality: f32,
29 pub tonality_slope: f32,
30 pub noisiness: f32,
31 pub activity: f32,
32 pub music_prob: f32,
33 pub music_prob_min: f32,
34 pub music_prob_max: f32,
35 pub bandwidth: i32,
36 pub activity_probability: f32,
37 pub max_pitch_ratio: f32,
38 pub leak_boost: [u8; LEAK_BANDS],
40}
41
42impl Default for AnalysisInfo {
43 fn default() -> Self {
44 AnalysisInfo {
45 valid: false,
46 tonality: 0.0,
47 tonality_slope: 0.0,
48 noisiness: 0.0,
49 activity: 0.0,
50 music_prob: 0.0,
51 music_prob_min: 0.0,
52 music_prob_max: 0.0,
53 bandwidth: 0,
54 activity_probability: 0.0,
55 max_pitch_ratio: 0.0,
56 leak_boost: [0; LEAK_BANDS],
57 }
58 }
59}
60
61const WEIGHTS_SCALE: f32 = 1.0 / 128.0;
64const MAX_NEURONS: usize = 32;
65
66fn tansig_approx(x: f32) -> f32 {
67 if !(x < 8.0) {
69 return 1.0;
70 }
71 if !(x > -8.0) {
72 return -1.0;
73 }
74 if x.is_nan() {
75 return 0.0;
76 }
77 let (x, sign) = if x < 0.0 { (-x, -1.0f32) } else { (x, 1.0f32) };
78 let i = (0.5 + 25.0 * x).floor() as usize;
79 let x = x - 0.04 * i as f32;
80 let y = TANSIG_TABLE[i];
81 let dy = 1.0 - y * y;
82 let y = y + x * dy * (1.0 - y * x);
83 sign * y
84}
85
86fn sigmoid_approx(x: f32) -> f32 {
87 0.5 + 0.5 * tansig_approx(0.5 * x)
88}
89
90fn gemm_accum(out: &mut [f32], weights: &[i8], rows: usize, cols: usize, col_stride: usize, x: &[f32]) {
91 for i in 0..rows {
92 for j in 0..cols {
93 out[i] += weights[j * col_stride + i] as f32 * x[j];
94 }
95 }
96}
97
98struct DenseLayer {
99 bias: &'static [i8],
100 input_weights: &'static [i8],
101 nb_inputs: usize,
102 nb_neurons: usize,
103 sigmoid: bool,
104}
105
106struct GruLayer {
107 bias: &'static [i8],
108 input_weights: &'static [i8],
109 recurrent_weights: &'static [i8],
110 nb_inputs: usize,
111 nb_neurons: usize,
112}
113
114const LAYER0: DenseLayer = DenseLayer {
115 bias: &LAYER0_BIAS,
116 input_weights: &LAYER0_WEIGHTS,
117 nb_inputs: 25,
118 nb_neurons: 32,
119 sigmoid: false,
120};
121const LAYER1: GruLayer = GruLayer {
122 bias: &LAYER1_BIAS,
123 input_weights: &LAYER1_WEIGHTS,
124 recurrent_weights: &LAYER1_RECUR_WEIGHTS,
125 nb_inputs: 32,
126 nb_neurons: 24,
127};
128const LAYER2: DenseLayer = DenseLayer {
129 bias: &LAYER2_BIAS,
130 input_weights: &LAYER2_WEIGHTS,
131 nb_inputs: 24,
132 nb_neurons: 2,
133 sigmoid: true,
134};
135
136fn compute_dense(layer: &DenseLayer, output: &mut [f32], input: &[f32]) {
137 let (m, n) = (layer.nb_inputs, layer.nb_neurons);
138 for i in 0..n {
139 output[i] = layer.bias[i] as f32;
140 }
141 gemm_accum(output, layer.input_weights, n, m, n, input);
142 for o in output.iter_mut().take(n) {
143 *o *= WEIGHTS_SCALE;
144 *o = if layer.sigmoid {
145 sigmoid_approx(*o)
146 } else {
147 tansig_approx(*o)
148 };
149 }
150}
151
152fn compute_gru(gru: &GruLayer, state: &mut [f32], input: &[f32]) {
153 let (m, n) = (gru.nb_inputs, gru.nb_neurons);
154 let stride = 3 * n;
155 let mut z = [0.0f32; MAX_NEURONS];
156 let mut r = [0.0f32; MAX_NEURONS];
157 let mut h = [0.0f32; MAX_NEURONS];
158 let mut tmp = [0.0f32; MAX_NEURONS];
159
160 for i in 0..n {
162 z[i] = gru.bias[i] as f32;
163 }
164 gemm_accum(&mut z, gru.input_weights, n, m, stride, input);
165 gemm_accum(&mut z, gru.recurrent_weights, n, n, stride, state);
166 for zi in z.iter_mut().take(n) {
167 *zi = sigmoid_approx(WEIGHTS_SCALE * *zi);
168 }
169
170 for i in 0..n {
172 r[i] = gru.bias[n + i] as f32;
173 }
174 gemm_accum(&mut r, &gru.input_weights[n..], n, m, stride, input);
175 gemm_accum(&mut r, &gru.recurrent_weights[n..], n, n, stride, state);
176 for ri in r.iter_mut().take(n) {
177 *ri = sigmoid_approx(WEIGHTS_SCALE * *ri);
178 }
179
180 for i in 0..n {
182 h[i] = gru.bias[2 * n + i] as f32;
183 }
184 for i in 0..n {
185 tmp[i] = state[i] * r[i];
186 }
187 gemm_accum(&mut h, &gru.input_weights[2 * n..], n, m, stride, input);
188 gemm_accum(&mut h, &gru.recurrent_weights[2 * n..], n, n, stride, &tmp);
189 for i in 0..n {
190 state[i] = z[i] * state[i] + (1.0 - z[i]) * tansig_approx(WEIGHTS_SCALE * h[i]);
191 }
192}
193
194fn fast_atan2f(y: f32, x: f32) -> f32 {
197 const CA: f32 = 0.43157974;
198 const CB: f32 = 0.67848403;
199 const CC: f32 = 0.08595542;
200 const CE: f32 = std::f32::consts::PI / 2.0;
201 let x2 = x * x;
202 let y2 = y * y;
203 if x2 + y2 < 1e-18 {
204 return 0.0;
205 }
206 if x2 < y2 {
207 let den = (y2 + CB * x2) * (y2 + CC * x2);
208 -x * y * (y2 + CA * x2) / den + if y < 0.0 { -CE } else { CE }
209 } else {
210 let den = (x2 + CB * y2) * (x2 + CC * y2);
211 x * y * (x2 + CA * y2) / den + if y < 0.0 { -CE } else { CE }
212 - if x * y < 0.0 { -CE } else { CE }
213 }
214}
215
216fn resampler_down2_hp(s: &mut [f32; 3], out: &mut [f32], input: &[f32]) -> f32 {
219 let len2 = input.len() / 2;
220 let mut hp_ener = 0.0f64;
221 for k in 0..len2 {
222 let in32 = input[2 * k];
223 let y = in32 - s[0];
224 let x = 0.6074371 * y;
225 let out32 = s[0] + x;
226 s[0] = in32 + x;
227 let mut out32_hp = out32;
228
229 let in32 = input[2 * k + 1];
230 let y = in32 - s[1];
231 let x = 0.15063 * y;
232 let out32 = out32 + s[1] + x;
233 s[1] = in32 + x;
234
235 let y = -in32 - s[2];
236 let x = 0.15063 * y;
237 out32_hp = out32_hp + s[2] + x;
238 s[2] = -in32 + x;
239
240 hp_ener += (out32_hp as f64) * (out32_hp as f64);
241 out[k] = 0.5 * out32;
242 }
243 hp_ener as f32
244}
245
246#[allow(clippy::too_many_arguments)]
250fn downmix_and_resample(
251 x: &[f32],
252 y: &mut [f32],
253 s: &mut [f32; 3],
254 subframe: usize,
255 offset: usize,
256 channels: usize,
257 fs: i32,
258) -> f32 {
259 if subframe == 0 {
260 return 0.0;
261 }
262 let (subframe, offset) = match fs {
263 48000 => (subframe * 2, offset * 2),
264 16000 => (subframe * 2 / 3, offset * 2 / 3),
265 _ => (subframe, offset),
266 };
267 let scale = 1.0f32 / channels as f32;
269 let mut tmp = vec![0.0f32; subframe];
270 for (j, t) in tmp.iter_mut().enumerate() {
271 let mut sum = 0.0f32;
272 for c in 0..channels {
273 sum += x[(offset + j) * channels + c];
274 }
275 *t = sum * scale;
276 }
277 match fs {
278 48000 => resampler_down2_hp(s, y, &tmp),
279 24000 => {
280 y[..subframe].copy_from_slice(&tmp);
281 0.0
282 }
283 16000 => {
284 let mut tmp3x = vec![0.0f32; 3 * subframe];
286 for j in 0..subframe {
287 tmp3x[3 * j] = tmp[j];
288 tmp3x[3 * j + 1] = tmp[j];
289 tmp3x[3 * j + 2] = tmp[j];
290 }
291 resampler_down2_hp(s, y, &tmp3x)
292 }
293 _ => 0.0,
294 }
295}
296
297pub struct TonalityAnalysisState {
300 pub fs: i32,
301 angle: [f32; 240],
302 d_angle: [f32; 240],
303 d2_angle: [f32; 240],
304 inmem: [f32; ANALYSIS_BUF_SIZE],
305 mem_fill: usize,
306 prev_band_tonality: [f32; NB_TBANDS],
307 prev_tonality: f32,
308 prev_bandwidth: i32,
309 e: [[f32; NB_TBANDS]; NB_FRAMES],
310 log_e: [[f32; NB_TBANDS]; NB_FRAMES],
311 low_e: [f32; NB_TBANDS],
312 high_e: [f32; NB_TBANDS],
313 mean_e: [f32; NB_TBANDS + 1],
314 mem: [f32; 32],
315 cmean: [f32; 8],
316 std: [f32; 9],
317 etracker: f32,
318 low_e_count: f32,
319 e_count: usize,
320 count: i32,
321 analysis_offset: i32,
322 write_pos: usize,
323 read_pos: usize,
324 read_subframe: i32,
325 hp_ener_accum: f32,
326 initialized: bool,
327 rnn_state: [f32; MAX_NEURONS],
328 downmix_state: [f32; 3],
329 info: [AnalysisInfo; DETECT_SIZE],
330}
331
332impl TonalityAnalysisState {
333 pub fn new(fs: i32) -> Self {
334 TonalityAnalysisState {
335 fs,
336 angle: [0.0; 240],
337 d_angle: [0.0; 240],
338 d2_angle: [0.0; 240],
339 inmem: [0.0; ANALYSIS_BUF_SIZE],
340 mem_fill: 0,
341 prev_band_tonality: [0.0; NB_TBANDS],
342 prev_tonality: 0.0,
343 prev_bandwidth: 0,
344 e: [[0.0; NB_TBANDS]; NB_FRAMES],
345 log_e: [[0.0; NB_TBANDS]; NB_FRAMES],
346 low_e: [0.0; NB_TBANDS],
347 high_e: [0.0; NB_TBANDS],
348 mean_e: [0.0; NB_TBANDS + 1],
349 mem: [0.0; 32],
350 cmean: [0.0; 8],
351 std: [0.0; 9],
352 etracker: 0.0,
353 low_e_count: 0.0,
354 e_count: 0,
355 count: 0,
356 analysis_offset: 0,
357 write_pos: 0,
358 read_pos: 0,
359 read_subframe: 0,
360 hp_ener_accum: 0.0,
361 initialized: false,
362 rnn_state: [0.0; MAX_NEURONS],
363 downmix_state: [0.0; 3],
364 info: [AnalysisInfo::default(); DETECT_SIZE],
365 }
366 }
367
368 pub fn initialized(&self) -> bool {
369 self.initialized
370 }
371
372 pub fn reset(&mut self) {
373 let fs = self.fs;
374 *self = TonalityAnalysisState::new(fs);
375 }
376}
377
378pub fn tonality_get_info(tonal: &mut TonalityAnalysisState, len: usize) -> AnalysisInfo {
382 let mut pos = tonal.read_pos as i32;
383 let mut curr_lookahead = tonal.write_pos as i32 - tonal.read_pos as i32;
384 if curr_lookahead < 0 {
385 curr_lookahead += DETECT_SIZE as i32;
386 }
387
388 tonal.read_subframe += len as i32 / (tonal.fs / 400);
389 while tonal.read_subframe >= 8 {
390 tonal.read_subframe -= 8;
391 tonal.read_pos += 1;
392 }
393 if tonal.read_pos >= DETECT_SIZE {
394 tonal.read_pos -= DETECT_SIZE;
395 }
396
397 if len as i32 > tonal.fs / 50 && pos != tonal.write_pos as i32 {
399 pos += 1;
400 if pos == DETECT_SIZE as i32 {
401 pos = 0;
402 }
403 }
404 if pos == tonal.write_pos as i32 {
405 pos -= 1;
406 }
407 if pos < 0 {
408 pos = DETECT_SIZE as i32 - 1;
409 }
410 let pos0 = pos;
411 let mut info = tonal.info[pos as usize];
412 if !info.valid {
413 return info;
414 }
415 let mut tonality_max = info.tonality;
416 let mut tonality_avg = info.tonality;
417 let mut tonality_count = 1;
418 let mut bandwidth_span = 6;
420 for _ in 0..3 {
422 pos += 1;
423 if pos == DETECT_SIZE as i32 {
424 pos = 0;
425 }
426 if pos == tonal.write_pos as i32 {
427 break;
428 }
429 tonality_max = tonality_max.max(tonal.info[pos as usize].tonality);
430 tonality_avg += tonal.info[pos as usize].tonality;
431 tonality_count += 1;
432 info.bandwidth = info.bandwidth.max(tonal.info[pos as usize].bandwidth);
433 bandwidth_span -= 1;
434 }
435 pos = pos0;
436 for _ in 0..bandwidth_span {
438 pos -= 1;
439 if pos < 0 {
440 pos = DETECT_SIZE as i32 - 1;
441 }
442 if pos == tonal.write_pos as i32 {
443 break;
444 }
445 info.bandwidth = info.bandwidth.max(tonal.info[pos as usize].bandwidth);
446 }
447 info.tonality = (tonality_avg / tonality_count as f32).max(tonality_max - 0.2);
448
449 let mut mpos = pos0;
450 let mut vpos = pos0;
451 if curr_lookahead > 15 {
454 mpos += 5;
455 if mpos >= DETECT_SIZE as i32 {
456 mpos -= DETECT_SIZE as i32;
457 }
458 vpos += 1;
459 if vpos >= DETECT_SIZE as i32 {
460 vpos -= DETECT_SIZE as i32;
461 }
462 }
463
464 let mut prob_min = 1.0f32;
466 let mut prob_max = 0.0f32;
467 let vad_prob = tonal.info[vpos as usize].activity_probability;
468 let mut prob_count = 0.1f32.max(vad_prob);
469 let mut prob_avg = 0.1f32.max(vad_prob) * tonal.info[mpos as usize].music_prob;
470 loop {
471 mpos += 1;
472 if mpos == DETECT_SIZE as i32 {
473 mpos = 0;
474 }
475 if mpos == tonal.write_pos as i32 {
476 break;
477 }
478 vpos += 1;
479 if vpos == DETECT_SIZE as i32 {
480 vpos = 0;
481 }
482 if vpos == tonal.write_pos as i32 {
483 break;
484 }
485 let pos_vad = tonal.info[vpos as usize].activity_probability;
486 prob_min = ((prob_avg - TRANSITION_PENALTY * (vad_prob - pos_vad)) / prob_count).min(prob_min);
487 prob_max = ((prob_avg + TRANSITION_PENALTY * (vad_prob - pos_vad)) / prob_count).max(prob_max);
488 prob_count += 0.1f32.max(pos_vad);
489 prob_avg += 0.1f32.max(pos_vad) * tonal.info[mpos as usize].music_prob;
490 }
491 info.music_prob = prob_avg / prob_count;
492 prob_min = (prob_avg / prob_count).min(prob_min);
493 prob_max = (prob_avg / prob_count).max(prob_max);
494 prob_min = prob_min.max(0.0);
495 prob_max = prob_max.min(1.0);
496
497 if curr_lookahead < 10 {
499 let mut pmin = prob_min;
500 let mut pmax = prob_max;
501 let mut pos = pos0;
502 for _ in 0..(tonal.count - 1).min(15).max(0) {
504 pos -= 1;
505 if pos < 0 {
506 pos = DETECT_SIZE as i32 - 1;
507 }
508 pmin = pmin.min(tonal.info[pos as usize].music_prob);
509 pmax = pmax.max(tonal.info[pos as usize].music_prob);
510 }
511 pmin = 0.0f32.max(pmin - 0.1 * vad_prob);
513 pmax = 1.0f32.min(pmax + 0.1 * vad_prob);
514 prob_min += (1.0 - 0.1 * curr_lookahead as f32) * (pmin - prob_min);
515 prob_max += (1.0 - 0.1 * curr_lookahead as f32) * (pmax - prob_max);
516 }
517 info.music_prob_min = prob_min;
518 info.music_prob_max = prob_max;
519 info
520}
521
522#[allow(clippy::needless_range_loop)]
525fn tonality_analysis(
526 tonal: &mut TonalityAnalysisState,
527 kfft: &KissFftState,
528 x: &[f32],
529 len: usize,
530 offset: usize,
531 channels: usize,
532 lsb_depth: i32,
533) {
534 const N: usize = 480;
535 const N2: usize = 240;
536 let pi4 = (std::f64::consts::PI.powi(4)) as f32;
537
538 if !tonal.initialized {
539 tonal.mem_fill = 240;
540 tonal.initialized = true;
541 }
542 let alpha = 1.0 / (10.min(1 + tonal.count) as f32);
543 let alpha_e = 1.0 / (25.min(1 + tonal.count) as f32);
544 let mut alpha_e2 = 1.0 / (100.min(1 + tonal.count) as f32);
546 if tonal.count <= 1 {
547 alpha_e2 = 1.0;
548 }
549
550 let (mut len, mut offset) = (len, offset);
551 if tonal.fs == 48000 {
552 len /= 2;
553 offset /= 2;
554 } else if tonal.fs == 16000 {
555 len = 3 * len / 2;
556 offset = 3 * offset / 2;
557 }
558
559 {
560 let fill = (len).min(ANALYSIS_BUF_SIZE - tonal.mem_fill);
561 let mut seg = vec![0.0f32; fill.max(1)];
562 let hp = downmix_and_resample(
563 x,
564 &mut seg,
565 &mut tonal.downmix_state,
566 fill,
567 offset,
568 channels,
569 tonal.fs,
570 );
571 tonal.hp_ener_accum += hp;
572 let mf = tonal.mem_fill;
573 tonal.inmem[mf..mf + fill].copy_from_slice(&seg[..fill]);
574 }
575
576 if tonal.mem_fill + len < ANALYSIS_BUF_SIZE {
577 tonal.mem_fill += len;
578 return;
580 }
581 let hp_ener = tonal.hp_ener_accum;
582 let write_pos_now = tonal.write_pos;
583 tonal.write_pos += 1;
584 if tonal.write_pos >= DETECT_SIZE {
585 tonal.write_pos -= DETECT_SIZE;
586 }
587
588 let silence_thresh = 1.0f32 / (1i64 << lsb_depth) as f32;
590 let is_silence = tonal
591 .inmem
592 .iter()
593 .fold(0.0f32, |m, &v| m.max(v.abs()))
594 <= silence_thresh;
595
596 let mut fft_in = vec![KissCpx::new(0.0, 0.0); N];
597 let mut fft_out = vec![KissCpx::new(0.0, 0.0); N];
598 let mut tonality = [0.0f32; 240];
599 let mut noisiness = [0.0f32; 240];
600 for i in 0..N2 {
601 let w = ANALYSIS_WINDOW[i];
602 fft_in[i] = KissCpx::new(w * tonal.inmem[i], w * tonal.inmem[N2 + i]);
603 fft_in[N - i - 1] = KissCpx::new(
604 w * tonal.inmem[N - i - 1],
605 w * tonal.inmem[N + N2 - i - 1],
606 );
607 }
608 tonal.inmem.copy_within(ANALYSIS_BUF_SIZE - 240..ANALYSIS_BUF_SIZE, 0);
609 let remaining = len - (ANALYSIS_BUF_SIZE - tonal.mem_fill);
610 {
611 let mut seg = vec![0.0f32; remaining.max(1)];
612 let hp = downmix_and_resample(
613 x,
614 &mut seg,
615 &mut tonal.downmix_state,
616 remaining,
617 offset + ANALYSIS_BUF_SIZE - tonal.mem_fill,
618 channels,
619 tonal.fs,
620 );
621 tonal.hp_ener_accum = hp;
622 tonal.inmem[240..240 + remaining].copy_from_slice(&seg[..remaining]);
623 }
624 tonal.mem_fill = 240 + remaining;
625
626 if is_silence {
627 let prev_pos = (write_pos_now + DETECT_SIZE - 1) % DETECT_SIZE;
629 tonal.info[write_pos_now] = tonal.info[prev_pos];
630 return;
631 }
632
633 let scale = kfft.scale();
635 for (i, v) in fft_in.iter().enumerate() {
636 fft_out[kfft.bitrev[i] as usize] = KissCpx::new(scale * v.r, scale * v.i);
637 }
638 opus_fft_impl(kfft, &mut fft_out);
639 let out = &fft_out;
640
641 let info_idx = write_pos_now;
642 if out[0].r.is_nan() {
643 tonal.info[info_idx].valid = false;
644 return;
645 }
646
647 let a = &mut tonal.angle;
648 let da = &mut tonal.d_angle;
649 let d2a = &mut tonal.d2_angle;
650 let mut tonality2 = [0.0f32; 240];
651 for i in 1..N2 {
652 let x1r = out[i].r + out[N - i].r;
653 let x1i = out[i].i - out[N - i].i;
654 let x2r = out[i].i + out[N - i].i;
655 let x2i = out[N - i].r - out[i].r;
656
657 let angle = (0.5 / std::f64::consts::PI) as f32 * fast_atan2f(x1i, x1r);
658 let d_angle = angle - a[i];
659 let d2_angle = d_angle - da[i];
660
661 let angle2 = (0.5 / std::f64::consts::PI) as f32 * fast_atan2f(x2i, x2r);
662 let d_angle2 = angle2 - angle;
663 let d2_angle2 = d_angle2 - d_angle;
664
665 let mut mod1 = d2_angle - d2_angle.round_ties_even();
666 noisiness[i] = mod1.abs();
667 mod1 *= mod1;
668 mod1 *= mod1;
669
670 let mut mod2 = d2_angle2 - d2_angle2.round_ties_even();
671 noisiness[i] += mod2.abs();
672 mod2 *= mod2;
673 mod2 *= mod2;
674
675 let avg_mod = 0.25 * (d2a[i] + mod1 + 2.0 * mod2);
676 tonality[i] = 1.0 / (1.0 + 40.0 * 16.0 * pi4 * avg_mod) - 0.015;
678 tonality2[i] = 1.0 / (1.0 + 40.0 * 16.0 * pi4 * mod2) - 0.015;
680
681 a[i] = angle2;
682 da[i] = d_angle2;
683 d2a[i] = mod2;
684 }
685 for i in 2..N2 - 1 {
686 let tt = tonality2[i].min(tonality2[i - 1].max(tonality2[i + 1]));
687 tonality[i] = 0.9 * tonality[i].max(tt - 0.1);
688 }
689
690 let mut frame_tonality = 0.0f32;
691 let mut max_frame_tonality = 0.0f32;
692 let mut frame_noisiness = 0.0f32;
693 let mut frame_stationarity = 0.0f32;
694 if tonal.count == 0 {
695 for b in 0..NB_TBANDS {
696 tonal.low_e[b] = 1e10;
697 tonal.high_e[b] = -1e10;
698 }
699 }
700 let mut relative_e = 0.0f32;
701 let mut frame_loudness = 0.0f32;
702 let mut log_e = [0.0f32; NB_TBANDS];
703 let mut band_log2 = [0.0f32; NB_TBANDS + 1];
704 let mut band_tonality = [0.0f32; NB_TBANDS];
705 let mut slope = 0.0f32;
706 {
708 let x1r = 2.0 * out[0].r;
709 let x2r = 2.0 * out[0].i;
710 let mut e = x1r * x1r + x2r * x2r;
711 for i in 1..4 {
712 let bin_e = out[i].r * out[i].r
713 + out[N - i].r * out[N - i].r
714 + out[i].i * out[i].i
715 + out[N - i].i * out[N - i].i;
716 e += bin_e;
717 }
718 band_log2[0] = 0.5 * 1.442695 * ((e + 1e-10) as f64).ln() as f32;
719 }
720 for b in 0..NB_TBANDS {
721 let mut e = 0.0f32;
722 let mut t_e = 0.0f32;
723 let mut n_e = 0.0f32;
724 for i in TBANDS[b]..TBANDS[b + 1] {
725 let bin_e = out[i].r * out[i].r
726 + out[N - i].r * out[N - i].r
727 + out[i].i * out[i].i
728 + out[N - i].i * out[N - i].i;
729 e += bin_e;
730 t_e += bin_e * 0.0f32.max(tonality[i]);
731 n_e += bin_e * 2.0 * (0.5 - noisiness[i]);
732 }
733 if !(e < 1e9) || e.is_nan() {
735 tonal.info[info_idx].valid = false;
736 return;
737 }
738
739 tonal.e[tonal.e_count][b] = e;
740 frame_noisiness += n_e / (1e-15 + e);
741
742 frame_loudness += ((e + 1e-10) as f64).sqrt() as f32;
743 log_e[b] = ((e + 1e-10) as f64).ln() as f32;
744 band_log2[b + 1] = 0.5 * 1.442695 * log_e[b];
745 tonal.log_e[tonal.e_count][b] = log_e[b];
746 if tonal.count == 0 {
747 tonal.high_e[b] = log_e[b];
748 tonal.low_e[b] = log_e[b];
749 }
750 if tonal.high_e[b] > tonal.low_e[b] + 7.5 {
751 if tonal.high_e[b] - log_e[b] > log_e[b] - tonal.low_e[b] {
752 tonal.high_e[b] -= 0.01;
753 } else {
754 tonal.low_e[b] += 0.01;
755 }
756 }
757 if log_e[b] > tonal.high_e[b] {
758 tonal.high_e[b] = log_e[b];
759 tonal.low_e[b] = tonal.low_e[b].max(tonal.high_e[b] - 15.0);
760 } else if log_e[b] < tonal.low_e[b] {
761 tonal.low_e[b] = log_e[b];
762 tonal.high_e[b] = tonal.high_e[b].min(tonal.low_e[b] + 15.0);
763 }
764 relative_e += (log_e[b] - tonal.low_e[b]) / (1e-5 + (tonal.high_e[b] - tonal.low_e[b]));
765
766 let mut l1 = 0.0f32;
767 let mut l2 = 0.0f32;
768 for i in 0..NB_FRAMES {
769 l1 += (tonal.e[i][b] as f64).sqrt() as f32;
770 l2 += tonal.e[i][b];
771 }
772
773 let mut stationarity = (l1 / (1e-15 + NB_FRAMES as f64 * l2 as f64).sqrt() as f32).min(0.99);
774 stationarity *= stationarity;
775 stationarity *= stationarity;
776 frame_stationarity += stationarity;
777 band_tonality[b] = (t_e / (1e-15 + e)).max(stationarity * tonal.prev_band_tonality[b]);
778 frame_tonality += band_tonality[b];
779 if b >= NB_TBANDS - NB_TONAL_SKIP_BANDS {
780 frame_tonality -= band_tonality[b + NB_TONAL_SKIP_BANDS - NB_TBANDS];
788 }
789 max_frame_tonality =
790 max_frame_tonality.max((1.0 + 0.03 * (b as f32 - NB_TBANDS as f32)) * frame_tonality);
791 slope += band_tonality[b] * (b as f32 - 8.0);
792 tonal.prev_band_tonality[b] = band_tonality[b];
793 }
794
795 let mut leakage_from = [0.0f32; NB_TBANDS + 1];
796 let mut leakage_to = [0.0f32; NB_TBANDS + 1];
797 leakage_from[0] = band_log2[0];
798 leakage_to[0] = band_log2[0] - LEAKAGE_OFFSET;
799 for b in 1..NB_TBANDS + 1 {
800 let leak_slope = LEAKAGE_SLOPE * (TBANDS[b] - TBANDS[b - 1]) as f32 / 4.0;
801 leakage_from[b] = (leakage_from[b - 1] + leak_slope).min(band_log2[b]);
802 leakage_to[b] = (leakage_to[b - 1] - leak_slope).max(band_log2[b] - LEAKAGE_OFFSET);
803 }
804 for b in (0..NB_TBANDS - 1).rev() {
805 let leak_slope = LEAKAGE_SLOPE * (TBANDS[b + 1] - TBANDS[b]) as f32 / 4.0;
806 leakage_from[b] = (leakage_from[b + 1] + leak_slope).min(leakage_from[b]);
807 leakage_to[b] = (leakage_to[b + 1] - leak_slope).max(leakage_to[b]);
808 }
809 for b in 0..NB_TBANDS + 1 {
810 let boost = 0.0f32.max(leakage_to[b] - band_log2[b])
813 + 0.0f32.max(band_log2[b] - (leakage_from[b] + LEAKAGE_OFFSET));
814 tonal.info[info_idx].leak_boost[b] = 255.min((0.5 + 64.0 * boost).floor() as i32) as u8;
815 }
816
817 let mut spec_variability = 0.0f32;
818 for i in 0..NB_FRAMES {
819 let mut mindist = 1e15f32;
820 for j in 0..NB_FRAMES {
821 let mut dist = 0.0f32;
822 for k in 0..NB_TBANDS {
823 let tmp = tonal.log_e[i][k] - tonal.log_e[j][k];
824 dist += tmp * tmp;
825 }
826 if j != i {
827 mindist = mindist.min(dist);
828 }
829 }
830 spec_variability += mindist;
831 }
832 spec_variability = ((spec_variability / NB_FRAMES as f32 / NB_TBANDS as f32) as f64).sqrt() as f32;
833
834 let mut bandwidth_mask = 0.0f32;
835 let mut bandwidth = 0i32;
836 let mut max_e = 0.0f32;
837 let lsb = 0.max(lsb_depth - 8);
838 let mut noise_floor = 5.7e-4 / (1u32 << lsb) as f32;
839 noise_floor *= noise_floor;
840 let mut below_max_pitch = 0.0f32;
841 let mut above_max_pitch = 0.0f32;
842 let mut is_masked = [false; NB_TBANDS + 1];
843 for b in 0..NB_TBANDS {
844 let band_start = TBANDS[b];
845 let band_end = TBANDS[b + 1];
846 let mut e = 0.0f32;
847 for i in band_start..band_end {
848 let bin_e = out[i].r * out[i].r
849 + out[N - i].r * out[N - i].r
850 + out[i].i * out[i].i
851 + out[N - i].i * out[N - i].i;
852 e += bin_e;
853 }
854 max_e = max_e.max(e);
855 if band_start < 64 {
856 below_max_pitch += e;
857 } else {
858 above_max_pitch += e;
859 }
860 tonal.mean_e[b] = ((1.0 - alpha_e2) * tonal.mean_e[b]).max(e);
861 let em = e.max(tonal.mean_e[b]);
862 if e * 1e9 > max_e
864 && (em > 3.0 * noise_floor * (band_end - band_start) as f32
865 || e > noise_floor * (band_end - band_start) as f32)
866 {
867 bandwidth = b as i32 + 1;
868 }
869 is_masked[b] = e
870 < (if tonal.prev_bandwidth >= b as i32 + 1 {
871 0.01
872 } else {
873 0.05
874 }) * bandwidth_mask;
875 bandwidth_mask = (0.05 * bandwidth_mask).max(e);
877 }
878 let bw_raw_dbg = bandwidth;
879 let (mut hp_e_dbg, mut hp_thresh_dbg) = (0.0f32, 0.0f32);
880 if tonal.fs == 48000 {
882 let noise_ratio = if tonal.prev_bandwidth == 20 { 10.0 } else { 30.0 };
883 let e = hp_ener * (1.0 / (60.0 * 60.0));
884 above_max_pitch += e;
885 tonal.mean_e[NB_TBANDS] = ((1.0 - alpha_e2) * tonal.mean_e[NB_TBANDS]).max(e);
886 let em = e.max(tonal.mean_e[NB_TBANDS]);
887 hp_e_dbg = e;
888 hp_thresh_dbg = noise_ratio * noise_floor * 160.0;
889 if em > 3.0 * noise_ratio * noise_floor * 160.0 || e > noise_ratio * noise_floor * 160.0 {
890 bandwidth = 20;
891 }
892 is_masked[NB_TBANDS] = e
893 < (if tonal.prev_bandwidth == 20 { 0.01 } else { 0.05 }) * bandwidth_mask;
894 }
895 let bw_before_mask_dbg = bandwidth;
896 tonal.info[info_idx].max_pitch_ratio = if above_max_pitch > below_max_pitch {
897 below_max_pitch / above_max_pitch
898 } else {
899 1.0
900 };
901 if bandwidth == 20 && is_masked[NB_TBANDS] {
903 bandwidth -= 2;
904 } else if bandwidth > 0 && bandwidth <= NB_TBANDS as i32 && is_masked[bandwidth as usize - 1] {
905 bandwidth -= 1;
906 }
907 let bw_before_warmup = bandwidth;
908 if tonal.count <= 2 {
909 bandwidth = 20;
910 }
911 if std::env::var_os("RUSTY_OPUS_BW_DEBUG").is_some() && tonal.count < 40 {
915 eprintln!(
916 "BWDBG count={:3} bw_raw={:2} bw_premask={:2} bw_final={:2} \
917 hp_ener={:.6e} hp_e={:.6e} thresh={:.6e} masked_hp={} noise_floor={:.3e}",
918 tonal.count,
919 bw_raw_dbg,
920 bw_before_mask_dbg,
921 bandwidth,
922 hp_ener,
923 hp_e_dbg,
924 hp_thresh_dbg,
925 is_masked[NB_TBANDS] as u8,
926 noise_floor,
927 );
928 let _ = bw_before_warmup;
929 }
930 frame_loudness = 20.0 * (frame_loudness as f64).log10() as f32;
931 tonal.etracker = (tonal.etracker - 0.003).max(frame_loudness);
932 tonal.low_e_count *= 1.0 - alpha_e;
933 if frame_loudness < tonal.etracker - 30.0 {
934 tonal.low_e_count += alpha_e;
935 }
936
937 let mut bfcc = [0.0f32; 8];
938 let mut mid_e = [0.0f32; 8];
939 for i in 0..8 {
940 let mut sum = 0.0f32;
941 for b in 0..16 {
942 sum += DCT_TABLE[i * 16 + b] * log_e[b];
943 }
944 bfcc[i] = sum;
945 }
946 for i in 0..8 {
947 let mut sum = 0.0f32;
948 for b in 0..16 {
949 sum += DCT_TABLE[i * 16 + b] * 0.5 * (tonal.high_e[b] + tonal.low_e[b]);
950 }
951 mid_e[i] = sum;
952 }
953
954 frame_stationarity /= NB_TBANDS as f32;
955 relative_e /= NB_TBANDS as f32;
956 if tonal.count < 10 {
957 relative_e = 0.5;
958 }
959 frame_noisiness /= NB_TBANDS as f32;
960 tonal.info[info_idx].activity = frame_noisiness + (1.0 - frame_noisiness) * relative_e;
961 let mut frame_tonality = max_frame_tonality / (NB_TBANDS - NB_TONAL_SKIP_BANDS) as f32;
962 frame_tonality = frame_tonality.max(tonal.prev_tonality * 0.8);
963 tonal.prev_tonality = frame_tonality;
964
965 slope /= 64.0;
966 tonal.info[info_idx].tonality_slope = slope;
967
968 tonal.e_count = (tonal.e_count + 1) % NB_FRAMES;
969 tonal.count = (tonal.count + 1).min(ANALYSIS_COUNT_MAX);
970 tonal.info[info_idx].tonality = frame_tonality;
971
972 let mut features = [0.0f32; 25];
973 for i in 0..4 {
974 features[i] = -0.12299 * (bfcc[i] + tonal.mem[i + 24])
975 + 0.49195 * (tonal.mem[i] + tonal.mem[i + 16])
976 + 0.69693 * tonal.mem[i + 8]
977 - 1.4349 * tonal.cmean[i];
978 }
979 for i in 0..4 {
980 tonal.cmean[i] = (1.0 - alpha) * tonal.cmean[i] + alpha * bfcc[i];
981 }
982 for i in 0..4 {
983 features[4 + i] =
984 0.63246 * (bfcc[i] - tonal.mem[i + 24]) + 0.31623 * (tonal.mem[i] - tonal.mem[i + 16]);
985 }
986 for i in 0..3 {
987 features[8 + i] = 0.53452 * (bfcc[i] + tonal.mem[i + 24])
988 - 0.26726 * (tonal.mem[i] + tonal.mem[i + 16])
989 - 0.53452 * tonal.mem[i + 8];
990 }
991
992 if tonal.count > 5 {
993 for i in 0..9 {
994 tonal.std[i] = (1.0 - alpha) * tonal.std[i] + alpha * features[i] * features[i];
995 }
996 }
997 for i in 0..4 {
998 features[i] = bfcc[i] - mid_e[i];
999 }
1000
1001 for i in 0..8 {
1002 tonal.mem[i + 24] = tonal.mem[i + 16];
1003 tonal.mem[i + 16] = tonal.mem[i + 8];
1004 tonal.mem[i + 8] = tonal.mem[i];
1005 tonal.mem[i] = bfcc[i];
1006 }
1007 for i in 0..9 {
1008 features[11 + i] = (tonal.std[i] as f64).sqrt() as f32 - STD_FEATURE_BIAS[i];
1009 }
1010 features[18] = spec_variability - 0.78;
1011 features[20] = tonal.info[info_idx].tonality - 0.154723;
1012 features[21] = tonal.info[info_idx].activity - 0.724643;
1013 features[22] = frame_stationarity - 0.743717;
1014 features[23] = tonal.info[info_idx].tonality_slope + 0.069216;
1015 features[24] = tonal.low_e_count - 0.067930;
1016
1017 let mut layer_out = [0.0f32; MAX_NEURONS];
1018 let mut frame_probs = [0.0f32; 2];
1019 compute_dense(&LAYER0, &mut layer_out, &features);
1020 let mut rnn_state = tonal.rnn_state;
1021 compute_gru(&LAYER1, &mut rnn_state, &layer_out);
1022 tonal.rnn_state = rnn_state;
1023 compute_dense(&LAYER2, &mut frame_probs, &tonal.rnn_state);
1024
1025 tonal.info[info_idx].activity_probability = frame_probs[1];
1027 tonal.info[info_idx].music_prob = frame_probs[0];
1028
1029 tonal.info[info_idx].bandwidth = bandwidth;
1030 tonal.prev_bandwidth = bandwidth;
1031 tonal.info[info_idx].noisiness = frame_noisiness;
1032 tonal.info[info_idx].valid = true;
1033}
1034
1035pub fn run_analysis(
1038 analysis: &mut TonalityAnalysisState,
1039 kfft: &KissFftState,
1040 analysis_pcm: &[f32],
1041 analysis_frame_size: usize,
1042 frame_size: usize,
1043 channels: usize,
1044 fs: i32,
1045 lsb_depth: i32,
1046) -> AnalysisInfo {
1047 let mut analysis_frame_size = analysis_frame_size & !1;
1048 analysis_frame_size = analysis_frame_size.min((DETECT_SIZE - 5) * fs as usize / 50);
1050
1051 let mut pcm_len = analysis_frame_size as i32 - analysis.analysis_offset;
1052 let mut offset = analysis.analysis_offset;
1053 while pcm_len > 0 {
1054 tonality_analysis(
1055 analysis,
1056 kfft,
1057 analysis_pcm,
1058 (fs as usize / 50).min(pcm_len as usize),
1059 offset as usize,
1060 channels,
1061 lsb_depth,
1062 );
1063 offset += fs / 50;
1064 pcm_len -= fs / 50;
1065 }
1066 analysis.analysis_offset = analysis_frame_size as i32;
1067 analysis.analysis_offset -= frame_size as i32;
1068
1069 tonality_get_info(analysis, frame_size)
1070}