Skip to main content

threers_codec/opus/
celt.rs

1//! CELT, as ffmpeg n7.1 decodes it (`opus/dec_celt.c`, `celt.c`, `pvq.c`).
2//!
3//! Every float expression keeps its C types: `M_PI`/`M_SQRT1_2`/`1.0`
4//! literals promote to double where the C does, the postfilter transition
5//! runs in double, and the C is not FMA-contracted. The aarch64 reference
6//! build's NEON kernels (`vector_fmul_window`, `vector_fmac_scalar`,
7//! `opus_postfilter`, `opus_deemphasis`) fuse their multiply-adds as the
8//! assembly does.
9//!
10//! The band quantiser's pointer arithmetic is kept as offsets into one
11//! arena (`Mem`) holding both channels' coefficients, the two folding
12//! norms and the scratch band.
13
14#![allow(clippy::too_many_arguments, clippy::needless_range_loop)]
15
16use super::mdct15::Mdct15;
17use super::rc::{ilog, Rc};
18use super::tab::*;
19
20pub const MAX_BANDS: usize = 21;
21
22/// Path counters for coverage checks (`OPUS_STATS=1` in opus_dump).
23pub static STATS: [std::sync::atomic::AtomicUsize; 12] =
24    [const { std::sync::atomic::AtomicUsize::new(0) }; 12];
25pub const STAT_NAMES: [&str; 12] = [
26    "transient",
27    "anticollapse",
28    "postfilter",
29    "intensity",
30    "dual_stereo",
31    "silence",
32    "tf_select",
33    "split_stereo_n2",
34    "fold_noise",
35    "inv_phase",
36    "redundancy_pre",
37    "redundancy_post",
38];
39#[inline]
40pub(crate) fn stat(i: usize) {
41    STATS[i].fetch_add(1, std::sync::atomic::Ordering::Relaxed);
42}
43const SHORT_BLOCKSIZE: usize = 120;
44const OVERLAP: usize = 120;
45const MAX_FRAME_SIZE: usize = 960;
46const VECTORS: i32 = 11;
47const ALLOC_STEPS: i32 = 6;
48const FINE_OFFSET: i32 = 21;
49const MAX_FINE_BITS: i32 = 8;
50const QTHETA_OFFSET: i32 = 4;
51const QTHETA_OFFSET_TWOPHASE: i32 = 16;
52const POSTFILTER_MINPERIOD: i32 = 15;
53const ENERGY_SILENCE: f32 = -28.0;
54
55const SPREAD_NONE: i32 = 0;
56const SPREAD_NORMAL: i32 = 2;
57const SPREAD_AGGRESSIVE: i32 = 3;
58
59const M_SQRT1_2: f64 = std::f64::consts::FRAC_1_SQRT_2;
60const M_SQRT2: f64 = std::f64::consts::SQRT_2;
61const M_PI: f64 = std::f64::consts::PI;
62
63// Arena layout (floats).
64const X0: usize = 0;
65const X1: usize = MAX_FRAME_SIZE;
66const NORM1: usize = 2 * MAX_FRAME_SIZE;
67const NORM2: usize = NORM1 + 8 * 100;
68const SCRATCH: usize = NORM2 + 8 * 100;
69const MEM_LEN: usize = SCRATCH + 8 * 22;
70
71type Mem = Vec<f32>;
72
73fn alpha_coef(size: usize) -> f32 {
74    [
75        29440.0f32 / 32768.0,
76        26112.0 / 32768.0,
77        21248.0 / 32768.0,
78        16384.0 / 32768.0,
79    ][size]
80}
81fn beta_coef(size: usize) -> f32 {
82    [
83        1.0f32 - (30147.0 / 32768.0),
84        1.0 - (22282.0 / 32768.0),
85        1.0 - (12124.0 / 32768.0),
86        1.0 - (6554.0 / 32768.0),
87    ][size]
88}
89
90/// `ff_opus_deemph_weights`: products of `CELT_EMPH_COEFF` in double.
91fn deemph_weights() -> [f32; 16] {
92    let c = 0.8500061035f64;
93    let p = [c, c * c, c * c * c, c * c * c * c];
94    let z = 0.0f64;
95    [
96        p[0], p[1], p[2], p[3], z, p[0], p[1], p[2], z, z, p[0], p[1], z, z, z, p[0],
97    ]
98    .map(|v| v as f32)
99}
100
101#[derive(Clone)]
102pub struct Block {
103    energy: [f32; MAX_BANDS],
104    prev_energy: [[f32; MAX_BANDS]; 2],
105    collapse_masks: [u8; MAX_BANDS],
106    buf: Vec<f32>,
107    pf_period_new: i32,
108    pf_gains_new: [f32; 3],
109    pf_period: i32,
110    pf_gains: [f32; 3],
111    pf_period_old: i32,
112    pf_gains_old: [f32; 3],
113    emph_coeff: f32,
114}
115
116impl Block {
117    fn new() -> Self {
118        Block {
119            energy: [0.0; MAX_BANDS],
120            prev_energy: [[0.0; MAX_BANDS]; 2],
121            collapse_masks: [0; MAX_BANDS],
122            buf: vec![0.0; 2048],
123            pf_period_new: 0,
124            pf_gains_new: [0.0; 3],
125            pf_period: 0,
126            pf_gains: [0.0; 3],
127            pf_period_old: 0,
128            pf_gains_old: [0.0; 3],
129            emph_coeff: 0.0,
130        }
131    }
132}
133
134pub struct Celt {
135    tx: [Mdct15; 4],
136    block: [Block; 2],
137    mem: Mem,
138    tmp: Vec<f32>,
139    hadamard_tmp: [f32; 256],
140    qcoeff: [i32; 256],
141    channels: usize,
142    output_channels: usize,
143    apply_phase_inv: bool,
144
145    size: usize,
146    start_band: usize,
147    end_band: usize,
148    coded_bands: usize,
149    transient: bool,
150    blocks: usize,
151    blocksize: usize,
152    silence: bool,
153    anticollapse_needed: i32,
154    anticollapse: u32,
155    intensity_stereo: usize,
156    dual_stereo: bool,
157    flushed: bool,
158    seed: u32,
159    spread: i32,
160    alloc_trim: i32,
161
162    framebits: i32,
163    remaining: i32,
164    remaining2: i32,
165    caps: [i32; MAX_BANDS],
166    fine_bits: [i32; MAX_BANDS],
167    fine_priority: [i32; MAX_BANDS],
168    pulses: [i32; MAX_BANDS],
169    tf_change: [i32; MAX_BANDS],
170    weights: [f32; 16],
171}
172
173impl Celt {
174    /// `ff_celt_init`.
175    pub fn new(output_channels: usize, apply_phase_inv: bool) -> Self {
176        let scale = -1.0f32 / 32768.0;
177        let mut c = Celt {
178            tx: [0, 1, 2, 3].map(|i| Mdct15::new(15 << (i + 3), scale)),
179            block: [Block::new(), Block::new()],
180            mem: vec![0.0; MEM_LEN],
181            tmp: Vec::new(),
182            hadamard_tmp: [0.0; 256],
183            qcoeff: [0; 256],
184            channels: 0,
185            output_channels,
186            apply_phase_inv,
187            size: 0,
188            start_band: 0,
189            end_band: 0,
190            coded_bands: 0,
191            transient: false,
192            blocks: 1,
193            blocksize: 0,
194            silence: false,
195            anticollapse_needed: 0,
196            anticollapse: 0,
197            intensity_stereo: 0,
198            dual_stereo: false,
199            flushed: false,
200            seed: 0,
201            spread: 0,
202            alloc_trim: 0,
203            framebits: 0,
204            remaining: 0,
205            remaining2: 0,
206            caps: [0; MAX_BANDS],
207            fine_bits: [0; MAX_BANDS],
208            fine_priority: [0; MAX_BANDS],
209            pulses: [0; MAX_BANDS],
210            tf_change: [0; MAX_BANDS],
211            weights: deemph_weights(),
212        };
213        c.flush();
214        c
215    }
216
217    /// `ff_celt_flush`.
218    pub fn flush(&mut self) {
219        if self.flushed {
220            return;
221        }
222        for b in self.block.iter_mut() {
223            b.prev_energy = [[ENERGY_SILENCE; MAX_BANDS]; 2];
224            b.energy = [0.0; MAX_BANDS];
225            b.buf.iter_mut().for_each(|v| *v = 0.0);
226            b.pf_gains = [0.0; 3];
227            b.pf_gains_old = [0.0; 3];
228            b.pf_gains_new = [0.0; 3];
229            b.emph_coeff = 0.0 / self.weights[0];
230        }
231        self.seed = 0;
232        self.flushed = true;
233    }
234
235    #[inline]
236    fn rng(&mut self) -> u32 {
237        self.seed = 1664525u32.wrapping_mul(self.seed).wrapping_add(1013904223);
238        self.seed
239    }
240
241    fn decode_coarse_energy(&mut self, rc: &mut Rc) {
242        let mut prev = [0f32; 2];
243        let mut alpha = alpha_coef(self.size);
244        let mut beta = beta_coef(self.size);
245        let mut model = &CELT_COARSE_ENERGY_DIST[self.size][0];
246        if rc.tell() as i32 + 3 <= self.framebits && rc.dec_log(3) != 0 {
247            alpha = 0.0;
248            beta = 1.0 - (4915.0f32 / 32768.0);
249            model = &CELT_COARSE_ENERGY_DIST[self.size][1];
250        }
251        for i in 0..MAX_BANDS {
252            for j in 0..self.channels {
253                if i < self.start_band || i >= self.end_band {
254                    self.block[j].energy[i] = 0.0;
255                    continue;
256                }
257                let available = self.framebits - rc.tell() as i32;
258                let value: f32 = if available >= 15 {
259                    let k = i.min(20) << 1;
260                    rc.dec_laplace((model[k] as u32) << 7, (model[k + 1] as i32) << 6) as f32
261                } else if available >= 2 {
262                    let x = rc.dec_cdf(&CELT_MODEL_ENERGY_SMALL) as i32;
263                    ((x >> 1) ^ -(x & 1)) as f32
264                } else if available >= 1 {
265                    -(rc.dec_log(1) as f32)
266                } else {
267                    -1.0
268                };
269                let e = &mut self.block[j].energy[i];
270                *e = (-9.0f32).max(*e) * alpha + prev[j] + value;
271                prev[j] += beta * value;
272            }
273        }
274    }
275
276    fn decode_fine_energy(&mut self, rc: &mut Rc) {
277        for i in self.start_band..self.end_band {
278            let fb = self.fine_bits[i];
279            if fb == 0 {
280                continue;
281            }
282            for j in 0..self.channels {
283                let q2 = rc.get_raw(fb as u32);
284                let offset =
285                    (q2 as f32 + 0.5f32) * (1i32 << (14 - fb)) as f32 / 16384.0f32 - 0.5f32;
286                self.block[j].energy[i] += offset;
287            }
288        }
289    }
290
291    fn decode_final_energy(&mut self, rc: &mut Rc) {
292        let mut bits_left = self.framebits - rc.tell() as i32;
293        for priority in 0..2 {
294            let mut i = self.start_band;
295            while i < self.end_band && bits_left >= self.channels as i32 {
296                if self.fine_priority[i] != priority || self.fine_bits[i] >= MAX_FINE_BITS {
297                    i += 1;
298                    continue;
299                }
300                for j in 0..self.channels {
301                    let q2 = rc.get_raw(1);
302                    let offset = (q2 as f32 - 0.5f32)
303                        * (1i32 << (14 - self.fine_bits[i] - 1)) as f32
304                        / 16384.0f32;
305                    self.block[j].energy[i] += offset;
306                    bits_left -= 1;
307                }
308                i += 1;
309            }
310        }
311    }
312
313    fn decode_tf_changes(&mut self, rc: &mut Rc) {
314        let (mut diff, mut tf_select, mut tf_changed) = (0i32, 0usize, 0i32);
315        let mut bits = if self.transient { 2 } else { 4 };
316        let mut consumed = rc.tell() as i32;
317        let tf_select_bit = (self.size != 0 && consumed + bits < self.framebits) as i32;
318        for i in self.start_band..self.end_band {
319            if consumed + bits + tf_select_bit <= self.framebits {
320                diff ^= rc.dec_log(bits as u32) as i32;
321                consumed = rc.tell() as i32;
322                tf_changed |= diff;
323            }
324            self.tf_change[i] = diff;
325            bits = if self.transient { 4 } else { 5 };
326        }
327        let t = &CELT_TF_SELECT[self.size][self.transient as usize];
328        if tf_select_bit != 0 && t[0][tf_changed as usize] != t[1][tf_changed as usize] {
329            tf_select = rc.dec_log(1) as usize;
330            stat(6);
331        }
332        for i in self.start_band..self.end_band {
333            self.tf_change[i] = t[tf_select][self.tf_change[i] as usize] as i32;
334        }
335    }
336
337    fn denormalize(&mut self, ch: usize) {
338        let base = if ch == 0 { X0 } else { X1 };
339        for i in self.start_band..self.end_band {
340            let dst = base + ((CELT_FREQ_BANDS[i] as usize) << self.size);
341            let log_norm = self.block[ch].energy[i] + CELT_MEAN_ENERGY[i];
342            let norm = log_norm.min(32.0).exp2();
343            for j in 0..(CELT_FREQ_RANGE[i] as usize) << self.size {
344                self.mem[dst + j] *= norm;
345            }
346        }
347    }
348
349    fn parse_postfilter(&mut self, rc: &mut Rc, consumed: i32) -> i32 {
350        let mut consumed = consumed;
351        self.block[0].pf_gains_new = [0.0; 3];
352        self.block[1].pf_gains_new = [0.0; 3];
353        if self.start_band == 0 && consumed + 16 <= self.framebits {
354            if rc.dec_log(1) != 0 {
355                stat(2);
356                let octave = rc.dec_uint(6);
357                let period = (16i32 << octave) + rc.get_raw(4 + octave) as i32 - 1;
358                let gain = 0.09375f32 * (rc.get_raw(3) + 1) as f32;
359                let tapset = if rc.tell() as i32 + 2 <= self.framebits {
360                    rc.dec_cdf(&CELT_MODEL_TAPSET) as usize
361                } else {
362                    0
363                };
364                for b in self.block.iter_mut() {
365                    b.pf_period_new = period.max(POSTFILTER_MINPERIOD);
366                    for k in 0..3 {
367                        b.pf_gains_new[k] = gain * CELT_POSTFILTER_TAPS[tapset][k];
368                    }
369                }
370            }
371            consumed = rc.tell() as i32;
372        }
373        consumed
374    }
375
376    fn process_anticollapse(&mut self, ch: usize) {
377        let base = if ch == 0 { X0 } else { X1 };
378        for i in self.start_band..self.end_band {
379            let mut renormalize = false;
380            let n = (CELT_FREQ_RANGE[i] as i32) << self.size;
381            let depth = (1 + self.pulses[i]) / n;
382            let thresh = ((-1.0f64 - (0.125f32 * depth as f32) as f64) as f32).exp2();
383            let sqrt_1 = 1.0f32 / (n as f32).sqrt();
384            let xptr = base + ((CELT_FREQ_BANDS[i] as usize) << self.size);
385            let mut prev = [
386                self.block[ch].prev_energy[0][i],
387                self.block[ch].prev_energy[1][i],
388            ];
389            if self.channels == 1 {
390                prev[0] = prev[0].max(self.block[1].prev_energy[0][i]);
391                prev[1] = prev[1].max(self.block[1].prev_energy[1][i]);
392            }
393            let ediff = (self.block[ch].energy[i] - prev[0].min(prev[1])).max(0.0);
394            let mut r = (1.0f32 - ediff).exp2();
395            if self.size == 3 {
396                r = (r as f64 * M_SQRT2) as f32;
397            }
398            r = thresh.min(r) * sqrt_1;
399            for k in 0..1usize << self.size {
400                if self.block[ch].collapse_masks[i] & (1 << k) == 0 {
401                    for j in 0..CELT_FREQ_RANGE[i] as usize {
402                        let v = if self.rng() & 0x8000 != 0 { r } else { -r };
403                        self.mem[xptr + (j << self.size) + k] = v;
404                    }
405                    renormalize = true;
406                }
407            }
408            if renormalize {
409                renormalize_vector(&mut self.mem[xptr..xptr + n as usize], 1.0);
410            }
411        }
412    }
413
414    /// `ff_celt_decode_frame`. `out[c]` receives `frame_size` samples for
415    /// each output channel.
416    pub fn decode_frame(
417        &mut self,
418        rc: &mut Rc,
419        out: &mut [&mut [f32]],
420        channels: usize,
421        frame_size: usize,
422        start_band: usize,
423        end_band: usize,
424    ) -> Result<(), String> {
425        if channels != 1 && channels != 2 {
426            return Err(format!("Invalid number of coded channels: {channels}"));
427        }
428        if start_band > end_band || end_band > MAX_BANDS {
429            return Err(format!("Invalid start/end band: {start_band} {end_band}"));
430        }
431        self.silence = false;
432        self.transient = false;
433        self.anticollapse = 0;
434        self.flushed = false;
435        self.channels = channels;
436        self.start_band = start_band;
437        self.end_band = end_band;
438        self.framebits = rc.rb_bytes() as i32 * 8;
439        let size = (frame_size / SHORT_BLOCKSIZE).max(1).ilog2() as usize;
440        if size > 3 || frame_size != SHORT_BLOCKSIZE << size {
441            return Err(format!("Invalid CELT frame size: {frame_size}"));
442        }
443        self.size = size;
444        if self.output_channels == 0 {
445            self.output_channels = channels;
446        }
447        for i in 0..self.channels {
448            let base = if i == 0 { X0 } else { X1 };
449            self.mem[base..base + MAX_FRAME_SIZE]
450                .iter_mut()
451                .for_each(|v| *v = 0.0);
452            self.block[i].collapse_masks = [0; MAX_BANDS];
453        }
454        let mut consumed = rc.tell() as i32;
455        if consumed >= self.framebits {
456            self.silence = true;
457        } else if consumed == 1 {
458            self.silence = rc.dec_log(15) != 0;
459        }
460        if self.silence {
461            stat(5);
462            consumed = self.framebits;
463            rc.total_bits = rc
464                .total_bits
465                .wrapping_add((self.framebits as u32).wrapping_sub(rc.tell()));
466        }
467        consumed = self.parse_postfilter(rc, consumed);
468        if self.size != 0 && consumed + 3 <= self.framebits {
469            self.transient = rc.dec_log(3) != 0;
470        }
471        if self.transient {
472            stat(0);
473        }
474        self.blocks = if self.transient { 1 << self.size } else { 1 };
475        self.blocksize = frame_size / self.blocks;
476        let txi = if self.transient { 0 } else { self.size };
477
478        if channels == 1 {
479            for i in 0..MAX_BANDS {
480                self.block[0].energy[i] = self.block[0].energy[i].max(self.block[1].energy[i]);
481            }
482        }
483
484        self.decode_coarse_energy(rc);
485        self.decode_tf_changes(rc);
486        self.bitalloc(rc);
487        self.decode_fine_energy(rc);
488        self.quant_bands(rc);
489        if self.anticollapse_needed != 0 {
490            self.anticollapse = rc.get_raw(1);
491        }
492        self.decode_final_energy(rc);
493
494        for i in 0..self.channels {
495            if self.anticollapse != 0 {
496                stat(1);
497                self.process_anticollapse(i);
498            }
499            self.denormalize(i);
500        }
501
502        let mut downmix = false;
503        if self.output_channels < self.channels {
504            // vector_fmac_scalar(coeffs0, coeffs1, 1.0, FFALIGN(frame_size, 16)), fused
505            for k in 0..(frame_size + 15) & !15 {
506                self.mem[X0 + k] = self.mem[X1 + k].mul_add(1.0, self.mem[X0 + k]);
507            }
508            downmix = true;
509        } else if self.output_channels > self.channels {
510            self.mem.copy_within(X0..X0 + frame_size, X1);
511        }
512
513        if self.silence {
514            for b in self.block.iter_mut() {
515                b.energy = [ENERGY_SILENCE; MAX_BANDS];
516            }
517            self.mem[X0..X0 + 2 * MAX_FRAME_SIZE]
518                .iter_mut()
519                .for_each(|v| *v = 0.0);
520        }
521
522        let window = &CELT_WINDOW_PADDED[8..8 + OVERLAP];
523        for i in 0..self.output_channels {
524            let base = if i == 0 { X0 } else { X1 };
525            for j in 0..self.blocks {
526                let dst = 1024 + j * self.blocksize;
527                let (tx, mem, tmp, block) =
528                    (&self.tx[txi], &self.mem, &mut self.tmp, &mut self.block[i]);
529                tx.run(
530                    &mut block.buf[dst + OVERLAP / 2..dst + OVERLAP / 2 + self.blocksize],
531                    &mem[base + j..],
532                    self.blocks,
533                    tmp,
534                );
535                fmul_window_inplace(&mut block.buf[dst..dst + OVERLAP], window, OVERLAP / 2);
536            }
537            let block = &mut self.block[i];
538            if downmix {
539                for v in &mut block.buf[1024..1024 + frame_size] {
540                    *v *= 0.5f32;
541                }
542            }
543            postfilter(block, self.blocksize * self.blocks);
544            block.emph_coeff = deemphasis(
545                &mut out[i][..frame_size],
546                &block.buf[1024 - frame_size..1024],
547                block.emph_coeff,
548                &self.weights,
549            );
550        }
551
552        if channels == 1 {
553            self.block[1].energy = self.block[0].energy;
554        }
555        for b in self.block.iter_mut() {
556            if !self.transient {
557                b.prev_energy[1] = b.prev_energy[0];
558                b.prev_energy[0] = b.energy;
559            } else {
560                for j in 0..MAX_BANDS {
561                    b.prev_energy[0][j] = b.prev_energy[0][j].min(b.energy[j]);
562                }
563            }
564            for j in (0..self.start_band).chain(self.end_band..MAX_BANDS) {
565                b.prev_energy[0][j] = ENERGY_SILENCE;
566                b.energy[j] = 0.0;
567            }
568        }
569        self.seed = rc.range;
570        Ok(())
571    }
572
573    // ---------------------------------------------------------------- celt.c
574
575    fn normc(&self, bits: i32) -> i32 {
576        (bits << (self.channels - 1) << self.size) >> 2
577    }
578
579    /// `ff_celt_bitalloc(f, rc, 0)`.
580    fn bitalloc(&mut self, rc: &mut Rc) {
581        let ch = self.channels as i32;
582        let size = self.size as i32;
583        let (start, end) = (self.start_band, self.end_band);
584        let mut skip_startband = start;
585        let mut skip_bit = 0;
586        let mut intensitystereo_bit = 0;
587        let mut dualstereo_bit = 0;
588        let mut dynalloc = 6;
589        let mut extrabits = 0;
590        let mut boost = [0i32; MAX_BANDS];
591        let mut trim_offset = [0i32; MAX_BANDS];
592        let mut threshold = [0i32; MAX_BANDS];
593        let mut bits1 = [0i32; MAX_BANDS];
594        let mut bits2 = [0i32; MAX_BANDS];
595        let fr = |i: usize| CELT_FREQ_RANGE[i] as i32;
596        let fb = |i: usize| CELT_FREQ_BANDS[i] as i32;
597
598        if rc.tell() as i32 + 4 <= self.framebits {
599            self.spread = rc.dec_cdf(&CELT_MODEL_SPREAD) as i32;
600        } else {
601            self.spread = SPREAD_NORMAL;
602        }
603        for i in 0..MAX_BANDS {
604            self.caps[i] =
605                self.normc((CELT_STATIC_CAPS[self.size][self.channels - 1][i] as i32 + 64) * fr(i));
606        }
607        let mut tbits_8ths = self.framebits << 3;
608        for i in start..end {
609            let mut quanta = fr(i) << (ch - 1) << size;
610            let mut b_dynalloc = dynalloc;
611            quanta = (quanta << 3).min((6 << 3).max(quanta));
612            while (rc.tell_frac() as i32) + (b_dynalloc << 3) < tbits_8ths
613                && boost[i] < self.caps[i]
614            {
615                let is_boost = rc.dec_log(b_dynalloc as u32);
616                if is_boost == 0 {
617                    break;
618                }
619                boost[i] += quanta;
620                tbits_8ths -= quanta;
621                b_dynalloc = 1;
622            }
623            if boost[i] != 0 {
624                dynalloc = (dynalloc - 1).max(2);
625            }
626        }
627        self.alloc_trim = 5;
628        if rc.tell_frac() as i32 + (6 << 3) <= tbits_8ths {
629            self.alloc_trim = rc.dec_cdf(&CELT_MODEL_ALLOC_TRIM) as i32;
630        }
631        tbits_8ths = (self.framebits << 3) - rc.tell_frac() as i32 - 1;
632        self.anticollapse_needed = 0;
633        if self.transient && size >= 2 && tbits_8ths >= ((size + 2) << 3) {
634            self.anticollapse_needed = 1 << 3;
635        }
636        tbits_8ths -= self.anticollapse_needed;
637        if tbits_8ths >= 1 << 3 {
638            skip_bit = 1 << 3;
639        }
640        tbits_8ths -= skip_bit;
641        if ch == 2 {
642            intensitystereo_bit = CELT_LOG2_FRAC[end - start] as i32;
643            if intensitystereo_bit <= tbits_8ths {
644                tbits_8ths -= intensitystereo_bit;
645                if tbits_8ths >= 1 << 3 {
646                    dualstereo_bit = 1 << 3;
647                    tbits_8ths -= 1 << 3;
648                }
649            } else {
650                intensitystereo_bit = 0;
651            }
652        }
653        for i in start..end {
654            let trim = self.alloc_trim - 5 - size;
655            let band = fr(i) * (end as i32 - i as i32 - 1);
656            let duration = size + 3;
657            let scale = duration + ch - 1;
658            threshold[i] = ((3 * fr(i)) << duration >> 4).max(ch << 3);
659            trim_offset[i] = (trim * (band << scale)) >> 6;
660            if fr(i) << size == 1 {
661                trim_offset[i] -= ch << 3;
662            }
663        }
664        let mut low = 1;
665        let mut high = VECTORS - 1;
666        while low <= high {
667            let center = (low + high) >> 1;
668            let mut done = false;
669            let mut total = 0;
670            for i in (start..end).rev() {
671                let mut bandbits = self.normc(fr(i) * CELT_STATIC_ALLOC[center as usize][i] as i32);
672                if bandbits != 0 {
673                    bandbits = (bandbits + trim_offset[i]).max(0);
674                }
675                bandbits += boost[i];
676                if bandbits >= threshold[i] || done {
677                    done = true;
678                    total += bandbits.min(self.caps[i]);
679                } else if bandbits >= ch << 3 {
680                    total += ch << 3;
681                }
682            }
683            if total > tbits_8ths {
684                high = center - 1;
685            } else {
686                low = center + 1;
687            }
688        }
689        high = low;
690        low -= 1;
691        for i in start..end {
692            bits1[i] = self.normc(fr(i) * CELT_STATIC_ALLOC[low as usize][i] as i32);
693            bits2[i] = if high >= VECTORS {
694                self.caps[i]
695            } else {
696                self.normc(fr(i) * CELT_STATIC_ALLOC[high as usize][i] as i32)
697            };
698            if bits1[i] != 0 {
699                bits1[i] = (bits1[i] + trim_offset[i]).max(0);
700            }
701            if bits2[i] != 0 {
702                bits2[i] = (bits2[i] + trim_offset[i]).max(0);
703            }
704            if low != 0 {
705                bits1[i] += boost[i];
706            }
707            bits2[i] += boost[i];
708            if boost[i] != 0 {
709                skip_startband = i;
710            }
711            bits2[i] = (bits2[i] - bits1[i]).max(0);
712        }
713        let mut low = 0;
714        let mut high = 1 << ALLOC_STEPS;
715        for _ in 0..ALLOC_STEPS {
716            let center = (low + high) >> 1;
717            let mut done = false;
718            let mut total = 0;
719            for j in (start..end).rev() {
720                let bandbits = bits1[j] + ((center * bits2[j]) >> ALLOC_STEPS);
721                if bandbits >= threshold[j] || done {
722                    done = true;
723                    total += bandbits.min(self.caps[j]);
724                } else if bandbits >= ch << 3 {
725                    total += ch << 3;
726                }
727            }
728            if total > tbits_8ths {
729                high = center;
730            } else {
731                low = center;
732            }
733        }
734        let mut done = false;
735        let mut total = 0;
736        for i in (start..end).rev() {
737            let mut bandbits = bits1[i] + ((low * bits2[i]) >> ALLOC_STEPS);
738            if bandbits >= threshold[i] || done {
739                done = true;
740            } else {
741                bandbits = if bandbits >= ch << 3 { ch << 3 } else { 0 };
742            }
743            bandbits = bandbits.min(self.caps[i]);
744            self.pulses[i] = bandbits;
745            total += bandbits;
746        }
747        self.coded_bands = end;
748        loop {
749            let j = self.coded_bands - 1;
750            if j == skip_startband {
751                tbits_8ths += skip_bit;
752                break;
753            }
754            let mut remaining = tbits_8ths - total;
755            let bandbits = remaining / (fb(j + 1) - fb(start));
756            remaining -= bandbits * (fb(j + 1) - fb(start));
757            let mut allocation = self.pulses[j] + bandbits * fr(j);
758            allocation += (remaining - (fb(j) - fb(start))).max(0);
759            if allocation >= threshold[j].max((ch + 1) << 3) {
760                if rc.dec_log(1) != 0 {
761                    break;
762                }
763                total += 1 << 3;
764                allocation -= 1 << 3;
765            }
766            total -= self.pulses[j];
767            if intensitystereo_bit != 0 {
768                total -= intensitystereo_bit;
769                intensitystereo_bit = CELT_LOG2_FRAC[j - start] as i32;
770                total += intensitystereo_bit;
771            }
772            self.pulses[j] = if allocation >= ch << 3 { ch << 3 } else { 0 };
773            total += self.pulses[j];
774            self.coded_bands -= 1;
775        }
776        self.intensity_stereo = 0;
777        self.dual_stereo = false;
778        if intensitystereo_bit != 0 {
779            self.intensity_stereo =
780                start + rc.dec_uint((self.coded_bands + 1 - start) as u32) as usize;
781            if self.intensity_stereo < self.coded_bands {
782                stat(3);
783            }
784        }
785        if self.intensity_stereo <= start {
786            tbits_8ths += dualstereo_bit;
787        } else if dualstereo_bit != 0 {
788            self.dual_stereo = rc.dec_log(1) != 0;
789            if self.dual_stereo {
790                stat(4);
791            }
792        }
793        let mut remaining = tbits_8ths - total;
794        let bandbits = remaining / (fb(self.coded_bands) - fb(start));
795        remaining -= bandbits * (fb(self.coded_bands) - fb(start));
796        for i in start..self.coded_bands {
797            let bits = remaining.min(fr(i));
798            self.pulses[i] += bits + bandbits * fr(i);
799            remaining -= bits;
800        }
801        let mut i = start;
802        while i < self.coded_bands {
803            let n = fr(i) << size;
804            let prev_extra = extrabits;
805            self.pulses[i] += extrabits;
806            if n > 1 {
807                extrabits = (self.pulses[i] - self.caps[i]).max(0);
808                self.pulses[i] -= extrabits;
809                let dof = n * ch
810                    + (ch == 2 && n > 2 && !self.dual_stereo && i < self.intensity_stereo) as i32;
811                let temp = dof * (CELT_LOG_FREQ_RANGE[i] as i32 + (size << 3));
812                let mut offset = (temp >> 1) - dof * FINE_OFFSET;
813                if n == 2 {
814                    offset += dof << 1;
815                }
816                if self.pulses[i] + offset < 2 * (dof << 3) {
817                    offset += temp >> 2;
818                } else if self.pulses[i] + offset < 3 * (dof << 3) {
819                    offset += temp >> 3;
820                }
821                let fine_bits = (self.pulses[i] + offset + (dof << 2)) / (dof << 3);
822                let max_bits = ((self.pulses[i] >> 3) >> (ch - 1)).clamp(0, MAX_FINE_BITS);
823                self.fine_bits[i] = fine_bits.clamp(0, max_bits);
824                self.fine_priority[i] =
825                    (self.fine_bits[i] * (dof << 3) >= self.pulses[i] + offset) as i32;
826                self.pulses[i] -= self.fine_bits[i] << (ch - 1) << 3;
827            } else {
828                extrabits = (self.pulses[i] - (ch << 3)).max(0);
829                self.pulses[i] -= extrabits;
830                self.fine_bits[i] = 0;
831                self.fine_priority[i] = 1;
832            }
833            if extrabits > 0 {
834                let mut fineextra = (extrabits >> (ch + 2)).min(MAX_FINE_BITS - self.fine_bits[i]);
835                self.fine_bits[i] += fineextra;
836                fineextra <<= ch + 2;
837                self.fine_priority[i] = (fineextra >= extrabits - prev_extra) as i32;
838                extrabits -= fineextra;
839            }
840            i += 1;
841        }
842        self.remaining = extrabits;
843        while i < end {
844            self.fine_bits[i] = self.pulses[i] >> (ch - 1) >> 3;
845            self.pulses[i] = 0;
846            self.fine_priority[i] = (self.fine_bits[i] < 1) as i32;
847            i += 1;
848        }
849    }
850
851    /// `ff_celt_quant_bands` (decoder).
852    fn quant_bands(&mut self, rc: &mut Rc) {
853        let totalbits = (self.framebits << 3) - self.anticollapse_needed;
854        let mut update_lowband = true;
855        let mut lowband_offset = 0usize;
856        let (start, end, size) = (self.start_band, self.end_band, self.size);
857        let fb = |i: usize| CELT_FREQ_BANDS[i] as usize;
858        let fr = |i: usize| CELT_FREQ_RANGE[i] as usize;
859        for i in start..end {
860            let full = (1u32 << self.blocks) - 1;
861            let mut cm = [full, full];
862            let band_offset = fb(i) << size;
863            let band_size = fr(i) << size;
864            let x = X0 + band_offset;
865            let y = if self.channels == 2 {
866                Some(X1 + band_offset)
867            } else {
868                None
869            };
870            let consumed = rc.tell_frac() as i32;
871            let mut effective_lowband: isize = -1;
872            let mut b = 0;
873            if i != start {
874                self.remaining -= consumed;
875            }
876            self.remaining2 = totalbits - consumed - 1;
877            if i < self.coded_bands {
878                let curr_balance = self.remaining / 3.min(self.coded_bands as i32 - i as i32);
879                b = (self.remaining2 + 1)
880                    .min(self.pulses[i] + curr_balance)
881                    .clamp(0, (1 << 14) - 1);
882            }
883            if (fb(i) as i32 - fr(i) as i32 >= fb(start) as i32 || i == start + 1)
884                && (update_lowband || lowband_offset == 0)
885            {
886                lowband_offset = i;
887            }
888            if i == start + 1 {
889                let count = (fr(i) - fr(i - 1)) << size;
890                self.mem.copy_within(
891                    NORM1 + band_offset - count..NORM1 + band_offset,
892                    NORM1 + band_offset,
893                );
894                if self.channels == 2 {
895                    self.mem.copy_within(
896                        NORM2 + band_offset - count..NORM2 + band_offset,
897                        NORM2 + band_offset,
898                    );
899                }
900            }
901            if lowband_offset != 0
902                && (self.spread != SPREAD_AGGRESSIVE || self.blocks > 1 || self.tf_change[i] < 0)
903            {
904                effective_lowband =
905                    (fb(start) as isize).max(fb(lowband_offset) as isize - fr(i) as isize);
906                let mut foldstart = lowband_offset;
907                loop {
908                    foldstart -= 1;
909                    if (fb(foldstart) as isize) <= effective_lowband {
910                        break;
911                    }
912                }
913                let mut foldend = lowband_offset - 1;
914                loop {
915                    foldend += 1;
916                    if !(foldend < i && (fb(foldend) as isize) < effective_lowband + fr(i) as isize)
917                    {
918                        break;
919                    }
920                }
921                cm = [0, 0];
922                for j in foldstart..foldend {
923                    cm[0] |= self.block[0].collapse_masks[j] as u32;
924                    cm[1] |= self.block[self.channels - 1].collapse_masks[j] as u32;
925                }
926            }
927            if self.dual_stereo && i == self.intensity_stereo {
928                self.dual_stereo = false;
929                for j in fb(start) << size..band_offset {
930                    self.mem[NORM1 + j] = (self.mem[NORM1 + j] + self.mem[NORM2 + j]) / 2.0f32;
931                }
932            }
933            let (nl1, nl2) = if effective_lowband != -1 {
934                let e = (effective_lowband as usize) << size;
935                (Some(NORM1 + e), Some(NORM2 + e))
936            } else {
937                (None, None)
938            };
939            if self.dual_stereo {
940                cm[0] = self.quant_band(
941                    rc,
942                    i,
943                    x,
944                    None,
945                    band_size,
946                    b >> 1,
947                    self.blocks as u32,
948                    nl1,
949                    size as i32,
950                    Some(NORM1 + band_offset),
951                    0,
952                    1.0,
953                    Some(SCRATCH),
954                    cm[0],
955                );
956                cm[1] = self.quant_band(
957                    rc,
958                    i,
959                    y.unwrap(),
960                    None,
961                    band_size,
962                    b >> 1,
963                    self.blocks as u32,
964                    nl2,
965                    size as i32,
966                    Some(NORM2 + band_offset),
967                    0,
968                    1.0,
969                    Some(SCRATCH),
970                    cm[1],
971                );
972            } else {
973                cm[0] = self.quant_band(
974                    rc,
975                    i,
976                    x,
977                    y,
978                    band_size,
979                    b,
980                    self.blocks as u32,
981                    nl1,
982                    size as i32,
983                    Some(NORM1 + band_offset),
984                    0,
985                    1.0,
986                    Some(SCRATCH),
987                    cm[0] | cm[1],
988                );
989                cm[1] = cm[0];
990            }
991            self.block[0].collapse_masks[i] = cm[0] as u8;
992            self.block[self.channels - 1].collapse_masks[i] = cm[1] as u8;
993            self.remaining += self.pulses[i] + consumed;
994            update_lowband = b > (band_size as i32) << 3;
995        }
996    }
997
998    // ---------------------------------------------------------------- pvq.c
999
1000    fn haar1(&mut self, x: usize, n0: usize, stride: usize) {
1001        let n0 = n0 >> 1;
1002        for i in 0..stride {
1003            for j in 0..n0 {
1004                let a = x + stride * (2 * j) + i;
1005                let b = x + stride * (2 * j + 1) + i;
1006                let (x0, x1) = (self.mem[a], self.mem[b]);
1007                self.mem[a] = ((x0 + x1) as f64 * M_SQRT1_2) as f32;
1008                self.mem[b] = ((x0 - x1) as f64 * M_SQRT1_2) as f32;
1009            }
1010        }
1011    }
1012
1013    fn hadamard_order(stride: usize, hadamard: bool) -> &'static [u8] {
1014        &CELT_HADAMARD_ORDER[if hadamard { stride - 2 } else { 30 }..]
1015    }
1016
1017    fn deinterleave_hadamard(&mut self, x: usize, n0: usize, stride: usize, hadamard: bool) {
1018        let order = Self::hadamard_order(stride, hadamard);
1019        let n = n0 * stride;
1020        for i in 0..stride {
1021            for j in 0..n0 {
1022                self.hadamard_tmp[order[i] as usize * n0 + j] = self.mem[x + j * stride + i];
1023            }
1024        }
1025        self.mem[x..x + n].copy_from_slice(&self.hadamard_tmp[..n]);
1026    }
1027
1028    fn interleave_hadamard(&mut self, x: usize, n0: usize, stride: usize, hadamard: bool) {
1029        let order = Self::hadamard_order(stride, hadamard);
1030        let n = n0 * stride;
1031        for i in 0..stride {
1032            for j in 0..n0 {
1033                self.hadamard_tmp[j * stride + i] = self.mem[x + order[i] as usize * n0 + j];
1034            }
1035        }
1036        self.mem[x..x + n].copy_from_slice(&self.hadamard_tmp[..n]);
1037    }
1038
1039    fn exp_rotation(&mut self, x: usize, len: u32, stride: u32, k: u32, spread: i32) {
1040        if 2 * k >= len || spread == SPREAD_NONE {
1041            return;
1042        }
1043        let gain = len as f32 / (len + (20 - 5 * spread as u32) * k) as f32;
1044        let theta = (M_PI * gain as f64 * gain as f64 / 4.0) as f32;
1045        let (s, c) = theta.sin_cos();
1046        let mut stride2 = 0u32;
1047        if len >= stride << 3 {
1048            stride2 = 1;
1049            while (stride2 * stride2 + stride2) * stride + (stride >> 2) < len {
1050                stride2 += 1;
1051            }
1052        }
1053        let len = len / stride;
1054        for i in 0..stride {
1055            let base = x + (i * len) as usize;
1056            if stride2 != 0 {
1057                rotation_impl(
1058                    &mut self.mem[base..base + len as usize],
1059                    stride2 as usize,
1060                    s,
1061                    c,
1062                );
1063            }
1064            rotation_impl(&mut self.mem[base..base + len as usize], 1, c, s);
1065        }
1066    }
1067
1068    fn alg_unquant(
1069        &mut self,
1070        rc: &mut Rc,
1071        x: usize,
1072        n: u32,
1073        k: u32,
1074        spread: i32,
1075        blocks: u32,
1076        gain: f32,
1077    ) -> u32 {
1078        let norm = decode_pulses(rc, &mut self.qcoeff[..n as usize], n, k);
1079        let gain = gain / (norm as f32).sqrt();
1080        for i in 0..n as usize {
1081            self.mem[x + i] = gain * self.qcoeff[i] as f32;
1082        }
1083        self.exp_rotation(x, n, blocks, k, spread);
1084        extract_collapse_mask(&self.qcoeff, n, blocks)
1085    }
1086
1087    fn stereo_merge(&mut self, x: usize, y: usize, mid: f32, n: usize) {
1088        let (mut xp, mut side) = (0f32, 0f32);
1089        for i in 0..n {
1090            xp += self.mem[x + i] * self.mem[y + i];
1091            side += self.mem[y + i] * self.mem[y + i];
1092        }
1093        xp *= mid;
1094        let mid2 = mid;
1095        let e0 = mid2 * mid2 + side - 2.0 * xp;
1096        let e1 = mid2 * mid2 + side + 2.0 * xp;
1097        if e0 < 6e-4f32 || e1 < 6e-4f32 {
1098            for i in 0..n {
1099                self.mem[y + i] = self.mem[x + i];
1100            }
1101            return;
1102        }
1103        let g0 = 1.0f32 / e0.sqrt();
1104        let g1 = 1.0f32 / e1.sqrt();
1105        for i in 0..n {
1106            let v0 = mid * self.mem[x + i];
1107            let v1 = self.mem[y + i];
1108            self.mem[x + i] = g0 * (v0 - v1);
1109            self.mem[y + i] = g1 * (v0 + v1);
1110        }
1111    }
1112
1113    /// `quant_band_template(..., quant = 0)`.
1114    fn quant_band(
1115        &mut self,
1116        rc: &mut Rc,
1117        band: usize,
1118        x: usize,
1119        y: Option<usize>,
1120        n: usize,
1121        b: i32,
1122        blocks: u32,
1123        lowband: Option<usize>,
1124        duration: i32,
1125        lowband_out: Option<usize>,
1126        level: i32,
1127        gain: f32,
1128        lowband_scratch: Option<usize>,
1129        fill: u32,
1130    ) -> u32 {
1131        let (mut n, mut b, mut blocks, mut lowband, mut duration, mut fill, mut y) =
1132            (n, b, blocks, lowband, duration, fill, y);
1133        let stereo = y.is_some();
1134        let mut split = stereo;
1135        let n0 = n;
1136        let mut n_b = n / blocks as usize;
1137        let mut n_b0 = n_b;
1138        let mut b0 = blocks;
1139        let mut time_divide = 0;
1140        let mut recombine = 0;
1141        let mut inv = false;
1142        let mut mid = 0f32;
1143        let side;
1144        let longblocks = b0 == 1;
1145        let mut cm: u32 = 0;
1146
1147        if n == 1 {
1148            let mut px = x;
1149            for i in 0..=stereo as usize {
1150                let mut sign = 0;
1151                if self.remaining2 >= 1 << 3 {
1152                    sign = rc.get_raw(1);
1153                    self.remaining2 -= 1 << 3;
1154                }
1155                self.mem[px] = 1.0f32 - 2.0f32 * sign as f32;
1156                if i == 0
1157                    && let Some(yy) = y
1158                {
1159                    px = yy;
1160                }
1161            }
1162            if let Some(lo) = lowband_out {
1163                self.mem[lo] = self.mem[x];
1164            }
1165            return 1;
1166        }
1167
1168        if !stereo && level == 0 {
1169            let mut tf_change = self.tf_change[band];
1170            if tf_change > 0 {
1171                recombine = tf_change;
1172            }
1173            if let Some(lb) = lowband
1174                && (recombine != 0 || ((n_b & 1) == 0 && tf_change < 0) || b0 > 1)
1175            {
1176                let sc = lowband_scratch.unwrap();
1177                self.mem.copy_within(lb..lb + n, sc);
1178                lowband = Some(sc);
1179            }
1180            for k in 0..recombine {
1181                if let Some(lb) = lowband {
1182                    self.haar1(lb, n >> k, 1 << k);
1183                }
1184                fill = CELT_BIT_INTERLEAVE[(fill & 0xF) as usize] as u32
1185                    | (CELT_BIT_INTERLEAVE[(fill >> 4) as usize] as u32) << 2;
1186            }
1187            blocks >>= recombine;
1188            n_b <<= recombine;
1189            while (n_b & 1) == 0 && tf_change < 0 {
1190                if let Some(lb) = lowband {
1191                    self.haar1(lb, n_b, blocks as usize);
1192                }
1193                fill |= fill << blocks;
1194                blocks <<= 1;
1195                n_b >>= 1;
1196                time_divide += 1;
1197                tf_change += 1;
1198            }
1199            b0 = blocks;
1200            n_b0 = n_b;
1201            if b0 > 1
1202                && let Some(lb) = lowband
1203            {
1204                self.deinterleave_hadamard(
1205                    lb,
1206                    n_b >> recombine,
1207                    (b0 << recombine) as usize,
1208                    longblocks,
1209                );
1210            }
1211        }
1212
1213        let cache_at = CELT_CACHE_INDEX[(duration + 1) as usize * MAX_BANDS + band] as usize;
1214        let cache = &CELT_CACHE_BITS[cache_at..];
1215        if !stereo && duration >= 0 && b > cache[cache[0] as usize] as i32 + 12 && n > 2 {
1216            n >>= 1;
1217            y = Some(x + n);
1218            split = true;
1219            duration -= 1;
1220            if blocks == 1 {
1221                fill = (fill & 1) | (fill << 1);
1222            }
1223            blocks = (blocks + 1) >> 1;
1224        }
1225
1226        if split {
1227            let mut itheta: i32 = 0;
1228            let pulse_cap = CELT_LOG_FREQ_RANGE[band] as i32 + duration * 8;
1229            let offset = (pulse_cap >> 1)
1230                - if stereo && n == 2 {
1231                    QTHETA_OFFSET_TWOPHASE
1232                } else {
1233                    QTHETA_OFFSET
1234                };
1235            let qn = if stereo && band >= self.intensity_stereo {
1236                1
1237            } else {
1238                compute_qn(n as i32, b, offset, pulse_cap, stereo)
1239            };
1240            let tell = rc.tell_frac() as i32;
1241            if qn != 1 {
1242                itheta = if stereo && n > 2 {
1243                    rc.dec_uint_step((qn / 2) as u32) as i32
1244                } else if stereo || b0 > 1 {
1245                    rc.dec_uint((qn + 1) as u32) as i32
1246                } else {
1247                    rc.dec_uint_tri(qn as u32) as i32
1248                };
1249                itheta = itheta * 16384 / qn;
1250            } else if stereo {
1251                inv = if b > 2 << 3 && self.remaining2 > 2 << 3 {
1252                    rc.dec_log(2) != 0
1253                } else {
1254                    false
1255                };
1256                inv = if self.apply_phase_inv { inv } else { false };
1257                if inv {
1258                    stat(9);
1259                }
1260                itheta = 0;
1261            }
1262            let qalloc = rc.tell_frac() as i32 - tell;
1263            b -= qalloc;
1264            let orig_fill = fill;
1265            let (imid, iside, mut delta);
1266            if itheta == 0 {
1267                imid = 32767;
1268                iside = 0;
1269                fill &= if blocks >= 32 {
1270                    u32::MAX
1271                } else {
1272                    (1u32 << blocks) - 1
1273                };
1274                delta = -16384;
1275            } else if itheta == 16384 {
1276                imid = 0;
1277                iside = 32767;
1278                fill &= ((1u32 << blocks) - 1) << blocks;
1279                delta = 16384;
1280            } else {
1281                imid = celt_cos(itheta as i16) as i32;
1282                iside = celt_cos((16384 - itheta) as i16) as i32;
1283                delta = round_mul16(((n as i32) - 1) << 7, celt_log2tan(iside, imid));
1284            }
1285            mid = imid as f32 / 32768.0f32;
1286            side = iside as f32 / 32768.0f32;
1287
1288            if n == 2 && stereo {
1289                stat(7);
1290                let yy = y.unwrap();
1291                let mut mbits = b;
1292                let sbits = if itheta != 0 && itheta != 16384 {
1293                    1 << 3
1294                } else {
1295                    0
1296                };
1297                mbits -= sbits;
1298                let c = itheta > 8192;
1299                self.remaining2 -= qalloc + sbits;
1300                let (x2, y2) = if c { (yy, x) } else { (x, yy) };
1301                let mut sign = 0;
1302                if sbits != 0 {
1303                    sign = rc.get_raw(1) as i32;
1304                }
1305                let sign = 1 - 2 * sign;
1306                cm = self.quant_band(
1307                    rc,
1308                    band,
1309                    x2,
1310                    None,
1311                    n,
1312                    mbits,
1313                    blocks,
1314                    lowband,
1315                    duration,
1316                    lowband_out,
1317                    level,
1318                    gain,
1319                    lowband_scratch,
1320                    orig_fill,
1321                );
1322                self.mem[y2] = -sign as f32 * self.mem[x2 + 1];
1323                self.mem[y2 + 1] = sign as f32 * self.mem[x2];
1324                self.mem[x] *= mid;
1325                self.mem[x + 1] *= mid;
1326                self.mem[yy] *= side;
1327                self.mem[yy + 1] *= side;
1328                let tmp = self.mem[x];
1329                self.mem[x] = tmp - self.mem[yy];
1330                self.mem[yy] += tmp;
1331                let tmp = self.mem[x + 1];
1332                self.mem[x + 1] = tmp - self.mem[yy + 1];
1333                self.mem[yy + 1] += tmp;
1334            } else {
1335                let yy = y.unwrap();
1336                let mut next_lowband2 = None;
1337                let mut next_lowband_out1 = None;
1338                let mut next_level = 0;
1339                if b0 > 1 && !stereo && (itheta & 0x3fff) != 0 {
1340                    if itheta > 8192 {
1341                        delta -= delta >> (4 - duration);
1342                    } else {
1343                        delta = 0.min(delta + (((n as i32) << 3) >> (5 - duration)));
1344                    }
1345                }
1346                let mut mbits = ((b - delta) / 2).clamp(0, b);
1347                let mut sbits = b - mbits;
1348                self.remaining2 -= qalloc;
1349                if let Some(lb) = lowband
1350                    && !stereo
1351                {
1352                    next_lowband2 = Some(lb + n);
1353                }
1354                if stereo {
1355                    next_lowband_out1 = lowband_out;
1356                } else {
1357                    next_level = level + 1;
1358                }
1359                let mut rebalance = self.remaining2;
1360                let shift = (b0 >> 1) & (if stereo { 0 } else { u32::MAX });
1361                if mbits >= sbits {
1362                    cm = self.quant_band(
1363                        rc,
1364                        band,
1365                        x,
1366                        None,
1367                        n,
1368                        mbits,
1369                        blocks,
1370                        lowband,
1371                        duration,
1372                        next_lowband_out1,
1373                        next_level,
1374                        if stereo { 1.0 } else { gain * mid },
1375                        lowband_scratch,
1376                        fill,
1377                    );
1378                    rebalance = mbits - (rebalance - self.remaining2);
1379                    if rebalance > 3 << 3 && itheta != 0 {
1380                        sbits += rebalance - (3 << 3);
1381                    }
1382                    let cmt = self.quant_band(
1383                        rc,
1384                        band,
1385                        yy,
1386                        None,
1387                        n,
1388                        sbits,
1389                        blocks,
1390                        next_lowband2,
1391                        duration,
1392                        None,
1393                        next_level,
1394                        gain * side,
1395                        None,
1396                        fill >> blocks,
1397                    );
1398                    cm |= cmt << shift;
1399                } else {
1400                    cm = self.quant_band(
1401                        rc,
1402                        band,
1403                        yy,
1404                        None,
1405                        n,
1406                        sbits,
1407                        blocks,
1408                        next_lowband2,
1409                        duration,
1410                        None,
1411                        next_level,
1412                        gain * side,
1413                        None,
1414                        fill >> blocks,
1415                    );
1416                    cm <<= shift;
1417                    rebalance = sbits - (rebalance - self.remaining2);
1418                    if rebalance > 3 << 3 && itheta != 16384 {
1419                        mbits += rebalance - (3 << 3);
1420                    }
1421                    cm |= self.quant_band(
1422                        rc,
1423                        band,
1424                        x,
1425                        None,
1426                        n,
1427                        mbits,
1428                        blocks,
1429                        lowband,
1430                        duration,
1431                        next_lowband_out1,
1432                        next_level,
1433                        if stereo { 1.0 } else { gain * mid },
1434                        lowband_scratch,
1435                        fill,
1436                    );
1437                }
1438            }
1439        } else {
1440            let mut q = bits2pulses(cache, b);
1441            let mut curr_bits = pulses2bits(cache, q);
1442            self.remaining2 -= curr_bits;
1443            while self.remaining2 < 0 && q > 0 {
1444                self.remaining2 += curr_bits;
1445                q -= 1;
1446                curr_bits = pulses2bits(cache, q);
1447                self.remaining2 -= curr_bits;
1448            }
1449            if q != 0 {
1450                let k = if q < 8 {
1451                    q
1452                } else {
1453                    (8 + (q & 7)) << ((q >> 3) - 1)
1454                };
1455                cm = self.alg_unquant(rc, x, n as u32, k as u32, self.spread, blocks, gain);
1456            } else {
1457                let cm_mask = (1u32 << blocks) - 1;
1458                fill &= cm_mask;
1459                if fill != 0 {
1460                    match lowband {
1461                        None => {
1462                            stat(8);
1463                            for i in 0..n {
1464                                self.mem[x + i] = ((self.rng() as i32) >> 20) as f32;
1465                            }
1466                            cm = cm_mask;
1467                        }
1468                        Some(lb) => {
1469                            for i in 0..n {
1470                                let d = if self.rng() & 0x8000 != 0 {
1471                                    1.0f32 / 256.0
1472                                } else {
1473                                    -1.0f32 / 256.0
1474                                };
1475                                self.mem[x + i] = self.mem[lb + i] + d;
1476                            }
1477                            cm = fill;
1478                        }
1479                    }
1480                    renormalize_vector(&mut self.mem[x..x + n], gain);
1481                } else {
1482                    self.mem[x..x + n].iter_mut().for_each(|v| *v = 0.0);
1483                }
1484            }
1485        }
1486
1487        if stereo {
1488            let yy = y.unwrap();
1489            if n > 2 {
1490                self.stereo_merge(x, yy, mid, n);
1491            }
1492            if inv {
1493                for i in 0..n {
1494                    self.mem[yy + i] *= -1.0;
1495                }
1496            }
1497        } else if level == 0 {
1498            if b0 > 1 {
1499                self.interleave_hadamard(
1500                    x,
1501                    n_b >> recombine,
1502                    (b0 << recombine) as usize,
1503                    longblocks,
1504                );
1505            }
1506            n_b = n_b0;
1507            blocks = b0;
1508            for _ in 0..time_divide {
1509                blocks >>= 1;
1510                n_b <<= 1;
1511                cm |= cm >> blocks;
1512                self.haar1(x, n_b, blocks as usize);
1513            }
1514            for k in 0..recombine {
1515                cm = CELT_BIT_DEINTERLEAVE[cm as usize] as u32;
1516                self.haar1(x, n0 >> k, 1 << k);
1517            }
1518            blocks <<= recombine;
1519            if let Some(lo) = lowband_out {
1520                let nn = (n0 as f32).sqrt();
1521                for i in 0..n0 {
1522                    self.mem[lo + i] = nn * self.mem[x + i];
1523                }
1524            }
1525            cm &= if blocks >= 32 {
1526                u32::MAX
1527            } else {
1528                (1u32 << blocks) - 1
1529            };
1530        }
1531        cm
1532    }
1533}
1534
1535fn renormalize_vector(x: &mut [f32], gain: f32) {
1536    let mut g = 1e-15f32;
1537    for v in x.iter() {
1538        g += v * v;
1539    }
1540    let g = gain / g.sqrt();
1541    for v in x.iter_mut() {
1542        *v *= g;
1543    }
1544}
1545
1546fn rotation_impl(x: &mut [f32], stride: usize, c: f32, s: f32) {
1547    let len = x.len();
1548    for i in 0..len - stride {
1549        let (x1, x2) = (x[i], x[i + stride]);
1550        x[i + stride] = c * x2 + s * x1;
1551        x[i] = c * x1 - s * x2;
1552    }
1553    let mut i = len as isize - 2 * stride as isize - 1;
1554    while i >= 0 {
1555        let k = i as usize;
1556        let (x1, x2) = (x[k], x[k + stride]);
1557        x[k + stride] = c * x2 + s * x1;
1558        x[k] = c * x1 - s * x2;
1559        i -= 1;
1560    }
1561}
1562
1563fn extract_collapse_mask(iy: &[i32], n: u32, b: u32) -> u32 {
1564    if b <= 1 {
1565        return 1;
1566    }
1567    let n0 = (n / b) as usize;
1568    let mut m = 0u32;
1569    for i in 0..b as usize {
1570        for j in 0..n0 {
1571            m |= ((iy[i * n0 + j] != 0) as u32) << i;
1572        }
1573    }
1574    m
1575}
1576
1577#[inline]
1578fn round_mul16(a: i32, b: i32) -> i32 {
1579    (a * b + 16384) >> 15
1580}
1581
1582fn celt_cos(x: i16) -> i16 {
1583    let x = (((x as i32) * (x as i32) + 4096) >> 13) as i16 as i32;
1584    let x =
1585        ((32767 - x) + round_mul16(x, -7651 + round_mul16(x, 8277 + round_mul16(-626, x)))) as i16;
1586    (x as i32 + 1) as i16
1587}
1588
1589fn celt_log2tan(isin: i32, icos: i32) -> i32 {
1590    let lc = ilog(icos as u32) as i32;
1591    let ls = ilog(isin as u32) as i32;
1592    let icos = icos << (15 - lc);
1593    let isin = isin << (15 - ls);
1594    (ls << 11) - (lc << 11) + round_mul16(isin, round_mul16(isin, -2597) + 7932)
1595        - round_mul16(icos, round_mul16(icos, -2597) + 7932)
1596}
1597
1598fn bits2pulses(cache: &[u8], bits: i32) -> i32 {
1599    let mut low = 0i32;
1600    let mut high = cache[0] as i32;
1601    let bits = bits - 1;
1602    for _ in 0..6 {
1603        let center = (low + high + 1) >> 1;
1604        if cache[center as usize] as i32 >= bits {
1605            high = center;
1606        } else {
1607            low = center;
1608        }
1609    }
1610    let lowv = if low == 0 {
1611        -1
1612    } else {
1613        cache[low as usize] as i32
1614    };
1615    if bits - lowv <= cache[high as usize] as i32 - bits {
1616        low
1617    } else {
1618        high
1619    }
1620}
1621
1622fn pulses2bits(cache: &[u8], pulses: i32) -> i32 {
1623    if pulses == 0 {
1624        0
1625    } else {
1626        cache[pulses as usize] as i32 + 1
1627    }
1628}
1629
1630fn compute_qn(n: i32, b: i32, offset: i32, pulse_cap: i32, stereo: bool) -> i32 {
1631    let mut n2 = 2 * n - 1;
1632    if stereo && n == 2 {
1633        n2 -= 1;
1634    }
1635    let qb = (b - pulse_cap - (4 << 3))
1636        .min((b + n2 * offset) / n2)
1637        .min(8 << 3);
1638    if qb < (1 << 3 >> 1) {
1639        1
1640    } else {
1641        (((CELT_QN_EXP2[(qb & 0x7) as usize] as i32 >> (14 - (qb >> 3))) + 1) >> 1) << 1
1642    }
1643}
1644
1645#[inline]
1646fn pvq_u(n: u32, k: u32) -> u32 {
1647    let (a, b) = (n.min(k), n.max(k));
1648    CELT_PVQ_U[PVQ_ROW[a as usize] + b as usize]
1649}
1650
1651const PVQ_ROW: [usize; 15] = [
1652    0, 176, 351, 525, 698, 870, 1041, 1131, 1178, 1207, 1226, 1240, 1248, 1254, 1257,
1653];
1654
1655#[inline]
1656fn row(n: u32, k: u32) -> u32 {
1657    CELT_PVQ_U[PVQ_ROW[n as usize] + k as usize]
1658}
1659
1660/// `celt_decode_pulses` -> `celt_cwrsi`.
1661fn decode_pulses(rc: &mut Rc, y: &mut [i32], n: u32, k: u32) -> u64 {
1662    let idx = rc.dec_uint(pvq_u(n, k) + pvq_u(n, k + 1));
1663    cwrsi(n, k, idx, y)
1664}
1665
1666fn cwrsi(mut n: u32, mut k: u32, mut i: u32, y: &mut [i32]) -> u64 {
1667    let mut norm: u64 = 0;
1668    let mut yi = 0;
1669    let mut push = |v: i32, norm: &mut u64| {
1670        *norm += (v * v) as u64;
1671        y[yi] = v;
1672        yi += 1;
1673    };
1674    while n > 2 {
1675        if k >= n {
1676            let p = row(n, k + 1);
1677            let s: i32 = -((i >= p) as i32);
1678            i -= p & s as u32;
1679            let k0 = k;
1680            let q = row(n, n);
1681            let mut p;
1682            if q > i {
1683                k = n;
1684                loop {
1685                    k -= 1;
1686                    p = row(k, n);
1687                    if p <= i {
1688                        break;
1689                    }
1690                }
1691            } else {
1692                p = row(n, k);
1693                while p > i {
1694                    k -= 1;
1695                    p = row(n, k);
1696                }
1697            }
1698            i -= p;
1699            let val = (k0 as i32 - k as i32 + s) ^ s;
1700            push(val, &mut norm);
1701        } else {
1702            let p = row(k, n);
1703            let q = row(k + 1, n);
1704            if p <= i && i < q {
1705                i -= p;
1706                push(0, &mut norm);
1707            } else {
1708                let s: i32 = -((i >= q) as i32);
1709                i -= q & s as u32;
1710                let k0 = k;
1711                let mut p;
1712                loop {
1713                    k -= 1;
1714                    p = row(k, n);
1715                    if p <= i {
1716                        break;
1717                    }
1718                }
1719                i -= p;
1720                let val = (k0 as i32 - k as i32 + s) ^ s;
1721                push(val, &mut norm);
1722            }
1723        }
1724        n -= 1;
1725    }
1726    // N == 2
1727    let p = 2 * k + 1;
1728    let s: i32 = -((i >= p) as i32);
1729    i -= p & s as u32;
1730    let k0 = k;
1731    k = i.div_ceil(2);
1732    if k != 0 {
1733        i -= 2 * k - 1;
1734    }
1735    let val = (k0 as i32 - k as i32 + s) ^ s;
1736    push(val, &mut norm);
1737    // N == 1
1738    let s = -(i as i32);
1739    let val = (k as i32 + s) ^ s;
1740    push(val, &mut norm);
1741    norm
1742}
1743
1744/// `ff_vector_fmul_window_neon(dst, dst, dst + len, win, len)` in place.
1745fn fmul_window_inplace(d: &mut [f32], win: &[f32], len: usize) {
1746    for k in 0..len {
1747        let (i, j) = (k, 2 * len - 1 - k);
1748        let s0 = d[k];
1749        let s1 = d[2 * len - 1 - k];
1750        let (wi, wj) = (win[i], win[j]);
1751        d[i] = (-s1).mul_add(wi, s0 * wj);
1752        d[j] = s1.mul_add(wj, s0 * wi);
1753    }
1754}
1755
1756/// `celt_postfilter` (+ `celt_postfilter_apply_transition` in double, and
1757/// `ff_opus_postfilter_neon`).
1758fn postfilter(b: &mut Block, len: usize) {
1759    let filter_len = len as isize - 2 * OVERLAP as isize;
1760    apply_transition(b, 1024);
1761    b.pf_period_old = b.pf_period;
1762    b.pf_gains_old = b.pf_gains;
1763    b.pf_period = b.pf_period_new;
1764    b.pf_gains = b.pf_gains_new;
1765    if len > OVERLAP {
1766        apply_transition(b, 1024 + OVERLAP);
1767        if b.pf_gains[0] > f32::EPSILON && filter_len > 0 {
1768            postfilter_neon(
1769                &mut b.buf,
1770                1024 + 2 * OVERLAP,
1771                b.pf_period as usize,
1772                b.pf_gains,
1773                filter_len as usize,
1774            );
1775        }
1776        b.pf_period_old = b.pf_period;
1777        b.pf_gains_old = b.pf_gains;
1778    }
1779    b.buf.copy_within(len..len + 1024 + OVERLAP / 2, 0);
1780}
1781
1782fn apply_transition(b: &mut Block, at: usize) {
1783    let t0 = b.pf_period_old as usize;
1784    let t1 = b.pf_period as usize;
1785    if b.pf_gains[0] == 0.0 && b.pf_gains_old[0] == 0.0 {
1786        return;
1787    }
1788    let [g00, g01, g02] = b.pf_gains_old;
1789    let [g10, g11, g12] = b.pf_gains;
1790    let d = &mut b.buf;
1791    let mut x1 = d[at - t1 + 1];
1792    let mut x2 = d[at - t1];
1793    let mut x3 = d[at - t1 - 1];
1794    let mut x4 = d[at - t1 - 2];
1795    for i in 0..OVERLAP {
1796        let w = CELT_WINDOW2[i];
1797        let x0 = d[at + i - t1 + 2];
1798        let iw = 1.0f64 - w as f64;
1799        let sum = iw * g00 as f64 * d[at + i - t0] as f64
1800            + iw * g01 as f64 * (d[at + i - t0 - 1] + d[at + i - t0 + 1]) as f64
1801            + iw * g02 as f64 * (d[at + i - t0 - 2] + d[at + i - t0 + 2]) as f64
1802            + (w * g10 * x2) as f64
1803            + (w * g11 * (x1 + x3)) as f64
1804            + (w * g12 * (x0 + x4)) as f64;
1805        d[at + i] = (d[at + i] as f64 + sum) as f32;
1806        x4 = x3;
1807        x3 = x2;
1808        x2 = x1;
1809        x1 = x0;
1810    }
1811}
1812
1813/// `ff_opus_postfilter_neon`, four lanes at a time (len is a multiple of 4).
1814fn postfilter_neon(d: &mut [f32], at: usize, period: usize, g: [f32; 3], len: usize) {
1815    let mut i = 0;
1816    while i < len {
1817        for l in 0..4 {
1818            let p = at + i + l;
1819            let x4 = d[p - period - 2];
1820            let x3 = d[p - period - 1];
1821            let x2 = d[p - period];
1822            let x1 = d[p - period + 1];
1823            let x0 = d[p - period + 2];
1824            let v3 = x0.mul_add(g[2], x4 * g[2]);
1825            let v6 = (x1 + x3) * g[1] + v3;
1826            let v4 = x2.mul_add(g[0], d[p]);
1827            d[p] = v4 + v6;
1828        }
1829        i += 4;
1830    }
1831}
1832
1833/// `ff_opus_deemphasis_neon`, eight samples per step.
1834fn deemphasis(y: &mut [f32], x: &[f32], coeff: f32, w: &[f32; 16]) -> f32 {
1835    let (v4, v5, v6, v7) = (&w[0..4], &w[4..8], &w[8..12], &w[12..16]);
1836    let mut v0 = [0f32; 4];
1837    for l in 0..4 {
1838        v0[l] = v4[l] * coeff;
1839    }
1840    let mut last = coeff;
1841    for (yc, xc) in y.chunks_mut(8).zip(x.chunks(8)) {
1842        let mut v1 = [xc[0], xc[1], xc[2], xc[3]];
1843        let mut v2 = [xc[4], xc[5], xc[6], xc[7]];
1844        let mut v3 = [0f32; 4];
1845        for l in 0..4 {
1846            v0[l] = v5[l].mul_add(v1[0], v0[l]);
1847            v3[l] = v7[l] * v2[2];
1848            v0[l] = v6[l].mul_add(v1[1], v0[l]);
1849            v3[l] = v6[l].mul_add(v2[1], v3[l]);
1850            v0[l] = v7[l].mul_add(v1[2], v0[l]);
1851            v3[l] = v5[l].mul_add(v2[0], v3[l]);
1852        }
1853        for l in 0..4 {
1854            v1[l] += v0[l];
1855            v2[l] += v3[l];
1856        }
1857        for l in 0..4 {
1858            v2[l] = v4[l].mul_add(v1[3], v2[l]);
1859        }
1860        yc[..4].copy_from_slice(&v1);
1861        yc[4..].copy_from_slice(&v2);
1862        for l in 0..4 {
1863            v0[l] = v4[l] * v2[3];
1864        }
1865        last = v2[3];
1866    }
1867    last
1868}