Skip to main content

threers_codec/vp9/
transform.rs

1//! 4×4 DCT / ADST, 8×8 DCT, 16×16 DCT, and 32×32 DCT transforms matching libvpx.
2//!
3//! Forward: `vpx_fdct4x4_c` / `vp9_fht4x4_c` / `vpx_fdct8x8_c` / `vpx_fdct16x16_c` /
4//! `vpx_fdct32x32_c`.
5//! Inverse: `vpx_idct4x4_16_add_c` / `vp9_iht4x4_16_add_c` / `vpx_idct8x8_64_add_c` /
6//! `vpx_idct16x16_256_add_c` / `vpx_idct32x32_1024_add_c`.
7//! Constants from `vpx_dsp/txfm_common.h` (libvpx v1.14.1).
8
9// The butterflies index a flat buffer as the spec indexes a 2D block, so the
10// row term is written out even when it is `0 * 4` or `1 * 4`. Collapsing those
11// to `input[i]` and `input[4 + i]` would save nothing and would stop the code
12// reading line-for-line against libvpx, which is the only way to check it.
13#![allow(clippy::erasing_op, clippy::identity_op)]
14
15use crate::vp9::tables::{DC_PRED, H_PRED, V_PRED};
16
17const DCT_CONST_BITS: u32 = 14;
18
19const COSPI_1_64: i64 = 16364;
20const COSPI_2_64: i64 = 16305;
21const COSPI_3_64: i64 = 16207;
22const COSPI_4_64: i64 = 16069;
23const COSPI_5_64: i64 = 15893;
24const COSPI_6_64: i64 = 15679;
25const COSPI_7_64: i64 = 15426;
26const COSPI_8_64: i64 = 15137;
27const COSPI_9_64: i64 = 14811;
28const COSPI_10_64: i64 = 14449;
29const COSPI_11_64: i64 = 14053;
30const COSPI_12_64: i64 = 13623;
31const COSPI_13_64: i64 = 13160;
32const COSPI_14_64: i64 = 12665;
33const COSPI_15_64: i64 = 12140;
34const COSPI_16_64: i64 = 11585;
35const COSPI_17_64: i64 = 11003;
36const COSPI_18_64: i64 = 10394;
37const COSPI_19_64: i64 = 9760;
38const COSPI_20_64: i64 = 9102;
39const COSPI_21_64: i64 = 8423;
40const COSPI_22_64: i64 = 7723;
41const COSPI_23_64: i64 = 7005;
42const COSPI_24_64: i64 = 6270;
43const COSPI_25_64: i64 = 5520;
44const COSPI_26_64: i64 = 4756;
45const COSPI_27_64: i64 = 3981;
46const COSPI_28_64: i64 = 3196;
47const COSPI_29_64: i64 = 2404;
48const COSPI_30_64: i64 = 1606;
49const COSPI_31_64: i64 = 804;
50
51const SINPI_1_9: i64 = 5283;
52const SINPI_2_9: i64 = 9929;
53const SINPI_3_9: i64 = 13377;
54const SINPI_4_9: i64 = 15212;
55
56/// VP9 `TX_TYPE` for 4×4 hybrid transforms.
57#[derive(Clone, Copy, Debug, PartialEq, Eq)]
58#[repr(u8)]
59pub enum TxType {
60    DctDct = 0,
61    /// ADST vertical, DCT horizontal — used by `V_PRED`.
62    AdstDct = 1,
63    /// DCT vertical, ADST horizontal — used by `H_PRED`.
64    DctAdst = 2,
65    AdstAdst = 3,
66}
67
68/// Map an intra prediction mode to its TX type (`intra_mode_to_tx_type_lookup`).
69pub fn tx_type_from_mode(mode: i8) -> TxType {
70    match mode {
71        V_PRED => TxType::AdstDct,
72        H_PRED => TxType::DctAdst,
73        DC_PRED => TxType::DctDct,
74        // D135/TM → ADST_ADST; other diagonals unused by this encoder.
75        _ => TxType::AdstAdst,
76    }
77}
78
79/// A 4-point transform: reads one row or column, writes the transformed one.
80type Transform4 = fn(&[i32; 4], &mut [i32; 4]);
81
82#[inline]
83fn round_power_of_two(value: i64, n: u32) -> i64 {
84    (value + (1i64 << (n - 1))) >> n
85}
86
87#[inline]
88fn fdct_round_shift(input: i64) -> i32 {
89    round_power_of_two(input, DCT_CONST_BITS) as i32
90}
91
92#[inline]
93fn dct_const_round_shift(input: i64) -> i32 {
94    round_power_of_two(input, DCT_CONST_BITS) as i32
95}
96
97/// Hardware-style wrap to signed 16-bit (`WRAPLOW` with `CONFIG_EMULATE_HARDWARE`).
98#[inline]
99fn wraplow(x: i32) -> i32 {
100    (x as i16) as i32
101}
102
103/// 1-D forward DCT4 (libvpx encoder `fdct4`).
104fn fdct4(input: &[i32; 4], output: &mut [i32; 4]) {
105    let step = [
106        input[0] as i64 + input[3] as i64,
107        input[1] as i64 + input[2] as i64,
108        input[1] as i64 - input[2] as i64,
109        input[0] as i64 - input[3] as i64,
110    ];
111    let temp1 = (step[0] + step[1]) * COSPI_16_64;
112    let temp2 = (step[0] - step[1]) * COSPI_16_64;
113    output[0] = fdct_round_shift(temp1);
114    output[2] = fdct_round_shift(temp2);
115    let temp1 = step[2] * COSPI_24_64 + step[3] * COSPI_8_64;
116    let temp2 = -step[2] * COSPI_8_64 + step[3] * COSPI_24_64;
117    output[1] = fdct_round_shift(temp1);
118    output[3] = fdct_round_shift(temp2);
119}
120
121/// 1-D forward ADST4 (libvpx encoder `fadst4`).
122fn fadst4(input: &[i32; 4], output: &mut [i32; 4]) {
123    let x0 = input[0] as i64;
124    let x1 = input[1] as i64;
125    let x2 = input[2] as i64;
126    let x3 = input[3] as i64;
127    if (x0 | x1 | x2 | x3) == 0 {
128        *output = [0; 4];
129        return;
130    }
131    let s0 = SINPI_1_9 * x0;
132    let s1 = SINPI_4_9 * x0;
133    let s2 = SINPI_2_9 * x1;
134    let s3 = SINPI_1_9 * x1;
135    let s4 = SINPI_3_9 * x2;
136    let s5 = SINPI_4_9 * x3;
137    let s6 = SINPI_2_9 * x3;
138    let s7 = x0 + x1 - x3;
139
140    let x0 = s0 + s2 + s5;
141    let x1 = SINPI_3_9 * s7;
142    let x2 = s1 - s3 + s6;
143    let x3 = s4;
144
145    let s0 = x0 + x3;
146    let s1 = x1;
147    let s2 = x2 - x3;
148    let s3 = x2 - x0 + x3;
149
150    output[0] = wraplow(fdct_round_shift(s0));
151    output[1] = wraplow(fdct_round_shift(s1));
152    output[2] = wraplow(fdct_round_shift(s2));
153    output[3] = wraplow(fdct_round_shift(s3));
154}
155
156/// Forward 4×4 DCT. `input` is a row-major residual block (stride 4).
157///
158/// Matches libvpx `vpx_fdct4x4_c`.
159pub fn fdct4x4(input: &[i16; 16], output: &mut [i32; 16]) {
160    fht4x4(input, output, TxType::DctDct);
161}
162
163/// Forward 4×4 hybrid transform (`vp9_fht4x4_c`).
164pub fn fht4x4(input: &[i16; 16], output: &mut [i32; 16], tx_type: TxType) {
165    if tx_type == TxType::DctDct {
166        // Dedicated path matches `vpx_fdct4x4_c` bit-exact (same math as FHT
167        // with DCT/DCT, kept separate for clarity with existing tests).
168        let mut intermediate = [0i32; 16];
169        {
170            let mut out_col = 0usize;
171            for i in 0..4 {
172                let mut in_high = [
173                    input[0 * 4 + i] as i64 * 16,
174                    input[1 * 4 + i] as i64 * 16,
175                    input[2 * 4 + i] as i64 * 16,
176                    input[3 * 4 + i] as i64 * 16,
177                ];
178                if i == 0 && in_high[0] != 0 {
179                    in_high[0] += 1;
180                }
181                let step = [
182                    in_high[0] + in_high[3],
183                    in_high[1] + in_high[2],
184                    in_high[1] - in_high[2],
185                    in_high[0] - in_high[3],
186                ];
187                let temp1 = (step[0] + step[1]) * COSPI_16_64;
188                let temp2 = (step[0] - step[1]) * COSPI_16_64;
189                intermediate[out_col] = fdct_round_shift(temp1);
190                intermediate[out_col + 2] = fdct_round_shift(temp2);
191                let temp1 = step[2] * COSPI_24_64 + step[3] * COSPI_8_64;
192                let temp2 = -step[2] * COSPI_8_64 + step[3] * COSPI_24_64;
193                intermediate[out_col + 1] = fdct_round_shift(temp1);
194                intermediate[out_col + 3] = fdct_round_shift(temp2);
195                out_col += 4;
196            }
197        }
198        {
199            let mut out_col = 0usize;
200            for i in 0..4 {
201                let in_high = [
202                    intermediate[0 * 4 + i] as i64,
203                    intermediate[1 * 4 + i] as i64,
204                    intermediate[2 * 4 + i] as i64,
205                    intermediate[3 * 4 + i] as i64,
206                ];
207                let step = [
208                    in_high[0] + in_high[3],
209                    in_high[1] + in_high[2],
210                    in_high[1] - in_high[2],
211                    in_high[0] - in_high[3],
212                ];
213                let temp1 = (step[0] + step[1]) * COSPI_16_64;
214                let temp2 = (step[0] - step[1]) * COSPI_16_64;
215                output[out_col] = fdct_round_shift(temp1);
216                output[out_col + 2] = fdct_round_shift(temp2);
217                let temp1 = step[2] * COSPI_24_64 + step[3] * COSPI_8_64;
218                let temp2 = -step[2] * COSPI_8_64 + step[3] * COSPI_24_64;
219                output[out_col + 1] = fdct_round_shift(temp1);
220                output[out_col + 3] = fdct_round_shift(temp2);
221                out_col += 4;
222            }
223        }
224        for v in output.iter_mut() {
225            *v = (*v + 1) >> 2;
226        }
227        return;
228    }
229
230    let (cols, rows): (Transform4, Transform4) = match tx_type {
231        TxType::AdstDct => (fadst4, fdct4),
232        TxType::DctAdst => (fdct4, fadst4),
233        TxType::AdstAdst => (fadst4, fadst4),
234        TxType::DctDct => unreachable!(),
235    };
236
237    let mut out = [0i32; 16];
238    // Columns
239    for i in 0..4 {
240        let mut temp_in = [
241            input[0 * 4 + i] as i32 * 16,
242            input[1 * 4 + i] as i32 * 16,
243            input[2 * 4 + i] as i32 * 16,
244            input[3 * 4 + i] as i32 * 16,
245        ];
246        if i == 0 && temp_in[0] != 0 {
247            temp_in[0] += 1;
248        }
249        let mut temp_out = [0i32; 4];
250        cols(&temp_in, &mut temp_out);
251        for j in 0..4 {
252            out[j * 4 + i] = temp_out[j];
253        }
254    }
255    // Rows
256    for i in 0..4 {
257        let temp_in = [out[i * 4], out[i * 4 + 1], out[i * 4 + 2], out[i * 4 + 3]];
258        let mut temp_out = [0i32; 4];
259        rows(&temp_in, &mut temp_out);
260        for j in 0..4 {
261            output[i * 4 + j] = (temp_out[j] + 1) >> 2;
262        }
263    }
264}
265
266/// 1-D IDCT4 (libvpx `idct4_c`).
267fn idct4(input: &[i32; 4], output: &mut [i32; 4]) {
268    let i0 = input[0] as i16 as i64;
269    let i1 = input[1] as i16 as i64;
270    let i2 = input[2] as i16 as i64;
271    let i3 = input[3] as i16 as i64;
272
273    let temp1 = (i0 + i2) * COSPI_16_64;
274    let temp2 = (i0 - i2) * COSPI_16_64;
275    let step0 = wraplow(dct_const_round_shift(temp1));
276    let step1 = wraplow(dct_const_round_shift(temp2));
277    let temp1 = i1 * COSPI_24_64 - i3 * COSPI_8_64;
278    let temp2 = i1 * COSPI_8_64 + i3 * COSPI_24_64;
279    let step2 = wraplow(dct_const_round_shift(temp1));
280    let step3 = wraplow(dct_const_round_shift(temp2));
281
282    output[0] = wraplow(step0 + step3);
283    output[1] = wraplow(step1 + step2);
284    output[2] = wraplow(step1 - step2);
285    output[3] = wraplow(step0 - step3);
286}
287
288/// 1-D IADST4 (libvpx `iadst4_c`).
289fn iadst4(input: &[i32; 4], output: &mut [i32; 4]) {
290    let x0 = input[0] as i16 as i32;
291    let x1 = input[1] as i16 as i32;
292    let x2 = input[2] as i16 as i32;
293    let x3 = input[3] as i16 as i32;
294    if (x0 | x1 | x2 | x3) == 0 {
295        *output = [0; 4];
296        return;
297    }
298
299    let s0 = SINPI_1_9 * x0 as i64;
300    let s1 = SINPI_2_9 * x0 as i64;
301    let s2 = SINPI_3_9 * x1 as i64;
302    let s3 = SINPI_4_9 * x2 as i64;
303    let s4 = SINPI_1_9 * x2 as i64;
304    let s5 = SINPI_2_9 * x3 as i64;
305    let s6 = SINPI_4_9 * x3 as i64;
306    let s7 = wraplow(x0 - x2 + x3) as i64;
307
308    let s0 = s0 + s3 + s5;
309    let s1 = s1 - s4 - s6;
310    let s3 = s2;
311    let s2 = SINPI_3_9 * s7;
312
313    output[0] = wraplow(dct_const_round_shift(s0 + s3));
314    output[1] = wraplow(dct_const_round_shift(s1 + s3));
315    output[2] = wraplow(dct_const_round_shift(s2));
316    output[3] = wraplow(dct_const_round_shift(s0 + s1 - s3));
317}
318
319#[inline]
320fn clip_pixel(val: i32) -> u8 {
321    val.clamp(0, 255) as u8
322}
323
324#[inline]
325fn clip_pixel_add(dest: u8, trans: i32) -> u8 {
326    clip_pixel(dest as i32 + wraplow(trans))
327}
328
329/// Inverse 4×4 DCT and add into `dest` (libvpx `vpx_idct4x4_16_add_c`).
330pub fn idct4x4_add(input: &[i32; 16], dest: &mut [u8], stride: usize) {
331    iht4x4_add(input, dest, stride, TxType::DctDct);
332}
333
334/// Inverse 4×4 hybrid transform and add (`vp9_iht4x4_16_add_c`).
335///
336/// libvpx's `transform_2d` is `{ cols, rows }`; IHT applies `.rows` first
337/// (horizontal), then `.cols` (vertical). For `ADST_DCT` that is idct-then-iadst.
338pub fn iht4x4_add(input: &[i32; 16], dest: &mut [u8], stride: usize, tx_type: TxType) {
339    assert!(dest.len() >= 3 * stride + 4);
340
341    // (cols, rows) — same packing as libvpx `transform_2d`.
342    let (cols, rows): (Transform4, Transform4) = match tx_type {
343        TxType::DctDct => (idct4, idct4),
344        TxType::AdstDct => (iadst4, idct4),
345        TxType::DctAdst => (idct4, iadst4),
346        TxType::AdstAdst => (iadst4, iadst4),
347    };
348
349    let mut out = [0i32; 16];
350    // Rows (horizontal) first.
351    for i in 0..4 {
352        let row_in = [
353            input[i * 4],
354            input[i * 4 + 1],
355            input[i * 4 + 2],
356            input[i * 4 + 3],
357        ];
358        let mut row_out = [0i32; 4];
359        rows(&row_in, &mut row_out);
360        out[i * 4..i * 4 + 4].copy_from_slice(&row_out);
361    }
362    // Columns (vertical) second.
363    for i in 0..4 {
364        let col_in = [
365            out[0 * 4 + i],
366            out[1 * 4 + i],
367            out[2 * 4 + i],
368            out[3 * 4 + i],
369        ];
370        let mut col_out = [0i32; 4];
371        cols(&col_in, &mut col_out);
372        for j in 0..4 {
373            let add = round_power_of_two(col_out[j] as i64, 4) as i32;
374            let idx = j * stride + i;
375            dest[idx] = clip_pixel_add(dest[idx], add);
376        }
377    }
378}
379
380/// Forward 8×8 DCT (`vpx_fdct8x8_c`). `input` is row-major residual (stride 8).
381pub fn fdct8x8(input: &[i16; 64], output: &mut [i32; 64]) {
382    let mut intermediate = [0i32; 64];
383    for pass in 0..2 {
384        for i in 0..8 {
385            let (s0, s1, s2, s3, s4, s5, s6, s7) = if pass == 0 {
386                (
387                    (input[0 * 8 + i] as i64 + input[7 * 8 + i] as i64) * 4,
388                    (input[1 * 8 + i] as i64 + input[6 * 8 + i] as i64) * 4,
389                    (input[2 * 8 + i] as i64 + input[5 * 8 + i] as i64) * 4,
390                    (input[3 * 8 + i] as i64 + input[4 * 8 + i] as i64) * 4,
391                    (input[3 * 8 + i] as i64 - input[4 * 8 + i] as i64) * 4,
392                    (input[2 * 8 + i] as i64 - input[5 * 8 + i] as i64) * 4,
393                    (input[1 * 8 + i] as i64 - input[6 * 8 + i] as i64) * 4,
394                    (input[0 * 8 + i] as i64 - input[7 * 8 + i] as i64) * 4,
395                )
396            } else {
397                (
398                    intermediate[0 * 8 + i] as i64 + intermediate[7 * 8 + i] as i64,
399                    intermediate[1 * 8 + i] as i64 + intermediate[6 * 8 + i] as i64,
400                    intermediate[2 * 8 + i] as i64 + intermediate[5 * 8 + i] as i64,
401                    intermediate[3 * 8 + i] as i64 + intermediate[4 * 8 + i] as i64,
402                    intermediate[3 * 8 + i] as i64 - intermediate[4 * 8 + i] as i64,
403                    intermediate[2 * 8 + i] as i64 - intermediate[5 * 8 + i] as i64,
404                    intermediate[1 * 8 + i] as i64 - intermediate[6 * 8 + i] as i64,
405                    intermediate[0 * 8 + i] as i64 - intermediate[7 * 8 + i] as i64,
406                )
407            };
408
409            let x0 = s0 + s3;
410            let x1 = s1 + s2;
411            let x2 = s1 - s2;
412            let x3 = s0 - s3;
413            let t0 = (x0 + x1) * COSPI_16_64;
414            let t1 = (x0 - x1) * COSPI_16_64;
415            let t2 = x2 * COSPI_24_64 + x3 * COSPI_8_64;
416            let t3 = -x2 * COSPI_8_64 + x3 * COSPI_24_64;
417
418            let dest = if pass == 0 {
419                &mut intermediate[i * 8..i * 8 + 8]
420            } else {
421                &mut output[i * 8..i * 8 + 8]
422            };
423            dest[0] = fdct_round_shift(t0);
424            dest[2] = fdct_round_shift(t2);
425            dest[4] = fdct_round_shift(t1);
426            dest[6] = fdct_round_shift(t3);
427
428            let t0 = (s6 - s5) * COSPI_16_64;
429            let t1 = (s6 + s5) * COSPI_16_64;
430            let t2 = fdct_round_shift(t0) as i64;
431            let t3 = fdct_round_shift(t1) as i64;
432
433            let x0 = s4 + t2;
434            let x1 = s4 - t2;
435            let x2 = s7 - t3;
436            let x3 = s7 + t3;
437
438            let t0 = x0 * COSPI_28_64 + x3 * COSPI_4_64;
439            let t1 = x1 * COSPI_12_64 + x2 * COSPI_20_64;
440            let t2 = x2 * COSPI_12_64 + x1 * (-COSPI_20_64);
441            let t3 = x3 * COSPI_28_64 + x0 * (-COSPI_4_64);
442            dest[1] = fdct_round_shift(t0);
443            dest[3] = fdct_round_shift(t2);
444            dest[5] = fdct_round_shift(t1);
445            dest[7] = fdct_round_shift(t3);
446        }
447    }
448    for v in output.iter_mut() {
449        *v /= 2;
450    }
451}
452
453/// One 1-D forward DCT16 pass (libvpx `vpx_fdct16x16_c` inner loop).
454fn fdct16_1d(in_high: [i64; 8], step1: [i64; 8], out: &mut [i32; 16]) {
455    let mut step2 = [0i64; 8];
456    let mut step3 = [0i64; 8];
457
458    // Even part (fdct8 on sums).
459    {
460        let s0 = in_high[0] + in_high[7];
461        let s1 = in_high[1] + in_high[6];
462        let s2 = in_high[2] + in_high[5];
463        let s3 = in_high[3] + in_high[4];
464        let s4 = in_high[3] - in_high[4];
465        let s5 = in_high[2] - in_high[5];
466        let s6 = in_high[1] - in_high[6];
467        let s7 = in_high[0] - in_high[7];
468
469        let x0 = s0 + s3;
470        let x1 = s1 + s2;
471        let x2 = s1 - s2;
472        let x3 = s0 - s3;
473        let t0 = (x0 + x1) * COSPI_16_64;
474        let t1 = (x0 - x1) * COSPI_16_64;
475        let t2 = x3 * COSPI_8_64 + x2 * COSPI_24_64;
476        let t3 = x3 * COSPI_24_64 - x2 * COSPI_8_64;
477        out[0] = fdct_round_shift(t0);
478        out[4] = fdct_round_shift(t2);
479        out[8] = fdct_round_shift(t1);
480        out[12] = fdct_round_shift(t3);
481
482        let t0 = (s6 - s5) * COSPI_16_64;
483        let t1 = (s6 + s5) * COSPI_16_64;
484        let t2 = fdct_round_shift(t0) as i64;
485        let t3 = fdct_round_shift(t1) as i64;
486
487        let x0 = s4 + t2;
488        let x1 = s4 - t2;
489        let x2 = s7 - t3;
490        let x3 = s7 + t3;
491
492        let t0 = x0 * COSPI_28_64 + x3 * COSPI_4_64;
493        let t1 = x1 * COSPI_12_64 + x2 * COSPI_20_64;
494        let t2 = x2 * COSPI_12_64 + x1 * (-COSPI_20_64);
495        let t3 = x3 * COSPI_28_64 + x0 * (-COSPI_4_64);
496        out[2] = fdct_round_shift(t0);
497        out[6] = fdct_round_shift(t2);
498        out[10] = fdct_round_shift(t1);
499        out[14] = fdct_round_shift(t3);
500    }
501
502    // Odd part (step1 -> odd_results).
503    {
504        let temp1 = (step1[5] - step1[2]) * COSPI_16_64;
505        let temp2 = (step1[4] - step1[3]) * COSPI_16_64;
506        step2[2] = fdct_round_shift(temp1) as i64;
507        step2[3] = fdct_round_shift(temp2) as i64;
508        let temp1 = (step1[4] + step1[3]) * COSPI_16_64;
509        let temp2 = (step1[5] + step1[2]) * COSPI_16_64;
510        step2[4] = fdct_round_shift(temp1) as i64;
511        step2[5] = fdct_round_shift(temp2) as i64;
512
513        step3[0] = step1[0] + step2[3];
514        step3[1] = step1[1] + step2[2];
515        step3[2] = step1[1] - step2[2];
516        step3[3] = step1[0] - step2[3];
517        step3[4] = step1[7] - step2[4];
518        step3[5] = step1[6] - step2[5];
519        step3[6] = step1[6] + step2[5];
520        step3[7] = step1[7] + step2[4];
521
522        let temp1 = step3[1] * (-COSPI_8_64) + step3[6] * COSPI_24_64;
523        let temp2 = step3[2] * COSPI_24_64 + step3[5] * COSPI_8_64;
524        step2[1] = fdct_round_shift(temp1) as i64;
525        step2[2] = fdct_round_shift(temp2) as i64;
526        let temp1 = step3[2] * COSPI_8_64 - step3[5] * COSPI_24_64;
527        let temp2 = step3[1] * COSPI_24_64 + step3[6] * COSPI_8_64;
528        step2[5] = fdct_round_shift(temp1) as i64;
529        step2[6] = fdct_round_shift(temp2) as i64;
530
531        let mut s1 = step1;
532        s1[0] = step3[0] + step2[1];
533        s1[1] = step3[0] - step2[1];
534        s1[2] = step3[3] + step2[2];
535        s1[3] = step3[3] - step2[2];
536        s1[4] = step3[4] - step2[5];
537        s1[5] = step3[4] + step2[5];
538        s1[6] = step3[7] - step2[6];
539        s1[7] = step3[7] + step2[6];
540
541        let temp1 = s1[0] * COSPI_30_64 + s1[7] * COSPI_2_64;
542        let temp2 = s1[1] * COSPI_14_64 + s1[6] * COSPI_18_64;
543        out[1] = fdct_round_shift(temp1);
544        out[9] = fdct_round_shift(temp2);
545        let temp1 = s1[2] * COSPI_22_64 + s1[5] * COSPI_10_64;
546        let temp2 = s1[3] * COSPI_6_64 + s1[4] * COSPI_26_64;
547        out[5] = fdct_round_shift(temp1);
548        out[13] = fdct_round_shift(temp2);
549        let temp1 = s1[3] * (-COSPI_26_64) + s1[4] * COSPI_6_64;
550        let temp2 = s1[2] * (-COSPI_10_64) + s1[5] * COSPI_22_64;
551        out[3] = fdct_round_shift(temp1);
552        out[11] = fdct_round_shift(temp2);
553        let temp1 = s1[1] * (-COSPI_18_64) + s1[6] * COSPI_14_64;
554        let temp2 = s1[0] * (-COSPI_2_64) + s1[7] * COSPI_30_64;
555        out[7] = fdct_round_shift(temp1);
556        out[15] = fdct_round_shift(temp2);
557    }
558}
559
560/// Forward 16×16 DCT (`vpx_fdct16x16_c`). `input` is row-major residual (stride 16).
561pub fn fdct16x16(input: &[i16; 256], output: &mut [i32; 256]) {
562    let mut intermediate = [0i32; 256];
563    for pass in 0..2 {
564        for i in 0..16 {
565            let (in_high, step1) = if pass == 0 {
566                let col = |row: usize| input[row * 16 + i] as i64;
567                (
568                    [
569                        (col(0) + col(15)) * 4,
570                        (col(1) + col(14)) * 4,
571                        (col(2) + col(13)) * 4,
572                        (col(3) + col(12)) * 4,
573                        (col(4) + col(11)) * 4,
574                        (col(5) + col(10)) * 4,
575                        (col(6) + col(9)) * 4,
576                        (col(7) + col(8)) * 4,
577                    ],
578                    [
579                        (col(7) - col(8)) * 4,
580                        (col(6) - col(9)) * 4,
581                        (col(5) - col(10)) * 4,
582                        (col(4) - col(11)) * 4,
583                        (col(3) - col(12)) * 4,
584                        (col(2) - col(13)) * 4,
585                        (col(1) - col(14)) * 4,
586                        (col(0) - col(15)) * 4,
587                    ],
588                )
589            } else {
590                let col = |row: usize| {
591                    let v = intermediate[row * 16 + i];
592                    ((v + 1) >> 2) as i64
593                };
594                (
595                    [
596                        col(0) + col(15),
597                        col(1) + col(14),
598                        col(2) + col(13),
599                        col(3) + col(12),
600                        col(4) + col(11),
601                        col(5) + col(10),
602                        col(6) + col(9),
603                        col(7) + col(8),
604                    ],
605                    [
606                        col(7) - col(8),
607                        col(6) - col(9),
608                        col(5) - col(10),
609                        col(4) - col(11),
610                        col(3) - col(12),
611                        col(2) - col(13),
612                        col(1) - col(14),
613                        col(0) - col(15),
614                    ],
615                )
616            };
617
618            let dest = if pass == 0 {
619                &mut intermediate[i * 16..i * 16 + 16]
620            } else {
621                &mut output[i * 16..i * 16 + 16]
622            };
623            fdct16_1d(in_high, step1, dest.try_into().unwrap());
624        }
625    }
626}
627
628fn idct8(input: &[i32; 8], output: &mut [i32; 8]) {
629    let mut step1 = [0i32; 8];
630    let mut step2 = [0i32; 8];
631
632    step1[0] = input[0] as i16 as i32;
633    step1[2] = input[4] as i16 as i32;
634    step1[1] = input[2] as i16 as i32;
635    step1[3] = input[6] as i16 as i32;
636    let temp1 = input[1] as i16 as i64 * COSPI_28_64 - input[7] as i16 as i64 * COSPI_4_64;
637    let temp2 = input[1] as i16 as i64 * COSPI_4_64 + input[7] as i16 as i64 * COSPI_28_64;
638    step1[4] = wraplow(dct_const_round_shift(temp1));
639    step1[7] = wraplow(dct_const_round_shift(temp2));
640    let temp1 = input[5] as i16 as i64 * COSPI_12_64 - input[3] as i16 as i64 * COSPI_20_64;
641    let temp2 = input[5] as i16 as i64 * COSPI_20_64 + input[3] as i16 as i64 * COSPI_12_64;
642    step1[5] = wraplow(dct_const_round_shift(temp1));
643    step1[6] = wraplow(dct_const_round_shift(temp2));
644
645    let temp1 = (step1[0] as i64 + step1[2] as i64) * COSPI_16_64;
646    let temp2 = (step1[0] as i64 - step1[2] as i64) * COSPI_16_64;
647    step2[0] = wraplow(dct_const_round_shift(temp1));
648    step2[1] = wraplow(dct_const_round_shift(temp2));
649    let temp1 = step1[1] as i64 * COSPI_24_64 - step1[3] as i64 * COSPI_8_64;
650    let temp2 = step1[1] as i64 * COSPI_8_64 + step1[3] as i64 * COSPI_24_64;
651    step2[2] = wraplow(dct_const_round_shift(temp1));
652    step2[3] = wraplow(dct_const_round_shift(temp2));
653    step2[4] = wraplow(step1[4] + step1[5]);
654    step2[5] = wraplow(step1[4] - step1[5]);
655    step2[6] = wraplow(-step1[6] + step1[7]);
656    step2[7] = wraplow(step1[6] + step1[7]);
657
658    step1[0] = wraplow(step2[0] + step2[3]);
659    step1[1] = wraplow(step2[1] + step2[2]);
660    step1[2] = wraplow(step2[1] - step2[2]);
661    step1[3] = wraplow(step2[0] - step2[3]);
662    step1[4] = step2[4];
663    let temp1 = (step2[6] as i64 - step2[5] as i64) * COSPI_16_64;
664    let temp2 = (step2[5] as i64 + step2[6] as i64) * COSPI_16_64;
665    step1[5] = wraplow(dct_const_round_shift(temp1));
666    step1[6] = wraplow(dct_const_round_shift(temp2));
667    step1[7] = step2[7];
668
669    output[0] = wraplow(step1[0] + step1[7]);
670    output[1] = wraplow(step1[1] + step1[6]);
671    output[2] = wraplow(step1[2] + step1[5]);
672    output[3] = wraplow(step1[3] + step1[4]);
673    output[4] = wraplow(step1[3] - step1[4]);
674    output[5] = wraplow(step1[2] - step1[5]);
675    output[6] = wraplow(step1[1] - step1[6]);
676    output[7] = wraplow(step1[0] - step1[7]);
677}
678
679/// 1-D IDCT16 (libvpx `idct16_c`).
680fn idct16(input: &[i32; 16], output: &mut [i32; 16]) {
681    let mut step1 = [0i32; 16];
682    let mut step2 = [0i32; 16];
683
684    // stage 1
685    step1[0] = input[0] as i16 as i32;
686    step1[1] = input[8] as i16 as i32;
687    step1[2] = input[4] as i16 as i32;
688    step1[3] = input[12] as i16 as i32;
689    step1[4] = input[2] as i16 as i32;
690    step1[5] = input[10] as i16 as i32;
691    step1[6] = input[6] as i16 as i32;
692    step1[7] = input[14] as i16 as i32;
693    step1[8] = input[1] as i16 as i32;
694    step1[9] = input[9] as i16 as i32;
695    step1[10] = input[5] as i16 as i32;
696    step1[11] = input[13] as i16 as i32;
697    step1[12] = input[3] as i16 as i32;
698    step1[13] = input[11] as i16 as i32;
699    step1[14] = input[7] as i16 as i32;
700    step1[15] = input[15] as i16 as i32;
701
702    // stage 2
703    step2[0] = step1[0];
704    step2[1] = step1[1];
705    step2[2] = step1[2];
706    step2[3] = step1[3];
707    step2[4] = step1[4];
708    step2[5] = step1[5];
709    step2[6] = step1[6];
710    step2[7] = step1[7];
711
712    let temp1 = step1[8] as i64 * COSPI_30_64 - step1[15] as i64 * COSPI_2_64;
713    let temp2 = step1[8] as i64 * COSPI_2_64 + step1[15] as i64 * COSPI_30_64;
714    step2[8] = wraplow(dct_const_round_shift(temp1));
715    step2[15] = wraplow(dct_const_round_shift(temp2));
716
717    let temp1 = step1[9] as i64 * COSPI_14_64 - step1[14] as i64 * COSPI_18_64;
718    let temp2 = step1[9] as i64 * COSPI_18_64 + step1[14] as i64 * COSPI_14_64;
719    step2[9] = wraplow(dct_const_round_shift(temp1));
720    step2[14] = wraplow(dct_const_round_shift(temp2));
721
722    let temp1 = step1[10] as i64 * COSPI_22_64 - step1[13] as i64 * COSPI_10_64;
723    let temp2 = step1[10] as i64 * COSPI_10_64 + step1[13] as i64 * COSPI_22_64;
724    step2[10] = wraplow(dct_const_round_shift(temp1));
725    step2[13] = wraplow(dct_const_round_shift(temp2));
726
727    let temp1 = step1[11] as i64 * COSPI_6_64 - step1[12] as i64 * COSPI_26_64;
728    let temp2 = step1[11] as i64 * COSPI_26_64 + step1[12] as i64 * COSPI_6_64;
729    step2[11] = wraplow(dct_const_round_shift(temp1));
730    step2[12] = wraplow(dct_const_round_shift(temp2));
731
732    // stage 3
733    step1[0] = step2[0];
734    step1[1] = step2[1];
735    step1[2] = step2[2];
736    step1[3] = step2[3];
737
738    let temp1 = step2[4] as i64 * COSPI_28_64 - step2[7] as i64 * COSPI_4_64;
739    let temp2 = step2[4] as i64 * COSPI_4_64 + step2[7] as i64 * COSPI_28_64;
740    step1[4] = wraplow(dct_const_round_shift(temp1));
741    step1[7] = wraplow(dct_const_round_shift(temp2));
742    let temp1 = step2[5] as i64 * COSPI_12_64 - step2[6] as i64 * COSPI_20_64;
743    let temp2 = step2[5] as i64 * COSPI_20_64 + step2[6] as i64 * COSPI_12_64;
744    step1[5] = wraplow(dct_const_round_shift(temp1));
745    step1[6] = wraplow(dct_const_round_shift(temp2));
746
747    step1[8] = wraplow(step2[8] + step2[9]);
748    step1[9] = wraplow(step2[8] - step2[9]);
749    step1[10] = wraplow(-step2[10] + step2[11]);
750    step1[11] = wraplow(step2[10] + step2[11]);
751    step1[12] = wraplow(step2[12] + step2[13]);
752    step1[13] = wraplow(step2[12] - step2[13]);
753    step1[14] = wraplow(-step2[14] + step2[15]);
754    step1[15] = wraplow(step2[14] + step2[15]);
755
756    // stage 4
757    let temp1 = (step1[0] as i64 + step1[1] as i64) * COSPI_16_64;
758    let temp2 = (step1[0] as i64 - step1[1] as i64) * COSPI_16_64;
759    step2[0] = wraplow(dct_const_round_shift(temp1));
760    step2[1] = wraplow(dct_const_round_shift(temp2));
761    let temp1 = step1[2] as i64 * COSPI_24_64 - step1[3] as i64 * COSPI_8_64;
762    let temp2 = step1[2] as i64 * COSPI_8_64 + step1[3] as i64 * COSPI_24_64;
763    step2[2] = wraplow(dct_const_round_shift(temp1));
764    step2[3] = wraplow(dct_const_round_shift(temp2));
765    step2[4] = wraplow(step1[4] + step1[5]);
766    step2[5] = wraplow(step1[4] - step1[5]);
767    step2[6] = wraplow(-step1[6] + step1[7]);
768    step2[7] = wraplow(step1[6] + step1[7]);
769
770    step2[8] = step1[8];
771    step2[15] = step1[15];
772    let temp1 = -step1[9] as i64 * COSPI_8_64 + step1[14] as i64 * COSPI_24_64;
773    let temp2 = step1[9] as i64 * COSPI_24_64 + step1[14] as i64 * COSPI_8_64;
774    step2[9] = wraplow(dct_const_round_shift(temp1));
775    step2[14] = wraplow(dct_const_round_shift(temp2));
776    let temp1 = -step1[10] as i64 * COSPI_24_64 - step1[13] as i64 * COSPI_8_64;
777    let temp2 = -step1[10] as i64 * COSPI_8_64 + step1[13] as i64 * COSPI_24_64;
778    step2[10] = wraplow(dct_const_round_shift(temp1));
779    step2[13] = wraplow(dct_const_round_shift(temp2));
780    step2[11] = step1[11];
781    step2[12] = step1[12];
782
783    // stage 5
784    step1[0] = wraplow(step2[0] + step2[3]);
785    step1[1] = wraplow(step2[1] + step2[2]);
786    step1[2] = wraplow(step2[1] - step2[2]);
787    step1[3] = wraplow(step2[0] - step2[3]);
788    step1[4] = step2[4];
789    let temp1 = (step2[6] as i64 - step2[5] as i64) * COSPI_16_64;
790    let temp2 = (step2[5] as i64 + step2[6] as i64) * COSPI_16_64;
791    step1[5] = wraplow(dct_const_round_shift(temp1));
792    step1[6] = wraplow(dct_const_round_shift(temp2));
793    step1[7] = step2[7];
794
795    step1[8] = wraplow(step2[8] + step2[11]);
796    step1[9] = wraplow(step2[9] + step2[10]);
797    step1[10] = wraplow(step2[9] - step2[10]);
798    step1[11] = wraplow(step2[8] - step2[11]);
799    step1[12] = wraplow(-step2[12] + step2[15]);
800    step1[13] = wraplow(-step2[13] + step2[14]);
801    step1[14] = wraplow(step2[13] + step2[14]);
802    step1[15] = wraplow(step2[12] + step2[15]);
803
804    // stage 6
805    step2[0] = wraplow(step1[0] + step1[7]);
806    step2[1] = wraplow(step1[1] + step1[6]);
807    step2[2] = wraplow(step1[2] + step1[5]);
808    step2[3] = wraplow(step1[3] + step1[4]);
809    step2[4] = wraplow(step1[3] - step1[4]);
810    step2[5] = wraplow(step1[2] - step1[5]);
811    step2[6] = wraplow(step1[1] - step1[6]);
812    step2[7] = wraplow(step1[0] - step1[7]);
813    step2[8] = step1[8];
814    step2[9] = step1[9];
815    let temp1 = (-step1[10] + step1[13]) as i64 * COSPI_16_64;
816    let temp2 = (step1[10] + step1[13]) as i64 * COSPI_16_64;
817    step2[10] = wraplow(dct_const_round_shift(temp1));
818    step2[13] = wraplow(dct_const_round_shift(temp2));
819    let temp1 = (-step1[11] + step1[12]) as i64 * COSPI_16_64;
820    let temp2 = (step1[11] + step1[12]) as i64 * COSPI_16_64;
821    step2[11] = wraplow(dct_const_round_shift(temp1));
822    step2[12] = wraplow(dct_const_round_shift(temp2));
823    step2[14] = step1[14];
824    step2[15] = step1[15];
825
826    // stage 7
827    output[0] = wraplow(step2[0] + step2[15]);
828    output[1] = wraplow(step2[1] + step2[14]);
829    output[2] = wraplow(step2[2] + step2[13]);
830    output[3] = wraplow(step2[3] + step2[12]);
831    output[4] = wraplow(step2[4] + step2[11]);
832    output[5] = wraplow(step2[5] + step2[10]);
833    output[6] = wraplow(step2[6] + step2[9]);
834    output[7] = wraplow(step2[7] + step2[8]);
835    output[8] = wraplow(step2[7] - step2[8]);
836    output[9] = wraplow(step2[6] - step2[9]);
837    output[10] = wraplow(step2[5] - step2[10]);
838    output[11] = wraplow(step2[4] - step2[11]);
839    output[12] = wraplow(step2[3] - step2[12]);
840    output[13] = wraplow(step2[2] - step2[13]);
841    output[14] = wraplow(step2[1] - step2[14]);
842    output[15] = wraplow(step2[0] - step2[15]);
843}
844
845/// Inverse 8×8 DCT and add into `dest` (`vpx_idct8x8_64_add_c`).
846pub fn idct8x8_add(input: &[i32; 64], dest: &mut [u8], stride: usize) {
847    assert!(dest.len() >= 7 * stride + 8);
848    let mut out = [0i32; 64];
849    for i in 0..8 {
850        let mut row_in = [0i32; 8];
851        row_in.copy_from_slice(&input[i * 8..i * 8 + 8]);
852        let mut row_out = [0i32; 8];
853        idct8(&row_in, &mut row_out);
854        out[i * 8..i * 8 + 8].copy_from_slice(&row_out);
855    }
856    for i in 0..8 {
857        let col_in = [
858            out[0 * 8 + i],
859            out[1 * 8 + i],
860            out[2 * 8 + i],
861            out[3 * 8 + i],
862            out[4 * 8 + i],
863            out[5 * 8 + i],
864            out[6 * 8 + i],
865            out[7 * 8 + i],
866        ];
867        let mut col_out = [0i32; 8];
868        idct8(&col_in, &mut col_out);
869        for j in 0..8 {
870            let add = round_power_of_two(col_out[j] as i64, 5) as i32;
871            let idx = j * stride + i;
872            dest[idx] = clip_pixel_add(dest[idx], add);
873        }
874    }
875}
876
877/// Inverse 16×16 DCT and add into `dest` (`vpx_idct16x16_256_add_c`).
878pub fn idct16x16_add(input: &[i32; 256], dest: &mut [u8], stride: usize) {
879    assert!(dest.len() >= 15 * stride + 16);
880    let mut out = [0i32; 256];
881    for i in 0..16 {
882        let row_in: [i32; 16] = input[i * 16..i * 16 + 16].try_into().unwrap();
883        let mut row_out = [0i32; 16];
884        idct16(&row_in, &mut row_out);
885        out[i * 16..i * 16 + 16].copy_from_slice(&row_out);
886    }
887    for i in 0..16 {
888        let col_in = [
889            out[0 * 16 + i],
890            out[1 * 16 + i],
891            out[2 * 16 + i],
892            out[3 * 16 + i],
893            out[4 * 16 + i],
894            out[5 * 16 + i],
895            out[6 * 16 + i],
896            out[7 * 16 + i],
897            out[8 * 16 + i],
898            out[9 * 16 + i],
899            out[10 * 16 + i],
900            out[11 * 16 + i],
901            out[12 * 16 + i],
902            out[13 * 16 + i],
903            out[14 * 16 + i],
904            out[15 * 16 + i],
905        ];
906        let mut col_out = [0i32; 16];
907        idct16(&col_in, &mut col_out);
908        for j in 0..16 {
909            let add = round_power_of_two(col_out[j] as i64, 6) as i32;
910            let idx = j * stride + i;
911            dest[idx] = clip_pixel_add(dest[idx], add);
912        }
913    }
914}
915
916#[inline]
917fn dct_32_round(input: i64) -> i32 {
918    fdct_round_shift(input)
919}
920
921#[inline]
922fn half_round_shift(x: i32) -> i32 {
923    (x + 1) >> 2
924}
925
926/// 1-D forward DCT32 (libvpx `vpx_fdct32`). `round` applies `half_round_shift` after stage 2.
927fn fdct32(input: &[i64; 32], output: &mut [i32; 32], round: bool) {
928    let mut step = [0i64; 32];
929
930    // Stage 1
931    for i in 0..16 {
932        step[i] = input[i] + input[31 - i];
933    }
934    for i in 0..16 {
935        step[16 + i] = -input[16 + i] + input[15 - i];
936    }
937
938    // Stage 2
939    output[0] = (step[0] + step[15]) as i32;
940    output[1] = (step[1] + step[14]) as i32;
941    output[2] = (step[2] + step[13]) as i32;
942    output[3] = (step[3] + step[12]) as i32;
943    output[4] = (step[4] + step[11]) as i32;
944    output[5] = (step[5] + step[10]) as i32;
945    output[6] = (step[6] + step[9]) as i32;
946    output[7] = (step[7] + step[8]) as i32;
947    output[8] = (-step[8] + step[7]) as i32;
948    output[9] = (-step[9] + step[6]) as i32;
949    output[10] = (-step[10] + step[5]) as i32;
950    output[11] = (-step[11] + step[4]) as i32;
951    output[12] = (-step[12] + step[3]) as i32;
952    output[13] = (-step[13] + step[2]) as i32;
953    output[14] = (-step[14] + step[1]) as i32;
954    output[15] = (-step[15] + step[0]) as i32;
955
956    output[16] = step[16] as i32;
957    output[17] = step[17] as i32;
958    output[18] = step[18] as i32;
959    output[19] = step[19] as i32;
960
961    output[20] = dct_32_round((-step[20] + step[27]) * COSPI_16_64);
962    output[21] = dct_32_round((-step[21] + step[26]) * COSPI_16_64);
963    output[22] = dct_32_round((-step[22] + step[25]) * COSPI_16_64);
964    output[23] = dct_32_round((-step[23] + step[24]) * COSPI_16_64);
965
966    output[24] = dct_32_round((step[24] + step[23]) * COSPI_16_64);
967    output[25] = dct_32_round((step[25] + step[22]) * COSPI_16_64);
968    output[26] = dct_32_round((step[26] + step[21]) * COSPI_16_64);
969    output[27] = dct_32_round((step[27] + step[20]) * COSPI_16_64);
970
971    output[28] = step[28] as i32;
972    output[29] = step[29] as i32;
973    output[30] = step[30] as i32;
974    output[31] = step[31] as i32;
975
976    if round {
977        for v in output.iter_mut() {
978            *v = half_round_shift(*v);
979        }
980    }
981
982    // Stage 3 — reload step from output
983    for i in 0..32 {
984        step[i] = output[i] as i64;
985    }
986    step[0] = output[0] as i64 + output[7] as i64;
987    step[1] = output[1] as i64 + output[6] as i64;
988    step[2] = output[2] as i64 + output[5] as i64;
989    step[3] = output[3] as i64 + output[4] as i64;
990    step[4] = -output[4] as i64 + output[3] as i64;
991    step[5] = -output[5] as i64 + output[2] as i64;
992    step[6] = -output[6] as i64 + output[1] as i64;
993    step[7] = -output[7] as i64 + output[0] as i64;
994    step[8] = output[8] as i64;
995    step[9] = output[9] as i64;
996    step[10] = dct_32_round((-output[10] as i64 + output[13] as i64) * COSPI_16_64) as i64;
997    step[11] = dct_32_round((-output[11] as i64 + output[12] as i64) * COSPI_16_64) as i64;
998    step[12] = dct_32_round((output[12] as i64 + output[11] as i64) * COSPI_16_64) as i64;
999    step[13] = dct_32_round((output[13] as i64 + output[10] as i64) * COSPI_16_64) as i64;
1000    step[14] = output[14] as i64;
1001    step[15] = output[15] as i64;
1002
1003    step[16] = output[16] as i64 + output[23] as i64;
1004    step[17] = output[17] as i64 + output[22] as i64;
1005    step[18] = output[18] as i64 + output[21] as i64;
1006    step[19] = output[19] as i64 + output[20] as i64;
1007    step[20] = -output[20] as i64 + output[19] as i64;
1008    step[21] = -output[21] as i64 + output[18] as i64;
1009    step[22] = -output[22] as i64 + output[17] as i64;
1010    step[23] = -output[23] as i64 + output[16] as i64;
1011    step[24] = -output[24] as i64 + output[31] as i64;
1012    step[25] = -output[25] as i64 + output[30] as i64;
1013    step[26] = -output[26] as i64 + output[29] as i64;
1014    step[27] = -output[27] as i64 + output[28] as i64;
1015    step[28] = output[28] as i64 + output[27] as i64;
1016    step[29] = output[29] as i64 + output[26] as i64;
1017    step[30] = output[30] as i64 + output[25] as i64;
1018    step[31] = output[31] as i64 + output[24] as i64;
1019
1020    // Stage 4
1021    output[0] = (step[0] + step[3]) as i32;
1022    output[1] = (step[1] + step[2]) as i32;
1023    output[2] = (-step[2] + step[1]) as i32;
1024    output[3] = (-step[3] + step[0]) as i32;
1025    output[4] = step[4] as i32;
1026    output[5] = dct_32_round((-step[5] + step[6]) * COSPI_16_64);
1027    output[6] = dct_32_round((step[6] + step[5]) * COSPI_16_64);
1028    output[7] = step[7] as i32;
1029    output[8] = (step[8] + step[11]) as i32;
1030    output[9] = (step[9] + step[10]) as i32;
1031    output[10] = (-step[10] + step[9]) as i32;
1032    output[11] = (-step[11] + step[8]) as i32;
1033    output[12] = (-step[12] + step[15]) as i32;
1034    output[13] = (-step[13] + step[14]) as i32;
1035    output[14] = (step[14] + step[13]) as i32;
1036    output[15] = (step[15] + step[12]) as i32;
1037
1038    output[16] = step[16] as i32;
1039    output[17] = step[17] as i32;
1040    output[18] = dct_32_round(step[18] * -COSPI_8_64 + step[29] * COSPI_24_64);
1041    output[19] = dct_32_round(step[19] * -COSPI_8_64 + step[28] * COSPI_24_64);
1042    output[20] = dct_32_round(step[20] * -COSPI_24_64 + step[27] * -COSPI_8_64);
1043    output[21] = dct_32_round(step[21] * -COSPI_24_64 + step[26] * -COSPI_8_64);
1044    output[22] = step[22] as i32;
1045    output[23] = step[23] as i32;
1046    output[24] = step[24] as i32;
1047    output[25] = step[25] as i32;
1048    output[26] = dct_32_round(step[26] * COSPI_24_64 + step[21] * -COSPI_8_64);
1049    output[27] = dct_32_round(step[27] * COSPI_24_64 + step[20] * -COSPI_8_64);
1050    output[28] = dct_32_round(step[28] * COSPI_8_64 + step[19] * COSPI_24_64);
1051    output[29] = dct_32_round(step[29] * COSPI_8_64 + step[18] * COSPI_24_64);
1052    output[30] = step[30] as i32;
1053    output[31] = step[31] as i32;
1054
1055    // Stage 5 — reload step from output
1056    for i in 0..32 {
1057        step[i] = output[i] as i64;
1058    }
1059    step[0] = dct_32_round((output[0] as i64 + output[1] as i64) * COSPI_16_64) as i64;
1060    step[1] = dct_32_round((-output[1] as i64 + output[0] as i64) * COSPI_16_64) as i64;
1061    step[2] = dct_32_round(output[2] as i64 * COSPI_24_64 + output[3] as i64 * COSPI_8_64) as i64;
1062    step[3] = dct_32_round(output[3] as i64 * COSPI_24_64 - output[2] as i64 * COSPI_8_64) as i64;
1063    step[4] = output[4] as i64 + output[5] as i64;
1064    step[5] = -output[5] as i64 + output[4] as i64;
1065    step[6] = -output[6] as i64 + output[7] as i64;
1066    step[7] = output[7] as i64 + output[6] as i64;
1067    step[8] = output[8] as i64;
1068    step[9] = dct_32_round(output[9] as i64 * -COSPI_8_64 + output[14] as i64 * COSPI_24_64) as i64;
1069    step[10] =
1070        dct_32_round(output[10] as i64 * -COSPI_24_64 + output[13] as i64 * -COSPI_8_64) as i64;
1071    step[11] = output[11] as i64;
1072    step[12] = output[12] as i64;
1073    step[13] =
1074        dct_32_round(output[13] as i64 * COSPI_24_64 + output[10] as i64 * -COSPI_8_64) as i64;
1075    step[14] = dct_32_round(output[14] as i64 * COSPI_8_64 + output[9] as i64 * COSPI_24_64) as i64;
1076    step[15] = output[15] as i64;
1077
1078    step[16] = output[16] as i64 + output[19] as i64;
1079    step[17] = output[17] as i64 + output[18] as i64;
1080    step[18] = -output[18] as i64 + output[17] as i64;
1081    step[19] = -output[19] as i64 + output[16] as i64;
1082    step[20] = -output[20] as i64 + output[23] as i64;
1083    step[21] = -output[21] as i64 + output[22] as i64;
1084    step[22] = output[22] as i64 + output[21] as i64;
1085    step[23] = output[23] as i64 + output[20] as i64;
1086    step[24] = output[24] as i64 + output[27] as i64;
1087    step[25] = output[25] as i64 + output[26] as i64;
1088    step[26] = -output[26] as i64 + output[25] as i64;
1089    step[27] = -output[27] as i64 + output[24] as i64;
1090    step[28] = -output[28] as i64 + output[31] as i64;
1091    step[29] = -output[29] as i64 + output[30] as i64;
1092    step[30] = output[30] as i64 + output[29] as i64;
1093    step[31] = output[31] as i64 + output[28] as i64;
1094
1095    // Stage 6
1096    output[0] = step[0] as i32;
1097    output[1] = step[1] as i32;
1098    output[2] = step[2] as i32;
1099    output[3] = step[3] as i32;
1100    output[4] = dct_32_round(step[4] * COSPI_28_64 + step[7] * COSPI_4_64);
1101    output[5] = dct_32_round(step[5] * COSPI_12_64 + step[6] * COSPI_20_64);
1102    output[6] = dct_32_round(step[6] * COSPI_12_64 + step[5] * -COSPI_20_64);
1103    output[7] = dct_32_round(step[7] * COSPI_28_64 + step[4] * -COSPI_4_64);
1104    output[8] = (step[8] + step[9]) as i32;
1105    output[9] = (-step[9] + step[8]) as i32;
1106    output[10] = (-step[10] + step[11]) as i32;
1107    output[11] = (step[11] + step[10]) as i32;
1108    output[12] = (step[12] + step[13]) as i32;
1109    output[13] = (-step[13] + step[12]) as i32;
1110    output[14] = (-step[14] + step[15]) as i32;
1111    output[15] = (step[15] + step[14]) as i32;
1112
1113    output[16] = step[16] as i32;
1114    output[17] = dct_32_round(step[17] * -COSPI_4_64 + step[30] * COSPI_28_64);
1115    output[18] = dct_32_round(step[18] * -COSPI_28_64 + step[29] * -COSPI_4_64);
1116    output[19] = step[19] as i32;
1117    output[20] = step[20] as i32;
1118    output[21] = dct_32_round(step[21] * -COSPI_20_64 + step[26] * COSPI_12_64);
1119    output[22] = dct_32_round(step[22] * -COSPI_12_64 + step[25] * -COSPI_20_64);
1120    output[23] = step[23] as i32;
1121    output[24] = step[24] as i32;
1122    output[25] = dct_32_round(step[25] * COSPI_12_64 + step[22] * -COSPI_20_64);
1123    output[26] = dct_32_round(step[26] * COSPI_20_64 + step[21] * COSPI_12_64);
1124    output[27] = step[27] as i32;
1125    output[28] = step[28] as i32;
1126    output[29] = dct_32_round(step[29] * COSPI_28_64 + step[18] * -COSPI_4_64);
1127    output[30] = dct_32_round(step[30] * COSPI_4_64 + step[17] * COSPI_28_64);
1128    output[31] = step[31] as i32;
1129
1130    // Stage 7 — reload step from output
1131    for i in 0..32 {
1132        step[i] = output[i] as i64;
1133    }
1134    step[0] = output[0] as i64;
1135    step[1] = output[1] as i64;
1136    step[2] = output[2] as i64;
1137    step[3] = output[3] as i64;
1138    step[4] = output[4] as i64;
1139    step[5] = output[5] as i64;
1140    step[6] = output[6] as i64;
1141    step[7] = output[7] as i64;
1142    step[8] = dct_32_round(output[8] as i64 * COSPI_30_64 + output[15] as i64 * COSPI_2_64) as i64;
1143    step[9] = dct_32_round(output[9] as i64 * COSPI_14_64 + output[14] as i64 * COSPI_18_64) as i64;
1144    step[10] =
1145        dct_32_round(output[10] as i64 * COSPI_22_64 + output[13] as i64 * COSPI_10_64) as i64;
1146    step[11] =
1147        dct_32_round(output[11] as i64 * COSPI_6_64 + output[12] as i64 * COSPI_26_64) as i64;
1148    step[12] =
1149        dct_32_round(output[12] as i64 * COSPI_6_64 + output[11] as i64 * -COSPI_26_64) as i64;
1150    step[13] =
1151        dct_32_round(output[13] as i64 * COSPI_22_64 + output[10] as i64 * -COSPI_10_64) as i64;
1152    step[14] =
1153        dct_32_round(output[14] as i64 * COSPI_14_64 + output[9] as i64 * -COSPI_18_64) as i64;
1154    step[15] =
1155        dct_32_round(output[15] as i64 * COSPI_30_64 + output[8] as i64 * -COSPI_2_64) as i64;
1156
1157    step[16] = output[16] as i64 + output[17] as i64;
1158    step[17] = -output[17] as i64 + output[16] as i64;
1159    step[18] = -output[18] as i64 + output[19] as i64;
1160    step[19] = output[19] as i64 + output[18] as i64;
1161    step[20] = output[20] as i64 + output[21] as i64;
1162    step[21] = -output[21] as i64 + output[20] as i64;
1163    step[22] = -output[22] as i64 + output[23] as i64;
1164    step[23] = output[23] as i64 + output[22] as i64;
1165    step[24] = output[24] as i64 + output[25] as i64;
1166    step[25] = -output[25] as i64 + output[24] as i64;
1167    step[26] = -output[26] as i64 + output[27] as i64;
1168    step[27] = output[27] as i64 + output[26] as i64;
1169    step[28] = output[28] as i64 + output[29] as i64;
1170    step[29] = -output[29] as i64 + output[28] as i64;
1171    step[30] = -output[30] as i64 + output[31] as i64;
1172    step[31] = output[31] as i64 + output[30] as i64;
1173
1174    // Final stage — bit-reversed output indices
1175    output[0] = step[0] as i32;
1176    output[16] = step[1] as i32;
1177    output[8] = step[2] as i32;
1178    output[24] = step[3] as i32;
1179    output[4] = step[4] as i32;
1180    output[20] = step[5] as i32;
1181    output[12] = step[6] as i32;
1182    output[28] = step[7] as i32;
1183    output[2] = step[8] as i32;
1184    output[18] = step[9] as i32;
1185    output[10] = step[10] as i32;
1186    output[26] = step[11] as i32;
1187    output[6] = step[12] as i32;
1188    output[22] = step[13] as i32;
1189    output[14] = step[14] as i32;
1190    output[30] = step[15] as i32;
1191
1192    output[1] = dct_32_round(step[16] * COSPI_31_64 + step[31] * COSPI_1_64);
1193    output[17] = dct_32_round(step[17] * COSPI_15_64 + step[30] * COSPI_17_64);
1194    output[9] = dct_32_round(step[18] * COSPI_23_64 + step[29] * COSPI_9_64);
1195    output[25] = dct_32_round(step[19] * COSPI_7_64 + step[28] * COSPI_25_64);
1196    output[5] = dct_32_round(step[20] * COSPI_27_64 + step[27] * COSPI_5_64);
1197    output[21] = dct_32_round(step[21] * COSPI_11_64 + step[26] * COSPI_21_64);
1198    output[13] = dct_32_round(step[22] * COSPI_19_64 + step[25] * COSPI_13_64);
1199    output[29] = dct_32_round(step[23] * COSPI_3_64 + step[24] * COSPI_29_64);
1200    output[3] = dct_32_round(step[24] * COSPI_3_64 + step[23] * -COSPI_29_64);
1201    output[19] = dct_32_round(step[25] * COSPI_19_64 + step[22] * -COSPI_13_64);
1202    output[11] = dct_32_round(step[26] * COSPI_11_64 + step[21] * -COSPI_21_64);
1203    output[27] = dct_32_round(step[27] * COSPI_27_64 + step[20] * -COSPI_5_64);
1204    output[7] = dct_32_round(step[28] * COSPI_7_64 + step[19] * -COSPI_25_64);
1205    output[23] = dct_32_round(step[29] * COSPI_23_64 + step[18] * -COSPI_9_64);
1206    output[15] = dct_32_round(step[30] * COSPI_15_64 + step[17] * -COSPI_17_64);
1207    output[31] = dct_32_round(step[31] * COSPI_31_64 + step[16] * -COSPI_1_64);
1208}
1209/// 1-D IDCT32 (libvpx `idct32_c`).
1210fn idct32(input: &[i32; 32], output: &mut [i32; 32]) {
1211    let mut step1 = [0i32; 32];
1212    let mut step2 = [0i32; 32];
1213    // stage 1
1214    step1[0] = input[0] as i16 as i32;
1215    step1[1] = input[16] as i16 as i32;
1216    step1[2] = input[8] as i16 as i32;
1217    step1[3] = input[24] as i16 as i32;
1218    step1[4] = input[4] as i16 as i32;
1219    step1[5] = input[20] as i16 as i32;
1220    step1[6] = input[12] as i16 as i32;
1221    step1[7] = input[28] as i16 as i32;
1222    step1[8] = input[2] as i16 as i32;
1223    step1[9] = input[18] as i16 as i32;
1224    step1[10] = input[10] as i16 as i32;
1225    step1[11] = input[26] as i16 as i32;
1226    step1[12] = input[6] as i16 as i32;
1227    step1[13] = input[22] as i16 as i32;
1228    step1[14] = input[14] as i16 as i32;
1229    step1[15] = input[30] as i16 as i32;
1230    {
1231        let temp1 = input[1] as i16 as i64 * COSPI_31_64 - input[31] as i16 as i64 * COSPI_1_64;
1232        let temp2 = input[1] as i16 as i64 * COSPI_1_64 + input[31] as i16 as i64 * COSPI_31_64;
1233        step1[16] = wraplow(dct_const_round_shift(temp1));
1234        step1[31] = wraplow(dct_const_round_shift(temp2));
1235    }
1236    {
1237        let temp1 = input[17] as i16 as i64 * COSPI_15_64 - input[15] as i16 as i64 * COSPI_17_64;
1238        let temp2 = input[17] as i16 as i64 * COSPI_17_64 + input[15] as i16 as i64 * COSPI_15_64;
1239        step1[17] = wraplow(dct_const_round_shift(temp1));
1240        step1[30] = wraplow(dct_const_round_shift(temp2));
1241    }
1242    {
1243        let temp1 = input[9] as i16 as i64 * COSPI_23_64 - input[23] as i16 as i64 * COSPI_9_64;
1244        let temp2 = input[9] as i16 as i64 * COSPI_9_64 + input[23] as i16 as i64 * COSPI_23_64;
1245        step1[18] = wraplow(dct_const_round_shift(temp1));
1246        step1[29] = wraplow(dct_const_round_shift(temp2));
1247    }
1248    {
1249        let temp1 = input[25] as i16 as i64 * COSPI_7_64 - input[7] as i16 as i64 * COSPI_25_64;
1250        let temp2 = input[25] as i16 as i64 * COSPI_25_64 + input[7] as i16 as i64 * COSPI_7_64;
1251        step1[19] = wraplow(dct_const_round_shift(temp1));
1252        step1[28] = wraplow(dct_const_round_shift(temp2));
1253    }
1254    {
1255        let temp1 = input[5] as i16 as i64 * COSPI_27_64 - input[27] as i16 as i64 * COSPI_5_64;
1256        let temp2 = input[5] as i16 as i64 * COSPI_5_64 + input[27] as i16 as i64 * COSPI_27_64;
1257        step1[20] = wraplow(dct_const_round_shift(temp1));
1258        step1[27] = wraplow(dct_const_round_shift(temp2));
1259    }
1260    {
1261        let temp1 = input[21] as i16 as i64 * COSPI_11_64 - input[11] as i16 as i64 * COSPI_21_64;
1262        let temp2 = input[21] as i16 as i64 * COSPI_21_64 + input[11] as i16 as i64 * COSPI_11_64;
1263        step1[21] = wraplow(dct_const_round_shift(temp1));
1264        step1[26] = wraplow(dct_const_round_shift(temp2));
1265    }
1266    {
1267        let temp1 = input[13] as i16 as i64 * COSPI_19_64 - input[19] as i16 as i64 * COSPI_13_64;
1268        let temp2 = input[13] as i16 as i64 * COSPI_13_64 + input[19] as i16 as i64 * COSPI_19_64;
1269        step1[22] = wraplow(dct_const_round_shift(temp1));
1270        step1[25] = wraplow(dct_const_round_shift(temp2));
1271    }
1272    {
1273        let temp1 = input[29] as i16 as i64 * COSPI_3_64 - input[3] as i16 as i64 * COSPI_29_64;
1274        let temp2 = input[29] as i16 as i64 * COSPI_29_64 + input[3] as i16 as i64 * COSPI_3_64;
1275        step1[23] = wraplow(dct_const_round_shift(temp1));
1276        step1[24] = wraplow(dct_const_round_shift(temp2));
1277    }
1278    // stage 2
1279    step2[0] = step1[0];
1280    step2[1] = step1[1];
1281    step2[2] = step1[2];
1282    step2[3] = step1[3];
1283    step2[4] = step1[4];
1284    step2[5] = step1[5];
1285    step2[6] = step1[6];
1286    step2[7] = step1[7];
1287    {
1288        let temp1 = step1[8] as i64 * COSPI_30_64 - step1[15] as i64 * COSPI_2_64;
1289        let temp2 = step1[8] as i64 * COSPI_2_64 + step1[15] as i64 * COSPI_30_64;
1290        step2[8] = wraplow(dct_const_round_shift(temp1));
1291        step2[15] = wraplow(dct_const_round_shift(temp2));
1292    }
1293    {
1294        let temp1 = step1[9] as i64 * COSPI_14_64 - step1[14] as i64 * COSPI_18_64;
1295        let temp2 = step1[9] as i64 * COSPI_18_64 + step1[14] as i64 * COSPI_14_64;
1296        step2[9] = wraplow(dct_const_round_shift(temp1));
1297        step2[14] = wraplow(dct_const_round_shift(temp2));
1298    }
1299    {
1300        let temp1 = step1[10] as i64 * COSPI_22_64 - step1[13] as i64 * COSPI_10_64;
1301        let temp2 = step1[10] as i64 * COSPI_10_64 + step1[13] as i64 * COSPI_22_64;
1302        step2[10] = wraplow(dct_const_round_shift(temp1));
1303        step2[13] = wraplow(dct_const_round_shift(temp2));
1304    }
1305    {
1306        let temp1 = step1[11] as i64 * COSPI_6_64 - step1[12] as i64 * COSPI_26_64;
1307        let temp2 = step1[11] as i64 * COSPI_26_64 + step1[12] as i64 * COSPI_6_64;
1308        step2[11] = wraplow(dct_const_round_shift(temp1));
1309        step2[12] = wraplow(dct_const_round_shift(temp2));
1310    }
1311    step2[16] = wraplow(step1[16] + step1[17]);
1312    step2[17] = wraplow(step1[16] - step1[17]);
1313    step2[18] = wraplow(-step1[18] + step1[19]);
1314    step2[19] = wraplow(step1[18] + step1[19]);
1315    step2[20] = wraplow(step1[20] + step1[21]);
1316    step2[21] = wraplow(step1[20] - step1[21]);
1317    step2[22] = wraplow(-step1[22] + step1[23]);
1318    step2[23] = wraplow(step1[22] + step1[23]);
1319    step2[24] = wraplow(step1[24] + step1[25]);
1320    step2[25] = wraplow(step1[24] - step1[25]);
1321    step2[26] = wraplow(-step1[26] + step1[27]);
1322    step2[27] = wraplow(step1[26] + step1[27]);
1323    step2[28] = wraplow(step1[28] + step1[29]);
1324    step2[29] = wraplow(step1[28] - step1[29]);
1325    step2[30] = wraplow(-step1[30] + step1[31]);
1326    step2[31] = wraplow(step1[30] + step1[31]);
1327    // stage 3
1328    step1[0] = step2[0];
1329    step1[1] = step2[1];
1330    step1[2] = step2[2];
1331    step1[3] = step2[3];
1332    {
1333        let temp1 = step2[4] as i64 * COSPI_28_64 - step2[7] as i64 * COSPI_4_64;
1334        let temp2 = step2[4] as i64 * COSPI_4_64 + step2[7] as i64 * COSPI_28_64;
1335        step1[4] = wraplow(dct_const_round_shift(temp1));
1336        step1[7] = wraplow(dct_const_round_shift(temp2));
1337    }
1338    {
1339        let temp1 = step2[5] as i64 * COSPI_12_64 - step2[6] as i64 * COSPI_20_64;
1340        let temp2 = step2[5] as i64 * COSPI_20_64 + step2[6] as i64 * COSPI_12_64;
1341        step1[5] = wraplow(dct_const_round_shift(temp1));
1342        step1[6] = wraplow(dct_const_round_shift(temp2));
1343    }
1344    step1[8] = wraplow(step2[8] + step2[9]);
1345    step1[9] = wraplow(step2[8] - step2[9]);
1346    step1[10] = wraplow(-step2[10] + step2[11]);
1347    step1[11] = wraplow(step2[10] + step2[11]);
1348    step1[12] = wraplow(step2[12] + step2[13]);
1349    step1[13] = wraplow(step2[12] - step2[13]);
1350    step1[14] = wraplow(-step2[14] + step2[15]);
1351    step1[15] = wraplow(step2[14] + step2[15]);
1352    step1[16] = step2[16];
1353    step1[31] = step2[31];
1354    {
1355        let temp1 = -step2[17] as i64 * COSPI_4_64 + step2[30] as i64 * COSPI_28_64;
1356        let temp2 = step2[17] as i64 * COSPI_28_64 + step2[30] as i64 * COSPI_4_64;
1357        step1[17] = wraplow(dct_const_round_shift(temp1));
1358        step1[30] = wraplow(dct_const_round_shift(temp2));
1359    }
1360    {
1361        let temp1 = -step2[18] as i64 * COSPI_28_64 - step2[29] as i64 * COSPI_4_64;
1362        let temp2 = -step2[18] as i64 * COSPI_4_64 + step2[29] as i64 * COSPI_28_64;
1363        step1[18] = wraplow(dct_const_round_shift(temp1));
1364        step1[29] = wraplow(dct_const_round_shift(temp2));
1365    }
1366    step1[19] = step2[19];
1367    step1[20] = step2[20];
1368    {
1369        let temp1 = -step2[21] as i64 * COSPI_20_64 + step2[26] as i64 * COSPI_12_64;
1370        let temp2 = step2[21] as i64 * COSPI_12_64 + step2[26] as i64 * COSPI_20_64;
1371        step1[21] = wraplow(dct_const_round_shift(temp1));
1372        step1[26] = wraplow(dct_const_round_shift(temp2));
1373    }
1374    {
1375        let temp1 = -step2[22] as i64 * COSPI_12_64 - step2[25] as i64 * COSPI_20_64;
1376        let temp2 = -step2[22] as i64 * COSPI_20_64 + step2[25] as i64 * COSPI_12_64;
1377        step1[22] = wraplow(dct_const_round_shift(temp1));
1378        step1[25] = wraplow(dct_const_round_shift(temp2));
1379    }
1380    step1[23] = step2[23];
1381    step1[24] = step2[24];
1382    step1[27] = step2[27];
1383    step1[28] = step2[28];
1384    // stage 4
1385    {
1386        let temp1 = (step1[0] as i64 + step1[1] as i64) * COSPI_16_64;
1387        let temp2 = (step1[0] as i64 - step1[1] as i64) * COSPI_16_64;
1388        step2[0] = wraplow(dct_const_round_shift(temp1));
1389        step2[1] = wraplow(dct_const_round_shift(temp2));
1390    }
1391    {
1392        let temp1 = step1[2] as i64 * COSPI_24_64 - step1[3] as i64 * COSPI_8_64;
1393        let temp2 = step1[2] as i64 * COSPI_8_64 + step1[3] as i64 * COSPI_24_64;
1394        step2[2] = wraplow(dct_const_round_shift(temp1));
1395        step2[3] = wraplow(dct_const_round_shift(temp2));
1396    }
1397    step2[4] = wraplow(step1[4] + step1[5]);
1398    step2[5] = wraplow(step1[4] - step1[5]);
1399    step2[6] = wraplow(-step1[6] + step1[7]);
1400    step2[7] = wraplow(step1[6] + step1[7]);
1401    step2[8] = step1[8];
1402    step2[15] = step1[15];
1403    {
1404        let temp1 = -step1[9] as i64 * COSPI_8_64 + step1[14] as i64 * COSPI_24_64;
1405        let temp2 = step1[9] as i64 * COSPI_24_64 + step1[14] as i64 * COSPI_8_64;
1406        step2[9] = wraplow(dct_const_round_shift(temp1));
1407        step2[14] = wraplow(dct_const_round_shift(temp2));
1408    }
1409    {
1410        let temp1 = -step1[10] as i64 * COSPI_24_64 - step1[13] as i64 * COSPI_8_64;
1411        let temp2 = -step1[10] as i64 * COSPI_8_64 + step1[13] as i64 * COSPI_24_64;
1412        step2[10] = wraplow(dct_const_round_shift(temp1));
1413        step2[13] = wraplow(dct_const_round_shift(temp2));
1414    }
1415    step2[11] = step1[11];
1416    step2[12] = step1[12];
1417    step2[16] = wraplow(step1[16] + step1[19]);
1418    step2[17] = wraplow(step1[17] + step1[18]);
1419    step2[18] = wraplow(step1[17] - step1[18]);
1420    step2[19] = wraplow(step1[16] - step1[19]);
1421    step2[20] = wraplow(-step1[20] + step1[23]);
1422    step2[21] = wraplow(-step1[21] + step1[22]);
1423    step2[22] = wraplow(step1[21] + step1[22]);
1424    step2[23] = wraplow(step1[20] + step1[23]);
1425    step2[24] = wraplow(step1[24] + step1[27]);
1426    step2[25] = wraplow(step1[25] + step1[26]);
1427    step2[26] = wraplow(step1[25] - step1[26]);
1428    step2[27] = wraplow(step1[24] - step1[27]);
1429    step2[28] = wraplow(-step1[28] + step1[31]);
1430    step2[29] = wraplow(-step1[29] + step1[30]);
1431    step2[30] = wraplow(step1[29] + step1[30]);
1432    step2[31] = wraplow(step1[28] + step1[31]);
1433    // stage 5
1434    step1[0] = wraplow(step2[0] + step2[3]);
1435    step1[1] = wraplow(step2[1] + step2[2]);
1436    step1[2] = wraplow(step2[1] - step2[2]);
1437    step1[3] = wraplow(step2[0] - step2[3]);
1438    step1[4] = step2[4];
1439    {
1440        let temp1 = (step2[6] as i64 - step2[5] as i64) * COSPI_16_64;
1441        let temp2 = (step2[5] as i64 + step2[6] as i64) * COSPI_16_64;
1442        step1[5] = wraplow(dct_const_round_shift(temp1));
1443        step1[6] = wraplow(dct_const_round_shift(temp2));
1444    }
1445    step1[7] = step2[7];
1446    step1[8] = wraplow(step2[8] + step2[11]);
1447    step1[9] = wraplow(step2[9] + step2[10]);
1448    step1[10] = wraplow(step2[9] - step2[10]);
1449    step1[11] = wraplow(step2[8] - step2[11]);
1450    step1[12] = wraplow(-step2[12] + step2[15]);
1451    step1[13] = wraplow(-step2[13] + step2[14]);
1452    step1[14] = wraplow(step2[13] + step2[14]);
1453    step1[15] = wraplow(step2[12] + step2[15]);
1454    step1[16] = step2[16];
1455    step1[17] = step2[17];
1456    {
1457        let temp1 = -step2[18] as i64 * COSPI_8_64 + step2[29] as i64 * COSPI_24_64;
1458        let temp2 = step2[18] as i64 * COSPI_24_64 + step2[29] as i64 * COSPI_8_64;
1459        step1[18] = wraplow(dct_const_round_shift(temp1));
1460        step1[29] = wraplow(dct_const_round_shift(temp2));
1461    }
1462    {
1463        let temp1 = -step2[19] as i64 * COSPI_8_64 + step2[28] as i64 * COSPI_24_64;
1464        let temp2 = step2[19] as i64 * COSPI_24_64 + step2[28] as i64 * COSPI_8_64;
1465        step1[19] = wraplow(dct_const_round_shift(temp1));
1466        step1[28] = wraplow(dct_const_round_shift(temp2));
1467    }
1468    {
1469        let temp1 = -step2[20] as i64 * COSPI_24_64 - step2[27] as i64 * COSPI_8_64;
1470        let temp2 = -step2[20] as i64 * COSPI_8_64 + step2[27] as i64 * COSPI_24_64;
1471        step1[20] = wraplow(dct_const_round_shift(temp1));
1472        step1[27] = wraplow(dct_const_round_shift(temp2));
1473    }
1474    {
1475        let temp1 = -step2[21] as i64 * COSPI_24_64 - step2[26] as i64 * COSPI_8_64;
1476        let temp2 = -step2[21] as i64 * COSPI_8_64 + step2[26] as i64 * COSPI_24_64;
1477        step1[21] = wraplow(dct_const_round_shift(temp1));
1478        step1[26] = wraplow(dct_const_round_shift(temp2));
1479    }
1480    step1[22] = step2[22];
1481    step1[23] = step2[23];
1482    step1[24] = step2[24];
1483    step1[25] = step2[25];
1484    step1[30] = step2[30];
1485    step1[31] = step2[31];
1486    // stage 6
1487    step2[0] = wraplow(step1[0] + step1[7]);
1488    step2[1] = wraplow(step1[1] + step1[6]);
1489    step2[2] = wraplow(step1[2] + step1[5]);
1490    step2[3] = wraplow(step1[3] + step1[4]);
1491    step2[4] = wraplow(step1[3] - step1[4]);
1492    step2[5] = wraplow(step1[2] - step1[5]);
1493    step2[6] = wraplow(step1[1] - step1[6]);
1494    step2[7] = wraplow(step1[0] - step1[7]);
1495    step2[8] = step1[8];
1496    step2[9] = step1[9];
1497    {
1498        let temp1 = (-step1[10] as i64 + step1[13] as i64) * COSPI_16_64;
1499        let temp2 = (step1[10] as i64 + step1[13] as i64) * COSPI_16_64;
1500        step2[10] = wraplow(dct_const_round_shift(temp1));
1501        step2[13] = wraplow(dct_const_round_shift(temp2));
1502    }
1503    {
1504        let temp1 = (-step1[11] as i64 + step1[12] as i64) * COSPI_16_64;
1505        let temp2 = (step1[11] as i64 + step1[12] as i64) * COSPI_16_64;
1506        step2[11] = wraplow(dct_const_round_shift(temp1));
1507        step2[12] = wraplow(dct_const_round_shift(temp2));
1508    }
1509    step2[14] = step1[14];
1510    step2[15] = step1[15];
1511    step2[16] = wraplow(step1[16] + step1[23]);
1512    step2[17] = wraplow(step1[17] + step1[22]);
1513    step2[18] = wraplow(step1[18] + step1[21]);
1514    step2[19] = wraplow(step1[19] + step1[20]);
1515    step2[20] = wraplow(step1[19] - step1[20]);
1516    step2[21] = wraplow(step1[18] - step1[21]);
1517    step2[22] = wraplow(step1[17] - step1[22]);
1518    step2[23] = wraplow(step1[16] - step1[23]);
1519    step2[24] = wraplow(-step1[24] + step1[31]);
1520    step2[25] = wraplow(-step1[25] + step1[30]);
1521    step2[26] = wraplow(-step1[26] + step1[29]);
1522    step2[27] = wraplow(-step1[27] + step1[28]);
1523    step2[28] = wraplow(step1[27] + step1[28]);
1524    step2[29] = wraplow(step1[26] + step1[29]);
1525    step2[30] = wraplow(step1[25] + step1[30]);
1526    step2[31] = wraplow(step1[24] + step1[31]);
1527    // stage 7
1528    step1[0] = wraplow(step2[0] + step2[15]);
1529    step1[1] = wraplow(step2[1] + step2[14]);
1530    step1[2] = wraplow(step2[2] + step2[13]);
1531    step1[3] = wraplow(step2[3] + step2[12]);
1532    step1[4] = wraplow(step2[4] + step2[11]);
1533    step1[5] = wraplow(step2[5] + step2[10]);
1534    step1[6] = wraplow(step2[6] + step2[9]);
1535    step1[7] = wraplow(step2[7] + step2[8]);
1536    step1[8] = wraplow(step2[7] - step2[8]);
1537    step1[9] = wraplow(step2[6] - step2[9]);
1538    step1[10] = wraplow(step2[5] - step2[10]);
1539    step1[11] = wraplow(step2[4] - step2[11]);
1540    step1[12] = wraplow(step2[3] - step2[12]);
1541    step1[13] = wraplow(step2[2] - step2[13]);
1542    step1[14] = wraplow(step2[1] - step2[14]);
1543    step1[15] = wraplow(step2[0] - step2[15]);
1544    step1[16] = step2[16];
1545    step1[17] = step2[17];
1546    step1[18] = step2[18];
1547    step1[19] = step2[19];
1548    {
1549        let temp1 = (-step2[20] as i64 + step2[27] as i64) * COSPI_16_64;
1550        let temp2 = (step2[20] as i64 + step2[27] as i64) * COSPI_16_64;
1551        step1[20] = wraplow(dct_const_round_shift(temp1));
1552        step1[27] = wraplow(dct_const_round_shift(temp2));
1553    }
1554    {
1555        let temp1 = (-step2[21] as i64 + step2[26] as i64) * COSPI_16_64;
1556        let temp2 = (step2[21] as i64 + step2[26] as i64) * COSPI_16_64;
1557        step1[21] = wraplow(dct_const_round_shift(temp1));
1558        step1[26] = wraplow(dct_const_round_shift(temp2));
1559    }
1560    {
1561        let temp1 = (-step2[22] as i64 + step2[25] as i64) * COSPI_16_64;
1562        let temp2 = (step2[22] as i64 + step2[25] as i64) * COSPI_16_64;
1563        step1[22] = wraplow(dct_const_round_shift(temp1));
1564        step1[25] = wraplow(dct_const_round_shift(temp2));
1565    }
1566    {
1567        let temp1 = (-step2[23] as i64 + step2[24] as i64) * COSPI_16_64;
1568        let temp2 = (step2[23] as i64 + step2[24] as i64) * COSPI_16_64;
1569        step1[23] = wraplow(dct_const_round_shift(temp1));
1570        step1[24] = wraplow(dct_const_round_shift(temp2));
1571    }
1572    step1[28] = step2[28];
1573    step1[29] = step2[29];
1574    step1[30] = step2[30];
1575    step1[31] = step2[31];
1576    // final stage
1577    output[0] = wraplow(step1[0] + step1[31]);
1578    output[1] = wraplow(step1[1] + step1[30]);
1579    output[2] = wraplow(step1[2] + step1[29]);
1580    output[3] = wraplow(step1[3] + step1[28]);
1581    output[4] = wraplow(step1[4] + step1[27]);
1582    output[5] = wraplow(step1[5] + step1[26]);
1583    output[6] = wraplow(step1[6] + step1[25]);
1584    output[7] = wraplow(step1[7] + step1[24]);
1585    output[8] = wraplow(step1[8] + step1[23]);
1586    output[9] = wraplow(step1[9] + step1[22]);
1587    output[10] = wraplow(step1[10] + step1[21]);
1588    output[11] = wraplow(step1[11] + step1[20]);
1589    output[12] = wraplow(step1[12] + step1[19]);
1590    output[13] = wraplow(step1[13] + step1[18]);
1591    output[14] = wraplow(step1[14] + step1[17]);
1592    output[15] = wraplow(step1[15] + step1[16]);
1593    output[16] = wraplow(step1[15] - step1[16]);
1594    output[17] = wraplow(step1[14] - step1[17]);
1595    output[18] = wraplow(step1[13] - step1[18]);
1596    output[19] = wraplow(step1[12] - step1[19]);
1597    output[20] = wraplow(step1[11] - step1[20]);
1598    output[21] = wraplow(step1[10] - step1[21]);
1599    output[22] = wraplow(step1[9] - step1[22]);
1600    output[23] = wraplow(step1[8] - step1[23]);
1601    output[24] = wraplow(step1[7] - step1[24]);
1602    output[25] = wraplow(step1[6] - step1[25]);
1603    output[26] = wraplow(step1[5] - step1[26]);
1604    output[27] = wraplow(step1[4] - step1[27]);
1605    output[28] = wraplow(step1[3] - step1[28]);
1606    output[29] = wraplow(step1[2] - step1[29]);
1607    output[30] = wraplow(step1[1] - step1[30]);
1608    output[31] = wraplow(step1[0] - step1[31]);
1609}
1610/// Forward 32×32 DCT (`vpx_fdct32x32_c`). `input` is row-major residual (stride 32).
1611pub fn fdct32x32(input: &[i16; 1024], output: &mut [i32; 1024]) {
1612    let mut intermediate = [0i32; 1024];
1613    // Columns
1614    for i in 0..32 {
1615        let mut temp_in = [0i64; 32];
1616        for j in 0..32 {
1617            temp_in[j] = input[j * 32 + i] as i64 * 4;
1618        }
1619        let mut temp_out = [0i32; 32];
1620        fdct32(&temp_in, &mut temp_out, false);
1621        for j in 0..32 {
1622            let v = temp_out[j];
1623            intermediate[j * 32 + i] = (v + 1 + i32::from(v > 0)) >> 2;
1624        }
1625    }
1626    // Rows
1627    for i in 0..32 {
1628        let mut temp_in = [0i64; 32];
1629        for j in 0..32 {
1630            temp_in[j] = intermediate[j + i * 32] as i64;
1631        }
1632        let mut temp_out = [0i32; 32];
1633        fdct32(&temp_in, &mut temp_out, false);
1634        for j in 0..32 {
1635            let v = temp_out[j];
1636            output[j + i * 32] = (v + 1 + i32::from(v < 0)) >> 2;
1637        }
1638    }
1639}
1640
1641/// Inverse 32×32 DCT and add into `dest` (`vpx_idct32x32_1024_add_c`).
1642pub fn idct32x32_add(input: &[i32; 1024], dest: &mut [u8], stride: usize) {
1643    assert!(dest.len() >= 31 * stride + 32);
1644    let mut out = [0i32; 1024];
1645    for i in 0..32 {
1646        let row = &input[i * 32..i * 32 + 32];
1647        if row.iter().any(|&x| x != 0) {
1648            let row_in: [i32; 32] = row.try_into().unwrap();
1649            let mut row_out = [0i32; 32];
1650            idct32(&row_in, &mut row_out);
1651            out[i * 32..i * 32 + 32].copy_from_slice(&row_out);
1652        } else {
1653            out[i * 32..i * 32 + 32].fill(0);
1654        }
1655    }
1656    for i in 0..32 {
1657        let mut col_in = [0i32; 32];
1658        for j in 0..32 {
1659            col_in[j] = out[j * 32 + i];
1660        }
1661        let mut col_out = [0i32; 32];
1662        idct32(&col_in, &mut col_out);
1663        for j in 0..32 {
1664            let add = round_power_of_two(col_out[j] as i64, 6) as i32;
1665            let idx = j * stride + i;
1666            dest[idx] = clip_pixel_add(dest[idx], add);
1667        }
1668    }
1669}
1670
1671#[cfg(test)]
1672mod tests {
1673    use super::*;
1674
1675    struct Rng(u32);
1676    impl Rng {
1677        fn next(&mut self) -> u32 {
1678            let mut x = self.0;
1679            x ^= x << 13;
1680            x ^= x >> 17;
1681            x ^= x << 5;
1682            self.0 = x;
1683            x
1684        }
1685    }
1686
1687    #[test]
1688    fn fdct_idct_roundtrip_small_residuals() {
1689        let mut rng = Rng(0xC0FF_EE42);
1690        for _ in 0..200 {
1691            let mut residual = [0i16; 16];
1692            for r in residual.iter_mut() {
1693                *r = (rng.next() % 31) as i16 - 15;
1694            }
1695            let mut coeffs = [0i32; 16];
1696            fdct4x4(&residual, &mut coeffs);
1697
1698            let mut recon = [0u8; 16];
1699            recon.fill(128);
1700            idct4x4_add(&coeffs, &mut recon, 4);
1701
1702            for i in 0..16 {
1703                let got = recon[i] as i32 - 128;
1704                let exp = residual[i] as i32;
1705                assert!(
1706                    (got - exp).abs() <= 2,
1707                    "pix {i}: got {got} expected {exp} (residual {:?}, coeffs {:?})",
1708                    residual,
1709                    coeffs
1710                );
1711            }
1712        }
1713    }
1714
1715    #[test]
1716    fn hybrid_roundtrip_all_tx_types() {
1717        let mut rng = Rng(0xAD57_4444);
1718        for &tx in &[
1719            TxType::DctDct,
1720            TxType::AdstDct,
1721            TxType::DctAdst,
1722            TxType::AdstAdst,
1723        ] {
1724            for _ in 0..80 {
1725                let mut residual = [0i16; 16];
1726                for r in residual.iter_mut() {
1727                    *r = (rng.next() % 21) as i16 - 10;
1728                }
1729                let mut coeffs = [0i32; 16];
1730                fht4x4(&residual, &mut coeffs, tx);
1731                let mut recon = [128u8; 16];
1732                iht4x4_add(&coeffs, &mut recon, 4, tx);
1733                for i in 0..16 {
1734                    let got = recon[i] as i32 - 128;
1735                    let exp = residual[i] as i32;
1736                    assert!(
1737                        (got - exp).abs() <= 3,
1738                        "tx={tx:?} pix {i}: got {got} expected {exp}"
1739                    );
1740                }
1741            }
1742        }
1743    }
1744
1745    #[test]
1746    fn mode_to_tx_type_matches_libvpx() {
1747        assert_eq!(tx_type_from_mode(DC_PRED), TxType::DctDct);
1748        assert_eq!(tx_type_from_mode(V_PRED), TxType::AdstDct);
1749        assert_eq!(tx_type_from_mode(H_PRED), TxType::DctAdst);
1750    }
1751
1752    #[test]
1753    fn fdct_zero_is_zero() {
1754        let input = [0i16; 16];
1755        let mut out = [1i32; 16];
1756        fdct4x4(&input, &mut out);
1757        assert_eq!(out, [0i32; 16]);
1758    }
1759
1760    #[test]
1761    fn fdct8_idct8_roundtrip_small_residuals() {
1762        let mut residual = [0i16; 64];
1763        for i in 0..64 {
1764            residual[i] = ((i as i16 * 3) % 17) - 8;
1765        }
1766        let mut coeffs = [0i32; 64];
1767        fdct8x8(&residual, &mut coeffs);
1768        let mut dest = [128u8; 64];
1769        idct8x8_add(&coeffs, &mut dest, 8);
1770        for i in 0..64 {
1771            let exp = (128 + residual[i] as i32).clamp(0, 255) as u8;
1772            assert!(
1773                dest[i].abs_diff(exp) <= 2,
1774                "pix {i}: got {} expected {exp}",
1775                dest[i]
1776            );
1777        }
1778    }
1779
1780    #[test]
1781    fn fdct8_zero_is_zero() {
1782        let input = [0i16; 64];
1783        let mut out = [1i32; 64];
1784        fdct8x8(&input, &mut out);
1785        assert_eq!(out, [0i32; 64]);
1786    }
1787
1788    #[test]
1789    fn fdct16_zero_is_zero() {
1790        let input = [0i16; 256];
1791        let mut out = [1i32; 256];
1792        fdct16x16(&input, &mut out);
1793        assert_eq!(out, [0i32; 256]);
1794    }
1795
1796    #[test]
1797    fn fdct16_idct16_roundtrip_small_residuals() {
1798        let mut rng = Rng(0x16_16_BEEF);
1799        for _ in 0..100 {
1800            let mut residual = [0i16; 256];
1801            for r in residual.iter_mut() {
1802                *r = (rng.next() % 31) as i16 - 15;
1803            }
1804            let mut coeffs = [0i32; 256];
1805            fdct16x16(&residual, &mut coeffs);
1806            let mut dest = [128u8; 256];
1807            idct16x16_add(&coeffs, &mut dest, 16);
1808            for i in 0..256 {
1809                let exp = (128 + residual[i] as i32).clamp(0, 255) as u8;
1810                assert!(
1811                    dest[i].abs_diff(exp) <= 2,
1812                    "pix {i}: got {} expected {exp}",
1813                    dest[i]
1814                );
1815            }
1816        }
1817    }
1818
1819    #[test]
1820    fn idct_add_onto_nonzero_base() {
1821        let residual = [4i16; 16];
1822        let mut coeffs = [0i32; 16];
1823        fdct4x4(&residual, &mut coeffs);
1824        let mut dest = [10u8; 16];
1825        idct4x4_add(&coeffs, &mut dest, 4);
1826        for &p in &dest {
1827            assert!((p as i32 - 14).abs() <= 2, "pixel {p}");
1828        }
1829    }
1830
1831    #[test]
1832    fn fdct32_zero_is_zero() {
1833        let input = [0i16; 1024];
1834        let mut out = [1i32; 1024];
1835        fdct32x32(&input, &mut out);
1836        assert_eq!(out, [0i32; 1024]);
1837    }
1838
1839    #[test]
1840    fn fdct32_idct32_roundtrip_small_residuals() {
1841        let mut rng = Rng(0x32_32_CAFE);
1842        for _ in 0..100 {
1843            let mut residual = [0i16; 1024];
1844            for r in residual.iter_mut() {
1845                *r = (rng.next() % 21) as i16 - 10;
1846            }
1847            let mut coeffs = [0i32; 1024];
1848            fdct32x32(&residual, &mut coeffs);
1849            let mut dest = [128u8; 1024];
1850            idct32x32_add(&coeffs, &mut dest, 32);
1851            for i in 0..1024 {
1852                let exp = (128 + residual[i] as i32).clamp(0, 255) as u8;
1853                assert!(
1854                    dest[i].abs_diff(exp) <= 2,
1855                    "pix {i}: got {} expected {exp}",
1856                    dest[i]
1857                );
1858            }
1859        }
1860    }
1861
1862    #[test]
1863    fn wraplow_sign_extends_16() {
1864        assert_eq!(wraplow(0x0001_0001), 1);
1865        assert_eq!(wraplow(0x0000_FFFF_u32 as i32), -1);
1866        assert_eq!(wraplow(32768), -32768);
1867    }
1868}