pxdct 0.3.2

/*
 * // Copyright (c) Radzivon Bartoshyk 1/2026. All rights reserved.
 * //
 * // Redistribution and use in source and binary forms, with or without modification,
 * // are permitted provided that the following conditions are met:
 * //
 * // 1.  Redistributions of source code must retain the above copyright notice, this
 * // list of conditions and the following disclaimer.
 * //
 * // 2.  Redistributions in binary form must reproduce the above copyright notice,
 * // this list of conditions and the following disclaimer in the documentation
 * // and/or other materials provided with the distribution.
 * //
 * // 3.  Neither the name of the copyright holder nor the names of its
 * // contributors may be used to endorse or promote products derived from
 * // this software without specific prior written permission.
 * //
 * // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
 * // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 * // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
use crate::avx::dct2::mixed_radix3d::{
    dct2_radix_n_cos_twiddles_avx_d, dct2_radix_n_rotation_twiddles_avx_d,
};
use crate::avx::stored::AvxStoreD;
use crate::avx::util::{boring_avx_mixed_radix, fma};
use crate::bidirectional::BidirectionalStore;
use crate::dct2::prime_butterflies::MixedRadix13Sample;
use crate::util::{DctSample, try_vec, validate_scratch};
use crate::{PxdctError, PxdctExecutor};
use num_traits::{AsPrimitive, One};
use std::sync::Arc;

pub(crate) struct AvxDct2MixedRadix13d {
    rotation_layer: Vec<AvxStoreD>,
    cos_twiddles: Vec<AvxStoreD>,
    inner_dct: Arc<dyn PxdctExecutor<f64> + Send + Sync>,
    inner_dct_scratch_size: usize,
    execution_length: usize,
    q_modules: usize,
}

impl AvxDct2MixedRadix13d {
    pub(crate) fn new(
        len: usize,
        inner_dct: Arc<dyn PxdctExecutor<f64> + Send + Sync>,
    ) -> Result<AvxDct2MixedRadix13d, PxdctError> {
        assert!(
            len.is_multiple_of(13),
            "Mixed radix 13 should not be called on sizes no divisible by 13"
        );

        let q_modules = len / 13;

        let inner_dct_scratch_size = inner_dct.scratch_size();

        Ok(AvxDct2MixedRadix13d {
            rotation_layer: unsafe { dct2_radix_n_rotation_twiddles_avx_d(13, q_modules, len) },
            cos_twiddles: unsafe { dct2_radix_n_cos_twiddles_avx_d(13, q_modules, len) },
            inner_dct,
            inner_dct_scratch_size,
            execution_length: len,
            q_modules: len / 13,
        })
    }
}

boring_avx_mixed_radix!(AvxDct2MixedRadix13d, f64);

impl AvxDct2MixedRadix13d {
    #[inline]
    #[target_feature(enable = "avx2", enable = "fma")]
    fn exec_block<S: BidirectionalStore<f64>, const N: usize>(
        &self,
        data: &mut S,
        a_buffer: &[f64],
        s_buffer: &[f64],
        c_buffer: &[f64],
        uk: usize,
        k: usize,
    ) {
        // Apply rotation twiddles to combine forward and inverted components
        let rotation_twiddle_re = unsafe { *self.rotation_layer.get_unchecked(uk) };
        let rotation_twiddle_im = unsafe { *self.rotation_layer.get_unchecked(uk + 1) };

        let c_forward = AvxStoreD::load_n::<N>(unsafe { c_buffer.get_unchecked(k..) });
        let s_forward = AvxStoreD::load_n::<N>(unsafe {
            s_buffer.get_unchecked(self.q_modules - k - (N - 1)..)
        })
        .reverse_n::<N>();

        let rotated_dc = fma(s_forward, rotation_twiddle_re, c_forward);

        let twiddle_re = unsafe { *self.cos_twiddles.get_unchecked(uk) };
        let twiddle_im = unsafe { *self.cos_twiddles.get_unchecked(uk + 1) };

        let twiddled_dc = rotated_dc * twiddle_re;

        let mut dc0 = twiddled_dc;
        let mut dc2 = twiddled_dc * f64::R13_EVEN_TWIDDLE_0;
        let mut dc4 = twiddled_dc * f64::R13_EVEN_TWIDDLE_5;
        let mut dc6 = twiddled_dc * f64::R13_EVEN_TWIDDLE_1;
        let mut dc8 = twiddled_dc * f64::R13_EVEN_TWIDDLE_4;
        let mut dc10 = twiddled_dc * f64::R13_EVEN_TWIDDLE_2;
        let mut dc12 = twiddled_dc * f64::R13_EVEN_TWIDDLE_3;

        let rotated_ds = fma(c_forward, rotation_twiddle_im, s_forward);

        let twiddled_ds = rotated_ds * twiddle_im;

        let mut ds1 = twiddled_ds * f64::R13_ODD_TWIDDLE_0;
        let mut ds3 = twiddled_ds * -f64::R13_ODD_TWIDDLE_1;
        let mut ds5 = twiddled_ds * f64::R13_ODD_TWIDDLE_2;
        let mut ds7 = twiddled_ds * -f64::R13_ODD_TWIDDLE_3;
        let mut ds9 = twiddled_ds * f64::R13_ODD_TWIDDLE_4;
        let mut ds11 = twiddled_ds * -f64::R13_ODD_TWIDDLE_5;

        {
            let c_forward =
                AvxStoreD::load_n::<N>(unsafe { c_buffer.get_unchecked(self.q_modules + k..) });
            let s_forward = AvxStoreD::load_n::<N>(unsafe {
                s_buffer.get_unchecked(self.q_modules * 2 - k - (N - 1)..)
            })
            .reverse_n::<N>();

            let rotation_twiddle_re = unsafe { *self.rotation_layer.get_unchecked(uk + 2) };
            let rotation_twiddle_im = unsafe { *self.rotation_layer.get_unchecked(uk + 3) };

            let twiddle_re = unsafe { *self.cos_twiddles.get_unchecked(uk + 2) };
            let twiddle_im = unsafe { *self.cos_twiddles.get_unchecked(uk + 3) };

            let rotated_dc1 = fma(s_forward, rotation_twiddle_re, c_forward);
            let rotated_ds2 = fma(c_forward, rotation_twiddle_im, s_forward);

            let twiddled_dc = twiddle_re * rotated_dc1;
            let twiddled_ds = twiddle_im * rotated_ds2;

            dc0 = twiddled_dc + dc0;
            dc2 = AvxStoreD::f64_mul_add(f64::R13_EVEN_TWIDDLE_1, twiddled_dc, dc2);
            dc4 = AvxStoreD::f64_mul_add(f64::R13_EVEN_TWIDDLE_3, twiddled_dc, dc4);
            dc6 = AvxStoreD::f64_mul_add(f64::R13_EVEN_TWIDDLE_4, twiddled_dc, dc6);
            dc8 = AvxStoreD::f64_mul_add(f64::R13_EVEN_TWIDDLE_0, twiddled_dc, dc8);
            dc10 = AvxStoreD::f64_mul_add(f64::R13_EVEN_TWIDDLE_5, twiddled_dc, dc10);
            dc12 = AvxStoreD::f64_mul_add(f64::R13_EVEN_TWIDDLE_2, twiddled_dc, dc12);

            ds1 = AvxStoreD::f64_mul_add(f64::R13_ODD_TWIDDLE_1, twiddled_ds, ds1);
            ds3 = AvxStoreD::f64_mul_add(-f64::R13_ODD_TWIDDLE_4, twiddled_ds, ds3);
            ds5 = AvxStoreD::f64_mul_add(-f64::R13_ODD_TWIDDLE_5, twiddled_ds, ds5);
            ds7 = AvxStoreD::f64_mul_add(f64::R13_ODD_TWIDDLE_2, twiddled_ds, ds7);
            ds9 = AvxStoreD::f64_mul_add(-f64::R13_ODD_TWIDDLE_0, twiddled_ds, ds9);
            ds11 = AvxStoreD::f64_mul_add(f64::R13_ODD_TWIDDLE_3, twiddled_ds, ds11);
        }

        {
            let c_forward =
                AvxStoreD::load_n::<N>(unsafe { c_buffer.get_unchecked(self.q_modules * 2 + k..) });
            let s_forward = AvxStoreD::load_n::<N>(unsafe {
                s_buffer.get_unchecked(self.q_modules * 3 - k - (N - 1)..)
            })
            .reverse_n::<N>();

            let rotation_twiddle_re = unsafe { *self.rotation_layer.get_unchecked(uk + 4) };
            let rotation_twiddle_im = unsafe { *self.rotation_layer.get_unchecked(uk + 5) };

            let twiddle_re = unsafe { *self.cos_twiddles.get_unchecked(uk + 4) };
            let twiddle_im = unsafe { *self.cos_twiddles.get_unchecked(uk + 5) };

            let rotated_dc1 = fma(s_forward, rotation_twiddle_re, c_forward);
            let rotated_ds2 = fma(c_forward, rotation_twiddle_im, s_forward);

            let twiddled_dc = twiddle_re * rotated_dc1;
            let twiddled_ds = twiddle_im * rotated_ds2;

            dc0 = twiddled_dc + dc0;
            dc2 = AvxStoreD::f64_mul_add(f64::R13_EVEN_TWIDDLE_2, twiddled_dc, dc2);
            dc4 = AvxStoreD::f64_mul_add(f64::R13_EVEN_TWIDDLE_1, twiddled_dc, dc4);
            dc6 = AvxStoreD::f64_mul_add(f64::R13_EVEN_TWIDDLE_5, twiddled_dc, dc6);
            dc8 = AvxStoreD::f64_mul_add(f64::R13_EVEN_TWIDDLE_3, twiddled_dc, dc8);
            dc10 = AvxStoreD::f64_mul_add(f64::R13_EVEN_TWIDDLE_0, twiddled_dc, dc10);
            dc12 = AvxStoreD::f64_mul_add(f64::R13_EVEN_TWIDDLE_4, twiddled_dc, dc12);

            ds1 = AvxStoreD::f64_mul_add(f64::R13_ODD_TWIDDLE_2, twiddled_ds, ds1);
            ds3 = AvxStoreD::f64_mul_add(f64::R13_ODD_TWIDDLE_5, twiddled_ds, ds3);
            ds5 = AvxStoreD::f64_mul_add(-f64::R13_ODD_TWIDDLE_0, twiddled_ds, ds5);
            ds7 = AvxStoreD::f64_mul_add(f64::R13_ODD_TWIDDLE_4, twiddled_ds, ds7);
            ds9 = AvxStoreD::f64_mul_add(f64::R13_ODD_TWIDDLE_3, twiddled_ds, ds9);
            ds11 = AvxStoreD::f64_mul_add(-f64::R13_ODD_TWIDDLE_1, twiddled_ds, ds11);
        }

        {
            let c_forward =
                AvxStoreD::load_n::<N>(unsafe { c_buffer.get_unchecked(self.q_modules * 3 + k..) });
            let s_forward = AvxStoreD::load_n::<N>(unsafe {
                s_buffer.get_unchecked(self.q_modules * 4 - k - (N - 1)..)
            })
            .reverse_n::<N>();

            let rotation_twiddle_re = unsafe { *self.rotation_layer.get_unchecked(uk + 6) };
            let rotation_twiddle_im = unsafe { *self.rotation_layer.get_unchecked(uk + 7) };

            let twiddle_re = unsafe { *self.cos_twiddles.get_unchecked(uk + 6) };
            let twiddle_im = unsafe { *self.cos_twiddles.get_unchecked(uk + 7) };

            let rotated_dc1 = fma(s_forward, rotation_twiddle_re, c_forward);
            let rotated_ds2 = fma(c_forward, rotation_twiddle_im, s_forward);

            let twiddled_dc = twiddle_re * rotated_dc1;
            let twiddled_ds = twiddle_im * rotated_ds2;

            dc0 = twiddled_dc + dc0;
            dc2 = AvxStoreD::f64_mul_add(f64::R13_EVEN_TWIDDLE_3, twiddled_dc, dc2);
            dc4 = AvxStoreD::f64_mul_add(f64::R13_EVEN_TWIDDLE_0, twiddled_dc, dc4);
            dc6 = AvxStoreD::f64_mul_add(f64::R13_EVEN_TWIDDLE_2, twiddled_dc, dc6);
            dc8 = AvxStoreD::f64_mul_add(f64::R13_EVEN_TWIDDLE_5, twiddled_dc, dc8);
            dc10 = AvxStoreD::f64_mul_add(f64::R13_EVEN_TWIDDLE_4, twiddled_dc, dc10);
            dc12 = AvxStoreD::f64_mul_add(f64::R13_EVEN_TWIDDLE_1, twiddled_dc, dc12);

            ds1 = AvxStoreD::f64_mul_add(f64::R13_ODD_TWIDDLE_3, twiddled_ds, ds1);
            ds3 = AvxStoreD::f64_mul_add(f64::R13_ODD_TWIDDLE_2, twiddled_ds, ds3);
            ds5 = AvxStoreD::f64_mul_add(-f64::R13_ODD_TWIDDLE_4, twiddled_ds, ds5);
            ds7 = AvxStoreD::f64_mul_add(-f64::R13_ODD_TWIDDLE_1, twiddled_ds, ds7);
            ds9 = AvxStoreD::f64_mul_add(f64::R13_ODD_TWIDDLE_5, twiddled_ds, ds9);
            ds11 = AvxStoreD::f64_mul_add(f64::R13_ODD_TWIDDLE_0, twiddled_ds, ds11);
        }

        {
            let c_forward =
                AvxStoreD::load_n::<N>(unsafe { c_buffer.get_unchecked(self.q_modules * 4 + k..) });
            let s_forward = AvxStoreD::load_n::<N>(unsafe {
                s_buffer.get_unchecked(self.q_modules * 5 - k - (N - 1)..)
            })
            .reverse_n::<N>();

            let rotation_twiddle_re = unsafe { *self.rotation_layer.get_unchecked(uk + 8) };
            let rotation_twiddle_im = unsafe { *self.rotation_layer.get_unchecked(uk + 9) };

            let twiddle_re = unsafe { *self.cos_twiddles.get_unchecked(uk + 8) };
            let twiddle_im = unsafe { *self.cos_twiddles.get_unchecked(uk + 9) };

            let rotated_dc1 = fma(s_forward, rotation_twiddle_re, c_forward);
            let rotated_ds2 = fma(c_forward, rotation_twiddle_im, s_forward);

            let twiddled_dc = twiddle_re * rotated_dc1;
            let twiddled_ds = twiddle_im * rotated_ds2;

            dc0 = twiddled_dc + dc0;
            dc2 = AvxStoreD::f64_mul_add(f64::R13_EVEN_TWIDDLE_4, twiddled_dc, dc2);
            dc4 = AvxStoreD::f64_mul_add(f64::R13_EVEN_TWIDDLE_2, twiddled_dc, dc4);
            dc6 = AvxStoreD::f64_mul_add(f64::R13_EVEN_TWIDDLE_0, twiddled_dc, dc6);
            dc8 = AvxStoreD::f64_mul_add(f64::R13_EVEN_TWIDDLE_1, twiddled_dc, dc8);
            dc10 = AvxStoreD::f64_mul_add(f64::R13_EVEN_TWIDDLE_3, twiddled_dc, dc10);
            dc12 = AvxStoreD::f64_mul_add(f64::R13_EVEN_TWIDDLE_5, twiddled_dc, dc12);

            ds1 = AvxStoreD::f64_mul_add(f64::R13_ODD_TWIDDLE_4, twiddled_ds, ds1);
            ds3 = AvxStoreD::f64_mul_add(f64::R13_ODD_TWIDDLE_0, twiddled_ds, ds3);
            ds5 = AvxStoreD::f64_mul_add(f64::R13_ODD_TWIDDLE_3, twiddled_ds, ds5);
            ds7 = AvxStoreD::f64_mul_add(-f64::R13_ODD_TWIDDLE_5, twiddled_ds, ds7);
            ds9 = AvxStoreD::f64_mul_add(-f64::R13_ODD_TWIDDLE_1, twiddled_ds, ds9);
            ds11 = AvxStoreD::f64_mul_add(-f64::R13_ODD_TWIDDLE_2, twiddled_ds, ds11);
        }

        {
            let c_forward =
                AvxStoreD::load_n::<N>(unsafe { c_buffer.get_unchecked(self.q_modules * 5 + k..) });
            let s_forward = AvxStoreD::load_n::<N>(unsafe {
                s_buffer.get_unchecked(self.q_modules * 6 - k - (N - 1)..)
            })
            .reverse_n::<N>();

            let rotation_twiddle_re = unsafe { *self.rotation_layer.get_unchecked(uk + 10) };
            let rotation_twiddle_im = unsafe { *self.rotation_layer.get_unchecked(uk + 11) };

            let twiddle_re = unsafe { *self.cos_twiddles.get_unchecked(uk + 10) };
            let twiddle_im = unsafe { *self.cos_twiddles.get_unchecked(uk + 11) };

            let rotated_dc1 = fma(s_forward, rotation_twiddle_re, c_forward);
            let rotated_ds2 = fma(c_forward, rotation_twiddle_im, s_forward);

            let twiddled_dc = twiddle_re * rotated_dc1;
            let twiddled_ds = twiddle_im * rotated_ds2;

            dc0 = twiddled_dc + dc0;
            dc2 = AvxStoreD::f64_mul_add(f64::R13_EVEN_TWIDDLE_5, twiddled_dc, dc2);
            dc4 = AvxStoreD::f64_mul_add(f64::R13_EVEN_TWIDDLE_4, twiddled_dc, dc4);
            dc6 = AvxStoreD::f64_mul_add(f64::R13_EVEN_TWIDDLE_3, twiddled_dc, dc6);
            dc8 = AvxStoreD::f64_mul_add(f64::R13_EVEN_TWIDDLE_2, twiddled_dc, dc8);
            dc10 = AvxStoreD::f64_mul_add(f64::R13_EVEN_TWIDDLE_1, twiddled_dc, dc10);
            dc12 = AvxStoreD::f64_mul_add(f64::R13_EVEN_TWIDDLE_0, twiddled_dc, dc12);

            ds1 = AvxStoreD::f64_mul_add(f64::R13_ODD_TWIDDLE_5, twiddled_ds, ds1);
            ds3 = AvxStoreD::f64_mul_add(f64::R13_ODD_TWIDDLE_3, twiddled_ds, ds3);
            ds5 = AvxStoreD::f64_mul_add(f64::R13_ODD_TWIDDLE_1, twiddled_ds, ds5);
            ds7 = AvxStoreD::f64_mul_add(f64::R13_ODD_TWIDDLE_0, twiddled_ds, ds7);
            ds9 = AvxStoreD::f64_mul_add(f64::R13_ODD_TWIDDLE_2, twiddled_ds, ds9);
            ds11 = AvxStoreD::f64_mul_add(f64::R13_ODD_TWIDDLE_4, twiddled_ds, ds11);
        }

        let a0 = AvxStoreD::load_n::<N>(unsafe { a_buffer.get_unchecked(k..) });
        let dc = dc0 + a0;

        dc2 = -(dc2 + a0);
        dc4 += a0;
        dc6 += a0;
        dc8 += a0;
        dc10 += a0;
        dc12 += a0;

        dc.write_n::<N>(data.slice_from_mut(k..));

        let dss1 = AvxStoreD::f64_mul_add(2., ds1, -dc);
        dss1.reverse_n::<N>()
            .write_n::<N>(data.slice_from_mut(self.q_modules * 2 - k - (N - 1)..));

        let idx1 = self.q_modules * 2 + k;
        dc2 = AvxStoreD::f64_mul_add(2f64.as_(), dc2, -dss1);
        dc2.write_n::<N>(data.slice_from_mut(idx1..));

        let dss3 = AvxStoreD::f64_mul_add(2f64.as_(), -ds3, -dc2);
        let idx = self.q_modules * 4 - k - (N - 1);
        dss3.reverse_n::<N>()
            .write_n::<N>(data.slice_from_mut(idx..));

        let idx1 = self.q_modules * 4 + k;
        let mdc4 = AvxStoreD::f64_mul_add(2f64.as_(), dc4, -dss3);
        mdc4.write_n::<N>(data.slice_from_mut(idx1..));

        let dss5 = AvxStoreD::f64_mul_add(2f64.as_(), ds5, -mdc4);
        let idx = self.q_modules * 6 - k - (N - 1);
        dss5.reverse_n::<N>()
            .write_n::<N>(data.slice_from_mut(idx..));

        dc6 = AvxStoreD::f64_mul_add(2., -dc6, -dss5);
        let idx = self.q_modules * 6 + k;
        dc6.write_n::<N>(data.slice_from_mut(idx..));

        let dss6 = AvxStoreD::f64_mul_add(2., -ds7, -dc6);
        let idx = self.q_modules * 8 - k - (N - 1);
        dss6.reverse_n::<N>()
            .write_n::<N>(data.slice_from_mut(idx..));

        dc8 = AvxStoreD::f64_mul_add(2., dc8, -dss6);
        let idx = self.q_modules * 8 + k;
        dc8.write_n::<N>(data.slice_from_mut(idx..));

        let dss9 = AvxStoreD::f64_mul_add(2., ds9, -dc8);
        let idx = self.q_modules * 10 - k - (N - 1);
        dss9.reverse_n::<N>()
            .write_n::<N>(data.slice_from_mut(idx..));

        dc10 = AvxStoreD::f64_mul_add(2., -dc10, -dss9);
        let idx = self.q_modules * 10 + k;
        dc10.write_n::<N>(data.slice_from_mut(idx..));

        let dss11 = AvxStoreD::f64_mul_add(2., -ds11, -dc10);
        let idx = self.q_modules * 12 - k - (N - 1);
        dss11
            .reverse_n::<N>()
            .write_n::<N>(data.slice_from_mut(idx..));

        dc12 = AvxStoreD::f64_mul_add(2., dc12, -dss11);
        let idx = self.q_modules * 12 + k;
        dc12.write_n::<N>(data.slice_from_mut(idx..));
    }

    #[inline]
    #[target_feature(enable = "avx2", enable = "fma")]
    fn execute_store<S: BidirectionalStore<f64>>(
        &self,
        data: &mut S,
        scratch: &mut [f64],
    ) -> Result<(), PxdctError> {
        let (scratch, inner_scratch) = scratch.split_at_mut(self.execution_length);
        let q_modules = self.execution_length / 13;
        let (a_buffer, c_s_buffer) = scratch.split_at_mut(q_modules);
        let (c_buffer, s_buffer) = c_s_buffer.split_at_mut(q_modules * 6);

        // Step 1: Decompose input into A (center), C (even-symmetric), S (odd-symmetric) buffers
        for (n, dst) in a_buffer.iter_mut().enumerate() {
            *dst = data[n * 13 + 6];
        }

        // Extract and combine symmetric pairs with sign alternation for S buffer
        for (m, (c_buffer, s_buffer)) in c_buffer
            .chunks_exact_mut(q_modules)
            .zip(s_buffer.chunks_exact_mut(q_modules))
            .enumerate()
        {
            let mut sign = f64::one();
            for (n, (c_dst, s_dst)) in c_buffer.iter_mut().zip(s_buffer.iter_mut()).enumerate() {
                let u0 = data[13 * n + m];
                let u1 = data[13 * n + 13 - m - 1];

                *c_dst = u0 + u1;
                *s_dst = (u0 - u1).mulsign(sign);

                sign = -sign;
            }
        }

        // Step 2: Apply DCT-II to all buffers (A, C₀, C₁, S₀, S₁)
        self.inner_dct
            .execute_with_scratch(scratch, inner_scratch)?;

        let (a_buffer, c_s_buffer) = scratch.split_at_mut(q_modules);
        let (c_buffer, s_buffer) = c_s_buffer.split_at_mut(q_modules * 6);

        {
            // Step 3: Recombine transformed buffers with twiddle factors

            // Handle k=0 case (DC and low frequencies)
            let qc = c_buffer[0];
            let mut c0 = qc; // Component C₀ (position 0)
            let mut c1 = qc * f64::R13_EVEN_TWIDDLE_0; // Component C₂ (position 2, uses j=2)
            let mut c2 = qc * f64::R13_EVEN_TWIDDLE_5; // Component C₄ (position 4, uses j=4)
            let mut c3 = qc * f64::R13_EVEN_TWIDDLE_1; // Component C6 (position 6, uses j=6)
            let mut c4 = qc * f64::R13_EVEN_TWIDDLE_4; // Component C8 (position 8, uses j=8)
            let mut c5 = qc * f64::R13_EVEN_TWIDDLE_2;
            let mut c6 = qc * f64::R13_EVEN_TWIDDLE_3;

            let s0_twiddled = s_buffer[0];

            let mut s0 = s0_twiddled * f64::R13_ODD_TWIDDLE_0;
            let mut s1 = s0_twiddled * -f64::R13_ODD_TWIDDLE_1;
            let mut s2 = s0_twiddled * f64::R13_ODD_TWIDDLE_2;
            let mut s3 = s0_twiddled * -f64::R13_ODD_TWIDDLE_3;
            let mut s4 = s0_twiddled * f64::R13_ODD_TWIDDLE_4;
            let mut s5 = s0_twiddled * -f64::R13_ODD_TWIDDLE_5;

            let ci = unsafe { *c_buffer.get_unchecked(q_modules) };
            let si = unsafe { *s_buffer.get_unchecked(q_modules) };

            let ci2 = unsafe { *c_buffer.get_unchecked(q_modules * 2) };
            let si2 = unsafe { *s_buffer.get_unchecked(q_modules * 2) };

            let ci3 = unsafe { *c_buffer.get_unchecked(q_modules * 3) };
            let si3 = unsafe { *s_buffer.get_unchecked(q_modules * 3) };

            let ci4 = unsafe { *c_buffer.get_unchecked(q_modules * 4) };
            let si4 = unsafe { *s_buffer.get_unchecked(q_modules * 4) };

            let ci5 = unsafe { *c_buffer.get_unchecked(q_modules * 5) };
            let si5 = unsafe { *s_buffer.get_unchecked(q_modules * 5) };

            c0 = ci + c0 + ci2 + ci3 + ci4 + ci5;

            let a0 = a_buffer[0];

            let dc = c0 + a0;
            data[0] = dc;

            c1 = fma(ci, f64::R13_EVEN_TWIDDLE_1, c1);
            c1 = fma(ci2, f64::R13_EVEN_TWIDDLE_2, c1);
            c1 = fma(ci3, f64::R13_EVEN_TWIDDLE_3, c1);
            c1 = fma(ci4, f64::R13_EVEN_TWIDDLE_4, c1);
            c1 = fma(ci5, f64::R13_EVEN_TWIDDLE_5, c1);

            c2 = fma(ci, f64::R13_EVEN_TWIDDLE_3, c2);
            c2 = fma(ci2, f64::R13_EVEN_TWIDDLE_1, c2);
            c2 = fma(ci3, f64::R13_EVEN_TWIDDLE_0, c2);
            c2 = fma(ci4, f64::R13_EVEN_TWIDDLE_2, c2);
            c2 = fma(ci5, f64::R13_EVEN_TWIDDLE_4, c2);

            let dc2 = c2 + a0;
            data[q_modules * 4] = dc2;

            c3 = fma(ci, f64::R13_EVEN_TWIDDLE_4, c3);
            c3 = fma(ci2, f64::R13_EVEN_TWIDDLE_5, c3);
            c3 = fma(ci3, f64::R13_EVEN_TWIDDLE_2, c3);
            c3 = fma(ci4, f64::R13_EVEN_TWIDDLE_0, c3);
            c3 = fma(ci5, f64::R13_EVEN_TWIDDLE_3, c3);

            let dc3 = c3 + a0;
            data[q_modules * 6] = -dc3;

            c4 = fma(ci, f64::R13_EVEN_TWIDDLE_0, c4);
            c4 = fma(ci2, f64::R13_EVEN_TWIDDLE_3, c4);
            c4 = fma(ci3, f64::R13_EVEN_TWIDDLE_5, c4);
            c4 = fma(ci4, f64::R13_EVEN_TWIDDLE_1, c4);
            c4 = fma(ci5, f64::R13_EVEN_TWIDDLE_2, c4);

            let dc4 = c4 + a0;
            data[q_modules * 8] = dc4;

            c5 = fma(ci, f64::R13_EVEN_TWIDDLE_5, c5);
            c5 = fma(ci2, f64::R13_EVEN_TWIDDLE_0, c5);
            c5 = fma(ci3, f64::R13_EVEN_TWIDDLE_4, c5);
            c5 = fma(ci4, f64::R13_EVEN_TWIDDLE_3, c5);
            c5 = fma(ci5, f64::R13_EVEN_TWIDDLE_1, c5);

            let dc5 = c5 + a0;
            data[q_modules * 10] = -dc5;

            c6 = fma(ci, f64::R13_EVEN_TWIDDLE_2, c6);
            c6 = fma(ci2, f64::R13_EVEN_TWIDDLE_4, c6);
            c6 = fma(ci3, f64::R13_EVEN_TWIDDLE_1, c6);
            c6 = fma(ci4, f64::R13_EVEN_TWIDDLE_5, c6);
            c6 = fma(ci5, f64::R13_EVEN_TWIDDLE_0, c6);

            let dc6 = c6 + a0;
            data[q_modules * 12] = dc6;

            s0 = fma(si, f64::R13_ODD_TWIDDLE_1, s0);
            s0 = fma(si2, f64::R13_ODD_TWIDDLE_2, s0);
            s0 = fma(si3, f64::R13_ODD_TWIDDLE_3, s0);
            s0 = fma(si4, f64::R13_ODD_TWIDDLE_4, s0);
            s0 = fma(si5, f64::R13_ODD_TWIDDLE_5, s0);

            s1 = fma(si, -f64::R13_ODD_TWIDDLE_4, s1);
            s1 = fma(si2, f64::R13_ODD_TWIDDLE_5, s1);
            s1 = fma(si3, f64::R13_ODD_TWIDDLE_2, s1);
            s1 = fma(si4, f64::R13_ODD_TWIDDLE_0, s1);
            s1 = fma(si5, f64::R13_ODD_TWIDDLE_3, s1);

            s2 = fma(si, -f64::R13_ODD_TWIDDLE_5, s2);
            s2 = fma(si2, -f64::R13_ODD_TWIDDLE_0, s2);
            s2 = fma(si3, -f64::R13_ODD_TWIDDLE_4, s2);
            s2 = fma(si4, f64::R13_ODD_TWIDDLE_3, s2);
            s2 = fma(si5, f64::R13_ODD_TWIDDLE_1, s2);

            s3 = fma(si, f64::R13_ODD_TWIDDLE_2, s3);
            s3 = fma(si2, f64::R13_ODD_TWIDDLE_4, s3);
            s3 = fma(si3, -f64::R13_ODD_TWIDDLE_1, s3);
            s3 = fma(si4, -f64::R13_ODD_TWIDDLE_5, s3);
            s3 = fma(si5, f64::R13_ODD_TWIDDLE_0, s3);

            s4 = fma(si, -f64::R13_ODD_TWIDDLE_0, s4);
            s4 = fma(si2, f64::R13_ODD_TWIDDLE_3, s4);
            s4 = fma(si3, f64::R13_ODD_TWIDDLE_5, s4);
            s4 = fma(si4, -f64::R13_ODD_TWIDDLE_1, s4);
            s4 = fma(si5, f64::R13_ODD_TWIDDLE_2, s4);

            s5 = fma(si, f64::R13_ODD_TWIDDLE_3, s5);
            s5 = fma(si2, -f64::R13_ODD_TWIDDLE_1, s5);
            s5 = fma(si3, f64::R13_ODD_TWIDDLE_0, s5);
            s5 = fma(si4, -f64::R13_ODD_TWIDDLE_2, s5);
            s5 = fma(si5, f64::R13_ODD_TWIDDLE_4, s5);

            data[q_modules * 3] = -s1;
            data[q_modules] = s0;
            let qid2 = -(c1 + a0); // negated 2j
            data[q_modules * 2] = qid2;
            data[q_modules * 5] = s2;
            data[q_modules * 7] = -s3;
            data[q_modules * 9] = s4;
            data[q_modules * 11] = -s5;

            let mut uk = 0usize;
            let mut k = 1usize;
            while k + 4 <= q_modules {
                self.exec_block::<S, 4>(data, a_buffer, s_buffer, c_buffer, uk, k);

                k += 4;
                uk += 12;
            }

            let remainder = q_modules - k;

            if remainder == 3 {
                self.exec_block::<S, 3>(data, a_buffer, s_buffer, c_buffer, uk, k);
            } else if remainder == 2 {
                self.exec_block::<S, 2>(data, a_buffer, s_buffer, c_buffer, uk, k);
            } else if remainder == 1 {
                self.exec_block::<S, 1>(data, a_buffer, s_buffer, c_buffer, uk, k);
            }
        }
        Ok(())
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::Pxdct;
    use crate::tests::naive_dct2;
    use crate::util::has_valid_avx;

    #[test]
    fn test_radix13_dct() {
        if !has_valid_avx() {
            return;
        }
        let mut input = vec![0.; 52];
        for (i, z) in input.iter_mut().enumerate() {
            *z = i as f64 + rand::random::<f64>() * 10.0;
        }
        let mut reference_input = input.clone();
        // let rr = Pxdct::make_dct2_f64(25).unwrap();
        // rr.execute(&mut reference_input).unwrap();
        reference_input = naive_dct2(&reference_input);
        let bf =
            AvxDct2MixedRadix13d::new(input.len(), Pxdct::make_dct2_f64(input.len() / 13).unwrap())
                .unwrap();
        bf.execute(&mut input).unwrap();
        println!(
            "{:?}",
            input
                .iter()
                .enumerate()
                .map(|(i, x)| format!("({i}) {}", x))
                .collect::<Vec<_>>()
        );
        println!(
            "{:?}",
            reference_input
                .iter()
                .enumerate()
                .map(|(i, x)| format!("({i}) {}", x))
                .collect::<Vec<_>>()
        );
        input
            .iter()
            .zip(reference_input.iter())
            .enumerate()
            .for_each(|(i, (&src, &r0))| {
                if (src - r0).abs() > 1e-1 {
                    println!(
                        "Difference must be < {}, but it was {}, at position {i}",
                        1e-1,
                        (src - r0).abs()
                    )
                }
            });
    }
}