pxdct 0.3.2

/*
 * // Copyright (c) Radzivon Bartoshyk 12/2025. All rights reserved.
 * //
 * // Redistribution and use in source and binary forms, with or without modification,
 * // are permitted provided that the following conditions are met:
 * //
 * // 1.  Redistributions of source code must retain the above copyright notice, this
 * // list of conditions and the following disclaimer.
 * //
 * // 2.  Redistributions in binary form must reproduce the above copyright notice,
 * // this list of conditions and the following disclaimer in the documentation
 * // and/or other materials provided with the distribution.
 * //
 * // 3.  Neither the name of the copyright holder nor the names of its
 * // contributors may be used to endorse or promote products derived from
 * // this software without specific prior written permission.
 * //
 * // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
 * // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 * // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
use crate::avx::dct2::{dct2_radix_n_cos_twiddles_avx_f, dct2_radix_n_rotation_twiddles_avx_f};
use crate::avx::storef::AvxStoreF;
use crate::avx::util::{define_avx_butterfly, fma};
use crate::bidirectional::{BidirectionalStore, InPlaceStore};
use crate::dct2::MixedRadix3Sample;
use crate::factory_dct2::Dct2Factory;
use crate::util::{DctConstants, DctSample, mixed_radix_inner_twiddle};
use crate::{PxdctError, PxdctExecutor};
use num_traits::{AsPrimitive, One, Zero};
use std::sync::Arc;

#[derive(Debug, Clone)]
pub(crate) struct AvxDct2Butterfly9<T: DctSample> {
    _phantom: std::marker::PhantomData<T>,
}

impl<T: DctSample> Default for AvxDct2Butterfly9<T>
where
    f64: AsPrimitive<T>,
    usize: AsPrimitive<T>,
{
    fn default() -> Self {
        Self {
            _phantom: std::marker::PhantomData,
        }
    }
}

impl<T: DctSample> AvxDct2Butterfly9<T>
where
    f64: AsPrimitive<T>,
{
    #[inline]
    #[target_feature(enable = "avx2", enable = "fma")]
    pub(crate) fn exec<S: BidirectionalStore<T>>(&self, data: &mut S) {
        // This is autogenerated Radix-Q where Q = 9.
        let a_buffer = [data[4]];
        let c_buffer = [
            data[0] + data[8],
            data[1] + data[7],
            data[2] + data[6],
            data[3] + data[5],
        ];
        let s_buffer = [
            data[0] - data[8],
            data[1] - data[7],
            data[2] - data[6],
            data[3] - data[5],
        ];
        let qc = c_buffer[0];
        let mut c0 = qc;
        let mut c1 = qc * T::R9_EVEN_TWIDDLE_0;
        let mut s0 = s_buffer[0];
        s0 *= T::R9_ODD_TWIDDLE_0;
        let ci1 = c_buffer[1];
        let si1 = s_buffer[1];
        c0 = ci1 + c0;
        c1 = fma(ci1, -T::HALF, c1);
        s0 = fma(si1, T::R9_ODD_TWIDDLE_1, s0);
        let ci2 = c_buffer[2];
        let si2 = s_buffer[2];
        c0 = ci2 + c0;
        c1 = fma(ci2, T::R9_EVEN_TWIDDLE_1, c1);
        s0 = fma(si2, T::R9_ODD_TWIDDLE_2, s0);
        let ci3 = c_buffer[3];
        let si3 = s_buffer[3];
        c0 = ci3 + c0;
        c1 = fma(ci3, T::R9_EVEN_TWIDDLE_2, c1);
        s0 = fma(si3, T::R9_ODD_TWIDDLE_3, s0);
        let a0 = a_buffer[0];
        let dc = c0 + a0;
        data[0] = dc;
        data[1] = s0;
        data[2] = -(c1 + a0);
        let mut c0 = qc * T::R9_EVEN_TWIDDLE_2;
        let mut s0 = s_buffer[0];
        s0 *= -T::R9_ODD_TWIDDLE_1;
        c0 = fma(ci1, -T::HALF, c0);
        c0 = fma(ci2, T::R9_EVEN_TWIDDLE_0, c0);
        s0 = fma(si2, T::R9_ODD_TWIDDLE_1, s0);
        c0 = fma(ci3, T::R9_EVEN_TWIDDLE_1, c0);
        s0 = fma(si3, T::R9_ODD_TWIDDLE_1, s0);
        let dc = c0 + a0;
        data[4] = dc;
        data[3] = -s0;
        let mut c0 = qc * -T::HALF;
        let mut s0 = s_buffer[0];
        s0 *= T::R9_ODD_TWIDDLE_2;
        c0 += ci1;
        s0 = fma(si1, -T::R9_ODD_TWIDDLE_1, s0);
        c0 = fma(ci2, -T::HALF, c0);
        s0 = fma(si2, -T::R9_ODD_TWIDDLE_3, s0);
        c0 = fma(ci3, -T::HALF, c0);
        s0 = fma(si3, T::R9_ODD_TWIDDLE_0, s0);
        let dc = c0 + a0;
        data[6] = -dc;
        data[5] = s0;
        let mut c0 = qc * T::R9_EVEN_TWIDDLE_1;
        let mut s0 = s_buffer[0];
        s0 *= -T::R9_ODD_TWIDDLE_3;
        c0 = fma(ci1, -T::HALF, c0);
        s0 = fma(si1, T::R9_ODD_TWIDDLE_1, s0);
        c0 = fma(ci2, T::R9_EVEN_TWIDDLE_2, c0);
        s0 = fma(si2, -T::R9_ODD_TWIDDLE_0, s0);
        c0 = fma(ci3, T::R9_EVEN_TWIDDLE_0, c0);
        s0 = fma(si3, T::R9_ODD_TWIDDLE_2, s0);
        let dc = c0 + a0;
        data[8] = dc;
        data[7] = -s0;
    }
}

define_avx_butterfly!(AvxDct2Butterfly9, 9);

#[target_feature(enable = "avx2")]
pub(crate) fn dct2_radix3_avx_groups(len: usize) -> Vec<AvxStoreF> {
    let inner_layer_groups = len / 3;
    let simd_groups = inner_layer_groups.div_ceil(8);
    let mut inner_layer = Vec::with_capacity(simd_groups * 8);

    for g in 0..simd_groups {
        let mut re0 = [0f32; 8];
        let mut im0 = [0f32; 8];
        let mut re1 = [0f32; 8];
        let mut im1 = [0f32; 8];

        for lane in 0..8 {
            let i = g * 8 + lane;
            if i < inner_layer_groups {
                let t0 = mixed_radix_inner_twiddle(2.0 * (i as f32) + 1.0, len);
                let t1 = mixed_radix_inner_twiddle(2.0 * (2.0 * (i as f32) + 1.0), len);

                re0[lane] = t0.re;
                im0[lane] = t0.im * f32::SQRT_3;

                re1[lane] = t1.re;
                im1[lane] = -t1.im * f32::SQRT_3;
            }
        }

        inner_layer.push(AvxStoreF::load(&re0));
        inner_layer.push(AvxStoreF::load(&im0));
        inner_layer.push(AvxStoreF::load(&re1));
        inner_layer.push(AvxStoreF::load(&im1));
    }
    inner_layer
}

#[derive(Debug, Clone)]
pub(crate) struct AvxDct2Butterfly27f {
    bf9: AvxDct2Butterfly9<f32>,
    inner_layer: [AvxStoreF; 8],
}

impl Default for AvxDct2Butterfly27f {
    fn default() -> Self {
        Self {
            bf9: AvxDct2Butterfly9::default(),
            inner_layer: unsafe { dct2_radix3_avx_groups(27).try_into().unwrap() },
        }
    }
}

impl AvxDct2Butterfly27f {
    #[inline]
    #[target_feature(enable = "avx2", enable = "fma")]
    pub(crate) fn exec<S: BidirectionalStore<f32>>(
        &self,
        data: &mut S,
        a_buffer: &mut [f32; 9],
        b_buffer: &mut [f32; 9],
        c_buffer: &mut [f32; 9],
    ) {
        {
            let ai = AvxStoreF::load(data.slice_from(0..));
            let bi = AvxStoreF::load(data.slice_from(18..));
            let mut ci = AvxStoreF::load(data.slice_from(18 - 8..));
            ci = ci.reverse();

            let cos_sin_ai_re = self.inner_layer[0];
            let cos_sin_ai_im = self.inner_layer[1];
            let cos_sin_2ai_re = self.inner_layer[2];
            let cos_sin_2ai_im = self.inner_layer[3];

            let bici = bi + ci;
            let a_comp = ai + bici;
            let second_layer_comp0 = AvxStoreF::f32_mul_sub(2., ai, bici);

            let d_ci_bi = ci - bi;

            let b0_b = fma(second_layer_comp0, cos_sin_ai_re, d_ci_bi * cos_sin_ai_im);
            let c0_b = fma(second_layer_comp0, cos_sin_2ai_re, d_ci_bi * cos_sin_2ai_im);

            a_comp.write(a_buffer);
            b0_b.write(b_buffer);
            c0_b.write(c_buffer);
        }

        {
            let j = 8;
            let ai = AvxStoreF::load1(data.slice_from(j..));
            let bi = AvxStoreF::load1(data.slice_from(18 + j..));
            let ci = AvxStoreF::load1(data.slice_from(18 - j - 1..));

            let cos_sin_ai_re = self.inner_layer[4];
            let cos_sin_ai_im = self.inner_layer[5];
            let cos_sin_2ai_re = self.inner_layer[6];
            let cos_sin_2ai_im = self.inner_layer[7];

            let bici = bi + ci;
            let a_comp = ai + bici;
            let second_layer_comp0 = AvxStoreF::f32_mul_sub(2., ai, bici);

            let d_ci_bi = ci - bi;

            let b0_b = fma(second_layer_comp0, cos_sin_ai_re, d_ci_bi * cos_sin_ai_im);
            let c0_b = fma(second_layer_comp0, cos_sin_2ai_re, d_ci_bi * cos_sin_2ai_im);

            unsafe {
                a_comp.write1(a_buffer.get_unchecked_mut(j..));
                b0_b.write1(b_buffer.get_unchecked_mut(j..));
                c0_b.write1(c_buffer.get_unchecked_mut(j..));
            }
        }

        self.bf9.exec(&mut InPlaceStore::new(a_buffer));
        self.bf9.exec(&mut InPlaceStore::new(b_buffer));
        self.bf9.exec(&mut InPlaceStore::new(c_buffer));

        data[0] = a_buffer[0];
        let b0 = b_buffer[0] * f32::HALF;
        data[1] = b0;
        let c0 = c_buffer[0] * f32::HALF;
        data[2] = c0;

        let mut last_b = c0;
        let mut last_c = b0;

        for k in 1..9 {
            data[3 * k] = a_buffer[k];

            let deferred_c = b_buffer[k] - last_b;
            data[3 * k + 1] = deferred_c;

            last_b = c_buffer[k] - last_c;
            data[3 * k + 2] = last_b;
            last_c = deferred_c;
        }
    }
}

impl AvxDct2Butterfly27f {
    #[target_feature(enable = "avx2", enable = "fma")]
    fn execute_impl(&self, data: &mut [f32]) -> Result<(), PxdctError> {
        if !data.len().is_multiple_of(27) {
            return Err(PxdctError::InvalidSizeMultiplier(data.len(), self.length()));
        }

        let mut a_buffer = [f32::zero(); 9];
        let mut b_buffer = [f32::zero(); 9];
        let mut c_buffer = [f32::zero(); 9];

        for chunk in data.chunks_exact_mut(27) {
            self.exec(
                &mut InPlaceStore::new(chunk),
                &mut a_buffer,
                &mut b_buffer,
                &mut c_buffer,
            );
        }
        Ok(())
    }

    #[target_feature(enable = "avx2", enable = "fma")]
    fn execute_into_impl(&self, input: &[f32], output: &mut [f32]) -> Result<(), PxdctError> {
        use crate::util::validate_oof_sizes;
        validate_oof_sizes!(input, output, 27);

        let mut a_buffer = [f32::zero(); 9];
        let mut b_buffer = [f32::zero(); 9];
        let mut c_buffer = [f32::zero(); 9];

        use crate::bidirectional::BiStore;
        for (src, dst) in input.chunks_exact(27).zip(output.chunks_exact_mut(27)) {
            self.exec(
                &mut BiStore::new(src, dst),
                &mut a_buffer,
                &mut b_buffer,
                &mut c_buffer,
            );
        }
        Ok(())
    }
}

impl PxdctExecutor<f32> for AvxDct2Butterfly27f {
    fn execute(&self, data: &mut [f32]) -> Result<(), PxdctError> {
        unsafe { self.execute_impl(data) }
    }

    fn execute_with_scratch(&self, data: &mut [f32], _: &mut [f32]) -> Result<(), PxdctError> {
        unsafe { self.execute_impl(data) }
    }

    fn execute_into(&self, input: &[f32], output: &mut [f32]) -> Result<(), PxdctError> {
        unsafe { self.execute_into_impl(input, output) }
    }

    fn execute_into_with_scratch(
        &self,
        input: &[f32],
        output: &mut [f32],
        _: &mut [f32],
    ) -> Result<(), PxdctError> {
        unsafe { self.execute_into_impl(input, output) }
    }

    fn length(&self) -> usize {
        27
    }

    fn scratch_size(&self) -> usize {
        0
    }
}

#[derive(Clone)]
pub(crate) struct AvxDct2Butterfly81f {
    bf27: Arc<dyn PxdctExecutor<f32> + Send + Sync>,
    inner_layer: [AvxStoreF; 16],
}

impl Default for AvxDct2Butterfly81f {
    fn default() -> Self {
        Self {
            bf27: f32::dct2_butterfly27(),
            inner_layer: unsafe { dct2_radix3_avx_groups(81).try_into().unwrap() },
        }
    }
}

impl AvxDct2Butterfly81f {
    #[inline]
    #[target_feature(enable = "avx2", enable = "fma")]
    fn exec<S: BidirectionalStore<f32>>(
        &self,
        data: &mut S,
        a_buffer: &mut [f32; 27],
        b_buffer: &mut [f32; 27],
        c_buffer: &mut [f32; 27],
    ) {
        for i in 0..3 {
            let ai = AvxStoreF::load(data.slice_from(i * 8..));
            let bi = AvxStoreF::load(data.slice_from(54 + i * 8..));
            let mut ci = AvxStoreF::load(data.slice_from(54 - i * 8 - 8..));
            ci = ci.reverse();

            let layer_idx = i * 4;

            let cos_sin_ai_re = self.inner_layer[layer_idx];
            let cos_sin_ai_im = self.inner_layer[layer_idx + 1];
            let cos_sin_2ai_re = self.inner_layer[layer_idx + 2];
            let cos_sin_2ai_im = self.inner_layer[layer_idx + 3];

            let bici = bi + ci;
            let a_comp = ai + bici;
            let second_layer_comp0 = AvxStoreF::f32_mul_sub(2., ai, bici);

            let d_ci_bi = ci - bi;

            let b0_b = fma(second_layer_comp0, cos_sin_ai_re, d_ci_bi * cos_sin_ai_im);
            let c0_b = fma(second_layer_comp0, cos_sin_2ai_re, d_ci_bi * cos_sin_2ai_im);

            unsafe {
                a_comp.write(a_buffer.get_unchecked_mut(i * 8..));
                b0_b.write(b_buffer.get_unchecked_mut(i * 8..));
                c0_b.write(c_buffer.get_unchecked_mut(i * 8..));
            }
        }

        {
            let i = 3;
            let ai = AvxStoreF::load3(data.slice_from(i * 8..));
            let bi = AvxStoreF::load3(data.slice_from(54 + i * 8..));
            let mut ci = AvxStoreF::load3(data.slice_from(54 - i * 8 - 3..));
            ci = ci.reverse3();

            let layer_idx = i * 4;

            let cos_sin_ai_re = self.inner_layer[layer_idx];
            let cos_sin_ai_im = self.inner_layer[layer_idx + 1];
            let cos_sin_2ai_re = self.inner_layer[layer_idx + 2];
            let cos_sin_2ai_im = self.inner_layer[layer_idx + 3];

            let bici = bi + ci;
            let a_comp = ai + bici;
            let second_layer_comp0 = AvxStoreF::f32_mul_sub(2., ai, bici);

            let d_ci_bi = ci - bi;

            let b0_b = fma(second_layer_comp0, cos_sin_ai_re, d_ci_bi * cos_sin_ai_im);
            let c0_b = fma(second_layer_comp0, cos_sin_2ai_re, d_ci_bi * cos_sin_2ai_im);

            unsafe {
                a_comp.write3(a_buffer.get_unchecked_mut(i * 8..));
                b0_b.write3(b_buffer.get_unchecked_mut(i * 8..));
                c0_b.write3(c_buffer.get_unchecked_mut(i * 8..));
            }
        }

        _ = self.bf27.execute(a_buffer.as_mut_slice());
        _ = self.bf27.execute(b_buffer.as_mut_slice());
        _ = self.bf27.execute(c_buffer.as_mut_slice());

        data[0] = a_buffer[0];
        let b0 = b_buffer[0] * f32::HALF;
        data[1] = b0;
        let c0 = c_buffer[0] * f32::HALF;
        data[2] = c0;

        let mut last_b = c0;
        let mut last_c = b0;

        for k in 1..27 {
            data[3 * k] = a_buffer[k];

            let deferred_c = b_buffer[k] - last_b;
            data[3 * k + 1] = deferred_c;

            last_b = c_buffer[k] - last_c;
            data[3 * k + 2] = last_b;
            last_c = deferred_c;
        }
    }
}

impl AvxDct2Butterfly81f {
    #[target_feature(enable = "avx2", enable = "fma")]
    fn execute_impl(&self, data: &mut [f32]) -> Result<(), PxdctError> {
        if !data.len().is_multiple_of(81) {
            return Err(PxdctError::InvalidSizeMultiplier(data.len(), self.length()));
        }

        let mut a_buffer = [f32::zero(); 27];
        let mut b_buffer = [f32::zero(); 27];
        let mut c_buffer = [f32::zero(); 27];

        for chunk in data.chunks_exact_mut(81) {
            self.exec(
                &mut InPlaceStore::new(chunk),
                &mut a_buffer,
                &mut b_buffer,
                &mut c_buffer,
            );
        }
        Ok(())
    }

    #[target_feature(enable = "avx2", enable = "fma")]
    fn execute_into_impl(&self, input: &[f32], output: &mut [f32]) -> Result<(), PxdctError> {
        use crate::util::validate_oof_sizes;
        validate_oof_sizes!(input, output, 81);

        let mut a_buffer = [f32::zero(); 27];
        let mut b_buffer = [f32::zero(); 27];
        let mut c_buffer = [f32::zero(); 27];

        use crate::bidirectional::BiStore;
        for (src, dst) in input.chunks_exact(81).zip(output.chunks_exact_mut(81)) {
            self.exec(
                &mut BiStore::new(src, dst),
                &mut a_buffer,
                &mut b_buffer,
                &mut c_buffer,
            );
        }
        Ok(())
    }
}

impl PxdctExecutor<f32> for AvxDct2Butterfly81f {
    fn execute(&self, data: &mut [f32]) -> Result<(), PxdctError> {
        unsafe { self.execute_impl(data) }
    }

    fn execute_with_scratch(&self, data: &mut [f32], _: &mut [f32]) -> Result<(), PxdctError> {
        unsafe { self.execute_impl(data) }
    }

    fn execute_into(&self, input: &[f32], output: &mut [f32]) -> Result<(), PxdctError> {
        unsafe { self.execute_into_impl(input, output) }
    }

    fn execute_into_with_scratch(
        &self,
        input: &[f32],
        output: &mut [f32],
        _: &mut [f32],
    ) -> Result<(), PxdctError> {
        unsafe { self.execute_into_impl(input, output) }
    }

    fn length(&self) -> usize {
        81
    }

    fn scratch_size(&self) -> usize {
        0
    }
}

#[derive(Clone)]
pub(crate) struct AvxDct2Butterfly243f {
    bf81: Arc<dyn PxdctExecutor<f32> + Send + Sync>,
    rotation_layer: [AvxStoreF; 20],
    cos_twiddles: [AvxStoreF; 20],
}

impl Default for AvxDct2Butterfly243f {
    fn default() -> Self {
        let rotation_layer = unsafe { dct2_radix_n_rotation_twiddles_avx_f(3, 243 / 3, 243) };

        // Precompute cosine twiddles for even components
        // Stored as Complex{re: even_twiddle, im: odd_twiddle} for cache efficiency
        let cos_twiddles = unsafe { dct2_radix_n_cos_twiddles_avx_f(3, 243 / 3, 243) };
        Self {
            bf81: f32::dct2_butterfly81(),
            rotation_layer: rotation_layer.try_into().unwrap(),
            cos_twiddles: cos_twiddles.try_into().unwrap(),
        }
    }
}

impl AvxDct2Butterfly243f {
    #[inline]
    #[target_feature(enable = "avx2", enable = "fma")]
    fn exec<S: BidirectionalStore<f32>>(
        &self,
        data: &mut S,
        a_buffer: &mut [f32; 81],
        c_buffer: &mut [f32; 81],
        s_buffer: &mut [f32; 81],
    ) {
        for n in 0..81 {
            a_buffer[n] = data[n * 3 + 1];
        }

        _ = self.bf81.execute(a_buffer);

        let mut sign = f32::one();
        for n in 0..81 {
            let u0 = data[3 * n];
            let u1 = data[3 * n + 3 - 1];

            c_buffer[n] = u0 + u1;
            s_buffer[n] = (u0 - u1).mulsign(sign);

            sign = -sign;
        }

        _ = self.bf81.execute(c_buffer);
        _ = self.bf81.execute(s_buffer);

        let q_modules = 81;

        {
            // Step 3: Recombine transformed buffers with twiddle factors

            // Handle k=0 case (DC and low frequencies)
            let qc = c_buffer[0];
            let c0 = qc; // Component C₀ (position 0)
            let c1 = qc * -f32::HALF; // Component C₂ (position 2, uses j=2)

            let s0_twiddled = s_buffer[0];

            let s0 = s0_twiddled * f32::SIN2PI_OVER_3;

            // Write output: C₀ (pos 0), S₁
            let a0 = a_buffer[0];
            let dc = c0 + a0;

            data[0] = dc;
            data[q_modules] = s0;

            let qid2 = -(c1 + a0); // negated 2j
            data[q_modules * 2] = qid2;

            let mut k = 1usize;
            let mut uk = 0usize;

            // Step 4: Handle k≥1 cases with rotation twiddles
            while k + 8 <= q_modules {
                // Apply rotation twiddles to combine forward and inverted components
                let rotation_twiddle_re = self.rotation_layer[uk];
                let rotation_twiddle_im = self.rotation_layer[uk + 1];

                let c_forward = AvxStoreF::load(&c_buffer[k..]);
                let s_forward = AvxStoreF::load(&s_buffer[q_modules - k - 7..]).reverse();

                let rotated_dc = fma(s_forward, rotation_twiddle_re, c_forward);

                let twiddle_re = self.cos_twiddles[uk];
                let twiddle_im = self.cos_twiddles[uk + 1];

                let twiddled_dc = rotated_dc * twiddle_re;

                let dc0 = twiddled_dc;
                let mut dc2 = -twiddled_dc * f32::HALF;

                let rotated_ds = fma(c_forward, rotation_twiddle_im, s_forward);

                let twiddled_ds = rotated_ds * twiddle_im;

                let ds1 = twiddled_ds * f32::SIN2PI_OVER_3;

                let a0 = AvxStoreF::load(&a_buffer[k..]);
                let dc = dc0 + a0;
                dc.write(data.slice_from_mut(k..));

                let dss1 = AvxStoreF::f32_mul_add(2., ds1, -dc);
                let q = dss1.reverse();
                q.write(data.slice_from_mut(q_modules * 2 - k - 7..));

                dc2 = -(dc2 + a0); // negated 2j
                dc2 = AvxStoreF::f32_mul_add(2., dc2, -dss1);
                dc2.write(data.slice_from_mut(q_modules * 2 + k..));

                k += 8;
                uk += 2;
            }
        }
    }
}

impl AvxDct2Butterfly243f {
    #[target_feature(enable = "avx2", enable = "fma")]
    fn execute_impl(&self, data: &mut [f32]) -> Result<(), PxdctError> {
        if !data.len().is_multiple_of(243) {
            return Err(PxdctError::InvalidSizeMultiplier(data.len(), self.length()));
        }

        let mut a_buffer = [f32::zero(); 81];
        let mut b_buffer = [f32::zero(); 81];
        let mut c_buffer = [f32::zero(); 81];

        for chunk in data.chunks_exact_mut(243) {
            self.exec(
                &mut InPlaceStore::new(chunk),
                &mut a_buffer,
                &mut c_buffer,
                &mut b_buffer,
            );
        }
        Ok(())
    }

    #[target_feature(enable = "avx2", enable = "fma")]
    fn execute_into_impl(&self, input: &[f32], output: &mut [f32]) -> Result<(), PxdctError> {
        use crate::util::validate_oof_sizes;
        validate_oof_sizes!(input, output, 243);

        let mut a_buffer = [f32::zero(); 81];
        let mut b_buffer = [f32::zero(); 81];
        let mut c_buffer = [f32::zero(); 81];

        use crate::bidirectional::BiStore;
        for (src, dst) in input.chunks_exact(243).zip(output.chunks_exact_mut(243)) {
            self.exec(
                &mut BiStore::new(src, dst),
                &mut a_buffer,
                &mut c_buffer,
                &mut b_buffer,
            );
        }
        Ok(())
    }
}

impl PxdctExecutor<f32> for AvxDct2Butterfly243f {
    fn execute(&self, data: &mut [f32]) -> Result<(), PxdctError> {
        unsafe { self.execute_impl(data) }
    }

    fn execute_with_scratch(&self, data: &mut [f32], _: &mut [f32]) -> Result<(), PxdctError> {
        unsafe { self.execute_impl(data) }
    }

    fn execute_into(&self, input: &[f32], output: &mut [f32]) -> Result<(), PxdctError> {
        unsafe { self.execute_into_impl(input, output) }
    }

    fn execute_into_with_scratch(
        &self,
        input: &[f32],
        output: &mut [f32],
        _: &mut [f32],
    ) -> Result<(), PxdctError> {
        unsafe { self.execute_into_impl(input, output) }
    }

    fn length(&self) -> usize {
        243
    }

    fn scratch_size(&self) -> usize {
        0
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::PxdctExecutor;
    use crate::avx::dct2_bf_power2::gen_test_avx_butterfly;
    use crate::tests::{naive_dct2, naive_dct2_f32};

    gen_test_avx_butterfly!(test_bf9, AvxDct2Butterfly9, 9, 1e-7, naive_dct2);
    gen_test_avx_butterfly!(test_bf27, AvxDct2Butterfly27f, 27, 1e-3, naive_dct2_f32);
    gen_test_avx_butterfly!(test_bf81, AvxDct2Butterfly81f, 81, 1e-3, naive_dct2_f32);
    gen_test_avx_butterfly!(test_bf243, AvxDct2Butterfly243f, 243, 1e-3, naive_dct2_f32);
}