pxdct 0.3.2

/*
 * // Copyright (c) Radzivon Bartoshyk 1/2026. All rights reserved.
 * //
 * // Redistribution and use in source and binary forms, with or without modification,
 * // are permitted provided that the following conditions are met:
 * //
 * // 1.  Redistributions of source code must retain the above copyright notice, this
 * // list of conditions and the following disclaimer.
 * //
 * // 2.  Redistributions in binary form must reproduce the above copyright notice,
 * // this list of conditions and the following disclaimer in the documentation
 * // and/or other materials provided with the distribution.
 * //
 * // 3.  Neither the name of the copyright holder nor the names of its
 * // contributors may be used to endorse or promote products derived from
 * // this software without specific prior written permission.
 * //
 * // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * // DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
 * // FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * // DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * // SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
 * // CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
 * // OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 * // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
use crate::avx::util::fma;
use crate::bidirectional::{BidirectionalStore, InPlaceStore};
use crate::dct4::{Dct4MixedRadix7Sample, Dct4MixedRadix11Sample, Dct4MixedRadix13Sample};
use crate::util::DctSample;
use crate::{PxdctError, PxdctExecutor};
use num_traits::AsPrimitive;
use std::marker::PhantomData;

#[derive(Debug, Clone)]
pub(crate) struct AvxDct4Butterfly7<T> {
    _phantom: PhantomData<T>,
}

impl<T: DctSample> Default for AvxDct4Butterfly7<T>
where
    f64: AsPrimitive<T>,
{
    fn default() -> Self {
        Self {
            _phantom: PhantomData,
        }
    }
}

impl<T: DctSample + Dct4MixedRadix7Sample> AvxDct4Butterfly7<T> {
    #[inline(always)]
    pub(crate) fn exec<S: BidirectionalStore<T>>(&self, data: &mut S) {
        // This is autogenerated Radix-Q where Q = 7.
        let a_buffer = [data[3] * T::FRAC_1_SQRT_2];
        let c_buffer = [
            (data[0] + data[6]) * T::FRAC_1_SQRT_2,
            (data[1] + data[5]) * T::FRAC_1_SQRT_2,
            (data[2] + data[4]) * T::FRAC_1_SQRT_2,
        ];
        let s_buffer = [
            (data[0] - data[6]) * T::FRAC_1_SQRT_2,
            (data[1] - data[5]) * T::FRAC_1_SQRT_2,
            (data[2] - data[4]) * T::FRAC_1_SQRT_2,
        ];
        let c0 = c_buffer[0];
        let s0 = s_buffer[0];
        let s0_tw0 = s0 * T::D4_R7_ROT_TWIDDLE_1;
        let s0_tw1 = -s0 * T::D4_R7_ROT_TWIDDLE_0;
        let iq0 = fma(c0, T::D4_R7_ROT_TWIDDLE_0, s0_tw0);
        let siq0 = fma(c0, T::D4_R7_ROT_TWIDDLE_1, s0_tw1);
        let mut u0 = iq0;
        let mut u1 = u0;
        let mut v0 = siq0;
        u1 *= T::D4_R7_ROT_TWIDDLE_2;
        v0 *= T::D4_R7_ROT_TWIDDLE_3;
        let c1 = c_buffer[1];
        let s1 = s_buffer[1];
        let iq1 = fma(c1, T::D4_R7_ROT_TWIDDLE_2, s1 * T::D4_R7_ROT_TWIDDLE_3);
        let siq1 = fma(c1, T::D4_R7_ROT_TWIDDLE_3, -s1 * T::D4_R7_ROT_TWIDDLE_2);
        u0 += iq1;
        u1 = fma(iq1, T::D4_R7_ROT_TWIDDLE_4, u1);
        v0 = fma(siq1, T::D4_R7_ROT_TWIDDLE_5, v0);
        let c2 = c_buffer[2];
        let s2 = s_buffer[2];
        let iq2 = fma(c2, T::D4_R7_ROT_TWIDDLE_5, s2 * T::D4_R7_ROT_TWIDDLE_4);
        let siq2 = fma(c2, T::D4_R7_ROT_TWIDDLE_4, -s2 * T::D4_R7_ROT_TWIDDLE_5);
        u0 += iq2;
        u1 = fma(iq2, -T::D4_R7_ROT_TWIDDLE_1, u1);
        v0 = fma(siq2, T::D4_R7_ROT_TWIDDLE_0, v0);
        let a0 = a_buffer[0];
        u0 += a0;
        u1 = u1 - a0;
        data[0] = u0;
        data[1] = u1 - v0;
        data[2] = u1 + v0;
        let mut u2 = iq0;
        let mut v2 = siq0;
        u2 *= T::D4_R7_ROT_TWIDDLE_1;
        v2 *= T::D4_R7_ROT_TWIDDLE_0;
        u2 = fma(iq1, -T::D4_R7_ROT_TWIDDLE_2, u2);
        v2 = fma(siq1, T::D4_R7_ROT_TWIDDLE_3, v2);
        u2 = fma(iq2, -T::D4_R7_ROT_TWIDDLE_4, u2);
        v2 = fma(siq2, -T::D4_R7_ROT_TWIDDLE_5, v2);
        u2 += a0;
        data[3] = u2 - v2;
        data[4] = u2 + v2;
        let mut u3 = iq0;
        let mut v3 = siq0;
        u3 *= T::D4_R7_ROT_TWIDDLE_4;
        v3 *= T::D4_R7_ROT_TWIDDLE_5;
        u3 = fma(iq1, -T::D4_R7_ROT_TWIDDLE_1, u3);
        v3 = fma(siq1, -T::D4_R7_ROT_TWIDDLE_0, v3);
        u3 = fma(iq2, T::D4_R7_ROT_TWIDDLE_2, u3);
        v3 = fma(siq2, T::D4_R7_ROT_TWIDDLE_3, v3);
        u3 = u3 - a0;
        data[5] = u3 - v3;
        data[6] = u3 + v3;
    }
}

impl<T: DctSample + Dct4MixedRadix7Sample> AvxDct4Butterfly7<T>
where
    f64: AsPrimitive<T>,
{
    #[target_feature(enable = "avx2", enable = "fma")]
    fn execute_impl(&self, data: &mut [T]) -> Result<(), PxdctError> {
        if !data.len().is_multiple_of(7) {
            return Err(PxdctError::InvalidSizeMultiplier(data.len(), self.length()));
        }
        for chunk in data.chunks_exact_mut(7) {
            self.exec(&mut InPlaceStore::new(chunk));
        }
        Ok(())
    }

    #[target_feature(enable = "avx2", enable = "fma")]
    fn execute_into_impl(&self, input: &[T], output: &mut [T]) -> Result<(), PxdctError> {
        use crate::util::validate_oof_sizes;
        validate_oof_sizes!(input, output, 7);
        use crate::bidirectional::BiStore;
        for (src, dst) in input.chunks_exact(7).zip(output.chunks_exact_mut(7)) {
            self.exec(&mut BiStore::new(src, dst));
        }
        Ok(())
    }
}

impl<T: DctSample + Dct4MixedRadix7Sample> PxdctExecutor<T> for AvxDct4Butterfly7<T>
where
    f64: AsPrimitive<T>,
{
    fn execute(&self, data: &mut [T]) -> Result<(), PxdctError> {
        unsafe { self.execute_impl(data) }
    }

    fn execute_with_scratch(&self, data: &mut [T], _: &mut [T]) -> Result<(), PxdctError> {
        unsafe { self.execute_impl(data) }
    }

    fn execute_into(&self, input: &[T], output: &mut [T]) -> Result<(), PxdctError> {
        unsafe { self.execute_into_impl(input, output) }
    }

    fn execute_into_with_scratch(
        &self,
        input: &[T],
        output: &mut [T],
        _: &mut [T],
    ) -> Result<(), PxdctError> {
        unsafe { self.execute_into_impl(input, output) }
    }

    fn length(&self) -> usize {
        7
    }

    fn scratch_size(&self) -> usize {
        0
    }
}

#[derive(Debug, Clone)]
pub(crate) struct AvxDct4Butterfly11<T> {
    _phantom: PhantomData<T>,
}

impl<T: DctSample> Default for AvxDct4Butterfly11<T>
where
    f64: AsPrimitive<T>,
{
    fn default() -> Self {
        Self {
            _phantom: PhantomData,
        }
    }
}

impl<T: DctSample + Dct4MixedRadix11Sample> AvxDct4Butterfly11<T> {
    #[inline(always)]
    pub(crate) fn exec<S: BidirectionalStore<T>>(&self, data: &mut S) {
        // This is autogenerated Radix-Q where Q = 11.
        let a_buffer = [data[5] * T::FRAC_1_SQRT_2];
        let c_buffer = [
            (data[0] + data[10]) * T::FRAC_1_SQRT_2,
            (data[1] + data[9]) * T::FRAC_1_SQRT_2,
            (data[2] + data[8]) * T::FRAC_1_SQRT_2,
            (data[3] + data[7]) * T::FRAC_1_SQRT_2,
            (data[4] + data[6]) * T::FRAC_1_SQRT_2,
        ];
        let s_buffer = [
            (data[0] - data[10]) * T::FRAC_1_SQRT_2,
            (data[1] - data[9]) * T::FRAC_1_SQRT_2,
            (data[2] - data[8]) * T::FRAC_1_SQRT_2,
            (data[3] - data[7]) * T::FRAC_1_SQRT_2,
            (data[4] - data[6]) * T::FRAC_1_SQRT_2,
        ];
        let c0 = c_buffer[0];
        let s0 = s_buffer[0];
        let s0_tw0 = s0 * T::D4_R11_ROT_TWIDDLE_1;
        let s0_tw1 = -s0 * T::D4_R11_ROT_TWIDDLE_0;
        let iq0 = fma(c0, T::D4_R11_ROT_TWIDDLE_0, s0_tw0);
        let siq0 = fma(c0, T::D4_R11_ROT_TWIDDLE_1, s0_tw1);
        let mut u0 = iq0;
        let mut u1 = u0;
        let mut v0 = siq0;
        u1 *= T::D4_R11_ROT_TWIDDLE_2;
        v0 *= T::D4_R11_ROT_TWIDDLE_3;
        let c1 = c_buffer[1];
        let s1 = s_buffer[1];
        let iq1 = fma(c1, T::D4_R11_ROT_TWIDDLE_4, s1 * T::D4_R11_ROT_TWIDDLE_5);
        let siq1 = fma(c1, T::D4_R11_ROT_TWIDDLE_5, -s1 * T::D4_R11_ROT_TWIDDLE_4);
        u0 += iq1;
        u1 = fma(iq1, T::D4_R11_ROT_TWIDDLE_1, u1);
        v0 = fma(siq1, T::D4_R11_ROT_TWIDDLE_0, v0);
        let c2 = c_buffer[2];
        let s2 = s_buffer[2];
        let iq2 = fma(c2, T::D4_R11_ROT_TWIDDLE_6, s2 * T::D4_R11_ROT_TWIDDLE_7);
        let siq2 = fma(c2, T::D4_R11_ROT_TWIDDLE_7, -s2 * T::D4_R11_ROT_TWIDDLE_6);
        u0 += iq2;
        u1 = fma(iq2, T::D4_R11_ROT_TWIDDLE_8, u1);
        v0 = fma(siq2, T::D4_R11_ROT_TWIDDLE_9, v0);
        let c3 = c_buffer[3];
        let s3 = s_buffer[3];
        let iq3 = fma(c3, T::D4_R11_ROT_TWIDDLE_2, s3 * T::D4_R11_ROT_TWIDDLE_3);
        let siq3 = fma(c3, T::D4_R11_ROT_TWIDDLE_3, -s3 * T::D4_R11_ROT_TWIDDLE_2);
        u0 += iq3;
        u1 = fma(iq3, -T::D4_R11_ROT_TWIDDLE_7, u1);
        v0 = fma(siq3, T::D4_R11_ROT_TWIDDLE_6, v0);
        let c4 = c_buffer[4];
        let s4 = s_buffer[4];
        let iq4 = fma(c4, T::D4_R11_ROT_TWIDDLE_9, s4 * T::D4_R11_ROT_TWIDDLE_8);
        let siq4 = fma(c4, T::D4_R11_ROT_TWIDDLE_8, -s4 * T::D4_R11_ROT_TWIDDLE_9);
        u0 += iq4;
        u1 = fma(iq4, -T::D4_R11_ROT_TWIDDLE_4, u1);
        v0 = fma(siq4, T::D4_R11_ROT_TWIDDLE_5, v0);
        let a0 = a_buffer[0];
        u0 += a0;
        u1 = u1 - a0;
        data[0] = u0;
        data[1] = u1 - v0;
        data[2] = u1 + v0;
        let mut u2 = iq0;
        let mut v2 = siq0;
        u2 *= T::D4_R11_ROT_TWIDDLE_4;
        v2 *= T::D4_R11_ROT_TWIDDLE_5;
        u2 = fma(iq1, -T::D4_R11_ROT_TWIDDLE_8, u2);
        v2 = fma(siq1, T::D4_R11_ROT_TWIDDLE_9, v2);
        u2 = fma(iq2, -T::D4_R11_ROT_TWIDDLE_2, u2);
        v2 = fma(siq2, T::D4_R11_ROT_TWIDDLE_3, v2);
        u2 = fma(iq3, -T::D4_R11_ROT_TWIDDLE_1, u2);
        v2 = fma(siq3, -T::D4_R11_ROT_TWIDDLE_0, v2);
        u2 = fma(iq4, T::D4_R11_ROT_TWIDDLE_7, u2);
        v2 = fma(siq4, -T::D4_R11_ROT_TWIDDLE_6, v2);
        u2 += a0;
        data[3] = u2 - v2;
        data[4] = u2 + v2;
        let mut u3 = iq0;
        let mut v3 = siq0;
        u3 *= T::D4_R11_ROT_TWIDDLE_1;
        v3 *= T::D4_R11_ROT_TWIDDLE_0;
        u3 = fma(iq1, -T::D4_R11_ROT_TWIDDLE_4, u3);
        v3 = fma(siq1, T::D4_R11_ROT_TWIDDLE_5, v3);
        u3 = fma(iq2, -T::D4_R11_ROT_TWIDDLE_7, u3);
        v3 = fma(siq2, -T::D4_R11_ROT_TWIDDLE_6, v3);
        u3 = fma(iq3, T::D4_R11_ROT_TWIDDLE_2, u3);
        v3 = fma(siq3, -T::D4_R11_ROT_TWIDDLE_3, v3);
        u3 = fma(iq4, T::D4_R11_ROT_TWIDDLE_8, u3);
        v3 = fma(siq4, T::D4_R11_ROT_TWIDDLE_9, v3);
        u3 = u3 - a0;
        data[5] = u3 - v3;
        data[6] = u3 + v3;
        let mut u4 = iq0;
        let mut v4 = siq0;
        u4 *= T::D4_R11_ROT_TWIDDLE_7;
        v4 *= T::D4_R11_ROT_TWIDDLE_6;
        u4 = fma(iq1, -T::D4_R11_ROT_TWIDDLE_2, u4);
        v4 = fma(siq1, -T::D4_R11_ROT_TWIDDLE_3, v4);
        u4 = fma(iq2, T::D4_R11_ROT_TWIDDLE_4, u4);
        v4 = fma(siq2, -T::D4_R11_ROT_TWIDDLE_5, v4);
        u4 = fma(iq3, -T::D4_R11_ROT_TWIDDLE_8, u4);
        v4 = fma(siq3, T::D4_R11_ROT_TWIDDLE_9, v4);
        u4 = fma(iq4, -T::D4_R11_ROT_TWIDDLE_1, u4);
        v4 = fma(siq4, -T::D4_R11_ROT_TWIDDLE_0, v4);
        u4 += a0;
        data[7] = u4 - v4;
        data[8] = u4 + v4;
        let mut u5 = iq0;
        let mut v5 = siq0;
        u5 *= T::D4_R11_ROT_TWIDDLE_8;
        v5 *= T::D4_R11_ROT_TWIDDLE_9;
        u5 = fma(iq1, -T::D4_R11_ROT_TWIDDLE_7, u5);
        v5 = fma(siq1, -T::D4_R11_ROT_TWIDDLE_6, v5);
        u5 = fma(iq2, T::D4_R11_ROT_TWIDDLE_1, u5);
        v5 = fma(siq2, T::D4_R11_ROT_TWIDDLE_0, v5);
        u5 = fma(iq3, -T::D4_R11_ROT_TWIDDLE_4, u5);
        v5 = fma(siq3, -T::D4_R11_ROT_TWIDDLE_5, v5);
        u5 = fma(iq4, T::D4_R11_ROT_TWIDDLE_2, u5);
        v5 = fma(siq4, T::D4_R11_ROT_TWIDDLE_3, v5);
        u5 = u5 - a0;
        data[9] = u5 - v5;
        data[10] = u5 + v5;
    }
}

impl<T: DctSample + Dct4MixedRadix11Sample> AvxDct4Butterfly11<T>
where
    f64: AsPrimitive<T>,
{
    #[target_feature(enable = "avx2", enable = "fma")]
    fn execute_impl(&self, data: &mut [T]) -> Result<(), PxdctError> {
        if !data.len().is_multiple_of(11) {
            return Err(PxdctError::InvalidSizeMultiplier(data.len(), self.length()));
        }
        for chunk in data.chunks_exact_mut(11) {
            self.exec(&mut InPlaceStore::new(chunk));
        }
        Ok(())
    }

    #[target_feature(enable = "avx2", enable = "fma")]
    fn execute_into_impl(&self, input: &[T], output: &mut [T]) -> Result<(), PxdctError> {
        use crate::util::validate_oof_sizes;
        validate_oof_sizes!(input, output, 11);
        use crate::bidirectional::BiStore;
        for (src, dst) in input.chunks_exact(11).zip(output.chunks_exact_mut(11)) {
            self.exec(&mut BiStore::new(src, dst));
        }
        Ok(())
    }
}

impl<T: DctSample + Dct4MixedRadix11Sample> PxdctExecutor<T> for AvxDct4Butterfly11<T>
where
    f64: AsPrimitive<T>,
{
    fn execute(&self, data: &mut [T]) -> Result<(), PxdctError> {
        unsafe { self.execute_impl(data) }
    }

    fn execute_with_scratch(&self, data: &mut [T], _: &mut [T]) -> Result<(), PxdctError> {
        unsafe { self.execute_impl(data) }
    }

    fn execute_into(&self, input: &[T], output: &mut [T]) -> Result<(), PxdctError> {
        unsafe { self.execute_into_impl(input, output) }
    }

    fn execute_into_with_scratch(
        &self,
        input: &[T],
        output: &mut [T],
        _: &mut [T],
    ) -> Result<(), PxdctError> {
        unsafe { self.execute_into_impl(input, output) }
    }

    fn length(&self) -> usize {
        11
    }

    fn scratch_size(&self) -> usize {
        0
    }
}

#[derive(Debug, Clone)]
pub(crate) struct AvxDct4Butterfly13<T> {
    _phantom: PhantomData<T>,
}

impl<T: DctSample> Default for AvxDct4Butterfly13<T>
where
    f64: AsPrimitive<T>,
{
    fn default() -> Self {
        Self {
            _phantom: PhantomData,
        }
    }
}

impl<T: DctSample + Dct4MixedRadix13Sample> AvxDct4Butterfly13<T> {
    #[inline(always)]
    pub(crate) fn exec<S: BidirectionalStore<T>>(&self, data: &mut S) {
        // This is autogenerated Radix-Q where Q = 13.
        let a_buffer = [data[6] * T::FRAC_1_SQRT_2];
        let c_buffer = [
            (data[0] + data[12]) * T::FRAC_1_SQRT_2,
            (data[1] + data[11]) * T::FRAC_1_SQRT_2,
            (data[2] + data[10]) * T::FRAC_1_SQRT_2,
            (data[3] + data[9]) * T::FRAC_1_SQRT_2,
            (data[4] + data[8]) * T::FRAC_1_SQRT_2,
            (data[5] + data[7]) * T::FRAC_1_SQRT_2,
        ];
        let s_buffer = [
            (data[0] - data[12]) * T::FRAC_1_SQRT_2,
            (data[1] - data[11]) * T::FRAC_1_SQRT_2,
            (data[2] - data[10]) * T::FRAC_1_SQRT_2,
            (data[3] - data[9]) * T::FRAC_1_SQRT_2,
            (data[4] - data[8]) * T::FRAC_1_SQRT_2,
            (data[5] - data[7]) * T::FRAC_1_SQRT_2,
        ];
        let c0 = c_buffer[0];
        let s0 = s_buffer[0];
        let s0_tw0 = s0 * T::D4_R13_ROT_TWIDDLE_1;
        let s0_tw1 = -s0 * T::D4_R13_ROT_TWIDDLE_0;
        let iq0 = fma(c0, T::D4_R13_ROT_TWIDDLE_0, s0_tw0);
        let siq0 = fma(c0, T::D4_R13_ROT_TWIDDLE_1, s0_tw1);
        let mut u0 = iq0;
        let mut u1 = u0;
        let mut v0 = siq0;
        u1 *= T::D4_R13_ROT_TWIDDLE_2;
        v0 *= T::D4_R13_ROT_TWIDDLE_3;
        let c1 = c_buffer[1];
        let s1 = s_buffer[1];
        let iq1 = fma(c1, T::D4_R13_ROT_TWIDDLE_4, s1 * T::D4_R13_ROT_TWIDDLE_5);
        let siq1 = fma(c1, T::D4_R13_ROT_TWIDDLE_5, -s1 * T::D4_R13_ROT_TWIDDLE_4);
        u0 += iq1;
        u1 = fma(iq1, T::D4_R13_ROT_TWIDDLE_0, u1);
        v0 = fma(siq1, T::D4_R13_ROT_TWIDDLE_1, v0);
        let c2 = c_buffer[2];
        let s2 = s_buffer[2];
        let iq2 = fma(c2, T::D4_R13_ROT_TWIDDLE_6, s2 * T::D4_R13_ROT_TWIDDLE_7);
        let siq2 = fma(c2, T::D4_R13_ROT_TWIDDLE_7, -s2 * T::D4_R13_ROT_TWIDDLE_6);
        u0 += iq2;
        u1 = fma(iq2, T::D4_R13_ROT_TWIDDLE_8, u1);
        v0 = fma(siq2, T::D4_R13_ROT_TWIDDLE_9, v0);
        let c3 = c_buffer[3];
        let s3 = s_buffer[3];
        let iq3 = fma(c3, T::D4_R13_ROT_TWIDDLE_9, s3 * T::D4_R13_ROT_TWIDDLE_8);
        let siq3 = fma(c3, T::D4_R13_ROT_TWIDDLE_8, -s3 * T::D4_R13_ROT_TWIDDLE_9);
        u0 += iq3;
        u1 = fma(iq3, T::D4_R13_ROT_TWIDDLE_10, u1);
        v0 = fma(siq3, T::D4_R13_ROT_TWIDDLE_11, v0);
        let c4 = c_buffer[4];
        let s4 = s_buffer[4];
        let iq4 = fma(c4, T::D4_R13_ROT_TWIDDLE_2, s4 * T::D4_R13_ROT_TWIDDLE_3);
        let siq4 = fma(c4, T::D4_R13_ROT_TWIDDLE_3, -s4 * T::D4_R13_ROT_TWIDDLE_2);
        u0 += iq4;
        u1 = fma(iq4, -T::D4_R13_ROT_TWIDDLE_5, u1);
        v0 = fma(siq4, T::D4_R13_ROT_TWIDDLE_4, v0);
        let c5 = c_buffer[5];
        let s5 = s_buffer[5];
        let iq5 = fma(c5, T::D4_R13_ROT_TWIDDLE_11, s5 * -T::D4_R13_ROT_TWIDDLE_10);
        let siq5 = fma(
            c5,
            -T::D4_R13_ROT_TWIDDLE_10,
            -s5 * T::D4_R13_ROT_TWIDDLE_11,
        );
        u0 += iq5;
        u1 = fma(iq5, -T::D4_R13_ROT_TWIDDLE_6, u1);
        v0 = fma(siq5, T::D4_R13_ROT_TWIDDLE_7, v0);
        let a0 = a_buffer[0];
        u0 += a0;
        u1 = u1 - a0;
        data[0] = u0;
        data[1] = u1 - v0;
        data[2] = u1 + v0;
        let mut u2 = iq0;
        let mut v2 = siq0;
        u2 *= T::D4_R13_ROT_TWIDDLE_6;
        v2 *= T::D4_R13_ROT_TWIDDLE_7;
        u2 = fma(iq1, -T::D4_R13_ROT_TWIDDLE_10, u2);
        v2 = fma(siq1, T::D4_R13_ROT_TWIDDLE_11, v2);
        u2 = fma(iq2, -T::D4_R13_ROT_TWIDDLE_0, u2);
        v2 = fma(siq2, T::D4_R13_ROT_TWIDDLE_1, v2);
        u2 = fma(iq3, -T::D4_R13_ROT_TWIDDLE_2, u2);
        v2 = fma(siq3, -T::D4_R13_ROT_TWIDDLE_3, v2);
        u2 = fma(iq4, -T::D4_R13_ROT_TWIDDLE_8, u2);
        v2 = fma(siq4, -T::D4_R13_ROT_TWIDDLE_9, v2);
        u2 = fma(iq5, T::D4_R13_ROT_TWIDDLE_5, u2);
        v2 = fma(siq5, -T::D4_R13_ROT_TWIDDLE_4, v2);
        u2 += a0;
        data[3] = u2 - v2;
        data[4] = u2 + v2;
        let mut u3 = iq0;
        let mut v3 = siq0;
        u3 *= T::D4_R13_ROT_TWIDDLE_0;
        v3 *= T::D4_R13_ROT_TWIDDLE_1;
        u3 = fma(iq1, -T::D4_R13_ROT_TWIDDLE_5, u3);
        v3 = fma(siq1, T::D4_R13_ROT_TWIDDLE_4, v3);
        u3 = fma(iq2, -T::D4_R13_ROT_TWIDDLE_6, u3);
        v3 = fma(siq2, -T::D4_R13_ROT_TWIDDLE_7, v3);
        u3 = fma(iq3, T::D4_R13_ROT_TWIDDLE_8, u3);
        v3 = fma(siq3, -T::D4_R13_ROT_TWIDDLE_9, v3);
        u3 = fma(iq4, T::D4_R13_ROT_TWIDDLE_2, u3);
        v3 = fma(siq4, T::D4_R13_ROT_TWIDDLE_3, v3);
        u3 = fma(iq5, T::D4_R13_ROT_TWIDDLE_10, u3);
        v3 = fma(siq5, T::D4_R13_ROT_TWIDDLE_11, v3);
        u3 = u3 - a0;
        data[5] = u3 - v3;
        data[6] = u3 + v3;
        let mut u4 = iq0;
        let mut v4 = siq0;
        u4 *= T::D4_R13_ROT_TWIDDLE_5;
        v4 *= T::D4_R13_ROT_TWIDDLE_4;
        u4 = fma(iq1, -T::D4_R13_ROT_TWIDDLE_2, u4);
        v4 = fma(siq1, T::D4_R13_ROT_TWIDDLE_3, v4);
        u4 = fma(iq2, -T::D4_R13_ROT_TWIDDLE_10, u4);
        v4 = fma(siq2, -T::D4_R13_ROT_TWIDDLE_11, v4);
        u4 = fma(iq3, T::D4_R13_ROT_TWIDDLE_6, u4);
        v4 = fma(siq3, T::D4_R13_ROT_TWIDDLE_7, v4);
        u4 = fma(iq4, -T::D4_R13_ROT_TWIDDLE_0, u4);
        v4 = fma(siq4, T::D4_R13_ROT_TWIDDLE_1, v4);
        u4 = fma(iq5, -T::D4_R13_ROT_TWIDDLE_8, u4);
        v4 = fma(siq5, -T::D4_R13_ROT_TWIDDLE_9, v4);
        u4 += a0;
        data[7] = u4 - v4;
        data[8] = u4 + v4;
        let mut u5 = iq0;
        let mut v5 = siq0;
        u5 *= T::D4_R13_ROT_TWIDDLE_8;
        v5 *= T::D4_R13_ROT_TWIDDLE_9;
        u5 = fma(iq1, -T::D4_R13_ROT_TWIDDLE_6, u5);
        v5 = fma(siq1, -T::D4_R13_ROT_TWIDDLE_7, v5);
        u5 = fma(iq2, T::D4_R13_ROT_TWIDDLE_2, u5);
        v5 = fma(siq2, -T::D4_R13_ROT_TWIDDLE_3, v5);
        u5 = fma(iq3, -T::D4_R13_ROT_TWIDDLE_5, u5);
        v5 = fma(siq3, T::D4_R13_ROT_TWIDDLE_4, v5);
        u5 = fma(iq4, T::D4_R13_ROT_TWIDDLE_10, u5);
        v5 = fma(siq4, -T::D4_R13_ROT_TWIDDLE_11, v5);
        u5 = fma(iq5, T::D4_R13_ROT_TWIDDLE_0, u5);
        v5 = fma(siq5, T::D4_R13_ROT_TWIDDLE_1, v5);
        u5 = u5 - a0;
        data[9] = u5 - v5;
        data[10] = u5 + v5;
        let mut u6 = iq0;
        let mut v6 = siq0;
        u6 *= -T::D4_R13_ROT_TWIDDLE_10;
        v6 *= T::D4_R13_ROT_TWIDDLE_11;
        u6 = fma(iq1, -T::D4_R13_ROT_TWIDDLE_8, u6);
        v6 = fma(siq1, -T::D4_R13_ROT_TWIDDLE_9, v6);
        u6 = fma(iq2, T::D4_R13_ROT_TWIDDLE_5, u6);
        v6 = fma(siq2, T::D4_R13_ROT_TWIDDLE_4, v6);
        u6 = fma(iq3, -T::D4_R13_ROT_TWIDDLE_0, u6);
        v6 = fma(siq3, -T::D4_R13_ROT_TWIDDLE_1, v6);
        u6 = fma(iq4, T::D4_R13_ROT_TWIDDLE_6, u6);
        v6 = fma(siq4, T::D4_R13_ROT_TWIDDLE_7, v6);
        u6 = fma(iq5, -T::D4_R13_ROT_TWIDDLE_2, u6);
        v6 = fma(siq5, -T::D4_R13_ROT_TWIDDLE_3, v6);
        u6 += a0;
        data[11] = u6 - v6;
        data[12] = u6 + v6;
    }
}

impl<T: DctSample + Dct4MixedRadix13Sample> AvxDct4Butterfly13<T>
where
    f64: AsPrimitive<T>,
{
    #[target_feature(enable = "avx2", enable = "fma")]
    fn execute_impl(&self, data: &mut [T]) -> Result<(), PxdctError> {
        if !data.len().is_multiple_of(13) {
            return Err(PxdctError::InvalidSizeMultiplier(data.len(), self.length()));
        }
        for chunk in data.chunks_exact_mut(13) {
            self.exec(&mut InPlaceStore::new(chunk));
        }
        Ok(())
    }

    #[target_feature(enable = "avx2", enable = "fma")]
    fn execute_into_impl(&self, input: &[T], output: &mut [T]) -> Result<(), PxdctError> {
        use crate::util::validate_oof_sizes;
        validate_oof_sizes!(input, output, 13);
        use crate::bidirectional::BiStore;
        for (src, dst) in input.chunks_exact(13).zip(output.chunks_exact_mut(13)) {
            self.exec(&mut BiStore::new(src, dst));
        }
        Ok(())
    }
}

impl<T: DctSample + Dct4MixedRadix13Sample> PxdctExecutor<T> for AvxDct4Butterfly13<T>
where
    f64: AsPrimitive<T>,
{
    fn execute(&self, data: &mut [T]) -> Result<(), PxdctError> {
        unsafe { self.execute_impl(data) }
    }

    fn execute_with_scratch(&self, data: &mut [T], _: &mut [T]) -> Result<(), PxdctError> {
        unsafe { self.execute_impl(data) }
    }

    fn execute_into(&self, input: &[T], output: &mut [T]) -> Result<(), PxdctError> {
        unsafe { self.execute_into_impl(input, output) }
    }

    fn execute_into_with_scratch(
        &self,
        input: &[T],
        output: &mut [T],
        _: &mut [T],
    ) -> Result<(), PxdctError> {
        unsafe { self.execute_into_impl(input, output) }
    }

    fn length(&self) -> usize {
        13
    }

    fn scratch_size(&self) -> usize {
        0
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::PxdctExecutor;
    use crate::avx::dct2_bf_power2::gen_test_avx_butterfly;
    use crate::tests::naive_dct4;

    gen_test_avx_butterfly!(test_bf_dct4_7, AvxDct4Butterfly7, 7, 1e-7, naive_dct4);
    gen_test_avx_butterfly!(test_bf_dct4_11, AvxDct4Butterfly11, 11, 1e-7, naive_dct4);
    gen_test_avx_butterfly!(test_bf_dct4_13, AvxDct4Butterfly13, 13, 1e-7, naive_dct4);
}