tenferro-internal-cpu-kernels 0.9.0

use super::*;

/// The Rust scalar type behind a preset variant name a macro received.
macro_rules! preset_scalar {
    (F32) => {
        f32
    };
    (F64) => {
        f64
    };
    (I32) => {
        i32
    };
    (I64) => {
        i64
    };
    (Bool) => {
        bool
    };
    (C32) => {
        num_complex::Complex32
    };
    (C64) => {
        num_complex::Complex64
    };
}
mod static_replay;

#[test]
fn materialize_tensor_read_covers_all_owned_dtypes() {
    macro_rules! check {
        ($variant:ident, $ty:ty, $value:expr) => {{
            let source = Tensor::from_typed::<preset_scalar!($variant)>(
                TypedTensor::<$ty>::from_vec_col_major(vec![1], vec![$value]).unwrap(),
            );
            let mut buffers = BufferPool::new();
            let materialized = crate::materialize_tensor_read(
                &mut buffers,
                "test",
                TensorRead::from_tensor(&source),
            )
            .unwrap();
            assert_eq!(materialized.dtype(), source.dtype());
        }};
    }

    check!(F32, f32, 1.0);
    check!(F64, f64, 2.0);
    check!(I32, i32, 3);
    check!(I64, i64, 4);
    check!(Bool, bool, true);
    check!(
        C32,
        num_complex::Complex32,
        num_complex::Complex32::new(5.0, 1.0)
    );
    check!(
        C64,
        num_complex::Complex64,
        num_complex::Complex64::new(6.0, 2.0)
    );
}

#[test]
fn integer_multiplication_wraps_owned_view_and_scalar_broadcast_paths() {
    let mut buffers = BufferPool::new();
    macro_rules! check {
        ($ty:ty, $variant:ident) => {{
            let max: $ty = <$ty>::MAX;
            let expected: $ty = -2;
            let scalar = TypedTensor::<$ty>::from_vec_col_major(vec![], vec![max]).unwrap();
            let other = TypedTensor::<$ty>::from_vec_col_major(vec![], vec![2]).unwrap();
            let owned = mul_read_with_pool(
                &mut buffers,
                TensorRead::from_tensor(&Tensor::from_typed::<preset_scalar!($variant)>(
                    scalar.duplicate().unwrap(),
                )),
                TensorRead::from_tensor(&Tensor::from_typed::<preset_scalar!($variant)>(
                    other.duplicate().unwrap(),
                )),
                Exec::serial(),
            )
            .unwrap();
            assert_eq!(owned.as_slice::<$ty>().unwrap(), &[expected]);
            let view = mul_read_with_pool(
                &mut buffers,
                TensorRead::from_view(TensorView::$variant(scalar.as_view())),
                TensorRead::from_view(TensorView::$variant(other.as_view())),
                Exec::serial(),
            )
            .unwrap();
            assert_eq!(view.as_slice::<$ty>().unwrap(), &[expected]);
            let broadcast = broadcast_multiply_read_with_pool(
                &mut buffers,
                TensorRead::from_tensor(&Tensor::from_typed::<preset_scalar!($variant)>(
                    scalar.duplicate().unwrap(),
                )),
                &[],
                &[],
                TensorRead::from_tensor(&Tensor::from_typed::<preset_scalar!($variant)>(other)),
                &[],
                &[],
                Exec::serial(),
            )
            .unwrap()
            .unwrap();
            assert_eq!(broadcast.as_slice::<$ty>().unwrap(), &[expected]);

            let vector = TypedTensor::<$ty>::from_vec_col_major(vec![3], vec![2, 3, 4]).unwrap();
            let expected_vector = [
                max.wrapping_mul(2),
                max.wrapping_mul(3),
                max.wrapping_mul(4),
            ];
            let owned_scalar_vector = mul_read_with_pool(
                &mut buffers,
                TensorRead::from_tensor(&Tensor::from_typed::<preset_scalar!($variant)>(
                    scalar.duplicate().unwrap(),
                )),
                TensorRead::from_tensor(&Tensor::from_typed::<preset_scalar!($variant)>(
                    vector.duplicate().unwrap(),
                )),
                Exec::serial(),
            )
            .unwrap();
            assert_eq!(owned_scalar_vector.shape(), &[3]);
            assert_eq!(
                owned_scalar_vector.as_slice::<$ty>().unwrap(),
                &expected_vector
            );
            let owned_vector_scalar = mul_read_with_pool(
                &mut buffers,
                TensorRead::from_tensor(&Tensor::from_typed::<preset_scalar!($variant)>(
                    vector.duplicate().unwrap(),
                )),
                TensorRead::from_tensor(&Tensor::from_typed::<preset_scalar!($variant)>(
                    scalar.duplicate().unwrap(),
                )),
                Exec::serial(),
            )
            .unwrap();
            assert_eq!(owned_vector_scalar.shape(), &[3]);
            assert_eq!(
                owned_vector_scalar.as_slice::<$ty>().unwrap(),
                &expected_vector
            );

            let view_scalar_vector = mul_read_with_pool(
                &mut buffers,
                TensorRead::from_view(TensorView::$variant(scalar.as_view())),
                TensorRead::from_view(TensorView::$variant(vector.as_view())),
                Exec::serial(),
            )
            .unwrap();
            assert_eq!(view_scalar_vector.shape(), &[3]);
            assert_eq!(
                view_scalar_vector.as_slice::<$ty>().unwrap(),
                &expected_vector
            );
            let view_vector_scalar = mul_read_with_pool(
                &mut buffers,
                TensorRead::from_view(TensorView::$variant(vector.as_view())),
                TensorRead::from_view(TensorView::$variant(scalar.as_view())),
                Exec::serial(),
            )
            .unwrap();
            assert_eq!(view_vector_scalar.shape(), &[3]);
            assert_eq!(
                view_vector_scalar.as_slice::<$ty>().unwrap(),
                &expected_vector
            );

            let broadcast_scalar_vector = broadcast_multiply_read_with_pool(
                &mut buffers,
                TensorRead::from_tensor(&Tensor::from_typed::<preset_scalar!($variant)>(
                    scalar.duplicate().unwrap(),
                )),
                &[3],
                &[],
                TensorRead::from_tensor(&Tensor::from_typed::<preset_scalar!($variant)>(
                    vector.duplicate().unwrap(),
                )),
                &[3],
                &[0],
                Exec::serial(),
            )
            .unwrap()
            .unwrap();
            assert_eq!(broadcast_scalar_vector.shape(), &[3]);
            assert_eq!(
                broadcast_scalar_vector.as_slice::<$ty>().unwrap(),
                &expected_vector
            );
            let broadcast_vector_scalar = broadcast_multiply_read_with_pool(
                &mut buffers,
                TensorRead::from_tensor(&Tensor::from_typed::<preset_scalar!($variant)>(vector)),
                &[3],
                &[0],
                TensorRead::from_tensor(&Tensor::from_typed::<preset_scalar!($variant)>(scalar)),
                &[3],
                &[],
                Exec::serial(),
            )
            .unwrap()
            .unwrap();
            assert_eq!(broadcast_vector_scalar.shape(), &[3]);
            assert_eq!(
                broadcast_vector_scalar.as_slice::<$ty>().unwrap(),
                &expected_vector
            );
        }};
    }
    check!(i32, I32);
    check!(i64, I64);
}

#[test]
fn broadcast_multiply_rank_n_outer_product_matches_reference() {
    let mut buffers = BufferPool::default();
    let lhs_data = [1.0_f64, 2.0, 3.0, 4.0, 5.0, 6.0];
    let rhs_data = [7.0_f64, 8.0, 9.0, 10.0];
    let lhs = TypedTensorView::from_slice([2, 3], [1, 2], 0, &lhs_data).unwrap();
    let rhs = TypedTensorView::from_slice([2, 2], [1, 2], 0, &rhs_data).unwrap();

    let out = broadcast_multiply_read_with_pool(
        &mut buffers,
        TensorRead::from_view(TensorView::F64(lhs)),
        &[2, 3, 2, 2],
        &[0, 1],
        TensorRead::from_view(TensorView::F64(rhs)),
        &[2, 3, 2, 2],
        &[2, 3],
        Exec::serial(),
    )
    .unwrap()
    .expect("f64 broadcast multiply is supported");

    assert_eq!(out.shape(), &[2, 3, 2, 2]);
    let expected: Vec<f64> = (0..2)
        .flat_map(|d| {
            (0..2).flat_map(move |c| {
                (0..3).flat_map(move |b| {
                    (0..2).map(move |a| lhs_data[a + 2 * b] * rhs_data[c + 2 * d])
                })
            })
        })
        .collect();
    assert_eq!(out.as_slice::<f64>().unwrap(), expected.as_slice());
}

#[test]
fn broadcast_multiply_shared_batch_outer_product_matches_reference() {
    let mut buffers = BufferPool::default();
    let lhs_data = [1.0_f64, 2.0, 3.0, 4.0, 5.0, 6.0];
    let rhs_data = [
        7.0_f64, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0,
    ];
    let lhs = TypedTensorView::from_slice([2, 3], [1, 2], 0, &lhs_data).unwrap();
    let rhs = TypedTensorView::from_slice([4, 3], [1, 4], 0, &rhs_data).unwrap();

    let out = broadcast_multiply_read_with_pool(
        &mut buffers,
        TensorRead::from_view(TensorView::F64(lhs)),
        &[2, 4, 3],
        &[0, 2],
        TensorRead::from_view(TensorView::F64(rhs)),
        &[2, 4, 3],
        &[1, 2],
        Exec::serial(),
    )
    .unwrap()
    .expect("f64 broadcast multiply is supported");

    assert_eq!(out.shape(), &[2, 4, 3]);
    let expected: Vec<f64> = (0..3)
        .flat_map(|t| {
            (0..4).flat_map(move |o| (0..2).map(move |j| lhs_data[j + 2 * t] * rhs_data[o + 4 * t]))
        })
        .collect();
    assert_eq!(out.as_slice::<f64>().unwrap(), expected.as_slice());
}

#[test]
fn broadcast_multiply_fallback_handles_permuted_elementwise_without_materialization() {
    let mut buffers = BufferPool::default();
    let lhs_data: Vec<f64> = (0..24).map(|i| (i + 1) as f64).collect();
    let rhs_data: Vec<f64> = (0..24).map(|i| (100 + i) as f64).collect();
    let lhs = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![2, 3, 4], lhs_data.clone()).unwrap(),
    );
    let rhs = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![4, 2, 3], rhs_data.clone()).unwrap(),
    );

    let out = broadcast_multiply_read_with_pool(
        &mut buffers,
        TensorRead::from_tensor(&lhs),
        &[2, 3, 4],
        &[0, 1, 2],
        TensorRead::from_tensor(&rhs),
        &[2, 3, 4],
        &[2, 0, 1],
        Exec::serial(),
    )
    .unwrap()
    .expect("same-rank permuted elementwise multiply should use fallback broadcast mul");

    let expected: Vec<f64> = (0..4)
        .flat_map(|k| {
            let lhs_data = &lhs_data;
            let rhs_data = &rhs_data;
            (0..3).flat_map(move |j| {
                (0..2).map(move |i| {
                    let lhs_offset = i + 2 * j + 6 * k;
                    let rhs_offset = k + 4 * i + 8 * j;
                    lhs_data[lhs_offset] * rhs_data[rhs_offset]
                })
            })
        })
        .collect();

    assert_eq!(out.shape(), &[2, 3, 4]);
    assert_eq!(out.as_slice::<f64>().unwrap(), expected.as_slice());
}

#[test]
fn broadcast_multiply_handles_scalar_full_output_pairs() {
    let mut buffers = BufferPool::default();
    let scalar =
        Tensor::from_typed::<f64>(TypedTensor::from_vec_col_major(vec![], vec![2.0]).unwrap());
    let vector = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![3], vec![1.0, 2.0, 3.0]).unwrap(),
    );

    let lhs_scalar = broadcast_multiply_read_with_pool(
        &mut buffers,
        TensorRead::from_tensor(&scalar),
        &[3],
        &[],
        TensorRead::from_tensor(&vector),
        &[3],
        &[0],
        Exec::serial(),
    )
    .unwrap()
    .expect("scalar lhs broadcast multiply should materialize");
    assert_eq!(lhs_scalar.as_slice::<f64>().unwrap(), &[2.0, 4.0, 6.0]);

    let rhs_scalar = broadcast_multiply_read_with_pool(
        &mut buffers,
        TensorRead::from_tensor(&vector),
        &[3],
        &[0],
        TensorRead::from_tensor(&scalar),
        &[3],
        &[],
        Exec::serial(),
    )
    .unwrap()
    .expect("scalar rhs broadcast multiply should materialize");
    assert_eq!(rhs_scalar.as_slice::<f64>().unwrap(), &[2.0, 4.0, 6.0]);

    let other_scalar =
        Tensor::from_typed::<f64>(TypedTensor::from_vec_col_major(vec![], vec![3.0]).unwrap());
    let both_scalar = broadcast_multiply_read_with_pool(
        &mut buffers,
        TensorRead::from_tensor(&scalar),
        &[3],
        &[],
        TensorRead::from_tensor(&other_scalar),
        &[3],
        &[],
        Exec::serial(),
    )
    .unwrap()
    .expect("scalar-scalar broadcast multiply should materialize");
    assert_eq!(both_scalar.as_slice::<f64>().unwrap(), &[6.0, 6.0, 6.0]);

    let complex_scalar = Tensor::from_typed::<tenferro_tensor::Complex64>(
        TypedTensor::from_vec_col_major(vec![], vec![c64(0.5, -1.5)]).unwrap(),
    );
    let complex_vector = Tensor::from_typed::<tenferro_tensor::Complex64>(
        TypedTensor::from_vec_col_major(vec![2], vec![c64(1.0, 2.0), c64(-3.0, 0.5)]).unwrap(),
    );
    let complex_value = broadcast_multiply_value_with_pool(
        &mut buffers,
        TensorRead::from_tensor(&complex_scalar),
        &[2],
        &[],
        TensorRead::from_tensor(&complex_vector),
        &[2],
        &[0],
        Exec::serial(),
    )
    .unwrap()
    .expect("complex scalar broadcast multiply should materialize");
    let complex_tensor =
        crate::materialize_tensor_read(&mut buffers, "test", complex_value.tensor_read()).unwrap();
    let complex_data = complex_tensor.as_slice::<Complex<f64>>().unwrap();
    assert_c64_close(complex_data[0], c64(3.5, -0.5));
    assert_c64_close(complex_data[1], c64(-0.75, 4.75));
}

#[test]
fn lazy_outer_product_lhs_prefix_preserves_logical_output_order() {
    let mut buffers = BufferPool::default();
    let lhs_data = [1.0_f64, 2.0, 3.0, 4.0, 5.0, 6.0];
    let rhs_data = [10.0_f64, 20.0, 30.0, 40.0];
    let lhs = TypedTensorView::from_slice([2, 3], [3, 1], 0, &lhs_data).unwrap();
    let rhs = TypedTensorView::from_slice([4], [1], 0, &rhs_data).unwrap();

    let out = try_lazy_outer_product_with_pool(
        &mut buffers,
        &lhs,
        &[2, 3, 4],
        &[0, 1],
        &rhs,
        &[2, 3, 4],
        &[2],
        Exec::serial(),
    )
    .unwrap()
    .expect("non-canonical lhs physical order should use lazy outer-product output");

    assert_ne!(
        out.strides.as_slice(),
        col_major_strides(&[2, 3, 4]).unwrap().as_slice()
    );
    let value = TensorValue::from_parts(
        Tensor::from_typed::<f64>(out.base),
        vec![2, 3, 4],
        out.strides,
        0,
    )
    .unwrap();
    let tensor = crate::materialize_tensor_read(&mut buffers, "test", value.tensor_read()).unwrap();
    let expected: Vec<f64> = (0..4)
        .flat_map(|k| {
            (0..3).flat_map(move |j| (0..2).map(move |i| lhs_data[i * 3 + j] * rhs_data[k]))
        })
        .collect();
    assert_eq!(tensor.shape(), &[2, 3, 4]);
    assert_eq!(tensor.as_slice::<f64>().unwrap(), expected.as_slice());
}

#[test]
fn lazy_outer_product_rhs_prefix_preserves_logical_output_order() {
    let mut buffers = BufferPool::default();
    let lhs_data = [1.0_f64, 2.0, 3.0, 4.0, 5.0, 6.0];
    let rhs_data = [10.0_f64, 20.0, 30.0, 40.0];
    let lhs = TypedTensorView::from_slice([2, 3], [3, 1], 0, &lhs_data).unwrap();
    let rhs = TypedTensorView::from_slice([4], [1], 0, &rhs_data).unwrap();

    let out = try_lazy_outer_product_with_pool(
        &mut buffers,
        &lhs,
        &[4, 2, 3],
        &[1, 2],
        &rhs,
        &[4, 2, 3],
        &[0],
        Exec::serial(),
    )
    .unwrap()
    .expect("rhs-prefix output should still support lazy non-canonical lhs order");

    assert_ne!(
        out.strides.as_slice(),
        col_major_strides(&[4, 2, 3]).unwrap().as_slice()
    );
    let value = TensorValue::from_parts(
        Tensor::from_typed::<f64>(out.base),
        vec![4, 2, 3],
        out.strides,
        0,
    )
    .unwrap();
    let tensor = crate::materialize_tensor_read(&mut buffers, "test", value.tensor_read()).unwrap();
    let expected: Vec<f64> = (0..3)
        .flat_map(|j| {
            (0..2).flat_map(move |i| (0..4).map(move |k| rhs_data[k] * lhs_data[i * 3 + j]))
        })
        .collect();
    assert_eq!(tensor.shape(), &[4, 2, 3]);
    assert_eq!(tensor.as_slice::<f64>().unwrap(), expected.as_slice());
}

fn c32(real: f32, imag: f32) -> Complex<f32> {
    Complex::new(real, imag)
}

fn c64(real: f64, imag: f64) -> Complex<f64> {
    Complex::new(real, imag)
}

fn assert_c32_close(actual: Complex<f32>, expected: Complex<f32>) {
    assert!((actual.re - expected.re).abs() < 1.0e-5);
    assert!((actual.im - expected.im).abs() < 1.0e-5);
}

fn assert_c64_close(actual: Complex<f64>, expected: Complex<f64>) {
    assert!((actual.re - expected.re).abs() < 1.0e-12);
    assert!((actual.im - expected.im).abs() < 1.0e-12);
}

fn assert_shape_mismatch<T>(result: crate::Result<T>, op: &'static str) {
    assert!(matches!(
        result,
        Err(crate::Error::Validation { op: actual, .. }) if actual == op
    ));
}

fn assert_dtype_mismatch<T>(result: crate::Result<T>, op: &'static str) {
    assert!(matches!(
        result,
        Err(crate::Error::Validation { op: actual, .. }) if actual == op
    ));
}

fn assert_unsupported<T>(result: crate::Result<T>, op: &'static str) {
    assert!(matches!(
        result,
        Err(crate::Error::Unsupported { op: actual, .. }) if actual == op
    ));
}

fn assert_unsupported_contains<T>(result: crate::Result<T>, op: &'static str, expected: &str) {
    assert!(matches!(
        result,
        Err(crate::Error::Unsupported {
            op: actual,
            message,
        }) if actual == op && message.contains(expected)
    ));
}

#[test]
fn complex_ordered_ops_are_explicitly_rejected() {
    let lhs = Tensor::from_typed::<tenferro_tensor::Complex64>(
        TypedTensor::from_vec_col_major(vec![1], vec![c64(1.0, 0.0)]).unwrap(),
    );
    let rhs = Tensor::from_typed::<tenferro_tensor::Complex64>(
        TypedTensor::from_vec_col_major(vec![1], vec![c64(0.0, 1.0)]).unwrap(),
    );

    for (op, result) in [
        ("maximum", maximum(&lhs, &rhs)),
        ("minimum", minimum(&lhs, &rhs)),
        ("compare", compare(&lhs, &rhs, &CompareDir::Le)),
    ] {
        assert!(matches!(
            result,
            Err(crate::Error::Unsupported { op: actual, message })
                if actual == op && message.contains("total order")
        ));
    }

    assert!(matches!(
        clamp(&lhs, &rhs, &lhs),
        Err(crate::Error::Unsupported { op: "clamp", message })
            if message.contains("total order")
    ));
}

#[test]
fn typed_view_helpers_cover_scalar_and_validation_paths() {
    let mut buffers = BufferPool::default();
    let lhs = TypedTensor::<f64>::from_vec_col_major(vec![2], vec![1.0, 4.0]).unwrap();
    let rhs = TypedTensor::<f64>::from_vec_col_major(vec![2], vec![10.0, 20.0]).unwrap();
    let scalar = TypedTensor::<f64>::from_vec_col_major(vec![], vec![2.0]).unwrap();
    let short = TypedTensor::<f64>::from_vec_col_major(vec![1], vec![99.0]).unwrap();
    let pred = TypedTensor::<bool>::from_vec_col_major(vec![2], vec![true, false]).unwrap();
    let lower = TypedTensor::<f64>::from_vec_col_major(vec![2], vec![0.0, 2.0]).unwrap();
    let upper = TypedTensor::<f64>::from_vec_col_major(vec![2], vec![3.0, 5.0]).unwrap();

    let lhs_view = lhs.as_view();
    let rhs_view = rhs.as_view();
    let scalar_view = scalar.as_view();
    let short_view = short.as_view();
    let pred_view = pred.as_view();
    let lower_view = lower.as_view();
    let upper_view = upper.as_view();

    let same_shape = zip_with_pool(
        "test_binary",
        ErasedZipOp::Add,
        &mut buffers,
        &lhs_view,
        &rhs_view,
        Exec::serial(),
    )
    .unwrap();
    assert_eq!(same_shape.as_slice().unwrap(), &[11.0, 24.0]);

    let scalar_lhs = zip_with_pool(
        "test_binary",
        ErasedZipOp::Multiply,
        &mut buffers,
        &scalar_view,
        &rhs_view,
        Exec::serial(),
    )
    .unwrap();
    assert_eq!(scalar_lhs.as_slice().unwrap(), &[20.0, 40.0]);

    let scalar_rhs = zip_with_pool(
        "test_binary",
        ErasedZipOp::Multiply,
        &mut buffers,
        &lhs_view,
        &scalar_view,
        Exec::serial(),
    )
    .unwrap();
    assert_eq!(scalar_rhs.as_slice().unwrap(), &[2.0, 8.0]);

    assert_shape_mismatch(
        zip_with_pool(
            "test_binary",
            ErasedZipOp::Add,
            &mut buffers,
            &lhs_view,
            &short_view,
            Exec::serial(),
        ),
        "test_binary",
    );

    let negated = map_with_pool::<f64, f64, _>(
        "test_unary",
        ErasedMapOp::Negate,
        &mut buffers,
        &lhs_view,
        Exec::serial(),
    )
    .unwrap();
    assert_eq!(negated.as_slice().unwrap(), &[-1.0, -4.0]);

    let compared = typed_compare_with_pool(
        &mut buffers,
        &lhs_view,
        &rhs_view,
        &CompareDir::Lt,
        Exec::serial(),
    )
    .unwrap();
    assert_eq!(compared.as_slice().unwrap(), &[true, true]);
    assert_shape_mismatch(
        typed_compare_with_pool(
            &mut buffers,
            &lhs_view,
            &short_view,
            &CompareDir::Lt,
            Exec::serial(),
        ),
        "compare",
    );

    let selected = typed_select_with_pool(
        &mut buffers,
        &pred_view,
        &lhs_view,
        &rhs_view,
        Exec::serial(),
    )
    .unwrap();
    assert_eq!(selected.as_slice().unwrap(), &[1.0, 20.0]);
    assert_shape_mismatch(
        typed_select_with_pool(
            &mut buffers,
            &pred_view,
            &short_view,
            &rhs_view,
            Exec::serial(),
        ),
        "select",
    );
    assert_shape_mismatch(
        typed_select_with_pool(
            &mut buffers,
            &pred_view,
            &lhs_view,
            &short_view,
            Exec::serial(),
        ),
        "select",
    );

    let clamped = typed_clamp_with_pool(
        &mut buffers,
        &lhs_view,
        &lower_view,
        &upper_view,
        Exec::serial(),
    )
    .unwrap();
    assert_eq!(clamped.as_slice().unwrap(), &[1.0, 4.0]);

    let degenerate_lower: TypedTensor<f64> =
        TypedTensor::from_vec_col_major(vec![2], vec![5.0_f64, 5.0]).unwrap();
    let degenerate_upper: TypedTensor<f64> =
        TypedTensor::from_vec_col_major(vec![2], vec![3.0_f64, 3.0]).unwrap();
    let degenerate = typed_clamp_with_pool(
        &mut buffers,
        &lhs_view,
        &degenerate_lower.as_view(),
        &degenerate_upper.as_view(),
        Exec::serial(),
    )
    .unwrap();
    assert_eq!(degenerate.as_slice().unwrap(), &[3.0, 3.0]);

    assert_shape_mismatch(
        typed_clamp_with_pool(
            &mut buffers,
            &lhs_view,
            &short_view,
            &upper_view,
            Exec::serial(),
        ),
        "clamp",
    );
    assert_shape_mismatch(
        typed_clamp_with_pool(
            &mut buffers,
            &lhs_view,
            &lower_view,
            &short_view,
            Exec::serial(),
        ),
        "clamp",
    );
}

#[test]
fn read_as_cpu_view_covers_tensor_and_view_variants() {
    let f32_tensor = Tensor::from_typed::<f32>(
        TypedTensor::from_vec_col_major(vec![2], vec![1.0_f32, 2.0]).unwrap(),
    );
    let f64_tensor = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![2], vec![1.0_f64, 2.0]).unwrap(),
    );
    let i32_tensor = Tensor::from_typed::<i32>(
        TypedTensor::from_vec_col_major(vec![2], vec![1_i32, 2]).unwrap(),
    );
    let i64_tensor = Tensor::from_typed::<i64>(
        TypedTensor::from_vec_col_major(vec![2], vec![1_i64, 2]).unwrap(),
    );
    let bool_tensor = Tensor::from_typed::<bool>(
        TypedTensor::from_vec_col_major(vec![2], vec![true, false]).unwrap(),
    );
    let c32_tensor = Tensor::from_typed::<tenferro_tensor::Complex32>(
        TypedTensor::from_vec_col_major(vec![2], vec![c32(1.0, 0.0), c32(0.0, 1.0)]).unwrap(),
    );
    let c64_tensor = Tensor::from_typed::<tenferro_tensor::Complex64>(
        TypedTensor::from_vec_col_major(vec![2], vec![c64(1.0, 0.0), c64(0.0, 1.0)]).unwrap(),
    );

    match read_as_cpu_view(TensorRead::from_tensor(&f32_tensor))
        .expect("tests exercise preset scalars")
    {
        CpuReadView::F32(view) => assert_eq!(view.shape(), &[2]),
        _ => panic!("expected f32 tensor read view"),
    }
    match read_as_cpu_view(TensorRead::from_tensor(&f64_tensor))
        .expect("tests exercise preset scalars")
    {
        CpuReadView::F64(view) => assert_eq!(view.shape(), &[2]),
        _ => panic!("expected f64 tensor read view"),
    }
    match read_as_cpu_view(TensorRead::from_tensor(&i32_tensor))
        .expect("tests exercise preset scalars")
    {
        CpuReadView::I32(view) => assert_eq!(view.shape(), &[2]),
        _ => panic!("expected i32 tensor read view"),
    }
    match read_as_cpu_view(TensorRead::from_tensor(&i64_tensor))
        .expect("tests exercise preset scalars")
    {
        CpuReadView::I64(view) => assert_eq!(view.shape(), &[2]),
        _ => panic!("expected i64 tensor read view"),
    }
    match read_as_cpu_view(TensorRead::from_tensor(&bool_tensor))
        .expect("tests exercise preset scalars")
    {
        CpuReadView::Bool(view) => assert_eq!(view.shape(), &[2]),
        _ => panic!("expected bool tensor read view"),
    }
    match read_as_cpu_view(TensorRead::from_tensor(&c32_tensor))
        .expect("tests exercise preset scalars")
    {
        CpuReadView::C32(view) => assert_eq!(view.shape(), &[2]),
        _ => panic!("expected c32 tensor read view"),
    }
    match read_as_cpu_view(TensorRead::from_tensor(&c64_tensor))
        .expect("tests exercise preset scalars")
    {
        CpuReadView::C64(view) => assert_eq!(view.shape(), &[2]),
        _ => panic!("expected c64 tensor read view"),
    }

    let f32_view_source = TypedTensor::<f32>::from_vec_col_major(vec![1], vec![3.0]).unwrap();
    let f64_view_source = TypedTensor::<f64>::from_vec_col_major(vec![1], vec![3.0]).unwrap();
    let i32_view_source = TypedTensor::<i32>::from_vec_col_major(vec![1], vec![3]).unwrap();
    let i64_view_source = TypedTensor::<i64>::from_vec_col_major(vec![1], vec![3]).unwrap();
    let bool_view_source = TypedTensor::<bool>::from_vec_col_major(vec![1], vec![true]).unwrap();
    let c32_view_source =
        TypedTensor::<Complex<f32>>::from_vec_col_major(vec![1], vec![c32(3.0, 0.0)]).unwrap();
    let c64_view_source =
        TypedTensor::<Complex<f64>>::from_vec_col_major(vec![1], vec![c64(3.0, 0.0)]).unwrap();

    match read_as_cpu_view(TensorRead::from_view(TensorView::F32(
        f32_view_source.as_view(),
    )))
    .expect("tests exercise preset scalars")
    {
        CpuReadView::F32(view) => assert_eq!(view.shape(), &[1]),
        _ => panic!("expected f32 borrowed read view"),
    }
    match read_as_cpu_view(TensorRead::from_view(TensorView::F64(
        f64_view_source.as_view(),
    )))
    .expect("tests exercise preset scalars")
    {
        CpuReadView::F64(view) => assert_eq!(view.shape(), &[1]),
        _ => panic!("expected f64 borrowed read view"),
    }
    match read_as_cpu_view(TensorRead::from_view(TensorView::I32(
        i32_view_source.as_view(),
    )))
    .expect("tests exercise preset scalars")
    {
        CpuReadView::I32(view) => assert_eq!(view.shape(), &[1]),
        _ => panic!("expected i32 borrowed read view"),
    }
    match read_as_cpu_view(TensorRead::from_view(TensorView::I64(
        i64_view_source.as_view(),
    )))
    .expect("tests exercise preset scalars")
    {
        CpuReadView::I64(view) => assert_eq!(view.shape(), &[1]),
        _ => panic!("expected i64 borrowed read view"),
    }
    match read_as_cpu_view(TensorRead::from_view(TensorView::Bool(
        bool_view_source.as_view(),
    )))
    .expect("tests exercise preset scalars")
    {
        CpuReadView::Bool(view) => assert_eq!(view.shape(), &[1]),
        _ => panic!("expected bool borrowed read view"),
    }
    match read_as_cpu_view(TensorRead::from_view(TensorView::C32(
        c32_view_source.as_view(),
    )))
    .expect("tests exercise preset scalars")
    {
        CpuReadView::C32(view) => assert_eq!(view.shape(), &[1]),
        _ => panic!("expected c32 borrowed read view"),
    }
    match read_as_cpu_view(TensorRead::from_view(TensorView::C64(
        c64_view_source.as_view(),
    )))
    .expect("tests exercise preset scalars")
    {
        CpuReadView::C64(view) => assert_eq!(view.shape(), &[1]),
        _ => panic!("expected c64 borrowed read view"),
    }
}

#[test]
fn tensor_read_elementwise_dispatch_covers_view_and_complex_scalar_branches() {
    let mut buffers = BufferPool::default();

    let f32_a = TypedTensor::<f32>::from_vec_col_major(vec![2], vec![1.0, -2.0]).unwrap();
    let f32_b = TypedTensor::<f32>::from_vec_col_major(vec![2], vec![3.0, 4.0]).unwrap();
    let f32_scalar = TypedTensor::<f32>::from_vec_col_major(vec![], vec![2.0]).unwrap();
    let f64_a = TypedTensor::<f64>::from_vec_col_major(vec![3], vec![-3.0, 0.0, 4.0]).unwrap();
    let f64_b = TypedTensor::<f64>::from_vec_col_major(vec![3], vec![1.0, 2.0, 5.0]).unwrap();
    let f64_scalar = TypedTensor::<f64>::from_vec_col_major(vec![], vec![2.0]).unwrap();
    let i32_a = TypedTensor::<i32>::from_vec_col_major(vec![2], vec![1, 4]).unwrap();
    let i32_b = TypedTensor::<i32>::from_vec_col_major(vec![2], vec![2, 3]).unwrap();
    let i64_a = TypedTensor::<i64>::from_vec_col_major(vec![2], vec![5, -1]).unwrap();
    let i64_b = TypedTensor::<i64>::from_vec_col_major(vec![2], vec![2, -1]).unwrap();
    let bool_a = TypedTensor::<bool>::from_vec_col_major(vec![2], vec![true, false]).unwrap();
    let bool_b = TypedTensor::<bool>::from_vec_col_major(vec![2], vec![false, false]).unwrap();
    let pred = TypedTensor::<bool>::from_vec_col_major(vec![2], vec![true, false]).unwrap();
    let c32_a = TypedTensor::<Complex<f32>>::from_vec_col_major(
        vec![2],
        vec![c32(3.0, 4.0), c32(0.0, 2.0)],
    )
    .unwrap();
    let c32_b = TypedTensor::<Complex<f32>>::from_vec_col_major(
        vec![2],
        vec![c32(1.0, 0.0), c32(0.0, 2.0)],
    )
    .unwrap();
    let c64_a = TypedTensor::<Complex<f64>>::from_vec_col_major(
        vec![2],
        vec![c64(3.0, 4.0), c64(0.0, 2.0)],
    )
    .unwrap();
    let c64_b = TypedTensor::<Complex<f64>>::from_vec_col_major(
        vec![2],
        vec![c64(1.0, 0.0), c64(0.0, 2.0)],
    )
    .unwrap();

    let f32_b_tensor = Tensor::from_typed::<f32>(f32_b.duplicate().unwrap());
    let f64_b_tensor = Tensor::from_typed::<f64>(f64_b.duplicate().unwrap());
    let c32_b_tensor = Tensor::from_typed::<tenferro_tensor::Complex32>(c32_b.duplicate().unwrap());
    let c64_b_tensor = Tensor::from_typed::<tenferro_tensor::Complex64>(c64_b.duplicate().unwrap());
    let f32_scalar_tensor = Tensor::from_typed::<f32>(f32_scalar.duplicate().unwrap());
    let f64_scalar_tensor = Tensor::from_typed::<f64>(f64_scalar.duplicate().unwrap());

    let add_f32 = add_read_with_pool(
        &mut buffers,
        TensorRead::from_view(TensorView::F32(f32_a.as_view())),
        TensorRead::from_tensor(&f32_b_tensor),
        Exec::serial(),
    )
    .unwrap();
    assert_eq!(add_f32.as_slice::<f32>().unwrap(), &[4.0, 2.0]);

    let add_c32_scalar = add_read_with_pool(
        &mut buffers,
        TensorRead::from_view(TensorView::F32(f32_scalar.as_view())),
        TensorRead::from_tensor(&c32_b_tensor),
        Exec::serial(),
    )
    .unwrap();
    assert_c32_close(
        add_c32_scalar.as_slice::<Complex<f32>>().unwrap()[0],
        c32(3.0, 0.0),
    );

    let add_c64_scalar = add_read_with_pool(
        &mut buffers,
        TensorRead::from_view(TensorView::C64(c64_b.as_view())),
        TensorRead::from_tensor(&f64_scalar_tensor),
        Exec::serial(),
    )
    .unwrap();
    assert_c64_close(
        add_c64_scalar.as_slice::<Complex<f64>>().unwrap()[1],
        c64(2.0, 2.0),
    );

    let mul_f64 = mul_read_with_pool(
        &mut buffers,
        TensorRead::from_view(TensorView::F64(f64_a.as_view())),
        TensorRead::from_tensor(&f64_b_tensor),
        Exec::serial(),
    )
    .unwrap();
    assert_eq!(mul_f64.as_slice::<f64>().unwrap(), &[-3.0, 0.0, 20.0]);

    let mul_c32_scalar = mul_read_with_pool(
        &mut buffers,
        TensorRead::from_tensor(&c32_b_tensor),
        TensorRead::from_view(TensorView::F32(f32_scalar.as_view())),
        Exec::serial(),
    )
    .unwrap();
    assert_c32_close(
        mul_c32_scalar.as_slice::<Complex<f32>>().unwrap()[1],
        c32(0.0, 4.0),
    );

    assert_dtype_mismatch(
        mul_read_with_pool(
            &mut buffers,
            TensorRead::from_view(TensorView::F32(f32_a.as_view())),
            TensorRead::from_view(TensorView::F64(f64_a.as_view())),
            Exec::serial(),
        ),
        "mul",
    );

    let div_c32 = div_read_with_pool(
        &mut buffers,
        TensorRead::from_view(TensorView::F32(f32_scalar.as_view())),
        TensorRead::from_view(TensorView::C32(c32_b.as_view())),
        Exec::serial(),
    )
    .unwrap();
    assert_c32_close(
        div_c32.as_slice::<Complex<f32>>().unwrap()[1],
        c32(0.0, -1.0),
    );

    let div_c64 = div_read_with_pool(
        &mut buffers,
        TensorRead::from_view(TensorView::C64(c64_a.as_view())),
        TensorRead::from_view(TensorView::F64(f64_scalar.as_view())),
        Exec::serial(),
    )
    .unwrap();
    assert_c64_close(
        div_c64.as_slice::<Complex<f64>>().unwrap()[0],
        c64(1.5, 2.0),
    );

    let div_f32 = div_read_with_pool(
        &mut buffers,
        TensorRead::from_tensor(&f32_b_tensor),
        TensorRead::from_tensor(&f32_scalar_tensor),
        Exec::serial(),
    )
    .unwrap();
    assert_eq!(div_f32.as_slice::<f32>().unwrap(), &[1.5, 2.0]);

    let div_f64 = div_read_with_pool(
        &mut buffers,
        TensorRead::from_tensor(&f64_b_tensor),
        TensorRead::from_view(TensorView::F64(f64_scalar.as_view())),
        Exec::serial(),
    )
    .unwrap();
    assert_eq!(div_f64.as_slice::<f64>().unwrap(), &[0.5, 1.0, 2.5]);

    let neg = neg_read_with_pool(
        &mut buffers,
        TensorRead::from_view(TensorView::C64(c64_a.as_view())),
        Exec::serial(),
    )
    .unwrap();
    assert_c64_close(neg.as_slice::<Complex<f64>>().unwrap()[0], c64(-3.0, -4.0));
    let neg_i32 = neg_read_with_pool(
        &mut buffers,
        TensorRead::from_view(TensorView::I32(i32_a.as_view())),
        Exec::serial(),
    )
    .unwrap();
    assert_eq!(neg_i32.as_slice::<i32>().unwrap(), &[-1, -4]);

    let conj = conj_read_with_pool(
        &mut buffers,
        TensorRead::from_view(TensorView::C32(c32_a.as_view())),
        Exec::serial(),
    )
    .unwrap();
    assert_c32_close(conj.as_slice::<Complex<f32>>().unwrap()[0], c32(3.0, -4.0));
    assert_unsupported(
        conj_read_with_pool(
            &mut buffers,
            TensorRead::from_view(TensorView::Bool(bool_a.as_view())),
            Exec::serial(),
        ),
        "conj",
    );

    let abs = abs_read_with_pool(
        &mut buffers,
        TensorRead::from_view(TensorView::C64(c64_a.as_view())),
        Exec::serial(),
    )
    .unwrap();
    assert_eq!(abs.as_slice::<f64>().unwrap()[0], 5.0);

    let sign = sign_read_with_pool(
        &mut buffers,
        TensorRead::from_view(TensorView::F64(f64_a.as_view())),
        Exec::serial(),
    )
    .unwrap();
    assert_eq!(sign.as_slice::<f64>().unwrap(), &[-1.0, 0.0, 1.0]);

    let sign_complex = sign_read_with_pool(
        &mut buffers,
        TensorRead::from_view(TensorView::C32(c32_a.as_view())),
        Exec::serial(),
    )
    .unwrap();
    assert_c32_close(
        sign_complex.as_slice::<Complex<f32>>().unwrap()[0],
        c32(0.6, 0.8),
    );

    // A magnitude whose square underflows must still produce a unit phase: the
    // modulus is computed with `hypot`, not by squaring the components.
    let tiny_complex =
        TypedTensor::<Complex<f64>>::from_vec_col_major(vec![1], vec![Complex::new(1.0e-200, 0.0)])
            .unwrap();
    let tiny_sign = sign_read_with_pool(
        &mut buffers,
        TensorRead::from_view(TensorView::C64(tiny_complex.as_view())),
        Exec::serial(),
    )
    .unwrap();
    assert_c64_close(
        tiny_sign.as_slice::<Complex<f64>>().unwrap()[0],
        c64(1.0, 0.0),
    );
    let zero_complex =
        TypedTensor::<Complex<f64>>::from_vec_col_major(vec![1], vec![Complex::new(0.0, 0.0)])
            .unwrap();
    let zero_sign = sign_read_with_pool(
        &mut buffers,
        TensorRead::from_view(TensorView::C64(zero_complex.as_view())),
        Exec::serial(),
    )
    .unwrap();
    assert_c64_close(
        zero_sign.as_slice::<Complex<f64>>().unwrap()[0],
        c64(0.0, 0.0),
    );

    assert_unsupported_contains(
        maximum_read_with_pool(
            &mut buffers,
            TensorRead::from_view(TensorView::C32(c32_a.as_view())),
            TensorRead::from_tensor(&c32_b_tensor),
            Exec::serial(),
        ),
        "maximum",
        "total order",
    );
    assert_unsupported_contains(
        minimum_read_with_pool(
            &mut buffers,
            TensorRead::from_view(TensorView::C64(c64_a.as_view())),
            TensorRead::from_tensor(&c64_b_tensor),
            Exec::serial(),
        ),
        "minimum",
        "total order",
    );

    let cmp_i32 = compare_read_with_pool(
        &mut buffers,
        TensorRead::from_view(TensorView::I32(i32_a.as_view())),
        TensorRead::from_view(TensorView::I32(i32_b.as_view())),
        &CompareDir::Lt,
        Exec::serial(),
    )
    .unwrap();
    assert_eq!(cmp_i32.as_slice::<bool>().unwrap(), &[true, false]);

    let cmp_i64 = compare_read_with_pool(
        &mut buffers,
        TensorRead::from_view(TensorView::I64(i64_a.as_view())),
        TensorRead::from_view(TensorView::I64(i64_b.as_view())),
        &CompareDir::Ge,
        Exec::serial(),
    )
    .unwrap();
    assert_eq!(cmp_i64.as_slice::<bool>().unwrap(), &[true, true]);

    let cmp_bool = compare_read_with_pool(
        &mut buffers,
        TensorRead::from_view(TensorView::Bool(bool_a.as_view())),
        TensorRead::from_view(TensorView::Bool(bool_b.as_view())),
        &CompareDir::Eq,
        Exec::serial(),
    )
    .unwrap();
    assert_eq!(cmp_bool.as_slice::<bool>().unwrap(), &[false, true]);

    assert_unsupported_contains(
        compare_read_with_pool(
            &mut buffers,
            TensorRead::from_view(TensorView::C32(c32_a.as_view())),
            TensorRead::from_view(TensorView::C32(c32_b.as_view())),
            &CompareDir::Gt,
            Exec::serial(),
        ),
        "compare",
        "total order",
    );

    let select_i64 = select_read_with_pool(
        &mut buffers,
        TensorRead::from_view(TensorView::Bool(pred.as_view())),
        TensorRead::from_view(TensorView::I64(i64_a.as_view())),
        TensorRead::from_view(TensorView::I64(i64_b.as_view())),
        Exec::serial(),
    )
    .unwrap();
    assert_eq!(select_i64.as_slice::<i64>().unwrap(), &[5, -1]);

    let select_bool = select_read_with_pool(
        &mut buffers,
        TensorRead::from_view(TensorView::Bool(pred.as_view())),
        TensorRead::from_view(TensorView::Bool(bool_a.as_view())),
        TensorRead::from_view(TensorView::Bool(bool_b.as_view())),
        Exec::serial(),
    )
    .unwrap();
    assert_eq!(select_bool.as_slice::<bool>().unwrap(), &[true, false]);

    let select_c32 = select_read_with_pool(
        &mut buffers,
        TensorRead::from_view(TensorView::Bool(pred.as_view())),
        TensorRead::from_view(TensorView::C32(c32_a.as_view())),
        TensorRead::from_view(TensorView::C32(c32_b.as_view())),
        Exec::serial(),
    )
    .unwrap();
    assert_c32_close(
        select_c32.as_slice::<Complex<f32>>().unwrap()[1],
        c32(0.0, 2.0),
    );

    assert_dtype_mismatch(
        select_read_with_pool(
            &mut buffers,
            TensorRead::from_view(TensorView::Bool(pred.as_view())),
            TensorRead::from_view(TensorView::F32(f32_a.as_view())),
            TensorRead::from_view(TensorView::F64(f64_a.as_view())),
            Exec::serial(),
        ),
        "select",
    );
    assert_dtype_mismatch(
        select_read_with_pool(
            &mut buffers,
            TensorRead::from_view(TensorView::F32(f32_a.as_view())),
            TensorRead::from_view(TensorView::F32(f32_a.as_view())),
            TensorRead::from_view(TensorView::F32(f32_b.as_view())),
            Exec::serial(),
        ),
        "select",
    );

    assert_unsupported_contains(
        clamp_read_with_pool(
            &mut buffers,
            TensorRead::from_view(TensorView::C32(c32_a.as_view())),
            TensorRead::from_view(TensorView::C32(c32_b.as_view())),
            TensorRead::from_view(TensorView::C32(c32_a.as_view())),
            Exec::serial(),
        ),
        "clamp",
        "total order",
    );
    assert_unsupported_contains(
        clamp_read_with_pool(
            &mut buffers,
            TensorRead::from_view(TensorView::C64(c64_a.as_view())),
            TensorRead::from_view(TensorView::C64(c64_b.as_view())),
            TensorRead::from_view(TensorView::C64(c64_a.as_view())),
            Exec::serial(),
        ),
        "clamp",
        "total order",
    );

    assert_dtype_mismatch(
        clamp_read_with_pool(
            &mut buffers,
            TensorRead::from_view(TensorView::F32(f32_a.as_view())),
            TensorRead::from_view(TensorView::F32(f32_b.as_view())),
            TensorRead::from_view(TensorView::F64(f64_b.as_view())),
            Exec::serial(),
        ),
        "clamp",
    );
}

#[test]
fn ordered_compare_fixed_dispatch_preserves_owned_and_view_semantics() {
    let lhs = Tensor::from_vec_col_major(vec![3], vec![1.0_f64, 2.0, f64::NAN]).unwrap();
    let rhs = Tensor::from_vec_col_major(vec![3], vec![1.0_f64, 3.0, 4.0]).unwrap();
    let Some(lhs_typed) = lhs.as_typed::<f64>() else {
        unreachable!("f64 fixture");
    };
    let Some(rhs_typed) = rhs.as_typed::<f64>() else {
        unreachable!("f64 fixture");
    };
    let cases = [
        (CompareDir::Eq, [true, false, false]),
        (CompareDir::Lt, [false, true, false]),
        (CompareDir::Le, [true, true, false]),
        (CompareDir::Gt, [false, false, false]),
        (CompareDir::Ge, [true, false, false]),
    ];
    let mut buffers = BufferPool::new();

    for (dir, expected) in cases {
        let owned = compare_with_pool(&mut buffers, &lhs, &rhs, &dir, Exec::serial()).unwrap();
        let borrowed = compare_read_with_pool(
            &mut buffers,
            TensorRead::from_view(TensorView::F64(lhs_typed.as_view())),
            TensorRead::from_view(TensorView::F64(rhs_typed.as_view())),
            &dir,
            Exec::serial(),
        )
        .unwrap();

        assert_eq!(owned.as_slice::<bool>().unwrap(), &expected);
        assert_eq!(borrowed.as_slice::<bool>().unwrap(), &expected);
    }
}

#[test]
fn ordered_compare_owned_and_read_routes_use_fixed_dispatch() {
    let source = include_str!("../elementwise.rs");
    let helper = source
        .split_once("fn typed_compare_with_pool")
        .and_then(|(_, suffix)| suffix.split_once("fn typed_select_with_pool"))
        .map(|(body, _)| body)
        .expect("fixed comparison helper must remain defined");
    assert!(
        helper.contains("erased_compare_into_uninit("),
        "fixed comparison helper must dispatch through strided erased_compare_into_uninit"
    );

    for (start, end, route) in [
        (
            "pub fn compare_with_pool(",
            "pub fn compare_read_with_pool(",
            "owned",
        ),
        (
            "pub fn compare_read_with_pool(",
            "/// Select values",
            "borrowed",
        ),
    ] {
        let body = source
            .split_once(start)
            .and_then(|(_, suffix)| suffix.split_once(end))
            .map(|(body, _)| body)
            .unwrap_or_else(|| panic!("{route} comparison route must remain defined"));
        assert!(
            body.matches("typed_compare_with_pool").count() >= 5,
            "{route} real, integer, and Bool comparisons must use fixed dispatch"
        );
    }
}

#[test]
fn broadcast_multiply_read_and_value_cover_dtypes_and_error_paths() {
    let mut buffers = BufferPool::default();
    let lhs_shape = [2, 3, 4];
    let lhs_dims = [0, 1];
    let rhs_dims = [2];

    let lhs_f32_data: Vec<f32> = (1..=6).map(|x| x as f32).collect();
    let rhs_f32_data = [10.0_f32, 20.0, 30.0, 40.0];
    let lhs_f32 = TypedTensorView::from_slice([2, 3], [3, 1], 0, &lhs_f32_data).unwrap();
    let rhs_f32 = TypedTensorView::from_slice([4], [1], 0, &rhs_f32_data).unwrap();
    let value_f32 = broadcast_multiply_value_with_pool(
        &mut buffers,
        TensorRead::from_view(TensorView::F32(lhs_f32.clone())),
        &lhs_shape,
        &lhs_dims,
        TensorRead::from_view(TensorView::F32(rhs_f32)),
        &lhs_shape,
        &rhs_dims,
        Exec::serial(),
    )
    .unwrap()
    .expect("non-canonical f32 outer product should stay lazy");
    assert!(value_f32.is_view());
    assert_eq!(
        crate::materialize_tensor_read(&mut buffers, "test", value_f32.tensor_read())
            .unwrap()
            .shape(),
        &lhs_shape
    );

    let lhs_i32_data = [1_i32, 2, 3, 4, 5, 6];
    let rhs_i32_data = [2_i32, 3, 4, 5];
    let lhs_i32 = TypedTensorView::from_slice([2, 3], [3, 1], 0, &lhs_i32_data).unwrap();
    let rhs_i32 = TypedTensorView::from_slice([4], [1], 0, &rhs_i32_data).unwrap();
    let value_i32 = broadcast_multiply_value_with_pool(
        &mut buffers,
        TensorRead::from_view(TensorView::I32(lhs_i32.clone())),
        &lhs_shape,
        &lhs_dims,
        TensorRead::from_view(TensorView::I32(rhs_i32)),
        &lhs_shape,
        &rhs_dims,
        Exec::serial(),
    )
    .unwrap()
    .expect("non-canonical i32 outer product should stay lazy");
    assert!(value_i32.is_view());

    let lhs_i64_data = [1_i64, 2, 3, 4, 5, 6];
    let rhs_i64_data = [2_i64, 3, 4, 5];
    let lhs_i64 = TypedTensorView::from_slice([2, 3], [3, 1], 0, &lhs_i64_data).unwrap();
    let rhs_i64 = TypedTensorView::from_slice([4], [1], 0, &rhs_i64_data).unwrap();
    let value_i64 = broadcast_multiply_value_with_pool(
        &mut buffers,
        TensorRead::from_view(TensorView::I64(lhs_i64.clone())),
        &lhs_shape,
        &lhs_dims,
        TensorRead::from_view(TensorView::I64(rhs_i64)),
        &lhs_shape,
        &rhs_dims,
        Exec::serial(),
    )
    .unwrap()
    .expect("non-canonical i64 outer product should stay lazy");
    assert!(value_i64.is_view());

    let lhs_c32_data = [
        c32(1.0, 0.0),
        c32(2.0, 0.0),
        c32(3.0, 0.0),
        c32(4.0, 0.0),
        c32(5.0, 0.0),
        c32(6.0, 0.0),
    ];
    let rhs_c32_data = [c32(2.0, 0.0), c32(3.0, 0.0), c32(4.0, 0.0), c32(5.0, 0.0)];
    let lhs_c32 = TypedTensorView::from_slice([2, 3], [3, 1], 0, &lhs_c32_data).unwrap();
    let rhs_c32 = TypedTensorView::from_slice([4], [1], 0, &rhs_c32_data).unwrap();
    let value_c32 = broadcast_multiply_value_with_pool(
        &mut buffers,
        TensorRead::from_view(TensorView::C32(lhs_c32.clone())),
        &lhs_shape,
        &lhs_dims,
        TensorRead::from_view(TensorView::C32(rhs_c32)),
        &lhs_shape,
        &rhs_dims,
        Exec::serial(),
    )
    .unwrap()
    .expect("non-canonical c32 outer product should stay lazy");
    assert!(value_c32.is_view());

    let lhs_c64_data = [
        c64(1.0, 0.0),
        c64(2.0, 0.0),
        c64(3.0, 0.0),
        c64(4.0, 0.0),
        c64(5.0, 0.0),
        c64(6.0, 0.0),
    ];
    let rhs_c64_data = [c64(2.0, 0.0), c64(3.0, 0.0), c64(4.0, 0.0), c64(5.0, 0.0)];
    let lhs_c64 = TypedTensorView::from_slice([2, 3], [3, 1], 0, &lhs_c64_data).unwrap();
    let rhs_c64 = TypedTensorView::from_slice([4], [1], 0, &rhs_c64_data).unwrap();
    let value_c64 = broadcast_multiply_value_with_pool(
        &mut buffers,
        TensorRead::from_view(TensorView::C64(lhs_c64.clone())),
        &lhs_shape,
        &lhs_dims,
        TensorRead::from_view(TensorView::C64(rhs_c64)),
        &lhs_shape,
        &rhs_dims,
        Exec::serial(),
    )
    .unwrap()
    .expect("non-canonical c64 outer product should stay lazy");
    assert!(value_c64.is_view());

    let same_shape_i64_lhs =
        Tensor::from_typed::<i64>(TypedTensor::from_vec_col_major(vec![2], vec![2, 3]).unwrap());
    let same_shape_i64_rhs =
        Tensor::from_typed::<i64>(TypedTensor::from_vec_col_major(vec![2], vec![4, 5]).unwrap());
    let materialized = broadcast_multiply_value_with_pool(
        &mut buffers,
        TensorRead::from_tensor(&same_shape_i64_lhs),
        &[2],
        &[0],
        TensorRead::from_tensor(&same_shape_i64_rhs),
        &[2],
        &[0],
        Exec::serial(),
    )
    .unwrap()
    .expect("same-shape multiply should materialize");
    assert!(materialized.as_tensor().is_some());
    assert_eq!(
        crate::materialize_tensor_read(&mut buffers, "test", materialized.tensor_read())
            .unwrap()
            .as_slice::<i64>()
            .unwrap(),
        &[8, 15]
    );

    let bool_lhs = Tensor::from_typed::<bool>(
        TypedTensor::from_vec_col_major(vec![2], vec![true, false]).unwrap(),
    );
    let bool_rhs = Tensor::from_typed::<bool>(
        TypedTensor::from_vec_col_major(vec![2], vec![true, true]).unwrap(),
    );
    assert!(broadcast_multiply_read_with_pool(
        &mut buffers,
        TensorRead::from_tensor(&bool_lhs),
        &[2],
        &[0],
        TensorRead::from_tensor(&bool_rhs),
        &[2],
        &[0],
        Exec::serial(),
    )
    .unwrap()
    .is_none());

    assert_shape_mismatch(
        typed_broadcast_mul_with_pool(
            &mut buffers,
            &lhs_f32,
            &[2, 3, 4],
            &[0, 1],
            &lhs_f32,
            &[2, 3, 5],
            &[0, 1],
            Exec::serial(),
        ),
        "broadcast_multiply",
    );
}

/// Exercises the erased elementwise tables through their public entry points, one case per preset
/// dtype, so the tag arms the shared dispatch introduced are covered rather than merely compiled.
///
/// Operations a dtype does not admit are called too: the refusal arm is part of the table, and the
/// test only asserts the dtype when the call succeeds.
#[test]
fn erased_tables_cover_every_preset_dtype() {
    macro_rules! cases {
        ($scalar:ty, $lhs:expr, $rhs:expr) => {{
            let make = |value: $scalar| {
                Tensor::from_typed(
                    TypedTensor::<$scalar>::from_vec_col_major(vec![1], vec![value]).unwrap(),
                )
            };
            let lhs = make($lhs);
            let rhs = make($rhs);
            // Arithmetic between one dtype and itself keeps that dtype.
            for (name, result) in [
                ("add", add(&lhs, &rhs)),
                ("sub", sub(&lhs, &rhs)),
                ("mul", mul(&lhs, &rhs)),
                ("div", div(&lhs, &rhs)),
                ("rem", rem(&lhs, &rhs)),
            ] {
                match result {
                    Ok(value) => assert_eq!(value.dtype(), lhs.dtype(), "{name} keeps the dtype"),
                    Err(error) => assert!(!error.to_string().is_empty(), "{name} refusal"),
                }
            }
            // Ordering and unary tables may widen: `abs` of a complex operand is its real magnitude.
            for (name, result) in [
                ("maximum", maximum(&lhs, &rhs)),
                ("minimum", minimum(&lhs, &rhs)),
            ] {
                if let Ok(value) = result {
                    assert!(!value.shape().is_empty(), "{name} result");
                }
            }
            for (name, result) in [
                ("neg", neg(&lhs)),
                ("conj", conj(&lhs)),
                ("abs", abs(&lhs)),
                ("sign", sign(&lhs)),
                ("compare", compare(&lhs, &rhs, &CompareDir::Lt)),
            ] {
                if let Ok(value) = result {
                    assert_eq!(value.shape().to_vec(), lhs.shape().to_vec(), "{name} shape");
                }
            }
        }};
    }

    cases!(f32, 3.0, 2.0);
    cases!(f64, 3.0, 2.0);
    cases!(i32, 3, 2);
    cases!(i64, 3, 2);
    cases!(bool, true, false);
    cases!(
        num_complex::Complex32,
        num_complex::Complex32::new(3.0, 0.0),
        num_complex::Complex32::new(2.0, 0.0)
    );
    cases!(
        num_complex::Complex64,
        num_complex::Complex64::new(3.0, 0.0),
        num_complex::Complex64::new(2.0, 0.0)
    );

    // The ternary table needs a boolean predicate and the operands it selects between.
    let pred =
        Tensor::from_typed(TypedTensor::<bool>::from_vec_col_major(vec![1], vec![true]).unwrap());
    let on_true =
        Tensor::from_typed(TypedTensor::<f64>::from_vec_col_major(vec![1], vec![1.0]).unwrap());
    let on_false =
        Tensor::from_typed(TypedTensor::<f64>::from_vec_col_major(vec![1], vec![2.0]).unwrap());
    let selected = select(&pred, &on_true, &on_false).expect("f64 select");
    assert_eq!(selected.dtype(), DType::F64);
}

/// Calls the tables with every ordered pair of preset dtypes, so the promotion arms and the refusal
/// arms execute rather than only the same-dtype arms the per-dtype test covers.
#[test]
fn erased_tables_cover_every_dtype_pair() {
    macro_rules! tensor {
        ($scalar:ty, $value:expr) => {
            Tensor::from_typed(
                TypedTensor::<$scalar>::from_vec_col_major(vec![2], vec![$value, $value]).unwrap(),
            )
        };
    }

    let scalars = vec![
        tensor!(f32, 3.0),
        tensor!(f64, 3.0),
        tensor!(i32, 3),
        tensor!(i64, 3),
        tensor!(bool, true),
        tensor!(
            num_complex::Complex32,
            num_complex::Complex32::new(3.0, 1.0)
        ),
        tensor!(
            num_complex::Complex64,
            num_complex::Complex64::new(3.0, 1.0)
        ),
    ];

    for lhs in &scalars {
        for rhs in &scalars {
            for (name, result) in [
                ("add", add(lhs, rhs)),
                ("sub", sub(lhs, rhs)),
                ("mul", mul(lhs, rhs)),
                ("div", div(lhs, rhs)),
                ("rem", rem(lhs, rhs)),
                ("maximum", maximum(lhs, rhs)),
                ("minimum", minimum(lhs, rhs)),
                ("compare", compare(lhs, rhs, &CompareDir::Lt)),
            ] {
                match result {
                    Ok(value) => assert_eq!(
                        value.shape().to_vec(),
                        lhs.shape().to_vec(),
                        "{name} on {:?}, {:?} keeps the shape",
                        lhs.dtype(),
                        rhs.dtype()
                    ),
                    Err(error) => assert!(
                        !error.to_string().is_empty(),
                        "{name} on {:?}, {:?} must refuse with a message",
                        lhs.dtype(),
                        rhs.dtype()
                    ),
                }
            }
            let selected = select(lhs, lhs, rhs);
            match selected {
                Ok(value) => assert_eq!(value.shape().to_vec(), lhs.shape().to_vec()),
                Err(error) => assert!(!error.to_string().is_empty()),
            }
        }
    }
}