tenferro-cpu 0.9.0

use super::*;
#[allow(unused_imports)]
use crate::test_support::RunErased;
use tenferro_tensor::TensorRead;

#[test]
fn test_dot_general_matmul() {
    let a = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![2, 3], vec![1.0, 4.0, 2.0, 5.0, 3.0, 6.0]).unwrap(),
    );
    let b = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(
            vec![3, 4],
            vec![
                1.0, 5.0, 9.0, 2.0, 6.0, 10.0, 3.0, 7.0, 11.0, 4.0, 8.0, 12.0,
            ],
        )
        .unwrap(),
    );
    let mut backend = crate::CpuEngine::serial();
    let c = crate::test_support::on_engine(&mut backend, |__s| {
        __s.dot_general_read(
            TensorRead::from_tensor(&a),
            TensorRead::from_tensor(&b),
            &DotGeneralConfig {
                lhs_contracting_dims: [1].as_slice().into(),
                rhs_contracting_dims: [0].as_slice().into(),
                lhs_batch_dims: [].as_slice().into(),
                rhs_batch_dims: [].as_slice().into(),
            },
        )
    })
    .unwrap()
    .unwrap();
    assert_eq!(c.shape(), &[2, 4]);
    assert_eq!(get_f64(&c, &[0, 0]), 38.0);
    assert_eq!(get_f64(&c, &[1, 0]), 83.0);
    assert_eq!(get_f64(&c, &[0, 1]), 44.0);
    assert_eq!(get_f64(&c, &[1, 1]), 98.0);
    assert_eq!(get_f64(&c, &[0, 3]), 56.0);
    assert_eq!(get_f64(&c, &[1, 3]), 128.0);
}

#[test]
fn test_dot_general_with_conj_matches_materialized_complex_matmul() {
    let lhs_data = vec![
        Complex64::new(1.0, 2.0),
        Complex64::new(-3.0, 0.5),
        Complex64::new(2.0, -1.0),
        Complex64::new(0.25, 4.0),
    ];
    let rhs_data = vec![
        Complex64::new(-2.0, 1.0),
        Complex64::new(1.5, -0.25),
        Complex64::new(0.5, 3.0),
        Complex64::new(-1.0, -2.0),
    ];
    let lhs = Tensor::from_typed::<tenferro_tensor::Complex64>(
        TypedTensor::from_vec_col_major(vec![2, 2], lhs_data.clone()).unwrap(),
    );
    let rhs = Tensor::from_typed::<tenferro_tensor::Complex64>(
        TypedTensor::from_vec_col_major(vec![2, 2], rhs_data.clone()).unwrap(),
    );
    let config = DotGeneralConfig {
        lhs_contracting_dims: [1].as_slice().into(),
        rhs_contracting_dims: [0].as_slice().into(),
        lhs_batch_dims: [].as_slice().into(),
        rhs_batch_dims: [].as_slice().into(),
    };
    let mut backend = crate::CpuEngine::serial();

    let out = crate::test_support::on_engine(&mut backend, |__s| {
        __s.dot_general_with_conj(&lhs, &rhs, &config, true, true)
    })
    .unwrap()
    .unwrap();

    let lhs_conj: Vec<Complex64> = lhs_data.iter().map(|value| value.conj()).collect();
    let rhs_conj: Vec<Complex64> = rhs_data.iter().map(|value| value.conj()).collect();
    let expected = matmul_c64(&lhs_conj, &rhs_conj, 2, 2, 2);
    for col in 0..2 {
        for row in 0..2 {
            assert_c64_close(
                get_c64(&out, &[row, col]),
                expected[col_major_index(2, row, col)],
            );
        }
    }
}

#[test]
fn test_dot_general_read_accepts_tensor_and_view_inputs() {
    let lhs =
        Tensor::from_vec_col_major(vec![2, 3], vec![1.0_f64, 2.0, 3.0, 4.0, 5.0, 6.0]).unwrap();
    let rhs_shape = [3usize, 2];
    let rhs_data = [1.0_f64, 2.0, 3.0, 4.0, 5.0, 6.0];
    let rhs_view = TensorView::f64(&rhs_shape, &rhs_data).unwrap();
    let config = DotGeneralConfig {
        lhs_contracting_dims: [1].as_slice().into(),
        rhs_contracting_dims: [0].as_slice().into(),
        lhs_batch_dims: [].as_slice().into(),
        rhs_batch_dims: [].as_slice().into(),
    };
    let mut backend = crate::CpuEngine::serial();

    let direct = crate::test_support::on_engine(&mut backend, |__s| {
        __s.dot_general_read(
            TensorRead::from_tensor(&lhs),
            TensorRead::from_view(rhs_view.clone()),
            &config,
        )
    })
    .unwrap()
    .unwrap();
    assert_eq!(direct.shape(), &[2, 2]);
    assert_eq!(direct.as_slice::<f64>().unwrap(), &[22.0, 28.0, 49.0, 64.0]);

    let session = crate::test_support::on_engine(&mut backend, |exec| {
        exec.dot_general_read(
            TensorRead::from_tensor(&lhs),
            TensorRead::from_view(rhs_view),
            &config,
        )
    })
    .unwrap();
    let session = session.unwrap();
    assert_eq!(
        session.as_slice::<f64>().unwrap(),
        &[22.0, 28.0, 49.0, 64.0]
    );
}

#[test]
fn test_dot_general_read_accepts_transposed_host_view_input() {
    let lhs_source =
        TypedTensor::<f64>::from_vec_col_major(vec![3, 2], vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0])
            .unwrap();
    let lhs_view = lhs_source.as_view().transpose_view([1, 0]).unwrap();
    let rhs =
        Tensor::from_vec_col_major(vec![3, 2], vec![7.0_f64, 8.0, 9.0, 10.0, 11.0, 12.0]).unwrap();
    let config = DotGeneralConfig {
        lhs_contracting_dims: [1].as_slice().into(),
        rhs_contracting_dims: [0].as_slice().into(),
        lhs_batch_dims: [].as_slice().into(),
        rhs_batch_dims: [].as_slice().into(),
    };
    let mut backend = crate::CpuEngine::serial();

    let out = crate::test_support::on_engine(&mut backend, |__s| {
        __s.dot_general_read(
            TensorRead::from_view(TensorView::F64(lhs_view)),
            TensorRead::from_tensor(&rhs),
            &config,
        )
    })
    .unwrap()
    .unwrap();

    assert_eq!(out.shape(), &[2, 2]);
    assert_eq!(out.as_slice::<f64>().unwrap(), &[50.0, 122.0, 68.0, 167.0]);
}

#[test]
fn test_dot_general_read_into_writes_compact_and_strided_outputs() {
    let lhs =
        Tensor::from_vec_col_major(vec![2, 3], vec![1.0_f64, 2.0, 3.0, 4.0, 5.0, 6.0]).unwrap();
    let rhs_shape = [3usize, 2];
    let rhs_data = [1.0_f64, 2.0, 3.0, 4.0, 5.0, 6.0];
    let config = DotGeneralConfig {
        lhs_contracting_dims: [1].as_slice().into(),
        rhs_contracting_dims: [0].as_slice().into(),
        lhs_batch_dims: [].as_slice().into(),
        rhs_batch_dims: [].as_slice().into(),
    };
    let mut backend = crate::CpuEngine::serial();

    let mut compact = Tensor::from_vec_col_major(vec![2, 2], vec![-1.0_f64; 4]).unwrap();
    crate::test_support::on_engine(&mut backend, |__s| {
        __s.dot_general_read_into(
            TensorRead::from_tensor(&lhs),
            TensorRead::from_view(TensorView::f64(&rhs_shape, &rhs_data).unwrap()),
            &config,
            TensorWrite::from_tensor(&mut compact),
        )
    })
    .unwrap()
    .unwrap();
    assert_eq!(
        compact.as_slice::<f64>().unwrap(),
        &[22.0, 28.0, 49.0, 64.0]
    );

    let mut strided_data = [-1.0_f64; 8];
    {
        let out_view = TensorViewMut::F64(
            TypedTensorViewMut::from_slice([2, 2], [1, 3], 1, &mut strided_data).unwrap(),
        );
        crate::test_support::on_engine(&mut backend, |exec| {
            exec.dot_general_read_into(
                TensorRead::from_tensor(&lhs),
                TensorRead::from_view(TensorView::f64(&rhs_shape, &rhs_data).unwrap()),
                &config,
                TensorWrite::from_view(out_view),
            )
        })
        .unwrap()
        .unwrap();
    }
    assert_eq!(
        strided_data,
        [-1.0, 22.0, 28.0, -1.0, 49.0, 64.0, -1.0, -1.0]
    );
}

#[test]
fn test_dot_general_read_into_rejects_output_shape_and_dtype_mismatch() {
    let lhs =
        Tensor::from_vec_col_major(vec![2, 3], vec![1.0_f64, 2.0, 3.0, 4.0, 5.0, 6.0]).unwrap();
    let rhs =
        Tensor::from_vec_col_major(vec![3, 2], vec![1.0_f64, 2.0, 3.0, 4.0, 5.0, 6.0]).unwrap();
    let config = DotGeneralConfig {
        lhs_contracting_dims: [1].as_slice().into(),
        rhs_contracting_dims: [0].as_slice().into(),
        lhs_batch_dims: [].as_slice().into(),
        rhs_batch_dims: [].as_slice().into(),
    };
    let mut backend = crate::CpuEngine::serial();

    let mut wrong_shape = Tensor::from_vec_col_major(vec![4], vec![0.0_f64; 4]).unwrap();
    let shape_err = crate::test_support::on_engine(&mut backend, |__s| {
        __s.dot_general_read_into(
            TensorRead::from_tensor(&lhs),
            TensorRead::from_tensor(&rhs),
            &config,
            TensorWrite::from_tensor(&mut wrong_shape),
        )
    })
    .unwrap()
    .unwrap_err();
    assert!(matches!(
        shape_err,
        Error::Validation {
            op: "dot_general",
            source: tenferro_tensor::ValidationError::ShapeMismatch(_),
        }
    ));

    let mut wrong_dtype = Tensor::from_vec_col_major(vec![2, 2], vec![0.0_f32; 4]).unwrap();
    let dtype_err = crate::test_support::on_engine(&mut backend, |__s| {
        __s.dot_general_read_into(
            TensorRead::from_tensor(&lhs),
            TensorRead::from_tensor(&rhs),
            &config,
            TensorWrite::from_tensor(&mut wrong_dtype),
        )
    })
    .unwrap()
    .unwrap_err();
    assert!(matches!(
        dtype_err,
        Error::Validation {
            op: "dot_general",
            source: tenferro_tensor::ValidationError::DTypeMismatch { .. },
        }
    ));
}

#[test]
fn test_dot_general_read_into_accum_updates_existing_output() {
    let lhs =
        Tensor::from_vec_col_major(vec![2, 3], vec![1.0_f64, 2.0, 3.0, 4.0, 5.0, 6.0]).unwrap();
    let rhs =
        Tensor::from_vec_col_major(vec![3, 2], vec![1.0_f64, 2.0, 3.0, 4.0, 5.0, 6.0]).unwrap();
    let initial = [10.0_f64, 20.0, 30.0, 40.0];
    let mut out = Tensor::from_vec_col_major(vec![2, 2], initial.to_vec()).unwrap();
    let config = DotGeneralConfig {
        lhs_contracting_dims: [1].as_slice().into(),
        rhs_contracting_dims: [0].as_slice().into(),
        lhs_batch_dims: [].as_slice().into(),
        rhs_batch_dims: [].as_slice().into(),
    };
    let accum = DotGeneralAccumulation {
        lhs_conj: false,
        rhs_conj: false,
        alpha: ContractionScalar::F64(2.0),
        beta: ContractionScalar::F64(-0.5),
    };
    let mut backend = crate::CpuEngine::serial();

    crate::test_support::on_engine(&mut backend, |__s| {
        __s.dot_general_read_into_accum(
            TensorRead::from_tensor(&lhs),
            TensorRead::from_tensor(&rhs),
            &config,
            accum,
            TensorWrite::from_tensor(&mut out),
        )
    })
    .unwrap()
    .unwrap();

    let expected_dot = [22.0, 28.0, 49.0, 64.0];
    for (actual, expected) in out
        .as_slice::<f64>()
        .unwrap()
        .iter()
        .zip(expected_dot.iter().zip(initial))
    {
        let (dot, previous) = expected;
        assert_f64_close(*actual, 2.0 * *dot - 0.5 * previous);
    }

    let mut strided_data = [-1.0_f64, 10.0, 20.0, -1.0, 30.0, 40.0, -1.0, -1.0];
    {
        let out_view = TensorViewMut::F64(
            TypedTensorViewMut::from_slice([2, 2], [1, 3], 1, &mut strided_data).unwrap(),
        );
        crate::test_support::on_engine(&mut backend, |exec| {
            exec.dot_general_read_into_accum_cached(
                Some(7),
                TensorRead::from_tensor(&lhs),
                TensorRead::from_tensor(&rhs),
                &config,
                accum,
                TensorWrite::from_view(out_view),
            )
        })
        .unwrap()
        .unwrap();
    }
    assert_eq!(strided_data[0], -1.0);
    assert_eq!(strided_data[3], -1.0);
    assert_eq!(strided_data[6], -1.0);
    assert_eq!(strided_data[7], -1.0);
    for (actual, expected) in [
        strided_data[1],
        strided_data[2],
        strided_data[4],
        strided_data[5],
    ]
    .into_iter()
    .zip(expected_dot.iter().zip(initial))
    {
        let (dot, previous) = expected;
        assert_f64_close(actual, 2.0 * *dot - 0.5 * previous);
    }
}

#[test]
fn test_dot_general_read_into_accum_applies_complex_conj_and_scalars() {
    let lhs_data = vec![
        Complex64::new(1.0, 2.0),
        Complex64::new(-3.0, 0.5),
        Complex64::new(2.0, -1.0),
        Complex64::new(0.25, 4.0),
    ];
    let rhs_data = vec![
        Complex64::new(-2.0, 1.0),
        Complex64::new(1.5, -0.25),
        Complex64::new(0.5, 3.0),
        Complex64::new(-1.0, -2.0),
    ];
    let initial = vec![
        Complex64::new(1.0, -1.0),
        Complex64::new(2.0, 0.5),
        Complex64::new(-3.0, 2.0),
        Complex64::new(0.25, -4.0),
    ];
    let lhs = Tensor::from_typed::<tenferro_tensor::Complex64>(
        TypedTensor::from_vec_col_major(vec![2, 2], lhs_data.clone()).unwrap(),
    );
    let rhs = Tensor::from_typed::<tenferro_tensor::Complex64>(
        TypedTensor::from_vec_col_major(vec![2, 2], rhs_data.clone()).unwrap(),
    );
    let mut out = Tensor::from_typed::<tenferro_tensor::Complex64>(
        TypedTensor::from_vec_col_major(vec![2, 2], initial.clone()).unwrap(),
    );
    let config = DotGeneralConfig {
        lhs_contracting_dims: [1].as_slice().into(),
        rhs_contracting_dims: [0].as_slice().into(),
        lhs_batch_dims: [].as_slice().into(),
        rhs_batch_dims: [].as_slice().into(),
    };
    let alpha = Complex64::new(0.5, -1.0);
    let beta = Complex64::new(-0.25, 0.75);
    let accum = DotGeneralAccumulation {
        lhs_conj: true,
        rhs_conj: true,
        alpha: ContractionScalar::C64(alpha),
        beta: ContractionScalar::C64(beta),
    };
    let mut backend = crate::CpuEngine::serial();

    crate::test_support::on_engine(&mut backend, |__s| {
        __s.dot_general_read_into_accum(
            TensorRead::from_tensor(&lhs),
            TensorRead::from_tensor(&rhs),
            &config,
            accum,
            TensorWrite::from_tensor(&mut out),
        )
    })
    .unwrap()
    .unwrap();

    let lhs_conj: Vec<Complex64> = lhs_data.iter().map(|value| value.conj()).collect();
    let rhs_conj: Vec<Complex64> = rhs_data.iter().map(|value| value.conj()).collect();
    let dot = matmul_c64(&lhs_conj, &rhs_conj, 2, 2, 2);
    for index in 0..4 {
        assert_c64_close(
            out.as_slice::<Complex64>().unwrap()[index],
            alpha * dot[index] + beta * initial[index],
        );
    }
}

#[test]
fn test_dot_general_read_into_accum_rejects_scalar_dtype_mismatch() {
    let lhs = Tensor::from_vec_col_major(vec![1, 1], vec![2.0_f64]).unwrap();
    let rhs = Tensor::from_vec_col_major(vec![1, 1], vec![3.0_f64]).unwrap();
    let mut out = Tensor::from_vec_col_major(vec![1, 1], vec![4.0_f64]).unwrap();
    let config = DotGeneralConfig {
        lhs_contracting_dims: [1].as_slice().into(),
        rhs_contracting_dims: [0].as_slice().into(),
        lhs_batch_dims: [].as_slice().into(),
        rhs_batch_dims: [].as_slice().into(),
    };
    let accum = DotGeneralAccumulation {
        lhs_conj: false,
        rhs_conj: false,
        alpha: ContractionScalar::F32(1.0),
        beta: ContractionScalar::F64(0.0),
    };
    let mut backend = crate::CpuEngine::serial();

    let err = crate::test_support::on_engine(&mut backend, |__s| {
        __s.dot_general_read_into_accum(
            TensorRead::from_tensor(&lhs),
            TensorRead::from_tensor(&rhs),
            &config,
            accum,
            TensorWrite::from_tensor(&mut out),
        )
    })
    .unwrap()
    .unwrap_err();

    assert!(matches!(
        err,
        Error::Validation {
            op: "dot_general",
            source: tenferro_tensor::ValidationError::DTypeMismatch { .. },
        }
    ));
}

#[test]
fn test_dot_general_read_into_accum_covers_supported_scalar_dtypes() {
    let config = DotGeneralConfig {
        lhs_contracting_dims: [1].as_slice().into(),
        rhs_contracting_dims: [0].as_slice().into(),
        lhs_batch_dims: [].as_slice().into(),
        rhs_batch_dims: [].as_slice().into(),
    };

    let lhs_f32 = Tensor::from_vec_col_major(vec![2, 2], vec![1.0_f32, 2.0, 3.0, 4.0]).unwrap();
    let rhs_f32 = Tensor::from_vec_col_major(vec![2, 1], vec![5.0_f32, 6.0]).unwrap();
    let mut out_f32 = Tensor::from_vec_col_major(vec![2, 1], vec![7.0_f32, 8.0]).unwrap();
    crate::CpuEngine::serial()
        .run_erased(|__s| {
            __s.dot_general_read_into_accum(
                TensorRead::from_tensor(&lhs_f32),
                TensorRead::from_tensor(&rhs_f32),
                &config,
                DotGeneralAccumulation {
                    lhs_conj: false,
                    rhs_conj: false,
                    alpha: ContractionScalar::F32(2.0),
                    beta: ContractionScalar::F32(-0.5),
                },
                TensorWrite::from_tensor(&mut out_f32),
            )
        })
        .unwrap()
        .unwrap();
    let out_f32 = out_f32.as_slice::<f32>().unwrap();
    assert!((out_f32[0] - 42.5).abs() < 1.0e-5);
    assert!((out_f32[1] - 64.0).abs() < 1.0e-5);

    let lhs_c32 = Tensor::from_typed::<tenferro_tensor::Complex32>(
        TypedTensor::from_vec_col_major(
            vec![1, 2],
            vec![Complex32::new(1.0, 1.0), Complex32::new(2.0, -1.0)],
        )
        .unwrap(),
    );
    let rhs_c32 = Tensor::from_typed::<tenferro_tensor::Complex32>(
        TypedTensor::from_vec_col_major(
            vec![2, 1],
            vec![Complex32::new(0.5, -1.0), Complex32::new(3.0, 0.25)],
        )
        .unwrap(),
    );
    let mut out_c32 = Tensor::from_typed::<tenferro_tensor::Complex32>(
        TypedTensor::from_vec_col_major(vec![1, 1], vec![Complex32::new(-2.0, 0.5)]).unwrap(),
    );
    let alpha = Complex32::new(1.5, -0.25);
    let beta = Complex32::new(0.25, 0.5);
    crate::CpuEngine::serial()
        .run_erased(|__s| {
            __s.dot_general_read_into_accum(
                TensorRead::from_tensor(&lhs_c32),
                TensorRead::from_tensor(&rhs_c32),
                &config,
                DotGeneralAccumulation {
                    lhs_conj: true,
                    rhs_conj: false,
                    alpha: ContractionScalar::C32(alpha),
                    beta: ContractionScalar::C32(beta),
                },
                TensorWrite::from_tensor(&mut out_c32),
            )
        })
        .unwrap()
        .unwrap();
    let dot = lhs_c32.as_slice::<Complex32>().unwrap()[0].conj()
        * rhs_c32.as_slice::<Complex32>().unwrap()[0]
        + lhs_c32.as_slice::<Complex32>().unwrap()[1].conj()
            * rhs_c32.as_slice::<Complex32>().unwrap()[1];
    let expected = alpha * dot + beta * Complex32::new(-2.0, 0.5);
    let actual = out_c32.as_slice::<Complex32>().unwrap()[0];
    assert!((actual.re - expected.re).abs() < 1.0e-5);
    assert!((actual.im - expected.im).abs() < 1.0e-5);
}

#[cfg(feature = "blas")]
#[test]
fn test_dot_general_read_blas_negative_stride_view_falls_back() {
    let lhs_source =
        TypedTensor::<f64>::from_vec_col_major(vec![2, 3], vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0])
            .unwrap();
    let lhs_view = lhs_source
        .as_view()
        .slice_axis_view(1, StridedSliceSpec::reverse())
        .unwrap();
    let rhs =
        Tensor::from_vec_col_major(vec![3, 2], vec![7.0_f64, 8.0, 9.0, 10.0, 11.0, 12.0]).unwrap();
    let config = DotGeneralConfig {
        lhs_contracting_dims: [1].as_slice().into(),
        rhs_contracting_dims: [0].as_slice().into(),
        lhs_batch_dims: [].as_slice().into(),
        rhs_batch_dims: [].as_slice().into(),
    };
    let mut backend = crate::CpuEngine::serial();

    let out = crate::test_support::on_engine(&mut backend, |session| {
        session.dot_general_read(
            TensorRead::from_view(TensorView::F64(lhs_view)),
            TensorRead::from_tensor(&rhs),
            &config,
        )
    })
    .unwrap()
    .unwrap();

    assert_eq!(out.shape(), &[2, 2]);
    assert_eq!(out.as_slice::<f64>().unwrap(), &[68.0, 92.0, 95.0, 128.0]);
}

#[test]
fn test_dot_general_inner_product_returns_rank0_scalar() {
    let a = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![3], vec![1.0, 2.0, 3.0]).unwrap(),
    );
    let b = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![3], vec![4.0, 5.0, 6.0]).unwrap(),
    );
    let mut backend = crate::CpuEngine::serial();
    let c = crate::test_support::on_engine(&mut backend, |__s| {
        __s.dot_general_read(
            TensorRead::from_tensor(&a),
            TensorRead::from_tensor(&b),
            &DotGeneralConfig {
                lhs_contracting_dims: [0].as_slice().into(),
                rhs_contracting_dims: [0].as_slice().into(),
                lhs_batch_dims: [].as_slice().into(),
                rhs_batch_dims: [].as_slice().into(),
            },
        )
    })
    .unwrap()
    .unwrap();
    assert!(c.shape().is_empty());
    assert_eq!(get_f64(&c, &[]), 32.0);
}

#[test]
fn test_dot_general_zero_sized_matmul_returns_empty_matrix() {
    let a =
        Tensor::from_typed::<f64>(TypedTensor::from_vec_col_major(vec![0, 0], Vec::new()).unwrap());
    let b =
        Tensor::from_typed::<f64>(TypedTensor::from_vec_col_major(vec![0, 0], Vec::new()).unwrap());
    let mut backend = crate::CpuEngine::serial();
    let c = crate::test_support::on_engine(&mut backend, |__s| {
        __s.dot_general_read(
            TensorRead::from_tensor(&a),
            TensorRead::from_tensor(&b),
            &DotGeneralConfig {
                lhs_contracting_dims: [1].as_slice().into(),
                rhs_contracting_dims: [0].as_slice().into(),
                lhs_batch_dims: [].as_slice().into(),
                rhs_batch_dims: [].as_slice().into(),
            },
        )
    })
    .unwrap()
    .unwrap();

    assert_eq!(c.shape(), &[0, 0]);
    assert!(c
        .as_typed::<f64>()
        .expect("expected F64 tensor")
        .host_data()
        .unwrap()
        .is_empty());
}

#[test]
fn test_dot_general_zero_contracting_dim_returns_zero_filled_output() {
    let a =
        Tensor::from_typed::<f64>(TypedTensor::from_vec_col_major(vec![2, 0], Vec::new()).unwrap());
    let b =
        Tensor::from_typed::<f64>(TypedTensor::from_vec_col_major(vec![0, 3], Vec::new()).unwrap());
    let mut backend = crate::CpuEngine::serial();
    let c = crate::test_support::on_engine(&mut backend, |__s| {
        __s.dot_general_read(
            TensorRead::from_tensor(&a),
            TensorRead::from_tensor(&b),
            &DotGeneralConfig {
                lhs_contracting_dims: [1].as_slice().into(),
                rhs_contracting_dims: [0].as_slice().into(),
                lhs_batch_dims: [].as_slice().into(),
                rhs_batch_dims: [].as_slice().into(),
            },
        )
    })
    .unwrap()
    .unwrap();

    assert_eq!(c.shape(), &[2, 3]);
    assert_eq!(
        c.as_typed::<f64>()
            .expect("expected F64 tensor")
            .host_data()
            .unwrap(),
        &[0.0; 6]
    );
}

#[test]
fn test_dot_general_falls_back_for_unfusable_lhs_batch_layout() {
    let a = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(
            vec![2, 2, 2, 2],
            vec![
                1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0,
                16.0,
            ],
        )
        .unwrap(),
    );
    let b = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(
            vec![2, 2, 2, 2],
            vec![
                1.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0, 1.0, 0.0, 0.0, 1.0,
            ],
        )
        .unwrap(),
    );
    let mut backend = crate::CpuEngine::serial();
    let c = crate::test_support::on_engine(&mut backend, |__s| {
        __s.dot_general_read(
            TensorRead::from_tensor(&a),
            TensorRead::from_tensor(&b),
            &DotGeneralConfig {
                lhs_contracting_dims: [3].as_slice().into(),
                rhs_contracting_dims: [0].as_slice().into(),
                lhs_batch_dims: [0, 2].as_slice().into(),
                rhs_batch_dims: [2, 3].as_slice().into(),
            },
        )
    })
    .unwrap()
    .unwrap();

    assert_eq!(c.shape(), &[2, 2, 2, 2]);
    assert_eq!(get_f64(&c, &[0, 0, 0, 0]), 1.0);
    assert_eq!(get_f64(&c, &[1, 0, 0, 0]), 3.0);
    assert_eq!(get_f64(&c, &[0, 1, 0, 0]), 9.0);
    assert_eq!(get_f64(&c, &[1, 1, 0, 0]), 11.0);
    assert_eq!(get_f64(&c, &[0, 0, 1, 0]), 2.0);
    assert_eq!(get_f64(&c, &[1, 0, 1, 0]), 4.0);
    assert_eq!(get_f64(&c, &[0, 1, 1, 0]), 10.0);
    assert_eq!(get_f64(&c, &[1, 1, 1, 0]), 12.0);
    assert_eq!(get_f64(&c, &[0, 0, 0, 1]), 5.0);
    assert_eq!(get_f64(&c, &[1, 0, 0, 1]), 7.0);
    assert_eq!(get_f64(&c, &[0, 1, 0, 1]), 13.0);
    assert_eq!(get_f64(&c, &[1, 1, 0, 1]), 15.0);
    assert_eq!(get_f64(&c, &[0, 0, 1, 1]), 6.0);
    assert_eq!(get_f64(&c, &[1, 0, 1, 1]), 8.0);
    assert_eq!(get_f64(&c, &[0, 1, 1, 1]), 14.0);
    assert_eq!(get_f64(&c, &[1, 1, 1, 1]), 16.0);
}

#[test]
fn test_dot_general_falls_back_for_mixed_batch_orders() {
    let lhs_storage = [1.0_f64, 2.0, 3.0, 4.0, 5.0, 6.0];
    let rhs_storage = [10.0_f64, 20.0, 30.0, 40.0, 50.0, 60.0];
    let lhs_view =
        TypedTensorView::from_slice([1, 1, 2, 3], [1, 1, 3, 1], 0, &lhs_storage).unwrap();
    let rhs_view =
        TypedTensorView::from_slice([1, 1, 2, 3], [1, 1, 1, 2], 0, &rhs_storage).unwrap();
    let config = DotGeneralConfig {
        lhs_contracting_dims: [1].as_slice().into(),
        rhs_contracting_dims: [0].as_slice().into(),
        lhs_batch_dims: [2, 3].as_slice().into(),
        rhs_batch_dims: [2, 3].as_slice().into(),
    };
    let mut backend = crate::CpuEngine::serial();

    let output = crate::test_support::on_engine(&mut backend, |__s| {
        __s.dot_general_read(
            TensorRead::from_view(TensorView::F64(lhs_view)),
            TensorRead::from_view(TensorView::F64(rhs_view)),
            &config,
        )
    })
    .unwrap()
    .unwrap();

    assert_eq!(output.shape(), &[1, 1, 2, 3]);
    for b1 in 0..3 {
        for b0 in 0..2 {
            let lhs = lhs_storage[b0 * 3 + b1];
            let rhs = rhs_storage[b0 + b1 * 2];
            assert_eq!(
                get_f64(&output, &[0, 0, b0, b1]),
                lhs * rhs,
                "mixed batch coordinate ({b0}, {b1})"
            );
        }
    }
}

#[test]
fn test_transpose() {
    let t = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![2, 3], vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]).unwrap(),
    );
    let tr = transpose(&t, &[1, 0]).unwrap();
    assert_eq!(tr.shape(), &[3, 2]);
    assert_eq!(get_f64(&tr, &[0, 0]), 1.0);
    assert_eq!(get_f64(&tr, &[0, 1]), 2.0);
    assert_eq!(get_f64(&tr, &[1, 0]), 3.0);
    assert_eq!(get_f64(&tr, &[1, 1]), 4.0);
    assert_eq!(get_f64(&tr, &[2, 0]), 5.0);
    assert_eq!(get_f64(&tr, &[2, 1]), 6.0);
}

#[test]
fn test_broadcast_in_dim() {
    let scalar =
        Tensor::from_typed::<f64>(TypedTensor::from_vec_col_major(vec![], vec![5.0]).unwrap());
    let broadcast = broadcast_in_dim(&scalar, &[3], &[]).unwrap();
    assert_eq!(broadcast.shape(), &[3]);
    assert_eq!(get_f64(&broadcast, &[0]), 5.0);
    assert_eq!(get_f64(&broadcast, &[1]), 5.0);
    assert_eq!(get_f64(&broadcast, &[2]), 5.0);

    let v = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![3], vec![1.0, 2.0, 3.0]).unwrap(),
    );
    let m = broadcast_in_dim(&v, &[3, 2], &[0]).unwrap();
    assert_eq!(m.shape(), &[3, 2]);
    for j in 0..2 {
        assert_eq!(get_f64(&m, &[0, j]), 1.0);
        assert_eq!(get_f64(&m, &[1, j]), 2.0);
        assert_eq!(get_f64(&m, &[2, j]), 3.0);
    }
}

#[test]
fn test_tril_3x3() {
    let t = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(
            vec![3, 3],
            vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0],
        )
        .unwrap(),
    );
    let lower = tril(&t, 0).unwrap();
    assert_eq!(lower.shape(), &[3, 3]);
    assert_eq!(
        lower
            .as_typed::<f64>()
            .expect("expected f64 tensor")
            .host_data()
            .unwrap(),
        &[1.0, 2.0, 3.0, 0.0, 5.0, 6.0, 0.0, 0.0, 9.0]
    );
}

#[test]
fn test_triu_3x3() {
    let t = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(
            vec![3, 3],
            vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0],
        )
        .unwrap(),
    );
    let upper = triu(&t, 0).unwrap();
    assert_eq!(upper.shape(), &[3, 3]);
    assert_eq!(
        upper
            .as_typed::<f64>()
            .expect("expected f64 tensor")
            .host_data()
            .unwrap(),
        &[1.0, 0.0, 0.0, 4.0, 5.0, 0.0, 7.0, 8.0, 9.0]
    );
}

#[test]
fn test_triangular_masks_rectangular_batched_nonzero_diagonal() {
    let t = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![2, 3, 2], (1..=12).map(f64::from).collect()).unwrap(),
    );

    let lower = tril(&t, 1).unwrap();
    assert_eq!(
        lower.as_slice::<f64>().unwrap(),
        &[1.0, 2.0, 3.0, 4.0, 0.0, 6.0, 7.0, 8.0, 9.0, 10.0, 0.0, 12.0]
    );

    let upper = triu(&t, 1).unwrap();
    assert_eq!(
        upper.as_slice::<f64>().unwrap(),
        &[0.0, 0.0, 3.0, 0.0, 5.0, 6.0, 0.0, 0.0, 9.0, 0.0, 11.0, 12.0]
    );
}

#[test]
fn test_tril_triu_zero_sized_batch_return_empty_tensor() {
    let t = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![2, 2, 0], Vec::new()).unwrap(),
    );

    let lower = tril(&t, 0).unwrap();
    assert_eq!(lower.shape(), &[2, 2, 0]);
    assert!(lower
        .as_typed::<f64>()
        .expect("expected f64 tensor")
        .host_data()
        .unwrap()
        .is_empty());

    let upper = triu(&t, 0).unwrap();
    assert_eq!(upper.shape(), &[2, 2, 0]);
    assert!(upper
        .as_typed::<f64>()
        .expect("expected f64 tensor")
        .host_data()
        .unwrap()
        .is_empty());
}

#[test]
fn test_tril_triu_extreme_offsets_do_not_overflow() {
    let t = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![2, 2], vec![1.0, 2.0, 3.0, 4.0]).unwrap(),
    );

    let lower_min = tril(&t, i64::MIN).unwrap();
    assert_eq!(
        lower_min
            .as_typed::<f64>()
            .expect("expected f64 tensor")
            .host_data()
            .unwrap(),
        &[0.0, 0.0, 0.0, 0.0]
    );

    let upper_min = triu(&t, i64::MIN).unwrap();
    assert_eq!(
        upper_min
            .as_typed::<f64>()
            .expect("expected f64 tensor")
            .host_data()
            .unwrap(),
        &[1.0, 2.0, 3.0, 4.0]
    );

    let lower_max = tril(&t, i64::MAX).unwrap();
    assert_eq!(
        lower_max
            .as_typed::<f64>()
            .expect("expected f64 tensor")
            .host_data()
            .unwrap(),
        &[1.0, 2.0, 3.0, 4.0]
    );

    let upper_max = triu(&t, i64::MAX).unwrap();
    assert_eq!(
        upper_max
            .as_typed::<f64>()
            .expect("expected f64 tensor")
            .host_data()
            .unwrap(),
        &[0.0, 0.0, 0.0, 0.0]
    );
}

#[test]
fn test_triangular_masks_use_checked_index_arithmetic_contract() {
    let source = include_str!("../../structural.rs");
    let section_start = source
        .find("fn typed_triangular_mask")
        .expect("typed_triangular_mask should exist");
    let section = &source[section_start..];

    for needle in [
        "checked_triangular_extent(op, tensor.shape(), rows, cols)?",
        "checked_triangular_offset(op, batch_idx, block_size, col, rows, row_idx)?",
    ] {
        assert!(
            section.contains(needle),
            "triangular masks should use checked index arithmetic: missing {needle}"
        );
    }
}

#[test]
fn test_neg_and_conj() {
    let t = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![2], vec![3.0, -7.0]).unwrap(),
    );
    let n = neg(&t).unwrap();
    assert_eq!(get_f64(&n, &[0]), -3.0);
    assert_eq!(get_f64(&n, &[1]), 7.0);

    let c = conj(&t).unwrap();
    assert_eq!(get_f64(&c, &[0]), 3.0);
    assert_eq!(get_f64(&c, &[1]), -7.0);
}

#[test]
fn test_cpu_backend_analytic_ops_real() {
    let mut backend = crate::CpuEngine::serial();

    let exp_input = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![2], vec![0.0, 1.0]).unwrap(),
    );
    let exp_out = crate::test_support::on_engine(&mut backend, |__s| {
        __s.exp_read(TensorRead::from_tensor(&exp_input))
    })
    .unwrap()
    .unwrap();
    assert_f64_close(get_f64(&exp_out, &[0]), 1.0);
    assert_f64_close(get_f64(&exp_out, &[1]), std::f64::consts::E);

    let log_input = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![2], vec![1.0, 4.0]).unwrap(),
    );
    let log_out = crate::test_support::on_engine(&mut backend, |__s| {
        __s.log_read(TensorRead::from_tensor(&log_input))
    })
    .unwrap()
    .unwrap();
    assert_f64_close(get_f64(&log_out, &[0]), 0.0);
    assert_f64_close(get_f64(&log_out, &[1]), 4.0_f64.ln());

    let trig_input = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![2], vec![0.0, std::f64::consts::FRAC_PI_2]).unwrap(),
    );
    let sin_out = crate::test_support::on_engine(&mut backend, |__s| {
        __s.sin_read(TensorRead::from_tensor(&trig_input))
    })
    .unwrap()
    .unwrap();
    let cos_out = crate::test_support::on_engine(&mut backend, |__s| {
        __s.cos_read(TensorRead::from_tensor(&trig_input))
    })
    .unwrap()
    .unwrap();
    assert_f64_close(get_f64(&sin_out, &[0]), 0.0);
    assert_f64_close(get_f64(&sin_out, &[1]), 1.0);
    assert_f64_close(get_f64(&cos_out, &[0]), 1.0);
    assert_f64_close(get_f64(&cos_out, &[1]), 0.0);

    let tanh_input = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![2], vec![0.0, 1.0]).unwrap(),
    );
    let tanh_out = crate::test_support::on_engine(&mut backend, |__s| {
        __s.tanh_read(TensorRead::from_tensor(&tanh_input))
    })
    .unwrap()
    .unwrap();
    assert_f64_close(get_f64(&tanh_out, &[0]), 0.0);
    assert_f64_close(get_f64(&tanh_out, &[1]), 1.0_f64.tanh());

    let sqrt_input = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![2], vec![1.0, 4.0]).unwrap(),
    );
    let sqrt_out = crate::test_support::on_engine(&mut backend, |__s| {
        __s.sqrt_read(TensorRead::from_tensor(&sqrt_input))
    })
    .unwrap()
    .unwrap();
    let rsqrt_out = crate::test_support::on_engine(&mut backend, |__s| {
        __s.rsqrt_read(TensorRead::from_tensor(&sqrt_input))
    })
    .unwrap()
    .unwrap();
    assert_f64_close(get_f64(&sqrt_out, &[0]), 1.0);
    assert_f64_close(get_f64(&sqrt_out, &[1]), 2.0);
    assert_f64_close(get_f64(&rsqrt_out, &[0]), 1.0);
    assert_f64_close(get_f64(&rsqrt_out, &[1]), 0.5);

    let expm1_out = crate::test_support::on_engine(&mut backend, |__s| {
        __s.expm1_read(tenferro_tensor::TensorRead::from_tensor(&exp_input))
    })
    .unwrap()
    .unwrap();
    let log1p_out = crate::test_support::on_engine(&mut backend, |__s| {
        __s.log1p_read(TensorRead::from_tensor(&log_input))
    })
    .unwrap()
    .unwrap();
    assert_f64_close(get_f64(&expm1_out, &[0]), 0.0);
    assert_f64_close(get_f64(&expm1_out, &[1]), 1.0_f64.exp_m1());
    assert_f64_close(get_f64(&log1p_out, &[0]), 2.0_f64.ln());
    assert_f64_close(get_f64(&log1p_out, &[1]), 5.0_f64.ln());

    let pow_base = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![2], vec![2.0, 9.0]).unwrap(),
    );
    let pow_exp = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![2], vec![3.0, 0.5]).unwrap(),
    );
    let pow_out = crate::test_support::on_engine(&mut backend, |__s| {
        __s.pow_read(
            TensorRead::from_tensor(&pow_base),
            TensorRead::from_tensor(&pow_exp),
        )
    })
    .unwrap()
    .unwrap();
    assert_f64_close(get_f64(&pow_out, &[0]), 8.0);
    assert_f64_close(get_f64(&pow_out, &[1]), 3.0);
}

#[test]
fn test_cpu_backend_analytic_ops_complex() {
    let mut backend = crate::CpuEngine::serial();

    let exp_input = Tensor::from_typed::<tenferro_tensor::Complex64>(
        TypedTensor::from_vec_col_major(
            vec![2],
            vec![Complex64::new(0.0, 0.0), Complex64::new(1.0, 1.0)],
        )
        .unwrap(),
    );
    let exp_out = crate::test_support::on_engine(&mut backend, |__s| {
        __s.exp_read(TensorRead::from_tensor(&exp_input))
    })
    .unwrap()
    .unwrap();
    assert_c64_close(get_c64(&exp_out, &[0]), Complex64::new(1.0, 0.0));
    assert_c64_close(get_c64(&exp_out, &[1]), Complex64::new(1.0, 1.0).exp());

    let log_input = Tensor::from_typed::<tenferro_tensor::Complex64>(
        TypedTensor::from_vec_col_major(
            vec![2],
            vec![Complex64::new(1.0, 0.0), Complex64::new(2.0, -0.5)],
        )
        .unwrap(),
    );
    let log_out = crate::test_support::on_engine(&mut backend, |__s| {
        __s.log_read(TensorRead::from_tensor(&log_input))
    })
    .unwrap()
    .unwrap();
    assert_c64_close(get_c64(&log_out, &[0]), Complex64::new(1.0, 0.0).ln());
    assert_c64_close(get_c64(&log_out, &[1]), Complex64::new(2.0, -0.5).ln());

    let trig_input = Tensor::from_typed::<tenferro_tensor::Complex64>(
        TypedTensor::from_vec_col_major(
            vec![2],
            vec![Complex64::new(0.0, 0.0), Complex64::new(0.5, -0.25)],
        )
        .unwrap(),
    );
    let sin_out = crate::test_support::on_engine(&mut backend, |__s| {
        __s.sin_read(TensorRead::from_tensor(&trig_input))
    })
    .unwrap()
    .unwrap();
    let cos_out = crate::test_support::on_engine(&mut backend, |__s| {
        __s.cos_read(TensorRead::from_tensor(&trig_input))
    })
    .unwrap()
    .unwrap();
    let tanh_out = crate::test_support::on_engine(&mut backend, |__s| {
        __s.tanh_read(TensorRead::from_tensor(&trig_input))
    })
    .unwrap()
    .unwrap();
    assert_c64_close(get_c64(&sin_out, &[0]), Complex64::new(0.0, 0.0).sin());
    assert_c64_close(get_c64(&sin_out, &[1]), Complex64::new(0.5, -0.25).sin());
    assert_c64_close(get_c64(&cos_out, &[0]), Complex64::new(0.0, 0.0).cos());
    assert_c64_close(get_c64(&cos_out, &[1]), Complex64::new(0.5, -0.25).cos());
    assert_c64_close(get_c64(&tanh_out, &[0]), Complex64::new(0.0, 0.0).tanh());
    assert_c64_close(get_c64(&tanh_out, &[1]), Complex64::new(0.5, -0.25).tanh());

    let sqrt_input = Tensor::from_typed::<tenferro_tensor::Complex64>(
        TypedTensor::from_vec_col_major(
            vec![2],
            vec![Complex64::new(1.0, 0.0), Complex64::new(4.0, 3.0)],
        )
        .unwrap(),
    );
    let sqrt_out = crate::test_support::on_engine(&mut backend, |__s| {
        __s.sqrt_read(TensorRead::from_tensor(&sqrt_input))
    })
    .unwrap()
    .unwrap();
    let rsqrt_out = crate::test_support::on_engine(&mut backend, |__s| {
        __s.rsqrt_read(TensorRead::from_tensor(&sqrt_input))
    })
    .unwrap()
    .unwrap();
    assert_c64_close(get_c64(&sqrt_out, &[0]), Complex64::new(1.0, 0.0).sqrt());
    assert_c64_close(get_c64(&sqrt_out, &[1]), Complex64::new(4.0, 3.0).sqrt());
    assert_c64_close_tol(
        get_c64(&rsqrt_out, &[0]),
        Complex64::new(1.0, 0.0) / Complex64::new(1.0, 0.0).sqrt(),
        1.0e-12,
    );
    assert_c64_close_tol(
        get_c64(&rsqrt_out, &[1]),
        Complex64::new(1.0, 0.0) / Complex64::new(4.0, 3.0).sqrt(),
        1.0e-12,
    );

    let expm1_out = crate::test_support::on_engine(&mut backend, |__s| {
        __s.expm1_read(tenferro_tensor::TensorRead::from_tensor(&exp_input))
    })
    .unwrap()
    .unwrap();
    let log1p_out = crate::test_support::on_engine(&mut backend, |__s| {
        __s.log1p_read(TensorRead::from_tensor(&log_input))
    })
    .unwrap()
    .unwrap();
    assert_c64_close(
        get_c64(&expm1_out, &[0]),
        Complex64::new(0.0, 0.0).exp() - Complex64::new(1.0, 0.0),
    );
    assert_c64_close(
        get_c64(&expm1_out, &[1]),
        Complex64::new(1.0, 1.0).exp() - Complex64::new(1.0, 0.0),
    );
    assert_c64_close(
        get_c64(&log1p_out, &[0]),
        (Complex64::new(1.0, 0.0) + Complex64::new(1.0, 0.0)).ln(),
    );
    assert_c64_close(
        get_c64(&log1p_out, &[1]),
        (Complex64::new(2.0, -0.5) + Complex64::new(1.0, 0.0)).ln(),
    );

    let pow_base = Tensor::from_typed::<tenferro_tensor::Complex64>(
        TypedTensor::from_vec_col_major(
            vec![2],
            vec![Complex64::new(1.0, 1.0), Complex64::new(2.0, -1.0)],
        )
        .unwrap(),
    );
    let pow_exp = Tensor::from_typed::<tenferro_tensor::Complex64>(
        TypedTensor::from_vec_col_major(
            vec![2],
            vec![Complex64::new(2.0, 0.0), Complex64::new(0.5, 0.25)],
        )
        .unwrap(),
    );
    let pow_out = crate::test_support::on_engine(&mut backend, |__s| {
        __s.pow_read(
            TensorRead::from_tensor(&pow_base),
            TensorRead::from_tensor(&pow_exp),
        )
    })
    .unwrap()
    .unwrap();
    assert_c64_close(
        get_c64(&pow_out, &[0]),
        Complex64::new(1.0, 1.0).powc(Complex64::new(2.0, 0.0)),
    );
    assert_c64_close(
        get_c64(&pow_out, &[1]),
        Complex64::new(2.0, -1.0).powc(Complex64::new(0.5, 0.25)),
    );
}

#[test]
fn test_extract_diagonal() {
    let square = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(
            vec![3, 3],
            vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0],
        )
        .unwrap(),
    );
    let d = extract_diagonal(&square, 0, 1).unwrap();
    assert_eq!(d.shape(), &[3]);
    assert_eq!(get_f64(&d, &[0]), 1.0);
    assert_eq!(get_f64(&d, &[1]), 5.0);
    assert_eq!(get_f64(&d, &[2]), 9.0);

    let cube = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![2, 3, 3], (1..=18).map(|x| x as f64).collect())
            .unwrap(),
    );
    let diag = extract_diagonal(&cube, 1, 2).unwrap();
    assert_eq!(diag.shape(), &[2, 3]);
    assert_eq!(get_f64(&diag, &[0, 0]), 1.0);
    assert_eq!(get_f64(&diag, &[1, 1]), 10.0);
    assert_eq!(get_f64(&diag, &[1, 2]), 18.0);

    // `axis_a > axis_b`: the diagonal axis is removed from the cotangent at
    // `axis_b` and the diagonal lands at `axis_a - 1` (shape_infer/CUDA
    // contract), not at the strided primitive's larger axis.
    let reversed = extract_diagonal(&cube, 2, 0).unwrap();
    assert_eq!(reversed.shape(), &[3, 2]);
    assert_eq!(
        (0..6)
            .map(|i| get_f64(&reversed, &[i % 3, i / 3]))
            .collect::<Vec<_>>(),
        vec![1.0, 3.0, 5.0, 8.0, 10.0, 12.0]
    );
}

#[test]
fn test_embed_diagonal() {
    let v = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![3], vec![1.0, 2.0, 3.0]).unwrap(),
    );
    let m = embed_diagonal(&v, 0, 1).unwrap();
    assert_eq!(m.shape(), &[3, 3]);
    assert_eq!(get_f64(&m, &[0, 0]), 1.0);
    assert_eq!(get_f64(&m, &[1, 1]), 2.0);
    assert_eq!(get_f64(&m, &[2, 2]), 3.0);
    assert_eq!(get_f64(&m, &[0, 1]), 0.0);
    assert_eq!(get_f64(&m, &[2, 0]), 0.0);
}

#[test]
fn test_cpu_backend_dispatches_tensor_backend_ops() {
    let a = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![2], vec![1.0, 2.0]).unwrap(),
    );
    let b = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![2], vec![3.0, 4.0]).unwrap(),
    );
    let mut backend = crate::CpuEngine::serial();
    let out = crate::test_support::on_engine(&mut backend, |__s| {
        __s.add_read(TensorRead::from_tensor(&a), TensorRead::from_tensor(&b))
    })
    .unwrap()
    .unwrap();
    assert_eq!(get_f64(&out, &[0]), 4.0);
    assert_eq!(get_f64(&out, &[1]), 6.0);
}

#[test]
fn test_tier2_elementwise_ops_real() {
    let lhs = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![3], vec![8.0, -2.0, 9.0]).unwrap(),
    );
    let rhs = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![3], vec![2.0, 5.0, 3.0]).unwrap(),
    );
    let pred = Tensor::from_typed::<bool>(
        TypedTensor::from_vec_col_major(vec![3], vec![false, true, true]).unwrap(),
    );
    let on_true = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![3], vec![10.0, 20.0, 30.0]).unwrap(),
    );
    let on_false = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![3], vec![1.0, 2.0, 3.0]).unwrap(),
    );
    let lower = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![3], vec![-1.0, -1.0, 0.0]).unwrap(),
    );
    let upper = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![3], vec![1.0, 0.25, 4.0]).unwrap(),
    );
    let mut backend = crate::CpuEngine::serial();

    let div = crate::test_support::on_engine(&mut backend, |__s| {
        __s.div_read(TensorRead::from_tensor(&lhs), TensorRead::from_tensor(&rhs))
    })
    .unwrap()
    .unwrap();
    assert_eq!(get_f64(&div, &[0]), 4.0);
    assert_eq!(get_f64(&div, &[1]), -0.4);
    assert_eq!(get_f64(&div, &[2]), 3.0);

    let abs = crate::test_support::on_engine(&mut backend, |__s| {
        __s.abs_read(TensorRead::from_tensor(&lhs))
    })
    .unwrap()
    .unwrap();
    assert_eq!(get_f64(&abs, &[0]), 8.0);
    assert_eq!(get_f64(&abs, &[1]), 2.0);
    assert_eq!(get_f64(&abs, &[2]), 9.0);

    let sign = crate::test_support::on_engine(&mut backend, |__s| {
        __s.sign_read(TensorRead::from_tensor(&lhs))
    })
    .unwrap()
    .unwrap();
    assert_eq!(get_f64(&sign, &[0]), 1.0);
    assert_eq!(get_f64(&sign, &[1]), -1.0);
    assert_eq!(get_f64(&sign, &[2]), 1.0);

    let maximum = crate::test_support::on_engine(&mut backend, |__s| {
        __s.maximum_read(TensorRead::from_tensor(&lhs), TensorRead::from_tensor(&rhs))
    })
    .unwrap()
    .unwrap();
    assert_eq!(get_f64(&maximum, &[0]), 8.0);
    assert_eq!(get_f64(&maximum, &[1]), 5.0);
    assert_eq!(get_f64(&maximum, &[2]), 9.0);

    let minimum = crate::test_support::on_engine(&mut backend, |__s| {
        __s.minimum_read(TensorRead::from_tensor(&lhs), TensorRead::from_tensor(&rhs))
    })
    .unwrap()
    .unwrap();
    assert_eq!(get_f64(&minimum, &[0]), 2.0);
    assert_eq!(get_f64(&minimum, &[1]), -2.0);
    assert_eq!(get_f64(&minimum, &[2]), 3.0);

    let eq = crate::test_support::on_engine(&mut backend, |__s| {
        __s.compare_read(
            TensorRead::from_tensor(&lhs),
            TensorRead::from_tensor(&rhs),
            &CompareDir::Eq,
        )
    })
    .unwrap()
    .unwrap();
    assert!(!get_bool(&eq, &[0]));
    assert!(!get_bool(&eq, &[1]));
    assert!(!get_bool(&eq, &[2]));

    let lt = crate::test_support::on_engine(&mut backend, |__s| {
        __s.compare_read(
            TensorRead::from_tensor(&lhs),
            TensorRead::from_tensor(&rhs),
            &CompareDir::Lt,
        )
    })
    .unwrap()
    .unwrap();
    assert!(!get_bool(&lt, &[0]));
    assert!(get_bool(&lt, &[1]));
    assert!(!get_bool(&lt, &[2]));

    let le = crate::test_support::on_engine(&mut backend, |__s| {
        __s.compare_read(
            TensorRead::from_tensor(&lhs),
            TensorRead::from_tensor(&rhs),
            &CompareDir::Le,
        )
    })
    .unwrap()
    .unwrap();
    assert!(!get_bool(&le, &[0]));
    assert!(get_bool(&le, &[1]));
    assert!(!get_bool(&le, &[2]));

    let gt = crate::test_support::on_engine(&mut backend, |__s| {
        __s.compare_read(
            TensorRead::from_tensor(&lhs),
            TensorRead::from_tensor(&rhs),
            &CompareDir::Gt,
        )
    })
    .unwrap()
    .unwrap();
    assert!(get_bool(&gt, &[0]));
    assert!(!get_bool(&gt, &[1]));
    assert!(get_bool(&gt, &[2]));

    let ge = crate::test_support::on_engine(&mut backend, |__s| {
        __s.compare_read(
            TensorRead::from_tensor(&lhs),
            TensorRead::from_tensor(&rhs),
            &CompareDir::Ge,
        )
    })
    .unwrap()
    .unwrap();
    assert!(get_bool(&ge, &[0]));
    assert!(!get_bool(&ge, &[1]));
    assert!(get_bool(&ge, &[2]));

    let select = crate::test_support::on_engine(&mut backend, |__s| {
        __s.select_read(
            TensorRead::from_tensor(&pred),
            TensorRead::from_tensor(&on_true),
            TensorRead::from_tensor(&on_false),
        )
    })
    .unwrap()
    .unwrap();
    assert_eq!(get_f64(&select, &[0]), 1.0);
    assert_eq!(get_f64(&select, &[1]), 20.0);
    assert_eq!(get_f64(&select, &[2]), 30.0);

    let clamp = crate::test_support::on_engine(&mut backend, |__s| {
        __s.clamp_read(
            TensorRead::from_tensor(&lhs),
            TensorRead::from_tensor(&lower),
            TensorRead::from_tensor(&upper),
        )
    })
    .unwrap()
    .unwrap();
    assert_eq!(get_f64(&clamp, &[0]), 1.0);
    assert_eq!(get_f64(&clamp, &[1]), -1.0);
    assert_eq!(get_f64(&clamp, &[2]), 4.0);
}

#[test]
fn test_tier2_elementwise_ops_complex() {
    let input = Tensor::from_typed::<tenferro_tensor::Complex64>(
        TypedTensor::from_vec_col_major(
            vec![2],
            vec![Complex64::new(3.0, 4.0), Complex64::new(0.0, 0.0)],
        )
        .unwrap(),
    );
    let lhs = Tensor::from_typed::<tenferro_tensor::Complex64>(
        TypedTensor::from_vec_col_major(
            vec![2],
            vec![Complex64::new(3.0, 4.0), Complex64::new(1.0, 0.0)],
        )
        .unwrap(),
    );
    let rhs = Tensor::from_typed::<tenferro_tensor::Complex64>(
        TypedTensor::from_vec_col_major(
            vec![2],
            vec![Complex64::new(1.0, 0.0), Complex64::new(0.0, 2.0)],
        )
        .unwrap(),
    );
    let mut backend = crate::CpuEngine::serial();

    let abs = crate::test_support::on_engine(&mut backend, |__s| {
        __s.abs_read(TensorRead::from_tensor(&input))
    })
    .unwrap()
    .unwrap();
    assert_eq!(abs.dtype(), DType::F64);
    assert_eq!(get_f64(&abs, &[0]), 5.0);
    assert_eq!(get_f64(&abs, &[1]), 0.0);

    let sign = crate::test_support::on_engine(&mut backend, |__s| {
        __s.sign_read(TensorRead::from_tensor(&input))
    })
    .unwrap()
    .unwrap();
    assert_c64_close(get_c64(&sign, &[0]), Complex64::new(0.6, 0.8));
    assert_c64_close(get_c64(&sign, &[1]), Complex64::new(0.0, 0.0));

    assert!(matches!(
        crate::test_support::on_engine(&mut backend, |__s| __s.maximum_read(TensorRead::from_tensor(&lhs), TensorRead::from_tensor(&rhs))).unwrap(),
        Err(crate::Error::Unsupported {
            op: "maximum",
            message,
        }) if message.contains("total order")
    ));
    assert!(matches!(
        crate::test_support::on_engine(&mut backend, |__s| __s.minimum_read(TensorRead::from_tensor(&lhs), TensorRead::from_tensor(&rhs))).unwrap(),
        Err(crate::Error::Unsupported {
            op: "minimum",
            message,
        }) if message.contains("total order")
    ));
}

/// The canonical fallback plans packed and borrowed canonical operands
/// directly: a borrowed operand at a nonzero offset and a strided destination
/// view must still produce the reference contraction (#1897).
#[test]
fn canonical_fallback_honors_operand_offsets_and_output_strides() {
    use tenferro_tensor::{
        DotGeneralConfig, TensorView, TensorViewMut, TensorWrite, TypedTensorView,
        TypedTensorViewMut,
    };

    // lhs [2, 3, 2] contracts its middle axis, so its free axes (0, 2) need
    // packing; rhs is a compact [3, 2] block at element offset 5 of a larger
    // buffer, already canonical and therefore borrowed.
    let lhs_data: Vec<f64> = (1..=12).map(f64::from).collect();
    let lhs = Tensor::from_vec_col_major(vec![2, 3, 2], lhs_data.clone()).unwrap();
    let rhs_storage: Vec<f64> = (0..11).map(|i| f64::from(i) * 0.5 - 1.0).collect();
    let rhs = TypedTensorView::from_slice([3, 2], [1, 3], 5, &rhs_storage).unwrap();
    let rhs_data = &rhs_storage[5..11];
    // The [2, 2, 2] output keeps its fused row group (i, l) compact but pads
    // each GEMM column to a leading dimension of 6 in a 12-element buffer; BLAS
    // also requires a unit row stride.
    let mut out_storage = vec![-7.0_f64; 12];
    let config = DotGeneralConfig {
        lhs_contracting_dims: [1].as_slice().into(),
        rhs_contracting_dims: [0].as_slice().into(),
        lhs_batch_dims: [].as_slice().into(),
        rhs_batch_dims: [].as_slice().into(),
    };

    let mut backend = crate::test_support::cpu_engine(1).unwrap();
    {
        let out =
            TypedTensorViewMut::from_slice([2, 2, 2], [1, 2, 6], 0, &mut out_storage).unwrap();
        crate::test_support::on_engine(&mut backend, |session| {
            session.dot_general_read_into(
                TensorRead::from_tensor(&lhs),
                TensorRead::from_view(TensorView::F64(rhs)),
                &config,
                TensorWrite::from_view(TensorViewMut::F64(out)),
            )
        })
        .unwrap()
        .unwrap();
    }

    // out[i, l, n] = sum_j lhs[i, j, l] * rhs[j, n] at storage i + 2l + 6n.
    for n in 0..2 {
        for l in 0..2 {
            for i in 0..2 {
                let expected: f64 = (0..3)
                    .map(|j| lhs_data[i + 2 * j + 6 * l] * rhs_data[j + 3 * n])
                    .sum();
                assert_eq!(out_storage[i + 2 * l + 6 * n], expected, "({i}, {l}, {n})");
            }
        }
    }
    // The padding slots of each column are untouched.
    assert!(out_storage
        .iter()
        .enumerate()
        .filter(|(index, _)| index % 6 >= 4)
        .all(|(_, &value)| value == -7.0));
}

#[test]
fn auto_batched_gemm_on_lanes_matches_reference_with_padded_batches() {
    use tenferro_tensor::{DotGeneralConfig, TensorViewMut, TensorWrite, TypedTensorViewMut};

    // 1024 tiny 4x4x4 products clear the Auto lane gate at 4 threads, so the
    // batch runs as one strided GEMM per lane. Each output item is padded to a
    // batch stride of 20, and the padding must survive the lane split.
    let (m, batch, pitch) = (4_usize, 1024_usize, 20_usize);
    let lhs_data: Vec<f64> = (0..m * m * batch).map(|i| (i % 13) as f64 - 6.0).collect();
    let rhs_data: Vec<f64> = (0..m * m * batch).map(|i| (i % 7) as f64 * 0.5).collect();
    let lhs = Tensor::from_vec_col_major(vec![m, m, batch], lhs_data.clone()).unwrap();
    let rhs = Tensor::from_vec_col_major(vec![m, m, batch], rhs_data.clone()).unwrap();
    let config = DotGeneralConfig {
        lhs_contracting_dims: [1].as_slice().into(),
        rhs_contracting_dims: [0].as_slice().into(),
        lhs_batch_dims: [2].as_slice().into(),
        rhs_batch_dims: [2].as_slice().into(),
    };

    // A nonzero view offset shifts every lane's chunk split.
    for offset in [0_usize, 3] {
        let mut out_storage = vec![-7.0_f64; offset + pitch * batch];
        let mut backend = crate::test_support::cpu_engine(4).unwrap();
        {
            let out = TypedTensorViewMut::from_slice(
                [m, m, batch],
                [1, m as isize, pitch as isize],
                offset as isize,
                &mut out_storage,
            )
            .unwrap();
            crate::test_support::on_engine(&mut backend, |session| {
                session.dot_general_read_into(
                    TensorRead::from_tensor(&lhs),
                    TensorRead::from_tensor(&rhs),
                    &config,
                    TensorWrite::from_view(TensorViewMut::F64(out)),
                )
            })
            .unwrap()
            .unwrap();
        }

        for b in 0..batch {
            for j in 0..m {
                for i in 0..m {
                    let expected: f64 = (0..m)
                        .map(|k| lhs_data[i + m * k + m * m * b] * rhs_data[k + m * j + m * m * b])
                        .sum();
                    assert_eq!(
                        out_storage[offset + i + m * j + pitch * b],
                        expected,
                        "offset {offset}: ({i}, {j}, {b})"
                    );
                }
            }
        }
        assert!(out_storage
            .iter()
            .enumerate()
            .filter(|(index, _)| *index < offset || (index - offset) % pitch >= m * m)
            .all(|(_, &value)| value == -7.0));
    }
}

#[test]
fn auto_allocated_batched_gemm_on_lanes_matches_reference() {
    use tenferro_tensor::DotGeneralConfig;

    // The allocating entry point writes an uninitialized pooled destination;
    // 1024 4x4x4 products clear the Auto lane gate at 4 threads, so each lane
    // fully overwrites its own chunk of that destination (#1898).
    let (m, batch) = (4_usize, 1024_usize);
    let lhs_data: Vec<f64> = (0..m * m * batch).map(|i| (i % 13) as f64 - 6.0).collect();
    let rhs_data: Vec<f64> = (0..m * m * batch).map(|i| (i % 7) as f64 * 0.5).collect();
    let lhs = Tensor::from_vec_col_major(vec![m, m, batch], lhs_data.clone()).unwrap();
    let rhs = Tensor::from_vec_col_major(vec![m, m, batch], rhs_data.clone()).unwrap();
    let config = DotGeneralConfig {
        lhs_contracting_dims: [1].as_slice().into(),
        rhs_contracting_dims: [0].as_slice().into(),
        lhs_batch_dims: [2].as_slice().into(),
        rhs_batch_dims: [2].as_slice().into(),
    };

    let mut backend = crate::test_support::cpu_engine(4).unwrap();
    let output = crate::test_support::on_engine(&mut backend, |session| {
        session.dot_general_read(
            TensorRead::from_tensor(&lhs),
            TensorRead::from_tensor(&rhs),
            &config,
        )
    })
    .unwrap()
    .unwrap();

    assert_eq!(output.shape(), &[m, m, batch]);
    let out = output.as_slice::<f64>().unwrap();
    for b in 0..batch {
        for j in 0..m {
            for i in 0..m {
                let expected: f64 = (0..m)
                    .map(|k| lhs_data[i + m * k + m * m * b] * rhs_data[k + m * j + m * m * b])
                    .sum();
                assert_eq!(out[i + m * j + m * m * b], expected, "({i}, {j}, {b})");
            }
        }
    }
}

/// #2214: the dot read hooks consume a view operand instead of staging a copy.
///
/// The counter is the canonicalization spy: `to_contiguous_read` increments it
/// only when it actually copies a view, so a hook that delegates to the
/// trait default (which canonicalizes both reads) fails this test. Provider
/// packing inside the contraction lane is a different layer and is not counted
/// here.
#[test]
fn dot_read_hooks_do_not_stage_view_operands() {
    use tenferro_tensor::{SessionCachedDot, TensorStructural, TensorValue};

    let lhs = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![2, 3], vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]).unwrap(),
    );
    let rhs = Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![3, 2], vec![1.0, 0.0, 0.0, 0.0, 1.0, 0.0]).unwrap(),
    );
    let config = DotGeneralConfig {
        lhs_contracting_dims: [1].as_slice().into(),
        rhs_contracting_dims: [0].as_slice().into(),
        lhs_batch_dims: [].as_slice().into(),
        rhs_batch_dims: [].as_slice().into(),
    };
    let mut engine = crate::CpuEngine::serial();

    // Views are the read this hook must not materialize.
    let lhs_value = TensorValue::from_tensor(lhs);
    let rhs_value = TensorValue::from_tensor(rhs);

    let out = engine
        .dot_general_read_cached(
            Some(0),
            TensorRead::from_view(lhs_value.tensor_view()),
            TensorRead::from_view(rhs_value.tensor_view()),
            &config,
        )
        .unwrap();
    assert_eq!(out.as_slice::<f64>().unwrap(), &[1.0, 2.0, 3.0, 4.0]);
    assert_eq!(
        engine.resources.read_stagings(),
        0,
        "dot_general_read_cached must not stage a view operand"
    );

    let conj = engine
        .dot_general_with_conj_read_cached(
            Some(0),
            TensorRead::from_view(lhs_value.tensor_view()),
            TensorRead::from_view(rhs_value.tensor_view()),
            &config,
            false,
            true,
        )
        .unwrap();
    assert_eq!(conj.as_slice::<f64>().unwrap(), &[1.0, 2.0, 3.0, 4.0]);
    assert_eq!(
        engine.resources.read_stagings(),
        0,
        "dot_general_with_conj_read_cached must not stage a view operand"
    );

    let eager_conj = engine
        .dot_general_with_conj_read(
            TensorRead::from_view(lhs_value.tensor_view()),
            TensorRead::from_view(rhs_value.tensor_view()),
            &config,
            true,
            false,
        )
        .unwrap();
    assert_eq!(eager_conj.as_slice::<f64>().unwrap(), &[1.0, 2.0, 3.0, 4.0]);
    assert_eq!(
        engine.resources.read_stagings(),
        0,
        "dot_general_with_conj_read must not stage a view operand"
    );

    // Positive control: canonicalizing a view is counted, so the assertions
    // above cannot pass because the counter is dead.
    let _ = engine
        .to_contiguous_read(TensorRead::from_view(lhs_value.tensor_view()))
        .unwrap();
    assert_eq!(engine.resources.read_stagings(), 1);
}

/// #2214: every read entry that claims to consume a view must not stage a copy.
///
/// The counter is raised only when `to_contiguous_read` actually canonicalizes a
/// view, so an entry that delegates to a materializing default fails here. Each
/// entry is exercised with borrowed views, which is the read a typed tensor
/// produces.
#[test]
fn read_entries_consume_views_without_staging() {
    use tenferro_tensor::{
        Engine, SessionCachedDot, SliceConfig, TensorAnalytic, TensorDot, TensorElementwise,
        TensorIndexing, TensorReduction, TensorStructural, TensorValue, TensorView,
        TypedTensorView,
    };

    let a = TensorValue::from_tensor(Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![2, 3], vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0]).unwrap(),
    ));
    let b = TensorValue::from_tensor(Tensor::from_typed::<f64>(
        TypedTensor::from_vec_col_major(vec![3, 2], vec![1.0, 0.0, 0.0, 0.0, 1.0, 0.0]).unwrap(),
    ));
    let dot_config = DotGeneralConfig {
        lhs_contracting_dims: [1].as_slice().into(),
        rhs_contracting_dims: [0].as_slice().into(),
        lhs_batch_dims: [].as_slice().into(),
        rhs_batch_dims: [].as_slice().into(),
    };
    let mut engine = crate::CpuEngine::serial();

    macro_rules! check {
        ($label:literal, $call:expr) => {{
            let before = engine.resources.read_stagings();
            let _ = $call.unwrap();
            assert_eq!(
                engine.resources.read_stagings(),
                before,
                concat!($label, " must not stage a view operand")
            );
        }};
    }

    check!(
        "add_read",
        engine.add_read(
            TensorRead::from_view(a.tensor_view()),
            TensorRead::from_view(a.tensor_view())
        )
    );
    check!(
        "exp_read",
        engine.exp_read(TensorRead::from_view(a.tensor_view()))
    );
    check!(
        "reduce_sum_read",
        engine.reduce_sum_read(TensorRead::from_view(a.tensor_view()), &[0])
    );
    check!(
        "transpose_read",
        engine.transpose_read(TensorRead::from_view(a.tensor_view()), &[1, 0])
    );
    check!(
        "reshape_read",
        engine.reshape_read(TensorRead::from_view(a.tensor_view()), &[3, 2])
    );
    check!(
        "broadcast_in_dim_read",
        engine.broadcast_in_dim_read(TensorRead::from_view(a.tensor_view()), &[2, 3, 2], &[0, 1])
    );
    check!(
        "vdot_read",
        engine.vdot_read(
            TensorRead::from_view(a.tensor_view()),
            TensorRead::from_view(a.tensor_view())
        )
    );
    check!(
        "norm_squared_read",
        engine.norm_squared_read(TensorRead::from_view(a.tensor_view()))
    );
    check!(
        "dot_general_read",
        engine.dot_general_read(
            TensorRead::from_view(a.tensor_view()),
            TensorRead::from_view(b.tensor_view()),
            &dot_config
        )
    );
    check!(
        "dot_general_read_cached",
        engine.dot_general_read_cached(
            Some(0),
            TensorRead::from_view(a.tensor_view()),
            TensorRead::from_view(b.tensor_view()),
            &dot_config
        )
    );

    // A non-contiguous (transposed) view reaches the slice kernel through its
    // own strides: it must neither stage a copy nor read the wrong elements.
    let transposed_data = [1.0_f64, 2.0, 3.0, 4.0, 5.0, 6.0];
    let transposed =
        TypedTensorView::from_slice(vec![3, 2], vec![2, 1], 0, &transposed_data).unwrap();
    let before = engine.resources.read_stagings();
    let sliced = engine
        .slice_read(
            TensorRead::from_view(TensorView::F64(transposed)),
            &SliceConfig {
                starts: vec![0, 0],
                limits: vec![2, 2],
                strides: vec![1, 1],
            },
        )
        .unwrap();
    assert_eq!(engine.resources.read_stagings(), before);
    assert_eq!(sliced.as_slice::<f64>().unwrap(), &[1.0, 3.0, 2.0, 4.0]);

    // The remaining indexing reads take the same descriptor, so a strided view
    // reaches each kernel unreduced.
    let view = || TypedTensorView::from_slice(vec![3, 2], vec![2, 1], 0, &transposed_data).unwrap();

    let before = engine.resources.read_stagings();
    let reversed = engine
        .reverse_read(TensorRead::from_view(TensorView::F64(view())), &[0])
        .unwrap();
    assert_eq!(engine.resources.read_stagings(), before);
    assert_eq!(
        reversed.as_slice::<f64>().unwrap(),
        &[5.0, 3.0, 1.0, 6.0, 4.0, 2.0]
    );

    let before = engine.resources.read_stagings();
    let padded = engine
        .pad_read(
            TensorRead::from_view(TensorView::F64(view())),
            &PadConfig {
                edge_padding_low: vec![1, 0],
                edge_padding_high: vec![0, 0],
                interior_padding: vec![0, 0],
            },
        )
        .unwrap();
    assert_eq!(engine.resources.read_stagings(), before);
    assert_eq!(padded.shape(), &[4, 2]);

    let before = engine.resources.read_stagings();
    let joined = engine
        .concatenate_read(
            &[
                TensorRead::from_view(TensorView::F64(view())),
                TensorRead::from_view(TensorView::F64(view())),
            ],
            0,
        )
        .unwrap();
    assert_eq!(engine.resources.read_stagings(), before);
    assert_eq!(joined.shape(), &[6, 2]);

    let indices = Tensor::from_vec_col_major(vec![1, 1], vec![1_i64]).unwrap();
    let before = engine.resources.read_stagings();
    let gathered = engine
        .gather_read(
            TensorRead::from_view(TensorView::F64(view())),
            TensorRead::from_tensor(&indices),
            &GatherConfig {
                offset_dims: vec![1],
                collapsed_slice_dims: vec![0],
                start_index_map: vec![0],
                index_vector_dim: 1,
                slice_sizes: vec![1, 2],
            },
        )
        .unwrap();
    assert_eq!(engine.resources.read_stagings(), before);
    assert_eq!(gathered.as_slice::<f64>().unwrap(), &[3.0, 4.0]);

    // Scatter takes both the operand and the updates as descriptors; the
    // updates view skips every other element, and an owned operand mixed with a
    // view update takes the same path.
    let update_data = [10.0_f64, 99.0, 20.0];
    let updates = || TypedTensorView::from_slice(vec![1, 2], vec![1, 2], 0, &update_data).unwrap();
    let scatter_config = ScatterConfig {
        update_window_dims: vec![1],
        inserted_window_dims: vec![0],
        scatter_dims_to_operand_dims: vec![0],
        index_vector_dim: 1,
    };
    let before = engine.resources.read_stagings();
    let scattered = engine
        .scatter_read(
            TensorRead::from_view(TensorView::F64(view())),
            TensorRead::from_tensor(&indices),
            TensorRead::from_view(TensorView::F64(updates())),
            &scatter_config,
        )
        .unwrap();
    assert_eq!(engine.resources.read_stagings(), before);
    assert_eq!(
        scattered.as_slice::<f64>().unwrap(),
        &[1.0, 13.0, 5.0, 2.0, 24.0, 6.0]
    );
    let owned_operand = Tensor::from_vec_col_major(vec![3, 2], vec![0.0_f64; 6]).unwrap();
    let before = engine.resources.read_stagings();
    let scattered = engine
        .scatter_read(
            TensorRead::from_tensor(&owned_operand),
            TensorRead::from_tensor(&indices),
            TensorRead::from_view(TensorView::F64(updates())),
            &scatter_config,
        )
        .unwrap();
    assert_eq!(engine.resources.read_stagings(), before);
    assert_eq!(
        scattered.as_slice::<f64>().unwrap(),
        &[0.0, 10.0, 0.0, 0.0, 20.0, 0.0]
    );
    let starts = Tensor::from_vec_col_major(vec![2], vec![1_i64, 0]).unwrap();
    let before = engine.resources.read_stagings();
    let updated = engine
        .dynamic_update_slice_read(
            TensorRead::from_view(TensorView::F64(view())),
            TensorRead::from_view(TensorView::F64(updates())),
            TensorRead::from_tensor(&starts),
        )
        .unwrap();
    assert_eq!(engine.resources.read_stagings(), before);
    assert_eq!(
        updated.as_slice::<f64>().unwrap(),
        &[1.0, 10.0, 5.0, 2.0, 20.0, 6.0]
    );

    // The structural reads share the same descriptor path.
    let before = engine.resources.read_stagings();
    let lower = engine
        .tril_read(TensorRead::from_view(TensorView::F64(view())), 0)
        .unwrap();
    assert_eq!(engine.resources.read_stagings(), before);
    assert_eq!(
        lower.as_slice::<f64>().unwrap(),
        &[1.0, 3.0, 5.0, 0.0, 4.0, 6.0]
    );

    let before = engine.resources.read_stagings();
    let upper = engine
        .triu_read(TensorRead::from_view(TensorView::F64(view())), 0)
        .unwrap();
    assert_eq!(engine.resources.read_stagings(), before);
    assert_eq!(
        upper.as_slice::<f64>().unwrap(),
        &[1.0, 0.0, 0.0, 2.0, 4.0, 0.0]
    );

    let before = engine.resources.read_stagings();
    let diagonal = engine
        .extract_diagonal_read(TensorRead::from_view(TensorView::F64(view())), 0, 1)
        .unwrap();
    assert_eq!(engine.resources.read_stagings(), before);
    assert_eq!(diagonal.as_slice::<f64>().unwrap(), &[1.0, 4.0]);

    let before = engine.resources.read_stagings();
    let embedded = engine
        .embed_diagonal_read(TensorRead::from_view(TensorView::F64(view())), 0, 1)
        .unwrap();
    assert_eq!(engine.resources.read_stagings(), before);
    assert_eq!(embedded.shape(), &[3, 3, 2]);

    // `to_contiguous_read` is the canonicalization itself: copying a view is its
    // contract, so it is the live positive control for the counter.
    let before = engine.resources.read_stagings();
    let _ = engine
        .to_contiguous_read(TensorRead::from_view(a.tensor_view()))
        .unwrap();
    assert_eq!(engine.resources.read_stagings(), before + 1);
}

/// #2214: each read entry reads a view through its real strides and offset.
///
/// Every layout below is a 3x2 view; each read must equal the owned entry on
/// the compact copy of the same logical values. All layouts share one engine,
/// so same-shape plans are reused across different strides and offsets.
#[test]
fn read_entries_follow_view_strides_and_offsets() {
    use tenferro_tensor::{
        Engine, SliceConfig, TensorIndexing, TensorStructural, TensorView, TypedTensorView,
    };

    struct Layout<'a> {
        data: &'a [f64],
        strides: [isize; 2],
        offset: isize,
        /// Logical values in column-major order.
        logical: [f64; 6],
    }
    let reversed = [0.0_f64, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0];
    let shifted = [9.0_f64, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0];
    let transposed = [1.0_f64, 2.0, 3.0, 4.0, 5.0, 6.0];
    let broadcast = [7.0_f64, 8.0];
    let layouts = [
        Layout {
            data: &transposed,
            strides: [2, 1],
            offset: 0,
            logical: [1.0, 3.0, 5.0, 2.0, 4.0, 6.0],
        },
        Layout {
            data: &reversed,
            strides: [-2, -1],
            offset: 6,
            logical: [6.0, 4.0, 2.0, 5.0, 3.0, 1.0],
        },
        Layout {
            data: &shifted,
            strides: [-2, -1],
            offset: 7,
            logical: [6.0, 4.0, 2.0, 5.0, 3.0, 1.0],
        },
        Layout {
            data: &broadcast,
            strides: [0, 1],
            offset: 0,
            logical: [7.0, 7.0, 7.0, 8.0, 8.0, 8.0],
        },
    ];
    // Updates [10, 20] read backwards from an offset.
    let update_data = [99.0_f64, 20.0, 99.0, 10.0];
    let update_view = || {
        TensorRead::from_view(TensorView::F64(
            TypedTensorView::from_slice(vec![1, 2], vec![1, -2], 3, &update_data).unwrap(),
        ))
    };
    let updates = Tensor::from_vec_col_major(vec![1, 2], vec![10.0_f64, 20.0]).unwrap();
    let indices = Tensor::from_vec_col_major(vec![1, 1], vec![1_i64]).unwrap();
    let starts = Tensor::from_vec_col_major(vec![2], vec![1_i64, 1]).unwrap();
    let update_starts = Tensor::from_vec_col_major(vec![2], vec![2_i64, 0]).unwrap();
    let gather = GatherConfig {
        offset_dims: vec![1],
        collapsed_slice_dims: vec![0],
        start_index_map: vec![0],
        index_vector_dim: 1,
        slice_sizes: vec![1, 2],
    };
    let scatter = ScatterConfig {
        update_window_dims: vec![1],
        inserted_window_dims: vec![0],
        scatter_dims_to_operand_dims: vec![0],
        index_vector_dim: 1,
    };
    let slice = SliceConfig {
        starts: vec![1, 0],
        limits: vec![3, 2],
        strides: vec![1, 1],
    };
    let pad = PadConfig {
        edge_padding_low: vec![1, 0],
        edge_padding_high: vec![0, 1],
        interior_padding: vec![1, 0],
    };
    let values = |tensor: Tensor| tensor.as_slice::<f64>().unwrap().to_vec();

    let mut engine = crate::CpuEngine::serial();
    for Layout {
        data,
        strides,
        offset,
        logical,
    } in layouts
    {
        let view = || {
            TensorRead::from_view(TensorView::F64(
                TypedTensorView::from_slice(vec![3, 2], strides, offset, data).unwrap(),
            ))
        };
        let owned = Tensor::from_vec_col_major(vec![3, 2], logical.to_vec()).unwrap();
        let idx = || TensorRead::from_tensor(&indices);
        let label = format!("strides {strides:?} offset {offset}");
        let before = engine.resources.read_stagings();

        macro_rules! same {
            ($name:literal, $read:expr, $owned:expr) => {
                assert_eq!(
                    values($read.unwrap()),
                    values($owned.unwrap()),
                    "{} on {label}",
                    $name
                );
            };
        }
        same!(
            "gather",
            engine.gather_read(view(), idx(), &gather),
            engine.gather(&owned, &indices, &gather)
        );
        same!(
            "scatter",
            engine.scatter_read(view(), idx(), update_view(), &scatter),
            engine.scatter(&owned, &indices, &updates, &scatter)
        );
        same!(
            "dynamic_update_slice",
            engine.dynamic_update_slice_read(
                view(),
                update_view(),
                TensorRead::from_tensor(&update_starts)
            ),
            engine.dynamic_update_slice(&owned, &updates, &update_starts)
        );
        same!(
            "slice",
            engine.slice_read(view(), &slice),
            engine.slice(&owned, &slice)
        );
        same!(
            "dynamic_slice",
            engine.dynamic_slice_read(view(), TensorRead::from_tensor(&starts), &[2, 1]),
            engine.dynamic_slice(&owned, &starts, &[2, 1])
        );
        same!(
            "pad",
            engine.pad_read(view(), &pad),
            engine.pad(&owned, &pad)
        );
        same!(
            "concatenate",
            engine.concatenate_read(&[view(), TensorRead::from_tensor(&owned)], 1),
            engine.concatenate(&[&owned, &owned], 1)
        );
        same!(
            "reverse",
            engine.reverse_read(view(), &[0, 1]),
            engine.reverse(&owned, &[0, 1])
        );
        same!(
            "tril",
            engine.tril_read(view(), -1),
            engine.tril(&owned, -1)
        );
        same!("triu", engine.triu_read(view(), 1), engine.triu(&owned, 1));
        same!(
            "extract_diagonal",
            engine.extract_diagonal_read(view(), 0, 1),
            engine.extract_diagonal(&owned, 0, 1)
        );
        same!(
            "embed_diagonal",
            engine.embed_diagonal_read(view(), 0, 1),
            engine.embed_diagonal(&owned, 0, 1)
        );
        assert_eq!(engine.resources.read_stagings(), before, "{label} staged");
    }
}

/// #2214: the read entry accepts what the owned entry accepts (Bool, a
/// borrowed `starts` vector) and reports the same errors.
#[test]
fn dynamic_update_slice_read_matches_the_owned_dispatch() {
    use tenferro_tensor::{TensorIndexing, TensorView, TypedTensorView};

    let mut engine = crate::CpuEngine::serial();
    let operand = Tensor::from_vec_col_major(vec![3], vec![false, false, false]).unwrap();
    let update = Tensor::from_vec_col_major(vec![1], vec![true]).unwrap();
    // `starts = [1]`, borrowed as a view with an offset.
    let start_data = [9_i64, 1];
    let starts = || {
        TensorRead::from_view(TensorView::I64(
            TypedTensorView::from_slice(vec![1], vec![1], 1, &start_data).unwrap(),
        ))
    };
    let flags = engine
        .dynamic_update_slice_read(
            TensorRead::from_tensor(&operand),
            TensorRead::from_tensor(&update),
            starts(),
        )
        .unwrap();
    assert_eq!(flags.as_slice::<bool>().unwrap(), &[false, true, false]);

    let real = Tensor::from_vec_col_major(vec![3], vec![0.0_f64; 3]).unwrap();
    let owned_err = engine
        .dynamic_update_slice(
            &real,
            &update,
            &Tensor::from_vec_col_major(vec![1], vec![1_i64]).unwrap(),
        )
        .unwrap_err();
    let read_err = engine
        .dynamic_update_slice_read(
            TensorRead::from_tensor(&real),
            TensorRead::from_tensor(&update),
            starts(),
        )
        .unwrap_err();
    assert_eq!(format!("{read_err:?}"), format!("{owned_err:?}"));
}

/// An empty axis whose extent does not fit in `i64` clamps its start instead
/// of panicking; the compact output layout for that shape then overflows,
/// which is reported as an error.
#[test]
fn dynamic_update_slice_read_clamps_on_a_huge_empty_axis() {
    use tenferro_tensor::{TensorIndexing, TensorView, TypedTensorView};

    let mut engine = crate::CpuEngine::serial();
    let empty: [f64; 0] = [];
    let operand = TypedTensorView::from_slice(vec![usize::MAX, 0], vec![0, 0], 0, &empty).unwrap();
    let update = Tensor::from_vec_col_major(vec![0, 0], Vec::<f64>::new()).unwrap();
    let starts = Tensor::from_vec_col_major(vec![2], vec![0_i64, 0]).unwrap();
    let err = engine
        .dynamic_update_slice_read(
            TensorRead::from_view(TensorView::F64(operand)),
            TensorRead::from_tensor(&update),
            TensorRead::from_tensor(&starts),
        )
        .unwrap_err();
    assert!(
        matches!(
            err,
            crate::Error::BackendSource {
                op: "dynamic_update_slice",
                ..
            }
        ),
        "{err:?}"
    );
}