fuel-dispatch 0.19.1

// SPDX-License-Identifier: MIT OR Apache-2.0
//! Dispatch wrappers + registration for baracuda-kernels-sys-backed
//! kernels. Sibling to `dispatch::register_cuda_kernels` (PTX path) —
//! each kernel here registers as an alternative at the same
//! `(op_kind, dtypes, BackendId::Cuda)` key. Phase 7.6 step 9a's
//! append-on-register semantics keep them composable; step 9b's
//! route picker ranks by `PrecisionGuarantee` + telemetry.
//!
//! The boilerplate-explosion in this file (50+ unary kinds × 4
//! dtypes, plus binary / ternary / etc.) is generated by per-family
//! macros — adding a new (op, dtype) entry is one manifest line
//! here + one matching line in `fuel-cuda-backend::baracuda`.

#![cfg(feature = "cuda")]

use std::sync::{Arc, RwLock};

use fuel_ir::DType;
use fuel_ir::dispatch::OpKind;
use fuel_ir::{Error, Layout, Result, probe::BackendId};

use crate::kernel::{KernelBindingTable, OpParams};
use fuel_memory::Storage;

// Re-use the storage-helpers from dispatch.rs so we read/write
// CudaStorageBytes the same way the PTX wrappers do.
use crate::dispatch::{cuda_input, cuda_output, read_storage, write_storage};

/// Generate one CUDA unary dispatch wrapper that adapts a baracuda
/// `fn(&CudaStorageBytes, Option<&Layout>) -> Result<CudaStorageBytes>`
/// kernel into the `KernelRef` signature the binding table expects.
///
/// Arguments:
/// - `$wrapper_name` — the function symbol to emit.
/// - `$baracuda_fn` — fully-qualified path to the baracuda kernel
///   function (e.g. `fuel_cuda_backend::baracuda::elementwise::unary_neg_f32`).
macro_rules! cuda_unary_baracuda_wrapper {
    ($wrapper_name:ident, $baracuda_fn:path) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            layouts: &[Layout],
            _params: &OpParams,
        ) -> Result<()> {
            if inputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(stringify!($wrapper_name), ": expected 1 input, got {}"),
                    inputs.len(),
                ))
                .bt());
            }
            if outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(stringify!($wrapper_name), ": expected 1 output, got {}"),
                    outputs.len(),
                ))
                .bt());
            }
            let layout = layouts.first();
            let in_guard = read_storage(&inputs[0])?;
            let mut out_guard = write_storage(&outputs[0])?;
            let src_cuda = cuda_input(&in_guard)?;
            let out_cuda = cuda_output(&mut out_guard)?;
            // Write-into-output (CapturedRun): the `_into` driver writes into
            // the executor-provided outputs[0] — no alloc + swap.
            $baracuda_fn(src_cuda, layout, out_cuda)
        }
    };
}

/// Generate one CUDA in-place unary dispatch wrapper. Phase 3e of
/// in-place ops infrastructure. The executor's `WorkItemKind::InplaceKernel`
/// arm passes the target Arc as `outputs[0]` with `inputs=[]`; the
/// wrapper acquires the target's write lock and calls the baracuda
/// in-place kernel (same baracuda symbol as the non-inplace cousin,
/// but with src_ptr == dst_ptr).
macro_rules! cuda_unary_inplace_baracuda_wrapper {
    ($wrapper_name:ident, $baracuda_fn:path) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            _layouts: &[Layout],
            _params: &OpParams,
        ) -> Result<()> {
            if !inputs.is_empty() || outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": expected 0 inputs + 1 output (target adopted by executor), got {} + {}"
                    ),
                    inputs.len(),
                    outputs.len(),
                ))
                .bt());
            }
            let mut out_guard = write_storage(&outputs[0])?;
            let out_cuda = cuda_output(&mut out_guard)?;
            $baracuda_fn(out_cuda)
        }
    };
}

/// Generate one CUDA binary dispatch wrapper. Writes the kernel output
/// directly into the executor-provided `outputs[0]` buffer via the
/// baracuda `_into` (write-into) driver — no internal alloc + swap.
///
/// Byte-identical to the prior alloc-and-replace for normal realizes (the
/// executor pre-sizes `outputs[0]` from the node's output shape), and it
/// saves one device allocation per op. Crucially, it is the enabler for
/// persistent-output CUDA-graph capture: in the executor's capture mode
/// `outputs[0]` is a FIXED-ADDRESS persistent buffer written in place, so
/// no device allocation happens inside the capture scope (where both
/// alloc and host sync are illegal).
macro_rules! cuda_binary_baracuda_wrapper {
    ($wrapper_name:ident, $baracuda_into_fn:path) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            layouts: &[Layout],
            _params: &OpParams,
        ) -> Result<()> {
            if inputs.len() != 2 {
                return Err(Error::Msg(format!(
                    concat!(stringify!($wrapper_name), ": expected 2 inputs, got {}"),
                    inputs.len(),
                ))
                .bt());
            }
            if outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(stringify!($wrapper_name), ": expected 1 output, got {}"),
                    outputs.len(),
                ))
                .bt());
            }
            let lhs_layout = layouts.first();
            let rhs_layout = layouts.get(1);
            let lhs_guard = read_storage(&inputs[0])?;
            let rhs_guard = read_storage(&inputs[1])?;
            let mut out_guard = write_storage(&outputs[0])?;
            let lhs_cuda = cuda_input(&lhs_guard)?;
            let rhs_cuda = cuda_input(&rhs_guard)?;
            let out_cuda = cuda_output(&mut out_guard)?;
            $baracuda_into_fn(lhs_cuda, rhs_cuda, lhs_layout, rhs_layout, out_cuda)
        }
    };
}

/// Generate a CUDA RoPE dispatch wrapper that pulls
/// `(outer_count, seq, head_dim)` from `OpParams::Rope` and
/// forwards to a baracuda RoPE kernel.
macro_rules! cuda_rope_baracuda_wrapper {
    ($wrapper_name:ident, $baracuda_fn:path) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            layouts: &[Layout],
            params: &OpParams,
        ) -> Result<()> {
            if inputs.len() != 1 || outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": expected 1 input + 1 output, got {} + {}",
                    ),
                    inputs.len(),
                    outputs.len(),
                ))
                .bt());
            }
            let (outer_count, seq, head_dim) = match params {
                OpParams::Rope {
                    outer_count,
                    seq,
                    head_dim,
                } => (*outer_count, *seq, *head_dim),
                other => {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": expected OpParams::Rope, got {:?}",
                        ),
                        other,
                    ))
                    .bt());
                }
            };
            let layout = layouts.first();
            let in_guard = read_storage(&inputs[0])?;
            let mut out_guard = write_storage(&outputs[0])?;
            let src_cuda = cuda_input(&in_guard)?;
            let out_cuda = cuda_output(&mut out_guard)?;
            // Write-into-output (CapturedRun): `_into` writes into outputs[0].
            $baracuda_fn(src_cuda, layout, outer_count, seq, head_dim, out_cuda)
        }
    };
}

/// Generate a CUDA index_select dispatch wrapper. Two inputs
/// `(src, indices)`; pulls the shape params from
/// `OpParams::IndexSelect`.
macro_rules! cuda_index_select_baracuda_wrapper {
    ($wrapper_name:ident, $baracuda_fn:path) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            _layouts: &[Layout],
            params: &OpParams,
        ) -> Result<()> {
            if inputs.len() != 2 || outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": expected 2 inputs + 1 output, got {} + {}",
                    ),
                    inputs.len(),
                    outputs.len(),
                ))
                .bt());
            }
            let (outer_count, source_dim_size, n_indices, inner_count) = match params {
                OpParams::IndexSelect {
                    outer_count,
                    source_dim_size,
                    n_indices,
                    inner_count,
                } => (*outer_count, *source_dim_size, *n_indices, *inner_count),
                other => {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": expected OpParams::IndexSelect, got {:?}",
                        ),
                        other,
                    ))
                    .bt());
                }
            };
            let src_guard = read_storage(&inputs[0])?;
            let idx_guard = read_storage(&inputs[1])?;
            if idx_guard.dtype != DType::U32 {
                return Err(Error::Msg(format!(
                    concat!(stringify!($wrapper_name), ": indices must be U32, got {:?}",),
                    idx_guard.dtype,
                ))
                .bt());
            }
            let mut out_guard = write_storage(&outputs[0])?;
            let src_cuda = cuda_input(&src_guard)?;
            let idx_cuda = cuda_input(&idx_guard)?;
            let out_cuda = cuda_output(&mut out_guard)?;
            // Write-into-output (CapturedRun): `_into` writes into outputs[0].
            $baracuda_fn(
                src_cuda,
                idx_cuda,
                outer_count,
                source_dim_size,
                n_indices,
                inner_count,
                out_cuda,
            )
        }
    };
}

// ===========================================================================
// Indexing — IndexSelect wrappers
// ===========================================================================

macro_rules! cuda_gather_baracuda_wrapper {
    ($wrapper_name:ident, $baracuda_fn:path) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            _layouts: &[Layout],
            params: &OpParams,
        ) -> Result<()> {
            if inputs.len() != 2 || outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": expected 2 inputs + 1 output, got {} + {}"
                    ),
                    inputs.len(),
                    outputs.len(),
                ))
                .bt());
            }
            let (source_shape, output_shape, dim) = match params {
                OpParams::Gather {
                    source_shape,
                    output_shape,
                    dim,
                } => (source_shape.clone(), output_shape.clone(), *dim),
                other => {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": expected OpParams::Gather, got {:?}"
                        ),
                        other,
                    ))
                    .bt());
                }
            };
            let src_guard = read_storage(&inputs[0])?;
            let idx_guard = read_storage(&inputs[1])?;
            if idx_guard.dtype != DType::U32 {
                return Err(Error::Msg(format!(
                    concat!(stringify!($wrapper_name), ": indices must be U32, got {:?}"),
                    idx_guard.dtype,
                ))
                .bt());
            }
            let mut out_guard = write_storage(&outputs[0])?;
            let src_cuda = cuda_input(&src_guard)?;
            let idx_cuda = cuda_input(&idx_guard)?;
            let result = $baracuda_fn(src_cuda, idx_cuda, &source_shape, &output_shape, dim)?;
            let out_cuda = cuda_output(&mut out_guard)?;
            *out_cuda = result;
            Ok(())
        }
    };
}

macro_rules! cuda_scatter_add_baracuda_wrapper {
    ($wrapper_name:ident, $baracuda_fn:path) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            _layouts: &[Layout],
            params: &OpParams,
        ) -> Result<()> {
            if inputs.len() != 3 || outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": expected 3 inputs + 1 output, got {} + {}"
                    ),
                    inputs.len(),
                    outputs.len(),
                ))
                .bt());
            }
            let (base_shape, src_shape, dim) = match params {
                OpParams::ScatterAdd {
                    base_shape,
                    src_shape,
                    dim,
                } => (base_shape.clone(), src_shape.clone(), *dim),
                other => {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": expected OpParams::ScatterAdd, got {:?}"
                        ),
                        other,
                    ))
                    .bt());
                }
            };
            let base_guard = read_storage(&inputs[0])?;
            let idx_guard = read_storage(&inputs[1])?;
            if idx_guard.dtype != DType::U32 {
                return Err(Error::Msg(format!(
                    concat!(stringify!($wrapper_name), ": indices must be U32, got {:?}"),
                    idx_guard.dtype,
                ))
                .bt());
            }
            let src_guard = read_storage(&inputs[2])?;
            let mut out_guard = write_storage(&outputs[0])?;
            let base_cuda = cuda_input(&base_guard)?;
            let idx_cuda = cuda_input(&idx_guard)?;
            let src_cuda = cuda_input(&src_guard)?;
            let result = $baracuda_fn(base_cuda, idx_cuda, src_cuda, &base_shape, &src_shape, dim)?;
            let out_cuda = cuda_output(&mut out_guard)?;
            *out_cuda = result;
            Ok(())
        }
    };
}

macro_rules! cuda_masked_fill_baracuda_wrapper {
    ($wrapper_name:ident, $baracuda_fn:path) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            _layouts: &[Layout],
            params: &OpParams,
        ) -> Result<()> {
            if inputs.len() != 2 || outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": expected 2 inputs + 1 output, got {} + {}"
                    ),
                    inputs.len(),
                    outputs.len(),
                ))
                .bt());
            }
            let fill_bytes = match params {
                OpParams::MaskedFill { fill_bytes } => fill_bytes.clone(),
                other => {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": expected OpParams::MaskedFill, got {:?}"
                        ),
                        other,
                    ))
                    .bt());
                }
            };
            let src_guard = read_storage(&inputs[0])?;
            let mask_guard = read_storage(&inputs[1])?;
            let mut out_guard = write_storage(&outputs[0])?;
            let src_cuda = cuda_input(&src_guard)?;
            let mask_cuda = cuda_input(&mask_guard)?;
            let result = $baracuda_fn(src_cuda, mask_cuda, &fill_bytes)?;
            let out_cuda = cuda_output(&mut out_guard)?;
            *out_cuda = result;
            Ok(())
        }
    };
}

pub mod indexing {
    use super::*;
    use fuel_cuda_backend::baracuda::indexing as bk;

    cuda_index_select_baracuda_wrapper!(index_select_f32, bk::index_select_f32_into);
    cuda_index_select_baracuda_wrapper!(index_select_f64, bk::index_select_f64_into);
    cuda_index_select_baracuda_wrapper!(index_select_i32, bk::index_select_i32_into);

    cuda_gather_baracuda_wrapper!(gather_f32, bk::gather_f32);
    cuda_gather_baracuda_wrapper!(gather_f64, bk::gather_f64);
    cuda_gather_baracuda_wrapper!(gather_i32, bk::gather_i32);

    cuda_scatter_add_baracuda_wrapper!(scatter_add_f32, bk::scatter_add_f32);
    cuda_scatter_add_baracuda_wrapper!(scatter_add_f64, bk::scatter_add_f64);

    cuda_masked_fill_baracuda_wrapper!(masked_fill_f32, bk::masked_fill_f32);
    cuda_masked_fill_baracuda_wrapper!(masked_fill_f64, bk::masked_fill_f64);
    cuda_masked_fill_baracuda_wrapper!(masked_fill_i32, bk::masked_fill_i32);
}

// ===========================================================================
// Int-GEMM — S8/U8 RRR Identity (W8A8 phase 1)
// ===========================================================================
//
// Baracuda's `gemm_{s8,u8}_rrr_sm80_run` are RRR-layout (row-major ×
// row-major → row-major), Identity-epilogue Int8 tensor-core GEMM.
// Mirror `Op::MatMul`'s shape contract directly — registering at
// `(MatMul, [I8, I8, I8], Cuda)` and `[U8, U8, U8]` keys, so the
// existing `OpParams::Matmul { m, n, k, lhs/rhs_batch_dims }` flows
// through unchanged.
//
// Bias / BiasRelu / BiasGelu / BiasSilu epilogues are not yet wired
// — those need a `FusedLinearInt` op (or extending FusedLinear with
// int dtypes), which is a separate session.

macro_rules! cuda_matmul_baracuda_wrapper {
    ($wrapper_name:ident, $baracuda_fn:path $(,)?) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            _layouts: &[Layout],
            params: &OpParams,
        ) -> Result<()> {
            if inputs.len() != 2 || outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": expected 2 inputs + 1 output, got {} + {}",
                    ),
                    inputs.len(),
                    outputs.len(),
                ))
                .bt());
            }
            let (lhs_batch_dims, rhs_batch_dims, m, n, k) = match params {
                OpParams::Matmul {
                    lhs_batch_dims,
                    rhs_batch_dims,
                    m,
                    n,
                    k,
                    ..
                } => (lhs_batch_dims.clone(), rhs_batch_dims.clone(), *m, *n, *k),
                other => {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": expected OpParams::Matmul, got {:?}"
                        ),
                        other,
                    ))
                    .bt());
                }
            };
            let lhs_guard = read_storage(&inputs[0])?;
            let rhs_guard = read_storage(&inputs[1])?;
            let mut out_guard = write_storage(&outputs[0])?;
            let lhs_cuda = cuda_input(&lhs_guard)?;
            let rhs_cuda = cuda_input(&rhs_guard)?;
            let result = $baracuda_fn(
                lhs_cuda,
                rhs_cuda,
                &lhs_batch_dims,
                &rhs_batch_dims,
                m,
                n,
                k,
            )?;
            let out_cuda = cuda_output(&mut out_guard)?;
            *out_cuda = result;
            Ok(())
        }
    };
}

/// Write-into-output dense-matmul wrapper (CapturedRun). Same as
/// `cuda_matmul_baracuda_wrapper!` but calls the `_into` gemm driver,
/// writing the product into the executor-provided `outputs[0]` — the
/// capture-safe path (no alloc + swap). Used for the FP dense matmuls
/// (`matmul_f32/f64/f16/bf16`); the int8 `gemm_int` path keeps the
/// allocating wrapper (not in the capturable decode set).
macro_rules! cuda_matmul_into_baracuda_wrapper {
    ($wrapper_name:ident, $baracuda_into_fn:path $(,)?) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            _layouts: &[Layout],
            params: &OpParams,
        ) -> Result<()> {
            if inputs.len() != 2 || outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": expected 2 inputs + 1 output, got {} + {}",
                    ),
                    inputs.len(),
                    outputs.len(),
                ))
                .bt());
            }
            let (lhs_batch_dims, rhs_batch_dims, m, n, k) = match params {
                OpParams::Matmul {
                    lhs_batch_dims,
                    rhs_batch_dims,
                    m,
                    n,
                    k,
                    ..
                } => (lhs_batch_dims.clone(), rhs_batch_dims.clone(), *m, *n, *k),
                other => {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": expected OpParams::Matmul, got {:?}"
                        ),
                        other,
                    ))
                    .bt());
                }
            };
            let lhs_guard = read_storage(&inputs[0])?;
            let rhs_guard = read_storage(&inputs[1])?;
            let mut out_guard = write_storage(&outputs[0])?;
            let lhs_cuda = cuda_input(&lhs_guard)?;
            let rhs_cuda = cuda_input(&rhs_guard)?;
            let out_cuda = cuda_output(&mut out_guard)?;
            $baracuda_into_fn(
                lhs_cuda,
                rhs_cuda,
                &lhs_batch_dims,
                &rhs_batch_dims,
                m,
                n,
                k,
                out_cuda,
            )
        }
    };
}

pub mod gemm_int {
    use super::*;
    use fuel_cuda_backend::baracuda::gemm_int as bk;

    cuda_matmul_baracuda_wrapper!(gemm_s8_rrr, bk::gemm_s8_rrr);
    cuda_matmul_baracuda_wrapper!(gemm_u8_rrr, bk::gemm_u8_rrr);
}

// ===========================================================================
// Dense FP GEMM — Phase 74 `gemm_dense` facade (alpha.67)
// ===========================================================================
//
// cuBLAS-backed dense matmul at `(MatMul, [T, T, T], Cuda)` for
// T ∈ {F32, F64, F16, BF16}. Answers Fuel's 2026-06-10 ask and
// retires the `register_cuda_kernels` matmul residue: the cuBLAS f32
// path and the CUTLASS bf16/f16 byte paths are replaced by this one
// family. f64 is net-new CUDA matmul coverage; GQA / broadcast
// batches are served uniformly at every dtype (the CUTLASS byte path
// rejected them). f32 is IEEE binary32 (cuBLAS default math mode,
// not TF32); f16/bf16 accumulate in f32.

pub mod gemm_dense {
    use super::*;
    use fuel_cuda_backend::baracuda::gemm_dense as bk;

    cuda_matmul_into_baracuda_wrapper!(matmul_f32, bk::matmul_f32_into);
    cuda_matmul_into_baracuda_wrapper!(matmul_f64, bk::matmul_f64_into);
    cuda_matmul_into_baracuda_wrapper!(matmul_f16, bk::matmul_f16_into);
    cuda_matmul_into_baracuda_wrapper!(matmul_bf16, bk::matmul_bf16_into);
}

// ===========================================================================
// Broadcast-reverse reductions — ReduceSumTo / ReduceMaxTo
// ===========================================================================
//
// Autograd's gradient-of-broadcast primitive. The baracuda
// `reduce_{sum,max}_to_*` symbols have shipped sys-only since
// alpha.46; the 2026-06-10 ask-reply surfaced them. Replaces the
// f32-only `register_cuda_kernels` residue with full 4-dtype
// coverage — f16/bf16/f64 training graphs with broadcasts no longer
// fall back to CPU at every gradient edge. The FFI takes the input's
// strides, so these register with `strided_input` caps and skip the
// auto-Contiguize gate.

macro_rules! cuda_reduce_to_baracuda_wrapper {
    ($wrapper_name:ident, $baracuda_fn:path, $variant:ident $(,)?) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            layouts: &[Layout],
            params: &OpParams,
        ) -> Result<()> {
            if inputs.len() != 1 || outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": expected 1 input + 1 output, got {} + {}",
                    ),
                    inputs.len(),
                    outputs.len(),
                ))
                .bt());
            }
            let (input_shape, output_shape) = match params {
                OpParams::$variant {
                    input_shape,
                    output_shape,
                } => (input_shape.clone(), output_shape.clone()),
                other => {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": expected OpParams::",
                            stringify!($variant),
                            ", got {:?}",
                        ),
                        other,
                    ))
                    .bt());
                }
            };
            let input_layout = layouts.first().ok_or_else(|| {
                Error::Msg(concat!(stringify!($wrapper_name), ": layouts empty").to_string()).bt()
            })?;
            let in_guard = read_storage(&inputs[0])?;
            let mut out_guard = write_storage(&outputs[0])?;
            let src_cuda = cuda_input(&in_guard)?;
            let result = $baracuda_fn(src_cuda, input_layout, &input_shape, &output_shape)?;
            let out_cuda = cuda_output(&mut out_guard)?;
            *out_cuda = result;
            Ok(())
        }
    };
}

pub mod reduce_to {
    use super::*;
    use fuel_cuda_backend::baracuda::reduce_to as bk;

    cuda_reduce_to_baracuda_wrapper!(sum_to_f32, bk::reduce_sum_to_f32, ReduceSumTo);
    cuda_reduce_to_baracuda_wrapper!(sum_to_f64, bk::reduce_sum_to_f64, ReduceSumTo);
    cuda_reduce_to_baracuda_wrapper!(sum_to_f16, bk::reduce_sum_to_f16, ReduceSumTo);
    cuda_reduce_to_baracuda_wrapper!(sum_to_bf16, bk::reduce_sum_to_bf16, ReduceSumTo);

    cuda_reduce_to_baracuda_wrapper!(max_to_f32, bk::reduce_max_to_f32, ReduceMaxTo);
    cuda_reduce_to_baracuda_wrapper!(max_to_f64, bk::reduce_max_to_f64, ReduceMaxTo);
    cuda_reduce_to_baracuda_wrapper!(max_to_f16, bk::reduce_max_to_f16, ReduceMaxTo);
    cuda_reduce_to_baracuda_wrapper!(max_to_bf16, bk::reduce_max_to_bf16, ReduceMaxTo);
}

// ===========================================================================
// CausalConv1d — baracuda Phase 50 via the Fuel-prepad bridge
// ===========================================================================
//
// Moved here from `register_cuda_kernels` (2026-06-10): the kernels
// were always baracuda-backed (`causal_conv1d_fuel_prepad_*` strips
// baracuda's `kernel - 1` leading pad timesteps via one `cuMemcpy2D`
// D→D so the output matches the CPU kernel bit-for-bit); this file is
// the architectural home for baracuda coverage.

macro_rules! cuda_causal_conv1d_baracuda_wrapper {
    ($wrapper_name:ident, $baracuda_fn:path $(,)?) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            _layouts: &[Layout],
            params: &OpParams,
        ) -> Result<()> {
            if inputs.len() != 3 {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": expected 3 inputs (x, weight, bias), got {}",
                    ),
                    inputs.len(),
                ))
                .bt());
            }
            if outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(stringify!($wrapper_name), ": expected 1 output, got {}"),
                    outputs.len(),
                ))
                .bt());
            }
            let (batch, channels, seq_in, seq_out, kernel, use_silu) = match params {
                OpParams::CausalConv1d {
                    batch,
                    channels,
                    seq_in,
                    seq_out,
                    kernel,
                    use_silu,
                } => (*batch, *channels, *seq_in, *seq_out, *kernel, *use_silu),
                other => {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": expected OpParams::CausalConv1d, got {:?}",
                        ),
                        other,
                    ))
                    .bt());
                }
            };
            let x_guard = read_storage(&inputs[0])?;
            let w_guard = read_storage(&inputs[1])?;
            let bias_guard = read_storage(&inputs[2])?;
            let mut out_guard = write_storage(&outputs[0])?;
            let x_cuda = cuda_input(&x_guard)?;
            let w_cuda = cuda_input(&w_guard)?;
            let bias_cuda = cuda_input(&bias_guard)?;
            let result = $baracuda_fn(
                x_cuda, w_cuda, bias_cuda, batch, channels, seq_in, seq_out, kernel, use_silu,
            )?;
            let out_cuda = cuda_output(&mut out_guard)?;
            *out_cuda = result;
            Ok(())
        }
    };
}

pub mod conv1d {
    use super::*;
    use fuel_cuda_backend::baracuda::mamba as bk;

    cuda_causal_conv1d_baracuda_wrapper!(causal_conv1d_f32, bk::causal_conv1d_fuel_prepad_f32);
    cuda_causal_conv1d_baracuda_wrapper!(causal_conv1d_f64, bk::causal_conv1d_fuel_prepad_f64);
    cuda_causal_conv1d_baracuda_wrapper!(causal_conv1d_bf16, bk::causal_conv1d_fuel_prepad_bf16);
    cuda_causal_conv1d_baracuda_wrapper!(causal_conv1d_f16, bk::causal_conv1d_fuel_prepad_f16);
}

// ===========================================================================
// ArgMaxDim / ArgMinDim — U32-output variants (alpha.28)
// ===========================================================================
//
// Pulls `dim` from `OpParams::Reduce { dims, keepdim: _ }` with the
// single-dim constraint Fuel's PTX argmax/argmin paths enforce.
// Output is U32 indices in the squeezed (rank N-1) shape.

macro_rules! cuda_arg_reduce_baracuda_wrapper {
    ($wrapper_name:ident, $baracuda_fn:path $(,)?) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            layouts: &[Layout],
            params: &OpParams,
        ) -> Result<()> {
            if inputs.len() != 1 || outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": expected 1 input + 1 output, got {} + {}"
                    ),
                    inputs.len(),
                    outputs.len(),
                ))
                .bt());
            }
            let dim = match params {
                OpParams::Reduce { dims, keepdim: _ } => {
                    if dims.len() != 1 {
                        return Err(Error::Msg(format!(
                            concat!(
                                stringify!($wrapper_name),
                                ": expected a single reduce dim, got {:?}"
                            ),
                            dims,
                        ))
                        .bt());
                    }
                    dims[0]
                }
                other => {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": expected OpParams::Reduce, got {:?}"
                        ),
                        other,
                    ))
                    .bt());
                }
            };
            let layout = layouts.first().ok_or_else(|| {
                Error::Msg(format!(concat!(
                    stringify!($wrapper_name),
                    ": layouts empty"
                ),))
                .bt()
            })?;
            let in_guard = read_storage(&inputs[0])?;
            let mut out_guard = write_storage(&outputs[0])?;
            let src_cuda = cuda_input(&in_guard)?;
            let result = $baracuda_fn(src_cuda, layout, dim)?;
            let out_cuda = cuda_output(&mut out_guard)?;
            *out_cuda = result;
            Ok(())
        }
    };
}

pub mod arg_reduce {
    use super::*;
    use fuel_cuda_backend::baracuda::arg_reduce as bk;

    cuda_arg_reduce_baracuda_wrapper!(argmax_dim_u32_f32, bk::argmax_dim_u32_f32);
    cuda_arg_reduce_baracuda_wrapper!(argmin_dim_u32_f32, bk::argmin_dim_u32_f32);
    cuda_arg_reduce_baracuda_wrapper!(argmax_dim_u32_f64, bk::argmax_dim_u32_f64);
    cuda_arg_reduce_baracuda_wrapper!(argmin_dim_u32_f64, bk::argmin_dim_u32_f64);
    cuda_arg_reduce_baracuda_wrapper!(argmax_dim_u32_f16, bk::argmax_dim_u32_f16);
    cuda_arg_reduce_baracuda_wrapper!(argmin_dim_u32_f16, bk::argmin_dim_u32_f16);
    cuda_arg_reduce_baracuda_wrapper!(argmax_dim_u32_bf16, bk::argmax_dim_u32_bf16);
    cuda_arg_reduce_baracuda_wrapper!(argmin_dim_u32_bf16, bk::argmin_dim_u32_bf16);
}

// ===========================================================================
// PowI — `y = x^n` with integer exponent (OpParams::PowI { exp })
// ===========================================================================
//
// Alpha.28 added `unary_powi_<dtype>_run` over f32/f64/f16/bf16 plus
// matching backward kernels. Power-by-squaring on device — correct on
// negative bases, no `pow(x, n as f32)` shim needed. Exponent ships
// via `p0: f32` (round-trips exactly for |n| ≤ 2^24).

macro_rules! cuda_powi_baracuda_wrapper {
    ($wrapper_name:ident, $baracuda_fn:path $(,)?) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            layouts: &[Layout],
            params: &OpParams,
        ) -> Result<()> {
            if inputs.len() != 1 || outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": expected 1 input + 1 output, got {} + {}"
                    ),
                    inputs.len(),
                    outputs.len(),
                ))
                .bt());
            }
            let exp = match params {
                OpParams::PowI { exp } => *exp,
                other => {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": expected OpParams::PowI, got {:?}"
                        ),
                        other,
                    ))
                    .bt());
                }
            };
            let layout = layouts.first();
            let in_guard = read_storage(&inputs[0])?;
            let mut out_guard = write_storage(&outputs[0])?;
            let src_cuda = cuda_input(&in_guard)?;
            let result = $baracuda_fn(src_cuda, layout, exp)?;
            let out_cuda = cuda_output(&mut out_guard)?;
            *out_cuda = result;
            Ok(())
        }
    };
}

pub mod powi {
    use super::*;
    use fuel_cuda_backend::baracuda::powi as bk;

    cuda_powi_baracuda_wrapper!(powi_f32, bk::powi_f32);
    cuda_powi_baracuda_wrapper!(powi_f64, bk::powi_f64);
    cuda_powi_baracuda_wrapper!(powi_f16, bk::powi_f16);
    cuda_powi_baracuda_wrapper!(powi_bf16, bk::powi_bf16);
}

// PowI backward — `(x, upstream) → grad_x = exp · x^(exp-1) ·
// upstream`. Two inputs (x at index 0, upstream at index 1); pulls
// the same `exp` as the forward through OpParams::PowI.
macro_rules! cuda_powi_backward_baracuda_wrapper {
    ($wrapper_name:ident, $baracuda_fn:path $(,)?) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            layouts: &[Layout],
            params: &OpParams,
        ) -> Result<()> {
            if inputs.len() != 2 || outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": expected 2 inputs + 1 output, got {} + {}"
                    ),
                    inputs.len(),
                    outputs.len(),
                ))
                .bt());
            }
            let exp = match params {
                OpParams::PowI { exp } => *exp,
                other => {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": expected OpParams::PowI, got {:?}"
                        ),
                        other,
                    ))
                    .bt());
                }
            };
            let layout = layouts.first();
            let x_guard = read_storage(&inputs[0])?;
            let up_guard = read_storage(&inputs[1])?;
            let mut out_guard = write_storage(&outputs[0])?;
            let x_cuda = cuda_input(&x_guard)?;
            let up_cuda = cuda_input(&up_guard)?;
            let result = $baracuda_fn(x_cuda, up_cuda, layout, exp)?;
            let out_cuda = cuda_output(&mut out_guard)?;
            *out_cuda = result;
            Ok(())
        }
    };
}

pub mod powi_backward {
    use super::*;
    use fuel_cuda_backend::baracuda::powi as bk;

    cuda_powi_backward_baracuda_wrapper!(powi_backward_f32, bk::powi_backward_f32);
    cuda_powi_backward_baracuda_wrapper!(powi_backward_f64, bk::powi_backward_f64);
    cuda_powi_backward_baracuda_wrapper!(powi_backward_f16, bk::powi_backward_f16);
    cuda_powi_backward_baracuda_wrapper!(powi_backward_bf16, bk::powi_backward_bf16);
}

// ===========================================================================
// Clamp — scalar (min, max) → broadcast-tensor ternary clamp
// ===========================================================================
//
// Fuel's `OpKind::ClampElementwise` carries `(min, max)` in
// `OpParams::Clamp`; baracuda's `ternary_clamp` accepts two bound
// tensors. The per-dtype wrapper allocates two 1-element device
// buffers (one per bound) inside the kernel module and invokes the
// strided variant with stride 0 on the bound axis.

macro_rules! cuda_clamp_baracuda_wrapper {
    ($wrapper_name:ident, $baracuda_fn:path, $scalar_ty:ty $(,)?) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            layouts: &[Layout],
            params: &OpParams,
        ) -> Result<()> {
            if inputs.len() != 1 || outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": expected 1 input + 1 output, got {} + {}"
                    ),
                    inputs.len(),
                    outputs.len(),
                ))
                .bt());
            }
            let (min_f64, max_f64) = match params {
                OpParams::Clamp { min, max } => (*min, *max),
                other => {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": expected OpParams::Clamp, got {:?}"
                        ),
                        other,
                    ))
                    .bt());
                }
            };
            let layout = layouts.first().ok_or_else(|| {
                Error::Msg(
                    concat!(
                        stringify!($wrapper_name),
                        ": Clamp requires an input layout (call with layouts[0])",
                    )
                    .to_string(),
                )
                .bt()
            })?;
            let in_guard = read_storage(&inputs[0])?;
            let mut out_guard = write_storage(&outputs[0])?;
            let src_cuda = cuda_input(&in_guard)?;
            let result = $baracuda_fn(
                src_cuda,
                layout,
                min_f64 as $scalar_ty,
                max_f64 as $scalar_ty,
            )?;
            let out_cuda = cuda_output(&mut out_guard)?;
            *out_cuda = result;
            Ok(())
        }
    };
}

pub mod clamp {
    use super::*;
    use fuel_cuda_backend::baracuda::clamp as bk;

    cuda_clamp_baracuda_wrapper!(clamp_f32, bk::clamp_f32, f32);
    cuda_clamp_baracuda_wrapper!(clamp_f64, bk::clamp_f64, f64);
    cuda_clamp_baracuda_wrapper!(clamp_f16, bk::clamp_f16, f32);
    cuda_clamp_baracuda_wrapper!(clamp_bf16, bk::clamp_bf16, f32);
}

// ===========================================================================
// ClampInplace + PowIInplace — scalar-param in-place op family (B)
// ===========================================================================
//
// Same shape contract as `affine_inplace_*`: the executor's
// `WorkItemKind::InplaceKernel` arm passes the target as `outputs[0]`
// with `inputs=[]`. The wrapper extracts `(min, max)` from
// `OpParams::Clamp` or `exp` from `OpParams::PowI` and dispatches the
// in-place baracuda function.

macro_rules! cuda_clamp_inplace_baracuda_wrapper {
    ($wrapper_name:ident, $baracuda_fn:path, $scalar_ty:ty $(,)?) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            _layouts: &[Layout],
            params: &OpParams,
        ) -> Result<()> {
            if !inputs.is_empty() || outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": expected 0 inputs + 1 output (target adopted by executor), got {} + {}"
                    ),
                    inputs.len(),
                    outputs.len(),
                ))
                .bt());
            }
            let (min_f64, max_f64) = match params {
                OpParams::Clamp { min, max } => (*min, *max),
                other => {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": expected OpParams::Clamp, got {:?}"
                        ),
                        other,
                    ))
                    .bt());
                }
            };
            let mut out_guard = write_storage(&outputs[0])?;
            let out_cuda = cuda_output(&mut out_guard)?;
            $baracuda_fn(out_cuda, min_f64 as $scalar_ty, max_f64 as $scalar_ty)
        }
    };
}

macro_rules! cuda_powi_inplace_baracuda_wrapper {
    ($wrapper_name:ident, $baracuda_fn:path $(,)?) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            _layouts: &[Layout],
            params: &OpParams,
        ) -> Result<()> {
            if !inputs.is_empty() || outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": expected 0 inputs + 1 output (target adopted by executor), got {} + {}"
                    ),
                    inputs.len(),
                    outputs.len(),
                ))
                .bt());
            }
            let exp = match params {
                OpParams::PowI { exp } => *exp,
                other => {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": expected OpParams::PowI, got {:?}"
                        ),
                        other,
                    ))
                    .bt());
                }
            };
            let mut out_guard = write_storage(&outputs[0])?;
            let out_cuda = cuda_output(&mut out_guard)?;
            $baracuda_fn(out_cuda, exp)
        }
    };
}

pub mod clamp_inplace {
    use super::*;
    use fuel_cuda_backend::baracuda::clamp as bk;

    cuda_clamp_inplace_baracuda_wrapper!(clamp_inplace_f32, bk::clamp_inplace_f32, f32);
    cuda_clamp_inplace_baracuda_wrapper!(clamp_inplace_f64, bk::clamp_inplace_f64, f64);
    cuda_clamp_inplace_baracuda_wrapper!(clamp_inplace_f16, bk::clamp_inplace_f16, f32);
    cuda_clamp_inplace_baracuda_wrapper!(clamp_inplace_bf16, bk::clamp_inplace_bf16, f32);
}

pub mod powi_inplace {
    use super::*;
    use fuel_cuda_backend::baracuda::powi as bk;

    cuda_powi_inplace_baracuda_wrapper!(powi_inplace_f32, bk::powi_inplace_f32);
    cuda_powi_inplace_baracuda_wrapper!(powi_inplace_f64, bk::powi_inplace_f64);
    cuda_powi_inplace_baracuda_wrapper!(powi_inplace_f16, bk::powi_inplace_f16);
    cuda_powi_inplace_baracuda_wrapper!(powi_inplace_bf16, bk::powi_inplace_bf16);
}

// ===========================================================================
// Concat — N-ary via N-1 chained baracuda concat2 calls
// ===========================================================================
//
// Baracuda only ships binary `concat2`. The wrapper chains N-1 calls
// to cover Fuel's `OpKind::Concat` N-input semantics. The route
// picker / Judge will tend to prefer the PTX 1-launch path for N > 2
// in practice; baracuda still wins on N == 2 where there's no
// launch-count tax.
//
// CapturedRun executor build-out (4b-γ): N==2 takes a WRITE-INTO path
// (`$baracuda_into_fn` — no allocation, correct in Record/Reuse/None
// executor modes per `op_kind_is_capture_writeinto`'s doc comment). N>2
// PRESERVES the original allocate-and-replace behavior byte-for-byte
// (this wrapper also serves ordinary, non-capture N-ary realize calls
// elsewhere in the codebase) — it still allocates internally for its N-2
// intermediates, then D2D-copies the chain's result into `outputs[0]` so
// the output Arc's IDENTITY is preserved (only its contents change) rather
// than replaced. `capture_decode`'s validation loop rejects N>2 Concat
// before Reuse mode is ever entered, so this N>2 path — despite still
// allocating — can never run mid-capture.

macro_rules! cuda_concat_baracuda_wrapper {
    ($wrapper_name:ident, $baracuda_fn:path, $baracuda_into_fn:path $(,)?) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            layouts: &[Layout],
            params: &OpParams,
        ) -> Result<()> {
            if inputs.is_empty() {
                return Err(Error::Msg(format!(concat!(
                    stringify!($wrapper_name),
                    ": at least 1 input required"
                ),))
                .bt());
            }
            if outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(stringify!($wrapper_name), ": expected 1 output, got {}"),
                    outputs.len(),
                ))
                .bt());
            }
            let (outer_count, input_dim_sizes, inner_count, axis) = match params {
                OpParams::Concat {
                    outer_count,
                    input_dim_sizes,
                    inner_count,
                    axis,
                } => (*outer_count, input_dim_sizes.clone(), *inner_count, *axis),
                other => {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": expected OpParams::Concat, got {:?}"
                        ),
                        other,
                    ))
                    .bt());
                }
            };
            if input_dim_sizes.len() != inputs.len() {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": OpParams declares {} inputs but the work item carries {}",
                    ),
                    input_dim_sizes.len(),
                    inputs.len(),
                ))
                .bt());
            }
            // When the executor supplies input layouts, take the
            // stride-aware path. The executor passes `layouts[0..N]`
            // as input layouts and `layouts[N]` (if present) as the
            // output's layout; we slice to the input prefix.
            let input_layouts = if layouts.len() >= inputs.len() {
                Some(&layouts[..inputs.len()])
            } else {
                None
            };
            let in_guards: Vec<_> = inputs
                .iter()
                .map(read_storage)
                .collect::<Result<Vec<_>>>()?;
            let mut in_cudas: Vec<&fuel_cuda_backend::CudaStorageBytes> =
                Vec::with_capacity(in_guards.len());
            for g in &in_guards {
                in_cudas.push(cuda_input(g)?);
            }
            if in_cudas.len() == 2 {
                // Write-into path (CapturedRun-safe): write directly into
                // outputs[0], no allocation.
                let mut out_guard = write_storage(&outputs[0])?;
                let out_cuda = cuda_output(&mut out_guard)?;
                $baracuda_into_fn(
                    &in_cudas,
                    input_layouts,
                    axis,
                    outer_count,
                    &input_dim_sizes,
                    inner_count,
                    out_cuda,
                )
            } else {
                // N>2: unchanged allocate-and-replace behavior (this
                // wrapper also serves ordinary N-ary realize calls outside
                // any capture scope) — D2D-copy the chain's result into
                // outputs[0] so its Arc identity is preserved.
                let result = $baracuda_fn(
                    &in_cudas,
                    input_layouts,
                    axis,
                    outer_count,
                    &input_dim_sizes,
                    inner_count,
                )?;
                let mut out_guard = write_storage(&outputs[0])?;
                let out_cuda = cuda_output(&mut out_guard)?;
                out_cuda.copy_from_device(&result)
            }
        }
    };
}

pub mod concat {
    use super::*;
    use fuel_cuda_backend::baracuda::concat as bk;

    cuda_concat_baracuda_wrapper!(concat_f32, bk::concat_f32, bk::concat_f32_into);
    cuda_concat_baracuda_wrapper!(concat_f64, bk::concat_f64, bk::concat_f64_into);
    cuda_concat_baracuda_wrapper!(concat_f16, bk::concat_f16, bk::concat_f16_into);
    cuda_concat_baracuda_wrapper!(concat_bf16, bk::concat_bf16, bk::concat_bf16_into);
}

// ===========================================================================
// WriteSlice — in-place rectangular slab assignment (Phase 7.6 step 9c E.3.2)
// ===========================================================================
//
// The pipelined executor's WorkItemKind::WriteSlice branch wires
// `inputs=[source]` and `outputs=[dest_arc]` where outputs[0]'s Arc IS
// the destination's Storage Arc (zero-copy in-place adoption). The
// wrapper holds a write lock on outputs[0] and a read lock on inputs[0],
// then asks baracuda's byte-width-dispatched kernel to mutate dest in
// place.
//
// Byte-width-keyed (not dtype-keyed): both source and dest dtype are
// uniform (canonicalized to `[T_src, T_out]` by build_lookup_dtypes),
// and the kernel only cares about element size — there's no arithmetic.
// One wrapper per byte width covers all aligned-element dtypes.

macro_rules! cuda_write_slice_baracuda_wrapper {
    ($wrapper_name:ident, $baracuda_fn:path) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            _layouts: &[Layout],
            params: &OpParams,
        ) -> Result<()> {
            if inputs.len() != 1 || outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": expected 1 input + 1 output, got {} + {}"
                    ),
                    inputs.len(),
                    outputs.len(),
                ))
                .bt());
            }
            let (dest_shape, ranges) = match params {
                OpParams::WriteSlice {
                    dest_shape, ranges, ..
                } => (dest_shape, ranges),
                other => {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": expected OpParams::WriteSlice, got {:?}"
                        ),
                        other,
                    ))
                    .bt());
                }
            };
            // Derive source_shape + range_start from the ranges. The
            // executor has already validated rank + bounds at compile
            // time; this is the kernel's last-line input shaping.
            let rank = dest_shape.len();
            let mut source_shape = Vec::with_capacity(rank);
            let mut range_start = Vec::with_capacity(rank);
            for &(start, end) in ranges.iter() {
                source_shape.push(end - start);
                range_start.push(start);
            }

            let in_guard = read_storage(&inputs[0])?;
            let mut out_guard = write_storage(&outputs[0])?;
            let src_cuda = cuda_input(&in_guard)?;
            let dest_cuda = cuda_output(&mut out_guard)?;
            $baracuda_fn(dest_cuda, src_cuda, dest_shape, &source_shape, &range_start)
        }
    };
}

pub mod write_slice {
    use super::*;
    use fuel_cuda_backend::baracuda::write_slice as bk;

    cuda_write_slice_baracuda_wrapper!(write_slice_b1, bk::write_slice_b1);
    cuda_write_slice_baracuda_wrapper!(write_slice_b2, bk::write_slice_b2);
    cuda_write_slice_baracuda_wrapper!(write_slice_b4, bk::write_slice_b4);
    cuda_write_slice_baracuda_wrapper!(write_slice_b8, bk::write_slice_b8);
    cuda_write_slice_baracuda_wrapper!(write_slice_b16, bk::write_slice_b16);
}

// ===========================================================================
// WriteSliceRotating — sliding-window KV cache writes (Phase C)
// ===========================================================================
//
// The pipelined executor wires `inputs=[source, position]` +
// `outputs=[dest_arc]`. The wrapper:
//   1. D2H-reads the U32 position scalar (4 bytes).
//   2. Computes `wrapped_start = position % modulus`.
//   3. Splits the write into at most two contiguous chunks across
//      the ring boundary.
//   4. Slices the source via `CudaStorageBytes::slot_copy_to_new`
//      for each chunk and reuses baracuda's existing
//      `write_slice_b{1,2,4,8,16}` kernel for the actual mutation.
//
// v1 constraint: rotating axis must be axis 0 (the leading dim).
// This makes the source-side byte split a simple prefix/suffix —
// matching Mistral / Phi-3 sliding-window K/V layout
// `[seq, n_kv_heads, head_dim]` where seq is the leading axis.

macro_rules! cuda_write_slice_rotating_baracuda_wrapper {
    ($wrapper_name:ident, $baracuda_fn:path, $dtype_size:expr) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            _layouts: &[Layout],
            params: &OpParams,
        ) -> Result<()> {
            if inputs.len() != 2 || outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": expected 2 inputs (source, position) + 1 output (dest), got {} + {}"
                    ),
                    inputs.len(),
                    outputs.len(),
                ))
                .bt());
            }
            let (dest_shape, axis, modulus, ranges) = match params {
                OpParams::WriteSliceRotating {
                    dest_shape,
                    axis,
                    modulus,
                    ranges,
                } => (dest_shape, *axis, *modulus, ranges),
                other => {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": expected OpParams::WriteSliceRotating, got {:?}"
                        ),
                        other,
                    ))
                    .bt());
                }
            };
            let rank = dest_shape.len();
            if ranges.len() != rank {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": ranges.len() {} != dest rank {}"
                    ),
                    ranges.len(),
                    rank,
                ))
                .bt());
            }

            // D2H the 4-byte position scalar.
            let pos_guard = read_storage(&inputs[1])?;
            let pos_cuda = cuda_input(&pos_guard)?;
            let pos_bytes = pos_cuda.to_cpu_bytes()?;
            if pos_bytes.len() < 4 {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": position storage has {} bytes, need >= 4"
                    ),
                    pos_bytes.len(),
                ))
                .bt());
            }
            let position =
                u32::from_ne_bytes([pos_bytes[0], pos_bytes[1], pos_bytes[2], pos_bytes[3]])
                    as usize;
            let wrapped_start = position % modulus;

            // Derive slab_shape from ranges; v1 forces axis=0 so the
            // source byte split is a prefix/suffix.
            let mut slab_shape: Vec<usize> = Vec::with_capacity(rank);
            for (i, &(start, end)) in ranges.iter().enumerate() {
                if end < start {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": ranges[{i}] = ({start}, {end}) has end < start"
                        ),
                        i = i,
                        start = start,
                        end = end,
                    ))
                    .bt());
                }
                let slab = end - start;
                if i == axis && slab > modulus {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": rotating-axis slab {slab} > modulus {modulus}"
                        ),
                        slab = slab,
                        modulus = modulus,
                    ))
                    .bt());
                }
                slab_shape.push(slab);
            }
            let slab_axis_len = slab_shape[axis];
            let slab_elems: usize = slab_shape.iter().copied().product();
            if slab_elems == 0 {
                return Ok(());
            }

            let first_len = slab_axis_len.min(modulus - wrapped_start);
            let second_len = slab_axis_len - first_len;

            // Outer/axis/inner decomposition for strided source extract.
            // For axis 0, outer_count is 1 and the strided extract
            // reduces to a single contiguous copy.
            let outer_count: usize = slab_shape[..axis].iter().copied().product();
            let inner_per_row: usize = slab_shape[axis + 1..].iter().copied().product();
            let row_bytes = inner_per_row * $dtype_size;
            let stride_bytes = slab_axis_len * row_bytes;

            let src_guard = read_storage(&inputs[0])?;
            let src_cuda = cuda_input(&src_guard)?;

            if first_len > 0 {
                let chunk_row_bytes = first_len * row_bytes;
                let src_first = src_cuda.extract_strided_to_new(
                    outer_count,
                    stride_bytes,
                    /* offset_in_outer */ 0,
                    chunk_row_bytes,
                )?;
                let mut sub_source_shape = slab_shape.clone();
                sub_source_shape[axis] = first_len;
                let mut sub_range_start: Vec<usize> = ranges.iter().map(|r| r.0).collect();
                sub_range_start[axis] = wrapped_start;

                let mut out_guard = write_storage(&outputs[0])?;
                let dest_cuda = cuda_output(&mut out_guard)?;
                $baracuda_fn(
                    dest_cuda,
                    &src_first,
                    dest_shape,
                    &sub_source_shape,
                    &sub_range_start,
                )?;
            }
            if second_len > 0 {
                let chunk_row_bytes = second_len * row_bytes;
                let offset_in_outer = first_len * row_bytes;
                let src_second = src_cuda.extract_strided_to_new(
                    outer_count,
                    stride_bytes,
                    offset_in_outer,
                    chunk_row_bytes,
                )?;
                let mut sub_source_shape = slab_shape.clone();
                sub_source_shape[axis] = second_len;
                let mut sub_range_start: Vec<usize> = ranges.iter().map(|r| r.0).collect();
                sub_range_start[axis] = 0;

                let mut out_guard = write_storage(&outputs[0])?;
                let dest_cuda = cuda_output(&mut out_guard)?;
                $baracuda_fn(
                    dest_cuda,
                    &src_second,
                    dest_shape,
                    &sub_source_shape,
                    &sub_range_start,
                )?;
            }
            Ok(())
        }
    };
}

pub mod write_slice_rotating {
    use super::*;
    use fuel_cuda_backend::baracuda::write_slice as bk;

    cuda_write_slice_rotating_baracuda_wrapper!(write_slice_rotating_b1, bk::write_slice_b1, 1);
    cuda_write_slice_rotating_baracuda_wrapper!(write_slice_rotating_b2, bk::write_slice_b2, 2);
    cuda_write_slice_rotating_baracuda_wrapper!(write_slice_rotating_b4, bk::write_slice_b4, 4);
    cuda_write_slice_rotating_baracuda_wrapper!(write_slice_rotating_b8, bk::write_slice_b8, 8);
    cuda_write_slice_rotating_baracuda_wrapper!(write_slice_rotating_b16, bk::write_slice_b16, 16);
}

// ===========================================================================
// WriteSliceDoff — device-resident-offset KV-cache append (CapturedRun)
// ===========================================================================
//
// Like write_slice_rotating's wrapper but WITHOUT the position D2H + host
// math + ring split (all of which are capture-illegal / capture-baking).
// The `offset` operand (inputs[1], rank-0 I64) stays on the device: its
// raw device pointer is threaded straight to baracuda's `_doff` launcher as
// `dyn_start_dev`, which reads the start device-side at kernel entry. That
// is exactly what lets a captured CUDA graph replay at the host-updated
// position (per-token fixed-address H2D bump of *offset). No wrap.

macro_rules! cuda_write_slice_doff_baracuda_wrapper {
    ($wrapper_name:ident, $baracuda_fn:path) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            _layouts: &[Layout],
            params: &OpParams,
        ) -> Result<()> {
            if inputs.len() != 2 || outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": expected 2 inputs (source, offset) + 1 output (dest), got {} + {}"
                    ),
                    inputs.len(),
                    outputs.len(),
                ))
                .bt());
            }
            let (dest_shape, axis, ranges) = match params {
                OpParams::WriteSliceDoff {
                    dest_shape,
                    axis,
                    ranges,
                } => (dest_shape, *axis, ranges),
                other => {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": expected OpParams::WriteSliceDoff, got {:?}"
                        ),
                        other,
                    ))
                    .bt());
                }
            };
            let rank = dest_shape.len();
            if ranges.len() != rank {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": ranges.len() {} != dest rank {}"
                    ),
                    ranges.len(),
                    rank,
                ))
                .bt());
            }
            if axis >= rank {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": axis {} out of bounds for rank {}"
                    ),
                    axis, rank,
                ))
                .bt());
            }
            // Derive source_shape (slab widths) + range_start (off-axis
            // starts; the axis slot is a placeholder — the true start is
            // read device-side from `offset`).
            let mut source_shape = Vec::with_capacity(rank);
            let mut range_start = Vec::with_capacity(rank);
            for &(start, end) in ranges.iter() {
                if end < start {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": range ({}, {}) has end < start"
                        ),
                        start, end,
                    ))
                    .bt());
                }
                source_shape.push(end - start);
                range_start.push(start);
            }

            // Acquire all three guards; hold them across the kernel call
            // so the device buffers stay live. `offset` is read for its
            // DEVICE pointer only (no D2H).
            let src_guard = read_storage(&inputs[0])?;
            let off_guard = read_storage(&inputs[1])?;
            let mut out_guard = write_storage(&outputs[0])?;
            let src_cuda = cuda_input(&src_guard)?;
            let off_cuda = cuda_input(&off_guard)?;
            if off_cuda.len_bytes() < 8 {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": offset storage has {} bytes, need >= 8 (I64)"
                    ),
                    off_cuda.len_bytes(),
                ))
                .bt());
            }
            let dyn_start_dev = off_cuda.buffer().as_raw().0 as *const i64;
            let dest_cuda = cuda_output(&mut out_guard)?;
            // SAFETY (GAP-337): `$baracuda_fn` (a `write_slice_doff_b*`) requires the
            // device start value `*dyn_start_dev` to satisfy `start + width <= dest
            // dim[axis]`; the kernel does not clamp, so an out-of-range start is an
            // out-of-bounds device write. On the in-tree decode paths that value is the
            // KV `cached_len`, bounded BEFORE this launch by the per-token check in
            // `build_decode_token_data` and the build guards in
            // `build_batched_decode_logits` / `forward_with_kv_context` /
            // `build_and_realize_first_decode_token`, and for paged KV by the block-pool
            // allocator (a valid physical slot). Those are the PRIMARY live paths, NOT an
            // exhaustive caller census. `dyn_start_dev` is the `offset` operand's live
            // device i64 pointer, checked non-null and >= 8 bytes above.
            unsafe {
                $baracuda_fn(
                    dest_cuda,
                    src_cuda,
                    dest_shape,
                    &source_shape,
                    &range_start,
                    axis,
                    dyn_start_dev,
                )
            }
        }
    };
}

pub mod write_slice_doff {
    use super::*;
    use fuel_cuda_backend::baracuda::write_slice as bk;

    cuda_write_slice_doff_baracuda_wrapper!(write_slice_doff_b1, bk::write_slice_doff_b1);
    cuda_write_slice_doff_baracuda_wrapper!(write_slice_doff_b2, bk::write_slice_doff_b2);
    cuda_write_slice_doff_baracuda_wrapper!(write_slice_doff_b4, bk::write_slice_doff_b4);
    cuda_write_slice_doff_baracuda_wrapper!(write_slice_doff_b8, bk::write_slice_doff_b8);
}

// ===========================================================================
// Triu / Tril — upper / lower triangular mask (Phase 7.6 step 9c E.3.2.4)
// ===========================================================================
//
// Per-dtype kernels (7 dtypes: f16/bf16/f32/f64/i32/i64/bool). Wrappers
// destructure OpParams::Triangular { batch_count, rows, cols, diagonal }
// and call the matching baracuda symbol. The kernel allocates a fresh
// output buffer (no in-place semantics).

macro_rules! cuda_triangular_baracuda_wrapper {
    ($wrapper_name:ident, $baracuda_fn:path) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            layouts: &[Layout],
            params: &OpParams,
        ) -> Result<()> {
            if inputs.len() != 1 || outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": expected 1 input + 1 output, got {} + {}"
                    ),
                    inputs.len(),
                    outputs.len(),
                ))
                .bt());
            }
            let (batch, rows, cols, diag) = match params {
                OpParams::Triangular {
                    batch_count,
                    rows,
                    cols,
                    diagonal,
                } => (*batch_count, *rows, *cols, *diagonal),
                other => {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": expected OpParams::Triangular, got {:?}"
                        ),
                        other,
                    ))
                    .bt());
                }
            };
            let layout = layouts.first();
            let in_guard = read_storage(&inputs[0])?;
            let mut out_guard = write_storage(&outputs[0])?;
            let src_cuda = cuda_input(&in_guard)?;
            let result = $baracuda_fn(src_cuda, layout, batch, rows, cols, diag)?;
            let out_cuda = cuda_output(&mut out_guard)?;
            *out_cuda = result;
            Ok(())
        }
    };
}

pub mod triangular {
    use super::*;
    use fuel_cuda_backend::baracuda::triangular as bk;

    // Triu
    cuda_triangular_baracuda_wrapper!(triu_f32, bk::triu_f32);
    cuda_triangular_baracuda_wrapper!(triu_f64, bk::triu_f64);
    cuda_triangular_baracuda_wrapper!(triu_f16, bk::triu_f16);
    cuda_triangular_baracuda_wrapper!(triu_bf16, bk::triu_bf16);
    cuda_triangular_baracuda_wrapper!(triu_i32, bk::triu_i32);
    cuda_triangular_baracuda_wrapper!(triu_i64, bk::triu_i64);
    cuda_triangular_baracuda_wrapper!(triu_bool, bk::triu_bool);

    // Tril
    cuda_triangular_baracuda_wrapper!(tril_f32, bk::tril_f32);
    cuda_triangular_baracuda_wrapper!(tril_f64, bk::tril_f64);
    cuda_triangular_baracuda_wrapper!(tril_f16, bk::tril_f16);
    cuda_triangular_baracuda_wrapper!(tril_bf16, bk::tril_bf16);
    cuda_triangular_baracuda_wrapper!(tril_i32, bk::tril_i32);
    cuda_triangular_baracuda_wrapper!(tril_i64, bk::tril_i64);
    cuda_triangular_baracuda_wrapper!(tril_bool, bk::tril_bool);
}

// ===========================================================================
// Flip — single-axis reverse (baracuda alpha.27, integrated alpha.29)
// ===========================================================================

macro_rules! cuda_flip_baracuda_wrapper {
    ($wrapper_name:ident, $baracuda_fn:path) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            layouts: &[Layout],
            params: &OpParams,
        ) -> Result<()> {
            if inputs.len() != 1 || outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": expected 1 input + 1 output, got {} + {}"
                    ),
                    inputs.len(),
                    outputs.len(),
                ))
                .bt());
            }
            let (outer, dim_size, inner, axis) = match params {
                OpParams::Flip {
                    outer_count,
                    dim_size,
                    inner_count,
                    axis,
                } => (*outer_count, *dim_size, *inner_count, *axis),
                other => {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": expected OpParams::Flip, got {:?}"
                        ),
                        other,
                    ))
                    .bt());
                }
            };
            let layout = layouts.first();
            let in_guard = read_storage(&inputs[0])?;
            let mut out_guard = write_storage(&outputs[0])?;
            let src_cuda = cuda_input(&in_guard)?;
            let result = $baracuda_fn(src_cuda, layout, axis, outer, dim_size, inner)?;
            let out_cuda = cuda_output(&mut out_guard)?;
            *out_cuda = result;
            Ok(())
        }
    };
}

pub mod flip {
    use super::*;
    use fuel_cuda_backend::baracuda::flip as bk;

    cuda_flip_baracuda_wrapper!(flip_f32, bk::flip_f32);
    cuda_flip_baracuda_wrapper!(flip_f64, bk::flip_f64);
    cuda_flip_baracuda_wrapper!(flip_f16, bk::flip_f16);
    cuda_flip_baracuda_wrapper!(flip_bf16, bk::flip_bf16);
}

// ===========================================================================
// Roll — single-axis cyclic shift (baracuda alpha.27, integrated alpha.29)
// ===========================================================================

macro_rules! cuda_roll_baracuda_wrapper {
    ($wrapper_name:ident, $baracuda_fn:path) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            layouts: &[Layout],
            params: &OpParams,
        ) -> Result<()> {
            if inputs.len() != 1 || outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": expected 1 input + 1 output, got {} + {}"
                    ),
                    inputs.len(),
                    outputs.len(),
                ))
                .bt());
            }
            let (outer, dim_size, inner, shift, axis) = match params {
                OpParams::Roll {
                    outer_count,
                    dim_size,
                    inner_count,
                    shift,
                    axis,
                } => (*outer_count, *dim_size, *inner_count, *shift, *axis),
                other => {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": expected OpParams::Roll, got {:?}"
                        ),
                        other,
                    ))
                    .bt());
                }
            };
            let layout = layouts.first();
            let in_guard = read_storage(&inputs[0])?;
            let mut out_guard = write_storage(&outputs[0])?;
            let src_cuda = cuda_input(&in_guard)?;
            let result = $baracuda_fn(src_cuda, layout, axis, outer, dim_size, inner, shift)?;
            let out_cuda = cuda_output(&mut out_guard)?;
            *out_cuda = result;
            Ok(())
        }
    };
}

pub mod roll {
    use super::*;
    use fuel_cuda_backend::baracuda::roll as bk;

    cuda_roll_baracuda_wrapper!(roll_f32, bk::roll_f32);
    cuda_roll_baracuda_wrapper!(roll_f64, bk::roll_f64);
    cuda_roll_baracuda_wrapper!(roll_f16, bk::roll_f16);
    cuda_roll_baracuda_wrapper!(roll_bf16, bk::roll_bf16);
}

// ===========================================================================
// CumSum — inclusive prefix sum along one axis (baracuda alpha.27,
//          integrated alpha.29).
// ===========================================================================

macro_rules! cuda_cumsum_baracuda_wrapper {
    ($wrapper_name:ident, $baracuda_fn:path) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            layouts: &[Layout],
            params: &OpParams,
        ) -> Result<()> {
            if inputs.len() != 1 || outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": expected 1 input + 1 output, got {} + {}"
                    ),
                    inputs.len(),
                    outputs.len(),
                ))
                .bt());
            }
            let (outer, dim_size, inner, axis) = match params {
                OpParams::CumSum {
                    outer_count,
                    dim_size,
                    inner_count,
                    axis,
                } => (*outer_count, *dim_size, *inner_count, *axis),
                other => {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": expected OpParams::CumSum, got {:?}"
                        ),
                        other,
                    ))
                    .bt());
                }
            };
            let layout = layouts.first();
            let in_guard = read_storage(&inputs[0])?;
            let mut out_guard = write_storage(&outputs[0])?;
            let src_cuda = cuda_input(&in_guard)?;
            let result = $baracuda_fn(src_cuda, layout, axis, outer, dim_size, inner)?;
            let out_cuda = cuda_output(&mut out_guard)?;
            *out_cuda = result;
            Ok(())
        }
    };
}

pub mod cumsum {
    use super::*;
    use fuel_cuda_backend::baracuda::cumsum as bk;

    cuda_cumsum_baracuda_wrapper!(cumsum_f32, bk::cumsum_f32);
    cuda_cumsum_baracuda_wrapper!(cumsum_f64, bk::cumsum_f64);
    cuda_cumsum_baracuda_wrapper!(cumsum_f16, bk::cumsum_f16);
    cuda_cumsum_baracuda_wrapper!(cumsum_bf16, bk::cumsum_bf16);
}

// ===========================================================================
// Pad — multi-dim padding with Constant/Reflect/Replicate modes
//       (baracuda alpha.29; Circular mode exists in baracuda but
//        fuel-graph's IR doesn't emit it yet).
// ===========================================================================

/// Decode the pre-encoded `fill_bytes` to a typed value. Constant-mode
/// Pad's OpParams carries fill_bytes already in the output's dtype
/// (encode_value_to_bytes does that at op_to_op_params time). Here we
/// reinterpret the bytes back to the matching scalar type.
fn decode_fill_f32(b: &[u8]) -> Result<f32> {
    if b.len() != 4 {
        return Err(Error::Msg(format!(
            "pad_constant_f32: fill_bytes.len()={} != 4",
            b.len(),
        ))
        .bt());
    }
    Ok(f32::from_le_bytes([b[0], b[1], b[2], b[3]]))
}
fn decode_fill_f64(b: &[u8]) -> Result<f64> {
    if b.len() != 8 {
        return Err(Error::Msg(format!(
            "pad_constant_f64: fill_bytes.len()={} != 8",
            b.len(),
        ))
        .bt());
    }
    Ok(f64::from_le_bytes([
        b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7],
    ]))
}
fn decode_fill_f16(b: &[u8]) -> Result<half::f16> {
    if b.len() != 2 {
        return Err(Error::Msg(format!(
            "pad_constant_f16: fill_bytes.len()={} != 2",
            b.len(),
        ))
        .bt());
    }
    Ok(half::f16::from_le_bytes([b[0], b[1]]))
}
fn decode_fill_bf16(b: &[u8]) -> Result<half::bf16> {
    if b.len() != 2 {
        return Err(Error::Msg(format!(
            "pad_constant_bf16: fill_bytes.len()={} != 2",
            b.len(),
        ))
        .bt());
    }
    Ok(half::bf16::from_le_bytes([b[0], b[1]]))
}

/// Generate per-dtype Pad-forward wrapper. Reads `OpParams::Pad`,
/// dispatches on `mode_tag` (0=Constant, 1=Reflect, 2=Replicate);
/// returns Unsupported for any other tag (baracuda has Circular but
/// fuel-graph doesn't emit it today).
macro_rules! cuda_pad_baracuda_wrapper {
    ($wrapper_name:ident, $const_fn:path, $reflect_fn:path, $replicate_fn:path, $decode_fn:path) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            _layouts: &[Layout],
            params: &OpParams,
        ) -> Result<()> {
            if inputs.len() != 1 || outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": expected 1 input + 1 output, got {} + {}"
                    ),
                    inputs.len(),
                    outputs.len(),
                ))
                .bt());
            }
            let (in_shape, out_shape, padding, mode_tag, fill_bytes) = match params {
                OpParams::Pad {
                    in_shape,
                    out_shape,
                    padding,
                    mode_tag,
                    fill_bytes,
                } => (in_shape, out_shape, padding, *mode_tag, fill_bytes),
                other => {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": expected OpParams::Pad, got {:?}"
                        ),
                        other,
                    ))
                    .bt());
                }
            };
            let in_guard = read_storage(&inputs[0])?;
            let mut out_guard = write_storage(&outputs[0])?;
            let src_cuda = cuda_input(&in_guard)?;
            let result = match mode_tag {
                0 => {
                    let value = $decode_fn(fill_bytes)?;
                    $const_fn(src_cuda, in_shape, out_shape, padding, value)?
                }
                1 => $reflect_fn(src_cuda, in_shape, out_shape, padding)?,
                2 => $replicate_fn(src_cuda, in_shape, out_shape, padding)?,
                other => {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": mode_tag {} not supported \
                         (Constant=0, Reflect=1, Replicate=2; Circular=3 not yet)"
                        ),
                        other,
                    ))
                    .bt());
                }
            };
            let out_cuda = cuda_output(&mut out_guard)?;
            *out_cuda = result;
            Ok(())
        }
    };
}

/// Generate per-dtype PadBackward wrapper. Only Constant-mode backward
/// is supported (the other modes' backwards aren't shipped in baracuda
/// alpha.29; the CPU path remains the only target for them).
macro_rules! cuda_pad_backward_baracuda_wrapper {
    ($wrapper_name:ident, $bw_fn:path) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            _layouts: &[Layout],
            params: &OpParams,
        ) -> Result<()> {
            if inputs.len() != 1 || outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": expected 1 input + 1 output, got {} + {}"
                    ),
                    inputs.len(),
                    outputs.len(),
                ))
                .bt());
            }
            let (in_shape, out_shape, padding, mode_tag) = match params {
                OpParams::PadBackward {
                    in_shape,
                    out_shape,
                    padding,
                    mode_tag,
                } => (in_shape, out_shape, padding, *mode_tag),
                other => {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": expected OpParams::PadBackward, got {:?}"
                        ),
                        other,
                    ))
                    .bt());
                }
            };
            if mode_tag != 0 {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": only Constant-mode backward (tag 0) supported \
                     on CUDA today, got mode_tag {}"
                    ),
                    mode_tag,
                ))
                .bt());
            }
            // OpParams::PadBackward layout: in_shape = forward x shape
            // (= dx shape), out_shape = forward y shape (= dy shape).
            // baracuda's pad_constant_backward takes input_shape (= dx
            // shape) and pad_low; dy_shape is implied. We pass both
            // through to fuel-cuda's pad_backward wrapper so its stride
            // arrays are computed correctly.
            let in_guard = read_storage(&inputs[0])?;
            let mut out_guard = write_storage(&outputs[0])?;
            let dy_cuda = cuda_input(&in_guard)?;
            let result = $bw_fn(dy_cuda, in_shape, out_shape, padding)?;
            let dx_cuda = cuda_output(&mut out_guard)?;
            *dx_cuda = result;
            Ok(())
        }
    };
}

pub mod pad {
    use super::*;
    use fuel_cuda_backend::baracuda::pad as bk;

    cuda_pad_baracuda_wrapper!(
        pad_f32,
        bk::pad_constant_f32,
        bk::pad_reflect_f32,
        bk::pad_replicate_f32,
        super::decode_fill_f32
    );
    cuda_pad_baracuda_wrapper!(
        pad_f64,
        bk::pad_constant_f64,
        bk::pad_reflect_f64,
        bk::pad_replicate_f64,
        super::decode_fill_f64
    );
    cuda_pad_baracuda_wrapper!(
        pad_f16,
        bk::pad_constant_f16,
        bk::pad_reflect_f16,
        bk::pad_replicate_f16,
        super::decode_fill_f16
    );
    cuda_pad_baracuda_wrapper!(
        pad_bf16,
        bk::pad_constant_bf16,
        bk::pad_reflect_bf16,
        bk::pad_replicate_bf16,
        super::decode_fill_bf16
    );

    cuda_pad_backward_baracuda_wrapper!(pad_backward_f32, bk::pad_backward_f32);
    cuda_pad_backward_baracuda_wrapper!(pad_backward_f64, bk::pad_backward_f64);
    cuda_pad_backward_baracuda_wrapper!(pad_backward_f16, bk::pad_backward_f16);
    cuda_pad_backward_baracuda_wrapper!(pad_backward_bf16, bk::pad_backward_bf16);
}

// ===========================================================================
// Affine — `y = mul * x + add` with scalar (mul, add) per OpParams::Affine
// ===========================================================================
//
// Per-dtype wrappers; baracuda's FFI carries (a, b) in the natural
// per-dtype arithmetic type (f32 for f16/bf16 storage; f64 for f64
// storage; per-element int type for i32/i64/u8). The wrappers cast
// Fuel's `(f64, f64)` OpParams down to the right type at the FFI
// boundary.

macro_rules! cuda_affine_baracuda_wrapper {
    ($wrapper_name:ident, $baracuda_fn:path, $scalar_cast:expr $(,)?) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            layouts: &[Layout],
            params: &OpParams,
        ) -> Result<()> {
            if inputs.len() != 1 || outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": expected 1 input + 1 output, got {} + {}"
                    ),
                    inputs.len(),
                    outputs.len(),
                ))
                .bt());
            }
            let (mul_f64, add_f64) = match params {
                OpParams::Affine { mul, add } => (*mul, *add),
                other => {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": expected OpParams::Affine, got {:?}"
                        ),
                        other,
                    ))
                    .bt());
                }
            };
            let cast = $scalar_cast;
            let mul = cast(mul_f64);
            let add = cast(add_f64);
            let layout = layouts.first();
            let in_guard = read_storage(&inputs[0])?;
            let mut out_guard = write_storage(&outputs[0])?;
            let src_cuda = cuda_input(&in_guard)?;
            let out_cuda = cuda_output(&mut out_guard)?;
            // Write-into-output (CapturedRun): `_into` writes into outputs[0].
            $baracuda_fn(src_cuda, layout, mul, add, out_cuda)
        }
    };
}

pub mod affine {
    use super::*;
    use fuel_cuda_backend::baracuda::affine as bk;

    cuda_affine_baracuda_wrapper!(affine_f32, bk::affine_f32_into, |v: f64| v as f32);
    cuda_affine_baracuda_wrapper!(affine_f64, bk::affine_f64_into, |v: f64| v);
    cuda_affine_baracuda_wrapper!(affine_f16, bk::affine_f16_into, |v: f64| v as f32);
    cuda_affine_baracuda_wrapper!(affine_bf16, bk::affine_bf16_into, |v: f64| v as f32);
    cuda_affine_baracuda_wrapper!(affine_i32, bk::affine_i32_into, |v: f64| v as i32);
    cuda_affine_baracuda_wrapper!(affine_i64, bk::affine_i64_into, |v: f64| v as i64);
    cuda_affine_baracuda_wrapper!(affine_u8, bk::affine_u8_into, |v: f64| v as u8);

    // In-place affine — Phase 3d of the in-place ops infrastructure.
    // baracuda's `affine_inplace_{f32,f64}_run` symbols take a single
    // pointer (`y` = read+write). The executor's
    // `WorkItemKind::InplaceKernel` arm passes the target Arc as
    // `outputs[0]` with `inputs=[]`; we acquire its write lock and
    // mutate through `bytes_mut()` (via `cuda_output` which returns
    // `&mut CudaStorageBytes`).
    //
    // alpha.62 ships all 4 FP dtypes (f32/f64/bf16/f16); the
    // half-precision variants pivot scalars through f32 at the FFI
    // boundary. Closes the only known in-place ops gap from the
    // 2026-05-30 baracuda ask (Item 1).
    macro_rules! cuda_affine_inplace_baracuda_wrapper {
        ($wrapper_name:ident, $baracuda_fn:path, $scalar_cast:expr $(,)?) => {
            pub fn $wrapper_name(
                inputs: &[Arc<RwLock<Storage>>],
                outputs: &mut [Arc<RwLock<Storage>>],
                _layouts: &[Layout],
                params: &OpParams,
            ) -> Result<()> {
                if !inputs.is_empty() || outputs.len() != 1 {
                    return Err(Error::Msg(format!(
                        concat!(stringify!($wrapper_name),
                            ": expected 0 inputs + 1 output (target adopted by executor), got {} + {}"),
                        inputs.len(), outputs.len(),
                    )).bt());
                }
                let (mul_f64, add_f64) = match params {
                    OpParams::Affine { mul, add } => (*mul, *add),
                    other => {
                        return Err(Error::Msg(format!(
                            concat!(stringify!($wrapper_name), ": expected OpParams::Affine, got {:?}"),
                            other,
                        )).bt());
                    }
                };
                let cast = $scalar_cast;
                let mul = cast(mul_f64);
                let add = cast(add_f64);
                let mut out_guard = write_storage(&outputs[0])?;
                let out_cuda = cuda_output(&mut out_guard)?;
                $baracuda_fn(out_cuda, mul, add)
            }
        };
    }

    cuda_affine_inplace_baracuda_wrapper!(affine_inplace_f32, bk::affine_inplace_f32, |v: f64| v

        as f32);
    cuda_affine_inplace_baracuda_wrapper!(affine_inplace_f64, bk::affine_inplace_f64, |v: f64| v);
    cuda_affine_inplace_baracuda_wrapper!(
        affine_inplace_bf16,
        bk::affine_inplace_bf16,
        |v: f64| v as f32

    );
    cuda_affine_inplace_baracuda_wrapper!(affine_inplace_f16, bk::affine_inplace_f16, |v: f64| v

        as f32);
}

// ===========================================================================
// Cast — single wrapper across every (src_dt, dst_dt) pair baracuda exposes
// ===========================================================================
//
// Cast's binding-table key is `[src_dt, dst_dt]` (two dtypes, not the
// usual unary `[dt, dt]`). The wrapper reads both dtypes from the
// input/output Storage and dispatches into
// `fuel_cuda_backend::baracuda::cast::dispatch`, which picks the right
// FFI symbol from the 8×8 baracuda surface. One wrapper, many
// (per-pair) binding-table entries — same pattern as the PTX-backed
// `cast_cuda_wrapper` in [`crate::dispatch`].

pub mod cast {
    use super::*;
    use fuel_cuda_backend::baracuda::cast as bk;

    pub fn cast_baracuda_wrapper(
        inputs: &[Arc<RwLock<Storage>>],
        outputs: &mut [Arc<RwLock<Storage>>],
        _layouts: &[Layout],
        _params: &OpParams,
    ) -> Result<()> {
        if inputs.len() != 1 || outputs.len() != 1 {
            return Err(Error::Msg(format!(
                "cast_baracuda_wrapper: expected 1 input + 1 output, got {} + {}",
                inputs.len(),
                outputs.len(),
            ))
            .bt());
        }
        let in_guard = read_storage(&inputs[0])?;
        let mut out_guard = write_storage(&outputs[0])?;
        let src_dtype = in_guard.dtype;
        let dst_dtype = out_guard.dtype;
        let src_cuda = cuda_input(&in_guard)?;
        let result = bk::dispatch(src_cuda, src_dtype, dst_dtype)?;
        let out_cuda = cuda_output(&mut out_guard)?;
        *out_cuda = result;
        Ok(())
    }
}

/// Generate a CUDA softmax-last-dim dispatch wrapper that pulls
/// `(outer_count, last_dim)` from `OpParams::SoftmaxLastDim`
/// (or `LogSoftmaxLastDim`) and forwards to a baracuda softmax
/// kernel.
macro_rules! cuda_softmax_last_dim_baracuda_wrapper {
    ($wrapper_name:ident, $baracuda_fn:path) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            layouts: &[Layout],
            params: &OpParams,
        ) -> Result<()> {
            if inputs.len() != 1 || outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": expected 1 input + 1 output, got {} + {}",
                    ),
                    inputs.len(),
                    outputs.len(),
                ))
                .bt());
            }
            // Validate params shape (outer_count + last_dim are now
            // implicit in the layout — the wrapper consumes only the
            // discriminant to confirm we got the expected OpKind).
            match params {
                OpParams::SoftmaxLastDim { .. } | OpParams::LogSoftmaxLastDim { .. } => {}
                other => {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": expected OpParams::SoftmaxLastDim/LogSoftmaxLastDim, got {:?}",
                        ),
                        other,
                    ))
                    .bt());
                }
            }
            let layout = layouts.first().ok_or_else(|| {
                Error::Msg(
                    concat!(
                        stringify!($wrapper_name),
                        ": SoftmaxLastDim requires an input layout (call with layouts[0])",
                    )
                    .to_string(),
                )
                .bt()
            })?;
            let in_guard = read_storage(&inputs[0])?;
            let mut out_guard = write_storage(&outputs[0])?;
            let src_cuda = cuda_input(&in_guard)?;
            let out_cuda = cuda_output(&mut out_guard)?;
            // Write-into-output (CapturedRun): `_into` writes into outputs[0].
            $baracuda_fn(src_cuda, layout, out_cuda)
        }
    };
}

// ===========================================================================
// Softmax wrappers (Softmax + LogSoftmax LastDim)
// ===========================================================================

pub mod softmax {
    use super::*;
    use fuel_cuda_backend::baracuda::softmax as bk;

    cuda_softmax_last_dim_baracuda_wrapper!(softmax_f32, bk::softmax_last_dim_f32_into);
    cuda_softmax_last_dim_baracuda_wrapper!(softmax_f16, bk::softmax_last_dim_f16_into);
    cuda_softmax_last_dim_baracuda_wrapper!(softmax_bf16, bk::softmax_last_dim_bf16_into);
    cuda_softmax_last_dim_baracuda_wrapper!(softmax_f64, bk::softmax_last_dim_f64_into);

    cuda_softmax_last_dim_baracuda_wrapper!(log_softmax_f32, bk::log_softmax_last_dim_f32_into);
    cuda_softmax_last_dim_baracuda_wrapper!(log_softmax_f16, bk::log_softmax_last_dim_f16_into);
    cuda_softmax_last_dim_baracuda_wrapper!(log_softmax_bf16, bk::log_softmax_last_dim_bf16_into);
    cuda_softmax_last_dim_baracuda_wrapper!(log_softmax_f64, bk::log_softmax_last_dim_f64_into);
}

// ===========================================================================
// Attention — RoPE wrappers
// ===========================================================================

pub mod attention {
    use super::*;
    use fuel_cuda_backend::baracuda::attention as bk;

    cuda_rope_baracuda_wrapper!(rope_f32, bk::rope_f32_into);
    cuda_rope_baracuda_wrapper!(rope_f16, bk::rope_f16_into);
    cuda_rope_baracuda_wrapper!(rope_bf16, bk::rope_bf16_into);
    cuda_rope_baracuda_wrapper!(rope_f64, bk::rope_f64_into);
}

// ===========================================================================
// Attention — FlashDecoding (decode-flash, seq_q==1, capacity-K)
// ===========================================================================
//
// The FIRST capacity-K CUDA `OpKind::FlashAttn` binding. Adapts the
// baracuda `flash_decoding_{f16,bf16}` decode kernel: pulls the geometry +
// math from `OpParams::FlashAttn` (`sk` = physical K/V capacity, `k_len` =
// logical attended prefix resolved from the SymEnv in pipelined.rs), and
// forwards the per-tensor input `Layout`s so the wrapper derives capacity
// strides directly (no Contiguize). Three inputs (q, k, v) — this is the
// NO-alibi decode arm; window / softcap / alibi are not implemented by the
// kernel and must be gated out statically (registered dtype key + cost fn).
// The `causal` flag is accepted-and-ignored: at `seq_q == 1` the single
// query attends the whole `[0, k_len)` prefix either way (the reply's
// "always attends the full prefix" == Fuel's decode model).

/// Generate one CUDA FlashDecoding dispatch wrapper adapting a
/// `fuel-cuda-backend::baracuda::attention::flash_decoding_*` kernel into
/// the `KernelRef` signature. The dtype gate is the registration key
/// (f16/bf16), not a runtime check; the wrapper pulls geometry from
/// `OpParams::FlashAttn` and forwards the per-tensor input `Layout`s.
macro_rules! cuda_flash_decoding_baracuda_wrapper {
    ($wrapper_name:ident, $baracuda_fn:path $(,)?) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            layouts: &[Layout],
            params: &OpParams,
        ) -> Result<()> {
            // Decode arm: exactly q, k, v (no alibi) + one output.
            if inputs.len() != 3 || outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": expected 3 inputs (q, k, v) + 1 output, got {} + {}",
                    ),
                    inputs.len(),
                    outputs.len(),
                ))
                .bt());
            }
            let (b, hq, hkv, sq, sk, d, k_len, scale) = match params {
                OpParams::FlashAttn {
                    b,
                    hq,
                    hkv,
                    sq,
                    sk,
                    d,
                    k_len,
                    softmax_scale,
                    causal: _,
                    window_size_left,
                    window_size_right,
                    softcap,
                } => {
                    // Window / softcap are unimplemented by the decode
                    // kernel; the ranker must exclude these (cost gate).
                    // Hard-error defensively (fail-fast dispatch).
                    if window_size_left.is_some()
                        || window_size_right.is_some()
                        || softcap.is_some()
                    {
                        return Err(Error::Msg(format!(
                            concat!(
                                stringify!($wrapper_name),
                                ": window/softcap not supported by flash_decoding \
                                 (window_left={:?}, window_right={:?}, softcap={:?}); \
                                 the ranker must route these to the decomposed base map",
                            ),
                            window_size_left, window_size_right, softcap,
                        ))
                        .bt());
                    }
                    (*b, *hq, *hkv, *sq, *sk, *d, *k_len, *softmax_scale)
                }
                other => {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": expected OpParams::FlashAttn, got {:?}",
                        ),
                        other,
                    ))
                    .bt());
                }
            };
            let q_layout = layouts.first();
            let k_layout = layouts.get(1);
            let v_layout = layouts.get(2);
            let q_guard = read_storage(&inputs[0])?;
            let k_guard = read_storage(&inputs[1])?;
            let v_guard = read_storage(&inputs[2])?;
            let mut out_guard = write_storage(&outputs[0])?;
            let q_cuda = cuda_input(&q_guard)?;
            let k_cuda = cuda_input(&k_guard)?;
            let v_cuda = cuda_input(&v_guard)?;
            let result = $baracuda_fn(
                q_cuda, k_cuda, v_cuda, q_layout, k_layout, v_layout, b, hq, hkv, sq, sk, d, k_len,
                scale,
            )?;
            let out_cuda = cuda_output(&mut out_guard)?;
            *out_cuda = result;
            Ok(())
        }
    };
}

pub mod flash_decoding {
    use super::*;
    use fuel_cuda_backend::baracuda::attention as bk;

    cuda_flash_decoding_baracuda_wrapper!(flash_decoding_f16, bk::flash_decoding_f16);
    cuda_flash_decoding_baracuda_wrapper!(flash_decoding_bf16, bk::flash_decoding_bf16);
}

/// Generate a CUDA norm-last-dim dispatch wrapper that pulls
/// `(outer_count, last_dim, eps)` from `OpParams::NormLastDim`
/// and forwards to a baracuda last-dim norm kernel.
macro_rules! cuda_norm_last_dim_baracuda_wrapper {
    ($wrapper_name:ident, $baracuda_fn:path) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            layouts: &[Layout],
            params: &OpParams,
        ) -> Result<()> {
            if inputs.len() != 1 || outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": expected 1 input + 1 output, got {} + {}",
                    ),
                    inputs.len(),
                    outputs.len(),
                ))
                .bt());
            }
            let eps = match params {
                OpParams::NormLastDim { eps, .. } => *eps,
                other => {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": expected OpParams::NormLastDim, got {:?}",
                        ),
                        other,
                    ))
                    .bt());
                }
            };
            let layout = layouts.first().ok_or_else(|| {
                Error::Msg(
                    concat!(
                        stringify!($wrapper_name),
                        ": NormLastDim requires an input layout (call with layouts[0])",
                    )
                    .to_string(),
                )
                .bt()
            })?;
            let in_guard = read_storage(&inputs[0])?;
            let mut out_guard = write_storage(&outputs[0])?;
            let src_cuda = cuda_input(&in_guard)?;
            let result = $baracuda_fn(src_cuda, layout, eps)?;
            let out_cuda = cuda_output(&mut out_guard)?;
            *out_cuda = result;
            Ok(())
        }
    };
}

/// Write-into-output RMS-norm wrapper (CapturedRun). Same as
/// `cuda_norm_last_dim_baracuda_wrapper!` but calls the `_into` rms driver
/// (RMS scratch from the device's fixed workspace), writing the normalized
/// result into outputs[0] — the capture-safe path. LayerNorm keeps the
/// allocating wrapper (two scratch buffers; not in the capturable decode set).
macro_rules! cuda_rms_norm_into_baracuda_wrapper {
    ($wrapper_name:ident, $baracuda_into_fn:path) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            layouts: &[Layout],
            params: &OpParams,
        ) -> Result<()> {
            if inputs.len() != 1 || outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(
                        stringify!($wrapper_name),
                        ": expected 1 input + 1 output, got {} + {}",
                    ),
                    inputs.len(),
                    outputs.len(),
                ))
                .bt());
            }
            let eps = match params {
                OpParams::NormLastDim { eps, .. } => *eps,
                other => {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": expected OpParams::NormLastDim, got {:?}",
                        ),
                        other,
                    ))
                    .bt());
                }
            };
            let layout = layouts.first().ok_or_else(|| {
                Error::Msg(
                    concat!(
                        stringify!($wrapper_name),
                        ": NormLastDim requires an input layout (call with layouts[0])",
                    )
                    .to_string(),
                )
                .bt()
            })?;
            let in_guard = read_storage(&inputs[0])?;
            let mut out_guard = write_storage(&outputs[0])?;
            let src_cuda = cuda_input(&in_guard)?;
            let out_cuda = cuda_output(&mut out_guard)?;
            // Write-into-output (CapturedRun): `_into` writes into outputs[0].
            $baracuda_into_fn(src_cuda, layout, eps, out_cuda)
        }
    };
}

// ===========================================================================
// Norm-last-dim wrappers (RmsNorm + LayerNorm)
// ===========================================================================

pub mod norm {
    use super::*;
    use fuel_cuda_backend::baracuda::norm as bk;

    cuda_rms_norm_into_baracuda_wrapper!(rms_f32, bk::rms_norm_last_dim_f32_into);
    cuda_rms_norm_into_baracuda_wrapper!(rms_f16, bk::rms_norm_last_dim_f16_into);
    cuda_rms_norm_into_baracuda_wrapper!(rms_bf16, bk::rms_norm_last_dim_bf16_into);
    cuda_rms_norm_into_baracuda_wrapper!(rms_f64, bk::rms_norm_last_dim_f64_into);

    cuda_norm_last_dim_baracuda_wrapper!(layer_f32, bk::layer_norm_last_dim_f32);
    cuda_norm_last_dim_baracuda_wrapper!(layer_f16, bk::layer_norm_last_dim_f16);
    cuda_norm_last_dim_baracuda_wrapper!(layer_bf16, bk::layer_norm_last_dim_bf16);
    cuda_norm_last_dim_baracuda_wrapper!(layer_f64, bk::layer_norm_last_dim_f64);
}

/// Generate one CUDA reduce dispatch wrapper. Reduce wrappers
/// destructure `OpParams::Reduce { dims, keepdim }` and forward
/// to the baracuda multi-axis driver.
macro_rules! cuda_reduce_baracuda_wrapper {
    ($wrapper_name:ident, $baracuda_fn:path) => {
        pub fn $wrapper_name(
            inputs: &[Arc<RwLock<Storage>>],
            outputs: &mut [Arc<RwLock<Storage>>],
            layouts: &[Layout],
            params: &OpParams,
        ) -> Result<()> {
            if inputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(stringify!($wrapper_name), ": expected 1 input, got {}"),
                    inputs.len(),
                ))
                .bt());
            }
            if outputs.len() != 1 {
                return Err(Error::Msg(format!(
                    concat!(stringify!($wrapper_name), ": expected 1 output, got {}"),
                    outputs.len(),
                ))
                .bt());
            }
            let (dims, keepdim) = match params {
                OpParams::Reduce { dims, keepdim } => (dims.clone(), *keepdim),
                other => {
                    return Err(Error::Msg(format!(
                        concat!(
                            stringify!($wrapper_name),
                            ": expected OpParams::Reduce, got {:?}",
                        ),
                        other,
                    ))
                    .bt());
                }
            };
            let layout = layouts.first().ok_or_else(|| {
                Error::Msg(format!(concat!(
                    stringify!($wrapper_name),
                    ": reduce requires an input layout (call with layouts[0])",
                ),))
                .bt()
            })?;
            let in_guard = read_storage(&inputs[0])?;
            let mut out_guard = write_storage(&outputs[0])?;
            let src_cuda = cuda_input(&in_guard)?;
            let result = $baracuda_fn(src_cuda, layout, &dims, keepdim)?;
            let out_cuda = cuda_output(&mut out_guard)?;
            *out_cuda = result;
            Ok(())
        }
    };
}

// ===========================================================================
// Reduce wrappers
// ===========================================================================

pub mod reduce {
    use super::*;
    use fuel_cuda_backend::baracuda::reduce as bk;

    cuda_reduce_baracuda_wrapper!(sum_f32, bk::reduce_sum_f32);
    cuda_reduce_baracuda_wrapper!(max_f32, bk::reduce_max_f32);
    cuda_reduce_baracuda_wrapper!(min_f32, bk::reduce_min_f32);
    cuda_reduce_baracuda_wrapper!(mean_f32, bk::reduce_mean_f32);

    cuda_reduce_baracuda_wrapper!(sum_f16, bk::reduce_sum_f16);
    cuda_reduce_baracuda_wrapper!(max_f16, bk::reduce_max_f16);
    cuda_reduce_baracuda_wrapper!(min_f16, bk::reduce_min_f16);
    cuda_reduce_baracuda_wrapper!(mean_f16, bk::reduce_mean_f16);

    cuda_reduce_baracuda_wrapper!(sum_bf16, bk::reduce_sum_bf16);
    cuda_reduce_baracuda_wrapper!(max_bf16, bk::reduce_max_bf16);
    cuda_reduce_baracuda_wrapper!(min_bf16, bk::reduce_min_bf16);
    cuda_reduce_baracuda_wrapper!(mean_bf16, bk::reduce_mean_bf16);

    cuda_reduce_baracuda_wrapper!(sum_f64, bk::reduce_sum_f64);
    cuda_reduce_baracuda_wrapper!(max_f64, bk::reduce_max_f64);
    cuda_reduce_baracuda_wrapper!(min_f64, bk::reduce_min_f64);
    cuda_reduce_baracuda_wrapper!(mean_f64, bk::reduce_mean_f64);
}

// ===========================================================================
// Elementwise binary wrappers
// ===========================================================================

pub mod binary {
    use super::*;
    use fuel_cuda_backend::baracuda::binary as bk;

    // --- comparison family (T, T -> U8) ---
    //
    // The generated wrapper is dtype-agnostic: it hands raw
    // `CudaStorageBytes` through, and the U8-vs-F32 width difference is
    // handled one layer down (`compare_kernel!` passes an output element
    // width of 1). Registering these is what lets `Op::PagedAttn`'s
    // primitive recipe resolve entirely on CUDA — `ge` on [F32,F32,U8] was
    // its single unresolvable node, which is why the paged graph carried
    // host round-trips.
    cuda_binary_baracuda_wrapper!(eq_f32_u8, bk::binary_cmp_eq_f32_into);
    cuda_binary_baracuda_wrapper!(eq_f64_u8, bk::binary_cmp_eq_f64_into);
    cuda_binary_baracuda_wrapper!(eq_f16_u8, bk::binary_cmp_eq_f16_into);
    cuda_binary_baracuda_wrapper!(eq_bf16_u8, bk::binary_cmp_eq_bf16_into);

    cuda_binary_baracuda_wrapper!(ne_f32_u8, bk::binary_cmp_ne_f32_into);
    cuda_binary_baracuda_wrapper!(ne_f64_u8, bk::binary_cmp_ne_f64_into);
    cuda_binary_baracuda_wrapper!(ne_f16_u8, bk::binary_cmp_ne_f16_into);
    cuda_binary_baracuda_wrapper!(ne_bf16_u8, bk::binary_cmp_ne_bf16_into);

    cuda_binary_baracuda_wrapper!(lt_f32_u8, bk::binary_cmp_lt_f32_into);
    cuda_binary_baracuda_wrapper!(lt_f64_u8, bk::binary_cmp_lt_f64_into);
    cuda_binary_baracuda_wrapper!(lt_f16_u8, bk::binary_cmp_lt_f16_into);
    cuda_binary_baracuda_wrapper!(lt_bf16_u8, bk::binary_cmp_lt_bf16_into);

    cuda_binary_baracuda_wrapper!(le_f32_u8, bk::binary_cmp_le_f32_into);
    cuda_binary_baracuda_wrapper!(le_f64_u8, bk::binary_cmp_le_f64_into);
    cuda_binary_baracuda_wrapper!(le_f16_u8, bk::binary_cmp_le_f16_into);
    cuda_binary_baracuda_wrapper!(le_bf16_u8, bk::binary_cmp_le_bf16_into);

    cuda_binary_baracuda_wrapper!(gt_f32_u8, bk::binary_cmp_gt_f32_into);
    cuda_binary_baracuda_wrapper!(gt_f64_u8, bk::binary_cmp_gt_f64_into);
    cuda_binary_baracuda_wrapper!(gt_f16_u8, bk::binary_cmp_gt_f16_into);
    cuda_binary_baracuda_wrapper!(gt_bf16_u8, bk::binary_cmp_gt_bf16_into);

    cuda_binary_baracuda_wrapper!(ge_f32_u8, bk::binary_cmp_ge_f32_into);
    cuda_binary_baracuda_wrapper!(ge_f64_u8, bk::binary_cmp_ge_f64_into);
    cuda_binary_baracuda_wrapper!(ge_f16_u8, bk::binary_cmp_ge_f16_into);
    cuda_binary_baracuda_wrapper!(ge_bf16_u8, bk::binary_cmp_ge_bf16_into);

    cuda_binary_baracuda_wrapper!(add_f32, bk::binary_add_f32_into);
    cuda_binary_baracuda_wrapper!(sub_f32, bk::binary_sub_f32_into);
    cuda_binary_baracuda_wrapper!(mul_f32, bk::binary_mul_f32_into);
    cuda_binary_baracuda_wrapper!(div_f32, bk::binary_div_f32_into);
    cuda_binary_baracuda_wrapper!(maximum_f32, bk::binary_maximum_f32_into);
    cuda_binary_baracuda_wrapper!(minimum_f32, bk::binary_minimum_f32_into);
    cuda_binary_baracuda_wrapper!(pow_f32, bk::binary_pow_f32_into);
    cuda_binary_baracuda_wrapper!(rem_f32, bk::binary_rem_f32_into);

    cuda_binary_baracuda_wrapper!(add_f16, bk::binary_add_f16_into);
    cuda_binary_baracuda_wrapper!(sub_f16, bk::binary_sub_f16_into);
    cuda_binary_baracuda_wrapper!(mul_f16, bk::binary_mul_f16_into);
    cuda_binary_baracuda_wrapper!(div_f16, bk::binary_div_f16_into);
    cuda_binary_baracuda_wrapper!(maximum_f16, bk::binary_maximum_f16_into);
    cuda_binary_baracuda_wrapper!(minimum_f16, bk::binary_minimum_f16_into);
    cuda_binary_baracuda_wrapper!(pow_f16, bk::binary_pow_f16_into);
    cuda_binary_baracuda_wrapper!(rem_f16, bk::binary_rem_f16_into);

    cuda_binary_baracuda_wrapper!(add_bf16, bk::binary_add_bf16_into);
    cuda_binary_baracuda_wrapper!(sub_bf16, bk::binary_sub_bf16_into);
    cuda_binary_baracuda_wrapper!(mul_bf16, bk::binary_mul_bf16_into);
    cuda_binary_baracuda_wrapper!(div_bf16, bk::binary_div_bf16_into);
    cuda_binary_baracuda_wrapper!(maximum_bf16, bk::binary_maximum_bf16_into);
    cuda_binary_baracuda_wrapper!(minimum_bf16, bk::binary_minimum_bf16_into);
    cuda_binary_baracuda_wrapper!(pow_bf16, bk::binary_pow_bf16_into);
    cuda_binary_baracuda_wrapper!(rem_bf16, bk::binary_rem_bf16_into);

    cuda_binary_baracuda_wrapper!(add_f64, bk::binary_add_f64_into);
    cuda_binary_baracuda_wrapper!(sub_f64, bk::binary_sub_f64_into);
    cuda_binary_baracuda_wrapper!(mul_f64, bk::binary_mul_f64_into);
    cuda_binary_baracuda_wrapper!(div_f64, bk::binary_div_f64_into);
    cuda_binary_baracuda_wrapper!(maximum_f64, bk::binary_maximum_f64_into);
    cuda_binary_baracuda_wrapper!(minimum_f64, bk::binary_minimum_f64_into);
    cuda_binary_baracuda_wrapper!(pow_f64, bk::binary_pow_f64_into);
    cuda_binary_baracuda_wrapper!(rem_f64, bk::binary_rem_f64_into);
}

// ===========================================================================
// Elementwise unary wrappers
// ===========================================================================
//
// Manifest: one line per (op, dtype) pair. Both halves of the
// `(*_wrapper, *_kernel)` mapping live here so the file is the single
// source of truth for "baracuda's coverage of Fuel's binding table."

pub mod unary {
    use super::*;
    use fuel_cuda_backend::baracuda::elementwise as bk;

    // f32 surface
    cuda_unary_baracuda_wrapper!(neg_f32, bk::unary_neg_f32_into);
    cuda_unary_baracuda_wrapper!(abs_f32, bk::unary_abs_f32_into);
    cuda_unary_baracuda_wrapper!(sign_f32, bk::unary_sign_f32_into);
    cuda_unary_baracuda_wrapper!(sqr_f32, bk::unary_sqr_f32_into);
    cuda_unary_baracuda_wrapper!(sqrt_f32, bk::unary_sqrt_f32_into);
    cuda_unary_baracuda_wrapper!(rsqrt_f32, bk::unary_rsqrt_f32_into);
    cuda_unary_baracuda_wrapper!(recip_f32, bk::unary_recip_f32_into);
    cuda_unary_baracuda_wrapper!(exp_f32, bk::unary_exp_f32_into);
    cuda_unary_baracuda_wrapper!(log_f32, bk::unary_log_f32_into);
    cuda_unary_baracuda_wrapper!(sin_f32, bk::unary_sin_f32_into);
    cuda_unary_baracuda_wrapper!(cos_f32, bk::unary_cos_f32_into);
    cuda_unary_baracuda_wrapper!(tanh_f32, bk::unary_tanh_f32_into);
    cuda_unary_baracuda_wrapper!(relu_f32, bk::unary_relu_f32_into);
    cuda_unary_baracuda_wrapper!(gelu_f32, bk::unary_gelu_f32_into);
    cuda_unary_baracuda_wrapper!(gelu_tanh_f32, bk::unary_gelu_tanh_f32_into);
    cuda_unary_baracuda_wrapper!(step_f32, bk::unary_step_f32_into);
    cuda_unary_baracuda_wrapper!(silu_f32, bk::unary_silu_f32_into);
    cuda_unary_baracuda_wrapper!(sigmoid_f32, bk::unary_sigmoid_f32_into);
    cuda_unary_baracuda_wrapper!(floor_f32, bk::unary_floor_f32_into);
    cuda_unary_baracuda_wrapper!(ceil_f32, bk::unary_ceil_f32_into);
    cuda_unary_baracuda_wrapper!(round_f32, bk::unary_round_f32_into);
    cuda_unary_baracuda_wrapper!(erf_f32, bk::unary_erf_f32_into);

    // In-place unary surface — Phase 3e of in-place ops infrastructure.
    // Reuses the same baracuda symbols but the wrapper passes the
    // target's pointer for both x + y. Full 4-dtype coverage matches
    // the non-inplace cousins above.
    cuda_unary_inplace_baracuda_wrapper!(relu_inplace_f32, bk::unary_inplace_relu_f32);
    cuda_unary_inplace_baracuda_wrapper!(silu_inplace_f32, bk::unary_inplace_silu_f32);
    cuda_unary_inplace_baracuda_wrapper!(gelu_inplace_f32, bk::unary_inplace_gelu_f32);
    cuda_unary_inplace_baracuda_wrapper!(tanh_inplace_f32, bk::unary_inplace_tanh_f32);
    cuda_unary_inplace_baracuda_wrapper!(sigmoid_inplace_f32, bk::unary_inplace_sigmoid_f32);

    cuda_unary_inplace_baracuda_wrapper!(relu_inplace_f64, bk::unary_inplace_relu_f64);
    cuda_unary_inplace_baracuda_wrapper!(silu_inplace_f64, bk::unary_inplace_silu_f64);
    cuda_unary_inplace_baracuda_wrapper!(gelu_inplace_f64, bk::unary_inplace_gelu_f64);
    cuda_unary_inplace_baracuda_wrapper!(tanh_inplace_f64, bk::unary_inplace_tanh_f64);
    cuda_unary_inplace_baracuda_wrapper!(sigmoid_inplace_f64, bk::unary_inplace_sigmoid_f64);

    cuda_unary_inplace_baracuda_wrapper!(relu_inplace_bf16, bk::unary_inplace_relu_bf16);
    cuda_unary_inplace_baracuda_wrapper!(silu_inplace_bf16, bk::unary_inplace_silu_bf16);
    cuda_unary_inplace_baracuda_wrapper!(gelu_inplace_bf16, bk::unary_inplace_gelu_bf16);
    cuda_unary_inplace_baracuda_wrapper!(tanh_inplace_bf16, bk::unary_inplace_tanh_bf16);
    cuda_unary_inplace_baracuda_wrapper!(sigmoid_inplace_bf16, bk::unary_inplace_sigmoid_bf16);

    cuda_unary_inplace_baracuda_wrapper!(relu_inplace_f16, bk::unary_inplace_relu_f16);
    cuda_unary_inplace_baracuda_wrapper!(silu_inplace_f16, bk::unary_inplace_silu_f16);
    cuda_unary_inplace_baracuda_wrapper!(gelu_inplace_f16, bk::unary_inplace_gelu_f16);
    cuda_unary_inplace_baracuda_wrapper!(tanh_inplace_f16, bk::unary_inplace_tanh_f16);
    cuda_unary_inplace_baracuda_wrapper!(sigmoid_inplace_f16, bk::unary_inplace_sigmoid_f16);

    // In-place unary op family expansion (2026-05-30) — 16 new ops
    // × 4 dtypes = 64 new wrappers. Each reuses the matching baracuda
    // forward symbol with same-pointer dispatch.
    cuda_unary_inplace_baracuda_wrapper!(neg_inplace_f32, bk::unary_inplace_neg_f32);
    cuda_unary_inplace_baracuda_wrapper!(neg_inplace_f64, bk::unary_inplace_neg_f64);
    cuda_unary_inplace_baracuda_wrapper!(neg_inplace_bf16, bk::unary_inplace_neg_bf16);
    cuda_unary_inplace_baracuda_wrapper!(neg_inplace_f16, bk::unary_inplace_neg_f16);

    cuda_unary_inplace_baracuda_wrapper!(abs_inplace_f32, bk::unary_inplace_abs_f32);
    cuda_unary_inplace_baracuda_wrapper!(abs_inplace_f64, bk::unary_inplace_abs_f64);
    cuda_unary_inplace_baracuda_wrapper!(abs_inplace_bf16, bk::unary_inplace_abs_bf16);
    cuda_unary_inplace_baracuda_wrapper!(abs_inplace_f16, bk::unary_inplace_abs_f16);

    cuda_unary_inplace_baracuda_wrapper!(sqr_inplace_f32, bk::unary_inplace_sqr_f32);
    cuda_unary_inplace_baracuda_wrapper!(sqr_inplace_f64, bk::unary_inplace_sqr_f64);
    cuda_unary_inplace_baracuda_wrapper!(sqr_inplace_bf16, bk::unary_inplace_sqr_bf16);
    cuda_unary_inplace_baracuda_wrapper!(sqr_inplace_f16, bk::unary_inplace_sqr_f16);

    cuda_unary_inplace_baracuda_wrapper!(sqrt_inplace_f32, bk::unary_inplace_sqrt_f32);
    cuda_unary_inplace_baracuda_wrapper!(sqrt_inplace_f64, bk::unary_inplace_sqrt_f64);
    cuda_unary_inplace_baracuda_wrapper!(sqrt_inplace_bf16, bk::unary_inplace_sqrt_bf16);
    cuda_unary_inplace_baracuda_wrapper!(sqrt_inplace_f16, bk::unary_inplace_sqrt_f16);

    cuda_unary_inplace_baracuda_wrapper!(rsqrt_inplace_f32, bk::unary_inplace_rsqrt_f32);
    cuda_unary_inplace_baracuda_wrapper!(rsqrt_inplace_f64, bk::unary_inplace_rsqrt_f64);
    cuda_unary_inplace_baracuda_wrapper!(rsqrt_inplace_bf16, bk::unary_inplace_rsqrt_bf16);
    cuda_unary_inplace_baracuda_wrapper!(rsqrt_inplace_f16, bk::unary_inplace_rsqrt_f16);

    cuda_unary_inplace_baracuda_wrapper!(recip_inplace_f32, bk::unary_inplace_recip_f32);
    cuda_unary_inplace_baracuda_wrapper!(recip_inplace_f64, bk::unary_inplace_recip_f64);
    cuda_unary_inplace_baracuda_wrapper!(recip_inplace_bf16, bk::unary_inplace_recip_bf16);
    cuda_unary_inplace_baracuda_wrapper!(recip_inplace_f16, bk::unary_inplace_recip_f16);

    cuda_unary_inplace_baracuda_wrapper!(exp_inplace_f32, bk::unary_inplace_exp_f32);
    cuda_unary_inplace_baracuda_wrapper!(exp_inplace_f64, bk::unary_inplace_exp_f64);
    cuda_unary_inplace_baracuda_wrapper!(exp_inplace_bf16, bk::unary_inplace_exp_bf16);
    cuda_unary_inplace_baracuda_wrapper!(exp_inplace_f16, bk::unary_inplace_exp_f16);

    cuda_unary_inplace_baracuda_wrapper!(log_inplace_f32, bk::unary_inplace_log_f32);
    cuda_unary_inplace_baracuda_wrapper!(log_inplace_f64, bk::unary_inplace_log_f64);
    cuda_unary_inplace_baracuda_wrapper!(log_inplace_bf16, bk::unary_inplace_log_bf16);
    cuda_unary_inplace_baracuda_wrapper!(log_inplace_f16, bk::unary_inplace_log_f16);

    cuda_unary_inplace_baracuda_wrapper!(sin_inplace_f32, bk::unary_inplace_sin_f32);
    cuda_unary_inplace_baracuda_wrapper!(sin_inplace_f64, bk::unary_inplace_sin_f64);
    cuda_unary_inplace_baracuda_wrapper!(sin_inplace_bf16, bk::unary_inplace_sin_bf16);
    cuda_unary_inplace_baracuda_wrapper!(sin_inplace_f16, bk::unary_inplace_sin_f16);

    cuda_unary_inplace_baracuda_wrapper!(cos_inplace_f32, bk::unary_inplace_cos_f32);
    cuda_unary_inplace_baracuda_wrapper!(cos_inplace_f64, bk::unary_inplace_cos_f64);
    cuda_unary_inplace_baracuda_wrapper!(cos_inplace_bf16, bk::unary_inplace_cos_bf16);
    cuda_unary_inplace_baracuda_wrapper!(cos_inplace_f16, bk::unary_inplace_cos_f16);

    cuda_unary_inplace_baracuda_wrapper!(sign_inplace_f32, bk::unary_inplace_sign_f32);
    cuda_unary_inplace_baracuda_wrapper!(sign_inplace_f64, bk::unary_inplace_sign_f64);
    cuda_unary_inplace_baracuda_wrapper!(sign_inplace_bf16, bk::unary_inplace_sign_bf16);
    cuda_unary_inplace_baracuda_wrapper!(sign_inplace_f16, bk::unary_inplace_sign_f16);

    cuda_unary_inplace_baracuda_wrapper!(floor_inplace_f32, bk::unary_inplace_floor_f32);
    cuda_unary_inplace_baracuda_wrapper!(floor_inplace_f64, bk::unary_inplace_floor_f64);
    cuda_unary_inplace_baracuda_wrapper!(floor_inplace_bf16, bk::unary_inplace_floor_bf16);
    cuda_unary_inplace_baracuda_wrapper!(floor_inplace_f16, bk::unary_inplace_floor_f16);

    cuda_unary_inplace_baracuda_wrapper!(ceil_inplace_f32, bk::unary_inplace_ceil_f32);
    cuda_unary_inplace_baracuda_wrapper!(ceil_inplace_f64, bk::unary_inplace_ceil_f64);
    cuda_unary_inplace_baracuda_wrapper!(ceil_inplace_bf16, bk::unary_inplace_ceil_bf16);
    cuda_unary_inplace_baracuda_wrapper!(ceil_inplace_f16, bk::unary_inplace_ceil_f16);

    cuda_unary_inplace_baracuda_wrapper!(round_inplace_f32, bk::unary_inplace_round_f32);
    cuda_unary_inplace_baracuda_wrapper!(round_inplace_f64, bk::unary_inplace_round_f64);
    cuda_unary_inplace_baracuda_wrapper!(round_inplace_bf16, bk::unary_inplace_round_bf16);
    cuda_unary_inplace_baracuda_wrapper!(round_inplace_f16, bk::unary_inplace_round_f16);

    cuda_unary_inplace_baracuda_wrapper!(erf_inplace_f32, bk::unary_inplace_erf_f32);
    cuda_unary_inplace_baracuda_wrapper!(erf_inplace_f64, bk::unary_inplace_erf_f64);
    cuda_unary_inplace_baracuda_wrapper!(erf_inplace_bf16, bk::unary_inplace_erf_bf16);
    cuda_unary_inplace_baracuda_wrapper!(erf_inplace_f16, bk::unary_inplace_erf_f16);

    cuda_unary_inplace_baracuda_wrapper!(gelu_erf_inplace_f32, bk::unary_inplace_gelu_erf_f32);
    cuda_unary_inplace_baracuda_wrapper!(gelu_erf_inplace_f64, bk::unary_inplace_gelu_erf_f64);
    cuda_unary_inplace_baracuda_wrapper!(gelu_erf_inplace_bf16, bk::unary_inplace_gelu_erf_bf16);
    cuda_unary_inplace_baracuda_wrapper!(gelu_erf_inplace_f16, bk::unary_inplace_gelu_erf_f16);

    // f16
    cuda_unary_baracuda_wrapper!(neg_f16, bk::unary_neg_f16_into);
    cuda_unary_baracuda_wrapper!(abs_f16, bk::unary_abs_f16_into);
    cuda_unary_baracuda_wrapper!(sqr_f16, bk::unary_sqr_f16_into);
    cuda_unary_baracuda_wrapper!(sqrt_f16, bk::unary_sqrt_f16_into);
    cuda_unary_baracuda_wrapper!(recip_f16, bk::unary_recip_f16_into);
    cuda_unary_baracuda_wrapper!(exp_f16, bk::unary_exp_f16_into);
    cuda_unary_baracuda_wrapper!(log_f16, bk::unary_log_f16_into);
    cuda_unary_baracuda_wrapper!(sin_f16, bk::unary_sin_f16_into);
    cuda_unary_baracuda_wrapper!(cos_f16, bk::unary_cos_f16_into);
    cuda_unary_baracuda_wrapper!(tanh_f16, bk::unary_tanh_f16_into);
    cuda_unary_baracuda_wrapper!(relu_f16, bk::unary_relu_f16_into);
    cuda_unary_baracuda_wrapper!(gelu_f16, bk::unary_gelu_f16_into);
    cuda_unary_baracuda_wrapper!(gelu_tanh_f16, bk::unary_gelu_tanh_f16_into);
    cuda_unary_baracuda_wrapper!(step_f16, bk::unary_step_f16_into);
    cuda_unary_baracuda_wrapper!(silu_f16, bk::unary_silu_f16_into);
    cuda_unary_baracuda_wrapper!(sigmoid_f16, bk::unary_sigmoid_f16_into);
    cuda_unary_baracuda_wrapper!(rsqrt_f16, bk::unary_rsqrt_f16_into);
    cuda_unary_baracuda_wrapper!(floor_f16, bk::unary_floor_f16_into);
    cuda_unary_baracuda_wrapper!(ceil_f16, bk::unary_ceil_f16_into);
    cuda_unary_baracuda_wrapper!(round_f16, bk::unary_round_f16_into);
    cuda_unary_baracuda_wrapper!(erf_f16, bk::unary_erf_f16_into);

    // bf16
    cuda_unary_baracuda_wrapper!(neg_bf16, bk::unary_neg_bf16_into);
    cuda_unary_baracuda_wrapper!(abs_bf16, bk::unary_abs_bf16_into);
    cuda_unary_baracuda_wrapper!(sqr_bf16, bk::unary_sqr_bf16_into);
    cuda_unary_baracuda_wrapper!(sqrt_bf16, bk::unary_sqrt_bf16_into);
    cuda_unary_baracuda_wrapper!(recip_bf16, bk::unary_recip_bf16_into);
    cuda_unary_baracuda_wrapper!(exp_bf16, bk::unary_exp_bf16_into);
    cuda_unary_baracuda_wrapper!(log_bf16, bk::unary_log_bf16_into);
    cuda_unary_baracuda_wrapper!(sin_bf16, bk::unary_sin_bf16_into);
    cuda_unary_baracuda_wrapper!(cos_bf16, bk::unary_cos_bf16_into);
    cuda_unary_baracuda_wrapper!(tanh_bf16, bk::unary_tanh_bf16_into);
    cuda_unary_baracuda_wrapper!(relu_bf16, bk::unary_relu_bf16_into);
    cuda_unary_baracuda_wrapper!(gelu_bf16, bk::unary_gelu_bf16_into);
    cuda_unary_baracuda_wrapper!(gelu_tanh_bf16, bk::unary_gelu_tanh_bf16_into);
    cuda_unary_baracuda_wrapper!(step_bf16, bk::unary_step_bf16_into);
    cuda_unary_baracuda_wrapper!(silu_bf16, bk::unary_silu_bf16_into);
    cuda_unary_baracuda_wrapper!(sigmoid_bf16, bk::unary_sigmoid_bf16_into);
    cuda_unary_baracuda_wrapper!(rsqrt_bf16, bk::unary_rsqrt_bf16_into);
    cuda_unary_baracuda_wrapper!(floor_bf16, bk::unary_floor_bf16_into);
    cuda_unary_baracuda_wrapper!(ceil_bf16, bk::unary_ceil_bf16_into);
    cuda_unary_baracuda_wrapper!(round_bf16, bk::unary_round_bf16_into);
    cuda_unary_baracuda_wrapper!(erf_bf16, bk::unary_erf_bf16_into);

    // f64
    cuda_unary_baracuda_wrapper!(neg_f64, bk::unary_neg_f64_into);
    cuda_unary_baracuda_wrapper!(abs_f64, bk::unary_abs_f64_into);
    cuda_unary_baracuda_wrapper!(sqr_f64, bk::unary_sqr_f64_into);
    cuda_unary_baracuda_wrapper!(sqrt_f64, bk::unary_sqrt_f64_into);
    cuda_unary_baracuda_wrapper!(recip_f64, bk::unary_recip_f64_into);
    cuda_unary_baracuda_wrapper!(exp_f64, bk::unary_exp_f64_into);
    cuda_unary_baracuda_wrapper!(log_f64, bk::unary_log_f64_into);
    cuda_unary_baracuda_wrapper!(sin_f64, bk::unary_sin_f64_into);
    cuda_unary_baracuda_wrapper!(cos_f64, bk::unary_cos_f64_into);
    cuda_unary_baracuda_wrapper!(tanh_f64, bk::unary_tanh_f64_into);
    cuda_unary_baracuda_wrapper!(relu_f64, bk::unary_relu_f64_into);
    cuda_unary_baracuda_wrapper!(gelu_f64, bk::unary_gelu_f64_into);
    cuda_unary_baracuda_wrapper!(gelu_tanh_f64, bk::unary_gelu_tanh_f64_into);
    cuda_unary_baracuda_wrapper!(step_f64, bk::unary_step_f64_into);
    cuda_unary_baracuda_wrapper!(silu_f64, bk::unary_silu_f64_into);
    cuda_unary_baracuda_wrapper!(sigmoid_f64, bk::unary_sigmoid_f64_into);
    cuda_unary_baracuda_wrapper!(rsqrt_f64, bk::unary_rsqrt_f64_into);
    cuda_unary_baracuda_wrapper!(floor_f64, bk::unary_floor_f64_into);
    cuda_unary_baracuda_wrapper!(ceil_f64, bk::unary_ceil_f64_into);
    cuda_unary_baracuda_wrapper!(round_f64, bk::unary_round_f64_into);
    cuda_unary_baracuda_wrapper!(erf_f64, bk::unary_erf_f64_into);

    // Wrappers are `pub fn` (per the macro); accessible as
    // `unary::<name>` from `register_baracuda_cuda_kernels` below.
}

/// The authored CUDA (baracuda) cast kernel contract, embedded into the binary
/// via `include_str!` (the PRODUCTION contract). Parsed + lowered by
/// [`register_cuda_cast_from_contract`].
const CUDA_CAST_CONTRACT: &str = include_str!("../../docs/kernel-contracts/cuda/cast.fkc.md");

/// Register the CUDA cast family (70 `(SRC, DST)` pairs → the ONE dtype-agnostic
/// `cast::cast_baracuda_wrapper`) by IMPORTING its FKC kernel contract — the
/// **first CUDA-backend FKC consumer**, the baracuda analogue of the CPU
/// `register_cpu_*_from_contract` / Vulkan `register_vulkan_*_from_contract`
/// families. FKC is unconditional core infrastructure, so this is the ONE
/// registration path for the family: the hand-written `table.register(OpKind::Cast,
/// &[src, dst], …)` calls (the 8×8 double-loop + the F8E4M3 legs) are DELETED.
///
/// The contract is authored per-DESTINATION (`## cast_to_<dst>`), each section
/// fanning its `src` operand over the accepted sources (§3.4 dtype fan-out) and
/// pinning a `fixed(DST)` output, so the importer resolves
/// `cast_to_<dst>_<src_suffix>` through the production
/// [`crate::fkc::CudaLinkRegistry`] and keys `[SRC, DST]` — byte-for-byte the
/// deleted regs. Because the CUDA cast is a SINGLE wrapper (it reads both dtypes
/// off the in/out Storage and dispatches into baracuda's 8×8 FFI), every one of
/// the 70 fanned symbols resolves to the SAME fn-pointer — a synthetic-base
/// umbrella (the Vulkan shape / pad-copy precedent). Distinct `(Cast, [SRC, DST])`
/// keys are legal sibling registrations of one wrapper, so `finalize` (which only
/// rejects the same `KernelRef` twice at ONE key) passes.
///
/// Behavior-preserving vs. the deleted hand-written path: identical wrapper +
/// contiguous-only caps (`awkward_layout_strategy: requires_contiguous` ⇒
/// `strided_input == false`, matching the deleted plain `table.register` default
/// caps). The binding's `kernel_source` becomes the contract's `"baracuda"` tag
/// (the deleted path stamped `""`) and its precision the contract's author seed
/// (`audited: false` ⇒ `PrecisionGuarantee::UNAUDITED`, byte-for-byte the deleted
/// regs' default). Cost is preserved because this runs BEFORE the
/// `fill_unset_cost_for_backend(Cuda, …)` pass in `register_optional_backends`
/// (`dispatch.rs`), which upgrades the imported `unknown_cost` sentinels to the
/// same OpKind cost fn every other CUDA primitive gets.
///
/// The family declares NO fused ops. Init-boundary fail-fast: a parse/lower/link
/// failure of the embedded, authored contract is a programmer error surfaced once
/// here via `expect` (mirroring the CPU / Vulkan convention); it cannot fail for a
/// runtime-data reason.
fn register_cuda_cast_from_contract(table: &mut KernelBindingTable) {
    let provider = crate::fkc::import_bundle_str(CUDA_CAST_CONTRACT, &crate::fkc::CudaLinkRegistry)
        .expect(
            "authored CUDA cast contract must import \
                 (embedded via include_str!, resolved through CudaLinkRegistry)",
        );
    debug_assert!(
        provider.fused.is_empty(),
        "cuda cast contract declares no fused ops",
    );
    let mut fused = crate::fused::FusedKernelRegistry::new();
    provider
        .register_into(table, &mut fused)
        .expect("CUDA cast contract must register into the binding table");
}

/// The authored CUDA (baracuda) FlashDecoding contract, embedded via
/// `include_str!`. Imported by [`register_cuda_flash_decoding_from_contract`].
const CUDA_FLASH_DECODING_CONTRACT: &str =
    include_str!("../../docs/kernel-contracts/cuda/flash_decoding.fkc.md");

/// Register the CUDA FlashDecoding decode arm (`OpKind::FlashAttn`, `[f16;4]` /
/// `[bf16;4]`, capacity-K) by IMPORTING its FKC kernel contract — the SOLE path,
/// replacing the two DELETED hand-written `table.register_full(FlashAttn, …)`
/// regs. The LAST CUDA family to migrate (CUDA now 31/31 contract-sourced).
///
/// **This is the §4.4 cost-fn trampoline (Task-F) consumer.** The contract's
/// `cost.cost_fn: cost_flash_decoding_cuda` is resolved through
/// [`crate::fkc::CudaLinkRegistry`]'s `CUDA_COST_FNS` table and pinned on the
/// binding, so the CUSTOM static infeasibility gate (`cost_flash_decoding_cuda`:
/// INFEASIBLE cost for `seq_q != 1` / `head_dim` outside `[1, 128]`) SURVIVES
/// the `fill_unset_cost_for_backend(Cuda, …)` pass — exactly the guarantee the
/// hand-written `register_full` (not `register_with_caps`) previously bought by
/// stamping the cost fn directly. Without the trampoline an FKC import would
/// stamp `unknown_cost` → `fill_unset` would clobber it with the shape-blind
/// shared `FlashAttn` cost fn (`cost_flash_attn_primitive_cpu`) → the gate lost.
///
/// Behavior-preserving vs. the deleted regs: the SAME `flash_decoding_{f16,bf16}`
/// wrappers (resolved through `CUDA_FLASH_DECODING_ENTRY_POINTS`), `strided_input`
/// caps (each operand accepts strided + broadcast ⇒ the §6 projection yields
/// `true`, byte-for-byte `KernelCaps::strided_input()`), and UNAUDITED precision
/// (`audited: false`). `kernel_source` becomes the contract's `"baracuda"` (the
/// deleted regs stamped `""`). Runs INSIDE `register_baracuda_cuda_kernels`,
/// BEFORE the `fill_unset_cost_for_backend(Cuda, …)` pass in
/// `register_optional_backends` (dispatch.rs). Init-boundary fail-fast `expect`.
fn register_cuda_flash_decoding_from_contract(table: &mut KernelBindingTable) {
    let provider =
        crate::fkc::import_bundle_str(CUDA_FLASH_DECODING_CONTRACT, &crate::fkc::CudaLinkRegistry)
            .expect(
                "authored CUDA flash_decoding contract must import \
                 (embedded via include_str!, resolved through CudaLinkRegistry)",
            );
    debug_assert!(
        provider.fused.is_empty(),
        "cuda flash_decoding contract declares no fused ops",
    );
    let mut fused = crate::fused::FusedKernelRegistry::new();
    provider
        .register_into(table, &mut fused)
        .expect("CUDA flash_decoding contract must register into the binding table");
}

// ===========================================================================
// GROUP 1 CUDA FKC families (per-(op,dtype) strided) - contract registration.
// ===========================================================================

/// The authored CUDA (baracuda) binary kernel contract (`include_str!`), imported by
/// [`register_cuda_binary_from_contract`] - the SOLE registration path (hand-written regs DELETED).
const CUDA_BINARY_CONTRACT: &str = include_str!("../../docs/kernel-contracts/cuda/binary.fkc.md");

/// The authored CUDA (baracuda) COMPARISON kernel contract (`include_str!`),
/// imported by [`register_cuda_compare_from_contract`].
///
/// Authoring the `.fkc.md` and registering its entry points is NOT sufficient
/// on its own: the link registry only resolves `symbol -> KernelRef`, and a
/// contract that is never `include_str!`d and imported here declares nothing
/// into the binding table. `Op::PagedAttn`'s recipe stays unplaceable until
/// all three layers exist.
const CUDA_COMPARE_CONTRACT: &str = include_str!("../../docs/kernel-contracts/cuda/compare.fkc.md");

/// Register the CUDA comparison family (6 ops x 4 float dtypes -> U8) FROM its
/// FKC contract. This is what makes `GreaterEqualElementwise[F32,F32,U8]`
/// resolvable on CUDA — the single node that kept `Op::PagedAttn`'s primitive
/// recipe off the device.
fn register_cuda_compare_from_contract(table: &mut KernelBindingTable) {
    let provider =
        crate::fkc::import_bundle_str(CUDA_COMPARE_CONTRACT, &crate::fkc::CudaLinkRegistry).expect(
            "authored CUDA compare contract must import (embedded include_str!, CudaLinkRegistry)",
        );
    debug_assert!(
        provider.fused.is_empty(),
        "cuda compare contract declares no fused ops"
    );
    let mut fused = crate::fused::FusedKernelRegistry::new();
    provider
        .register_into(table, &mut fused)
        .expect("CUDA compare contract must register into the binding table");
}

/// Register the CUDA binary family FROM its FKC contract (per-(op,dtype), fanned
/// `<op>_<dtype>` -> the distinct production wrappers via [`crate::fkc::CudaLinkRegistry`]).
/// Runs BEFORE `fill_unset_cost_for_backend(Cuda, ..)`; caps/precision ride through the import.
fn register_cuda_binary_from_contract(table: &mut KernelBindingTable) {
    let provider =
        crate::fkc::import_bundle_str(CUDA_BINARY_CONTRACT, &crate::fkc::CudaLinkRegistry).expect(
            "authored CUDA binary contract must import (embedded include_str!, CudaLinkRegistry)",
        );
    debug_assert!(
        provider.fused.is_empty(),
        "cuda binary contract declares no fused ops"
    );
    let mut fused = crate::fused::FusedKernelRegistry::new();
    provider
        .register_into(table, &mut fused)
        .expect("CUDA binary contract must register into the binding table");
}

/// The authored CUDA (baracuda) reduce kernel contract (`include_str!`), imported by
/// [`register_cuda_reduce_from_contract`] - the SOLE registration path (hand-written regs DELETED).
const CUDA_REDUCE_CONTRACT: &str = include_str!("../../docs/kernel-contracts/cuda/reduce.fkc.md");

/// Register the CUDA reduce family FROM its FKC contract (per-(op,dtype), fanned
/// `<op>_<dtype>` -> the distinct production wrappers via [`crate::fkc::CudaLinkRegistry`]).
/// Runs BEFORE `fill_unset_cost_for_backend(Cuda, ..)`; caps/precision ride through the import.
fn register_cuda_reduce_from_contract(table: &mut KernelBindingTable) {
    let provider =
        crate::fkc::import_bundle_str(CUDA_REDUCE_CONTRACT, &crate::fkc::CudaLinkRegistry).expect(
            "authored CUDA reduce contract must import (embedded include_str!, CudaLinkRegistry)",
        );
    debug_assert!(
        provider.fused.is_empty(),
        "cuda reduce contract declares no fused ops"
    );
    let mut fused = crate::fused::FusedKernelRegistry::new();
    provider
        .register_into(table, &mut fused)
        .expect("CUDA reduce contract must register into the binding table");
}

/// The authored CUDA (baracuda) norm kernel contract (`include_str!`), imported by
/// [`register_cuda_norm_from_contract`] - the SOLE registration path (hand-written regs DELETED).
const CUDA_NORM_CONTRACT: &str = include_str!("../../docs/kernel-contracts/cuda/norm.fkc.md");

/// Register the CUDA norm family FROM its FKC contract (per-(op,dtype), fanned
/// `<op>_<dtype>` -> the distinct production wrappers via [`crate::fkc::CudaLinkRegistry`]).
/// Runs BEFORE `fill_unset_cost_for_backend(Cuda, ..)`; caps/precision ride through the import.
fn register_cuda_norm_from_contract(table: &mut KernelBindingTable) {
    let provider = crate::fkc::import_bundle_str(CUDA_NORM_CONTRACT, &crate::fkc::CudaLinkRegistry)
        .expect(
            "authored CUDA norm contract must import (embedded include_str!, CudaLinkRegistry)",
        );
    debug_assert!(
        provider.fused.is_empty(),
        "cuda norm contract declares no fused ops"
    );
    let mut fused = crate::fused::FusedKernelRegistry::new();
    provider
        .register_into(table, &mut fused)
        .expect("CUDA norm contract must register into the binding table");
}

/// The authored CUDA (baracuda) softmax kernel contract (`include_str!`), imported by
/// [`register_cuda_softmax_from_contract`] - the SOLE registration path (hand-written regs DELETED).
const CUDA_SOFTMAX_CONTRACT: &str = include_str!("../../docs/kernel-contracts/cuda/softmax.fkc.md");

/// Register the CUDA softmax family FROM its FKC contract (per-(op,dtype), fanned
/// `<op>_<dtype>` -> the distinct production wrappers via [`crate::fkc::CudaLinkRegistry`]).
/// Runs BEFORE `fill_unset_cost_for_backend(Cuda, ..)`; caps/precision ride through the import.
fn register_cuda_softmax_from_contract(table: &mut KernelBindingTable) {
    let provider =
        crate::fkc::import_bundle_str(CUDA_SOFTMAX_CONTRACT, &crate::fkc::CudaLinkRegistry).expect(
            "authored CUDA softmax contract must import (embedded include_str!, CudaLinkRegistry)",
        );
    debug_assert!(
        provider.fused.is_empty(),
        "cuda softmax contract declares no fused ops"
    );
    let mut fused = crate::fused::FusedKernelRegistry::new();
    provider
        .register_into(table, &mut fused)
        .expect("CUDA softmax contract must register into the binding table");
}

/// The authored CUDA (baracuda) powi kernel contract (`include_str!`), imported by
/// [`register_cuda_powi_from_contract`] - the SOLE registration path (hand-written regs DELETED).
const CUDA_POWI_CONTRACT: &str = include_str!("../../docs/kernel-contracts/cuda/powi.fkc.md");

/// Register the CUDA powi family FROM its FKC contract (per-(op,dtype), fanned
/// `<op>_<dtype>` -> the distinct production wrappers via [`crate::fkc::CudaLinkRegistry`]).
/// Runs BEFORE `fill_unset_cost_for_backend(Cuda, ..)`; caps/precision ride through the import.
fn register_cuda_powi_from_contract(table: &mut KernelBindingTable) {
    let provider = crate::fkc::import_bundle_str(CUDA_POWI_CONTRACT, &crate::fkc::CudaLinkRegistry)
        .expect(
            "authored CUDA powi contract must import (embedded include_str!, CudaLinkRegistry)",
        );
    debug_assert!(
        provider.fused.is_empty(),
        "cuda powi contract declares no fused ops"
    );
    let mut fused = crate::fused::FusedKernelRegistry::new();
    provider
        .register_into(table, &mut fused)
        .expect("CUDA powi contract must register into the binding table");
}

/// The authored CUDA (baracuda) powi_backward kernel contract (`include_str!`), imported by
/// [`register_cuda_powi_backward_from_contract`] - the SOLE registration path (hand-written regs DELETED).
const CUDA_POWI_BACKWARD_CONTRACT: &str =
    include_str!("../../docs/kernel-contracts/cuda/powi_backward.fkc.md");

/// Register the CUDA powi_backward family FROM its FKC contract (per-(op,dtype), fanned
/// `<op>_<dtype>` -> the distinct production wrappers via [`crate::fkc::CudaLinkRegistry`]).
/// Runs BEFORE `fill_unset_cost_for_backend(Cuda, ..)`; caps/precision ride through the import.
fn register_cuda_powi_backward_from_contract(table: &mut KernelBindingTable) {
    let provider = crate::fkc::import_bundle_str(CUDA_POWI_BACKWARD_CONTRACT, &crate::fkc::CudaLinkRegistry)
        .expect("authored CUDA powi_backward contract must import (embedded include_str!, CudaLinkRegistry)");
    debug_assert!(
        provider.fused.is_empty(),
        "cuda powi_backward contract declares no fused ops"
    );
    let mut fused = crate::fused::FusedKernelRegistry::new();
    provider
        .register_into(table, &mut fused)
        .expect("CUDA powi_backward contract must register into the binding table");
}

/// The authored CUDA (baracuda) clamp kernel contract (`include_str!`), imported by
/// [`register_cuda_clamp_from_contract`] - the SOLE registration path (hand-written regs DELETED).
const CUDA_CLAMP_CONTRACT: &str = include_str!("../../docs/kernel-contracts/cuda/clamp.fkc.md");

/// Register the CUDA clamp family FROM its FKC contract (per-(op,dtype), fanned
/// `<op>_<dtype>` -> the distinct production wrappers via [`crate::fkc::CudaLinkRegistry`]).
/// Runs BEFORE `fill_unset_cost_for_backend(Cuda, ..)`; caps/precision ride through the import.
fn register_cuda_clamp_from_contract(table: &mut KernelBindingTable) {
    let provider =
        crate::fkc::import_bundle_str(CUDA_CLAMP_CONTRACT, &crate::fkc::CudaLinkRegistry).expect(
            "authored CUDA clamp contract must import (embedded include_str!, CudaLinkRegistry)",
        );
    debug_assert!(
        provider.fused.is_empty(),
        "cuda clamp contract declares no fused ops"
    );
    let mut fused = crate::fused::FusedKernelRegistry::new();
    provider
        .register_into(table, &mut fused)
        .expect("CUDA clamp contract must register into the binding table");
}

/// The authored CUDA (baracuda) flip kernel contract (`include_str!`), imported by
/// [`register_cuda_flip_from_contract`] - the SOLE registration path (hand-written regs DELETED).
const CUDA_FLIP_CONTRACT: &str = include_str!("../../docs/kernel-contracts/cuda/flip.fkc.md");

/// Register the CUDA flip family FROM its FKC contract (per-(op,dtype), fanned
/// `<op>_<dtype>` -> the distinct production wrappers via [`crate::fkc::CudaLinkRegistry`]).
/// Runs BEFORE `fill_unset_cost_for_backend(Cuda, ..)`; caps/precision ride through the import.
fn register_cuda_flip_from_contract(table: &mut KernelBindingTable) {
    let provider = crate::fkc::import_bundle_str(CUDA_FLIP_CONTRACT, &crate::fkc::CudaLinkRegistry)
        .expect(
            "authored CUDA flip contract must import (embedded include_str!, CudaLinkRegistry)",
        );
    debug_assert!(
        provider.fused.is_empty(),
        "cuda flip contract declares no fused ops"
    );
    let mut fused = crate::fused::FusedKernelRegistry::new();
    provider
        .register_into(table, &mut fused)
        .expect("CUDA flip contract must register into the binding table");
}

/// The authored CUDA (baracuda) roll kernel contract (`include_str!`), imported by
/// [`register_cuda_roll_from_contract`] - the SOLE registration path (hand-written regs DELETED).
const CUDA_ROLL_CONTRACT: &str = include_str!("../../docs/kernel-contracts/cuda/roll.fkc.md");

/// Register the CUDA roll family FROM its FKC contract (per-(op,dtype), fanned
/// `<op>_<dtype>` -> the distinct production wrappers via [`crate::fkc::CudaLinkRegistry`]).
/// Runs BEFORE `fill_unset_cost_for_backend(Cuda, ..)`; caps/precision ride through the import.
fn register_cuda_roll_from_contract(table: &mut KernelBindingTable) {
    let provider = crate::fkc::import_bundle_str(CUDA_ROLL_CONTRACT, &crate::fkc::CudaLinkRegistry)
        .expect(
            "authored CUDA roll contract must import (embedded include_str!, CudaLinkRegistry)",
        );
    debug_assert!(
        provider.fused.is_empty(),
        "cuda roll contract declares no fused ops"
    );
    let mut fused = crate::fused::FusedKernelRegistry::new();
    provider
        .register_into(table, &mut fused)
        .expect("CUDA roll contract must register into the binding table");
}

/// The authored CUDA (baracuda) cumsum kernel contract (`include_str!`), imported by
/// [`register_cuda_cumsum_from_contract`] - the SOLE registration path (hand-written regs DELETED).
const CUDA_CUMSUM_CONTRACT: &str = include_str!("../../docs/kernel-contracts/cuda/cumsum.fkc.md");

/// Register the CUDA cumsum family FROM its FKC contract (per-(op,dtype), fanned
/// `<op>_<dtype>` -> the distinct production wrappers via [`crate::fkc::CudaLinkRegistry`]).
/// Runs BEFORE `fill_unset_cost_for_backend(Cuda, ..)`; caps/precision ride through the import.
fn register_cuda_cumsum_from_contract(table: &mut KernelBindingTable) {
    let provider =
        crate::fkc::import_bundle_str(CUDA_CUMSUM_CONTRACT, &crate::fkc::CudaLinkRegistry).expect(
            "authored CUDA cumsum contract must import (embedded include_str!, CudaLinkRegistry)",
        );
    debug_assert!(
        provider.fused.is_empty(),
        "cuda cumsum contract declares no fused ops"
    );
    let mut fused = crate::fused::FusedKernelRegistry::new();
    provider
        .register_into(table, &mut fused)
        .expect("CUDA cumsum contract must register into the binding table");
}

/// The authored CUDA (baracuda) triangular kernel contract (`include_str!`), imported by
/// [`register_cuda_triangular_from_contract`] - the SOLE registration path (hand-written regs DELETED).
const CUDA_TRIANGULAR_CONTRACT: &str =
    include_str!("../../docs/kernel-contracts/cuda/triangular.fkc.md");

/// Register the CUDA triangular family FROM its FKC contract (per-(op,dtype), fanned
/// `<op>_<dtype>` -> the distinct production wrappers via [`crate::fkc::CudaLinkRegistry`]).
/// Runs BEFORE `fill_unset_cost_for_backend(Cuda, ..)`; caps/precision ride through the import.
fn register_cuda_triangular_from_contract(table: &mut KernelBindingTable) {
    let provider = crate::fkc::import_bundle_str(
        CUDA_TRIANGULAR_CONTRACT,
        &crate::fkc::CudaLinkRegistry,
    )
    .expect(
        "authored CUDA triangular contract must import (embedded include_str!, CudaLinkRegistry)",
    );
    debug_assert!(
        provider.fused.is_empty(),
        "cuda triangular contract declares no fused ops"
    );
    let mut fused = crate::fused::FusedKernelRegistry::new();
    provider
        .register_into(table, &mut fused)
        .expect("CUDA triangular contract must register into the binding table");
}

/// The authored CUDA (baracuda) arg_reduce kernel contract (`include_str!`), imported by
/// [`register_cuda_arg_reduce_from_contract`] - the SOLE registration path (hand-written regs DELETED).
const CUDA_ARG_REDUCE_CONTRACT: &str =
    include_str!("../../docs/kernel-contracts/cuda/arg_reduce.fkc.md");

/// Register the CUDA arg_reduce family FROM its FKC contract (per-(op,dtype), fanned
/// `<op>_<dtype>` -> the distinct production wrappers via [`crate::fkc::CudaLinkRegistry`]).
/// Runs BEFORE `fill_unset_cost_for_backend(Cuda, ..)`; caps/precision ride through the import.
fn register_cuda_arg_reduce_from_contract(table: &mut KernelBindingTable) {
    let provider = crate::fkc::import_bundle_str(
        CUDA_ARG_REDUCE_CONTRACT,
        &crate::fkc::CudaLinkRegistry,
    )
    .expect(
        "authored CUDA arg_reduce contract must import (embedded include_str!, CudaLinkRegistry)",
    );
    debug_assert!(
        provider.fused.is_empty(),
        "cuda arg_reduce contract declares no fused ops"
    );
    let mut fused = crate::fused::FusedKernelRegistry::new();
    provider
        .register_into(table, &mut fused)
        .expect("CUDA arg_reduce contract must register into the binding table");
}

/// The authored CUDA (baracuda) reduce_to kernel contract (`include_str!`), imported by
/// [`register_cuda_reduce_to_from_contract`] - the SOLE registration path (hand-written regs DELETED).
const CUDA_REDUCE_TO_CONTRACT: &str =
    include_str!("../../docs/kernel-contracts/cuda/reduce_to.fkc.md");

/// Register the CUDA reduce_to family FROM its FKC contract (per-(op,dtype), fanned
/// `<op>_<dtype>` -> the distinct production wrappers via [`crate::fkc::CudaLinkRegistry`]).
/// Runs BEFORE `fill_unset_cost_for_backend(Cuda, ..)`; caps/precision ride through the import.
fn register_cuda_reduce_to_from_contract(table: &mut KernelBindingTable) {
    let provider = crate::fkc::import_bundle_str(
        CUDA_REDUCE_TO_CONTRACT,
        &crate::fkc::CudaLinkRegistry,
    )
    .expect(
        "authored CUDA reduce_to contract must import (embedded include_str!, CudaLinkRegistry)",
    );
    debug_assert!(
        provider.fused.is_empty(),
        "cuda reduce_to contract declares no fused ops"
    );
    let mut fused = crate::fused::FusedKernelRegistry::new();
    provider
        .register_into(table, &mut fused)
        .expect("CUDA reduce_to contract must register into the binding table");
}

/// The authored CUDA (baracuda) rope kernel contract (`include_str!`), imported by
/// [`register_cuda_rope_from_contract`] - the SOLE registration path (hand-written regs DELETED).
const CUDA_ROPE_CONTRACT: &str = include_str!("../../docs/kernel-contracts/cuda/rope.fkc.md");

/// Register the CUDA rope family FROM its FKC contract (per-(op,dtype), fanned
/// `<op>_<dtype>` -> the distinct production wrappers via [`crate::fkc::CudaLinkRegistry`]).
/// Runs BEFORE `fill_unset_cost_for_backend(Cuda, ..)`; caps/precision ride through the import.
fn register_cuda_rope_from_contract(table: &mut KernelBindingTable) {
    let provider = crate::fkc::import_bundle_str(CUDA_ROPE_CONTRACT, &crate::fkc::CudaLinkRegistry)
        .expect(
            "authored CUDA rope contract must import (embedded include_str!, CudaLinkRegistry)",
        );
    debug_assert!(
        provider.fused.is_empty(),
        "cuda rope contract declares no fused ops"
    );
    let mut fused = crate::fused::FusedKernelRegistry::new();
    provider
        .register_into(table, &mut fused)
        .expect("CUDA rope contract must register into the binding table");
}

// ===========================================================================
// FUSED FusedOps::ROPE — CapturedRun 4b-resume: REVERTED (2026-07-13).
// ===========================================================================
//
// baracuda's `rope_apply_<dt>` was registered here as the CUDA impl of
// `FusedOps::ROPE` (commit 3be28ab1) and then reverted: `rope_apply` uses the
// INTERLEAVED pairing `(2k, 2k+1)` (GPT-J) while Fuel's `FusedOps::ROPE`
// (`Tensor::rope_with_tables`) is ROTATE-HALF `(j, j+head_dim/2)` (Llama/HF)
// — different functions (confirmed by baracuda's own `rope_fp.cu` "rotates
// pairs (2i, 2i+1)" and a GPU numerical test in
// `fuel_cuda_backend::baracuda::attention::fused_rope_tests`). CUDA rope runs
// via the correct rotate-half primitive decompose; the staged (unwired)
// `rope_apply_fused_<dt>_into` driver + `RopeTableCache` remain in
// `fuel-cuda-backend`, ready to wire once baracuda ships a rotate-half
// table-driven `rope_apply` variant.

// ===========================================================================
// GROUP3 CUDA FKC families - contract registration.
// ===========================================================================

/// The authored CUDA (baracuda) unary kernel contract (`include_str!`), imported by
/// [`register_cuda_unary_from_contract`] - the SOLE registration path (hand-written regs DELETED).
const CUDA_UNARY_CONTRACT: &str = include_str!("../../docs/kernel-contracts/cuda/unary.fkc.md");

/// Register the CUDA unary family FROM its FKC contract (per-(op,dtype), fanned
/// `<op>_<dtype>` -> the distinct production wrappers via [`crate::fkc::CudaLinkRegistry`]).
/// Runs BEFORE `fill_unset_cost_for_backend(Cuda, ..)`; caps/precision ride through the import.
fn register_cuda_unary_from_contract(table: &mut KernelBindingTable) {
    let provider =
        crate::fkc::import_bundle_str(CUDA_UNARY_CONTRACT, &crate::fkc::CudaLinkRegistry).expect(
            "authored CUDA unary contract must import (embedded include_str!, CudaLinkRegistry)",
        );
    debug_assert!(
        provider.fused.is_empty(),
        "cuda unary contract declares no fused ops"
    );
    let mut fused = crate::fused::FusedKernelRegistry::new();
    provider
        .register_into(table, &mut fused)
        .expect("CUDA unary contract must register into the binding table");
}

// ===========================================================================
// GROUP2 CUDA FKC families - contract registration.
// ===========================================================================

/// The authored CUDA (baracuda) clamp_inplace kernel contract (`include_str!`), imported by
/// [`register_cuda_clamp_inplace_from_contract`] - the SOLE registration path (hand-written regs DELETED).
const CUDA_CLAMP_INPLACE_CONTRACT: &str =
    include_str!("../../docs/kernel-contracts/cuda/clamp_inplace.fkc.md");

/// Register the CUDA clamp_inplace family FROM its FKC contract (per-(op,dtype), fanned
/// `<op>_<dtype>` -> the distinct production wrappers via [`crate::fkc::CudaLinkRegistry`]).
/// Runs BEFORE `fill_unset_cost_for_backend(Cuda, ..)`; caps/precision ride through the import.
fn register_cuda_clamp_inplace_from_contract(table: &mut KernelBindingTable) {
    let provider = crate::fkc::import_bundle_str(CUDA_CLAMP_INPLACE_CONTRACT, &crate::fkc::CudaLinkRegistry)
        .expect("authored CUDA clamp_inplace contract must import (embedded include_str!, CudaLinkRegistry)");
    debug_assert!(
        provider.fused.is_empty(),
        "cuda clamp_inplace contract declares no fused ops"
    );
    let mut fused = crate::fused::FusedKernelRegistry::new();
    provider
        .register_into(table, &mut fused)
        .expect("CUDA clamp_inplace contract must register into the binding table");
}

/// The authored CUDA (baracuda) powi_inplace kernel contract (`include_str!`), imported by
/// [`register_cuda_powi_inplace_from_contract`] - the SOLE registration path (hand-written regs DELETED).
const CUDA_POWI_INPLACE_CONTRACT: &str =
    include_str!("../../docs/kernel-contracts/cuda/powi_inplace.fkc.md");

/// Register the CUDA powi_inplace family FROM its FKC contract (per-(op,dtype), fanned
/// `<op>_<dtype>` -> the distinct production wrappers via [`crate::fkc::CudaLinkRegistry`]).
/// Runs BEFORE `fill_unset_cost_for_backend(Cuda, ..)`; caps/precision ride through the import.
fn register_cuda_powi_inplace_from_contract(table: &mut KernelBindingTable) {
    let provider = crate::fkc::import_bundle_str(
        CUDA_POWI_INPLACE_CONTRACT,
        &crate::fkc::CudaLinkRegistry,
    )
    .expect(
        "authored CUDA powi_inplace contract must import (embedded include_str!, CudaLinkRegistry)",
    );
    debug_assert!(
        provider.fused.is_empty(),
        "cuda powi_inplace contract declares no fused ops"
    );
    let mut fused = crate::fused::FusedKernelRegistry::new();
    provider
        .register_into(table, &mut fused)
        .expect("CUDA powi_inplace contract must register into the binding table");
}

/// The authored CUDA (baracuda) inplace_unary kernel contract (`include_str!`), imported by
/// [`register_cuda_inplace_unary_from_contract`] - the SOLE registration path (hand-written regs DELETED).
const CUDA_INPLACE_UNARY_CONTRACT: &str =
    include_str!("../../docs/kernel-contracts/cuda/inplace_unary.fkc.md");

/// Register the CUDA inplace_unary family FROM its FKC contract (per-(op,dtype), fanned
/// `<op>_<dtype>` -> the distinct production wrappers via [`crate::fkc::CudaLinkRegistry`]).
/// Runs BEFORE `fill_unset_cost_for_backend(Cuda, ..)`; caps/precision ride through the import.
fn register_cuda_inplace_unary_from_contract(table: &mut KernelBindingTable) {
    let provider = crate::fkc::import_bundle_str(CUDA_INPLACE_UNARY_CONTRACT, &crate::fkc::CudaLinkRegistry)
        .expect("authored CUDA inplace_unary contract must import (embedded include_str!, CudaLinkRegistry)");
    debug_assert!(
        provider.fused.is_empty(),
        "cuda inplace_unary contract declares no fused ops"
    );
    let mut fused = crate::fused::FusedKernelRegistry::new();
    provider
        .register_into(table, &mut fused)
        .expect("CUDA inplace_unary contract must register into the binding table");
}

// ===========================================================================
// GROUP4 CUDA FKC families - contract registration.
// ===========================================================================

/// The authored CUDA (baracuda) write_slice kernel contract (`include_str!`), imported by
/// [`register_cuda_write_slice_from_contract`] - the SOLE registration path (hand-written regs DELETED).
const CUDA_WRITE_SLICE_CONTRACT: &str =
    include_str!("../../docs/kernel-contracts/cuda/write_slice.fkc.md");

/// Register the CUDA write_slice family FROM its FKC contract (per-(op,dtype), fanned
/// `<op>_<dtype>` -> the distinct production wrappers via [`crate::fkc::CudaLinkRegistry`]).
/// Runs BEFORE `fill_unset_cost_for_backend(Cuda, ..)`; caps/precision ride through the import.
fn register_cuda_write_slice_from_contract(table: &mut KernelBindingTable) {
    let provider = crate::fkc::import_bundle_str(
        CUDA_WRITE_SLICE_CONTRACT,
        &crate::fkc::CudaLinkRegistry,
    )
    .expect(
        "authored CUDA write_slice contract must import (embedded include_str!, CudaLinkRegistry)",
    );
    debug_assert!(
        provider.fused.is_empty(),
        "cuda write_slice contract declares no fused ops"
    );
    let mut fused = crate::fused::FusedKernelRegistry::new();
    provider
        .register_into(table, &mut fused)
        .expect("CUDA write_slice contract must register into the binding table");
}

/// The authored CUDA (baracuda) write_slice_rotating kernel contract (`include_str!`), imported by
/// [`register_cuda_write_slice_rotating_from_contract`] - the SOLE registration path (hand-written regs DELETED).
const CUDA_WRITE_SLICE_ROTATING_CONTRACT: &str =
    include_str!("../../docs/kernel-contracts/cuda/write_slice_rotating.fkc.md");

/// Register the CUDA write_slice_rotating family FROM its FKC contract (per-(op,dtype), fanned
/// `<op>_<dtype>` -> the distinct production wrappers via [`crate::fkc::CudaLinkRegistry`]).
/// Runs BEFORE `fill_unset_cost_for_backend(Cuda, ..)`; caps/precision ride through the import.
fn register_cuda_write_slice_rotating_from_contract(table: &mut KernelBindingTable) {
    let provider = crate::fkc::import_bundle_str(CUDA_WRITE_SLICE_ROTATING_CONTRACT, &crate::fkc::CudaLinkRegistry)
        .expect("authored CUDA write_slice_rotating contract must import (embedded include_str!, CudaLinkRegistry)");
    debug_assert!(
        provider.fused.is_empty(),
        "cuda write_slice_rotating contract declares no fused ops"
    );
    let mut fused = crate::fused::FusedKernelRegistry::new();
    provider
        .register_into(table, &mut fused)
        .expect("CUDA write_slice_rotating contract must register into the binding table");
}

/// The authored CUDA (baracuda) write_slice_doff kernel contract (`include_str!`), imported by
/// [`register_cuda_write_slice_doff_from_contract`] - the SOLE registration path.
const CUDA_WRITE_SLICE_DOFF_CONTRACT: &str =
    include_str!("../../docs/kernel-contracts/cuda/write_slice_doff.fkc.md");

/// Register the CUDA write_slice_doff family FROM its FKC contract (per-(op,dtype), fanned
/// `<op>_<dtype>` -> the b1/b2/b4/b8 production wrappers via [`crate::fkc::CudaLinkRegistry`]).
/// Runs BEFORE `fill_unset_cost_for_backend(Cuda, ..)`; caps/precision ride through the import.
fn register_cuda_write_slice_doff_from_contract(table: &mut KernelBindingTable) {
    let provider = crate::fkc::import_bundle_str(CUDA_WRITE_SLICE_DOFF_CONTRACT, &crate::fkc::CudaLinkRegistry)
        .expect("authored CUDA write_slice_doff contract must import (embedded include_str!, CudaLinkRegistry)");
    debug_assert!(
        provider.fused.is_empty(),
        "cuda write_slice_doff contract declares no fused ops"
    );
    let mut fused = crate::fused::FusedKernelRegistry::new();
    provider
        .register_into(table, &mut fused)
        .expect("CUDA write_slice_doff contract must register into the binding table");
}

// ===========================================================================
// GROUP5 CUDA FKC families - contract registration.
// ===========================================================================

/// The authored CUDA (baracuda) concat kernel contract (`include_str!`), imported by
/// [`register_cuda_concat_from_contract`] - the SOLE registration path (hand-written regs DELETED).
const CUDA_CONCAT_CONTRACT: &str = include_str!("../../docs/kernel-contracts/cuda/concat.fkc.md");

/// Register the CUDA concat family FROM its FKC contract (per-(op,dtype), fanned
/// `<op>_<dtype>` -> the distinct production wrappers via [`crate::fkc::CudaLinkRegistry`]).
/// Runs BEFORE `fill_unset_cost_for_backend(Cuda, ..)`; caps/precision ride through the import.
fn register_cuda_concat_from_contract(table: &mut KernelBindingTable) {
    let provider =
        crate::fkc::import_bundle_str(CUDA_CONCAT_CONTRACT, &crate::fkc::CudaLinkRegistry).expect(
            "authored CUDA concat contract must import (embedded include_str!, CudaLinkRegistry)",
        );
    debug_assert!(
        provider.fused.is_empty(),
        "cuda concat contract declares no fused ops"
    );
    let mut fused = crate::fused::FusedKernelRegistry::new();
    provider
        .register_into(table, &mut fused)
        .expect("CUDA concat contract must register into the binding table");
}

/// The authored CUDA (baracuda) affine kernel contract (`include_str!`), imported by
/// [`register_cuda_affine_from_contract`] - the SOLE registration path (hand-written regs DELETED).
const CUDA_AFFINE_CONTRACT: &str = include_str!("../../docs/kernel-contracts/cuda/affine.fkc.md");

/// Register the CUDA affine family FROM its FKC contract (per-(op,dtype), fanned
/// `<op>_<dtype>` -> the distinct production wrappers via [`crate::fkc::CudaLinkRegistry`]).
/// Runs BEFORE `fill_unset_cost_for_backend(Cuda, ..)`; caps/precision ride through the import.
fn register_cuda_affine_from_contract(table: &mut KernelBindingTable) {
    let provider =
        crate::fkc::import_bundle_str(CUDA_AFFINE_CONTRACT, &crate::fkc::CudaLinkRegistry).expect(
            "authored CUDA affine contract must import (embedded include_str!, CudaLinkRegistry)",
        );
    debug_assert!(
        provider.fused.is_empty(),
        "cuda affine contract declares no fused ops"
    );
    let mut fused = crate::fused::FusedKernelRegistry::new();
    provider
        .register_into(table, &mut fused)
        .expect("CUDA affine contract must register into the binding table");
}

/// The authored CUDA (baracuda) inplace_affine kernel contract (`include_str!`), imported by
/// [`register_cuda_inplace_affine_from_contract`] - the SOLE registration path (hand-written regs DELETED).
const CUDA_INPLACE_AFFINE_CONTRACT: &str =
    include_str!("../../docs/kernel-contracts/cuda/inplace_affine.fkc.md");

/// Register the CUDA inplace_affine family FROM its FKC contract (per-(op,dtype), fanned
/// `<op>_<dtype>` -> the distinct production wrappers via [`crate::fkc::CudaLinkRegistry`]).
/// Runs BEFORE `fill_unset_cost_for_backend(Cuda, ..)`; caps/precision ride through the import.
fn register_cuda_inplace_affine_from_contract(table: &mut KernelBindingTable) {
    let provider = crate::fkc::import_bundle_str(CUDA_INPLACE_AFFINE_CONTRACT, &crate::fkc::CudaLinkRegistry)
        .expect("authored CUDA inplace_affine contract must import (embedded include_str!, CudaLinkRegistry)");
    debug_assert!(
        provider.fused.is_empty(),
        "cuda inplace_affine contract declares no fused ops"
    );
    let mut fused = crate::fused::FusedKernelRegistry::new();
    provider
        .register_into(table, &mut fused)
        .expect("CUDA inplace_affine contract must register into the binding table");
}

/// The authored CUDA (baracuda) pad kernel contract (`include_str!`), imported by
/// [`register_cuda_pad_from_contract`] - the SOLE registration path (hand-written regs DELETED).
const CUDA_PAD_CONTRACT: &str = include_str!("../../docs/kernel-contracts/cuda/pad.fkc.md");

/// Register the CUDA pad family FROM its FKC contract (per-(op,dtype), fanned
/// `<op>_<dtype>` -> the distinct production wrappers via [`crate::fkc::CudaLinkRegistry`]).
/// Runs BEFORE `fill_unset_cost_for_backend(Cuda, ..)`; caps/precision ride through the import.
fn register_cuda_pad_from_contract(table: &mut KernelBindingTable) {
    let provider = crate::fkc::import_bundle_str(CUDA_PAD_CONTRACT, &crate::fkc::CudaLinkRegistry)
        .expect("authored CUDA pad contract must import (embedded include_str!, CudaLinkRegistry)");
    debug_assert!(
        provider.fused.is_empty(),
        "cuda pad contract declares no fused ops"
    );
    let mut fused = crate::fused::FusedKernelRegistry::new();
    provider
        .register_into(table, &mut fused)
        .expect("CUDA pad contract must register into the binding table");
}

/// The authored CUDA (baracuda) pad_backward kernel contract (`include_str!`), imported by
/// [`register_cuda_pad_backward_from_contract`] - the SOLE registration path (hand-written regs DELETED).
const CUDA_PAD_BACKWARD_CONTRACT: &str =
    include_str!("../../docs/kernel-contracts/cuda/pad_backward.fkc.md");

/// Register the CUDA pad_backward family FROM its FKC contract (per-(op,dtype), fanned
/// `<op>_<dtype>` -> the distinct production wrappers via [`crate::fkc::CudaLinkRegistry`]).
/// Runs BEFORE `fill_unset_cost_for_backend(Cuda, ..)`; caps/precision ride through the import.
fn register_cuda_pad_backward_from_contract(table: &mut KernelBindingTable) {
    let provider = crate::fkc::import_bundle_str(
        CUDA_PAD_BACKWARD_CONTRACT,
        &crate::fkc::CudaLinkRegistry,
    )
    .expect(
        "authored CUDA pad_backward contract must import (embedded include_str!, CudaLinkRegistry)",
    );
    debug_assert!(
        provider.fused.is_empty(),
        "cuda pad_backward contract declares no fused ops"
    );
    let mut fused = crate::fused::FusedKernelRegistry::new();
    provider
        .register_into(table, &mut fused)
        .expect("CUDA pad_backward contract must register into the binding table");
}

/// The authored CUDA (baracuda) causal_conv1d kernel contract (`include_str!`), imported by
/// [`register_cuda_causal_conv1d_from_contract`] - the SOLE registration path (hand-written regs DELETED).
const CUDA_CAUSAL_CONV1D_CONTRACT: &str =
    include_str!("../../docs/kernel-contracts/cuda/causal_conv1d.fkc.md");

/// Register the CUDA causal_conv1d family FROM its FKC contract (per-(op,dtype), fanned
/// `<op>_<dtype>` -> the distinct production wrappers via [`crate::fkc::CudaLinkRegistry`]).
/// Runs BEFORE `fill_unset_cost_for_backend(Cuda, ..)`; caps/precision ride through the import.
fn register_cuda_causal_conv1d_from_contract(table: &mut KernelBindingTable) {
    let provider = crate::fkc::import_bundle_str(CUDA_CAUSAL_CONV1D_CONTRACT, &crate::fkc::CudaLinkRegistry)
        .expect("authored CUDA causal_conv1d contract must import (embedded include_str!, CudaLinkRegistry)");
    debug_assert!(
        provider.fused.is_empty(),
        "cuda causal_conv1d contract declares no fused ops"
    );
    let mut fused = crate::fused::FusedKernelRegistry::new();
    provider
        .register_into(table, &mut fused)
        .expect("CUDA causal_conv1d contract must register into the binding table");
}

// ===========================================================================
// GROUP6 CUDA FKC families - contract registration.
// ===========================================================================

/// The authored CUDA (baracuda) gemm_dense kernel contract (`include_str!`), imported by
/// [`register_cuda_gemm_dense_from_contract`] - the SOLE registration path (hand-written regs DELETED).
const CUDA_GEMM_DENSE_CONTRACT: &str =
    include_str!("../../docs/kernel-contracts/cuda/gemm_dense.fkc.md");

/// Register the CUDA gemm_dense family FROM its FKC contract (per-(op,dtype), fanned
/// `<op>_<dtype>` -> the distinct production wrappers via [`crate::fkc::CudaLinkRegistry`]).
/// Runs BEFORE `fill_unset_cost_for_backend(Cuda, ..)`; caps/precision ride through the import.
fn register_cuda_gemm_dense_from_contract(table: &mut KernelBindingTable) {
    let provider = crate::fkc::import_bundle_str(
        CUDA_GEMM_DENSE_CONTRACT,
        &crate::fkc::CudaLinkRegistry,
    )
    .expect(
        "authored CUDA gemm_dense contract must import (embedded include_str!, CudaLinkRegistry)",
    );
    debug_assert!(
        provider.fused.is_empty(),
        "cuda gemm_dense contract declares no fused ops"
    );
    let mut fused = crate::fused::FusedKernelRegistry::new();
    provider
        .register_into(table, &mut fused)
        .expect("CUDA gemm_dense contract must register into the binding table");
}

/// The authored CUDA (baracuda) gemm_int kernel contract (`include_str!`), imported by
/// [`register_cuda_gemm_int_from_contract`] - the SOLE registration path (hand-written regs DELETED).
const CUDA_GEMM_INT_CONTRACT: &str =
    include_str!("../../docs/kernel-contracts/cuda/gemm_int.fkc.md");

/// Register the CUDA gemm_int family FROM its FKC contract (per-(op,dtype), fanned
/// `<op>_<dtype>` -> the distinct production wrappers via [`crate::fkc::CudaLinkRegistry`]).
/// Runs BEFORE `fill_unset_cost_for_backend(Cuda, ..)`; caps/precision ride through the import.
fn register_cuda_gemm_int_from_contract(table: &mut KernelBindingTable) {
    let provider = crate::fkc::import_bundle_str(
        CUDA_GEMM_INT_CONTRACT,
        &crate::fkc::CudaLinkRegistry,
    )
    .expect(
        "authored CUDA gemm_int contract must import (embedded include_str!, CudaLinkRegistry)",
    );
    debug_assert!(
        provider.fused.is_empty(),
        "cuda gemm_int contract declares no fused ops"
    );
    let mut fused = crate::fused::FusedKernelRegistry::new();
    provider
        .register_into(table, &mut fused)
        .expect("CUDA gemm_int contract must register into the binding table");
}

/// The authored CUDA (baracuda) indexing kernel contract (`include_str!`), imported by
/// [`register_cuda_indexing_from_contract`] - the SOLE registration path (hand-written regs DELETED).
const CUDA_INDEXING_CONTRACT: &str =
    include_str!("../../docs/kernel-contracts/cuda/indexing.fkc.md");

/// Register the CUDA indexing family (IndexSelect / Gather / MaskedFill over
/// `{F32, F64, I32}` = 9 keys + ScatterAdd over `{F32, F64}` = 2 keys = 11
/// bindings) FROM its FKC contract — the LAST clean CUDA family, closing the
/// CUDA FKC arc. IndexSelect / Gather / MaskedFill fan their DATA operand
/// (§3.4 base `entry_point` → `<op>_<dtype>`) with the `U32` index / `U8` mask
/// as a FIXED single-dtype slot (no independent fan axis, no
/// `FanoutDtypeMismatch`), each fanned symbol resolving to its OWN per-dtype
/// baracuda wrapper via [`crate::fkc::CudaLinkRegistry`]; ScatterAdd is
/// per-dtype (single-dtype `entry_point` resolved AS-IS). Keys `[T, U32, T]` /
/// `[T, U8, T]` / `[T, U32, T, T]` — byte-for-byte the deleted hand-written
/// `table.register(IndexSelect/Gather/MaskedFill/ScatterAdd, …)` regs. Runs
/// BEFORE `fill_unset_cost_for_backend(Cuda, ..)`; contiguous-only caps
/// (`requires_contiguous` ⇒ `strided_input == false`) + UNAUDITED precision seed
/// ride through the import.
fn register_cuda_indexing_from_contract(table: &mut KernelBindingTable) {
    let provider = crate::fkc::import_bundle_str(
        CUDA_INDEXING_CONTRACT,
        &crate::fkc::CudaLinkRegistry,
    )
    .expect(
        "authored CUDA indexing contract must import (embedded include_str!, CudaLinkRegistry)",
    );
    debug_assert!(
        provider.fused.is_empty(),
        "cuda indexing contract declares no fused ops"
    );
    let mut fused = crate::fused::FusedKernelRegistry::new();
    provider
        .register_into(table, &mut fused)
        .expect("CUDA indexing contract must register into the binding table");
}

/// Register every baracuda-backed CUDA kernel into the binding table
/// as an alternative at its `(OpKind, dtypes, Cuda)` decision point.
/// Sits alongside the PTX-backed `register_cuda_kernels` — both can
/// be called on the same table; step 9a's append-on-register makes
/// them coexist.
///
/// Coverage spans unary / binary / reduce / norm / softmax /
/// attention / indexing / GGUF / int-GEMM / cast / conv across
/// F32/F16/BF16/F64 (see the per-family sections below). The
/// 2026-06-11 sweep closed the Judge's cuda:0 elementwise gap:
/// Rsqrt/Floor/Ceil/Round/Erf unary + Pow/Rem binary.
pub fn register_baracuda_cuda_kernels(table: &mut KernelBindingTable) {
    // ----- Attention — FlashDecoding (decode-flash, seq_q==1, capacity-K):
    // the LAST CUDA family to migrate off its hand-written regs (CUDA now
    // 31/31 contract-sourced). Baracuda `flash_decoding_{f16,bf16}` (alpha.72),
    // NO-alibi decode arm keyed [q, k, v, out] = [T;4] over {F16, BF16}, is now
    // registered FROM `docs/kernel-contracts/cuda/flash_decoding.fkc.md` via
    // register_cuda_flash_decoding_from_contract — the SOLE path. The two
    // hand-written `table.register_full(FlashAttn, [f16;4]/[bf16;4], …,
    // cost_flash_decoding_cuda)` regs are DELETED.
    //
    // The CUSTOM static infeasibility gate (`cost_flash_decoding_cuda`:
    // INFEASIBLE cost for seq_q!=1 / head_dim outside [1,128], keeping the
    // ranker off unsupported shapes) is PRESERVED across the migration by the
    // §4.4 cost-fn trampoline (Task-F): the contract PINS `cost.cost_fn:
    // cost_flash_decoding_cuda`, resolved through CudaLinkRegistry's
    // CUDA_COST_FNS table, so the pinned cost fn SURVIVES the
    // fill_unset_cost_for_backend(Cuda, …) pass (which only replaces
    // unknown_cost) — where a plain FKC import would have let fill_unset
    // clobber the gate with the shape-blind shared FlashAttn cost fn.
    //
    // GATING BOUNDARY (see cost_flash_decoding_cuda + attention.rs): the cost
    // gate deprioritizes unsupported shapes at `compile_plan` placement (an
    // unpinned KV-resident node routes to CPU-decompose), but the executor's
    // binding lookup is key-only + fail-fast — a HARD-PINNED-CUDA prefill node
    // would still reach this kernel and Err. The guaranteed clean decline is
    // graph-construction gating (only ever CREATE a CUDA FlashAttn node for
    // decode shapes: the step-3 fusion pathfinder / lazy.rs decode lowering,
    // symbolic-extents-and-persistent-decode §0 step 2), which owns arm
    // selection — this binding is the implementation that arm resolves to.
    //
    // Runs BEFORE the fill_unset_cost_for_backend(Cuda, …) pass in
    // register_optional_backends (dispatch.rs), like every other from_contract
    // family below.
    register_cuda_flash_decoding_from_contract(table);

    // ----- IndexSelect / Gather / MaskedFill / ScatterAdd are now registered
    // FROM the cuda indexing FKC contract (docs/kernel-contracts/cuda/indexing.fkc.md)
    // via register_cuda_indexing_from_contract in the GROUP 5 block below — the
    // SOLE path. The hand-written table.register(IndexSelect/Gather/MaskedFill/
    // ScatterAdd, …) regs (keys [T,U32,T] / [T,U8,T] / [T,U32,T,T], plain
    // default caps) are DELETED. IndexSelect/Gather/MaskedFill fan their DATA
    // operand with the U32 index / U8 mask a FIXED single-dtype slot (no fan
    // axis); ScatterAdd is per-dtype (entry_point AS-IS).

    // ----- Cast — the FULL 70-pair family (8×8 over {F32,F64,F16,BF16,I32,
    // U32,I64,U8} + F8E4M3 ↔ {F32,F16,BF16}) — registered FROM its FKC kernel
    // contract, the SOLE path (the hand-written 8×8 double-loop + F8E4M3 legs
    // are DELETED). The FIRST CUDA-backend FKC consumer. Runs BEFORE the
    // `fill_unset_cost_for_backend(Cuda, …)` pass in `register_optional_backends`
    // (dispatch.rs) so its imported `unknown_cost` sentinels are upgraded to the
    // same OpKind cost fn every other CUDA primitive gets. ONE dtype-agnostic
    // wrapper (`cast::cast_baracuda_wrapper`, dispatching on the in/out dtype
    // pair over baracuda's 8×8 FFI) across all 70 (SRC, DST) keys — a
    // synthetic-base umbrella; contiguous-only (default caps ⇒ strided_input
    // == false). The full per-destination numeric detail (RNE/saturate,
    // int↔float, F8E4M3 decode/saturate) is contracted in
    // `docs/kernel-contracts/cuda/cast.fkc.md`. -----
    register_cuda_cast_from_contract(table);

    // ----- GROUP 1 CUDA FKC families (per-(op,dtype) strided): each
    // replaces its deleted hand-written register_with_caps(..., strided)
    // block; all run BEFORE fill_unset_cost_for_backend(Cuda, ..) so the
    // imported unknown_cost sentinels upgrade to the shared per-OpKind cost. -----
    register_cuda_binary_from_contract(table);
    register_cuda_compare_from_contract(table);
    register_cuda_reduce_from_contract(table);
    register_cuda_norm_from_contract(table);
    register_cuda_softmax_from_contract(table);
    register_cuda_powi_from_contract(table);
    register_cuda_powi_backward_from_contract(table);
    register_cuda_clamp_from_contract(table);
    register_cuda_flip_from_contract(table);
    register_cuda_roll_from_contract(table);
    register_cuda_cumsum_from_contract(table);
    register_cuda_triangular_from_contract(table);
    register_cuda_arg_reduce_from_contract(table);
    register_cuda_reduce_to_from_contract(table);
    register_cuda_rope_from_contract(table);

    // ----- GROUP 3 forward elementwise-unary (strided) - replaces the
    // deleted hand-written register_with_caps(..., strided) unary block. -----
    register_cuda_unary_from_contract(table);

    // ----- GROUP 2 contiguous in-place families - replace the deleted
    // hand-written table.register(..) in-place blocks (default caps). -----
    register_cuda_clamp_inplace_from_contract(table);
    register_cuda_powi_inplace_from_contract(table);
    register_cuda_inplace_unary_from_contract(table);

    // ----- GROUP 4 byte-width umbrellas (WriteSlice / WriteSliceRotating):
    // replace the deleted hand-written per-dtype table.register(..) blocks. -----
    register_cuda_write_slice_from_contract(table);
    register_cuda_write_slice_rotating_from_contract(table);
    register_cuda_write_slice_doff_from_contract(table);

    // ----- GROUP 5 multi-operand / param-ride families (concat/affine +
    // inplace_affine/pad/pad_backward/causal_conv1d + indexing); replace the
    // deleted hand-written blocks. Indexing (IndexSelect/Gather/MaskedFill
    // data-dtype fan + ScatterAdd per-dtype) models the U32 index / U8 mask as
    // a FIXED single-dtype slot (no independent fan axis), so it migrates
    // cleanly — the LAST clean CUDA family. -----
    register_cuda_concat_from_contract(table);
    register_cuda_affine_from_contract(table);
    register_cuda_inplace_affine_from_contract(table);
    register_cuda_pad_from_contract(table);
    register_cuda_pad_backward_from_contract(table);
    register_cuda_causal_conv1d_from_contract(table);
    register_cuda_indexing_from_contract(table);

    // ----- GROUP 6 MatMul facades (gemm_dense f32/f64/f16/bf16 + gemm_int
    // s8/u8) at the shared OpKind::MatMul; replace the deleted regs. -----
    register_cuda_gemm_dense_from_contract(table);
    register_cuda_gemm_int_from_contract(table);
}

// ===========================================================================
// FKC contract-migration tests (born-red gate for the CUDA cast family)
// ===========================================================================
//
// The whole file is `#![cfg(feature = "cuda")]`, so this module is compiled
// only under `--features cuda` — no device is touched (registration is pure
// binding-table population; `register_baracuda_cuda_kernels` never probes
// hardware, and no live GPU / cudnn is required for a registration-only test).

#[cfg(test)]
mod cast_contract_tests {
    use super::*;
    use crate::kernel::{KernelBindingTable, KernelRef};

    /// FIRST CUDA-BACKEND FKC CONSUMER (born-red gate). `register_baracuda_cuda_kernels`
    /// registers the whole `OpKind::Cast` family (70 `(SRC, DST)` pairs) FROM ITS
    /// KERNEL CONTRACT (`docs/kernel-contracts/cuda/cast.fkc.md`) via
    /// `register_cuda_cast_from_contract` — the sole registration path, now that the
    /// hand-written `table.register(OpKind::Cast, &[src, dst], …)` calls (the 8×8
    /// double-loop + the F8E4M3 legs) are DELETED.
    ///
    /// For each `(Cast, [SRC, DST], Cuda)` key this asserts:
    ///  - the binding resolves to the EXACT production wrapper fn-pointer
    ///    (behavior-preserving — the SAME single `cast_baracuda_wrapper` the
    ///    deleted regs used, every pair sharing it via the synthetic-base umbrella),
    ///  - `kernel_source == "baracuda"` — the contract's provenance tag (the deleted
    ///    hand-written path stamped `""`). THIS is the discriminator that makes the
    ///    test go red: while the hand-written regs are present (kernel_source `""`)
    ///    OR with them deleted + the import absent (the key missing entirely), the
    ///    assertion fails,
    ///  - caps stay contiguous-only (`strided_input == false`), byte-for-byte the
    ///    deleted plain `table.register` default caps.
    ///
    /// Precision is contract-sourced (`audited: false` ⇒ `PrecisionGuarantee::UNAUDITED`,
    /// the deleted regs' default) and NOT asserted here. The 70 pairs are
    /// reconstructed exactly as production built them: the 8×8 cross product over
    /// `{F32, F64, F16, BF16, I32, U32, I64, U8}` + `F8E4M3 → {F32, F16, BF16}` +
    /// `{F32, F16, BF16} → F8E4M3`.
    #[test]
    fn register_baracuda_binds_cast_family_from_contract() {
        let mut table = KernelBindingTable::new();
        register_baracuda_cuda_kernels(&mut table);
        let cuda = BackendId::Cuda;
        let expected: KernelRef = cast::cast_baracuda_wrapper;

        // Reconstruct production's 70 (src, dst) keys exactly (the deleted
        // hand-written double-loop + F8E4M3 legs).
        let base = [
            DType::F32,
            DType::F64,
            DType::F16,
            DType::BF16,
            DType::I32,
            DType::U32,
            DType::I64,
            DType::U8,
        ];
        let mut pairs: Vec<(DType, DType)> = Vec::new();
        for &src in &base {
            for &dst in &base {
                pairs.push((src, dst));
            }
        }
        for dst in [DType::F32, DType::F16, DType::BF16] {
            pairs.push((DType::F8E4M3, dst));
            pairs.push((dst, DType::F8E4M3));
        }

        let mut checked = 0usize;
        for (src, dst) in pairs {
            let alts = table.lookup_alternatives(OpKind::Cast, &[src, dst], cuda);
            let entry = alts
                .iter()
                .find(|e| e.kernel as usize == expected as usize)
                .unwrap_or_else(|| {
                    panic!(
                        "Cast [{src:?}, {dst:?}]/Cuda: the production wrapper must be bound \
                         FROM the cuda cast contract in register_baracuda_cuda_kernels — found {} \
                         alternative(s) with sources {:?}",
                        alts.len(),
                        alts.iter().map(|e| e.kernel_source).collect::<Vec<_>>(),
                    )
                });
            assert_eq!(
                entry.kernel_source, "baracuda",
                "Cast [{src:?}, {dst:?}]: cast family must be contract-sourced \
                 (kernel_source=\"baracuda\"); got {:?}",
                entry.kernel_source,
            );
            assert!(
                !entry.caps.strided_input,
                "Cast [{src:?}, {dst:?}]: caps preserved (contiguous-only, strided_input=false)",
            );
            checked += 1;
        }
        assert_eq!(checked, 70, "all 70 (SRC, DST) cast pairs checked");
    }

    /// GROUP 1 (born-red gate). Every trivial per-(op,dtype) STRIDED CUDA family
    /// (binary / reduce / norm / softmax / powi / powi_backward / clamp / flip /
    /// roll / cumsum / triangular / arg_reduce / reduce_to / rope) is registered
    /// FROM its FKC contract (`docs/kernel-contracts/cuda/<fam>.fkc.md`) via the
    /// `register_cuda_<fam>_from_contract` importers - the SOLE path (hand-written
    /// `register_with_caps(..., strided)` regs DELETED). For each `(op, dtypes,
    /// Cuda)` key this asserts the EXACT production wrapper is bound,
    /// `kernel_source == "baracuda"` (RED while the hand-written regs stamp `""`),
    /// and `caps.strided_input == true` (byte-for-byte the deleted strided caps).
    #[test]
    fn register_baracuda_binds_group1_families_from_contract() {
        let mut table = KernelBindingTable::new();
        register_baracuda_cuda_kernels(&mut table);
        let cuda = BackendId::Cuda;
        let cases: &[(OpKind, &[DType], KernelRef, bool)] = &[
            (
                OpKind::AddElementwise,
                &[DType::F32, DType::F32, DType::F32][..],
                binary::add_f32 as KernelRef,
                true,
            ),
            (
                OpKind::AddElementwise,
                &[DType::F16, DType::F16, DType::F16][..],
                binary::add_f16 as KernelRef,
                true,
            ),
            (
                OpKind::AddElementwise,
                &[DType::BF16, DType::BF16, DType::BF16][..],
                binary::add_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::AddElementwise,
                &[DType::F64, DType::F64, DType::F64][..],
                binary::add_f64 as KernelRef,
                true,
            ),
            (
                OpKind::SubElementwise,
                &[DType::F32, DType::F32, DType::F32][..],
                binary::sub_f32 as KernelRef,
                true,
            ),
            (
                OpKind::SubElementwise,
                &[DType::F16, DType::F16, DType::F16][..],
                binary::sub_f16 as KernelRef,
                true,
            ),
            (
                OpKind::SubElementwise,
                &[DType::BF16, DType::BF16, DType::BF16][..],
                binary::sub_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::SubElementwise,
                &[DType::F64, DType::F64, DType::F64][..],
                binary::sub_f64 as KernelRef,
                true,
            ),
            (
                OpKind::MulElementwise,
                &[DType::F32, DType::F32, DType::F32][..],
                binary::mul_f32 as KernelRef,
                true,
            ),
            (
                OpKind::MulElementwise,
                &[DType::F16, DType::F16, DType::F16][..],
                binary::mul_f16 as KernelRef,
                true,
            ),
            (
                OpKind::MulElementwise,
                &[DType::BF16, DType::BF16, DType::BF16][..],
                binary::mul_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::MulElementwise,
                &[DType::F64, DType::F64, DType::F64][..],
                binary::mul_f64 as KernelRef,
                true,
            ),
            (
                OpKind::DivElementwise,
                &[DType::F32, DType::F32, DType::F32][..],
                binary::div_f32 as KernelRef,
                true,
            ),
            (
                OpKind::DivElementwise,
                &[DType::F16, DType::F16, DType::F16][..],
                binary::div_f16 as KernelRef,
                true,
            ),
            (
                OpKind::DivElementwise,
                &[DType::BF16, DType::BF16, DType::BF16][..],
                binary::div_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::DivElementwise,
                &[DType::F64, DType::F64, DType::F64][..],
                binary::div_f64 as KernelRef,
                true,
            ),
            (
                OpKind::MaximumElementwise,
                &[DType::F32, DType::F32, DType::F32][..],
                binary::maximum_f32 as KernelRef,
                true,
            ),
            (
                OpKind::MaximumElementwise,
                &[DType::F16, DType::F16, DType::F16][..],
                binary::maximum_f16 as KernelRef,
                true,
            ),
            (
                OpKind::MaximumElementwise,
                &[DType::BF16, DType::BF16, DType::BF16][..],
                binary::maximum_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::MaximumElementwise,
                &[DType::F64, DType::F64, DType::F64][..],
                binary::maximum_f64 as KernelRef,
                true,
            ),
            (
                OpKind::MinimumElementwise,
                &[DType::F32, DType::F32, DType::F32][..],
                binary::minimum_f32 as KernelRef,
                true,
            ),
            (
                OpKind::MinimumElementwise,
                &[DType::F16, DType::F16, DType::F16][..],
                binary::minimum_f16 as KernelRef,
                true,
            ),
            (
                OpKind::MinimumElementwise,
                &[DType::BF16, DType::BF16, DType::BF16][..],
                binary::minimum_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::MinimumElementwise,
                &[DType::F64, DType::F64, DType::F64][..],
                binary::minimum_f64 as KernelRef,
                true,
            ),
            (
                OpKind::PowElementwise,
                &[DType::F32, DType::F32, DType::F32][..],
                binary::pow_f32 as KernelRef,
                true,
            ),
            (
                OpKind::PowElementwise,
                &[DType::F16, DType::F16, DType::F16][..],
                binary::pow_f16 as KernelRef,
                true,
            ),
            (
                OpKind::PowElementwise,
                &[DType::BF16, DType::BF16, DType::BF16][..],
                binary::pow_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::PowElementwise,
                &[DType::F64, DType::F64, DType::F64][..],
                binary::pow_f64 as KernelRef,
                true,
            ),
            (
                OpKind::RemElementwise,
                &[DType::F32, DType::F32, DType::F32][..],
                binary::rem_f32 as KernelRef,
                true,
            ),
            (
                OpKind::RemElementwise,
                &[DType::F16, DType::F16, DType::F16][..],
                binary::rem_f16 as KernelRef,
                true,
            ),
            (
                OpKind::RemElementwise,
                &[DType::BF16, DType::BF16, DType::BF16][..],
                binary::rem_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::RemElementwise,
                &[DType::F64, DType::F64, DType::F64][..],
                binary::rem_f64 as KernelRef,
                true,
            ),
            (
                OpKind::SumReduce,
                &[DType::F32, DType::F32][..],
                reduce::sum_f32 as KernelRef,
                true,
            ),
            (
                OpKind::SumReduce,
                &[DType::F16, DType::F16][..],
                reduce::sum_f16 as KernelRef,
                true,
            ),
            (
                OpKind::SumReduce,
                &[DType::BF16, DType::BF16][..],
                reduce::sum_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::SumReduce,
                &[DType::F64, DType::F64][..],
                reduce::sum_f64 as KernelRef,
                true,
            ),
            (
                OpKind::MaxReduce,
                &[DType::F32, DType::F32][..],
                reduce::max_f32 as KernelRef,
                true,
            ),
            (
                OpKind::MaxReduce,
                &[DType::F16, DType::F16][..],
                reduce::max_f16 as KernelRef,
                true,
            ),
            (
                OpKind::MaxReduce,
                &[DType::BF16, DType::BF16][..],
                reduce::max_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::MaxReduce,
                &[DType::F64, DType::F64][..],
                reduce::max_f64 as KernelRef,
                true,
            ),
            (
                OpKind::MinReduce,
                &[DType::F32, DType::F32][..],
                reduce::min_f32 as KernelRef,
                true,
            ),
            (
                OpKind::MinReduce,
                &[DType::F16, DType::F16][..],
                reduce::min_f16 as KernelRef,
                true,
            ),
            (
                OpKind::MinReduce,
                &[DType::BF16, DType::BF16][..],
                reduce::min_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::MinReduce,
                &[DType::F64, DType::F64][..],
                reduce::min_f64 as KernelRef,
                true,
            ),
            (
                OpKind::MeanReduce,
                &[DType::F32, DType::F32][..],
                reduce::mean_f32 as KernelRef,
                true,
            ),
            (
                OpKind::MeanReduce,
                &[DType::F16, DType::F16][..],
                reduce::mean_f16 as KernelRef,
                true,
            ),
            (
                OpKind::MeanReduce,
                &[DType::BF16, DType::BF16][..],
                reduce::mean_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::MeanReduce,
                &[DType::F64, DType::F64][..],
                reduce::mean_f64 as KernelRef,
                true,
            ),
            (
                OpKind::RmsNormLastDim,
                &[DType::F32, DType::F32][..],
                norm::rms_f32 as KernelRef,
                true,
            ),
            (
                OpKind::RmsNormLastDim,
                &[DType::F16, DType::F16][..],
                norm::rms_f16 as KernelRef,
                true,
            ),
            (
                OpKind::RmsNormLastDim,
                &[DType::BF16, DType::BF16][..],
                norm::rms_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::RmsNormLastDim,
                &[DType::F64, DType::F64][..],
                norm::rms_f64 as KernelRef,
                true,
            ),
            (
                OpKind::LayerNormLastDim,
                &[DType::F32, DType::F32][..],
                norm::layer_f32 as KernelRef,
                true,
            ),
            (
                OpKind::LayerNormLastDim,
                &[DType::F16, DType::F16][..],
                norm::layer_f16 as KernelRef,
                true,
            ),
            (
                OpKind::LayerNormLastDim,
                &[DType::BF16, DType::BF16][..],
                norm::layer_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::LayerNormLastDim,
                &[DType::F64, DType::F64][..],
                norm::layer_f64 as KernelRef,
                true,
            ),
            (
                OpKind::SoftmaxLastDim,
                &[DType::F32, DType::F32][..],
                softmax::softmax_f32 as KernelRef,
                true,
            ),
            (
                OpKind::SoftmaxLastDim,
                &[DType::F16, DType::F16][..],
                softmax::softmax_f16 as KernelRef,
                true,
            ),
            (
                OpKind::SoftmaxLastDim,
                &[DType::BF16, DType::BF16][..],
                softmax::softmax_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::SoftmaxLastDim,
                &[DType::F64, DType::F64][..],
                softmax::softmax_f64 as KernelRef,
                true,
            ),
            (
                OpKind::LogSoftmaxLastDim,
                &[DType::F32, DType::F32][..],
                softmax::log_softmax_f32 as KernelRef,
                true,
            ),
            (
                OpKind::LogSoftmaxLastDim,
                &[DType::F16, DType::F16][..],
                softmax::log_softmax_f16 as KernelRef,
                true,
            ),
            (
                OpKind::LogSoftmaxLastDim,
                &[DType::BF16, DType::BF16][..],
                softmax::log_softmax_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::LogSoftmaxLastDim,
                &[DType::F64, DType::F64][..],
                softmax::log_softmax_f64 as KernelRef,
                true,
            ),
            (
                OpKind::PowIElementwise,
                &[DType::F32, DType::F32][..],
                powi::powi_f32 as KernelRef,
                true,
            ),
            (
                OpKind::PowIElementwise,
                &[DType::F64, DType::F64][..],
                powi::powi_f64 as KernelRef,
                true,
            ),
            (
                OpKind::PowIElementwise,
                &[DType::F16, DType::F16][..],
                powi::powi_f16 as KernelRef,
                true,
            ),
            (
                OpKind::PowIElementwise,
                &[DType::BF16, DType::BF16][..],
                powi::powi_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::PowIElementwiseBackward,
                &[DType::F32, DType::F32, DType::F32][..],
                powi_backward::powi_backward_f32 as KernelRef,
                true,
            ),
            (
                OpKind::PowIElementwiseBackward,
                &[DType::F64, DType::F64, DType::F64][..],
                powi_backward::powi_backward_f64 as KernelRef,
                true,
            ),
            (
                OpKind::PowIElementwiseBackward,
                &[DType::F16, DType::F16, DType::F16][..],
                powi_backward::powi_backward_f16 as KernelRef,
                true,
            ),
            (
                OpKind::PowIElementwiseBackward,
                &[DType::BF16, DType::BF16, DType::BF16][..],
                powi_backward::powi_backward_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::ClampElementwise,
                &[DType::F32, DType::F32][..],
                clamp::clamp_f32 as KernelRef,
                true,
            ),
            (
                OpKind::ClampElementwise,
                &[DType::F64, DType::F64][..],
                clamp::clamp_f64 as KernelRef,
                true,
            ),
            (
                OpKind::ClampElementwise,
                &[DType::F16, DType::F16][..],
                clamp::clamp_f16 as KernelRef,
                true,
            ),
            (
                OpKind::ClampElementwise,
                &[DType::BF16, DType::BF16][..],
                clamp::clamp_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::Flip,
                &[DType::F32, DType::F32][..],
                flip::flip_f32 as KernelRef,
                true,
            ),
            (
                OpKind::Flip,
                &[DType::F16, DType::F16][..],
                flip::flip_f16 as KernelRef,
                true,
            ),
            (
                OpKind::Flip,
                &[DType::BF16, DType::BF16][..],
                flip::flip_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::Flip,
                &[DType::F64, DType::F64][..],
                flip::flip_f64 as KernelRef,
                true,
            ),
            (
                OpKind::Roll,
                &[DType::F32, DType::F32][..],
                roll::roll_f32 as KernelRef,
                true,
            ),
            (
                OpKind::Roll,
                &[DType::F16, DType::F16][..],
                roll::roll_f16 as KernelRef,
                true,
            ),
            (
                OpKind::Roll,
                &[DType::BF16, DType::BF16][..],
                roll::roll_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::Roll,
                &[DType::F64, DType::F64][..],
                roll::roll_f64 as KernelRef,
                true,
            ),
            (
                OpKind::CumSum,
                &[DType::F32, DType::F32][..],
                cumsum::cumsum_f32 as KernelRef,
                true,
            ),
            (
                OpKind::CumSum,
                &[DType::F16, DType::F16][..],
                cumsum::cumsum_f16 as KernelRef,
                true,
            ),
            (
                OpKind::CumSum,
                &[DType::BF16, DType::BF16][..],
                cumsum::cumsum_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::CumSum,
                &[DType::F64, DType::F64][..],
                cumsum::cumsum_f64 as KernelRef,
                true,
            ),
            (
                OpKind::Triu,
                &[DType::F32, DType::F32][..],
                triangular::triu_f32 as KernelRef,
                true,
            ),
            (
                OpKind::Triu,
                &[DType::F64, DType::F64][..],
                triangular::triu_f64 as KernelRef,
                true,
            ),
            (
                OpKind::Triu,
                &[DType::F16, DType::F16][..],
                triangular::triu_f16 as KernelRef,
                true,
            ),
            (
                OpKind::Triu,
                &[DType::BF16, DType::BF16][..],
                triangular::triu_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::Triu,
                &[DType::I32, DType::I32][..],
                triangular::triu_i32 as KernelRef,
                true,
            ),
            (
                OpKind::Triu,
                &[DType::I64, DType::I64][..],
                triangular::triu_i64 as KernelRef,
                true,
            ),
            (
                OpKind::Tril,
                &[DType::F32, DType::F32][..],
                triangular::tril_f32 as KernelRef,
                true,
            ),
            (
                OpKind::Tril,
                &[DType::F64, DType::F64][..],
                triangular::tril_f64 as KernelRef,
                true,
            ),
            (
                OpKind::Tril,
                &[DType::F16, DType::F16][..],
                triangular::tril_f16 as KernelRef,
                true,
            ),
            (
                OpKind::Tril,
                &[DType::BF16, DType::BF16][..],
                triangular::tril_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::Tril,
                &[DType::I32, DType::I32][..],
                triangular::tril_i32 as KernelRef,
                true,
            ),
            (
                OpKind::Tril,
                &[DType::I64, DType::I64][..],
                triangular::tril_i64 as KernelRef,
                true,
            ),
            (
                OpKind::ArgMaxDim,
                &[DType::F32, DType::U32][..],
                arg_reduce::argmax_dim_u32_f32 as KernelRef,
                true,
            ),
            (
                OpKind::ArgMaxDim,
                &[DType::F64, DType::U32][..],
                arg_reduce::argmax_dim_u32_f64 as KernelRef,
                true,
            ),
            (
                OpKind::ArgMaxDim,
                &[DType::F16, DType::U32][..],
                arg_reduce::argmax_dim_u32_f16 as KernelRef,
                true,
            ),
            (
                OpKind::ArgMaxDim,
                &[DType::BF16, DType::U32][..],
                arg_reduce::argmax_dim_u32_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::ArgMinDim,
                &[DType::F32, DType::U32][..],
                arg_reduce::argmin_dim_u32_f32 as KernelRef,
                true,
            ),
            (
                OpKind::ArgMinDim,
                &[DType::F64, DType::U32][..],
                arg_reduce::argmin_dim_u32_f64 as KernelRef,
                true,
            ),
            (
                OpKind::ArgMinDim,
                &[DType::F16, DType::U32][..],
                arg_reduce::argmin_dim_u32_f16 as KernelRef,
                true,
            ),
            (
                OpKind::ArgMinDim,
                &[DType::BF16, DType::U32][..],
                arg_reduce::argmin_dim_u32_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::ReduceSumTo,
                &[DType::F32, DType::F32][..],
                reduce_to::sum_to_f32 as KernelRef,
                true,
            ),
            (
                OpKind::ReduceSumTo,
                &[DType::F16, DType::F16][..],
                reduce_to::sum_to_f16 as KernelRef,
                true,
            ),
            (
                OpKind::ReduceSumTo,
                &[DType::BF16, DType::BF16][..],
                reduce_to::sum_to_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::ReduceSumTo,
                &[DType::F64, DType::F64][..],
                reduce_to::sum_to_f64 as KernelRef,
                true,
            ),
            (
                OpKind::ReduceMaxTo,
                &[DType::F32, DType::F32][..],
                reduce_to::max_to_f32 as KernelRef,
                true,
            ),
            (
                OpKind::ReduceMaxTo,
                &[DType::F16, DType::F16][..],
                reduce_to::max_to_f16 as KernelRef,
                true,
            ),
            (
                OpKind::ReduceMaxTo,
                &[DType::BF16, DType::BF16][..],
                reduce_to::max_to_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::ReduceMaxTo,
                &[DType::F64, DType::F64][..],
                reduce_to::max_to_f64 as KernelRef,
                true,
            ),
            (
                OpKind::Rope,
                &[DType::F32, DType::F32][..],
                attention::rope_f32 as KernelRef,
                true,
            ),
            (
                OpKind::Rope,
                &[DType::F16, DType::F16][..],
                attention::rope_f16 as KernelRef,
                true,
            ),
            (
                OpKind::Rope,
                &[DType::BF16, DType::BF16][..],
                attention::rope_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::Rope,
                &[DType::F64, DType::F64][..],
                attention::rope_f64 as KernelRef,
                true,
            ),
        ];
        let mut checked = 0usize;
        for (op, key, expected, want_strided) in cases {
            let alts = table.lookup_alternatives(*op, key, cuda);
            let entry = alts
                .iter()
                .find(|e| e.kernel as usize == *expected as usize)
                .unwrap_or_else(|| {
                    panic!(
                        "{op:?} {key:?}/Cuda: the production wrapper must be bound FROM the cuda \
                         contract in register_baracuda_cuda_kernels - found {} alternative(s) with \
                         sources {:?}",
                        alts.len(),
                        alts.iter().map(|e| e.kernel_source).collect::<Vec<_>>(),
                    )
                });
            assert_eq!(
                entry.kernel_source, "baracuda",
                "{op:?} {key:?}: family must be contract-sourced (kernel_source=\"baracuda\"); got {:?}",
                entry.kernel_source,
            );
            assert_eq!(
                entry.caps.strided_input, *want_strided,
                "{op:?} {key:?}: caps preserved (strided_input == {want_strided})",
            );
            checked += 1;
        }
        assert_eq!(checked, 120, "all 120 Group-1 (op, dtype) keys checked");
    }

    /// GROUP 3 (born-red gate). The forward elementwise-unary family (21 ops x 4
    /// dtypes + f32-only Sign = 85 keys) is registered FROM its FKC contract via
    /// `register_cuda_unary_from_contract` - the SOLE path (hand-written
    /// `register_with_caps(..., strided)` regs DELETED). The Gelu<->gelu_tanh name
    /// inversion + f32-only Sign are LINK-TABLE handled (base symbol == wrapper).
    /// Asserts the EXACT production wrapper is bound, `kernel_source == "baracuda"`
    /// (RED while hand-written stamps `""`), and `caps.strided_input == true`.
    #[test]
    fn register_baracuda_binds_group3_forward_unary_from_contract() {
        let mut table = KernelBindingTable::new();
        register_baracuda_cuda_kernels(&mut table);
        let cuda = BackendId::Cuda;
        let cases: &[(OpKind, &[DType], KernelRef, bool)] = &[
            (
                OpKind::NegElementwise,
                &[DType::F32, DType::F32][..],
                unary::neg_f32 as KernelRef,
                true,
            ),
            (
                OpKind::NegElementwise,
                &[DType::F16, DType::F16][..],
                unary::neg_f16 as KernelRef,
                true,
            ),
            (
                OpKind::NegElementwise,
                &[DType::BF16, DType::BF16][..],
                unary::neg_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::NegElementwise,
                &[DType::F64, DType::F64][..],
                unary::neg_f64 as KernelRef,
                true,
            ),
            (
                OpKind::AbsElementwise,
                &[DType::F32, DType::F32][..],
                unary::abs_f32 as KernelRef,
                true,
            ),
            (
                OpKind::AbsElementwise,
                &[DType::F16, DType::F16][..],
                unary::abs_f16 as KernelRef,
                true,
            ),
            (
                OpKind::AbsElementwise,
                &[DType::BF16, DType::BF16][..],
                unary::abs_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::AbsElementwise,
                &[DType::F64, DType::F64][..],
                unary::abs_f64 as KernelRef,
                true,
            ),
            (
                OpKind::SqrElementwise,
                &[DType::F32, DType::F32][..],
                unary::sqr_f32 as KernelRef,
                true,
            ),
            (
                OpKind::SqrElementwise,
                &[DType::F16, DType::F16][..],
                unary::sqr_f16 as KernelRef,
                true,
            ),
            (
                OpKind::SqrElementwise,
                &[DType::BF16, DType::BF16][..],
                unary::sqr_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::SqrElementwise,
                &[DType::F64, DType::F64][..],
                unary::sqr_f64 as KernelRef,
                true,
            ),
            (
                OpKind::SqrtElementwise,
                &[DType::F32, DType::F32][..],
                unary::sqrt_f32 as KernelRef,
                true,
            ),
            (
                OpKind::SqrtElementwise,
                &[DType::F16, DType::F16][..],
                unary::sqrt_f16 as KernelRef,
                true,
            ),
            (
                OpKind::SqrtElementwise,
                &[DType::BF16, DType::BF16][..],
                unary::sqrt_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::SqrtElementwise,
                &[DType::F64, DType::F64][..],
                unary::sqrt_f64 as KernelRef,
                true,
            ),
            (
                OpKind::RecipElementwise,
                &[DType::F32, DType::F32][..],
                unary::recip_f32 as KernelRef,
                true,
            ),
            (
                OpKind::RecipElementwise,
                &[DType::F16, DType::F16][..],
                unary::recip_f16 as KernelRef,
                true,
            ),
            (
                OpKind::RecipElementwise,
                &[DType::BF16, DType::BF16][..],
                unary::recip_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::RecipElementwise,
                &[DType::F64, DType::F64][..],
                unary::recip_f64 as KernelRef,
                true,
            ),
            (
                OpKind::ExpElementwise,
                &[DType::F32, DType::F32][..],
                unary::exp_f32 as KernelRef,
                true,
            ),
            (
                OpKind::ExpElementwise,
                &[DType::F16, DType::F16][..],
                unary::exp_f16 as KernelRef,
                true,
            ),
            (
                OpKind::ExpElementwise,
                &[DType::BF16, DType::BF16][..],
                unary::exp_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::ExpElementwise,
                &[DType::F64, DType::F64][..],
                unary::exp_f64 as KernelRef,
                true,
            ),
            (
                OpKind::LogElementwise,
                &[DType::F32, DType::F32][..],
                unary::log_f32 as KernelRef,
                true,
            ),
            (
                OpKind::LogElementwise,
                &[DType::F16, DType::F16][..],
                unary::log_f16 as KernelRef,
                true,
            ),
            (
                OpKind::LogElementwise,
                &[DType::BF16, DType::BF16][..],
                unary::log_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::LogElementwise,
                &[DType::F64, DType::F64][..],
                unary::log_f64 as KernelRef,
                true,
            ),
            (
                OpKind::SinElementwise,
                &[DType::F32, DType::F32][..],
                unary::sin_f32 as KernelRef,
                true,
            ),
            (
                OpKind::SinElementwise,
                &[DType::F16, DType::F16][..],
                unary::sin_f16 as KernelRef,
                true,
            ),
            (
                OpKind::SinElementwise,
                &[DType::BF16, DType::BF16][..],
                unary::sin_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::SinElementwise,
                &[DType::F64, DType::F64][..],
                unary::sin_f64 as KernelRef,
                true,
            ),
            (
                OpKind::CosElementwise,
                &[DType::F32, DType::F32][..],
                unary::cos_f32 as KernelRef,
                true,
            ),
            (
                OpKind::CosElementwise,
                &[DType::F16, DType::F16][..],
                unary::cos_f16 as KernelRef,
                true,
            ),
            (
                OpKind::CosElementwise,
                &[DType::BF16, DType::BF16][..],
                unary::cos_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::CosElementwise,
                &[DType::F64, DType::F64][..],
                unary::cos_f64 as KernelRef,
                true,
            ),
            (
                OpKind::TanhElementwise,
                &[DType::F32, DType::F32][..],
                unary::tanh_f32 as KernelRef,
                true,
            ),
            (
                OpKind::TanhElementwise,
                &[DType::F16, DType::F16][..],
                unary::tanh_f16 as KernelRef,
                true,
            ),
            (
                OpKind::TanhElementwise,
                &[DType::BF16, DType::BF16][..],
                unary::tanh_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::TanhElementwise,
                &[DType::F64, DType::F64][..],
                unary::tanh_f64 as KernelRef,
                true,
            ),
            (
                OpKind::ReluElementwise,
                &[DType::F32, DType::F32][..],
                unary::relu_f32 as KernelRef,
                true,
            ),
            (
                OpKind::ReluElementwise,
                &[DType::F16, DType::F16][..],
                unary::relu_f16 as KernelRef,
                true,
            ),
            (
                OpKind::ReluElementwise,
                &[DType::BF16, DType::BF16][..],
                unary::relu_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::ReluElementwise,
                &[DType::F64, DType::F64][..],
                unary::relu_f64 as KernelRef,
                true,
            ),
            (
                OpKind::GeluTanhElementwise,
                &[DType::F32, DType::F32][..],
                unary::gelu_tanh_f32 as KernelRef,
                true,
            ),
            (
                OpKind::GeluTanhElementwise,
                &[DType::F16, DType::F16][..],
                unary::gelu_tanh_f16 as KernelRef,
                true,
            ),
            (
                OpKind::GeluTanhElementwise,
                &[DType::BF16, DType::BF16][..],
                unary::gelu_tanh_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::GeluTanhElementwise,
                &[DType::F64, DType::F64][..],
                unary::gelu_tanh_f64 as KernelRef,
                true,
            ),
            (
                OpKind::GeluElementwise,
                &[DType::F32, DType::F32][..],
                unary::gelu_f32 as KernelRef,
                true,
            ),
            (
                OpKind::GeluElementwise,
                &[DType::F16, DType::F16][..],
                unary::gelu_f16 as KernelRef,
                true,
            ),
            (
                OpKind::GeluElementwise,
                &[DType::BF16, DType::BF16][..],
                unary::gelu_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::GeluElementwise,
                &[DType::F64, DType::F64][..],
                unary::gelu_f64 as KernelRef,
                true,
            ),
            (
                OpKind::StepElementwise,
                &[DType::F32, DType::F32][..],
                unary::step_f32 as KernelRef,
                true,
            ),
            (
                OpKind::StepElementwise,
                &[DType::F16, DType::F16][..],
                unary::step_f16 as KernelRef,
                true,
            ),
            (
                OpKind::StepElementwise,
                &[DType::BF16, DType::BF16][..],
                unary::step_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::StepElementwise,
                &[DType::F64, DType::F64][..],
                unary::step_f64 as KernelRef,
                true,
            ),
            (
                OpKind::SiluElementwise,
                &[DType::F32, DType::F32][..],
                unary::silu_f32 as KernelRef,
                true,
            ),
            (
                OpKind::SiluElementwise,
                &[DType::F16, DType::F16][..],
                unary::silu_f16 as KernelRef,
                true,
            ),
            (
                OpKind::SiluElementwise,
                &[DType::BF16, DType::BF16][..],
                unary::silu_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::SiluElementwise,
                &[DType::F64, DType::F64][..],
                unary::silu_f64 as KernelRef,
                true,
            ),
            (
                OpKind::SigmoidElementwise,
                &[DType::F32, DType::F32][..],
                unary::sigmoid_f32 as KernelRef,
                true,
            ),
            (
                OpKind::SigmoidElementwise,
                &[DType::F16, DType::F16][..],
                unary::sigmoid_f16 as KernelRef,
                true,
            ),
            (
                OpKind::SigmoidElementwise,
                &[DType::BF16, DType::BF16][..],
                unary::sigmoid_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::SigmoidElementwise,
                &[DType::F64, DType::F64][..],
                unary::sigmoid_f64 as KernelRef,
                true,
            ),
            (
                OpKind::RsqrtElementwise,
                &[DType::F32, DType::F32][..],
                unary::rsqrt_f32 as KernelRef,
                true,
            ),
            (
                OpKind::RsqrtElementwise,
                &[DType::F16, DType::F16][..],
                unary::rsqrt_f16 as KernelRef,
                true,
            ),
            (
                OpKind::RsqrtElementwise,
                &[DType::BF16, DType::BF16][..],
                unary::rsqrt_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::RsqrtElementwise,
                &[DType::F64, DType::F64][..],
                unary::rsqrt_f64 as KernelRef,
                true,
            ),
            (
                OpKind::FloorElementwise,
                &[DType::F32, DType::F32][..],
                unary::floor_f32 as KernelRef,
                true,
            ),
            (
                OpKind::FloorElementwise,
                &[DType::F16, DType::F16][..],
                unary::floor_f16 as KernelRef,
                true,
            ),
            (
                OpKind::FloorElementwise,
                &[DType::BF16, DType::BF16][..],
                unary::floor_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::FloorElementwise,
                &[DType::F64, DType::F64][..],
                unary::floor_f64 as KernelRef,
                true,
            ),
            (
                OpKind::CeilElementwise,
                &[DType::F32, DType::F32][..],
                unary::ceil_f32 as KernelRef,
                true,
            ),
            (
                OpKind::CeilElementwise,
                &[DType::F16, DType::F16][..],
                unary::ceil_f16 as KernelRef,
                true,
            ),
            (
                OpKind::CeilElementwise,
                &[DType::BF16, DType::BF16][..],
                unary::ceil_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::CeilElementwise,
                &[DType::F64, DType::F64][..],
                unary::ceil_f64 as KernelRef,
                true,
            ),
            (
                OpKind::RoundElementwise,
                &[DType::F32, DType::F32][..],
                unary::round_f32 as KernelRef,
                true,
            ),
            (
                OpKind::RoundElementwise,
                &[DType::F16, DType::F16][..],
                unary::round_f16 as KernelRef,
                true,
            ),
            (
                OpKind::RoundElementwise,
                &[DType::BF16, DType::BF16][..],
                unary::round_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::RoundElementwise,
                &[DType::F64, DType::F64][..],
                unary::round_f64 as KernelRef,
                true,
            ),
            (
                OpKind::ErfElementwise,
                &[DType::F32, DType::F32][..],
                unary::erf_f32 as KernelRef,
                true,
            ),
            (
                OpKind::ErfElementwise,
                &[DType::F16, DType::F16][..],
                unary::erf_f16 as KernelRef,
                true,
            ),
            (
                OpKind::ErfElementwise,
                &[DType::BF16, DType::BF16][..],
                unary::erf_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::ErfElementwise,
                &[DType::F64, DType::F64][..],
                unary::erf_f64 as KernelRef,
                true,
            ),
            (
                OpKind::SignElementwise,
                &[DType::F32, DType::F32][..],
                unary::sign_f32 as KernelRef,
                true,
            ),
        ];
        let mut checked = 0usize;
        for (op, key, expected, want_strided) in cases {
            let alts = table.lookup_alternatives(*op, key, cuda);
            let entry = alts
                .iter()
                .find(|e| e.kernel as usize == *expected as usize)
                .unwrap_or_else(|| {
                    panic!(
                        "{op:?} {key:?}/Cuda: the production wrapper must be bound FROM the cuda \
                         contract in register_baracuda_cuda_kernels - found {} alternative(s) with \
                         sources {:?}",
                        alts.len(),
                        alts.iter().map(|e| e.kernel_source).collect::<Vec<_>>(),
                    )
                });
            assert_eq!(
                entry.kernel_source, "baracuda",
                "{op:?} {key:?}: family must be contract-sourced (kernel_source=\"baracuda\"); got {:?}",
                entry.kernel_source,
            );
            assert_eq!(
                entry.caps.strided_input, *want_strided,
                "{op:?} {key:?}: caps preserved (strided_input == {want_strided})",
            );
            checked += 1;
        }
        assert_eq!(checked, 85, "all 85 group3 keys checked");
    }

    /// GROUP 2 (born-red gate). Every CONTIGUOUS in-place CUDA family
    /// (clamp_inplace / powi_inplace / the 5 in-place activations + the 16-op
    /// in-place unary expansion) is registered FROM its FKC contract via the
    /// `register_cuda_<fam>_from_contract` importers - the SOLE path (hand-written
    /// `table.register(..)` regs DELETED). Asserts the EXACT production wrapper is
    /// bound, `kernel_source == "baracuda"` (RED while hand-written stamps `""`),
    /// and `caps.strided_input == false` (default caps - contiguous in-place).
    #[test]
    fn register_baracuda_binds_group2_inplace_families_from_contract() {
        let mut table = KernelBindingTable::new();
        register_baracuda_cuda_kernels(&mut table);
        let cuda = BackendId::Cuda;
        let cases: &[(OpKind, &[DType], KernelRef, bool)] = &[
            (
                OpKind::ClampInplace,
                &[DType::F32, DType::F32][..],
                clamp_inplace::clamp_inplace_f32 as KernelRef,
                false,
            ),
            (
                OpKind::ClampInplace,
                &[DType::F64, DType::F64][..],
                clamp_inplace::clamp_inplace_f64 as KernelRef,
                false,
            ),
            (
                OpKind::ClampInplace,
                &[DType::BF16, DType::BF16][..],
                clamp_inplace::clamp_inplace_bf16 as KernelRef,
                false,
            ),
            (
                OpKind::ClampInplace,
                &[DType::F16, DType::F16][..],
                clamp_inplace::clamp_inplace_f16 as KernelRef,
                false,
            ),
            (
                OpKind::PowIInplace,
                &[DType::F32, DType::F32][..],
                powi_inplace::powi_inplace_f32 as KernelRef,
                false,
            ),
            (
                OpKind::PowIInplace,
                &[DType::F64, DType::F64][..],
                powi_inplace::powi_inplace_f64 as KernelRef,
                false,
            ),
            (
                OpKind::PowIInplace,
                &[DType::BF16, DType::BF16][..],
                powi_inplace::powi_inplace_bf16 as KernelRef,
                false,
            ),
            (
                OpKind::PowIInplace,
                &[DType::F16, DType::F16][..],
                powi_inplace::powi_inplace_f16 as KernelRef,
                false,
            ),
            (
                OpKind::ReluInplace,
                &[DType::F32, DType::F32][..],
                unary::relu_inplace_f32 as KernelRef,
                false,
            ),
            (
                OpKind::ReluInplace,
                &[DType::F64, DType::F64][..],
                unary::relu_inplace_f64 as KernelRef,
                false,
            ),
            (
                OpKind::ReluInplace,
                &[DType::BF16, DType::BF16][..],
                unary::relu_inplace_bf16 as KernelRef,
                false,
            ),
            (
                OpKind::ReluInplace,
                &[DType::F16, DType::F16][..],
                unary::relu_inplace_f16 as KernelRef,
                false,
            ),
            (
                OpKind::SiluInplace,
                &[DType::F32, DType::F32][..],
                unary::silu_inplace_f32 as KernelRef,
                false,
            ),
            (
                OpKind::SiluInplace,
                &[DType::F64, DType::F64][..],
                unary::silu_inplace_f64 as KernelRef,
                false,
            ),
            (
                OpKind::SiluInplace,
                &[DType::BF16, DType::BF16][..],
                unary::silu_inplace_bf16 as KernelRef,
                false,
            ),
            (
                OpKind::SiluInplace,
                &[DType::F16, DType::F16][..],
                unary::silu_inplace_f16 as KernelRef,
                false,
            ),
            (
                OpKind::GeluTanhInplace,
                &[DType::F32, DType::F32][..],
                unary::gelu_inplace_f32 as KernelRef,
                false,
            ),
            (
                OpKind::GeluTanhInplace,
                &[DType::F64, DType::F64][..],
                unary::gelu_inplace_f64 as KernelRef,
                false,
            ),
            (
                OpKind::GeluTanhInplace,
                &[DType::BF16, DType::BF16][..],
                unary::gelu_inplace_bf16 as KernelRef,
                false,
            ),
            (
                OpKind::GeluTanhInplace,
                &[DType::F16, DType::F16][..],
                unary::gelu_inplace_f16 as KernelRef,
                false,
            ),
            (
                OpKind::TanhInplace,
                &[DType::F32, DType::F32][..],
                unary::tanh_inplace_f32 as KernelRef,
                false,
            ),
            (
                OpKind::TanhInplace,
                &[DType::F64, DType::F64][..],
                unary::tanh_inplace_f64 as KernelRef,
                false,
            ),
            (
                OpKind::TanhInplace,
                &[DType::BF16, DType::BF16][..],
                unary::tanh_inplace_bf16 as KernelRef,
                false,
            ),
            (
                OpKind::TanhInplace,
                &[DType::F16, DType::F16][..],
                unary::tanh_inplace_f16 as KernelRef,
                false,
            ),
            (
                OpKind::SigmoidInplace,
                &[DType::F32, DType::F32][..],
                unary::sigmoid_inplace_f32 as KernelRef,
                false,
            ),
            (
                OpKind::SigmoidInplace,
                &[DType::F64, DType::F64][..],
                unary::sigmoid_inplace_f64 as KernelRef,
                false,
            ),
            (
                OpKind::SigmoidInplace,
                &[DType::BF16, DType::BF16][..],
                unary::sigmoid_inplace_bf16 as KernelRef,
                false,
            ),
            (
                OpKind::SigmoidInplace,
                &[DType::F16, DType::F16][..],
                unary::sigmoid_inplace_f16 as KernelRef,
                false,
            ),
            (
                OpKind::NegInplace,
                &[DType::F32, DType::F32][..],
                unary::neg_inplace_f32 as KernelRef,
                false,
            ),
            (
                OpKind::NegInplace,
                &[DType::F64, DType::F64][..],
                unary::neg_inplace_f64 as KernelRef,
                false,
            ),
            (
                OpKind::NegInplace,
                &[DType::BF16, DType::BF16][..],
                unary::neg_inplace_bf16 as KernelRef,
                false,
            ),
            (
                OpKind::NegInplace,
                &[DType::F16, DType::F16][..],
                unary::neg_inplace_f16 as KernelRef,
                false,
            ),
            (
                OpKind::AbsInplace,
                &[DType::F32, DType::F32][..],
                unary::abs_inplace_f32 as KernelRef,
                false,
            ),
            (
                OpKind::AbsInplace,
                &[DType::F64, DType::F64][..],
                unary::abs_inplace_f64 as KernelRef,
                false,
            ),
            (
                OpKind::AbsInplace,
                &[DType::BF16, DType::BF16][..],
                unary::abs_inplace_bf16 as KernelRef,
                false,
            ),
            (
                OpKind::AbsInplace,
                &[DType::F16, DType::F16][..],
                unary::abs_inplace_f16 as KernelRef,
                false,
            ),
            (
                OpKind::SqrInplace,
                &[DType::F32, DType::F32][..],
                unary::sqr_inplace_f32 as KernelRef,
                false,
            ),
            (
                OpKind::SqrInplace,
                &[DType::F64, DType::F64][..],
                unary::sqr_inplace_f64 as KernelRef,
                false,
            ),
            (
                OpKind::SqrInplace,
                &[DType::BF16, DType::BF16][..],
                unary::sqr_inplace_bf16 as KernelRef,
                false,
            ),
            (
                OpKind::SqrInplace,
                &[DType::F16, DType::F16][..],
                unary::sqr_inplace_f16 as KernelRef,
                false,
            ),
            (
                OpKind::SqrtInplace,
                &[DType::F32, DType::F32][..],
                unary::sqrt_inplace_f32 as KernelRef,
                false,
            ),
            (
                OpKind::SqrtInplace,
                &[DType::F64, DType::F64][..],
                unary::sqrt_inplace_f64 as KernelRef,
                false,
            ),
            (
                OpKind::SqrtInplace,
                &[DType::BF16, DType::BF16][..],
                unary::sqrt_inplace_bf16 as KernelRef,
                false,
            ),
            (
                OpKind::SqrtInplace,
                &[DType::F16, DType::F16][..],
                unary::sqrt_inplace_f16 as KernelRef,
                false,
            ),
            (
                OpKind::RsqrtInplace,
                &[DType::F32, DType::F32][..],
                unary::rsqrt_inplace_f32 as KernelRef,
                false,
            ),
            (
                OpKind::RsqrtInplace,
                &[DType::F64, DType::F64][..],
                unary::rsqrt_inplace_f64 as KernelRef,
                false,
            ),
            (
                OpKind::RsqrtInplace,
                &[DType::BF16, DType::BF16][..],
                unary::rsqrt_inplace_bf16 as KernelRef,
                false,
            ),
            (
                OpKind::RsqrtInplace,
                &[DType::F16, DType::F16][..],
                unary::rsqrt_inplace_f16 as KernelRef,
                false,
            ),
            (
                OpKind::RecipInplace,
                &[DType::F32, DType::F32][..],
                unary::recip_inplace_f32 as KernelRef,
                false,
            ),
            (
                OpKind::RecipInplace,
                &[DType::F64, DType::F64][..],
                unary::recip_inplace_f64 as KernelRef,
                false,
            ),
            (
                OpKind::RecipInplace,
                &[DType::BF16, DType::BF16][..],
                unary::recip_inplace_bf16 as KernelRef,
                false,
            ),
            (
                OpKind::RecipInplace,
                &[DType::F16, DType::F16][..],
                unary::recip_inplace_f16 as KernelRef,
                false,
            ),
            (
                OpKind::ExpInplace,
                &[DType::F32, DType::F32][..],
                unary::exp_inplace_f32 as KernelRef,
                false,
            ),
            (
                OpKind::ExpInplace,
                &[DType::F64, DType::F64][..],
                unary::exp_inplace_f64 as KernelRef,
                false,
            ),
            (
                OpKind::ExpInplace,
                &[DType::BF16, DType::BF16][..],
                unary::exp_inplace_bf16 as KernelRef,
                false,
            ),
            (
                OpKind::ExpInplace,
                &[DType::F16, DType::F16][..],
                unary::exp_inplace_f16 as KernelRef,
                false,
            ),
            (
                OpKind::LogInplace,
                &[DType::F32, DType::F32][..],
                unary::log_inplace_f32 as KernelRef,
                false,
            ),
            (
                OpKind::LogInplace,
                &[DType::F64, DType::F64][..],
                unary::log_inplace_f64 as KernelRef,
                false,
            ),
            (
                OpKind::LogInplace,
                &[DType::BF16, DType::BF16][..],
                unary::log_inplace_bf16 as KernelRef,
                false,
            ),
            (
                OpKind::LogInplace,
                &[DType::F16, DType::F16][..],
                unary::log_inplace_f16 as KernelRef,
                false,
            ),
            (
                OpKind::SinInplace,
                &[DType::F32, DType::F32][..],
                unary::sin_inplace_f32 as KernelRef,
                false,
            ),
            (
                OpKind::SinInplace,
                &[DType::F64, DType::F64][..],
                unary::sin_inplace_f64 as KernelRef,
                false,
            ),
            (
                OpKind::SinInplace,
                &[DType::BF16, DType::BF16][..],
                unary::sin_inplace_bf16 as KernelRef,
                false,
            ),
            (
                OpKind::SinInplace,
                &[DType::F16, DType::F16][..],
                unary::sin_inplace_f16 as KernelRef,
                false,
            ),
            (
                OpKind::CosInplace,
                &[DType::F32, DType::F32][..],
                unary::cos_inplace_f32 as KernelRef,
                false,
            ),
            (
                OpKind::CosInplace,
                &[DType::F64, DType::F64][..],
                unary::cos_inplace_f64 as KernelRef,
                false,
            ),
            (
                OpKind::CosInplace,
                &[DType::BF16, DType::BF16][..],
                unary::cos_inplace_bf16 as KernelRef,
                false,
            ),
            (
                OpKind::CosInplace,
                &[DType::F16, DType::F16][..],
                unary::cos_inplace_f16 as KernelRef,
                false,
            ),
            (
                OpKind::SignInplace,
                &[DType::F32, DType::F32][..],
                unary::sign_inplace_f32 as KernelRef,
                false,
            ),
            (
                OpKind::SignInplace,
                &[DType::F64, DType::F64][..],
                unary::sign_inplace_f64 as KernelRef,
                false,
            ),
            (
                OpKind::SignInplace,
                &[DType::BF16, DType::BF16][..],
                unary::sign_inplace_bf16 as KernelRef,
                false,
            ),
            (
                OpKind::SignInplace,
                &[DType::F16, DType::F16][..],
                unary::sign_inplace_f16 as KernelRef,
                false,
            ),
            (
                OpKind::FloorInplace,
                &[DType::F32, DType::F32][..],
                unary::floor_inplace_f32 as KernelRef,
                false,
            ),
            (
                OpKind::FloorInplace,
                &[DType::F64, DType::F64][..],
                unary::floor_inplace_f64 as KernelRef,
                false,
            ),
            (
                OpKind::FloorInplace,
                &[DType::BF16, DType::BF16][..],
                unary::floor_inplace_bf16 as KernelRef,
                false,
            ),
            (
                OpKind::FloorInplace,
                &[DType::F16, DType::F16][..],
                unary::floor_inplace_f16 as KernelRef,
                false,
            ),
            (
                OpKind::CeilInplace,
                &[DType::F32, DType::F32][..],
                unary::ceil_inplace_f32 as KernelRef,
                false,
            ),
            (
                OpKind::CeilInplace,
                &[DType::F64, DType::F64][..],
                unary::ceil_inplace_f64 as KernelRef,
                false,
            ),
            (
                OpKind::CeilInplace,
                &[DType::BF16, DType::BF16][..],
                unary::ceil_inplace_bf16 as KernelRef,
                false,
            ),
            (
                OpKind::CeilInplace,
                &[DType::F16, DType::F16][..],
                unary::ceil_inplace_f16 as KernelRef,
                false,
            ),
            (
                OpKind::RoundInplace,
                &[DType::F32, DType::F32][..],
                unary::round_inplace_f32 as KernelRef,
                false,
            ),
            (
                OpKind::RoundInplace,
                &[DType::F64, DType::F64][..],
                unary::round_inplace_f64 as KernelRef,
                false,
            ),
            (
                OpKind::RoundInplace,
                &[DType::BF16, DType::BF16][..],
                unary::round_inplace_bf16 as KernelRef,
                false,
            ),
            (
                OpKind::RoundInplace,
                &[DType::F16, DType::F16][..],
                unary::round_inplace_f16 as KernelRef,
                false,
            ),
            (
                OpKind::ErfInplace,
                &[DType::F32, DType::F32][..],
                unary::erf_inplace_f32 as KernelRef,
                false,
            ),
            (
                OpKind::ErfInplace,
                &[DType::F64, DType::F64][..],
                unary::erf_inplace_f64 as KernelRef,
                false,
            ),
            (
                OpKind::ErfInplace,
                &[DType::BF16, DType::BF16][..],
                unary::erf_inplace_bf16 as KernelRef,
                false,
            ),
            (
                OpKind::ErfInplace,
                &[DType::F16, DType::F16][..],
                unary::erf_inplace_f16 as KernelRef,
                false,
            ),
            (
                OpKind::GeluInplace,
                &[DType::F32, DType::F32][..],
                unary::gelu_erf_inplace_f32 as KernelRef,
                false,
            ),
            (
                OpKind::GeluInplace,
                &[DType::F64, DType::F64][..],
                unary::gelu_erf_inplace_f64 as KernelRef,
                false,
            ),
            (
                OpKind::GeluInplace,
                &[DType::BF16, DType::BF16][..],
                unary::gelu_erf_inplace_bf16 as KernelRef,
                false,
            ),
            (
                OpKind::GeluInplace,
                &[DType::F16, DType::F16][..],
                unary::gelu_erf_inplace_f16 as KernelRef,
                false,
            ),
        ];
        let mut checked = 0usize;
        for (op, key, expected, want_strided) in cases {
            let alts = table.lookup_alternatives(*op, key, cuda);
            let entry = alts
                .iter()
                .find(|e| e.kernel as usize == *expected as usize)
                .unwrap_or_else(|| {
                    panic!(
                        "{op:?} {key:?}/Cuda: the production wrapper must be bound FROM the cuda \
                         contract in register_baracuda_cuda_kernels - found {} alternative(s) with \
                         sources {:?}",
                        alts.len(),
                        alts.iter().map(|e| e.kernel_source).collect::<Vec<_>>(),
                    )
                });
            assert_eq!(
                entry.kernel_source, "baracuda",
                "{op:?} {key:?}: family must be contract-sourced (kernel_source=\"baracuda\"); got {:?}",
                entry.kernel_source,
            );
            assert_eq!(
                entry.caps.strided_input, *want_strided,
                "{op:?} {key:?}: caps preserved (strided_input == {want_strided})",
            );
            checked += 1;
        }
        assert_eq!(checked, 92, "all 92 group2 keys checked");
    }

    /// GROUP 4 (born-red gate). The byte-width-umbrella families (WriteSlice /
    /// WriteSliceRotating): 9 dtypes each fan to one of b1/b2/b4/b8 wrappers by
    /// element byte-width, registered FROM their FKC contracts via
    /// `register_cuda_write_slice{,_rotating}_from_contract` - the SOLE path
    /// (hand-written `table.register(..)` regs DELETED). Asserts the EXACT
    /// byte-width wrapper is bound at each dtype key, `kernel_source == "baracuda"`
    /// (RED while hand-written stamps `""`), and `caps.strided_input == false`.
    #[test]
    fn register_baracuda_binds_group4_byte_width_umbrellas_from_contract() {
        let mut table = KernelBindingTable::new();
        register_baracuda_cuda_kernels(&mut table);
        let cuda = BackendId::Cuda;
        let cases: &[(OpKind, &[DType], KernelRef, bool)] = &[
            (
                OpKind::WriteSlice,
                &[DType::F32, DType::F32][..],
                write_slice::write_slice_b4 as KernelRef,
                false,
            ),
            (
                OpKind::WriteSlice,
                &[DType::F64, DType::F64][..],
                write_slice::write_slice_b8 as KernelRef,
                false,
            ),
            (
                OpKind::WriteSlice,
                &[DType::F16, DType::F16][..],
                write_slice::write_slice_b2 as KernelRef,
                false,
            ),
            (
                OpKind::WriteSlice,
                &[DType::BF16, DType::BF16][..],
                write_slice::write_slice_b2 as KernelRef,
                false,
            ),
            (
                OpKind::WriteSlice,
                &[DType::I32, DType::I32][..],
                write_slice::write_slice_b4 as KernelRef,
                false,
            ),
            (
                OpKind::WriteSlice,
                &[DType::I64, DType::I64][..],
                write_slice::write_slice_b8 as KernelRef,
                false,
            ),
            (
                OpKind::WriteSlice,
                &[DType::U32, DType::U32][..],
                write_slice::write_slice_b4 as KernelRef,
                false,
            ),
            (
                OpKind::WriteSlice,
                &[DType::U8, DType::U8][..],
                write_slice::write_slice_b1 as KernelRef,
                false,
            ),
            (
                OpKind::WriteSlice,
                &[DType::I8, DType::I8][..],
                write_slice::write_slice_b1 as KernelRef,
                false,
            ),
            (
                OpKind::WriteSliceRotating,
                &[DType::F32, DType::F32][..],
                write_slice_rotating::write_slice_rotating_b4 as KernelRef,
                false,
            ),
            (
                OpKind::WriteSliceRotating,
                &[DType::F64, DType::F64][..],
                write_slice_rotating::write_slice_rotating_b8 as KernelRef,
                false,
            ),
            (
                OpKind::WriteSliceRotating,
                &[DType::F16, DType::F16][..],
                write_slice_rotating::write_slice_rotating_b2 as KernelRef,
                false,
            ),
            (
                OpKind::WriteSliceRotating,
                &[DType::BF16, DType::BF16][..],
                write_slice_rotating::write_slice_rotating_b2 as KernelRef,
                false,
            ),
            (
                OpKind::WriteSliceRotating,
                &[DType::I32, DType::I32][..],
                write_slice_rotating::write_slice_rotating_b4 as KernelRef,
                false,
            ),
            (
                OpKind::WriteSliceRotating,
                &[DType::I64, DType::I64][..],
                write_slice_rotating::write_slice_rotating_b8 as KernelRef,
                false,
            ),
            (
                OpKind::WriteSliceRotating,
                &[DType::U32, DType::U32][..],
                write_slice_rotating::write_slice_rotating_b4 as KernelRef,
                false,
            ),
            (
                OpKind::WriteSliceRotating,
                &[DType::U8, DType::U8][..],
                write_slice_rotating::write_slice_rotating_b1 as KernelRef,
                false,
            ),
            (
                OpKind::WriteSliceRotating,
                &[DType::I8, DType::I8][..],
                write_slice_rotating::write_slice_rotating_b1 as KernelRef,
                false,
            ),
            (
                OpKind::WriteSliceDoff,
                &[DType::F32, DType::F32][..],
                write_slice_doff::write_slice_doff_b4 as KernelRef,
                false,
            ),
            (
                OpKind::WriteSliceDoff,
                &[DType::F64, DType::F64][..],
                write_slice_doff::write_slice_doff_b8 as KernelRef,
                false,
            ),
            (
                OpKind::WriteSliceDoff,
                &[DType::F16, DType::F16][..],
                write_slice_doff::write_slice_doff_b2 as KernelRef,
                false,
            ),
            (
                OpKind::WriteSliceDoff,
                &[DType::BF16, DType::BF16][..],
                write_slice_doff::write_slice_doff_b2 as KernelRef,
                false,
            ),
            (
                OpKind::WriteSliceDoff,
                &[DType::I32, DType::I32][..],
                write_slice_doff::write_slice_doff_b4 as KernelRef,
                false,
            ),
            (
                OpKind::WriteSliceDoff,
                &[DType::I64, DType::I64][..],
                write_slice_doff::write_slice_doff_b8 as KernelRef,
                false,
            ),
            (
                OpKind::WriteSliceDoff,
                &[DType::U32, DType::U32][..],
                write_slice_doff::write_slice_doff_b4 as KernelRef,
                false,
            ),
            (
                OpKind::WriteSliceDoff,
                &[DType::U8, DType::U8][..],
                write_slice_doff::write_slice_doff_b1 as KernelRef,
                false,
            ),
            (
                OpKind::WriteSliceDoff,
                &[DType::I8, DType::I8][..],
                write_slice_doff::write_slice_doff_b1 as KernelRef,
                false,
            ),
        ];
        let mut checked = 0usize;
        for (op, key, expected, want_strided) in cases {
            let alts = table.lookup_alternatives(*op, key, cuda);
            let entry = alts
                .iter()
                .find(|e| e.kernel as usize == *expected as usize)
                .unwrap_or_else(|| {
                    panic!(
                        "{op:?} {key:?}/Cuda: the production wrapper must be bound FROM the cuda \
                         contract in register_baracuda_cuda_kernels - found {} alternative(s) with \
                         sources {:?}",
                        alts.len(),
                        alts.iter().map(|e| e.kernel_source).collect::<Vec<_>>(),
                    )
                });
            assert_eq!(
                entry.kernel_source, "baracuda",
                "{op:?} {key:?}: family must be contract-sourced (kernel_source=\"baracuda\"); got {:?}",
                entry.kernel_source,
            );
            assert_eq!(
                entry.caps.strided_input, *want_strided,
                "{op:?} {key:?}: caps preserved (strided_input == {want_strided})",
            );
            checked += 1;
        }
        assert_eq!(checked, 27, "all 27 group4 keys checked");
    }

    /// GROUP 5 (born-red gate). concat + affine (strided) and inplace_affine /
    /// pad / pad_backward / causal_conv1d (contiguous) are registered FROM their
    /// FKC contracts via `register_cuda_<fam>_from_contract` - the SOLE path
    /// (hand-written regs DELETED). Mixed caps: each case carries its own
    /// `want_strided`. Asserts the EXACT production wrapper is bound,
    /// `kernel_source == "baracuda"` (RED while hand-written stamps `""`), and
    /// caps. (indexing is DEFERRED - mixed fan/fixed-index operand.)
    #[test]
    fn register_baracuda_binds_group5_multiop_families_from_contract() {
        let mut table = KernelBindingTable::new();
        register_baracuda_cuda_kernels(&mut table);
        let cuda = BackendId::Cuda;
        let cases: &[(OpKind, &[DType], KernelRef, bool)] = &[
            (
                OpKind::Concat,
                &[DType::F32, DType::F32][..],
                concat::concat_f32 as KernelRef,
                true,
            ),
            (
                OpKind::Concat,
                &[DType::F16, DType::F16][..],
                concat::concat_f16 as KernelRef,
                true,
            ),
            (
                OpKind::Concat,
                &[DType::BF16, DType::BF16][..],
                concat::concat_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::Concat,
                &[DType::F64, DType::F64][..],
                concat::concat_f64 as KernelRef,
                true,
            ),
            (
                OpKind::Affine,
                &[DType::F32, DType::F32][..],
                affine::affine_f32 as KernelRef,
                true,
            ),
            (
                OpKind::Affine,
                &[DType::F64, DType::F64][..],
                affine::affine_f64 as KernelRef,
                true,
            ),
            (
                OpKind::Affine,
                &[DType::F16, DType::F16][..],
                affine::affine_f16 as KernelRef,
                true,
            ),
            (
                OpKind::Affine,
                &[DType::BF16, DType::BF16][..],
                affine::affine_bf16 as KernelRef,
                true,
            ),
            (
                OpKind::Affine,
                &[DType::I32, DType::I32][..],
                affine::affine_i32 as KernelRef,
                true,
            ),
            (
                OpKind::Affine,
                &[DType::I64, DType::I64][..],
                affine::affine_i64 as KernelRef,
                true,
            ),
            (
                OpKind::Affine,
                &[DType::U8, DType::U8][..],
                affine::affine_u8 as KernelRef,
                true,
            ),
            (
                OpKind::InplaceAffine,
                &[DType::F32, DType::F32][..],
                affine::affine_inplace_f32 as KernelRef,
                false,
            ),
            (
                OpKind::InplaceAffine,
                &[DType::F64, DType::F64][..],
                affine::affine_inplace_f64 as KernelRef,
                false,
            ),
            (
                OpKind::InplaceAffine,
                &[DType::BF16, DType::BF16][..],
                affine::affine_inplace_bf16 as KernelRef,
                false,
            ),
            (
                OpKind::InplaceAffine,
                &[DType::F16, DType::F16][..],
                affine::affine_inplace_f16 as KernelRef,
                false,
            ),
            (
                OpKind::Pad,
                &[DType::F32, DType::F32][..],
                pad::pad_f32 as KernelRef,
                false,
            ),
            (
                OpKind::Pad,
                &[DType::F16, DType::F16][..],
                pad::pad_f16 as KernelRef,
                false,
            ),
            (
                OpKind::Pad,
                &[DType::BF16, DType::BF16][..],
                pad::pad_bf16 as KernelRef,
                false,
            ),
            (
                OpKind::Pad,
                &[DType::F64, DType::F64][..],
                pad::pad_f64 as KernelRef,
                false,
            ),
            (
                OpKind::PadBackward,
                &[DType::F32, DType::F32][..],
                pad::pad_backward_f32 as KernelRef,
                false,
            ),
            (
                OpKind::PadBackward,
                &[DType::F16, DType::F16][..],
                pad::pad_backward_f16 as KernelRef,
                false,
            ),
            (
                OpKind::PadBackward,
                &[DType::BF16, DType::BF16][..],
                pad::pad_backward_bf16 as KernelRef,
                false,
            ),
            (
                OpKind::PadBackward,
                &[DType::F64, DType::F64][..],
                pad::pad_backward_f64 as KernelRef,
                false,
            ),
            (
                OpKind::CausalConv1d,
                &[DType::F32, DType::F32, DType::F32, DType::F32][..],
                conv1d::causal_conv1d_f32 as KernelRef,
                false,
            ),
            (
                OpKind::CausalConv1d,
                &[DType::F64, DType::F64, DType::F64, DType::F64][..],
                conv1d::causal_conv1d_f64 as KernelRef,
                false,
            ),
            (
                OpKind::CausalConv1d,
                &[DType::BF16, DType::BF16, DType::BF16, DType::BF16][..],
                conv1d::causal_conv1d_bf16 as KernelRef,
                false,
            ),
            (
                OpKind::CausalConv1d,
                &[DType::F16, DType::F16, DType::F16, DType::F16][..],
                conv1d::causal_conv1d_f16 as KernelRef,
                false,
            ),
        ];
        let mut checked = 0usize;
        for (op, key, expected, want_strided) in cases {
            let alts = table.lookup_alternatives(*op, key, cuda);
            let entry = alts
                .iter()
                .find(|e| e.kernel as usize == *expected as usize)
                .unwrap_or_else(|| {
                    panic!(
                        "{op:?} {key:?}/Cuda: the production wrapper must be bound FROM the cuda \
                         contract in register_baracuda_cuda_kernels - found {} alternative(s) with \
                         sources {:?}",
                        alts.len(),
                        alts.iter().map(|e| e.kernel_source).collect::<Vec<_>>(),
                    )
                });
            assert_eq!(
                entry.kernel_source, "baracuda",
                "{op:?} {key:?}: family must be contract-sourced (kernel_source=\"baracuda\"); got {:?}",
                entry.kernel_source,
            );
            assert_eq!(
                entry.caps.strided_input, *want_strided,
                "{op:?} {key:?}: caps preserved (strided_input == {want_strided})",
            );
            checked += 1;
        }
        assert_eq!(checked, 27, "all 27 group5 keys checked");
    }

    /// GROUP 6 (born-red gate). The MatMul facades gemm_dense (f32/f64/f16/bf16)
    /// + gemm_int (i8->gemm_s8_rrr, u8->gemm_u8_rrr) - 6 combos at the SHARED
    /// OpKind::MatMul - are registered FROM their FKC contracts via
    /// `register_cuda_gemm_{dense,int}_from_contract` - the SOLE path (hand-written
    /// regs DELETED). Asserts the EXACT production wrapper is bound at each dtype
    /// key, `kernel_source == "baracuda"` (RED while hand-written stamps `""`),
    /// and `caps.strided_input == false` (packed row-major, default caps).
    #[test]
    fn register_baracuda_binds_group6_matmul_facades_from_contract() {
        let mut table = KernelBindingTable::new();
        register_baracuda_cuda_kernels(&mut table);
        let cuda = BackendId::Cuda;
        let cases: &[(OpKind, &[DType], KernelRef, bool)] = &[
            (
                OpKind::MatMul,
                &[DType::F32, DType::F32, DType::F32][..],
                gemm_dense::matmul_f32 as KernelRef,
                false,
            ),
            (
                OpKind::MatMul,
                &[DType::F16, DType::F16, DType::F16][..],
                gemm_dense::matmul_f16 as KernelRef,
                false,
            ),
            (
                OpKind::MatMul,
                &[DType::BF16, DType::BF16, DType::BF16][..],
                gemm_dense::matmul_bf16 as KernelRef,
                false,
            ),
            (
                OpKind::MatMul,
                &[DType::F64, DType::F64, DType::F64][..],
                gemm_dense::matmul_f64 as KernelRef,
                false,
            ),
            (
                OpKind::MatMul,
                &[DType::I8, DType::I8, DType::I8][..],
                gemm_int::gemm_s8_rrr as KernelRef,
                false,
            ),
            (
                OpKind::MatMul,
                &[DType::U8, DType::U8, DType::U8][..],
                gemm_int::gemm_u8_rrr as KernelRef,
                false,
            ),
        ];
        let mut checked = 0usize;
        for (op, key, expected, want_strided) in cases {
            let alts = table.lookup_alternatives(*op, key, cuda);
            let entry = alts
                .iter()
                .find(|e| e.kernel as usize == *expected as usize)
                .unwrap_or_else(|| {
                    panic!(
                        "{op:?} {key:?}/Cuda: the production wrapper must be bound FROM the cuda \
                         contract in register_baracuda_cuda_kernels - found {} alternative(s) with \
                         sources {:?}",
                        alts.len(),
                        alts.iter().map(|e| e.kernel_source).collect::<Vec<_>>(),
                    )
                });
            assert_eq!(
                entry.kernel_source, "baracuda",
                "{op:?} {key:?}: family must be contract-sourced (kernel_source=\"baracuda\"); got {:?}",
                entry.kernel_source,
            );
            assert_eq!(
                entry.caps.strided_input, *want_strided,
                "{op:?} {key:?}: caps preserved (strided_input == {want_strided})",
            );
            checked += 1;
        }
        assert_eq!(checked, 6, "all 6 group6 keys checked");
    }

    /// INDEXING (born-red gate). The LAST clean CUDA family — IndexSelect /
    /// Gather / MaskedFill over `{F32, F64, I32}` (9 keys) + ScatterAdd over
    /// `{F32, F64}` (2 keys) = 11 bindings — is registered FROM its FKC contract
    /// (`docs/kernel-contracts/cuda/indexing.fkc.md`) via
    /// `register_cuda_indexing_from_contract` — the SOLE path, now that the
    /// hand-written `table.register(IndexSelect/Gather/MaskedFill/ScatterAdd, …)`
    /// regs are DELETED.
    ///
    /// IndexSelect / Gather / MaskedFill fan their DATA operand (§3.4 base
    /// `entry_point` → `<op>_<dtype>`) with the `U32` index / `Bool` mask a FIXED
    /// single-dtype slot (no fan axis), each fanned symbol resolving to its OWN
    /// per-dtype baracuda wrapper; ScatterAdd is per-dtype (entry_point AS-IS).
    /// Keys `[T, U32, T]` / `[T, Bool, T]` / `[T, U32, T, T]`. (The MaskedFill mask
    /// was `U8` before GAP-168(c); this test kept the `U8` key after the contract
    /// moved to `[BOOL]`, so it asserted a binding that no longer exists. Measured:
    /// it failed with "found 0 alternative(s)" for `[F32, U8, F32]`.) Asserts the EXACT
    /// production wrapper is bound, `kernel_source == "baracuda"` (RED while the
    /// deleted hand-written regs stamp `""`), and `caps.strided_input == false`
    /// (contiguous-only, plain-register default caps).
    #[test]
    fn register_baracuda_binds_indexing_from_contract() {
        let mut table = KernelBindingTable::new();
        register_baracuda_cuda_kernels(&mut table);
        let cuda = BackendId::Cuda;
        let u32 = DType::U32;
        // GAP-168(c): the MaskedFill mask is Bool (the contract block says `[BOOL]`).
        let mask = DType::Bool;
        let cases: &[(OpKind, &[DType], KernelRef)] = &[
            (
                OpKind::IndexSelect,
                &[DType::F32, u32, DType::F32][..],
                indexing::index_select_f32 as KernelRef,
            ),
            (
                OpKind::IndexSelect,
                &[DType::F64, u32, DType::F64][..],
                indexing::index_select_f64 as KernelRef,
            ),
            (
                OpKind::IndexSelect,
                &[DType::I32, u32, DType::I32][..],
                indexing::index_select_i32 as KernelRef,
            ),
            (
                OpKind::Gather,
                &[DType::F32, u32, DType::F32][..],
                indexing::gather_f32 as KernelRef,
            ),
            (
                OpKind::Gather,
                &[DType::F64, u32, DType::F64][..],
                indexing::gather_f64 as KernelRef,
            ),
            (
                OpKind::Gather,
                &[DType::I32, u32, DType::I32][..],
                indexing::gather_i32 as KernelRef,
            ),
            (
                OpKind::MaskedFill,
                &[DType::F32, mask, DType::F32][..],
                indexing::masked_fill_f32 as KernelRef,
            ),
            (
                OpKind::MaskedFill,
                &[DType::F64, mask, DType::F64][..],
                indexing::masked_fill_f64 as KernelRef,
            ),
            (
                OpKind::MaskedFill,
                &[DType::I32, mask, DType::I32][..],
                indexing::masked_fill_i32 as KernelRef,
            ),
            (
                OpKind::ScatterAdd,
                &[DType::F32, u32, DType::F32, DType::F32][..],
                indexing::scatter_add_f32 as KernelRef,
            ),
            (
                OpKind::ScatterAdd,
                &[DType::F64, u32, DType::F64, DType::F64][..],
                indexing::scatter_add_f64 as KernelRef,
            ),
        ];
        let mut checked = 0usize;
        for (op, key, expected) in cases {
            let alts = table.lookup_alternatives(*op, key, cuda);
            let entry = alts
                .iter()
                .find(|e| e.kernel as usize == *expected as usize)
                .unwrap_or_else(|| {
                    panic!(
                        "{op:?} {key:?}/Cuda: the production wrapper must be bound FROM the cuda \
                         indexing contract in register_baracuda_cuda_kernels - found {} alternative(s) \
                         with sources {:?}",
                        alts.len(),
                        alts.iter().map(|e| e.kernel_source).collect::<Vec<_>>(),
                    )
                });
            assert_eq!(
                entry.kernel_source, "baracuda",
                "{op:?} {key:?}: family must be contract-sourced (kernel_source=\"baracuda\"); got {:?}",
                entry.kernel_source,
            );
            assert_eq!(
                entry.caps.strided_input, false,
                "{op:?} {key:?}: caps preserved (strided_input == false)",
            );
            checked += 1;
        }
        assert_eq!(checked, 11, "all 11 indexing keys checked");
    }

    /// TASK-F born-red gate (the §4.4 cost-fn trampoline consumer). The
    /// FlashDecoding decode arm (`OpKind::FlashAttn`, `[f16;4]` / `[bf16;4]`,
    /// Cuda) is now registered FROM its FKC contract
    /// (`docs/kernel-contracts/cuda/flash_decoding.fkc.md`) via
    /// `register_cuda_flash_decoding_from_contract` — the SOLE path (the two
    /// hand-written `register_full(FlashAttn, …)` regs DELETED). CUDA is now
    /// 31/31 contract-sourced.
    ///
    /// For each of the two keys this asserts:
    ///  - the EXACT production wrapper (`flash_decoding_{f16,bf16}`) is bound,
    ///  - `kernel_source == "baracuda"` — RED while the hand-written regs (which
    ///    stamp `""`) are present; GREEN once contract-sourced,
    ///  - `caps.strided_input == true` (byte-for-byte `KernelCaps::strided_input()`),
    ///  - **THE INFEASIBILITY GATE IS PRESERVED**: the binding's `CostFn` is the
    ///    contract-pinned `cost_flash_decoding_cuda` (pointer identity) AND it
    ///    returns an INFEASIBLE cost (`flops == u64::MAX`) for an unsupported
    ///    shape (`seq_q != 1` or `head_dim > 128`), while a supported decode
    ///    shape is feasible. This proves the §4.4 cost-fn trampoline kept the
    ///    custom gate — a plain FKC import would have stamped `unknown_cost` and
    ///    `fill_unset_cost_for_backend(Cuda, …)` would have clobbered it with the
    ///    shape-blind `cost_flash_attn_primitive_cpu` (gate LOST → this assertion
    ///    RED).
    #[test]
    fn register_baracuda_binds_flash_decoding_from_contract_with_cost_gate() {
        let mut table = KernelBindingTable::new();
        register_baracuda_cuda_kernels(&mut table);
        let cuda = BackendId::Cuda;

        // Minimal caps — `cost_flash_decoding_cuda` ignores the caps arg.
        let caps = fuel_ir::backend::BackendCapabilities {
            backend_id: BackendId::Cuda,
            device_location: fuel_ir::DeviceLocation::Cpu,
            op_dtype_support: std::collections::HashSet::new(),
            required_alignment: 1,
            access_granularity_bits: 8,
            transfer_paths: vec![],
            storage_substrate: fuel_ir::backend::SubstrateClass::HostBytes,
            compute_throughput_flops_per_ns: 1.0,
            mem_bandwidth_bytes_per_ns: 4.0,
        };

        let cases: &[(&[DType], KernelRef)] = &[
            (
                &[DType::F16, DType::F16, DType::F16, DType::F16][..],
                flash_decoding::flash_decoding_f16 as KernelRef,
            ),
            (
                &[DType::BF16, DType::BF16, DType::BF16, DType::BF16][..],
                flash_decoding::flash_decoding_bf16 as KernelRef,
            ),
        ];

        for (key, expected) in cases {
            // (1) EXACT wrapper bound + contract-sourced.
            let alts = table.lookup_alternatives(OpKind::FlashAttn, key, cuda);
            let entry = alts
                .iter()
                .find(|e| e.kernel as usize == *expected as usize)
                .unwrap_or_else(|| {
                    panic!(
                        "FlashAttn {key:?}/Cuda: the flash_decoding wrapper must be bound FROM the \
                         cuda flash_decoding contract - found {} alternative(s) with sources {:?}",
                        alts.len(),
                        alts.iter().map(|e| e.kernel_source).collect::<Vec<_>>(),
                    )
                });
            assert_eq!(
                entry.kernel_source, "baracuda",
                "FlashAttn {key:?}: flash_decoding must be contract-sourced \
                 (kernel_source=\"baracuda\"); got {:?}",
                entry.kernel_source,
            );
            assert!(
                entry.caps.strided_input,
                "FlashAttn {key:?}: caps preserved (strided_input == true, capacity-strided)",
            );

            // (2) THE GATE: the §4.4-pinned cost fn survived fill_unset (pointer
            // identity) AND still returns INFEASIBLE for unsupported shapes.
            assert_eq!(
                entry.cost as usize,
                crate::cost::cost_flash_decoding_cuda as usize,
                "FlashAttn {key:?}: the contract-pinned cost_flash_decoding_cuda must survive \
                 fill_unset (Task-F trampoline) — else fill_unset clobbered the gate",
            );

            let dt = key[0];
            let out = fuel_ir::Shape::from_dims(&[1, 8, 1, 64]);

            // seq_q != 1 ⇒ INFEASIBLE (the static gate).
            let bad_sq = OpParams::FlashAttn {
                b: 1,
                hq: 8,
                hkv: 8,
                sq: 2,
                sk: 16,
                d: 64,
                k_len: 16,
                softmax_scale: 1.0,
                causal: true,
                window_size_left: None,
                window_size_right: None,
                softcap: None,
            };
            assert_eq!(
                (entry.cost)(&[out.clone()], &[dt], &bad_sq, &caps).flops,
                u64::MAX,
                "FlashAttn {key:?}: seq_q!=1 must be INFEASIBLE (gate preserved)",
            );

            // head_dim > 128 ⇒ INFEASIBLE.
            let bad_d = OpParams::FlashAttn {
                b: 1,
                hq: 8,
                hkv: 8,
                sq: 1,
                sk: 16,
                d: 256,
                k_len: 16,
                softmax_scale: 1.0,
                causal: true,
                window_size_left: None,
                window_size_right: None,
                softcap: None,
            };
            assert_eq!(
                (entry.cost)(&[out.clone()], &[dt], &bad_d, &caps).flops,
                u64::MAX,
                "FlashAttn {key:?}: head_dim>128 must be INFEASIBLE (gate preserved)",
            );

            // Supported decode shape (seq_q==1, head_dim<=128) ⇒ FEASIBLE.
            let good = OpParams::FlashAttn {
                b: 1,
                hq: 8,
                hkv: 8,
                sq: 1,
                sk: 16,
                d: 64,
                k_len: 16,
                softmax_scale: 1.0,
                causal: true,
                window_size_left: None,
                window_size_right: None,
                softcap: None,
            };
            assert_ne!(
                (entry.cost)(&[out], &[dt], &good, &caps).flops,
                u64::MAX,
                "FlashAttn {key:?}: a supported decode shape must be FEASIBLE (real gate, not a constant)",
            );
        }
    }
}