cubek_convolution/kernels/forward/

args.rs

use cubecl::{
    client::Client,
    prelude::*,
    std::tensor::{
        launch::ViewArg,
        layout::{
            VirtualLayoutLaunch,
            chain::{Chain, ChainLaunch},
        },
    },
    zspace::{shape, strides},
};
use cubecl_ir::EnumSet;
use cubek_matmul::multi_level::stage::SwizzleMode;
use cubek_matmul::{
    definition::MatmulElems,
    multi_level::{
        BatchMatmulRoutine,
        args::*,
        components::global::memory::{GlobalLayoutConfig, NoopLayout, NoopLayoutLaunch},
        definition::{BatchMatmulBlueprint, Blueprint},
    },
};
use cubek_std::{InputBinding, MatrixLayout, launch::tma::remap_storage_for_tma};

use crate::components::{
    ConvolutionParams, ConvolutionProblem,
    global::{
        args::{RuntimeArgs, RuntimeArgsLaunch},
        layout::{
            BiasLayout, Im2colLayout, Im2colLayoutLaunch, NhwcCheck, NhwcLayout, NhwcLayoutLaunch,
            OutLayout, OutLayoutLaunch, TmaIm2colLayout, TmaIm2colLayoutLaunch, WeightLayout,
            WeightLayoutLaunch,
        },
    },
};

pub trait ConcreteArgs<A: BatchMatmulRoutine<RuntimeArgs>>:
    MatmulArgs<
        Input<Vector<Lhs, LhsSize>, Vector<Rhs, RhsSize>, Vector<Acc, AccSize>>: ConcreteInputsFactory<A>,
        Output<Vector<Acc, AccSize>>: ConcreteOutputFactory<A>,
        Config = RuntimeArgs,
    >
{
    fn adjust_problem(
        client: &Client,
        problem: ConvolutionProblem,
        blueprint: &A::Blueprint,
        dtypes: &MatmulElems,
    ) -> ConvolutionProblem;
}

impl<A: BatchMatmulRoutine<RuntimeArgs>> ConcreteArgs<A> for TensorArgs<RuntimeArgs> {
    fn adjust_problem(
        client: &Client,
        mut problem: ConvolutionProblem,
        _blueprint: &A::Blueprint,
        dtypes: &MatmulElems,
    ) -> ConvolutionProblem {
        let io_width = client.properties().io_width();
        let channel_align = io_width as usize / dtypes.lhs_global.size_bits();
        let padded_channels = problem.channels.next_multiple_of(channel_align);
        let shape_k = problem.kernel_size.iter().product::<u32>() as usize * padded_channels;

        problem.k = shape_k;
        problem.padded_channels = padded_channels;

        problem
    }
}

impl<A: BatchMatmulRoutine<RuntimeArgs, Blueprint = BatchMatmulBlueprint>> ConcreteArgs<A>
    for TensorMapArgs<RuntimeArgs>
{
    fn adjust_problem(
        _client: &Client,
        mut problem: ConvolutionProblem,
        blueprint: &BatchMatmulBlueprint,
        _dtypes: &MatmulElems,
    ) -> ConvolutionProblem {
        let channel_align = match blueprint.swizzle_modes.lhs {
            SwizzleMode::None => blueprint.tiling_scheme.tile_size.k() as usize,
            _ => blueprint.tiling_scheme.elements_per_stage_along_k() as usize,
        };
        let padded_channels = problem.channels.next_multiple_of(channel_align);
        let shape_k = problem.kernel_size.iter().product::<u32>() as usize * padded_channels;

        problem.k = shape_k;
        problem.padded_channels = padded_channels;

        problem
    }
}

/// Create the input runtime arguments for a matmul kernel that works on concrete inputs and
/// output (not fused).
pub trait ConcreteInputsFactory<A: BatchMatmulRoutine<RuntimeArgs>>: LaunchArg {
    #[allow(clippy::too_many_arguments)]
    fn create(
        lhs: InputBinding,
        rhs: InputBinding,
        bias: Option<InputBinding>,
        blueprint: &A::Blueprint,
        problem: &ConvolutionProblem,
        dtypes: &MatmulElems,
    ) -> (Self::RuntimeArg, RuntimeArgsLaunch);
}

/// Create the output runtime arguments for a matmul kernel that works on concrete inputs and
/// output (not fused).
pub trait ConcreteOutputFactory<A: BatchMatmulRoutine<RuntimeArgs>>: LaunchArg {
    fn create(
        out: TensorBinding,
        blueprint: &A::Blueprint,
        problem: &ConvolutionProblem,
        dtypes: &MatmulElems,
    ) -> Self::RuntimeArg;
}

impl<Lhs: CubePrimitive, Rhs: CubePrimitive, EO: CubePrimitive, A: BatchMatmulRoutine<RuntimeArgs>>
    ConcreteInputsFactory<A> for TensorInputs<Lhs, Rhs, EO>
{
    fn create(
        lhs: InputBinding,
        rhs: InputBinding,
        bias: Option<InputBinding>,
        blueprint: &A::Blueprint,
        problem: &ConvolutionProblem,
        _dtypes: &MatmulElems,
    ) -> (Self::RuntimeArg, RuntimeArgsLaunch) {
        type LhsLayout = Chain<NhwcLayout, Im2colLayout>;
        type RhsLayout = Chain<NhwcLayout, WeightLayout>;

        let padded_channels = problem.padded_channels as u32;
        let conv_params = ConvolutionParams::from_problem(problem);

        let layout_lhs = Im2colLayoutLaunch::from_args(
            problem,
            conv_params,
            blueprint.lhs_global_layout_config(),
        );
        let layout_rhs =
            WeightLayoutLaunch::from_args(problem, blueprint.rhs_global_layout_config());

        let layout_lhs = {
            let mut checks = EnumSet::empty();
            if problem.should_check_spatial_bounds() {
                checks.insert(NhwcCheck::Spatial);
            }
            if problem.should_check_channel() {
                checks.insert(NhwcCheck::Channel);
            }
            let global = NhwcLayoutLaunch::checked(checks);
            ChainLaunch::new(global, layout_lhs)
        };
        let layout_rhs = {
            let mut checks = EnumSet::empty();
            if problem.should_check_channel() {
                checks.insert(NhwcCheck::Channel);
            }
            let global = NhwcLayoutLaunch::checked(checks);
            ChainLaunch::new(global, layout_rhs)
        };

        let inputs = TensorInputsLaunch::new(
            VirtualLayoutLaunch::new::<NoopLayout>(NoopLayoutLaunch::new()),
            ViewArg::new_tensor::<LhsLayout>(lhs.into_data().into_tensor_arg(), layout_lhs),
            VirtualLayoutLaunch::new::<NoopLayout>(NoopLayoutLaunch::new()),
            ViewArg::new_tensor::<RhsLayout>(rhs.into_data().into_tensor_arg(), layout_rhs),
            bias.as_ref()
                .map(|_| VirtualLayoutLaunch::new::<NoopLayout>(NoopLayoutLaunch::new()))
                .into(),
            bias.map(|bias| {
                ViewArg::new_tensor::<BiasLayout>(bias.into_data().into_tensor_arg(), ())
            })
            .into(),
        );

        let runtime_args = RuntimeArgsLaunch::new(
            problem.k as u32,
            problem.channels as u32,
            padded_channels,
            conv_params.operation,
        );

        (inputs, runtime_args)
    }
}

impl<EG: CubePrimitive, A: BatchMatmulRoutine<RuntimeArgs>> ConcreteOutputFactory<A>
    for TensorOutput<EG>
{
    fn create(
        out: TensorBinding,
        blueprint: &A::Blueprint,
        problem: &ConvolutionProblem,
        _dtypes: &MatmulElems,
    ) -> Self::RuntimeArg {
        type Layout = Chain<NhwcLayout, OutLayout>;

        let global = NhwcLayoutLaunch::unchecked();
        let layout = OutLayoutLaunch::from_args(problem, blueprint.out_global_layout_config());
        let layout = ChainLaunch::new(global, layout);
        let view = ViewArg::new_tensor::<Layout>(out.into_tensor_arg(), layout);
        let batch = VirtualLayoutLaunch::new::<NoopLayout>(NoopLayoutLaunch::new());
        TensorOutputLaunch::new(view, batch)
    }
}

impl<
    Lhs: CubePrimitive,
    Rhs: CubePrimitive,
    EO: CubePrimitive,
    A: BatchMatmulRoutine<RuntimeArgs, Blueprint = BatchMatmulBlueprint>,
> ConcreteInputsFactory<A> for TensorMapInputs<Lhs, Rhs, EO>
{
    fn create(
        lhs: InputBinding,
        rhs: InputBinding,
        bias: Option<InputBinding>,
        blueprint: &BatchMatmulBlueprint,
        problem: &ConvolutionProblem,
        dtypes: &MatmulElems,
    ) -> (Self::RuntimeArg, RuntimeArgsLaunch) {
        let tiling_scheme = blueprint.tiling_scheme;
        let stage_m = tiling_scheme.elements_per_stage_along_m();
        let stage_n = tiling_scheme.elements_per_stage_along_n();

        let tile_size_k = match blueprint.swizzle_modes.lhs {
            SwizzleMode::None => tiling_scheme.tile_size.k,
            _ => tiling_scheme.elements_per_stage_along_k(),
        };

        let mut stage_size_rhs = shape![1; problem.dimensionality.num_dims()];
        stage_size_rhs.insert(0, stage_n as usize);
        stage_size_rhs.push(tile_size_k as usize);

        let lhs_elem = remap_storage_for_tma(dtypes.lhs_stage);

        let mut elem_stride = strides![1; 2 + problem.stride.len()];

        for (i, stride) in problem.stride.iter().enumerate() {
            elem_stride[i + 1] = *stride as usize;
        }

        let lhs = TensorMapArg::new(
            Im2colArgs {
                pixel_box_lower_corner: calculate_lower_corner(&problem.padding),
                pixel_box_upper_corner: calculate_upper_corner(problem),
                channels_per_pixel: tile_size_k,
                pixels_per_column: stage_m,
            },
            lhs.clone().into_data().into_tensor_arg(),
            lhs_elem,
        )
        .with_elem_stride(elem_stride)
        .with_swizzle(blueprint.swizzle_modes.lhs.into());

        let rhs = TensorMapArg::new(
            TiledArgs {
                tile_size: stage_size_rhs,
            },
            rhs.clone().into_data().into_tensor_arg(),
            dtypes.rhs_global,
        )
        .with_swizzle(blueprint.swizzle_modes.rhs.into());

        let padded_channels = problem.padded_channels as u32;
        let shape_k = problem.k as u32;

        // Im2col needs extra checking because if `k` is OOB it wraps around the kernel and can load
        // in-bounds but not in-kernel elements. Other TMA layouts are always outside the shape if
        // any matrix dim is out of bounds.
        let stages_lhs = A::num_stages().lhs;
        let stages_size_k = blueprint.tiling_scheme.elements_per_stage_along_k() * stages_lhs;
        let check_kernel = !shape_k.is_multiple_of(stages_size_k);
        let lhs_layout = TmaIm2colLayoutLaunch::from_args(problem, check_kernel);
        let rhs_layout = WeightLayoutLaunch::from_args(
            problem,
            GlobalLayoutConfig {
                check_row_bounds: false,
                check_col_bounds: false,
                matrix_layout: MatrixLayout::ColMajor,
            },
        );

        let bias = bias
            .map(|bias| ViewArg::new_tensor::<BiasLayout>(bias.into_data().into_tensor_arg(), ()));

        let inputs = TensorMapInputsLaunch::new(
            ViewArg::new_tensor_map_im2col::<TmaIm2colLayout, _, _>(lhs, lhs_layout),
            ViewArg::new_tensor_map_tiled::<WeightLayout>(rhs, rhs_layout),
            bias.into(),
            ComptimeOptionArgs::Some(VirtualLayoutLaunch::new::<NoopLayout>(
                NoopLayoutLaunch::new(),
            )),
        );

        let runtime_args = RuntimeArgsLaunch::new(
            shape_k,
            problem.channels as u32,
            padded_channels,
            problem.operation,
        );

        (inputs, runtime_args)
    }
}

fn calculate_lower_corner(padding: &[i32]) -> Vec<i32> {
    padding.iter().map(|padding| -*padding).collect()
}

/// The upper corner bounds the pixel box the im2col tensor map may be addressed in: a base
/// coordinate is legal only while it stays under `in_shape + upper_corner`.
pub(crate) fn calculate_upper_corner(problem: &ConvolutionProblem) -> Vec<i32> {
    let ConvolutionProblem {
        padding,
        kernel_size,
        dilation,
        stride,
        in_shape,
        out_shape,
        ..
    } = problem;

    (0..padding.len())
        .map(|d| {
            let symmetric = padding[d] - (kernel_size[d] - 1) as i32 * dilation[d] as i32;
            // `problem.padding` is the padding at the *beginning* of each dimension, so it cannot bound the
            // far edge on its own. Where padding is asymmetric, `out_shape` is derived from the leading *and*
            // trailing padding while the corner would only see the leading half, and the last output position
            // addresses one pixel past the box. So take whichever is larger.
            let max_coord = (out_shape[d] as i32 - 1) * stride[d] as i32 - padding[d];
            let required = max_coord - in_shape[d] as i32 + 1;
            symmetric.max(required)
        })
        .collect()
}