cubek-matmul 0.2.0

CubeK: Matrix Multiplication Kernels
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111

//! Strategy variants that are only exposed for the test suite.
//!
//! These routines are exercised by the full test tree but are not part of the
//! publicly supported [`Strategy`] enum: either they are experimental
//! or they use loading combinations that are not wired into the production selector.

use std::fmt::Display;

use cubecl::{Runtime, client::ComputeClient, prelude::TensorBinding};
use cubek_std::InputBinding;

use crate::{
    components::{
        global::read::{
            async_full_cooperative::AsyncFullCooperativeLoading,
            async_full_cyclic::AsyncFullCyclicLoading,
        },
        stage::ColMajorTilingOrder,
        tile::TileMatmulKind,
    },
    definition::{MatmulElems, MatmulSetupError},
    launch::launch_tiling,
    routines::{
        BlueprintStrategy, Routine, TilingArgs, interleaved::InterleavedAlgorithm,
        simple::SimpleBarrierAlgorithm,
    },
};

/// Non-public strategy variants reserved for test coverage.
#[allow(clippy::type_complexity)]
#[derive(Clone)]
pub enum TestStrategy {
    SimpleBarrierCooperativeCmma(
        BlueprintStrategy<(), SimpleBarrierAlgorithm<AsyncFullCooperativeLoading>>,
    ),
    SimpleBarrierCooperativeMma(
        BlueprintStrategy<(), SimpleBarrierAlgorithm<AsyncFullCooperativeLoading>>,
    ),
    SimpleBarrierCyclicCmma(
        BlueprintStrategy<(), SimpleBarrierAlgorithm<AsyncFullCyclicLoading<ColMajorTilingOrder>>>,
    ),
    SimpleBarrierCyclicMma(
        BlueprintStrategy<(), SimpleBarrierAlgorithm<AsyncFullCyclicLoading<ColMajorTilingOrder>>>,
    ),
    Interleaved(BlueprintStrategy<(), InterleavedAlgorithm>),
}

impl Display for TestStrategy {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            Self::SimpleBarrierCooperativeCmma(s) => {
                write!(f, "matmul_simple_barrier_cooperative_cmma{}", s)
            }
            Self::SimpleBarrierCooperativeMma(s) => {
                write!(f, "matmul_simple_barrier_cooperative_mma{}", s)
            }
            Self::SimpleBarrierCyclicCmma(s) => {
                write!(f, "matmul_simple_barrier_cyclic_cmma{}", s)
            }
            Self::SimpleBarrierCyclicMma(s) => {
                write!(f, "matmul_simple_barrier_cyclic_mma{}", s)
            }
            Self::Interleaved(s) => write!(f, "matmul_interleaved{}", s),
        }
    }
}

fn with_kind<RC, A>(
    sel: &BlueprintStrategy<RC, A>,
    kind: TileMatmulKind,
) -> BlueprintStrategy<RC, A>
where
    RC: crate::launch::RuntimeConfig,
    A: Routine<RC>,
    A::Strategy: TilingArgs,
{
    let mut sel = sel.clone();
    if let BlueprintStrategy::Inferred(args) = &mut sel {
        args.set_tile_matmul(kind);
    }
    sel
}

#[allow(clippy::result_large_err)]
impl TestStrategy {
    pub fn launch_ref<R: Runtime>(
        &self,
        client: &ComputeClient<R>,
        lhs: InputBinding<R>,
        rhs: InputBinding<R>,
        out: TensorBinding<R>,
        dtypes: &mut MatmulElems,
    ) -> Result<(), MatmulSetupError> {
        use TileMatmulKind::{Cmma, Mma};
        match self {
            Self::SimpleBarrierCooperativeCmma(sel) => {
                launch_tiling::launch_ref(client, lhs, rhs, out, &with_kind(sel, Cmma), dtypes)
            }
            Self::SimpleBarrierCooperativeMma(sel) => {
                launch_tiling::launch_ref(client, lhs, rhs, out, &with_kind(sel, Mma), dtypes)
            }
            Self::SimpleBarrierCyclicCmma(sel) => {
                launch_tiling::launch_ref(client, lhs, rhs, out, &with_kind(sel, Cmma), dtypes)
            }
            Self::SimpleBarrierCyclicMma(sel) => {
                launch_tiling::launch_ref(client, lhs, rhs, out, &with_kind(sel, Mma), dtypes)
            }
            Self::Interleaved(sel) => launch_tiling::launch_ref(client, lhs, rhs, out, sel, dtypes),
        }
    }
}