burn-cubecl 0.21.0-pre.2

Generic backend that can be compiled just-in-time to any shader language target
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124

use burn_backend::{DType, QTensorPrimitive, TensorMetadata};
use cubecl::quant::scheme::{QuantStore, QuantValue};
use cubecl::server::AllocationKind;

use crate::{CubeRuntime, ops::empty_qtensor, tensor::CubeTensor};

/// Make a jit tensor contiguous.
pub fn into_contiguous<R: CubeRuntime>(tensor: CubeTensor<R>) -> CubeTensor<R> {
    if tensor.is_contiguous() {
        return tensor;
    }

    if tensor.qparams.is_some() {
        return into_contiguous_quantized(tensor, AllocationKind::Contiguous);
    }

    let output = cubecl::std::tensor::into_contiguous_ref(
        &tensor.client,
        &tensor.as_handle_ref(),
        tensor.dtype.into(),
    )
    .expect("Kernel to never fail");

    CubeTensor::new(
        tensor.client,
        output.handle,
        *output.metadata,
        tensor.device,
        tensor.dtype,
    )
}

/// Make a jit tensor contiguous with an aligned last stride. Tensor is considered already contiguous
/// if runtime can read it as is. This is equivalent in practice.
#[cfg_attr(
    feature = "tracing",
    tracing::instrument(level = "trace", skip(tensor))
)]
pub fn into_contiguous_aligned<R: CubeRuntime>(tensor: CubeTensor<R>) -> CubeTensor<R> {
    if R::can_read_tensor(tensor.meta.shape(), tensor.meta.strides()) {
        return tensor;
    }

    if tensor.qparams.is_some() {
        return into_contiguous_quantized(tensor, AllocationKind::Optimized);
    }

    let output = cubecl::std::tensor::into_contiguous_pitched_ref(
        &tensor.client,
        &tensor.as_handle_ref(),
        tensor.dtype.into(),
    )
    .expect("Kernel to never fail");

    CubeTensor::new(
        tensor.client,
        output.handle,
        *output.metadata,
        tensor.device,
        tensor.dtype,
    )
}

#[cfg_attr(
    feature = "tracing",
    tracing::instrument(level = "trace", skip(tensor))
)]
fn into_contiguous_quantized<R: CubeRuntime>(
    tensor: CubeTensor<R>,
    kind: AllocationKind,
) -> CubeTensor<R> {
    let scheme = tensor.scheme();
    let output = empty_qtensor(tensor.shape(), *tensor.scheme(), &tensor.device, kind);
    let (values, scales) = tensor.quantized_handles().unwrap();
    let (out_values, out_scales) = output.quantized_handles().unwrap();

    match scheme.store {
        QuantStore::PackedU32(packed_dim) => {
            cubecl::std::tensor::into_contiguous_packed_ref(
                &values.client,
                &values.as_handle_ref(),
                &out_values.as_handle_ref(),
                packed_dim,
                tensor.meta.shape(),
                scheme.num_quants(),
                DType::U32.into(),
            )
            .expect("Kernel to never fail");
        }
        // e2m1 is special because it has a native packed representation, `e2m1x2`.
        // It's internally stored as `u8` with a packing factor of 2.
        QuantStore::PackedNative(packed_dim) if scheme.value == QuantValue::E2M1 => {
            cubecl::std::tensor::into_contiguous_packed_ref(
                &values.client,
                &values.as_handle_ref(),
                &out_values.as_handle_ref(),
                packed_dim,
                tensor.meta.shape(),
                scheme.num_quants(),
                DType::U8.into(),
            )
            .expect("Kernel to never fail");
        }
        _ => {
            cubecl::std::tensor::copy_into(
                &values.client,
                &values.as_handle_ref(),
                &out_values.as_handle_ref(),
                values.dtype.into(),
            )
            .expect("Kernel to never fail");
        }
    }

    cubecl::std::tensor::copy_into(
        &scales.client,
        &scales.as_handle_ref(),
        &out_scales.as_handle_ref(),
        scales.dtype.into(),
    )
    .expect("Kernel to never fail");

    output
}