Skip to main content

cubecl_spirv/
target.rs

1use cubecl_core::{
2    WgpuCompilationOptions,
3    ir::{
4        ExpandValue, Scope, UIntKind,
5        attributes::{ATTR_BUFFER_BINDING, BufferBindingAttr, EntrypointInterface, FuncInterface},
6        ident,
7        interfaces::TypedExt,
8        metadata::Info,
9        prelude::*,
10    },
11};
12use cubecl_ir::{
13    attributes::{ATTR_BUFFER_IO, BufferIOAttr},
14    dialect::BlockPtrExt,
15};
16use pliron::{
17    builtin::ops::FuncOp,
18    graph::walkers::uninterruptible::immutable::walk_op,
19    identifier::Identifier,
20    irbuild::{inserter::Inserter, listener::DummyListener, match_rewrite::apply_match_rewrite},
21    printable::Printable,
22    std_deps::sync::LazyLock,
23};
24use pliron_spirv::{
25    decorations::{DecoratableOp, DecorationInfo},
26    interfaces::{VerCapExtOpInterface, VerCapExtTypeInterface},
27    ops::{AddressOfOp, GlobalVariableOp, InBoundsAccessChainOp, LoadOp, SpirvModuleOp},
28    types::{ArrayType, PointerType, RuntimeArrayType, StructType},
29};
30use rspirv::{
31    dr::Operand,
32    spirv::{Capability, Decoration, MemoryAccess, StorageClass},
33};
34
35use crate::types::ty_to_spirv_dialect_explicit_layout;
36
37pub static PARAMS_NAME: LazyLock<Identifier> = LazyLock::new(|| ident("_spirv_params"));
38
39/// Lower op that depends on info struct
40#[op_interface]
41pub trait LowerInfoOp {
42    verify_op_succ!();
43    fn lower(&self, ctx: &mut Context, rewriter: &mut MatchRewriter, info_st: Value) -> Value;
44}
45
46/// Run on: `SpirvModuleOp`
47pub struct ConvertArgsPass;
48
49#[pass_name]
50impl Pass for ConvertArgsPass {
51    fn run(
52        &mut self,
53        op: Ptr<Operation>,
54        ctx: &mut Context,
55        _analyses: &mut AnalysisManager,
56    ) -> Result<PassResult> {
57        let mut module = op
58            .as_op::<SpirvModuleOp>(ctx)
59            .expect("Should run on SPIR-V module");
60        visit_all_ops_of_type_mut::<FuncOp, _>(ctx, &mut module, op, |ctx, module, func| {
61            if func.get_entrypoint_abi(ctx).is_some() {
62                Self::convert_func(ctx, *module, func).unwrap();
63            }
64        });
65        let mut res = PassResult::default();
66        res.ir_changed = IRStatus::Changed;
67        Ok(res)
68    }
69}
70
71impl ConvertArgsPass {
72    pub fn convert_func(ctx: &mut Context, module: SpirvModuleOp, func: FuncOp) -> Result<()> {
73        let func_op = func.get_operation();
74        let info = ctx.aux_ty::<Info>().clone();
75        let entry = func.get_entry_block(ctx);
76        let module_body = module.get_body(ctx, 0);
77        let args = entry.arguments(ctx);
78        let mut buffers = vec![];
79
80        for (i, arg) in args.iter().enumerate() {
81            let binding = func.get_arg_attr::<BufferBindingAttr>(ctx, i, &ATTR_BUFFER_BINDING);
82            let io = func.get_arg_attr::<BufferIOAttr>(ctx, i, &ATTR_BUFFER_IO);
83
84            if let Some(binding) = binding {
85                let io = io.expect("Buffers should have IO attribute");
86                buffers.insert(binding.buffer_pos, (*arg, buffer_ty(ctx, *arg, *io)));
87            } else {
88                panic!("Expected all kernel inputs to be bindings")
89            }
90        }
91
92        let mut addr_struct = StructType::default();
93        addr_struct.decorate_type(DecorationInfo::unit(Decoration::Block));
94
95        for (i, (_, ty)) in buffers.iter().copied().enumerate() {
96            addr_struct.field_types.push(ty);
97            addr_struct.offsets.push((i * size_of::<u64>()) as u32);
98        }
99
100        if info.has_info() {
101            let offset = buffers.len() * size_of::<u64>();
102            addr_struct.field_types.push(info_ty(ctx));
103            addr_struct.offsets.push(offset as u32);
104        }
105
106        let addr_struct = Type::instantiate(addr_struct, ctx).to_handle();
107        let storage_class = params_storage_class(ctx, buffers.len());
108        let addr_struct_ptr = PointerType::get(ctx, addr_struct, storage_class).into();
109        let ptrs_var =
110            GlobalVariableOp::new(ctx, addr_struct, storage_class, PARAMS_NAME.clone(), None);
111        ptrs_var.get_operation().insert_at_front(module_body, ctx);
112
113        if !matches!(storage_class, StorageClass::PushConstant) {
114            ptrs_var.set_decoration_descriptor_set(ctx, 0.into());
115            ptrs_var.set_decoration_binding(ctx, 0.into());
116        }
117
118        let mut rewriter = IRRewriter::<DummyListener>::default();
119        rewriter.set_insertion_point_to_block_start(entry);
120        let scope = Scope::from_context_and_inserter(ctx, &mut rewriter);
121        let ptrs = scope.register_with_result(&AddressOfOp::new(
122            ctx,
123            addr_struct_ptr,
124            PARAMS_NAME.clone(),
125        ));
126
127        for (i, (buffer, ..)) in buffers.iter().enumerate() {
128            load_buffer_array(ctx, &scope, ptrs, i, *buffer, storage_class);
129        }
130
131        if info.has_info() {
132            let info = load_buffer(ctx, &scope, ptrs, buffers.len(), storage_class);
133            apply_match_rewrite(ctx, &mut LowerInfoOps(info), Default::default(), func_op)?;
134        }
135
136        let num_args = entry.deref(ctx).get_num_arguments();
137
138        for _ in 0..num_args {
139            func.pop_argument(ctx);
140        }
141        Ok(())
142    }
143}
144
145fn buffer_ty(ctx: &Context, buffer: Value, io: BufferIOAttr) -> TypeHandle {
146    let ty = buffer.element_ty(ctx);
147    let ty_size = ty.size(ctx);
148    let ty = ty_to_spirv_dialect_explicit_layout(ctx, ty);
149    let array = RuntimeArrayType::get(ctx, ty, Some(ty_size as u32));
150
151    let mut struct_ = StructType::default();
152    struct_.decorate_type(DecorationInfo::unit(Decoration::Block));
153    struct_.field_types.push(array.into());
154    struct_.offsets.push(0);
155    if !io.is_readable() {
156        struct_.decorate_member(0, DecorationInfo::unit(Decoration::NonReadable));
157    }
158    if !io.is_writable() {
159        struct_.decorate_member(0, DecorationInfo::unit(Decoration::NonWritable));
160    }
161
162    let struct_ = Type::instantiate(struct_, ctx).to_handle();
163    PointerType::get(ctx, struct_, StorageClass::PhysicalStorageBuffer).into()
164}
165
166fn info_ty(ctx: &Context) -> TypeHandle {
167    let address_ty = ctx.address_type();
168    let info = ctx.aux_ty::<Info>().clone();
169
170    let mut struct_ = StructType::default();
171    struct_
172        .type_decorations
173        .push(DecorationInfo::unit(Decoration::Block));
174
175    for scalar in info.scalars {
176        let ty = ty_to_spirv_dialect_explicit_layout(ctx, scalar.ty.to_type(ctx));
177        let ty_size = scalar.ty.expand_size(ctx.address_type()) as u32;
178        let array = ArrayType::get(ctx, scalar.count as u32, ty, Some(ty_size));
179        struct_.field_types.push(array.into());
180        struct_.offsets.push(scalar.offset as u32);
181    }
182
183    if let Some(field) = info.sized_meta {
184        let ty = ty_to_spirv_dialect_explicit_layout(ctx, field.ty.to_type(ctx));
185        let ty_size = field.ty.expand_size(ctx.address_type()) as u32;
186        let array = ArrayType::get(ctx, field.count as u32, ty, Some(ty_size));
187        struct_.field_types.push(array.into());
188        struct_.offsets.push(field.offset as u32);
189    }
190
191    if info.has_dynamic_meta {
192        let address_ty =
193            ty_to_spirv_dialect_explicit_layout(ctx, address_ty.unsigned_type().to_type(ctx));
194        let ty_size = address_ty.size(ctx) as u32;
195        let array = RuntimeArrayType::get(ctx, address_ty, Some(ty_size));
196        struct_.field_types.push(array.into());
197        struct_.offsets.push(info.dynamic_meta_offset as u32);
198    }
199
200    // Info is only ever read.
201    for i in 0..struct_.field_types.len() {
202        struct_.decorate_member(i, DecorationInfo::unit(Decoration::NonWritable));
203    }
204
205    let struct_ = Type::instantiate(struct_, ctx).to_handle();
206    PointerType::get(ctx, struct_, StorageClass::PhysicalStorageBuffer).into()
207}
208
209fn load_buffer_array(
210    ctx: &mut Context,
211    scope: &Scope,
212    ptrs: Value,
213    idx: usize,
214    cube_buffer: Value,
215    storage_class: StorageClass,
216) {
217    let buffer = load_buffer(ctx, scope, ptrs, idx, storage_class);
218    let buffer_ty = TypedHandle::<PointerType>::from_handle(buffer.get_type(ctx), ctx).unwrap();
219    let buffer_ty =
220        TypedHandle::<StructType>::from_handle(buffer_ty.deref(ctx).element_type, ctx).unwrap();
221    let array_ty = buffer_ty.deref(ctx).field_types[0];
222    let array_ptr_ty = PointerType::get(ctx, array_ty, StorageClass::PhysicalStorageBuffer);
223
224    let zero = ExpandValue::constant(0.into(), UIntKind::U32).value(scope);
225    let array = InBoundsAccessChainOp::new(ctx, array_ptr_ty.into(), buffer, vec![zero]);
226    let array = scope.register_with_result(&array);
227    cube_buffer.replace_all_uses_with(ctx, &array);
228}
229
230fn load_buffer(
231    ctx: &mut Context,
232    scope: &Scope,
233    ptrs: Value,
234    idx: usize,
235    storage_class: StorageClass,
236) -> Value {
237    let struct_ptr = TypedHandle::<PointerType>::from_handle(ptrs.get_type(ctx), ctx).unwrap();
238    let struct_ =
239        TypedHandle::<StructType>::from_handle(struct_ptr.deref(ctx).element_type, ctx).unwrap();
240    let buffer_ty = struct_.deref(ctx).field_types[idx];
241    let buffer_ptr_ty = PointerType::get(ctx, buffer_ty, storage_class);
242
243    let idx = ExpandValue::constant(idx.into(), UIntKind::U32).value(scope);
244    let buffer_ptr = InBoundsAccessChainOp::new(ctx, buffer_ptr_ty.into(), ptrs, vec![idx]);
245    let buffer_ptr = scope.register_with_result(&buffer_ptr);
246    let load = LoadOp::new(ctx, buffer_ty, buffer_ptr, MemoryAccess::NONE, None);
247    scope.register_with_result(&load)
248}
249
250pub fn params_storage_class(ctx: &Context, num_buffers: usize) -> StorageClass {
251    let num_addresses = match ctx.aux_ty::<Info>().has_info() {
252        true => num_buffers + 1,
253        false => num_buffers,
254    };
255    let comp_options = &ctx.aux_ty::<WgpuCompilationOptions>().vulkan;
256    if num_addresses > comp_options.push_constant_size / size_of::<u64>() {
257        StorageClass::Uniform
258    } else {
259        StorageClass::PushConstant
260    }
261}
262
263struct LowerInfoOps(Value);
264
265impl MatchRewrite for LowerInfoOps {
266    fn r#match(&mut self, ctx: &Context, op: Ptr<Operation>) -> bool {
267        op.impls::<dyn LowerInfoOp>(ctx)
268    }
269
270    fn rewrite(
271        &mut self,
272        ctx: &mut Context,
273        rewriter: &mut MatchRewriter,
274        op: Ptr<Operation>,
275    ) -> Result<()> {
276        let dyn_op = op.dyn_op(ctx);
277        let lower_info = op_cast::<dyn LowerInfoOp>(&*dyn_op).unwrap();
278        let value = lower_info.lower(ctx, rewriter, self.0);
279        rewriter.replace_operation_with_values(ctx, op, vec![value]);
280        Ok(())
281    }
282}
283
284#[derive(Default)]
285pub struct CollectVerCapExtPass;
286
287#[pass_name]
288impl Pass for CollectVerCapExtPass {
289    fn run(
290        &mut self,
291        op: Ptr<Operation>,
292        ctx: &mut Context,
293        _analyses: &mut AnalysisManager,
294    ) -> Result<PassResult> {
295        let mut res = PassResult::default();
296        res.ir_changed = IRStatus::Changed;
297        let mut module = op.as_op::<SpirvModuleOp>(ctx).unwrap();
298        let conf = &WALKCONFIG_PREORDER_FORWARD;
299        walk_op(ctx, &mut module, conf, op, update_ver_cap_ext);
300        while update_capability_requirements_recursive(ctx, module) {}
301        Ok(res)
302    }
303}
304
305#[op_interface]
306pub trait CustomCapabilitiesOp {
307    fn custom_capabilities(&self, ctx: &Context) -> Vec<Capability>;
308    fn verify(_op: &dyn Op, _ctx: &Context) -> Result<()>
309    where
310        Self: Sized,
311    {
312        Ok(())
313    }
314}
315
316// These are restricted to OpenCL so remove from consideration
317#[rustfmt::skip]
318const EXCLUDED_CAPS: &[Capability] = {
319    use Capability::*;
320    &[Kernel, Vector16, Float16Buffer, ImageBasic, ImageReadWrite, ImageMipmap, Pipes,
321    DeviceEnqueue, LiteralSampler, SubgroupDispatch, NamedBarrier, PipeStorage]
322};
323const PREFERRED_CAPS: &[Capability] = &[Capability::GroupNonUniformArithmetic];
324const PREFERRED_EXTS: &[&str] = &["SPV_KHR_physical_storage_buffer"];
325
326/// Capabilities can require other capabilities which require other capabilities, and each can
327/// require extensions/a minimum version. So add them all recursively.
328fn update_capability_requirements_recursive(ctx: &Context, module: SpirvModuleOp) -> bool {
329    let op = module.get_operation();
330    let caps_start = module.get_vce(ctx).capabilities;
331    for cap in caps_start.iter().copied().map(Operand::from) {
332        let min_version = cap.minimum_version();
333        let extensions = cap.required_extensions();
334        let caps = cap.required_capabilities();
335        update_ver_cap_ext_impl(ctx, module, min_version, extensions, caps, op);
336    }
337    module.get_vce(ctx).capabilities != caps_start
338}
339
340fn update_ver_cap_ext(ctx: &Context, module: &mut SpirvModuleOp, node: IRNode) {
341    match node {
342        IRNode::BasicBlock(block) => {
343            let args = block.arguments(ctx);
344            for arg in args {
345                let ty = arg.get_type(ctx).deref(ctx);
346                if let Some(ver_cap_ext) = type_cast(&*ty) {
347                    update_ver_cap_ext_ty(ctx, module, ver_cap_ext);
348                }
349            }
350        }
351        IRNode::Operation(op) => {
352            let dyn_op = op.dyn_op(ctx);
353            if let Some(custom) = op_cast::<dyn CustomCapabilitiesOp>(&*dyn_op) {
354                for cap in custom.custom_capabilities(ctx) {
355                    module.insert_capability(ctx, cap);
356                }
357            }
358            if let Some(ver_cap_ext) = op_cast(&*dyn_op) {
359                update_ver_cap_ext_op(ctx, module, ver_cap_ext);
360            }
361            for res in op.results(ctx) {
362                let ty = res.get_type(ctx).deref(ctx);
363                if let Some(ver_cap_ext) = type_cast(&*ty) {
364                    update_ver_cap_ext_ty(ctx, module, ver_cap_ext);
365                }
366            }
367        }
368        _ => {}
369    }
370}
371
372fn update_ver_cap_ext_op(ctx: &Context, module: &mut SpirvModuleOp, op: &dyn VerCapExtOpInterface) {
373    let min_version = op.min_version(ctx);
374    let extensions = op.required_extensions(ctx);
375    let caps = op.required_capabilities(ctx);
376    let op = op.get_operation();
377    update_ver_cap_ext_impl(ctx, *module, min_version, extensions, caps, op);
378}
379
380fn update_ver_cap_ext_ty(
381    ctx: &Context,
382    module: &mut SpirvModuleOp,
383    ty: &dyn VerCapExtTypeInterface,
384) {
385    let min_version = ty.min_version(ctx);
386    let extensions = ty.required_extensions(ctx);
387    let caps = ty.required_capabilities(ctx);
388    let ty = ty.get_self_handle(ctx);
389    update_ver_cap_ext_impl(ctx, *module, min_version, extensions, caps, ty);
390}
391
392fn update_ver_cap_ext_impl(
393    ctx: &Context,
394    module: SpirvModuleOp,
395    min_version: Option<(u8, u8)>,
396    extensions: Vec<Vec<&'static str>>,
397    caps: Vec<Vec<Capability>>,
398    obj: impl Printable,
399) {
400    let max_ver = ctx
401        .aux_ty::<WgpuCompilationOptions>()
402        .vulkan
403        .max_spirv_version;
404    let extensions = if min_version.is_some_and(|ver| ver <= max_ver) {
405        vec![]
406    } else {
407        extensions
408    };
409
410    for mut cap_set in caps.into_iter().filter(|set| !set.is_empty()) {
411        cap_set.retain(|cap| !EXCLUDED_CAPS.contains(cap));
412        if cap_set.len() == 1 {
413            module.insert_capability(ctx, cap_set[0]);
414        } else if cap_set.iter().any(|cap| module.has_capability(ctx, cap)) {
415            continue;
416        } else if let Some(preferred) = cap_set.iter().find(|cap| PREFERRED_CAPS.contains(cap)) {
417            module.insert_capability(ctx, *preferred);
418        } else {
419            panic!(
420                "Need custom rule for multi-capability node {}, lists capabilities: {:?}",
421                obj.disp(ctx),
422                cap_set
423            );
424        }
425    }
426
427    for ext_set in extensions.into_iter().filter(|set| !set.is_empty()) {
428        if ext_set.len() == 1 {
429            module.insert_extension(ctx, ext_set[0]);
430        } else if ext_set.iter().any(|ext| module.has_extension(ctx, *ext)) {
431            continue;
432        } else if let Some(preferred) = ext_set.iter().find(|cap| PREFERRED_EXTS.contains(cap)) {
433            module.insert_extension(ctx, *preferred);
434        } else {
435            panic!(
436                "Need custom rule for multi-extension node {}, lists extensions: {:?}",
437                obj.disp(ctx),
438                ext_set
439            );
440        }
441    }
442}