1use cubecl_core::{
2 WgpuCompilationOptions,
3 ir::{
4 ExpandValue, Scope, UIntKind,
5 attributes::{ATTR_BUFFER_BINDING, BufferBindingAttr, EntrypointInterface, FuncInterface},
6 ident,
7 interfaces::TypedExt,
8 metadata::Info,
9 prelude::*,
10 },
11};
12use cubecl_ir::{
13 attributes::{ATTR_BUFFER_IO, BufferIOAttr},
14 dialect::BlockPtrExt,
15};
16use pliron::{
17 builtin::ops::FuncOp,
18 graph::walkers::uninterruptible::immutable::walk_op,
19 identifier::Identifier,
20 irbuild::{inserter::Inserter, listener::DummyListener, match_rewrite::apply_match_rewrite},
21 printable::Printable,
22 std_deps::sync::LazyLock,
23};
24use pliron_spirv::{
25 decorations::{DecoratableOp, DecorationInfo},
26 interfaces::{VerCapExtOpInterface, VerCapExtTypeInterface},
27 ops::{AddressOfOp, GlobalVariableOp, InBoundsAccessChainOp, LoadOp, SpirvModuleOp},
28 types::{ArrayType, PointerType, RuntimeArrayType, StructType},
29};
30use rspirv::{
31 dr::Operand,
32 spirv::{Capability, Decoration, MemoryAccess, StorageClass},
33};
34
35use crate::types::ty_to_spirv_dialect_explicit_layout;
36
37pub static PARAMS_NAME: LazyLock<Identifier> = LazyLock::new(|| ident("_spirv_params"));
38
39#[op_interface]
41pub trait LowerInfoOp {
42 verify_op_succ!();
43 fn lower(&self, ctx: &mut Context, rewriter: &mut MatchRewriter, info_st: Value) -> Value;
44}
45
46pub struct ConvertArgsPass;
48
49#[pass_name]
50impl Pass for ConvertArgsPass {
51 fn run(
52 &mut self,
53 op: Ptr<Operation>,
54 ctx: &mut Context,
55 _analyses: &mut AnalysisManager,
56 ) -> Result<PassResult> {
57 let mut module = op
58 .as_op::<SpirvModuleOp>(ctx)
59 .expect("Should run on SPIR-V module");
60 visit_all_ops_of_type_mut::<FuncOp, _>(ctx, &mut module, op, |ctx, module, func| {
61 if func.get_entrypoint_abi(ctx).is_some() {
62 Self::convert_func(ctx, *module, func).unwrap();
63 }
64 });
65 let mut res = PassResult::default();
66 res.ir_changed = IRStatus::Changed;
67 Ok(res)
68 }
69}
70
71impl ConvertArgsPass {
72 pub fn convert_func(ctx: &mut Context, module: SpirvModuleOp, func: FuncOp) -> Result<()> {
73 let func_op = func.get_operation();
74 let info = ctx.aux_ty::<Info>().clone();
75 let entry = func.get_entry_block(ctx);
76 let module_body = module.get_body(ctx, 0);
77 let args = entry.arguments(ctx);
78 let mut buffers = vec![];
79
80 for (i, arg) in args.iter().enumerate() {
81 let binding = func.get_arg_attr::<BufferBindingAttr>(ctx, i, &ATTR_BUFFER_BINDING);
82 let io = func.get_arg_attr::<BufferIOAttr>(ctx, i, &ATTR_BUFFER_IO);
83
84 if let Some(binding) = binding {
85 let io = io.expect("Buffers should have IO attribute");
86 buffers.insert(binding.buffer_pos, (*arg, buffer_ty(ctx, *arg, *io)));
87 } else {
88 panic!("Expected all kernel inputs to be bindings")
89 }
90 }
91
92 let mut addr_struct = StructType::default();
93 addr_struct.decorate_type(DecorationInfo::unit(Decoration::Block));
94
95 for (i, (_, ty)) in buffers.iter().copied().enumerate() {
96 addr_struct.field_types.push(ty);
97 addr_struct.offsets.push((i * size_of::<u64>()) as u32);
98 }
99
100 if info.has_info() {
101 let offset = buffers.len() * size_of::<u64>();
102 addr_struct.field_types.push(info_ty(ctx));
103 addr_struct.offsets.push(offset as u32);
104 }
105
106 let addr_struct = Type::instantiate(addr_struct, ctx).to_handle();
107 let storage_class = params_storage_class(ctx, buffers.len());
108 let addr_struct_ptr = PointerType::get(ctx, addr_struct, storage_class).into();
109 let ptrs_var =
110 GlobalVariableOp::new(ctx, addr_struct, storage_class, PARAMS_NAME.clone(), None);
111 ptrs_var.get_operation().insert_at_front(module_body, ctx);
112
113 if !matches!(storage_class, StorageClass::PushConstant) {
114 ptrs_var.set_decoration_descriptor_set(ctx, 0.into());
115 ptrs_var.set_decoration_binding(ctx, 0.into());
116 }
117
118 let mut rewriter = IRRewriter::<DummyListener>::default();
119 rewriter.set_insertion_point_to_block_start(entry);
120 let scope = Scope::from_context_and_inserter(ctx, &mut rewriter);
121 let ptrs = scope.register_with_result(&AddressOfOp::new(
122 ctx,
123 addr_struct_ptr,
124 PARAMS_NAME.clone(),
125 ));
126
127 for (i, (buffer, ..)) in buffers.iter().enumerate() {
128 load_buffer_array(ctx, &scope, ptrs, i, *buffer, storage_class);
129 }
130
131 if info.has_info() {
132 let info = load_buffer(ctx, &scope, ptrs, buffers.len(), storage_class);
133 apply_match_rewrite(ctx, &mut LowerInfoOps(info), Default::default(), func_op)?;
134 }
135
136 let num_args = entry.deref(ctx).get_num_arguments();
137
138 for _ in 0..num_args {
139 func.pop_argument(ctx);
140 }
141 Ok(())
142 }
143}
144
145fn buffer_ty(ctx: &Context, buffer: Value, io: BufferIOAttr) -> TypeHandle {
146 let ty = buffer.element_ty(ctx);
147 let ty_size = ty.size(ctx);
148 let ty = ty_to_spirv_dialect_explicit_layout(ctx, ty);
149 let array = RuntimeArrayType::get(ctx, ty, Some(ty_size as u32));
150
151 let mut struct_ = StructType::default();
152 struct_.decorate_type(DecorationInfo::unit(Decoration::Block));
153 struct_.field_types.push(array.into());
154 struct_.offsets.push(0);
155 if !io.is_readable() {
156 struct_.decorate_member(0, DecorationInfo::unit(Decoration::NonReadable));
157 }
158 if !io.is_writable() {
159 struct_.decorate_member(0, DecorationInfo::unit(Decoration::NonWritable));
160 }
161
162 let struct_ = Type::instantiate(struct_, ctx).to_handle();
163 PointerType::get(ctx, struct_, StorageClass::PhysicalStorageBuffer).into()
164}
165
166fn info_ty(ctx: &Context) -> TypeHandle {
167 let address_ty = ctx.address_type();
168 let info = ctx.aux_ty::<Info>().clone();
169
170 let mut struct_ = StructType::default();
171 struct_
172 .type_decorations
173 .push(DecorationInfo::unit(Decoration::Block));
174
175 for scalar in info.scalars {
176 let ty = ty_to_spirv_dialect_explicit_layout(ctx, scalar.ty.to_type(ctx));
177 let ty_size = scalar.ty.expand_size(ctx.address_type()) as u32;
178 let array = ArrayType::get(ctx, scalar.count as u32, ty, Some(ty_size));
179 struct_.field_types.push(array.into());
180 struct_.offsets.push(scalar.offset as u32);
181 }
182
183 if let Some(field) = info.sized_meta {
184 let ty = ty_to_spirv_dialect_explicit_layout(ctx, field.ty.to_type(ctx));
185 let ty_size = field.ty.expand_size(ctx.address_type()) as u32;
186 let array = ArrayType::get(ctx, field.count as u32, ty, Some(ty_size));
187 struct_.field_types.push(array.into());
188 struct_.offsets.push(field.offset as u32);
189 }
190
191 if info.has_dynamic_meta {
192 let address_ty =
193 ty_to_spirv_dialect_explicit_layout(ctx, address_ty.unsigned_type().to_type(ctx));
194 let ty_size = address_ty.size(ctx) as u32;
195 let array = RuntimeArrayType::get(ctx, address_ty, Some(ty_size));
196 struct_.field_types.push(array.into());
197 struct_.offsets.push(info.dynamic_meta_offset as u32);
198 }
199
200 for i in 0..struct_.field_types.len() {
202 struct_.decorate_member(i, DecorationInfo::unit(Decoration::NonWritable));
203 }
204
205 let struct_ = Type::instantiate(struct_, ctx).to_handle();
206 PointerType::get(ctx, struct_, StorageClass::PhysicalStorageBuffer).into()
207}
208
209fn load_buffer_array(
210 ctx: &mut Context,
211 scope: &Scope,
212 ptrs: Value,
213 idx: usize,
214 cube_buffer: Value,
215 storage_class: StorageClass,
216) {
217 let buffer = load_buffer(ctx, scope, ptrs, idx, storage_class);
218 let buffer_ty = TypedHandle::<PointerType>::from_handle(buffer.get_type(ctx), ctx).unwrap();
219 let buffer_ty =
220 TypedHandle::<StructType>::from_handle(buffer_ty.deref(ctx).element_type, ctx).unwrap();
221 let array_ty = buffer_ty.deref(ctx).field_types[0];
222 let array_ptr_ty = PointerType::get(ctx, array_ty, StorageClass::PhysicalStorageBuffer);
223
224 let zero = ExpandValue::constant(0.into(), UIntKind::U32).value(scope);
225 let array = InBoundsAccessChainOp::new(ctx, array_ptr_ty.into(), buffer, vec![zero]);
226 let array = scope.register_with_result(&array);
227 cube_buffer.replace_all_uses_with(ctx, &array);
228}
229
230fn load_buffer(
231 ctx: &mut Context,
232 scope: &Scope,
233 ptrs: Value,
234 idx: usize,
235 storage_class: StorageClass,
236) -> Value {
237 let struct_ptr = TypedHandle::<PointerType>::from_handle(ptrs.get_type(ctx), ctx).unwrap();
238 let struct_ =
239 TypedHandle::<StructType>::from_handle(struct_ptr.deref(ctx).element_type, ctx).unwrap();
240 let buffer_ty = struct_.deref(ctx).field_types[idx];
241 let buffer_ptr_ty = PointerType::get(ctx, buffer_ty, storage_class);
242
243 let idx = ExpandValue::constant(idx.into(), UIntKind::U32).value(scope);
244 let buffer_ptr = InBoundsAccessChainOp::new(ctx, buffer_ptr_ty.into(), ptrs, vec![idx]);
245 let buffer_ptr = scope.register_with_result(&buffer_ptr);
246 let load = LoadOp::new(ctx, buffer_ty, buffer_ptr, MemoryAccess::NONE, None);
247 scope.register_with_result(&load)
248}
249
250pub fn params_storage_class(ctx: &Context, num_buffers: usize) -> StorageClass {
251 let num_addresses = match ctx.aux_ty::<Info>().has_info() {
252 true => num_buffers + 1,
253 false => num_buffers,
254 };
255 let comp_options = &ctx.aux_ty::<WgpuCompilationOptions>().vulkan;
256 if num_addresses > comp_options.push_constant_size / size_of::<u64>() {
257 StorageClass::Uniform
258 } else {
259 StorageClass::PushConstant
260 }
261}
262
263struct LowerInfoOps(Value);
264
265impl MatchRewrite for LowerInfoOps {
266 fn r#match(&mut self, ctx: &Context, op: Ptr<Operation>) -> bool {
267 op.impls::<dyn LowerInfoOp>(ctx)
268 }
269
270 fn rewrite(
271 &mut self,
272 ctx: &mut Context,
273 rewriter: &mut MatchRewriter,
274 op: Ptr<Operation>,
275 ) -> Result<()> {
276 let dyn_op = op.dyn_op(ctx);
277 let lower_info = op_cast::<dyn LowerInfoOp>(&*dyn_op).unwrap();
278 let value = lower_info.lower(ctx, rewriter, self.0);
279 rewriter.replace_operation_with_values(ctx, op, vec![value]);
280 Ok(())
281 }
282}
283
284#[derive(Default)]
285pub struct CollectVerCapExtPass;
286
287#[pass_name]
288impl Pass for CollectVerCapExtPass {
289 fn run(
290 &mut self,
291 op: Ptr<Operation>,
292 ctx: &mut Context,
293 _analyses: &mut AnalysisManager,
294 ) -> Result<PassResult> {
295 let mut res = PassResult::default();
296 res.ir_changed = IRStatus::Changed;
297 let mut module = op.as_op::<SpirvModuleOp>(ctx).unwrap();
298 let conf = &WALKCONFIG_PREORDER_FORWARD;
299 walk_op(ctx, &mut module, conf, op, update_ver_cap_ext);
300 while update_capability_requirements_recursive(ctx, module) {}
301 Ok(res)
302 }
303}
304
305#[op_interface]
306pub trait CustomCapabilitiesOp {
307 fn custom_capabilities(&self, ctx: &Context) -> Vec<Capability>;
308 fn verify(_op: &dyn Op, _ctx: &Context) -> Result<()>
309 where
310 Self: Sized,
311 {
312 Ok(())
313 }
314}
315
316#[rustfmt::skip]
318const EXCLUDED_CAPS: &[Capability] = {
319 use Capability::*;
320 &[Kernel, Vector16, Float16Buffer, ImageBasic, ImageReadWrite, ImageMipmap, Pipes,
321 DeviceEnqueue, LiteralSampler, SubgroupDispatch, NamedBarrier, PipeStorage]
322};
323const PREFERRED_CAPS: &[Capability] = &[Capability::GroupNonUniformArithmetic];
324const PREFERRED_EXTS: &[&str] = &["SPV_KHR_physical_storage_buffer"];
325
326fn update_capability_requirements_recursive(ctx: &Context, module: SpirvModuleOp) -> bool {
329 let op = module.get_operation();
330 let caps_start = module.get_vce(ctx).capabilities;
331 for cap in caps_start.iter().copied().map(Operand::from) {
332 let min_version = cap.minimum_version();
333 let extensions = cap.required_extensions();
334 let caps = cap.required_capabilities();
335 update_ver_cap_ext_impl(ctx, module, min_version, extensions, caps, op);
336 }
337 module.get_vce(ctx).capabilities != caps_start
338}
339
340fn update_ver_cap_ext(ctx: &Context, module: &mut SpirvModuleOp, node: IRNode) {
341 match node {
342 IRNode::BasicBlock(block) => {
343 let args = block.arguments(ctx);
344 for arg in args {
345 let ty = arg.get_type(ctx).deref(ctx);
346 if let Some(ver_cap_ext) = type_cast(&*ty) {
347 update_ver_cap_ext_ty(ctx, module, ver_cap_ext);
348 }
349 }
350 }
351 IRNode::Operation(op) => {
352 let dyn_op = op.dyn_op(ctx);
353 if let Some(custom) = op_cast::<dyn CustomCapabilitiesOp>(&*dyn_op) {
354 for cap in custom.custom_capabilities(ctx) {
355 module.insert_capability(ctx, cap);
356 }
357 }
358 if let Some(ver_cap_ext) = op_cast(&*dyn_op) {
359 update_ver_cap_ext_op(ctx, module, ver_cap_ext);
360 }
361 for res in op.results(ctx) {
362 let ty = res.get_type(ctx).deref(ctx);
363 if let Some(ver_cap_ext) = type_cast(&*ty) {
364 update_ver_cap_ext_ty(ctx, module, ver_cap_ext);
365 }
366 }
367 }
368 _ => {}
369 }
370}
371
372fn update_ver_cap_ext_op(ctx: &Context, module: &mut SpirvModuleOp, op: &dyn VerCapExtOpInterface) {
373 let min_version = op.min_version(ctx);
374 let extensions = op.required_extensions(ctx);
375 let caps = op.required_capabilities(ctx);
376 let op = op.get_operation();
377 update_ver_cap_ext_impl(ctx, *module, min_version, extensions, caps, op);
378}
379
380fn update_ver_cap_ext_ty(
381 ctx: &Context,
382 module: &mut SpirvModuleOp,
383 ty: &dyn VerCapExtTypeInterface,
384) {
385 let min_version = ty.min_version(ctx);
386 let extensions = ty.required_extensions(ctx);
387 let caps = ty.required_capabilities(ctx);
388 let ty = ty.get_self_handle(ctx);
389 update_ver_cap_ext_impl(ctx, *module, min_version, extensions, caps, ty);
390}
391
392fn update_ver_cap_ext_impl(
393 ctx: &Context,
394 module: SpirvModuleOp,
395 min_version: Option<(u8, u8)>,
396 extensions: Vec<Vec<&'static str>>,
397 caps: Vec<Vec<Capability>>,
398 obj: impl Printable,
399) {
400 let max_ver = ctx
401 .aux_ty::<WgpuCompilationOptions>()
402 .vulkan
403 .max_spirv_version;
404 let extensions = if min_version.is_some_and(|ver| ver <= max_ver) {
405 vec![]
406 } else {
407 extensions
408 };
409
410 for mut cap_set in caps.into_iter().filter(|set| !set.is_empty()) {
411 cap_set.retain(|cap| !EXCLUDED_CAPS.contains(cap));
412 if cap_set.len() == 1 {
413 module.insert_capability(ctx, cap_set[0]);
414 } else if cap_set.iter().any(|cap| module.has_capability(ctx, cap)) {
415 continue;
416 } else if let Some(preferred) = cap_set.iter().find(|cap| PREFERRED_CAPS.contains(cap)) {
417 module.insert_capability(ctx, *preferred);
418 } else {
419 panic!(
420 "Need custom rule for multi-capability node {}, lists capabilities: {:?}",
421 obj.disp(ctx),
422 cap_set
423 );
424 }
425 }
426
427 for ext_set in extensions.into_iter().filter(|set| !set.is_empty()) {
428 if ext_set.len() == 1 {
429 module.insert_extension(ctx, ext_set[0]);
430 } else if ext_set.iter().any(|ext| module.has_extension(ctx, *ext)) {
431 continue;
432 } else if let Some(preferred) = ext_set.iter().find(|cap| PREFERRED_EXTS.contains(cap)) {
433 module.insert_extension(ctx, *preferred);
434 } else {
435 panic!(
436 "Need custom rule for multi-extension node {}, lists extensions: {:?}",
437 obj.disp(ctx),
438 ext_set
439 );
440 }
441 }
442}