1use crate::{
2 CollectVerCapExtPass, ConvertArgsPass, PARAMS_NAME, SpirvKernel,
3 lower::LowerOpsSpirvPass,
4 ops::{
5 branch::BranchToSpirvConversionPass,
6 builtin::{BUILTINS_NAME, LowerBuiltinsPass},
7 memory::lower_shared,
8 to_spirv_dialect::ToSpirvDialectPass,
9 },
10 params_storage_class,
11};
12use cubecl_core::{
13 Compiler, WgpuCompilationOptions,
14 ir::{
15 ContextExt, attributes::FuncInterface, features::EnumSet, ident, metadata::Info,
16 rewrite::SimplifyOpsPass,
17 },
18 post_processing::{
19 bitwise::PromoteBitwisePass,
20 checked_io::{CheckedIo, CheckedIoPass},
21 minifloat::{
22 Fp8Container, LowerMinifloatCast, LowerMinifloatCastPass, LowerMinifloatCompare,
23 LowerMinifloatComparePass,
24 },
25 saturating::LowerSaturatingArithmeticPass,
26 unroll::UnrollPass,
27 },
28 prelude::{KernelDefinition, Visibility},
29};
30use cubecl_environment::backtrace::BackTrace;
31use cubecl_ir::{
32 attributes::{ATTR_BUFFER_IO, BufferIOAttr, EntrypointInterface},
33 dialect::{scf::BranchToSCFPass, ssa_matrix::MatrixToSSAPass},
34 prelude::{OperationPtrExt, SingleBlockRegionInterface, SymbolOpInterface},
35 rewrite::{CanonicalizePass, visit_all_ops_of_type_mut},
36 settings::{Dim3, KernelSettings},
37};
38use cubecl_opt::passes::{
39 alloc_shared_memory::AllocateSharedMemoryBlockPass,
40 annotate_buffer_visibility::AnnotateGlobalVisibilityPass,
41 inst_combine::InstCombinePass,
42 mem2reg::Mem2RegPass,
43 sccp::SCCPPass,
44 simple_cse::SimpleCSEPass,
45 sroa::SROAPass,
46 uniformity::{DYNAMICALLY_UNIFORM_ATTR, MarkDynamicallyUniformPass, UniformAttr},
47};
48use cubecl_runtime::compiler::CompilationError;
49use pliron::{
50 basic_block::BasicBlock,
51 builtin::{
52 op_interfaces::OneRegionInterface,
53 ops::{FuncOp, ModuleOp},
54 },
55 context::{Context, Ptr},
56 identifier::Identifier,
57 irbuild::{
58 inserter::BlockInsertionPoint,
59 listener::DummyListener,
60 rewriter::{IRRewriter, Rewriter},
61 },
62 op::{Op, op_cast},
63 operation::{Operation, verify_operation},
64 opts::{dce::DCEPass, simplify_cfg::SimplifyCFGPass},
65 pass::{AnalysisManager, NestedOpsPass, OpPass, PMConfig, Pass, Passes},
66};
67use pliron_spirv::{
68 PlironBuilder, ToSpirvOp,
69 attrs::VerCapExtAttr,
70 decorations::{DecoratableOp, set_decoration_uniform, set_decoration_uniform_id},
71 ops::{EntryPointOp, ExecutionModeOp, SpirvModuleOp},
72};
73use rspirv::{
74 binary::Assemble,
75 dr::Module,
76 spirv::{
77 AddressingModel, Capability, ExecutionMode, ExecutionModel, MemoryModel, Scope,
78 StorageClass,
79 },
80};
81use std::{fmt::Debug, sync::Arc};
82
83pub struct KernelInfo {
84 pub cube_dim: Dim3,
85}
86
87#[derive(Clone, Copy, Default)]
88pub struct SpirvCompiler;
89
90impl Compiler for SpirvCompiler {
91 type Representation = SpirvKernel;
92 type CompilationOptions = WgpuCompilationOptions;
93
94 fn buffer_io(repr: &Self::Representation) -> Option<Vec<BufferIOAttr>> {
95 repr.io.clone()
96 }
97
98 fn compile(
99 &mut self,
100 value: KernelDefinition,
101 compilation_options: &Self::CompilationOptions,
102 ) -> Result<Self::Representation, CompilationError> {
103 let errors = value.body.pop_errors();
104 if !errors.is_empty() {
105 let mut reason = "Can't compile spirv kernel".to_string();
106 for error in errors {
107 reason += error.as_str();
108 reason += "\n";
109 }
110
111 return Err(CompilationError::Validation {
112 reason,
113 backtrace: BackTrace::capture(),
114 });
115 }
116
117 #[cfg(feature = "pliron-dump")]
118 let ir_printing_dir = kernel_dir_name(&value.settings.kernel_name);
119
120 let entry_func = value.body.state().entry_func;
121 let module = value.body.state().module;
122
123 let mut ctx = value.body.into_context().expect("Should be unique");
124 ctx.set_aux_ty::<Info>(value.info);
125 ctx.set_aux_ty::<WgpuCompilationOptions>(*compilation_options);
126 ctx.set_aux_ty::<KernelInfo>(KernelInfo {
127 cube_dim: value.settings.cube_dim,
128 });
129
130 let (module, bindings, io, shared_size) = self.compile_kernel(
131 &mut ctx,
132 module,
133 entry_func,
134 value.settings.clone(),
135 #[cfg(feature = "pliron-dump")]
136 ir_printing_dir,
137 )?;
138
139 let info_visibility = Visibility::Read;
140 let immediate_size = match params_storage_class(&ctx, bindings.len()) {
141 StorageClass::PushConstant => Some((bindings.len() + 1) * size_of::<u64>()),
142 _ => None,
143 };
144
145 let kernel = SpirvKernel {
146 assembled_module: module.assemble(),
147 module: Some(Arc::new(module)),
148 bindings,
149 io: Some(io),
150 shared_size,
151 immediate_size,
152 info_visibility,
153 };
154
155 #[cfg(feature = "pliron-dump")]
156 dump_spirv(&kernel, &value.settings.kernel_name);
157
158 Ok(kernel)
159 }
160
161 fn extension(&self) -> &'static str {
162 "spv"
163 }
164
165 fn lang_tag(&self) -> &'static str {
166 "spirv"
167 }
168}
169
170impl Debug for SpirvCompiler {
171 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
172 f.write_str("spirv")
173 }
174}
175
176impl SpirvCompiler {
177 pub fn compile_kernel(
178 &mut self,
179 ctx: &mut Context,
180 module: ModuleOp,
181 entry_func: FuncOp,
182 settings: KernelSettings,
183 #[cfg(feature = "pliron-dump")] ir_printing_dir: Option<std::path::PathBuf>,
184 ) -> Result<(Module, Vec<Visibility>, Vec<BufferIOAttr>, usize), CompilationError> {
185 let entry = entry_func.get_entry_block(ctx);
186 let comp_opts = ctx.aux_ty::<WgpuCompilationOptions>();
187 let module_op = module.get_operation();
188
189 #[cfg(feature = "pliron-dump")]
190 if let Some(print_dir) = &ir_printing_dir {
191 use pliron::printable::Printable;
192 let str = std::format!("{}", module_op.disp(ctx));
193 std::fs::write(print_dir.join("initial.plir"), &str).unwrap();
194 }
195
196 verify_operation(module.get_operation(), ctx)?;
197
198 let config = PMConfig {
199 print_after_all: cfg!(feature = "pliron-dump"),
200 #[cfg(feature = "pliron-dump")]
201 ir_printing_dir,
202 ..Default::default()
203 };
204
205 let mut analyses = AnalysisManager::default();
206 analyses.set_config(config);
207
208 let mut passes = OpPass::<ModuleOp, Passes>::default();
209
210 let mut func_passes = OpPass::<FuncOp, Passes>::default();
211 func_passes.add_pass(SROAPass);
212 func_passes.add_pass(CheckedIoPass::new(CheckedIo::new(
213 settings.execution_mode,
214 settings.kernel_name,
215 )));
216 func_passes.add_pass(UnrollPass::new(comp_opts.vulkan.max_vector_size));
217 func_passes.add_pass(AllocateSharedMemoryBlockPass);
218 func_passes.add_pass(LowerSaturatingArithmeticPass::default());
219 let native_fp8 = match comp_opts.vulkan.supports_float8 {
223 true => {
224 cubecl_core::ir::types::Fp8Format::E4M3 | cubecl_core::ir::types::Fp8Format::E5M2
225 }
226 false => EnumSet::empty(),
227 };
228 func_passes.add_pass(LowerMinifloatCastPass::new(LowerMinifloatCast::new(
229 native_fp8,
230 Fp8Container::Bytes,
231 )));
232 func_passes.add_pass(LowerMinifloatComparePass::new(LowerMinifloatCompare::new(
233 Fp8Container::Bytes,
234 )));
235 func_passes.add_pass(BranchToSCFPass::default());
236 func_passes.add_pass(MatrixToSSAPass::default());
237
238 passes.add_pass(NestedOpsPass::new(func_passes));
239 passes.add_pass(MarkDynamicallyUniformPass);
240 passes.add_pass(LowerBuiltinsPass);
241
242 let mut func_passes = OpPass::<FuncOp, Passes>::default();
243 func_passes.add_pass(SCCPPass);
244 func_passes.add_pass(InstCombinePass::default());
245 func_passes.add_pass(SimpleCSEPass::without_memory());
246 func_passes.add_pass(SimplifyOpsPass::default());
247 func_passes.add_pass(PromoteBitwisePass);
248 func_passes.add_pass(LowerOpsSpirvPass::default());
249 func_passes.add_pass(DCEPass);
250 func_passes.add_pass(SROAPass);
251
252 func_passes.add_pass(Mem2RegPass);
253
254 func_passes.add_pass(SROAPass);
255 func_passes.add_pass(SCCPPass);
256 func_passes.add_pass(SimpleCSEPass::with_memory());
257 func_passes.add_pass(DCEPass);
258 func_passes.add_pass(CanonicalizePass::default());
259
260 passes.add_pass(NestedOpsPass::new(func_passes));
261 passes.add_pass(AnnotateGlobalVisibilityPass);
262
263 passes.run(module_op, ctx, &mut analyses)?;
264
265 let bindings = (0..entry.deref(ctx).get_num_arguments()).map(|i| {
266 let io = entry_func.get_arg_attr::<BufferIOAttr>(ctx, i, &ATTR_BUFFER_IO);
267 match io.expect("Should have IO attr").is_writable() {
268 false => Visibility::Read,
269 true => Visibility::ReadWrite,
270 }
271 });
272 let bindings: Vec<Visibility> = bindings.collect();
273 let io = cubecl_core::ir::attributes::buffer_io_by_position(ctx, entry_func)
276 .into_iter()
277 .collect::<Vec<BufferIOAttr>>();
278
279 verify_operation(module_op, ctx)?;
280
281 let mut passes = OpPass::<ModuleOp, Passes>::default();
282 let mut func_passes = OpPass::<FuncOp, Passes>::default();
283
284 func_passes.add_pass(BranchToSpirvConversionPass::default());
285 func_passes.add_pass(Mem2RegPass);
286 func_passes.add_pass(DCEPass);
287 func_passes.add_pass(SCCPPass);
288 func_passes.add_pass(SimplifyCFGPass);
289 func_passes.add_pass(DCEPass);
290
291 passes.add_pass(NestedOpsPass::new(func_passes));
292 passes.run(module_op, ctx, &mut analyses)?;
293
294 verify_operation(module_op, ctx)?;
295
296 let spirv_module = insert_spirv_module(ctx, module);
297 let spirv_module_op = spirv_module.get_operation();
298
299 let mut passes = OpPass::<SpirvModuleOp, Passes>::default();
300 let mut func_passes = OpPass::<FuncOp, Passes>::default();
301
302 func_passes.add_pass(DCEPass);
303 func_passes.add_pass(ToSpirvDialectPass::default());
304
305 passes.add_pass(ConvertArgsPass);
306 passes.add_pass(NestedOpsPass::new(func_passes));
307
308 passes.run(spirv_module_op, ctx, &mut analyses)?;
312
313 let (shared_size, shared_args) = lower_shared(ctx, spirv_module);
314 declare_entry_point(ctx, spirv_module, shared_args);
315
316 OpPass::<SpirvModuleOp, CollectVerCapExtPass>::default()
318 .run(spirv_module_op, ctx, &mut analyses)
319 .unwrap();
320
321 let mut builder = PlironBuilder::default();
326 spirv_module.to_spirv(ctx, &mut builder)?;
327 let module = builder.module();
328
329 Ok((module, bindings, io, shared_size))
330 }
331}
332
333fn insert_spirv_module(ctx: &mut Context, module: ModuleOp) -> SpirvModuleOp {
334 let mut rewriter = IRRewriter::<DummyListener>::default();
335 let comp_opts = ctx.aux_ty::<WgpuCompilationOptions>().vulkan;
336
337 let spirv_module = SpirvModuleOp::new(
338 ctx,
339 ident("kernel"),
340 AddressingModel::PhysicalStorageBuffer64,
341 MemoryModel::Vulkan,
342 );
343 rewriter.inline_region(
344 ctx,
345 module.get_region(ctx),
346 BlockInsertionPoint::AtRegionStart(spirv_module.get_region(ctx)),
347 );
348 let module_body = BasicBlock::new(ctx, None, vec![]);
349 module_body.insert_at_front(module.get_region(ctx), ctx);
350 spirv_module
351 .get_operation()
352 .insert_at_front(module_body, ctx);
353 let vce = VerCapExtAttr::new(
354 comp_opts.max_spirv_version,
355 vec![Capability::Shader],
356 vec![],
357 );
358 spirv_module.set_attr_spirv_module_vce(ctx, vce);
359 spirv_module
360}
361
362fn declare_entry_point(ctx: &mut Context, module: SpirvModuleOp, shared_args: Vec<Identifier>) {
363 let op = module.get_operation();
364 visit_all_ops_of_type_mut::<FuncOp, _>(
365 ctx,
366 &mut (module, shared_args),
367 op,
368 |ctx, (module, shared_args), func| {
369 let Some(entry) = func.get_entrypoint_abi(ctx) else {
370 return;
371 };
372 let block = module.get_body(ctx, 0);
373 let func_name = func.get_symbol_name(ctx);
374 let mut interface = vec![PARAMS_NAME.clone(), BUILTINS_NAME.clone()];
375 interface.extend(shared_args.clone());
376 let entry_point = EntryPointOp::new(
377 ctx,
378 ExecutionModel::GLCompute,
379 func_name.clone(),
380 func_name.to_string(),
381 interface,
382 );
383 entry_point.get_operation().insert_at_front(block, ctx);
384 let (x, y, z) = entry.cube_dim.into();
385 let execution_mode =
386 ExecutionModeOp::new(ctx, func_name, ExecutionMode::LocalSize, vec![x, y, z]);
387 execution_mode.get_operation().insert_at_front(block, ctx);
388 },
389 );
390}
391
392pub(crate) fn decorate_uniform(ctx: &Context, op: Ptr<Operation>, uniformity: Option<UniformAttr>) {
393 let Some(uniformity) = uniformity else {
394 return;
395 };
396 let props = ctx.aux_ty::<WgpuCompilationOptions>().vulkan;
397 if let Some(can_decorate) = op_cast::<dyn DecoratableOp>(&*op.dyn_op(ctx)) {
398 if props.max_spirv_version >= (1, 4) {
399 let scope = match uniformity {
402 UniformAttr::Device | UniformAttr::Cube => Scope::Workgroup,
403 UniformAttr::Plane => Scope::Subgroup,
404 };
405 set_decoration_uniform_id(can_decorate, ctx, scope.into());
406 } else {
407 set_decoration_uniform(can_decorate, ctx);
408 }
409 } else {
410 op.set_attr(ctx, &DYNAMICALLY_UNIFORM_ATTR, uniformity);
411 }
412}
413
414#[cfg(feature = "pliron-dump")]
415pub fn kernel_dir_name(name: &str) -> Option<std::path::PathBuf> {
416 if let Ok(dir) = std::env::var("CUBECL_DEBUG_PLIRON") {
417 let path = sanitize_filename::sanitize_with_options(
418 name,
419 sanitize_filename::Options {
420 replacement: "_",
421 ..Default::default()
422 },
423 );
424 let dir = std::path::PathBuf::from(dir).join(&path);
425 std::fs::create_dir_all(&dir).unwrap();
426 Some(dir)
427 } else {
428 None
429 }
430}
431
432#[cfg(feature = "pliron-dump")]
433pub(crate) fn dump_spirv(repr: &SpirvKernel, name: &str) {
434 use std::fs;
435
436 if let Some(dir) = kernel_dir_name(name) {
437 let kernel = &repr.assembled_module;
438 let kernel = kernel
439 .iter()
440 .flat_map(|it| it.to_le_bytes())
441 .collect::<Vec<_>>();
442 let kernel_path = dir.join("module.spv");
443 fs::write(kernel_path, kernel).unwrap();
444 }
445}