Skip to main content

cubecl_spirv/
compiler.rs

1use crate::{
2    CollectVerCapExtPass, ConvertArgsPass, PARAMS_NAME, SpirvKernel,
3    lower::LowerOpsSpirvPass,
4    ops::{
5        branch::BranchToSpirvConversionPass,
6        builtin::{BUILTINS_NAME, LowerBuiltinsPass},
7        memory::lower_shared,
8        to_spirv_dialect::ToSpirvDialectPass,
9    },
10    params_storage_class,
11};
12use cubecl_core::{
13    Compiler, WgpuCompilationOptions,
14    ir::{
15        ContextExt, attributes::FuncInterface, features::EnumSet, ident, metadata::Info,
16        rewrite::SimplifyOpsPass,
17    },
18    post_processing::{
19        bitwise::PromoteBitwisePass,
20        checked_io::{CheckedIo, CheckedIoPass},
21        minifloat::{
22            Fp8Container, LowerMinifloatCast, LowerMinifloatCastPass, LowerMinifloatCompare,
23            LowerMinifloatComparePass,
24        },
25        saturating::LowerSaturatingArithmeticPass,
26        unroll::UnrollPass,
27    },
28    prelude::{KernelDefinition, Visibility},
29};
30use cubecl_environment::backtrace::BackTrace;
31use cubecl_ir::{
32    attributes::{ATTR_BUFFER_IO, BufferIOAttr, EntrypointInterface},
33    dialect::{scf::BranchToSCFPass, ssa_matrix::MatrixToSSAPass},
34    prelude::{OperationPtrExt, SingleBlockRegionInterface, SymbolOpInterface},
35    rewrite::{CanonicalizePass, visit_all_ops_of_type_mut},
36    settings::{Dim3, KernelSettings},
37};
38use cubecl_opt::passes::{
39    alloc_shared_memory::AllocateSharedMemoryBlockPass,
40    annotate_buffer_visibility::AnnotateGlobalVisibilityPass,
41    inst_combine::InstCombinePass,
42    mem2reg::Mem2RegPass,
43    sccp::SCCPPass,
44    simple_cse::SimpleCSEPass,
45    sroa::SROAPass,
46    uniformity::{DYNAMICALLY_UNIFORM_ATTR, MarkDynamicallyUniformPass, UniformAttr},
47};
48use cubecl_runtime::compiler::CompilationError;
49use pliron::{
50    basic_block::BasicBlock,
51    builtin::{
52        op_interfaces::OneRegionInterface,
53        ops::{FuncOp, ModuleOp},
54    },
55    context::{Context, Ptr},
56    identifier::Identifier,
57    irbuild::{
58        inserter::BlockInsertionPoint,
59        listener::DummyListener,
60        rewriter::{IRRewriter, Rewriter},
61    },
62    op::{Op, op_cast},
63    operation::{Operation, verify_operation},
64    opts::{dce::DCEPass, simplify_cfg::SimplifyCFGPass},
65    pass::{AnalysisManager, NestedOpsPass, OpPass, PMConfig, Pass, Passes},
66};
67use pliron_spirv::{
68    PlironBuilder, ToSpirvOp,
69    attrs::VerCapExtAttr,
70    decorations::{DecoratableOp, set_decoration_uniform, set_decoration_uniform_id},
71    ops::{EntryPointOp, ExecutionModeOp, SpirvModuleOp},
72};
73use rspirv::{
74    binary::Assemble,
75    dr::Module,
76    spirv::{
77        AddressingModel, Capability, ExecutionMode, ExecutionModel, MemoryModel, Scope,
78        StorageClass,
79    },
80};
81use std::{fmt::Debug, sync::Arc};
82
83pub struct KernelInfo {
84    pub cube_dim: Dim3,
85}
86
87#[derive(Clone, Copy, Default)]
88pub struct SpirvCompiler;
89
90impl Compiler for SpirvCompiler {
91    type Representation = SpirvKernel;
92    type CompilationOptions = WgpuCompilationOptions;
93
94    fn buffer_io(repr: &Self::Representation) -> Option<Vec<BufferIOAttr>> {
95        repr.io.clone()
96    }
97
98    fn compile(
99        &mut self,
100        value: KernelDefinition,
101        compilation_options: &Self::CompilationOptions,
102    ) -> Result<Self::Representation, CompilationError> {
103        let errors = value.body.pop_errors();
104        if !errors.is_empty() {
105            let mut reason = "Can't compile spirv kernel".to_string();
106            for error in errors {
107                reason += error.as_str();
108                reason += "\n";
109            }
110
111            return Err(CompilationError::Validation {
112                reason,
113                backtrace: BackTrace::capture(),
114            });
115        }
116
117        #[cfg(feature = "pliron-dump")]
118        let ir_printing_dir = kernel_dir_name(&value.settings.kernel_name);
119
120        let entry_func = value.body.state().entry_func;
121        let module = value.body.state().module;
122
123        let mut ctx = value.body.into_context().expect("Should be unique");
124        ctx.set_aux_ty::<Info>(value.info);
125        ctx.set_aux_ty::<WgpuCompilationOptions>(*compilation_options);
126        ctx.set_aux_ty::<KernelInfo>(KernelInfo {
127            cube_dim: value.settings.cube_dim,
128        });
129
130        let (module, bindings, io, shared_size) = self.compile_kernel(
131            &mut ctx,
132            module,
133            entry_func,
134            value.settings.clone(),
135            #[cfg(feature = "pliron-dump")]
136            ir_printing_dir,
137        )?;
138
139        let info_visibility = Visibility::Read;
140        let immediate_size = match params_storage_class(&ctx, bindings.len()) {
141            StorageClass::PushConstant => Some((bindings.len() + 1) * size_of::<u64>()),
142            _ => None,
143        };
144
145        let kernel = SpirvKernel {
146            assembled_module: module.assemble(),
147            module: Some(Arc::new(module)),
148            bindings,
149            io: Some(io),
150            shared_size,
151            immediate_size,
152            info_visibility,
153        };
154
155        #[cfg(feature = "pliron-dump")]
156        dump_spirv(&kernel, &value.settings.kernel_name);
157
158        Ok(kernel)
159    }
160
161    fn extension(&self) -> &'static str {
162        "spv"
163    }
164
165    fn lang_tag(&self) -> &'static str {
166        "spirv"
167    }
168}
169
170impl Debug for SpirvCompiler {
171    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
172        f.write_str("spirv")
173    }
174}
175
176impl SpirvCompiler {
177    pub fn compile_kernel(
178        &mut self,
179        ctx: &mut Context,
180        module: ModuleOp,
181        entry_func: FuncOp,
182        settings: KernelSettings,
183        #[cfg(feature = "pliron-dump")] ir_printing_dir: Option<std::path::PathBuf>,
184    ) -> Result<(Module, Vec<Visibility>, Vec<BufferIOAttr>, usize), CompilationError> {
185        let entry = entry_func.get_entry_block(ctx);
186        let comp_opts = ctx.aux_ty::<WgpuCompilationOptions>();
187        let module_op = module.get_operation();
188
189        #[cfg(feature = "pliron-dump")]
190        if let Some(print_dir) = &ir_printing_dir {
191            use pliron::printable::Printable;
192            let str = std::format!("{}", module_op.disp(ctx));
193            std::fs::write(print_dir.join("initial.plir"), &str).unwrap();
194        }
195
196        verify_operation(module.get_operation(), ctx)?;
197
198        let config = PMConfig {
199            print_after_all: cfg!(feature = "pliron-dump"),
200            #[cfg(feature = "pliron-dump")]
201            ir_printing_dir,
202            ..Default::default()
203        };
204
205        let mut analyses = AnalysisManager::default();
206        analyses.set_config(config);
207
208        let mut passes = OpPass::<ModuleOp, Passes>::default();
209
210        let mut func_passes = OpPass::<FuncOp, Passes>::default();
211        func_passes.add_pass(SROAPass);
212        func_passes.add_pass(CheckedIoPass::new(CheckedIo::new(
213            settings.execution_mode,
214            settings.kernel_name,
215        )));
216        func_passes.add_pass(UnrollPass::new(comp_opts.vulkan.max_vector_size));
217        func_passes.add_pass(AllocateSharedMemoryBlockPass);
218        func_passes.add_pass(LowerSaturatingArithmeticPass::default());
219        // The two the extension actually provides, named rather than `EnumSet::all()`: `ue8m0`
220        // is also an `Fp8Format`, and `VK_EXT_shader_float8` does not cover it. Claiming it here
221        // would emit a conversion the driver has no encoding for.
222        let native_fp8 = match comp_opts.vulkan.supports_float8 {
223            true => {
224                cubecl_core::ir::types::Fp8Format::E4M3 | cubecl_core::ir::types::Fp8Format::E5M2
225            }
226            false => EnumSet::empty(),
227        };
228        func_passes.add_pass(LowerMinifloatCastPass::new(LowerMinifloatCast::new(
229            native_fp8,
230            Fp8Container::Bytes,
231        )));
232        func_passes.add_pass(LowerMinifloatComparePass::new(LowerMinifloatCompare::new(
233            Fp8Container::Bytes,
234        )));
235        func_passes.add_pass(BranchToSCFPass::default());
236        func_passes.add_pass(MatrixToSSAPass::default());
237
238        passes.add_pass(NestedOpsPass::new(func_passes));
239        passes.add_pass(MarkDynamicallyUniformPass);
240        passes.add_pass(LowerBuiltinsPass);
241
242        let mut func_passes = OpPass::<FuncOp, Passes>::default();
243        func_passes.add_pass(SCCPPass);
244        func_passes.add_pass(InstCombinePass::default());
245        func_passes.add_pass(SimpleCSEPass::without_memory());
246        func_passes.add_pass(SimplifyOpsPass::default());
247        func_passes.add_pass(PromoteBitwisePass);
248        func_passes.add_pass(LowerOpsSpirvPass::default());
249        func_passes.add_pass(DCEPass);
250        func_passes.add_pass(SROAPass);
251
252        func_passes.add_pass(Mem2RegPass);
253
254        func_passes.add_pass(SROAPass);
255        func_passes.add_pass(SCCPPass);
256        func_passes.add_pass(SimpleCSEPass::with_memory());
257        func_passes.add_pass(DCEPass);
258        func_passes.add_pass(CanonicalizePass::default());
259
260        passes.add_pass(NestedOpsPass::new(func_passes));
261        passes.add_pass(AnnotateGlobalVisibilityPass);
262
263        passes.run(module_op, ctx, &mut analyses)?;
264
265        let bindings = (0..entry.deref(ctx).get_num_arguments()).map(|i| {
266            let io = entry_func.get_arg_attr::<BufferIOAttr>(ctx, i, &ATTR_BUFFER_IO);
267            match io.expect("Should have IO attr").is_writable() {
268                false => Visibility::Read,
269                true => Visibility::ReadWrite,
270            }
271        });
272        let bindings: Vec<Visibility> = bindings.collect();
273        // The four-state answer, by buffer position, before anything widens
274        // or collapses it: what the launch path's taint bookkeeping consumes.
275        let io = cubecl_core::ir::attributes::buffer_io_by_position(ctx, entry_func)
276            .into_iter()
277            .collect::<Vec<BufferIOAttr>>();
278
279        verify_operation(module_op, ctx)?;
280
281        let mut passes = OpPass::<ModuleOp, Passes>::default();
282        let mut func_passes = OpPass::<FuncOp, Passes>::default();
283
284        func_passes.add_pass(BranchToSpirvConversionPass::default());
285        func_passes.add_pass(Mem2RegPass);
286        func_passes.add_pass(DCEPass);
287        func_passes.add_pass(SCCPPass);
288        func_passes.add_pass(SimplifyCFGPass);
289        func_passes.add_pass(DCEPass);
290
291        passes.add_pass(NestedOpsPass::new(func_passes));
292        passes.run(module_op, ctx, &mut analyses)?;
293
294        verify_operation(module_op, ctx)?;
295
296        let spirv_module = insert_spirv_module(ctx, module);
297        let spirv_module_op = spirv_module.get_operation();
298
299        let mut passes = OpPass::<SpirvModuleOp, Passes>::default();
300        let mut func_passes = OpPass::<FuncOp, Passes>::default();
301
302        func_passes.add_pass(DCEPass);
303        func_passes.add_pass(ToSpirvDialectPass::default());
304
305        passes.add_pass(ConvertArgsPass);
306        passes.add_pass(NestedOpsPass::new(func_passes));
307
308        // The conversion pass reports ops it must not compile (e.g.
309        // cube.poison) as errors, not bugs; surface them as a compilation
310        // error instead of panicking the device thread.
311        passes.run(spirv_module_op, ctx, &mut analyses)?;
312
313        let (shared_size, shared_args) = lower_shared(ctx, spirv_module);
314        declare_entry_point(ctx, spirv_module, shared_args);
315
316        // Make sure this is the last pass so it catches all ops
317        OpPass::<SpirvModuleOp, CollectVerCapExtPass>::default()
318            .run(spirv_module_op, ctx, &mut analyses)
319            .unwrap();
320
321        // Something weird with the validation rules, can't be bothered to debug for the MVP.
322        // Try to figure this out later.
323        // verify_operation(module_op, ctx).expect("Failed to verify after passes");
324
325        let mut builder = PlironBuilder::default();
326        spirv_module.to_spirv(ctx, &mut builder)?;
327        let module = builder.module();
328
329        Ok((module, bindings, io, shared_size))
330    }
331}
332
333fn insert_spirv_module(ctx: &mut Context, module: ModuleOp) -> SpirvModuleOp {
334    let mut rewriter = IRRewriter::<DummyListener>::default();
335    let comp_opts = ctx.aux_ty::<WgpuCompilationOptions>().vulkan;
336
337    let spirv_module = SpirvModuleOp::new(
338        ctx,
339        ident("kernel"),
340        AddressingModel::PhysicalStorageBuffer64,
341        MemoryModel::Vulkan,
342    );
343    rewriter.inline_region(
344        ctx,
345        module.get_region(ctx),
346        BlockInsertionPoint::AtRegionStart(spirv_module.get_region(ctx)),
347    );
348    let module_body = BasicBlock::new(ctx, None, vec![]);
349    module_body.insert_at_front(module.get_region(ctx), ctx);
350    spirv_module
351        .get_operation()
352        .insert_at_front(module_body, ctx);
353    let vce = VerCapExtAttr::new(
354        comp_opts.max_spirv_version,
355        vec![Capability::Shader],
356        vec![],
357    );
358    spirv_module.set_attr_spirv_module_vce(ctx, vce);
359    spirv_module
360}
361
362fn declare_entry_point(ctx: &mut Context, module: SpirvModuleOp, shared_args: Vec<Identifier>) {
363    let op = module.get_operation();
364    visit_all_ops_of_type_mut::<FuncOp, _>(
365        ctx,
366        &mut (module, shared_args),
367        op,
368        |ctx, (module, shared_args), func| {
369            let Some(entry) = func.get_entrypoint_abi(ctx) else {
370                return;
371            };
372            let block = module.get_body(ctx, 0);
373            let func_name = func.get_symbol_name(ctx);
374            let mut interface = vec![PARAMS_NAME.clone(), BUILTINS_NAME.clone()];
375            interface.extend(shared_args.clone());
376            let entry_point = EntryPointOp::new(
377                ctx,
378                ExecutionModel::GLCompute,
379                func_name.clone(),
380                func_name.to_string(),
381                interface,
382            );
383            entry_point.get_operation().insert_at_front(block, ctx);
384            let (x, y, z) = entry.cube_dim.into();
385            let execution_mode =
386                ExecutionModeOp::new(ctx, func_name, ExecutionMode::LocalSize, vec![x, y, z]);
387            execution_mode.get_operation().insert_at_front(block, ctx);
388        },
389    );
390}
391
392pub(crate) fn decorate_uniform(ctx: &Context, op: Ptr<Operation>, uniformity: Option<UniformAttr>) {
393    let Some(uniformity) = uniformity else {
394        return;
395    };
396    let props = ctx.aux_ty::<WgpuCompilationOptions>().vulkan;
397    if let Some(can_decorate) = op_cast::<dyn DecoratableOp>(&*op.dyn_op(ctx)) {
398        if props.max_spirv_version >= (1, 4) {
399            // The spec theoretically allows any but Vulkan has a hardcoded set of just `Workgroup`
400            // and `Subgroup` for the allowed values
401            let scope = match uniformity {
402                UniformAttr::Device | UniformAttr::Cube => Scope::Workgroup,
403                UniformAttr::Plane => Scope::Subgroup,
404            };
405            set_decoration_uniform_id(can_decorate, ctx, scope.into());
406        } else {
407            set_decoration_uniform(can_decorate, ctx);
408        }
409    } else {
410        op.set_attr(ctx, &DYNAMICALLY_UNIFORM_ATTR, uniformity);
411    }
412}
413
414#[cfg(feature = "pliron-dump")]
415pub fn kernel_dir_name(name: &str) -> Option<std::path::PathBuf> {
416    if let Ok(dir) = std::env::var("CUBECL_DEBUG_PLIRON") {
417        let path = sanitize_filename::sanitize_with_options(
418            name,
419            sanitize_filename::Options {
420                replacement: "_",
421                ..Default::default()
422            },
423        );
424        let dir = std::path::PathBuf::from(dir).join(&path);
425        std::fs::create_dir_all(&dir).unwrap();
426        Some(dir)
427    } else {
428        None
429    }
430}
431
432#[cfg(feature = "pliron-dump")]
433pub(crate) fn dump_spirv(repr: &SpirvKernel, name: &str) {
434    use std::fs;
435
436    if let Some(dir) = kernel_dir_name(name) {
437        let kernel = &repr.assembled_module;
438        let kernel = kernel
439            .iter()
440            .flat_map(|it| it.to_le_bytes())
441            .collect::<Vec<_>>();
442        let kernel_path = dir.join("module.spv");
443        fs::write(kernel_path, kernel).unwrap();
444    }
445}