use cubecl_core::{self as cubecl, prelude::*};
use cubecl_ir::{
Builtin, ContextExt,
dialect::general::ReadBuiltinOp,
prelude::{
AnalysisManager, Context, MatchRewrite, MatchRewriter, OneResultInterface, Operation,
OperationPtrExt, PassResult, Ptr, Result, Rewriter, SingleBlockRegionInterface, pass_name,
},
rewrite::visit_all_ops_of_type_mut,
types::scalar::PoisonType,
};
use cubecl_opt::passes::uniformity::{DEVICE_UNIFORM, UniformAttr, op_dyn_uniformity};
use hashbrown::HashMap;
use pliron::{
builtin::{
attributes::IntegerAttr,
ops::{ConstantOp, FuncOp, ModuleOp},
types::{IntegerType, Signedness},
},
dict_key,
irbuild::match_rewrite::apply_match_rewrite,
op::Op,
pass::Pass,
r#type::{TypeHandle, Typed},
utils::apint::{APInt, bw},
value::Value,
};
use pliron_spirv::{
attrs::BuiltInAttr,
decorations::DecorationInfo,
ops::{AddressOfOp, GlobalVariableOp, InBoundsAccessChainOp, LoadOp},
types::{MemberDecorationInfo, PointerType, StructType, VectorType},
};
use rspirv::spirv::{BuiltIn, Decoration, MemoryAccess, StorageClass};
use crate::{KernelInfo, decorate_uniform};
dict_key!(BUILTINS_NAME, "_spirv_builtins");
pub struct LowerBuiltinsPass;
#[pass_name]
impl Pass for LowerBuiltinsPass {
fn run(
&mut self,
op: Ptr<Operation>,
ctx: &mut Context,
_analyses: &mut AnalysisManager,
) -> Result<PassResult> {
let mut res = PassResult::default();
let module = op.as_op::<ModuleOp>(ctx).expect("Should run on module");
let module = module.get_body(ctx, 0);
let mut offsets = HashMap::new();
let mut funcs = vec![];
visit_all_ops_of_type_mut::<FuncOp, _>(ctx, &mut funcs, op, |ctx, funcs, func| {
let entry = func.get_entry_block(ctx);
let dummy_ty = PoisonType::get(ctx).to_handle();
let tmp_builtins = AddressOfOp::new(ctx, dummy_ty, BUILTINS_NAME.clone());
tmp_builtins.get_operation().insert_at_front(entry, ctx);
funcs.push((func, tmp_builtins.get_result(ctx)))
});
for (func, struct_val) in funcs.iter().copied() {
let op = func.get_operation();
let mut rewriter = RewriteBuiltins {
struct_val,
offsets: &mut offsets,
};
res.ir_changed |= apply_match_rewrite(ctx, &mut rewriter, Default::default(), op)?;
}
if offsets.is_empty() {
offsets.insert(BuiltIn::LocalInvocationIndex, 0);
}
let mut used_builtins = offsets.into_iter().collect::<Vec<_>>();
used_builtins.sort_by_key(|(_, offset)| *offset);
let mut field_types = vec![];
let mut member_dec = vec![];
let type_dec = vec![DecorationInfo::unit(Decoration::Block)];
for (builtin, id) in used_builtins {
let decoration =
DecorationInfo::new(Decoration::BuiltIn, Box::new(BuiltInAttr::new(builtin)));
field_types.push(builtin_ty(ctx, builtin));
member_dec.push(MemberDecorationInfo::new(id, decoration));
}
let builtins_struct =
StructType::get(ctx, field_types, vec![], member_dec, type_dec).into();
let builtins_struct_ptr =
PointerType::get(ctx, builtins_struct, StorageClass::Input).into();
let builtins_var = GlobalVariableOp::new(
ctx,
builtins_struct,
StorageClass::Input,
BUILTINS_NAME.clone(),
None,
);
builtins_var.get_operation().insert_at_front(module, ctx);
for (_, struct_val) in funcs.iter() {
struct_val.set_type(ctx, builtins_struct_ptr);
}
Ok(res)
}
}
fn builtin_ty(ctx: &Context, builtin: BuiltIn) -> TypeHandle {
let u32 = IntegerType::get(ctx, 32, Signedness::Signless).to_handle();
let dim3 = VectorType::get(ctx, 3, u32).to_handle();
match builtin {
BuiltIn::NumWorkgroups => dim3,
BuiltIn::WorkgroupId => dim3,
BuiltIn::LocalInvocationId => dim3,
BuiltIn::GlobalInvocationId => dim3,
BuiltIn::LocalInvocationIndex => u32,
BuiltIn::SubgroupSize => u32,
BuiltIn::SubgroupId => u32,
BuiltIn::SubgroupLocalInvocationId => u32,
_ => unreachable!("Unsupported builtin"),
}
}
struct RewriteBuiltins<'a> {
struct_val: Value,
offsets: &'a mut HashMap<BuiltIn, u32>,
}
impl RewriteBuiltins<'_> {
fn get_offset(&mut self, scope: &Scope, builtin: BuiltIn) -> Value {
let value = if let Some(&existing) = self.offsets.get(&builtin) {
existing
} else {
let offset = self.offsets.len() as u32;
self.offsets.insert(builtin, offset);
offset
};
const_int32(scope, value)
}
}
impl MatchRewrite for RewriteBuiltins<'_> {
fn r#match(&mut self, ctx: &Context, op: Ptr<Operation>) -> bool {
op.is_op::<ReadBuiltinOp>(ctx)
}
fn rewrite(
&mut self,
ctx: &mut Context,
rewriter: &mut MatchRewriter,
op: Ptr<Operation>,
) -> Result<()> {
let builtin = op.as_op::<ReadBuiltinOp>(ctx).unwrap();
let uniformity = op_dyn_uniformity(ctx, op);
let scope = Scope::from_context_and_inserter(ctx, rewriter);
let ty = builtin.get_result(ctx).get_type(ctx);
let builtin = builtin.builtin(ctx).0;
let value = self.lower_builtin(&scope, ty, builtin, uniformity);
rewriter.replace_operation_with_values(ctx, op, vec![value]);
Ok(())
}
}
#[cube]
fn absolute_pos() -> usize {
cube_pos() * CUBE_DIM as usize + UNIT_POS as usize
}
#[cube]
fn cube_count() -> usize {
CUBE_COUNT_X as usize * CUBE_COUNT_Y as usize * CUBE_COUNT_Z as usize
}
#[cube]
fn cube_pos() -> usize {
CUBE_POS_Z as usize * CUBE_COUNT_Y as usize * CUBE_COUNT_X as usize
+ CUBE_POS_Y as usize * CUBE_COUNT_X as usize
+ CUBE_POS_X as usize
}
#[cube]
fn constant(#[comptime] value: u32) -> u32 {
value
}
impl RewriteBuiltins<'_> {
fn lower_builtin(
&mut self,
scope: &Scope,
ty: TypeHandle,
builtin: Builtin,
uniformity: Option<UniformAttr>,
) -> Value {
let cube_dim = scope.ctx().aux_ty::<KernelInfo>().cube_dim;
match builtin {
Builtin::UnitPos if cube_dim.num_elems() == 1 => {
constant::expand(scope, 0).value(scope)
}
Builtin::UnitPosX if cube_dim.x == 1 => constant::expand(scope, 0).value(scope),
Builtin::UnitPosY if cube_dim.y == 1 => constant::expand(scope, 0).value(scope),
Builtin::UnitPosZ if cube_dim.z == 1 => constant::expand(scope, 0).value(scope),
Builtin::UnitPos => {
self.read_scalar_builtin(scope, ty, BuiltIn::LocalInvocationIndex, uniformity)
}
Builtin::UnitPosX => {
self.read_dim3_builtin(scope, ty, BuiltIn::LocalInvocationId, 0, uniformity)
}
Builtin::UnitPosY => {
self.read_dim3_builtin(scope, ty, BuiltIn::LocalInvocationId, 1, uniformity)
}
Builtin::UnitPosZ => {
self.read_dim3_builtin(scope, ty, BuiltIn::LocalInvocationId, 2, uniformity)
}
Builtin::CubePosCluster => constant::expand(scope, 0).value(scope),
Builtin::CubePosClusterX => constant::expand(scope, 0).value(scope),
Builtin::CubePosClusterY => constant::expand(scope, 0).value(scope),
Builtin::CubePosClusterZ => constant::expand(scope, 0).value(scope),
Builtin::CubePos => cube_pos::expand(scope).value(scope),
Builtin::CubePosX => {
self.read_dim3_builtin(scope, ty, BuiltIn::WorkgroupId, 0, uniformity)
}
Builtin::CubePosY => {
self.read_dim3_builtin(scope, ty, BuiltIn::WorkgroupId, 1, uniformity)
}
Builtin::CubePosZ => {
self.read_dim3_builtin(scope, ty, BuiltIn::WorkgroupId, 2, uniformity)
}
Builtin::CubeDim => constant::expand(scope, cube_dim.num_elems()).value(scope),
Builtin::CubeDimX => constant::expand(scope, cube_dim.x).value(scope),
Builtin::CubeDimY => constant::expand(scope, cube_dim.y).value(scope),
Builtin::CubeDimZ => constant::expand(scope, cube_dim.z).value(scope),
Builtin::CubeClusterDim => constant::expand(scope, 1).value(scope),
Builtin::CubeClusterDimX => constant::expand(scope, 1).value(scope),
Builtin::CubeClusterDimY => constant::expand(scope, 1).value(scope),
Builtin::CubeClusterDimZ => constant::expand(scope, 1).value(scope),
Builtin::CubeCount => cube_count::expand(scope).value(scope),
Builtin::CubeCountX => {
self.read_dim3_builtin(scope, ty, BuiltIn::NumWorkgroups, 0, uniformity)
}
Builtin::CubeCountY => {
self.read_dim3_builtin(scope, ty, BuiltIn::NumWorkgroups, 1, uniformity)
}
Builtin::CubeCountZ => {
self.read_dim3_builtin(scope, ty, BuiltIn::NumWorkgroups, 2, uniformity)
}
Builtin::PlaneDim => {
self.read_scalar_builtin(scope, ty, BuiltIn::SubgroupSize, uniformity)
}
Builtin::PlanePos => {
self.read_scalar_builtin(scope, ty, BuiltIn::SubgroupId, uniformity)
}
Builtin::UnitPosPlane => {
self.read_scalar_builtin(scope, ty, BuiltIn::SubgroupLocalInvocationId, uniformity)
}
Builtin::AbsolutePos => absolute_pos::expand(scope).value(scope),
Builtin::AbsolutePosX => {
self.read_dim3_builtin(scope, ty, BuiltIn::GlobalInvocationId, 0, uniformity)
}
Builtin::AbsolutePosY => {
self.read_dim3_builtin(scope, ty, BuiltIn::GlobalInvocationId, 1, uniformity)
}
Builtin::AbsolutePosZ => {
self.read_dim3_builtin(scope, ty, BuiltIn::GlobalInvocationId, 2, uniformity)
}
}
}
fn read_scalar_builtin(
&mut self,
scope: &Scope,
ty: TypeHandle,
builtin: BuiltIn,
uniformity: Option<UniformAttr>,
) -> Value {
let ctx = scope.ctx_mut();
let offset = self.get_offset(scope, builtin);
let ptr_ty = PointerType::get(ctx, ty, StorageClass::Input).to_handle();
let access_chain = InBoundsAccessChainOp::new(ctx, ptr_ty, self.struct_val, vec![offset]);
decorate_uniform(ctx, access_chain.get_operation(), Some(DEVICE_UNIFORM));
let offset_ptr = scope.register_with_result(&access_chain);
let load = LoadOp::new(ctx, ty, offset_ptr, MemoryAccess::NONE, None);
decorate_uniform(ctx, load.get_operation(), uniformity);
scope.register_with_result(&load)
}
fn read_dim3_builtin(
&mut self,
scope: &Scope,
ty: TypeHandle,
builtin: BuiltIn,
dim: u32,
uniformity: Option<UniformAttr>,
) -> Value {
let ctx = scope.ctx_mut();
let offset = self.get_offset(scope, builtin);
let dim_const = const_int32(scope, dim);
let ptr_ty = PointerType::get(ctx, ty, StorageClass::Input).to_handle();
let access_chain =
InBoundsAccessChainOp::new(ctx, ptr_ty, self.struct_val, vec![offset, dim_const]);
decorate_uniform(ctx, access_chain.get_operation(), Some(DEVICE_UNIFORM));
let offset_ptr = scope.register_with_result(&access_chain);
let load = LoadOp::new(ctx, ty, offset_ptr, MemoryAccess::NONE, None);
decorate_uniform(ctx, load.get_operation(), uniformity);
scope.register_with_result(&load)
}
}
pub(crate) fn const_int32(scope: &Scope, value: u32) -> Value {
let constant = const_op_int32(scope.ctx_mut(), value);
scope.register_with_result(&constant)
}
pub(crate) fn const_op_int32(ctx: &mut Context, value: u32) -> ConstantOp {
let ty = IntegerType::get(ctx, 32, Signedness::Signless);
let value = IntegerAttr::new(ty, APInt::from_u32(value, bw(32)));
ConstantOp::new(ctx, Box::new(value))
}