cubecl-spirv 0.11.0

SPIR-V compiler for CubeCL
use cubecl_core::{
    self as cubecl, WgpuCompilationOptions, frontend::polyfills::bitwise::*, num_traits::Zero,
    prelude::*,
};
use cubecl_ir::{
    dialect::bitwise::{self, *},
    interfaces::TypedExt,
    prelude::*,
};
use cubecl_opt::passes::uniformity::op_dyn_uniformity;
use pliron::{
    builtin::{
        attributes::{IntegerAttr, VecAttr},
        ops::ConstantOp,
        types::IntegerType,
    },
    utils::apint::{APInt, bw},
};
use pliron_spirv::{
    attrs::CompositeAttr,
    ext::gl,
    ops::{self, IAddOp, ISubOp, ShiftRightArithmeticOp, ShiftRightLogicalOp},
    types::VectorType,
};

use crate::{
    decorate_uniform,
    lower::lower_unop,
    ops::{
        base::{binop_to_spirv_dialect, unop_to_spirv_dialect},
        to_spirv_dialect::ToSpirvDialectOp,
    },
    types::ty_to_spirv_dialect,
};

define_size!(N);

binop_to_spirv_dialect!(bitwise::BitwiseAndOp => ops::BitwiseAndOp);
binop_to_spirv_dialect!(bitwise::BitwiseOrOp => ops::BitwiseOrOp);
binop_to_spirv_dialect!(bitwise::BitwiseXorOp => ops::BitwiseXorOp);
binop_to_spirv_dialect!(bitwise::ShiftLeftOp => ops::ShiftLeftLogicalOp);

unop_to_spirv_dialect!(bitwise::BitwiseNotOp => ops::NotOp);
unop_to_spirv_dialect!(bitwise::CountOnesOp => ops::BitCountOp);
unop_to_spirv_dialect!(bitwise::ReverseBitsOp => ops::BitReverseOp);

pub fn const_int_maybe_vec(scope: &Scope, value: u32, ref_ty: TypeHandle) -> Value {
    let ty = ref_ty.deref(scope.ctx());
    if let Some(vector) = ty.downcast_ref::<VectorType>() {
        let elem_ty =
            TypedHandle::<IntegerType>::from_handle(vector.element_type, scope.ctx()).unwrap();
        let width = bw(elem_ty.deref(scope.ctx()).width() as usize);
        let value = IntegerAttr::new(elem_ty, APInt::from_u32(value, width));
        let values = VecAttr(vec![Box::new(value); vector.count as usize]);
        let value = CompositeAttr::new(values, ref_ty);
        scope.register_with_result(&ConstantOp::new(scope.ctx_mut(), Box::new(value)))
    } else {
        let ref_ty = TypedHandle::<IntegerType>::from_handle(ref_ty, scope.ctx()).unwrap();
        let width = bw(ref_ty.deref(scope.ctx()).width() as usize);
        let value = IntegerAttr::new(ref_ty, APInt::from_u32(value, width));

        scope.register_with_result(&ConstantOp::new(scope.ctx_mut(), Box::new(value)))
    }
}

#[op_interface_impl]
impl ToSpirvDialectOp for bitwise::ShiftRightOp {
    fn to_spirv_dialect(
        &self,
        ctx: &mut Context,
        rewriter: &mut DialectConversionRewriter,
        _operands_info: &OperandsInfo,
    ) -> Result<()> {
        let op = self.get_operation();
        let uniformity = op_dyn_uniformity(ctx, op);
        let lhs = op.operand(ctx, 0);
        let rhs = op.operand(ctx, 1);
        let out_ty = ty_to_spirv_dialect(ctx, self.get_result(ctx).get_type(ctx));
        if self.get_result(ctx).scalar_ty(ctx).is_signed_int(ctx) {
            let new_op = ShiftRightArithmeticOp::new(ctx, out_ty, lhs, rhs);
            rewriter.append_op(ctx, &new_op);
            rewriter.replace_operation(ctx, op, new_op.get_operation());
            decorate_uniform(ctx, new_op.get_operation(), uniformity);
        } else {
            let new_op = ShiftRightLogicalOp::new(ctx, out_ty, lhs, rhs);
            rewriter.append_op(ctx, &new_op);
            rewriter.replace_operation(ctx, op, new_op.get_operation());
            decorate_uniform(ctx, new_op.get_operation(), uniformity);
        }

        Ok(())
    }
}

#[op_interface_impl]
impl ToSpirvDialectOp for bitwise::LeadingZerosBitsOp {
    fn to_spirv_dialect(
        &self,
        ctx: &mut Context,
        rewriter: &mut DialectConversionRewriter,
        _operands_info: &OperandsInfo,
    ) -> Result<()> {
        let scope = Scope::from_context_and_inserter(ctx, rewriter);
        let op = self.get_operation();
        let uniformity = op_dyn_uniformity(ctx, op);
        let inp = self.input(ctx);
        let out_ty = ty_to_spirv_dialect(ctx, self.get_result(ctx).get_type(ctx));

        // Indices are zero based, so subtract 1 from u32 width
        let width = const_int_maybe_vec(&scope, 31, inp.get_type(ctx));
        let msb = if self.get_result(ctx).scalar_ty(ctx).is_signed_int(ctx) {
            let op = gl::FindSMsbOp::new(ctx, out_ty, inp);
            decorate_uniform(ctx, op.get_operation(), uniformity);
            scope.register_with_result(&op)
        } else {
            let op = gl::FindUMsbOp::new(ctx, out_ty, inp);
            decorate_uniform(ctx, op.get_operation(), uniformity);
            scope.register_with_result(&op)
        };
        let sub = ISubOp::new(ctx, out_ty, width, msb);
        decorate_uniform(ctx, sub.get_operation(), uniformity);
        let value = scope.register_with_result(&sub);
        rewriter.replace_operation_with_values(ctx, op, vec![value]);

        Ok(())
    }
}

#[op_interface_impl]
impl ToSpirvDialectOp for bitwise::FindFirstSetOp {
    fn to_spirv_dialect(
        &self,
        ctx: &mut Context,
        rewriter: &mut DialectConversionRewriter,
        _operands_info: &OperandsInfo,
    ) -> Result<()> {
        let scope = Scope::from_context_and_inserter(ctx, rewriter);
        let op = self.get_operation();
        let uniformity = op_dyn_uniformity(ctx, op);
        let inp = self.input(ctx);
        let out_ty = ty_to_spirv_dialect(ctx, self.get_result(ctx).get_type(ctx));

        let one = const_int_maybe_vec(&scope, 1, inp.get_type(ctx));
        let find_lsb = gl::FindILsbOp::new(ctx, out_ty, inp);
        decorate_uniform(ctx, find_lsb.get_operation(), uniformity);
        let lsb = scope.register_with_result(&find_lsb);
        // Normalize to CUDA/POSIX convention of 1 based index, with 0 meaning not found
        let add = IAddOp::new(ctx, out_ty, lsb, one);
        decorate_uniform(ctx, add.get_operation(), uniformity);
        let value = scope.register_with_result(&add);
        rewriter.replace_operation_with_values(ctx, op, vec![value]);

        Ok(())
    }
}

#[op_interface_impl]
impl ToSpirvDialectOp for bitwise::TrailingZerosBitsOp {
    fn to_spirv_dialect(
        &self,
        ctx: &mut Context,
        rewriter: &mut DialectConversionRewriter,
        _operands_info: &OperandsInfo,
    ) -> Result<()> {
        let scope = Scope::from_context_and_inserter(ctx, rewriter);
        let op = self.get_operation();
        let uniformity = op_dyn_uniformity(ctx, op);
        let inp = self.input(ctx);
        let out_ty = ty_to_spirv_dialect(ctx, self.result_type(ctx));

        let find_lsb = gl::FindILsbOp::new(ctx, out_ty, inp);
        decorate_uniform(ctx, find_lsb.get_operation(), uniformity);
        let lsb = scope.register_with_result(&find_lsb);
        scope.register_value_type::<(), N>(self.result_type(ctx));
        let value = trailing_zeros_adjust::expand(&scope, inp.into(), lsb.into()).value(&scope);
        decorate_uniform(ctx, value.defining_op().unwrap(), uniformity);
        rewriter.replace_operation_with_values(ctx, op, vec![value]);

        Ok(())
    }
}

lower_unop!(CountOnesOp, u64_count_bits, |op, ctx| {
    op.input(ctx).scalar_ty(ctx).is_int_of_width(ctx, 64) && !has_u64_bitwise(ctx)
});
lower_unop!(ReverseBitsOp, u64_reverse, |op, ctx| {
    op.input(ctx).scalar_ty(ctx).is_int_of_width(ctx, 64) && !has_u64_bitwise(ctx)
});
lower_unop!(LeadingZerosBitsOp, u64_leading_zeros, |op, ctx| {
    op.input(ctx).scalar_ty(ctx).is_int_of_width(ctx, 64)
});
lower_unop!(FindFirstSetOp, u64_ffs, |op, ctx| {
    op.input(ctx).scalar_ty(ctx).is_int_of_width(ctx, 64)
});
lower_unop!(TrailingZerosBitsOp, u64_trailing_zeros, |op, ctx| {
    op.input(ctx).scalar_ty(ctx).is_int_of_width(ctx, 64)
});

fn has_u64_bitwise(ctx: &Context) -> bool {
    ctx.aux_ty::<WgpuCompilationOptions>()
        .vulkan
        .supports_arbitrary_bitwise
}

#[cube]
fn u64_reverse<I: Int, N: Size>(x: Vector<I, N>) -> Vector<I, N> {
    let shift = Vector::new(I::new(32));

    let low = Vector::<u32, N>::cast_from(x);
    let high = Vector::<u32, N>::cast_from(x >> shift);

    let low_rev = Vector::reverse_bits(low);
    let high_rev = Vector::reverse_bits(high);
    // Swap low and high values
    let high = Vector::cast_from(low_rev) << shift;
    high | Vector::cast_from(high_rev)
}

#[cube]
fn u64_count_bits<I: Int, N: Size>(x: Vector<I, N>) -> Vector<u32, N> {
    let shift = Vector::new(I::new(32));

    let low = Vector::<u32, N>::cast_from(x);
    let high = Vector::<u32, N>::cast_from(x >> shift);

    let low_cnt = Vector::<u32, N>::cast_from(Vector::count_ones(low));
    let high_cnt = Vector::<u32, N>::cast_from(Vector::count_ones(high));
    low_cnt + high_cnt
}

// find_lsb returns -1 (0xFFFFFFFF) for zero input
// trailing_zeros should return bit_width for zero input
#[cube]
fn trailing_zeros_adjust(inp: Vector<u32, N>, lsb: Vector<u32, N>) -> Vector<u32, N> {
    let width = Vector::new(32);
    select_many(inp.equal(&Vector::zero()), width, lsb)
}