use cubecl_core::{
self as cubecl, WgpuCompilationOptions, frontend::polyfills::bitwise::*, num_traits::Zero,
prelude::*,
};
use cubecl_ir::{
dialect::bitwise::{self, *},
interfaces::TypedExt,
prelude::*,
};
use cubecl_opt::passes::uniformity::op_dyn_uniformity;
use pliron::{
builtin::{
attributes::{IntegerAttr, VecAttr},
ops::ConstantOp,
types::IntegerType,
},
utils::apint::{APInt, bw},
};
use pliron_spirv::{
attrs::CompositeAttr,
ext::gl,
ops::{self, IAddOp, ISubOp, ShiftRightArithmeticOp, ShiftRightLogicalOp},
types::VectorType,
};
use crate::{
decorate_uniform,
lower::lower_unop,
ops::{
base::{binop_to_spirv_dialect, unop_to_spirv_dialect},
to_spirv_dialect::ToSpirvDialectOp,
},
types::ty_to_spirv_dialect,
};
define_size!(N);
binop_to_spirv_dialect!(bitwise::BitwiseAndOp => ops::BitwiseAndOp);
binop_to_spirv_dialect!(bitwise::BitwiseOrOp => ops::BitwiseOrOp);
binop_to_spirv_dialect!(bitwise::BitwiseXorOp => ops::BitwiseXorOp);
binop_to_spirv_dialect!(bitwise::ShiftLeftOp => ops::ShiftLeftLogicalOp);
unop_to_spirv_dialect!(bitwise::BitwiseNotOp => ops::NotOp);
unop_to_spirv_dialect!(bitwise::CountOnesOp => ops::BitCountOp);
unop_to_spirv_dialect!(bitwise::ReverseBitsOp => ops::BitReverseOp);
pub fn const_int_maybe_vec(scope: &Scope, value: u32, ref_ty: TypeHandle) -> Value {
let ty = ref_ty.deref(scope.ctx());
if let Some(vector) = ty.downcast_ref::<VectorType>() {
let elem_ty =
TypedHandle::<IntegerType>::from_handle(vector.element_type, scope.ctx()).unwrap();
let width = bw(elem_ty.deref(scope.ctx()).width() as usize);
let value = IntegerAttr::new(elem_ty, APInt::from_u32(value, width));
let values = VecAttr(vec![Box::new(value); vector.count as usize]);
let value = CompositeAttr::new(values, ref_ty);
scope.register_with_result(&ConstantOp::new(scope.ctx_mut(), Box::new(value)))
} else {
let ref_ty = TypedHandle::<IntegerType>::from_handle(ref_ty, scope.ctx()).unwrap();
let width = bw(ref_ty.deref(scope.ctx()).width() as usize);
let value = IntegerAttr::new(ref_ty, APInt::from_u32(value, width));
scope.register_with_result(&ConstantOp::new(scope.ctx_mut(), Box::new(value)))
}
}
#[op_interface_impl]
impl ToSpirvDialectOp for bitwise::ShiftRightOp {
fn to_spirv_dialect(
&self,
ctx: &mut Context,
rewriter: &mut DialectConversionRewriter,
_operands_info: &OperandsInfo,
) -> Result<()> {
let op = self.get_operation();
let uniformity = op_dyn_uniformity(ctx, op);
let lhs = op.operand(ctx, 0);
let rhs = op.operand(ctx, 1);
let out_ty = ty_to_spirv_dialect(ctx, self.get_result(ctx).get_type(ctx));
if self.get_result(ctx).scalar_ty(ctx).is_signed_int(ctx) {
let new_op = ShiftRightArithmeticOp::new(ctx, out_ty, lhs, rhs);
rewriter.append_op(ctx, &new_op);
rewriter.replace_operation(ctx, op, new_op.get_operation());
decorate_uniform(ctx, new_op.get_operation(), uniformity);
} else {
let new_op = ShiftRightLogicalOp::new(ctx, out_ty, lhs, rhs);
rewriter.append_op(ctx, &new_op);
rewriter.replace_operation(ctx, op, new_op.get_operation());
decorate_uniform(ctx, new_op.get_operation(), uniformity);
}
Ok(())
}
}
#[op_interface_impl]
impl ToSpirvDialectOp for bitwise::LeadingZerosBitsOp {
fn to_spirv_dialect(
&self,
ctx: &mut Context,
rewriter: &mut DialectConversionRewriter,
_operands_info: &OperandsInfo,
) -> Result<()> {
let scope = Scope::from_context_and_inserter(ctx, rewriter);
let op = self.get_operation();
let uniformity = op_dyn_uniformity(ctx, op);
let inp = self.input(ctx);
let out_ty = ty_to_spirv_dialect(ctx, self.get_result(ctx).get_type(ctx));
let width = const_int_maybe_vec(&scope, 31, inp.get_type(ctx));
let msb = if self.get_result(ctx).scalar_ty(ctx).is_signed_int(ctx) {
let op = gl::FindSMsbOp::new(ctx, out_ty, inp);
decorate_uniform(ctx, op.get_operation(), uniformity);
scope.register_with_result(&op)
} else {
let op = gl::FindUMsbOp::new(ctx, out_ty, inp);
decorate_uniform(ctx, op.get_operation(), uniformity);
scope.register_with_result(&op)
};
let sub = ISubOp::new(ctx, out_ty, width, msb);
decorate_uniform(ctx, sub.get_operation(), uniformity);
let value = scope.register_with_result(&sub);
rewriter.replace_operation_with_values(ctx, op, vec![value]);
Ok(())
}
}
#[op_interface_impl]
impl ToSpirvDialectOp for bitwise::FindFirstSetOp {
fn to_spirv_dialect(
&self,
ctx: &mut Context,
rewriter: &mut DialectConversionRewriter,
_operands_info: &OperandsInfo,
) -> Result<()> {
let scope = Scope::from_context_and_inserter(ctx, rewriter);
let op = self.get_operation();
let uniformity = op_dyn_uniformity(ctx, op);
let inp = self.input(ctx);
let out_ty = ty_to_spirv_dialect(ctx, self.get_result(ctx).get_type(ctx));
let one = const_int_maybe_vec(&scope, 1, inp.get_type(ctx));
let find_lsb = gl::FindILsbOp::new(ctx, out_ty, inp);
decorate_uniform(ctx, find_lsb.get_operation(), uniformity);
let lsb = scope.register_with_result(&find_lsb);
let add = IAddOp::new(ctx, out_ty, lsb, one);
decorate_uniform(ctx, add.get_operation(), uniformity);
let value = scope.register_with_result(&add);
rewriter.replace_operation_with_values(ctx, op, vec![value]);
Ok(())
}
}
#[op_interface_impl]
impl ToSpirvDialectOp for bitwise::TrailingZerosBitsOp {
fn to_spirv_dialect(
&self,
ctx: &mut Context,
rewriter: &mut DialectConversionRewriter,
_operands_info: &OperandsInfo,
) -> Result<()> {
let scope = Scope::from_context_and_inserter(ctx, rewriter);
let op = self.get_operation();
let uniformity = op_dyn_uniformity(ctx, op);
let inp = self.input(ctx);
let out_ty = ty_to_spirv_dialect(ctx, self.result_type(ctx));
let find_lsb = gl::FindILsbOp::new(ctx, out_ty, inp);
decorate_uniform(ctx, find_lsb.get_operation(), uniformity);
let lsb = scope.register_with_result(&find_lsb);
scope.register_value_type::<(), N>(self.result_type(ctx));
let value = trailing_zeros_adjust::expand(&scope, inp.into(), lsb.into()).value(&scope);
decorate_uniform(ctx, value.defining_op().unwrap(), uniformity);
rewriter.replace_operation_with_values(ctx, op, vec![value]);
Ok(())
}
}
lower_unop!(CountOnesOp, u64_count_bits, |op, ctx| {
op.input(ctx).scalar_ty(ctx).is_int_of_width(ctx, 64) && !has_u64_bitwise(ctx)
});
lower_unop!(ReverseBitsOp, u64_reverse, |op, ctx| {
op.input(ctx).scalar_ty(ctx).is_int_of_width(ctx, 64) && !has_u64_bitwise(ctx)
});
lower_unop!(LeadingZerosBitsOp, u64_leading_zeros, |op, ctx| {
op.input(ctx).scalar_ty(ctx).is_int_of_width(ctx, 64)
});
lower_unop!(FindFirstSetOp, u64_ffs, |op, ctx| {
op.input(ctx).scalar_ty(ctx).is_int_of_width(ctx, 64)
});
lower_unop!(TrailingZerosBitsOp, u64_trailing_zeros, |op, ctx| {
op.input(ctx).scalar_ty(ctx).is_int_of_width(ctx, 64)
});
fn has_u64_bitwise(ctx: &Context) -> bool {
ctx.aux_ty::<WgpuCompilationOptions>()
.vulkan
.supports_arbitrary_bitwise
}
#[cube]
fn u64_reverse<I: Int, N: Size>(x: Vector<I, N>) -> Vector<I, N> {
let shift = Vector::new(I::new(32));
let low = Vector::<u32, N>::cast_from(x);
let high = Vector::<u32, N>::cast_from(x >> shift);
let low_rev = Vector::reverse_bits(low);
let high_rev = Vector::reverse_bits(high);
let high = Vector::cast_from(low_rev) << shift;
high | Vector::cast_from(high_rev)
}
#[cube]
fn u64_count_bits<I: Int, N: Size>(x: Vector<I, N>) -> Vector<u32, N> {
let shift = Vector::new(I::new(32));
let low = Vector::<u32, N>::cast_from(x);
let high = Vector::<u32, N>::cast_from(x >> shift);
let low_cnt = Vector::<u32, N>::cast_from(Vector::count_ones(low));
let high_cnt = Vector::<u32, N>::cast_from(Vector::count_ones(high));
low_cnt + high_cnt
}
#[cube]
fn trailing_zeros_adjust(inp: Vector<u32, N>, lsb: Vector<u32, N>) -> Vector<u32, N> {
let width = Vector::new(32);
select_many(inp.equal(&Vector::zero()), width, lsb)
}