cubecl-spirv 0.11.0

SPIR-V compiler for CubeCL
use cubecl_core::{self as cubecl, frontend::polyfills::*, prelude::*};
use cubecl_ir::{dialect::math, interfaces::TypedExt, prelude::*};
use cubecl_opt::passes::uniformity::op_dyn_uniformity;
use pliron_spirv::{
    attrs::PackedVectorFormatAttr, ext::gl, ops, spirv::PackedVectorFormat, types::StructType,
};

use crate::{
    decorate_uniform,
    lower::{LowerOp, lower_binop, lower_unop},
    ops::{
        base::{binop_to_spirv_dialect, ternop_to_spirv_dialect, unop_to_spirv_dialect},
        to_spirv_dialect::ToSpirvDialectOp,
    },
    types::ty_to_spirv_dialect,
};

unop_to_spirv_dialect!(math::SNegOp => ops::SNegateOp);
unop_to_spirv_dialect!(math::FNegOp => ops::FNegateOp);

binop_to_spirv_dialect!(math::IAddOp => ops::IAddOp);
binop_to_spirv_dialect!(math::FAddOp => ops::FAddOp);
binop_to_spirv_dialect!(math::ISubOp => ops::ISubOp);
binop_to_spirv_dialect!(math::FSubOp => ops::FSubOp);
binop_to_spirv_dialect!(math::IMulOp => ops::IMulOp);
binop_to_spirv_dialect!(math::FMulOp => ops::FMulOp);
binop_to_spirv_dialect!(math::SDivOp => ops::SDivOp);
binop_to_spirv_dialect!(math::UDivOp => ops::UDivOp);
binop_to_spirv_dialect!(math::FDivOp => ops::FDivOp);
lower_binop!(math::SRemOp, signed_remainder);
binop_to_spirv_dialect!(math::URemOp => ops::UModOp);
binop_to_spirv_dialect!(math::FRemOp => ops::FRemOp);
lower_binop!(math::SModFloorOp, signed_mod_floor);
binop_to_spirv_dialect!(math::FModFloorOp => ops::FModOp);

// Vulkan only defines OpSRem/OpSMod for negative operands when maintenance8 is
// enabled. Reconstruct the truncating remainder using integer arithmetic so
// signed inputs work on devices without that feature too.
#[cube]
fn signed_remainder<I: Int, N: Size>(lhs: Vector<I, N>, rhs: Vector<I, N>) -> Vector<I, N> {
    // Division by zero and MIN / -1 are outside the portable contract.
    lhs - (lhs / rhs) * rhs
}

#[cube]
fn signed_mod_floor<I: Int, N: Size>(lhs: Vector<I, N>, rhs: Vector<I, N>) -> Vector<I, N> {
    let remainder = signed_remainder(lhs, rhs);
    let zero = Vector::new(I::from_int(0));
    let different_signs = (remainder ^ rhs).less_than(&zero);
    let adjust = remainder.not_equal(&zero).vec_and(different_signs);
    remainder + select_many(adjust, rhs, zero)
}

#[op_interface_impl]
impl LowerOp for math::Dp4aOp {
    fn should_lower(&self, ctx: &Context) -> bool {
        !ctx.aux_ty::<cubecl_core::WgpuCompilationOptions>()
            .vulkan
            .supports_dp4a
    }

    fn lower(&self, scope: &Scope) -> Vec<Value> {
        let ctx = scope.ctx();
        vec![expand_dp4a_polyfill(
            scope,
            self.a(ctx),
            self.b(ctx),
            self.c(ctx),
        )]
    }
}

#[op_interface_impl]
impl ToSpirvDialectOp for math::Dp4aOp {
    fn to_spirv_dialect(
        &self,
        ctx: &mut Context,
        rewriter: &mut DialectConversionRewriter,
        _operands_info: &OperandsInfo,
    ) -> Result<()> {
        let op = self.get_operation();
        let out_ty = ty_to_spirv_dialect(ctx, self.get_result(ctx).get_type(ctx));
        let packed = PackedVectorFormatAttr::new(PackedVectorFormat::PackedVectorFormat4x8Bit);
        let dot = ops::SDotOp::new(ctx, out_ty, self.a(ctx), self.b(ctx), Some(packed));
        rewriter.append_op(ctx, &dot);
        let add = ops::IAddOp::new(ctx, out_ty, dot.get_result(ctx), self.c(ctx));
        rewriter.append_op(ctx, &add);
        rewriter.replace_operation(ctx, op, add.get_operation());
        Ok(())
    }
}

binop_to_spirv_dialect!(SimplePowOp => gl::PowOp);

unop_to_spirv_dialect!(math::SAbsOp => gl::SAbsOp);
unop_to_spirv_dialect!(math::FAbsOp => gl::FAbsOp);
unop_to_spirv_dialect!(math::ExpOp => gl::ExpOp);
unop_to_spirv_dialect!(math::LogOp => gl::LogOp);
unop_to_spirv_dialect!(math::IsNanOp => ops::IsNanOp);
unop_to_spirv_dialect!(math::IsInfOp => ops::IsInfOp);

unop_to_spirv_dialect!(math::SinOp => gl::SinOp);
unop_to_spirv_dialect!(math::CosOp => gl::CosOp);
unop_to_spirv_dialect!(math::TanOp => gl::TanOp);
unop_to_spirv_dialect!(math::SinhOp => gl::SinhOp);
unop_to_spirv_dialect!(math::CoshOp => gl::CoshOp);
unop_to_spirv_dialect!(math::TanhOp => gl::TanhOp);
unop_to_spirv_dialect!(math::ArcSinOp => gl::AsinOp);
unop_to_spirv_dialect!(math::ArcCosOp => gl::AcosOp);
unop_to_spirv_dialect!(math::ArcTanOp => gl::AtanOp);
unop_to_spirv_dialect!(math::ArcSinhOp => gl::AsinhOp);
unop_to_spirv_dialect!(math::ArcCoshOp => gl::AcoshOp);
unop_to_spirv_dialect!(math::ArcTanhOp => gl::AtanhOp);
binop_to_spirv_dialect!(math::ArcTan2Op => gl::Atan2Op);

unop_to_spirv_dialect!(math::DegreesOp => gl::DegreesOp);
unop_to_spirv_dialect!(math::RadiansOp => gl::RadiansOp);

unop_to_spirv_dialect!(math::SqrtOp => gl::SqrtOp);
unop_to_spirv_dialect!(math::RsqrtOp => gl::InverseSqrtOp);

unop_to_spirv_dialect!(math::RoundOp => gl::RoundOp);
unop_to_spirv_dialect!(math::FloorOp => gl::FloorOp);
unop_to_spirv_dialect!(math::CeilOp => gl::CeilOp);
unop_to_spirv_dialect!(math::TruncOp => gl::TruncOp);

#[op_interface_impl]
impl ToSpirvDialectOp for math::SMulHiOp {
    fn to_spirv_dialect(
        &self,
        ctx: &mut Context,
        rewriter: &mut DialectConversionRewriter,
        _operands_info: &OperandsInfo,
    ) -> Result<()> {
        let op = self.get_operation();
        let uniformity = op_dyn_uniformity(ctx, op);
        let lhs = self.lhs(ctx);
        let rhs = self.rhs(ctx);
        let out_ty = ty_to_spirv_dialect(ctx, self.get_result(ctx).get_type(ctx));
        let out_st = StructType::get(ctx, vec![out_ty, out_ty], vec![], vec![], vec![]).into();
        let mul = ops::SMulExtendedOp::new(ctx, out_st, lhs, rhs);
        decorate_uniform(ctx, mul.get_operation(), uniformity);
        rewriter.append_op(ctx, &mul);
        let new_op = ops::CompositeExtractOp::new(ctx, out_ty, mul.get_result(ctx), vec![1.into()]);
        decorate_uniform(ctx, new_op.get_operation(), uniformity);
        rewriter.append_op(ctx, &new_op);
        rewriter.replace_operation(ctx, op, new_op.get_operation());

        Ok(())
    }
}

#[op_interface_impl]
impl ToSpirvDialectOp for math::UMulHiOp {
    fn to_spirv_dialect(
        &self,
        ctx: &mut Context,
        rewriter: &mut DialectConversionRewriter,
        _operands_info: &OperandsInfo,
    ) -> Result<()> {
        let op = self.get_operation();
        let uniformity = op_dyn_uniformity(ctx, op);
        let lhs = self.lhs(ctx);
        let rhs = self.rhs(ctx);
        let out_ty = ty_to_spirv_dialect(ctx, self.get_result(ctx).get_type(ctx));
        let out_st = StructType::get(ctx, vec![out_ty, out_ty], vec![], vec![], vec![]).into();
        let mul = ops::UMulExtendedOp::new(ctx, out_st, lhs, rhs);
        decorate_uniform(ctx, mul.get_operation(), uniformity);
        rewriter.append_op(ctx, &mul);
        let new_op = ops::CompositeExtractOp::new(ctx, out_ty, mul.get_result(ctx), vec![1.into()]);
        decorate_uniform(ctx, new_op.get_operation(), uniformity);
        rewriter.append_op(ctx, &new_op);
        rewriter.replace_operation(ctx, op, new_op.get_operation());

        Ok(())
    }
}

#[op_interface_impl]
impl ToSpirvDialectOp for gl::PowOp {
    fn should_convert(&self, ctx: &Context) -> bool {
        ty_to_spirv_dialect(ctx, self.result_type(ctx)) != self.result_type(ctx)
    }
    fn to_spirv_dialect(
        &self,
        ctx: &mut Context,
        rewriter: &mut DialectConversionRewriter,
        _operands_info: &OperandsInfo,
    ) -> Result<()> {
        let ty = ty_to_spirv_dialect(ctx, self.result_type(ctx));
        rewriter.set_value_type(ctx, self.get_result(ctx), ty);
        Ok(())
    }
}

lower_unop!(math::RecipOp, recip);
lower_unop!(math::Log1pOp, log1p);
lower_unop!(math::Expm1Op, expm1);
lower_unop!(math::ErfOp, erf);
lower_binop!(math::HypotOp, hypot);
lower_binop!(math::RhypotOp, rhypot);

lower_binop!(math::PowfOp, powf);

#[op_interface_impl]
impl LowerOp for math::PowiOp {
    fn lower(&self, scope: &Scope) -> Vec<Value> {
        define_scalar!(T);
        define_size!(S);
        let lhs = self.lhs(scope.ctx());
        let rhs = self.rhs(scope.ctx());
        scope.register_value_type::<T, S>(lhs);

        let ty = lhs.scalar_ty(scope.ctx());
        let result = if ty.is_int(scope.ctx()) || ty.is_index(scope.ctx()) {
            powi_int::expand::<T, S>(scope, lhs.into(), rhs.into()).read_value(scope)
        } else {
            powi::expand::<T, S>(scope, lhs.into(), rhs.into()).read_value(scope)
        };
        vec![result]
    }
}

ternop_to_spirv_dialect!(math::FmaOp => gl::FmaOp);

/// `gl::FmaOp` is defined for floats only, so an integer fma keeps its multiply and add.
#[op_interface_impl]
impl LowerOp for math::FmaOp {
    fn should_lower(&self, ctx: &Context) -> bool {
        self.get_result(ctx).scalar_ty(ctx).is_int(ctx)
    }

    fn lower(&self, scope: &Scope) -> Vec<Value> {
        define_scalar!(T);
        define_size!(S);
        let a = self.a(scope.ctx());
        let b = self.b(scope.ctx());
        let c = self.c(scope.ctx());
        scope.register_value_type::<T, S>(a);
        vec![int_fma::expand::<T, S>(scope, a.into(), b.into(), c.into()).read_value(scope)]
    }
}

#[cube]
fn int_fma<T: Int, N: Size>(a: Vector<T, N>, b: Vector<T, N>, c: Vector<T, N>) -> Vector<T, N> {
    a * b + c
}