tract-gpu 0.23.6

use crate::tensor::{DeviceTensor, DeviceTensorExt};
use derive_new::new;
use tract_core::internal::*;

/// Fused scale + mask + softmax over the last axis.  When the mask is float
/// it is added in log-space (`out = softmax(x*scale + mask)`); when it is
/// bool, masked positions are substituted with `-inf` before softmax.
///
/// If `post_softmax_mask` is true (bool mask only), fully-masked rows — whose
/// softmax would otherwise be NaN — are written as `0` instead.  Partially-
/// masked rows are unaffected.
pub type DispatchScaledMaskedSoftmaxFn = fn(
    input: &DeviceTensor,
    scale: &Tensor,
    mask: &DeviceTensor,
    post_softmax_mask: bool,
    output: &DeviceTensor,
) -> TractResult<()>;

#[derive(Clone, new)]
pub struct GpuScaledMaskedSoftmax {
    pub scale: Arc<Tensor>,
    pub post_softmax_mask: bool,
    pub backend_name: &'static str,
    pub dispatch: DispatchScaledMaskedSoftmaxFn,
}

impl std::fmt::Debug for GpuScaledMaskedSoftmax {
    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
        write!(f, "{}ScaledMaskedSoftmax", self.backend_name)
    }
}

impl PartialEq for GpuScaledMaskedSoftmax {
    fn eq(&self, other: &Self) -> bool {
        self.backend_name == other.backend_name
            && self.scale == other.scale
            && self.post_softmax_mask == other.post_softmax_mask
    }
}
impl Eq for GpuScaledMaskedSoftmax {}

impl std::hash::Hash for GpuScaledMaskedSoftmax {
    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
        self.backend_name.hash(state);
        self.scale.hash(state);
        self.post_softmax_mask.hash(state);
    }
}

impl Op for GpuScaledMaskedSoftmax {
    fn name(&self) -> StaticName {
        format!("{}ScaledMaskedSoftmax", self.backend_name).into()
    }
    op_as_typed_op!();
}

impl EvalOp for GpuScaledMaskedSoftmax {
    op_out_of_plan!();

    fn eval(&self, ctx: &EvalContext, inputs: TVec<TValue>) -> TractResult<TVec<TValue>> {
        let (input_val, mask_val) = args_2!(inputs);
        let input = input_val.to_device_tensor()?;
        let mask = mask_val.to_device_tensor()?;
        let output =
            crate::turn_handler::make_tensor_for_node(ctx, input.datum_type(), input.shape())?;
        (self.dispatch)(input, &self.scale, mask, self.post_softmax_mask, &output)?;
        Ok(tvec!(output.into_tensor().into_tvalue()))
    }
}

impl TypedOp for GpuScaledMaskedSoftmax {
    fn output_facts(&self, inputs: &[&TypedFact]) -> TractResult<TVec<TypedFact>> {
        crate::utils::facts_to_device_facts(inputs, |facts| {
            ensure!(facts.len() == 2);
            let dt = facts[0].datum_type;
            let mask_dt = facts[1].datum_type;
            ensure!(mask_dt == dt || mask_dt == bool::datum_type());
            // post_softmax_mask is bool-mask-only per the CPU contract.
            ensure!(!self.post_softmax_mask || mask_dt == bool::datum_type());
            ensure!(facts[0].rank() <= 5);
            ensure!(facts[0].rank() >= 2);
            ensure!(facts[0].rank() == facts[1].rank());
            let fact = dt.fact(facts[0].shape.clone());
            Ok(tvec!(fact))
        })
        .with_context(|| format!("Error while computing facts for {:?}", self.name()))
    }
    as_op!();
}