leto-ops 0.45.2

Math, reduction, SIMD, and parallel kernels for Leto arrays
Documentation
use super::{AttentionError, AttentionGradients, AttentionMask, AttentionOperand, AttentionResult};
use crate::domain::real::RealScalar;
use eunomia::{NumericElement, RealField};
use leto::{ArrayView, ArrayViewMut, LetoError};

#[derive(Debug, Clone, Copy)]
pub(super) struct AttentionPlan {
    pub(super) batch: usize,
    pub(super) query_len: usize,
    pub(super) key_len: usize,
    pub(super) key_dim: usize,
    pub(super) value_dim: usize,
}

impl AttentionPlan {
    pub(super) fn score_elements(self) -> AttentionResult<usize> {
        self.batch
            .checked_mul(self.query_len)
            .and_then(|value| value.checked_mul(self.key_len))
            .ok_or(AttentionError::WorkspaceOverflow)
    }
}

fn validate_view<T, const N: usize>(
    operand: AttentionOperand,
    view: &ArrayView<'_, T, N>,
) -> AttentionResult<()> {
    view.layout()
        .validate_storage_len(view.data().len())
        .map_err(|source| AttentionError::Layout { operand, source })
}

fn validate_mut_view<T>(
    operand: AttentionOperand,
    view: &mut ArrayViewMut<'_, T, 3>,
) -> AttentionResult<()> {
    view.layout()
        .validate_storage_len(view.data().len())
        .map_err(|source| AttentionError::Layout { operand, source })?;
    if !view
        .layout()
        .is_injective()
        .map_err(|source| AttentionError::Layout { operand, source })?
    {
        return Err(AttentionError::Layout {
            operand,
            source: LetoError::StorageError {
                reason: "attention mutable view contains aliased logical offsets".to_string(),
            },
        });
    }
    Ok(())
}

fn validate_finite<T: RealScalar, const N: usize>(
    operand: AttentionOperand,
    view: &ArrayView<'_, T, N>,
) -> AttentionResult<()> {
    if view
        .iter()
        .copied()
        .any(|value| !NumericElement::is_finite(value))
    {
        return Err(AttentionError::NonFinite { operand });
    }
    Ok(())
}

fn expect_shape(
    operand: AttentionOperand,
    actual: [usize; 3],
    expected: [usize; 3],
) -> AttentionResult<()> {
    if actual != expected {
        return Err(AttentionError::Shape {
            operand,
            expected,
            actual,
        });
    }
    Ok(())
}

fn validate_mask<T: RealScalar>(
    mask: AttentionMask<'_, T>,
    target: [usize; 3],
) -> AttentionResult<()> {
    if let Some(grouped) = mask.grouped() {
        validate_view(AttentionOperand::Mask, &grouped.view)?;
        validate_finite(AttentionOperand::Mask, &grouped.view)?;
        let actual = grouped.view.shape();
        let Some(grouped_batches) = actual[0].checked_mul(grouped.batches_per_group.get()) else {
            return Err(AttentionError::WorkspaceOverflow);
        };
        if grouped_batches != target[0] || actual[1] != target[2] {
            return Err(AttentionError::MaskShape {
                actual: [actual[0], 1, actual[1]],
                target,
            });
        }
        return Ok(());
    }
    let Some(mask_view) = mask.view() else {
        return Ok(());
    };
    validate_view(AttentionOperand::Mask, &mask_view)?;
    validate_finite(AttentionOperand::Mask, &mask_view)?;
    let actual = mask_view.shape();
    if actual
        .iter()
        .zip(target)
        .any(|(&dimension, target_dimension)| dimension != 1 && dimension != target_dimension)
    {
        return Err(AttentionError::MaskShape { actual, target });
    }
    Ok(())
}

fn read<T: Copy>(view: &ArrayView<'_, T, 3>, index: [usize; 3]) -> T {
    *view
        .get(index)
        .expect("invariant: validated attention input index is in bounds")
}

pub(super) fn mask_is_active<T: RealScalar>(
    mask: AttentionMask<'_, T>,
    batch: usize,
    query: usize,
    key: usize,
) -> bool {
    if mask.is_causal() && key > query {
        return false;
    }
    if let Some(grouped) = mask.grouped() {
        let group = batch / grouped.batches_per_group.get();
        return *grouped
            .view
            .get([group, key])
            .expect("invariant: validated grouped attention mask index is in bounds")
            != <T as NumericElement>::ZERO;
    }
    let Some(mask_view) = mask.view() else {
        return true;
    };
    let shape = mask_view.shape();
    let index = [
        if shape[0] == 1 { 0 } else { batch },
        if shape[1] == 1 { 0 } else { query },
        if shape[2] == 1 { 0 } else { key },
    ];
    read(&mask_view, index) != <T as NumericElement>::ZERO
}

fn validate_forward_arithmetic<T: RealScalar>(
    query: &ArrayView<'_, T, 3>,
    key: &ArrayView<'_, T, 3>,
    mask: AttentionMask<'_, T>,
    scale: T,
    plan: AttentionPlan,
) -> AttentionResult<()> {
    for batch in 0..plan.batch {
        for query_index in 0..plan.query_len {
            let mut active_keys = 0usize;
            for key_index in 0..plan.key_len {
                if !mask_is_active(mask, batch, query_index, key_index) {
                    continue;
                }
                active_keys = active_keys
                    .checked_add(1)
                    .ok_or(AttentionError::WorkspaceOverflow)?;
                let mut score = <T as NumericElement>::ZERO;
                for feature in 0..plan.key_dim {
                    score += read(query, [batch, query_index, feature])
                        * read(key, [batch, key_index, feature]);
                }
                score *= scale;
                if !NumericElement::is_finite(score) {
                    return Err(AttentionError::ArithmeticNonFinite {
                        operand: AttentionOperand::Weights,
                    });
                }
            }
            if active_keys > 0 {
                let support_bound = active_keys
                    .checked_next_power_of_two()
                    .ok_or(AttentionError::WorkspaceOverflow)?;
                if !NumericElement::is_finite(T::from_count(support_bound)) {
                    return Err(AttentionError::ArithmeticNonFinite {
                        operand: AttentionOperand::Weights,
                    });
                }
            }
        }
    }
    Ok(())
}

fn validate_probability_weights<T: RealScalar + RealField>(
    weights: &ArrayView<'_, T, 3>,
    plan: AttentionPlan,
) -> AttentionResult<()> {
    let width = T::from_count(plan.key_len);
    let tolerance = <T as RealField>::EPSILON * width * T::from_count(4);
    if !NumericElement::is_finite(tolerance) || tolerance >= T::from_f64(0.5) {
        return Err(AttentionError::ArithmeticNonFinite {
            operand: AttentionOperand::Weights,
        });
    }
    for batch in 0..plan.batch {
        for query in 0..plan.query_len {
            let mut sum = <T as NumericElement>::ZERO;
            for key in 0..plan.key_len {
                let weight = read(weights, [batch, query, key]);
                if weight < <T as NumericElement>::ZERO || weight > <T as NumericElement>::ONE {
                    return Err(AttentionError::InvalidWeights { batch, query });
                }
                sum += weight;
            }
            if sum != <T as NumericElement>::ZERO
                && NumericElement::abs(sum - <T as NumericElement>::ONE) > tolerance
            {
                return Err(AttentionError::InvalidWeights { batch, query });
            }
        }
    }
    Ok(())
}

pub(super) fn validate_forward<T: RealScalar>(
    query: &ArrayView<'_, T, 3>,
    key: &ArrayView<'_, T, 3>,
    value: &ArrayView<'_, T, 3>,
    mask: AttentionMask<'_, T>,
    scale: T,
    output: &mut ArrayViewMut<'_, T, 3>,
    weights: &mut ArrayViewMut<'_, T, 3>,
) -> AttentionResult<AttentionPlan> {
    for (operand, view) in [
        (AttentionOperand::Query, query),
        (AttentionOperand::Key, key),
        (AttentionOperand::Value, value),
    ] {
        validate_view(operand, view)?;
        validate_finite(operand, view)?;
    }
    if !NumericElement::is_finite(scale) {
        return Err(AttentionError::NonFinite {
            operand: AttentionOperand::Scale,
        });
    }

    let [batch, query_len, key_dim] = query.shape();
    let [key_batch, key_len, actual_key_dim] = key.shape();
    let [value_batch, value_len, value_dim] = value.shape();
    if key_len == 0 {
        return Err(AttentionError::EmptyKeySequence);
    }
    expect_shape(
        AttentionOperand::Key,
        key.shape(),
        [batch, key_len, key_dim],
    )?;
    expect_shape(
        AttentionOperand::Value,
        value.shape(),
        [batch, key_len, value_dim],
    )?;
    debug_assert_eq!(key_batch, batch);
    debug_assert_eq!(actual_key_dim, key_dim);
    debug_assert_eq!(value_batch, batch);
    debug_assert_eq!(value_len, key_len);

    expect_shape(
        AttentionOperand::Output,
        output.shape(),
        [batch, query_len, value_dim],
    )?;
    expect_shape(
        AttentionOperand::Weights,
        weights.shape(),
        [batch, query_len, key_len],
    )?;
    validate_mut_view(AttentionOperand::Output, output)?;
    validate_mut_view(AttentionOperand::Weights, weights)?;
    validate_mask(mask, [batch, query_len, key_len])?;

    let plan = AttentionPlan {
        batch,
        query_len,
        key_len,
        key_dim,
        value_dim,
    };
    validate_forward_arithmetic(query, key, mask, scale, plan)?;
    Ok(plan)
}

#[allow(clippy::too_many_arguments)]
pub(super) fn validate_backward<T: RealScalar + RealField>(
    output_gradient: &ArrayView<'_, T, 3>,
    query: &ArrayView<'_, T, 3>,
    key: &ArrayView<'_, T, 3>,
    value: &ArrayView<'_, T, 3>,
    weights: &ArrayView<'_, T, 3>,
    scale: T,
    gradients: &mut AttentionGradients<'_, T>,
) -> AttentionResult<AttentionPlan> {
    if !gradients.has_any() {
        return Err(AttentionError::NoGradientTargets);
    }
    for (operand, view) in [
        (AttentionOperand::OutputGradient, output_gradient),
        (AttentionOperand::Query, query),
        (AttentionOperand::Key, key),
        (AttentionOperand::Value, value),
        (AttentionOperand::Weights, weights),
    ] {
        validate_view(operand, view)?;
        validate_finite(operand, view)?;
    }
    if !NumericElement::is_finite(scale) {
        return Err(AttentionError::NonFinite {
            operand: AttentionOperand::Scale,
        });
    }

    let [batch, query_len, key_dim] = query.shape();
    let [_, key_len, _] = key.shape();
    let [_, _, value_dim] = value.shape();
    if key_len == 0 {
        return Err(AttentionError::EmptyKeySequence);
    }
    expect_shape(
        AttentionOperand::Key,
        key.shape(),
        [batch, key_len, key_dim],
    )?;
    expect_shape(
        AttentionOperand::Value,
        value.shape(),
        [batch, key_len, value_dim],
    )?;
    expect_shape(
        AttentionOperand::Weights,
        weights.shape(),
        [batch, query_len, key_len],
    )?;
    expect_shape(
        AttentionOperand::OutputGradient,
        output_gradient.shape(),
        [batch, query_len, value_dim],
    )?;

    for (operand, view, expected) in [
        (
            AttentionOperand::QueryGradient,
            gradients.query.as_mut(),
            [batch, query_len, key_dim],
        ),
        (
            AttentionOperand::KeyGradient,
            gradients.key.as_mut(),
            [batch, key_len, key_dim],
        ),
        (
            AttentionOperand::ValueGradient,
            gradients.value.as_mut(),
            [batch, key_len, value_dim],
        ),
    ] {
        if let Some(view) = view {
            expect_shape(operand, view.shape(), expected)?;
            validate_mut_view(operand, view)?;
            if view
                .as_view()
                .iter()
                .copied()
                .any(|value| !NumericElement::is_finite(value))
            {
                return Err(AttentionError::NonFinite { operand });
            }
        }
    }

    let plan = AttentionPlan {
        batch,
        query_len,
        key_len,
        key_dim,
        value_dim,
    };
    if gradients.query.is_some() || gradients.key.is_some() {
        plan.score_elements()?;
    }
    validate_probability_weights(weights, plan)?;
    Ok(plan)
}