cubecl_runtime/tune/

operation.rs

use alloc::boxed::Box;
use alloc::string::String;
use alloc::sync::Arc;
use alloc::vec::Vec;
use core::fmt::{Debug, Display};
use core::hash::Hash;
use cubecl_common::hash::{StableHash, StableHasher};

use alloc::format;

use crate::tune::{Bounds, BoundsGenerator, Eviction, Evictor};

use super::{
    AutotuneError, input_generator::InputGenerator, key_generator::KeyGenerator,
    tune_inputs::TuneInputs,
};
use super::{Tunable, TunePlan};

/// A type-erased delegate for a tunable function.
///
/// The lifetime `'inp` is the lifetime of the input data, the function must be defined such that
/// it can be called for any lifetime `inp` and produce a `Result<Out, AutotuneError>`.
type TuneDelegate<I, Out> =
    dyn for<'inp> Fn(<I as TuneInputs>::At<'inp>) -> Result<Out, AutotuneError> + Send + Sync;

/// A named, type-erased tunable function stored in a [`TunableSet`]. Constructed via
/// [`Tunable::new`](super::Tunable::new); callers don't name this type directly.
#[derive(new)]
pub struct TuneFn<I: TuneInputs, Out> {
    pub(crate) name: String,
    func: Box<TuneDelegate<I, Out>>,
}

impl<I: TuneInputs, Out: 'static> TuneFn<I, Out> {
    /// Run the wrapped function on the given inputs.
    pub fn execute<'a>(&self, inputs: <I as TuneInputs>::At<'a>) -> Result<Out, AutotuneError> {
        (self.func)(inputs)
    }
}

/// A set of candidate tunable functions for autotune, sharing a key generator and an
/// input generator. See [`TuneInputs`] for the `F` parameter.
///
/// `Id` is what its tunables are [identified](Tunable::identified) by, inferred from the
/// tunables it is given: `()` for tunables built by [`Tunable::new`], named and nothing more.
pub struct TunableSet<K: AutotuneKey, F: TuneInputs, Output: 'static, Id = ()> {
    tunables: Vec<Tunable<K, F, Output, Id>>,
    /// The low half of the [checksum](Self::compute_checksum), kept in step with `tunables`: every
    /// launch looks its tuner up by it.
    checksum: u64,
    key_gen: Arc<dyn KeyGenerator<K, F> + Send + Sync>,
    input_gen: Arc<dyn InputGenerator<K, F> + Send + Sync>,
    bounds_gen: Option<Arc<dyn BoundsGenerator<K, F> + Send + Sync>>,
    eviction: Option<Arc<dyn Eviction<K, F> + Send + Sync>>,
    short_circuit: bool,
}

impl<K: AutotuneKey, F: TuneInputs, Output: 'static, Id> TunableSet<K, F, Output, Id> {
    /// Create a tunable set from a key generator and an input generator.
    pub fn new(key_gen: impl KeyGenerator<K, F>, input_gen: impl InputGenerator<K, F>) -> Self {
        let mut set = Self {
            tunables: Default::default(),
            checksum: 0,
            input_gen: Arc::new(input_gen),
            key_gen: Arc::new(key_gen),
            bounds_gen: None,
            eviction: None,
            short_circuit: true,
        };
        set.checksum = set.names_hash() as u64;
        set
    }

    /// Shorthand for [`new`](Self::new) with a [`CloneInputGenerator`](super::CloneInputGenerator): benchmarks run
    /// on clones of the real call inputs.
    pub fn new_cloning_inputs(key_gen: impl KeyGenerator<K, F>) -> Self {
        Self::new(key_gen, super::CloneInputGenerator)
    }

    /// The number of tunables in the set.
    pub fn len(&self) -> usize {
        self.tunables.len()
    }

    /// Whether this set contains no tunables.
    pub fn is_empty(&self) -> bool {
        self.tunables.is_empty()
    }

    /// Register a tunable with this tunable set.
    pub fn with(mut self, tunable: Tunable<K, F, Output, Id>) -> Self {
        self.tunables.push(tunable);
        self.checksum = self.names_hash() as u64;
        self
    }

    /// Sets the autotune bounds for this set.
    pub fn with_bounds(mut self, bounds: Arc<dyn BoundsGenerator<K, F> + Send + Sync>) -> Self {
        self.bounds_gen = Some(bounds);
        self
    }

    /// Sets what runs before every measured sample, so the candidates are timed reading
    /// memory rather than the cache the previous sample left warm. See [`Eviction`].
    pub fn with_eviction(mut self, eviction: Arc<dyn Eviction<K, F> + Send + Sync>) -> Self {
        self.eviction = Some(eviction);
        self
    }

    /// Set whether bounds-based short-circuiting is enabled.
    pub fn with_short_circuit(mut self, enabled: bool) -> Self {
        self.short_circuit = enabled;
        self
    }

    /// Whether short-circuiting is enabled for this set.
    pub fn is_short_circuit_enabled(&self) -> bool {
        self.short_circuit
    }

    /// All candidate operations in this set, in registration order.
    pub fn autotunables(&self) -> impl Iterator<Item = &TuneFn<F, Output>> {
        self.tunables.iter().map(|tunable| &tunable.function)
    }

    /// Returns the [autotune plan](TunePlan) for the given set.
    pub(crate) fn plan(&self, key: &K) -> TunePlan {
        TunePlan::new(key, &self.tunables)
    }

    /// Returns the operation for the given index, matching the order returned by
    /// `autotunables`. Tunables are tried in order, so index 0 should be a good default.
    pub fn fastest(&self, fastest_index: usize) -> &TuneFn<F, Output> {
        &self.tunables[fastest_index].function
    }

    /// What the tunable at `index` is [identified](Tunable::identified) by. The index is one
    /// the tuner stored for this set, so it panics out of range as [`fastest`](Self::fastest)
    /// does.
    pub(crate) fn identity(&self, index: usize) -> &Id {
        self.tunables[index].identity()
    }

    /// Compute a checksum that invalidates outdated cached auto-tune results when the
    /// set of tunable names changes.
    pub fn compute_checksum(&self) -> String {
        format!("{:x}", self.names_hash())
    }

    /// The low half of the [checksum](Self::compute_checksum), without hashing the names again.
    pub(crate) fn checksum(&self) -> u64 {
        self.checksum
    }

    fn names_hash(&self) -> StableHash {
        // Each name is hashed with its own terminator, so `["a", "bc"]` and `["abc"]` differ.
        let mut hasher = StableHasher::new();
        for tune in &self.tunables {
            tune.function.name.hash(&mut hasher);
        }
        hasher.finalize()
    }

    /// Generate a key from a set of inputs
    pub fn generate_key<'a>(&self, inputs: &F::At<'a>) -> K {
        self.key_gen.generate(inputs)
    }

    /// Generate a set of test inputs from a key and reference inputs.
    pub fn generate_inputs<'a>(&self, key: &K, inputs: &F::At<'a>) -> F::At<'a> {
        self.input_gen.generate(key, inputs)
    }

    /// The eviction registered on this set, bound to `key` and the reference `inputs`, if
    /// any: what a benchmark loop runs before each of its samples.
    pub(crate) fn evictor<'i>(&self, key: &K, inputs: &F::At<'i>) -> Option<Box<Evictor<'i>>> {
        let eviction = self.eviction.clone()?;
        let key = key.clone();
        let inputs = inputs.clone();
        Some(Box::new(move || eviction.evict(&key, &inputs)))
    }

    /// The throughput bounds registered on this set, if any.
    pub fn bounds(&self, key: &K, inputs: &F::At<'_>) -> Option<Bounds> {
        self.bounds_gen.as_ref().map(|f| f.generate(key, inputs))
    }
}

#[cfg(serializable)]
/// Trait alias, serializable for the persistent cache and the autotune log
pub trait AutotuneKey:
    Clone
    + Debug
    + PartialEq
    + Eq
    + Hash
    + Display
    + serde::Serialize
    + serde::de::DeserializeOwned
    + Send
    + Sync
    + 'static
{
}
#[cfg(not(serializable))]
/// Trait alias
pub trait AutotuneKey:
    Clone + Debug + PartialEq + Eq + Hash + Display + Send + Sync + 'static
{
}

impl AutotuneKey for String {}