use super::*;
pub struct OperationCatalogEntry {
pub operation: OperationKind,
pub name: &'static str,
pub family: OperationKind,
pub profile: SemanticProfile,
pub descriptor: &'static str,
pub attributes: &'static str,
pub input_arity: core::ops::RangeInclusive<usize>,
pub output_arity: core::ops::RangeInclusive<usize>,
pub accepted_ranks: core::ops::RangeInclusive<usize>,
pub broadcasting: BroadcastingRule,
pub dtype: DTypeRule,
pub output: OutputRule,
pub same_device: bool,
pub empty: EmptyRule,
pub numeric: NumericRule,
pub gradient: GradientRule,
pub deterministic: bool,
pub layout: LayoutRule,
pub legacy_source: &'static str,
pub capture_eligible: bool,
pub site: ExecutionSite,
}
const fn execution_site(operation: OperationKind) -> ExecutionSite {
match operation {
OperationKind::AddInPlace
| OperationKind::SubInPlace
| OperationKind::MulInPlace
| OperationKind::DivInPlace
| OperationKind::ZeroInPlace
| OperationKind::FillInPlace
| OperationKind::SgdStep
| OperationKind::AdamStep
| OperationKind::AdamWStep => ExecutionSite::Mutation,
OperationKind::ToDevice => ExecutionSite::DeviceTransfer,
OperationKind::RequireGrad | OperationKind::Detach | OperationKind::Backward => {
ExecutionSite::GraphState
}
OperationKind::TensorToBytes
| OperationKind::ToHostFloatScalar
| OperationKind::ToHostFloatVec
| OperationKind::ToHostIntScalar
| OperationKind::ToHostIntVec => ExecutionSite::HostReadback,
OperationKind::Sample | OperationKind::Rnn | OperationKind::Lstm => ExecutionSite::Composed,
OperationKind::TensorFromData
| OperationKind::TensorFromBytes
| OperationKind::Zeros
| OperationKind::Ones
| OperationKind::UniformRandom
| OperationKind::NormalRandom
| OperationKind::VariableZeros
| OperationKind::VariableOnes
| OperationKind::VariableUniformRandom
| OperationKind::VariableNormalRandom
| OperationKind::Full
| OperationKind::Arange
| OperationKind::Linspace => ExecutionSite::Creation,
_ => ExecutionSite::Kernel,
}
}
const fn profile_semantics(
profile: SemanticProfile,
) -> (
BroadcastingRule,
DTypeRule,
OutputRule,
EmptyRule,
NumericRule,
GradientRule,
LayoutRule,
) {
use SemanticProfile::*;
match profile {
Creation => (
BroadcastingRule::None,
DTypeRule::ExplicitOutput,
OutputRule::Created,
EmptyRule::Allowed,
NumericRule::TypedContract,
GradientRule::None,
LayoutRule::FreshContiguous,
),
UnaryFloat => (
BroadcastingRule::None,
DTypeRule::Floating,
OutputRule::Preserve,
EmptyRule::Allowed,
NumericRule::IeeePropagate,
GradientRule::Defined,
LayoutRule::FreshContiguous,
),
BinaryBroadcast => (
BroadcastingRule::Numpy,
DTypeRule::NumericSame,
OutputRule::Broadcast,
EmptyRule::Allowed,
NumericRule::IeeePropagate,
GradientRule::Defined,
LayoutRule::FreshContiguous,
),
Comparison => (
BroadcastingRule::Numpy,
DTypeRule::BooleanResult,
OutputRule::Broadcast,
EmptyRule::Allowed,
NumericRule::IeeePropagate,
GradientRule::None,
LayoutRule::FreshContiguous,
),
Logical => (
BroadcastingRule::Numpy,
DTypeRule::Boolean,
OutputRule::Broadcast,
EmptyRule::Allowed,
NumericRule::NotApplicable,
GradientRule::None,
LayoutRule::FreshContiguous,
),
Mutation => (
BroadcastingRule::None,
DTypeRule::NumericSame,
OutputRule::Preserve,
EmptyRule::Allowed,
NumericRule::IeeePropagate,
GradientRule::Defined,
LayoutRule::PreserveOrMaterialize,
),
Shape => (
BroadcastingRule::TypedContract,
DTypeRule::Preserve,
OutputRule::ShapeAttributes,
EmptyRule::Allowed,
NumericRule::NotApplicable,
GradientRule::Defined,
LayoutRule::ViewWhenPossible,
),
Selection => (
BroadcastingRule::Numpy,
DTypeRule::TypedContract,
OutputRule::Broadcast,
EmptyRule::Allowed,
NumericRule::TypedContract,
GradientRule::Piecewise,
LayoutRule::FreshContiguous,
),
Indexing => (
BroadcastingRule::None,
DTypeRule::TypedContract,
OutputRule::Indexing,
EmptyRule::TypedContract,
NumericRule::CheckedInteger,
GradientRule::Defined,
LayoutRule::FreshContiguous,
),
MatMul => (
BroadcastingRule::TypedContract,
DTypeRule::Floating,
OutputRule::MatMul,
EmptyRule::IdentityOrDefined,
NumericRule::StableAccumulation,
GradientRule::Defined,
LayoutRule::FreshContiguous,
),
Attention => (
BroadcastingRule::TypedContract,
DTypeRule::Floating,
OutputRule::TypedInference,
EmptyRule::TypedContract,
NumericRule::StableAccumulation,
GradientRule::Defined,
LayoutRule::FreshContiguous,
),
Reduction => (
BroadcastingRule::None,
DTypeRule::Floating,
OutputRule::Reduction,
EmptyRule::RejectedWhenReductionIsEmpty,
NumericRule::StableAccumulation,
GradientRule::Defined,
LayoutRule::FreshContiguous,
),
IndexReduction => (
BroadcastingRule::None,
DTypeRule::IndexResult,
OutputRule::Reduction,
EmptyRule::RejectedWhenReductionIsEmpty,
NumericRule::IeeePropagate,
GradientRule::None,
LayoutRule::FreshContiguous,
),
Scan => (
BroadcastingRule::None,
DTypeRule::Preserve,
OutputRule::Preserve,
EmptyRule::Allowed,
NumericRule::StableAccumulation,
GradientRule::Defined,
LayoutRule::FreshContiguous,
),
Normalization => (
BroadcastingRule::TypedContract,
DTypeRule::Floating,
OutputRule::Preserve,
EmptyRule::RejectedWhenReductionIsEmpty,
NumericRule::StableAccumulation,
GradientRule::Defined,
LayoutRule::FreshContiguous,
),
Module | Composite => (
BroadcastingRule::TypedContract,
DTypeRule::TypedContract,
OutputRule::TypedInference,
EmptyRule::TypedContract,
NumericRule::TypedContract,
GradientRule::Defined,
LayoutRule::FreshContiguous,
),
Loss => (
BroadcastingRule::TypedContract,
DTypeRule::Floating,
OutputRule::Reduction,
EmptyRule::RejectedWhenReductionIsEmpty,
NumericRule::StableAccumulation,
GradientRule::Defined,
LayoutRule::FreshContiguous,
),
Quantized => (
BroadcastingRule::TypedContract,
DTypeRule::Quantized,
OutputRule::TypedInference,
EmptyRule::TypedContract,
NumericRule::TypedContract,
GradientRule::None,
LayoutRule::FreshContiguous,
),
Optimizer => (
BroadcastingRule::None,
DTypeRule::Floating,
OutputRule::TypedInference,
EmptyRule::Allowed,
NumericRule::TypedContract,
GradientRule::None,
LayoutRule::FreshContiguous,
),
Transfer => (
BroadcastingRule::None,
DTypeRule::TypedContract,
OutputRule::TypedInference,
EmptyRule::Allowed,
NumericRule::NotApplicable,
GradientRule::TypedContract,
LayoutRule::PreserveOrMaterialize,
),
Autograd => (
BroadcastingRule::None,
DTypeRule::Preserve,
OutputRule::Preserve,
EmptyRule::Allowed,
NumericRule::NotApplicable,
GradientRule::None,
LayoutRule::PreserveOrMaterialize,
),
}
}
#[allow(clippy::too_many_arguments)]
pub(super) const fn entry(
operation: OperationKind,
name: &'static str,
family: OperationKind,
profile: SemanticProfile,
descriptor: &'static str,
attributes: &'static str,
min_arity: usize,
max_arity: usize,
legacy_source: &'static str,
) -> OperationCatalogEntry {
let (mut broadcasting, dtype, mut output, mut empty, numeric, mut gradient, layout) =
profile_semantics(profile);
let accepted_ranks = match operation {
OperationKind::MatMulExact | OperationKind::QuantizedMatMul => 2..=usize::MAX,
OperationKind::Dot | OperationKind::Outer => 1..=1,
OperationKind::BatchedMatMul | OperationKind::Addmm => 2..=3,
OperationKind::Rnn | OperationKind::Lstm => 2..=3,
OperationKind::Conv1dExact => 2..=3,
OperationKind::Conv2dExact
| OperationKind::ConvTranspose2d
| OperationKind::MaxPool2d
| OperationKind::AvgPool2d
| OperationKind::AdaptiveAvgPool2dExact => 3..=4,
OperationKind::Softmax
| OperationKind::GroupNorm
| OperationKind::InstanceNorm
| OperationKind::LayerNorm
| OperationKind::BatchNorm
| OperationKind::RmsNorm => 1..=usize::MAX,
_ => 0..=usize::MAX,
};
let output_arity = match operation {
OperationKind::TopK => 2..=2,
OperationKind::Chunk | OperationKind::Split => 0..=usize::MAX,
OperationKind::Rnn => 2..=2,
OperationKind::Lstm => 3..=3,
OperationKind::ToHostFloatScalar
| OperationKind::ToHostFloatVec
| OperationKind::ToHostIntScalar
| OperationKind::ToHostIntVec
| OperationKind::TensorToBytes => 0..=0,
OperationKind::Backward => 0..=0,
OperationKind::AdamStep | OperationKind::AdamWStep => 3..=3,
_ => 1..=1,
};
if matches!(
operation,
OperationKind::BroadcastAs | OperationKind::BroadcastLeft
) {
broadcasting = BroadcastingRule::ExplicitTarget;
}
if matches!(operation, OperationKind::ToDType) {
output = OutputRule::ExplicitDType;
} else if matches!(operation, OperationKind::ToDevice) {
output = OutputRule::Preserve;
} else if matches!(
operation,
OperationKind::Dot
| OperationKind::Outer
| OperationKind::Addmm
| OperationKind::ScaledDotProductAttention
| OperationKind::Linear
) {
output = OutputRule::TypedInference;
} else if matches!(operation, OperationKind::QuantizedMatMul) {
output = OutputRule::MatMul;
} else if matches!(
operation,
OperationKind::Conv1dExact
| OperationKind::Conv2dExact
| OperationKind::ConvTranspose2d
| OperationKind::MaxPool2d
| OperationKind::AvgPool2d
| OperationKind::AdaptiveAvgPool2dExact
| OperationKind::Rnn
| OperationKind::Lstm
) {
output = OutputRule::ShapeAttributes;
} else if matches!(
operation,
OperationKind::LayerNorm
| OperationKind::BatchNorm
| OperationKind::GroupNorm
| OperationKind::InstanceNorm
| OperationKind::RmsNorm
| OperationKind::Dropout
| OperationKind::MaskedFill
| OperationKind::Scatter
) {
output = OutputRule::Preserve;
} else if matches!(
operation,
OperationKind::ToHostFloatScalar
| OperationKind::ToHostFloatVec
| OperationKind::ToHostIntScalar
| OperationKind::ToHostIntVec
| OperationKind::TensorToBytes
) {
output = OutputRule::HostValue;
}
if matches!(
operation,
OperationKind::Step
| OperationKind::Sign
| OperationKind::Floor
| OperationKind::Ceil
| OperationKind::Round
| OperationKind::Trunc
| OperationKind::Frac
) {
gradient = GradientRule::Undefined;
}
if matches!(operation, OperationKind::Scatter) {
gradient = GradientRule::Undefined;
} else if matches!(
operation,
OperationKind::TensorToBytes
| OperationKind::ToHostFloatScalar
| OperationKind::ToHostFloatVec
| OperationKind::ToHostIntScalar
| OperationKind::ToHostIntVec
) {
gradient = GradientRule::None;
} else if matches!(operation, OperationKind::ToDevice) {
gradient = GradientRule::Defined;
}
if matches!(
operation,
OperationKind::SumAll
| OperationKind::SumDim
| OperationKind::SumKeepDim
| OperationKind::ProdAll
| OperationKind::ProdDim
| OperationKind::Cumsum
) {
empty = EmptyRule::IdentityOrDefined;
}
OperationCatalogEntry {
operation,
name,
family,
profile,
descriptor,
attributes,
input_arity: min_arity..=max_arity,
output_arity,
accepted_ranks,
broadcasting,
dtype,
output,
same_device: !matches!(
profile,
SemanticProfile::Creation | SemanticProfile::Transfer
),
empty,
numeric,
gradient,
deterministic: !matches!(
operation,
OperationKind::UniformRandom
| OperationKind::NormalRandom
| OperationKind::VariableUniformRandom
| OperationKind::VariableNormalRandom
| OperationKind::Sample
| OperationKind::Dropout
| OperationKind::Scatter
| OperationKind::TopK
| OperationKind::Argsort
),
layout,
legacy_source,
capture_eligible: !matches!(
profile,
SemanticProfile::Optimizer | SemanticProfile::Transfer
) && !matches!(
operation,
OperationKind::Sample | OperationKind::Backward
),
site: execution_site(operation),
}
}
macro_rules! define_catalog {
($(($variant:ident, $name:literal, $family:ident, $profile:ident, $attrs:ident, $min:expr, $max:expr, $legacy:literal),)*) => {
pub static OPERATION_CATALOG: &[OperationCatalogEntry] = &[
$(entry(
OperationKind::$variant,
$name,
OperationKind::$family,
SemanticProfile::$profile,
concat!("Descriptor<op::", stringify!($variant), ">"),
stringify!($attrs),
$min,
$max,
$legacy,
),)*
];
pub mod op {
$(
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash)]
pub struct $variant;
)*
}
$(
impl Operation for op::$variant {
type Attributes = $attrs;
const KEY: OperationKey = OperationKey {
namespace: Cow::Borrowed("incin"),
name: Cow::Borrowed($name),
version: 1,
};
fn infer_outputs(
attributes: &Self::Attributes,
inputs: &[LogicalTensorMeta],
) -> Result<Vec<LogicalTensorMeta>, DescriptorError> {
let row = catalog_entry(OperationKind::$variant).ok_or(
DescriptorError::MissingCatalogEntry {
operation: OperationKind::$variant,
},
)?;
infer_outputs(OperationKind::$variant, row, attributes, inputs)
}
fn infer_invocation(
attributes: Self::Attributes,
inputs: Vec<LogicalTensorMeta>,
) -> Result<ValidatedInvocation<Self>, DescriptorError> {
ValidatedInvocation::<Self>::infer_runtime(attributes, inputs)
}
fn infer_invocation_typed<S: crate::shapes::Shape>(
attributes: Self::Attributes,
inputs: Vec<LogicalTensorMeta>,
expected: &crate::shapes::ShapeValue<S>,
) -> Result<ValidatedInvocation<Self>, DescriptorError> {
ValidatedInvocation::<Self>::infer_typed(attributes, inputs, expected)
}
}
impl CanonicalOperation for op::$variant {
const ID: OperationKind = OperationKind::$variant;
}
)*
};
}
pub(crate) use define_catalog;