use alloc::vec::Vec;
use half::{bf16, f16};
use typenum::{U1, U2};
use crate::dist::collective::{CollectiveError, CollectiveReductionDType, StreamId};
use crate::dist::mesh::{Data, DeviceMesh, MeshAxis, MeshSpec, Pipeline, TensorParallel};
use crate::dist::placement::{Partial, PlacementAxis, PlacementKind, Replicated, Sharded, Sum};
use crate::dist::plan::{
CollectivePlan, CollectivePlanBuilder, CollectiveTag, PlanError, SequenceToken,
};
use crate::dist::rule::ShardDivisible;
use crate::shapes::Dyn;
use crate::tensor::dtype::{BuiltinDType, ConstDType, DTypeId};
pub type TwoRankTensorParallel = MeshSpec<Data<U1>, TensorParallel<U2>, Pipeline<U1>>;
pub trait TensorParallelDType: CollectiveReductionDType<Sum> {}
impl TensorParallelDType for f32 {}
impl TensorParallelDType for f64 {}
impl TensorParallelDType for f16 {}
impl TensorParallelDType for bf16 {}
impl TensorParallelDType for Dyn {}
pub trait TwoWayShard: ShardDivisible<U2> {}
impl<T> TwoWayShard for T where T: ShardDivisible<U2> {}
pub const fn validate_tensor_parallel_dtype(dtype: DTypeId) -> Result<(), TensorParallelError> {
match dtype {
DTypeId::BF16 | DTypeId::F16 | DTypeId::F32 | DTypeId::F64 => Ok(()),
DTypeId::U8 | DTypeId::U32 | DTypeId::I64 | DTypeId::Q8_0 | DTypeId::Bool => {
Err(TensorParallelError::UnsupportedTensorDType { dtype })
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum TensorParallelDimension {
OutputFeatures,
InputFeatures,
AttentionHeads,
}
pub const fn validate_two_way_extent(
dimension: TensorParallelDimension,
extent: usize,
) -> Result<usize, TensorParallelError> {
if !extent.is_multiple_of(2) {
Err(TensorParallelError::NonDivisible {
dimension,
extent,
ranks: 2,
})
} else {
Ok(extent / 2)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct TensorParallelId(u64);
impl TensorParallelId {
const MAX: u64 = u64::MAX >> 2;
pub const fn new(value: u64) -> Result<Self, TensorParallelError> {
if value == 0 {
Err(TensorParallelError::ReservedOperationId)
} else if value > Self::MAX {
Err(TensorParallelError::OperationIdTooLarge {
maximum: Self::MAX,
found: value,
})
} else {
Ok(Self(value))
}
}
#[must_use]
pub const fn get(self) -> u64 {
self.0
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum TensorParallelCollective {
ColumnOutputGather {
tensor_axis: usize,
},
RowOutputSum,
AttentionHeadGather {
tensor_axis: usize,
},
}
impl TensorParallelCollective {
const fn tag_code(self) -> u64 {
match self {
Self::ColumnOutputGather { .. } => 1,
Self::RowOutputSum => 2,
Self::AttentionHeadGather { .. } => 3,
}
}
#[must_use]
pub const fn plan_tag(self, id: TensorParallelId) -> CollectiveTag {
CollectiveTag::new((id.get() << 2) | self.tag_code())
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct TensorParallelDescriptor {
id: TensorParallelId,
collective: TensorParallelCollective,
local_elements: usize,
global_elements: usize,
dtype: DTypeId,
sequence: SequenceToken,
}
impl TensorParallelDescriptor {
#[must_use]
pub const fn id(self) -> TensorParallelId {
self.id
}
#[must_use]
pub const fn collective(self) -> TensorParallelCollective {
self.collective
}
#[must_use]
pub const fn local_elements(self) -> usize {
self.local_elements
}
#[must_use]
pub const fn global_elements(self) -> usize {
self.global_elements
}
#[must_use]
pub const fn dtype(self) -> DTypeId {
self.dtype
}
#[must_use]
pub const fn sequence(self) -> SequenceToken {
self.sequence
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TensorParallelPlan {
collective: CollectivePlan,
operations: Vec<TensorParallelDescriptor>,
}
impl TensorParallelPlan {
#[must_use]
pub const fn collective_plan(&self) -> &CollectivePlan {
&self.collective
}
#[must_use]
pub fn operations(&self) -> &[TensorParallelDescriptor] {
&self.operations
}
#[must_use]
pub fn into_collective_plan(self) -> CollectivePlan {
self.collective
}
}
pub struct TensorParallelPlanBuilder<'a> {
inner: CollectivePlanBuilder<'a, TwoRankTensorParallel>,
rank: usize,
previous: Option<SequenceToken>,
operations: Vec<TensorParallelDescriptor>,
}
impl<'a> TensorParallelPlanBuilder<'a> {
#[must_use]
pub fn new(mesh: &'a DeviceMesh<TwoRankTensorParallel>, rank: usize) -> Self {
Self {
inner: CollectivePlanBuilder::new(mesh),
rank,
previous: None,
operations: Vec::new(),
}
}
pub fn push_column_static<K, Axis, Out>(
&mut self,
id: TensorParallelId,
outer_elements: usize,
stream: StreamId,
) -> Result<SequenceToken, TensorParallelError>
where
K: ConstDType + BuiltinDType + TensorParallelDType,
Axis: PlacementAxis,
Out: TwoWayShard,
{
let local_elements = checked_product(
TensorParallelDimension::OutputFeatures,
outer_elements,
<Out as ShardDivisible<U2>>::LOCAL,
)?;
let global_elements = checked_double(local_elements)?;
let rank = self.rank;
let collective = TensorParallelCollective::ColumnOutputGather {
tensor_axis: Axis::INDEX,
};
self.push_common(
id,
collective,
local_elements,
global_elements,
K::DTYPE,
|inner, tag, dependency| {
inner.push_static_tagged::<
K,
Sharded<TwoRankTensorParallel, Axis>,
Replicated<TwoRankTensorParallel>,
>(
tag,
MeshAxis::Tensor,
rank,
local_elements,
stream,
dependency,
)
},
)
}
pub fn push_column_dyn(
&mut self,
id: TensorParallelId,
global_shape: &[usize],
output_axis: usize,
dtype: DTypeId,
stream: StreamId,
) -> Result<SequenceToken, TensorParallelError> {
validate_tensor_parallel_dtype(dtype)?;
let (local_elements, global_elements) = dynamic_sharded_elements(
TensorParallelDimension::OutputFeatures,
global_shape,
output_axis,
)?;
let rank = self.rank;
let collective = TensorParallelCollective::ColumnOutputGather {
tensor_axis: output_axis,
};
self.push_common(
id,
collective,
local_elements,
global_elements,
dtype,
|inner, tag, dependency| {
inner.push_dyn_tagged(
tag,
MeshAxis::Tensor,
rank,
local_elements,
dtype,
PlacementKind::Sharded { axis: output_axis },
PlacementKind::Replicated,
stream,
dependency,
)
},
)
}
pub fn push_row_static<K, In>(
&mut self,
id: TensorParallelId,
output_elements: usize,
stream: StreamId,
) -> Result<SequenceToken, TensorParallelError>
where
K: ConstDType + BuiltinDType + TensorParallelDType,
In: TwoWayShard,
{
let _ = <In as ShardDivisible<U2>>::LOCAL;
let rank = self.rank;
self.push_common(
id,
TensorParallelCollective::RowOutputSum,
output_elements,
output_elements,
K::DTYPE,
|inner, tag, dependency| {
inner.push_static_tagged::<
K,
Partial<TwoRankTensorParallel, Sum>,
Replicated<TwoRankTensorParallel>,
>(
tag,
MeshAxis::Tensor,
rank,
output_elements,
stream,
dependency,
)
},
)
}
pub fn push_row_dyn(
&mut self,
id: TensorParallelId,
global_input_features: usize,
output_elements: usize,
dtype: DTypeId,
stream: StreamId,
) -> Result<SequenceToken, TensorParallelError> {
validate_tensor_parallel_dtype(dtype)?;
validate_two_way_extent(
TensorParallelDimension::InputFeatures,
global_input_features,
)?;
let rank = self.rank;
self.push_common(
id,
TensorParallelCollective::RowOutputSum,
output_elements,
output_elements,
dtype,
|inner, tag, dependency| {
inner.push_dyn_tagged(
tag,
MeshAxis::Tensor,
rank,
output_elements,
dtype,
PlacementKind::Partial {
reduction: crate::exec::ReduceOp::Sum,
},
PlacementKind::Replicated,
stream,
dependency,
)
},
)
}
pub fn push_attention_static<K, Axis, Heads>(
&mut self,
id: TensorParallelId,
elements_per_head: usize,
stream: StreamId,
) -> Result<SequenceToken, TensorParallelError>
where
K: ConstDType + BuiltinDType + TensorParallelDType,
Axis: PlacementAxis,
Heads: TwoWayShard,
{
let local_elements = checked_product(
TensorParallelDimension::AttentionHeads,
elements_per_head,
<Heads as ShardDivisible<U2>>::LOCAL,
)?;
let global_elements = checked_double(local_elements)?;
let rank = self.rank;
let collective = TensorParallelCollective::AttentionHeadGather {
tensor_axis: Axis::INDEX,
};
self.push_common(
id,
collective,
local_elements,
global_elements,
K::DTYPE,
|inner, tag, dependency| {
inner.push_static_tagged::<
K,
Sharded<TwoRankTensorParallel, Axis>,
Replicated<TwoRankTensorParallel>,
>(
tag,
MeshAxis::Tensor,
rank,
local_elements,
stream,
dependency,
)
},
)
}
pub fn push_attention_dyn(
&mut self,
id: TensorParallelId,
global_shape: &[usize],
head_axis: usize,
dtype: DTypeId,
stream: StreamId,
) -> Result<SequenceToken, TensorParallelError> {
validate_tensor_parallel_dtype(dtype)?;
let (local_elements, global_elements) = dynamic_sharded_elements(
TensorParallelDimension::AttentionHeads,
global_shape,
head_axis,
)?;
let rank = self.rank;
let collective = TensorParallelCollective::AttentionHeadGather {
tensor_axis: head_axis,
};
self.push_common(
id,
collective,
local_elements,
global_elements,
dtype,
|inner, tag, dependency| {
inner.push_dyn_tagged(
tag,
MeshAxis::Tensor,
rank,
local_elements,
dtype,
PlacementKind::Sharded { axis: head_axis },
PlacementKind::Replicated,
stream,
dependency,
)
},
)
}
#[allow(clippy::too_many_arguments)]
fn push_common(
&mut self,
id: TensorParallelId,
collective: TensorParallelCollective,
local_elements: usize,
global_elements: usize,
dtype: DTypeId,
append: impl FnOnce(
&mut CollectivePlanBuilder<'a, TwoRankTensorParallel>,
CollectiveTag,
Option<SequenceToken>,
) -> Result<SequenceToken, PlanError>,
) -> Result<SequenceToken, TensorParallelError> {
if self.operations.iter().any(|operation| operation.id == id) {
return Err(TensorParallelError::DuplicateOperation { id });
}
let tag = collective.plan_tag(id);
let sequence = append(&mut self.inner, tag, self.previous)?;
self.operations.push(TensorParallelDescriptor {
id,
collective,
local_elements,
global_elements,
dtype,
sequence,
});
self.previous = Some(sequence);
Ok(sequence)
}
pub fn finish(self) -> Result<TensorParallelPlan, TensorParallelError> {
if self.operations.is_empty() {
return Err(TensorParallelError::NoOperations);
}
Ok(TensorParallelPlan {
collective: self.inner.finish(),
operations: self.operations,
})
}
}
fn dynamic_sharded_elements(
dimension: TensorParallelDimension,
global_shape: &[usize],
axis: usize,
) -> Result<(usize, usize), TensorParallelError> {
let Some(&extent) = global_shape.get(axis) else {
return Err(TensorParallelError::AxisOutOfBounds {
axis,
rank: global_shape.len(),
});
};
validate_two_way_extent(dimension, extent)?;
let global_elements = checked_numel(global_shape)?;
Ok((global_elements / 2, global_elements))
}
fn checked_numel(shape: &[usize]) -> Result<usize, TensorParallelError> {
shape.iter().copied().try_fold(1usize, |elements, extent| {
elements
.checked_mul(extent)
.ok_or(TensorParallelError::ElementCountOverflow)
})
}
fn checked_product(
_dimension: TensorParallelDimension,
lhs: usize,
rhs: usize,
) -> Result<usize, TensorParallelError> {
lhs.checked_mul(rhs)
.ok_or(TensorParallelError::ElementCountOverflow)
}
fn checked_double(elements: usize) -> Result<usize, TensorParallelError> {
elements
.checked_mul(2)
.ok_or(TensorParallelError::ElementCountOverflow)
}
#[non_exhaustive]
#[derive(thiserror::Error, Debug, Clone, PartialEq, Eq)]
pub enum TensorParallelError {
#[error("tensor-parallel operation identity zero is reserved")]
ReservedOperationId,
#[error("tensor-parallel operation identity {found} exceeds maximum {maximum}")]
OperationIdTooLarge {
maximum: u64,
found: u64,
},
#[error("tensor-parallel operation identity {id:?} appears more than once")]
DuplicateOperation {
id: TensorParallelId,
},
#[error("a tensor-parallel plan must contain at least one operation")]
NoOperations,
#[error("dtype {dtype:?} cannot represent tensor-parallel linear or attention values")]
UnsupportedTensorDType {
dtype: DTypeId,
},
#[error("{dimension:?} extent {extent} is not divisible by {ranks} tensor-parallel ranks")]
NonDivisible {
dimension: TensorParallelDimension,
extent: usize,
ranks: usize,
},
#[error("tensor axis {axis} is outside rank {rank}")]
AxisOutOfBounds {
axis: usize,
rank: usize,
},
#[error("tensor-parallel element count overflows usize")]
ElementCountOverflow,
#[error(transparent)]
Plan(#[from] PlanError),
#[error(transparent)]
Collective(#[from] CollectiveError),
}