use alloc::vec::Vec;
use half::{bf16, f16};
use typenum::{U1, U2};
use crate::dist::collective::{CollectiveError, CollectiveReductionDType, StreamId};
use crate::dist::mesh::{Data, DeviceMesh, MeshAxis, MeshSpec, Pipeline, TensorParallel};
use crate::dist::placement::{Mean, Partial, PlacementKind, Replicated};
use crate::dist::plan::{
CollectivePlan, CollectivePlanBuilder, CollectiveTag, PlanError, SequenceToken,
};
use crate::shapes::Dyn;
use crate::tensor::dtype::{BuiltinDType, ConstDType, DTypeId};
pub type TwoRankDataParallel = MeshSpec<Data<U2>, TensorParallel<U1>, Pipeline<U1>>;
pub trait DataParallelDType: CollectiveReductionDType<Mean> {}
impl DataParallelDType for f32 {}
impl DataParallelDType for f64 {}
impl DataParallelDType for f16 {}
impl DataParallelDType for bf16 {}
impl DataParallelDType for Dyn {}
pub const fn validate_data_parallel_dtype(dtype: DTypeId) -> Result<(), DataParallelError> {
match dtype {
DTypeId::BF16 | DTypeId::F16 | DTypeId::F32 | DTypeId::F64 => Ok(()),
DTypeId::U8 | DTypeId::U32 | DTypeId::I64 | DTypeId::Q8_0 | DTypeId::Bool => {
Err(DataParallelError::UnsupportedGradientDType { dtype })
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct GradientId(u64);
impl GradientId {
pub const fn new(value: u64) -> Result<Self, DataParallelError> {
if value == 0 {
Err(DataParallelError::ReservedGradientId)
} else {
Ok(Self(value))
}
}
#[must_use]
pub const fn get(self) -> u64 {
self.0
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct GradientDescriptor {
id: GradientId,
elements: usize,
dtype: DTypeId,
sequence: SequenceToken,
}
impl GradientDescriptor {
#[must_use]
pub const fn id(self) -> GradientId {
self.id
}
#[must_use]
pub const fn elements(self) -> usize {
self.elements
}
#[must_use]
pub const fn dtype(self) -> DTypeId {
self.dtype
}
#[must_use]
pub const fn sequence(self) -> SequenceToken {
self.sequence
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DataParallelPlan {
collective: CollectivePlan,
gradients: Vec<GradientDescriptor>,
}
impl DataParallelPlan {
#[must_use]
pub const fn collective_plan(&self) -> &CollectivePlan {
&self.collective
}
#[must_use]
pub fn gradients(&self) -> &[GradientDescriptor] {
&self.gradients
}
#[must_use]
pub fn into_collective_plan(self) -> CollectivePlan {
self.collective
}
}
pub struct DataParallelPlanBuilder<'a> {
inner: CollectivePlanBuilder<'a, TwoRankDataParallel>,
rank: usize,
previous: Option<SequenceToken>,
gradients: Vec<GradientDescriptor>,
}
impl<'a> DataParallelPlanBuilder<'a> {
#[must_use]
pub fn new(mesh: &'a DeviceMesh<TwoRankDataParallel>, rank: usize) -> Self {
Self {
inner: CollectivePlanBuilder::new(mesh),
rank,
previous: None,
gradients: Vec::new(),
}
}
pub fn push_static<K>(
&mut self,
id: GradientId,
elements: usize,
stream: StreamId,
) -> Result<SequenceToken, DataParallelError>
where
K: ConstDType + BuiltinDType + DataParallelDType,
{
let rank = self.rank;
self.push_common(id, elements, K::DTYPE, |inner, tag, dependency| {
inner.push_static_tagged::<
K,
Partial<TwoRankDataParallel, Mean>,
Replicated<TwoRankDataParallel>,
>(
tag,
MeshAxis::Data,
rank,
elements,
stream,
dependency,
)
})
}
pub fn push_dyn(
&mut self,
id: GradientId,
elements: usize,
dtype: DTypeId,
stream: StreamId,
) -> Result<SequenceToken, DataParallelError> {
validate_data_parallel_dtype(dtype)?;
let rank = self.rank;
self.push_common(id, elements, dtype, |inner, tag, dependency| {
inner.push_dyn_tagged(
tag,
MeshAxis::Data,
rank,
elements,
dtype,
PlacementKind::Partial {
reduction: crate::exec::ReduceOp::Mean,
},
PlacementKind::Replicated,
stream,
dependency,
)
})
}
fn push_common(
&mut self,
id: GradientId,
elements: usize,
dtype: DTypeId,
append: impl FnOnce(
&mut CollectivePlanBuilder<'a, TwoRankDataParallel>,
CollectiveTag,
Option<SequenceToken>,
) -> Result<SequenceToken, PlanError>,
) -> Result<SequenceToken, DataParallelError> {
if self.gradients.iter().any(|gradient| gradient.id == id) {
return Err(DataParallelError::DuplicateGradient { id });
}
let sequence = append(&mut self.inner, CollectiveTag::new(id.get()), self.previous)?;
self.gradients.push(GradientDescriptor {
id,
elements,
dtype,
sequence,
});
self.previous = Some(sequence);
Ok(sequence)
}
pub fn finish(self) -> Result<DataParallelPlan, DataParallelError> {
if self.gradients.is_empty() {
return Err(DataParallelError::NoGradients);
}
Ok(DataParallelPlan {
collective: self.inner.finish(),
gradients: self.gradients,
})
}
}
#[non_exhaustive]
#[derive(thiserror::Error, Debug, Clone, PartialEq, Eq)]
pub enum DataParallelError {
#[error("gradient identity zero is reserved for unlabelled collectives")]
ReservedGradientId,
#[error("gradient identity {id:?} appears more than once")]
DuplicateGradient {
id: GradientId,
},
#[error("a data-parallel step must contain at least one gradient")]
NoGradients,
#[error("dtype {dtype:?} cannot represent a data-parallel mean gradient")]
UnsupportedGradientDType {
dtype: DTypeId,
},
#[error(transparent)]
Plan(#[from] PlanError),
#[error(transparent)]
Collective(#[from] CollectiveError),
}