use super::*;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct SequenceToken(u64);
impl SequenceToken {
#[must_use]
pub const fn get(self) -> u64 {
self.0
}
}
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct CollectiveTag(u64);
impl CollectiveTag {
pub const ANONYMOUS: Self = Self(0);
#[must_use]
pub const fn new(value: u64) -> Self {
Self(value)
}
#[must_use]
pub const fn get(self) -> u64 {
self.0
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CollectiveDescriptor {
tag: CollectiveTag,
group: GroupId,
kind: CollectiveKind,
input_elements: usize,
output_elements: usize,
input_bytes: usize,
output_bytes: usize,
dtype: DTypeId,
source: PlacementKind,
destination: PlacementKind,
sequence: SequenceToken,
stream: StreamId,
depends_on: Option<SequenceToken>,
}
impl CollectiveDescriptor {
#[must_use]
pub const fn tag(&self) -> CollectiveTag {
self.tag
}
#[must_use]
pub const fn group(&self) -> GroupId {
self.group
}
#[must_use]
pub const fn kind(&self) -> CollectiveKind {
self.kind
}
#[must_use]
pub const fn input_elements(&self) -> usize {
self.input_elements
}
#[must_use]
pub const fn output_elements(&self) -> usize {
self.output_elements
}
#[must_use]
pub const fn input_bytes(&self) -> usize {
self.input_bytes
}
#[must_use]
pub const fn output_bytes(&self) -> usize {
self.output_bytes
}
#[must_use]
pub const fn dtype(&self) -> DTypeId {
self.dtype
}
#[must_use]
pub const fn source(&self) -> PlacementKind {
self.source
}
#[must_use]
pub const fn destination(&self) -> PlacementKind {
self.destination
}
#[must_use]
pub const fn sequence(&self) -> SequenceToken {
self.sequence
}
#[must_use]
pub const fn stream(&self) -> StreamId {
self.stream
}
pub const fn depends_on(&self) -> Option<SequenceToken> {
self.depends_on
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CollectivePlan {
mesh: MeshId,
descriptors: Vec<CollectiveDescriptor>,
hash: u64,
}
impl CollectivePlan {
#[must_use]
pub const fn mesh_id(&self) -> MeshId {
self.mesh
}
#[must_use]
pub fn descriptors(&self) -> &[CollectiveDescriptor] {
&self.descriptors
}
#[must_use]
pub const fn hash(&self) -> u64 {
self.hash
}
#[must_use]
pub fn summary(&self) -> PlanSummary {
PlanSummary {
mesh: self.mesh,
hash: self.hash,
collectives: self.descriptors.len(),
}
}
}
#[derive(Debug)]
pub struct CollectivePlanBuilder<'a, M: ValidMesh> {
mesh: &'a DeviceMesh<M>,
descriptors: Vec<CollectiveDescriptor>,
}
#[diagnostic::on_unimplemented(
message = "`{Self}` to `{To}` is not a collective supported for dtype `{K}`",
label = "no statically valid collective transition",
note = "integer mean reductions require a floating dtype; local-only placement changes do not belong in a collective plan"
)]
pub trait PlannedCollectiveTransition<K: CollectiveDType, To: Placement>:
LegalTransition<To>
{
}
impl<K, M, Axis> PlannedCollectiveTransition<K, Replicated<M>> for Sharded<M, Axis>
where
K: CollectiveDType,
M: ValidMesh,
Axis: PlacementAxis,
{
}
impl<K, M, R> PlannedCollectiveTransition<K, Replicated<M>> for Partial<M, R>
where
K: CollectiveReductionDType<R>,
M: ValidMesh,
R: PartialReduction,
{
}
impl<K, M, R, Axis> PlannedCollectiveTransition<K, Sharded<M, Axis>> for Partial<M, R>
where
K: CollectiveReductionDType<R>,
M: ValidMesh,
R: PartialReduction,
Axis: PlacementAxis,
{
}
impl<'a, M: ValidMesh> CollectivePlanBuilder<'a, M> {
#[must_use]
pub fn new(mesh: &'a DeviceMesh<M>) -> Self {
Self {
mesh,
descriptors: Vec::new(),
}
}
pub fn push_static<K, From, To>(
&mut self,
axis: MeshAxis,
rank: usize,
input_elements: usize,
stream: StreamId,
depends_on: Option<SequenceToken>,
) -> Result<SequenceToken, PlanError>
where
K: ConstDType + BuiltinDType + CollectiveDType,
From: ConstPlacement + PlannedCollectiveTransition<K, To> + PlacementOn<M>,
To: ConstPlacement + PlacementOn<M>,
{
self.push_static_tagged::<K, From, To>(
CollectiveTag::ANONYMOUS,
axis,
rank,
input_elements,
stream,
depends_on,
)
}
#[allow(clippy::too_many_arguments)]
pub fn push_static_tagged<K, From, To>(
&mut self,
tag: CollectiveTag,
axis: MeshAxis,
rank: usize,
input_elements: usize,
stream: StreamId,
depends_on: Option<SequenceToken>,
) -> Result<SequenceToken, PlanError>
where
K: ConstDType + BuiltinDType + CollectiveDType,
From: ConstPlacement + PlannedCollectiveTransition<K, To> + PlacementOn<M>,
To: ConstPlacement + PlacementOn<M>,
{
self.push_checked(
tag,
axis,
rank,
input_elements,
K::DTYPE,
From::PLACEMENT,
To::PLACEMENT,
From::TRANSITION,
stream,
depends_on,
)
}
#[allow(clippy::too_many_arguments)]
pub fn push_dyn(
&mut self,
axis: MeshAxis,
rank: usize,
input_elements: usize,
dtype: DTypeId,
source: PlacementKind,
destination: PlacementKind,
stream: StreamId,
depends_on: Option<SequenceToken>,
) -> Result<SequenceToken, PlanError> {
self.push_dyn_tagged(
CollectiveTag::ANONYMOUS,
axis,
rank,
input_elements,
dtype,
source,
destination,
stream,
depends_on,
)
}
#[allow(clippy::too_many_arguments)]
pub fn push_dyn_tagged(
&mut self,
tag: CollectiveTag,
axis: MeshAxis,
rank: usize,
input_elements: usize,
dtype: DTypeId,
source: PlacementKind,
destination: PlacementKind,
stream: StreamId,
depends_on: Option<SequenceToken>,
) -> Result<SequenceToken, PlanError> {
validate_collective_dtype(dtype)?;
let transition = crate::dist::validate_transition(source, destination)?;
self.push_checked(
tag,
axis,
rank,
input_elements,
dtype,
source,
destination,
transition,
stream,
depends_on,
)
}
#[allow(clippy::too_many_arguments)]
fn push_checked(
&mut self,
tag: CollectiveTag,
axis: MeshAxis,
rank: usize,
input_elements: usize,
dtype: DTypeId,
source: PlacementKind,
destination: PlacementKind,
transition: PlacementTransition,
stream: StreamId,
depends_on: Option<SequenceToken>,
) -> Result<SequenceToken, PlanError> {
validate_collective_dtype(dtype)?;
let runtime_transition = crate::dist::validate_transition(source, destination)?;
if runtime_transition != transition {
return Err(PlanError::TransitionMismatch {
typed: transition,
runtime: runtime_transition,
});
}
if let Some(dependency) = depends_on
&& dependency.0 >= self.descriptors.len() as u64
{
return Err(PlanError::UnknownDependency {
dependency,
next: self.descriptors.len(),
});
}
let members = self
.mesh
.groups()
.group(axis, rank)
.ok_or(PlanError::RankOutOfRange {
rank,
world: M::WORLD,
})?;
let group = GroupId::new(group_token(self.mesh.id(), axis, &members), members.len())?;
let kind = kind_for_transition(transition, source)?;
if let CollectiveKind::AllReduce(op) | CollectiveKind::ReduceScatter(op) = kind {
validate_collective_reduction(dtype, op)?;
}
if matches!(
kind,
CollectiveKind::AllGather | CollectiveKind::ReduceScatter(_)
) && axis != MeshAxis::Tensor
{
return Err(PlanError::WrongAxis {
kind,
expected: MeshAxis::Tensor,
found: axis,
});
}
let output_elements = output_elements(kind, input_elements, group.ranks())?;
let input_bytes = dtype.size_bytes(input_elements, OperationKind::Storage)?;
let output_bytes = dtype.size_bytes(output_elements, OperationKind::Storage)?;
let sequence = SequenceToken(
u64::try_from(self.descriptors.len()).map_err(|_| PlanError::SequenceOverflow)?,
);
self.descriptors.push(CollectiveDescriptor {
tag,
group,
kind,
input_elements,
output_elements,
input_bytes,
output_bytes,
dtype,
source,
destination,
sequence,
stream,
depends_on,
});
Ok(sequence)
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn push_send_recv_tagged(
&mut self,
tag: CollectiveTag,
rank: usize,
input_elements: usize,
dtype: DTypeId,
source_rank: usize,
destination_rank: usize,
source_stage: usize,
destination_stage: usize,
stream: StreamId,
depends_on: Option<SequenceToken>,
) -> Result<SequenceToken, PlanError> {
validate_collective_dtype(dtype)?;
if let Some(dependency) = depends_on
&& dependency.0 >= self.descriptors.len() as u64
{
return Err(PlanError::UnknownDependency {
dependency,
next: self.descriptors.len(),
});
}
let members = self.mesh.groups().group(MeshAxis::Pipeline, rank).ok_or(
PlanError::RankOutOfRange {
rank,
world: M::WORLD,
},
)?;
let group = GroupId::new(
group_token(self.mesh.id(), MeshAxis::Pipeline, &members),
members.len(),
)?;
validate_peer("source", source_rank, group.ranks())?;
validate_peer("destination", destination_rank, group.ranks())?;
if source_rank == destination_rank {
return Err(CollectiveError::SamePeer { rank: source_rank }.into());
}
let input_bytes = dtype.size_bytes(input_elements, OperationKind::Storage)?;
let sequence = SequenceToken(
u64::try_from(self.descriptors.len()).map_err(|_| PlanError::SequenceOverflow)?,
);
self.descriptors.push(CollectiveDescriptor {
tag,
group,
kind: CollectiveKind::SendRecv {
source: source_rank,
destination: destination_rank,
},
input_elements,
output_elements: input_elements,
input_bytes,
output_bytes: input_bytes,
dtype,
source: PlacementKind::PipelineStage {
index: source_stage,
},
destination: PlacementKind::PipelineStage {
index: destination_stage,
},
sequence,
stream,
depends_on,
});
Ok(sequence)
}
#[must_use]
pub fn finish(self) -> CollectivePlan {
let hash = plan_hash(self.mesh.id(), &self.descriptors);
CollectivePlan {
mesh: self.mesh.id(),
descriptors: self.descriptors,
hash,
}
}
}