use alloc::vec::Vec;
use crate::dist::collective::CollectiveReductionDType;
use crate::dist::data_parallel::DataParallelDType;
use crate::shapes::error::OperationKind;
use crate::tensor::dtype::{BuiltinDType, ConstDType, DTypeId};
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum ZeROStage {
ZeRO1,
ZeRO2,
ZeRO3,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct FsdpParameterId(u64);
impl FsdpParameterId {
pub const fn new(value: u64) -> Result<Self, FsdpError> {
if value == 0 {
Err(FsdpError::ReservedParameterId)
} else {
Ok(Self(value))
}
}
#[must_use]
pub const fn get(self) -> u64 {
self.0
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct FsdpParameterDescriptor {
id: FsdpParameterId,
unsharded_elements: usize,
sharded_elements: usize,
dtype: DTypeId,
layer_id: usize,
}
impl FsdpParameterDescriptor {
#[must_use]
pub const fn id(self) -> FsdpParameterId {
self.id
}
#[must_use]
pub const fn unsharded_elements(self) -> usize {
self.unsharded_elements
}
#[must_use]
pub const fn sharded_elements(self) -> usize {
self.sharded_elements
}
#[must_use]
pub const fn dtype(self) -> DTypeId {
self.dtype
}
#[must_use]
pub const fn layer_id(self) -> usize {
self.layer_id
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct FsdpMemoryReport {
pub persistent_bytes: usize,
pub transient_bytes: usize,
pub unsharded_full_bytes: usize,
pub memory_reduction_ratio: f64,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FsdpPlan {
stage: ZeROStage,
world_size: usize,
parameters: Vec<FsdpParameterDescriptor>,
}
impl FsdpPlan {
#[must_use]
pub const fn stage(&self) -> ZeROStage {
self.stage
}
#[must_use]
pub const fn world_size(&self) -> usize {
self.world_size
}
#[must_use]
pub fn parameters(&self) -> &[FsdpParameterDescriptor] {
&self.parameters
}
#[must_use]
pub fn memory_report(&self) -> FsdpMemoryReport {
let mut total_unsharded_bytes = 0usize;
let mut total_sharded_bytes = 0usize;
let mut max_layer_transient_bytes = 0usize;
let mut current_layer_id = usize::MAX;
let mut current_layer_bytes = 0usize;
for p in &self.parameters {
let unsharded_b = p
.dtype
.size_bytes(p.unsharded_elements, OperationKind::Storage)
.unwrap_or(0);
let sharded_b = p
.dtype
.size_bytes(p.sharded_elements, OperationKind::Storage)
.unwrap_or(0);
total_unsharded_bytes += unsharded_b;
total_sharded_bytes += sharded_b;
if p.layer_id != current_layer_id {
if current_layer_bytes > max_layer_transient_bytes {
max_layer_transient_bytes = current_layer_bytes;
}
current_layer_id = p.layer_id;
current_layer_bytes = unsharded_b;
} else {
current_layer_bytes += unsharded_b;
}
}
if current_layer_bytes > max_layer_transient_bytes {
max_layer_transient_bytes = current_layer_bytes;
}
let persistent_bytes = match self.stage {
ZeROStage::ZeRO1 => total_unsharded_bytes * 2 + (total_sharded_bytes * 2),
ZeROStage::ZeRO2 => total_unsharded_bytes + (total_sharded_bytes * 3),
ZeROStage::ZeRO3 => total_sharded_bytes * 4,
};
let transient_bytes = match self.stage {
ZeROStage::ZeRO1 | ZeROStage::ZeRO2 => 0,
ZeROStage::ZeRO3 => max_layer_transient_bytes,
};
let unsharded_full_bytes = total_unsharded_bytes * 4;
let memory_reduction_ratio = if persistent_bytes > 0 {
unsharded_full_bytes as f64 / persistent_bytes as f64
} else {
1.0
};
FsdpMemoryReport {
persistent_bytes,
transient_bytes,
unsharded_full_bytes,
memory_reduction_ratio,
}
}
pub fn verify_memory_parity(&self) -> Result<(), FsdpError> {
let report = self.memory_report();
if self.stage == ZeROStage::ZeRO3
&& self.world_size > 1
&& report.persistent_bytes >= report.unsharded_full_bytes
{
return Err(FsdpError::ParityViolation {
reason: "ZeRO-3 persistent bytes must be strictly less than full DP bytes",
});
}
Ok(())
}
}
#[derive(Debug)]
pub struct FsdpPlanBuilder {
stage: ZeROStage,
parameters: Vec<FsdpParameterDescriptor>,
}
impl FsdpPlanBuilder {
#[must_use]
pub fn new(stage: ZeROStage) -> Self {
Self {
stage,
parameters: Vec::new(),
}
}
pub fn push_static<K>(
&mut self,
id: FsdpParameterId,
unsharded_elements: usize,
layer_id: usize,
world_size: usize,
) -> Result<(), FsdpError>
where
K: ConstDType
+ BuiltinDType
+ DataParallelDType
+ CollectiveReductionDType<crate::dist::placement::Mean>,
{
self.push_dyn(id, unsharded_elements, K::DTYPE, layer_id, world_size)
}
pub fn push_dyn(
&mut self,
id: FsdpParameterId,
unsharded_elements: usize,
dtype: DTypeId,
layer_id: usize,
world_size: usize,
) -> Result<(), FsdpError> {
if world_size == 0 {
return Err(FsdpError::InvalidWorldSize);
}
if self.parameters.iter().any(|p| p.id == id) {
return Err(FsdpError::DuplicateParameter { id });
}
let sharded_elements = unsharded_elements.div_ceil(world_size);
self.parameters.push(FsdpParameterDescriptor {
id,
unsharded_elements,
sharded_elements,
dtype,
layer_id,
});
Ok(())
}
pub fn finish(self, world_size: usize) -> Result<FsdpPlan, FsdpError> {
if world_size == 0 {
return Err(FsdpError::InvalidWorldSize);
}
if self.parameters.is_empty() {
return Err(FsdpError::NoParameters);
}
let plan = FsdpPlan {
stage: self.stage,
world_size,
parameters: self.parameters,
};
plan.verify_memory_parity()?;
Ok(plan)
}
}
#[non_exhaustive]
#[derive(thiserror::Error, Debug, Clone, PartialEq, Eq)]
pub enum FsdpError {
#[error("parameter identity zero is reserved")]
ReservedParameterId,
#[error("FSDP world size must be greater than zero")]
InvalidWorldSize,
#[error("parameter identity {id:?} registered more than once")]
DuplicateParameter {
id: FsdpParameterId,
},
#[error("an FSDP plan must contain at least one parameter")]
NoParameters,
#[error("FSDP memory parity violation: {reason}")]
ParityViolation {
reason: &'static str,
},
}