Struct ZeroOptimizationConfig 

pub struct ZeroOptimizationConfig {

    pub stage: ZeroStage,
    pub allgather_bucket_size: Option<u64>,
    pub reduce_bucket_size: Option<u64>,
    pub overlap_comm: Option<bool>,
    pub contiguous_gradients: Option<bool>,
    pub sub_group_size: Option<u32>,
    pub reduce_scatter: Option<bool>,
    pub allgather_partitions: Option<bool>,
    pub stage3_max_live_parameters: Option<u64>,
    pub stage3_max_reuse_distance: Option<u64>,
    pub stage3_prefetch_bucket_size: Option<u64>,
    pub stage3_param_persistence_threshold: Option<u64>,
    pub offload_optimizer: Option<OffloadOptimizerConfig>,
    pub offload_param: Option<OffloadParamConfig>,
}

ZeRO optimization configuration

Fields

stage: ZeroStage

ZeRO stage (0, 1, 2, or 3)

allgather_bucket_size: Option<u64>

Allgather bucket size

reduce_bucket_size: Option<u64>

Reduce bucket size

overlap_comm: Option<bool>

Overlap communications

contiguous_gradients: Option<bool>

Contiguous gradients

sub_group_size: Option<u32>

Sub-group size

reduce_scatter: Option<bool>

Reduce scatter

allgather_partitions: Option<bool>

Allgather partitions

stage3_max_live_parameters: Option<u64>

Stage 3 max live parameters

stage3_max_reuse_distance: Option<u64>

Stage 3 max reuse distance

stage3_prefetch_bucket_size: Option<u64>

Stage 3 prefetch bucket size

stage3_param_persistence_threshold: Option<u64>

Stage 3 parameter persistence threshold

offload_optimizer: Option<OffloadOptimizerConfig>

CPU offloading configuration

offload_param: Option<OffloadParamConfig>

Parameter offloading configuration

Trait Implementations

impl Clone for ZeroOptimizationConfig

fn clone(&self) -> ZeroOptimizationConfig

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for ZeroOptimizationConfig

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for ZeroOptimizationConfig

fn default() -> Self

Returns the “default value” for a type.

impl<'de> Deserialize<'de> for ZeroOptimizationConfig

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl Serialize for ZeroOptimizationConfig

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.