Struct DistributedConfig 

pub struct DistributedConfig {
    pub world_size: usize,
    pub rank: usize,
    pub num_shards: usize,
    pub shuffle_shards: bool,
    pub seed: Option<u64>,
    pub drop_last: bool,
    pub enable_distributed_cache: bool,
}

Configuration for distributed loading

Fields

world_size: usize

Total number of workers/processes

rank: usize

Rank of this worker (0 to world_size-1)

num_shards: usize

Number of shards to create

shuffle_shards: bool

Whether to shuffle shards

seed: Option<u64>

Random seed for shuffling

drop_last: bool

Whether to drop last incomplete batch

enable_distributed_cache: bool

Whether to use distributed caching

Implementations

impl DistributedConfig

pub fn new(world_size: usize, rank: usize) -> Result<Self>

Create a new distributed configuration

Arguments

• world_size - Total number of workers
• rank - Rank of this worker

Returns

• DistributedConfig - Configuration instance

pub fn with_shards(self, num_shards: usize) -> Self

Set number of shards

pub fn with_shuffle(self, shuffle: bool, seed: Option<u64>) -> Self

Enable shard shuffling

pub fn with_drop_last(self, drop_last: bool) -> Self

Set drop_last behavior

Trait Implementations

impl Clone for DistributedConfig

fn clone(&self) -> DistributedConfig

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for DistributedConfig

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.