Struct EnhancedDistributedSampler 

pub struct EnhancedDistributedSampler { /* private fields */ }

Enhanced distributed sampler with multi-node support

Implementations

impl EnhancedDistributedSampler

pub fn new( num_replicas: usize, rank: usize, config: DistributedLoadingConfig, ) -> Result<Self>

Create a new enhanced distributed sampler

pub fn initialize(&mut self) -> Result<()>

Initialize the distributed environment and establish connections

pub fn sample_indices_distributed( &self, dataset_len: usize, ) -> Result<Box<dyn Iterator<Item = usize> + Send>>

Sample indices with enhanced distributed coordination

pub fn load_batch_distributed<T, D>( &self, dataset: &D, indices: &[usize], ) -> Result<BatchResult<T>>

where T: Clone + Default + Send + Sync + 'static + Pod + Zeroable + Serialize + for<'de> Deserialize<'de> + Zero, D: Dataset<T> + Send + Sync,

Load data with multi-node coordination and RDMA optimization

pub fn collective_operation( &self, op_type: CollectiveOpType, data: Option<Vec<u8>>, ) -> Result<Option<Vec<u8>>>

Perform collective operation across all nodes

pub fn get_statistics(&self) -> DistributedLoadingStats

Get performance statistics

pub fn shutdown(&mut self) -> Result<()>

Shutdown distributed loading and cleanup resources

Trait Implementations

impl Sampler for EnhancedDistributedSampler

fn sample_indices(&self, len: usize) -> Box<dyn Iterator<Item = usize> + Send>

Generate an iterator over sample indices

fn is_random(&self) -> bool

Check if this sampler produces indices in random order

fn set_seed(&mut self, seed: Option<u64>)

Set random seed if applicable