Struct StreamingConfig 

pub struct StreamingConfig {
    pub world_size: usize,
    pub rank: usize,
    pub partition_strategy: PartitionStrategy,
    pub prefetch_buffer_size: usize,
    pub shuffle_seed: Option<u64>,
    pub checkpoint_interval: Option<usize>,
    pub fault_tolerant: bool,
    pub replication_factor: usize,
    pub dynamic_balancing: bool,
}

Configuration for distributed streaming

Fields

world_size: usize

Total number of workers in the distributed system

rank: usize

Current worker rank (0-indexed)

partition_strategy: PartitionStrategy

Partition strategy for distributing data

prefetch_buffer_size: usize

Buffer size for prefetching

shuffle_seed: Option<u64>

Enable deterministic shuffling with seed

checkpoint_interval: Option<usize>

Checkpoint interval (number of samples)

fault_tolerant: bool

Enable fault tolerance

replication_factor: usize

Replication factor for fault tolerance

dynamic_balancing: bool

Dynamic load balancing enabled

Implementations

impl StreamingConfig

pub fn new(world_size: usize, rank: usize) -> Result<Self>

Create a new streaming configuration

pub fn with_partition_strategy(self, strategy: PartitionStrategy) -> Self

Set the partition strategy

pub fn with_prefetch_buffer_size(self, size: usize) -> Self

Set the prefetch buffer size

pub fn with_shuffle_seed(self, seed: u64) -> Self

Set the shuffle seed for deterministic shuffling

pub fn with_checkpointing(self, interval: usize) -> Self

Enable checkpointing with specified interval

pub fn with_fault_tolerance(self, replication_factor: usize) -> Self

Enable fault tolerance with replication

pub fn with_dynamic_balancing(self, enabled: bool) -> Self

Enable dynamic load balancing

pub fn validate(&self) -> Result<()>

Validate the configuration

Trait Implementations

impl Clone for StreamingConfig

fn clone(&self) -> StreamingConfig

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for StreamingConfig

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for StreamingConfig

fn default() -> Self

Returns the “default value” for a type.

impl<'de> Deserialize<'de> for StreamingConfig

fn deserialize<__D>(__deserializer: __D) -> Result<Self, __D::Error>

where __D: Deserializer<'de>,

Deserialize this value from the given Serde deserializer.

impl Serialize for StreamingConfig

fn serialize<__S>(&self, __serializer: __S) -> Result<__S::Ok, __S::Error>

where __S: Serializer,

Serialize this value into the given Serde serializer.