Trait DistributedDataset 

pub trait DistributedDataset: Send + Sync {
    type Item;
    type PartitionStrategy;

    // Required methods
    fn size(&self) -> u64;
    fn partition_count(&self) -> u32;
    fn partition<'a>(
        &'a mut self,
        cluster: &'a dyn DistributedCluster,
        strategy: Self::PartitionStrategy,
    ) -> BoxFuture<'a, Result<Vec<DistributedPartition<Self::Item>>>>;
    fn get_partition(
        &self,
        partition_id: u32,
    ) -> BoxFuture<'_, Result<DistributedPartition<Self::Item>>>;
    fn repartition<'a>(
        &'a mut self,
        cluster: &'a dyn DistributedCluster,
        new_strategy: Self::PartitionStrategy,
    ) -> BoxFuture<'a, Result<()>>;
    fn collect(
        &self,
        cluster: &dyn DistributedCluster,
    ) -> BoxFuture<'_, Result<Vec<Self::Item>>>;
    fn partition_assignment(&self) -> HashMap<NodeId, Vec<u32>>;
}

Trait for datasets that can be distributed across cluster nodes

Required Associated Types

type Item

Associated type for data items

type PartitionStrategy

Associated type for partitioning strategy

Required Methods

fn size(&self) -> u64

Get the total size of the dataset

fn partition_count(&self) -> u32

Get the number of partitions

fn partition<'a>( &'a mut self, cluster: &'a dyn DistributedCluster, strategy: Self::PartitionStrategy, ) -> BoxFuture<'a, Result<Vec<DistributedPartition<Self::Item>>>>

Partition the dataset across cluster nodes

fn get_partition( &self, partition_id: u32, ) -> BoxFuture<'_, Result<DistributedPartition<Self::Item>>>

Get a specific partition

fn repartition<'a>( &'a mut self, cluster: &'a dyn DistributedCluster, new_strategy: Self::PartitionStrategy, ) -> BoxFuture<'a, Result<()>>

Repartition the dataset with a new strategy

fn collect( &self, cluster: &dyn DistributedCluster, ) -> BoxFuture<'_, Result<Vec<Self::Item>>>

Collect all partitions back to coordinator

fn partition_assignment(&self) -> HashMap<NodeId, Vec<u32>>

Get partition assignment for nodes

Implementors

impl DistributedDataset for DistributedNumericalDataset

type Item = Vec<f64>

type PartitionStrategy = PartitioningStrategy