pub trait ShuffleReader: Send + Sync {
// Required methods
fn read_partition<'life0, 'async_trait>(
&'life0 self,
partition_id: usize,
) -> Pin<Box<dyn Future<Output = Result<Option<Box<dyn RecordBatchStream + Unpin + 'static>>>> + Send + 'async_trait>>
where Self: 'async_trait,
'life0: 'async_trait;
fn partition_size(&self, partition_id: usize) -> Result<usize>;
fn total_loss(&self) -> Option<f64>;
}
Expand description
A reader that can read the shuffled partitions.
Required Methods§
Sourcefn read_partition<'life0, 'async_trait>(
&'life0 self,
partition_id: usize,
) -> Pin<Box<dyn Future<Output = Result<Option<Box<dyn RecordBatchStream + Unpin + 'static>>>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
fn read_partition<'life0, 'async_trait>(
&'life0 self,
partition_id: usize,
) -> Pin<Box<dyn Future<Output = Result<Option<Box<dyn RecordBatchStream + Unpin + 'static>>>> + Send + 'async_trait>>where
Self: 'async_trait,
'life0: 'async_trait,
Read a partition by partition_id will return Ok(None) if partition_size is 0 check reader.partition_size(partition_id) before calling this function
Sourcefn partition_size(&self, partition_id: usize) -> Result<usize>
fn partition_size(&self, partition_id: usize) -> Result<usize>
Get the size of the partition by partition_id
Sourcefn total_loss(&self) -> Option<f64>
fn total_loss(&self) -> Option<f64>
Get the total loss, if the loss is not available, return None, in such case, the caller should sum up the losses from each batch’s metadata. Must be called after all partitions are read.