Struct DistributedWriter 

pub struct DistributedWriter { /* private fields */ }

Distributed writer for parallel file writing

Implementations

impl DistributedWriter

pub fn new<P: AsRef<Path>>(output_dir: P) -> Self

Create a new distributed writer

pub fn num_partitions(self, num: usize) -> Self

Set number of partitions

pub fn partition_naming<F>(self, naming: F) -> Self

where F: Fn(usize) -> String + Send + Sync + 'static,

Set partition naming function

pub fn merge_strategy(self, strategy: MergeStrategy) -> Self

Set merge strategy

pub fn write_parallel<T, F>( &self, data: Vec<T>, writer: F, ) -> Result<Vec<PathBuf>>

where T: Send + 'static + Clone, F: Fn(&T, &mut File) -> Result<()> + Send + Sync + 'static,

Write data in parallel