Struct ScanBuilder

pub struct ScanBuilder<A> { /* private fields */ }

A struct for building a scan operation.

Implementations

impl<A: 'static + Send> ScanBuilder<A>

pub fn with_filter(self, filter: ExprRef) -> Self

pub fn with_some_filter(self, filter: Option<ExprRef>) -> Self

pub fn with_projection(self, projection: ExprRef) -> Self

pub fn with_row_range(self, row_range: Range<u64>) -> Self

pub fn with_selection(self, selection: Selection) -> Self

pub fn with_row_indices(self, row_indices: Buffer<u64>) -> Self

pub fn with_row_offset(self, row_offset: u64) -> Self

pub fn with_split_by(self, split_by: SplitBy) -> Self

pub fn with_concurrency(self, concurrency: usize) -> Self

The number of row splits to make progress on concurrently, must be greater than 0.

pub fn with_tokio_executor(self, handle: Handle) -> Self

Spawn each CPU task onto the given Tokio runtime.

Note that this is an odd use of the Tokio runtime. Typically, it is used predominantly for I/O bound tasks.

pub fn with_executor(self, executor: Arc<dyn TaskExecutor>) -> Self

pub fn with_metrics(self, metrics: VortexMetrics) -> Self

pub fn with_limit(self, limit: usize) -> Self

pub fn dtype(&self) -> VortexResult<DType>

The DType returned by the scan, after applying the projection.

pub fn map<B: 'static>( self, map_fn: impl Fn(A) -> VortexResult<B> + 'static + Send + Sync, ) -> ScanBuilder<B>

Map each split of the scan. The function will be run on the spawned task.

pub fn build( self, ) -> VortexResult<Vec<BoxFuture<'static, VortexResult<Option<A>>>>>

Constructs a task per row split of the scan, returned as a vector of futures.

pub fn into_stream( self, ) -> VortexResult<impl Stream<Item = VortexResult<A>> + 'static>

pub fn into_thread_pool_iter( self, pool: ThreadPool, ) -> VortexResult<impl Iterator<Item = VortexResult<A>> + Send + 'static>

Returns a blocking iterator over the scan, where tasks perform CPU work on the provided thread pool.

impl ScanBuilder<ArrayRef>

pub fn new(layout_reader: Arc<dyn LayoutReader>) -> Self

pub fn map_to_record_batch(self, schema: SchemaRef) -> ScanBuilder<RecordBatch>

Map the scan into a stream of Arrow RecordBatch.

pub fn into_array_stream(self) -> VortexResult<impl ArrayStream + 'static>

Returns a stream over the scan with each CPU task polled on the current thread as per the behaviour of futures::stream::Buffered.

pub fn into_array_iter( self, ) -> VortexResult<impl ArrayIterator + Send + Clone + 'static>

Returns a thread-safe ArrayIterator that can be cloned and passed to other threads to make progress on the same scan concurrently.

Within each thread, the array chunks will be emitted in the original order they are within the scan. Between threads, the order is not guaranteed.