Trait DataSource

pub trait DataSource:
    Send
    + Sync
    + Debug {
    // Required methods
    fn open(
        &self,
        partition: usize,
        context: Arc<TaskContext>,
    ) -> Result<SendableRecordBatchStream>;
    fn as_any(&self) -> &dyn Any;
    fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter<'_>) -> Result;
    fn output_partitioning(&self) -> Partitioning;
    fn eq_properties(&self) -> EquivalenceProperties;
    fn statistics(&self) -> Result<Statistics>;
    fn with_fetch(&self, _limit: Option<usize>) -> Option<Arc<dyn DataSource>>;
    fn fetch(&self) -> Option<usize>;
    fn try_swapping_with_projection(
        &self,
        _projection: &ProjectionExec,
    ) -> Result<Option<Arc<dyn ExecutionPlan>>>;

    // Provided methods
    fn repartitioned(
        &self,
        _target_partitions: usize,
        _repartition_file_min_size: usize,
        _output_ordering: Option<LexOrdering>,
    ) -> Result<Option<Arc<dyn DataSource>>> { ... }
    fn metrics(&self) -> ExecutionPlanMetricsSet { ... }
}

Common behaviors in Data Sources for both from Files and Memory.

See Also

• DataSourceExec for physical plan implementation
• FileSource for file format implementations (Parquet, Json, etc)

Notes

Requires Debug to assist debugging

Required Methods

fn open( &self, partition: usize, context: Arc<TaskContext>, ) -> Result<SendableRecordBatchStream>

fn as_any(&self) -> &dyn Any

fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter<'_>) -> Result

Format this source for display in explain plans

fn output_partitioning(&self) -> Partitioning

fn eq_properties(&self) -> EquivalenceProperties

fn statistics(&self) -> Result<Statistics>

fn with_fetch(&self, _limit: Option<usize>) -> Option<Arc<dyn DataSource>>

Return a copy of this DataSource with a new fetch limit

fn fetch(&self) -> Option<usize>

fn try_swapping_with_projection( &self, _projection: &ProjectionExec, ) -> Result<Option<Arc<dyn ExecutionPlan>>>

Provided Methods

fn repartitioned( &self, _target_partitions: usize, _repartition_file_min_size: usize, _output_ordering: Option<LexOrdering>, ) -> Result<Option<Arc<dyn DataSource>>>

Return a copy of this DataSource with a new partitioning scheme

fn metrics(&self) -> ExecutionPlanMetricsSet

Implementors

impl DataSource for FileScanConfig

impl DataSource for MemorySourceConfig