Trait FileSource

pub trait FileSource: Send + Sync {
    // Required methods
    fn create_file_opener(
        &self,
        object_store: Arc<dyn ObjectStore>,
        base_config: &FileScanConfig,
        partition: usize,
    ) -> Arc<dyn FileOpener>;
    fn as_any(&self) -> &dyn Any;
    fn with_batch_size(&self, batch_size: usize) -> Arc<dyn FileSource>;
    fn with_schema(&self, schema: SchemaRef) -> Arc<dyn FileSource>;
    fn with_projection(&self, config: &FileScanConfig) -> Arc<dyn FileSource>;
    fn with_statistics(&self, statistics: Statistics) -> Arc<dyn FileSource>;
    fn metrics(&self) -> &ExecutionPlanMetricsSet;
    fn statistics(&self) -> Result<Statistics>;
    fn file_type(&self) -> &str;

    // Provided methods
    fn fmt_extra(&self, _t: DisplayFormatType, _f: &mut Formatter<'_>) -> Result { ... }
    fn repartitioned(
        &self,
        target_partitions: usize,
        repartition_file_min_size: usize,
        output_ordering: Option<LexOrdering>,
        config: &FileScanConfig,
    ) -> Result<Option<FileScanConfig>> { ... }
    fn try_pushdown_filters(
        &self,
        filters: Vec<Arc<dyn PhysicalExpr>>,
        _config: &ConfigOptions,
    ) -> Result<FilterPushdownPropagation<Arc<dyn FileSource>>> { ... }
    fn with_schema_adapter_factory(
        &self,
        _factory: Arc<dyn SchemaAdapterFactory>,
    ) -> Result<Arc<dyn FileSource>> { ... }
    fn schema_adapter_factory(&self) -> Option<Arc<dyn SchemaAdapterFactory>> { ... }
}

file format specific behaviors for elements in DataSource

See more details on specific implementations:

• ArrowSource
• AvroSource
• CsvSource
• JsonSource
• ParquetSource

Required Methods

fn create_file_opener( &self, object_store: Arc<dyn ObjectStore>, base_config: &FileScanConfig, partition: usize, ) -> Arc<dyn FileOpener>

Creates a dyn FileOpener based on given parameters

fn as_any(&self) -> &dyn Any

Any

fn with_batch_size(&self, batch_size: usize) -> Arc<dyn FileSource>

Initialize new type with batch size configuration

fn with_schema(&self, schema: SchemaRef) -> Arc<dyn FileSource>

Initialize new instance with a new schema

fn with_projection(&self, config: &FileScanConfig) -> Arc<dyn FileSource>

Initialize new instance with projection information

fn with_statistics(&self, statistics: Statistics) -> Arc<dyn FileSource>

Initialize new instance with projected statistics

fn metrics(&self) -> &ExecutionPlanMetricsSet

Return execution plan metrics

fn statistics(&self) -> Result<Statistics>

Return projected statistics

fn file_type(&self) -> &str

String representation of file source such as “csv”, “json”, “parquet”

Provided Methods

fn fmt_extra(&self, _t: DisplayFormatType, _f: &mut Formatter<'_>) -> Result

Format FileType specific information

fn repartitioned( &self, target_partitions: usize, repartition_file_min_size: usize, output_ordering: Option<LexOrdering>, config: &FileScanConfig, ) -> Result<Option<FileScanConfig>>

If supported by the FileSource, redistribute files across partitions according to their size. Allows custom file formats to implement their own repartitioning logic.

The default implementation uses FileGroupPartitioner. See that struct for more details.

fn try_pushdown_filters( &self, filters: Vec<Arc<dyn PhysicalExpr>>, _config: &ConfigOptions, ) -> Result<FilterPushdownPropagation<Arc<dyn FileSource>>>

Try to push down filters into this FileSource. See ExecutionPlan::handle_child_pushdown_result for more details.

fn with_schema_adapter_factory( &self, _factory: Arc<dyn SchemaAdapterFactory>, ) -> Result<Arc<dyn FileSource>>

Set optional schema adapter factory.

SchemaAdapterFactory allows user to specify how fields from the file get mapped to that of the table schema. If you implement this method, you should also implement schema_adapter_factory.

The default implementation returns a not implemented error.

fn schema_adapter_factory(&self) -> Option<Arc<dyn SchemaAdapterFactory>>

Returns the current schema adapter factory if set

Default implementation returns None.

Implementors