Struct CsvSource 

pub struct CsvSource { /* private fields */ }

A Config for CsvOpener

Example: create a DataSourceExec for CSV

let options = CsvOptions {
    has_header: Some(true),
    delimiter: b',',
    quote: b'"',
    newlines_in_values: Some(true), // The file contains newlines in values
    ..Default::default()
};
let source = Arc::new(CsvSource::new(file_schema.clone())
    .with_csv_options(options)
    .with_terminator(Some(b'#'))
);
// Create a DataSourceExec for reading the first 100MB of `file1.csv`
let config = FileScanConfigBuilder::new(object_store_url, source)
    .with_file(PartitionedFile::new("file1.csv", 100*1024*1024))
    .build();
let exec = (DataSourceExec::from_data_source(config));

Implementations

impl CsvSource

pub fn new(table_schema: impl Into<TableSchema>) -> Self

Returns a CsvSource

pub fn with_csv_options(self, options: CsvOptions) -> Self

Sets the CSV options

pub fn has_header(&self) -> bool

true if the first line of each file is a header

pub fn truncate_rows(&self) -> bool

pub fn delimiter(&self) -> u8

A column delimiter

pub fn quote(&self) -> u8

The quote character

pub fn terminator(&self) -> Option<u8>

The line terminator

pub fn comment(&self) -> Option<u8>

Lines beginning with this byte are ignored.

pub fn escape(&self) -> Option<u8>

The escape character

pub fn with_escape(&self, escape: Option<u8>) -> Self

Initialize a CsvSource with escape

pub fn with_terminator(&self, terminator: Option<u8>) -> Self

Initialize a CsvSource with terminator

pub fn with_comment(&self, comment: Option<u8>) -> Self

Initialize a CsvSource with comment

pub fn with_truncate_rows(&self, truncate_rows: bool) -> Self

Whether to support truncate rows when read csv file

pub fn newlines_in_values(&self) -> bool

Whether values may contain newline characters

impl CsvSource

pub fn try_from_proto( node: &PhysicalPlanNode, ctx: &ExecutionPlanDecodeCtx<'_>, ) -> Result<Arc<dyn ExecutionPlan>>

Reconstructs a DataSourceExec from a protobuf CsvScan.

Custom line terminators are not represented in the wire format.

Trait Implementations

impl Clone for CsvSource

fn clone(&self) -> CsvSource

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for CsvSource

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl FileSource for CsvSource

fn try_to_proto( &self, base: &FileScanConfig, ctx: &ExecutionPlanEncodeCtx<'_>, ) -> Result<Option<PhysicalPlanNode>>

Emit a CsvScan node wrapping the shared base config and CSV options.

fn create_file_opener( &self, object_store: Arc<dyn ObjectStore>, base_config: &FileScanConfig, partition_index: usize, ) -> Result<Arc<dyn FileOpener>>

Creates a dyn FileOpener based on given parameters.

fn table_schema(&self) -> &TableSchema

Returns the table schema for the overall table (including partition columns, if any)

fn with_batch_size(&self, batch_size: usize) -> Arc<dyn FileSource>

Initialize new type with batch size configuration

fn try_pushdown_projection( &self, projection: &ProjectionExprs, ) -> Result<Option<Arc<dyn FileSource>>>

Try to push down a projection into this FileSource.

fn projection(&self) -> Option<&ProjectionExprs>

Return the projection that will be applied to the output stream on top of Self::table_schema.

fn metrics(&self) -> &ExecutionPlanMetricsSet

Return execution plan metrics

fn file_type(&self) -> &str

String representation of file source such as “csv”, “json”, “parquet”

fn supports_repartitioning(&self) -> bool

Returns whether this file source supports repartitioning files by byte ranges.

fn fmt_extra(&self, t: DisplayFormatType, f: &mut Formatter<'_>) -> Result

Format FileType specific information

fn apply_expressions( &self, f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>, ) -> Result<TreeNodeRecursion>

Apply a function to all physical expressions used by this file source.

fn create_morselizer( &self, object_store: Arc<dyn ObjectStore>, base_config: &FileScanConfig, partition: usize, ) -> Result<Box<dyn Morselizer>, DataFusionError>

Creates a dyn Morselizer based on given parameters.

fn filter(&self) -> Option<Arc<dyn PhysicalExpr>>

Returns the filter expression that will be applied during the file scan.

fn repartitioned( &self, target_partitions: usize, repartition_file_min_size: usize, output_ordering: Option<LexOrdering>, config: &FileScanConfig, ) -> Result<Option<FileScanConfig>, DataFusionError>

If supported by the FileSource, redistribute files across partitions according to their size. Allows custom file formats to implement their own repartitioning logic.

fn try_pushdown_filters( &self, filters: Vec<Arc<dyn PhysicalExpr>>, _config: &ConfigOptions, ) -> Result<FilterPushdownPropagation<Arc<dyn FileSource>>, DataFusionError>

Try to push down filters into this FileSource.

fn try_pushdown_sort( &self, order: &[PhysicalSortExpr], eq_properties: &EquivalenceProperties, ) -> Result<SortOrderPushdownResult<Arc<dyn FileSource>>, DataFusionError>

Try to create a new FileSource that can produce data in the specified sort order.

fn try_reverse_output( &self, _order: &[PhysicalSortExpr], _eq_properties: &EquivalenceProperties, ) -> Result<SortOrderPushdownResult<Arc<dyn FileSource>>, DataFusionError>

👎Deprecated since 53.0.0:

Renamed to try_pushdown_sort. This method was never limited to reversing output. It will be removed in 59.0.0 or later.

Deprecated: Renamed to Self::try_pushdown_sort.

fn reorder_files(&self, files: Vec<PartitionedFile>) -> Vec<PartitionedFile>

Reorder files in the shared work queue to optimize query performance.

fn with_schema_adapter_factory( &self, _factory: Arc<dyn SchemaAdapterFactory>, ) -> Result<Arc<dyn FileSource>, DataFusionError>

👎Deprecated since 53.0.0:

SchemaAdapterFactory has been removed. Use PhysicalExprAdapterFactory instead. See upgrading.md for more details.

Deprecated: Set optional schema adapter factory.

fn schema_adapter_factory(&self) -> Option<Arc<dyn SchemaAdapterFactory>>

👎Deprecated since 53.0.0:

SchemaAdapterFactory has been removed. Use PhysicalExprAdapterFactory instead. See upgrading.md for more details.

Deprecated: Returns the current schema adapter factory if set.

impl From<CsvSource> for Arc<dyn FileSource>

fn from(source: CsvSource) -> Self

Converts to this type from the input type.