Struct deltalake::datafusion::prelude::CsvReadOptions

pub struct CsvReadOptions<'a> {
    pub has_header: bool,
    pub delimiter: u8,
    pub quote: u8,
    pub escape: Option<u8>,
    pub schema: Option<&'a Schema>,
    pub schema_infer_max_records: usize,
    pub file_extension: &'a str,
    pub table_partition_cols: Vec<(String, DataType)>,
    pub file_compression_type: FileCompressionType,
    pub file_sort_order: Vec<Vec<Expr>>,
}

Options that control the reading of CSV files.

Note this structure is supplied when a datasource is created and can not not vary from statement to statement. For settings that can vary statement to statement see ConfigOptions.

Fields

has_header: bool

Does the CSV file have a header?

If schema inference is run on a file with no headers, default column names are created.

delimiter: u8

An optional column delimiter. Defaults to b','.

quote: u8

An optional quote character. Defaults to b'"'.

escape: Option<u8>

An optional escape character. Defaults to None.

schema: Option<&'a Schema>

An optional schema representing the CSV files. If None, CSV reader will try to infer it based on data in file.

schema_infer_max_records: usize

Max number of rows to read from CSV files for schema inference if needed. Defaults to DEFAULT_SCHEMA_INFER_MAX_RECORD.

file_extension: &'a str

File extension; only files with this extension are selected for data input. Defaults to FileType::CSV.get_ext().as_str().

table_partition_cols: Vec<(String, DataType)>

Partition Columns

file_compression_type: FileCompressionType

File compression type

file_sort_order: Vec<Vec<Expr>>

Indicates how the file is sorted

Implementations

impl<'a> CsvReadOptions<'a>

pub fn new() -> CsvReadOptions<'a>

Create a CSV read option with default presets

pub fn has_header(self, has_header: bool) -> CsvReadOptions<'a>

Configure has_header setting

pub fn delimiter(self, delimiter: u8) -> CsvReadOptions<'a>

Specify delimiter to use for CSV read

pub fn quote(self, quote: u8) -> CsvReadOptions<'a>

Specify quote to use for CSV read

pub fn escape(self, escape: u8) -> CsvReadOptions<'a>

Specify delimiter to use for CSV read

pub fn file_extension(self, file_extension: &'a str) -> CsvReadOptions<'a>

Specify the file extension for CSV file selection

pub fn delimiter_option(self, delimiter: Option<u8>) -> CsvReadOptions<'a>

Configure delimiter setting with Option, None value will be ignored

pub fn schema(self, schema: &'a Schema) -> CsvReadOptions<'a>

Specify schema to use for CSV read

pub fn table_partition_cols( self, table_partition_cols: Vec<(String, DataType)> ) -> CsvReadOptions<'a>

Specify table_partition_cols for partition pruning

pub fn schema_infer_max_records(self, max_records: usize) -> CsvReadOptions<'a>

Configure number of max records to read for schema inference

pub fn file_compression_type( self, file_compression_type: FileCompressionType ) -> CsvReadOptions<'a>

Configure file compression type

pub fn file_sort_order( self, file_sort_order: Vec<Vec<Expr>> ) -> CsvReadOptions<'a>

Configure if file has known sort order

Trait Implementations

impl<'a> Clone for CsvReadOptions<'a>

fn clone(&self) -> CsvReadOptions<'a>

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl<'a> Default for CsvReadOptions<'a>

fn default() -> CsvReadOptions<'a>

Returns the “default value” for a type.

impl ReadOptions<'_> for CsvReadOptions<'_>

fn to_listing_options( &self, config: &SessionConfig, table_options: TableOptions ) -> ListingOptions

Helper to convert these user facing options to ListingTable options

fn get_resolved_schema<'life0, 'life1, 'async_trait>( &'life0 self, config: &'life1 SessionConfig, state: SessionState, table_path: ListingTableUrl ) -> Pin<Box<dyn Future<Output = Result<Arc<Schema>, DataFusionError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, CsvReadOptions<'_>: 'async_trait,

Infer and resolve the schema from the files/sources provided.

fn _get_resolved_schema<'life0, 'async_trait>( &'a self, config: &'life0 SessionConfig, state: SessionState, table_path: ListingTableUrl, schema: Option<&'a Schema> ) -> Pin<Box<dyn Future<Output = Result<Arc<Schema>, DataFusionError>> + Send + 'async_trait>>

where 'a: 'async_trait, 'a: 'async_trait, 'life0: 'async_trait, Self: Sync + 'async_trait,

helper function to reduce repetitive code. Infers the schema from sources if not provided. Infinite data sources not supported through this function.