Struct CsvSource

pub struct CsvSource { /* private fields */ }

A Config for CsvOpener

Example: create a DataSourceExec for CSV

let source = Arc::new(CsvSource::new(
        true,
        b',',
        b'"',
    )
    .with_terminator(Some(b'#')
));
// Create a DataSourceExec for reading the first 100MB of `file1.csv`
let config = FileScanConfigBuilder::new(object_store_url, file_schema, source)
    .with_file(PartitionedFile::new("file1.csv", 100*1024*1024))
    .with_newlines_in_values(true) // The file contains newlines in values;
    .build();
let exec = (DataSourceExec::from_data_source(config));

Implementations

impl CsvSource

pub fn new(has_header: bool, delimiter: u8, quote: u8) -> Self

Returns a CsvSource

pub fn has_header(&self) -> bool

true if the first line of each file is a header

pub fn delimiter(&self) -> u8

A column delimiter

pub fn quote(&self) -> u8

The quote character

pub fn terminator(&self) -> Option<u8>

The line terminator

pub fn comment(&self) -> Option<u8>

Lines beginning with this byte are ignored.

pub fn escape(&self) -> Option<u8>

The escape character

pub fn with_escape(&self, escape: Option<u8>) -> Self

Initialize a CsvSource with escape

pub fn with_terminator(&self, terminator: Option<u8>) -> Self

Initialize a CsvSource with terminator

pub fn with_comment(&self, comment: Option<u8>) -> Self

Initialize a CsvSource with comment

Trait Implementations

impl Clone for CsvSource

fn clone(&self) -> CsvSource

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for CsvSource

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for CsvSource

fn default() -> CsvSource

Returns the “default value” for a type.

impl FileSource for CsvSource

fn create_file_opener( &self, object_store: Arc<dyn ObjectStore>, base_config: &FileScanConfig, _partition: usize, ) -> Arc<dyn FileOpener>

Creates a dyn FileOpener based on given parameters

fn as_any(&self) -> &dyn Any

Any

fn with_batch_size(&self, batch_size: usize) -> Arc<dyn FileSource>

Initialize new type with batch size configuration

fn with_schema(&self, schema: SchemaRef) -> Arc<dyn FileSource>

Initialize new instance with a new schema

fn with_statistics(&self, statistics: Statistics) -> Arc<dyn FileSource>

Initialize new instance with projected statistics

fn with_projection(&self, config: &FileScanConfig) -> Arc<dyn FileSource>

Initialize new instance with projection information

fn metrics(&self) -> &ExecutionPlanMetricsSet

Return execution plan metrics

fn statistics(&self) -> Result<Statistics>

Return projected statistics

fn file_type(&self) -> &str

String representation of file source such as “csv”, “json”, “parquet”

fn fmt_extra(&self, t: DisplayFormatType, f: &mut Formatter<'_>) -> Result

Format FileType specific information

fn repartitioned( &self, target_partitions: usize, repartition_file_min_size: usize, output_ordering: Option<LexOrdering>, config: &FileScanConfig, ) -> Result<Option<FileScanConfig>, DataFusionError>

If supported by the FileSource, redistribute files across partitions according to their size. Allows custom file formats to implement their own repartitioning logic.