Struct ParquetOptions

pub struct ParquetOptions {
    pub compression: Option<String>,
    pub merge_schema: Option<bool>,
    pub datetime_rebase_mode: Option<String>,
    pub int96_rebase_mode: Option<String>,
    pub common: CommonFileOptions,
}

A struct that represents options for configuring Parquet file parsing.

Options

• merge_schema: Merge schemas from all Parquet files.
• datetime_rebase_mode: Controls how Spark handles rebase of datetime fields.
• int96_rebase_mode: Controls how Spark handles rebase of INT96 fields.
• common: Common file options that are shared across multiple file formats.

Example

let options = ParquetOptions::new()
    .merge_schema(true)
    .common.path_glob_filter("*.parquet".to_string())
    .common.recursive_file_lookup(true);

let df = spark.read().parquet(["/path/to/parquet"], options)?;

Fields

compression: Option<String>

merge_schema: Option<bool>

datetime_rebase_mode: Option<String>

int96_rebase_mode: Option<String>

common: CommonFileOptions

Implementations

impl ParquetOptions

pub fn compression(self, value: &str) -> Self

pub fn merge_schema(self, value: bool) -> Self

pub fn datetime_rebase_mode(self, value: &str) -> Self

pub fn int96_rebase_mode(self, value: &str) -> Self

Trait Implementations

impl Clone for ParquetOptions

fn clone(&self) -> ParquetOptions

Returns a copy of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl ConfigOpts for ParquetOptions

fn to_options(&self) -> HashMap<String, String>

impl Debug for ParquetOptions

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for ParquetOptions

fn default() -> Self

Returns the “default value” for a type.