use parquet::arrow::arrow_reader::ArrowReaderMetadata;
use std::sync::Arc;
use arrow_schema::SchemaRef;
use parquet::file::metadata::ParquetMetaData;
use serde::Deserialize;
use crate::column_types::models::ColumnType;
use crate::file_store::models::{OpenFolder, StoredFile};
use crate::files_source::constants::DEFAULT_CSV_ENCODING;
use crate::schema_plan::models::DesiredColumn;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DataFile {
pub file: StoredFile,
pub partitions: Vec<(String, String)>,
}
#[derive(Debug, Clone, Default)]
pub struct PathValues {
pub partition_columns: Vec<String>,
pub template: Option<PathTemplate>,
pub unmatched: UnmatchedPaths,
pub constants: Vec<(String, String)>,
}
impl PathValues {
pub fn names(&self) -> Vec<&str> {
self.partition_columns
.iter()
.map(String::as_str)
.chain(
self.template
.iter()
.flat_map(|template| template.captures.iter())
.map(|(name, _)| name.as_str()),
)
.chain(self.constants.iter().map(|(name, _)| name.as_str()))
.collect()
}
}
#[derive(Debug, Clone)]
pub struct PathTemplate {
pub text: String,
pub pattern: regex::Regex,
pub captures: Vec<(String, ColumnType)>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum UnmatchedPaths {
#[default]
Fail,
Skip,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FileColumn {
pub name: String,
pub column_type: Result<ColumnType, String>,
pub direct_refusal: Option<String>,
pub stored_read: StoredRead,
pub lists: Option<bool>,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub enum StoredRead {
#[default]
Plain,
Nanos,
Epoch(u8),
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FilesTable {
pub files: Vec<DataFile>,
pub columns: Vec<FileColumn>,
pub problems: Vec<String>,
pub skipped: usize,
pub unmatched: usize,
pub dropped: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FilesRead {
pub files: Vec<DataFile>,
pub encoding: FilesEncoding,
pub checks: TextChecks,
pub columns: Vec<DesiredColumn>,
pub nanos: Vec<String>,
pub batch_rows: usize,
}
#[derive(Debug, Clone, Default)]
pub struct BatchFooters {
pub parquet: std::collections::BTreeMap<String, ArrowReaderMetadata>,
pub headers: std::collections::BTreeMap<String, Vec<String>>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct TextChecks {
pub known: Vec<String>,
pub exclude: Vec<String>,
pub allow: Vec<String>,
pub inferred: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct LoadedFile {
pub name: String,
pub version: String,
}
#[derive(Debug, Clone, Copy)]
pub struct FileSelection<'a> {
pub files: &'a [DataFile],
pub location: &'a str,
pub loaded: &'a [LoadedFile],
pub removed: bool,
pub cursor: Option<(&'a str, &'a ColumnType)>,
pub from: Option<&'a str>,
pub to: Option<&'a str>,
}
#[derive(Debug, Clone)]
pub struct Footer {
pub relative_key: String,
pub schema: SchemaRef,
pub metadata: Arc<ParquetMetaData>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum SourceFormat {
#[default]
Parquet,
Ndjson,
Json,
Csv,
Xlsx,
Xml,
}
impl std::fmt::Display for SourceFormat {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str(match self {
Self::Parquet => "parquet",
Self::Ndjson => "ndjson",
Self::Json => "json",
Self::Csv => "csv",
Self::Xlsx => "xlsx",
Self::Xml => "xml",
})
}
}
#[derive(Debug, Clone, PartialEq, Eq, Default, Deserialize)]
#[serde(rename_all = "snake_case", deny_unknown_fields)]
pub enum AfterLoad {
#[default]
Keep,
Delete,
MoveTo(String),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Compression {
#[default]
Auto,
None,
Gzip,
Zstd,
Zip,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Codec {
Plain,
Gzip,
Zstd,
}
#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct CsvOptions {
pub delimiter: char,
pub header: bool,
pub quote: char,
pub null_values: Vec<String>,
pub encoding: String,
pub skip_rows: u64,
}
impl Default for CsvOptions {
fn default() -> Self {
Self {
delimiter: ',',
header: true,
quote: '"',
null_values: vec![String::new()],
encoding: DEFAULT_CSV_ENCODING.to_owned(),
skip_rows: 0,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Charset {
Utf8,
Latin1,
Ascii,
Other(&'static encoding_rs::Encoding),
}
#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct XlsxOptions {
pub sheet: Option<String>,
pub header: bool,
pub skip_rows: u32,
}
impl Default for XlsxOptions {
fn default() -> Self {
Self {
sheet: None,
header: true,
skip_rows: 0,
}
}
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct XmlOptions {
pub record_path: String,
pub lists: Vec<String>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct FilesEncoding {
pub format: SourceFormat,
pub compression: Compression,
pub csv: CsvOptions,
pub xlsx: XlsxOptions,
pub xml: XmlOptions,
pub members: Option<String>,
pub record: Option<String>,
pub flatten_objects: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum RecordShape {
#[default]
Columns,
Record,
}
impl std::fmt::Display for FilesEncoding {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match (self.format, self.compression) {
(SourceFormat::Parquet, _) => write!(formatter, "{}", self.format),
(format, Compression::Auto) => write!(formatter, "{format}, compression by file name"),
(format, Compression::None) => write!(formatter, "{format}, uncompressed"),
(format, Compression::Gzip) => write!(formatter, "{format}, gzip"),
(format, Compression::Zstd) => write!(formatter, "{format}, zstd"),
(format, Compression::Zip) => write!(formatter, "{format} in ZIP archives"),
}?;
match &self.record {
Some(column) => write!(formatter, ", each record whole in column {column}"),
None => Ok(()),
}
}
}
#[derive(Debug)]
pub(crate) struct ListedFiles {
pub(crate) files: Vec<(DataFile, Codec)>,
pub(crate) skipped: usize,
pub(crate) unmatched: usize,
pub(crate) problems: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct Cell {
pub(crate) text: String,
pub(crate) kind: ColumnType,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct CellRecord {
pub(crate) line: u64,
pub(crate) cells: Vec<(String, Cell)>,
}
#[derive(Debug)]
pub(crate) struct FileRead<'a> {
pub(crate) folder: &'a OpenFolder,
pub(crate) request: &'a FilesRead,
pub(crate) data: &'a DataFile,
pub(crate) schema: &'a SchemaRef,
pub(crate) file_name: String,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize)]
pub struct AfterLoadReport {
pub action: &'static str,
pub status: &'static str,
pub files: u64,
pub failed: Vec<AfterLoadFailure>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize)]
pub struct AfterLoadFailure {
pub file: String,
pub error: String,
}