use std::collections::{BTreeMap, BTreeSet};
use std::time::Duration;
use arrow_schema::SchemaRef;
use chrono::NaiveDate;
use parquet::arrow::ArrowWriter;
use crate::adapters::contract::models::WriteMode;
use crate::adapters::files::_helpers::layout::swap::{local_folder, staged_stem};
use crate::column_types::models::ColumnType;
use crate::file_store::models::{OpenFolder, StoredFile};
use crate::project::models::{OutputEncoding, WriteStrategy};
use crate::schema_plan::models::DesiredColumn;
use crate::store_paths::main::escape_partition_value::escape_partition_value;
use crate::store_paths::main::unescape_partition_value::unescape_partition_value;
use crate::store_paths::models::{Location, StoreKind};
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FilesDestination {
pub connection: String,
pub store: StoreKind,
pub max_footer_reads: usize,
pub encoding: OutputEncoding,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PartitionBy {
Value,
Date,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PartitionFolder {
pub prefix: String,
pub suffix: String,
pub by: PartitionBy,
}
impl PartitionFolder {
pub fn name(&self, value: &str) -> String {
format!(
"{}{}{}",
self.prefix,
escape_partition_value(value),
self.suffix
)
}
pub fn value_of(&self, name: &str) -> Option<String> {
name.strip_prefix(self.prefix.as_str())?
.strip_suffix(self.suffix.as_str())
.filter(|value| !value.is_empty())
.map(unescape_partition_value)
}
}
impl std::fmt::Display for PartitionFolder {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let placeholder = match self.by {
PartitionBy::Value => "{cursor}",
PartitionBy::Date => "{cursor:date}",
};
write!(formatter, "{}{placeholder}{}/", self.prefix, self.suffix)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TablePath {
pub folder: Location,
pub partition: Option<PartitionFolder>,
}
impl TablePath {
pub fn staged_folder(&self, mode: &WriteMode, load_id: &str) -> Option<Location> {
let rewrites = matches!(mode, WriteMode::Replace | WriteMode::Recreate);
let (_, parent, name) = local_folder(&self.folder).filter(|_| rewrites)?;
Some(Location {
store: self.folder.store.clone(),
prefix: format!("{parent}{}{load_id}/", staged_stem(name)),
})
}
}
#[derive(Debug, Clone, Copy)]
pub struct TablePathRequest<'a> {
pub template: &'a str,
pub connection: &'a str,
pub store: &'a StoreKind,
pub source: &'a str,
pub schema: &'a str,
pub table: &'a str,
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
pub enum PartitionKey {
Int(i64),
Date(NaiveDate),
Text(String),
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Partition {
pub value: Option<String>,
pub files: Vec<StoredFile>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct TableFiles {
pub partitions: Vec<Partition>,
pub problems: Vec<String>,
}
impl TableFiles {
pub fn keys(&self) -> Vec<String> {
self.partitions
.iter()
.flat_map(|partition| partition.files.iter())
.map(|file| file.relative_key.clone())
.collect()
}
}
#[derive(Debug, Clone, Copy)]
pub struct PartitionWindowRequest<'a> {
pub by: PartitionBy,
pub cursor_column: &'a str,
pub cursor_type: &'a ColumnType,
pub newest: Option<&'a str>,
pub lookback: Option<Duration>,
pub start: Option<&'a str>,
pub end: Option<&'a str>,
pub cursor_start: Option<&'a str>,
pub reload: bool,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PartitionWindow {
pub from: Option<String>,
pub to: Option<String>,
pub note: String,
}
#[derive(Debug, Clone)]
pub struct WriterRequest<'a> {
pub folder: OpenFolder,
pub path: &'a TablePath,
pub cursor_column: Option<&'a str>,
pub columns: &'a [DesiredColumn],
pub load_id: &'a str,
pub file_id: &'a str,
pub loaded_at: &'a str,
pub encoding: &'a OutputEncoding,
}
#[derive(Debug)]
pub struct OpenFile {
pub body: FileBody,
pub rows: usize,
}
#[derive(Debug)]
pub enum FileBody {
Parquet(Box<ArrowWriter<Vec<u8>>>),
Csv(CsvOut),
}
#[derive(Debug)]
pub enum CsvOut {
Plain(Vec<u8>),
Gzip(Box<flate2::write::GzEncoder<Vec<u8>>>),
Zstd {
done: Vec<u8>,
pending: Vec<u8>,
},
}
#[derive(Debug)]
pub struct FilesWriter {
pub folder: OpenFolder,
pub encoding: OutputEncoding,
pub single_name: String,
pub partition: Option<PartitionFolder>,
pub cursor_column: Option<String>,
pub schema: SchemaRef,
pub load_id: String,
pub file_id: String,
pub loaded_at_micros: i64,
pub open: BTreeMap<String, OpenFile>,
pub written: Vec<String>,
pub partitions: BTreeSet<String>,
pub rows: u64,
}
#[derive(Debug, Clone, Copy)]
pub struct OutputCheck<'a> {
pub strategy: WriteStrategy,
pub cursor_column: Option<&'a str>,
pub cursor_type: Option<&'a ColumnType>,
pub partition: Option<&'a PartitionFolder>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PendingPhase {
Writing,
Replacing,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PendingLoad {
pub load_id: String,
pub phase: PendingPhase,
pub deletes: Vec<String>,
pub run: Option<PendingRun>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Marker {
pub phase: PendingPhase,
pub keys: Vec<String>,
pub run: Option<PendingRun>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PendingRun {
pub prefix: String,
pub name: String,
pub json: String,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct TableState {
pub listed: TableFiles,
pub pending: Vec<PendingLoad>,
pub exists: bool,
pub loaded: Vec<crate::files_source::models::LoadedFile>,
pub stale_loaded: Vec<String>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ReplacedFiles {
pub always: Vec<String>,
pub by_partition: BTreeMap<String, Vec<String>>,
}
impl ReplacedFiles {
pub fn keys(&self, written: &BTreeSet<String>) -> Vec<String> {
let mut keys = self.always.clone();
for value in written {
keys.extend(self.by_partition.get(value).into_iter().flatten().cloned());
}
keys
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FolderClaim {
pub id: String,
pub name: String,
pub folder: Location,
pub template: String,
pub schema: String,
pub source: Option<Location>,
}
#[derive(Debug, Clone, Copy)]
pub struct ReplaceRequest<'a> {
pub mode: &'a WriteMode,
pub listed: &'a TableFiles,
pub partition: Option<&'a PartitionFolder>,
pub cursor_type: Option<&'a ColumnType>,
}
#[derive(Debug, Clone, Copy, Default)]
pub struct ParquetFiles;