use std::time::Duration;
use crate::column_types::models::ColumnType;
use crate::file_store::models::OpenFolder;
use crate::files_source::models::{BatchFooters, DataFile, FilesRead};
use crate::load_naming::models::Compat;
use crate::loader::models::TableSpec;
use crate::postgres_source::models::ReadColumn;
use crate::schema_plan::models::{DesiredColumn, LiveColumn};
use crate::snowflake_source::models::UnloadedFiles;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TableName {
pub database: Option<String>,
pub schema: String,
pub table: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum WriteMode {
Append,
Merge {
unique_key: Vec<String>,
dedupe: Option<MergeDedupe>,
not_null_keys: Vec<String>,
},
DeleteInsert {
cursor_column: Option<String>,
from: Option<String>,
to: Option<String>,
values: Vec<String>,
replace_key: Vec<String>,
not_null_keys: Vec<String>,
},
Replace,
Recreate,
Changes(ChangeWrite),
MarkMissing(MarkWrite),
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TombstoneWrite {
pub tombstones: TableName,
pub deleted_at: String,
pub cursor: Vec<String>,
pub unique_key: Vec<String>,
pub not_null_keys: Vec<String>,
pub soft: bool,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MarkWrite {
pub unique_key: Vec<String>,
pub not_null_keys: Vec<String>,
pub soft: bool,
pub before: String,
pub max_drop: u8,
}
impl WriteMode {
pub fn staged_key(&self) -> Option<(&[String], &'static str)> {
match self {
WriteMode::Merge { unique_key, .. } => Some((unique_key, "unique_key")),
WriteMode::DeleteInsert { replace_key, .. } if !replace_key.is_empty() => {
Some((replace_key, "replace_key"))
}
_ => None,
}
}
pub fn matched_keys(&self) -> Vec<(&str, bool)> {
let (unique_key, not_null_keys) = match self {
WriteMode::Merge {
unique_key,
not_null_keys,
..
} => (unique_key, not_null_keys),
WriteMode::Changes(change) => (&change.unique_key, &change.not_null_keys),
WriteMode::MarkMissing(mark) => (&mark.unique_key, &mark.not_null_keys),
WriteMode::DeleteInsert {
replace_key,
not_null_keys,
..
} => (replace_key, not_null_keys),
_ => return Vec::new(),
};
unique_key
.iter()
.map(|key| (key.as_str(), !not_null_keys.contains(key)))
.collect()
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MergeDedupe {
pub ordinal: String,
pub newest_by: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ChangeWrite {
pub apply: ChangeApply,
pub unique_key: Vec<String>,
pub not_null_keys: Vec<String>,
pub truncated: Option<Truncation>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ChangeApply {
SoftDeletes,
HardDeletes,
Changelog,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Truncation {
pub lsn: i64,
pub committed_at: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct WriteStatementsRequest<'a> {
pub table: &'a TableName,
pub columns: &'a [DesiredColumn],
pub mode: &'a WriteMode,
pub hashed: &'a [String],
pub built: Option<&'a [String]>,
pub spared_load: Option<(&'a str, &'a str)>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TableShape {
pub definition: String,
pub indexes: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CursorQuery<'a> {
pub table: &'a TableName,
pub column: &'a str,
pub coalesce: &'a [String],
pub column_type: &'a ColumnType,
pub lookback: Option<Duration>,
pub scope: &'a [(String, ColumnType, String)],
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum NowShift {
Back(Duration),
Ahead(Duration),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CursorValue {
EpochMicros { zoned: bool },
Date,
Plain,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum BoundParameter {
Text(String),
Timestamp(chrono::NaiveDateTime, Option<u16>),
Date(chrono::NaiveDate),
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CursorRead {
pub sql: String,
pub value: CursorValue,
}
#[derive(Debug, Clone, Copy)]
pub struct WriteTarget<'a> {
pub table: &'a TableName,
pub staging: Option<&'a TableName>,
pub columns: &'a [DesiredColumn],
pub write_columns: &'a [DesiredColumn],
pub compat: Compat,
pub load_id: &'a str,
pub batch: u64,
pub direct: bool,
pub builds: bool,
pub constants: &'a [(String, ColumnType, String)],
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DirectLoaded {
Committed(u64),
Staged(u64),
}
#[derive(Debug, Clone, Copy)]
pub struct DirectBatch<'a> {
pub spec: &'a TableSpec,
pub read: &'a FilesRead,
pub write_columns: &'a [DesiredColumn],
pub target: WriteTarget<'a>,
pub loaded_at: &'a str,
pub files: &'a [DataFile],
pub footers: &'a BatchFooters,
pub mode: &'a WriteMode,
pub live: Option<&'a [LiveColumn]>,
pub source_folder: Option<&'a OpenFolder>,
}
#[derive(Debug, Clone, Copy)]
pub struct UnloadedBatch<'a> {
pub files: &'a UnloadedFiles,
pub read: &'a [ReadColumn],
pub write_columns: &'a [DesiredColumn],
pub target: WriteTarget<'a>,
pub loaded_at: &'a str,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CheckQuery<'a> {
Counts(&'a [String]),
Duplicates(&'a [String]),
DuplicateExamples(&'a [String]),
KeyDuplicates(&'a [String]),
KeyDuplicateExamples(&'a [String]),
NullKeyExamples(&'a [String]),
KeyValues(&'a [String], u64),
KeySources {
key: &'a [String],
staged: &'a TableName,
file_column: &'a str,
file: &'a str,
},
}