use crate::adapters::contract::models::BoundParameter;
use crate::adapters::contract::models::ChangeApply;
use crate::column_parse::models::ColumnParse;
use crate::load_events::models::PhaseTiming;
use crate::load_naming::models::{Compat, LoadedNames, NameMemo, NameRules, Naming};
use crate::row_batches::models::{Flatten, Unnest};
use std::collections::{BTreeMap, BTreeSet};
use std::time::Duration;
use crate::adapters::contract::models::{TableName, WriteMode};
use crate::adapters::dispatch::main::dialect::sql_adapter::sql_adapter;
use crate::adapters::dispatch::models::{Destination, Writer};
use crate::adapters::files::models::{
FilesWriter, PartitionFolder, PendingLoad, ReplacedFiles, TableFiles, TablePath, TableState,
};
use crate::adapters::snowflake::models::PlannedStage;
use crate::adapters::snowflake::models::StageRoute;
use crate::batch_plan::models::{FileGroup, WindowPlan};
use crate::column_types::models::{ColumnType, Warehouse};
use crate::file_store::models::{OpenFolder, StoredFile};
use crate::files_source::models::{
AfterLoad, AfterLoadReport, FilesEncoding, FilesOrder, FilesRead, FilesTable, Footer,
LoadedFile, PathValues, TextChecks,
};
use crate::loader::_helpers::render::problem_item;
use crate::loader::_helpers::sources::PLANNED_SOURCES;
use crate::masking::models::MaskRules;
use crate::postgres_source::models::{ReadColumn, SourceTable};
use crate::project::models::{
BatchStep, Contract, Deletes, NewTables, NullCursor, NullKeys, OnCursorOutOfRange,
OnDuplicateKeys, OnSchemaChange, OutputEncoding, Policy, ProjectConfig, SendMode, TableChecks,
WriteStrategy,
};
use crate::python_sources::types::PythonHost;
use crate::routing::models::{Route, Store};
use crate::run_log::main::finished_record::finished_record;
use crate::run_log::main::later_cursor::later_cursor;
use crate::run_log::main::next_stamp::next_stamp;
use crate::run_log::models::{RunOutcome, RunRecord, UnreadableRecord};
use crate::schema_plan::models::{DesiredColumn, LiveColumn, SchemaPlan};
use crate::snowflake_source::models::SnowflakeReader;
use crate::store_paths::models::Location;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SettingOrigin {
Table,
Source,
}
impl std::fmt::Display for SettingOrigin {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str(match self {
Self::Table => "table",
Self::Source => "source",
})
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CursorSpec {
pub column: String,
pub start: Option<String>,
pub lookback: Option<Duration>,
pub lookback_units: Option<i64>,
pub lookahead: Option<Duration>,
pub from_now: bool,
pub inclusive: bool,
pub literals: bool,
pub coalesce: Vec<String>,
pub null_cursor: NullCursor,
pub out_of_range: OnCursorOutOfRange,
}
impl CursorSpec {
pub fn columns(&self) -> impl Iterator<Item = &str> {
std::iter::once(self.column.as_str()).chain(self.coalesce.iter().map(String::as_str))
}
pub fn names(&self, name: &str) -> bool {
self.columns()
.any(|column| column.eq_ignore_ascii_case(name))
}
pub fn label(&self) -> String {
match self.coalesce.is_empty() {
true => self.column.clone(),
false => format!(
"coalesce({})",
self.columns().collect::<Vec<_>>().join(", ")
),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum CursorBinding {
Literals,
Parameters(Option<u16>),
Escapes(Option<u16>),
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TableSpec {
pub source_name: String,
pub source_table: SourceTable,
pub target: String,
pub destination: TableName,
pub description: String,
pub warehouse: Warehouse,
pub strategy: WriteStrategy,
pub strategy_origin: SettingOrigin,
pub unique_key: Vec<String>,
pub key_from_primary_key: bool,
pub settings_at: Option<String>,
pub unique_key_at: Option<String>,
pub on_duplicate_keys: OnDuplicateKeys,
pub null_keys: NullKeys,
pub not_null_keys: Vec<String>,
pub replace_key: Vec<String>,
pub newest_by: Vec<String>,
pub cursor: Option<CursorSpec>,
pub batch: Option<BatchStep>,
pub batch_concurrency: Option<std::num::NonZeroUsize>,
pub constants: Vec<(String, ColumnType, String)>,
pub shared_destination: bool,
pub test_loader_route: bool,
pub source_end: Option<SourceEnd>,
pub pinned: BTreeMap<String, ColumnType>,
pub child_pinned: BTreeMap<String, ColumnType>,
pub excluded_columns: Vec<(String, String)>,
pub masked: MaskRules,
pub contract: Contract,
pub on_schema_change: OnSchemaChange,
pub auto_widen: bool,
pub infer_types: bool,
pub variant_json: bool,
pub variants: Vec<String>,
pub type_change_rebuild: Policy,
pub allow_empty: bool,
pub logged: bool,
pub after_load: AfterLoad,
pub files_order: FilesOrder,
pub approved: bool,
pub new_tables: NewTables,
pub files: Option<FilesSource>,
pub destination_path: Option<String>,
pub flatten: Flatten,
pub row_id: bool,
pub root_rows: bool,
pub unnest: Unnest,
pub max_depth: u32,
pub python: Option<PythonTable>,
pub sheet: Option<crate::sheets_source::models::SheetRead>,
pub cdc: Option<CdcSpec>,
pub deletes: Option<DeletesSpec>,
pub tombstones_of: Option<String>,
pub setting_problems: Vec<String>,
pub tunnel: Option<String>,
pub checks: Option<TableChecks>,
pub warehouse_source: Option<WarehouseSource>,
pub row_filter: Option<String>,
pub layout: TableLayout,
pub naming: Naming,
pub compat: Compat,
pub send: SendMode,
pub staging_dir: Option<std::path::PathBuf>,
pub names: LoadedNames,
pub name_rules: NameRules,
pub listed: Option<Box<SourceTable>>,
pub transform: Option<TableTransform>,
pub parses: BTreeMap<String, ColumnParse>,
pub timezone: Option<(chrono_tz::Tz, SettingOrigin)>,
}
impl TableSpec {
pub fn soft_deletes(&self) -> bool {
self.deletes
.as_ref()
.is_some_and(|deletes| deletes.apply == Deletes::Soft)
}
pub fn query_sql(&self) -> Option<&str> {
self.warehouse_source
.as_ref()
.and_then(|source| source.sql.as_deref())
}
pub fn read_filter(&self, filter: Option<String>) -> Option<String> {
match (&self.row_filter, filter) {
(Some(row_filter), Some(filter)) => Some(format!("({row_filter}\n) AND ({filter})")),
(Some(row_filter), None) => Some(format!("({row_filter}\n)")),
(None, filter) => filter,
}
}
pub fn zone(&self) -> chrono_tz::Tz {
self.timezone.map_or(chrono_tz::Tz::UTC, |(zone, _)| zone)
}
pub fn loaded_type(&self, name: &str, column_type: ColumnType) -> ColumnType {
match column_type {
ColumnType::Json
if self.variant_json || self.variants.iter().any(|pin| pin == name) =>
{
ColumnType::Native(crate::column_types::constants::VARIANT_TYPE.to_owned())
}
other => other,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TableTransform {
pub function: String,
pub version: String,
pub older: usize,
pub creates_log: bool,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct WarehouseSource {
pub system: &'static str,
pub sql: Option<String>,
pub route: WarehouseRoute,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum WarehouseRoute {
Unload { store: Store, stage: String },
Query,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct TableLayout {
pub engine: Option<String>,
pub order_by: Option<Vec<String>>,
pub partition_by: Option<String>,
pub primary_key: Option<Vec<String>>,
pub table_settings: std::collections::BTreeMap<String, crate::project::models::LayoutSetting>,
}
impl TableLayout {
pub fn is_empty(&self) -> bool {
self == &Self::default()
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DeletesSpec {
pub apply: Deletes,
pub max_drop: u8,
pub scan: bool,
pub from: Option<TombstoneSpec>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TombstoneSpec {
pub source_table: String,
pub loaded: Option<(TableName, String)>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CdcSpec {
pub apply: ChangeApply,
pub slot: String,
pub publication: String,
pub batch_interval: std::time::Duration,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PythonTable {
pub module: String,
pub declared: Vec<String>,
pub lists: crate::row_batches::types::ListChoices,
pub literal_paths: Option<crate::http_source::models::LiteralPaths>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ReadWindow {
pub column: String,
pub from: Option<String>,
pub to: Option<String>,
pub inclusive: bool,
pub maximum: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FilesSource {
pub location: Location,
pub table: FilesTable,
pub partition_types: Vec<(String, ColumnType)>,
pub constants: Vec<String>,
pub paths: PathValues,
pub encoding: FilesEncoding,
pub checks: TextChecks,
pub notes: Vec<String>,
pub copy_to_internal_stage: bool,
pub transform: Option<FilesTransform>,
pub archive: Option<Location>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FilesTransform {
pub input: Vec<DesiredColumn>,
pub nanos: Vec<String>,
pub checks: TextChecks,
pub sample: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SkippedTable {
pub source_name: String,
pub source_table: String,
pub destination: String,
pub reason: String,
}
#[derive(Debug)]
pub struct FoundSource<'a> {
pub name: &'a String,
pub source: &'a crate::project::models::SourceConfig,
pub resolved: ResolvedTables,
pub files: Option<FilesSource>,
}
#[derive(Debug, Default)]
pub struct FoundSources<'a> {
pub sources: Vec<FoundSource<'a>>,
pub claims: Vec<crate::adapters::files::models::FolderClaim>,
pub configured: Vec<TableSpec>,
pub plan: ProjectPlan,
pub unresolved: Vec<String>,
pub drained: Vec<String>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ResolvedTables {
pub specs: Vec<TableSpec>,
pub skipped: Vec<SkippedTable>,
pub problems: Vec<String>,
pub problem_names: Vec<(String, String)>,
pub problem_places: Vec<(String, Option<String>)>,
pub warnings: Vec<String>,
pub drained: Option<DrainedFiles>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DrainedFiles {
pub in_scope: bool,
pub target: String,
pub table: TableName,
pub skipped: SkippedTable,
pub problem: String,
pub spec: Option<TableSpec>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct SelectedTables {
pub tables: Vec<SelectedTable>,
pub problems: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SelectedTable {
pub spec: TableSpec,
pub source_columns: Result<Vec<DesiredColumn>, Vec<String>>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TablePlan {
pub spec: TableSpec,
pub read_columns: Vec<ReadColumn>,
pub live: Option<Vec<LiveColumn>>,
pub schema: SchemaPlan,
pub mode: WriteMode,
pub source_filter: Option<String>,
pub source_parameters: Vec<BoundParameter>,
pub cursor_note: Option<String>,
pub window: Option<ReadWindow>,
pub route: Option<Route>,
pub files: Option<FilesRead>,
pub output: Option<FilesOutput>,
pub exists: bool,
pub logged: Option<LoggedPlan>,
pub commits: CommitRule,
pub stage: Option<StageRoute>,
pub planned_stage: Option<PlannedStage>,
pub fetch_memory: Option<FetchMemory>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FetchMemory {
pub budget: usize,
pub estimate: Option<usize>,
pub memory: String,
pub budget_note: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct LoggedPlan {
pub records: Vec<RunRecord>,
pub current: Option<RunRecord>,
pub disagreement: Option<String>,
pub unreadable: Vec<UnreadableRecord>,
pub runs_table: bool,
}
impl RunEntry {
pub fn finished(&self, outcome: RunOutcome) -> RunRecord {
let before = self.before.clone().filter(|_| !self.replaces);
let outcome = RunOutcome {
recorded_at: next_stamp(&outcome.recorded_at, self.newest.as_deref()),
..outcome
};
finished_record(&self.record, self.cursor_type.as_ref(), before, outcome)
}
pub fn later(&self, maximum: Option<String>, batch_maximum: Option<String>) -> Option<String> {
match &self.cursor_type {
Some(column_type) => later_cursor(maximum, batch_maximum, column_type),
None => None,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RunEntry {
pub record: RunRecord,
pub cursor_type: Option<ColumnType>,
pub before: Option<String>,
pub replaces: bool,
pub newest: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FilesOutput {
pub path: TablePath,
pub encoding: OutputEncoding,
pub listed: usize,
pub footers_read: usize,
pub max_footer_reads: usize,
pub notes: Vec<String>,
pub pending: Vec<PendingLoad>,
pub commits: BTreeSet<String>,
}
impl FilesOutput {
pub fn unfinished(&self) -> impl Iterator<Item = &PendingLoad> {
self.pending
.iter()
.filter(|load| !(load.expired && load.deletes.is_empty()))
}
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ProjectPlan {
pub tables: Vec<TablePlan>,
pub skipped: Vec<SkippedTable>,
pub problems: Vec<String>,
pub warnings: Vec<String>,
pub retryable_problems: Vec<String>,
pub failed_sources: std::collections::BTreeSet<String>,
pub problem_names: Vec<(String, String)>,
pub problem_places: Vec<(String, Option<String>)>,
pub unread_sources: BTreeMap<String, UnreadSource>,
pub unread_sharing: Vec<UnreadSharing>,
pub phases: Vec<PhaseTiming>,
pub memory: Option<String>,
pub memory_asks: bool,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct UnreadSource {
pub tables: Vec<String>,
pub patterns: bool,
pub connection: String,
pub variables: Vec<String>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct UnreadSharing {
pub warning: String,
pub sources: Vec<String>,
pub tables: Vec<String>,
}
impl ProjectPlan {
pub fn only_retryable_problems(&self) -> bool {
self.tables
.iter()
.all(|table| table.schema.blocked.is_none())
&& self
.problems
.iter()
.all(|problem| self.retryable_problems.contains(problem))
}
pub fn failing_sources(&self) -> Vec<String> {
let mut sources: Vec<String> = Vec::new();
let named = self.problems.iter().filter_map(|problem| {
let subject = problem_item(problem).name;
let inner = subject.strip_prefix("[sources.")?;
Some(inner.split(['.', ']']).next().unwrap_or(inner).to_owned())
});
let blocked = self
.tables
.iter()
.filter(|table| table.schema.blocked.is_some())
.map(|table| table.spec.source_name.clone());
for source in blocked
.chain(named)
.chain(self.failed_sources.iter().cloned())
{
if !sources.contains(&source) {
sources.push(source);
}
}
sources
}
pub fn planned_sources() -> usize {
PLANNED_SOURCES.load(std::sync::atomic::Ordering::Relaxed)
}
pub fn uncaptured(&self) -> Self {
Self {
tables: self
.tables
.iter()
.filter(|table| table.spec.cdc.is_none())
.cloned()
.collect(),
..self.clone()
}
}
}
impl TablePlan {
pub fn origin(&self) -> OutcomeOrigin {
OutcomeOrigin {
source: self.spec.source_name.clone(),
target: self.spec.target.clone(),
strategy: self.spec.strategy,
}
}
pub fn outcome_name(&self) -> String {
let adapter = sql_adapter(self.spec.warehouse);
match &self.output {
Some(output) if !adapter.runs_sql() => output.path.folder.to_string(),
_ => adapter.render_display_name(&self.spec.destination),
}
}
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct PlanOptions {
pub select: Vec<String>,
pub exclude: Vec<String>,
pub rebuild_allowed: bool,
pub reload: bool,
pub start_cursor: Option<String>,
pub end_cursor: Option<String>,
pub typed_cursors: TypedCursors,
pub range_flags: RangeFlags,
pub concurrency: Option<usize>,
pub memory_limit: Option<u64>,
pub batch_bytes: Option<usize>,
pub resnapshot: bool,
pub sync_deletes: bool,
pub fail_fast: bool,
pub memory_per_table: bool,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct TypedCursors {
pub start_ts: Option<String>,
pub end_ts: Option<String>,
pub start_int: Option<String>,
pub end_int: Option<String>,
}
impl PlanOptions {
pub fn ranged(&self) -> bool {
let typed = &self.typed_cursors;
[
&self.start_cursor,
&self.end_cursor,
&typed.start_ts,
&typed.end_ts,
&typed.start_int,
&typed.end_int,
]
.iter()
.any(|value| value.is_some())
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct RangeFlags {
pub start: &'static str,
pub end: &'static str,
}
impl Default for RangeFlags {
fn default() -> Self {
Self {
start: "--start-cursor",
end: "--end-cursor",
}
}
}
#[derive(Debug)]
pub struct PythonLoad<'a> {
pub project: &'a ProjectConfig,
pub table: &'a TablePlan,
pub destination: &'a Destination,
pub host: std::sync::Arc<dyn PythonHost>,
pub options: &'a PlanOptions,
}
#[derive(Debug)]
pub struct LoadRun<'a> {
pub project: &'a ProjectConfig,
pub destinations: &'a BTreeMap<String, Destination>,
pub host: Option<std::sync::Arc<dyn PythonHost>>,
pub options: &'a PlanOptions,
}
#[derive(Debug)]
pub struct SharedRun {
pub project: ProjectConfig,
pub destinations: BTreeMap<String, Destination>,
pub host: Option<std::sync::Arc<dyn PythonHost>>,
pub options: PlanOptions,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TableOutcome {
pub table: String,
pub origin: Option<OutcomeOrigin>,
pub loaded: Result<TableLoaded, String>,
pub retryable: bool,
pub elapsed: std::time::Duration,
pub skipped: bool,
pub peak_memory_bytes: Option<u64>,
}
impl TableOutcome {
pub fn skipped(table: &TablePlan, failed: &str) -> Self {
Self {
table: table.outcome_name(),
origin: Some(table.origin()),
loaded: Err(format!(
"not started: --fail-fast stopped the load after {failed} failed"
)),
retryable: true,
elapsed: std::time::Duration::ZERO,
skipped: true,
peak_memory_bytes: None,
}
}
pub fn passes_on_rerun(&self) -> bool {
match &self.loaded {
Ok(_) => true,
Err(_) => self.retryable,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct OutcomeOrigin {
pub source: String,
pub target: String,
pub strategy: WriteStrategy,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct TableLoaded {
pub rows: u64,
pub changes: Vec<String>,
pub note: Option<String>,
pub after_load: Option<AfterLoadReport>,
pub replaced_keys: Option<ReplacedKeys>,
pub transforms: Vec<crate::python_sources::models::TransformTime>,
pub names: Vec<String>,
pub warnings: Vec<String>,
}
#[derive(Debug, Clone)]
pub struct TransformRun {
pub host: std::sync::Arc<dyn PythonHost>,
pub times: std::sync::Arc<std::sync::Mutex<Vec<crate::python_sources::models::TransformTime>>>,
}
impl TransformRun {
pub fn new(host: std::sync::Arc<dyn PythonHost>) -> Self {
Self {
host,
times: std::sync::Arc::default(),
}
}
}
#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize)]
pub struct ReplacedKeys {
pub columns: Vec<String>,
pub values: std::collections::BTreeSet<String>,
pub truncated: bool,
}
impl ReplacedKeys {
pub fn added(kept: Option<Self>, (columns, values): (Vec<String>, Vec<String>)) -> Self {
let mut kept = kept.unwrap_or(Self {
columns,
..Self::default()
});
kept.values.extend(values);
while kept.values.len() > crate::loader::constants::REPLACED_KEYS_SHOWN {
kept.values.pop_last();
kept.truncated = true;
}
kept
}
}
#[derive(Debug)]
pub struct ReplannedColumns {
pub spec: TableSpec,
pub read_columns: Vec<ReadColumn>,
pub live: Option<Vec<LiveColumn>>,
pub schema: SchemaPlan,
}
#[derive(Debug)]
pub struct PythonFeed {
pub table: TablePlan,
pub merged: arrow_schema::Schema,
pub rows: u64,
pub outside: crate::batch_plan::models::OutsideRows,
pub warnings: Vec<String>,
pub unshown: (u64, u64),
pub null_cursors: u64,
pub publisher: Option<Publisher>,
pub changes: Vec<String>,
pub state: Option<String>,
pub reconciled: Option<String>,
pub flattened: std::collections::BTreeMap<String, bool>,
pub names: NameMemo,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum SourceEnd {
Empty,
Maximum { maximum: String, end: String },
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum CommitRule {
Windows(WindowPlan),
Ordered {
column: String,
cursor_type: ColumnType,
},
Size,
FileGroups(Vec<FileGroup>),
Whole(WholeReason),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum WholeReason {
Unordered,
TextCursor,
AppendWithoutCursor,
Chosen,
KeyScan,
SingleFile,
Blocked,
ReplaceKey,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum CommitRange {
Planned,
Window {
from: Option<String>,
to: Option<String>,
},
Values(Vec<String>),
}
#[derive(Debug)]
pub struct MergeRebuild {
pub build: TableName,
pub records: Vec<RunRecord>,
}
#[derive(Debug)]
pub struct Publisher {
pub table: TablePlan,
pub destination: Destination,
pub run: Option<RunEntry>,
pub load_id: String,
pub loaded_at: String,
pub direct: bool,
pub at_end: bool,
pub builds: bool,
pub indexes: Vec<String>,
pub staging: Option<TableName>,
pub batch: u64,
pub writer: Writer,
pub arrow_bytes: usize,
pub batch_rows: u64,
pub progressed_at: std::time::Instant,
pub child: bool,
pub staged_ordinal: i64,
pub waiting_rows: u64,
pub waiting_range: Option<CommitRange>,
pub committed_rows: u64,
pub commits: u64,
pub maximum: Option<String>,
pub state: Option<String>,
pub committed_state: Option<String>,
pub batch_bytes: usize,
pub files: Option<FilesSink>,
pub merged_file: Option<String>,
pub replaced: Option<ReplacedKeys>,
pub marked: Option<u64>,
pub older: u64,
pub run_keys: std::collections::BTreeMap<String, String>,
pub fed_files: Vec<StoredFile>,
pub settled_files: Vec<StoredFile>,
pub children: Option<ChildTables>,
pub rebuild: Option<MergeRebuild>,
pub transform: Option<(TransformRun, usize)>,
pub spool: Option<crate::commit_spool::models::Spool>,
pub pieces: Pieces,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Cuts {
Anywhere,
Between(Vec<String>),
Never(&'static str),
}
#[derive(Debug, Default)]
pub struct Pieces {
pub held: u64,
pub run: u64,
pub last_key: Option<String>,
pub committed: u64,
pub unfinished: bool,
pub piece: bool,
pub group: Option<RowGroupSeen>,
pub rows: u64,
pub large: u64,
pub copies: u64,
pub overflow: bool,
pub statement: Option<String>,
}
#[derive(Debug, Clone, Copy, Default)]
pub struct RowGroupSeen {
pub before: u64,
pub full: u64,
pub earlier: u64,
pub filled: u64,
pub large_at: u64,
pub last_large: u64,
}
#[derive(Debug)]
pub struct ChildTables {
pub choices: crate::row_batches::types::ListChoices,
pub live: Vec<String>,
pub destination: Destination,
pub options: PlanOptions,
pub feeds: Vec<ChildFeed>,
pub ignored: Vec<String>,
pub changes: Vec<String>,
pub merged: Vec<String>,
}
#[derive(Debug)]
pub struct ChildFeed {
pub path: String,
pub table: TablePlan,
pub merged: arrow_schema::Schema,
pub publisher: Option<Publisher>,
}
#[derive(Debug)]
pub struct FilesSink {
pub output: FilesOutput,
pub folder: OpenFolder,
pub replaced: ReplacedFiles,
pub writer: Option<FilesWriter>,
pub deleted: std::collections::BTreeSet<String>,
pub reading: Vec<LoadedFile>,
pub lease: crate::table_lease::models::Lease,
pub keeper: Option<(tokio_util::sync::DropGuard, tokio::task::JoinHandle<()>)>,
pub swap: Option<FolderSwap>,
}
#[derive(Debug)]
pub struct FolderSwap {
pub live: OpenFolder,
pub replaced: std::collections::BTreeSet<String>,
pub held: std::fs::File,
}
#[derive(Debug)]
pub enum SourceReader {
Postgres(tokio_postgres::Client, Vec<tokio_postgres::Client>),
Files(OpenFolder),
Snowflake(Box<SnowflakeReader>),
BigQuery(Box<crate::bigquery_source::models::BigQueryReader>),
Databricks(Box<crate::databricks_source::models::DatabricksReader>),
#[cfg(feature = "odbc")]
Odbc(Box<crate::odbc_source::models::OdbcReader>),
Sheets(crate::sheets_source::models::SheetsClient, String),
}
#[derive(Debug)]
pub struct FilesWindowStarts<'a> {
pub start: Option<String>,
pub end: Option<String>,
pub cursor_start: Option<String>,
pub newest: Option<&'a str>,
}
#[derive(Debug, Clone, Copy)]
pub struct FilesWindowRequest<'a> {
pub spec: &'a TableSpec,
pub options: &'a PlanOptions,
pub cursor_type: Option<&'a ColumnType>,
pub partition: Option<&'a PartitionFolder>,
pub listed: &'a TableFiles,
pub footers: &'a [Footer],
pub read_columns: &'a [ReadColumn],
}
#[derive(Debug, Clone)]
pub struct LiveFiles {
pub footers: Vec<Footer>,
pub live: Option<Vec<LiveColumn>>,
}
#[derive(Debug, Clone)]
pub struct FilesContext {
pub read_columns: Vec<ReadColumn>,
pub desired: Vec<DesiredColumn>,
pub path: TablePath,
pub cursor_type: Option<ColumnType>,
pub folder: OpenFolder,
pub state: TableState,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum StateKind {
Show,
Verify,
Reconcile,
Reset,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct StateRequest {
pub kind: StateKind,
pub append: bool,
pub to: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
pub struct PlanItem {
pub name: String,
pub action: String,
pub details: Vec<PlanDetail>,
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
pub struct PlanDetail {
pub label: String,
pub text: String,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize)]
pub struct PlanView {
pub ok: bool,
pub retryable: bool,
pub summary: String,
pub selected: usize,
#[serde(skip)]
pub destinations: Vec<String>,
pub objects: Vec<PlanItem>,
pub loads: Vec<PlanItem>,
pub skipped: Vec<PlanItem>,
pub errors: Vec<PlanItem>,
pub warnings: Vec<PlanItem>,
pub phases: Vec<PhaseTiming>,
pub memory: Option<String>,
#[serde(skip)]
pub memory_asks: bool,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TallyLine {
pub verb: &'static str,
pub name: String,
pub detail: String,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct FilesColumns {
pub folder: String,
pub live: Option<Vec<LiveColumn>>,
pub problems: Vec<String>,
pub files: Vec<FileSchema>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FileSchema {
pub key: String,
pub columns: Vec<LiveColumn>,
}
#[derive(Debug, Clone, Copy)]
pub struct RunReport<'a> {
pub command: &'static str,
pub applied: &'a [String],
pub outcomes: &'a [TableOutcome],
pub started: chrono::DateTime<chrono::Utc>,
pub elapsed: std::time::Duration,
pub peak_memory_bytes: Option<u64>,
}
#[derive(Debug, Clone, PartialEq, serde::Serialize)]
pub struct RunResult {
pub schema_version: u32,
pub command: &'static str,
pub ok: bool,
pub retryable: bool,
pub started_at: String,
pub finished_at: String,
pub duration_seconds: f64,
pub summary: RunSummary,
pub plan: PlanView,
pub objects: Vec<ObjectChange>,
pub tables: Vec<TableResult>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize)]
pub struct RunSummary {
pub created: usize,
pub altered: usize,
pub recreated: usize,
pub loaded: usize,
pub failed: usize,
pub skipped: usize,
pub rows: u64,
#[serde(skip_serializing_if = "Option::is_none")]
pub peak_memory_bytes: Option<u64>,
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
pub struct ObjectChange {
pub action: &'static str,
pub name: String,
pub detail: String,
}
#[derive(Debug, Clone, PartialEq, serde::Serialize)]
pub struct TableResult {
pub table: String,
pub source: Option<String>,
pub target: Option<String>,
pub write_strategy: Option<String>,
pub status: &'static str,
pub rows: Option<u64>,
pub duration_seconds: f64,
pub note: Option<String>,
pub error: Option<String>,
pub retryable: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub after_load: Option<AfterLoadReport>,
#[serde(skip_serializing_if = "Option::is_none")]
pub replaced_keys: Option<ReplacedKeys>,
#[serde(skip_serializing_if = "Vec::is_empty")]
pub transforms: Vec<crate::python_sources::models::TransformTime>,
#[serde(skip_serializing_if = "Vec::is_empty")]
pub names: Vec<String>,
#[serde(skip_serializing_if = "Vec::is_empty")]
pub warnings: Vec<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub peak_memory_bytes: Option<u64>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SourceProblem {
pub message: String,
pub retryable: bool,
}
impl From<String> for SourceProblem {
fn from(message: String) -> Self {
Self {
message,
retryable: false,
}
}
}
impl From<crate::loader::errors::LoaderError> for SourceProblem {
fn from(error: crate::loader::errors::LoaderError) -> Self {
Self {
retryable: error.is_retryable(),
message: error.to_string(),
}
}
}