use crate::adapters::contract::models::ChangeApply;
use std::collections::BTreeMap;
use std::time::Duration;
use crate::adapters::contract::models::{TableName, WriteMode};
use crate::adapters::dispatch::main::dialect::sql_adapter::sql_adapter;
use crate::adapters::dispatch::models::{Destination, Writer};
use crate::adapters::files::models::{
FilesWriter, PartitionFolder, PendingLoad, ReplacedFiles, TableFiles, TablePath, TableState,
};
use crate::adapters::snowflake::models::PlannedStage;
use crate::adapters::snowflake::models::StageRoute;
use crate::batch_plan::models::{FileGroup, WindowPlan};
use crate::column_types::models::{ColumnType, Warehouse};
use crate::file_store::models::OpenFolder;
use crate::files_source::models::{
FilesEncoding, FilesRead, FilesTable, Footer, LoadedFile, TextChecks,
};
use crate::masking::models::MaskRules;
use crate::postgres_source::models::{ReadColumn, SourceTable};
use crate::project::models::{
BatchStep, Contract, NewTables, OnSchemaChange, OutputEncoding, Policy, ProjectConfig,
TableChecks, WriteStrategy,
};
use crate::python_sources::types::PythonHost;
use crate::routing::models::{Route, Store};
use crate::run_log::main::finished_record::finished_record;
use crate::run_log::main::later_cursor::later_cursor;
use crate::run_log::main::next_stamp::next_stamp;
use crate::run_log::models::{RunOutcome, RunRecord, UnreadableRecord};
use crate::schema_plan::models::{DesiredColumn, LiveColumn, SchemaPlan};
use crate::snowflake_source::models::SnowflakeReader;
use crate::store_paths::models::Location;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SettingOrigin {
Table,
Source,
Default,
}
impl std::fmt::Display for SettingOrigin {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str(match self {
Self::Table => "table",
Self::Source => "source",
Self::Default => "default",
})
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CursorSpec {
pub column: String,
pub start: Option<String>,
pub lookback: Option<Duration>,
pub lookahead: Option<Duration>,
pub from_now: bool,
pub inclusive: bool,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TableSpec {
pub source_name: String,
pub source_table: SourceTable,
pub target: String,
pub destination: TableName,
pub description: String,
pub warehouse: Warehouse,
pub strategy: WriteStrategy,
pub strategy_origin: SettingOrigin,
pub unique_key: Vec<String>,
pub cursor: Option<CursorSpec>,
pub batch: Option<BatchStep>,
pub test_loader_route: bool,
pub source_end: Option<SourceEnd>,
pub pinned: BTreeMap<String, ColumnType>,
pub excluded_columns: Vec<(String, String)>,
pub masked: MaskRules,
pub contract: Contract,
pub on_schema_change: OnSchemaChange,
pub auto_widen: bool,
pub type_change_rebuild: Policy,
pub allow_empty: bool,
pub logged: bool,
pub approved: bool,
pub new_tables: NewTables,
pub files: Option<FilesSource>,
pub destination_path: Option<String>,
pub flatten: Vec<String>,
pub python: Option<PythonTable>,
pub sheet: Option<crate::sheets_source::models::SheetRead>,
pub cdc: Option<CdcSpec>,
pub tunnel: Option<String>,
pub checks: Option<TableChecks>,
pub warehouse_source: Option<WarehouseSource>,
pub layout: TableLayout,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct WarehouseSource {
pub system: &'static str,
pub sql: Option<String>,
pub route: WarehouseRoute,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum WarehouseRoute {
Unload { store: Store, stage: String },
Query,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct TableLayout {
pub engine: Option<String>,
pub order_by: Option<Vec<String>>,
pub partition_by: Option<String>,
pub primary_key: Option<Vec<String>>,
pub table_settings: std::collections::BTreeMap<String, crate::project::models::LayoutSetting>,
}
impl TableLayout {
pub fn is_empty(&self) -> bool {
self == &Self::default()
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct CdcSpec {
pub apply: ChangeApply,
pub slot: String,
pub publication: String,
pub batch_interval: std::time::Duration,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PythonTable {
pub module: String,
pub declared: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ReadWindow {
pub column: String,
pub from: Option<String>,
pub to: Option<String>,
pub inclusive: bool,
pub maximum: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FilesSource {
pub location: Location,
pub table: FilesTable,
pub partition_types: Vec<(String, ColumnType)>,
pub constants: Vec<String>,
pub encoding: FilesEncoding,
pub checks: TextChecks,
pub notes: Vec<String>,
pub copy_to_internal_stage: bool,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SkippedTable {
pub source_name: String,
pub source_table: String,
pub reason: String,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ResolvedTables {
pub specs: Vec<TableSpec>,
pub skipped: Vec<SkippedTable>,
pub problems: Vec<String>,
pub warnings: Vec<String>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct SelectedTables {
pub tables: Vec<SelectedTable>,
pub problems: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SelectedTable {
pub spec: TableSpec,
pub source_columns: Result<Vec<DesiredColumn>, Vec<String>>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TablePlan {
pub spec: TableSpec,
pub read_columns: Vec<ReadColumn>,
pub live: Option<Vec<LiveColumn>>,
pub schema: SchemaPlan,
pub mode: WriteMode,
pub source_filter: Option<String>,
pub source_parameters: Vec<String>,
pub cursor_note: Option<String>,
pub window: Option<ReadWindow>,
pub route: Option<Route>,
pub files: Option<FilesRead>,
pub output: Option<FilesOutput>,
pub exists: bool,
pub logged: Option<LoggedPlan>,
pub commits: CommitRule,
pub stage: Option<StageRoute>,
pub planned_stage: Option<PlannedStage>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct LoggedPlan {
pub records: Vec<RunRecord>,
pub current: Option<RunRecord>,
pub disagreement: Option<String>,
pub unreadable: Vec<UnreadableRecord>,
pub runs_table: bool,
}
impl RunEntry {
pub fn finished(&self, outcome: RunOutcome) -> RunRecord {
let before = self.before.clone().filter(|_| !self.replaces);
let outcome = RunOutcome {
recorded_at: next_stamp(&outcome.recorded_at, self.newest.as_deref()),
..outcome
};
finished_record(&self.record, self.cursor_type.as_ref(), before, outcome)
}
pub fn later(&self, maximum: Option<String>, batch_maximum: Option<String>) -> Option<String> {
match &self.cursor_type {
Some(column_type) => later_cursor(maximum, batch_maximum, column_type),
None => None,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RunEntry {
pub record: RunRecord,
pub cursor_type: Option<ColumnType>,
pub before: Option<String>,
pub replaces: bool,
pub newest: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FilesOutput {
pub path: TablePath,
pub encoding: OutputEncoding,
pub listed: usize,
pub footers_read: usize,
pub max_footer_reads: usize,
pub notes: Vec<String>,
pub pending: Vec<PendingLoad>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct ProjectPlan {
pub tables: Vec<TablePlan>,
pub skipped: Vec<SkippedTable>,
pub problems: Vec<String>,
pub warnings: Vec<String>,
pub retryable_problems: Vec<String>,
}
impl ProjectPlan {
pub fn only_retryable_problems(&self) -> bool {
self.tables
.iter()
.all(|table| table.schema.blocked.is_none())
&& self
.problems
.iter()
.all(|problem| self.retryable_problems.contains(problem))
}
pub fn uncaptured(&self) -> Self {
Self {
tables: self
.tables
.iter()
.filter(|table| table.spec.cdc.is_none())
.cloned()
.collect(),
..self.clone()
}
}
}
impl TablePlan {
pub fn origin(&self) -> OutcomeOrigin {
OutcomeOrigin {
source: self.spec.source_name.clone(),
target: self.spec.target.clone(),
strategy: self.spec.strategy,
}
}
pub fn outcome_name(&self) -> String {
let adapter = sql_adapter(self.spec.warehouse);
match &self.output {
Some(output) if !adapter.runs_sql() => output.path.folder.to_string(),
_ => adapter.render_display_name(&self.spec.destination),
}
}
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct PlanOptions {
pub select: Vec<String>,
pub exclude: Vec<String>,
pub rebuild_allowed: bool,
pub reload: bool,
pub start_cursor: Option<String>,
pub end_cursor: Option<String>,
pub concurrency: Option<usize>,
pub batch_bytes: Option<usize>,
pub resnapshot: bool,
}
#[derive(Debug)]
pub struct PythonLoad<'a> {
pub project: &'a ProjectConfig,
pub table: &'a TablePlan,
pub destination: &'a Destination,
pub host: std::sync::Arc<dyn PythonHost>,
pub options: &'a PlanOptions,
}
#[derive(Debug)]
pub struct LoadRun<'a> {
pub project: &'a ProjectConfig,
pub destinations: &'a BTreeMap<String, Destination>,
pub host: Option<std::sync::Arc<dyn PythonHost>>,
pub options: &'a PlanOptions,
}
#[derive(Debug)]
pub struct SharedRun {
pub project: ProjectConfig,
pub destinations: BTreeMap<String, Destination>,
pub host: Option<std::sync::Arc<dyn PythonHost>>,
pub options: PlanOptions,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TableOutcome {
pub table: String,
pub origin: Option<OutcomeOrigin>,
pub loaded: Result<TableLoaded, String>,
pub retryable: bool,
pub elapsed: std::time::Duration,
}
impl TableOutcome {
pub fn passes_on_rerun(&self) -> bool {
match &self.loaded {
Ok(_) => true,
Err(_) => self.retryable,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct OutcomeOrigin {
pub source: String,
pub target: String,
pub strategy: WriteStrategy,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct TableLoaded {
pub rows: u64,
pub changes: Vec<String>,
pub note: Option<String>,
}
#[derive(Debug)]
pub struct ReplannedColumns {
pub spec: TableSpec,
pub read_columns: Vec<ReadColumn>,
pub live: Option<Vec<LiveColumn>>,
pub schema: SchemaPlan,
}
#[derive(Debug)]
pub struct PythonFeed {
pub table: TablePlan,
pub merged: arrow_schema::Schema,
pub rows: u64,
pub outside: u64,
pub publisher: Option<Publisher>,
pub changes: Vec<String>,
pub state: Option<String>,
pub reconciled: Option<String>,
pub flattened: std::collections::BTreeMap<String, bool>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum SourceEnd {
Empty,
Maximum { maximum: String, end: String },
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum CommitRule {
Windows(WindowPlan),
Ordered {
column: String,
cursor_type: ColumnType,
},
Size,
FileGroups(Vec<FileGroup>),
Whole(WholeReason),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum WholeReason {
Unordered,
TextCursor,
AppendWithoutCursor,
Chosen,
SingleFile,
Blocked,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum CommitRange {
Planned,
Window {
from: Option<String>,
to: Option<String>,
},
Values(Vec<String>),
}
#[derive(Debug)]
pub struct Publisher {
pub table: TablePlan,
pub destination: Destination,
pub run: Option<RunEntry>,
pub load_id: String,
pub loaded_at: String,
pub direct: bool,
pub at_end: bool,
pub builds: bool,
pub indexes: Vec<String>,
pub staging: Option<TableName>,
pub batch: u64,
pub writer: Writer,
pub arrow_bytes: usize,
pub batch_rows: u64,
pub waiting_rows: u64,
pub waiting_range: Option<CommitRange>,
pub committed_rows: u64,
pub commits: u64,
pub maximum: Option<String>,
pub state: Option<String>,
pub committed_state: Option<String>,
pub batch_bytes: usize,
pub files: Option<FilesSink>,
}
#[derive(Debug)]
pub struct FilesSink {
pub output: FilesOutput,
pub folder: OpenFolder,
pub replaced: ReplacedFiles,
pub writer: Option<FilesWriter>,
pub deleted: std::collections::BTreeSet<String>,
pub reading: Vec<LoadedFile>,
pub lease: crate::table_lease::models::Lease,
pub keeper: Option<(tokio_util::sync::DropGuard, tokio::task::JoinHandle<()>)>,
pub swap: Option<FolderSwap>,
}
#[derive(Debug)]
pub struct FolderSwap {
pub live: OpenFolder,
pub replaced: std::collections::BTreeSet<String>,
}
#[derive(Debug)]
pub enum SourceReader {
Postgres(tokio_postgres::Client, Vec<tokio_postgres::Client>),
Files(OpenFolder),
Snowflake(Box<SnowflakeReader>),
BigQuery(Box<crate::bigquery_source::models::BigQueryReader>),
#[cfg(feature = "odbc")]
Odbc(Box<crate::odbc_source::models::OdbcReader>),
Sheets(crate::sheets_source::models::SheetsClient, String),
}
#[derive(Debug, Clone, Copy)]
pub struct FilesWindowRequest<'a> {
pub spec: &'a TableSpec,
pub options: &'a PlanOptions,
pub cursor_type: Option<&'a ColumnType>,
pub partition: Option<&'a PartitionFolder>,
pub listed: &'a TableFiles,
pub footers: &'a [Footer],
pub read_columns: &'a [ReadColumn],
}
#[derive(Debug, Clone)]
pub struct LiveFiles {
pub footers: Vec<Footer>,
pub live: Option<Vec<LiveColumn>>,
}
#[derive(Debug, Clone)]
pub struct FilesContext {
pub read_columns: Vec<ReadColumn>,
pub desired: Vec<DesiredColumn>,
pub path: TablePath,
pub cursor_type: Option<ColumnType>,
pub folder: OpenFolder,
pub state: TableState,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum StateKind {
Show,
Verify,
Reconcile,
Reset,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct StateRequest {
pub kind: StateKind,
pub append: bool,
pub to: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
pub struct PlanItem {
pub name: String,
pub action: String,
pub details: Vec<PlanDetail>,
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
pub struct PlanDetail {
pub label: String,
pub text: String,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize)]
pub struct PlanView {
pub ok: bool,
pub retryable: bool,
pub summary: String,
pub selected: usize,
pub objects: Vec<PlanItem>,
pub loads: Vec<PlanItem>,
pub skipped: Vec<PlanItem>,
pub errors: Vec<PlanItem>,
pub warnings: Vec<PlanItem>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TallyLine {
pub verb: &'static str,
pub name: String,
pub detail: String,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct FilesColumns {
pub folder: String,
pub live: Option<Vec<LiveColumn>>,
pub problems: Vec<String>,
pub files: Vec<FileSchema>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FileSchema {
pub key: String,
pub columns: Vec<LiveColumn>,
}
#[derive(Debug, Clone, Copy)]
pub struct RunReport<'a> {
pub command: &'static str,
pub applied: &'a [String],
pub outcomes: &'a [TableOutcome],
pub started: chrono::DateTime<chrono::Utc>,
pub elapsed: std::time::Duration,
}
#[derive(Debug, Clone, PartialEq, serde::Serialize)]
pub struct RunResult {
pub schema_version: u32,
pub command: &'static str,
pub ok: bool,
pub retryable: bool,
pub started_at: String,
pub finished_at: String,
pub duration_seconds: f64,
pub summary: RunSummary,
pub plan: PlanView,
pub objects: Vec<ObjectChange>,
pub tables: Vec<TableResult>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Serialize)]
pub struct RunSummary {
pub created: usize,
pub altered: usize,
pub recreated: usize,
pub loaded: usize,
pub failed: usize,
pub rows: u64,
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
pub struct ObjectChange {
pub action: &'static str,
pub name: String,
pub detail: String,
}
#[derive(Debug, Clone, PartialEq, serde::Serialize)]
pub struct TableResult {
pub table: String,
pub source: Option<String>,
pub target: Option<String>,
pub write_strategy: Option<String>,
pub status: &'static str,
pub rows: Option<u64>,
pub duration_seconds: f64,
pub note: Option<String>,
pub error: Option<String>,
pub retryable: bool,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SourceProblem {
pub message: String,
pub retryable: bool,
}
impl From<String> for SourceProblem {
fn from(message: String) -> Self {
Self {
message,
retryable: false,
}
}
}
impl From<crate::loader::errors::LoaderError> for SourceProblem {
fn from(error: crate::loader::errors::LoaderError) -> Self {
Self {
retryable: error.is_retryable(),
message: error.to_string(),
}
}
}