use std::collections::BTreeMap;
use std::path::PathBuf;
use std::time::Duration;
use serde::Deserialize;
use crate::config::models::AuthConfig;
use crate::files_source::constants::RECORD_COLUMN;
use crate::files_source::models::{
AfterLoad, Compression, CsvOptions, FilesEncoding, RecordShape, SourceFormat, UnmatchedPaths,
XlsxOptions, XmlOptions,
};
use crate::load_naming::models::Naming;
use crate::project::_helpers::defaults;
use crate::project::_helpers::serde_values::{
deserialize_optional_batch, deserialize_optional_duration, deserialize_optional_keys,
deserialize_optional_percent,
};
use crate::project::constants::{
DEFAULT_DESTINATION_PATH, DEFAULT_FILES_TABLE_NAME, DEFAULT_MAX_FOOTER_READS,
DEFAULT_TABLE_NAME,
};
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ProjectConfig {
#[serde(default)]
pub python_paths: Vec<PathBuf>,
#[serde(default)]
pub default_target: Option<String>,
#[serde(default)]
pub concurrency: Option<usize>,
#[serde(default)]
pub connections: BTreeMap<String, ConnectionConfig>,
#[serde(default)]
pub targets: BTreeMap<String, TargetConfig>,
#[serde(default)]
pub sources: BTreeMap<String, SourceConfig>,
#[serde(skip)]
pub connection_settings: serde_json::Map<String, serde_json::Value>,
#[serde(skip)]
pub python_modules: BTreeMap<String, String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
pub enum LoadMethod {
ExternalStage {
url: String,
#[serde(default)]
storage_integration: Option<String>,
#[serde(default)]
name: Option<String>,
#[serde(default)]
region: Option<String>,
#[serde(default)]
endpoint: Option<String>,
#[serde(default)]
keep_staged_files: bool,
},
InternalUserStage {},
InternalTableStage {},
InternalNamedStage { name: String },
Streaming {},
Insert {},
}
#[derive(Debug, Clone, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum ConnectionConfig {
Snowflake {
account: String,
user: String,
auth: AuthConfig,
#[serde(default)]
role: Option<String>,
#[serde(default)]
warehouse: Option<String>,
database: String,
#[serde(default)]
base_url: Option<String>,
#[serde(default)]
load_method: Option<Box<LoadMethod>>,
},
Databricks {
host: String,
client_id: String,
client_secret: crate::auth::models::Secret,
warehouse_id: String,
catalog: String,
volume: String,
},
#[serde(rename = "bigquery")]
BigQuery {
#[serde(default)]
project: Option<String>,
location: String,
service_account_key: crate::auth::models::Secret,
#[serde(default)]
base_url: Option<String>,
#[serde(default)]
storage_url: Option<String>,
},
#[serde(rename = "clickhouse")]
ClickHouse {
host: String,
#[serde(default)]
port: Option<u16>,
#[serde(default = "defaults::clickhouse_user")]
user: String,
#[serde(default)]
password: Option<crate::auth::models::Secret>,
#[serde(default)]
secure: bool,
},
Postgres {
host: String,
#[serde(default = "defaults::postgres_port")]
port: u16,
user: String,
#[serde(default)]
password: Option<crate::auth::models::Secret>,
database: String,
#[serde(default)]
sslmode: crate::postgres_source::models::SslMode,
#[serde(default)]
sslrootcert: Option<PathBuf>,
#[serde(default)]
sslcert: Option<PathBuf>,
#[serde(default)]
sslkey: Option<PathBuf>,
#[serde(default)]
ssh: Option<crate::ssh_tunnel::models::SshTunnel>,
},
S3 {
bucket: String,
#[serde(default)]
region: Option<String>,
#[serde(default)]
endpoint: Option<String>,
#[serde(default)]
profile: Option<String>,
},
Filesystem { root: String },
Sftp {
host: String,
#[serde(default)]
port: Option<u16>,
user: String,
#[serde(default)]
private_key_file: Option<PathBuf>,
#[serde(default)]
private_key_passphrase: Option<crate::auth::models::Secret>,
#[serde(default)]
password: Option<crate::auth::models::Secret>,
known_hosts: PathBuf,
},
#[serde(rename = "duckdb")]
DuckDb {
database: String,
},
#[serde(rename = "motherduck")]
MotherDuck {
#[serde(default)]
database: Option<String>,
#[serde(default)]
token: Option<crate::auth::models::Secret>,
},
Odbc {
#[serde(default)]
connection_string: Option<crate::auth::models::Secret>,
#[serde(default)]
dsn: Option<String>,
#[serde(default)]
user: Option<String>,
#[serde(default)]
password: Option<crate::auth::models::Secret>,
},
GoogleSheets {
#[serde(default)]
service_account_file: Option<PathBuf>,
#[serde(default)]
service_account_json: Option<crate::auth::models::Secret>,
#[serde(default)]
service_account_json_base64: Option<crate::auth::models::Secret>,
#[serde(default)]
client_id: Option<String>,
#[serde(default)]
client_secret: Option<crate::auth::models::Secret>,
#[serde(default)]
refresh_token: Option<crate::auth::models::Secret>,
#[serde(default)]
base_url: Option<String>,
#[serde(default)]
token_url: Option<String>,
},
Http {
base_url: String,
#[serde(default)]
token: Option<crate::auth::models::Secret>,
#[serde(default)]
username: Option<String>,
#[serde(default)]
password: Option<crate::auth::models::Secret>,
#[serde(default)]
client_id: Option<String>,
#[serde(default)]
client_secret: Option<crate::auth::models::Secret>,
#[serde(default)]
refresh_token: Option<crate::auth::models::Secret>,
#[serde(default)]
headers: BTreeMap<String, String>,
},
}
impl ConnectionConfig {
pub fn kind(&self) -> &'static str {
match self {
Self::Snowflake { .. } => "snowflake",
Self::Databricks { .. } => "databricks",
Self::BigQuery { .. } => "bigquery",
Self::ClickHouse { .. } => "clickhouse",
Self::Postgres { .. } => "postgres",
Self::DuckDb { .. } => "duckdb",
Self::MotherDuck { .. } => "motherduck",
Self::S3 { .. } => "s3",
Self::Filesystem { .. } => "filesystem",
Self::Sftp { .. } => "sftp",
Self::Odbc { .. } => "odbc",
Self::GoogleSheets { .. } => "google_sheets",
Self::Http { .. } => "http",
}
}
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct TargetConfig {
pub connection: String,
#[serde(default)]
pub database: Option<String>,
#[serde(default)]
pub loader_schema: Option<String>,
#[serde(default)]
pub path: Option<String>,
#[serde(default)]
pub format: Option<FilesFormat>,
#[serde(default)]
pub compression: Option<OutputCompression>,
#[serde(default)]
pub csv: Option<CsvOutput>,
#[serde(default)]
pub single_file: Option<bool>,
#[serde(default)]
pub max_footer_reads: Option<usize>,
#[serde(default)]
pub naming: Option<Naming>,
}
impl TargetConfig {
pub fn path_template(&self) -> &str {
self.path.as_deref().unwrap_or(DEFAULT_DESTINATION_PATH)
}
pub fn output_encoding(&self) -> OutputEncoding {
OutputEncoding {
format: self.format.unwrap_or_default(),
compression: self.compression.unwrap_or_default(),
csv: self.csv.clone().unwrap_or_default(),
single_file: self.single_file.unwrap_or_default(),
}
}
pub fn footer_read_limit(&self) -> usize {
self.max_footer_reads.unwrap_or(DEFAULT_MAX_FOOTER_READS)
}
}
#[derive(Debug, Clone, Deserialize)]
pub struct SourceConfig {
pub kind: SourceKind,
#[serde(default)]
pub shared_client: bool,
#[serde(default)]
pub parallel_tables: bool,
#[serde(default)]
pub connection: String,
#[serde(default)]
pub description: Option<String>,
#[serde(default)]
pub destination: Option<String>,
#[serde(default)]
pub schema: Option<String>,
#[serde(default)]
pub include: Vec<String>,
#[serde(default)]
pub exclude: Vec<String>,
#[serde(default)]
pub new_tables: NewTables,
#[serde(default)]
pub table_name: Option<String>,
#[serde(default)]
pub path: Option<String>,
#[serde(default)]
pub format: SourceFormat,
#[serde(default)]
pub compression: Option<Compression>,
#[serde(default)]
pub csv: Option<CsvOptions>,
#[serde(default)]
pub xlsx: Option<XlsxOptions>,
#[serde(default)]
pub xml: Option<XmlOptions>,
#[serde(default)]
pub members: Option<String>,
#[serde(default)]
pub after_load: AfterLoad,
#[serde(default)]
pub shape: RecordShape,
#[serde(default)]
pub record_column: Option<String>,
#[serde(default)]
pub destination_path: Option<String>,
#[serde(default)]
pub partition_columns: BTreeMap<String, String>,
#[serde(default)]
pub path_columns: Option<String>,
#[serde(default)]
pub unmatched_paths: Option<UnmatchedPaths>,
#[serde(default)]
pub copy_to_internal_stage: bool,
#[serde(default)]
pub exclude_columns: Vec<String>,
#[serde(default)]
pub skip_tables_with_columns: Vec<String>,
#[serde(default)]
pub allow_columns: Vec<String>,
#[serde(default)]
pub mask_key: Option<crate::auth::models::Secret>,
#[serde(flatten)]
pub defaults: TableSettings,
#[serde(default)]
pub tables: BTreeMap<String, TableOverride>,
#[serde(default)]
pub publication: Option<String>,
#[serde(default)]
pub slot: Option<String>,
#[serde(default)]
pub spreadsheet_id: Option<String>,
#[serde(default)]
pub ranges: BTreeMap<String, String>,
#[serde(default)]
pub cdc_mode: CdcMode,
#[serde(default)]
pub deletes: CdcDeletes,
#[serde(default, deserialize_with = "deserialize_optional_duration")]
pub batch_interval: Option<Duration>,
#[serde(default)]
pub warehouse: Option<String>,
#[serde(default)]
pub stage: Option<UnloadStage>,
#[serde(default)]
pub max_text_size: Option<usize>,
#[serde(default)]
pub max_binary_size: Option<usize>,
#[serde(default)]
pub project: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct UnloadStage {
#[serde(default)]
pub url: Option<String>,
#[serde(default)]
pub storage_integration: Option<String>,
#[serde(default)]
pub name: Option<String>,
#[serde(default)]
pub region: Option<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum CdcMode {
#[default]
Merge,
Changelog,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum CdcDeletes {
#[default]
Soft,
Hard,
}
impl SourceConfig {
pub fn table_constants(&self, table: &str) -> BTreeMap<String, toml::Value> {
self.tables
.get(table)
.and_then(|overrides| overrides.settings.constant_columns.as_ref())
.or(self.defaults.constant_columns.as_ref())
.map(|constants| constants.0.clone())
.unwrap_or_default()
}
pub fn discovers_tables(&self) -> bool {
matches!(
self.kind,
SourceKind::Postgres
| SourceKind::PostgresCdc
| SourceKind::Snowflake
| SourceKind::Odbc
| SourceKind::BigQuery
)
}
pub fn files_encoding(&self) -> FilesEncoding {
let mut xml = self.xml.clone().unwrap_or_default();
for path in self.tables.values().flat_map(|table| match &table.unnest {
Some(UnnestSetting::Paths(paths)) => paths.as_slice(),
Some(UnnestSetting::Word(_)) | None => &[],
}) {
let element = path.rsplit('.').next().unwrap_or_default().to_owned();
if !xml.lists.contains(&element) {
xml.lists.push(element);
}
}
FilesEncoding {
format: self.format,
compression: self.compression.unwrap_or_default(),
csv: self.csv.clone().unwrap_or_default(),
xlsx: self.xlsx.clone().unwrap_or_default(),
xml,
members: self.members.clone(),
record: (self.shape == RecordShape::Record).then(|| {
self.record_column
.clone()
.unwrap_or_else(|| RECORD_COLUMN.to_owned())
}),
flatten_objects: false,
}
}
pub fn table_name_template(&self) -> &str {
match (&self.table_name, self.kind) {
(Some(template), _) => template,
(
None,
SourceKind::Postgres
| SourceKind::PostgresCdc
| SourceKind::Snowflake
| SourceKind::Odbc
| SourceKind::BigQuery
| SourceKind::Python,
) => DEFAULT_TABLE_NAME,
(None, SourceKind::Files) => DEFAULT_FILES_TABLE_NAME,
(None, SourceKind::GoogleSheets) => DEFAULT_TABLE_NAME,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum SourceKind {
Postgres,
PostgresCdc,
Files,
Snowflake,
Odbc,
#[serde(rename = "bigquery")]
BigQuery,
Python,
GoogleSheets,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum FilesFormat {
#[default]
Parquet,
Csv,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum OutputCompression {
#[default]
None,
Gzip,
Zstd,
}
#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct CsvOutput {
pub delimiter: char,
pub header: bool,
pub quote: char,
pub null_value: String,
}
impl Default for CsvOutput {
fn default() -> Self {
Self {
delimiter: ',',
header: true,
quote: '"',
null_value: String::new(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub struct OutputEncoding {
pub format: FilesFormat,
pub compression: OutputCompression,
pub csv: CsvOutput,
pub single_file: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum NewTables {
#[default]
Sync,
Skip,
Fail,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize)]
pub struct TableSettings {
#[serde(default)]
pub write_strategy: Option<WriteStrategy>,
#[serde(default, deserialize_with = "deserialize_optional_keys")]
pub unique_key: Option<Vec<String>>,
#[serde(default)]
pub cursor_column: Option<String>,
#[serde(default)]
pub cursor_start: Option<String>,
#[serde(default, deserialize_with = "deserialize_optional_duration")]
pub lookback: Option<Duration>,
#[serde(default, deserialize_with = "deserialize_optional_duration")]
pub lookahead: Option<Duration>,
#[serde(default)]
pub lookback_from: Option<LookbackFrom>,
#[serde(default)]
pub append_cursor_inclusive: Option<bool>,
#[serde(default)]
pub contract: Option<Contract>,
#[serde(default)]
pub on_schema_change: Option<OnSchemaChange>,
#[serde(default)]
pub auto_widen: Option<bool>,
#[serde(default)]
pub type_change_rebuild: Option<Policy>,
#[serde(default)]
pub allow_empty: Option<bool>,
#[serde(default)]
pub state: Option<StateMode>,
#[cfg(feature = "container-tests")]
#[serde(default)]
pub test_loader_route: Option<bool>,
#[serde(default, deserialize_with = "deserialize_optional_batch")]
pub batch_size: Option<BatchStep>,
#[serde(default)]
pub checks: Option<TableChecks>,
#[serde(default)]
pub engine: Option<String>,
#[serde(default, deserialize_with = "deserialize_optional_keys")]
pub order_by: Option<Vec<String>>,
#[serde(default)]
pub partition_by: Option<String>,
#[serde(default, deserialize_with = "deserialize_optional_keys")]
pub primary_key: Option<Vec<String>>,
#[serde(default)]
pub table_settings: Option<BTreeMap<String, LayoutSetting>>,
#[serde(default)]
pub header: Option<bool>,
#[serde(default)]
pub header_row: Option<u32>,
#[serde(default)]
pub constant_columns: Option<ConstantColumns>,
#[serde(default)]
pub on_duplicate_keys: Option<OnDuplicateKeys>,
#[serde(default)]
pub dedup_sort: Option<BTreeMap<String, SortDirection>>,
#[serde(default)]
pub naming: Option<Naming>,
#[serde(default)]
pub row_id: Option<bool>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum OnDuplicateKeys {
#[default]
Dedupe,
Fail,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum SortDirection {
Asc,
Desc,
}
impl std::fmt::Display for SortDirection {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str(match self {
Self::Asc => "asc",
Self::Desc => "desc",
})
}
}
#[derive(Debug, Clone, Default, PartialEq, Deserialize)]
#[serde(transparent)]
pub struct ConstantColumns(pub BTreeMap<String, toml::Value>);
impl Eq for ConstantColumns {}
#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
#[serde(untagged)]
pub enum LayoutSetting {
Bool(bool),
Integer(i64),
Text(String),
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct TableChecks {
#[serde(default)]
pub unique: Option<Vec<String>>,
#[serde(default)]
pub not_null: Vec<String>,
#[serde(default)]
pub min_rows: Option<u64>,
#[serde(default, deserialize_with = "deserialize_optional_percent")]
pub max_row_drop: Option<u8>,
#[serde(default)]
pub warn: Vec<CheckName>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum CheckName {
Unique,
NotNull,
MinRows,
MaxRowDrop,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum LookbackFrom {
#[default]
Destination,
Now,
}
#[derive(Debug, Clone, Default, Deserialize)]
pub struct TableOverride {
#[serde(flatten)]
pub settings: TableSettings,
#[serde(default)]
pub description: Option<String>,
#[serde(default)]
pub name: Option<String>,
#[serde(default)]
pub destination_path: Option<String>,
#[serde(default)]
pub sql: Option<String>,
#[serde(default)]
pub columns: BTreeMap<String, String>,
#[serde(default)]
pub mask: BTreeMap<String, MaskMethod>,
#[serde(default)]
pub exclude_columns: Vec<String>,
#[serde(default)]
pub allow_columns: Vec<String>,
#[serde(default)]
pub flatten: Vec<String>,
#[serde(default)]
pub unnest: Option<UnnestSetting>,
#[serde(default)]
pub max_depth: Option<u32>,
}
#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
#[serde(untagged)]
pub enum UnnestSetting {
Word(String),
Paths(Vec<String>),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum MaskMethod {
Hash,
Hmac,
}
impl std::fmt::Display for MaskMethod {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str(match self {
Self::Hash => "hash",
Self::Hmac => "hmac",
})
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum WriteStrategy {
Append,
Merge,
DeleteInsert,
Replace,
}
impl std::fmt::Display for WriteStrategy {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str(match self {
Self::Append => "append",
Self::Merge => "merge",
Self::DeleteInsert => "delete_insert",
Self::Replace => "replace",
})
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BatchStep {
Duration(std::time::Duration),
Integer(u64),
Whole,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum StateMode {
#[default]
Stateless,
Logged,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Contract {
#[default]
None,
Enforced,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum OnSchemaChange {
#[default]
AppendNewColumns,
Ignore,
Fail,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Policy {
Allow,
#[default]
RequireConfirmation,
Deny,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct MotherDuckPath<'a> {
pub prefixed: bool,
pub name: Option<&'a str>,
pub options: &'a str,
}