use std::collections::BTreeMap;
use std::path::PathBuf;
use std::time::Duration;
use serde::Deserialize;
use crate::config::models::AuthConfig;
use crate::files_source::models::{Compression, CsvOptions, FilesEncoding, SourceFormat};
use crate::project::_helpers::defaults;
use crate::project::_helpers::serde_values::{
deserialize_optional_batch, deserialize_optional_duration, deserialize_optional_keys,
};
use crate::project::constants::{
DEFAULT_DESTINATION_PATH, DEFAULT_FILES_TABLE_NAME, DEFAULT_MAX_FOOTER_READS,
DEFAULT_TABLE_NAME,
};
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ProjectConfig {
#[serde(default)]
pub python_paths: Vec<PathBuf>,
#[serde(default)]
pub default_target: Option<String>,
#[serde(default)]
pub concurrency: Option<usize>,
#[serde(default)]
pub connections: BTreeMap<String, ConnectionConfig>,
#[serde(default)]
pub targets: BTreeMap<String, TargetConfig>,
#[serde(default)]
pub sources: BTreeMap<String, SourceConfig>,
#[serde(skip)]
pub connection_settings: serde_json::Map<String, serde_json::Value>,
#[serde(skip)]
pub python_modules: BTreeMap<String, String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
pub enum LoadMethod {
ExternalStage {
url: String,
#[serde(default)]
storage_integration: Option<String>,
#[serde(default)]
name: Option<String>,
#[serde(default)]
region: Option<String>,
#[serde(default)]
endpoint: Option<String>,
#[serde(default)]
keep_staged_files: bool,
},
InternalUserStage {},
InternalTableStage {},
InternalNamedStage { name: String },
Streaming {},
Insert {},
}
#[derive(Debug, Clone, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum ConnectionConfig {
Snowflake {
account: String,
user: String,
auth: AuthConfig,
#[serde(default)]
role: Option<String>,
#[serde(default)]
warehouse: Option<String>,
database: String,
#[serde(default)]
base_url: Option<String>,
#[serde(default)]
load_method: Option<Box<LoadMethod>>,
},
Databricks {
host: String,
client_id: String,
client_secret: crate::auth::models::Secret,
warehouse_id: String,
catalog: String,
volume: String,
},
Postgres {
host: String,
#[serde(default = "defaults::postgres_port")]
port: u16,
user: String,
#[serde(default)]
password: Option<crate::auth::models::Secret>,
database: String,
},
S3 {
bucket: String,
#[serde(default)]
region: Option<String>,
#[serde(default)]
endpoint: Option<String>,
},
Filesystem { root: String },
#[serde(rename = "duckdb")]
DuckDb {
database: String,
},
Http {
base_url: String,
#[serde(default)]
token: Option<crate::auth::models::Secret>,
#[serde(default)]
username: Option<String>,
#[serde(default)]
password: Option<crate::auth::models::Secret>,
#[serde(default)]
headers: BTreeMap<String, String>,
},
}
impl ConnectionConfig {
pub fn kind(&self) -> &'static str {
match self {
Self::Snowflake { .. } => "snowflake",
Self::Databricks { .. } => "databricks",
Self::Postgres { .. } => "postgres",
Self::DuckDb { .. } => "duckdb",
Self::S3 { .. } => "s3",
Self::Filesystem { .. } => "filesystem",
Self::Http { .. } => "http",
}
}
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct TargetConfig {
pub connection: String,
#[serde(default)]
pub database: Option<String>,
#[serde(default)]
pub loader_schema: Option<String>,
#[serde(default)]
pub path: Option<String>,
#[serde(default)]
pub format: Option<FilesFormat>,
#[serde(default)]
pub max_footer_reads: Option<usize>,
}
impl TargetConfig {
pub fn path_template(&self) -> &str {
self.path.as_deref().unwrap_or(DEFAULT_DESTINATION_PATH)
}
pub fn footer_read_limit(&self) -> usize {
self.max_footer_reads.unwrap_or(DEFAULT_MAX_FOOTER_READS)
}
}
#[derive(Debug, Clone, Deserialize)]
pub struct SourceConfig {
pub kind: SourceKind,
#[serde(default)]
pub shared_client: bool,
#[serde(default)]
pub parallel_tables: bool,
#[serde(default)]
pub connection: String,
#[serde(default)]
pub description: Option<String>,
#[serde(default)]
pub destination: Option<String>,
#[serde(default)]
pub schema: Option<String>,
#[serde(default)]
pub include: Vec<String>,
#[serde(default)]
pub exclude: Vec<String>,
#[serde(default)]
pub new_tables: NewTables,
#[serde(default)]
pub table_name: Option<String>,
#[serde(default)]
pub path: Option<String>,
#[serde(default)]
pub format: SourceFormat,
#[serde(default)]
pub compression: Option<Compression>,
#[serde(default)]
pub csv: Option<CsvOptions>,
#[serde(default)]
pub destination_path: Option<String>,
#[serde(default)]
pub partition_columns: BTreeMap<String, String>,
#[serde(default)]
pub copy_to_internal_stage: bool,
#[serde(default)]
pub exclude_columns: Vec<String>,
#[serde(default)]
pub skip_tables_with_columns: Vec<String>,
#[serde(default)]
pub allow_columns: Vec<String>,
#[serde(default)]
pub mask_key: Option<crate::auth::models::Secret>,
#[serde(flatten)]
pub defaults: TableSettings,
#[serde(default)]
pub tables: BTreeMap<String, TableOverride>,
}
impl SourceConfig {
pub fn files_encoding(&self) -> FilesEncoding {
FilesEncoding {
format: self.format,
compression: self.compression.unwrap_or_default(),
csv: self.csv.clone().unwrap_or_default(),
}
}
pub fn table_name_template(&self) -> &str {
match (&self.table_name, self.kind) {
(Some(template), _) => template,
(None, SourceKind::Postgres | SourceKind::Python) => DEFAULT_TABLE_NAME,
(None, SourceKind::Files) => DEFAULT_FILES_TABLE_NAME,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum SourceKind {
Postgres,
Files,
Python,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum FilesFormat {
#[default]
Parquet,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum NewTables {
#[default]
Sync,
Skip,
Fail,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize)]
pub struct TableSettings {
#[serde(default)]
pub write_strategy: Option<WriteStrategy>,
#[serde(default, deserialize_with = "deserialize_optional_keys")]
pub unique_key: Option<Vec<String>>,
#[serde(default)]
pub cursor_column: Option<String>,
#[serde(default)]
pub cursor_start: Option<String>,
#[serde(default, deserialize_with = "deserialize_optional_duration")]
pub lookback: Option<Duration>,
#[serde(default)]
pub append_cursor_inclusive: Option<bool>,
#[serde(default)]
pub contract: Option<Contract>,
#[serde(default)]
pub on_schema_change: Option<OnSchemaChange>,
#[serde(default)]
pub auto_widen: Option<bool>,
#[serde(default)]
pub type_change_rebuild: Option<Policy>,
#[serde(default)]
pub allow_empty: Option<bool>,
#[serde(default)]
pub state: Option<StateMode>,
#[cfg(feature = "container-tests")]
#[serde(default)]
pub test_loader_route: Option<bool>,
#[serde(default, deserialize_with = "deserialize_optional_batch")]
pub batch_size: Option<BatchStep>,
}
#[derive(Debug, Clone, Default, Deserialize)]
pub struct TableOverride {
#[serde(flatten)]
pub settings: TableSettings,
#[serde(default)]
pub description: Option<String>,
#[serde(default)]
pub name: Option<String>,
#[serde(default)]
pub destination_path: Option<String>,
#[serde(default)]
pub columns: BTreeMap<String, String>,
#[serde(default)]
pub mask: BTreeMap<String, MaskMethod>,
#[serde(default)]
pub exclude_columns: Vec<String>,
#[serde(default)]
pub allow_columns: Vec<String>,
#[serde(default)]
pub flatten: Vec<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum MaskMethod {
Hash,
Hmac,
}
impl std::fmt::Display for MaskMethod {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str(match self {
Self::Hash => "hash",
Self::Hmac => "hmac",
})
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum WriteStrategy {
Append,
Merge,
DeleteInsert,
Replace,
}
impl std::fmt::Display for WriteStrategy {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str(match self {
Self::Append => "append",
Self::Merge => "merge",
Self::DeleteInsert => "delete_insert",
Self::Replace => "replace",
})
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BatchStep {
Duration(std::time::Duration),
Integer(u64),
Whole,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum StateMode {
#[default]
Stateless,
Logged,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Contract {
#[default]
None,
Enforced,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum OnSchemaChange {
#[default]
AppendNewColumns,
Ignore,
Fail,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Policy {
Allow,
#[default]
RequireConfirmation,
Deny,
}