use std::collections::BTreeMap;
use std::path::PathBuf;
use std::time::Duration;
use serde::Deserialize;
use crate::config::models::AuthConfig;
use crate::project::_helpers::defaults;
use crate::project::_helpers::serde_values::{
deserialize_optional_duration, deserialize_optional_keys,
};
use crate::project::constants::{DEFAULT_FILES_TABLE_NAME, DEFAULT_TABLE_NAME};
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ProjectConfig {
#[serde(default)]
pub python_paths: Vec<PathBuf>,
#[serde(default)]
pub default_target: Option<String>,
#[serde(default)]
pub connections: BTreeMap<String, ConnectionConfig>,
#[serde(default)]
pub targets: BTreeMap<String, TargetConfig>,
#[serde(default)]
pub sources: BTreeMap<String, SourceConfig>,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
pub enum ConnectionConfig {
Snowflake {
account: String,
user: String,
auth: AuthConfig,
#[serde(default)]
role: Option<String>,
#[serde(default)]
warehouse: Option<String>,
database: String,
#[serde(default)]
base_url: Option<String>,
},
Postgres {
host: String,
#[serde(default = "defaults::postgres_port")]
port: u16,
user: String,
#[serde(default)]
password: Option<crate::auth::models::Secret>,
database: String,
},
S3 {
bucket: String,
#[serde(default)]
region: Option<String>,
#[serde(default)]
endpoint: Option<String>,
},
Filesystem { root: String },
#[serde(rename = "duckdb")]
DuckDb {
path: String,
},
}
impl ConnectionConfig {
pub fn kind(&self) -> &'static str {
match self {
Self::Snowflake { .. } => "snowflake",
Self::Postgres { .. } => "postgres",
Self::DuckDb { .. } => "duckdb",
Self::S3 { .. } => "s3",
Self::Filesystem { .. } => "filesystem",
}
}
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct TargetConfig {
pub connection: String,
#[serde(default)]
pub database: Option<String>,
pub loader_schema: String,
}
#[derive(Debug, Clone, Deserialize)]
pub struct SourceConfig {
pub kind: SourceKind,
pub connection: String,
#[serde(default)]
pub description: Option<String>,
#[serde(default)]
pub destination: Option<String>,
#[serde(default)]
pub schema: Option<String>,
#[serde(default)]
pub include: Vec<String>,
#[serde(default)]
pub exclude: Vec<String>,
#[serde(default)]
pub new_tables: NewTables,
#[serde(default)]
pub table_name: Option<String>,
#[serde(default)]
pub path: Option<String>,
#[serde(default)]
pub format: FilesFormat,
#[serde(default)]
pub partition_columns: BTreeMap<String, String>,
#[serde(default)]
pub exclude_columns: Vec<String>,
#[serde(default)]
pub skip_tables_with_columns: Vec<String>,
#[serde(default)]
pub allow_columns: Vec<String>,
#[serde(default)]
pub mask_key: Option<crate::auth::models::Secret>,
#[serde(flatten)]
pub defaults: TableSettings,
#[serde(default)]
pub tables: BTreeMap<String, TableOverride>,
}
impl SourceConfig {
pub fn table_name_template(&self) -> &str {
match (&self.table_name, self.kind) {
(Some(template), _) => template,
(None, SourceKind::Postgres) => DEFAULT_TABLE_NAME,
(None, SourceKind::Files) => DEFAULT_FILES_TABLE_NAME,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum SourceKind {
Postgres,
Files,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum FilesFormat {
#[default]
Parquet,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum NewTables {
#[default]
Sync,
Skip,
Fail,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize)]
pub struct TableSettings {
#[serde(default)]
pub write_strategy: Option<WriteStrategy>,
#[serde(default, deserialize_with = "deserialize_optional_keys")]
pub unique_key: Option<Vec<String>>,
#[serde(default)]
pub cursor_column: Option<String>,
#[serde(default)]
pub cursor_start: Option<String>,
#[serde(default, deserialize_with = "deserialize_optional_duration")]
pub lookback: Option<Duration>,
#[serde(default)]
pub append_cursor_inclusive: Option<bool>,
#[serde(default)]
pub contract: Option<Contract>,
#[serde(default)]
pub on_schema_change: Option<OnSchemaChange>,
#[serde(default)]
pub auto_widen: Option<bool>,
#[serde(default)]
pub type_change_rebuild: Option<Policy>,
}
#[derive(Debug, Clone, Default, Deserialize)]
pub struct TableOverride {
#[serde(flatten)]
pub settings: TableSettings,
#[serde(default)]
pub description: Option<String>,
#[serde(default)]
pub name: Option<String>,
#[serde(default)]
pub columns: BTreeMap<String, String>,
#[serde(default)]
pub mask: BTreeMap<String, MaskMethod>,
#[serde(default)]
pub exclude_columns: Vec<String>,
#[serde(default)]
pub allow_columns: Vec<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum MaskMethod {
Hash,
Hmac,
}
impl std::fmt::Display for MaskMethod {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str(match self {
Self::Hash => "hash",
Self::Hmac => "hmac",
})
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum WriteStrategy {
Append,
Merge,
DeleteInsert,
Replace,
}
impl std::fmt::Display for WriteStrategy {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str(match self {
Self::Append => "append",
Self::Merge => "merge",
Self::DeleteInsert => "delete_insert",
Self::Replace => "replace",
})
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Contract {
#[default]
None,
Enforced,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum OnSchemaChange {
#[default]
AppendNewColumns,
Ignore,
Fail,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Policy {
Allow,
#[default]
RequireConfirmation,
Deny,
}