use std::collections::BTreeMap;
use std::net::SocketAddr;
use std::path::PathBuf;
use std::time::Duration;
use serde::{Deserialize, Deserializer};
use crate::auth::models::Secret;
use crate::config::_helpers::defaults;
use crate::config::_helpers::units::{
deserialize_duration, deserialize_optional_duration, parse_byte_size,
};
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Config {
pub source: SourceConfig,
pub destination: DestinationConfig,
pub staging: StagingConfig,
#[serde(default)]
pub batching: BatchingConfig,
#[serde(default)]
pub envelope: EnvelopeConfig,
#[serde(default)]
pub naming: NamingConfig,
#[serde(default)]
pub topics: Vec<TopicOverride>,
#[serde(default)]
pub metrics: MetricsConfig,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct SourceConfig {
pub kind: SourceKind,
pub brokers: String,
pub include: Vec<String>,
#[serde(default)]
pub exclude: Vec<String>,
#[serde(default = "defaults::consumer_group")]
pub consumer_group: String,
#[serde(default)]
pub start_from: StartFrom,
#[serde(
default = "defaults::discovery_interval",
deserialize_with = "deserialize_duration"
)]
pub discovery_interval: Duration,
#[serde(default = "defaults::shards")]
pub shards: u32,
#[serde(default)]
pub shard: u32,
#[serde(default = "defaults::prefetch_budget")]
pub prefetch_budget: ByteSize,
#[serde(default = "defaults::startup_batch")]
pub startup_batch: usize,
#[serde(
default = "defaults::startup_interval",
deserialize_with = "deserialize_duration"
)]
pub startup_interval: Duration,
#[serde(default)]
pub properties: BTreeMap<String, String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum SourceKind {
Kafka,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum StartFrom {
Earliest,
#[default]
Latest,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct DestinationConfig {
pub kind: DestinationKind,
pub account: String,
pub user: String,
pub auth: AuthConfig,
#[serde(default)]
pub role: Option<String>,
#[serde(default)]
pub warehouse: Option<String>,
pub database: String,
pub schema: String,
#[serde(default)]
pub base_url: Option<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum DestinationKind {
Snowflake,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(tag = "method", rename_all = "snake_case", deny_unknown_fields)]
pub enum AuthConfig {
Pat {
token: Secret,
},
KeyPair {
private_key_file: PathBuf,
},
OauthAuthorizationCode {
#[serde(default)]
client_id: Option<String>,
#[serde(default)]
client_secret: Option<Secret>,
#[serde(default)]
redirect_uri: Option<String>,
},
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct StagingConfig {
pub kind: StagingKind,
#[serde(
default = "defaults::retain",
deserialize_with = "deserialize_duration"
)]
pub retain: Duration,
#[serde(default)]
pub load: LoadMethod,
#[serde(default)]
pub format: FileFormat,
#[serde(default = "defaults::max_in_flight")]
pub max_in_flight: usize,
#[serde(default = "defaults::max_pending_files")]
pub max_pending_files: usize,
#[serde(default)]
pub on_load_error: LoadErrorPolicy,
#[serde(
default = "defaults::pending_check_after",
deserialize_with = "deserialize_duration"
)]
pub pending_check_after: Duration,
#[serde(
default = "defaults::pending_recheck_every",
deserialize_with = "deserialize_duration"
)]
pub pending_recheck_every: Duration,
#[serde(default)]
pub s3: Option<S3Config>,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum LoadErrorPolicy {
#[default]
Halt,
Skip,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum FileFormat {
#[default]
Json,
Parquet,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum StagingKind {
None,
Internal,
S3,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum LoadMethod {
#[default]
Auto,
Snowpipe,
Copy,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct S3Config {
pub bucket: String,
#[serde(default)]
pub prefix: String,
pub region: String,
pub stage: String,
#[serde(default = "defaults::s3_path")]
pub path: String,
#[serde(default)]
pub endpoint: Option<String>,
#[serde(default)]
pub path_style: bool,
#[serde(default = "defaults::auto_ingest")]
pub auto_ingest: bool,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct BatchingConfig {
#[serde(default = "defaults::max_bytes")]
pub max_bytes: ByteSize,
#[serde(default = "defaults::max_rows")]
pub max_rows: usize,
#[serde(
default = "defaults::max_age",
deserialize_with = "deserialize_duration"
)]
pub max_age: Duration,
#[serde(default = "defaults::max_total_bytes")]
pub max_total_bytes: ByteSize,
#[serde(default = "defaults::max_request_bytes")]
pub max_request_bytes: ByteSize,
#[serde(default)]
pub oversized: OversizedPolicy,
}
impl Default for BatchingConfig {
fn default() -> Self {
Self {
max_bytes: defaults::max_bytes(),
max_rows: defaults::max_rows(),
max_age: defaults::max_age(),
max_total_bytes: defaults::max_total_bytes(),
max_request_bytes: defaults::max_request_bytes(),
oversized: OversizedPolicy::default(),
}
}
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum OversizedPolicy {
#[default]
Fail,
Skip,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct EnvelopeConfig {
#[serde(default = "defaults::columns")]
pub columns: Vec<ColumnConfig>,
#[serde(default = "defaults::error_logging")]
pub error_logging: bool,
#[serde(default = "defaults::time_zone")]
pub time_zone: String,
}
impl Default for EnvelopeConfig {
fn default() -> Self {
Self {
columns: defaults::columns(),
error_logging: defaults::error_logging(),
time_zone: defaults::time_zone(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ColumnConfig {
pub name: String,
pub field: EnvelopeField,
#[serde(rename = "type")]
pub sql_type: String,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum EnvelopeField {
Topic,
Partition,
Offset,
Key,
Timestamp,
TimestampType,
Value,
Headers,
ExportedAt,
Date,
Null,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct NamingConfig {
#[serde(default)]
pub strip_prefix: Option<String>,
#[serde(default)]
pub table_prefix: String,
#[serde(default)]
pub fallback: Option<String>,
#[serde(default)]
pub rules: Vec<NamingRule>,
#[serde(default)]
pub map: BTreeMap<String, String>,
#[serde(default)]
pub unmatched: UnmatchedPolicy,
#[serde(default = "defaults::stage_template")]
pub stage: String,
#[serde(default = "defaults::pipe_template")]
pub pipe: String,
#[serde(default = "defaults::streaming_pipe_template")]
pub streaming_pipe: String,
}
impl Default for NamingConfig {
fn default() -> Self {
Self {
strip_prefix: None,
table_prefix: String::new(),
fallback: None,
rules: Vec::new(),
map: BTreeMap::new(),
unmatched: UnmatchedPolicy::default(),
stage: defaults::stage_template(),
pipe: defaults::pipe_template(),
streaming_pipe: defaults::streaming_pipe_template(),
}
}
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct NamingRule {
#[serde(rename = "match")]
pub pattern: String,
#[serde(default)]
pub table: Option<String>,
#[serde(default)]
pub skip: bool,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum UnmatchedPolicy {
#[default]
Default,
Fail,
}
#[derive(Debug, Clone, Default, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct TopicOverride {
#[serde(rename = "match")]
pub pattern: String,
#[serde(default)]
pub start_from: Option<StartFrom>,
#[serde(default)]
pub max_bytes: Option<ByteSize>,
#[serde(default)]
pub max_rows: Option<usize>,
#[serde(default, deserialize_with = "deserialize_optional_duration")]
pub max_age: Option<Duration>,
#[serde(default)]
pub oversized: Option<OversizedPolicy>,
#[serde(default)]
pub unwrap: BTreeMap<String, String>,
}
#[derive(Debug, Clone, Default, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct MetricsConfig {
#[serde(default)]
pub listen: Option<SocketAddr>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct StreamSettings {
pub start_from: StartFrom,
pub max_bytes: u64,
pub max_rows: usize,
pub max_age: Duration,
pub oversized: OversizedPolicy,
pub unwrap: BTreeMap<String, String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub struct ByteSize(pub u64);
#[derive(Debug, Deserialize)]
#[serde(untagged)]
enum RawSize {
Number(u64),
Text(String),
}
impl<'de> Deserialize<'de> for ByteSize {
fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
match RawSize::deserialize(deserializer)? {
RawSize::Number(bytes) => Ok(Self(bytes)),
RawSize::Text(text) => parse_byte_size(&text)
.map(Self)
.map_err(serde::de::Error::custom),
}
}
}