faucet-source-oracle-cdc 1.2.2

Oracle Database CDC (LogMiner) source for the faucet-stream ecosystem
Documentation
//! Configuration for [`OracleCdcSource`](crate::OracleCdcSource).

use std::time::Duration;

use faucet_common_oracle::{OracleConnectionConfig, quote_ident_oracle, split_table};
use faucet_core::{DEFAULT_BATCH_SIZE, FaucetError, validate_batch_size};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};

fn default_poll_interval() -> Duration {
    Duration::from_secs(1)
}
fn default_idle_timeout() -> Duration {
    Duration::from_secs(30)
}
fn default_batch_size() -> usize {
    DEFAULT_BATCH_SIZE
}
fn default_max_connections() -> u32 {
    2
}
fn default_statement_timeout_secs() -> u64 {
    600
}
fn default_max_scn_window() -> u64 {
    500_000
}
fn default_flush_table() -> String {
    "FAUCET_LOGMNR_FLUSH".into()
}

/// Where to start mining on a fresh run (no persisted bookmark).
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum StartPosition {
    /// The database's current SCN — only changes committed from now on
    /// (transactions already open are captured from their start). Default.
    #[default]
    Current,
    /// The oldest SCN still present in the available redo (online + archived).
    Earliest,
}

/// What to do with a change LogMiner cannot render as SQL for a tracked table
/// (`UNSUPPORTED` rows, or a *dictionary mismatch* after the table's structure
/// changed since the redo was written).
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum OnUnsupported {
    /// Fail the run (default) — never silently drop a change.
    #[default]
    Fail,
    /// Log a warning and skip the change.
    Skip,
}

/// Configuration for the Oracle LogMiner CDC source.
#[derive(Clone, Serialize, Deserialize, JsonSchema)]
#[schemars(extend("x-faucet-follows-servers" = true))]
pub struct OracleCdcSourceConfig {
    /// Connection settings. Connect to the PDB that owns the tables (Oracle
    /// 21c+ mines per PDB), or to a non-CDB / the CDB root.
    #[serde(flatten)]
    pub connection: OracleConnectionConfig,
    /// Tables to capture as `OWNER.TABLE`, in data-dictionary case (usually
    /// upper case). **Required and non-empty.**
    pub tables: Vec<String>,
    /// Start position on a fresh run. Default `current`.
    #[serde(default)]
    pub start_position: StartPosition,
    /// Seconds to wait between polls that find nothing new. Default 1s.
    #[serde(
        default = "default_poll_interval",
        with = "faucet_core::config::duration_secs"
    )]
    #[schemars(with = "u64")]
    pub poll_interval: Duration,
    /// End the fetch cycle after this long without a captured change, counted
    /// only once mining has reached the current SCN. Default 30s. A
    /// long-running runtime re-invokes the source to keep tailing.
    #[serde(
        default = "default_idle_timeout",
        with = "faucet_core::config::duration_secs"
    )]
    #[schemars(with = "u64")]
    pub idle_timeout: Duration,
    /// Largest SCN range mined per LogMiner session. Default 500000.
    #[serde(default = "default_max_scn_window")]
    pub max_scn_window: u64,
    /// Abort when one in-flight transaction buffers more than this many
    /// changes (bounded memory). Defaults to 1,000,000; `null` = unbounded.
    #[serde(default = "faucet_core::replication::default_max_staged_records")]
    #[schemars(extend("x-faucet-caller-max" = 1_000_000))]
    pub max_staged_records: Option<usize>,
    /// Abort when all open transactions together buffer more than this many
    /// changes, counting LOB pieces and folded updates. Unset or `null` =
    /// unbounded.
    #[serde(default)]
    #[schemars(extend("x-faucet-caller-max" = 1_000_000, "x-faucet-caller-set" = true))]
    pub max_staged_records_total: Option<usize>,
    /// `0` aggregates committed transactions into pages of up to 100,000
    /// records, emitted at least every `idle_timeout`, each with its bookmark;
    /// otherwise each committed transaction is its own page. Default
    /// [`DEFAULT_BATCH_SIZE`].
    #[serde(default = "default_batch_size")]
    pub batch_size: usize,
    /// Maximum pooled sessions. Default 2.
    #[serde(default = "default_max_connections")]
    pub max_connections: u32,
    /// Per-round-trip call timeout in seconds (`0` disables). Default 600.
    #[serde(default = "default_statement_timeout_secs")]
    pub statement_timeout_secs: u64,
    /// Table the source commits to before each mining window, forcing the log
    /// writer to flush redo up to the window's end SCN. Created in the
    /// connecting user's schema when missing. Default `FAUCET_LOGMNR_FLUSH`.
    #[serde(default = "default_flush_table")]
    pub flush_table: String,
    /// Handling of changes LogMiner cannot render. Default `fail`.
    #[serde(default)]
    pub on_unsupported: OnUnsupported,
    /// Explicit state-store key. Derived as `oracle-cdc:<database>:<table>`
    /// (or a digest of several tables) when unset.
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub state_key: Option<String>,
}

impl std::fmt::Debug for OracleCdcSourceConfig {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("OracleCdcSourceConfig")
            .field("connection", &self.connection)
            .field("tables", &self.tables)
            .field("start_position", &self.start_position)
            .field("poll_interval", &self.poll_interval)
            .field("idle_timeout", &self.idle_timeout)
            .field("max_scn_window", &self.max_scn_window)
            .field("max_staged_records", &self.max_staged_records)
            .field("max_staged_records_total", &self.max_staged_records_total)
            .field("batch_size", &self.batch_size)
            .field("max_connections", &self.max_connections)
            .field("statement_timeout_secs", &self.statement_timeout_secs)
            .field("flush_table", &self.flush_table)
            .field("on_unsupported", &self.on_unsupported)
            .field("state_key", &self.state_key)
            .finish()
    }
}

impl OracleCdcSourceConfig {
    /// A config capturing `tables` with defaults elsewhere.
    pub fn new(connection: OracleConnectionConfig, tables: Vec<String>) -> Self {
        Self {
            connection,
            tables,
            start_position: StartPosition::default(),
            poll_interval: default_poll_interval(),
            idle_timeout: default_idle_timeout(),
            max_scn_window: default_max_scn_window(),
            max_staged_records: faucet_core::replication::default_max_staged_records(),
            max_staged_records_total: None,
            batch_size: default_batch_size(),
            max_connections: default_max_connections(),
            statement_timeout_secs: default_statement_timeout_secs(),
            flush_table: default_flush_table(),
            on_unsupported: OnUnsupported::default(),
            state_key: None,
        }
    }

    /// Validate the fail-fast invariants.
    pub fn validate(&self) -> Result<(), FaucetError> {
        self.connection.validate()?;
        if self.tables.is_empty() {
            return Err(FaucetError::Config(
                "oracle-cdc: `tables` must list at least one OWNER.TABLE".into(),
            ));
        }
        let mut seen = std::collections::BTreeSet::new();
        for t in &self.tables {
            let (owner, name) = split_table(t)?;
            let Some(owner) = owner else {
                return Err(FaucetError::Config(format!(
                    "oracle-cdc: table {t:?} must be owner-qualified (OWNER.TABLE)"
                )));
            };
            quote_ident_oracle(&owner)?;
            quote_ident_oracle(&name)?;
            if !seen.insert(t) {
                return Err(FaucetError::Config(format!(
                    "oracle-cdc: duplicate table {t:?} in `tables`"
                )));
            }
        }
        quote_ident_oracle(&self.flush_table)?;
        if self.poll_interval.is_zero() || self.idle_timeout.is_zero() {
            return Err(FaucetError::Config(
                "oracle-cdc: poll_interval and idle_timeout must be > 0".into(),
            ));
        }
        if self.max_scn_window == 0 {
            return Err(FaucetError::Config(
                "oracle-cdc: max_scn_window must be > 0".into(),
            ));
        }
        validate_batch_size(self.batch_size)?;
        faucet_core::state::validate_state_key(&self.resolved_state_key())?;
        Ok(())
    }

    /// `(owner, table)` pairs of the captured tables.
    pub fn table_pairs(&self) -> Vec<(String, String)> {
        self.tables
            .iter()
            .filter_map(|t| match split_table(t) {
                Ok((Some(o), n)) => Some((o, n)),
                _ => None,
            })
            .collect()
    }

    /// The state-store key for the SCN bookmark.
    pub fn resolved_state_key(&self) -> String {
        if let Some(k) = &self.state_key {
            return k.clone();
        }
        self.key_for_scope(self.connection.scope_label())
    }

    /// Keys the SCN bookmark was stored under by releases whose scope did not
    /// name the host and port; empty when the key is configured explicitly.
    pub fn legacy_state_keys(&self) -> Vec<String> {
        if self.state_key.is_some() {
            return Vec::new();
        }
        self.connection
            .legacy_scope_label()
            .map(|scope| vec![self.key_for_scope(scope)])
            .unwrap_or_default()
    }

    fn key_for_scope(&self, scope: String) -> String {
        let scope = if scope.is_empty() {
            "oracle".into()
        } else {
            scope
        };
        if self.tables.len() == 1 {
            let t: String = self.tables[0]
                .chars()
                .map(|c| {
                    if c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | '.') {
                        c
                    } else {
                        '_'
                    }
                })
                .collect();
            return format!("oracle-cdc:{scope}:{t}");
        }
        let mut sorted: Vec<&str> = self.tables.iter().map(String::as_str).collect();
        sorted.sort_unstable();
        format!("oracle-cdc:{scope}:{}", fnv1a_hex(&sorted.join(",")))
    }
}

fn fnv1a_hex(s: &str) -> String {
    let mut hash: u64 = 0xcbf2_9ce4_8422_2325;
    for b in s.bytes() {
        hash ^= u64::from(b);
        hash = hash.wrapping_mul(0x0000_0100_0000_01b3);
    }
    format!("{hash:016x}")
}

#[cfg(test)]
mod tests;