faucet-source-oracle-cdc 1.2.2

Oracle Database CDC (LogMiner) source for the faucet-stream ecosystem
Documentation
//! The durable SCN bookmark.
//!
//! ```json
//! { "commit_scn": 2145738, "restart_scn": 2145697, "committed_xids": ["07000600F3010000"] }
//! ```
//!
//! - `commit_scn` — every transaction committed below it has been emitted.
//! - `committed_xids` — transactions committed *at* `commit_scn` already
//!   emitted (several can share one SCN).
//! - `restart_scn` — where mining resumes: the first SCN of the oldest
//!   transaction still open at the bookmark, so its changes are rebuilt from
//!   the start rather than lost.
//! - `resetlogs_scn` — the database incarnation (`RESETLOGS_CHANGE#`) the
//!   bookmark was taken in; a resume in another incarnation fails, since the
//!   SCNs after a RESETLOGS name different changes.

use faucet_core::FaucetError;
use serde_json::{Value, json};

/// A resume position.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Position {
    /// Highest emitted commit SCN.
    pub commit_scn: u64,
    /// First SCN to mine on resume.
    pub restart_scn: u64,
    /// Transactions already emitted at exactly `commit_scn`.
    pub committed_xids: Vec<String>,
    /// `RESETLOGS_CHANGE#` of the incarnation the position belongs to; absent
    /// in bookmarks written before it was recorded.
    pub resetlogs_scn: Option<u64>,
    /// `INCARNATION_SCN` of the pluggable database's current incarnation
    /// (`V$PDB_INCARNATION`), when the source is connected to a PDB: a PDB
    /// point-in-time recovery or flashback starts a new one without a
    /// database-wide RESETLOGS.
    pub pdb_incarnation_scn: Option<u64>,
}

impl Position {
    /// A position at `scn` with nothing emitted there, resuming from
    /// `restart` (which must not exceed `scn + 1`).
    pub fn at(scn: u64, restart: u64) -> Self {
        Self {
            commit_scn: scn,
            restart_scn: restart.min(scn.saturating_add(1)),
            committed_xids: Vec::new(),
            resetlogs_scn: None,
            pdb_incarnation_scn: None,
        }
    }

    /// Tie the position to the database incarnation whose `RESETLOGS_CHANGE#`
    /// is `current`, refusing a position taken in another incarnation (after
    /// a point-in-time recovery or flashback its later SCNs name changes that
    /// no longer exist, and the new branch reuses them).
    pub fn bind_incarnation(&mut self, current: u64) -> Result<(), FaucetError> {
        match self.resetlogs_scn {
            Some(r) if r != current => Err(FaucetError::Source(format!(
                "oracle-cdc: the bookmark was taken in the database incarnation that began at \
                 SCN {r}, but the database was opened with RESETLOGS at SCN {current}; its \
                 changes after the branch point may no longer exist. Re-snapshot the tables \
                 and reset the bookmark"
            ))),
            _ => {
                if self.legacy_incarnation_unverified(current) {
                    tracing::warn!(
                        connector = "oracle-cdc",
                        commit_scn = self.commit_scn,
                        resetlogs_scn = current,
                        "the bookmark predates incarnation tracking and lies after the current \
                         RESETLOGS SCN, so a RESETLOGS before the upgrade cannot be detected; it is \
                         bound to the current incarnation from now on"
                    );
                }
                self.resetlogs_scn = Some(current);
                Ok(())
            }
        }
    }

    /// Tie the position to the PDB incarnation that began at `current`
    /// (`None` outside a PDB, or when it could not be read), refusing a
    /// position taken in another one — a PDB point-in-time recovery or
    /// flashback, which leaves the database's RESETLOGS SCN unchanged.
    pub fn bind_pdb_incarnation(&mut self, current: Option<u64>) -> Result<(), FaucetError> {
        let Some(current) = current else {
            return Ok(());
        };
        match self.pdb_incarnation_scn {
            Some(r) if r != current => Err(FaucetError::Source(format!(
                "oracle-cdc: the bookmark was taken in the pluggable database incarnation that \
                 began at SCN {r}, but the PDB's current incarnation began at SCN {current} (a \
                 point-in-time recovery or flashback); its changes after the branch point may \
                 no longer exist. Re-snapshot the tables and reset the bookmark"
            ))),
            Some(_) => Ok(()),
            None => {
                if self.commit_scn >= current {
                    tracing::warn!(
                        connector = "oracle-cdc",
                        commit_scn = self.commit_scn,
                        pdb_incarnation_scn = current,
                        "the bookmark predates PDB incarnation tracking and lies after the PDB's \
                         current incarnation SCN, so a PDB recovery before the upgrade cannot be \
                         detected; it is bound to the current PDB incarnation from now on"
                    );
                }
                self.pdb_incarnation_scn = Some(current);
                Ok(())
            }
        }
    }

    /// True for a bookmark with no recorded incarnation whose position lies
    /// at or after the current incarnation's branch point, where it may have
    /// been taken in an earlier incarnation.
    fn legacy_incarnation_unverified(&self, current: u64) -> bool {
        self.resetlogs_scn.is_none() && self.commit_scn >= current
    }

    /// Where to anchor a fresh capture: at `current_scn`, reaching back to the
    /// oldest open transaction's start so it is captured whole. A start of 0
    /// is no position at all (#843) and anchors at `current_scn` like no open
    /// transaction does.
    pub fn resume_from(current_scn: u64, oldest_open_start: Option<u64>) -> Self {
        let restart = oldest_open_start
            .filter(|&s| s > 0)
            .map_or(current_scn + 1, |s| s.min(current_scn + 1));
        Self::at(current_scn, restart)
    }

    /// True when the transaction `xid` committing at `commit_scn` was already
    /// emitted before this position.
    pub fn already_emitted(&self, commit_scn: u64, xid: &str) -> bool {
        commit_scn < self.commit_scn
            || (commit_scn == self.commit_scn && self.committed_xids.iter().any(|x| x == xid))
    }

    /// Record that `xid` committed at `commit_scn` was emitted, with the
    /// oldest still-open transaction starting at `oldest_open` (if any).
    pub fn record_commit(&mut self, commit_scn: u64, xid: &str, oldest_open: Option<u64>) {
        if commit_scn > self.commit_scn {
            self.commit_scn = commit_scn;
            self.committed_xids.clear();
        }
        if commit_scn == self.commit_scn && !self.committed_xids.iter().any(|x| x == xid) {
            self.committed_xids.push(xid.to_string());
        }
        self.restart_scn = oldest_open.map_or(self.commit_scn, |s| s.min(self.commit_scn));
    }

    /// Advance past a fully-mined window ending at `end_scn` (inclusive).
    pub fn advance_to(&mut self, end_scn: u64, oldest_open: Option<u64>) {
        if end_scn > self.commit_scn {
            self.commit_scn = end_scn;
            self.committed_xids.clear();
        }
        let next = self.commit_scn.saturating_add(1);
        self.restart_scn = oldest_open.map_or(next, |s| s.min(next));
    }

    /// Serialize for the state store.
    pub fn to_value(&self) -> Value {
        let mut v = json!({
            "commit_scn": self.commit_scn,
            "restart_scn": self.restart_scn,
            "committed_xids": self.committed_xids,
        });
        if let Some(r) = self.resetlogs_scn {
            v["resetlogs_scn"] = json!(r);
        }
        if let Some(r) = self.pdb_incarnation_scn {
            v["pdb_incarnation_scn"] = json!(r);
        }
        v
    }

    /// Parse a stored bookmark.
    pub fn from_value(v: &Value) -> Result<Self, FaucetError> {
        let bad = |what: &str| FaucetError::State(format!("oracle-cdc bookmark: {what} (got {v})"));
        let obj = v.as_object().ok_or_else(|| bad("expected an object"))?;
        let scn = |k: &str| {
            obj.get(k)
                .and_then(Value::as_u64)
                .ok_or_else(|| bad(&format!("`{k}` must be a non-negative integer")))
        };
        let xids = match obj.get("committed_xids") {
            None | Some(Value::Null) => Vec::new(),
            Some(Value::Array(a)) => a
                .iter()
                .map(|x| {
                    x.as_str()
                        .map(str::to_string)
                        .ok_or_else(|| bad("xids must be strings"))
                })
                .collect::<Result<_, _>>()?,
            Some(_) => return Err(bad("`committed_xids` must be an array")),
        };
        let resetlogs_scn = match obj.get("resetlogs_scn") {
            None | Some(Value::Null) => None,
            Some(_) => Some(scn("resetlogs_scn")?),
        };
        let pdb_incarnation_scn = match obj.get("pdb_incarnation_scn") {
            None | Some(Value::Null) => None,
            Some(_) => Some(scn("pdb_incarnation_scn")?),
        };
        Ok(Self {
            commit_scn: scn("commit_scn")?,
            restart_scn: scn("restart_scn")?,
            committed_xids: xids,
            resetlogs_scn,
            pdb_incarnation_scn,
        })
    }
}

/// Order two stored positions: by commit SCN, and at one SCN by the set of
/// transactions already emitted there. `None` when either is not a position.
pub fn position_le(a: &Value, b: &Value) -> Option<bool> {
    let (a, b) = (Position::from_value(a).ok()?, Position::from_value(b).ok()?);
    Some(
        a.commit_scn < b.commit_scn
            || (a.commit_scn == b.commit_scn
                && a.committed_xids
                    .iter()
                    .all(|x| b.committed_xids.contains(x))),
    )
}

/// Whether a change record's transaction is already emitted at `position`:
/// committed below its SCN, or one of the transactions emitted at exactly it.
pub fn record_covered(record: &Value, position: &Value) -> Option<bool> {
    let at = Position::from_value(position).ok()?;
    let scn = record.get("commit_scn")?.as_u64()?;
    let xid = record.get("xid")?.as_str()?;
    Some(
        scn < at.commit_scn || (scn == at.commit_scn && at.committed_xids.iter().any(|x| x == xid)),
    )
}

#[cfg(test)]
mod tests {
    #[test]
    fn incarnation_is_recorded_and_enforced() {
        let mut p = super::Position::at(10, 5);
        p.bind_incarnation(100).unwrap();
        assert_eq!(p.resetlogs_scn, Some(100));
        let v = p.to_value();
        assert_eq!(v["resetlogs_scn"], 100);
        let mut back = super::Position::from_value(&v).unwrap();
        back.bind_incarnation(100).unwrap();
        let err = back.bind_incarnation(200).unwrap_err();
        assert!(err.to_string().contains("RESETLOGS at SCN 200"), "{err}");
        let legacy = super::Position::from_value(&serde_json::json!({
            "commit_scn": 1, "restart_scn": 1, "resetlogs_scn": null
        }))
        .unwrap();
        assert_eq!(legacy.resetlogs_scn, None);
        assert!(legacy.legacy_incarnation_unverified(1));
        assert!(!legacy.legacy_incarnation_unverified(2));
        assert!(!p.legacy_incarnation_unverified(1));
        assert!(
            super::Position::from_value(&serde_json::json!({
                "commit_scn": 1, "restart_scn": 1, "resetlogs_scn": "x"
            }))
            .is_err()
        );
    }

    use super::*;

    #[test]
    fn a_pdb_incarnation_is_recorded_and_enforced() {
        let mut p = Position::at(10, 5);
        p.bind_pdb_incarnation(None).unwrap();
        assert_eq!(p.pdb_incarnation_scn, None);
        assert!(p.to_value().get("pdb_incarnation_scn").is_none());
        p.bind_pdb_incarnation(Some(7)).unwrap();
        let v = p.to_value();
        assert_eq!(v["pdb_incarnation_scn"], 7);
        let mut back = Position::from_value(&v).unwrap();
        back.bind_pdb_incarnation(Some(7)).unwrap();
        back.bind_pdb_incarnation(None).unwrap();
        let err = back.bind_pdb_incarnation(Some(42)).unwrap_err();
        assert!(err.to_string().contains("began at SCN 42"), "{err}");
        let mut legacy = Position::at(50, 50);
        legacy.bind_pdb_incarnation(Some(40)).unwrap();
        assert_eq!(legacy.pdb_incarnation_scn, Some(40));
        assert!(
            Position::from_value(&json!({
                "commit_scn": 1, "restart_scn": 1, "pdb_incarnation_scn": "x"
            }))
            .is_err()
        );
    }

    #[test]
    fn a_record_is_covered_by_a_lower_scn_or_an_emitted_transaction_at_it() {
        let at = json!({"commit_scn": 10, "restart_scn": 8, "committed_xids": ["A"]});
        let rec = |scn: u64, xid: &str| json!({"commit_scn": scn, "xid": xid});
        assert_eq!(record_covered(&rec(9, "Z"), &at), Some(true));
        assert_eq!(record_covered(&rec(10, "A"), &at), Some(true));
        assert_eq!(record_covered(&rec(10, "B"), &at), Some(false));
        assert_eq!(record_covered(&rec(11, "A"), &at), Some(false));
        assert_eq!(record_covered(&json!({"xid": "A"}), &at), None);
        assert_eq!(record_covered(&rec(1, "A"), &json!({"nope": 1})), None);
    }

    #[test]
    fn positions_order_by_scn_then_emitted_transactions() {
        let p = |scn: u64, xids: &[&str]| json!({"commit_scn": scn, "restart_scn": scn, "committed_xids": xids});
        assert_eq!(position_le(&p(10, &["a"]), &p(11, &[])), Some(true));
        assert_eq!(position_le(&p(11, &[]), &p(10, &["a"])), Some(false));
        assert_eq!(position_le(&p(10, &["a"]), &p(10, &["a", "b"])), Some(true));
        assert_eq!(
            position_le(&p(10, &["a", "b"]), &p(10, &["a"])),
            Some(false)
        );
        assert_eq!(position_le(&p(10, &[]), &json!("x")), None);
    }

    #[test]
    fn resume_reaches_back_to_open_transactions() {
        assert_eq!(Position::resume_from(100, None).restart_scn, 101);
        assert_eq!(Position::resume_from(100, Some(40)).restart_scn, 40);
        assert_eq!(Position::resume_from(100, Some(400)).restart_scn, 101);
        // #843: an open transaction reporting START_SCN 0 never anchors the
        // capture at SCN 0.
        let p = Position::resume_from(100, Some(0));
        assert_eq!(p.restart_scn, 101);
        assert_eq!(p.commit_scn, Position::resume_from(100, None).commit_scn);
        assert_eq!(Position::at(5, 9).restart_scn, 6);
    }

    #[test]
    fn skip_rules() {
        let mut p = Position::at(100, 90);
        assert!(p.already_emitted(99, "a"));
        assert!(!p.already_emitted(100, "a"));
        assert!(!p.already_emitted(101, "a"));
        p.record_commit(100, "a", Some(95));
        assert!(p.already_emitted(100, "a"));
        assert!(!p.already_emitted(100, "b"));
        assert_eq!(p.restart_scn, 95);
        p.record_commit(100, "b", None);
        p.record_commit(100, "b", None);
        assert_eq!(p.committed_xids, vec!["a".to_string(), "b".to_string()]);
        assert_eq!(p.restart_scn, 100);
        p.record_commit(120, "c", Some(200));
        assert_eq!(p.commit_scn, 120);
        assert_eq!(p.committed_xids, vec!["c".to_string()]);
        assert_eq!(p.restart_scn, 120);
    }

    #[test]
    fn advancing_windows() {
        let mut p = Position::at(100, 101);
        p.record_commit(110, "x", None);
        p.advance_to(110, None);
        assert_eq!(
            p.committed_xids,
            vec!["x".to_string()],
            "same-SCN xids survive"
        );
        assert_eq!(p.restart_scn, 111);
        p.advance_to(150, Some(130));
        assert_eq!(p.commit_scn, 150);
        assert!(p.committed_xids.is_empty());
        assert_eq!(p.restart_scn, 130);
        p.advance_to(140, None);
        assert_eq!(p.commit_scn, 150, "never moves backwards");
    }

    #[test]
    fn value_round_trip_and_errors() {
        let mut p = Position::at(7, 3);
        p.committed_xids.push("X".into());
        assert_eq!(Position::from_value(&p.to_value()).unwrap(), p);
        let bare = json!({"commit_scn": 1, "restart_scn": 1});
        assert!(
            Position::from_value(&bare)
                .unwrap()
                .committed_xids
                .is_empty()
        );
        for bad in [
            json!(5),
            json!({"restart_scn": 1}),
            json!({"commit_scn": -1, "restart_scn": 1}),
            json!({"commit_scn": 1, "restart_scn": 1, "committed_xids": "x"}),
            json!({"commit_scn": 1, "restart_scn": 1, "committed_xids": [1]}),
        ] {
            assert!(Position::from_value(&bad).is_err(), "{bad}");
        }
    }
}