Skip to main content

faucet_source_oracle_cdc/
state.rs

1//! The durable SCN bookmark.
2//!
3//! ```json
4//! { "commit_scn": 2145738, "restart_scn": 2145697, "committed_xids": ["07000600F3010000"] }
5//! ```
6//!
7//! - `commit_scn` — every transaction committed below it has been emitted.
8//! - `committed_xids` — transactions committed *at* `commit_scn` already
9//!   emitted (several can share one SCN).
10//! - `restart_scn` — where mining resumes: the first SCN of the oldest
11//!   transaction still open at the bookmark, so its changes are rebuilt from
12//!   the start rather than lost.
13//! - `resetlogs_scn` — the database incarnation (`RESETLOGS_CHANGE#`) the
14//!   bookmark was taken in; a resume in another incarnation fails, since the
15//!   SCNs after a RESETLOGS name different changes.
16
17use faucet_core::FaucetError;
18use serde_json::{Value, json};
19
20/// A resume position.
21#[derive(Debug, Clone, PartialEq, Eq)]
22pub struct Position {
23    /// Highest emitted commit SCN.
24    pub commit_scn: u64,
25    /// First SCN to mine on resume.
26    pub restart_scn: u64,
27    /// Transactions already emitted at exactly `commit_scn`.
28    pub committed_xids: Vec<String>,
29    /// `RESETLOGS_CHANGE#` of the incarnation the position belongs to; absent
30    /// in bookmarks written before it was recorded.
31    pub resetlogs_scn: Option<u64>,
32    /// `INCARNATION_SCN` of the pluggable database's current incarnation
33    /// (`V$PDB_INCARNATION`), when the source is connected to a PDB: a PDB
34    /// point-in-time recovery or flashback starts a new one without a
35    /// database-wide RESETLOGS.
36    pub pdb_incarnation_scn: Option<u64>,
37}
38
39impl Position {
40    /// A position at `scn` with nothing emitted there, resuming from
41    /// `restart` (which must not exceed `scn + 1`).
42    pub fn at(scn: u64, restart: u64) -> Self {
43        Self {
44            commit_scn: scn,
45            restart_scn: restart.min(scn.saturating_add(1)),
46            committed_xids: Vec::new(),
47            resetlogs_scn: None,
48            pdb_incarnation_scn: None,
49        }
50    }
51
52    /// Tie the position to the database incarnation whose `RESETLOGS_CHANGE#`
53    /// is `current`, refusing a position taken in another incarnation (after
54    /// a point-in-time recovery or flashback its later SCNs name changes that
55    /// no longer exist, and the new branch reuses them).
56    pub fn bind_incarnation(&mut self, current: u64) -> Result<(), FaucetError> {
57        match self.resetlogs_scn {
58            Some(r) if r != current => Err(FaucetError::Source(format!(
59                "oracle-cdc: the bookmark was taken in the database incarnation that began at \
60                 SCN {r}, but the database was opened with RESETLOGS at SCN {current}; its \
61                 changes after the branch point may no longer exist. Re-snapshot the tables \
62                 and reset the bookmark"
63            ))),
64            _ => {
65                if self.legacy_incarnation_unverified(current) {
66                    tracing::warn!(
67                        connector = "oracle-cdc",
68                        commit_scn = self.commit_scn,
69                        resetlogs_scn = current,
70                        "the bookmark predates incarnation tracking and lies after the current \
71                         RESETLOGS SCN, so a RESETLOGS before the upgrade cannot be detected; it is \
72                         bound to the current incarnation from now on"
73                    );
74                }
75                self.resetlogs_scn = Some(current);
76                Ok(())
77            }
78        }
79    }
80
81    /// Tie the position to the PDB incarnation that began at `current`
82    /// (`None` outside a PDB, or when it could not be read), refusing a
83    /// position taken in another one — a PDB point-in-time recovery or
84    /// flashback, which leaves the database's RESETLOGS SCN unchanged.
85    pub fn bind_pdb_incarnation(&mut self, current: Option<u64>) -> Result<(), FaucetError> {
86        let Some(current) = current else {
87            return Ok(());
88        };
89        match self.pdb_incarnation_scn {
90            Some(r) if r != current => Err(FaucetError::Source(format!(
91                "oracle-cdc: the bookmark was taken in the pluggable database incarnation that \
92                 began at SCN {r}, but the PDB's current incarnation began at SCN {current} (a \
93                 point-in-time recovery or flashback); its changes after the branch point may \
94                 no longer exist. Re-snapshot the tables and reset the bookmark"
95            ))),
96            Some(_) => Ok(()),
97            None => {
98                if self.commit_scn >= current {
99                    tracing::warn!(
100                        connector = "oracle-cdc",
101                        commit_scn = self.commit_scn,
102                        pdb_incarnation_scn = current,
103                        "the bookmark predates PDB incarnation tracking and lies after the PDB's \
104                         current incarnation SCN, so a PDB recovery before the upgrade cannot be \
105                         detected; it is bound to the current PDB incarnation from now on"
106                    );
107                }
108                self.pdb_incarnation_scn = Some(current);
109                Ok(())
110            }
111        }
112    }
113
114    /// True for a bookmark with no recorded incarnation whose position lies
115    /// at or after the current incarnation's branch point, where it may have
116    /// been taken in an earlier incarnation.
117    fn legacy_incarnation_unverified(&self, current: u64) -> bool {
118        self.resetlogs_scn.is_none() && self.commit_scn >= current
119    }
120
121    /// Where to anchor a fresh capture: at `current_scn`, reaching back to the
122    /// oldest open transaction's start so it is captured whole. A start of 0
123    /// is no position at all (#843) and anchors at `current_scn` like no open
124    /// transaction does.
125    pub fn resume_from(current_scn: u64, oldest_open_start: Option<u64>) -> Self {
126        let restart = oldest_open_start
127            .filter(|&s| s > 0)
128            .map_or(current_scn + 1, |s| s.min(current_scn + 1));
129        Self::at(current_scn, restart)
130    }
131
132    /// True when the transaction `xid` committing at `commit_scn` was already
133    /// emitted before this position.
134    pub fn already_emitted(&self, commit_scn: u64, xid: &str) -> bool {
135        commit_scn < self.commit_scn
136            || (commit_scn == self.commit_scn && self.committed_xids.iter().any(|x| x == xid))
137    }
138
139    /// Record that `xid` committed at `commit_scn` was emitted, with the
140    /// oldest still-open transaction starting at `oldest_open` (if any).
141    pub fn record_commit(&mut self, commit_scn: u64, xid: &str, oldest_open: Option<u64>) {
142        if commit_scn > self.commit_scn {
143            self.commit_scn = commit_scn;
144            self.committed_xids.clear();
145        }
146        if commit_scn == self.commit_scn && !self.committed_xids.iter().any(|x| x == xid) {
147            self.committed_xids.push(xid.to_string());
148        }
149        self.restart_scn = oldest_open.map_or(self.commit_scn, |s| s.min(self.commit_scn));
150    }
151
152    /// Advance past a fully-mined window ending at `end_scn` (inclusive).
153    pub fn advance_to(&mut self, end_scn: u64, oldest_open: Option<u64>) {
154        if end_scn > self.commit_scn {
155            self.commit_scn = end_scn;
156            self.committed_xids.clear();
157        }
158        let next = self.commit_scn.saturating_add(1);
159        self.restart_scn = oldest_open.map_or(next, |s| s.min(next));
160    }
161
162    /// Serialize for the state store.
163    pub fn to_value(&self) -> Value {
164        let mut v = json!({
165            "commit_scn": self.commit_scn,
166            "restart_scn": self.restart_scn,
167            "committed_xids": self.committed_xids,
168        });
169        if let Some(r) = self.resetlogs_scn {
170            v["resetlogs_scn"] = json!(r);
171        }
172        if let Some(r) = self.pdb_incarnation_scn {
173            v["pdb_incarnation_scn"] = json!(r);
174        }
175        v
176    }
177
178    /// Parse a stored bookmark.
179    pub fn from_value(v: &Value) -> Result<Self, FaucetError> {
180        let bad = |what: &str| FaucetError::State(format!("oracle-cdc bookmark: {what} (got {v})"));
181        let obj = v.as_object().ok_or_else(|| bad("expected an object"))?;
182        let scn = |k: &str| {
183            obj.get(k)
184                .and_then(Value::as_u64)
185                .ok_or_else(|| bad(&format!("`{k}` must be a non-negative integer")))
186        };
187        let xids = match obj.get("committed_xids") {
188            None | Some(Value::Null) => Vec::new(),
189            Some(Value::Array(a)) => a
190                .iter()
191                .map(|x| {
192                    x.as_str()
193                        .map(str::to_string)
194                        .ok_or_else(|| bad("xids must be strings"))
195                })
196                .collect::<Result<_, _>>()?,
197            Some(_) => return Err(bad("`committed_xids` must be an array")),
198        };
199        let resetlogs_scn = match obj.get("resetlogs_scn") {
200            None | Some(Value::Null) => None,
201            Some(_) => Some(scn("resetlogs_scn")?),
202        };
203        let pdb_incarnation_scn = match obj.get("pdb_incarnation_scn") {
204            None | Some(Value::Null) => None,
205            Some(_) => Some(scn("pdb_incarnation_scn")?),
206        };
207        Ok(Self {
208            commit_scn: scn("commit_scn")?,
209            restart_scn: scn("restart_scn")?,
210            committed_xids: xids,
211            resetlogs_scn,
212            pdb_incarnation_scn,
213        })
214    }
215}
216
217/// Order two stored positions: by commit SCN, and at one SCN by the set of
218/// transactions already emitted there. `None` when either is not a position.
219pub fn position_le(a: &Value, b: &Value) -> Option<bool> {
220    let (a, b) = (Position::from_value(a).ok()?, Position::from_value(b).ok()?);
221    Some(
222        a.commit_scn < b.commit_scn
223            || (a.commit_scn == b.commit_scn
224                && a.committed_xids
225                    .iter()
226                    .all(|x| b.committed_xids.contains(x))),
227    )
228}
229
230/// Whether a change record's transaction is already emitted at `position`:
231/// committed below its SCN, or one of the transactions emitted at exactly it.
232pub fn record_covered(record: &Value, position: &Value) -> Option<bool> {
233    let at = Position::from_value(position).ok()?;
234    let scn = record.get("commit_scn")?.as_u64()?;
235    let xid = record.get("xid")?.as_str()?;
236    Some(
237        scn < at.commit_scn || (scn == at.commit_scn && at.committed_xids.iter().any(|x| x == xid)),
238    )
239}
240
241#[cfg(test)]
242mod tests {
243    #[test]
244    fn incarnation_is_recorded_and_enforced() {
245        let mut p = super::Position::at(10, 5);
246        p.bind_incarnation(100).unwrap();
247        assert_eq!(p.resetlogs_scn, Some(100));
248        let v = p.to_value();
249        assert_eq!(v["resetlogs_scn"], 100);
250        let mut back = super::Position::from_value(&v).unwrap();
251        back.bind_incarnation(100).unwrap();
252        let err = back.bind_incarnation(200).unwrap_err();
253        assert!(err.to_string().contains("RESETLOGS at SCN 200"), "{err}");
254        let legacy = super::Position::from_value(&serde_json::json!({
255            "commit_scn": 1, "restart_scn": 1, "resetlogs_scn": null
256        }))
257        .unwrap();
258        assert_eq!(legacy.resetlogs_scn, None);
259        assert!(legacy.legacy_incarnation_unverified(1));
260        assert!(!legacy.legacy_incarnation_unverified(2));
261        assert!(!p.legacy_incarnation_unverified(1));
262        assert!(
263            super::Position::from_value(&serde_json::json!({
264                "commit_scn": 1, "restart_scn": 1, "resetlogs_scn": "x"
265            }))
266            .is_err()
267        );
268    }
269
270    use super::*;
271
272    #[test]
273    fn a_pdb_incarnation_is_recorded_and_enforced() {
274        let mut p = Position::at(10, 5);
275        p.bind_pdb_incarnation(None).unwrap();
276        assert_eq!(p.pdb_incarnation_scn, None);
277        assert!(p.to_value().get("pdb_incarnation_scn").is_none());
278        p.bind_pdb_incarnation(Some(7)).unwrap();
279        let v = p.to_value();
280        assert_eq!(v["pdb_incarnation_scn"], 7);
281        let mut back = Position::from_value(&v).unwrap();
282        back.bind_pdb_incarnation(Some(7)).unwrap();
283        back.bind_pdb_incarnation(None).unwrap();
284        let err = back.bind_pdb_incarnation(Some(42)).unwrap_err();
285        assert!(err.to_string().contains("began at SCN 42"), "{err}");
286        let mut legacy = Position::at(50, 50);
287        legacy.bind_pdb_incarnation(Some(40)).unwrap();
288        assert_eq!(legacy.pdb_incarnation_scn, Some(40));
289        assert!(
290            Position::from_value(&json!({
291                "commit_scn": 1, "restart_scn": 1, "pdb_incarnation_scn": "x"
292            }))
293            .is_err()
294        );
295    }
296
297    #[test]
298    fn a_record_is_covered_by_a_lower_scn_or_an_emitted_transaction_at_it() {
299        let at = json!({"commit_scn": 10, "restart_scn": 8, "committed_xids": ["A"]});
300        let rec = |scn: u64, xid: &str| json!({"commit_scn": scn, "xid": xid});
301        assert_eq!(record_covered(&rec(9, "Z"), &at), Some(true));
302        assert_eq!(record_covered(&rec(10, "A"), &at), Some(true));
303        assert_eq!(record_covered(&rec(10, "B"), &at), Some(false));
304        assert_eq!(record_covered(&rec(11, "A"), &at), Some(false));
305        assert_eq!(record_covered(&json!({"xid": "A"}), &at), None);
306        assert_eq!(record_covered(&rec(1, "A"), &json!({"nope": 1})), None);
307    }
308
309    #[test]
310    fn positions_order_by_scn_then_emitted_transactions() {
311        let p = |scn: u64, xids: &[&str]| json!({"commit_scn": scn, "restart_scn": scn, "committed_xids": xids});
312        assert_eq!(position_le(&p(10, &["a"]), &p(11, &[])), Some(true));
313        assert_eq!(position_le(&p(11, &[]), &p(10, &["a"])), Some(false));
314        assert_eq!(position_le(&p(10, &["a"]), &p(10, &["a", "b"])), Some(true));
315        assert_eq!(
316            position_le(&p(10, &["a", "b"]), &p(10, &["a"])),
317            Some(false)
318        );
319        assert_eq!(position_le(&p(10, &[]), &json!("x")), None);
320    }
321
322    #[test]
323    fn resume_reaches_back_to_open_transactions() {
324        assert_eq!(Position::resume_from(100, None).restart_scn, 101);
325        assert_eq!(Position::resume_from(100, Some(40)).restart_scn, 40);
326        assert_eq!(Position::resume_from(100, Some(400)).restart_scn, 101);
327        // #843: an open transaction reporting START_SCN 0 never anchors the
328        // capture at SCN 0.
329        let p = Position::resume_from(100, Some(0));
330        assert_eq!(p.restart_scn, 101);
331        assert_eq!(p.commit_scn, Position::resume_from(100, None).commit_scn);
332        assert_eq!(Position::at(5, 9).restart_scn, 6);
333    }
334
335    #[test]
336    fn skip_rules() {
337        let mut p = Position::at(100, 90);
338        assert!(p.already_emitted(99, "a"));
339        assert!(!p.already_emitted(100, "a"));
340        assert!(!p.already_emitted(101, "a"));
341        p.record_commit(100, "a", Some(95));
342        assert!(p.already_emitted(100, "a"));
343        assert!(!p.already_emitted(100, "b"));
344        assert_eq!(p.restart_scn, 95);
345        p.record_commit(100, "b", None);
346        p.record_commit(100, "b", None);
347        assert_eq!(p.committed_xids, vec!["a".to_string(), "b".to_string()]);
348        assert_eq!(p.restart_scn, 100);
349        p.record_commit(120, "c", Some(200));
350        assert_eq!(p.commit_scn, 120);
351        assert_eq!(p.committed_xids, vec!["c".to_string()]);
352        assert_eq!(p.restart_scn, 120);
353    }
354
355    #[test]
356    fn advancing_windows() {
357        let mut p = Position::at(100, 101);
358        p.record_commit(110, "x", None);
359        p.advance_to(110, None);
360        assert_eq!(
361            p.committed_xids,
362            vec!["x".to_string()],
363            "same-SCN xids survive"
364        );
365        assert_eq!(p.restart_scn, 111);
366        p.advance_to(150, Some(130));
367        assert_eq!(p.commit_scn, 150);
368        assert!(p.committed_xids.is_empty());
369        assert_eq!(p.restart_scn, 130);
370        p.advance_to(140, None);
371        assert_eq!(p.commit_scn, 150, "never moves backwards");
372    }
373
374    #[test]
375    fn value_round_trip_and_errors() {
376        let mut p = Position::at(7, 3);
377        p.committed_xids.push("X".into());
378        assert_eq!(Position::from_value(&p.to_value()).unwrap(), p);
379        let bare = json!({"commit_scn": 1, "restart_scn": 1});
380        assert!(
381            Position::from_value(&bare)
382                .unwrap()
383                .committed_xids
384                .is_empty()
385        );
386        for bad in [
387            json!(5),
388            json!({"restart_scn": 1}),
389            json!({"commit_scn": -1, "restart_scn": 1}),
390            json!({"commit_scn": 1, "restart_scn": 1, "committed_xids": "x"}),
391            json!({"commit_scn": 1, "restart_scn": 1, "committed_xids": [1]}),
392        ] {
393            assert!(Position::from_value(&bad).is_err(), "{bad}");
394        }
395    }
396}