1use std::collections::HashMap;
20use std::pin::Pin;
21use std::sync::Mutex;
22use std::time::{Duration, Instant};
23
24use async_trait::async_trait;
25use faucet_core::check::{CheckContext, CheckReport, Probe};
26use faucet_core::{FaucetError, Source, Stream, StreamPage};
27use futures::TryStreamExt;
28use serde_json::Value;
29use tiberius::{QueryItem, ToSql};
30
31use faucet_common_mssql::{MssqlPool, MssqlPooledConnection, build_pool, with_statement_timeout};
32
33use crate::change::{
34 LSN_ALIAS, OP_COLUMN, OpAction, PollPlan, SEQVAL_ALIAS, build_change_envelope,
35 business_columns, op_action, plan_poll,
36};
37use crate::config::MssqlCdcSourceConfig;
38use crate::decode::row_to_json;
39use crate::lsn::Lsn;
40use crate::state::Bookmarks;
41
42pub struct MssqlCdcSource {
44 config: MssqlCdcSourceConfig,
45 pool: MssqlPool,
46 state_key_value: String,
47 tables: HashMap<String, (String, String)>,
50 pending_bookmark: Mutex<Option<Bookmarks>>,
53}
54
55impl MssqlCdcSource {
56 pub async fn new(config: MssqlCdcSourceConfig) -> Result<Self, FaucetError> {
60 config.validate()?;
61 let pool = build_pool(&config.connection, config.max_connections).await?;
62 let state_key_value = config.resolved_state_key();
63
64 let mut conn = pool
65 .get()
66 .await
67 .map_err(|e| FaucetError::Source(format!("mssql-cdc: pool checkout failed: {e}")))?;
68
69 let (db_name, cdc_enabled) = fetch_db_cdc_status(&mut conn).await?;
71 if !cdc_enabled {
72 return Err(FaucetError::Source(format!(
73 "mssql-cdc: change data capture is not enabled on database {db_name:?}; \
74 run `EXEC sys.sp_cdc_enable_db;` (requires sysadmin)"
75 )));
76 }
77
78 let tables = fetch_change_tables(&mut conn).await?;
80 let missing: Vec<&str> = config
81 .capture_instances
82 .iter()
83 .filter(|ci| !tables.contains_key(ci.as_str()))
84 .map(String::as_str)
85 .collect();
86 if !missing.is_empty() {
87 return Err(FaucetError::Source(format!(
88 "mssql-cdc: capture instance(s) {missing:?} not found in cdc.change_tables on \
89 database {db_name:?}; enable them with \
90 `EXEC sys.sp_cdc_enable_table @source_schema=..., @source_name=..., \
91 @role_name=NULL, @capture_instance=...;`"
92 )));
93 }
94 drop(conn);
95
96 Ok(Self {
97 config,
98 pool,
99 state_key_value,
100 tables,
101 pending_bookmark: Mutex::new(None),
102 })
103 }
104
105 fn timeout(&self) -> Option<Duration> {
106 match self.config.statement_timeout_secs {
107 0 => None,
108 secs => Some(Duration::from_secs(secs)),
109 }
110 }
111}
112
113#[async_trait]
114impl Source for MssqlCdcSource {
115 async fn fetch_with_context(
118 &self,
119 ctx: &HashMap<String, Value>,
120 ) -> Result<Vec<Value>, FaucetError> {
121 use futures::StreamExt;
122 let mut pages = self.stream_pages_impl(ctx, 0);
123 let mut all = Vec::new();
124 while let Some(page) = pages.next().await {
125 all.extend(page?.records);
126 }
127 Ok(all)
128 }
129
130 fn stream_pages<'a>(
134 &'a self,
135 ctx: &'a HashMap<String, Value>,
136 _batch_size: usize,
137 ) -> Pin<Box<dyn Stream<Item = Result<StreamPage, FaucetError>> + Send + 'a>> {
138 self.stream_pages_impl(ctx, self.config.batch_size)
139 }
140
141 fn config_schema(&self) -> Value {
142 serde_json::to_value(schemars::schema_for!(MssqlCdcSourceConfig)).unwrap_or(Value::Null)
143 }
144
145 fn state_key(&self) -> Option<String> {
146 Some(self.state_key_value.clone())
147 }
148
149 async fn apply_start_bookmark(&self, bookmark: Value) -> Result<(), FaucetError> {
150 let marks = Bookmarks::from_value(bookmark)?;
151 *self
152 .pending_bookmark
153 .lock()
154 .expect("pending_bookmark mutex poisoned") = Some(marks);
155 Ok(())
156 }
157
158 async fn capture_resume_position(&self) -> Result<Option<Value>, FaucetError> {
162 let mut conn = self.pool.get().await.map_err(|e| {
163 FaucetError::Source(format!("mssql-cdc: capture_position checkout failed: {e}"))
164 })?;
165 let max_lsn = match self.query_max_lsn(&mut conn).await? {
166 Some(lsn) => lsn,
167 None => return Ok(None),
170 };
171 let mut marks = Bookmarks::new();
172 for ci in &self.config.capture_instances {
173 marks.set(ci.clone(), max_lsn);
174 }
175 Ok(Some(marks.to_value()?))
176 }
177
178 fn supports_exactly_once(&self) -> bool {
179 true
180 }
181
182 fn connector_name(&self) -> &'static str {
183 "mssql-cdc"
184 }
185
186 fn dataset_uri(&self) -> String {
187 let conn = self
188 .config
189 .connection
190 .connection_url
191 .as_deref()
192 .or(self.config.connection.connection_string.as_deref())
193 .unwrap_or("");
194 format!(
195 "{}?capture_instances={}",
196 faucet_core::redact_uri_credentials(conn),
197 self.config.capture_instances.join(",")
198 )
199 }
200
201 async fn check(&self, ctx: &CheckContext) -> Result<CheckReport, FaucetError> {
204 let start = Instant::now();
205
206 let probe_result = tokio::time::timeout(ctx.timeout, async {
207 let mut conn = self.pool.get().await.map_err(|e| {
208 Probe::fail_hint(
209 "connection",
210 start.elapsed(),
211 format!("could not check out a connection: {e}"),
212 "verify connection_url / credentials / TLS and that the server is reachable",
213 )
214 })?;
215 let connection = Probe::pass("connection", start.elapsed());
216
217 let cdc = match fetch_db_cdc_status(&mut conn).await {
218 Ok((_db, true)) => Probe::pass("cdc-enabled", start.elapsed()),
219 Ok((db, false)) => Probe::fail_hint(
220 "cdc-enabled",
221 start.elapsed(),
222 format!("CDC is not enabled on database {db:?}"),
223 "run `EXEC sys.sp_cdc_enable_db;` (requires sysadmin)",
224 ),
225 Err(e) => Probe::fail_hint(
226 "cdc-enabled",
227 start.elapsed(),
228 e.to_string(),
229 "the CDC status query failed — check permissions on sys.databases",
230 ),
231 };
232
233 let instances = match fetch_change_tables(&mut conn).await {
234 Ok(tables) => {
235 let missing: Vec<&str> = self
236 .config
237 .capture_instances
238 .iter()
239 .filter(|ci| !tables.contains_key(ci.as_str()))
240 .map(String::as_str)
241 .collect();
242 if missing.is_empty() {
243 Probe::pass("capture-instances", start.elapsed())
244 } else {
245 Probe::fail_hint(
246 "capture-instances",
247 start.elapsed(),
248 format!("capture instance(s) not found: {missing:?}"),
249 "enable them with `EXEC sys.sp_cdc_enable_table ...`",
250 )
251 }
252 }
253 Err(e) => Probe::fail_hint(
254 "capture-instances",
255 start.elapsed(),
256 e.to_string(),
257 "reading cdc.change_tables failed — check CDC is enabled and permissions",
258 ),
259 };
260
261 Ok::<Vec<Probe>, Probe>(vec![connection, cdc, instances])
262 })
263 .await;
264
265 match probe_result {
266 Ok(Ok(probes)) => Ok(CheckReport { probes }),
267 Ok(Err(probe)) => Ok(CheckReport::single(probe)),
268 Err(_elapsed) => Ok(CheckReport::single(Probe::fail_hint(
269 "connection",
270 start.elapsed(),
271 "connection timed out",
272 "the database did not respond within the check timeout",
273 ))),
274 }
275 }
276}
277
278impl MssqlCdcSource {
283 async fn query_max_lsn(
286 &self,
287 conn: &mut MssqlPooledConnection<'_>,
288 ) -> Result<Option<Lsn>, FaucetError> {
289 const SQL: &str = "SELECT CONVERT(VARCHAR(20), sys.fn_cdc_get_max_lsn(), 2) AS max_lsn";
290 let rows = self.run_collect(conn, SQL, &[]).await?;
291 let Some(row) = rows.first() else {
292 return Ok(None);
293 };
294 opt_lsn(row, "max_lsn")
295 }
296
297 async fn query_lsn_bounds(
300 &self,
301 conn: &mut MssqlPooledConnection<'_>,
302 capture_instance: &str,
303 ) -> Result<(Option<Lsn>, Option<Lsn>), FaucetError> {
304 const SQL: &str = "SELECT CONVERT(VARCHAR(20), sys.fn_cdc_get_min_lsn(@P1), 2) AS min_lsn, \
305 CONVERT(VARCHAR(20), sys.fn_cdc_get_max_lsn(), 2) AS max_lsn";
306 let ci_owned = capture_instance.to_string();
309 let ci: &dyn ToSql = &ci_owned;
310 let rows = self.run_collect(conn, SQL, &[ci]).await?;
311 let Some(row) = rows.first() else {
312 return Ok((None, None));
313 };
314 Ok((opt_lsn(row, "min_lsn")?, opt_lsn(row, "max_lsn")?))
315 }
316
317 async fn run_collect(
320 &self,
321 conn: &mut MssqlPooledConnection<'_>,
322 sql: &str,
323 params: &[&dyn ToSql],
324 ) -> Result<Vec<tiberius::Row>, FaucetError> {
325 let run = async {
326 conn.query(sql, params)
327 .await
328 .map_err(|e| FaucetError::Source(format!("mssql-cdc: query failed: {e}")))?
329 .into_first_result()
330 .await
331 .map_err(|e| FaucetError::Source(format!("mssql-cdc: result read failed: {e}")))
332 };
333 match self.timeout() {
334 Some(t) => {
335 with_statement_timeout(t, run, || {
336 FaucetError::Source("mssql-cdc: query timed out".into())
337 })
338 .await
339 }
340 None => run.await,
341 }
342 }
343}
344
345async fn fetch_db_cdc_status(
347 conn: &mut MssqlPooledConnection<'_>,
348) -> Result<(String, bool), FaucetError> {
349 const SQL: &str = "SELECT DB_NAME() AS db, \
350 CONVERT(INT, is_cdc_enabled) AS enabled FROM sys.databases WHERE database_id = DB_ID()";
351 let rows = conn
352 .query(SQL, &[])
353 .await
354 .map_err(|e| FaucetError::Source(format!("mssql-cdc: CDC-status query failed: {e}")))?
355 .into_first_result()
356 .await
357 .map_err(|e| FaucetError::Source(format!("mssql-cdc: CDC-status read failed: {e}")))?;
358 let Some(row) = rows.first() else {
359 return Err(FaucetError::Source(
360 "mssql-cdc: could not resolve the current database (sys.databases returned no row)"
361 .into(),
362 ));
363 };
364 let db = row
365 .try_get::<&str, _>("db")
366 .map_err(|e| FaucetError::Source(format!("mssql-cdc: DB_NAME decode failed: {e}")))?
367 .unwrap_or("")
368 .to_string();
369 let enabled = row
370 .try_get::<i32, _>("enabled")
371 .map_err(|e| FaucetError::Source(format!("mssql-cdc: is_cdc_enabled decode failed: {e}")))?
372 .unwrap_or(0)
373 != 0;
374 Ok((db, enabled))
375}
376
377async fn fetch_change_tables(
380 conn: &mut MssqlPooledConnection<'_>,
381) -> Result<HashMap<String, (String, String)>, FaucetError> {
382 const SQL: &str = "SELECT ct.capture_instance AS ci, s.name AS src_schema, o.name AS src_table \
383 FROM cdc.change_tables ct \
384 JOIN sys.objects o ON o.object_id = ct.source_object_id \
385 JOIN sys.schemas s ON s.schema_id = o.schema_id";
386 let rows = conn
387 .query(SQL, &[])
388 .await
389 .map_err(|e| FaucetError::Source(format!("mssql-cdc: change_tables query failed: {e}")))?
390 .into_first_result()
391 .await
392 .map_err(|e| FaucetError::Source(format!("mssql-cdc: change_tables read failed: {e}")))?;
393
394 let mut map = HashMap::with_capacity(rows.len());
395 for row in &rows {
396 let get = |col: &str| -> Result<String, FaucetError> {
397 row.try_get::<&str, _>(col)
398 .map_err(|e| {
399 FaucetError::Source(format!("mssql-cdc: change_tables decode ({col}): {e}"))
400 })?
401 .map(str::to_string)
402 .ok_or_else(|| FaucetError::Source(format!("mssql-cdc: change_tables null {col}")))
403 };
404 map.insert(get("ci")?, (get("src_schema")?, get("src_table")?));
405 }
406 Ok(map)
407}
408
409fn opt_lsn(row: &tiberius::Row, col: &str) -> Result<Option<Lsn>, FaucetError> {
411 match row
412 .try_get::<&str, _>(col)
413 .map_err(|e| FaucetError::Source(format!("mssql-cdc: {col} decode failed: {e}")))?
414 {
415 Some(hex) => Ok(Some(Lsn::from_hex(hex)?)),
416 None => Ok(None),
417 }
418}
419
420impl MssqlCdcSource {
425 fn stream_pages_impl<'a>(
426 &'a self,
427 _ctx: &'a HashMap<String, Value>,
428 batch_size: usize,
429 ) -> Pin<Box<dyn Stream<Item = Result<StreamPage, FaucetError>> + Send + 'a>> {
430 let per_transaction = batch_size != 0;
431 let poll_interval = self.config.poll_interval;
432 let idle_timeout = self.config.idle_timeout;
433 let start_position = self.config.start_position;
434 let max_staged = self.config.max_staged_records;
435
436 Box::pin(async_stream::try_stream! {
437 let mut marks = self
439 .pending_bookmark
440 .lock()
441 .expect("pending_bookmark mutex poisoned")
442 .take()
443 .unwrap_or_default();
444
445 let mut conn = self
446 .pool
447 .get()
448 .await
449 .map_err(|e| FaucetError::Source(format!("mssql-cdc: pool checkout failed: {e}")))?;
450
451 let mut agg: Vec<Value> = Vec::new();
453 let mut agg_dirty = false;
454
455 let mut last_activity = Instant::now();
456
457 loop {
458 let mut any_rows = false;
459
460 for ci in &self.config.capture_instances {
461 let (schema, table) = self
462 .tables
463 .get(ci)
464 .cloned()
465 .unwrap_or_else(|| ("".to_string(), ci.clone()));
466
467 let (min_lsn, max_lsn) = self.query_lsn_bounds(&mut conn, ci).await?;
468 let plan = plan_poll(marks.get(ci), min_lsn, max_lsn, start_position);
469
470 match plan {
471 PollPlan::NoChanges { set_bookmark } => {
472 if let Some(anchor) = set_bookmark
475 && marks.get(ci).is_none()
476 {
477 marks.set(ci.clone(), anchor);
478 if per_transaction {
479 yield StreamPage {
480 records: Vec::new(),
481 bookmark: Some(marks.to_value()?),
482 };
483 } else {
484 agg_dirty = true;
485 }
486 }
487 }
488 PollPlan::Query { from, to, gap } => {
489 if gap {
490 tracing::warn!(
491 connector = "mssql-cdc",
492 capture_instance = %ci,
493 "resume point predates the retained minimum LSN; the CDC \
494 cleanup job purged changes before they were read — resuming \
495 from the earliest retained change (a data gap is possible)"
496 );
497 }
498
499 let sql = changes_sql(ci);
500 let from_hex = from.to_hex();
501 let to_hex = to.to_hex();
502
503 let mut stream = {
506 let params: [&dyn ToSql; 2] = [&from_hex, &to_hex];
511 let query_fut = conn.query(&sql, ¶ms);
512 match self.timeout() {
513 Some(t) => {
514 with_statement_timeout(t, async {
515 query_fut.await.map_err(|e| {
516 FaucetError::Source(format!(
517 "mssql-cdc: get_all_changes failed for {ci}: {e}"
518 ))
519 })
520 }, || FaucetError::Source(
521 "mssql-cdc: get_all_changes timed out".into()
522 ))
523 .await?
524 }
525 None => query_fut.await.map_err(|e| {
526 FaucetError::Source(format!(
527 "mssql-cdc: get_all_changes failed for {ci}: {e}"
528 ))
529 })?,
530 }
531 };
532
533 let mut buffer: Vec<Value> = Vec::new();
534 let mut cur_lsn: Option<Lsn> = None;
535
536 while let Some(item) = stream.try_next().await.map_err(|e| {
537 FaucetError::Source(format!(
538 "mssql-cdc: change row stream failed for {ci}: {e}"
539 ))
540 })? {
541 let QueryItem::Row(row) = item else { continue };
542 let decoded = row_to_json(&row)?;
543
544 let lsn_hex = decoded
545 .get(LSN_ALIAS)
546 .and_then(Value::as_str)
547 .ok_or_else(|| FaucetError::Source(
548 "mssql-cdc: change row missing __$start_lsn".into()
549 ))?;
550 let row_lsn = Lsn::from_hex(lsn_hex)?;
551 let seqval_hex = decoded
552 .get(SEQVAL_ALIAS)
553 .and_then(Value::as_str)
554 .map(str::to_string);
555 let op_code = decoded
556 .get(OP_COLUMN)
557 .and_then(Value::as_i64)
558 .ok_or_else(|| FaucetError::Source(
559 "mssql-cdc: change row missing __$operation".into()
560 ))?;
561
562 if let Some(prev) = cur_lsn
566 && prev != row_lsn
567 {
568 marks.set(ci.clone(), prev);
569 let recs = std::mem::take(&mut buffer);
570 if per_transaction {
571 yield StreamPage {
572 records: recs,
573 bookmark: Some(marks.to_value()?),
574 };
575 } else {
576 agg.extend(recs);
577 agg_dirty = true;
578 }
579 }
580 cur_lsn = Some(row_lsn);
581
582 match op_action(op_code)? {
583 OpAction::Skip => continue,
584 OpAction::Emit(op) => {
585 if let Some(max) = max_staged
586 && buffer.len() >= max
587 {
588 Err(FaucetError::Source(format!(
589 "mssql-cdc: in-progress transaction for {ci} exceeded \
590 max_staged_records ({max}); aborting to avoid \
591 unbounded memory growth. Raise max_staged_records \
592 or reduce the source transaction size."
593 )))?;
594 }
595 let cols = business_columns(&decoded);
596 let env = build_change_envelope(
597 op,
598 &schema,
599 &table,
600 lsn_hex,
601 seqval_hex.as_deref(),
602 cols,
603 );
604 buffer.push(env);
605 any_rows = true;
606 }
607 }
608 }
609 drop(stream);
612
613 marks.set(ci.clone(), to);
616 let recs = std::mem::take(&mut buffer);
617 if per_transaction {
618 yield StreamPage {
619 records: recs,
620 bookmark: Some(marks.to_value()?),
621 };
622 } else {
623 agg.extend(recs);
624 agg_dirty = true;
625 }
626 }
627 }
628 }
629
630 if any_rows {
631 last_activity = Instant::now();
632 }
633
634 if last_activity.elapsed() >= idle_timeout {
637 break;
638 }
639 tokio::time::sleep(poll_interval).await;
640 }
641
642 if !per_transaction && (agg_dirty || !agg.is_empty()) {
644 yield StreamPage {
645 records: std::mem::take(&mut agg),
646 bookmark: Some(marks.to_value()?),
647 };
648 }
649
650 tracing::info!(
651 connector = "mssql-cdc",
652 state_key = %self.state_key_value,
653 "mssql-cdc fetch cycle complete",
654 );
655 })
656 }
657}
658
659fn changes_sql(capture_instance: &str) -> String {
663 format!(
664 "SELECT CONVERT(VARCHAR(20), __$start_lsn, 2) AS {lsn}, \
665 CONVERT(VARCHAR(20), __$seqval, 2) AS {seq}, * \
666 FROM cdc.fn_cdc_get_all_changes_{ci}(\
667 CONVERT(BINARY(10), @P1, 2), CONVERT(BINARY(10), @P2, 2), N'all') \
668 ORDER BY __$start_lsn, __$seqval, __$operation",
669 lsn = LSN_ALIAS,
670 seq = SEQVAL_ALIAS,
671 ci = capture_instance,
672 )
673}
674
675#[cfg(test)]
676mod tests {
677 use super::*;
678
679 #[test]
680 fn changes_sql_embeds_validated_instance_and_aliases() {
681 let sql = changes_sql("dbo_Orders");
682 assert!(
683 sql.contains("cdc.fn_cdc_get_all_changes_dbo_Orders("),
684 "{sql}"
685 );
686 assert!(sql.contains("AS __faucet_lsn"), "{sql}");
687 assert!(sql.contains("AS __faucet_seqval"), "{sql}");
688 assert!(sql.contains("N'all'"), "{sql}");
689 assert!(
690 sql.contains("ORDER BY __$start_lsn, __$seqval, __$operation"),
691 "{sql}"
692 );
693 assert!(sql.contains("@P1") && sql.contains("@P2"), "{sql}");
695 }
696}