1use std::collections::HashMap;
5use std::hash::{Hash, Hasher};
6use std::pin::Pin;
7use std::sync::Mutex;
8use std::time::Duration;
9
10use async_trait::async_trait;
11use faucet_core::check::{CheckContext, CheckReport, Probe};
12use faucet_core::replication::{filter_incremental, max_replication_value, max_value};
13use faucet_core::shard::{
14 PkShardBounds, ShardSpec, parse_pk_shard, pk_bounds_query, pk_shards_from_bounds,
15};
16use faucet_core::{FaucetError, Source, StreamPage};
17use futures::{Stream, TryStreamExt};
18use serde_json::Value;
19use tiberius::{QueryItem, ToSql};
20
21use faucet_common_mssql::{MssqlPool, build_pool, quote_ident_mssql, with_statement_timeout};
22
23use crate::config::{MssqlReplication, MssqlSourceConfig};
24use crate::convert::row_to_json;
25
26pub struct MssqlSource {
28 config: MssqlSourceConfig,
29 pool: MssqlPool,
30 start_bookmark: Mutex<Option<Value>>,
33 applied_shard: Mutex<Option<PkShardBounds>>,
37}
38
39impl MssqlSource {
40 pub async fn new(config: MssqlSourceConfig) -> Result<Self, FaucetError> {
42 config.validate()?;
43 let pool = build_pool(&config.connection, config.max_connections).await?;
44 Ok(Self {
45 config,
46 pool,
47 start_bookmark: Mutex::new(None),
48 applied_shard: Mutex::new(None),
49 })
50 }
51
52 fn timeout(&self) -> Option<Duration> {
53 match self.config.statement_timeout_secs {
54 0 => None,
55 secs => Some(Duration::from_secs(secs)),
56 }
57 }
58
59 fn current_start(&self) -> Option<Value> {
60 self.start_bookmark
61 .lock()
62 .expect("start_bookmark mutex poisoned")
63 .clone()
64 }
65
66 fn shard_wrap(&self, query: String) -> String {
70 match &*self.applied_shard.lock().expect("shard mutex poisoned") {
71 Some(bounds) => bounds.wrap(&query, bracket_quote),
72 None => query,
73 }
74 }
75}
76
77fn bracket_quote(name: &str) -> String {
81 format!("[{}]", name.replace(']', "]]"))
82}
83
84#[derive(Debug, Clone, PartialEq)]
86struct IncrementalCtx {
87 column: String,
88 start: Value,
89}
90
91fn build_query_and_params(
98 config: &MssqlSourceConfig,
99 context: &HashMap<String, Value>,
100 start_bookmark: Option<&Value>,
101) -> (String, Vec<Value>, Option<IncrementalCtx>) {
102 let (mut query, mut values) = if context.is_empty() {
104 (config.query.clone(), config.params.clone())
105 } else {
106 let (q, ctx_values) = faucet_core::util::substitute_context_bind_params(
107 &config.query,
108 context,
109 config.params.len() + 1,
110 |i| format!("@P{i}"),
111 );
112 let mut v = config.params.clone();
113 v.extend(ctx_values);
114 (q, v)
115 };
116
117 let incremental = match &config.replication {
118 MssqlReplication::Full => None,
119 MssqlReplication::Incremental {
120 column,
121 initial_value,
122 } => {
123 let start = start_bookmark
124 .cloned()
125 .unwrap_or_else(|| initial_value.clone());
126 if query.contains("@bookmark") {
129 let idx = values.len() + 1;
130 query = query.replace("@bookmark", &format!("@P{idx}"));
131 values.push(start.clone());
132 }
133 Some(IncrementalCtx {
134 column: column.clone(),
135 start,
136 })
137 }
138 };
139
140 (query, values, incremental)
141}
142
143enum OwnedParam {
146 I64(i64),
147 F64(f64),
148 Bool(bool),
149 Str(String),
150 Null(Option<i32>),
151}
152
153impl OwnedParam {
154 fn from_value(v: &Value) -> Self {
155 match v {
156 Value::String(s) => OwnedParam::Str(s.clone()),
157 Value::Number(n) if n.is_i64() => OwnedParam::I64(n.as_i64().unwrap()),
158 Value::Number(n) if n.is_u64() => OwnedParam::Str(n.as_u64().unwrap().to_string()),
165 Value::Number(n) => OwnedParam::F64(n.as_f64().unwrap_or(0.0)),
166 Value::Bool(b) => OwnedParam::Bool(*b),
167 Value::Null => OwnedParam::Null(None),
168 other => OwnedParam::Str(other.to_string()),
169 }
170 }
171
172 fn as_tosql(&self) -> &dyn ToSql {
173 match self {
174 OwnedParam::I64(v) => v,
175 OwnedParam::F64(v) => v,
176 OwnedParam::Bool(v) => v,
177 OwnedParam::Str(v) => v,
178 OwnedParam::Null(v) => v,
179 }
180 }
181}
182
183fn default_state_key(config: &MssqlSourceConfig) -> String {
186 let host = config
187 .connection
188 .connection_url
189 .as_deref()
190 .and_then(|u| url::Url::parse(u).ok())
191 .and_then(|u| u.host_str().map(|h| h.to_string()))
192 .unwrap_or_else(|| "mssql".to_string());
193
194 let mut hasher = std::collections::hash_map::DefaultHasher::new();
195 config.query.hash(&mut hasher);
196 let fingerprint = hasher.finish();
197 let host: String = host
199 .chars()
200 .map(|c| {
201 if c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | '.') {
202 c
203 } else {
204 '_'
205 }
206 })
207 .collect();
208 format!("mssql:{host}:{fingerprint:016x}")
209}
210
211#[async_trait]
212impl Source for MssqlSource {
213 async fn fetch_with_context(
214 &self,
215 context: &HashMap<String, Value>,
216 ) -> Result<Vec<Value>, FaucetError> {
217 Ok(self.collect_all(context).await?.0)
218 }
219
220 async fn fetch_with_context_incremental(
221 &self,
222 context: &HashMap<String, Value>,
223 ) -> Result<(Vec<Value>, Option<Value>), FaucetError> {
224 self.collect_all(context).await
225 }
226
227 fn stream_pages<'a>(
228 &'a self,
229 context: &'a HashMap<String, Value>,
230 _batch_size: usize,
231 ) -> Pin<Box<dyn Stream<Item = Result<StreamPage, FaucetError>> + Send + 'a>> {
232 let batch_size = self.config.batch_size;
233 let chunk = if batch_size == 0 {
234 usize::MAX
235 } else {
236 batch_size
237 };
238 let cap = if batch_size == 0 { 1024 } else { batch_size };
239 let start = self.current_start();
240 let (query, values, incr) = build_query_and_params(&self.config, context, start.as_ref());
241 let query = self.shard_wrap(query);
242
243 Box::pin(async_stream::try_stream! {
244 let mut conn = self
245 .pool
246 .get()
247 .await
248 .map_err(|e| FaucetError::Source(format!("MSSQL pool checkout failed: {e}")))?;
249
250 let mut stream = {
253 let owned: Vec<OwnedParam> = values.iter().map(OwnedParam::from_value).collect();
254 let refs: Vec<&dyn ToSql> = owned.iter().map(OwnedParam::as_tosql).collect();
255 let query_fut = conn.query(&query, &refs);
256 match self.timeout() {
257 Some(t) => {
258 with_statement_timeout(t, async {
259 query_fut.await.map_err(|e| {
260 FaucetError::Source(format!("MSSQL query failed: {e}"))
261 })
262 }, || FaucetError::Source("MSSQL query timed out".into()))
263 .await?
264 }
265 None => query_fut
266 .await
267 .map_err(|e| FaucetError::Source(format!("MSSQL query failed: {e}")))?,
268 }
269 };
270
271 let mut buffer: Vec<Value> = Vec::with_capacity(cap);
272 let mut running_max: Option<Value> = None;
273 let mut total = 0usize;
274
275 while let Some(item) = stream
276 .try_next()
277 .await
278 .map_err(|e| FaucetError::Source(format!("MSSQL row stream failed: {e}")))?
279 {
280 let QueryItem::Row(row) = item else { continue };
281 buffer.push(row_to_json(&row)?);
282 if buffer.len() >= chunk {
283 let page = std::mem::replace(&mut buffer, Vec::with_capacity(cap));
284 let kept = apply_incremental(page, incr.as_ref(), &mut running_max);
285 total += kept.len();
286 if !kept.is_empty() {
287 yield StreamPage { records: kept, bookmark: None };
288 }
289 }
290 }
291
292 let kept = apply_incremental(buffer, incr.as_ref(), &mut running_max);
295 total += kept.len();
296 let bookmark = if incr.is_some() { running_max.clone() } else { None };
297 if !kept.is_empty() || bookmark.is_some() {
298 yield StreamPage { records: kept, bookmark };
299 }
300
301 tracing::info!(rows = total, query = %self.config.query, "MSSQL source stream complete");
302 })
303 }
304
305 fn config_schema(&self) -> Value {
306 serde_json::to_value(faucet_core::schema_for!(MssqlSourceConfig))
307 .expect("schema serialization")
308 }
309
310 fn connector_name(&self) -> &'static str {
311 "mssql"
312 }
313
314 fn dataset_uri(&self) -> String {
315 let conn = self
316 .config
317 .connection
318 .connection_url
319 .as_deref()
320 .or(self.config.connection.connection_string.as_deref())
321 .unwrap_or("");
322 format!(
323 "{}?query={}",
324 faucet_core::redact_uri_credentials(conn),
325 self.config.query
326 )
327 }
328
329 fn state_key(&self) -> Option<String> {
330 match &self.config.replication {
331 MssqlReplication::Full => None,
332 MssqlReplication::Incremental { .. } => Some(
333 self.config
334 .state_key
335 .clone()
336 .unwrap_or_else(|| default_state_key(&self.config)),
337 ),
338 }
339 }
340
341 async fn apply_start_bookmark(&self, bookmark: Value) -> Result<(), FaucetError> {
342 *self
343 .start_bookmark
344 .lock()
345 .expect("start_bookmark mutex poisoned") = Some(bookmark);
346 Ok(())
347 }
348
349 async fn check(&self, ctx: &CheckContext) -> Result<CheckReport, FaucetError> {
350 let started = std::time::Instant::now();
351 let probe = match tokio::time::timeout(ctx.timeout, self.pool.get()).await {
352 Ok(Ok(_conn)) => Probe::pass("connect", started.elapsed()),
353 Ok(Err(e)) => Probe::fail_hint(
354 "connect",
355 started.elapsed(),
356 e.to_string(),
357 "check connection_url / credentials / TLS / that the server is reachable",
358 ),
359 Err(_) => Probe::fail_hint(
360 "connect",
361 started.elapsed(),
362 "timed out",
363 "check connection_url / credentials / TLS / that the server is reachable",
364 ),
365 };
366 Ok(CheckReport::single(probe))
367 }
368
369 fn is_shardable(&self) -> bool {
371 self.config.shard.is_some()
372 }
373
374 async fn enumerate_shards(&self, target: usize) -> Result<Vec<ShardSpec>, FaucetError> {
384 let Some(shard_cfg) = &self.config.shard else {
385 return Ok(vec![ShardSpec::whole()]);
386 };
387
388 let start = self.current_start();
389 let (inner, values, _incr) =
390 build_query_and_params(&self.config, &HashMap::new(), start.as_ref());
391 let key = quote_ident_mssql(&shard_cfg.key)?;
392 let bounds_sql = pk_bounds_query(&inner, &key, "BIGINT");
393
394 let mut conn = self
395 .pool
396 .get()
397 .await
398 .map_err(|e| FaucetError::Source(format!("MSSQL pool checkout failed: {e}")))?;
399
400 let rows = {
401 let owned: Vec<OwnedParam> = values.iter().map(OwnedParam::from_value).collect();
402 let refs: Vec<&dyn ToSql> = owned.iter().map(OwnedParam::as_tosql).collect();
403 let run = async {
404 conn.query(&bounds_sql, &refs)
405 .await
406 .map_err(|e| {
407 FaucetError::Source(format!(
408 "mssql: failed to compute shard bounds for key {:?} \
409 (it must be an integer-typed column, and the query must \
410 not end in a top-level ORDER BY): {e}",
411 shard_cfg.key
412 ))
413 })?
414 .into_first_result()
415 .await
416 .map_err(|e| {
417 FaucetError::Source(format!(
418 "mssql: failed to compute shard bounds for key {:?} \
419 (it must be an integer-typed column, and the query must \
420 not end in a top-level ORDER BY): {e}",
421 shard_cfg.key
422 ))
423 })
424 };
425 match self.timeout() {
426 Some(t) => {
427 with_statement_timeout(t, run, || {
428 FaucetError::Source("MSSQL shard-bounds query timed out".into())
429 })
430 .await?
431 }
432 None => run.await?,
433 }
434 };
435
436 let Some(row) = rows.first() else {
437 return Ok(vec![ShardSpec::whole()]);
438 };
439 let decoded = row_to_json(row)?;
440 Ok(pk_shards_from_bounds(
441 &shard_cfg.key,
442 decoded["lo"].as_i64(),
443 decoded["hi"].as_i64(),
444 target,
445 ))
446 }
447
448 async fn apply_shard(&self, shard: &ShardSpec) -> Result<(), FaucetError> {
451 let bounds = parse_pk_shard(shard, "mssql")?;
452 if let Some(b) = &bounds {
453 quote_ident_mssql(&b.key)?;
456 }
457 *self.applied_shard.lock().expect("shard mutex poisoned") = bounds;
458 Ok(())
459 }
460}
461
462impl MssqlSource {
463 async fn collect_all(
466 &self,
467 context: &HashMap<String, Value>,
468 ) -> Result<(Vec<Value>, Option<Value>), FaucetError> {
469 let start = self.current_start();
470 let (query, values, incr) = build_query_and_params(&self.config, context, start.as_ref());
471 let query = self.shard_wrap(query);
472
473 let mut conn = self
474 .pool
475 .get()
476 .await
477 .map_err(|e| FaucetError::Source(format!("MSSQL pool checkout failed: {e}")))?;
478
479 let rows = {
480 let owned: Vec<OwnedParam> = values.iter().map(OwnedParam::from_value).collect();
481 let refs: Vec<&dyn ToSql> = owned.iter().map(OwnedParam::as_tosql).collect();
482 let run = async {
483 conn.query(&query, &refs)
484 .await
485 .map_err(|e| FaucetError::Source(format!("MSSQL query failed: {e}")))?
486 .into_first_result()
487 .await
488 .map_err(|e| FaucetError::Source(format!("MSSQL result read failed: {e}")))
489 };
490 match self.timeout() {
491 Some(t) => {
492 with_statement_timeout(t, run, || {
493 FaucetError::Source("MSSQL query timed out".into())
494 })
495 .await?
496 }
497 None => run.await?,
498 }
499 };
500
501 let mut records = Vec::with_capacity(rows.len());
502 for row in &rows {
503 records.push(row_to_json(row)?);
504 }
505
506 let mut running_max: Option<Value> = None;
507 let records = apply_incremental(records, incr.as_ref(), &mut running_max);
508 let bookmark = if incr.is_some() { running_max } else { None };
509 Ok((records, bookmark))
510 }
511}
512
513fn apply_incremental(
516 page: Vec<Value>,
517 incr: Option<&IncrementalCtx>,
518 running_max: &mut Option<Value>,
519) -> Vec<Value> {
520 match incr {
521 None => page,
522 Some(ctx) => {
523 let kept = filter_incremental(page, &ctx.column, &ctx.start);
524 if let Some(m) = max_replication_value(&kept, &ctx.column) {
525 let m = m.clone();
526 *running_max = Some(match running_max.take() {
527 Some(prev) => max_value(prev, m),
528 None => m,
529 });
530 }
531 kept
532 }
533 }
534}
535
536#[cfg(test)]
537mod tests {
538 use super::*;
539 use faucet_core::shard::plan_pk_shards;
540 use serde_json::json;
541
542 fn full_cfg() -> MssqlSourceConfig {
543 MssqlSourceConfig::new("mssql://sa:pw@db.example.com:1433/sales", "SELECT * FROM t")
544 }
545
546 #[test]
547 fn owned_param_binds_large_u64_as_exact_string_not_wrapped_i64() {
548 let big = u64::MAX; match OwnedParam::from_value(&json!(big)) {
552 OwnedParam::Str(s) => assert_eq!(s, big.to_string()),
553 OwnedParam::I64(v) => panic!("u64::MAX bound as wrapped i64 {v}"),
554 _ => panic!("expected a string-bound param for a large u64"),
555 }
556 match OwnedParam::from_value(&json!(42u64)) {
558 OwnedParam::I64(v) => assert_eq!(v, 42),
559 _ => panic!("small u64 should bind as i64"),
560 }
561 match OwnedParam::from_value(&json!(i64::MAX)) {
563 OwnedParam::I64(v) => assert_eq!(v, i64::MAX),
564 _ => panic!("i64::MAX should bind as i64"),
565 }
566 }
567
568 #[test]
569 fn build_full_returns_query_and_params_unchanged() {
570 let mut cfg = full_cfg();
571 cfg.params = vec![json!(1), json!("x")];
572 let (q, v, incr) = build_query_and_params(&cfg, &HashMap::new(), None);
573 assert_eq!(q, "SELECT * FROM t");
574 assert_eq!(v, vec![json!(1), json!("x")]);
575 assert!(incr.is_none());
576 }
577
578 #[test]
579 fn build_incremental_binds_bookmark_token() {
580 let cfg = MssqlSourceConfig {
581 query: "SELECT * FROM t WHERE updated_at > @bookmark".into(),
582 replication: MssqlReplication::Incremental {
583 column: "updated_at".into(),
584 initial_value: json!("1970-01-01"),
585 },
586 ..full_cfg()
587 };
588 let (q, v, incr) = build_query_and_params(&cfg, &HashMap::new(), None);
589 assert_eq!(q, "SELECT * FROM t WHERE updated_at > @P1");
590 assert_eq!(v, vec![json!("1970-01-01")]);
591 assert_eq!(
592 incr,
593 Some(IncrementalCtx {
594 column: "updated_at".into(),
595 start: json!("1970-01-01")
596 })
597 );
598 }
599
600 #[test]
601 fn build_incremental_uses_stored_bookmark_over_initial() {
602 let cfg = MssqlSourceConfig {
603 query: "SELECT * FROM t WHERE c > @bookmark".into(),
604 params: vec![json!("p0")],
605 replication: MssqlReplication::Incremental {
606 column: "c".into(),
607 initial_value: json!(0),
608 },
609 ..full_cfg()
610 };
611 let stored = json!(500);
612 let (q, v, incr) = build_query_and_params(&cfg, &HashMap::new(), Some(&stored));
613 assert_eq!(q, "SELECT * FROM t WHERE c > @P2");
615 assert_eq!(v, vec![json!("p0"), json!(500)]);
616 assert_eq!(incr.unwrap().start, json!(500));
617 }
618
619 #[test]
620 fn build_incremental_without_token_still_returns_filter_ctx() {
621 let cfg = MssqlSourceConfig {
622 query: "SELECT * FROM t".into(),
623 replication: MssqlReplication::Incremental {
624 column: "c".into(),
625 initial_value: json!(0),
626 },
627 ..full_cfg()
628 };
629 let (q, v, incr) = build_query_and_params(&cfg, &HashMap::new(), None);
630 assert_eq!(q, "SELECT * FROM t");
631 assert!(v.is_empty());
632 assert!(incr.is_some(), "client-side filter must still run");
633 }
634
635 #[test]
636 fn owned_param_classifies_json() {
637 assert!(matches!(
638 OwnedParam::from_value(&json!("s")),
639 OwnedParam::Str(_)
640 ));
641 assert!(matches!(
642 OwnedParam::from_value(&json!(7)),
643 OwnedParam::I64(7)
644 ));
645 assert!(matches!(
646 OwnedParam::from_value(&json!(1.5)),
647 OwnedParam::F64(_)
648 ));
649 assert!(matches!(
650 OwnedParam::from_value(&json!(true)),
651 OwnedParam::Bool(true)
652 ));
653 assert!(matches!(
654 OwnedParam::from_value(&Value::Null),
655 OwnedParam::Null(None)
656 ));
657 assert!(matches!(
658 OwnedParam::from_value(&json!({"a":1})),
659 OwnedParam::Str(_)
660 ));
661 }
662
663 #[test]
664 fn apply_incremental_filters_and_tracks_max() {
665 let ctx = IncrementalCtx {
666 column: "c".into(),
667 start: json!(10),
668 };
669 let mut running = None;
670 let page = vec![json!({"c": 5}), json!({"c": 15}), json!({"c": 20})];
671 let kept = apply_incremental(page, Some(&ctx), &mut running);
672 assert_eq!(kept.len(), 2);
673 assert_eq!(running, Some(json!(20)));
674 }
675
676 #[test]
677 fn apply_incremental_full_passes_through() {
678 let mut running = None;
679 let page = vec![json!({"c": 1}), json!({"c": 2})];
680 let kept = apply_incremental(page, None, &mut running);
681 assert_eq!(kept.len(), 2);
682 assert_eq!(running, None);
683 }
684
685 #[test]
686 fn default_state_key_is_stable_and_valid() {
687 let cfg = full_cfg();
688 let k1 = default_state_key(&cfg);
689 let k2 = default_state_key(&cfg);
690 assert_eq!(k1, k2);
691 assert!(k1.starts_with("mssql:db.example.com:"));
692 faucet_core::state::validate_state_key(&k1).expect("derived key must be valid");
693 }
694
695 #[test]
698 fn dataset_uri_redacts_connection_url() {
699 let cfg = full_cfg();
700 let conn = cfg
701 .connection
702 .connection_url
703 .as_deref()
704 .or(cfg.connection.connection_string.as_deref())
705 .unwrap_or("");
706 let uri = format!(
707 "{}?query={}",
708 faucet_core::redact_uri_credentials(conn),
709 cfg.query
710 );
711 assert_eq!(
712 uri,
713 "mssql://db.example.com:1433/sales?query=SELECT * FROM t"
714 );
715 }
716
717 #[test]
720 fn shard_wrap_uses_bracket_quoting() {
721 let spec = faucet_core::ShardSpec::new(
722 "1",
723 json!({"key": "id", "lo": 100, "hi": 200, "lo_unbounded": false, "hi_unbounded": false}),
724 );
725 let bounds = PkShardBounds::from_spec(&spec).unwrap();
726 let sql = bounds.wrap("SELECT * FROM t", bracket_quote);
727 assert!(sql.contains("(SELECT * FROM t) AS _faucet_shard"), "{sql}");
728 assert!(sql.contains("[id] >= 100"), "bracket-quoted key: {sql}");
729 assert!(sql.contains("[id] < 200"), "half-open upper bound: {sql}");
730 }
731
732 #[test]
733 fn last_shard_wrap_covers_null_keys() {
734 let shards = plan_pk_shards("id", 0, 99, 3);
735 let last = PkShardBounds::from_spec(shards.last().unwrap()).unwrap();
736 let sql = last.wrap("SELECT * FROM t", bracket_quote);
737 assert!(
738 sql.contains("[id] IS NULL"),
739 "last shard must match NULL keys: {sql}"
740 );
741 }
742
743 #[test]
744 fn shard_wrap_preserves_positional_bind_markers() {
745 let cfg = MssqlSourceConfig {
748 query: "SELECT * FROM t WHERE c > @bookmark".into(),
749 replication: MssqlReplication::Incremental {
750 column: "c".into(),
751 initial_value: json!(0),
752 },
753 ..full_cfg()
754 };
755 let (q, v, _incr) = build_query_and_params(&cfg, &HashMap::new(), None);
756 let spec = faucet_core::ShardSpec::new(
757 "0",
758 json!({"key": "id", "lo": 0, "hi": 10, "lo_unbounded": false, "hi_unbounded": false}),
759 );
760 let wrapped = PkShardBounds::from_spec(&spec)
761 .unwrap()
762 .wrap(&q, bracket_quote);
763 assert!(wrapped.contains("@P1"), "bind marker survives: {wrapped}");
764 assert_eq!(v.len(), 1, "one bound value for the bookmark");
765 }
766}