1use arrow_tiberius::{SqlExecutionOutcome, TableName, create_table_sql_from_mappings};
8use async_trait::async_trait;
9
10use crate::DeltaFunnelError;
11
12use super::{
13 LoadMode, MssqlConnectionSource, MssqlConnectionSummary, MssqlLifecycleExecutionGuardrail,
14 MssqlTargetCleanupStatus, MssqlTargetOutputPlan, MssqlTargetTable, MssqlTargetTableState,
15 MssqlWriteFailureContext, MssqlWritePhase,
16};
17
18#[allow(dead_code)]
20#[async_trait]
21pub(crate) trait MssqlTargetLifecycleClient: Send {
22 async fn table_exists(&mut self, table: &TableName) -> arrow_tiberius::Result<bool>;
24
25 async fn execute_statement(&mut self, sql: &str)
27 -> arrow_tiberius::Result<SqlExecutionOutcome>;
28}
29
30#[async_trait]
31impl MssqlTargetLifecycleClient for arrow_tiberius::ConnectedMssqlClient {
32 async fn table_exists(&mut self, table: &TableName) -> arrow_tiberius::Result<bool> {
33 arrow_tiberius::ConnectedMssqlClient::table_exists(self, table).await
34 }
35
36 async fn execute_statement(
37 &mut self,
38 sql: &str,
39 ) -> arrow_tiberius::Result<SqlExecutionOutcome> {
40 arrow_tiberius::ConnectedMssqlClient::execute_statement(self, sql).await
41 }
42}
43
44#[allow(dead_code)]
46pub(crate) struct MssqlConnectedLifecycleClient<'client> {
47 output_plan: &'client MssqlTargetOutputPlan,
48 client: &'client mut arrow_tiberius::ConnectedMssqlClient,
49}
50
51impl<'client> MssqlConnectedLifecycleClient<'client> {
52 #[must_use]
54 pub(crate) fn new(
55 output_plan: &'client MssqlTargetOutputPlan,
56 client: &'client mut arrow_tiberius::ConnectedMssqlClient,
57 ) -> Self {
58 Self {
59 output_plan,
60 client,
61 }
62 }
63
64 #[allow(dead_code)]
66 #[must_use]
67 pub(crate) const fn output_plan(&self) -> &MssqlTargetOutputPlan {
68 self.output_plan
69 }
70}
71
72#[async_trait]
73impl MssqlTargetLifecycleClient for MssqlConnectedLifecycleClient<'_> {
74 async fn table_exists(&mut self, table: &TableName) -> arrow_tiberius::Result<bool> {
75 MssqlTargetLifecycleClient::table_exists(self.client, table).await
76 }
77
78 async fn execute_statement(
79 &mut self,
80 sql: &str,
81 ) -> arrow_tiberius::Result<SqlExecutionOutcome> {
82 MssqlTargetLifecycleClient::execute_statement(self.client, sql).await
83 }
84}
85
86#[allow(dead_code)]
88pub(crate) async fn prepare_mssql_target_lifecycle<C>(
89 output_plan: &MssqlTargetOutputPlan,
90 client: &mut C,
91) -> Result<MssqlPreparedTarget, DeltaFunnelError>
92where
93 C: MssqlTargetLifecycleClient + ?Sized,
94{
95 match output_plan.load_mode() {
96 LoadMode::AppendExisting => prepare_mssql_append_existing_target(output_plan, client).await,
97 LoadMode::CreateAndLoad => prepare_mssql_create_and_load_target(output_plan, client).await,
98 LoadMode::Replace => prepare_mssql_replace_target(output_plan, client).await,
99 }
100}
101
102pub(crate) async fn prepare_mssql_append_existing_target<C>(
104 output_plan: &MssqlTargetOutputPlan,
105 client: &mut C,
106) -> Result<MssqlPreparedTarget, DeltaFunnelError>
107where
108 C: MssqlTargetLifecycleClient + ?Sized,
109{
110 ensure_append_existing_output(output_plan)?;
111 ensure_lifecycle_guardrail(
112 output_plan,
113 MssqlLifecycleExecutionGuardrail::TargetTableExistenceProbe,
114 )?;
115 let table_name = table_name_from_target(output_plan.output_name(), output_plan.target_table())?;
116 let exists = client
117 .table_exists(&table_name)
118 .await
119 .map_err(|source| prepare_target_lifecycle_error(output_plan, source.to_string()))?;
120
121 if !exists {
122 return Err(prepare_target_lifecycle_error(
123 output_plan,
124 format!(
125 "append-existing target table {} does not exist",
126 table_name.quoted_sql()
127 ),
128 ));
129 }
130
131 MssqlPreparedTarget::from_output_plan(output_plan, MssqlPreparedTargetAction::VerifiedExisting)
132}
133
134pub(crate) async fn prepare_mssql_create_and_load_target<C>(
136 output_plan: &MssqlTargetOutputPlan,
137 client: &mut C,
138) -> Result<MssqlPreparedTarget, DeltaFunnelError>
139where
140 C: MssqlTargetLifecycleClient + ?Sized,
141{
142 ensure_create_and_load_output(output_plan)?;
143 ensure_lifecycle_guardrail(
144 output_plan,
145 MssqlLifecycleExecutionGuardrail::TargetTableExistenceProbe,
146 )?;
147 ensure_lifecycle_guardrail(
148 output_plan,
149 MssqlLifecycleExecutionGuardrail::CreateTableDdlExecution,
150 )?;
151 let table_name = table_name_from_target(output_plan.output_name(), output_plan.target_table())?;
152 let exists = client
153 .table_exists(&table_name)
154 .await
155 .map_err(|source| prepare_target_lifecycle_error(output_plan, source.to_string()))?;
156
157 if exists {
158 return Err(prepare_target_lifecycle_error(
159 output_plan,
160 format!(
161 "create-and-load target table {} already exists",
162 table_name.quoted_sql()
163 ),
164 ));
165 }
166
167 let create_table_sql = output_plan.create_table_sql().ok_or_else(|| {
168 prepare_target_lifecycle_error(
169 output_plan,
170 "create-and-load target preparation requires planned create-table SQL",
171 )
172 })?;
173 client
174 .execute_statement(create_table_sql)
175 .await
176 .map_err(|source| prepare_target_lifecycle_error(output_plan, source.to_string()))?;
177
178 MssqlPreparedTarget::from_output_plan(output_plan, MssqlPreparedTargetAction::CreatedTable)
179}
180
181pub(crate) async fn prepare_mssql_replace_target<C>(
183 output_plan: &MssqlTargetOutputPlan,
184 client: &mut C,
185) -> Result<MssqlPreparedTarget, DeltaFunnelError>
186where
187 C: MssqlTargetLifecycleClient + ?Sized,
188{
189 ensure_replace_output(output_plan)?;
190 ensure_lifecycle_guardrail(
191 output_plan,
192 MssqlLifecycleExecutionGuardrail::TargetTableExistenceProbe,
193 )?;
194 ensure_lifecycle_guardrail(
195 output_plan,
196 MssqlLifecycleExecutionGuardrail::CreateTableDdlExecution,
197 )?;
198 let final_table_name =
199 table_name_from_target(output_plan.output_name(), output_plan.target_table())?;
200 client
201 .table_exists(&final_table_name)
202 .await
203 .map_err(|source| prepare_target_lifecycle_error(output_plan, source.to_string()))?;
204
205 let staging_table_name = available_replace_staging_table(output_plan, client).await?;
206 let create_table_sql =
207 create_table_sql_from_mappings(&staging_table_name, output_plan.schema_mappings());
208 client
209 .execute_statement(&create_table_sql)
210 .await
211 .map_err(|source| prepare_target_lifecycle_error(output_plan, source.to_string()))?;
212
213 MssqlPreparedTarget::from_table_name(
214 output_plan,
215 staging_table_name,
216 MssqlPreparedTargetAction::CreatedStagingTable,
217 )
218}
219
220#[allow(dead_code)]
222pub(crate) async fn cleanup_mssql_prepared_target<C>(
223 output_plan: &MssqlTargetOutputPlan,
224 prepared_target: Option<&MssqlPreparedTarget>,
225 client: &mut C,
226) -> Result<MssqlTargetCleanupStatus, DeltaFunnelError>
227where
228 C: MssqlTargetLifecycleClient + ?Sized,
229{
230 let Some(prepared_target) = prepared_target else {
231 return Ok(cleanup_before_target_creation(output_plan));
232 };
233 ensure_prepared_target_matches_output_plan(output_plan, prepared_target)?;
234
235 match prepared_target.report().action() {
236 MssqlPreparedTargetAction::VerifiedExisting => Ok(MssqlTargetCleanupStatus::NotApplicable),
237 MssqlPreparedTargetAction::CreatedTable
238 | MssqlPreparedTargetAction::CreatedStagingTable => {
239 let drop_table_sql = format!("DROP TABLE {}", prepared_target.quoted_table_sql());
240 client
241 .execute_statement(&drop_table_sql)
242 .await
243 .map_err(|source| cleanup_error(output_plan, source.to_string()))?;
244
245 Ok(MssqlTargetCleanupStatus::Succeeded)
246 }
247 }
248}
249
250#[allow(dead_code)]
252pub(crate) async fn swap_mssql_replace_target<C>(
253 output_plan: &MssqlTargetOutputPlan,
254 prepared_target: &MssqlPreparedTarget,
255 client: &mut C,
256) -> Result<(), DeltaFunnelError>
257where
258 C: MssqlTargetLifecycleClient + ?Sized,
259{
260 ensure_replace_output(output_plan)?;
261 if !prepared_target_matches_output_plan(output_plan, prepared_target) {
262 return Err(swap_target_error(
263 output_plan,
264 "prepared target does not match output plan",
265 false,
266 ));
267 }
268 if prepared_target.report().action() != MssqlPreparedTargetAction::CreatedStagingTable {
269 return Err(swap_target_error(
270 output_plan,
271 "replace swap requires a prepared staging table",
272 false,
273 ));
274 }
275
276 let final_table_name =
277 table_name_from_target(output_plan.output_name(), output_plan.target_table())?;
278 let backup_table_name = available_replace_backup_table(output_plan, client).await?;
279 let swap_sql = replace_swap_sql(
280 &final_table_name,
281 prepared_target.table_name(),
282 &backup_table_name,
283 );
284
285 client
286 .execute_statement(&swap_sql)
287 .await
288 .map_err(|source| swap_target_error(output_plan, source.to_string(), true))?;
289
290 Ok(())
291}
292
293#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
295pub enum MssqlPreparedTargetAction {
296 VerifiedExisting,
298 CreatedTable,
300 CreatedStagingTable,
302}
303
304impl MssqlPreparedTargetAction {
305 const fn cleanup_status(self) -> MssqlTargetCleanupStatus {
306 match self {
307 Self::VerifiedExisting => MssqlTargetCleanupStatus::NotApplicable,
308 Self::CreatedTable | Self::CreatedStagingTable => {
309 MssqlTargetCleanupStatus::NotAttempted
310 }
311 }
312 }
313}
314
315#[derive(Debug, Clone, PartialEq, Eq)]
317pub struct MssqlPreparedTargetReport {
318 output_name: String,
319 target_table: MssqlTargetTable,
320 load_mode: LoadMode,
321 connection_source: MssqlConnectionSource,
322 connection: MssqlConnectionSummary,
323 expected_target_state: MssqlTargetTableState,
324 action: MssqlPreparedTargetAction,
325 cleanup: MssqlTargetCleanupStatus,
326}
327
328impl MssqlPreparedTargetReport {
329 pub fn from_output_plan(
331 output_plan: &MssqlTargetOutputPlan,
332 action: MssqlPreparedTargetAction,
333 ) -> Result<Self, DeltaFunnelError> {
334 ensure_action_matches_load_mode(output_plan, action)?;
335
336 Ok(Self {
337 output_name: output_plan.output_name().to_owned(),
338 target_table: output_plan.target_table().clone(),
339 load_mode: output_plan.load_mode(),
340 connection_source: output_plan.connection_source(),
341 connection: output_plan.connection().clone(),
342 expected_target_state: output_plan.lifecycle_plan().expected_target_state(),
343 action,
344 cleanup: action.cleanup_status(),
345 })
346 }
347
348 #[must_use]
350 pub fn output_name(&self) -> &str {
351 &self.output_name
352 }
353
354 #[must_use]
356 pub const fn target_table(&self) -> &MssqlTargetTable {
357 &self.target_table
358 }
359
360 #[must_use]
362 pub const fn load_mode(&self) -> LoadMode {
363 self.load_mode
364 }
365
366 #[must_use]
368 pub const fn connection_source(&self) -> MssqlConnectionSource {
369 self.connection_source
370 }
371
372 #[must_use]
374 pub const fn connection(&self) -> &MssqlConnectionSummary {
375 &self.connection
376 }
377
378 #[must_use]
380 pub const fn expected_target_state(&self) -> MssqlTargetTableState {
381 self.expected_target_state
382 }
383
384 #[must_use]
386 pub const fn action(&self) -> MssqlPreparedTargetAction {
387 self.action
388 }
389
390 #[must_use]
392 pub const fn cleanup(&self) -> MssqlTargetCleanupStatus {
393 self.cleanup
394 }
395}
396
397#[derive(Debug, Clone, PartialEq, Eq)]
399pub struct MssqlPreparedTarget {
400 table_name: TableName,
401 report: MssqlPreparedTargetReport,
402}
403
404impl MssqlPreparedTarget {
405 pub fn from_output_plan(
407 output_plan: &MssqlTargetOutputPlan,
408 action: MssqlPreparedTargetAction,
409 ) -> Result<Self, DeltaFunnelError> {
410 let table_name =
411 table_name_from_target(output_plan.output_name(), output_plan.target_table())?;
412 Self::from_table_name(output_plan, table_name, action)
413 }
414
415 pub(crate) fn from_table_name(
416 output_plan: &MssqlTargetOutputPlan,
417 table_name: TableName,
418 action: MssqlPreparedTargetAction,
419 ) -> Result<Self, DeltaFunnelError> {
420 let report = MssqlPreparedTargetReport::from_output_plan(output_plan, action)?;
421
422 Ok(Self { table_name, report })
423 }
424
425 #[must_use]
427 pub const fn table_name(&self) -> &TableName {
428 &self.table_name
429 }
430
431 #[must_use]
433 pub fn quoted_table_sql(&self) -> String {
434 self.table_name.quoted_sql()
435 }
436
437 #[must_use]
439 pub const fn report(&self) -> &MssqlPreparedTargetReport {
440 &self.report
441 }
442}
443
444pub(crate) fn table_name_from_target(
445 output_name: &str,
446 table: &MssqlTargetTable,
447) -> Result<TableName, DeltaFunnelError> {
448 match table.schema() {
449 Some(schema) => TableName::new(schema, table.table()),
450 None => TableName::unqualified(table.table()),
451 }
452 .map_err(|source| DeltaFunnelError::MssqlDdlTargetIdentifier {
453 output_name: output_name.to_owned(),
454 source,
455 })
456}
457
458fn ensure_action_matches_load_mode(
459 output_plan: &MssqlTargetOutputPlan,
460 action: MssqlPreparedTargetAction,
461) -> Result<(), DeltaFunnelError> {
462 let allowed = matches!(
463 (output_plan.load_mode(), action),
464 (
465 LoadMode::AppendExisting,
466 MssqlPreparedTargetAction::VerifiedExisting
467 ) | (
468 LoadMode::CreateAndLoad,
469 MssqlPreparedTargetAction::CreatedTable
470 ) | (
471 LoadMode::Replace,
472 MssqlPreparedTargetAction::CreatedStagingTable
473 )
474 );
475
476 if allowed {
477 return Ok(());
478 }
479
480 Err(DeltaFunnelError::MssqlLifecyclePlanning {
481 output_name: output_plan.output_name().to_owned(),
482 message: "prepared target action does not match load mode".to_owned(),
483 })
484}
485
486#[allow(dead_code)]
487fn ensure_append_existing_output(
488 output_plan: &MssqlTargetOutputPlan,
489) -> Result<(), DeltaFunnelError> {
490 if output_plan.load_mode() == LoadMode::AppendExisting {
491 return Ok(());
492 }
493
494 Err(DeltaFunnelError::MssqlLifecyclePlanning {
495 output_name: output_plan.output_name().to_owned(),
496 message: "append-existing target preparation requires append-existing load mode".to_owned(),
497 })
498}
499
500#[allow(dead_code)]
501fn ensure_create_and_load_output(
502 output_plan: &MssqlTargetOutputPlan,
503) -> Result<(), DeltaFunnelError> {
504 if output_plan.load_mode() == LoadMode::CreateAndLoad {
505 return Ok(());
506 }
507
508 Err(DeltaFunnelError::MssqlLifecyclePlanning {
509 output_name: output_plan.output_name().to_owned(),
510 message: "create-and-load target preparation requires create-and-load load mode".to_owned(),
511 })
512}
513
514#[allow(dead_code)]
515fn ensure_replace_output(output_plan: &MssqlTargetOutputPlan) -> Result<(), DeltaFunnelError> {
516 if output_plan.load_mode() == LoadMode::Replace {
517 return Ok(());
518 }
519
520 Err(DeltaFunnelError::MssqlLifecyclePlanning {
521 output_name: output_plan.output_name().to_owned(),
522 message: "replace target preparation requires replace load mode".to_owned(),
523 })
524}
525
526#[allow(dead_code)]
527fn ensure_lifecycle_guardrail(
528 output_plan: &MssqlTargetOutputPlan,
529 guardrail: MssqlLifecycleExecutionGuardrail,
530) -> Result<(), DeltaFunnelError> {
531 if output_plan
532 .lifecycle_plan()
533 .execution_guardrails()
534 .contains(&guardrail)
535 {
536 return Ok(());
537 }
538
539 Err(DeltaFunnelError::MssqlLifecyclePlanning {
540 output_name: output_plan.output_name().to_owned(),
541 message: format!("target lifecycle plan does not allow {guardrail:?}"),
542 })
543}
544
545#[allow(dead_code)]
546fn ensure_prepared_target_matches_output_plan(
547 output_plan: &MssqlTargetOutputPlan,
548 prepared_target: &MssqlPreparedTarget,
549) -> Result<(), DeltaFunnelError> {
550 if prepared_target_matches_output_plan(output_plan, prepared_target) {
551 return Ok(());
552 }
553
554 Err(cleanup_error(
555 output_plan,
556 "prepared target does not match output plan",
557 ))
558}
559
560fn prepared_target_matches_output_plan(
561 output_plan: &MssqlTargetOutputPlan,
562 prepared_target: &MssqlPreparedTarget,
563) -> bool {
564 let report = prepared_target.report();
565 report.output_name() == output_plan.output_name()
566 && report.target_table() == output_plan.target_table()
567 && report.load_mode() == output_plan.load_mode()
568 && report.connection_source() == output_plan.connection_source()
569 && report.connection() == output_plan.connection()
570 && report.expected_target_state() == output_plan.lifecycle_plan().expected_target_state()
571}
572
573#[allow(dead_code)]
574fn prepare_target_lifecycle_error(
575 output_plan: &MssqlTargetOutputPlan,
576 message: impl Into<String>,
577) -> DeltaFunnelError {
578 DeltaFunnelError::MssqlWritePhase {
579 context: Box::new(MssqlWriteFailureContext::from_output_plan(
580 output_plan,
581 MssqlWritePhase::PrepareTargetLifecycle,
582 0,
583 0,
584 0,
585 false,
586 cleanup_before_target_creation(output_plan),
587 )),
588 message: message.into(),
589 }
590}
591
592#[allow(dead_code)]
593fn cleanup_error(
594 output_plan: &MssqlTargetOutputPlan,
595 message: impl Into<String>,
596) -> DeltaFunnelError {
597 DeltaFunnelError::MssqlWritePhase {
598 context: Box::new(MssqlWriteFailureContext::from_output_plan(
599 output_plan,
600 MssqlWritePhase::Cleanup,
601 0,
602 0,
603 0,
604 false,
605 MssqlTargetCleanupStatus::Failed,
606 )),
607 message: message.into(),
608 }
609}
610
611#[allow(dead_code)]
612fn swap_target_error(
613 output_plan: &MssqlTargetOutputPlan,
614 message: impl Into<String>,
615 partial_write_possible: bool,
616) -> DeltaFunnelError {
617 DeltaFunnelError::MssqlWritePhase {
618 context: Box::new(MssqlWriteFailureContext::from_output_plan(
619 output_plan,
620 MssqlWritePhase::SwapTarget,
621 0,
622 0,
623 0,
624 partial_write_possible,
625 MssqlTargetCleanupStatus::NotAttempted,
626 )),
627 message: message.into(),
628 }
629}
630
631#[allow(dead_code)]
632fn cleanup_before_target_creation(output_plan: &MssqlTargetOutputPlan) -> MssqlTargetCleanupStatus {
633 match output_plan.load_mode() {
634 LoadMode::AppendExisting => MssqlTargetCleanupStatus::NotApplicable,
635 LoadMode::CreateAndLoad | LoadMode::Replace => MssqlTargetCleanupStatus::NotAttempted,
636 }
637}
638
639async fn available_replace_staging_table<C>(
640 output_plan: &MssqlTargetOutputPlan,
641 client: &mut C,
642) -> Result<TableName, DeltaFunnelError>
643where
644 C: MssqlTargetLifecycleClient + ?Sized,
645{
646 for attempt in 0..100_u8 {
647 let table_name = replace_staging_table_name(output_plan, attempt)?;
648 let exists = client
649 .table_exists(&table_name)
650 .await
651 .map_err(|source| prepare_target_lifecycle_error(output_plan, source.to_string()))?;
652 if !exists {
653 return Ok(table_name);
654 }
655 }
656
657 Err(prepare_target_lifecycle_error(
658 output_plan,
659 "could not find an available replace staging table name",
660 ))
661}
662
663fn replace_staging_table_name(
664 output_plan: &MssqlTargetOutputPlan,
665 attempt: u8,
666) -> Result<TableName, DeltaFunnelError> {
667 let suffix = format!("__df_replace_{attempt}");
668 replace_auxiliary_table_name(output_plan, &suffix)
669}
670
671async fn available_replace_backup_table<C>(
672 output_plan: &MssqlTargetOutputPlan,
673 client: &mut C,
674) -> Result<TableName, DeltaFunnelError>
675where
676 C: MssqlTargetLifecycleClient + ?Sized,
677{
678 for attempt in 0..100_u8 {
679 let table_name = replace_backup_table_name(output_plan, attempt)?;
680 let exists = client
681 .table_exists(&table_name)
682 .await
683 .map_err(|source| swap_target_error(output_plan, source.to_string(), false))?;
684 if !exists {
685 return Ok(table_name);
686 }
687 }
688
689 Err(swap_target_error(
690 output_plan,
691 "could not find an available replace backup table name",
692 false,
693 ))
694}
695
696fn replace_backup_table_name(
697 output_plan: &MssqlTargetOutputPlan,
698 attempt: u8,
699) -> Result<TableName, DeltaFunnelError> {
700 let suffix = format!("__df_backup_{attempt}");
701 replace_auxiliary_table_name(output_plan, &suffix)
702}
703
704fn replace_auxiliary_table_name(
705 output_plan: &MssqlTargetOutputPlan,
706 suffix: &str,
707) -> Result<TableName, DeltaFunnelError> {
708 const MAX_IDENTIFIER_CHARS: usize = 128;
709 let max_base_chars = MAX_IDENTIFIER_CHARS.saturating_sub(suffix.chars().count());
710 let base = output_plan
711 .target_table()
712 .table()
713 .chars()
714 .take(max_base_chars)
715 .collect::<String>();
716 let table_name = format!("{base}{suffix}");
717
718 match output_plan.target_table().schema() {
719 Some(schema) => TableName::new(schema, table_name),
720 None => TableName::unqualified(table_name),
721 }
722 .map_err(|source| DeltaFunnelError::MssqlDdlTargetIdentifier {
723 output_name: output_plan.output_name().to_owned(),
724 source,
725 })
726}
727
728fn replace_swap_sql(
729 final_table: &TableName,
730 staging_table: &TableName,
731 backup_table: &TableName,
732) -> String {
733 let lock_resource = replace_lock_resource(final_table);
734 let final_object = final_table.quoted_sql();
735 let staging_object = staging_table.quoted_sql();
736 let backup_object = backup_table.quoted_sql();
737 let final_table_name = final_table.table().as_str();
738 let backup_table_name = backup_table.table().as_str();
739
740 format!(
741 concat!(
742 "SET XACT_ABORT ON;\n",
743 "BEGIN TRY\n",
744 " BEGIN TRANSACTION;\n",
745 " DECLARE @delta_funnel_lock_result int;\n",
746 " EXEC @delta_funnel_lock_result = sys.sp_getapplock ",
747 "@Resource = {lock_resource}, ",
748 "@LockMode = 'Exclusive', ",
749 "@LockOwner = 'Transaction', ",
750 "@LockTimeout = 0;\n",
751 " IF @delta_funnel_lock_result < 0\n",
752 " THROW 51000, 'DeltaFunnel replace lock was not acquired', 1;\n",
753 " IF OBJECT_ID({staging_object}, N'U') IS NULL\n",
754 " THROW 51002, 'DeltaFunnel replace staging table is missing', 1;\n",
755 " IF OBJECT_ID({backup_object}, N'U') IS NOT NULL\n",
756 " THROW 51003, 'DeltaFunnel replace backup table already exists', 1;\n",
757 " IF OBJECT_ID({final_object}, N'U') IS NOT NULL\n",
758 " BEGIN\n",
759 " EXEC sys.sp_rename @objname = {final_object}, ",
760 "@newname = {backup_table_name}, @objtype = 'OBJECT';\n",
761 " EXEC sys.sp_rename @objname = {staging_object}, ",
762 "@newname = {final_table_name}, @objtype = 'OBJECT';\n",
763 " DROP TABLE {backup_table};\n",
764 " END\n",
765 " ELSE\n",
766 " BEGIN\n",
767 " EXEC sys.sp_rename @objname = {staging_object}, ",
768 "@newname = {final_table_name}, @objtype = 'OBJECT';\n",
769 " END\n",
770 " COMMIT TRANSACTION;\n",
771 "END TRY\n",
772 "BEGIN CATCH\n",
773 " IF @@TRANCOUNT > 0\n",
774 " ROLLBACK TRANSACTION;\n",
775 " THROW;\n",
776 "END CATCH;"
777 ),
778 lock_resource = sql_nvarchar_literal(&lock_resource),
779 final_object = sql_nvarchar_literal(&final_object),
780 staging_object = sql_nvarchar_literal(&staging_object),
781 backup_object = sql_nvarchar_literal(&backup_object),
782 backup_table_name = sql_nvarchar_literal(backup_table_name),
783 final_table_name = sql_nvarchar_literal(final_table_name),
784 backup_table = backup_object,
785 )
786}
787
788fn replace_lock_resource(final_table: &TableName) -> String {
789 const MAX_RESOURCE_CHARS: usize = 255;
790 const HASH_CHARS: usize = 16;
791 const SEPARATOR_CHARS: usize = 1;
792
793 let resource = format!("delta-funnel:replace:{}", final_table.quoted_sql());
794 if resource.chars().count() <= MAX_RESOURCE_CHARS {
795 return resource;
796 }
797
798 let prefix_chars = MAX_RESOURCE_CHARS - HASH_CHARS - SEPARATOR_CHARS;
799 let prefix = resource.chars().take(prefix_chars).collect::<String>();
800 format!("{prefix}:{:016x}", stable_resource_hash(&resource))
801}
802
803fn stable_resource_hash(value: &str) -> u64 {
804 const FNV_OFFSET_BASIS: u64 = 0xcbf29ce484222325;
805 const FNV_PRIME: u64 = 0x100000001b3;
806
807 let mut hash = FNV_OFFSET_BASIS;
808 for byte in value.as_bytes() {
809 hash ^= u64::from(*byte);
810 hash = hash.wrapping_mul(FNV_PRIME);
811 }
812 hash
813}
814
815fn sql_nvarchar_literal(value: &str) -> String {
816 format!("N'{}'", value.replace('\'', "''"))
817}
818
819#[cfg(test)]
820mod tests {
821 use arrow_schema::{DataType, Field, Schema};
822 use arrow_tiberius::PlanOptions;
823 use async_trait::async_trait;
824
825 use super::*;
826 use crate::{
827 MssqlConnectionConfig, MssqlConnectionSource, MssqlTargetConfig,
828 MssqlTargetResolutionContext, plan_mssql_output_schema, plan_mssql_target_output,
829 };
830
831 fn secret_connection(label: &str) -> Result<MssqlConnectionConfig, DeltaFunnelError> {
832 Ok(MssqlConnectionConfig::new(
833 "server=tcp:sql.example.com;database=warehouse;user=admin;password=secret-token",
834 )?
835 .with_display_label(label))
836 }
837
838 fn output_plan(
839 output_name: &str,
840 load_mode: LoadMode,
841 table: MssqlTargetTable,
842 ) -> Result<MssqlTargetOutputPlan, DeltaFunnelError> {
843 let connection = secret_connection("warehouse-primary")?;
844 let target = MssqlTargetConfig::new(table)
845 .with_load_mode(load_mode)
846 .resolve(MssqlTargetResolutionContext {
847 output_name: Some(output_name),
848 default_connection: Some(&connection),
849 })?;
850 let schema = Schema::new(vec![
851 Field::new("order_id", DataType::Int64, false),
852 Field::new("status", DataType::Utf8, true),
853 ]);
854 let schema_plan = plan_mssql_output_schema(&schema, &target, PlanOptions::default())?;
855
856 plan_mssql_target_output(schema_plan)
857 }
858
859 fn test_table_name(schema: &str, table: &str) -> Result<TableName, DeltaFunnelError> {
860 TableName::new(schema, table).map_err(|error| DeltaFunnelError::Config {
861 message: error.to_string(),
862 })
863 }
864
865 fn replace_staging_prepared_target(
866 output_plan: &MssqlTargetOutputPlan,
867 table: &str,
868 ) -> Result<MssqlPreparedTarget, DeltaFunnelError> {
869 MssqlPreparedTarget::from_table_name(
870 output_plan,
871 test_table_name("dbo", table)?,
872 MssqlPreparedTargetAction::CreatedStagingTable,
873 )
874 }
875
876 #[derive(Default)]
877 struct RecordingLifecycleClient {
878 calls: Vec<String>,
879 exists: bool,
880 existence_results: Vec<bool>,
881 rows_affected: Vec<u64>,
882 table_exists_error: Option<String>,
883 execute_statement_error: Option<String>,
884 }
885
886 #[async_trait]
887 impl MssqlTargetLifecycleClient for RecordingLifecycleClient {
888 async fn table_exists(&mut self, table: &TableName) -> arrow_tiberius::Result<bool> {
889 self.calls.push(format!("probe {}", table.quoted_sql()));
890 if let Some(reason) = self.table_exists_error.take() {
891 return Err(arrow_tiberius::Error::TableExistsUnexpectedResult { reason });
892 }
893
894 if !self.existence_results.is_empty() {
895 return Ok(self.existence_results.remove(0));
896 }
897
898 Ok(self.exists)
899 }
900
901 async fn execute_statement(
902 &mut self,
903 sql: &str,
904 ) -> arrow_tiberius::Result<SqlExecutionOutcome> {
905 self.calls.push(format!("execute {sql}"));
906 if let Some(reason) = self.execute_statement_error.take() {
907 return Err(arrow_tiberius::Error::InvalidIdentifier { reason });
908 }
909
910 Ok(SqlExecutionOutcome {
911 rows_affected: self.rows_affected.clone(),
912 })
913 }
914 }
915
916 #[test]
917 fn connected_arrow_tiberius_client_implements_lifecycle_client_boundary() {
918 fn assert_lifecycle_client<C: MssqlTargetLifecycleClient>() {}
919
920 assert_lifecycle_client::<arrow_tiberius::ConnectedMssqlClient>();
921 }
922
923 #[tokio::test]
924 async fn lifecycle_client_boundary_is_fakeable() -> arrow_tiberius::Result<()> {
925 let table = TableName::new("dbo", "orders")?;
926 let mut client = RecordingLifecycleClient {
927 exists: true,
928 rows_affected: vec![1],
929 ..RecordingLifecycleClient::default()
930 };
931
932 let exists = MssqlTargetLifecycleClient::table_exists(&mut client, &table).await?;
933 let outcome = MssqlTargetLifecycleClient::execute_statement(
934 &mut client,
935 "CREATE TABLE [dbo].[orders] ([id] bigint)",
936 )
937 .await?;
938
939 assert!(exists);
940 assert_eq!(outcome.rows_affected, vec![1]);
941 assert_eq!(
942 client.calls,
943 vec![
944 "probe [dbo].[orders]".to_owned(),
945 "execute CREATE TABLE [dbo].[orders] ([id] bigint)".to_owned(),
946 ]
947 );
948 Ok(())
949 }
950
951 #[tokio::test]
952 async fn target_lifecycle_preparation_dispatches_append_existing()
953 -> Result<(), DeltaFunnelError> {
954 let output_plan = output_plan(
955 "orders_output",
956 LoadMode::AppendExisting,
957 MssqlTargetTable::new("dbo", "orders")?,
958 )?;
959 let mut client = RecordingLifecycleClient {
960 exists: true,
961 ..RecordingLifecycleClient::default()
962 };
963
964 let prepared = prepare_mssql_target_lifecycle(&output_plan, &mut client).await?;
965
966 assert_eq!(
967 prepared.report().action(),
968 MssqlPreparedTargetAction::VerifiedExisting
969 );
970 assert_eq!(client.calls, vec!["probe [dbo].[orders]".to_owned()]);
971 Ok(())
972 }
973
974 #[tokio::test]
975 async fn target_lifecycle_preparation_dispatches_create_and_load()
976 -> Result<(), DeltaFunnelError> {
977 let output_plan = output_plan(
978 "orders_output",
979 LoadMode::CreateAndLoad,
980 MssqlTargetTable::new("dbo", "orders")?,
981 )?;
982 let mut client = RecordingLifecycleClient {
983 exists: false,
984 ..RecordingLifecycleClient::default()
985 };
986
987 let prepared = prepare_mssql_target_lifecycle(&output_plan, &mut client).await?;
988
989 assert_eq!(
990 prepared.report().action(),
991 MssqlPreparedTargetAction::CreatedTable
992 );
993 assert_eq!(client.calls.len(), 2);
994 assert_eq!(client.calls[0], "probe [dbo].[orders]");
995 assert!(client.calls[1].starts_with("execute CREATE TABLE [dbo].[orders]"));
996 Ok(())
997 }
998
999 #[tokio::test]
1000 async fn target_lifecycle_preparation_dispatches_replace_to_staging()
1001 -> Result<(), DeltaFunnelError> {
1002 let output_plan = output_plan(
1003 "orders_output",
1004 LoadMode::Replace,
1005 MssqlTargetTable::new("dbo", "orders")?,
1006 )?;
1007 let mut client = RecordingLifecycleClient {
1008 existence_results: vec![true, false],
1009 ..RecordingLifecycleClient::default()
1010 };
1011
1012 let prepared = prepare_mssql_target_lifecycle(&output_plan, &mut client).await?;
1013
1014 assert_eq!(prepared.quoted_table_sql(), "[dbo].[orders__df_replace_0]");
1015 assert_eq!(
1016 prepared.report().action(),
1017 MssqlPreparedTargetAction::CreatedStagingTable
1018 );
1019 assert_eq!(
1020 prepared.report().cleanup(),
1021 MssqlTargetCleanupStatus::NotAttempted
1022 );
1023 assert_eq!(client.calls.len(), 3);
1024 assert_eq!(client.calls[0], "probe [dbo].[orders]");
1025 assert_eq!(client.calls[1], "probe [dbo].[orders__df_replace_0]");
1026 assert!(client.calls[2].starts_with("execute CREATE TABLE [dbo].[orders__df_replace_0]"));
1027 Ok(())
1028 }
1029
1030 #[tokio::test]
1031 async fn append_existing_preparation_verifies_existing_target() -> Result<(), DeltaFunnelError>
1032 {
1033 let output_plan = output_plan(
1034 "orders_output",
1035 LoadMode::AppendExisting,
1036 MssqlTargetTable::new("dbo", "orders")?,
1037 )?;
1038 let mut client = RecordingLifecycleClient {
1039 exists: true,
1040 ..RecordingLifecycleClient::default()
1041 };
1042
1043 let prepared = prepare_mssql_append_existing_target(&output_plan, &mut client).await?;
1044
1045 assert_eq!(prepared.quoted_table_sql(), "[dbo].[orders]");
1046 assert_eq!(
1047 prepared.report().action(),
1048 MssqlPreparedTargetAction::VerifiedExisting
1049 );
1050 assert_eq!(
1051 prepared.report().cleanup(),
1052 MssqlTargetCleanupStatus::NotApplicable
1053 );
1054 assert_eq!(client.calls, vec!["probe [dbo].[orders]".to_owned()]);
1055 Ok(())
1056 }
1057
1058 #[tokio::test]
1059 async fn append_existing_preparation_fails_when_target_is_absent()
1060 -> Result<(), DeltaFunnelError> {
1061 let output_plan = output_plan(
1062 "orders_output",
1063 LoadMode::AppendExisting,
1064 MssqlTargetTable::new("dbo", "orders")?,
1065 )?;
1066 let mut client = RecordingLifecycleClient {
1067 exists: false,
1068 ..RecordingLifecycleClient::default()
1069 };
1070
1071 let error = prepare_mssql_append_existing_target(&output_plan, &mut client)
1072 .await
1073 .err()
1074 .ok_or_else(|| DeltaFunnelError::Config {
1075 message: "expected absent append-existing target error".to_owned(),
1076 })?;
1077
1078 assert_prepare_target_lifecycle_error(
1079 error,
1080 "append-existing target table [dbo].[orders] does not exist",
1081 MssqlTargetCleanupStatus::NotApplicable,
1082 )?;
1083 assert_eq!(client.calls, vec!["probe [dbo].[orders]".to_owned()]);
1084 Ok(())
1085 }
1086
1087 #[tokio::test]
1088 async fn append_existing_preparation_maps_probe_errors_to_lifecycle_phase()
1089 -> Result<(), DeltaFunnelError> {
1090 let output_plan = output_plan(
1091 "orders_output",
1092 LoadMode::AppendExisting,
1093 MssqlTargetTable::new("dbo", "orders")?,
1094 )?;
1095 let mut client = RecordingLifecycleClient {
1096 table_exists_error: Some("metadata query failed\nfor test".to_owned()),
1097 ..RecordingLifecycleClient::default()
1098 };
1099
1100 let error = prepare_mssql_append_existing_target(&output_plan, &mut client)
1101 .await
1102 .err()
1103 .ok_or_else(|| DeltaFunnelError::Config {
1104 message: "expected probe error".to_owned(),
1105 })?;
1106
1107 assert_prepare_target_lifecycle_error(
1108 error,
1109 r"metadata query failed\nfor test",
1110 MssqlTargetCleanupStatus::NotApplicable,
1111 )?;
1112 assert_eq!(client.calls, vec!["probe [dbo].[orders]".to_owned()]);
1113 Ok(())
1114 }
1115
1116 #[tokio::test]
1117 async fn append_existing_preparation_rejects_create_and_load_before_probe()
1118 -> Result<(), DeltaFunnelError> {
1119 let output_plan = output_plan(
1120 "orders_output",
1121 LoadMode::CreateAndLoad,
1122 MssqlTargetTable::new("dbo", "orders")?,
1123 )?;
1124 let mut client = RecordingLifecycleClient {
1125 exists: true,
1126 ..RecordingLifecycleClient::default()
1127 };
1128
1129 let error = prepare_mssql_append_existing_target(&output_plan, &mut client)
1130 .await
1131 .err()
1132 .ok_or_else(|| DeltaFunnelError::Config {
1133 message: "expected append-only preparation error".to_owned(),
1134 })?;
1135
1136 assert!(matches!(
1137 error,
1138 DeltaFunnelError::MssqlLifecyclePlanning { .. }
1139 ));
1140 assert!(error.to_string().contains("requires append-existing"));
1141 assert!(client.calls.is_empty());
1142 Ok(())
1143 }
1144
1145 #[tokio::test]
1146 async fn create_and_load_preparation_creates_absent_target() -> Result<(), DeltaFunnelError> {
1147 let output_plan = output_plan(
1148 "orders_output",
1149 LoadMode::CreateAndLoad,
1150 MssqlTargetTable::new("dbo", "orders")?,
1151 )?;
1152 let create_table_sql = output_plan
1153 .create_table_sql()
1154 .ok_or_else(|| DeltaFunnelError::Config {
1155 message: "expected planned create-table SQL".to_owned(),
1156 })?
1157 .to_owned();
1158 let mut client = RecordingLifecycleClient {
1159 exists: false,
1160 rows_affected: vec![0],
1161 ..RecordingLifecycleClient::default()
1162 };
1163
1164 let prepared = prepare_mssql_create_and_load_target(&output_plan, &mut client).await?;
1165
1166 assert_eq!(prepared.quoted_table_sql(), "[dbo].[orders]");
1167 assert_eq!(
1168 prepared.report().action(),
1169 MssqlPreparedTargetAction::CreatedTable
1170 );
1171 assert_eq!(
1172 prepared.report().cleanup(),
1173 MssqlTargetCleanupStatus::NotAttempted
1174 );
1175 assert_eq!(
1176 client.calls,
1177 vec![
1178 "probe [dbo].[orders]".to_owned(),
1179 format!("execute {create_table_sql}"),
1180 ]
1181 );
1182 Ok(())
1183 }
1184
1185 #[tokio::test]
1186 async fn create_and_load_preparation_fails_when_target_already_exists()
1187 -> Result<(), DeltaFunnelError> {
1188 let output_plan = output_plan(
1189 "orders_output",
1190 LoadMode::CreateAndLoad,
1191 MssqlTargetTable::new("dbo", "orders")?,
1192 )?;
1193 let mut client = RecordingLifecycleClient {
1194 exists: true,
1195 ..RecordingLifecycleClient::default()
1196 };
1197
1198 let error = prepare_mssql_create_and_load_target(&output_plan, &mut client)
1199 .await
1200 .err()
1201 .ok_or_else(|| DeltaFunnelError::Config {
1202 message: "expected existing create-and-load target error".to_owned(),
1203 })?;
1204
1205 assert_prepare_target_lifecycle_error(
1206 error,
1207 "create-and-load target table [dbo].[orders] already exists",
1208 MssqlTargetCleanupStatus::NotAttempted,
1209 )?;
1210 assert_eq!(client.calls, vec!["probe [dbo].[orders]".to_owned()]);
1211 Ok(())
1212 }
1213
1214 #[tokio::test]
1215 async fn create_and_load_preparation_maps_probe_errors_to_lifecycle_phase()
1216 -> Result<(), DeltaFunnelError> {
1217 let output_plan = output_plan(
1218 "orders_output",
1219 LoadMode::CreateAndLoad,
1220 MssqlTargetTable::new("dbo", "orders")?,
1221 )?;
1222 let mut client = RecordingLifecycleClient {
1223 table_exists_error: Some("metadata query failed\nfor test".to_owned()),
1224 ..RecordingLifecycleClient::default()
1225 };
1226
1227 let error = prepare_mssql_create_and_load_target(&output_plan, &mut client)
1228 .await
1229 .err()
1230 .ok_or_else(|| DeltaFunnelError::Config {
1231 message: "expected create-and-load probe error".to_owned(),
1232 })?;
1233
1234 assert_prepare_target_lifecycle_error(
1235 error,
1236 r"metadata query failed\nfor test",
1237 MssqlTargetCleanupStatus::NotAttempted,
1238 )?;
1239 assert_eq!(client.calls, vec!["probe [dbo].[orders]".to_owned()]);
1240 Ok(())
1241 }
1242
1243 #[tokio::test]
1244 async fn create_and_load_preparation_maps_ddl_errors_to_lifecycle_phase()
1245 -> Result<(), DeltaFunnelError> {
1246 let output_plan = output_plan(
1247 "orders_output",
1248 LoadMode::CreateAndLoad,
1249 MssqlTargetTable::new("dbo", "orders")?,
1250 )?;
1251 let mut client = RecordingLifecycleClient {
1252 exists: false,
1253 execute_statement_error: Some("DDL failed\nfor test".to_owned()),
1254 ..RecordingLifecycleClient::default()
1255 };
1256
1257 let error = prepare_mssql_create_and_load_target(&output_plan, &mut client)
1258 .await
1259 .err()
1260 .ok_or_else(|| DeltaFunnelError::Config {
1261 message: "expected create-and-load DDL error".to_owned(),
1262 })?;
1263
1264 assert_prepare_target_lifecycle_error(
1265 error,
1266 r"DDL failed\nfor test",
1267 MssqlTargetCleanupStatus::NotAttempted,
1268 )?;
1269 assert_eq!(client.calls.len(), 2);
1270 assert_eq!(client.calls[0], "probe [dbo].[orders]");
1271 assert!(
1272 client.calls[1].starts_with("execute CREATE TABLE [dbo].[orders]"),
1273 "unexpected DDL call: {}",
1274 client.calls[1]
1275 );
1276 Ok(())
1277 }
1278
1279 #[tokio::test]
1280 async fn create_and_load_preparation_rejects_append_existing_before_probe()
1281 -> Result<(), DeltaFunnelError> {
1282 let output_plan = output_plan(
1283 "orders_output",
1284 LoadMode::AppendExisting,
1285 MssqlTargetTable::new("dbo", "orders")?,
1286 )?;
1287 let mut client = RecordingLifecycleClient {
1288 exists: false,
1289 ..RecordingLifecycleClient::default()
1290 };
1291
1292 let error = prepare_mssql_create_and_load_target(&output_plan, &mut client)
1293 .await
1294 .err()
1295 .ok_or_else(|| DeltaFunnelError::Config {
1296 message: "expected create-and-load mode error".to_owned(),
1297 })?;
1298
1299 assert!(matches!(
1300 error,
1301 DeltaFunnelError::MssqlLifecyclePlanning { .. }
1302 ));
1303 assert!(error.to_string().contains("requires create-and-load"));
1304 assert!(client.calls.is_empty());
1305 Ok(())
1306 }
1307
1308 #[tokio::test]
1309 async fn replace_preparation_creates_available_staging_target() -> Result<(), DeltaFunnelError>
1310 {
1311 let output_plan = output_plan(
1312 "orders_output",
1313 LoadMode::Replace,
1314 MssqlTargetTable::new("dbo", "orders")?,
1315 )?;
1316 let mut client = RecordingLifecycleClient {
1317 existence_results: vec![true, false],
1318 rows_affected: vec![0],
1319 ..RecordingLifecycleClient::default()
1320 };
1321
1322 let prepared = prepare_mssql_replace_target(&output_plan, &mut client).await?;
1323
1324 assert_eq!(prepared.quoted_table_sql(), "[dbo].[orders__df_replace_0]");
1325 assert_eq!(
1326 prepared.report().action(),
1327 MssqlPreparedTargetAction::CreatedStagingTable
1328 );
1329 assert_eq!(client.calls.len(), 3);
1330 assert_eq!(client.calls[0], "probe [dbo].[orders]");
1331 assert_eq!(client.calls[1], "probe [dbo].[orders__df_replace_0]");
1332 assert!(client.calls[2].starts_with("execute CREATE TABLE [dbo].[orders__df_replace_0]"));
1333 assert!(client.calls[2].contains("[order_id] bigint NOT NULL"));
1334 assert!(client.calls[2].contains("[status] nvarchar(max) NULL"));
1335 Ok(())
1336 }
1337
1338 #[tokio::test]
1339 async fn replace_preparation_creates_staging_when_final_target_is_absent()
1340 -> Result<(), DeltaFunnelError> {
1341 let output_plan = output_plan(
1342 "orders_output",
1343 LoadMode::Replace,
1344 MssqlTargetTable::new("dbo", "orders")?,
1345 )?;
1346 let mut client = RecordingLifecycleClient {
1347 existence_results: vec![false, false],
1348 rows_affected: vec![0],
1349 ..RecordingLifecycleClient::default()
1350 };
1351
1352 let prepared = prepare_mssql_replace_target(&output_plan, &mut client).await?;
1353
1354 assert_eq!(prepared.quoted_table_sql(), "[dbo].[orders__df_replace_0]");
1355 assert_eq!(
1356 prepared.report().action(),
1357 MssqlPreparedTargetAction::CreatedStagingTable
1358 );
1359 assert_eq!(
1360 client.calls,
1361 vec![
1362 "probe [dbo].[orders]",
1363 "probe [dbo].[orders__df_replace_0]",
1364 "execute CREATE TABLE [dbo].[orders__df_replace_0] (\n [order_id] bigint NOT NULL,\n [status] nvarchar(max) NULL\n);"
1365 ]
1366 );
1367 Ok(())
1368 }
1369
1370 #[tokio::test]
1371 async fn replace_preparation_skips_existing_staging_name() -> Result<(), DeltaFunnelError> {
1372 let output_plan = output_plan(
1373 "orders_output",
1374 LoadMode::Replace,
1375 MssqlTargetTable::new("dbo", "orders")?,
1376 )?;
1377 let mut client = RecordingLifecycleClient {
1378 existence_results: vec![true, true, false],
1379 ..RecordingLifecycleClient::default()
1380 };
1381
1382 let prepared = prepare_mssql_replace_target(&output_plan, &mut client).await?;
1383
1384 assert_eq!(prepared.quoted_table_sql(), "[dbo].[orders__df_replace_1]");
1385 assert_eq!(client.calls.len(), 4);
1386 assert_eq!(
1387 &client.calls[..3],
1388 [
1389 "probe [dbo].[orders]",
1390 "probe [dbo].[orders__df_replace_0]",
1391 "probe [dbo].[orders__df_replace_1]",
1392 ]
1393 );
1394 assert!(client.calls[3].starts_with("execute CREATE TABLE [dbo].[orders__df_replace_1]"));
1395 Ok(())
1396 }
1397
1398 #[tokio::test]
1399 async fn replace_preparation_rejects_create_and_load_before_probe()
1400 -> Result<(), DeltaFunnelError> {
1401 let output_plan = output_plan(
1402 "orders_output",
1403 LoadMode::CreateAndLoad,
1404 MssqlTargetTable::new("dbo", "orders")?,
1405 )?;
1406 let mut client = RecordingLifecycleClient {
1407 exists: true,
1408 ..RecordingLifecycleClient::default()
1409 };
1410
1411 let error = prepare_mssql_replace_target(&output_plan, &mut client)
1412 .await
1413 .err()
1414 .ok_or_else(|| DeltaFunnelError::Config {
1415 message: "expected replace preparation mode error".to_owned(),
1416 })?;
1417
1418 assert!(matches!(
1419 error,
1420 DeltaFunnelError::MssqlLifecyclePlanning { .. }
1421 ));
1422 assert!(error.to_string().contains("requires replace"));
1423 assert!(client.calls.is_empty());
1424 Ok(())
1425 }
1426
1427 #[tokio::test]
1428 async fn cleanup_returns_not_applicable_without_append_existing_target_creation()
1429 -> Result<(), DeltaFunnelError> {
1430 let output_plan = output_plan(
1431 "orders_output",
1432 LoadMode::AppendExisting,
1433 MssqlTargetTable::new("dbo", "orders")?,
1434 )?;
1435 let prepared = MssqlPreparedTarget::from_output_plan(
1436 &output_plan,
1437 MssqlPreparedTargetAction::VerifiedExisting,
1438 )?;
1439 let mut client = RecordingLifecycleClient::default();
1440
1441 let cleanup =
1442 cleanup_mssql_prepared_target(&output_plan, Some(&prepared), &mut client).await?;
1443
1444 assert_eq!(cleanup, MssqlTargetCleanupStatus::NotApplicable);
1445 assert!(client.calls.is_empty());
1446 Ok(())
1447 }
1448
1449 #[tokio::test]
1450 async fn cleanup_returns_not_attempted_before_create_and_load_target_creation()
1451 -> Result<(), DeltaFunnelError> {
1452 let output_plan = output_plan(
1453 "orders_output",
1454 LoadMode::CreateAndLoad,
1455 MssqlTargetTable::new("dbo", "orders")?,
1456 )?;
1457 let mut client = RecordingLifecycleClient::default();
1458
1459 let cleanup = cleanup_mssql_prepared_target(&output_plan, None, &mut client).await?;
1460
1461 assert_eq!(cleanup, MssqlTargetCleanupStatus::NotAttempted);
1462 assert!(client.calls.is_empty());
1463 Ok(())
1464 }
1465
1466 #[tokio::test]
1467 async fn cleanup_drops_deltafunnel_created_target_with_quoted_table_name()
1468 -> Result<(), DeltaFunnelError> {
1469 let output_plan = output_plan(
1470 "orders_output",
1471 LoadMode::CreateAndLoad,
1472 MssqlTargetTable::new("dbo.part", "target]part")?,
1473 )?;
1474 let prepared = MssqlPreparedTarget::from_output_plan(
1475 &output_plan,
1476 MssqlPreparedTargetAction::CreatedTable,
1477 )?;
1478 let mut client = RecordingLifecycleClient::default();
1479
1480 let cleanup =
1481 cleanup_mssql_prepared_target(&output_plan, Some(&prepared), &mut client).await?;
1482
1483 assert_eq!(cleanup, MssqlTargetCleanupStatus::Succeeded);
1484 assert_eq!(
1485 client.calls,
1486 vec!["execute DROP TABLE [dbo.part].[target]]part]".to_owned()]
1487 );
1488 Ok(())
1489 }
1490
1491 #[tokio::test]
1492 async fn cleanup_drops_deltafunnel_created_replace_staging_target()
1493 -> Result<(), DeltaFunnelError> {
1494 let output_plan = output_plan(
1495 "orders_output",
1496 LoadMode::Replace,
1497 MssqlTargetTable::new("dbo", "orders")?,
1498 )?;
1499 let staging_table = TableName::new("dbo", "orders__df_replace_0").map_err(|error| {
1500 DeltaFunnelError::Config {
1501 message: error.to_string(),
1502 }
1503 })?;
1504 let prepared = MssqlPreparedTarget::from_table_name(
1505 &output_plan,
1506 staging_table,
1507 MssqlPreparedTargetAction::CreatedStagingTable,
1508 )?;
1509 let mut client = RecordingLifecycleClient::default();
1510
1511 let cleanup =
1512 cleanup_mssql_prepared_target(&output_plan, Some(&prepared), &mut client).await?;
1513
1514 assert_eq!(cleanup, MssqlTargetCleanupStatus::Succeeded);
1515 assert_eq!(
1516 client.calls,
1517 vec!["execute DROP TABLE [dbo].[orders__df_replace_0]".to_owned()]
1518 );
1519 Ok(())
1520 }
1521
1522 #[tokio::test]
1523 async fn cleanup_errors_map_to_cleanup_phase() -> Result<(), DeltaFunnelError> {
1524 let output_plan = output_plan(
1525 "orders_output",
1526 LoadMode::CreateAndLoad,
1527 MssqlTargetTable::new("dbo", "orders")?,
1528 )?;
1529 let prepared = MssqlPreparedTarget::from_output_plan(
1530 &output_plan,
1531 MssqlPreparedTargetAction::CreatedTable,
1532 )?;
1533 let mut client = RecordingLifecycleClient {
1534 execute_statement_error: Some("DROP failed\nfor test".to_owned()),
1535 ..RecordingLifecycleClient::default()
1536 };
1537
1538 let error = cleanup_mssql_prepared_target(&output_plan, Some(&prepared), &mut client)
1539 .await
1540 .err()
1541 .ok_or_else(|| DeltaFunnelError::Config {
1542 message: "expected cleanup error".to_owned(),
1543 })?;
1544
1545 assert_cleanup_error(error, r"DROP failed\nfor test")?;
1546 assert_eq!(
1547 client.calls,
1548 vec!["execute DROP TABLE [dbo].[orders]".to_owned()]
1549 );
1550 Ok(())
1551 }
1552
1553 #[tokio::test]
1554 async fn cleanup_rejects_prepared_target_from_different_output_without_drop()
1555 -> Result<(), DeltaFunnelError> {
1556 let current_output_plan = output_plan(
1557 "orders_output",
1558 LoadMode::CreateAndLoad,
1559 MssqlTargetTable::new("dbo", "orders")?,
1560 )?;
1561 let other_output_plan = output_plan(
1562 "other_output",
1563 LoadMode::CreateAndLoad,
1564 MssqlTargetTable::new("dbo", "other_orders")?,
1565 )?;
1566 let prepared = MssqlPreparedTarget::from_output_plan(
1567 &other_output_plan,
1568 MssqlPreparedTargetAction::CreatedTable,
1569 )?;
1570 let mut client = RecordingLifecycleClient::default();
1571
1572 let error =
1573 cleanup_mssql_prepared_target(¤t_output_plan, Some(&prepared), &mut client)
1574 .await
1575 .err()
1576 .ok_or_else(|| DeltaFunnelError::Config {
1577 message: "expected mismatched cleanup target error".to_owned(),
1578 })?;
1579
1580 assert_cleanup_error(error, "prepared target does not match output plan")?;
1581 assert!(client.calls.is_empty());
1582 Ok(())
1583 }
1584
1585 #[test]
1586 fn qualified_target_table_converts_to_arrow_tiberius_table_name() -> Result<(), DeltaFunnelError>
1587 {
1588 let table = MssqlTargetTable::new("dbo", "orders")?;
1589
1590 let table_name = table_name_from_target("orders_output", &table)?;
1591
1592 assert_eq!(table_name.quoted_sql(), "[dbo].[orders]");
1593 Ok(())
1594 }
1595
1596 #[test]
1597 fn unqualified_target_table_keeps_identifier_unqualified() -> Result<(), DeltaFunnelError> {
1598 let table = MssqlTargetTable::unqualified("orders")?;
1599
1600 let table_name = table_name_from_target("orders_output", &table)?;
1601
1602 assert_eq!(table_name.quoted_sql(), "[orders]");
1603 Ok(())
1604 }
1605
1606 #[test]
1607 fn special_target_identifiers_use_arrow_tiberius_escaping() -> Result<(), DeltaFunnelError> {
1608 let table = MssqlTargetTable::new("dbo.part", "target]part")?;
1609
1610 let table_name = table_name_from_target("orders_output", &table)?;
1611
1612 assert_eq!(table_name.quoted_sql(), "[dbo.part].[target]]part]");
1613 Ok(())
1614 }
1615
1616 #[test]
1617 fn append_existing_prepared_target_reports_verified_existing() -> Result<(), DeltaFunnelError> {
1618 let output_plan = output_plan(
1619 "orders_output",
1620 LoadMode::AppendExisting,
1621 MssqlTargetTable::new("dbo", "orders")?,
1622 )?;
1623
1624 let prepared = MssqlPreparedTarget::from_output_plan(
1625 &output_plan,
1626 MssqlPreparedTargetAction::VerifiedExisting,
1627 )?;
1628 let report = prepared.report();
1629
1630 assert_eq!(prepared.quoted_table_sql(), "[dbo].[orders]");
1631 assert_eq!(report.output_name(), "orders_output");
1632 assert_eq!(report.target_table().schema(), Some("dbo"));
1633 assert_eq!(report.target_table().table(), "orders");
1634 assert_eq!(report.load_mode(), LoadMode::AppendExisting);
1635 assert_eq!(
1636 report.connection_source(),
1637 MssqlConnectionSource::ContextDefault
1638 );
1639 assert_eq!(
1640 report.connection().display_label(),
1641 Some("warehouse-primary")
1642 );
1643 assert_eq!(
1644 report.expected_target_state(),
1645 MssqlTargetTableState::Exists
1646 );
1647 assert_eq!(report.action(), MssqlPreparedTargetAction::VerifiedExisting);
1648 assert_eq!(report.cleanup(), MssqlTargetCleanupStatus::NotApplicable);
1649 Ok(())
1650 }
1651
1652 #[test]
1653 fn create_and_load_prepared_target_reports_created_table() -> Result<(), DeltaFunnelError> {
1654 let output_plan = output_plan(
1655 "orders_output",
1656 LoadMode::CreateAndLoad,
1657 MssqlTargetTable::unqualified("orders")?,
1658 )?;
1659
1660 let prepared = MssqlPreparedTarget::from_output_plan(
1661 &output_plan,
1662 MssqlPreparedTargetAction::CreatedTable,
1663 )?;
1664 let report = prepared.report();
1665
1666 assert_eq!(prepared.table_name().quoted_sql(), "[orders]");
1667 assert_eq!(report.load_mode(), LoadMode::CreateAndLoad);
1668 assert_eq!(
1669 report.expected_target_state(),
1670 MssqlTargetTableState::Absent
1671 );
1672 assert_eq!(report.action(), MssqlPreparedTargetAction::CreatedTable);
1673 assert_eq!(report.cleanup(), MssqlTargetCleanupStatus::NotAttempted);
1674 Ok(())
1675 }
1676
1677 #[test]
1678 fn replace_prepared_target_reports_created_staging_table() -> Result<(), DeltaFunnelError> {
1679 let output_plan = output_plan(
1680 "orders_output",
1681 LoadMode::Replace,
1682 MssqlTargetTable::unqualified("orders")?,
1683 )?;
1684 let staging_table = TableName::unqualified("orders__df_replace_0").map_err(|error| {
1685 DeltaFunnelError::Config {
1686 message: error.to_string(),
1687 }
1688 })?;
1689
1690 let prepared = MssqlPreparedTarget::from_table_name(
1691 &output_plan,
1692 staging_table,
1693 MssqlPreparedTargetAction::CreatedStagingTable,
1694 )?;
1695 let report = prepared.report();
1696
1697 assert_eq!(prepared.table_name().quoted_sql(), "[orders__df_replace_0]");
1698 assert_eq!(report.target_table().table(), "orders");
1699 assert_eq!(report.load_mode(), LoadMode::Replace);
1700 assert_eq!(
1701 report.expected_target_state(),
1702 MssqlTargetTableState::ExistsOrAbsent
1703 );
1704 assert_eq!(
1705 report.action(),
1706 MssqlPreparedTargetAction::CreatedStagingTable
1707 );
1708 assert_eq!(report.cleanup(), MssqlTargetCleanupStatus::NotAttempted);
1709 Ok(())
1710 }
1711
1712 #[test]
1713 fn replace_swap_sql_uses_transactional_rename_and_escaped_literals()
1714 -> Result<(), DeltaFunnelError> {
1715 let final_table = test_table_name("dbo", "orders")?;
1716 let staging_table = test_table_name("dbo", "orders__df_replace_0")?;
1717 let backup_table = test_table_name("dbo", "orders__df_backup_0")?;
1718
1719 let sql = replace_swap_sql(&final_table, &staging_table, &backup_table);
1720
1721 assert!(sql.starts_with("SET XACT_ABORT ON;"));
1722 assert!(sql.contains("BEGIN TRANSACTION;"));
1723 assert!(sql.contains("sys.sp_getapplock"));
1724 assert!(sql.contains("@LockOwner = 'Transaction'"));
1725 assert!(sql.contains("@LockTimeout = 0"));
1726 assert!(sql.contains("ROLLBACK TRANSACTION;"));
1727 assert!(!sql.contains("replace target table is missing"));
1728 assert!(sql.contains("IF OBJECT_ID(N'[dbo].[orders]', N'U') IS NOT NULL"));
1729 assert!(sql.contains("ELSE"));
1730 assert!(sql.contains(
1731 "EXEC sys.sp_rename @objname = N'[dbo].[orders]', @newname = N'orders__df_backup_0', @objtype = 'OBJECT';"
1732 ));
1733 assert!(sql.contains(
1734 "EXEC sys.sp_rename @objname = N'[dbo].[orders__df_replace_0]', @newname = N'orders', @objtype = 'OBJECT';"
1735 ));
1736 assert!(sql.contains("DROP TABLE [dbo].[orders__df_backup_0];"));
1737 assert_eq!(
1738 sql_nvarchar_literal("tenant's.orders"),
1739 "N'tenant''s.orders'"
1740 );
1741 Ok(())
1742 }
1743
1744 #[test]
1745 fn replace_lock_resource_keeps_long_target_names_distinct() -> Result<(), DeltaFunnelError> {
1746 let schema = "s".repeat(128);
1747 let table_prefix = "t".repeat(127);
1748 let first_table = test_table_name(&schema, &format!("{table_prefix}a"))?;
1749 let second_table = test_table_name(&schema, &format!("{table_prefix}b"))?;
1750
1751 let first_lock = replace_lock_resource(&first_table);
1752 let second_lock = replace_lock_resource(&second_table);
1753
1754 assert_eq!(first_lock.chars().count(), 255);
1755 assert_eq!(second_lock.chars().count(), 255);
1756 assert_ne!(first_lock, second_lock);
1757 Ok(())
1758 }
1759
1760 #[tokio::test]
1761 async fn replace_swap_skips_existing_backup_name_and_executes_batch()
1762 -> Result<(), DeltaFunnelError> {
1763 let output_plan = output_plan(
1764 "orders_output",
1765 LoadMode::Replace,
1766 MssqlTargetTable::new("dbo", "orders")?,
1767 )?;
1768 let prepared = replace_staging_prepared_target(&output_plan, "orders__df_replace_0")?;
1769 let mut client = RecordingLifecycleClient {
1770 existence_results: vec![true, false],
1771 rows_affected: vec![0],
1772 ..RecordingLifecycleClient::default()
1773 };
1774
1775 swap_mssql_replace_target(&output_plan, &prepared, &mut client).await?;
1776
1777 assert_eq!(client.calls.len(), 3);
1778 assert_eq!(client.calls[0], "probe [dbo].[orders__df_backup_0]");
1779 assert_eq!(client.calls[1], "probe [dbo].[orders__df_backup_1]");
1780 assert!(client.calls[2].contains(
1781 "EXEC sys.sp_rename @objname = N'[dbo].[orders]', @newname = N'orders__df_backup_1'"
1782 ));
1783 assert!(client.calls[2].contains("DROP TABLE [dbo].[orders__df_backup_1];"));
1784 Ok(())
1785 }
1786
1787 #[tokio::test]
1788 async fn replace_swap_maps_execute_errors_to_swap_phase() -> Result<(), DeltaFunnelError> {
1789 let output_plan = output_plan(
1790 "orders_output",
1791 LoadMode::Replace,
1792 MssqlTargetTable::new("dbo", "orders")?,
1793 )?;
1794 let prepared = replace_staging_prepared_target(&output_plan, "orders__df_replace_0")?;
1795 let mut client = RecordingLifecycleClient {
1796 existence_results: vec![false],
1797 execute_statement_error: Some("swap failed\nfor test".to_owned()),
1798 ..RecordingLifecycleClient::default()
1799 };
1800
1801 let error = swap_mssql_replace_target(&output_plan, &prepared, &mut client)
1802 .await
1803 .err()
1804 .ok_or_else(|| DeltaFunnelError::Config {
1805 message: "expected swap execution error".to_owned(),
1806 })?;
1807
1808 assert_swap_target_error(error, r"swap failed\nfor test", true)?;
1809 assert_eq!(client.calls.len(), 2);
1810 assert_eq!(client.calls[0], "probe [dbo].[orders__df_backup_0]");
1811 assert!(client.calls[1].starts_with("execute SET XACT_ABORT ON;"));
1812 Ok(())
1813 }
1814
1815 #[tokio::test]
1816 async fn replace_swap_rejects_mismatched_prepared_target_without_sql()
1817 -> Result<(), DeltaFunnelError> {
1818 let current_output_plan = output_plan(
1819 "orders_output",
1820 LoadMode::Replace,
1821 MssqlTargetTable::new("dbo", "orders")?,
1822 )?;
1823 let other_output_plan = output_plan(
1824 "other_output",
1825 LoadMode::Replace,
1826 MssqlTargetTable::new("dbo", "other_orders")?,
1827 )?;
1828 let prepared =
1829 replace_staging_prepared_target(&other_output_plan, "other_orders__df_replace_0")?;
1830 let mut client = RecordingLifecycleClient::default();
1831
1832 let error = swap_mssql_replace_target(¤t_output_plan, &prepared, &mut client)
1833 .await
1834 .err()
1835 .ok_or_else(|| DeltaFunnelError::Config {
1836 message: "expected mismatched swap target error".to_owned(),
1837 })?;
1838
1839 assert_swap_target_error(error, "prepared target does not match output plan", false)?;
1840 assert!(client.calls.is_empty());
1841 Ok(())
1842 }
1843
1844 #[test]
1845 fn prepared_target_action_must_match_load_mode() -> Result<(), DeltaFunnelError> {
1846 let output_plan = output_plan(
1847 "orders_output",
1848 LoadMode::AppendExisting,
1849 MssqlTargetTable::new("dbo", "orders")?,
1850 )?;
1851
1852 let error = MssqlPreparedTarget::from_output_plan(
1853 &output_plan,
1854 MssqlPreparedTargetAction::CreatedTable,
1855 )
1856 .err()
1857 .ok_or_else(|| DeltaFunnelError::Config {
1858 message: "expected mismatched prepared target action error".to_owned(),
1859 })?;
1860
1861 assert!(matches!(
1862 error,
1863 DeltaFunnelError::MssqlLifecyclePlanning { .. }
1864 ));
1865 assert!(error.to_string().contains("does not match load mode"));
1866 Ok(())
1867 }
1868
1869 #[test]
1870 fn prepared_target_reports_do_not_expose_connection_secrets() -> Result<(), DeltaFunnelError> {
1871 let output_plan = output_plan(
1872 "orders_output",
1873 LoadMode::CreateAndLoad,
1874 MssqlTargetTable::new("dbo", "orders")?,
1875 )?;
1876
1877 let prepared = MssqlPreparedTarget::from_output_plan(
1878 &output_plan,
1879 MssqlPreparedTargetAction::CreatedTable,
1880 )?;
1881 let combined = format!("{output_plan:?}\n{prepared:?}");
1882
1883 assert!(!combined.contains("secret-token"));
1884 assert!(!combined.contains("password"));
1885 assert!(!combined.contains("server=tcp"));
1886 assert!(combined.contains("warehouse-primary"));
1887 Ok(())
1888 }
1889
1890 fn assert_prepare_target_lifecycle_error(
1891 error: DeltaFunnelError,
1892 expected_message: &str,
1893 expected_cleanup: MssqlTargetCleanupStatus,
1894 ) -> Result<(), DeltaFunnelError> {
1895 let display = error.to_string();
1896 assert!(display.contains("orders_output"));
1897 assert!(display.contains("prepare target lifecycle"));
1898 assert!(display.contains(expected_message));
1899 assert!(!display.contains("secret-token"));
1900 assert!(!display.contains("server=tcp"));
1901 let DeltaFunnelError::MssqlWritePhase { context, .. } = error else {
1902 return Err(DeltaFunnelError::Config {
1903 message: "expected MssqlWritePhase error".to_owned(),
1904 });
1905 };
1906
1907 assert_eq!(context.phase(), MssqlWritePhase::PrepareTargetLifecycle);
1908 assert_eq!(context.output_name(), "orders_output");
1909 assert_eq!(context.stats().rows_written(), 0);
1910 assert_eq!(context.stats().batches_written(), 0);
1911 assert!(!context.partial_write_possible());
1912 assert_eq!(context.cleanup(), expected_cleanup);
1913 Ok(())
1914 }
1915
1916 fn assert_cleanup_error(
1917 error: DeltaFunnelError,
1918 expected_message: &str,
1919 ) -> Result<(), DeltaFunnelError> {
1920 let display = error.to_string();
1921 assert!(display.contains("orders_output"));
1922 assert!(display.contains("cleanup"));
1923 assert!(display.contains(expected_message));
1924 assert!(!display.contains("secret-token"));
1925 assert!(!display.contains("server=tcp"));
1926 let DeltaFunnelError::MssqlWritePhase { context, .. } = error else {
1927 return Err(DeltaFunnelError::Config {
1928 message: "expected MssqlWritePhase error".to_owned(),
1929 });
1930 };
1931
1932 assert_eq!(context.phase(), MssqlWritePhase::Cleanup);
1933 assert_eq!(context.output_name(), "orders_output");
1934 assert_eq!(context.stats().rows_written(), 0);
1935 assert_eq!(context.stats().batches_written(), 0);
1936 assert!(!context.partial_write_possible());
1937 assert_eq!(context.cleanup(), MssqlTargetCleanupStatus::Failed);
1938 Ok(())
1939 }
1940
1941 fn assert_swap_target_error(
1942 error: DeltaFunnelError,
1943 expected_message: &str,
1944 expected_partial_write_possible: bool,
1945 ) -> Result<(), DeltaFunnelError> {
1946 let display = error.to_string();
1947 assert!(display.contains("orders_output"));
1948 assert!(display.contains("swap target"));
1949 assert!(display.contains(expected_message));
1950 assert!(!display.contains("secret-token"));
1951 assert!(!display.contains("server=tcp"));
1952 let DeltaFunnelError::MssqlWritePhase { context, .. } = error else {
1953 return Err(DeltaFunnelError::Config {
1954 message: "expected MssqlWritePhase error".to_owned(),
1955 });
1956 };
1957
1958 assert_eq!(context.phase(), MssqlWritePhase::SwapTarget);
1959 assert_eq!(context.output_name(), "orders_output");
1960 assert_eq!(context.stats().rows_written(), 0);
1961 assert_eq!(context.stats().batches_written(), 0);
1962 assert_eq!(
1963 context.partial_write_possible(),
1964 expected_partial_write_possible
1965 );
1966 assert_eq!(context.cleanup(), MssqlTargetCleanupStatus::NotAttempted);
1967 Ok(())
1968 }
1969}