delta_funnel/
lib.rs

1//! Core library for Delta Funnel.
2//!
3//! This crate will own the high-level export orchestration from table formats
4//! such as Delta Lake into Microsoft SQL Server. Low-level Arrow to TDS bulk
5//! loading is expected to stay in `arrow-tiberius`.
6//!
7//! Module boundaries should follow the load workflow. `table_formats` owns
8//! upstream table-format integrations, starting with Delta source configuration
9//! and snapshot loading. Later DataFusion provider, query execution, SQL Server
10//! sink, and orchestration work should land in their own modules when the first
11//! real implementation slice needs them.
12
13pub mod error;
14mod observability;
15mod orchestrator;
16mod pipeline;
17pub(crate) mod query_engine;
18mod report;
19mod sql_server;
20mod support;
21mod table_formats;
22
23pub use error::{DeltaFunnelError, SqlTablePhase};
24pub use orchestrator::{
25    DeltaFunnelRuntime, DeltaFunnelSession, LazyTable, LazyTableKind, MssqlOutputTarget,
26    OutputWritePlan, PlannedMssqlOutput, RegisteredDerivedTable, RegisteredSessionSource, RunMode,
27    SessionOptions, WriteAllCacheMode, WriteAllOptions,
28};
29pub use pipeline::{
30    BatchHandoffError, BatchHandoffOutcome, BatchHandoffStats, BatchPipelinePhase,
31    RecordBatchConsumer, handoff_datafusion_query_output, handoff_record_batch_stream,
32};
33pub use query_engine::{
34    DeltaProviderReadStatsSnapshot, DeltaProviderReaderBackend, DeltaProviderScanExecutionOptions,
35    DeltaScanPartitionTargetDiagnosticInput, DeltaScanPartitionTargetDiagnosticOutput,
36    DeltaScanPartitionTargetDiagnosticSource, DeltaScanPartitionTargetLocalEnvironmentDiagnostic,
37    DeltaScanPartitionTargetLocalUnixFileDescriptorLimitStatus, DeltaTableProviderConfig,
38    QueryOptions, RegisteredDeltaSource, RegisteredDeltaSources, collect_delta_provider_read_stats,
39    datafusion_query_output_stream, datafusion_session_config, datafusion_session_context,
40    delta_scan_partition_target_local_environment_diagnostic,
41    derive_delta_scan_partition_target_diagnostic, register_delta_sources,
42    register_delta_sources_with_scan_execution_options,
43};
44pub use report::{
45    DeltaProtocolReport, DeltaProviderSchedulingReport, DeltaSourceReport, DryRunScanSummaryMode,
46    FileCount, FileCountKind, MssqlDryRunOutputFieldReport, MssqlDryRunOutputReport,
47    MssqlDryRunSqlIdentityReport, MssqlDryRunSqlIdentityState, MssqlDryRunWorkflowReport,
48    OutputStatus, OutputStatusKind, PhaseStatus, PhaseStatusKind, PhaseTimingReport,
49    ReportReasonCode, RowCount, RowCountKind, SourceUsageStatus, TargetValidationMode,
50    ValidationOptions, ValidationStatus, ValidationStatusKind, WorkflowStatus, WorkflowStatusKind,
51    WriteAllCacheAliasReport, WriteAllCacheAliasStatus, WriteAllCacheCandidateSkip,
52    WriteAllCacheCandidateSkipReason, WriteAllCacheReport, WriteAllNoCacheReason, WriteAllReport,
53    duration_to_micros_saturating, u128_to_u64_saturating, usize_to_u64_saturating,
54};
55pub use sql_server::{
56    LoadMode, MssqlBatchShapingReport, MssqlBinaryPolicy, MssqlConnectionConfig,
57    MssqlConnectionSource, MssqlConnectionSummary, MssqlDate64Policy, MssqlDdlPlan,
58    MssqlDecimal256Policy, MssqlDecimalPolicy, MssqlFloatPolicy, MssqlLifecycleExecutionGuardrail,
59    MssqlLifecycleGuardrailPolicy, MssqlLifecyclePlan, MssqlNanosecondPolicy,
60    MssqlOutputBatchStream, MssqlOutputBatchStreamFactory, MssqlOutputBatchValidationReport,
61    MssqlOutputFieldReport, MssqlOutputWriteJob, MssqlOutputWriteStatus, MssqlPreparedTarget,
62    MssqlPreparedTargetAction, MssqlPreparedTargetReport, MssqlSchemaDiagnostic,
63    MssqlSchemaDiagnosticField, MssqlSchemaPlan, MssqlSchemaPlanOptions, MssqlStringPolicy,
64    MssqlTableName, MssqlTargetCleanupStatus, MssqlTargetConfig, MssqlTargetOutputPlan,
65    MssqlTargetResolutionContext, MssqlTargetSummary, MssqlTargetTable, MssqlTargetTableState,
66    MssqlTimestampPolicy, MssqlTimezonePolicy, MssqlUInt64Policy, MssqlWorkflowWriteOptions,
67    MssqlWorkflowWriteReport, MssqlWriteBackend, MssqlWriteFailureContext, MssqlWriteFailureReport,
68    MssqlWritePhase, MssqlWriteReport, MssqlWriteSkippedReason, MssqlWriteSkippedReport,
69    MssqlWriteStats, ResolvedMssqlTarget, connect_mssql_client_from_ado_string,
70    default_mssql_write_backend, mssql_schema_diagnostic_reports,
71    mssql_write_backend_for_output_plan, plan_mssql_create_table_ddl, plan_mssql_lifecycle,
72    plan_mssql_output_schema, plan_mssql_target_for_output, plan_mssql_target_for_resolved_output,
73    plan_mssql_target_output, validate_mssql_output_record_batch, validate_mssql_output_schema,
74    write_mssql_outputs_to_mssql, write_output_batches_to_mssql,
75};
76pub(crate) use sql_server::{
77    MssqlWorkflowOutputWriter, write_mssql_outputs_with_writer,
78    write_output_batches_to_mssql_with_validation_options,
79};
80pub use support::sanitize_uri_for_display;
81pub use table_formats::{
82    DeltaSourceConfig, DeltaStorageOptions, PlannedDeltaSource, ProtocolPreflight,
83    load_delta_source, load_delta_sources, preflight_delta_protocol, preflight_delta_sources,
84};
85
86/// Current crate version.
87pub const VERSION: &str = env!("CARGO_PKG_VERSION");
88
89/// Returns the current crate version.
90#[must_use]
91pub fn version() -> &'static str {
92    VERSION
93}
delta_funnel/lib.rs

delta_funnel/
lib.rs
