use crate::BatchPipelinePhase;
use crate::support::{sanitize_text_for_display, sanitize_uri_for_display};
use snafu::Snafu;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DeltaScanFileReadPhase {
TableUriParsing,
FileMetadataConversion,
FilePathResolution,
ObjectStoreEngineConstruction,
ParquetReadSetup,
ParquetBatchRead,
RowIndexGeneration,
PredicateEvaluation,
ArrowConversion,
TransformApplication,
UnsupportedReadMode,
DeletionVectorPredicateRejection,
DeletionVectorMasking,
}
impl std::fmt::Display for DeltaScanFileReadPhase {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str(match self {
Self::TableUriParsing => "table URI parsing",
Self::FileMetadataConversion => "file metadata conversion",
Self::FilePathResolution => "file path resolution",
Self::ObjectStoreEngineConstruction => "object store engine construction",
Self::ParquetReadSetup => "Parquet read setup",
Self::ParquetBatchRead => "Parquet batch read",
Self::RowIndexGeneration => "row-index generation",
Self::PredicateEvaluation => "physical predicate evaluation",
Self::ArrowConversion => "Arrow conversion",
Self::TransformApplication => "physical-to-logical transform application",
Self::UnsupportedReadMode => "unsupported read mode",
Self::DeletionVectorPredicateRejection => "deletion-vector predicate read rejection",
Self::DeletionVectorMasking => "deletion-vector masking",
})
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DeltaScanDeletionVectorPhase {
TableUriParsing,
ObjectStoreEngineConstruction,
DescriptorAccess,
PayloadRead,
SelectionVectorLengthMismatch,
SelectionVectorExhaustion,
}
impl std::fmt::Display for DeltaScanDeletionVectorPhase {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str(match self {
Self::TableUriParsing => "table URI parsing",
Self::ObjectStoreEngineConstruction => "object store engine construction",
Self::DescriptorAccess => "deletion-vector descriptor access",
Self::PayloadRead => "deletion-vector payload read",
Self::SelectionVectorLengthMismatch => "selection-vector length mismatch",
Self::SelectionVectorExhaustion => "selection-vector exhaustion",
})
}
}
#[derive(Debug, Snafu)]
#[snafu(visibility(pub(crate)))]
pub enum DeltaFunnelError {
#[snafu(display("configuration error: {message}"))]
Config {
message: String,
},
#[snafu(display(
"invalid Delta source name `{}`: {reason}",
sanitize_source_name_for_display(name)
))]
InvalidSourceName {
name: String,
reason: &'static str,
},
#[snafu(display(
"duplicate Delta source name `{}`",
sanitize_source_name_for_display(name)
))]
DuplicateSourceName {
name: String,
},
#[snafu(display("invalid Delta source URI: {reason}"))]
InvalidSourceUri {
reason: &'static str,
},
#[snafu(display("Delta source engine error: {reason}"))]
DeltaSourceEngine {
reason: &'static str,
},
#[snafu(display("Delta snapshot load error: {reason}"))]
DeltaSnapshotLoad {
reason: String,
},
#[snafu(display(
"Delta protocol compatibility error for source `{}` at snapshot version {snapshot_version} ({}): {reason}",
sanitize_source_name_for_display(source_name),
sanitize_uri_for_display(table_uri)
))]
DeltaProtocolCompatibility {
source_name: String,
table_uri: String,
snapshot_version: u64,
reason: String,
},
#[snafu(display(
"Delta source schema error for source `{}` ({}): {}",
sanitize_source_name_for_display(source_name),
sanitize_uri_for_display(table_uri),
sanitize_reason_for_display(reason)
))]
DeltaSourceSchema {
source_name: String,
table_uri: String,
reason: String,
},
#[snafu(display(
"DataFusion registration error for source `{}` ({}): {}",
sanitize_source_name_for_display(source_name),
sanitize_uri_for_display(table_uri),
sanitize_reason_for_display(reason)
))]
DataFusionRegistration {
source_name: String,
table_uri: String,
reason: String,
},
#[snafu(display(
"SQL table error during {phase}: {}",
sanitize_reason_for_display(message)
))]
SqlTable {
phase: SqlTablePhase,
message: String,
},
#[snafu(display(
"Delta scan projection error for source `{}` ({}): {}",
sanitize_source_name_for_display(source_name),
sanitize_uri_for_display(table_uri),
sanitize_reason_for_display(reason)
))]
DeltaScanProjection {
source_name: String,
table_uri: String,
reason: String,
},
#[snafu(display(
"Delta scan filter error for source `{}` ({}): {}",
sanitize_source_name_for_display(source_name),
sanitize_uri_for_display(table_uri),
sanitize_reason_for_display(reason)
))]
DeltaScanFilter {
source_name: String,
table_uri: String,
reason: String,
},
#[snafu(display(
"Delta scan construction error for source `{}` ({}): {}",
sanitize_source_name_for_display(source_name),
sanitize_uri_for_display(table_uri),
sanitize_reason_for_display(&source.to_string())
))]
DeltaScanConstruction {
source_name: String,
table_uri: String,
#[snafu(source(from(delta_kernel::Error, Box::new)))]
source: Box<delta_kernel::Error>,
},
#[snafu(display(
"Delta scan metadata expansion error for source `{}` at snapshot version {snapshot_version} ({}): {}",
sanitize_source_name_for_display(source_name),
sanitize_uri_for_display(table_uri),
sanitize_reason_for_display(&source.to_string())
))]
DeltaScanMetadataExpansion {
source_name: String,
table_uri: String,
snapshot_version: u64,
#[snafu(source(from(delta_kernel::Error, Box::new)))]
source: Box<delta_kernel::Error>,
},
#[snafu(display(
"Delta scan file task planning error for source `{}` at snapshot version {snapshot_version} ({}), file `{}`: {}",
sanitize_source_name_for_display(source_name),
sanitize_uri_for_display(table_uri),
sanitize_reason_for_display(path),
sanitize_reason_for_display(reason)
))]
DeltaScanFileTaskPlanning {
source_name: String,
table_uri: String,
snapshot_version: u64,
path: String,
reason: String,
},
#[snafu(display(
"Delta scan file task partition planning error for source `{}` at snapshot version {snapshot_version} ({}): {}",
sanitize_source_name_for_display(source_name),
sanitize_uri_for_display(table_uri),
sanitize_reason_for_display(reason)
))]
DeltaScanFileTaskPartitionPlanning {
source_name: String,
table_uri: String,
snapshot_version: u64,
reason: String,
},
#[snafu(display(
"Delta scan file read error for source `{}` at snapshot version {snapshot_version} ({}), file `{}` during {phase}: {}",
sanitize_source_name_for_display(source_name),
sanitize_uri_for_display(table_uri),
sanitize_reason_for_display(path),
sanitize_reason_for_display(&source.to_string())
))]
DeltaScanFileRead {
source_name: String,
table_uri: String,
snapshot_version: u64,
path: String,
phase: DeltaScanFileReadPhase,
#[snafu(source(from(delta_kernel::Error, Box::new)))]
source: Box<delta_kernel::Error>,
},
#[snafu(display(
"Delta scan deletion-vector error for source `{}` at snapshot version {snapshot_version} ({}), file `{}` during {phase}: {}",
sanitize_source_name_for_display(source_name),
sanitize_uri_for_display(table_uri),
sanitize_reason_for_display(path),
sanitize_reason_for_display(&source.to_string())
))]
DeltaScanDeletionVector {
source_name: String,
table_uri: String,
snapshot_version: u64,
path: String,
phase: DeltaScanDeletionVectorPhase,
#[snafu(source(from(delta_kernel::Error, Box::new)))]
source: Box<delta_kernel::Error>,
},
#[snafu(display("dependency compatibility error: {message}"))]
DependencyCompatibility {
message: String,
},
#[snafu(display(
"batch pipeline {phase} error for option `{option}`: {}",
sanitize_reason_for_display(message)
))]
BatchPipeline {
phase: BatchPipelinePhase,
option: &'static str,
message: String,
},
#[snafu(display(
"MSSQL target configuration error for option `{option}`: {}",
sanitize_reason_for_display(message)
))]
MssqlTargetConfig {
option: &'static str,
message: String,
},
#[snafu(display(
"MSSQL target for output `{}` has no effective connection",
sanitize_text_for_display(output_name)
))]
MissingMssqlConnection {
output_name: String,
},
#[snafu(display(
"MSSQL schema planning error for output `{}`: {reason}",
sanitize_text_for_display(output_name)
))]
InvalidMssqlOutputIdentity {
output_name: String,
reason: &'static str,
},
#[snafu(display(
"MSSQL schema planning error for output `{}`: duplicate field name `{}` at indexes {first_index} and {duplicate_index}",
sanitize_text_for_display(output_name),
sanitize_text_for_display(field_name)
))]
DuplicateMssqlOutputField {
output_name: String,
field_name: String,
first_index: usize,
duplicate_index: usize,
},
#[snafu(display(
"MSSQL schema planning error for output `{}`: arrow-tiberius returned {} diagnostic(s)",
sanitize_text_for_display(output_name),
diagnostics.len()
))]
MssqlSchemaPlanning {
output_name: String,
diagnostics: arrow_tiberius::DiagnosticSet,
},
#[snafu(display(
"MSSQL schema planning error for output `{}`: {}",
sanitize_text_for_display(output_name),
sanitize_reason_for_display(&source.to_string())
))]
MssqlSchemaPlanningFailed {
output_name: String,
source: arrow_tiberius::Error,
},
#[snafu(display(
"MSSQL DDL planning error for output `{}`: {}",
sanitize_text_for_display(output_name),
sanitize_reason_for_display(&source.to_string())
))]
MssqlDdlTargetIdentifier {
output_name: String,
source: arrow_tiberius::Error,
},
#[snafu(display(
"MSSQL DDL planning error for output `{}`: {}",
sanitize_text_for_display(output_name),
sanitize_reason_for_display(message)
))]
MssqlDdlPlanning {
output_name: String,
message: String,
},
#[snafu(display(
"MSSQL lifecycle planning error for output `{}`: {}",
sanitize_text_for_display(output_name),
sanitize_reason_for_display(message)
))]
MssqlLifecyclePlanning {
output_name: String,
message: String,
},
#[snafu(display(
"MSSQL write error: {}",
sanitize_reason_for_display(&source.to_string())
))]
MssqlWrite {
source: arrow_tiberius::Error,
},
#[snafu(display(
"MSSQL write error for output `{}` during {}: {}",
sanitize_text_for_display(context.output_name()),
context.phase(),
sanitize_reason_for_display(message)
))]
MssqlWritePhase {
context: Box<crate::MssqlWriteFailureContext>,
message: String,
},
#[snafu(display(
"MSSQL write error for output `{}` during {}: {}",
sanitize_text_for_display(context.output_name()),
context.phase(),
sanitize_reason_for_display(&source.to_string())
))]
MssqlBatchSchemaValidation {
context: Box<crate::MssqlWriteFailureContext>,
source: arrow_tiberius::Error,
},
#[snafu(display(
"MSSQL workflow planning error: {}",
sanitize_reason_for_display(message)
))]
MssqlWorkflowPlanning {
message: String,
},
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SqlTablePhase {
ValidateSql,
PlanSql,
RegisterDerivedAlias,
}
impl std::fmt::Display for SqlTablePhase {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter.write_str(match self {
Self::ValidateSql => "SQL validation",
Self::PlanSql => "SQL planning",
Self::RegisterDerivedAlias => "derived alias registration",
})
}
}
fn sanitize_source_name_for_display(name: &str) -> String {
sanitize_text_for_display(name)
}
fn sanitize_reason_for_display(reason: &str) -> String {
sanitize_text_for_display(reason)
}
#[cfg(test)]
mod tests {
use std::error::Error;
use super::DeltaFunnelError;
#[test]
fn config_error_has_sanitized_display() {
let error = DeltaFunnelError::Config {
message: "max_concurrent_file_reads_per_scan must be greater than zero".to_owned(),
};
assert_eq!(
error.to_string(),
"configuration error: max_concurrent_file_reads_per_scan must be greater than zero"
);
}
#[test]
fn dependency_error_has_sanitized_display() {
let error = DeltaFunnelError::DependencyCompatibility {
message: "delta_kernel API smoke test failed".to_owned(),
};
assert_eq!(
error.to_string(),
"dependency compatibility error: delta_kernel API smoke test failed"
);
}
#[test]
fn batch_pipeline_error_has_sanitized_display() {
let error = DeltaFunnelError::BatchPipeline {
phase: super::BatchPipelinePhase::Configuration,
option: "output_batch_size",
message: "must be greater than zero".to_owned(),
};
assert_eq!(
error.to_string(),
"batch pipeline configuration error for option `output_batch_size`: must be greater than zero"
);
}
#[test]
fn batch_pipeline_error_display_escapes_control_characters() {
let error = DeltaFunnelError::BatchPipeline {
phase: super::BatchPipelinePhase::HandoffSetup,
option: "consumer_capacity",
message: "invalid\nvalue\tprovided".to_owned(),
};
let display = error.to_string();
assert!(!display.contains('\n'));
assert!(!display.contains('\t'));
assert!(display.contains(r"invalid\nvalue\tprovided"));
}
#[test]
fn mssql_write_error_has_sanitized_display() {
let error = DeltaFunnelError::MssqlWrite {
source: arrow_tiberius::Error::BackendUnavailable {
backend: arrow_tiberius::WriteBackend::DirectRawBulk,
reason: "not available\nfor test".to_owned(),
},
};
let display = error.to_string();
assert!(!display.contains('\n'));
assert!(display.contains(r"not available\nfor test"));
}
#[test]
fn mssql_write_phase_error_has_sanitized_display_and_context() -> Result<(), DeltaFunnelError> {
let connection = crate::MssqlConnectionConfig::new(
"server=tcp:sql.example.com;database=warehouse;user=admin;password=secret-token",
)?
.with_display_label("warehouse-primary");
let target_config =
crate::MssqlTargetConfig::new(crate::MssqlTargetTable::new("dbo", "orders")?);
let schema = arrow_schema::Schema::new(vec![arrow_schema::Field::new(
"order_id",
arrow_schema::DataType::Int64,
false,
)]);
let output_plan = crate::plan_mssql_target_for_output(
schema,
"orders_output",
&target_config,
Some(&connection),
arrow_tiberius::PlanOptions::default(),
)?;
let context = crate::MssqlWriteFailureContext::from_output_plan(
&output_plan,
crate::MssqlWritePhase::WriteBatch,
42,
3,
125,
true,
crate::MssqlTargetCleanupStatus::NotApplicable,
);
let error = DeltaFunnelError::MssqlWritePhase {
context: Box::new(context),
message: "batch failed\nwhile writing".to_owned(),
};
let display = error.to_string();
assert!(display.contains("orders_output"));
assert!(display.contains("write batch"));
assert!(!display.contains('\n'));
assert!(display.contains(r"batch failed\nwhile writing"));
assert!(!display.contains("secret-token"));
assert!(!display.contains("server=tcp"));
let DeltaFunnelError::MssqlWritePhase { context, .. } = error else {
return Err(DeltaFunnelError::Config {
message: "expected MssqlWritePhase error".to_owned(),
});
};
assert_eq!(context.phase(), crate::MssqlWritePhase::WriteBatch);
assert_eq!(context.output_name(), "orders_output");
assert_eq!(context.stats().rows_written(), 42);
assert!(context.partial_write_possible());
Ok(())
}
#[test]
fn mssql_batch_schema_validation_error_has_sanitized_display_and_context()
-> Result<(), DeltaFunnelError> {
let connection = crate::MssqlConnectionConfig::new(
"server=tcp:sql.example.com;database=warehouse;user=admin;password=secret-token",
)?
.with_display_label("warehouse-primary");
let target_config =
crate::MssqlTargetConfig::new(crate::MssqlTargetTable::new("dbo", "orders")?);
let schema = arrow_schema::Schema::new(vec![arrow_schema::Field::new(
"order_id",
arrow_schema::DataType::Int64,
false,
)]);
let output_plan = crate::plan_mssql_target_for_output(
schema,
"orders_output",
&target_config,
Some(&connection),
arrow_tiberius::PlanOptions::default(),
)?;
let context = crate::MssqlWriteFailureContext::from_output_plan(
&output_plan,
crate::MssqlWritePhase::ValidateBatchSchema,
0,
0,
0,
false,
crate::MssqlTargetCleanupStatus::NotApplicable,
);
let error = DeltaFunnelError::MssqlBatchSchemaValidation {
context: Box::new(context),
source: arrow_tiberius::Error::BackendUnavailable {
backend: arrow_tiberius::WriteBackend::DirectRawBulk,
reason: "schema mismatch\nfor test".to_owned(),
},
};
let display = error.to_string();
assert!(display.contains("orders_output"));
assert!(display.contains("validate batch schema"));
assert!(!display.contains('\n'));
assert!(display.contains(r"schema mismatch\nfor test"));
assert!(!display.contains("secret-token"));
assert!(!display.contains("server=tcp"));
let DeltaFunnelError::MssqlBatchSchemaValidation { context, source } = error else {
return Err(DeltaFunnelError::Config {
message: "expected MssqlBatchSchemaValidation error".to_owned(),
});
};
assert_eq!(context.phase(), crate::MssqlWritePhase::ValidateBatchSchema);
assert_eq!(context.output_name(), "orders_output");
assert_eq!(context.stats().rows_written(), 0);
assert!(!context.partial_write_possible());
assert!(matches!(
source,
arrow_tiberius::Error::BackendUnavailable { .. }
));
Ok(())
}
#[test]
fn invalid_source_name_error_has_sanitized_display() {
let error = DeltaFunnelError::InvalidSourceName {
name: "orders.latest".to_owned(),
reason: "source names may contain only ASCII letters, digits, and underscores",
};
assert_eq!(
error.to_string(),
"invalid Delta source name `orders.latest`: source names may contain only ASCII letters, digits, and underscores"
);
}
#[test]
fn invalid_source_name_display_escapes_control_characters() {
let error = DeltaFunnelError::InvalidSourceName {
name: "orders\nlatest\tname".to_owned(),
reason: "source names may contain only ASCII letters, digits, and underscores",
};
let display = error.to_string();
assert!(!display.contains('\n'));
assert!(!display.contains('\t'));
assert!(display.contains(r"orders\nlatest\tname"));
}
#[test]
fn duplicate_source_name_error_has_sanitized_display() {
let error = DeltaFunnelError::DuplicateSourceName {
name: "Orders".to_owned(),
};
assert_eq!(error.to_string(), "duplicate Delta source name `Orders`");
}
#[test]
fn invalid_source_uri_error_has_sanitized_display() {
let error = DeltaFunnelError::InvalidSourceUri {
reason: "table location could not be parsed or normalized",
};
assert_eq!(
error.to_string(),
"invalid Delta source URI: table location could not be parsed or normalized"
);
}
#[test]
fn source_engine_error_has_sanitized_display() {
let error = DeltaFunnelError::DeltaSourceEngine {
reason: "object store engine could not be constructed",
};
assert_eq!(
error.to_string(),
"Delta source engine error: object store engine could not be constructed"
);
}
#[test]
fn snapshot_load_error_has_sanitized_display() {
let error = DeltaFunnelError::DeltaSnapshotLoad {
reason: "snapshot could not be loaded".to_owned(),
};
assert_eq!(
error.to_string(),
"Delta snapshot load error: snapshot could not be loaded"
);
}
#[test]
fn protocol_compatibility_error_has_sanitized_display() {
let error = DeltaFunnelError::DeltaProtocolCompatibility {
source_name: "orders\nlatest".to_owned(),
table_uri: "s3://user:password@example.com/table?token=secret".to_owned(),
snapshot_version: 7,
reason: "unsupported Delta reader feature `deletionVectors`".to_owned(),
};
let display = error.to_string();
assert!(display.contains(r"orders\nlatest"));
assert!(display.contains("snapshot version 7"));
assert!(display.contains("s3://example.com/table"));
assert!(display.contains("deletionVectors"));
assert!(!display.contains('\n'));
assert!(!display.contains("user"));
assert!(!display.contains("password"));
assert!(!display.contains("token"));
assert!(!display.contains("secret"));
}
#[test]
fn source_schema_error_has_sanitized_display() {
let error = DeltaFunnelError::DeltaSourceSchema {
source_name: "orders\nlatest".to_owned(),
table_uri: "s3://user:password@example.com/table?token=secret".to_owned(),
reason: "field\nname could not be converted".to_owned(),
};
let display = error.to_string();
assert!(display.contains(r"orders\nlatest"));
assert!(display.contains("s3://example.com/table"));
assert!(display.contains(r"field\nname"));
assert!(!display.contains('\n'));
assert!(!display.contains("user"));
assert!(!display.contains("password"));
assert!(!display.contains("token"));
assert!(!display.contains("secret"));
}
#[test]
fn datafusion_registration_error_has_sanitized_display() {
let error = DeltaFunnelError::DataFusionRegistration {
source_name: "orders\nlatest".to_owned(),
table_uri: "s3://user:password@example.com/table?token=secret".to_owned(),
reason: "table\nalready exists".to_owned(),
};
let display = error.to_string();
assert!(display.contains(r"orders\nlatest"));
assert!(display.contains("s3://example.com/table"));
assert!(display.contains(r"table\nalready exists"));
assert!(!display.contains('\n'));
assert!(!display.contains("user"));
assert!(!display.contains("password"));
assert!(!display.contains("token"));
assert!(!display.contains("secret"));
}
#[test]
fn scan_metadata_expansion_error_has_sanitized_display()
-> Result<(), Box<dyn std::error::Error>> {
let error = DeltaFunnelError::DeltaScanMetadataExpansion {
source_name: "orders\nlatest".to_owned(),
table_uri: "s3://user:password@example.com/table?token=secret".to_owned(),
snapshot_version: 7,
source: Box::new(delta_kernel::Error::generic(
"scan\nmetadata expansion failed",
)),
};
let display = error.to_string();
assert!(display.contains(r"orders\nlatest"));
assert!(display.contains("snapshot version 7"));
assert!(display.contains("s3://example.com/table"));
assert!(display.contains(r"scan\nmetadata expansion failed"));
assert!(!display.contains('\n'));
assert!(!display.contains("user"));
assert!(!display.contains("password"));
assert!(!display.contains("token"));
assert!(!display.contains("secret"));
let source = Error::source(&error)
.ok_or("metadata expansion error must preserve its kernel source")?;
assert!(
source
.to_string()
.contains("scan\nmetadata expansion failed")
);
Ok(())
}
#[test]
fn scan_file_task_planning_error_has_sanitized_display() {
let error = DeltaFunnelError::DeltaScanFileTaskPlanning {
source_name: "orders\nlatest".to_owned(),
table_uri: "s3://user:password@example.com/table?token=secret".to_owned(),
snapshot_version: 7,
path: "part\n00000.parquet".to_owned(),
reason: "kernel\nsize was negative".to_owned(),
};
let display = error.to_string();
assert!(display.contains(r"orders\nlatest"));
assert!(display.contains("snapshot version 7"));
assert!(display.contains("s3://example.com/table"));
assert!(display.contains(r"part\n00000.parquet"));
assert!(display.contains(r"kernel\nsize was negative"));
assert!(!display.contains('\n'));
assert!(!display.contains("user"));
assert!(!display.contains("password"));
assert!(!display.contains("token"));
assert!(!display.contains("secret"));
}
#[test]
fn scan_file_task_partition_planning_error_has_sanitized_display() {
let error = DeltaFunnelError::DeltaScanFileTaskPartitionPlanning {
source_name: "orders\nlatest".to_owned(),
table_uri: "s3://user:password@example.com/table?token=secret".to_owned(),
snapshot_version: 7,
reason: "target\npartitions was zero".to_owned(),
};
let display = error.to_string();
assert!(display.contains(r"orders\nlatest"));
assert!(display.contains("snapshot version 7"));
assert!(display.contains("s3://example.com/table"));
assert!(display.contains(r"target\npartitions was zero"));
assert!(!display.contains('\n'));
assert!(!display.contains("user"));
assert!(!display.contains("password"));
assert!(!display.contains("token"));
assert!(!display.contains("secret"));
}
}