use crate::files_source::constants::LOADER_COLUMNS;
use crate::files_source::main::constant_columns::constant_columns;
use crate::files_source::main::path_template::path_template;
use crate::files_source::main::text_charset::text_charset;
use crate::files_source::models::{
AfterLoad, Compression, CsvQuote, FilePattern, FilesOrder, OnEmpty, RecordShape, SourceFormat,
};
use crate::load_naming::constants::SQLLOAD_PREFIX;
use crate::project::_helpers::targets::{
destination_kind_problems, is_files_target, stage_problems, target_problems,
};
use crate::project::_helpers::tombstones::tombstone_problems;
use crate::project::constants::{
FILES_CONNECTION_KINDS, MAX_POSTGRES_NAME_BYTES, MIN_BATCH_INTERVAL, STAGE_NAME_PARTS,
TABLE_PLACEHOLDER,
};
use crate::project::models::{
CdcMode, ConnectionConfig, ConstantColumns, Deletes, ProjectConfig, RenameSetting,
SourceConfig, SourceKind, TableSettings, WriteStrategy,
};
use crate::sheets_source::main::a1_range::a1_range;
use crate::store_paths::main::resolve_location::resolve_location;
use crate::store_paths::models::{Location, StoreKind};
pub(crate) fn reference_problems(project: &ProjectConfig) -> Vec<String> {
let mut problems: Vec<String> = Vec::new();
if let Some(default_target) = &project.default_target
&& !project.targets.contains_key(default_target)
{
problems.push(format!(
"default_target {default_target:?} is not a [targets] entry"
));
}
for (name, target) in &project.targets {
problems.extend(target_problems(project, name, target));
}
for (name, connection) in &project.connections {
let targeted = project
.targets
.values()
.any(|target| target.connection == *name);
problems.extend(stage_problems(name, connection, targeted));
problems.extend(odbc_connection_problems(name, connection));
if let ConnectionConfig::S3 {
profile: Some(profile),
..
} = connection
&& profile.is_empty()
{
problems.push(format!(
"[connections.{name}] profile is empty; omit it to use AWS_PROFILE, else default"
));
}
}
for (name, source) in &project.sources {
problems.extend(source_problems(project, name, source));
}
problems
}
fn odbc_connection_problems(name: &str, connection: &ConnectionConfig) -> Vec<String> {
let ConnectionConfig::Odbc {
connection_string,
dsn,
user,
password,
} = connection
else {
return Vec::new();
};
let place = format!("[connections.{name}]");
let mut problems: Vec<String> = Vec::new();
match (connection_string.is_some(), dsn.is_some()) {
(false, false) => problems.push(format!("{place} needs connection_string, or dsn")),
(true, true) => problems.push(format!(
"{place} takes connection_string or dsn, not both"
)),
(true, false) if user.is_some() || password.is_some() => problems.push(format!(
"{place} user and password go with dsn; with connection_string, put them in it (UID=...;PWD=...)"
)),
_ => {}
}
let unbraced = [dsn.as_deref(), user.as_deref()];
if unbraced
.into_iter()
.flatten()
.any(|value| value.contains([';', '{', '}']))
{
problems.push(format!(
"{place} dsn and user cannot contain ';', '{{' or '}}'; use connection_string instead"
));
}
problems
}
fn source_problems(project: &ProjectConfig, name: &str, source: &SourceConfig) -> Vec<String> {
let mut problems: Vec<String> = Vec::new();
let place = source_place(project, name);
problems.extend(after_load_problems(project, (name, &place), source));
problems.extend(tombstone_problems(&place, source));
let (kinds, expected): (&[&str], &str) = match source.kind {
SourceKind::Postgres | SourceKind::PostgresCdc => (&["postgres"], "postgres"),
SourceKind::Snowflake => (&["snowflake"], "snowflake"),
SourceKind::Odbc => (&["odbc"], "odbc"),
SourceKind::BigQuery => (&["bigquery"], "bigquery"),
SourceKind::Databricks => (&["databricks"], "databricks"),
SourceKind::Files => (&["s3", "filesystem", "sftp"], "s3, filesystem or sftp"),
SourceKind::Python => (&[], "any"),
SourceKind::GoogleSheets => (&["google_sheets"], "google_sheets"),
};
match project.connections.get(&source.connection) {
None if source.connection.is_empty() && source.kind == SourceKind::Python => {}
None if source.connection.is_empty() => {
problems.push(format!("{place} needs a connection"));
}
None => problems.push(format!(
"{place} connection {:?} is not a [connections] entry",
source.connection
)),
Some(connection) if source.kind == SourceKind::Python => {
problems.extend(shared_warehouse_problems(project, name, source, connection));
}
Some(connection) if !kinds.contains(&connection.kind()) => {
problems.push(format!(
"{place} has kind = {:?} but connection {:?} is a {} connection (expected {expected})",
source.kind.name(),
source.connection,
connection.kind()
));
}
Some(ConnectionConfig::Sftp { .. }) if source.after_load != AfterLoad::Keep => {
problems.push(format!(
"{place} after_load is not for an sftp connection: SQLLoad only reads from SFTP servers, so it cannot delete or move their files"
));
}
Some(ConnectionConfig::Sftp { .. }) if source.copy_to_internal_stage => {
problems.push(format!(
"{place} copy_to_internal_stage is not for an sftp connection: files on an SFTP server always pass through SQLLoad's reader"
));
}
Some(_) => {}
}
let only_target = (project.targets.len() == 1)
.then(|| project.targets.keys().next())
.flatten();
match source
.destination
.as_ref()
.or(project.default_target.as_ref())
.or(only_target)
{
Some(target) if !project.targets.contains_key(target) => {
problems.push(format!("{place} destination {target:?} is not a [targets] entry"));
}
Some(target) => problems.extend(destination_kind_problems(
&place,
source,
is_files_target(project, target),
)),
None => problems.push(format!(
"{place} needs a destination: set destination, or default_target when there are several targets"
)),
}
problems.extend(kind_problems(&place, source));
problems.extend(transform_problems(project, &place, source));
problems.extend(rename_problems(project, &place, source));
if source.kind == SourceKind::PostgresCdc {
problems.extend(cdc_name_problems(&place, name, source));
}
problems
}
fn shared_warehouse_problems(
project: &ProjectConfig,
name: &str,
source: &SourceConfig,
connection: &ConnectionConfig,
) -> Vec<String> {
if FILES_CONNECTION_KINDS.contains(&connection.kind()) {
return Vec::new();
}
project
.targets
.iter()
.filter(|(_, target)| target.connection == source.connection)
.map(|(target, _)| {
format!(
"Python source {name} uses connection {}, which target {target} loads into; Python code never gets warehouse credentials — give the source its own connection",
source.connection
)
})
.collect()
}
fn source_place(project: &ProjectConfig, name: &str) -> String {
match project.python_modules.get(name) {
Some(module) => format!("Python source {name} ({module})"),
None => format!("[sources.{name}]"),
}
}
fn kind_problems(place: &str, source: &SourceConfig) -> Vec<String> {
let mut problems: Vec<String> = Vec::new();
let template = source.table_name_template();
let cdc_only = source.publication.is_some()
|| source.slot.is_some()
|| source.cdc_mode != CdcMode::Merge
|| source.batch_interval.is_some();
if cdc_only && source.kind != SourceKind::PostgresCdc {
problems.push(format!(
"{place} publication, slot, cdc_mode and batch_interval are for kind = \"postgres_cdc\""
));
}
if source.kind == SourceKind::PostgresCdc {
problems.extend(cdc_problems(place, source));
}
problems.extend(constant_problems(place, source));
let queries = source.tables.values().any(|table| table.sql.is_some());
if !matches!(
source.kind,
SourceKind::Snowflake | SourceKind::Odbc | SourceKind::BigQuery | SourceKind::Databricks
) && queries
{
problems.push(format!(
"{place} sql is for kind = \"snowflake\", \"odbc\", \"bigquery\" and \"databricks\""
));
}
if source.kind != SourceKind::BigQuery && source.project.is_some() {
problems.push(format!("{place} project is for kind = \"bigquery\""));
}
if source.kind != SourceKind::Snowflake
&& (source.warehouse.is_some() || source.stage.is_some())
{
problems.push(format!(
"{place} warehouse and stage are for kind = \"snowflake\""
));
}
if source.kind != SourceKind::Odbc
&& (source.max_text_size.is_some() || source.max_binary_size.is_some())
{
problems.push(format!(
"{place} max_text_size and max_binary_size are for kind = \"odbc\""
));
}
if source.kind != SourceKind::Odbc
&& (source.long_column_threshold.is_some() || source.max_batch_memory.is_some())
{
problems.push(format!(
"{place} long_column_threshold and max_batch_memory are for kind = \"odbc\""
));
}
if source.max_batch_memory.is_some_and(|size| size.0 == 0) {
problems.push(format!("{place} max_batch_memory must be more than 0"));
}
if source.kind != SourceKind::Odbc
&& sheet_settings(source).any(|settings| settings.cursor_bounds.is_some())
{
problems.push(format!(
"{place} cursor_bounds is for kind = \"odbc\", whose drivers may refuse a bound cursor value"
));
}
if source.kind == SourceKind::Snowflake {
problems.extend(unload_stage_problems(place, source));
}
let selects = matches!(
source.kind,
SourceKind::Postgres
| SourceKind::Snowflake
| SourceKind::Odbc
| SourceKind::BigQuery
| SourceKind::Databricks
);
if !selects && sheet_settings(source).any(|settings| settings.row_filter.is_some()) {
problems.push(format!(
"{place} where is for kind = \"postgres\", \"snowflake\", \"odbc\", \"bigquery\" and \"databricks\", whose tables SQLLoad reads with a SELECT"
));
}
problems.extend(
(source.tables.iter())
.filter(|(_, table)| table.sql.is_some() && table.settings.row_filter.is_some())
.map(|(name, _)| {
format!("{place} [tables.{name}] where narrows a table SQLLoad selects from, and {name} is a query; put the condition in its sql")
}),
);
let sheets_only = source.spreadsheet_id.is_some()
|| !source.ranges.is_empty()
|| sheet_settings(source)
.any(|settings| settings.header.is_some() || settings.header_row.is_some());
problems.extend(column_names_problems(place, source));
if sheets_only && source.kind != SourceKind::GoogleSheets {
problems.push(format!(
"{place} spreadsheet_id, ranges, header and header_row are for kind = \"google_sheets\""
));
}
match source.kind {
SourceKind::Postgres
| SourceKind::PostgresCdc
| SourceKind::Snowflake
| SourceKind::Odbc
| SourceKind::BigQuery
| SourceKind::Databricks => {
if source.include.is_empty() && !queries {
let tables = match source.kind {
SourceKind::Snowflake
| SourceKind::Odbc
| SourceKind::BigQuery
| SourceKind::Databricks => {
"; list tables in include, or give a table a sql query"
}
_ => "",
};
problems.push(format!(
"{place} include is empty, so it selects no tables{tables}"
));
}
if !template.contains(TABLE_PLACEHOLDER) {
problems.push(format!(
"{place} table_name {template:?} must contain {TABLE_PLACEHOLDER}, or every table gets the same name"
));
}
problems.extend(files_only_problems(place, source));
}
SourceKind::Files => {
if source.path.is_none() {
problems.push(format!("{place} kind = \"files\" needs a path"));
}
if !source.include.is_empty() || !source.exclude.is_empty() {
problems.push(format!("{place} include and exclude are for database sources; a files source is one table"));
}
problems.extend(encoding_problems(place, source));
problems.extend(path_value_problems(place, source));
}
SourceKind::GoogleSheets => {
if source.ranges.len() != 1 && !template.contains(TABLE_PLACEHOLDER) {
problems.push(format!(
"{place} table_name {template:?} must contain {TABLE_PLACEHOLDER}, or every table gets the same name"
));
}
problems.extend(files_only_problems(place, source));
problems.extend(sheets_problems(place, source));
}
SourceKind::Python => {
if source.tables.len() > 1 && !template.contains(TABLE_PLACEHOLDER) {
problems.push(format!(
"{place} table_name {template:?} must contain {TABLE_PLACEHOLDER}, or every table gets the same name"
));
}
}
}
problems
}
fn unload_stage_problems(place: &str, source: &SourceConfig) -> Vec<String> {
let Some(stage) = &source.stage else {
return Vec::new();
};
let place = format!("{place} stage");
let mut problems: Vec<String> = Vec::new();
match (&stage.url, &stage.storage_integration, &stage.name) {
(Some(url), Some(_), None) => match resolve_location(None, url) {
Ok(Location {
store: StoreKind::S3 { .. },
..
}) => {}
Ok(_) => problems.push(format!("{place} url {url:?} must be an s3:// URL")),
Err(error) => problems.push(format!("{place} url {url:?}: {error}")),
},
(Some(_), None, None) => problems.push(format!(
"{place} url needs the storage_integration Snowflake writes to it with"
)),
(None, None, Some(name))
if name.split('.').filter(|part| !part.is_empty()).count() != STAGE_NAME_PARTS =>
{
problems.push(format!("{place} name {name:?} must be DATABASE.SCHEMA.STAGE"));
}
(None, None, Some(_)) => {}
_ => problems.push(format!(
"{place} takes either url and storage_integration (an S3 folder) or name (an existing stage)"
)),
}
problems
}
fn files_only_problems(place: &str, source: &SourceConfig) -> Vec<String> {
let mut problems: Vec<String> = Vec::new();
if source.path.is_some()
|| !source.partition_columns.is_empty()
|| source.path_columns.is_some()
|| source.unmatched_paths.is_some()
|| source.copy_to_internal_stage
|| source.after_load != AfterLoad::Keep
|| source.order != FilesOrder::default()
|| source.on_empty != OnEmpty::default()
{
problems.push(format!(
"{place} path, partition_columns, path_columns, unmatched_paths, copy_to_internal_stage, after_load, order and on_empty are for kind = \"files\""
));
}
if source.format != SourceFormat::Parquet
|| source.compression.is_some()
|| source.csv.is_some()
|| source.xlsx.is_some()
|| source.xml.is_some()
|| source.members.is_some()
|| source.pattern.is_some()
|| source.shape != RecordShape::Columns
|| source.record_column.is_some()
{
problems.push(format!(
"{place} format, compression, csv, xlsx, xml, members, pattern, shape and record_column are for kind = \"files\""
));
}
problems
}
fn transform_problems(project: &ProjectConfig, place: &str, source: &SourceConfig) -> Vec<String> {
let transforms: Vec<&String> = sheet_settings(source)
.filter_map(|settings| settings.transform.as_ref())
.collect();
let mut problems: Vec<String> = Vec::new();
if transforms.is_empty() {
return problems;
}
if !matches!(source.kind, SourceKind::Files | SourceKind::GoogleSheets) {
problems.push(format!(
"{place} transform is for kind = \"files\" and kind = \"google_sheets\" tables; an sl.http job takes transform= on paginate or fan_out"
));
}
if transforms
.iter()
.any(|transform| transform.trim().is_empty())
{
problems.push(format!(
"{place} transform is empty; give the name @sl.transform registered the function under"
));
}
if project.python_paths.is_empty() {
problems.push(format!(
"{place} transform needs python_paths, naming the folder that holds the @sl.transform function"
));
}
problems
}
fn rename_problems(project: &ProjectConfig, place: &str, source: &SourceConfig) -> Vec<String> {
let functions: Vec<&String> = sheet_settings(source)
.filter_map(|settings| match &settings.rename {
Some(RenameSetting::Function(function)) => Some(function),
_ => None,
})
.collect();
let mut problems: Vec<String> = Vec::new();
if functions.iter().any(|function| function.trim().is_empty()) {
problems.push(format!(
"{place} rename is empty; give a map of source names to loaded names, or the name @sl.rename registered a function under"
));
}
if !functions.is_empty() && project.python_paths.is_empty() {
problems.push(format!(
"{place} rename names a function, which needs python_paths, naming the folder that holds the @sl.rename function"
));
}
problems
}
fn sheet_settings(source: &SourceConfig) -> impl Iterator<Item = &TableSettings> {
std::iter::once(&source.defaults).chain(source.tables.values().map(|table| &table.settings))
}
fn column_names_problems(place: &str, source: &SourceConfig) -> Vec<String> {
let mut problems: Vec<String> = Vec::new();
let named = sheet_settings(source).any(|settings| settings.column_names.is_some());
let fits = match source.kind {
SourceKind::GoogleSheets => true,
SourceKind::Files => {
matches!(source.format, SourceFormat::Csv | SourceFormat::Xlsx)
&& source.shape != RecordShape::Record
}
_ => false,
};
if named && !fits {
problems.push(format!(
"{place} column_names is for columns of data without a header row: kind = \"files\" with format = \"csv\" or \"xlsx\" (not shape = \"record\"), or kind = \"google_sheets\""
));
return problems;
}
let header = match source.kind {
SourceKind::GoogleSheets => "header = false",
_ if source.format == SourceFormat::Csv => "csv = { header = false }",
_ => "xlsx = { header = false }",
};
if named && headed_tables_named(source) {
problems.push(format!(
"{place} column_names names the columns of data without a header row; set {header}, or leave the names to the header"
));
}
for names in sheet_settings(source).filter_map(|settings| settings.column_names.as_ref()) {
for (position, name) in names.iter().enumerate() {
if name.trim().is_empty() {
problems.push(format!(
"{place} column_names has an empty name in place {}",
position + 1
));
} else if names[..position]
.iter()
.any(|earlier| earlier.eq_ignore_ascii_case(name))
{
problems.push(format!(
"{place} column_names names {name} twice (ignoring case)"
));
}
}
}
problems
}
fn headed_tables_named(source: &SourceConfig) -> bool {
if source.kind != SourceKind::GoogleSheets {
return match source.format {
SourceFormat::Csv => source.csv.as_ref().is_none_or(|csv| csv.header),
_ => source.xlsx.as_ref().is_none_or(|xlsx| xlsx.header),
};
}
let defaults = &source.defaults;
let entries = source
.tables
.iter()
.map(|(name, table)| (name, Some(&table.settings)));
let ranged = source
.ranges
.keys()
.filter(|name| !source.tables.contains_key(*name))
.map(|name| (name, None));
let unlisted = source.ranges.is_empty().then_some(defaults);
entries
.chain(ranged)
.map(|(_, settings)| settings)
.chain(unlisted.map(Some))
.any(|settings| {
let pick = |value: fn(&TableSettings) -> Option<bool>| {
settings.and_then(value).or(value(defaults))
};
let named = settings
.and_then(|settings| settings.column_names.as_ref())
.or(defaults.column_names.as_ref())
.is_some();
named && pick(|settings| settings.header).unwrap_or(true)
})
}
fn sheets_problems(place: &str, source: &SourceConfig) -> Vec<String> {
let mut problems: Vec<String> = Vec::new();
if source
.spreadsheet_id
.as_deref()
.is_none_or(|id| id.trim().is_empty())
{
problems.push(format!("{place} kind = \"google_sheets\" needs a spreadsheet_id, the ID in the spreadsheet's URL"));
}
if !source.include.is_empty() || !source.exclude.is_empty() {
problems.push(format!("{place} include and exclude are not for kind = \"google_sheets\"; name its tables in ranges"));
}
for settings in sheet_settings(source) {
if settings.cursor_column.is_some()
|| settings.batch_size.is_some()
|| settings.write_strategy == Some(WriteStrategy::DeleteInsert)
{
problems.push(format!(
"{place} a sheet has no cursor, so cursor_column, batch_size and write_strategy = \"delete_insert\" do not apply; use replace, merge on a unique_key, or append"
));
break;
}
}
if sheet_settings(source).any(|settings| settings.header_row == Some(0)) {
problems.push(format!("{place} header_row counts from 1"));
}
problems
}
fn path_value_problems(place: &str, source: &SourceConfig) -> Vec<String> {
let mut problems: Vec<String> = Vec::new();
let mut names: Vec<(String, &str)> = source
.partition_columns
.keys()
.map(|name| (name.clone(), "partition_columns"))
.collect();
match source
.path_columns
.as_deref()
.map(|text| path_template(text, &[]))
{
None if source.unmatched_paths.is_some() => problems.push(format!(
"{place} unmatched_paths is for path_columns, which is not set"
)),
None => {}
Some(Ok(template)) => names.extend(
template
.captures
.into_iter()
.map(|(name, _)| (name, "path_columns")),
),
Some(Err(problem)) => problems.push(format!("{place} {problem}")),
}
if let Some(record) = &source.files_encoding().record {
names.push((record.clone(), "record_column"));
}
problems.extend(clashes(place, &names, 0));
let filled = names.len();
for constants in
sheet_settings(source).filter_map(|settings| settings.constant_columns.as_ref())
{
let mut named = names.clone();
named.extend(
constant_columns(&constants.0, &[])
.into_iter()
.flatten()
.map(|(name, _, _)| (name, "constant_columns")),
);
problems.extend(clashes(place, &named, filled));
}
problems
}
fn clashes(place: &str, names: &[(String, &str)], from: usize) -> Vec<String> {
(from..names.len())
.filter_map(|index| clash(place, &names[..=index]))
.collect()
}
fn clash(place: &str, names: &[(String, &str)]) -> Option<String> {
let ((name, setting), earlier) = names.split_last()?;
let (_, first) = earlier
.iter()
.find(|(seen, _)| seen.eq_ignore_ascii_case(name))?;
Some(format!(
"{place} column {name} is in both {first} and {setting}; give it one source"
))
}
fn constant_problems(place: &str, source: &SourceConfig) -> Vec<String> {
let mut problems: Vec<String> = Vec::new();
let maps: Vec<&ConstantColumns> = sheet_settings(source)
.filter_map(|settings| settings.constant_columns.as_ref())
.collect();
if source.kind == SourceKind::PostgresCdc && !maps.is_empty() {
problems.push(format!(
"{place} constant_columns is not supported for kind = \"postgres_cdc\""
));
}
for constants in maps {
if let Err(problem) = constant_columns(&constants.0, &[]) {
problems.push(format!("{place} {problem}"));
}
problems.extend(
constants
.0
.keys()
.filter(|name| name.to_ascii_lowercase().starts_with(SQLLOAD_PREFIX))
.map(|name| {
format!("{place} constant column {name}: names starting with {SQLLOAD_PREFIX} are SQLLoad's own")
}),
);
}
problems
}
fn after_load_problems(
project: &ProjectConfig,
(name, place): (&str, &str),
source: &SourceConfig,
) -> Vec<String> {
if source.kind != SourceKind::Files
|| source.after_load == AfterLoad::Keep
|| source.compression != Some(Compression::Zip)
{
return Vec::new();
}
let folder = |source: &SourceConfig| {
let path = source.path.as_deref().unwrap_or_default();
let path = path.strip_prefix("./").unwrap_or(path);
path.trim_matches('/').to_owned()
};
let inside = |inner: &str, outer: &str| {
outer.is_empty() || inner == outer || inner.starts_with(&format!("{outer}/"))
};
let mine = folder(source);
project
.sources
.iter()
.filter(|(other_name, other)| {
other_name.as_str() != name
&& other.kind == SourceKind::Files
&& other.connection == source.connection
&& other.compression == Some(Compression::Zip)
})
.filter(|(_, other)| {
let theirs = folder(other);
inside(&mine, &theirs) || inside(&theirs, &mine)
})
.map(|(other_name, other)| {
format!(
"{place} after_load moves or deletes each ZIP archive once its members load, and [sources.{other_name}] reads ZIP archives in {:?} on the same connection, which may still need them; remove after_load, or give the sources folders apart",
other.path.as_deref().unwrap_or_default()
)
})
.collect()
}
fn container_problems(place: &str, source: &SourceConfig) -> Vec<String> {
let mut problems: Vec<String> = Vec::new();
let zip = source.compression == Some(Compression::Zip);
if source.xlsx.is_some() && source.format != SourceFormat::Xlsx {
problems.push(format!("{place} xlsx is for format = \"xlsx\""));
}
if let Some(xlsx) = &source.xlsx {
if let Some(range) = &xlsx.range {
if xlsx.sheet.is_some() || xlsx.skip_rows != 0 {
problems.push(format!(
"{place} xlsx.range names the sheet and the first row itself; drop xlsx.sheet and xlsx.skip_rows"
));
}
match (a1_range(range), xlsx.header_row) {
(Err(problem), _) => problems.push(format!("{place} xlsx.{problem}")),
(Ok(range), Some(header_row)) => {
let rows = range.last_row.map(|last| last - range.first_row + 1);
if rows.is_some_and(|rows| header_row > rows) {
problems.push(format!(
"{place} xlsx.header_row {header_row} is past the range's {} rows",
rows.unwrap_or_default()
));
}
}
(Ok(_), None) => {}
}
}
if xlsx.header_row == Some(0) {
problems.push(format!("{place} xlsx.header_row counts from 1"));
}
}
match (&source.xml, source.format) {
(Some(_), format) if format != SourceFormat::Xml => {
problems.push(format!("{place} xml is for format = \"xml\""));
}
(Some(xml), _) if !xml.record_path.trim_matches('/').is_empty() => {}
(_, SourceFormat::Xml) => problems.push(format!(
"{place} format = \"xml\" needs xml.record_path, the element names from the root to each row's element, such as \"orders/order\""
)),
_ => {}
}
if let Some(Err(error)) = source.pattern.as_deref().map(FilePattern::new) {
problems.push(format!("{place} pattern: {error}"));
}
match (&source.members, zip) {
(Some(_), false) => problems.push(format!("{place} members is for compression = \"zip\"")),
(Some(pattern), true) => {
if let Err(error) = globset::Glob::new(pattern) {
problems.push(format!("{place} members {pattern:?}: {error}"));
}
}
(None, _) => {}
}
let loader_only = zip || matches!(source.format, SourceFormat::Xlsx | SourceFormat::Xml);
if source.copy_to_internal_stage && loader_only {
problems.push(format!(
"{place} copy_to_internal_stage is for files Snowflake reads itself; SQLLoad reads xlsx and xml files and ZIP archives' members"
));
}
problems
}
fn encoding_problems(place: &str, source: &SourceConfig) -> Vec<String> {
let mut problems: Vec<String> = Vec::new();
match (source.shape, source.format, &source.record_column) {
(RecordShape::Record, SourceFormat::Parquet, _) => problems.push(format!(
"{place} shape = \"record\" is for ndjson, json and csv files; Parquet files already have typed columns"
)),
(RecordShape::Record, format @ (SourceFormat::Xlsx | SourceFormat::Xml), _) => {
problems.push(format!(
"{place} shape = \"record\" is for ndjson, json and csv files, not {format}"
));
}
(RecordShape::Columns, _, Some(_)) => problems.push(format!(
"{place} record_column is for shape = \"record\", which is not set"
)),
(_, _, Some(name)) if name.trim().is_empty() => {
problems.push(format!("{place} record_column must not be empty"));
}
(_, _, Some(name))
if LOADER_COLUMNS
.iter()
.any(|loader| loader.eq_ignore_ascii_case(name)) =>
{
problems.push(format!(
"{place} record_column {name} is a column SQLLoad adds itself; choose another name"
));
}
_ => {}
}
if let Some(record) = &source.files_encoding().record {
for (table, overrides) in &source.tables {
let pinned: Vec<&str> = overrides
.columns
.keys()
.map(String::as_str)
.filter(|name| name != record)
.collect();
if !pinned.is_empty() {
problems.push(format!(
"{place} shape = \"record\" loads each record into one json column, {record}, so [tables.{table}] columns cannot pin {}",
pinned.join(", ")
));
}
}
}
problems.extend(container_problems(place, source));
if source.format == SourceFormat::Parquet && source.compression.is_some() {
problems.push(format!(
"{place} compression is for ndjson, json, csv, xlsx and xml files; Parquet files compress inside the file, and SQLLoad does not read them from ZIP archives"
));
}
match (&source.csv, source.format) {
(None, _) => {}
(Some(csv), SourceFormat::Csv) => {
let quote = csv.quote.character();
let characters = [
("delimiter", Some(csv.delimiter)),
("quote", quote),
("escape", csv.escape),
];
for (setting, character) in characters {
if let Some(character) = character
&& (!character.is_ascii() || character == '\n' || character == '\r')
{
problems.push(format!(
"{place} csv {setting} must be one ASCII character other than a line break, not {character:?}"
));
}
}
if quote == Some(csv.delimiter) || csv.escape == Some(csv.delimiter) {
problems.push(format!(
"{place} csv delimiter must differ from quote and escape"
));
}
match (csv.quote, csv.escape) {
(CsvQuote::Off(true), _) => problems.push(format!(
"{place} csv quote = true names no character; write it, such as '\"', or false to read every quote as a plain character"
)),
(CsvQuote::Off(false), Some(_)) => problems.push(format!(
"{place} csv escape works inside quoted fields, and quote = false has none; remove escape"
)),
_ => {}
}
if let Err(problem) = text_charset(&csv.encoding) {
problems.push(format!("{place} {problem}"));
}
}
(Some(_), format) => problems.push(format!(
"{place} csv is for format = \"csv\", not {format:?}",
format = format.to_string()
)),
}
problems
}
fn cdc_problems(place: &str, source: &SourceConfig) -> Vec<String> {
let mut problems: Vec<String> = Vec::new();
let settings = std::iter::once(&source.defaults)
.chain(source.tables.values().map(|table| &table.settings));
for settings in settings {
if settings.write_strategy.is_some()
|| settings.cursor_column.is_some()
|| settings.batch_size.is_some()
|| settings.state.is_some()
|| settings.checks.is_some()
|| settings.on_duplicate_keys.is_some()
|| settings.null_keys.is_some()
|| settings.replace_key.is_some()
|| settings.newest_by.is_some()
{
problems.push(format!(
"{place} kind = \"postgres_cdc\" writes every captured change, so it takes no write_strategy, cursor_column, batch_size, state, checks, on_duplicate_keys, null_keys, replace_key or newest_by; choose cdc_mode = \"merge\" or \"changelog\""
));
break;
}
}
if source
.batch_interval
.is_some_and(|interval| interval < MIN_BATCH_INTERVAL)
{
problems.push(format!(
"{place} batch_interval must be at least 1s, or `load --follow` would write a batch for almost every change"
));
}
let hard = std::iter::once(&source.defaults)
.chain(source.tables.values().map(|table| &table.settings))
.any(|settings| settings.deletes == Some(Deletes::Hard));
if hard && source.cdc_mode == CdcMode::Changelog {
problems.push(format!(
"{place} deletes = \"hard\" applies to cdc_mode = \"merge\"; a changelog keeps every delete as a row"
));
}
problems
}
fn cdc_name_problems(place: &str, name: &str, source: &SourceConfig) -> Vec<String> {
let default_name = format!("sqlload_{}", name.to_lowercase());
let mut problems: Vec<String> = Vec::new();
for (setting, chosen) in [("slot", &source.slot), ("publication", &source.publication)] {
let effective = chosen.as_deref().unwrap_or(&default_name);
let valid = !effective.is_empty()
&& effective.len() <= MAX_POSTGRES_NAME_BYTES
&& effective
.chars()
.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_');
if !valid {
problems.push(format!(
"{place} {setting} {effective:?} must be lower-case letters, digits and underscores, at most {MAX_POSTGRES_NAME_BYTES} characters; set {setting} = \"...\""
));
}
}
problems
}