sqlload 0.53.0

SQL data loading tooling.
use crate::project::constants::{
    CLICKHOUSE_CONNECTION_KIND, FILES_CONNECTION_KINDS, STAGE_NAME_PARTS,
    WAREHOUSE_CONNECTION_KINDS,
};
use crate::project::main::motherduck_path::motherduck_path;
use crate::project::models::{
    ConnectionConfig, FilesFormat, LoadMethod, ProjectConfig, SourceConfig, TargetConfig,
};
use crate::store_paths::main::resolve_location::resolve_location;
use crate::store_paths::models::{Location, StoreKind};

/// Problems with where a warehouse connection stages files: a Databricks `volume` must be `catalog.schema.volume`; a Snowflake `load_method` is required when a target loads into the connection (`targeted`), and an `external_stage` needs an `s3://` URL and exactly one of `storage_integration` (SQLLoad creates the stage) or `name` (an existing one).
pub(crate) fn stage_problems(
    name: &str,
    connection: &ConnectionConfig,
    targeted: bool,
) -> Vec<String> {
    let load_method = match connection {
        ConnectionConfig::Snowflake { load_method, .. } => load_method,
        ConnectionConfig::Databricks { volume, .. } => {
            return match volume.split('.').filter(|part| !part.is_empty()).count() {
                STAGE_NAME_PARTS => Vec::new(),
                _ => vec![format!(
                    "[connections.{name}] volume {volume:?} must be CATALOG.SCHEMA.VOLUME, an existing Unity Catalog volume"
                )],
            };
        }
        _ => return Vec::new(),
    };
    let Some(load_method) = load_method else {
        if !targeted {
            return Vec::new();
        }
        return vec![format!(
            "connection {name}: load_method is required: how should data get into Snowflake? Fix: add one of these to [connections.{name}]:\nload_method = {{ type = \"external_stage\", url = \"s3://...\", storage_integration = \"...\" }}   (fastest)\nload_method = {{ type = \"internal_user_stage\" }}   (no setup)"
        )];
    };
    let LoadMethod::ExternalStage {
        url,
        storage_integration,
        name: stage_name,
        ..
    } = load_method.as_ref()
    else {
        return Vec::new();
    };
    let place = format!("[connections.{name}] load_method");
    let mut problems: Vec<String> = Vec::new();
    match resolve_location(None, url) {
        Ok(Location {
            store: StoreKind::S3 { .. },
            ..
        }) => {}
        Ok(_) => problems.push(format!("{place} url {url:?} must be an s3:// URL")),
        Err(error) => problems.push(format!("{place} url {url:?}: {error}")),
    }
    match (storage_integration, stage_name) {
        (Some(_), Some(_)) | (None, None) => problems.push(format!(
            "{place} needs exactly one of storage_integration (SQLLoad creates _SQLLOAD_STAGE in each loader schema) or name (an existing stage)"
        )),
        (None, Some(stage_name))
            if stage_name.split('.').filter(|part| !part.is_empty()).count() != STAGE_NAME_PARTS =>
        {
            problems.push(format!(
                "{place} name {stage_name:?} must be DATABASE.SCHEMA.STAGE"
            ));
        }
        _ => {}
    }
    problems
}

/// Whether target `name` writes files (its connection is s3 or filesystem).
pub(crate) fn is_files_target(project: &ProjectConfig, name: &str) -> bool {
    project
        .targets
        .get(name)
        .and_then(|target| project.connections.get(&target.connection))
        .is_some_and(|connection| FILES_CONNECTION_KINDS.contains(&connection.kind()))
}

/// Problems with one target: its connection, and the settings its kind requires or rejects.
pub(crate) fn target_problems(
    project: &ProjectConfig,
    name: &str,
    target: &TargetConfig,
) -> Vec<String> {
    let place = format!("[targets.{name}]");
    let Some(connection) = project.connections.get(&target.connection) else {
        return vec![format!(
            "{place} connection {:?} is not a [connections] entry",
            target.connection
        )];
    };
    let kind = connection.kind();
    let files_settings = target.path.is_some()
        || target.format.is_some()
        || target.max_footer_reads.is_some()
        || target.compression.is_some()
        || target.csv.is_some()
        || target.single_file.is_some();
    let mut problems: Vec<String> = Vec::new();
    match (
        WAREHOUSE_CONNECTION_KINDS.contains(&kind),
        FILES_CONNECTION_KINDS.contains(&kind),
    ) {
        (true, _) => {
            if target.loader_schema.is_none() {
                problems.push(format!("{place} needs a loader_schema"));
            }
            if let ConnectionConfig::MotherDuck { database, .. } = connection
                && target
                    .database
                    .as_deref()
                    .or(database.as_deref())
                    .and_then(|database| motherduck_path(database).name)
                    .is_none()
            {
                problems.push(format!(
                    "{place} loads into MotherDuck, which attaches every database in the account: name the database, in the target or its connection"
                ));
            }
            if kind == CLICKHOUSE_CONNECTION_KIND && target.database.is_some() {
                problems.push(format!(
                    "{place} loads into ClickHouse, which has no level above its databases: drop database and name the ClickHouse database in loader_schema"
                ));
            }
            if files_settings {
                problems.push(format!(
                    "{place} path, format, compression, csv, single_file and max_footer_reads are for targets on an s3 or filesystem connection"
                ));
            }
        }
        (false, true) => {
            if target.loader_schema.is_some() || target.database.is_some() {
                problems.push(format!(
                    "{place} writes files, so it takes no loader_schema or database; set path instead"
                ));
            }
            if target.max_footer_reads == Some(0) {
                problems.push(format!("{place} max_footer_reads must be at least 1"));
            }
            problems.extend(csv_output_problems(&place, target));
        }
        (false, false) => problems.push(format!(
            "{place} connection {:?} is a {kind} connection; targets load into {} or write files to {}",
            target.connection,
            WAREHOUSE_CONNECTION_KINDS.join(", "),
            FILES_CONNECTION_KINDS.join(" or ")
        )),
    }
    problems
}

/// Problems with a source's settings for the kind of target it loads into.
pub(crate) fn destination_kind_problems(
    place: &str,
    source: &SourceConfig,
    files_target: bool,
) -> Vec<String> {
    let sets_path = source.destination_path.is_some()
        || source
            .tables
            .values()
            .any(|table| table.destination_path.is_some());
    let mut problems: Vec<String> = Vec::new();
    if sets_path && !files_target {
        problems.push(format!(
            "{place} destination_path is for targets on an s3 or filesystem connection"
        ));
    }
    if source.schema.is_some() && files_target {
        problems.push(format!(
            "{place} schema is for warehouse targets; a files target places tables with path or destination_path"
        ));
    }
    problems
}

/// A files target's CSV settings that do not fit its format, or that would make a CSV file ambiguous.
fn csv_output_problems(place: &str, target: &TargetConfig) -> Vec<String> {
    let mut problems: Vec<String> = Vec::new();
    if target.format.unwrap_or_default() != FilesFormat::Csv {
        if target.compression.is_some() || target.csv.is_some() {
            problems.push(format!(
                "{place} compression and csv are for format = \"csv\"; Parquet files compress inside the file"
            ));
        }
        return problems;
    }
    let Some(csv) = &target.csv else {
        return problems;
    };
    for (setting, character) in [("delimiter", csv.delimiter), ("quote", csv.quote)] {
        if !character.is_ascii() || character == '\n' || character == '\r' {
            problems.push(format!(
                "{place} csv {setting} must be one ASCII character other than a line break, not {character:?}"
            ));
        }
    }
    if csv.delimiter == csv.quote {
        problems.push(format!("{place} csv delimiter and quote must differ"));
    }
    if csv
        .null_value
        .chars()
        .any(|character| [csv.delimiter, csv.quote, '\n', '\r'].contains(&character))
    {
        problems.push(format!(
            "{place} csv null_value {:?} must not hold the delimiter, the quote or a line break, or it could not be told apart from a value",
            csv.null_value
        ));
    }
    problems
}