sqlload 0.77.1

SQL data loading tooling.
//! A files batch loaded by Databricks `COPY`ing the source's own Parquet files from S3.

use crate::adapters::contract::main::_batch_direct_problem::batch_direct_problem;
use crate::adapters::contract::main::_footer_rows::footer_rows;
use crate::adapters::contract::models::{DirectBatch, DirectLoaded, TableName};
use crate::adapters::contract::types::SqlAdapter;
use crate::adapters::databricks::_helpers::write::s3_copy::{
    S3Batch, S3File, copy_groups, s3_copy_statement,
};
use crate::adapters::databricks::constants::{INSERTED_ROWS_COLUMN, MAX_FILES_PER_COPY};
use crate::adapters::databricks::main::stage::_direct_problem::direct_problem;
use crate::adapters::databricks::models::{DatabricksSql, DatabricksWriter};
use crate::databricks_api::main::execute_statement::execute_statement;
use crate::files_source::constants::{SOURCE_FILE_COLUMN, SOURCE_VERSION_COLUMN};
use crate::files_source::main::normal_value::normal_value;
use crate::files_source::models::DataFile;
use crate::load_naming::main::metadata_name::metadata_name;
use crate::load_naming::models::MetadataRole;
use crate::loader::errors::LoaderError;
use crate::schema_plan::models::DesiredColumn;
use crate::sql_api::models::SqlRows;
use arrow_schema::Schema;

impl DatabricksWriter {
    /// Load `batch` by having Databricks `COPY` its Parquet files where they lie in S3, through the external location that covers them, into the staging or build table the load publishes from, or into the table itself: one `COPY INTO` (one transaction) per [`copy_groups`] group, as `FILES` takes at most 1000 files and `mergeSchema` won't merge one column held as two types. A batch whose own footers hold a column a cast would not keep exactly fails, and so does one whose inserted rows, as `COPY INTO` reports them, differ from its footers'; an append into the table itself first takes back what it committed. Files never fall back to SQLLoad mid-load. `None` when the table's files aren't in S3 under an external location.
    pub(in crate::adapters) async fn copy_files(
        &mut self,
        batch: &DirectBatch<'_>,
    ) -> Result<Option<DirectLoaded>, LoaderError> {
        let Some(source) = batch.spec.files.as_ref() else {
            return Ok(None);
        };
        let folder = source.location.to_string();
        let covered = self
            .destination
            .external_locations
            .get()
            .is_some_and(|locations| locations.iter().any(|url| folder.starts_with(url.as_str())));
        if !covered {
            return Ok(None);
        }
        if let Some(problem) =
            batch_direct_problem(batch, source, direct_problem(batch.spec, batch.read))
        {
            return Err(LoaderError::Invalid(format!(
                "Databricks cannot load this batch's files exactly by reading them itself, and they never go through SQLLoad instead: {problem}"
            )));
        }
        let target = &batch.target;
        let (into, columns) = match (target.staging, target.builds) {
            (Some(staging), false) => (staging, target.columns),
            (staging, _) => (staging.unwrap_or(target.table), batch.write_columns),
        };
        let no_footer = Schema::empty();
        let schemas: Vec<(&DataFile, &Schema)> = batch
            .files
            .iter()
            .map(|data| {
                let schema = batch
                    .footers
                    .parquet
                    .get(&data.file.store_key())
                    .map_or(&no_footer, |footer| footer.schema().as_ref());
                (data, schema)
            })
            .collect();
        let mut inserted = 0;
        let mut copied = false;
        for group in copy_groups(schemas, MAX_FILES_PER_COPY) {
            let statement = group_statement(batch, &group, (into, columns), folder.clone())?;
            let result = match execute_statement(&self.destination.api, &statement).await {
                Ok(result) => {
                    copied = true;
                    inserted_rows(&result)
                }
                Err(error) => Err(error.into()),
            };
            match result {
                Ok(rows) => inserted += rows,
                Err(error) if copied => return Err(self.undo(batch, error).await),
                Err(error) => return Err(error),
            }
        }
        let expected = footer_rows(batch.footers);
        if inserted != expected {
            let error = LoaderError::Invalid(format!(
                "Databricks loaded {inserted} rows from this batch's files, but their footers hold {expected}"
            ));
            return Err(self.undo(batch, error).await);
        }
        Ok(Some(match target.staging {
            Some(_) => DirectLoaded::Staged(expected),
            None => DirectLoaded::Committed(expected),
        }))
    }
}

impl DatabricksWriter {
    /// After `error`, take back the rows a batch going straight into the table committed: its load's rows from its files, which a later run would otherwise skip as loaded. A batch landing in a staging table needs nothing, as the load drops it.
    async fn undo(&self, batch: &DirectBatch<'_>, error: LoaderError) -> LoaderError {
        let target = &batch.target;
        if target.staging.is_some() {
            return error;
        }
        let Some(source) = batch.spec.files.as_ref() else {
            return error;
        };
        let literal = |text: &str| DatabricksSql.render_loader_value_literal(text);
        let paths: Vec<String> = batch
            .files
            .iter()
            .map(|data| literal(&format!("{}{}", source.location, data.file.relative_key)))
            .collect();
        let statement = format!(
            "DELETE FROM {} WHERE {} = {} AND {} IN ({})",
            DatabricksSql.render_qualified_name(target.table),
            DatabricksSql.render_identifier(metadata_name(target.compat, MetadataRole::LoadId)),
            literal(target.load_id),
            DatabricksSql.render_identifier(SOURCE_FILE_COLUMN),
            paths.join(", ")
        );
        match execute_statement(&self.destination.api, &statement).await {
            Ok(_) => error,
            Err(undo) => LoaderError::Invalid(format!(
                "{error}; taking back the rows it committed then failed too, so the table holds them: {undo}"
            )),
        }
    }
}

/// The `COPY INTO` that loads `group`, files of `batch` below `folder`, into `into`'s `columns`, reading each table column the group's files hold by its name in them.
fn group_statement(
    batch: &DirectBatch<'_>,
    group: &[&DataFile],
    (into, columns): (&TableName, &[DesiredColumn]),
    folder: String,
) -> Result<String, LoaderError> {
    let mut files = Vec::with_capacity(group.len());
    let mut in_files: Vec<(String, String)> = Vec::new();
    for data in group {
        files.push(s3_file(batch, data, columns)?);
        let Some(footer) = batch.footers.parquet.get(&data.file.store_key()) else {
            continue;
        };
        for field in footer.schema().fields() {
            let loaded = batch.spec.source_table.loaded_as(field.name()).to_owned();
            if !in_files.iter().any(|(name, _)| *name == loaded) {
                in_files.push((loaded, field.name().clone()));
            }
        }
    }
    Ok(s3_copy_statement(
        (&DatabricksSql.render_qualified_name(into), columns),
        &batch.target,
        &S3Batch {
            folder,
            files,
            in_files,
        },
        batch.loaded_at,
    ))
}

/// The rows a `COPY INTO`'s `result` reports it inserted.
pub(in crate::adapters) fn inserted_rows(result: &SqlRows) -> Result<u64, LoaderError> {
    let reported = result
        .columns
        .iter()
        .position(|column| column == INSERTED_ROWS_COLUMN)
        .and_then(|index| result.rows.first()?.get(index)?.clone());
    let Some(text) = reported else {
        return Err(LoaderError::Invalid(format!(
            "COPY INTO reported no {INSERTED_ROWS_COLUMN}"
        )));
    };
    text.parse::<u64>().map_err(|error| {
        LoaderError::Invalid(format!(
            "COPY INTO reported {INSERTED_ROWS_COLUMN} = {text:?}: {error}"
        ))
    })
}

/// The values SQLLoad gives every row of `data`: its name and version as the files source records them, and its partition values (constants among them), each converted to its column's type as SQLLoad's reader would, so a value that does not convert fails here rather than in Databricks' cast.
fn s3_file(
    batch: &DirectBatch<'_>,
    data: &DataFile,
    columns: &[DesiredColumn],
) -> Result<S3File, LoaderError> {
    let Some(source) = batch.spec.files.as_ref() else {
        return Err(LoaderError::Invalid("not a files table".to_owned()));
    };
    let mut constants = vec![
        (
            SOURCE_FILE_COLUMN.to_owned(),
            Some(format!("{}{}", source.location, data.file.relative_key)),
        ),
        (SOURCE_VERSION_COLUMN.to_owned(), Some(data.file.version())),
    ];
    for (name, _) in &source.partition_types {
        let value = data
            .partitions
            .iter()
            .find(|(partition, _)| partition == name)
            .map(|(_, value)| value.clone());
        let loaded = batch.spec.source_table.loaded_as(name);
        let column_type = columns
            .iter()
            .find(|column| column.name == loaded)
            .map(|column| &column.column_type);
        let value = match (value, column_type) {
            (Some(value), Some(column_type)) => {
                Some(normal_value(&value, column_type).map_err(|reason| {
                    LoaderError::Invalid(format!(
                        "{}: {name} = {value:?} does not load as {column_type}: {reason}",
                        data.file.relative_key
                    ))
                })?)
            }
            (value, _) => value,
        };
        constants.push((loaded.to_owned(), value));
    }
    Ok(S3File {
        key: data.file.relative_key.clone(),
        constants,
    })
}