use crate::adapters::contract::models::WriteTarget;
use crate::adapters::contract::types::SqlAdapter;
use crate::adapters::databricks::_helpers::write::copy_select::typed_value;
use crate::adapters::databricks::constants::S3_COPY_OPTIONS;
use crate::adapters::databricks::models::DatabricksSql;
use crate::files_source::constants::SOURCE_ROW_COLUMN;
use crate::load_naming::main::metadata_role::metadata_role;
use crate::load_naming::models::MetadataRole;
use crate::schema_plan::models::DesiredColumn;
use arrow_schema::{Field, Schema};
pub(crate) const FILE_PATH: &str = "url_decode(replace(_metadata.file_path, '+', '%2B'))";
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct S3Batch {
pub folder: String,
pub files: Vec<S3File>,
pub in_files: Vec<(String, String)>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct S3File {
pub key: String,
pub constants: Vec<(String, Option<String>)>,
}
pub(crate) fn s3_copy_statement(
(into, columns): (&str, &[DesiredColumn]),
target: &WriteTarget<'_>,
batch: &S3Batch,
loaded_at: &str,
) -> String {
let literal = |text: &str| DatabricksSql.render_loader_value_literal(text);
let mut values: Vec<String> = Vec::with_capacity(columns.len());
for column in columns {
values.push(format!(
"{} AS {}",
typed_value(
target,
column,
&column_value(target, batch, column, loaded_at)
),
DatabricksSql.render_identifier(&column.name)
));
}
let keys: Vec<String> = batch.files.iter().map(|file| literal(&file.key)).collect();
format!(
"COPY INTO {into} FROM (SELECT {} FROM {}) FILEFORMAT = PARQUET FILES = ({}) {S3_COPY_OPTIONS}",
values.join(", "),
literal(&batch.folder),
keys.join(", ")
)
}
fn column_value(
target: &WriteTarget<'_>,
batch: &S3Batch,
column: &DesiredColumn,
loaded_at: &str,
) -> String {
let literal = |text: &str| DatabricksSql.render_loader_value_literal(text);
let mut given: Vec<(&S3File, Option<&String>)> = Vec::new();
for file in &batch.files {
if let Some((_, value)) = file.constants.iter().find(|(name, _)| *name == column.name) {
given.push((file, value.as_ref()));
}
}
let held = batch.in_files.iter().find(|(name, _)| *name == column.name);
match (metadata_role(target.compat, &column.name), held) {
_ if !given.is_empty() => per_file(batch, &given),
(Some(MetadataRole::LoadId), _) => literal(target.load_id),
(Some(MetadataRole::LoadedAt), _) => literal(loaded_at),
_ if column.name == SOURCE_ROW_COLUMN => "_metadata.row_index + 1".to_owned(),
(_, Some((_, held))) => DatabricksSql.render_exact_identifier(held),
(_, None) => "NULL".to_owned(),
}
}
fn per_file(batch: &S3Batch, given: &[(&S3File, Option<&String>)]) -> String {
let literal = |text: &str| DatabricksSql.render_loader_value_literal(text);
let picks: Vec<String> = given
.iter()
.map(|(file, value)| {
format!(
"WHEN {} THEN {}",
literal(&format!("{}{}", batch.folder, file.key)),
value.map_or_else(|| "NULL".to_owned(), |value| literal(value))
)
})
.collect();
format!(
"CASE {FILE_PATH} {} ELSE raise_error(concat('sqlload: COPY INTO read a file outside the batch: ', _metadata.file_path)) END",
picks.join(" ")
)
}
pub(crate) fn copy_groups<T>(files: Vec<(T, &Schema)>, limit: usize) -> Vec<Vec<T>> {
let mut groups: Vec<(Vec<T>, Vec<&Field>)> = Vec::new();
for (file, schema) in files {
let joins = groups
.iter()
.position(|(members, held)| members.len() < limit && agrees(held, schema));
let index = joins.unwrap_or_else(|| {
groups.push((Vec::new(), Vec::new()));
groups.len() - 1
});
let (members, held) = &mut groups[index];
members.push(file);
for field in schema.fields() {
if !held
.iter()
.any(|other| other.name().eq_ignore_ascii_case(field.name()))
{
held.push(field);
}
}
}
groups.into_iter().map(|(members, _)| members).collect()
}
fn agrees(held: &[&Field], schema: &Schema) -> bool {
for field in schema.fields() {
for other in held {
if other.name().eq_ignore_ascii_case(field.name())
&& other.data_type() != field.data_type()
{
return false;
}
}
}
true
}