sqlload 0.57.0

SQL data loading tooling.
//! The statement that has ClickHouse read a files table's Parquet file from S3 itself.

use crate::adapters::clickhouse::_helpers::dialect::write::typed_literal;
use crate::adapters::clickhouse::constants::{
    KEY_ID_PARAMETER, ORDERED_READ_SETTINGS, SECRET_PARAMETER, TOKEN_PARAMETER,
};
use crate::adapters::clickhouse::models::ClickHouseSql;
use crate::adapters::contract::types::SqlAdapter;
use crate::column_types::models::ColumnType;
use crate::files_source::constants::SOURCE_ROW_COLUMN;
use crate::load_naming::main::metadata_role::metadata_role;
use crate::load_naming::models::{Compat, MetadataRole};
use crate::schema_plan::models::DesiredColumn;

/// One Parquet file ClickHouse reads: its URL, the columns its own footer holds (each table column with its name in the file, which `naming`, `rename` or numbering may have changed), and the values SQLLoad gives every row of it (its name and version as the files source records them, and its partition values), by column (`None` for a partition folder the file lacks).
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct PulledFile {
    pub url: String,
    pub in_file: Vec<(String, String)>,
    pub constants: Vec<(String, Option<String>)>,
}

/// The `s3()` credentials a pull signs its requests with.
#[derive(Clone, PartialEq, Eq)]
pub(crate) struct PullCredentials {
    pub key_id: String,
    pub secret: String,
    pub token: Option<String>,
}

impl std::fmt::Debug for PullCredentials {
    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        formatter
            .debug_struct("PullCredentials")
            .field("key_id", &self.key_id)
            .finish_non_exhaustive()
    }
}

/// How a pull types each column it inserts: the staging table's type, or the build table's.
pub(crate) type SqlTypeOf<'a> = &'a dyn Fn(&ColumnType) -> String;

/// The query parameters a pull's statement names its credentials by, with their values: sent beside the SQL, never in it, so no statement text, log or error echo holds them.
pub(crate) fn credential_parameters(
    credentials: Option<&PullCredentials>,
) -> Vec<(String, String)> {
    let Some(credentials) = credentials else {
        return Vec::new();
    };
    let mut parameters = vec![
        (
            format!("param_{KEY_ID_PARAMETER}"),
            escaped(&credentials.key_id),
        ),
        (
            format!("param_{SECRET_PARAMETER}"),
            escaped(&credentials.secret),
        ),
    ];
    parameters.extend(
        credentials
            .token
            .as_deref()
            .map(|token| (format!("param_{TOKEN_PARAMETER}"), escaped(token))),
    );
    parameters
}

/// `value` as ClickHouse's escaped text, which it unescapes into the parameter's value: a backslash, tab or newline as `\\`, `\t` or `\n`.
fn escaped(value: &str) -> String {
    value
        .replace('\\', "\\\\")
        .replace('\t', "\\t")
        .replace('\n', "\\n")
}

/// `INSERT INTO into` every one of `columns` (each typed by `sql_type`) from `file`, read by ClickHouse's `s3()` signed with `credentials` (named as [query parameters](credential_parameters)), or unsigned (`NOSIGN`) when there are none. A column takes the file's constant when it has one of that name, the file's own column when its footer holds it, and NULL otherwise; `_sqlload_source_row` numbers the file's rows from 1 in stored order (read by one thread, in order); the load id and load time are the batch's. Binary columns go through `substring`, which keeps a fixed-size value's trailing zero bytes that a cast to `String` drops.
pub(crate) fn pull_insert(
    into: &str,
    (columns, sql_type): (&[DesiredColumn], SqlTypeOf<'_>),
    (file, credentials): (&PulledFile, Option<&PullCredentials>),
    (compat, load_id, loaded_at): (Compat, &str, &str),
) -> String {
    let literal = |text: &str| ClickHouseSql.render_loader_value_literal(text);
    let identifier = |name: &str| ClickHouseSql.render_identifier(name);
    let cast = |value: String, column: &DesiredColumn| {
        format!("CAST({value} AS {})", sql_type(&column.column_type))
    };
    let mut values: Vec<String> = Vec::with_capacity(columns.len());
    for column in columns {
        let constant = file.constants.iter().find(|(name, _)| *name == column.name);
        let held = file
            .in_file
            .iter()
            .find(|(held, _)| *held == column.name)
            .map(|(_, name)| name.as_str());
        let value = match (column.name.as_str(), constant, held) {
            (_, Some((_, Some(value))), _) => typed_literal(value, &column.column_type),
            (_, Some((_, None)), _) => "NULL".to_owned(),
            (_, None, _) if metadata_role(compat, &column.name) == Some(MetadataRole::LoadId) => {
                typed_literal(load_id, &column.column_type)
            }
            (_, None, _) if metadata_role(compat, &column.name) == Some(MetadataRole::LoadedAt) => {
                typed_literal(loaded_at, &column.column_type)
            }
            (SOURCE_ROW_COLUMN, None, _) => "rowNumberInAllBlocks() + 1".to_owned(),
            (_, None, Some(name)) => match column.column_type {
                ColumnType::Binary => format!("substring({}, 1)", identifier(name)),
                _ => identifier(name),
            },
            (_, None, None) => "NULL".to_owned(),
        };
        values.push(cast(value, column));
    }
    let targets: Vec<String> = columns
        .iter()
        .map(|column| identifier(&column.name))
        .collect();
    let parameter = |name: &str| format!("{{{name}:String}}");
    let mut signed = vec![literal(&file.url)];
    match credentials {
        Some(credentials) => {
            signed.extend([parameter(KEY_ID_PARAMETER), parameter(SECRET_PARAMETER)]);
            signed.extend(
                credentials
                    .token
                    .as_ref()
                    .map(|_| parameter(TOKEN_PARAMETER)),
            );
        }
        None => signed.push("NOSIGN".to_owned()),
    }
    signed.push(literal("Parquet"));
    let numbered = columns
        .iter()
        .any(|column| column.name == SOURCE_ROW_COLUMN);
    let settings = match numbered {
        true => format!(" SETTINGS {ORDERED_READ_SETTINGS}"),
        false => String::new(),
    };
    format!(
        "INSERT INTO {into} ({}) SELECT {} FROM s3({}){settings}",
        targets.join(", "),
        values.join(", "),
        signed.join(", ")
    )
}

/// The HTTP URL ClickHouse reads `key` of `bucket` at: under `endpoint` by path when there is one (or with the bucket already in it, `in_endpoint`, for virtual-hosted requests to a custom endpoint), otherwise on AWS, in `region` when known, by virtual host unless the bucket's name holds a dot, which AWS's certificate does not cover. Each path segment is percent-encoded, which ClickHouse decodes before signing.
pub(crate) fn object_url(
    (bucket, key): (&str, &str),
    region: Option<&str>,
    (endpoint, in_endpoint): (Option<&str>, bool),
) -> Result<String, String> {
    let aws = match region {
        Some(region) => format!("s3.{region}.amazonaws.com"),
        None => "s3.amazonaws.com".to_owned(),
    };
    let (base, by_path): (String, bool) = match endpoint {
        Some(endpoint) => (endpoint.trim_end_matches('/').to_owned(), !in_endpoint),
        None if bucket.contains('.') => (format!("https://{aws}"), true),
        None => (format!("https://{bucket}.{aws}"), false),
    };
    let bucket_segment = by_path.then_some(bucket);
    let mut url = reqwest::Url::parse(&base).map_err(|error| format!("{base}: {error}"))?;
    url.path_segments_mut()
        .map_err(|()| format!("{base} cannot hold a path"))?
        .pop_if_empty()
        .extend(bucket_segment.into_iter().chain(key.split('/')));
    Ok(url.to_string())
}