sqlload 0.34.0

SQL data loading tooling.
//! Statements as the runner carries them out: sent as they are, or a publish step it carries out itself.

use crate::adapters::clickhouse::_helpers::dialect::publish::{
    partition_key_columns_query, partition_key_query, partition_rows_query, replace_partition,
    sorted_insert,
};
use crate::adapters::clickhouse::_helpers::session::http::send;
use crate::adapters::clickhouse::_helpers::session::rows::text_rows;
use crate::adapters::clickhouse::constants::{
    EXCHANGE_UNSUPPORTED_CODES, LIGHTWEIGHT_DELETE_MODE, LIGHTWEIGHT_DELETE_REFUSED_CODE,
    MEMORY_LIMIT_EXCEEDED_CODE, PUBLISH_PREFIX,
};
use crate::adapters::clickhouse::models::{ClickHouseDestination, ClickHouseSql, PublishStep};
use crate::adapters::contract::types::SqlAdapter;
use crate::http::errors::ApiError;

/// The publish step `statement` stands for, if it is one.
pub(crate) fn publish_step(statement: &str) -> Result<Option<PublishStep>, ApiError> {
    let Some(json) = statement.strip_prefix(PUBLISH_PREFIX) else {
        return Ok(None);
    };
    serde_json::from_str(json)
        .map(Some)
        .map_err(|error| ApiError::Protocol(format!("a malformed publish step: {error}")))
}

/// Run one statement, or the exchange, partition replace, row delete or sorted insert it stands for; a partition publish here is a mistake, as it only follows from a write.
pub(crate) async fn run_step(
    destination: &ClickHouseDestination,
    statement: &str,
) -> Result<(), ApiError> {
    match publish_step(statement)? {
        None => send(destination, None, statement.as_bytes().to_vec())
            .await
            .map(|_| ()),
        Some(PublishStep::Exchange {
            target,
            build,
            aside,
        }) => exchange(destination, (&target, &build, &aside)).await,
        Some(PublishStep::ReplacePartition {
            target,
            copy,
            partition,
        }) => replace_or_drop(destination, (&target, &copy, &partition)).await,
        Some(PublishStep::DeleteRows { table, condition }) => {
            delete_rows(destination, (&table, &condition)).await
        }
        Some(PublishStep::SortedInsert {
            database,
            table,
            columns,
            insert,
        }) => insert_sorted(destination, (&database, &table), &columns, &insert).await,
        Some(PublishStep::Partitions { .. }) => Err(ApiError::Protocol(
            "a partition publish cannot run inside another".to_owned(),
        )),
    }
}

/// Run `insert` into `database`.`table`, ordered by the partition key the server reports for the table when every column it reads is among `columns` (see [`sorted_insert`]); a sort that runs out of memory is replaced by the unsorted insert into the emptied table.
async fn insert_sorted(
    destination: &ClickHouseDestination,
    (database, table): (&str, &str),
    columns: &[String],
    insert: &str,
) -> Result<(), ApiError> {
    let first_values = |query: String| async move {
        let text = send(
            destination,
            None,
            format!("{query} FORMAT JSONCompactStrings").into_bytes(),
        )
        .await?;
        Ok::<Vec<String>, ApiError>(
            text_rows(&text)?
                .into_iter()
                .filter_map(|row| row.into_iter().next().flatten())
                .collect(),
        )
    };
    let key = first_values(partition_key_query(database, table)).await?;
    let key_columns = first_values(partition_key_columns_query(database, table)).await?;
    let key = key.first().map(String::as_str).unwrap_or_default();
    let sorted = sorted_insert(insert, key, &key_columns, columns);
    match send(destination, None, sorted.clone().into_bytes()).await {
        Ok(_) => Ok(()),
        Err(error) if sorted != insert && error.code() == Some(MEMORY_LIMIT_EXCEEDED_CODE) => {
            tracing::debug!(%error, "the sorted insert ran out of memory; inserting unsorted");
            let delta = format!(
                "{}.{}",
                ClickHouseSql.render_identifier(database),
                ClickHouseSql.render_identifier(table)
            );
            send(
                destination,
                None,
                format!("TRUNCATE TABLE {delta}").into_bytes(),
            )
            .await?;
            send(destination, None, insert.as_bytes().to_vec())
                .await
                .map(|_| ())
        }
        Err(error) => Err(error),
    }
}

/// Swap `build` in for `target` with `EXCHANGE TABLES`, atomic in an `Atomic` database; where the database cannot exchange tables, by one `RENAME TABLE` moving `target` to `aside`, `build` to `target` and `aside` to `build`, so `build` holds the old table either way.
pub(crate) async fn exchange(
    destination: &ClickHouseDestination,
    (target, build, aside): (&str, &str, &str),
) -> Result<(), ApiError> {
    let exchanged = send(
        destination,
        None,
        format!("EXCHANGE TABLES {target} AND {build}").into_bytes(),
    )
    .await;
    match exchanged {
        Ok(_) => Ok(()),
        Err(error)
            if error
                .code()
                .is_some_and(|code| EXCHANGE_UNSUPPORTED_CODES.contains(&code)) =>
        {
            tracing::debug!(%error, "EXCHANGE TABLES is unsupported here; swapping by renames");
            send(
                destination,
                None,
                format!(
                    "RENAME TABLE {target} TO {aside}, {build} TO {target}, {aside} TO {build}"
                )
                .into_bytes(),
            )
            .await
            .map(|_| ())
        }
        Err(error) => Err(error),
    }
}

/// Replace `target`'s `partition` with `copy`'s, or drop it when `copy` holds none of its rows, as ClickHouse 26.6 and later refuse to replace a partition from a table with no parts in it; either way one statement, so readers see the old partition or the new.
async fn replace_or_drop(
    destination: &ClickHouseDestination,
    (target, copy, partition): (&str, &str, &str),
) -> Result<(), ApiError> {
    let counted = send(
        destination,
        None,
        partition_rows_query(copy, partition).into_bytes(),
    )
    .await?;
    let statement = match counted.trim() {
        "0" => format!(
            "ALTER TABLE {target} DROP PARTITION ID {}",
            ClickHouseSql.render_loader_value_literal(partition)
        ),
        _ => replace_partition(target, copy, partition),
    };
    send(destination, None, statement.into_bytes())
        .await
        .map(|_| ())
}

/// Delete `table`'s rows matching `condition` with a lightweight `DELETE`, which only marks them (with [`LIGHTWEIGHT_DELETE_MODE`], and finished before it returns), or by rewriting their parts with `ALTER TABLE ... DELETE` where the table refuses one.
async fn delete_rows(
    destination: &ClickHouseDestination,
    (table, condition): (&str, &str),
) -> Result<(), ApiError> {
    let deleted = send(
        destination,
        None,
        format!(
            "DELETE FROM {table} WHERE {condition} SETTINGS {} = '{}', lightweight_deletes_sync = 2",
            LIGHTWEIGHT_DELETE_MODE.0, LIGHTWEIGHT_DELETE_MODE.1
        )
        .into_bytes(),
    )
    .await;
    match deleted {
        Ok(_) => Ok(()),
        Err(error) if error.code() == Some(LIGHTWEIGHT_DELETE_REFUSED_CODE) => {
            tracing::debug!(%error, "a lightweight DELETE is refused here; rewriting the parts");
            send(
                destination,
                None,
                format!("ALTER TABLE {table} DELETE WHERE {condition}").into_bytes(),
            )
            .await
            .map(|_| ())
        }
        Err(error) => Err(error),
    }
}