use crate::adapters::clickhouse::_helpers::dialect::publish::{
partition_key_columns_query, partition_key_query, partition_rows_query, replace_partition,
sorted_insert,
};
use crate::adapters::clickhouse::_helpers::session::http::send;
use crate::adapters::clickhouse::_helpers::session::rows::text_rows;
use crate::adapters::clickhouse::constants::{
EXCHANGE_UNSUPPORTED_CODES, LIGHTWEIGHT_DELETE_MODE, LIGHTWEIGHT_DELETE_REFUSED_CODE,
MEMORY_LIMIT_EXCEEDED_CODE, PUBLISH_PREFIX,
};
use crate::adapters::clickhouse::models::{ClickHouseDestination, ClickHouseSql, PublishStep};
use crate::adapters::contract::types::SqlAdapter;
use crate::http::errors::ApiError;
pub(crate) fn publish_step(statement: &str) -> Result<Option<PublishStep>, ApiError> {
let Some(json) = statement.strip_prefix(PUBLISH_PREFIX) else {
return Ok(None);
};
serde_json::from_str(json)
.map(Some)
.map_err(|error| ApiError::Protocol(format!("a malformed publish step: {error}")))
}
pub(crate) async fn run_step(
destination: &ClickHouseDestination,
statement: &str,
) -> Result<(), ApiError> {
match publish_step(statement)? {
None => send(destination, None, statement.as_bytes().to_vec())
.await
.map(|_| ()),
Some(PublishStep::Exchange {
target,
build,
aside,
}) => exchange(destination, (&target, &build, &aside)).await,
Some(PublishStep::ReplacePartition {
target,
copy,
partition,
}) => replace_or_drop(destination, (&target, ©, &partition)).await,
Some(PublishStep::DeleteRows { table, condition }) => {
delete_rows(destination, (&table, &condition)).await
}
Some(PublishStep::SortedInsert {
database,
table,
columns,
insert,
}) => insert_sorted(destination, (&database, &table), &columns, &insert).await,
Some(PublishStep::Partitions { .. }) => Err(ApiError::Protocol(
"a partition publish cannot run inside another".to_owned(),
)),
}
}
async fn insert_sorted(
destination: &ClickHouseDestination,
(database, table): (&str, &str),
columns: &[String],
insert: &str,
) -> Result<(), ApiError> {
let first_values = |query: String| async move {
let text = send(
destination,
None,
format!("{query} FORMAT JSONCompactStrings").into_bytes(),
)
.await?;
Ok::<Vec<String>, ApiError>(
text_rows(&text)?
.into_iter()
.filter_map(|row| row.into_iter().next().flatten())
.collect(),
)
};
let key = first_values(partition_key_query(database, table)).await?;
let key_columns = first_values(partition_key_columns_query(database, table)).await?;
let key = key.first().map(String::as_str).unwrap_or_default();
let sorted = sorted_insert(insert, key, &key_columns, columns);
match send(destination, None, sorted.clone().into_bytes()).await {
Ok(_) => Ok(()),
Err(error) if sorted != insert && error.code() == Some(MEMORY_LIMIT_EXCEEDED_CODE) => {
tracing::debug!(%error, "the sorted insert ran out of memory; inserting unsorted");
let delta = format!(
"{}.{}",
ClickHouseSql.render_identifier(database),
ClickHouseSql.render_identifier(table)
);
send(
destination,
None,
format!("TRUNCATE TABLE {delta}").into_bytes(),
)
.await?;
send(destination, None, insert.as_bytes().to_vec())
.await
.map(|_| ())
}
Err(error) => Err(error),
}
}
pub(crate) async fn exchange(
destination: &ClickHouseDestination,
(target, build, aside): (&str, &str, &str),
) -> Result<(), ApiError> {
let exchanged = send(
destination,
None,
format!("EXCHANGE TABLES {target} AND {build}").into_bytes(),
)
.await;
match exchanged {
Ok(_) => Ok(()),
Err(error)
if error
.code()
.is_some_and(|code| EXCHANGE_UNSUPPORTED_CODES.contains(&code)) =>
{
tracing::debug!(%error, "EXCHANGE TABLES is unsupported here; swapping by renames");
send(
destination,
None,
format!(
"RENAME TABLE {target} TO {aside}, {build} TO {target}, {aside} TO {build}"
)
.into_bytes(),
)
.await
.map(|_| ())
}
Err(error) => Err(error),
}
}
async fn replace_or_drop(
destination: &ClickHouseDestination,
(target, copy, partition): (&str, &str, &str),
) -> Result<(), ApiError> {
let counted = send(
destination,
None,
partition_rows_query(copy, partition).into_bytes(),
)
.await?;
let statement = match counted.trim() {
"0" => format!(
"ALTER TABLE {target} DROP PARTITION ID {}",
ClickHouseSql.render_loader_value_literal(partition)
),
_ => replace_partition(target, copy, partition),
};
send(destination, None, statement.into_bytes())
.await
.map(|_| ())
}
async fn delete_rows(
destination: &ClickHouseDestination,
(table, condition): (&str, &str),
) -> Result<(), ApiError> {
let deleted = send(
destination,
None,
format!(
"DELETE FROM {table} WHERE {condition} SETTINGS {} = '{}', lightweight_deletes_sync = 2",
LIGHTWEIGHT_DELETE_MODE.0, LIGHTWEIGHT_DELETE_MODE.1
)
.into_bytes(),
)
.await;
match deleted {
Ok(_) => Ok(()),
Err(error) if error.code() == Some(LIGHTWEIGHT_DELETE_REFUSED_CODE) => {
tracing::debug!(%error, "a lightweight DELETE is refused here; rewriting the parts");
send(
destination,
None,
format!("ALTER TABLE {table} DELETE WHERE {condition}").into_bytes(),
)
.await
.map(|_| ())
}
Err(error) => Err(error),
}
}