#[cfg(feature = "duckdb")]
use std::collections::{BTreeMap, BTreeSet};
use crate::adapters::contract::models::TableName;
#[cfg(feature = "duckdb")]
use crate::adapters::dispatch::main::dialect::sql_adapter::sql_adapter;
#[cfg(feature = "duckdb")]
use crate::adapters::dispatch::main::session::execute::execute;
#[cfg(feature = "duckdb")]
use crate::adapters::dispatch::main::session::text_values::text_values;
use crate::adapters::dispatch::models::Destination;
#[cfg(feature = "duckdb")]
use crate::column_types::models::Warehouse;
use crate::loader::errors::LoaderError;
#[cfg(feature = "duckdb")]
use crate::run_log::constants::{RUNS_TABLE, UNFINISHED_SEPARATOR, UNFINISHED_TABLE};
#[cfg(feature = "duckdb")]
use crate::run_log::main::unfinished_recovery::unfinished_recovery;
pub async fn recover_unfinished(
destination: &Destination,
tables: &[&TableName],
) -> Result<Vec<String>, LoaderError> {
#[cfg(feature = "duckdb")]
if let Destination::DuckDb(_) = destination {
return recover_duckdb(destination, tables).await;
}
let _ = (destination, tables);
Ok(Vec::new())
}
#[cfg(feature = "duckdb")]
async fn recover_duckdb(
destination: &Destination,
tables: &[&TableName],
) -> Result<Vec<String>, LoaderError> {
let mut schemas: BTreeMap<(Option<String>, String), BTreeSet<String>> = BTreeMap::new();
for table in tables {
schemas
.entry((table.database.clone(), table.schema.clone()))
.or_default()
.insert(table.table.clone());
}
let mut removed: Vec<String> = Vec::new();
for (place, names) in schemas {
removed.extend(recover_schema(destination, place, &names).await?);
}
Ok(removed)
}
#[cfg(feature = "duckdb")]
async fn recover_schema(
destination: &Destination,
(database, schema): (Option<String>, String),
names: &BTreeSet<String>,
) -> Result<Vec<String>, LoaderError> {
let adapter = sql_adapter(Warehouse::DuckDb);
let text = |value: &str| adapter.render_loader_value_literal(value);
let catalog = database
.as_deref()
.map_or_else(|| "current_database()".to_owned(), text);
let listed: Vec<String> = [UNFINISHED_TABLE, RUNS_TABLE]
.into_iter()
.chain(names.iter().map(String::as_str))
.map(text)
.collect();
let query = format!(
"SELECT table_name FROM information_schema.tables WHERE table_catalog = {catalog} AND table_schema = {} AND table_name IN ({})",
text(&schema),
listed.join(", ")
);
let present: BTreeSet<String> = text_values(destination, query).await?.into_iter().collect();
if !present.contains(UNFINISHED_TABLE) {
return Ok(Vec::new());
}
let markers = TableName {
database,
schema: schema.clone(),
table: UNFINISHED_TABLE.to_owned(),
};
let marked: Vec<String> = names.iter().map(|name| text(name)).collect();
let query = format!(
"SELECT DISTINCT table_name || chr(31) || load_id || chr(31) || load_id_column FROM {} WHERE table_name IN ({})",
adapter.render_qualified_name(&markers),
marked.join(", ")
);
let mut removed: Vec<String> = Vec::new();
for line in text_values(destination, query).await? {
let fields: Vec<&str> = line.split(UNFINISHED_SEPARATOR).collect();
let [table, load_id, column] = fields.as_slice() else {
continue;
};
let name = TableName {
table: (*table).to_owned(),
..markers.clone()
};
let exists = (
present.contains(*table),
present.contains(RUNS_TABLE),
Warehouse::DuckDb,
);
execute(
destination,
unfinished_recovery(&name, (column, load_id), exists),
)
.await?;
removed.push(format!(
"{schema}.{table}: removed the rows of load {load_id}, which stopped before its last commit, so this load reads from before it"
));
}
Ok(removed)
}