use std::collections::{BTreeMap, BTreeSet};
use crate::adapters::contract::models::WriteMode;
use crate::adapters::dispatch::main::dialect::files_destination::files_destination;
use crate::adapters::dispatch::main::dialect::sql_adapter::sql_adapter;
use crate::adapters::dispatch::main::session::clean_stage::clean_stage;
use crate::adapters::dispatch::main::session::drop_stale_staging::drop_stale_staging;
use crate::adapters::dispatch::models::Destination;
use crate::adapters::files::constants::PARQUET_SUFFIX;
use crate::adapters::files::main::clear_pending::clear_pending;
use crate::adapters::files::main::mark_pending::mark_pending;
use crate::adapters::files::main::newest_load_files::newest_load_files;
use crate::adapters::files::main::table_path::table_path;
use crate::adapters::files::models::{
FilesDestination, FolderClaim, TableFiles, TablePath, TablePathRequest,
};
use crate::adapters::files::models::{PartitionFolder, PendingPhase, PendingRun, ReplacedFiles};
use crate::column_types::models::ColumnType;
use crate::file_store::main::delete_files::delete_files;
use crate::file_store::main::write_file::write_file;
use crate::file_store::models::{OpenFolder, StoredFile};
use crate::files_source::constants::NANOSECOND_SCALE;
use crate::files_source::main::footer_columns::footer_columns;
use crate::files_source::main::read_footers::read_footers;
use crate::files_source::models::{FileColumn, StoredRead};
use crate::loader::_helpers::resolve::display_name;
use crate::loader::_helpers::table_plan::blocked_plan;
use crate::loader::errors::LoaderError;
use crate::loader::models::{
FilesSink, FilesSource, LiveFiles, LoadRun, ProjectPlan, TablePlan, TableSpec, UnreadSharing,
};
use crate::project::constants::DEFAULT_DESTINATION_PATH;
use crate::project::models::WriteStrategy;
use crate::schema_plan::models::{LiveColumn, TableAction};
use crate::store_paths::models::Location;
use crate::table_lease::constants::LEASE_REFRESH;
use crate::table_lease::main::keep_lease::keep_lease;
use crate::table_lease::main::lease_now::lease_now;
use crate::table_lease::main::release_lease::release_lease;
use crate::table_lease::models::Lease;
use bytes::Bytes;
use tokio_util::sync::CancellationToken;
pub(crate) async fn clean_staging(run: &LoadRun<'_>, plan: &ProjectPlan) {
let mut schemas: BTreeMap<(String, Option<String>, String), &TablePlan> = BTreeMap::new();
for table in plan
.tables
.iter()
.filter(|table| table.schema.blocked.is_none() && table.output.is_none())
{
let name = &table.spec.destination;
schemas
.entry((
table.spec.target.clone(),
name.database.clone(),
name.schema.clone(),
))
.or_insert(table);
}
for table in schemas.into_values() {
let Some(destination) = run.destinations.get(&table.spec.target) else {
continue;
};
let name = &table.spec.destination;
match drop_stale_staging(destination, name, table.spec.warehouse).await {
Ok(0) => {}
Ok(dropped) => {
tracing::info!(dropped, schema = %name.schema, "dropped staging tables crashed loads left")
}
Err(error) => {
tracing::warn!(%error, schema = %name.schema, "cannot clean up old staging tables")
}
}
}
}
pub(crate) async fn clean_stages(run: &LoadRun<'_>, plan: &ProjectPlan) {
let mut seen: BTreeSet<String> = BTreeSet::new();
for table in plan
.tables
.iter()
.filter(|table| table.schema.blocked.is_none())
{
let Some(destination) = run.destinations.get(&table.spec.target) else {
continue;
};
let stage = (
table.spec.target.as_str(),
table.stage.as_ref(),
&table.spec.destination,
);
let copies_files = table
.spec
.files
.as_ref()
.is_some_and(|files| files.copy_to_internal_stage);
let (checked, cleaned) = clean_stage(destination, stage, copies_files, seen).await;
seen = checked;
for (stage, removed) in cleaned {
match removed {
Ok(0) => {}
Ok(removed) => {
tracing::info!(removed, %stage, "removed load folders failed runs left on the stage")
}
Err(error) => tracing::warn!(%error, %stage, "cannot clean up the stage"),
}
}
}
}
pub(crate) fn table_folder(
spec: &TableSpec,
destination: &FilesDestination,
) -> Result<TablePath, String> {
table_path(&TablePathRequest {
template: spec
.destination_path
.as_deref()
.unwrap_or(DEFAULT_DESTINATION_PATH),
connection: &destination.connection,
store: &destination.store,
source: &spec.source_name,
schema: &spec.source_table.schema,
table: &spec.source_table.name,
})
}
pub(crate) async fn live_schema(
folder: &OpenFolder,
listed: &TableFiles,
max_footer_reads: usize,
) -> Result<Result<LiveFiles, Vec<String>>, LoaderError> {
let newest: Vec<StoredFile> = listed
.partitions
.last()
.map(|partition| newest_load_files(&parquet_files(&partition.files)))
.unwrap_or_default();
if newest.len() > max_footer_reads {
return Ok(Err(vec![format!(
"the newest load in {} wrote {} files, more footers than max_footer_reads = {max_footer_reads} allows reading for its schema; raise max_footer_reads on the target",
folder.location,
newest.len()
)]));
}
let footers = read_footers(folder, &newest).await?;
let (columns, conflicts) = footer_columns(&footers);
if !conflicts.is_empty() {
return Ok(Err(conflicts));
}
let live: Option<Vec<LiveColumn>> = (!footers.is_empty()).then(|| {
columns
.into_iter()
.map(|column| LiveColumn {
name: column.name,
column_type: column.column_type.unwrap_or_else(ColumnType::Native),
fraction_digits: None,
})
.collect()
});
Ok(Ok(LiveFiles { footers, live }))
}
pub(crate) fn parquet_files(files: &[StoredFile]) -> Vec<StoredFile> {
files
.iter()
.filter(|file| {
file.relative_key
.to_ascii_lowercase()
.ends_with(PARQUET_SUFFIX)
})
.cloned()
.collect()
}
pub(crate) fn block_shared_destinations(
mut plan: ProjectPlan,
configured: &[TableSpec],
) -> ProjectPlan {
for table in plan
.tables
.iter_mut()
.filter(|table| table.schema.blocked.is_none())
{
let others: Vec<&TableSpec> = configured
.iter()
.filter(|other| shares_destination(&table.spec, other))
.collect();
let opted_in = !table.spec.constants.is_empty()
|| others.iter().any(|other| !other.constants.is_empty());
if others.is_empty() || !opted_in {
continue;
}
let reasons = shared_problems(table, &others);
if !reasons.is_empty() {
table.schema = blocked_plan(reasons);
}
}
plan
}
pub(crate) fn mark_shared_destinations(
specs: Vec<TableSpec>,
configured: &[TableSpec],
unknown: bool,
) -> Vec<TableSpec> {
specs
.into_iter()
.map(|spec| TableSpec {
shared_destination: shared(&spec, configured)
|| (unknown && !spec.constants.is_empty()),
..spec
})
.collect()
}
pub(crate) fn unknown_sharing(
specs: &[TableSpec],
configured: &[TableSpec],
unresolved: &[String],
) -> Option<UnreadSharing> {
let unknown: Vec<&TableSpec> = specs
.iter()
.filter(|spec| spec.shared_destination && !shared(spec, configured))
.collect();
let names: Vec<String> = unknown
.iter()
.map(|spec| {
format!(
"{}.{}",
spec.source_name,
spec.source_table.qualified_name()
)
})
.collect();
(!names.is_empty()).then(|| UnreadSharing {
warning: format!(
"{} could not be read, so {} may share a destination table with it: their incremental cursors read only rows with their own constant values",
unresolved.join(", "),
names.join(", ")
),
sources: unresolved.to_vec(),
tables: unknown
.iter()
.map(|spec| display_name(spec.warehouse, &spec.destination))
.collect(),
})
}
fn shared(spec: &TableSpec, configured: &[TableSpec]) -> bool {
configured
.iter()
.any(|other| shares_destination(spec, other))
}
fn shares_destination(spec: &TableSpec, other: &TableSpec) -> bool {
other.target == spec.target
&& other.destination == spec.destination
&& (other.source_name != spec.source_name
|| other.source_table.qualified_name() != spec.source_table.qualified_name())
}
fn shared_problems(table: &TablePlan, others: &[&TableSpec]) -> Vec<String> {
let spec = &table.spec;
let names: Vec<String> = others
.iter()
.map(|other| {
format!(
"{}.{}",
other.source_name,
other.source_table.qualified_name()
)
})
.collect();
let place = format!(
"{} is also loaded by {}, and constant_columns tell their rows apart",
spec.destination.table,
names.join(", ")
);
let mut problems: Vec<String> = Vec::new();
let removes = std::iter::once(spec)
.chain(others.iter().copied())
.find(|each| {
matches!(
each.strategy,
WriteStrategy::Replace | WriteStrategy::DeleteInsert
)
});
if let Some(removing) = removes {
problems.push(format!(
"{place}; {}.{} uses write_strategy = \"{}\", which would remove the other tables' rows: use merge on a unique_key that includes a constant column, or append",
removing.source_name,
removing.source_table.qualified_name(),
removing.strategy
));
}
if table.mode == WriteMode::Recreate || table.schema.action == TableAction::Recreate {
problems.push(format!(
"{place}; --reload would drop the other tables' rows"
));
}
if spec.logged {
problems.push(format!(
"{place}; state = \"logged\" keeps one position per destination table, which they would share"
));
}
let keyed = spec.unique_key.iter().any(|key| is_constant(spec, key));
if spec.strategy == WriteStrategy::Merge && !keyed {
problems.push(format!(
"{place}; its unique_key ({}) has no constant column, so its merge could overwrite their rows: put a constant column in it",
spec.unique_key.join(", ")
));
}
let merges = spec.strategy == WriteStrategy::Merge;
let alike = others.iter().find(|other| !told_apart(spec, other, merges));
if let Some(other) = alike.filter(|_| keyed || !merges) {
problems.push(format!(
"{place}; {}.{} has the same values for {}, so their rows and incremental cursors can't be told apart: give each table a different value",
other.source_name,
other.source_table.qualified_name(),
match merges {
true => "the constant columns in the unique_key",
false => "every constant column",
}
));
}
problems
}
fn told_apart(spec: &TableSpec, other: &TableSpec, in_key: bool) -> bool {
spec.constants
.iter()
.filter(|(name, _, _)| !in_key || keyed_by(spec, name))
.any(|(name, _, value)| differs(other, name, value))
}
fn keyed_by(spec: &TableSpec, name: &str) -> bool {
spec.unique_key
.iter()
.any(|key| key.eq_ignore_ascii_case(name))
}
fn differs(other: &TableSpec, name: &str, value: &str) -> bool {
other
.constants
.iter()
.any(|(theirs, _, their_value)| theirs.eq_ignore_ascii_case(name) && their_value != value)
}
fn is_constant(spec: &TableSpec, name: &str) -> bool {
spec.constants
.iter()
.any(|(constant, _, _)| constant.eq_ignore_ascii_case(name))
}
pub(crate) fn folder_claims(
specs: &[TableSpec],
files: Option<&FilesSource>,
destinations: &BTreeMap<String, Destination>,
) -> Vec<FolderClaim> {
let mut claims: Vec<FolderClaim> = Vec::new();
for spec in specs {
let Some(target) = destinations.get(&spec.target).and_then(files_destination) else {
continue;
};
let name = format!(
"{}.{}",
spec.source_name,
spec.source_table.qualified_name()
);
if let Ok(path) = table_folder(spec, target) {
claims.push(FolderClaim {
id: name.clone(),
name,
folder: path.folder,
template: spec.destination_path.clone().unwrap_or_default(),
schema: spec.source_table.schema.clone(),
source: files.map(|files| files.location.clone()),
});
}
}
claims
}
pub(crate) fn keep_in_background(
folder: &OpenFolder,
lease: &Lease,
) -> (tokio_util::sync::DropGuard, tokio::task::JoinHandle<()>) {
let stop = CancellationToken::new();
let (stopped, folder, lease) = (stop.clone(), folder.clone(), lease.clone());
let renewal = tokio::spawn(async move {
loop {
tokio::select! {
() = stopped.cancelled() => return,
() = tokio::time::sleep(LEASE_REFRESH) => {}
}
match keep_lease(&folder, &lease, lease_now()).await {
Ok(Ok(())) => {}
Ok(Err(lost)) => {
tracing::warn!(folder = %folder.location, "{lost}");
return;
}
Err(error) => {
tracing::warn!(%error, folder = %folder.location, "cannot refresh the table's lease; retrying");
}
}
}
});
(stop.drop_guard(), renewal)
}
pub(crate) fn leased_folder(sink: &FilesSink) -> &OpenFolder {
sink.swap.as_ref().map_or(&sink.folder, |swap| &swap.live)
}
pub(crate) async fn give_back(folder: &OpenFolder, lease: &Lease) {
if let Err(error) = release_lease(folder, lease).await {
tracing::warn!(%error, folder = %folder.location, "cannot release the table's lease; it expires on its own");
}
}
pub(crate) fn nanosecond_columns(
spec: &TableSpec,
columns: &[FileColumn],
live: Option<&[LiveColumn]>,
) -> Vec<String> {
match sql_adapter(spec.warehouse).keeps_nanoseconds() {
true => columns
.iter()
.filter(|column| {
matches!(
column.stored_read,
StoredRead::Nanos | StoredRead::Epoch(NANOSECOND_SCALE)
)
})
.filter(|column| {
let loaded = spec.source_table.loaded_as(&column.name);
!spec.pinned.contains_key(loaded) && fewer_digits(live, loaded).is_none()
})
.map(|column| column.name.clone())
.collect(),
false => Vec::new(),
}
}
pub(crate) fn truncated_nanoseconds(spec: &TableSpec, live: Option<&[LiveColumn]>) -> Vec<String> {
let Some(files) = &spec.files else {
return Vec::new();
};
files
.table
.columns
.iter()
.filter(|column| {
matches!(
column.stored_read,
StoredRead::Nanos | StoredRead::Epoch(NANOSECOND_SCALE)
) && !spec
.pinned
.contains_key(spec.source_table.loaded_as(&column.name))
})
.filter_map(|column| truncated_warning(live, spec.source_table.loaded_as(&column.name)))
.collect()
}
fn truncated_warning(live: Option<&[LiveColumn]>, name: &str) -> Option<String> {
let digits = fewer_digits(live, name)?;
Some(format!(
"{name}: nanoseconds in the files, but the column holds {digits} digits after the second, so they are truncated.\nRebuild the column at (9) (--reload) to keep them."
))
}
fn fewer_digits(live: Option<&[LiveColumn]>, name: &str) -> Option<i64> {
live.into_iter()
.flatten()
.find(|existing| existing.name.eq_ignore_ascii_case(name))
.and_then(|existing| existing.fraction_digits)
.filter(|digits| *digits < i64::from(NANOSECOND_SCALE))
}
pub(crate) fn commit_deletes(
replaced: &ReplacedFiles,
partition: Option<&PartitionFolder>,
(written, last): (&BTreeSet<String>, bool),
deleted: &BTreeSet<String>,
) -> Vec<String> {
let folders: Vec<String> = match partition {
Some(partition) => written
.iter()
.map(|value| format!("{}/", partition.name(value)))
.collect(),
None => Vec::new(),
};
let mut keys: Vec<String> = written
.iter()
.flat_map(|value| replaced.by_partition.get(value).into_iter().flatten())
.cloned()
.collect();
for key in &replaced.always {
let written_folder = folders.iter().any(|folder| key.starts_with(folder));
if last || written_folder {
keys.push(key.clone());
}
}
keys.retain(|key| !deleted.contains(key));
keys.sort();
keys.dedup();
keys
}
pub(crate) async fn replace_files(
(folder, (table_folder, lease)): (&OpenFolder, (&OpenFolder, &Lease)),
file_id: &str,
(keys, run): (&[String], Option<&PendingRun>),
) -> Result<(), LoaderError> {
keep_lease(table_folder, lease, lease_now())
.await?
.map_err(LoaderError::Invalid)?;
mark_pending(folder, file_id, PendingPhase::Replacing, (keys, run)).await?;
if let Some(run) = run {
let records = OpenFolder {
store: folder.store.clone(),
location: Location {
store: folder.location.store.clone(),
prefix: run.prefix.clone(),
},
};
write_file(&records, &run.name, Bytes::from(run.json.clone())).await?;
}
delete_files(folder, keys).await?;
Ok(clear_pending(folder, file_id).await?)
}