use crate::adapters::contract::models::WriteMode;
use crate::adapters::files::main::plan_replaced_files::plan_replaced_files;
use crate::adapters::files::main::read_table_state::read_table_state;
use crate::adapters::files::main::repair_table::repair_table;
use crate::adapters::files::models::{ReplaceRequest, ReplacedFiles, TablePath, TableState};
use crate::column_types::models::ColumnType;
use crate::file_store::main::delete_files::delete_files;
use crate::file_store::main::open_folder::open_folder;
use crate::file_store::models::OpenFolder;
use crate::loader::errors::LoaderError;
use crate::loader::models::{FilesOutput, TablePlan};
use crate::table_lease::models::Lease;
pub(in crate::loader) async fn files_to_replace(
(folder, lease): (&OpenFolder, &Lease),
table: &TablePlan,
output: &FilesOutput,
) -> Result<(ReplacedFiles, Option<(OpenFolder, std::fs::File)>), LoaderError> {
let cursor_type = table.spec.cursor.as_ref().and_then(|cursor| {
table
.schema
.write_columns
.iter()
.find(|column| column.name == cursor.column)
.map(|column| column.column_type.clone())
});
let name = &table.spec.destination.table;
let state = repaired_state((folder, lease), (&output.path, name), cursor_type.as_ref()).await?;
let follows_plan = matches!(
table.mode,
WriteMode::Append | WriteMode::DeleteInsert { .. }
);
if follows_plan && state.commits != output.commits {
return Err(LoaderError::Transient(format!(
"{}: another load changed the table's files in {} after this load was planned, so what it would read may already be loaded; stopped before writing. Run the load again",
table.spec.destination.table, output.path.folder
)));
}
let replaced = plan_replaced_files(&ReplaceRequest {
mode: &table.mode,
listed: &state.listed,
partition: output.path.partition.as_ref(),
cursor_type: cursor_type.as_ref(),
})
.map_err(LoaderError::Invalid)?;
Ok((
replaced,
files_staging(output, &table.mode, &lease.load_id)?,
))
}
fn files_staging(
output: &FilesOutput,
mode: &WriteMode,
load_id: &str,
) -> Result<Option<(OpenFolder, std::fs::File)>, LoaderError> {
let staged = output.path.staging(mode, load_id).map_err(|error| {
let message = format!(
"{}: cannot check whether the filesystem swaps folders in one rename: {error}",
output.path.folder
);
match error.kind() == std::io::ErrorKind::NotFound {
true => LoaderError::Transient(message),
false => LoaderError::Invalid(message),
}
})?;
staged
.map(|(staged, held)| Ok((open_folder(&staged)?, held)))
.transpose()
}
async fn repaired_state(
(folder, lease): (&OpenFolder, &Lease),
(path, name): (&TablePath, &str),
cursor_type: Option<&ColumnType>,
) -> Result<TableState, LoaderError> {
let state = read_table_state(folder, path, cursor_type).await?;
repair_table((folder, lease), (&state.pending, &state.records))
.await?
.map_err(|lost| LoaderError::Transient(format!("{name}: {lost}")))?;
let state = match state.pending.is_empty() {
true => state,
false => read_table_state(folder, path, cursor_type).await?,
};
delete_files(folder, &state.stale_loaded).await?;
match state.listed.problems.first() {
Some(problem) => Err(LoaderError::Invalid(problem.clone())),
None => Ok(state),
}
}