use crate::adapters::contract::models::WriteMode;
use crate::adapters::files::main::plan_replaced_files::plan_replaced_files;
use crate::adapters::files::main::read_table_state::read_table_state;
use crate::adapters::files::main::repair_table::repair_table;
use crate::adapters::files::models::{ReplaceRequest, ReplacedFiles, TablePath, TableState};
use crate::column_types::models::ColumnType;
use crate::file_store::main::delete_files::delete_files;
use crate::file_store::models::OpenFolder;
use crate::loader::errors::LoaderError;
use crate::loader::models::{FilesOutput, TablePlan};
pub(in crate::loader) async fn files_to_replace(
folder: &OpenFolder,
table: &TablePlan,
output: &FilesOutput,
) -> Result<ReplacedFiles, LoaderError> {
let cursor_type = table.spec.cursor.as_ref().and_then(|cursor| {
table
.schema
.write_columns
.iter()
.find(|column| column.name == cursor.column)
.map(|column| column.column_type.clone())
});
let state = repaired_state(folder, &output.path, cursor_type.as_ref()).await?;
let follows_plan = matches!(
table.mode,
WriteMode::Append | WriteMode::DeleteInsert { .. }
);
if follows_plan && state.commits != output.commits {
return Err(LoaderError::Transient(format!(
"{}: another load changed the table's files in {} after this load was planned, so what it would read may already be loaded; stopped before writing. Run the load again",
table.spec.destination.table, output.path.folder
)));
}
plan_replaced_files(&ReplaceRequest {
mode: &table.mode,
listed: &state.listed,
partition: output.path.partition.as_ref(),
cursor_type: cursor_type.as_ref(),
})
.map_err(LoaderError::Invalid)
}
async fn repaired_state(
folder: &OpenFolder,
path: &TablePath,
cursor_type: Option<&ColumnType>,
) -> Result<TableState, LoaderError> {
let state = read_table_state(folder, path, cursor_type).await?;
repair_table(folder, (&state.pending, &state.records)).await?;
let state = match state.pending.is_empty() && !state.listed.keys().is_empty() {
true => state,
false => read_table_state(folder, path, cursor_type).await?,
};
delete_files(folder, &state.stale_loaded).await?;
match state.listed.problems.first() {
Some(problem) => Err(LoaderError::Invalid(problem.clone())),
None => Ok(state),
}
}