use crate::file_store::main::delete_files::delete_files;
use crate::file_store::main::open_folder::open_folder;
use crate::file_store::models::OpenFolder;
use crate::files_destination::main::clear_pending::clear_pending;
use crate::files_destination::main::close_writer::close_writer;
use crate::files_destination::main::mark_pending::mark_pending;
use crate::files_destination::main::open_writer::open_writer;
use crate::files_destination::main::write_rows::write_rows;
use crate::files_destination::models::{FilesWriter, PendingPhase, WriterRequest};
use crate::loader::_helpers::clock::now_utc_text;
use crate::loader::_helpers::sources::read_source;
use crate::loader::errors::LoaderError;
use crate::loader::main::_files_repair::files_to_replace;
use crate::loader::models::{FilesOutput, SourceReader, TablePlan};
use crate::masking::main::mask_batch::mask_batch;
use crate::write_sql::models::WriteMode;
pub(in crate::loader) async fn load_files_table(
reader: &SourceReader,
table: &TablePlan,
output: &FilesOutput,
) -> Result<u64, LoaderError> {
let spec = &table.spec;
let folder = open_folder(&output.path.folder)?;
let replaced = files_to_replace(&folder, table, output).await?;
let load_id = uuid::Uuid::new_v4().to_string();
let loaded_at = now_utc_text();
let mut writer = open_writer(WriterRequest {
folder: folder.clone(),
path: &output.path,
cursor_column: spec.cursor.as_ref().map(|cursor| cursor.column.as_str()),
columns: &table.schema.write_columns,
load_id: &load_id,
loaded_at: &loaded_at,
})
.map_err(LoaderError::Invalid)?;
let keep_schema = matches!(table.mode, WriteMode::Replace | WriteMode::Recreate);
mark_pending(&folder, &load_id, PendingPhase::Writing, &[]).await?;
let written = async {
let rows = read_source(reader, table, async |batch| {
let batch = mask_batch(batch, &spec.masked)?;
write_rows(&mut writer, batch)
.await
.map(|_| ())
.map_err(|error| error.to_string())
})
.await?;
close_writer(&mut writer, keep_schema).await?;
Ok::<u64, LoaderError>(rows)
}
.await;
let rows = match written {
Ok(rows) => rows,
Err(error) => {
abandon(&folder, &writer).await;
return Err(error);
}
};
replace_files(&folder, &load_id, &replaced.keys(&writer.partitions)).await?;
Ok(rows)
}
async fn replace_files(
folder: &OpenFolder,
load_id: &str,
keys: &[String],
) -> Result<(), LoaderError> {
mark_pending(folder, load_id, PendingPhase::Replacing, keys).await?;
delete_files(folder, keys).await?;
Ok(clear_pending(folder, load_id).await?)
}
async fn abandon(folder: &OpenFolder, writer: &FilesWriter) {
if let Err(error) = delete_files(folder, &writer.written).await {
tracing::warn!(%error, folder = %folder.location, "cannot delete the files of a failed load; the next plan or load deletes them");
}
}