use crate::adapters::contract::models::WriteMode;
use crate::adapters::files::main::read_table_state::read_table_state;
use crate::adapters::files::main::write_notes::write_notes;
use crate::adapters::files::models::FilesDestination;
use crate::file_store::main::open_folder::open_folder;
use crate::files_source::models::{FilesRead, LoadedFile};
use crate::loader::_helpers::destination::{live_schema, table_folder};
use crate::loader::_helpers::resolve::table_range;
use crate::loader::_helpers::sources::{FilesPlanRequest, LoadedSource, plan_files};
use crate::loader::_helpers::table_plan::{CommitRequest, Window, commit_rule};
use crate::loader::_helpers::table_plan::{
blocked_plan, blocked_table, captured_columns, captured_mode, captured_problems, check_output,
desired_columns, encoding_problems, read_columns, read_window,
};
use crate::loader::errors::LoaderError;
use crate::loader::main::_files_schema::files_schema;
use crate::loader::main::_files_window::files_window;
use crate::loader::models::{
CommitRule, FilesContext, FilesOutput, FilesWindowRequest, LiveFiles, PlanOptions, TablePlan,
TableSpec, WholeReason,
};
use crate::project::models::WriteStrategy;
use crate::schema_plan::models::{LiveColumn, SchemaPlan};
use crate::table_lease::main::lease_holder::lease_holder;
use crate::table_lease::main::lease_now::lease_now;
pub(in crate::loader) async fn plan_files_table(
spec: &TableSpec,
destination: &FilesDestination,
options: &PlanOptions,
) -> Result<TablePlan, LoaderError> {
let FilesContext {
read_columns,
desired,
path,
cursor_type,
folder,
state,
} = match files_context(spec, destination, options).await? {
Ok(context) => context,
Err(blocked) => return Ok(blocked),
};
let LiveFiles { footers, live } =
match live_schema(&folder, &state.listed, destination.max_footer_reads).await? {
Ok(current) => current,
Err(problems) => {
return Ok(TablePlan {
exists: state.exists,
..blocked_table(spec, problems)
});
}
};
let schema = files_schema(spec, &desired, live.as_deref(), options);
let (schema, read_columns) = captured_columns(spec, options, schema, read_columns);
let window_request = FilesWindowRequest {
spec,
options,
cursor_type: cursor_type.as_ref(),
partition: path.partition.as_ref(),
listed: &state.listed,
footers: &footers,
read_columns: &read_columns,
};
let (mut schema, window, files) =
planned_read(&window_request, (&state.loaded, live.as_deref()), schema).await?;
let read = window.as_ref().and_then(|window| read_window(spec, window));
let (source_filter, mode, cursor_note) = match window {
Some(window) => (window.filter, window.mode, window.note),
None => (None, WriteMode::Append, None),
};
let mode = captured_mode(spec, mode);
let notes = write_notes(&mode, &path.folder);
let commits = commit_rule(&CommitRequest {
spec,
window: read.as_ref(),
cursor_type: cursor_type.as_ref(),
read_columns: &read_columns,
files: files.as_ref(),
mode: &mode,
});
let commits = match commits {
Ok(_) if destination.encoding.single_file => CommitRule::Whole(WholeReason::SingleFile),
Ok(commits) => commits,
Err(problem) => {
if schema.blocked.is_none() {
schema = blocked_plan(vec![problem]);
}
CommitRule::Whole(WholeReason::Blocked)
}
};
Ok(TablePlan {
spec: spec.clone(),
read_columns,
live,
schema,
mode,
source_filter,
source_parameters: vec![],
cursor_note,
window: read,
route: None,
files,
exists: state.exists,
logged: None,
commits,
stage: None,
planned_stage: None,
fetch_memory: None,
output: Some(FilesOutput {
path,
encoding: destination.encoding.clone(),
listed: state.listed.keys().len(),
footers_read: footers.len(),
max_footer_reads: destination.max_footer_reads,
notes,
pending: state.pending,
commits: state.commits,
}),
})
}
async fn planned_read(
request: &FilesWindowRequest<'_>,
(loaded, live): (&[LoadedFile], Option<&[LiveColumn]>),
schema: SchemaPlan,
) -> Result<(SchemaPlan, Option<Window>, Option<FilesRead>), LoaderError> {
if schema.blocked.is_some() {
return Ok((schema, None, None));
}
let ranged = match table_range(request.options, request.spec, request.cursor_type, true) {
Ok(ranged) => ranged,
Err(problem) => return Ok((blocked_plan(vec![problem]), None, None)),
};
let request = &FilesWindowRequest {
options: &ranged,
..*request
};
let window = match files_window(request) {
Ok(window) => window,
Err(problem) => return Ok((blocked_plan(vec![problem]), None, None)),
};
let Some(files) = &request.spec.files else {
return Ok((schema, Some(window), None));
};
let written: Vec<String> = schema
.write_columns
.iter()
.map(|column| column.name.clone())
.collect();
let plan_request = FilesPlanRequest {
spec: request.spec,
files,
loaded: LoadedSource::Recorded(loaded),
live,
read_columns: request.read_columns,
written: &written,
bounds: (window.bounds.0.as_deref(), window.bounds.1.as_deref()),
options: request.options,
};
let planned = plan_files(&plan_request).await?;
let window = Window {
filter: None,
..window
};
Ok(match planned {
Ok(read) => (schema, Some(window), Some(read)),
Err(problem) => (blocked_plan(vec![problem]), Some(window), None),
})
}
async fn files_context(
spec: &TableSpec,
destination: &FilesDestination,
options: &PlanOptions,
) -> Result<Result<FilesContext, TablePlan>, LoaderError> {
let read_columns = match read_columns(spec) {
Ok(read) => read,
Err(problems) => return Ok(Err(blocked_table(spec, problems))),
};
let desired = desired_columns(spec, &read_columns);
let path = match table_folder(spec, destination) {
Ok(path) => path,
Err(problem) => return Ok(Err(blocked_table(spec, vec![problem]))),
};
let (cursor_type, mut problems) = check_output(spec, &path, &desired);
problems.extend(captured_problems(spec, &path, options));
problems.extend(encoding_problems(spec, &path, &destination.encoding));
let folder = open_folder(&path.folder)?;
let state = read_table_state(&folder, &path, cursor_type.as_ref()).await?;
problems.extend(state.listed.problems.clone());
let reads_records = match spec.strategy {
WriteStrategy::Append => true,
WriteStrategy::DeleteInsert => spec.cursor.is_some() && options.start_cursor.is_none(),
_ => false,
};
problems.extend(
state
.old_records
.clone()
.filter(|_| reads_records && !options.reload),
);
problems.extend(lease_holder(&folder, lease_now()).await?);
if !problems.is_empty() {
return Ok(Err(TablePlan {
exists: state.exists,
..blocked_table(spec, problems)
}));
}
Ok(Ok(FilesContext {
read_columns,
desired,
path,
cursor_type,
folder,
state,
}))
}