sqlload 0.61.0

SQL data loading tooling.
//! Plan one table into a files target: its folder, unfinished loads, the live schema from the newest load's footers, the cursor window, and for a files source the files it reads.

use crate::adapters::contract::models::WriteMode;
use crate::adapters::files::main::read_table_state::read_table_state;
use crate::adapters::files::main::write_notes::write_notes;
use crate::adapters::files::models::FilesDestination;
use crate::file_store::main::open_folder::open_folder;
use crate::files_source::models::{FilesRead, LoadedFile};
use crate::loader::_helpers::destination::{live_schema, table_folder};
use crate::loader::_helpers::resolve::table_range;
use crate::loader::_helpers::sources::{FilesPlanRequest, LoadedSource, plan_files};
use crate::loader::_helpers::table_plan::{CommitRequest, Window, commit_rule};
use crate::loader::_helpers::table_plan::{
    blocked_plan, blocked_table, captured_columns, captured_mode, captured_problems, check_output,
    desired_columns, encoding_problems, read_columns, read_window,
};
use crate::loader::errors::LoaderError;
use crate::loader::main::_files_schema::files_schema;
use crate::loader::main::_files_window::files_window;
use crate::loader::models::{
    CommitRule, FilesContext, FilesOutput, FilesWindowRequest, LiveFiles, PlanOptions, TablePlan,
    TableSpec, WholeReason,
};
use crate::project::models::WriteStrategy;
use crate::schema_plan::models::{LiveColumn, SchemaPlan};
use crate::table_lease::main::lease_holder::lease_holder;
use crate::table_lease::main::lease_now::lease_now;

/// One table's plan against the files already in its folder.
pub(in crate::loader) async fn plan_files_table(
    spec: &TableSpec,
    destination: &FilesDestination,
    options: &PlanOptions,
) -> Result<TablePlan, LoaderError> {
    let FilesContext {
        read_columns,
        desired,
        path,
        cursor_type,
        folder,
        state,
    } = match files_context(spec, destination, options).await? {
        Ok(context) => context,
        Err(blocked) => return Ok(blocked),
    };
    let LiveFiles { footers, live } =
        match live_schema(&folder, &state.listed, destination.max_footer_reads).await? {
            Ok(current) => current,
            Err(problems) => {
                return Ok(TablePlan {
                    exists: state.exists,
                    ..blocked_table(spec, problems)
                });
            }
        };
    let schema = files_schema(spec, &desired, live.as_deref(), options);
    let (schema, read_columns) = captured_columns(spec, options, schema, read_columns);
    let window_request = FilesWindowRequest {
        spec,
        options,
        cursor_type: cursor_type.as_ref(),
        partition: path.partition.as_ref(),
        listed: &state.listed,
        footers: &footers,
        read_columns: &read_columns,
    };
    let (mut schema, window, files) =
        planned_read(&window_request, (&state.loaded, live.as_deref()), schema).await?;
    let read = window.as_ref().and_then(|window| read_window(spec, window));
    let (source_filter, mode, cursor_note) = match window {
        Some(window) => (window.filter, window.mode, window.note),
        None => (None, WriteMode::Append, None),
    };
    let mode = captured_mode(spec, mode);
    let notes = write_notes(&mode, &path.folder);
    let commits = commit_rule(&CommitRequest {
        spec,
        window: read.as_ref(),
        cursor_type: cursor_type.as_ref(),
        read_columns: &read_columns,
        files: files.as_ref(),
        mode: &mode,
    });
    let commits = match commits {
        Ok(_) if destination.encoding.single_file => CommitRule::Whole(WholeReason::SingleFile),
        Ok(commits) => commits,
        Err(problem) => {
            if schema.blocked.is_none() {
                schema = blocked_plan(vec![problem]);
            }
            CommitRule::Whole(WholeReason::Blocked)
        }
    };
    Ok(TablePlan {
        spec: spec.clone(),
        read_columns,
        live,
        schema,
        mode,
        source_filter,
        source_parameters: vec![],
        cursor_note,
        window: read,
        route: None,
        files,
        exists: state.exists,
        logged: None,
        commits,
        stage: None,
        planned_stage: None,
        fetch_memory: None,
        output: Some(FilesOutput {
            path,
            encoding: destination.encoding.clone(),
            listed: state.listed.keys().len(),
            footers_read: footers.len(),
            max_footer_reads: destination.max_footer_reads,
            notes,
            pending: state.pending,
            commits: state.commits,
        }),
    })
}

/// The window `request` reads in (from the target's newest partition or files, over the range of the cursor's family when typed range flags are given), and for a files source the files it reads, skipping those the target's records hold (`loaded`), with no source filter, as the files are chosen instead; a problem in either blocks `schema` instead. Nothing is planned for a plan already blocked.
async fn planned_read(
    request: &FilesWindowRequest<'_>,
    (loaded, live): (&[LoadedFile], Option<&[LiveColumn]>),
    schema: SchemaPlan,
) -> Result<(SchemaPlan, Option<Window>, Option<FilesRead>), LoaderError> {
    if schema.blocked.is_some() {
        return Ok((schema, None, None));
    }
    let ranged = match table_range(request.options, request.spec, request.cursor_type, true) {
        Ok(ranged) => ranged,
        Err(problem) => return Ok((blocked_plan(vec![problem]), None, None)),
    };
    let request = &FilesWindowRequest {
        options: &ranged,
        ..*request
    };
    let window = match files_window(request) {
        Ok(window) => window,
        Err(problem) => return Ok((blocked_plan(vec![problem]), None, None)),
    };
    let Some(files) = &request.spec.files else {
        return Ok((schema, Some(window), None));
    };
    let written: Vec<String> = schema
        .write_columns
        .iter()
        .map(|column| column.name.clone())
        .collect();
    let plan_request = FilesPlanRequest {
        spec: request.spec,
        files,
        loaded: LoadedSource::Recorded(loaded),
        live,
        read_columns: request.read_columns,
        written: &written,
        bounds: (window.bounds.0.as_deref(), window.bounds.1.as_deref()),
        options: request.options,
    };
    let planned = plan_files(&plan_request).await?;
    let window = Window {
        filter: None,
        ..window
    };
    Ok(match planned {
        Ok(read) => (schema, Some(window), Some(read)),
        Err(problem) => (blocked_plan(vec![problem]), Some(window), None),
    })
}

/// The table's context, or its blocked plan (marked existing when its folder holds data files), blocked too while another load holds its lease, and while it holds records an older SQLLoad wrote that this load would read (`append`, or `delete_insert` checking files below its window; never `--reload`, which rewrites them).
async fn files_context(
    spec: &TableSpec,
    destination: &FilesDestination,
    options: &PlanOptions,
) -> Result<Result<FilesContext, TablePlan>, LoaderError> {
    let read_columns = match read_columns(spec) {
        Ok(read) => read,
        Err(problems) => return Ok(Err(blocked_table(spec, problems))),
    };
    let desired = desired_columns(spec, &read_columns);
    let path = match table_folder(spec, destination) {
        Ok(path) => path,
        Err(problem) => return Ok(Err(blocked_table(spec, vec![problem]))),
    };
    let (cursor_type, mut problems) = check_output(spec, &path, &desired);
    problems.extend(captured_problems(spec, &path, options));
    problems.extend(encoding_problems(spec, &path, &destination.encoding));
    let folder = open_folder(&path.folder)?;
    let state = read_table_state(&folder, &path, cursor_type.as_ref()).await?;
    problems.extend(state.listed.problems.clone());
    let reads_records = match spec.strategy {
        WriteStrategy::Append => true,
        WriteStrategy::DeleteInsert => spec.cursor.is_some() && options.start_cursor.is_none(),
        _ => false,
    };
    problems.extend(
        state
            .old_records
            .clone()
            .filter(|_| reads_records && !options.reload),
    );
    problems.extend(lease_holder(&folder, lease_now()).await?);
    if !problems.is_empty() {
        return Ok(Err(TablePlan {
            exists: state.exists,
            ..blocked_table(spec, problems)
        }));
    }
    Ok(Ok(FilesContext {
        read_columns,
        desired,
        path,
        cursor_type,
        folder,
        state,
    }))
}