sqlload 0.63.0

SQL data loading tooling.
//! Choose a files table's cursor window: whole partitions for delete_insert, rows after the newest files' maximum for append.

use crate::adapters::contract::main::cursor_bound::cursor_bound;
use crate::adapters::contract::main::cursor_text::cursor_text;
use crate::adapters::contract::models::{CursorValue, WriteMode};
use crate::adapters::files::main::check_window::check_window;
use crate::adapters::files::main::cursor_maximum::cursor_maximum;
use crate::adapters::files::main::partition_window::partition_window;
use crate::adapters::files::models::{PartitionBy, PartitionWindowRequest};
use crate::column_types::models::ColumnType;
use crate::loader::_helpers::table_plan::{Window, source_cursor, source_filter};
use crate::loader::constants::{EXCLUSIVE_BOUND, INCLUSIVE_BOUND};
use crate::loader::models::CursorBinding;
use crate::loader::models::FilesWindowRequest;
use crate::project::models::WriteStrategy;

/// The source filter, write mode and `plan` note: partitions from the newest one (`delete_insert`) or rows after the newest files' maximum (`append`; from it for a files source, which picks whole files of the newest partition on).
pub(in crate::loader) fn files_window(request: &FilesWindowRequest<'_>) -> Result<Window, String> {
    let (spec, options) = (request.spec, request.options);
    let backfill = options.start_cursor.is_some() || options.end_cursor.is_some();
    let (Some(cursor), Some(cursor_type), Some(partition)) =
        (&spec.cursor, request.cursor_type, request.partition)
    else {
        let mode = match (options.reload, spec.strategy) {
            (true, _) => WriteMode::Recreate,
            (false, WriteStrategy::Append) => WriteMode::Append,
            (false, _) => WriteMode::Replace,
        };
        return match backfill {
            true => Err("--start-cursor/--end-cursor need a cursor_column".to_owned()),
            false => Ok(Window {
                filter: None,
                parameters: Vec::new(),
                bounds: (None, None),
                mode,
                note: None,
                inclusive: true,
                maximum: None,
            }),
        };
    };
    if backfill && options.reload {
        return Err(
            "--reload rewrites every file, so it takes no --start-cursor/--end-cursor, nor their -ts and -int forms".to_owned(),
        );
    }
    if cursor.lookahead.is_some() || cursor.from_now {
        return Err(
            "lookahead and lookback_from = \"now\" apply to database and warehouse targets; a files target's window follows its newest partition"
                .to_owned(),
        );
    }
    let flag = |value: &Option<String>| {
        value
            .as_deref()
            .map(|text| cursor_text(text, cursor_type, spec.zone()))
            .transpose()
    };
    let (start, end) = (flag(&options.start_cursor)?, flag(&options.end_cursor)?);
    let newest = request
        .listed
        .partitions
        .last()
        .and_then(|partition| partition.value.as_deref())
        .filter(|_| !options.reload);
    let column = cursor.column.as_str();
    let expression = source_cursor(spec, request.read_columns, Some(cursor_type))
        .unwrap_or_else(|| column.to_owned());
    let mut data_maximum: Option<String> = None;
    let (from, to, note, operator) = match spec.strategy {
        WriteStrategy::DeleteInsert => {
            let window = partition_window(&PartitionWindowRequest {
                by: partition.by,
                cursor_column: column,
                cursor_type,
                newest,
                lookback: cursor.lookback,
                start: start.as_deref(),
                start_flag: options.range_flags.start,
                end: end.as_deref(),
                cursor_start: cursor.start.as_deref(),
                reload: options.reload,
            })?;
            (window.from, window.to, window.note, INCLUSIVE_BOUND)
        }
        _ => {
            let maximum = match (&start, newest) {
                (None, Some(newest)) => {
                    newest_maximum(request, partition.by, newest)?.map(|(maximum, from)| {
                        let bound = match cursor.inclusive || request.spec.files.is_some() {
                            true => "from",
                            false => "after",
                        };
                        let note = format!("{column} {bound} {maximum} ({from})");
                        (maximum, note)
                    })
                }
                _ => None,
            };
            let (from, note, operator) = match (start, maximum, &cursor.start) {
                (Some(start), _, _) => (
                    Some(start.clone()),
                    format!("{column} from {start} ({})", options.range_flags.start),
                    INCLUSIVE_BOUND,
                ),
                (None, Some((maximum, note)), _) => {
                    data_maximum = Some(maximum.clone());
                    let operator = match cursor.inclusive {
                        true => INCLUSIVE_BOUND,
                        false => EXCLUSIVE_BOUND,
                    };
                    (Some(maximum), note, operator)
                }
                (None, None, Some(start)) => (
                    Some(start.clone()),
                    format!("{column} first run, from cursor_start {start}"),
                    INCLUSIVE_BOUND,
                ),
                (None, None, None) => (
                    None,
                    format!("{column} first run, everything"),
                    INCLUSIVE_BOUND,
                ),
            };
            (from, end, note, operator)
        }
    };
    let mode = match (options.reload, spec.strategy) {
        (true, _) => WriteMode::Recreate,
        (false, WriteStrategy::DeleteInsert) => WriteMode::DeleteInsert {
            cursor_column: Some(column.to_owned()),
            from: from.clone(),
            to: to.clone(),
            values: vec![],
            replace_key: spec.replace_key.clone(),
            not_null_keys: spec.not_null_keys.clone(),
        },
        (false, _) => WriteMode::Append,
    };
    check_window(partition.by, cursor_type, (from.as_deref(), to.as_deref()))?;
    Ok(Window {
        filter: source_filter(
            &expression,
            (Some(cursor_type), None),
            (from.as_deref().map(|from| (operator, from)), to.as_deref()),
            CursorBinding::Literals,
        )
        .0,
        parameters: Vec::new(),
        bounds: (from, to),
        mode,
        note: Some(note),
        inclusive: operator == INCLUSIVE_BOUND,
        maximum: data_maximum,
    })
}

/// An append cursor's maximum so far, and where it came from: the newest folder's value for `{cursor}` partitions (exact), the newest load's statistics for `{cursor:date}`.
fn newest_maximum(
    request: &FilesWindowRequest<'_>,
    by: PartitionBy,
    newest: &str,
) -> Result<Option<(String, &'static str)>, String> {
    let (Some(cursor), Some(cursor_type)) = (&request.spec.cursor, request.cursor_type) else {
        return Ok(None);
    };
    let value = match cursor_type {
        ColumnType::TimestampTz => CursorValue::EpochMicros { zoned: true },
        ColumnType::TimestampNtz => CursorValue::EpochMicros { zoned: false },
        ColumnType::Date => CursorValue::Date,
        _ => CursorValue::Plain,
    };
    match by {
        PartitionBy::Value => Ok(Some((newest.to_owned(), "the newest partition"))),
        PartitionBy::Date => {
            let raw = cursor_maximum(request.footers, &cursor.column, cursor_type)?;
            let maximum = raw.map(|raw| cursor_bound(&raw, value, None)).transpose()?;
            Ok(maximum.map(|maximum| (maximum, "maximum in the newest load's files")))
        }
    }
}