//! The warehouse adapter contract, after sqlbuild's `StrictAdapter` and `BaseAdapter`: the shared rendering is the default, and each warehouse overrides what its SQL does differently.
use crate::adapters::contract::_helpers::changes::change_statements;
use arrow_array::RecordBatch;
use chrono::NaiveDateTime;
use crate::adapters::contract::_helpers::identifiers::{
exact_identifier, qualified, source_expression_cast, staging_type, string_literal,
};
use crate::adapters::contract::_helpers::logged::{commit_statements, logged_statements};
use crate::adapters::contract::_helpers::promote::same_value;
use crate::adapters::contract::_helpers::promote::{
drop_and_rename, merge_statement, swap_statements,
};
use crate::adapters::contract::_helpers::queries::{
check_query, child_tables_query, cursor_query, insert_rows, loaded_files_query,
staging_tables_query, staging_time, stale_staging,
};
use crate::adapters::contract::_helpers::recreate::recreate_statements;
use crate::adapters::contract::_helpers::schema::{
alter_statements, build_table, create_schema, create_table, staging_changes, staging_name,
staging_table,
};
use crate::adapters::contract::_helpers::type_fit::type_fit;
use crate::adapters::contract::_helpers::write::{
mark_statement, marked_counts, replaced_rows, tombstone_statements, write_statements,
};
use crate::adapters::contract::main::_dropped_ordinals::dropped_ordinals;
use crate::adapters::contract::main::_kept_row_ids::kept_row_ids;
use crate::adapters::contract::main::_older_rows::older_rows;
use crate::adapters::contract::main::_rekeyed_row_ids::rekeyed_row_ids;
use crate::adapters::contract::models::{
ChangeApply, ChangeWrite, CheckQuery, CursorQuery, CursorRead, CursorValue, MarkWrite,
TableName, TombstoneWrite, WriteMode, WriteStatementsRequest, WriteTarget,
};
use crate::column_types::models::{ColumnType, LiveColumnType, TypeFit, Warehouse, WarehouseType};
use crate::loader::errors::LoaderError;
use crate::loader::models::{TableLayout, TableSpec};
use crate::schema_plan::models::{ColumnChange, DesiredColumn, LiveColumn};
/// What a warehouse's SQL looks like. Pure rendering, with no I/O; names follow sqlbuild's adapter contract.
pub trait SqlAdapter: Sync + std::fmt::Debug {
/// The warehouse this adapter renders for.
fn warehouse(&self) -> Warehouse;
/// Whether the destination runs SQL; a files target writes Parquet and alters nothing in place.
fn runs_sql(&self) -> bool {
true
}
/// What a load's peak memory on this target scales with, for `plan`, where it doesn't follow the source's fetch budget; `None` elsewhere.
fn memory_note(&self) -> Option<&'static str> {
None
}
/// Whether a merge table's `deletes` (`load --sync-deletes`, `deletes_from`) may mark its rows in place: only on targets where its mark statements have run, so plan refuses it elsewhere rather than run them untested.
fn marks_deleted_rows(&self) -> bool {
false
}
/// Whether a `postgres_cdc` source applying its changes by `apply` may load into the destination: every SQL warehouse its changes have been applied to live; a files target only as a changelog, having no tables to merge into.
fn captures_changes(&self, apply: ChangeApply) -> bool {
self.runs_sql() || apply == ChangeApply::Changelog
}
/// The statements deleting published table `table`'s rows matching `condition` (a merge clearing the items of the roots it replaces from a child table): one `DELETE`. `scratch` names the tables a warehouse that publishes by swaps builds beside it.
fn render_clear_rows(
&self,
table: &TableName,
condition: &str,
_scratch: &TableName,
) -> Vec<String> {
vec![format!(
"DELETE FROM {} WHERE {condition}",
self.render_qualified_name(table)
)]
}
/// The statement deleting staging table `staging`'s rows matching `condition`.
fn render_delete_staged(&self, staging: &TableName, condition: &str) -> String {
format!(
"DELETE FROM {} WHERE {condition}",
self.render_qualified_name(staging)
)
}
/// The longest table or column name, in characters, that dlt's naming gives on the warehouse (the shorter of dlt's table and column limits), past which it shortens a name with a tag; `None` where it gives no limit.
fn max_identifier_length(&self) -> Option<usize> {
None
}
/// Whether a created table is sorted by its `unique_key` when it sets no `order_by`: for a table that doesn't merge, the primary key its source reports.
fn sorts_by_unique_key(&self) -> bool {
false
}
/// Whether SQLLoad never drops or recreates the warehouse's tables and never alters one it did not create: `replace` empties and refills the table in one transaction instead of swapping a new one in, `--reload` is refused, and an existing table without `_sqlload_load_id` is loaded as it is.
fn keeps_tables(&self) -> bool {
false
}
/// Whether the warehouse bills for the time it runs queries, with a minimum or cooldown after the last, so a load sends each commit's rows in one burst (`send = "at_commit"`) unless its target says otherwise.
fn bills_while_running(&self) -> bool {
false
}
/// Whether a `delete_insert` read in windows or cursor-ordered batches is staged whole and published in one commit at the end, not one commit per batch: each publish here rewrites the partitions it touches, so batches sharing a partition rewrite it again each time, and readers would see the run half published between commits.
fn publishes_ranges_once(&self) -> bool {
false
}
/// Whether a column SQLLoad keeps as it is (see [`keeps_tables`](SqlAdapter::keeps_tables)), of type `live`, takes every `desired` value exactly, or refuses one that does not fit rather than rounding it.
fn assigns_exactly(&self, desired: &ColumnType, live: &ColumnType) -> bool {
self.type_fit(desired, live) == TypeFit::Fits
}
/// Whether a `--reload` build table starts `LIKE` the table to keep its grants, and so keeps the live columns.
fn builds_like_target(&self) -> bool {
false
}
/// Whether SQLLoad's reader keeps nanosecond times and timestamps whole for the warehouse, instead of cutting them to microseconds.
fn keeps_nanoseconds(&self) -> bool {
true
}
/// How the warehouse spells `column_type` in DDL, and what is lost if it cannot hold it exactly.
fn render_loader_logical_type(&self, column_type: &ColumnType) -> WarehouseType;
/// Why the warehouse cannot load a column of `column_type` yet, if it cannot.
fn type_problem(&self, _column_type: &ColumnType) -> Option<&'static str> {
None
}
/// The portable type of a live column the information schema reports; `native(...)` when it has none.
fn live_type(&self, live: &LiveColumnType) -> ColumnType {
ColumnType::Native(live.data_type.clone())
}
/// Whether a column pinned `native("VARIANT")` reads as `json` and loads its values parsed into the warehouse's `VARIANT` (DuckDB), rather than passing the type through.
fn parses_variant_pins(&self) -> bool {
false
}
/// Whether the warehouse changes a `live` column to `target` without rewriting the table.
fn widens_in_place(&self, _live: &ColumnType, _target: &ColumnType) -> bool {
false
}
/// Whether `live` holds every `desired` value, can be widened to, or cannot.
fn type_fit(&self, desired: &ColumnType, live: &ColumnType) -> TypeFit {
type_fit(self, desired, live)
}
/// `name` as the warehouse stores it and reports it in its information schema.
fn stored_name(&self, name: &str) -> String {
name.to_owned()
}
/// How the warehouse stores the table or schema name `name`, which a catalog lists it by: as [`stored_name`](Self::stored_name), unless it folds table names apart from columns.
fn stored_table_name(&self, name: &str) -> String {
self.stored_name(name)
}
/// Whether the live column `live` is the one stored as `stored`.
fn is_stored_as(&self, live: &str, stored: &str) -> bool {
live.eq_ignore_ascii_case(stored)
}
/// `name` quoted, as the warehouse stores it.
fn render_identifier(&self, name: &str) -> String {
exact_identifier(&self.stored_name(name))
}
/// A live name, quoted exactly as the warehouse reports it.
fn render_exact_identifier(&self, name: &str) -> String {
exact_identifier(name)
}
/// A fully qualified table name.
fn render_qualified_name(&self, table: &TableName) -> String {
qualified(self, table)
}
/// `text` as a SQL string literal.
fn render_loader_value_literal(&self, text: &str) -> String {
string_literal(text)
}
/// A numeric cursor's bound `value`, a plain decimal number, as a source filter compares it with a column of `column_type`: bare.
fn render_cursor_number(&self, value: &str, _column_type: &ColumnType) -> String {
value.to_owned()
}
/// How `plan` names the warehouse reading files itself: the verb for reading them in place, and the step loading them from a stage.
fn render_read_words(&self) -> (&'static str, &'static str) {
("reads", "reads them")
}
/// `SCHEMA.TABLE` as `plan` prints it, the way the warehouse stores the names.
fn render_display_name(&self, table: &TableName) -> String {
format!("{}.{}", table.schema, table.table)
}
/// The type a staging column has: what the loaded files or batches carry (JSON and UUIDs as text).
fn render_staging_type(&self, column_type: &ColumnType) -> String {
staging_type(self, column_type)
}
/// The expression turning staging column `name` into the table's type.
fn render_source_expression_cast(&self, name: &str, column_type: &ColumnType) -> String {
source_expression_cast(self, name, column_type)
}
/// `left` and `right`, the two sides' values of one key column a merge matches rows on, compared: with `=` when the staged values are never NULL (`nullable` false), otherwise NULL-safely, a NULL matching a NULL (`IS NOT DISTINCT FROM`).
fn render_key_equality(&self, (left, right): (&str, &str), nullable: bool) -> String {
match nullable {
true => format!("{left} IS NOT DISTINCT FROM {right}"),
false => format!("{left} = {right}"),
}
}
/// Under `newest_by`, the rows of `ranked` (a subquery aliased `__sqlload_ranked`, its rows ranked by `__sqlload_rank`) ranked first of their key whose key's row in `table` (aliased `__sqlload_newer`, matched on `keys`, each rendered with whether it may be NULL) isn't `newer`; or (`older`) only those whose row is. A subquery aliased `__sqlload_kept` selecting `names`.
fn render_newest_kept(
&self,
table: &TableName,
(ranked, names): (&str, &[String]),
(keys, newer): (&[(String, bool)], &str),
older: bool,
) -> String {
let join: Vec<String> = keys
.iter()
.map(|(name, nullable)| {
let pair = (
format!("__sqlload_newer.{name}"),
format!("__sqlload_ranked.{name}"),
);
same_value(pair, *nullable)
})
.collect();
let exists = format!(
"EXISTS (SELECT 1 FROM {} AS __sqlload_newer WHERE {} AND {newer})",
self.render_qualified_name(table),
join.join(" AND ")
);
let keep = match older {
true => exists,
false => format!("NOT {exists}"),
};
format!(
"(SELECT {} FROM {ranked} WHERE __sqlload_rank = 1 AND {keep}) AS __sqlload_kept",
names.join(", ")
)
}
/// `value`, a merge key of `column_type`, as a window can be partitioned by it.
fn render_partition_key(&self, value: &str, _column_type: &ColumnType) -> String {
value.to_owned()
}
/// `expression` as text.
fn render_text_cast(&self, expression: &str) -> String {
format!("CAST({expression} AS VARCHAR)")
}
/// The query `check` runs on `relation`, returning its values as text.
fn render_check_query(&self, relation: &TableName, check: CheckQuery<'_>) -> String {
check_query(self, relation, check)
}
/// A query returning a row when `table` holds one.
fn render_any_row_query(&self, table: &TableName) -> String {
format!(
"SELECT 'x' FROM {} LIMIT 1",
self.render_qualified_name(table)
)
}
/// `CREATE SCHEMA IF NOT EXISTS` for `table`'s schema.
fn render_create_schema(&self, table: &TableName) -> String {
create_schema(self, table)
}
/// `CREATE TABLE IF NOT EXISTS` with every column nullable.
fn render_create_table(&self, table: &TableName, columns: &[DesiredColumn]) -> String {
create_table(self, table, columns)
}
/// Problems with a table's layout settings (`engine`, `order_by`, `partition_by`, `primary_key`, `table_settings`), which only ClickHouse takes.
fn layout_problems(&self, layout: &TableLayout) -> Vec<String> {
match layout.is_empty() {
true => Vec::new(),
false => vec![format!(
"engine, order_by, partition_by, primary_key and table_settings lay out ClickHouse tables; {} tables take none",
self.warehouse()
)],
}
}
/// `CREATE TABLE IF NOT EXISTS` for a loaded table: the table `spec` names with `columns`, laid out as it says where the warehouse takes a layout.
fn render_create_loaded_table(&self, spec: &TableSpec, columns: &[DesiredColumn]) -> String {
self.render_create_table(&spec.destination, columns)
}
/// `COMMENT ON TABLE`, giving `table` its description; none for a table without one.
fn render_table_comment(&self, table: &TableName, description: &str) -> Option<String> {
(!description.is_empty()).then(|| {
format!(
"COMMENT ON TABLE {} IS {}",
self.render_qualified_name(table),
self.render_loader_value_literal(description)
)
})
}
/// The statement creating the unique index a `merge` in `mode` needs, for a table SQLLoad creates; none where `merge` needs no index.
fn render_key_index(&self, _table: &TableName, _mode: &WriteMode) -> Option<String> {
None
}
/// The table kind a load's staging table is created as.
fn staging_table_kind(&self) -> &'static str {
"TABLE"
}
/// The staging or build table for `table` under `staging_id`.
fn staging_relation_name(&self, table: &TableName, staging_id: &str) -> TableName {
staging_name(table, staging_id)
}
/// The staging table for `table` under `staging_id`, and the statement creating it with `columns` (the `_sqlload_*` metadata included, which the staged batches carry).
fn render_create_staging_table(
&self,
table: &TableName,
columns: &[DesiredColumn],
staging_id: &str,
) -> (TableName, String) {
staging_table(self, table, columns, staging_id)
}
/// The statements creating `build` in `table`'s own shape, so batches land in their final form, what the destination added stays, and the swap copies nothing: from the table's `definition` by default. A `--reload` passes the `differing` live columns, as it rebuilds with the current columns.
fn render_create_build_table(
&self,
build: &TableName,
table: &TableName,
shape: (&[DesiredColumn], Option<&[LiveColumn]>, Option<&str>),
) -> Vec<String> {
build_table(self, build, table, shape)
}
/// The statement changing `column` of `target` to `sql_type` in place.
fn render_alter_column_type(&self, target: &str, column: &str, sql_type: &str) -> String {
format!("ALTER TABLE {target} ALTER COLUMN {column} TYPE {sql_type}")
}
/// The statements applying `changes` to `table`, whose current columns are `live`.
fn render_change_columns(
&self,
table: &TableName,
changes: &[ColumnChange],
live: &[LiveColumn],
) -> Vec<String> {
alter_statements(self, table, changes, live)
}
/// The select-list entry keeping column `name` of `column_type` in a rebuild that declares each column's type, where its value must coerce to that type.
fn render_typed_rebuild_column(&self, name: &str, _column_type: &ColumnType) -> String {
name.to_owned()
}
/// The statement rewriting `target` from its own rows through `select`, in place and keeping its grants; `columns` defines each column as it is to be, `name TYPE`, for a warehouse whose query results lose a type's parameters.
fn render_rebuild_table(&self, target: &str, (select, _columns): (&str, &str)) -> String {
format!("CREATE OR REPLACE TABLE {target} COPY GRANTS AS SELECT {select} FROM {target}")
}
/// The statement emptying staging table `staging` for the next batches.
fn render_empty_staging(&self, staging: &TableName) -> String {
format!("DELETE FROM {}", self.render_qualified_name(staging))
}
/// Whether a staging table's changed types are cast by rewriting it rather than altered in place.
fn widens_staging_by_rebuild(&self) -> bool {
false
}
/// The statements changing `staging`, made for `old` columns, to hold `new` ones; none when nothing changed. A build table (`built`, the table's live columns it started with) holds the table's own types, and those live columns stay.
fn render_staging_changes(
&self,
staging: &TableName,
columns: (&[DesiredColumn], &[DesiredColumn]),
built: Option<&[LiveColumn]>,
) -> Vec<String> {
staging_changes(self, staging, columns, built)
}
/// Promote the finished build table in place of `target` with no row copied (sqlbuild's staged promotion): by default the old table is dropped and the new one renamed in one transaction, then `indexes` are re-created.
fn render_swap(
&self,
target: &TableName,
build: &TableName,
indexes: &[String],
) -> Vec<String> {
drop_and_rename(self, target, build, indexes)
}
/// Under `newest_by`, a query counting, as text, the keys of `staging` whose newest row the merge of `request` skips, as the table holds a newer one; `None` without `newest_by`.
fn render_older_rows(
&self,
request: &WriteStatementsRequest<'_>,
staging: &TableName,
) -> Option<String> {
older_rows(self, request, staging)
}
/// A query of the ids (`row_id`) of the rows of `staging` the merge of `request` writes, whose child tables' rows it replaces.
fn render_kept_row_ids(
&self,
request: &WriteStatementsRequest<'_>,
staging: &TableName,
row_id: &str,
) -> String {
kept_row_ids(self, request, (staging, row_id))
}
/// Under a merge that dedupes, a query of the ordinal (as text) of each row of `staging` the merge of `request` drops, so the rows hanging off them (child tables' rows) are dropped too; `None` without `dedupe`.
fn render_dropped_ordinals(
&self,
request: &WriteStatementsRequest<'_>,
staging: &TableName,
) -> Option<String> {
dropped_ordinals(self, request, staging)
}
/// Under `delete_insert`, the condition choosing the rows its commit replaces, which their child tables' rows go with; `None` when it replaces every row.
fn render_replaced_rows(&self, request: &WriteStatementsRequest<'_>) -> Option<String> {
replaced_rows(self, request)
}
/// Under a merge, a query of the `row_id` of each row of the table that a row of `staging` shares its key with but whose `row_id` differs (an id not derived from the key, as dlt's `delete-insert` merge gives), so the rows hanging off it can be replaced too; `None` without a merge.
fn render_rekeyed_row_ids(
&self,
request: &WriteStatementsRequest<'_>,
staging: &TableName,
row_id: &str,
) -> Option<String> {
rekeyed_row_ids(self, request, staging, row_id)
}
/// One `MERGE`: key matches update every other column, the rest are inserted.
fn render_merge(
&self,
request: &WriteStatementsRequest<'_>,
unique_key: &[String],
source: &str,
values: &[String],
) -> String {
merge_statement(self, request, unique_key, (source, values))
}
/// `replace` from rows that arrived as text: build a copy beside the table, then swap it in.
fn render_replace(
&self,
request: &WriteStatementsRequest<'_>,
staging: &TableName,
insert_from: &str,
) -> Vec<String> {
swap_statements(self, request, staging, insert_from)
}
/// Rebuild the table from `source` in one statement, so readers never see it empty (`--reload`).
fn render_replace_table_from_relation(
&self,
request: &WriteStatementsRequest<'_>,
source: &str,
values: &[String],
) -> Vec<String> {
recreate_statements(self, request, source, values)
}
/// Statements applying a batch of captured changes from `staging`, guarded by `_sqlload_lsn` so a replay changes nothing; in one transaction where the warehouse has them.
fn render_changes(
&self,
request: &WriteStatementsRequest<'_>,
change: &ChangeWrite,
staging: &TableName,
) -> Vec<String> {
change_statements(self, request, change, staging)
}
/// A key scan's (`mark`) one statement marking the table's rows of the keys `staging` doesn't hold.
fn render_mark(
&self,
request: &WriteStatementsRequest<'_>,
mark: &MarkWrite,
staging: &TableName,
) -> String {
mark_statement(self, request, mark, staging)
}
/// Under a key scan (`mark`), the queries counting the table's rows it would mark and those it may mark from, each one count as text, for its `max_row_drop` limit.
fn render_marked_counts(
&self,
request: &WriteStatementsRequest<'_>,
mark: &MarkWrite,
staging: &TableName,
) -> [String; 2] {
marked_counts(self, request, mark, staging)
}
/// The statement a soft `deletes_from` marks rows with: `target` (the table aliased `__target`) gets `column` set from `marks`, a subquery aliased `__marks` holding each key's latest tombstone time as `column`, joined on `join`, for the rows `condition` chooses. `matched` chooses the same rows of those `marks` joins without a subquery: unmarked, with no cursor or one at or before the key's latest tombstone, which `latest_marks` (`marks` with that time also uncast, as `__sqlload_latest`) holds, so it compares as the tombstone's own time does.
fn render_soft_tombstones(
&self,
(target, column): (&str, &str),
((marks, _latest_marks), join): ((&str, &str), &str),
(condition, _matched): (&str, &str),
) -> String {
format!(
"UPDATE {target} SET {column} = __marks.{column} FROM {marks} WHERE {join} AND {condition}"
)
}
/// A table's tombstones (`deletes_from`): the query counting the rows they mark, as text, then the statement marking them.
fn render_tombstones(&self, table: &TableName, tombstones: &TombstoneWrite) -> [String; 2] {
tombstone_statements(self, table, tombstones)
}
/// Atomic statements applying `request.mode` from `staging`, which holds every column, the `_sqlload_*` metadata included; the caller drops `staging` and its swap table.
fn render_write(
&self,
request: &WriteStatementsRequest<'_>,
staging: &TableName,
) -> Vec<String> {
write_statements(self, request, staging)
}
/// Whether a `merge` batch's truncate is a `WHEN NOT MATCHED BY SOURCE` clause of its `MERGE` rather than a statement before it, so a warehouse that commits every statement alone still applies the batch in one commit.
fn truncates_in_merge(&self) -> bool {
false
}
/// Whether a files source may `merge` into this warehouse, file by file with a run record in each file's commit (tested on DuckDB and Postgres so far).
fn merges_files(&self) -> bool {
false
}
/// Whether `statement`, run alone, commits by itself, so a run record cannot share its transaction.
fn commits_alone(&self, _statement: &str) -> bool {
false
}
/// `statements` with `insert` inside their transaction where the warehouse allows it, so the record commits with the data; otherwise after them.
fn render_logged(&self, statements: Vec<String>, insert: String) -> Vec<String> {
logged_statements(self, statements, insert)
}
/// The write of `request` from `staging`, with a logged table's run record insert and (unless the staging table is swapped in or rebuilt from at the end) the emptying of the staging table for the next batches.
fn render_commit(
&self,
request: &WriteStatementsRequest<'_>,
staging: &TableName,
record: (Option<String>, bool),
) -> Vec<String> {
commit_statements(self, request, staging, record)
}
/// `INSERT` statements adding `rows` (each value as text, in `columns`' order) to staging table `table`, in statements of bounded size, each value cast to its column's type.
fn render_insert_rows(
&self,
table: &TableName,
columns: &[DesiredColumn],
rows: &[Vec<Option<String>>],
) -> Vec<String> {
insert_rows(self, table, columns, rows)
}
/// The text of a cursor's maximum `column`, read as `value`, independent of session time zones and output formats.
fn render_cursor_text(&self, column: &str, value: CursorValue) -> String {
match value {
CursorValue::EpochMicros { .. } => format!("CAST(epoch_us(MAX({column})) AS VARCHAR)"),
CursorValue::Date => format!("strftime(MAX({column}), '%Y-%m-%d')"),
CursorValue::Plain => format!("CAST(MAX({column}) AS VARCHAR)"),
}
}
/// Text `expression` compared bytewise (in code point order), whatever collation its column or database has, as the source's cursor filter and SQLLoad itself compare text cursors.
fn render_binary_text(&self, expression: &str) -> String {
expression.to_owned()
}
/// A query for a cursor's maximum and how to read it; `None` if a lookback needs a time column.
fn render_max_cursor(&self, request: &CursorQuery<'_>) -> Option<CursorRead> {
cursor_query(self, request)
}
/// `SELECT DISTINCT file || '|' || version`; without a version column, just the file names.
fn render_loaded_files_query(
&self,
table: &TableName,
file_column: &str,
version_column: Option<&str>,
) -> String {
loaded_files_query(self, table, file_column, version_column)
}
/// The condition choosing staging tables a crashed load may have left, by name (`_` matches any character).
fn render_staging_filter(&self, suffix: &str) -> String {
format!(
"table_name ILIKE {}",
self.render_loader_value_literal(&format!("%{suffix}%"))
)
}
/// The relation listing the columns of `table`'s schema's tables, with `table_schema`, `table_name` and `column_name` columns.
fn render_schema_columns(&self, _table: &TableName) -> String {
"information_schema.columns".to_owned()
}
/// The query listing the tables named as `parent`'s child tables are, in its schema, each marked by whether it has a parent-id column (see [`child_tables_query`]).
fn list_child_tables_sql(&self, parent: &TableName) -> String {
child_tables_query(self, (parent, &self.render_schema_columns(parent)))
}
/// The query listing staging tables a crashed load may have left in `table`'s schema; `database` is the connection's, for warehouses that need one.
fn list_staging_relations_sql(&self, table: &TableName, database: Option<&str>) -> String {
staging_tables_query(self, table, database)
}
/// Whether the listed staging table `name` is a crashed load's: by default, the time in its name is before `before`.
fn is_stale_staging(&self, name: &str, before: NaiveDateTime) -> bool {
staging_time(name).is_some_and(|time| time < before)
}
/// `DROP` statements for the stale staging tables among `found`, in `schema_of`'s schema.
fn render_stale_staging(
&self,
schema_of: &TableName,
found: &[String],
before: NaiveDateTime,
) -> Vec<String> {
stale_staging(self, schema_of, found, before)
}
/// `CREATE TABLE IF NOT EXISTS` for a run log of `columns`, and a fallback to try when the warehouse refuses the first.
fn render_create_run_log_table_sql(
&self,
runs: &TableName,
columns: &[DesiredColumn],
) -> (String, Option<String>) {
(self.render_create_table(runs, columns), None)
}
}
/// One table's write session on its destination: how its batches get in. The loader keeps the commit cycle (staging and build tables, commit points, run records); the writer owns the transport.
pub trait TableWriter {
/// Whether an unlogged `append` may write batches straight into the table rather than through a staging table.
fn writes_directly(&self) -> bool;
/// Add one batch, the load's metadata included, where `target`'s batches go.
fn write_batch(
&mut self,
target: &WriteTarget<'_>,
batch: RecordBatch,
) -> impl Future<Output = Result<(), LoaderError>> + Send;
/// The bytes of files the batch has staged so far, where it goes through files; `None` when its Arrow bytes measure it.
fn staged_bytes(&self) -> Option<usize>;
/// Stage what the batch still holds in memory, so every row it was given is in a file or sent.
fn finish_files(
&mut self,
target: &WriteTarget<'_>,
) -> impl Future<Output = Result<(), LoaderError>> + Send;
/// Load a batch that has rows where it goes: `COPY` its staged files, or commit its open transaction.
fn load_batch(
&mut self,
target: &WriteTarget<'_>,
) -> impl Future<Output = Result<(), LoaderError>> + Send;
/// Start the next batch, with nothing staged.
fn next_batch(&mut self);
/// Wait until every row sent without files is in the staging table.
fn settle_rows(&mut self) -> impl Future<Output = Result<(), LoaderError>> + Send;
/// Open the row channel again on its next use, after a schema change replaced the staging table.
fn reopen_rows(&mut self);
/// Roll back a batch's open transaction, before its staging table is dropped; best effort.
fn rollback_batch(&mut self) -> impl Future<Output = ()> + Send;
/// Remove the staged files of a batch that did not load, unless the stage keeps them; best effort.
fn remove_batch_files(&mut self, target: &WriteTarget<'_>) -> impl Future<Output = ()> + Send;
}