use crate::adapters::dispatch::main::dialect::sql_adapter::sql_adapter;
use arrow_array::RecordBatch;
use std::collections::BTreeMap;
use tokio_postgres::Client;
use crate::adapters::dispatch::main::dialect::files_destination::files_destination;
use crate::adapters::dispatch::main::session::live_columns::live_columns;
use crate::adapters::dispatch::main::session::text_values::text_values;
use crate::adapters::dispatch::models::Destination;
use crate::column_types::main::parse_type::parse_type;
use crate::column_types::models::{ColumnType, Warehouse};
use crate::file_store::main::open_folder::open_folder;
use crate::files_source::constants::{
NANOSECOND_SCALE, SOURCE_FILE_COLUMN, SOURCE_ROW_COLUMN, SOURCE_VERSION_COLUMN,
VERSION_SEPARATOR,
};
use crate::files_source::main::discover_files::discover_files;
use crate::files_source::main::read_files::read_files;
use crate::files_source::main::select_files::select_files;
use crate::files_source::models::{
BatchFooters, DataFile, FileColumn, FileSelection, FilesRead, FilesTable, LoadedFile,
SourceFormat, StoredRead, TextChecks,
};
use crate::loader::_helpers::python::python_tables;
use crate::loader::_helpers::resolve::{resolve_tables, source_target};
use crate::loader::_helpers::specs::{missing_description, table_name};
use crate::loader::_helpers::table_plan::{blocked_plan, plan_table};
use crate::loader::constants::{PARALLEL_READ_MIN_BYTES, PARALLEL_READERS, SOURCE_BATCH_ROWS};
use crate::loader::errors::LoaderError;
use crate::loader::models::{
CommitRule, FilesSource, PlanOptions, ProjectPlan, ResolvedTables, SkippedTable, SourceEnd,
SourceReader, TablePlan, TableSpec,
};
use crate::postgres_source::constants::DEFAULT_BATCH_BYTES;
use crate::postgres_source::main::connect_postgres::connect_postgres;
use crate::postgres_source::main::copy_table::copy_table;
use crate::postgres_source::main::cursor_end::cursor_end;
use crate::postgres_source::main::read_catalog::read_catalog;
use crate::postgres_source::models::{
CopyRequest, PgType, PostgresAddress, ReadColumn, SourceColumn, SourceTable, WireType,
};
use crate::project::models::{
BatchStep, ConnectionConfig, NewTables, ProjectConfig, SourceConfig, SourceKind, TableOverride,
WriteStrategy,
};
use crate::schema_plan::models::{ColumnOrigin, DesiredColumn, LiveColumn};
use crate::similar_names::main::close_name::close_name;
use crate::similar_names::main::nearest_name::nearest_name;
use crate::store_paths::main::resolve_location::resolve_location;
use crate::store_paths::models::{Location, StoreKind};
pub(crate) async fn connect_source(
project: &ProjectConfig,
source: &SourceConfig,
) -> Result<Client, LoaderError> {
let Some(ConnectionConfig::Postgres {
host,
port,
user,
password,
database,
}) = project.connections.get(&source.connection)
else {
return Err(LoaderError::Invalid(format!(
"connection {:?} is not a Postgres connection",
source.connection
)));
};
let address = PostgresAddress {
host,
port: *port,
user,
password: password.as_ref().map(|secret| secret.expose()),
database,
};
Ok(connect_postgres(address).await?)
}
pub(crate) fn files_location(
project: &ProjectConfig,
source: &SourceConfig,
) -> Result<Location, LoaderError> {
let store = match project.connections.get(&source.connection) {
Some(ConnectionConfig::S3 {
bucket,
region,
endpoint,
}) => StoreKind::S3 {
bucket: bucket.clone(),
region: region.clone(),
endpoint: endpoint.clone(),
},
Some(ConnectionConfig::Filesystem { root }) => StoreKind::Filesystem { root: root.clone() },
_ => {
return Err(LoaderError::Invalid(format!(
"connection {:?} is not an s3 or filesystem connection",
source.connection
)));
}
};
let path = source.path.as_deref().unwrap_or_default();
resolve_location(Some((source.connection.as_str(), &store)), path)
.map_err(|error| LoaderError::Invalid(error.to_string()))
}
pub(crate) fn source_readers(table: &TablePlan) -> usize {
let whole = matches!(table.commits, CommitRule::Size | CommitRule::Whole(_));
let postgres = table.spec.python.is_none() && table.spec.files.is_none();
match whole && postgres && table.spec.source_table.estimated_bytes >= PARALLEL_READ_MIN_BYTES {
true => PARALLEL_READERS,
false => 1,
}
}
pub(crate) async fn open_source(
project: &ProjectConfig,
source: &SourceConfig,
readers: usize,
) -> Result<SourceReader, LoaderError> {
match source.kind {
SourceKind::Postgres | SourceKind::PostgresCdc => {
let client = connect_source(project, source).await?;
let mut helpers = Vec::with_capacity(readers.saturating_sub(1));
for _ in 1..readers {
match connect_source(project, source).await {
Ok(helper) => helpers.push(helper),
Err(_) => break,
}
}
Ok(SourceReader::Postgres(client, helpers))
}
SourceKind::Files => Ok(SourceReader::Files(open_folder(&files_location(
project, source,
)?)?)),
SourceKind::Python => Err(LoaderError::Invalid(
"a Python source is read by calling its table functions".to_owned(),
)),
}
}
pub(crate) async fn source_tables(
project: &ProjectConfig,
source_name: &str,
source: &SourceConfig,
) -> Result<Result<(Vec<SourceTable>, Option<FilesSource>), String>, LoaderError> {
match source.kind {
SourceKind::Postgres | SourceKind::PostgresCdc => {
let client = match connect_source(project, source).await {
Ok(client) => client,
Err(error) => return Ok(Err(error.to_string())),
};
Ok(
match read_catalog(&client, &source.include, &source.exclude).await {
Ok(discovered) => Ok((discovered, None)),
Err(error) => Err(LoaderError::from(error).to_string()),
},
)
}
SourceKind::Files => {
let warehouse = match source_target(project, source) {
Ok(target) => target.warehouse,
Err(problem) => return Ok(Err(problem)),
};
Ok(
match discover_files_table(project, source_name, source, warehouse).await {
Ok((table, files)) => Ok((vec![table], Some(files))),
Err(error) => Err(error.to_string()),
},
)
}
SourceKind::Python => Ok(Ok((python_tables(source), None))),
}
}
pub(crate) async fn discover_files_table(
project: &ProjectConfig,
source_name: &str,
source: &SourceConfig,
warehouse: Warehouse,
) -> Result<(SourceTable, FilesSource), LoaderError> {
let location = files_location(project, source)?;
let folder = open_folder(&location)?;
let partition_names: Vec<String> = source.partition_columns.keys().cloned().collect();
let encoding = source.files_encoding();
let table = discover_files(&folder, &partition_names, &encoding).await?;
if let Some(problem) = table.problems.first() {
return Err(LoaderError::Invalid(problem.clone()));
}
files_source_parts(source_name, source, warehouse, (location, table))
}
pub(crate) fn files_source_parts(
source_name: &str,
source: &SourceConfig,
warehouse: Warehouse,
(location, table): (Location, FilesTable),
) -> Result<(SourceTable, FilesSource), LoaderError> {
let encoding = source.files_encoding();
let mut partition_types: Vec<(String, ColumnType)> = Vec::new();
for (name, text) in &source.partition_columns {
let parsed = parse_type(text, &[warehouse])
.map_err(|error| LoaderError::Invalid(format!("partition column {name}: {error}")))?;
partition_types.push((name.clone(), parsed.column_type));
}
let added: Vec<&str> = partition_types
.iter()
.map(|(name, _)| name.as_str())
.chain([SOURCE_FILE_COLUMN, SOURCE_VERSION_COLUMN, SOURCE_ROW_COLUMN])
.collect();
let overrides = source.tables.get(source_name);
let pinned_only = pinned_unsampled(overrides, &table, &added, encoding.format)?;
let dropped = (!table.dropped.is_empty()).then(|| {
format!(
"dropped {} from the files (replaced by this load's)",
table.dropped.join(", ")
)
});
let notes: Vec<String> = dropped
.into_iter()
.chain(pinned_only.iter().map(|name| {
format!("pinned column {name} was not seen in the sampled files (check the name)")
}))
.collect();
let typed = table
.columns
.iter()
.filter(|column| !added.contains(&column.name.as_str()))
.map(|column| (column.name.clone(), column.column_type.clone()))
.chain(
pinned_only
.into_iter()
.map(|name| (name, Ok(ColumnType::Text))),
)
.chain(
partition_types
.iter()
.map(|(name, column_type)| (name.clone(), Ok(column_type.clone()))),
)
.chain([
(SOURCE_FILE_COLUMN.to_owned(), Ok(ColumnType::Text)),
(SOURCE_VERSION_COLUMN.to_owned(), Ok(ColumnType::Text)),
(SOURCE_ROW_COLUMN.to_owned(), Ok(ColumnType::Int)),
]);
let columns: Vec<SourceColumn> = typed
.map(|(name, column_type)| SourceColumn {
name,
postgres_type: encoding.format.to_string(),
pg_type: PgType::Other,
column_type,
nullable: true,
})
.collect();
let checks = TextChecks {
known: columns.iter().map(|column| column.name.clone()).collect(),
exclude: [
source.exclude_columns.as_slice(),
overrides.map_or(&[], |table| table.exclude_columns.as_slice()),
]
.concat(),
allow: [
source.allow_columns.as_slice(),
overrides.map_or(&[], |table| table.allow_columns.as_slice()),
]
.concat(),
inferred: Vec::new(),
};
let source_table = SourceTable {
schema: String::new(),
name: source_name.to_owned(),
columns,
primary_key: Vec::new(),
indexed_columns: Vec::new(),
estimated_rows: -1,
comment: None,
estimated_bytes: table
.files
.iter()
.map(|data| i64::try_from(data.file.size).unwrap_or(i64::MAX))
.sum(),
};
Ok((
source_table,
FilesSource {
location,
table,
partition_types,
encoding,
checks,
notes,
copy_to_internal_stage: source.copy_to_internal_stage,
},
))
}
fn pinned_unsampled(
overrides: Option<&TableOverride>,
table: &FilesTable,
added: &[&str],
format: SourceFormat,
) -> Result<Vec<String>, LoaderError> {
let Some(overrides) = overrides.filter(|_| format != SourceFormat::Parquet) else {
return Ok(Vec::new());
};
let sampled: Vec<&str> = table
.columns
.iter()
.map(|column| column.name.as_str())
.collect();
let mut unsampled: Vec<String> = Vec::new();
for name in overrides.columns.keys() {
if added.contains(&name.as_str()) || sampled.contains(&name.as_str()) {
continue;
}
let known = sampled.iter().chain(added).copied();
if let Some(close) = close_name(name, known) {
return Err(LoaderError::Invalid(format!(
"pinned column {name} was not seen in the sampled files; did you mean {close:?}? (pin a name exactly as it appears in the files)"
)));
}
unsampled.push(name.clone());
}
Ok(unsampled)
}
pub(crate) fn files_read_columns(spec: &TableSpec) -> Result<Vec<ReadColumn>, Vec<String>> {
let mut problems: Vec<String> = Vec::new();
let mut columns: Vec<ReadColumn> = Vec::new();
for column in &spec.source_table.columns {
let excluded = spec
.excluded_columns
.iter()
.any(|(excluded, _)| *excluded == column.name);
let output = match (spec.pinned.get(&column.name), &column.column_type) {
_ if excluded => continue,
(Some(pinned), _) => pinned.clone(),
(None, Ok(column_type)) => column_type.clone(),
(None, Err(reason)) => {
problems.push(format!("column {}: {reason}", column.name));
continue;
}
};
columns.push(ReadColumn {
name: column.name.clone(),
expression: column.name.clone(),
wire: WireType::Text,
output,
});
}
match problems.is_empty() {
true => Ok(columns),
false => Err(problems),
}
}
pub(crate) struct FilesPlanRequest<'a> {
pub(crate) spec: &'a TableSpec,
pub(crate) files: &'a FilesSource,
pub(crate) destination: &'a Destination,
pub(crate) live: Option<&'a [LiveColumn]>,
pub(crate) read_columns: &'a [ReadColumn],
pub(crate) written: &'a [String],
pub(crate) bounds: (Option<&'a str>, Option<&'a str>),
pub(crate) options: &'a PlanOptions,
}
fn loaded_file(text: String, versioned: bool) -> LoadedFile {
match (versioned, text.rsplit_once(VERSION_SEPARATOR)) {
(true, Some((name, version))) => LoadedFile {
name: name.to_owned(),
version: version.to_owned(),
},
_ => LoadedFile {
name: text,
version: String::new(),
},
}
}
fn live_has(live: Option<&[LiveColumn]>, column: &str) -> bool {
match live {
Some(live) => live
.iter()
.any(|existing| existing.name.eq_ignore_ascii_case(column)),
None => false,
}
}
pub(crate) fn nanosecond_columns(
spec: &TableSpec,
columns: &[FileColumn],
live: Option<&[LiveColumn]>,
) -> Vec<String> {
match sql_adapter(spec.warehouse).keeps_nanoseconds() {
true => columns
.iter()
.filter(|column| {
matches!(
column.stored_read,
StoredRead::Nanos | StoredRead::Epoch(NANOSECOND_SCALE)
)
})
.filter(|column| !spec.pinned.contains_key(&column.name))
.filter(|column| fewer_digits(live, &column.name).is_none())
.map(|column| column.name.clone())
.collect(),
false => Vec::new(),
}
}
pub(crate) fn truncated_nanoseconds(spec: &TableSpec, live: Option<&[LiveColumn]>) -> Vec<String> {
let Some(files) = &spec.files else {
return Vec::new();
};
files
.table
.columns
.iter()
.filter(|column| {
matches!(
column.stored_read,
StoredRead::Nanos | StoredRead::Epoch(NANOSECOND_SCALE)
) && !spec.pinned.contains_key(&column.name)
})
.filter_map(|column| truncated_warning(live, &column.name))
.collect()
}
fn truncated_warning(live: Option<&[LiveColumn]>, name: &str) -> Option<String> {
let digits = fewer_digits(live, name)?;
Some(format!(
"{name}: nanoseconds in the files, but the column holds {digits} digits after the second, so they are truncated.\nRebuild the column at (9) (--reload) to keep them."
))
}
fn fewer_digits(live: Option<&[LiveColumn]>, name: &str) -> Option<i64> {
live.into_iter()
.flatten()
.find(|existing| existing.name.eq_ignore_ascii_case(name))
.and_then(|existing| existing.fraction_digits)
.filter(|digits| *digits < i64::from(NANOSECOND_SCALE))
}
pub(crate) async fn plan_files(
request: &FilesPlanRequest<'_>,
) -> Result<Result<FilesRead, String>, LoaderError> {
let spec = request.spec;
let cursor = match &spec.cursor {
None => None,
Some(cursor) => match request
.files
.partition_types
.iter()
.find(|(name, _)| *name == cursor.column)
{
Some((name, column_type)) => Some((name.as_str(), column_type)),
None => {
return Ok(Err(format!(
"cursor_column {} must be one of the partition_columns for a files source",
cursor.column
)));
}
},
};
if spec.strategy == WriteStrategy::Merge {
return Ok(Err(
"merge is not supported for files sources yet (loaded files would be re-read once their rows are updated); use delete_insert on a partition cursor, or append".to_owned(),
));
}
let skips_loaded = spec.strategy == WriteStrategy::Append;
if skips_loaded
&& request.live.is_some()
&& !live_has(request.live, SOURCE_FILE_COLUMN)
&& !request.options.reload
{
return Ok(Err(format!(
"the table has no {SOURCE_FILE_COLUMN} column, so append cannot tell which files are loaded and would load them all again; run it once with --reload"
)));
}
if skips_loaded
&& !request
.written
.iter()
.any(|name| name == SOURCE_FILE_COLUMN)
{
return Ok(Err(format!(
"{SOURCE_FILE_COLUMN} must be loaded: append uses it to skip files already loaded"
)));
}
let live_has = |column: &str| live_has(request.live, column);
let loaded: Vec<LoadedFile> =
match skips_loaded && live_has(SOURCE_FILE_COLUMN) && !request.options.reload {
true => {
let version = live_has(SOURCE_VERSION_COLUMN).then_some(SOURCE_VERSION_COLUMN);
let query = sql_adapter(spec.warehouse).render_loaded_files_query(
&spec.destination,
SOURCE_FILE_COLUMN,
version,
);
text_values(request.destination, query)
.await?
.into_iter()
.map(|text| loaded_file(text, version.is_some()))
.collect()
}
false => Vec::new(),
};
let location = request.files.location.to_string();
let selection = FileSelection {
files: &request.files.table.files,
location: &location,
loaded: &loaded,
cursor,
from: request.bounds.0,
to: request.bounds.1,
};
let columns: Vec<DesiredColumn> = request
.read_columns
.iter()
.map(|column| DesiredColumn {
name: column.name.clone(),
column_type: column.output.clone(),
origin: ColumnOrigin::Source,
})
.collect();
let constants: Vec<&str> = request
.files
.partition_types
.iter()
.map(|(name, _)| name.as_str())
.chain([SOURCE_FILE_COLUMN, SOURCE_VERSION_COLUMN, SOURCE_ROW_COLUMN])
.collect();
let checks = TextChecks {
inferred: request
.read_columns
.iter()
.map(|column| column.name.clone())
.filter(|name| !spec.pinned.contains_key(name) && !constants.contains(&name.as_str()))
.collect(),
..request.files.checks.clone()
};
let nanos = nanosecond_columns(spec, &request.files.table.columns, request.live);
Ok(select_files(&selection).map(|files| FilesRead {
files,
encoding: request.files.encoding.clone(),
checks,
columns,
nanos,
batch_rows: SOURCE_BATCH_ROWS,
}))
}
#[derive(Debug, Clone, Default)]
pub(crate) struct ReadPart {
pub(crate) filter: Option<String>,
pub(crate) order_by: Option<String>,
pub(crate) files: Option<Vec<DataFile>>,
pub(crate) footers: BatchFooters,
pub(crate) batch_bytes: Option<usize>,
}
pub(crate) async fn read_source(
reader: &SourceReader,
table: &TablePlan,
part: ReadPart,
on_batch: impl AsyncFnMut(RecordBatch) -> Result<(), String>,
) -> Result<u64, LoaderError> {
match (reader, &table.files) {
(SourceReader::Postgres(client, helpers), _) => {
let whole = part.filter.is_none() && part.order_by.is_none();
let request = CopyRequest {
table: &table.spec.source_table,
columns: table.read_columns.clone(),
filter: part.filter.or_else(|| table.source_filter.clone()),
order_by: part.order_by,
batch_rows: SOURCE_BATCH_ROWS,
batch_bytes: part
.batch_bytes
.map_or(DEFAULT_BATCH_BYTES, |bytes| bytes.min(DEFAULT_BATCH_BYTES)),
helpers: match whole {
true => helpers,
false => &[],
},
};
Ok(copy_table(client, request, on_batch).await?)
}
(SourceReader::Files(folder), Some(files)) => {
let read = match part.files.clone() {
Some(subset) => FilesRead {
files: subset,
..files.clone()
},
None => files.clone(),
};
Ok(read_files(folder, &read, &part.footers, on_batch).await?)
}
(SourceReader::Files(_), None) => Err(LoaderError::Invalid(format!(
"{}: a files source has no files planned",
table.spec.source_table.qualified_name()
))),
}
}
pub(crate) async fn source_ends(
project: &ProjectConfig,
(source_name, source): (&str, &SourceConfig),
mut specs: Vec<TableSpec>,
options: &PlanOptions,
) -> Result<Vec<TableSpec>, LoaderError> {
if source.kind != SourceKind::Postgres || options.end_cursor.is_some() {
return Ok(specs);
}
let mut client: Option<Client> = None;
for spec in &mut specs {
let integer_cursor = spec.cursor.as_ref().and_then(|cursor| {
spec.source_table
.columns
.iter()
.find(|column| column.name == cursor.column)
.filter(|column| {
matches!(
column.column_type,
Ok(ColumnType::Int | ColumnType::SmallInt)
)
})
.map(|column| column.name.clone())
});
let wanted = matches!(spec.batch, Some(BatchStep::Integer(_)))
&& selected(
options,
(
source_name,
&spec.destination.table,
&spec.source_table.qualified_name(),
),
);
let Some(column) = integer_cursor.filter(|_| wanted) else {
continue;
};
if client.is_none() {
client = Some(connect_source(project, source).await?);
}
let Some(connected) = &client else {
continue;
};
spec.source_end = Some(
match cursor_end(connected, &spec.source_table, &column).await? {
Some((maximum, end)) => SourceEnd::Maximum { maximum, end },
None => SourceEnd::Empty,
},
);
}
Ok(specs)
}
pub(crate) fn unmatched_selectors(
project: &ProjectConfig,
options: &PlanOptions,
names: &[(String, String)],
) -> Vec<String> {
let known: Vec<&str> = names
.iter()
.flat_map(|(destination, source_table)| [destination.as_str(), source_table.as_str()])
.chain(project.sources.keys().map(String::as_str))
.collect();
options
.select
.iter()
.chain(&options.exclude)
.filter(|selector| !known.contains(&selector.as_str()))
.map(
|selector| match nearest_name(selector, known.iter().copied()) {
Some(nearest) => format!("no table matches {selector}; did you mean {nearest}?"),
None => format!("no table matches {selector}"),
},
)
.collect()
}
pub(crate) fn selected(
options: &PlanOptions,
(source, destination, source_table): (&str, &str, &str),
) -> bool {
let picks = |selector: &String| {
selector == source || selector == destination || selector == source_table
};
(options.select.is_empty() || options.select.iter().any(picks))
&& !options.exclude.iter().any(picks)
}
pub(crate) async fn scoped_tables(
(project, options): (&ProjectConfig, &PlanOptions),
(source_name, source): (&str, &SourceConfig),
) -> Result<Result<(ResolvedTables, Option<FilesSource>), Vec<String>>, LoaderError> {
let (discovered, files) = match source_tables(project, source_name, source).await? {
Ok(found) => found,
Err(problem) => {
let in_scope = source_in_scope((project, options), (source_name, source), None);
return Ok(Err(in_scope
.then(|| format!("[sources.{source_name}]: {problem}"))
.into_iter()
.collect()));
}
};
let mut resolved = resolve_tables(project, source_name, &discovered);
if !source_in_scope((project, options), (source_name, source), Some(&discovered)) {
resolved.problems.clear();
}
Ok(Ok((resolved, files)))
}
pub(crate) fn source_in_scope(
(project, options): (&ProjectConfig, &PlanOptions),
(source_name, source): (&str, &SourceConfig),
listed: Option<&[SourceTable]>,
) -> bool {
let Some(discovered) = listed else {
return selected(options, (source_name, "", ""))
|| (!options
.exclude
.iter()
.any(|selector| selector == source_name)
&& options
.select
.iter()
.any(|selector| !project.sources.contains_key(selector)));
};
let found: Vec<String> = discovered.iter().map(SourceTable::qualified_name).collect();
let named = source
.tables
.iter()
.filter(|(qualified, _)| !found.contains(qualified))
.map(|(qualified, overrides)| {
let destination = match overrides.name.as_deref() {
Some(name) if !name.contains('{') => name,
_ => "",
};
(destination.to_owned(), qualified.clone())
});
let tables: Vec<(String, String)> = discovered
.iter()
.map(|table| {
let name = source
.tables
.get(&table.qualified_name())
.and_then(|overrides| overrides.name.as_deref());
(
table_name((source_name, source), name, table),
table.qualified_name(),
)
})
.chain(named)
.collect();
(tables.is_empty() && selected(options, (source_name, "", "")))
|| tables.iter().any(|(destination, qualified)| {
selected(options, (source_name, destination, qualified))
})
}
pub(crate) async fn admit_tables(
mut plan: ProjectPlan,
(source_name, source): (&String, &SourceConfig),
(planned, unselected): (Vec<TablePlan>, &[TableSpec]),
found: (&BTreeMap<String, Destination>, &PlanOptions),
) -> Result<ProjectPlan, LoaderError> {
let first_load = first_load(&planned, unselected, found).await?;
let mut planned = planned;
if let Some(problem) = partial_capture(source_name, source.kind, (&planned, unselected)) {
for table in &mut planned {
table.schema = blocked_plan(vec![problem.clone()]);
}
}
for mut table in planned {
let spec = &table.spec;
let qualified = spec.source_table.qualified_name();
let is_new = !first_load && !table.exists && !spec.approved;
let has_settings = source.tables.contains_key(&qualified);
match (is_new, spec.new_tables, has_settings) {
(true, NewTables::Skip, true) => plan.skipped.push(SkippedTable {
source_name: source_name.clone(),
source_table: qualified,
reason: "new table (new_tables = \"skip\") has settings but is not approved; add it to include".to_owned(),
}),
(true, NewTables::Skip, false) => plan.skipped.push(SkippedTable {
source_name: source_name.clone(),
source_table: qualified,
reason: "new table (new_tables = \"skip\"); add it to include to sync it".to_owned(),
}),
(true, NewTables::Fail, _) => plan.problems.push(format!(
"{qualified}: new table and new_tables = \"fail\"; add it to include or exclude it"
)),
_ => {
if let Some(problem) = missing_description(source_name, source, &spec.source_table)
{
table.schema = blocked_plan(vec![problem]);
}
plan.tables.push(table);
}
}
}
Ok(plan)
}
fn partial_capture(
name: &str,
kind: SourceKind,
(planned, unselected): (&[TablePlan], &[TableSpec]),
) -> Option<String> {
let partial = kind == SourceKind::PostgresCdc && !planned.is_empty() && !unselected.is_empty();
partial.then(|| {
let left_out: Vec<String> = unselected
.iter()
.map(|spec| spec.source_table.qualified_name())
.collect();
format!(
"this selection leaves out {} of source {name}, whose changes share its one slot and would be skipped; select the whole source, or move those tables to another postgres_cdc source",
left_out.join(", ")
)
})
}
async fn first_load(
planned: &[TablePlan],
unselected: &[TableSpec],
(destinations, options): (&BTreeMap<String, Destination>, &PlanOptions),
) -> Result<bool, LoaderError> {
if planned.iter().any(|table| table.exists) {
return Ok(false);
}
for spec in unselected {
let Some(destination) = destinations.get(&spec.target) else {
continue;
};
let exists = match files_destination(destination) {
Some(_) => plan_table(spec, destination, options)
.await
.is_ok_and(|table| table.exists),
None => live_columns(destination, &spec.destination)
.await
.is_ok_and(|live| live.is_some()),
};
if exists {
return Ok(false);
}
}
Ok(true)
}