use crate::adapters::dispatch::main::dialect::sql_adapter::sql_adapter;
use arrow_array::RecordBatch;
use std::collections::{BTreeMap, HashMap};
use tokio_postgres::Client;
use crate::adapters::dispatch::main::dialect::files_destination::files_destination;
use crate::adapters::dispatch::main::session::live_columns::live_columns;
use crate::adapters::dispatch::main::session::text_values::text_values;
use crate::adapters::dispatch::models::Destination;
use crate::adapters::files::models::FolderClaim;
use crate::auth::constants::{GOOGLE_AUTHORIZE_URL, GOOGLE_TOKEN_URL};
use crate::auth::main::google_token_cache_file::google_token_cache_file;
use crate::auth::models::GoogleUser;
use crate::bigquery_source::main::bigquery_reader::bigquery_reader;
use crate::bigquery_source::main::read_catalog::read_catalog as read_bigquery_catalog;
use crate::bigquery_source::main::read_rows::read_rows as read_bigquery_rows;
use crate::bigquery_source::models::{BigQueryReadRequest, BigQueryReader};
use crate::column_types::main::arrow_field::arrow_field;
use crate::column_types::main::parse_type::parse_type;
use crate::column_types::models::{ColumnType, Warehouse};
use crate::config::models::{DestinationConfig, DestinationKind};
use crate::file_store::main::open_folder::open_folder;
use crate::files_source::constants::{
NANOSECOND_SCALE, SOURCE_FILE_COLUMN, SOURCE_ROW_COLUMN, SOURCE_VERSION_COLUMN,
VERSION_SEPARATOR,
};
use crate::files_source::main::constant_columns::constant_columns;
use crate::files_source::main::discover_files::discover_files;
use crate::files_source::main::merge_order::merge_order;
use crate::files_source::main::path_template::path_template;
use crate::files_source::main::read_files::read_files;
use crate::files_source::main::select_files::select_files;
use crate::files_source::models::{
BatchFooters, DataFile, FileColumn, FileSelection, FilesRead, FilesTable, LoadedFile,
PathValues, SourceFormat, StoredRead, TextChecks,
};
use crate::http::main::http_client::build_http_client;
use crate::http::models::ConnectionCounter;
use crate::loader::_helpers::destination::table_folder;
use crate::loader::_helpers::python::{fitting_type, python_tables};
use crate::loader::_helpers::resolve::{resolve_tables, source_target};
use crate::loader::_helpers::run_log::merged_files;
use crate::loader::_helpers::specs::{missing_description, table_name};
use crate::loader::_helpers::table_plan::{blocked_plan, plan_table};
use crate::loader::constants::{
KEPT_TABLES_COLUMN_FIX, LOCALHOST, PARALLEL_READ_MIN_BYTES, PARALLEL_READERS, RELOAD_FIX,
SOURCE_BATCH_ROWS,
};
use crate::loader::errors::LoaderError;
use crate::loader::models::{
CommitRule, FilesSource, PlanOptions, ProjectPlan, ResolvedTables, SkippedTable, SourceEnd,
SourceProblem, SourceReader, TablePlan, TableSpec,
};
use crate::loader::models::{FoundSource, FoundSources};
use crate::postgres_source::constants::DEFAULT_BATCH_BYTES;
use crate::postgres_source::main::connect_postgres::connect_postgres;
use crate::postgres_source::main::copy_table::copy_table;
use crate::postgres_source::main::cursor_end::cursor_end;
use crate::postgres_source::main::read_catalog::read_catalog;
use crate::postgres_source::models::{
CopyRequest, Encryption, PgType, PostgresAddress, ReadColumn, SourceColumn, SourceTable,
SslMode, WireType,
};
use crate::project::main::postgres_address::postgres_address;
use crate::project::models::{
BatchStep, ConnectionConfig, NewTables, ProjectConfig, SourceConfig, SourceKind, TableOverride,
TableSettings, WriteStrategy,
};
use crate::schema_plan::models::{ColumnOrigin, DesiredColumn, LiveColumn};
use crate::sheets_source::constants::{EMPTY_SHEET_COLUMN_KIND, SCOPE, SHEET_COLUMN_KIND};
use crate::sheets_source::main::duplicate_key::duplicate_key;
use crate::sheets_source::main::read_grids::read_grids;
use crate::sheets_source::main::sheet_batch::sheet_batch;
use crate::sheets_source::main::sheet_table::sheet_table;
use crate::sheets_source::main::sheets_client::sheets_client;
use crate::sheets_source::main::tab_name::tab_name;
use crate::sheets_source::main::tab_ranges::tab_ranges;
use crate::sheets_source::main::user_sheets_client::user_sheets_client;
use crate::sheets_source::models::{Grid, HeaderRule, KeySource, SheetsClient};
use crate::similar_names::main::close_name::close_name;
use crate::similar_names::main::nearest_name::nearest_name;
use crate::snowflake_source::constants::SESSION_SCHEMA;
use crate::snowflake_source::main::connect_snowflake_source::connect_snowflake_source;
use crate::snowflake_source::main::read_catalog::read_catalog as read_snowflake_catalog;
use crate::snowflake_source::main::unload_files::unload_files;
use crate::snowflake_source::main::unload_rows::unload_rows;
use crate::snowflake_source::models::{UnloadRequest, UnloadedFiles};
use crate::ssh_tunnel::models::SshTunnel;
use crate::store_paths::main::resolve_location::resolve_location;
use crate::store_paths::models::{Location, StoreKind};
pub(crate) async fn connect_source(
project: &ProjectConfig,
source: &SourceConfig,
) -> Result<(Client, Encryption), LoaderError> {
let address = postgres_address(project, &source.connection).map_err(LoaderError::Invalid)?;
Ok(connect_postgres(address).await?)
}
pub(crate) fn files_location(
project: &ProjectConfig,
source: &SourceConfig,
) -> Result<Location, LoaderError> {
let store = match project.connections.get(&source.connection) {
Some(ConnectionConfig::S3 {
bucket,
region,
endpoint,
profile,
}) => StoreKind::S3 {
bucket: bucket.clone(),
region: region.clone(),
endpoint: endpoint.clone(),
profile: profile.clone(),
},
Some(ConnectionConfig::Filesystem { root }) => StoreKind::Filesystem { root: root.clone() },
Some(ConnectionConfig::Sftp {
host,
port,
user,
private_key_file,
private_key_passphrase,
password,
known_hosts,
}) => StoreKind::Sftp {
server: Box::new(SshTunnel {
host: host.clone(),
port: *port,
user: user.clone(),
private_key_file: private_key_file.clone(),
private_key_passphrase: private_key_passphrase.clone(),
password: password.clone(),
known_hosts: Some(known_hosts.clone()),
}),
root: String::new(),
},
_ => {
return Err(LoaderError::Invalid(format!(
"connection {:?} is not an s3, filesystem or sftp connection",
source.connection
)));
}
};
let path = source.path.as_deref().unwrap_or_default();
resolve_location(Some((source.connection.as_str(), &store)), path)
.map_err(|error| LoaderError::Invalid(error.to_string()))
}
pub(crate) fn source_readers(table: &TablePlan) -> usize {
let whole = matches!(table.commits, CommitRule::Size | CommitRule::Whole(_));
let postgres = table.spec.python.is_none()
&& table.spec.files.is_none()
&& table.spec.sheet.is_none()
&& table.spec.warehouse_source.is_none();
match whole && postgres && table.spec.source_table.estimated_bytes >= PARALLEL_READ_MIN_BYTES {
true => PARALLEL_READERS,
false => 1,
}
}
pub(crate) async fn open_source(
project: &ProjectConfig,
source: &SourceConfig,
readers: usize,
) -> Result<SourceReader, LoaderError> {
match source.kind {
SourceKind::Postgres | SourceKind::PostgresCdc => {
let (client, _) = connect_source(project, source).await?;
let mut helpers = Vec::with_capacity(readers.saturating_sub(1));
for _ in 1..readers {
match connect_source(project, source).await {
Ok((helper, _)) => helpers.push(helper),
Err(_) => break,
}
}
Ok(SourceReader::Postgres(client, helpers))
}
SourceKind::Files => Ok(SourceReader::Files(open_folder(&files_location(
project, source,
)?)?)),
SourceKind::Snowflake => Ok(SourceReader::Snowflake(Box::new(connect_snowflake_source(
snowflake_session(project, source)?,
source.stage.clone(),
)?))),
SourceKind::Odbc => Ok(open_odbc(project, source)?),
SourceKind::BigQuery => Ok(SourceReader::BigQuery(Box::new(bigquery_source_reader(
project, source,
)?))),
SourceKind::Python => Err(LoaderError::Invalid(
"a Python source is read by calling its table functions".to_owned(),
)),
SourceKind::GoogleSheets => {
let (client, spreadsheet_id) = sheets_reader(project, source)?;
Ok(SourceReader::Sheets(client, spreadsheet_id))
}
}
}
pub(crate) async fn source_tables(
project: &ProjectConfig,
source_name: &str,
source: &SourceConfig,
) -> Result<Result<(Vec<SourceTable>, Option<FilesSource>, Vec<String>), SourceProblem>, LoaderError>
{
match source.kind {
SourceKind::Postgres | SourceKind::PostgresCdc => {
let address = match postgres_address(project, &source.connection) {
Ok(address) => address,
Err(problem) => return Ok(Err(problem.into())),
};
let (client, encryption) = match connect_postgres(address.clone()).await {
Ok(connected) => connected,
Err(error) => return Ok(Err(LoaderError::from(error).into())),
};
let warnings: Vec<String> =
plain_text_warning((source_name, &source.connection), &address, encryption)
.into_iter()
.collect();
Ok(
match read_catalog(&client, &source.include, &source.exclude).await {
Ok(discovered) => Ok((discovered, None, warnings)),
Err(error) => Err(LoaderError::from(error).into()),
},
)
}
SourceKind::Files => {
let warehouse = match source_target(project, source) {
Ok(target) => target.warehouse,
Err(problem) => return Ok(Err(problem.into())),
};
Ok(
match discover_files_table(project, source_name, source, warehouse).await {
Ok((table, files)) => Ok((vec![table], Some(files), Vec::new())),
Err(error) => Err(error.into()),
},
)
}
SourceKind::Snowflake => {
let reader = match snowflake_session(project, source)
.and_then(|session| Ok(connect_snowflake_source(session, None)?))
{
Ok(reader) => reader,
Err(error) => return Ok(Err(error.into())),
};
let queries: Vec<(String, String)> = source
.tables
.iter()
.filter_map(|(name, table)| Some((name.clone(), table.sql.clone()?)))
.collect();
Ok(
match read_snowflake_catalog(&reader, (&source.include, &source.exclude), &queries)
.await
{
Ok(discovered) => Ok((discovered, None, Vec::new())),
Err(error) => Err(LoaderError::from(error).into()),
},
)
}
SourceKind::Odbc => {
let queries: Vec<(String, String)> = source
.tables
.iter()
.filter_map(|(name, table)| Some((name.clone(), table.sql.clone()?)))
.collect();
Ok(discover_odbc(project, source, &queries)
.await
.map(|discovered| (discovered, None, Vec::new()))
.map_err(SourceProblem::from))
}
SourceKind::BigQuery => {
let reader = match bigquery_source_reader(project, source) {
Ok(reader) => reader,
Err(error) => return Ok(Err(error.into())),
};
let queries: Vec<(String, String)> = source
.tables
.iter()
.filter_map(|(name, table)| Some((name.clone(), table.sql.clone()?)))
.collect();
Ok(
match read_bigquery_catalog(&reader, (&source.include, &source.exclude), &queries)
.await
{
Ok(discovered) => Ok((discovered, None, Vec::new())),
Err(error) => Err(LoaderError::from(error).into()),
},
)
}
SourceKind::Python => Ok(Ok((python_tables(source), None, Vec::new()))),
SourceKind::GoogleSheets => Ok(match sheets_tables(project, source).await {
Ok((tables, warnings)) => Ok((tables, None, warnings)),
Err(error) => Err(error.into()),
}),
}
}
#[cfg(feature = "odbc")]
pub(crate) fn odbc_source_reader(
project: &ProjectConfig,
source: &SourceConfig,
) -> Result<crate::odbc_source::models::OdbcReader, LoaderError> {
let Some(ConnectionConfig::Odbc {
connection_string,
dsn,
user,
password,
}) = project.connections.get(&source.connection)
else {
return Err(LoaderError::Invalid(format!(
"connection {:?} is not an odbc connection",
source.connection
)));
};
let limits = crate::odbc_source::models::ValueLimits {
max_text_size: source.max_text_size,
max_binary_size: source.max_binary_size,
};
crate::odbc_source::main::odbc_reader::odbc_reader(
(connection_string.as_ref(), dsn.as_deref()),
(user.as_deref(), password.as_ref()),
limits,
)
.map_err(LoaderError::Invalid)
}
#[cfg(feature = "odbc")]
fn open_odbc(project: &ProjectConfig, source: &SourceConfig) -> Result<SourceReader, LoaderError> {
Ok(SourceReader::Odbc(Box::new(odbc_source_reader(
project, source,
)?)))
}
#[cfg(not(feature = "odbc"))]
fn open_odbc(_: &ProjectConfig, _: &SourceConfig) -> Result<SourceReader, LoaderError> {
Err(LoaderError::Invalid(
crate::loader::constants::NO_ODBC.to_owned(),
))
}
#[cfg(feature = "odbc")]
async fn discover_odbc(
project: &ProjectConfig,
source: &SourceConfig,
queries: &[(String, String)],
) -> Result<Vec<SourceTable>, String> {
let reader = odbc_source_reader(project, source).map_err(|error| error.to_string())?;
crate::odbc_source::main::read_catalog::read_catalog(
&reader,
(&source.include, &source.exclude),
queries,
)
.await
.map_err(|error| error.to_string())
}
#[cfg(not(feature = "odbc"))]
async fn discover_odbc(
_: &ProjectConfig,
_: &SourceConfig,
_: &[(String, String)],
) -> Result<Vec<SourceTable>, String> {
Err(crate::loader::constants::NO_ODBC.to_owned())
}
pub(crate) fn bigquery_source_reader(
project: &ProjectConfig,
source: &SourceConfig,
) -> Result<BigQueryReader, LoaderError> {
let connection = project.connections.get(&source.connection).ok_or_else(|| {
LoaderError::Invalid(format!(
"connection {:?} is not a [connections] entry",
source.connection
))
})?;
Ok(bigquery_reader(connection, source.project.as_deref())?)
}
pub(crate) fn sheets_reader(
project: &ProjectConfig,
source: &SourceConfig,
) -> Result<(SheetsClient, String), LoaderError> {
let Some(ConnectionConfig::GoogleSheets {
service_account_file,
service_account_json,
service_account_json_base64,
client_id,
client_secret,
refresh_token,
base_url,
token_url,
}) = project.connections.get(&source.connection)
else {
return Err(LoaderError::Invalid(format!(
"connection {:?} is not a google_sheets connection",
source.connection
)));
};
let given = [
service_account_json.is_some(),
service_account_json_base64.is_some(),
service_account_file.is_some(),
client_id.is_some(),
];
if given.into_iter().filter(|given| *given).count() > 1 {
return Err(LoaderError::Invalid(format!(
"connection {:?} sets more than one of service_account_json, service_account_json_base64, service_account_file and client_id; keep one",
source.connection
)));
}
let http = build_http_client(&ConnectionCounter::default())?;
let client = match (client_id, client_secret) {
(Some(client_id), Some(client_secret)) => {
let cache_file = google_token_cache_file().ok_or_else(|| {
LoaderError::Invalid(
"no cache folder for the Google sign-in: set HOME or XDG_CACHE_HOME (LOCALAPPDATA on Windows)".to_owned(),
)
})?;
let user = GoogleUser {
client_id: client_id.clone(),
client_secret: client_secret.clone(),
refresh_token: refresh_token.clone(),
scope: SCOPE.to_owned(),
connection: source.connection.clone(),
authorize_url: GOOGLE_AUTHORIZE_URL.to_owned(),
token_url: token_url.as_deref().unwrap_or(GOOGLE_TOKEN_URL).to_owned(),
cache_file,
};
user_sheets_client(http, user, base_url.as_deref())
}
(Some(_), None) => {
return Err(LoaderError::Invalid(format!(
"connection {:?} sets client_id without client_secret; set both, from the OAuth client's JSON file",
source.connection
)));
}
(None, _) if client_secret.is_some() || refresh_token.is_some() => {
return Err(LoaderError::Invalid(format!(
"connection {:?} sets client_secret or refresh_token without client_id; set client_id to read as a Google user",
source.connection
)));
}
(None, _) => {
let key = match (
service_account_json,
service_account_json_base64,
service_account_file,
) {
(Some(json), _, _) => KeySource::Json(json.clone()),
(None, Some(encoded), _) => KeySource::Base64(encoded.clone()),
(None, None, Some(path)) => KeySource::File(path.clone()),
(None, None, None) => KeySource::Environment,
};
sheets_client(http, &key, base_url.as_deref())?
}
};
let spreadsheet_id = source.spreadsheet_id.clone().ok_or_else(|| {
LoaderError::Invalid("a google_sheets source needs a spreadsheet_id".to_owned())
})?;
Ok((client, spreadsheet_id))
}
pub(crate) fn sheet_rule(source: &SourceConfig, name: &str) -> HeaderRule {
let table = source.tables.get(name).map(|table| &table.settings);
let pick = |value: fn(&TableSettings) -> Option<u32>| {
table.and_then(value).or(value(&source.defaults))
};
HeaderRule {
header: pick(|settings| settings.header.map(u32::from)).is_none_or(|header| header == 1),
header_row: pick(|settings| settings.header_row).unwrap_or(1),
}
}
async fn sheets_tables(
project: &ProjectConfig,
source: &SourceConfig,
) -> Result<(Vec<SourceTable>, Vec<String>), LoaderError> {
let (client, spreadsheet_id) = sheets_reader(project, source)?;
let mut warnings: Vec<String> = Vec::new();
let mut named: Vec<(String, Grid)> = Vec::new();
match source.ranges.is_empty() {
true => {
let (_, grids) = read_grids(&client, &spreadsheet_id, None).await?;
for grid in grids {
match grid.rows.is_empty() {
true => warnings.push(format!(
"tab {:?} is empty, so it loads no table",
grid.title
)),
false => named.push((tab_name(&grid.title), grid)),
}
}
}
false => {
for (name, range) in &source.ranges {
let (_, grids) = read_grids(&client, &spreadsheet_id, Some(range)).await?;
let grid = grids
.into_iter()
.next()
.ok_or_else(|| LoaderError::Invalid(format!("range {range:?} read no tab")))?;
named.push((name.clone(), grid));
}
}
}
let mut problems: Vec<String> = Vec::new();
let mut tables: Vec<SourceTable> = Vec::with_capacity(named.len());
for (name, grid) in named {
if tables.iter().any(|table| table.name == name) {
problems.push(format!(
"tab {:?} loads as table {name}, as an earlier tab does; rename one, or name the tabs in ranges",
grid.title
));
continue;
}
let mut read = match sheet_table(&grid, sheet_rule(source, &name)) {
Ok(read) => read,
Err(found) => {
problems.extend(found);
continue;
}
};
let settings = source.tables.get(&name).map(|table| &table.settings);
let key = settings
.and_then(|settings| settings.unique_key.as_ref())
.or(source.defaults.unique_key.as_ref());
let merge = settings
.and_then(|settings| settings.write_strategy)
.or(source.defaults.write_strategy)
== Some(WriteStrategy::Merge);
let duplicate = key.filter(|_| merge).and_then(|key| {
let problem = duplicate_key(&grid, sheet_rule(source, &name), &read, key)?;
Some((problem, key.first()?.clone()))
});
if let Some((problem, first)) = duplicate
&& let Some(column) = read
.columns
.iter_mut()
.find(|column| column.name.eq_ignore_ascii_case(&first))
{
column.column_type = Err(problem);
}
warnings.extend(read.warnings);
tables.push(SourceTable {
schema: String::new(),
name,
columns: read
.columns
.into_iter()
.map(|column| SourceColumn {
name: column.name,
postgres_type: match column.has_values {
true => SHEET_COLUMN_KIND,
false => EMPTY_SHEET_COLUMN_KIND,
}
.to_owned(),
pg_type: PgType::Other,
column_type: column.column_type,
nullable: true,
source_spelling: None,
})
.collect(),
primary_key: Vec::new(),
indexed_columns: Vec::new(),
estimated_rows: read.rows as i64,
estimated_bytes: 0,
comment: None,
});
}
match problems.is_empty() {
true => Ok((tables, warnings)),
false => Err(LoaderError::Invalid(problems.join("; "))),
}
}
pub(crate) fn with_live_types(spec: &TableSpec, live: Option<&[LiveColumn]>) -> Option<TableSpec> {
spec.sheet.as_ref()?;
let live = live?;
let mut typed = spec.clone();
let mut changed = false;
for column in &mut typed.source_table.columns {
let Some(live_type) = live
.iter()
.find(|live| live.name.eq_ignore_ascii_case(&column.name))
.map(|live| live.column_type.clone())
else {
continue;
};
let Ok(inferred) = &column.column_type else {
continue;
};
let column_type = match (column.postgres_type.as_str(), &live_type, inferred) {
(EMPTY_SHEET_COLUMN_KIND, _, _) | (_, ColumnType::TimestampNtz, ColumnType::Date) => {
live_type
}
_ => fitting_type(inferred.clone(), Some(live_type)),
};
changed |= *inferred != column_type;
column.column_type = Ok(column_type);
}
changed.then_some(typed)
}
async fn sheet_rows(
client: &SheetsClient,
spreadsheet_id: &str,
table: &TablePlan,
) -> Result<RecordBatch, LoaderError> {
let name = &table.spec.source_table.name;
let read = table
.spec
.sheet
.as_ref()
.ok_or_else(|| LoaderError::Invalid(format!("{name} is not a sheet's table")))?;
let range = match &read.range {
Some(range) => range.clone(),
None => {
let (_, tabs) = tab_ranges(client, spreadsheet_id).await?;
tabs.into_iter()
.find(|(title, _)| tab_name(title) == *name)
.map(|(_, range)| range)
.ok_or_else(|| {
LoaderError::Invalid(format!(
"the spreadsheet has no tab for table {name} any more; run plan again"
))
})?
}
};
let (_, grids) = read_grids(client, spreadsheet_id, Some(&range)).await?;
let grid = grids
.into_iter()
.next()
.ok_or_else(|| LoaderError::Invalid(format!("range {range:?} read no tab")))?;
let columns: Vec<(String, ColumnType)> = table
.read_columns
.iter()
.map(|column| (column.name.clone(), column.output.clone()))
.collect();
sheet_batch(&grid, read.rule, &columns).map_err(LoaderError::Invalid)
}
fn plain_text_warning(
(source_name, connection): (&str, &str),
address: &PostgresAddress<'_>,
encryption: Encryption,
) -> Option<String> {
let local = address.host == LOCALHOST
|| address.host.starts_with('/')
|| address
.host
.parse::<std::net::IpAddr>()
.is_ok_and(|ip| ip.is_loopback());
(encryption == Encryption::None && address.tls.mode == SslMode::Prefer && !local).then(|| {
format!(
"[sources.{source_name}] reads {}:{} in plain text: the server does not offer TLS, so sslmode = \"prefer\" fell back; set sslmode = \"require\" (or \"verify-full\") on connection {connection} to refuse plain text, or \"disable\" to accept it",
address.host, address.port
)
})
}
fn snowflake_session(
project: &ProjectConfig,
source: &SourceConfig,
) -> Result<DestinationConfig, LoaderError> {
let Some(ConnectionConfig::Snowflake {
account,
user,
auth,
role,
warehouse,
database,
base_url,
..
}) = project.connections.get(&source.connection)
else {
return Err(LoaderError::Invalid(format!(
"connection {:?} is not a snowflake connection",
source.connection
)));
};
Ok(DestinationConfig {
kind: DestinationKind::Snowflake,
account: account.clone(),
user: user.clone(),
auth: auth.clone(),
role: role.clone(),
warehouse: source.warehouse.clone().or_else(|| warehouse.clone()),
database: database.clone(),
schema: SESSION_SCHEMA.to_owned(),
base_url: base_url.clone(),
})
}
pub(crate) async fn discover_files_table(
project: &ProjectConfig,
source_name: &str,
source: &SourceConfig,
warehouse: Warehouse,
) -> Result<(SourceTable, FilesSource), LoaderError> {
let location = files_location(project, source)?;
let folder = open_folder(&location)?;
let (values, _) = path_values((source_name, source), warehouse)?;
let encoding = source.files_encoding();
let table = discover_files(&folder, &values, &encoding).await?;
if let Some(problem) = table.problems.first() {
return Err(LoaderError::Invalid(problem.clone()));
}
files_source_parts(source_name, source, warehouse, (location, table))
}
fn path_values(
(source_name, source): (&str, &SourceConfig),
warehouse: Warehouse,
) -> Result<(PathValues, Vec<(String, ColumnType)>), LoaderError> {
let mut types: Vec<(String, ColumnType)> = Vec::new();
for (name, text) in &source.partition_columns {
let parsed = parse_type(text, &[warehouse])
.map_err(|error| LoaderError::Invalid(format!("partition column {name}: {error}")))?;
if matches!(parsed.column_type, ColumnType::Array(_)) {
return Err(LoaderError::Invalid(format!(
"partition column {name}: an array cannot come from a path"
)));
}
types.push((name.clone(), parsed.column_type));
}
let template = source
.path_columns
.as_deref()
.map(|text| path_template(text, &[warehouse]))
.transpose()
.map_err(LoaderError::Invalid)?;
types.extend(
template
.iter()
.flat_map(|template| template.captures.clone()),
);
let constants = constant_columns(&source.table_constants(source_name), &[warehouse])
.map_err(LoaderError::Invalid)?;
let typed = types
.iter()
.map(|(name, column_type)| (name, column_type))
.chain(
constants
.iter()
.map(|(name, column_type, _)| (name, column_type)),
);
for (name, column_type) in typed {
if arrow_field(name, column_type).is_none() {
return Err(LoaderError::Invalid(format!(
"column {name}: {column_type} has no Arrow type, so a partition, path or constant column cannot be one; pin a portable type"
)));
}
}
let values = PathValues {
partition_columns: source.partition_columns.keys().cloned().collect(),
template,
unmatched: source.unmatched_paths.unwrap_or_default(),
constants: constants
.iter()
.map(|(name, _, text)| (name.clone(), text.clone()))
.collect(),
};
types.extend(
constants
.into_iter()
.map(|(name, column_type, _)| (name, column_type)),
);
Ok((values, types))
}
pub(crate) fn files_source_parts(
source_name: &str,
source: &SourceConfig,
warehouse: Warehouse,
(location, table): (Location, FilesTable),
) -> Result<(SourceTable, FilesSource), LoaderError> {
let encoding = source.files_encoding();
let (values, partition_types) = path_values((source_name, source), warehouse)?;
let added: Vec<&str> = partition_types
.iter()
.map(|(name, _)| name.as_str())
.chain([SOURCE_FILE_COLUMN, SOURCE_VERSION_COLUMN, SOURCE_ROW_COLUMN])
.collect();
let overrides = source.tables.get(source_name);
let pinned_only = pinned_unsampled(overrides, &table, &added, encoding.format)?;
let dropped = (!table.dropped.is_empty()).then(|| {
format!(
"dropped {} from the files (replaced by this load's)",
table.dropped.join(", ")
)
});
let notes: Vec<String> = dropped
.into_iter()
.chain(pinned_only.iter().map(|name| {
format!("pinned column {name} was not seen in the sampled files (check the name)")
}))
.collect();
let typed = table
.columns
.iter()
.filter(|column| !added.contains(&column.name.as_str()))
.map(|column| (column.name.clone(), column.column_type.clone()))
.chain(
pinned_only
.into_iter()
.map(|name| (name, Ok(ColumnType::Text))),
)
.chain(
partition_types
.iter()
.map(|(name, column_type)| (name.clone(), Ok(column_type.clone()))),
)
.chain([
(SOURCE_FILE_COLUMN.to_owned(), Ok(ColumnType::Text)),
(SOURCE_VERSION_COLUMN.to_owned(), Ok(ColumnType::Text)),
(SOURCE_ROW_COLUMN.to_owned(), Ok(ColumnType::Int)),
]);
let columns: Vec<SourceColumn> = typed
.map(|(name, column_type)| SourceColumn {
name,
postgres_type: encoding.format.to_string(),
pg_type: PgType::Other,
column_type,
nullable: true,
source_spelling: None,
})
.collect();
let checks = TextChecks {
known: columns.iter().map(|column| column.name.clone()).collect(),
exclude: [
source.exclude_columns.as_slice(),
overrides.map_or(&[], |table| table.exclude_columns.as_slice()),
]
.concat(),
allow: [
source.allow_columns.as_slice(),
overrides.map_or(&[], |table| table.allow_columns.as_slice()),
]
.concat(),
inferred: Vec::new(),
};
let source_table = SourceTable {
schema: String::new(),
name: source_name.to_owned(),
columns,
primary_key: Vec::new(),
indexed_columns: Vec::new(),
estimated_rows: -1,
comment: None,
estimated_bytes: table
.files
.iter()
.map(|data| i64::try_from(data.file.size).unwrap_or(i64::MAX))
.sum(),
};
Ok((
source_table,
FilesSource {
location,
table,
partition_types,
constants: values.constants.into_iter().map(|(name, _)| name).collect(),
encoding,
checks,
notes,
copy_to_internal_stage: source.copy_to_internal_stage,
},
))
}
fn pinned_unsampled(
overrides: Option<&TableOverride>,
table: &FilesTable,
added: &[&str],
format: SourceFormat,
) -> Result<Vec<String>, LoaderError> {
let Some(overrides) = overrides.filter(|_| format != SourceFormat::Parquet) else {
return Ok(Vec::new());
};
let sampled: Vec<&str> = table
.columns
.iter()
.map(|column| column.name.as_str())
.collect();
let mut unsampled: Vec<String> = Vec::new();
for name in overrides.columns.keys() {
if added.contains(&name.as_str()) || sampled.contains(&name.as_str()) {
continue;
}
let known = sampled.iter().chain(added).copied();
if let Some(close) = close_name(name, known) {
return Err(LoaderError::Invalid(format!(
"pinned column {name} was not seen in the sampled files; did you mean {close:?}? (pin a name exactly as it appears in the files)"
)));
}
unsampled.push(name.clone());
}
Ok(unsampled)
}
pub(crate) fn files_read_columns(spec: &TableSpec) -> Result<Vec<ReadColumn>, Vec<String>> {
let mut problems: Vec<String> = Vec::new();
let mut columns: Vec<ReadColumn> = Vec::new();
for column in &spec.source_table.columns {
let excluded = spec
.excluded_columns
.iter()
.any(|(excluded, _)| *excluded == column.name);
let output = match (spec.pinned.get(&column.name), &column.column_type) {
_ if excluded => continue,
(Some(pinned), _) => pinned.clone(),
(None, Ok(column_type)) => column_type.clone(),
(None, Err(reason)) => {
problems.push(format!("column {}: {reason}", column.name));
continue;
}
};
columns.push(ReadColumn {
name: column.name.clone(),
expression: column.name.clone(),
wire: WireType::Text,
output,
});
}
match problems.is_empty() {
true => Ok(columns),
false => Err(problems),
}
}
pub(crate) struct FilesPlanRequest<'a> {
pub(crate) spec: &'a TableSpec,
pub(crate) files: &'a FilesSource,
pub(crate) loaded: LoadedSource<'a>,
pub(crate) live: Option<&'a [LiveColumn]>,
pub(crate) read_columns: &'a [ReadColumn],
pub(crate) written: &'a [String],
pub(crate) bounds: (Option<&'a str>, Option<&'a str>),
pub(crate) options: &'a PlanOptions,
}
#[derive(Debug, Clone, Copy)]
pub(crate) enum LoadedSource<'a> {
Query(&'a Destination),
Recorded(&'a [LoadedFile]),
}
async fn merge_ledger(
request: &FilesPlanRequest<'_>,
) -> Result<Result<Vec<LoadedFile>, String>, LoaderError> {
let spec = request.spec;
let destination = match request.loaded {
LoadedSource::Query(destination) => destination,
LoadedSource::Recorded(_) => {
return Ok(Err("merge is not supported for files targets".to_owned()));
}
};
if !sql_adapter(spec.warehouse).merges_files() {
return Ok(Err(format!(
"merge from a files source loads into DuckDB and Postgres targets so far, not {}; use append, or delete_insert on a partition cursor",
spec.warehouse
)));
}
if request.options.reload {
return Ok(Err(
"--reload would merge every file in one commit and lose their order; drop the table and load again to merge every file anew".to_owned(),
));
}
if spec.logged {
return Ok(Err(
"state = \"logged\" does not apply to a files source's merge, which records each file it merges".to_owned(),
));
}
if let Some(cursor) = &spec.cursor {
return Ok(Err(format!(
"cursor_column {} does not apply to a files source's merge, which picks the files it has not merged from the run log, whatever their partition; remove it and set write_strategy = \"merge\", or keep it and use delete_insert",
cursor.column
)));
}
let merged = match request.live {
None => return Ok(Ok(Vec::new())),
Some(_) => merged_files(spec, destination).await?,
};
let location = request.files.location.to_string();
let listed: HashMap<String, String> = request
.files
.table
.files
.iter()
.map(|data| {
(
format!("{location}{}", data.file.relative_key),
data.file.version(),
)
})
.collect();
Ok(merged.map(|merged| unchanged_files(merged, &listed)))
}
fn unchanged_files(merged: Vec<LoadedFile>, listed: &HashMap<String, String>) -> Vec<LoadedFile> {
let mut newest: HashMap<String, LoadedFile> = HashMap::new();
for file in merged {
newest.insert(file.name.clone(), file);
}
newest
.into_values()
.filter(|file| {
listed
.get(&file.name)
.is_none_or(|version| *version == file.version)
})
.collect()
}
pub(crate) fn loaded_file(text: String, versioned: bool) -> LoadedFile {
match (versioned, text.rsplit_once(VERSION_SEPARATOR)) {
(true, Some((name, version))) => LoadedFile {
name: name.to_owned(),
version: version.to_owned(),
},
_ => LoadedFile {
name: text,
version: String::new(),
},
}
}
fn live_has(live: Option<&[LiveColumn]>, column: &str) -> bool {
match live {
Some(live) => live
.iter()
.any(|existing| existing.name.eq_ignore_ascii_case(column)),
None => false,
}
}
pub(crate) fn nanosecond_columns(
spec: &TableSpec,
columns: &[FileColumn],
live: Option<&[LiveColumn]>,
) -> Vec<String> {
match sql_adapter(spec.warehouse).keeps_nanoseconds() {
true => columns
.iter()
.filter(|column| {
matches!(
column.stored_read,
StoredRead::Nanos | StoredRead::Epoch(NANOSECOND_SCALE)
)
})
.filter(|column| !spec.pinned.contains_key(&column.name))
.filter(|column| fewer_digits(live, &column.name).is_none())
.map(|column| column.name.clone())
.collect(),
false => Vec::new(),
}
}
pub(crate) fn truncated_nanoseconds(spec: &TableSpec, live: Option<&[LiveColumn]>) -> Vec<String> {
let Some(files) = &spec.files else {
return Vec::new();
};
files
.table
.columns
.iter()
.filter(|column| {
matches!(
column.stored_read,
StoredRead::Nanos | StoredRead::Epoch(NANOSECOND_SCALE)
) && !spec.pinned.contains_key(&column.name)
})
.filter_map(|column| truncated_warning(live, &column.name))
.collect()
}
fn truncated_warning(live: Option<&[LiveColumn]>, name: &str) -> Option<String> {
let digits = fewer_digits(live, name)?;
Some(format!(
"{name}: nanoseconds in the files, but the column holds {digits} digits after the second, so they are truncated.\nRebuild the column at (9) (--reload) to keep them."
))
}
fn fewer_digits(live: Option<&[LiveColumn]>, name: &str) -> Option<i64> {
live.into_iter()
.flatten()
.find(|existing| existing.name.eq_ignore_ascii_case(name))
.and_then(|existing| existing.fraction_digits)
.filter(|digits| *digits < i64::from(NANOSECOND_SCALE))
}
pub(crate) fn column_fix(warehouse: Warehouse) -> &'static str {
match sql_adapter(warehouse).keeps_tables() {
true => KEPT_TABLES_COLUMN_FIX,
false => RELOAD_FIX,
}
}
pub(crate) async fn plan_files(
request: &FilesPlanRequest<'_>,
) -> Result<Result<FilesRead, String>, LoaderError> {
let spec = request.spec;
let cursor = match &spec.cursor {
None => None,
Some(cursor) => match request
.files
.partition_types
.iter()
.find(|(name, _)| *name == cursor.column)
{
Some((name, column_type)) => Some((name.as_str(), column_type)),
None => {
return Ok(Err(format!(
"cursor_column {} must be one of the partition_columns or path_columns for a files source",
cursor.column
)));
}
},
};
if let Some((name, _)) = cursor
&& request
.files
.constants
.iter()
.any(|constant| constant == name)
{
return Ok(Err(format!(
"cursor_column {name} is a constant column, the same in every file, so it cannot pick the files to read; use a partition_columns or path_columns column"
)));
}
let merged = match spec.strategy == WriteStrategy::Merge {
true => match merge_ledger(request).await? {
Ok(merged) => Some(merged),
Err(problem) => return Ok(Err(problem)),
},
false => None,
};
let skips_loaded = spec.strategy == WriteStrategy::Append;
if skips_loaded
&& request.live.is_some()
&& !live_has(request.live, SOURCE_FILE_COLUMN)
&& !request.options.reload
{
return Ok(Err(format!(
"the table has no {SOURCE_FILE_COLUMN} column, so append cannot tell which files are loaded and would load them all again; {}",
column_fix(spec.warehouse)
)));
}
if skips_loaded
&& !request
.written
.iter()
.any(|name| name == SOURCE_FILE_COLUMN)
{
return Ok(Err(format!(
"{SOURCE_FILE_COLUMN} must be loaded: append uses it to skip files already loaded"
)));
}
let live_has = |column: &str| live_has(request.live, column);
let loaded: Vec<LoadedFile> =
match skips_loaded && live_has(SOURCE_FILE_COLUMN) && !request.options.reload {
true => match request.loaded {
LoadedSource::Query(destination) => {
let version = live_has(SOURCE_VERSION_COLUMN).then_some(SOURCE_VERSION_COLUMN);
let query = sql_adapter(spec.warehouse).render_loaded_files_query(
&spec.destination,
SOURCE_FILE_COLUMN,
version,
);
text_values(destination, query)
.await?
.into_iter()
.map(|text| loaded_file(text, version.is_some()))
.collect()
}
LoadedSource::Recorded(recorded) => recorded.to_vec(),
},
false => Vec::new(),
};
let loaded = merged.clone().unwrap_or(loaded);
let location = request.files.location.to_string();
let selection = FileSelection {
files: &request.files.table.files,
location: &location,
loaded: &loaded,
cursor,
from: request.bounds.0,
to: request.bounds.1,
};
let columns: Vec<DesiredColumn> = request
.read_columns
.iter()
.map(|column| DesiredColumn {
name: column.name.clone(),
column_type: column.output.clone(),
origin: ColumnOrigin::Source,
})
.collect();
let constants: Vec<&str> = request
.files
.partition_types
.iter()
.map(|(name, _)| name.as_str())
.chain([SOURCE_FILE_COLUMN, SOURCE_VERSION_COLUMN, SOURCE_ROW_COLUMN])
.collect();
let checks = TextChecks {
inferred: request
.read_columns
.iter()
.map(|column| column.name.clone())
.filter(|name| !spec.pinned.contains_key(name) && !constants.contains(&name.as_str()))
.collect(),
..request.files.checks.clone()
};
let nanos = nanosecond_columns(spec, &request.files.table.columns, request.live);
let ordered = select_files(&selection).map(|files| match merged {
Some(_) => merge_order(files),
None => files,
});
Ok(ordered.map(|files| FilesRead {
files,
encoding: request.files.encoding.clone(),
checks,
columns,
nanos,
batch_rows: SOURCE_BATCH_ROWS,
}))
}
#[derive(Debug, Clone, Default)]
pub(crate) struct ReadPart {
pub(crate) filter: Option<String>,
pub(crate) order_by: Option<String>,
pub(crate) files: Option<Vec<DataFile>>,
pub(crate) footers: BatchFooters,
pub(crate) batch_bytes: Option<usize>,
}
pub(crate) async fn read_source(
reader: &SourceReader,
table: &TablePlan,
part: ReadPart,
mut on_batch: impl AsyncFnMut(RecordBatch) -> Result<(), LoaderError>,
) -> Result<u64, LoaderError> {
let mut failed: Option<LoaderError> = None;
let read = read_each(reader, table, part, async |batch| {
on_batch(batch).await.map_err(|error| {
let text = error.to_string();
failed = Some(error);
text
})
})
.await;
match (read, failed) {
(Err(read), Some(failed)) if failed.is_retryable() => {
Err(LoaderError::Transient(read.to_string()))
}
(read, _) => read,
}
}
async fn read_each(
reader: &SourceReader,
table: &TablePlan,
part: ReadPart,
on_batch: impl AsyncFnMut(RecordBatch) -> Result<(), String>,
) -> Result<u64, LoaderError> {
match (reader, &table.files) {
(SourceReader::Postgres(client, helpers), _) => {
let whole = part.filter.is_none() && part.order_by.is_none();
let request = CopyRequest {
table: &table.spec.source_table,
columns: table.read_columns.clone(),
filter: part.filter.or_else(|| table.source_filter.clone()),
order_by: part.order_by,
batch_rows: SOURCE_BATCH_ROWS,
batch_bytes: part
.batch_bytes
.map_or(DEFAULT_BATCH_BYTES, |bytes| bytes.min(DEFAULT_BATCH_BYTES)),
helpers: match whole {
true => helpers,
false => &[],
},
};
Ok(copy_table(client, request, on_batch).await?)
}
(SourceReader::Files(folder), Some(files)) => {
let read = match part.files.clone() {
Some(subset) => FilesRead {
files: subset,
..files.clone()
},
None => files.clone(),
};
Ok(read_files(folder, &read, &part.footers, on_batch).await?)
}
(SourceReader::Snowflake(snowflake), _) => {
let filter = part.filter.or_else(|| table.source_filter.clone());
let request = unload_request(table, filter.as_deref());
Ok(unload_rows(snowflake, &request, on_batch).await?)
}
#[cfg(feature = "odbc")]
(SourceReader::Odbc(reader), _) => {
let (filter, parameters) = match part.filter {
Some(filter) => (Some(filter), &[][..]),
None => (table.source_filter.clone(), &table.source_parameters[..]),
};
let request = crate::odbc_source::models::OdbcReadRequest {
table: &table.spec.source_table,
sql: table
.spec
.warehouse_source
.as_ref()
.and_then(|source| source.sql.as_deref()),
columns: &table.read_columns,
filter: filter.as_deref(),
parameters,
batch_rows: SOURCE_BATCH_ROWS,
};
Ok(crate::odbc_source::main::read_rows::read_rows(reader, &request, on_batch).await?)
}
(SourceReader::BigQuery(reader), _) => {
let filter = part.filter.or_else(|| table.source_filter.clone());
let request = BigQueryReadRequest {
table: &table.spec.source_table,
sql: table
.spec
.warehouse_source
.as_ref()
.and_then(|source| source.sql.as_deref()),
columns: &table.read_columns,
filter: filter.as_deref(),
};
Ok(read_bigquery_rows(reader, &request, on_batch).await?)
}
(SourceReader::Sheets(client, spreadsheet_id), _) => {
let batch = sheet_rows(client, spreadsheet_id, table).await?;
let rows = batch.num_rows() as u64;
let mut on_batch = on_batch;
on_batch(batch).await.map_err(LoaderError::Invalid)?;
Ok(rows)
}
(SourceReader::Files(_), None) => Err(LoaderError::Invalid(format!(
"{}: a files source has no files planned",
table.spec.source_table.qualified_name()
))),
}
}
pub(crate) async fn unload_source<T>(
reader: &SourceReader,
table: &TablePlan,
part: ReadPart,
load: impl AsyncFnOnce(&UnloadedFiles) -> Result<T, LoaderError>,
) -> Result<T, LoaderError> {
let SourceReader::Snowflake(snowflake) = reader else {
return Err(LoaderError::Invalid(format!(
"{}: only a warehouse source unloads files",
table.spec.source_table.qualified_name()
)));
};
let filter = part.filter.or_else(|| table.source_filter.clone());
unload_files(snowflake, &unload_request(table, filter.as_deref()), load).await
}
fn unload_request<'a>(table: &'a TablePlan, filter: Option<&'a str>) -> UnloadRequest<'a> {
UnloadRequest {
table: &table.spec.source_table,
sql: table
.spec
.warehouse_source
.as_ref()
.and_then(|source| source.sql.as_deref()),
columns: &table.read_columns,
filter,
batch_rows: SOURCE_BATCH_ROWS,
}
}
pub(crate) async fn source_ends(
project: &ProjectConfig,
(source_name, source): (&str, &SourceConfig),
mut specs: Vec<TableSpec>,
options: &PlanOptions,
) -> Result<Vec<TableSpec>, LoaderError> {
if source.kind != SourceKind::Postgres || options.end_cursor.is_some() {
return Ok(specs);
}
let mut client: Option<Client> = None;
for spec in &mut specs {
let integer_cursor = spec.cursor.as_ref().and_then(|cursor| {
spec.source_table
.columns
.iter()
.find(|column| column.name == cursor.column)
.filter(|column| {
matches!(
column.column_type,
Ok(ColumnType::Int | ColumnType::SmallInt)
)
})
.map(|column| column.name.clone())
});
let wanted = matches!(spec.batch, Some(BatchStep::Integer(_)))
&& selected(
options,
(
source_name,
&spec.destination.table,
&spec.source_table.qualified_name(),
),
);
let Some(column) = integer_cursor.filter(|_| wanted) else {
continue;
};
if client.is_none() {
client = Some(connect_source(project, source).await?.0);
}
let Some(connected) = &client else {
continue;
};
spec.source_end = Some(
match cursor_end(connected, &spec.source_table, &column).await? {
Some((maximum, end)) => SourceEnd::Maximum { maximum, end },
None => SourceEnd::Empty,
},
);
}
Ok(specs)
}
pub(crate) fn destination_names(specs: &[TableSpec]) -> Vec<(String, String)> {
specs
.iter()
.map(|spec| {
(
spec.destination.table.clone(),
spec.source_table.qualified_name(),
)
})
.collect()
}
pub(crate) fn unmatched_selectors(
project: &ProjectConfig,
options: &PlanOptions,
names: &[(String, String)],
) -> Vec<String> {
let known: Vec<&str> = names
.iter()
.flat_map(|(destination, source_table)| [destination.as_str(), source_table.as_str()])
.chain(project.sources.keys().map(String::as_str))
.collect();
options
.select
.iter()
.chain(&options.exclude)
.filter(|selector| !known.contains(&selector.as_str()))
.map(
|selector| match nearest_name(selector, known.iter().copied()) {
Some(nearest) => format!("no table matches {selector}; did you mean {nearest}?"),
None => format!("no table matches {selector}"),
},
)
.collect()
}
pub(crate) fn selected(
options: &PlanOptions,
(source, destination, source_table): (&str, &str, &str),
) -> bool {
let picks = |selector: &String| {
selector == source || selector == destination || selector == source_table
};
(options.select.is_empty() || options.select.iter().any(picks))
&& !options.exclude.iter().any(picks)
}
pub(crate) async fn scoped_tables(
(project, options): (&ProjectConfig, &PlanOptions),
(source_name, source): (&str, &SourceConfig),
) -> Result<Result<(ResolvedTables, Option<FilesSource>), Option<SourceProblem>>, LoaderError> {
let (discovered, files, warnings) = match source_tables(project, source_name, source).await? {
Ok(found) => found,
Err(problem) => {
let in_scope = source_in_scope((project, options), (source_name, source), None);
return Ok(Err(in_scope.then(|| SourceProblem {
message: format!("[sources.{source_name}]: {}", problem.message),
retryable: problem.retryable,
})));
}
};
let mut resolved = resolve_tables(project, source_name, &discovered);
match source_in_scope((project, options), (source_name, source), Some(&discovered)) {
true => resolved.warnings = warnings,
false => resolved.problems.clear(),
}
Ok(Ok((resolved, files)))
}
pub(crate) fn source_in_scope(
(project, options): (&ProjectConfig, &PlanOptions),
(source_name, source): (&str, &SourceConfig),
listed: Option<&[SourceTable]>,
) -> bool {
let Some(discovered) = listed else {
return selected(options, (source_name, "", ""))
|| (!options
.exclude
.iter()
.any(|selector| selector == source_name)
&& options
.select
.iter()
.any(|selector| !project.sources.contains_key(selector)));
};
let found: Vec<String> = discovered.iter().map(SourceTable::qualified_name).collect();
let named = source
.tables
.iter()
.filter(|(qualified, _)| !found.contains(qualified))
.map(|(qualified, overrides)| {
let destination = match overrides.name.as_deref() {
Some(name) if !name.contains('{') => name,
_ => "",
};
(destination.to_owned(), qualified.clone())
});
let tables: Vec<(String, String)> = discovered
.iter()
.map(|table| {
let name = source
.tables
.get(&table.qualified_name())
.and_then(|overrides| overrides.name.as_deref());
(
table_name((source_name, source), name, table),
table.qualified_name(),
)
})
.chain(named)
.collect();
(tables.is_empty() && selected(options, (source_name, "", "")))
|| tables.iter().any(|(destination, qualified)| {
selected(options, (source_name, destination, qualified))
})
}
pub(crate) async fn admit_tables(
mut plan: ProjectPlan,
(source_name, source): (&String, &SourceConfig),
(planned, unselected): (Vec<TablePlan>, &[TableSpec]),
found: (&BTreeMap<String, Destination>, &PlanOptions),
) -> Result<ProjectPlan, LoaderError> {
let first_load = first_load(&planned, unselected, found).await?;
let mut planned = planned;
if let Some(problem) = partial_capture(source_name, source.kind, (&planned, unselected)) {
for table in &mut planned {
table.schema = blocked_plan(vec![problem.clone()]);
}
}
for mut table in planned {
let spec = &table.spec;
let qualified = spec.source_table.qualified_name();
let is_new = !first_load && !table.exists && !spec.approved;
let has_settings = source.tables.contains_key(&qualified);
match (is_new, spec.new_tables, has_settings) {
(true, NewTables::Skip, true) => plan.skipped.push(SkippedTable {
source_name: source_name.clone(),
source_table: qualified,
reason: "new table (new_tables = \"skip\") has settings but is not approved; add it to include".to_owned(),
}),
(true, NewTables::Skip, false) => plan.skipped.push(SkippedTable {
source_name: source_name.clone(),
source_table: qualified,
reason: "new table (new_tables = \"skip\"); add it to include to sync it".to_owned(),
}),
(true, NewTables::Fail, _) => plan.problems.push(format!(
"{qualified}: new table and new_tables = \"fail\"; add it to include or exclude it"
)),
_ => {
if let Some(problem) = missing_description(source_name, source, &spec.source_table)
{
table.schema = blocked_plan(vec![problem]);
}
plan.tables.push(table);
}
}
}
Ok(plan)
}
fn partial_capture(
name: &str,
kind: SourceKind,
(planned, unselected): (&[TablePlan], &[TableSpec]),
) -> Option<String> {
let partial = kind == SourceKind::PostgresCdc && !planned.is_empty() && !unselected.is_empty();
partial.then(|| {
let left_out: Vec<String> = unselected
.iter()
.map(|spec| spec.source_table.qualified_name())
.collect();
format!(
"this selection leaves out {} of source {name}, whose changes share its one slot and would be skipped; select the whole source, or move those tables to another postgres_cdc source",
left_out.join(", ")
)
})
}
async fn first_load(
planned: &[TablePlan],
unselected: &[TableSpec],
(destinations, options): (&BTreeMap<String, Destination>, &PlanOptions),
) -> Result<bool, LoaderError> {
if planned.iter().any(|table| table.exists) {
return Ok(false);
}
for spec in unselected {
let Some(destination) = destinations.get(&spec.target) else {
continue;
};
let exists = match files_destination(destination) {
Some(_) => plan_table(spec, destination, options)
.await
.is_ok_and(|table| table.exists),
None => live_columns(destination, &spec.destination)
.await
.is_ok_and(|live| live.is_some()),
};
if exists {
return Ok(false);
}
}
Ok(true)
}
pub(crate) async fn found_sources<'a>(
project: &'a ProjectConfig,
destinations: &BTreeMap<String, Destination>,
options: &PlanOptions,
) -> Result<FoundSources<'a>, LoaderError> {
let mut found = FoundSources::default();
for (source_name, source) in &project.sources {
let (resolved, files) =
match scoped_tables((project, options), (source_name, source)).await? {
Ok(tables) => tables,
Err(problem) => {
found.unresolved.push(source_name.clone());
if let Some(problem) = problem {
if problem.retryable {
found.plan.retryable_problems.push(problem.message.clone());
}
found.plan.problems.push(problem.message);
}
continue;
}
};
found
.claims
.extend(folder_claims(&resolved.specs, files.as_ref(), destinations));
found.configured.extend(resolved.specs.iter().cloned());
found.sources.push(FoundSource {
name: source_name,
source,
resolved,
files,
});
}
Ok(found)
}
fn folder_claims(
specs: &[TableSpec],
files: Option<&FilesSource>,
destinations: &BTreeMap<String, Destination>,
) -> Vec<FolderClaim> {
let mut claims: Vec<FolderClaim> = Vec::new();
for spec in specs {
let Some(target) = destinations.get(&spec.target).and_then(files_destination) else {
continue;
};
let name = format!(
"{}.{}",
spec.source_name,
spec.source_table.qualified_name()
);
if let Ok(path) = table_folder(spec, target) {
claims.push(FolderClaim {
id: name.clone(),
name,
folder: path.folder,
template: spec.destination_path.clone().unwrap_or_default(),
schema: spec.source_table.schema.clone(),
source: files.map(|files| files.location.clone()),
});
}
}
claims
}