use std::collections::BTreeMap;
use std::time::Duration;
use pgwire_replication::client::ReplicationEvent;
use pgwire_replication::{Lsn, ReplicationClient, ReplicationConfig, TlsConfig};
use tokio_postgres::Client;
use crate::adapters::contract::models::{ChangeApply, ChangeWrite, WriteMode};
use crate::adapters::dispatch::main::dialect::sql_adapter::sql_adapter;
use crate::adapters::dispatch::models::Destination;
use crate::cdc::_helpers::batch::changes_batch;
use crate::cdc::_helpers::buffer::buffer_change;
use crate::cdc::_helpers::snapshot_log::SnapshotLog;
use crate::cdc::constants::{
BATCH_CHANGES, LOST_WAL_STATUS, REPLICATION_OPTIONS, SNAPSHOT_SETTINGS, STOP_MESSAGE_PREFIX,
};
use crate::cdc::main::metadata_read_columns::metadata_read_columns;
use crate::cdc::models::{ChangeBuffer, StreamStep, TableChanges};
use crate::loader::constants::SOURCE_BATCH_ROWS;
use crate::loader::errors::LoaderError;
use crate::loader::models::{
CdcSpec, CommitRule, LoadRun, ProjectPlan, Publisher, TableLoaded, TableOutcome, TablePlan,
WholeReason,
};
use crate::postgres_source::constants::DEFAULT_BATCH_BYTES;
use crate::postgres_source::errors::PostgresError;
use crate::postgres_source::main::connect_postgres::connect_postgres;
use crate::postgres_source::main::copy_table::copy_table;
use crate::postgres_source::models::{CopyRequest, PostgresAddress, ReadColumn};
use crate::project::models::{ConnectionConfig, SourceConfig};
use crate::schema_plan::constants::{CDC_PREFIX, CDC_UNCHANGED_COLUMN};
const STATUS_INTERVAL: Duration = Duration::from_millis(100);
const CONFIRM_WAIT: Duration = Duration::from_secs(10);
pub(crate) struct CdcSource<'a> {
pub(crate) run: &'a LoadRun<'a>,
pub(crate) name: &'a str,
pub(crate) source: &'a SourceConfig,
pub(crate) cdc: &'a CdcSpec,
pub(crate) tables: Vec<&'a TablePlan>,
pub(crate) destination: &'a Destination,
}
fn crash_point(point: &str) {
#[cfg(feature = "container-tests")]
if std::env::var("SQLLOAD_TEST_CRASH_AT").is_ok_and(|wanted| wanted == point) {
std::process::abort();
}
#[cfg(not(feature = "container-tests"))]
let _ = point;
}
fn postgres(error: tokio_postgres::Error) -> LoaderError {
LoaderError::Postgres(PostgresError::from(error))
}
fn replication(error: impl std::fmt::Display) -> LoaderError {
LoaderError::Invalid(format!("logical replication: {error}"))
}
fn list(names: &[&String]) -> String {
names
.iter()
.map(|name| name.as_str())
.collect::<Vec<&str>>()
.join(", ")
}
fn quoted(name: &str) -> String {
format!("\"{}\"", name.replace('"', "\"\""))
}
impl CdcSource<'_> {
fn address(&self) -> Result<PostgresAddress<'_>, LoaderError> {
let Some(ConnectionConfig::Postgres {
host,
port,
user,
password,
database,
}) = self.run.project.connections.get(&self.source.connection)
else {
return Err(LoaderError::Invalid(format!(
"connection {:?} is not a Postgres connection",
self.source.connection
)));
};
Ok(PostgresAddress {
host,
port: *port,
user,
password: password.as_ref().map(|secret| secret.expose()),
database,
})
}
async fn connect(&self) -> Result<Client, LoaderError> {
Ok(connect_postgres(self.address()?).await?)
}
fn replication_config(&self) -> Result<ReplicationConfig, LoaderError> {
let address = self.address()?;
Ok(ReplicationConfig::new(
address.host,
address.user,
address.password.unwrap_or_default(),
address.database,
self.cdc.slot.clone(),
self.cdc.publication.clone(),
)
.with_port(address.port)
.with_tls(TlsConfig::disabled())
.with_status_interval(STATUS_INTERVAL)
.with_options(REPLICATION_OPTIONS)
.with_application_name("sqlload"))
}
fn log(&self) -> Result<SnapshotLog<'_>, LoaderError> {
let first = self
.tables
.first()
.map(|table| &table.spec)
.ok_or_else(|| {
LoaderError::Invalid(format!("source {} selects no tables", self.name))
})?;
Ok(SnapshotLog {
destination: self.destination,
warehouse: first.warehouse,
beside: &first.destination,
source: self.name,
slot: &self.cdc.slot,
})
}
async fn ensure_publication(&self, control: &Client) -> Result<(), LoaderError> {
let publication = &self.cdc.publication;
let names: Vec<String> = self
.tables
.iter()
.map(|table| {
let source = &table.spec.source_table;
format!("{}.{}", quoted(&source.schema), quoted(&source.name))
})
.collect();
self.refuse_unpublishable(control, &names).await?;
let exists = control
.query_opt(
"SELECT 1 FROM pg_publication WHERE pubname = $1",
&[publication],
)
.await
.map_err(postgres)?
.is_some();
let made_here = *publication == format!("sqlload_{}", self.name.to_lowercase());
let (name, tables) = (quoted(publication), names.join(", "));
let statement = match (exists, made_here && self.run.options.resnapshot) {
(false, _) => Some(format!("CREATE PUBLICATION {name} FOR TABLE {tables}")),
(true, true) => Some(format!("ALTER PUBLICATION {name} SET TABLE {tables}")),
(true, false) => None,
};
if let Some(statement) = statement {
control.batch_execute(&statement).await.map_err(postgres)?;
return Ok(());
}
let published: Vec<String> = control
.query(
"SELECT '\"' || replace(schemaname, '\"', '\"\"') || '\".\"' || replace(tablename, '\"', '\"\"') || '\"' FROM pg_publication_tables WHERE pubname = $1",
&[publication],
)
.await
.map_err(postgres)?
.iter()
.map(|row| row.get(0))
.collect();
let missing: Vec<&String> = names
.iter()
.filter(|name| !published.contains(name))
.collect();
let extra: Vec<&String> = published
.iter()
.filter(|name| !names.contains(name))
.collect();
match (missing.is_empty(), extra.is_empty()) {
(true, true) => Ok(()),
(false, _) => Err(LoaderError::Invalid(format!(
"publication {publication} does not publish {}; add them with ALTER PUBLICATION {} ADD TABLE {}",
list(&missing),
quoted(publication),
list(&missing)
))),
(true, false) => Err(LoaderError::Invalid(format!(
"publication {publication} also publishes {}, which source {} does not load, so the slot would stop at their first change; publish only the source's tables (ALTER PUBLICATION {} DROP TABLE {}), then run `sqlload load --resnapshot`",
list(&extra),
self.name,
quoted(publication),
list(&extra)
))),
}
}
async fn refuse_unpublishable(
&self,
control: &Client,
names: &[String],
) -> Result<(), LoaderError> {
let nothing: Vec<String> = control
.query(
"SELECT name FROM (SELECT '\"' || replace(n.nspname, '\"', '\"\"') || '\".\"' || replace(c.relname, '\"', '\"\"') || '\"' AS name, c.relreplident FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace) AS tables WHERE relreplident = 'n' AND name = ANY($1)",
&[&names],
)
.await
.map_err(postgres)?
.iter()
.map(|row| row.get(0))
.collect();
match nothing.is_empty() {
true => Ok(()),
false => Err(LoaderError::Invalid(format!(
"{} {} REPLICA IDENTITY NOTHING, so publishing it would make Postgres refuse its updates and deletes; set REPLICA IDENTITY DEFAULT, or exclude it",
nothing.join(", "),
match nothing.len() {
1 => "has",
_ => "have",
}
))),
}
}
async fn slot_state(&self, control: &Client) -> Result<Option<(String, String)>, LoaderError> {
Ok(control
.query_opt(
"SELECT confirmed_flush_lsn::text, COALESCE(wal_status, '') FROM pg_replication_slots WHERE slot_name = $1 AND database = current_database()",
&[&self.cdc.slot],
)
.await
.map_err(postgres)?
.map(|row| (row.get::<_, Option<String>>(0).unwrap_or_default(), row.get(1))))
}
async fn resume_problem(&self, wal_status: &str) -> Result<Option<String>, LoaderError> {
let slot = &self.cdc.slot;
if wal_status == LOST_WAL_STATUS {
return Ok(Some(format!(
"slot {slot} lost WAL it had not confirmed, so changes are missing; run `sqlload load --resnapshot` to drop and recreate it and copy every table again"
)));
}
let Some((_, snapshotted)) = self.log()?.current().await? else {
return Ok(Some(format!(
"slot {slot} exists but SQLLoad has no record of creating it, so it cannot tell which changes the tables already hold; drop it, or run `sqlload load --resnapshot` to drop and recreate it and copy every table again"
)));
};
let loading: Vec<&str> = self
.tables
.iter()
.map(|table| table.spec.destination.table.as_str())
.collect();
let left_out: Vec<&str> = snapshotted
.iter()
.map(|(name, _)| name.as_str())
.filter(|name| !loading.contains(name))
.collect();
if !left_out.is_empty() {
return Ok(Some(format!(
"this load leaves out {} of source {}, whose tables share slot {slot}; load the whole source (no --select or --exclude of its tables, and none blocked), or their changes would be skipped",
left_out.join(", "),
self.name
)));
}
let mut reshaped: Vec<&str> = Vec::new();
for table in &self.tables {
let name = table.spec.destination.table.as_str();
let recorded = snapshotted.iter().find(|(done, _)| done == name);
if recorded.is_some_and(|(_, columns)| *columns != data_columns(&table.read_columns)) {
reshaped.push(name);
}
}
if !reshaped.is_empty() {
return Ok(Some(format!(
"the columns of {} changed since their snapshot from slot {slot}; schema changes come in a later release, so run `sqlload load --resnapshot` to copy every table again",
reshaped.join(", ")
)));
}
let recorded: Vec<&str> = snapshotted.iter().map(|(done, _)| done.as_str()).collect();
let unfinished: Vec<&str> = loading
.into_iter()
.filter(|name| !recorded.contains(name))
.collect();
Ok((!unfinished.is_empty()).then(|| {
format!(
"{} {} no snapshot from slot {slot} (a snapshot stopped part-way, or the table joined since); run `sqlload load --resnapshot` to drop and recreate the slot and copy every table again",
unfinished.join(", "),
match unfinished.len() {
1 => "has",
_ => "have",
}
)
}))
}
pub(crate) async fn load(&self) -> Result<BTreeMap<String, u64>, LoaderError> {
let control = self.connect().await?;
self.ensure_publication(&control).await?;
self.log()?.ensure().await?;
let mut slot = self.slot_state(&control).await?;
if self.run.options.resnapshot && slot.is_some() {
control
.execute("SELECT pg_drop_replication_slot($1)", &[&self.cdc.slot])
.await
.map_err(postgres)?;
slot = None;
}
let (mut stream, mut rows) = match slot {
Some((_, wal_status)) => {
if let Some(problem) = self.resume_problem(&wal_status).await? {
return Err(LoaderError::Invalid(problem));
}
let session = ReplicationClient::connect(self.replication_config()?)
.await
.map_err(replication)?;
(session, BTreeMap::new())
}
None => {
let recreated = self.run.options.resnapshot;
if !recreated && self.log()?.current().await?.is_some() {
return Err(LoaderError::Invalid(format!(
"slot {} is gone (dropped, or never kept after a failover), so the changes since the last load are lost; run `sqlload load --resnapshot` to recreate it and copy every table again",
self.cdc.slot
)));
}
self.snapshot().await?
}
};
let stop: String = control
.query_one(
"SELECT pg_logical_emit_message(false, $1, '')::text",
&[&STOP_MESSAGE_PREFIX],
)
.await
.map_err(postgres)?
.get(0);
let stop = Lsn::parse(&stop).map_err(replication)?;
let streamed = self.stream(&mut stream, stop).await;
let confirmed = match streamed {
Ok((end, written)) => {
for (table, count) in written {
*rows.entry(table).or_default() += count;
}
match end {
Some(end) => self.await_confirmed(&control, end).await,
None => Ok(()),
}
}
Err(error) => Err(error),
};
stream.stop();
let _ = stream.shutdown().await;
confirmed.map(|()| rows)
}
async fn snapshot(&self) -> Result<(ReplicationClient, BTreeMap<String, u64>), LoaderError> {
let mut rows: BTreeMap<String, u64> = BTreeMap::new();
let mut session = ReplicationClient::connect_gated(self.replication_config()?)
.await
.map_err(replication)?;
let created = session
.simple_query(&format!(
"CREATE_REPLICATION_SLOT {} LOGICAL pgoutput (SNAPSHOT 'export')",
quoted(&self.cdc.slot)
))
.await
.map_err(replication)?;
let field = |index: usize| {
created
.first()
.and_then(|row| row.get(index))
.and_then(Option::as_ref)
.map(|bytes| String::from_utf8_lossy(bytes).into_owned())
.ok_or_else(|| replication("CREATE_REPLICATION_SLOT returned no snapshot"))
};
let consistent = Lsn::parse(&field(1)?).map_err(replication)?;
let snapshot_name = field(2)?;
let snapshot_lsn = i64::try_from(consistent.as_u64()).map_err(replication)?;
let log = self.log()?;
log.record("", snapshot_lsn, "").await?;
let client = self.connect().await?;
client
.batch_execute(&format!(
"{SNAPSHOT_SETTINGS}; BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY; SET TRANSACTION SNAPSHOT '{}'",
snapshot_name.replace('\'', "''")
))
.await
.map_err(postgres)?;
for (index, table) in self.tables.iter().enumerate() {
if index == 1 {
crash_point("snapshot-after-first-table");
}
let mut plan = (*table).clone();
plan.mode = WriteMode::Replace;
plan.commits = CommitRule::Whole(WholeReason::Chosen);
plan.read_columns =
with_snapshot_columns(&plan.read_columns, self.cdc, snapshot_lsn - 1);
let written = self.copy_table(&client, plan).await?;
log.record(
&table.spec.destination.table,
snapshot_lsn,
&data_columns(&table.read_columns),
)
.await?;
rows.insert(table.spec.destination.table.clone(), written);
}
client.batch_execute("COMMIT").await.map_err(postgres)?;
session.start().await.map_err(replication)?;
session.wait_started().await.map_err(replication)?;
Ok((session, rows))
}
async fn copy_table(&self, client: &Client, plan: TablePlan) -> Result<u64, LoaderError> {
let request = CopyRequest {
table: &plan.spec.source_table,
columns: plan.read_columns.clone(),
filter: None,
order_by: None,
batch_rows: SOURCE_BATCH_ROWS,
batch_bytes: DEFAULT_BATCH_BYTES,
helpers: &[],
};
let mut publisher =
Publisher::open(plan.clone(), (self.destination, self.run.options), None).await?;
let copied = async {
copy_table(client, request, async |batch| {
publisher
.stage(batch, None)
.await
.map_err(|error| error.to_string())
})
.await?;
publisher.finish(None).await
}
.await;
publisher.close().await;
copied
}
async fn stream(
&self,
stream: &mut ReplicationClient,
stop: Lsn,
) -> Result<(Option<u64>, BTreeMap<String, u64>), LoaderError> {
let mut rows: BTreeMap<String, u64> = BTreeMap::new();
let mut buffer = ChangeBuffer::default();
let mut confirmed: Option<u64> = None;
loop {
let Some(event) = stream.recv().await.map_err(replication)? else {
return Err(replication("the stream ended before the load caught up"));
};
let step = match &event {
ReplicationEvent::Begin { .. } => Some(StreamStep::Begin),
ReplicationEvent::XLogData { data, .. } => Some(StreamStep::Message(data)),
ReplicationEvent::Commit {
lsn,
end_lsn,
commit_time_micros,
} => Some(StreamStep::Commit {
lsn: lsn.as_u64(),
end_lsn: end_lsn.as_u64(),
time: *commit_time_micros,
}),
_ => None,
};
if let Some(step) = step {
buffer = buffer_change(buffer, step).map_err(LoaderError::Invalid)?;
}
let at_stop = matches!(
&event,
ReplicationEvent::Message { prefix, lsn, .. }
if prefix == STOP_MESSAGE_PREFIX && *lsn >= stop
);
let full =
matches!(event, ReplicationEvent::Commit { .. }) && buffer.count >= BATCH_CHANGES;
if full || at_stop {
let tables = std::mem::take(&mut buffer.tables);
buffer.count = 0;
for (table, count) in self.flush(tables).await? {
*rows.entry(table).or_default() += count;
}
if let Some(end) = buffer.end_lsn {
stream.update_applied_lsn(Lsn::from_u64(end));
confirmed = Some(end);
}
}
if at_stop {
stream.update_applied_lsn(stop);
return Ok((Some(stop.as_u64()).max(confirmed), rows));
}
}
}
async fn flush(
&self,
tables: BTreeMap<u32, TableChanges>,
) -> Result<BTreeMap<String, u64>, LoaderError> {
let mut rows: BTreeMap<String, u64> = BTreeMap::new();
let mut written = 0;
for changes in tables.values() {
let relation = &changes.relation;
let Some(table) = self.tables.iter().find(|table| {
let source = &table.spec.source_table;
source.schema == relation.schema && source.name == relation.table
}) else {
return Err(LoaderError::Invalid(format!(
"publication {} sends changes of {}.{}, which source {} does not load; its changes would be skipped, so add the table to include or drop it from the publication",
self.cdc.publication, relation.schema, relation.table, self.name
)));
};
let (batch, truncated) = changes_batch(
changes,
&table.read_columns,
(self.cdc.apply, &table.spec.unique_key),
)
.map_err(LoaderError::Invalid)?;
let mut plan = (*table).clone();
plan.mode = WriteMode::Changes(ChangeWrite {
apply: self.cdc.apply,
unique_key: table.spec.unique_key.clone(),
truncated,
});
let mut publisher =
Publisher::open(plan, (self.destination, self.run.options), None).await?;
let result = async {
publisher.stage(batch, None).await?;
if written == 0 {
crash_point("batch-before-commit");
}
publisher.finish(None).await
}
.await;
publisher.close().await;
let count = result?;
written += 1;
if written == 1 {
crash_point("batch-after-first-table");
}
rows.insert(table.spec.destination.table.clone(), count);
}
crash_point("batch-before-confirm");
Ok(rows)
}
async fn await_confirmed(&self, control: &Client, end: u64) -> Result<(), LoaderError> {
let wanted = Lsn::from_u64(end).to_pg_string();
let started = std::time::Instant::now();
loop {
let confirmed: bool = control
.query_one(
"SELECT COALESCE(confirmed_flush_lsn >= $2::text::pg_lsn, false) FROM pg_replication_slots WHERE slot_name = $1",
&[&self.cdc.slot, &wanted],
)
.await
.map_err(postgres)?
.get(0);
if confirmed {
return Ok(());
}
if started.elapsed() > CONFIRM_WAIT {
return Err(replication(format!(
"slot {} did not confirm {wanted}; the changes are written and the next load skips them",
self.cdc.slot
)));
}
tokio::time::sleep(STATUS_INTERVAL).await;
}
}
}
fn with_snapshot_columns(read: &[ReadColumn], cdc: &CdcSpec, snapshot_lsn: i64) -> Vec<ReadColumn> {
let mut columns: Vec<ReadColumn> = read
.iter()
.filter(|column| !column.name.starts_with(CDC_PREFIX))
.cloned()
.collect();
let merged = cdc.apply != ChangeApply::Changelog;
columns.extend(
metadata_read_columns(cdc.apply, snapshot_lsn)
.into_iter()
.filter(|column| !(merged && column.name == CDC_UNCHANGED_COLUMN)),
);
columns
}
pub(crate) fn captured_sources(plan: &ProjectPlan) -> BTreeMap<&str, CapturedTables<'_>> {
let mut sources: BTreeMap<&str, CapturedTables<'_>> = BTreeMap::new();
for table in plan.tables.iter().filter(|table| table.spec.cdc.is_some()) {
let entry = sources.entry(table.spec.source_name.as_str()).or_default();
match table.schema.blocked {
None => entry.0.push(table),
Some(_) => entry.1.push(table.spec.destination.table.clone()),
}
}
sources
}
pub(crate) type CapturedTables<'a> = (Vec<&'a TablePlan>, Vec<String>);
pub(crate) async fn load_source(
run: &LoadRun<'_>,
name: &str,
tables: &[&TablePlan],
) -> Result<BTreeMap<String, u64>, String> {
let first = tables.first().map(|table| &table.spec);
let (Some(source), Some(cdc), Some(destination)) = (
run.project.sources.get(name),
first.and_then(|spec| spec.cdc.as_ref()),
first.and_then(|spec| run.destinations.get(&spec.target)),
) else {
return Err(format!(
"source {name} is not a connected postgres_cdc source"
));
};
let unit = CdcSource {
run,
name,
source,
cdc,
tables: tables.to_vec(),
destination,
};
unit.load().await.map_err(|error| error.to_string())
}
pub(crate) fn source_outcomes(
tables: &[&TablePlan],
loaded: &Result<BTreeMap<String, u64>, String>,
elapsed: Duration,
) -> Vec<TableOutcome> {
let mut outcomes: Vec<TableOutcome> = Vec::with_capacity(tables.len());
for table in tables {
let spec = &table.spec;
let rows = loaded.as_ref().map(|rows| {
rows.get(&spec.destination.table)
.copied()
.unwrap_or_default()
});
outcomes.push(TableOutcome {
table: sql_adapter(spec.warehouse).render_display_name(&spec.destination),
loaded: rows
.map(|rows| TableLoaded {
rows,
..TableLoaded::default()
})
.map_err(Clone::clone),
elapsed,
});
}
outcomes
}
fn data_columns(read: &[ReadColumn]) -> String {
let mut columns = String::from(",");
for column in read
.iter()
.filter(|column| !column.name.starts_with(CDC_PREFIX))
{
columns.push_str(&column.name);
columns.push(',');
}
columns
}