use crate::cdc::_helpers::pgoutput::decode_message;
use crate::cdc::_helpers::spill::SpillFile;
use crate::cdc::constants::{
CHANGE_OVERHEAD_BYTES, FULL_IDENTITY, POSTGRES_EPOCH_MICROS, VALUE_OVERHEAD_BYTES,
};
use crate::cdc::models::{
Change, ChangeBuffer, ChangeOp, PgOutputMessage, Relation, RelationColumn, StreamStep,
TableChanges, TupleValue,
};
pub(crate) fn buffer_change(
mut buffer: ChangeBuffer,
step: StreamStep<'_>,
) -> Result<ChangeBuffer, String> {
match step {
StreamStep::Begin => {
buffer.bytes = buffer.bytes.saturating_sub(open_bytes(&buffer.open));
buffer.open.clear();
buffer.open_spill = None;
}
StreamStep::Message(bytes) => match decode_message(bytes)? {
PgOutputMessage::Other(_) => {}
_ => buffer = hold_message(buffer, bytes)?,
},
StreamStep::Commit { lsn, end_lsn, time } => {
let commit = Commit {
lsn: i64::try_from(lsn).map_err(|error| error.to_string())?,
committed_at: time.saturating_add(POSTGRES_EPOCH_MICROS),
};
buffer.bytes = buffer.bytes.saturating_sub(open_bytes(&buffer.open));
let held = std::mem::take(&mut buffer.open);
let mut index = 0;
if let Some(mut spilled) = buffer.open_spill.take() {
let mut reader = spilled.read()?;
while let Some(message) = reader.next_bytes()? {
let message = decode_message(&message)?;
index += usize::from(!matches!(message, PgOutputMessage::Relation(_)));
buffer = file_message(buffer, &message, (commit, index))?;
}
}
for message in held {
let message = decode_message(&message)?;
index += usize::from(!matches!(message, PgOutputMessage::Relation(_)));
buffer = file_message(buffer, &message, (commit, index))?;
}
buffer.end_lsn = Some(end_lsn);
}
}
Ok(buffer)
}
#[derive(Debug, Clone, Copy)]
struct Commit {
lsn: i64,
committed_at: i64,
}
fn hold_message(mut buffer: ChangeBuffer, bytes: &[u8]) -> Result<ChangeBuffer, String> {
if let Some(spilled) = &mut buffer.open_spill {
spilled.push_bytes(bytes)?;
return Ok(buffer);
}
buffer.open.push(bytes.to_vec());
buffer.bytes += bytes.len();
if buffer.bytes > buffer.spill_bytes {
let mut spilled = SpillFile::new()?;
for message in &buffer.open {
spilled.push_bytes(message)?;
}
buffer.bytes = buffer.bytes.saturating_sub(open_bytes(&buffer.open));
buffer.open.clear();
buffer.open_spill = Some(spilled);
tracing::info!(
limit = buffer.spill_bytes,
"a transaction outgrew the memory limit; spilling it to local disk"
);
}
Ok(buffer)
}
fn open_bytes(open: &[Vec<u8>]) -> usize {
open.iter().map(Vec::len).sum()
}
fn file_message(
mut buffer: ChangeBuffer,
message: &PgOutputMessage,
(commit, index): (Commit, usize),
) -> Result<ChangeBuffer, String> {
let seq = i64::try_from(index).unwrap_or(i64::MAX);
let change = |op, values: &[TupleValue], old_key: &Option<Vec<TupleValue>>| Change {
op,
values: values.to_vec(),
old_key: old_key.clone(),
lsn: commit.lsn,
seq,
committed_at: commit.committed_at,
};
let filed: Vec<(u32, Change)> = match message {
PgOutputMessage::Insert { relation, new } => {
vec![(*relation, change(ChangeOp::Insert, new, &None))]
}
PgOutputMessage::Update { relation, old, new } => {
vec![(*relation, change(ChangeOp::Update, new, old))]
}
PgOutputMessage::Delete { relation, old } => {
vec![(*relation, change(ChangeOp::Delete, old, &None))]
}
PgOutputMessage::Truncate { relations, .. } => relations
.iter()
.map(|relation| (*relation, change(ChangeOp::Truncate, &[], &None)))
.collect(),
PgOutputMessage::Relation(relation) => {
if let Some(table) = buffer.tables.get_mut(&relation.oid) {
table.relation = union_shape(&table.relation, relation);
}
buffer.relations.insert(relation.oid, relation.clone());
Vec::new()
}
PgOutputMessage::Other(_) => Vec::new(),
};
for (oid, mut change) in filed {
let Some(relation) = buffer.relations.get(&oid) else {
return Err(format!(
"a change refers to relation {oid}, which the stream never described"
));
};
let table = buffer.tables.entry(oid).or_insert_with(|| TableChanges {
relation: relation.clone(),
changes: Vec::new(),
});
if table.relation.columns != relation.columns {
change.values = fit_values(&change.values, &table.relation, relation);
change.old_key = change
.old_key
.map(|old| fit_values(&old, &table.relation, relation));
}
buffer.bytes += change_bytes(&change);
table.changes.push(change);
buffer.count += 1;
}
match buffer.bytes > buffer.spill_bytes {
true => spill_tables(buffer),
false => Ok(buffer),
}
}
fn union_shape(current: &Relation, newest: &Relation) -> Relation {
let mut columns: Vec<RelationColumn> = current
.columns
.iter()
.map(|column| {
named(newest, &column.name)
.and_then(|position| newest.columns.get(position))
.map_or_else(
|| RelationColumn {
key: false,
..column.clone()
},
|now| RelationColumn {
key: column.key && now.key,
..now.clone()
},
)
})
.collect();
for column in &newest.columns {
if !current
.columns
.iter()
.any(|known| known.name == column.name)
{
columns.push(column.clone());
}
}
let replica_identity = match current.replica_identity {
FULL_IDENTITY => newest.replica_identity,
narrower => narrower,
};
Relation {
columns,
replica_identity,
..newest.clone()
}
}
fn fit_values(values: &[TupleValue], union: &Relation, shape: &Relation) -> Vec<TupleValue> {
union
.columns
.iter()
.map(|column| {
named(shape, &column.name)
.and_then(|position| values.get(position).cloned())
.unwrap_or(TupleValue::Absent)
})
.collect()
}
fn named(relation: &Relation, name: &str) -> Option<usize> {
relation
.columns
.iter()
.position(|column| column.name == name)
}
fn change_bytes(change: &Change) -> usize {
let text = |values: &[TupleValue]| -> usize {
values
.iter()
.map(|value| match value {
TupleValue::Text(text) => text.len() + VALUE_OVERHEAD_BYTES,
_ => VALUE_OVERHEAD_BYTES,
})
.sum()
};
CHANGE_OVERHEAD_BYTES + text(&change.values) + change.old_key.as_deref().map_or(0, text)
}
fn spill_tables(mut buffer: ChangeBuffer) -> Result<ChangeBuffer, String> {
if buffer.spilled.is_empty() {
tracing::info!(
limit = buffer.spill_bytes,
"a batch outgrew the memory limit; spilling its changes to local disk"
);
}
for (oid, table) in &mut buffer.tables {
let spilled = match buffer.spilled.entry(*oid) {
std::collections::btree_map::Entry::Occupied(file) => file.into_mut(),
std::collections::btree_map::Entry::Vacant(slot) => slot.insert(SpillFile::new()?),
};
for change in table.changes.drain(..) {
spilled.push_change(&change)?;
}
}
buffer.bytes = open_bytes(&buffer.open);
Ok(buffer)
}