use std::sync::Arc;
use arrow_array::{ArrayRef, Int64Array, RecordBatch, StringArray, TimestampMicrosecondArray};
use arrow_schema::{Field, Schema};
use crate::adapters::contract::models::{ChangeApply, Truncation};
use crate::cdc::_helpers::fold::{Folder, fold_changes, key_of, key_positions};
use crate::cdc::_helpers::spill::{SpillFile, SpillReader};
use crate::cdc::_helpers::text_values::{arrow_type, micros_text, text_array};
use crate::cdc::constants::{
DELETE_OP, INSERT_OP, NOT_STREAMED, SPILL_CHUNK_CHANGES, SPILL_PARTITIONS, TRUNCATE_OP,
UPDATE_OP,
};
use crate::cdc::models::{Change, ChangeOp, Relation, TableChanges, TupleValue};
use crate::column_types::main::arrow_field::arrow_field;
use crate::postgres_source::models::{ReadColumn, SourceTable};
use crate::schema_plan::constants::{
CDC_COMMITTED_AT_COLUMN, CDC_LSN_COLUMN, CDC_OP_COLUMN, CDC_PREFIX, CDC_SEQ_COLUMN,
CDC_UNCHANGED_COLUMN,
};
pub(crate) fn rows_batch(
changes: &[Change],
relation: &Relation,
read: &[ReadColumn],
) -> Result<RecordBatch, String> {
let positions = column_positions(relation, read);
let data_positions: Vec<(&str, usize)> = read
.iter()
.zip(&positions)
.filter_map(|(column, position)| Some((column.name.as_str(), (*position)?)))
.collect();
let mut fields: Vec<Field> = Vec::with_capacity(read.len());
let mut arrays: Vec<ArrayRef> = Vec::with_capacity(read.len());
for (column, position) in read.iter().zip(&positions) {
let array = match position {
Some(position) => data_array(changes, *position, column)?,
None => metadata_array(changes, column, &data_positions)?,
};
let field = Field::new(&column.name, array.data_type().clone(), true);
fields.push(match arrow_field(&column.name, &column.output) {
Some(planned) if planned.data_type() == array.data_type() => {
field.with_metadata(planned.metadata().clone())
}
_ => field,
});
arrays.push(array);
}
RecordBatch::try_new(Arc::new(Schema::new(fields)), arrays).map_err(|error| error.to_string())
}
#[derive(Debug)]
pub(crate) enum ChangeParts {
Ready(Option<Vec<Change>>),
Log {
reader: SpillReader,
memory: std::vec::IntoIter<Change>,
},
Partitions {
files: std::collections::btree_map::IntoIter<u64, SpillFile>,
positions: Vec<usize>,
partitions: u64,
},
}
impl ChangeParts {
pub(crate) fn next_changes(&mut self) -> Result<Option<Vec<Change>>, String> {
match self {
Self::Ready(changes) => Ok(changes.take()),
Self::Log { reader, memory } => {
let mut part: Vec<Change> = Vec::new();
while part.len() < SPILL_CHUNK_CHANGES {
match reader.next_change()? {
Some(change) => part.push(change),
None => break,
}
}
part.extend(memory.take(SPILL_CHUNK_CHANGES - part.len()));
Ok((!part.is_empty()).then_some(part))
}
Self::Partitions {
files,
positions,
partitions,
} => {
let Some((partition, mut file)) = files.next() else {
return Ok(None);
};
let mut reader = file.read()?;
let mut folder = Folder::default();
while let Some(change) = reader.next_change()? {
folder.push(change, positions)?;
}
let mut folded = folder.finish();
folded.retain(|change| {
key_partition(&key_of(&change.values, positions), *partitions) == partition
});
Ok(Some(folded))
}
}
}
}
pub(crate) fn change_parts(
table: &TableChanges,
(spilled, spill_bytes): (Option<SpillFile>, usize),
(apply, unique_key): (ChangeApply, &[String]),
) -> Result<(ChangeParts, Option<Truncation>), String> {
let Some(mut spilled) = spilled else {
let (changes, truncated) = applied_changes(table, apply, unique_key)?;
return Ok((ChangeParts::Ready(Some(changes)), truncated));
};
let memory = table.changes.clone().into_iter();
if apply == ChangeApply::Changelog {
let reader = spilled.read()?;
return Ok((ChangeParts::Log { reader, memory }, None));
}
let mut last: Option<Change> = None;
let mut reader = spilled.read()?;
while let Some(change) = reader.next_change()? {
if change.op == ChangeOp::Truncate {
last = Some(change);
}
}
if let Some(change) = table
.changes
.iter()
.rev()
.find(|change| change.op == ChangeOp::Truncate)
{
last = Some(change.clone());
}
let after = last.as_ref().map(|change| (change.lsn, change.seq));
let positions = key_positions(&table.relation, unique_key)?;
let spilled_bytes = u64::try_from(spilled.bytes).unwrap_or(u64::MAX);
let limit = u64::try_from(spill_bytes.max(1)).unwrap_or(u64::MAX);
let partitions = spilled_bytes
.saturating_mul(4)
.div_ceil(limit)
.clamp(1, SPILL_PARTITIONS);
let mut files = PartitionFiles::default();
let mut reader = spilled.read()?;
let mut memory = memory;
loop {
let change = match reader.next_change()? {
Some(change) => change,
None => match memory.next() {
Some(change) => change,
None => break,
},
};
if after.is_some_and(|after| (change.lsn, change.seq) <= after) {
continue;
}
let key = key_of(&change.values, &positions);
if let Some(old) = change
.old_key
.as_ref()
.filter(|old| key_of(old, &positions) != key)
{
let deleted = Change {
op: ChangeOp::Delete,
values: old.clone(),
old_key: None,
..change.clone()
};
files.push(
key_partition(&key_of(old, &positions), partitions),
&deleted,
)?;
}
files.push(key_partition(&key, partitions), &change)?;
}
let truncated = last.map(|change| Truncation {
lsn: change.lsn,
committed_at: micros_text(change.committed_at),
});
let parts = ChangeParts::Partitions {
files: files.0.into_iter(),
positions,
partitions,
};
Ok((parts, truncated))
}
fn key_partition(key: &[TupleValue], partitions: u64) -> u64 {
let mut hasher = std::hash::DefaultHasher::new();
std::hash::Hash::hash(key, &mut hasher);
std::hash::Hasher::finish(&hasher) % partitions
}
#[derive(Debug, Default)]
struct PartitionFiles(std::collections::BTreeMap<u64, SpillFile>);
impl PartitionFiles {
fn push(&mut self, partition: u64, change: &Change) -> Result<(), String> {
let file = match self.0.entry(partition) {
std::collections::btree_map::Entry::Occupied(file) => file.into_mut(),
std::collections::btree_map::Entry::Vacant(slot) => slot.insert(SpillFile::new()?),
};
file.push_change(change)
}
}
fn applied_changes(
table: &TableChanges,
apply: ChangeApply,
unique_key: &[String],
) -> Result<(Vec<Change>, Option<Truncation>), String> {
if apply == ChangeApply::Changelog {
return Ok((table.changes.clone(), None));
}
let last_truncate = table
.changes
.iter()
.rposition(|change| change.op == ChangeOp::Truncate);
let truncated = last_truncate
.and_then(|index| table.changes.get(index))
.map(|change| Truncation {
lsn: change.lsn,
committed_at: micros_text(change.committed_at),
});
let after = last_truncate.map_or(0, |index| index + 1);
let positions = key_positions(&table.relation, unique_key)?;
let kept = table.changes.get(after..).unwrap_or_default();
Ok((fold_changes(kept, &positions)?, truncated))
}
pub(crate) fn loaded_relation(
relation: &Relation,
planned: &SourceTable,
) -> Result<Relation, String> {
let mut loaded = relation.clone();
let mut seen: std::collections::BTreeMap<String, String> = std::collections::BTreeMap::new();
for column in &mut loaded.columns {
let name = planned.loaded_as(&column.name).to_owned();
if let Some(other) = seen.insert(name.clone(), column.name.clone()) {
return Err(format!(
"{}.{}: columns {other} and {} both load as {name}; one was added during the load, so run the load again to plan it, and rename or exclude one of them",
relation.schema, relation.table, column.name
));
}
column.name = name;
}
Ok(loaded)
}
pub(crate) fn unplanned_columns(relation: &Relation, planned: &SourceTable) -> Vec<String> {
let known: std::collections::BTreeSet<&str> = planned
.columns
.iter()
.map(|column| column.name.as_str())
.collect();
relation
.columns
.iter()
.map(|column| column.name.clone())
.filter(|name| !known.contains(name.as_str()))
.collect()
}
fn column_positions(relation: &Relation, read: &[ReadColumn]) -> Vec<Option<usize>> {
let streamed: std::collections::BTreeMap<&str, usize> = relation
.columns
.iter()
.enumerate()
.map(|(position, column)| (column.name.as_str(), position))
.collect();
read.iter()
.map(|column| {
(!column.name.starts_with(CDC_PREFIX)).then(|| {
streamed
.get(column.name.as_str())
.copied()
.unwrap_or(NOT_STREAMED)
})
})
.collect()
}
pub(crate) fn refuse_renamed(
table: &TableChanges,
spilled: Option<&mut SpillFile>,
(read, gone): (&[ReadColumn], &[String]),
) -> Result<(), String> {
let relation = &table.relation;
let gone_positions: Vec<usize> = relation
.columns
.iter()
.enumerate()
.filter(|(_, column)| gone.contains(&column.name))
.map(|(position, _)| position)
.collect();
if gone_positions.is_empty() {
return Ok(());
}
let read_positions: Vec<usize> = column_positions(relation, read)
.into_iter()
.flatten()
.collect();
let misplaced = |change: &Change| {
matches!(change.op, ChangeOp::Insert | ChangeOp::Update)
&& !all_left_out(change, &gone_positions)
&& any_left_out(change, &read_positions)
};
let mut renamed = table.changes.iter().any(misplaced);
if let Some(file) = spilled {
let mut reader = file.read()?;
while let Some(change) = reader.next_change()? {
renamed = renamed || misplaced(&change);
}
}
match renamed {
true => Err(format!(
"{}.{} renamed or replaced column {} while changes to it were not yet loaded, so their values cannot be placed; run `sqlload load --resnapshot` to copy every table again",
relation.schema,
relation.table,
gone.join(", ")
)),
false => Ok(()),
}
}
fn left_out(change: &Change, position: usize) -> bool {
change
.values
.get(position)
.is_none_or(|value| *value == TupleValue::Absent)
}
fn all_left_out(change: &Change, positions: &[usize]) -> bool {
positions.iter().all(|position| left_out(change, *position))
}
fn any_left_out(change: &Change, positions: &[usize]) -> bool {
positions.iter().any(|position| left_out(change, *position))
}
fn data_array(
changes: &[Change],
position: usize,
column: &ReadColumn,
) -> Result<ArrayRef, String> {
let mut values: Vec<Option<&str>> = Vec::with_capacity(changes.len());
for change in changes {
values.push(match change.values.get(position) {
Some(TupleValue::Text(text)) => Some(text.as_str()),
_ => None,
});
}
text_array(&values, column).map_err(|problem| format!("{}: {problem}", column.name))
}
fn metadata_array(
changes: &[Change],
column: &ReadColumn,
data_positions: &[(&str, usize)],
) -> Result<ArrayRef, String> {
Ok(match column.name.as_str() {
CDC_LSN_COLUMN => Arc::new(Int64Array::from_iter_values(
changes.iter().map(|change| change.lsn),
)),
CDC_SEQ_COLUMN => Arc::new(Int64Array::from_iter_values(
changes.iter().map(|change| change.seq),
)),
CDC_OP_COLUMN => Arc::new(StringArray::from_iter_values(
changes.iter().map(|change| op_name(change.op)),
)),
CDC_COMMITTED_AT_COLUMN => Arc::new(
TimestampMicrosecondArray::from_iter_values(
changes.iter().map(|change| change.committed_at),
)
.with_timezone("UTC"),
),
CDC_UNCHANGED_COLUMN => Arc::new(StringArray::from_iter(
changes
.iter()
.map(|change| unchanged_flags(change, data_positions)),
)),
_ => arrow_array::new_null_array(&arrow_type(&column.output)?, changes.len()),
})
}
fn op_name(op: ChangeOp) -> &'static str {
match op {
ChangeOp::Insert => INSERT_OP,
ChangeOp::Update => UPDATE_OP,
ChangeOp::Delete => DELETE_OP,
ChangeOp::Truncate => TRUNCATE_OP,
}
}
fn unchanged_flags(change: &Change, data_positions: &[(&str, usize)]) -> Option<String> {
let writes = matches!(change.op, ChangeOp::Insert | ChangeOp::Update);
let mut names = String::new();
for (name, position) in data_positions {
let value = change.values.get(*position).unwrap_or(&TupleValue::Absent);
if *value == TupleValue::Unchanged || (writes && *value == TupleValue::Absent) {
names.push(',');
names.push_str(name);
}
}
(!names.is_empty()).then(|| format!("{names},"))
}