use std::collections::{BTreeMap, BTreeSet};
use diesel::{PgConnection, RunQueryDsl as _, sql_query};
use super::super::{quote_ident, quote_literal};
use super::qualified_ident as qualified;
const MAX_PASSES: usize = 100_000;
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum SampleAmount {
Percent(f64),
Count(u64),
}
#[derive(Debug, Clone, PartialEq)]
pub struct SampleSpec {
pub table: String,
pub amount: SampleAmount,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct SampleRules {
#[serde(default)]
pub always_include: Vec<String>,
#[serde(default)]
pub never_include: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ForeignKeyConstraint {
pub name: String,
pub child_table: String,
pub child_columns: Vec<String>,
pub parent_table: String,
pub parent_columns: Vec<String>,
pub match_full: bool,
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum SampleRole {
Root(SampleAmount),
AlwaysInclude,
NeverInclude,
Related,
}
#[derive(Debug, PartialEq, Eq)]
pub enum SampleError {
InvalidSpec {
spec: String,
detail: String,
},
UnknownRoot {
tables: Vec<String>,
},
DuplicateRoot {
table: String,
},
RootNeverIncluded {
tables: Vec<String>,
},
RootAlwaysIncluded {
tables: Vec<String>,
},
RuleContradiction {
tables: Vec<String>,
},
StaleRule {
tables: Vec<String>,
},
FrameworkRule {
tables: Vec<String>,
},
UncoveredTables {
tables: Vec<String>,
},
NeverIncludeReferenced {
edges: Vec<String>,
},
OutsideTableReferencesSampled {
edges: Vec<String>,
},
PartitionLocalForeignKey {
edges: Vec<String>,
},
PurgeOrderContradiction {
tables: Vec<String>,
},
RetainedReferencesPurged {
edges: Vec<String>,
},
NoRowKey {
tables: Vec<String>,
},
ForeignKeyCycle {
tables: Vec<String>,
},
IterationLimit,
KeyArityMismatch {
keys: Vec<String>,
},
NotEmptied {
tables: Vec<String>,
},
SampleMutatedByRefresh {
tables: Vec<String>,
},
IntegrityViolation {
violations: Vec<String>,
},
}
impl std::fmt::Display for SampleError {
#[allow(clippy::too_many_lines)]
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::InvalidSpec { spec, detail } => write!(
f,
"--sample {spec:?} is not a valid sample spec: {detail}\n \
Write it as `<table>=<count>` or `<table>=<percent>%`, for example \
`--sample users=1%` or `--sample users=500`."
),
Self::UnknownRoot { tables } => write!(
f,
"--sample names {} table(s) that are not part of the sampled \
universe:\n{}\n \
A root must be one of the app's own tables. Framework-owned tables are \
emptied with `[framework] purge` instead, and a partition is sampled \
through its parent table.",
tables.len(),
bullets(tables),
),
Self::DuplicateRoot { table } => write!(
f,
"--sample names {table:?} more than once.\n \
A root table takes exactly one size; give the one you mean."
),
Self::RootNeverIncluded { tables } => write!(
f,
"--sample names {} table(s) that {SAMPLE_SECTION} excludes:\n{}\n \
A root is what the sample is built from, so it cannot also be dropped. \
Remove it from `never_include`, or sample a different root.",
tables.len(),
bullets(tables),
),
Self::RootAlwaysIncluded { tables } => write!(
f,
"--sample names {} table(s) that {SAMPLE_SECTION} copies whole:\n{}\n \
A size and \"every row\" are two different answers. Remove it from \
`always_include` to subset it, or drop the --sample root to keep it \
whole.",
tables.len(),
bullets(tables),
),
Self::RuleContradiction { tables } => write!(
f,
"{} table(s) are declared both `always_include` and `never_include` in \
{SAMPLE_SECTION}:\n{}\n Pick one.",
tables.len(),
bullets(tables),
),
Self::StaleRule { tables } => write!(
f,
"{SAMPLE_SECTION} names {} table(s) the database does not have:\n{}\n \
The declaration has drifted from the schema — remove or rename the stale \
entries.",
tables.len(),
bullets(tables),
),
Self::FrameworkRule { tables } => write!(
f,
"{SAMPLE_SECTION} names {} framework-owned table(s):\n{}\n \
Sampling covers the app's own tables only — framework-owned rows are \
emptied with `[framework] purge = [...]` instead.",
tables.len(),
bullets(tables),
),
Self::UncoveredTables { tables } => write!(
f,
"{} table(s) cannot be reached from any --sample root through the foreign \
key graph:\n{}\n \
Sampling would empty them without saying so. Name one as a root \
(`--sample <table>=<n>`), copy it whole (`[sample] always_include`), or \
drop it deliberately (`[sample] never_include`).\n \
Being connected to a root is not enough: the walk descends only out of a \
root and out of the tables it descended into. A table hanging off one it \
merely ASCENDED into \u{2014} a shared org a kept user points at \u{2014} or off \
an `always_include` lookup table, is not reachable: descending from one \
such row would drag the whole database back in.",
tables.len(),
bullets(tables),
),
Self::NeverIncludeReferenced { edges } => write!(
f,
"{} foreign key(s) point at a table {SAMPLE_SECTION} excludes:\n{}\n \
Emptying the parent would leave those references dangling. Drop the \
referencing table too (`never_include`), or stop excluding the parent.",
edges.len(),
bullets(edges),
),
Self::OutsideTableReferencesSampled { edges } => write!(
f,
"{} reference(s) come from a table outside the sample:\n{}\n \
Nothing removes rows from a framework-owned table, so removing the rows \
it points at would break it. Empty it in the same run with \
`[framework] purge = [...]`.",
edges.len(),
bullets(edges),
),
Self::PartitionLocalForeignKey { edges } => write!(
f,
"{} foreign key(s) are declared on a partition rather than on its \
partitioned parent:\n{}\n \
The sample selects and removes a partition's rows through that parent, \
whose rows span every partition, so it cannot honour a key that binds \
one partition only. Declare the foreign key on the partitioned parent \
so PostgreSQL clones it to each partition, or drop the table with \
`never_include` in {SAMPLE_SECTION}.",
edges.len(),
bullets(edges),
),
Self::PurgeOrderContradiction { tables } => write!(
f,
"{} purged table(s) would have to be emptied both before and after \
the sample:\n{}\n \
A purge normally runs first, so removing the rows it points at is \
safe; but one the sample's own emptied rows reference has to wait \
for the sample instead. A table needing both leaves no order the \
run can take. Stop purging it, or drop the table that references \
it with `never_include` in {SAMPLE_SECTION}.",
tables.len(),
bullets(tables),
),
Self::RetainedReferencesPurged { edges } => write!(
f,
"{} reference(s) point from rows the sample keeps into a table \
`[framework] purge` empties:\n{}\n \
Emptying the parent would leave the kept rows dangling, and no order of \
the two removals avoids it. Stop purging that table, or drop the \
referencing table with `never_include` in {SAMPLE_SECTION}.",
edges.len(),
bullets(edges),
),
Self::NoRowKey { tables } => write!(
f,
"{} table(s) in the sample have no primary key:\n{}\n \
Without one the sampler has no row identity to select on, and a \
full-copy table still needs one to join its parents. Add a primary key, \
or drop the table with `never_include` in {SAMPLE_SECTION}.",
tables.len(),
bullets(tables),
),
Self::ForeignKeyCycle { tables } => write!(
f,
"{} table(s) reference each other in a cycle:\n{}\n \
The row removals then have no order that keeps every foreign key \
satisfied at each step. Copy one of them whole (`[sample] \
always_include`), which takes it out of the removals altogether, or \
break the cycle on the copy before sampling.",
tables.len(),
bullets(tables),
),
Self::KeyArityMismatch { keys } => write!(
f,
"{} foreign key(s) report a different number of columns on each side:\n{}\n \
The walk joins the two sides component by component, so a mismatch would \
silently follow a weaker key than the constraint declares \u{2014} keeping or \
removing rows the database never related. The catalog should never report \
this; if it does, the schema is worth inspecting before any sample is taken \
from it.",
keys.len(),
bullets(keys),
),
Self::IterationLimit => write!(
f,
"The sample selection did not settle after {MAX_PASSES} passes over the \
foreign key graph.\n \
Nothing was written. Please report this with the schema that produced it."
),
Self::NotEmptied { tables } => write!(
f,
"{} table(s) the run promised would be empty still hold rows after \
every write:\n{}\n \
Something wrote to them after they were emptied, so the rows are back, \
carrying whatever they carried before. Two things can: a trigger fired by \
one of the emptying statements, inserting into a table an earlier one \
already emptied — an `ON DELETE` archive trigger between two of them will \
do it — or a materialized view whose query calls a function that INSERTs, \
since `REFRESH` runs that query. Nothing is committed. Disable the \
triggers on the copy before scrubbing, drop the write from the view's \
function, or stop promising a table is emptied when something refills it.",
tables.len(),
bullets(tables),
),
Self::SampleMutatedByRefresh { tables } => write!(
f,
"{} sampled table(s) were changed by a materialized view's \
refresh:\n{}\n \
The refreshes are the last writes in the transaction, so rows a \
view's function adds land after the subset was selected, after the \
rewrites that remove PII, and after the counts this run reports — \
measured, a view whose function INSERTs into a sampled table left \
the run reporting `403 -> 4 row(s)` over a table holding 7, three \
of them never rewritten. Nothing is committed. Drop the write from \
the view's function, or take the view out of the database this run \
scrubs.",
tables.len(),
bullets(tables),
),
Self::IntegrityViolation { violations } => write!(
f,
"The sampled database failed its own foreign key check:\n{}\n \
Nothing was written — the whole run was rolled back. This means the \
selection walk missed a reference; please report it.",
bullets(violations),
),
}
}
}
const SAMPLE_SECTION: &str = "`[sample]` in scrub.toml";
fn bullets(items: &[String]) -> String {
items
.iter()
.map(|item| format!(" - {item}"))
.collect::<Vec<_>>()
.join("\n")
}
#[derive(Debug, Clone, PartialEq)]
pub struct SampleTable {
pub table: String,
pub role: SampleRole,
pub key: Vec<String>,
pub keep: String,
pub descend: String,
}
#[derive(Debug, Clone, PartialEq)]
pub struct SamplePlan {
pub seed: u64,
pub tables: Vec<SampleTable>,
pub walk_edges: Vec<ForeignKeyConstraint>,
pub verify_edges: Vec<ForeignKeyConstraint>,
pub purge_after: BTreeSet<String>,
}
pub struct SampleInputs<'a> {
pub roots: &'a [SampleSpec],
pub seed: u64,
pub rules: &'a SampleRules,
pub tables: &'a [(String, Vec<String>)],
pub foreign_keys: &'a [ForeignKeyConstraint],
pub framework_tables: &'a BTreeSet<String>,
pub purged: &'a BTreeSet<String>,
pub partitions: &'a BTreeMap<String, String>,
}
pub fn parse_spec(raw: &str) -> Result<SampleSpec, SampleError> {
let invalid = |detail: &str| SampleError::InvalidSpec {
spec: raw.to_owned(),
detail: detail.to_owned(),
};
let (table, amount) = raw
.rsplit_once('=')
.ok_or_else(|| invalid("no `=` between the table and the amount"))?;
if table.is_empty() {
return Err(invalid("the table name is empty"));
}
let amount = amount.trim();
if amount.is_empty() {
return Err(invalid("the amount is empty"));
}
let amount = if let Some(percent) = amount.strip_suffix('%') {
let value: f64 = percent
.parse()
.map_err(|_| invalid("the percentage is not a number"))?;
if !(value.is_finite() && value > 0.0 && value <= 100.0) {
return Err(invalid(
"a percentage must be greater than 0 and at most 100",
));
}
SampleAmount::Percent(value)
} else {
let value: u64 = amount
.parse()
.map_err(|_| invalid("the count is not a whole number (add `%` for a percentage)"))?;
if value == 0 {
return Err(invalid("a root of 0 rows would select nothing"));
}
SampleAmount::Count(value)
};
Ok(SampleSpec {
table: table.to_owned(),
amount,
})
}
impl SampleAmount {
#[must_use]
pub fn rows(self, total: i64) -> i64 {
let total = total.max(0);
match self {
Self::Count(n) => i64::try_from(n).unwrap_or(i64::MAX).min(total),
#[allow(clippy::cast_possible_truncation, clippy::cast_precision_loss)]
Self::Percent(pct) => {
let wanted = (total as f64 * pct / 100.0).ceil();
let rows = (wanted as i64).clamp(0, total);
if total > 0 && pct > 0.0 {
rows.max(1)
} else {
rows
}
}
}
}
}
impl SampleRole {
fn is_subsetted(self) -> bool {
match self {
Self::Root(SampleAmount::Percent(p)) => p < 100.0,
Self::Root(SampleAmount::Count(_)) | Self::Related | Self::NeverInclude => true,
Self::AlwaysInclude => false,
}
}
const fn descends(self) -> bool {
matches!(self, Self::Root(_) | Self::Related)
}
const fn descends_from_seed(self) -> bool {
matches!(self, Self::Root(_))
}
const fn as_str(self) -> &'static str {
match self {
Self::Root(_) => "root",
Self::AlwaysInclude => "always-include",
Self::NeverInclude => "never-include",
Self::Related => "related",
}
}
}
pub fn build_plan(inputs: &SampleInputs<'_>) -> Result<SamplePlan, SampleError> {
let keys: BTreeMap<&str, &Vec<String>> = inputs
.tables
.iter()
.map(|(name, key)| (name.as_str(), key))
.collect();
check_rules(inputs, &keys)?;
let roles = resolve_roles(inputs, &keys)?;
let (internal, walk, purge_after) = classify_edges(inputs, &roles)?;
check_coverage(&roles, &walk)?;
check_row_keys(&roles, &keys)?;
let numbered: BTreeMap<&str, usize> = keys
.keys()
.enumerate()
.map(|(index, name)| (*name, index))
.collect();
let ordered = delete_order(&roles, &internal)?;
let tables = ordered
.into_iter()
.map(|name| {
let index = numbered[name.as_str()];
SampleTable {
role: roles[&name],
key: keys[name.as_str()].clone(),
keep: format!("_autumn_sample_keep_{index}"),
descend: format!("_autumn_sample_desc_{index}"),
table: name,
}
})
.collect();
Ok(SamplePlan {
seed: inputs.seed,
tables,
walk_edges: walk,
verify_edges: internal,
purge_after,
})
}
fn check_rules(
inputs: &SampleInputs<'_>,
keys: &BTreeMap<&str, &Vec<String>>,
) -> Result<(), SampleError> {
let declared: Vec<&String> = inputs
.rules
.always_include
.iter()
.chain(&inputs.rules.never_include)
.collect();
let framework = sorted_unique(
declared
.iter()
.filter(|t| inputs.framework_tables.contains(**t))
.map(|t| (*t).clone()),
);
if !framework.is_empty() {
return Err(SampleError::FrameworkRule { tables: framework });
}
let stale = sorted_unique(
declared
.iter()
.filter(|t| !keys.contains_key(t.as_str()))
.map(|t| (*t).clone()),
);
if !stale.is_empty() {
return Err(SampleError::StaleRule { tables: stale });
}
let never: BTreeSet<&String> = inputs.rules.never_include.iter().collect();
let both = sorted_unique(
inputs
.rules
.always_include
.iter()
.filter(|t| never.contains(*t))
.cloned(),
);
if !both.is_empty() {
return Err(SampleError::RuleContradiction { tables: both });
}
Ok(())
}
fn resolve_roles(
inputs: &SampleInputs<'_>,
keys: &BTreeMap<&str, &Vec<String>>,
) -> Result<BTreeMap<String, SampleRole>, SampleError> {
let mut seen: BTreeSet<&str> = BTreeSet::new();
for spec in inputs.roots {
if !seen.insert(spec.table.as_str()) {
return Err(SampleError::DuplicateRoot {
table: spec.table.clone(),
});
}
}
let unknown = sorted_unique(inputs.roots.iter().filter_map(|s| {
if keys.contains_key(s.table.as_str()) {
return None;
}
let name = &s.table;
let trimmed = name.trim();
Some(if trimmed == name || !keys.contains_key(trimmed) {
format!("{name:?}")
} else {
format!("{name:?} \u{2014} did you mean {trimmed:?}? the table is taken verbatim, since one may begin or end with a space")
})
}));
if !unknown.is_empty() {
return Err(SampleError::UnknownRoot { tables: unknown });
}
let never: BTreeSet<&String> = inputs.rules.never_include.iter().collect();
let excluded = sorted_unique(
inputs
.roots
.iter()
.filter(|s| never.contains(&s.table))
.map(|s| s.table.clone()),
);
if !excluded.is_empty() {
return Err(SampleError::RootNeverIncluded { tables: excluded });
}
let always: BTreeSet<&String> = inputs.rules.always_include.iter().collect();
let copied_whole = sorted_unique(
inputs
.roots
.iter()
.filter(|s| always.contains(&s.table))
.map(|s| s.table.clone()),
);
if !copied_whole.is_empty() {
return Err(SampleError::RootAlwaysIncluded {
tables: copied_whole,
});
}
let mut roles: BTreeMap<String, SampleRole> = keys
.keys()
.map(|name| {
let owned = (*name).to_owned();
let role = if never.contains(&owned) {
SampleRole::NeverInclude
} else if always.contains(&owned) {
SampleRole::AlwaysInclude
} else {
SampleRole::Related
};
(owned, role)
})
.collect();
for spec in inputs.roots {
roles.insert(spec.table.clone(), SampleRole::Root(spec.amount));
}
Ok(roles)
}
type ClassifiedEdges = (
Vec<ForeignKeyConstraint>,
Vec<ForeignKeyConstraint>,
BTreeSet<String>,
);
fn check_key_arity(foreign_keys: &[ForeignKeyConstraint]) -> Result<(), SampleError> {
let mut keys: Vec<String> = foreign_keys
.iter()
.filter(|edge| edge.child_columns.len() != edge.parent_columns.len())
.map(|edge| {
format!(
"{} ({} -> {}): {} vs {} column(s)",
comment_safe(&edge.name),
comment_safe(&edge.child_table),
comment_safe(&edge.parent_table),
edge.child_columns.len(),
edge.parent_columns.len(),
)
})
.collect();
if keys.is_empty() {
return Ok(());
}
keys.sort();
Err(SampleError::KeyArityMismatch { keys })
}
enum PartitionVerdict {
Classify,
Refuse,
Ignore,
}
fn describe(edge: &ForeignKeyConstraint) -> String {
format!(
"{} -> {} ({})",
edge.child_table, edge.parent_table, edge.name
)
}
fn defer_purge_for_excluded_leaf(
inputs: &SampleInputs<'_>,
roles: &BTreeMap<String, SampleRole>,
edge: &ForeignKeyConstraint,
purge_after: &mut BTreeSet<String>,
purged_after_edges: &mut BTreeMap<String, Vec<String>>,
purged_before: &mut BTreeMap<String, Vec<String>>,
) {
if inputs.purged.contains(&edge.parent_table) {
purge_after.insert(edge.parent_table.clone());
purged_after_edges
.entry(edge.parent_table.clone())
.or_default()
.push(describe(edge));
}
if let Some(root) = inputs.partitions.get(&edge.child_table)
&& inputs.purged.contains(root)
&& roles
.get(&edge.parent_table)
.is_some_and(|role| role.is_subsetted())
{
purged_before
.entry(root.clone())
.or_default()
.push(describe(edge));
}
}
fn partition_verdict(
inputs: &SampleInputs<'_>,
roles: &BTreeMap<String, SampleRole>,
edge: &ForeignKeyConstraint,
) -> PartitionVerdict {
let excluded_leaf = |table: &String| {
inputs.partitions.get(table).is_some_and(|root| {
roles.get(root) == Some(&SampleRole::NeverInclude) || inputs.purged.contains(root)
})
};
if excluded_leaf(&edge.child_table) {
return PartitionVerdict::Ignore;
}
if inputs.partitions.contains_key(&edge.child_table)
|| inputs.partitions.contains_key(&edge.parent_table)
{
return PartitionVerdict::Refuse;
}
PartitionVerdict::Classify
}
fn classify_edges(
inputs: &SampleInputs<'_>,
roles: &BTreeMap<String, SampleRole>,
) -> Result<ClassifiedEdges, SampleError> {
check_key_arity(inputs.foreign_keys)?;
let mut outside_refs = Vec::new();
let mut dangling = Vec::new();
let mut partition_local = Vec::new();
let mut retained_into_purged = Vec::new();
let mut purge_after = BTreeSet::new();
let mut purged_before: BTreeMap<String, Vec<String>> = BTreeMap::new();
let mut purged_after_edges: BTreeMap<String, Vec<String>> = BTreeMap::new();
let mut purged_internal: Vec<(String, String, String)> = Vec::new();
let mut verify_only: Vec<ForeignKeyConstraint> = Vec::new();
let mut internal = Vec::new();
for edge in inputs.foreign_keys {
match partition_verdict(inputs, roles, edge) {
PartitionVerdict::Refuse => {
partition_local.push(describe(edge));
continue;
}
PartitionVerdict::Ignore => {
defer_purge_for_excluded_leaf(
inputs,
roles,
edge,
&mut purge_after,
&mut purged_after_edges,
&mut purged_before,
);
continue;
}
PartitionVerdict::Classify => {}
}
let child = roles.get(&edge.child_table);
let parent = roles.get(&edge.parent_table);
match (child, parent) {
(Some(child_role), Some(parent_role)) => {
if *parent_role == SampleRole::NeverInclude
&& *child_role != SampleRole::NeverInclude
{
dangling.push(describe(edge));
}
internal.push(edge.clone());
}
(None, Some(parent_role))
if parent_role.is_subsetted() && !inputs.purged.contains(&edge.child_table) =>
{
outside_refs.push(describe(edge));
}
(None, Some(parent_role)) if parent_role.is_subsetted() => {
purged_before
.entry(edge.child_table.clone())
.or_default()
.push(describe(edge));
}
(None, Some(_)) if !inputs.purged.contains(&edge.child_table) => {
verify_only.push(edge.clone());
}
(Some(child_role), None) if inputs.purged.contains(&edge.parent_table) => {
if *child_role == SampleRole::NeverInclude {
purge_after.insert(edge.parent_table.clone());
purged_after_edges
.entry(edge.parent_table.clone())
.or_default()
.push(describe(edge));
} else {
retained_into_purged.push(describe(edge));
}
}
(Some(child_role), None) if *child_role != SampleRole::NeverInclude => {
verify_only.push(edge.clone());
}
(None, None)
if inputs.purged.contains(&edge.child_table)
&& inputs.purged.contains(&edge.parent_table) =>
{
purged_internal.push((
edge.child_table.clone(),
edge.parent_table.clone(),
describe(edge),
));
}
_ => {}
}
}
propagate_deferrals(&purged_internal, &mut purge_after, &mut purged_after_edges);
raise_edge_refusals(EdgeRefusals {
partition_local,
retained_into_purged,
outside_refs,
dangling,
})?;
check_purge_order(&purge_after, &purged_before, &purged_after_edges)?;
let walk = internal
.iter()
.filter(|edge| {
roles[&edge.child_table] != SampleRole::NeverInclude
&& roles[&edge.parent_table] != SampleRole::NeverInclude
})
.cloned()
.collect();
internal.extend(verify_only);
Ok((internal, walk, purge_after))
}
struct EdgeRefusals {
partition_local: Vec<String>,
retained_into_purged: Vec<String>,
outside_refs: Vec<String>,
dangling: Vec<String>,
}
fn raise_edge_refusals(mut found: EdgeRefusals) -> Result<(), SampleError> {
for (edges, build) in [
(
&mut found.partition_local,
(|e| SampleError::PartitionLocalForeignKey { edges: e }) as fn(Vec<String>) -> _,
),
(
&mut found.retained_into_purged,
(|e| SampleError::RetainedReferencesPurged { edges: e }) as fn(Vec<String>) -> _,
),
(
&mut found.outside_refs,
(|e| SampleError::OutsideTableReferencesSampled { edges: e }) as fn(Vec<String>) -> _,
),
(
&mut found.dangling,
(|e| SampleError::NeverIncludeReferenced { edges: e }) as fn(Vec<String>) -> _,
),
] {
if !edges.is_empty() {
edges.sort();
return Err(build(std::mem::take(edges)));
}
}
Ok(())
}
fn propagate_deferrals(
edges: &[(String, String, String)],
purge_after: &mut BTreeSet<String>,
after_edges: &mut BTreeMap<String, Vec<String>>,
) {
let mut settled = false;
while !settled {
settled = true;
for (child, parent, edge) in edges {
if purge_after.contains(child) && !purge_after.contains(parent) {
purge_after.insert(parent.clone());
after_edges
.entry(parent.clone())
.or_default()
.push(edge.clone());
settled = false;
}
}
}
}
fn check_purge_order(
purge_after: &BTreeSet<String>,
before: &BTreeMap<String, Vec<String>>,
after: &BTreeMap<String, Vec<String>>,
) -> Result<(), SampleError> {
let mut contradictions: Vec<String> = purge_after
.iter()
.filter_map(|table| {
Some(format!(
"{table}: before {}, yet after {}",
before.get(table)?.join(", "),
after.get(table)?.join(", "),
))
})
.collect();
if contradictions.is_empty() {
return Ok(());
}
contradictions.sort();
Err(SampleError::PurgeOrderContradiction {
tables: contradictions,
})
}
fn check_coverage(
roles: &BTreeMap<String, SampleRole>,
walk: &[ForeignKeyConstraint],
) -> Result<(), SampleError> {
let mut descend: BTreeSet<&str> = roles
.iter()
.filter(|(_, role)| role.descends_from_seed())
.map(|(name, _)| name.as_str())
.collect();
let mut covered: BTreeSet<&str> = roles
.iter()
.filter(|(_, role)| matches!(role, SampleRole::Root(_) | SampleRole::AlwaysInclude))
.map(|(name, _)| name.as_str())
.collect();
let mut grew = true;
while grew {
grew = false;
for edge in walk {
let (child, parent) = (edge.child_table.as_str(), edge.parent_table.as_str());
if descend.contains(parent) && roles[child].descends() {
grew |= descend.insert(child);
}
if descend.contains(parent) {
grew |= covered.insert(child);
}
if covered.contains(child) {
grew |= covered.insert(parent);
}
}
}
let uncovered = sorted_unique(
roles
.iter()
.filter(|(name, role)| {
**role == SampleRole::Related && !covered.contains(name.as_str())
})
.map(|(name, _)| name.clone()),
);
if uncovered.is_empty() {
Ok(())
} else {
Err(SampleError::UncoveredTables { tables: uncovered })
}
}
fn check_row_keys(
roles: &BTreeMap<String, SampleRole>,
keys: &BTreeMap<&str, &Vec<String>>,
) -> Result<(), SampleError> {
let missing = sorted_unique(
roles
.iter()
.filter(|(name, role)| {
**role != SampleRole::NeverInclude && keys[name.as_str()].is_empty()
})
.map(|(name, _)| name.clone()),
);
if missing.is_empty() {
Ok(())
} else {
Err(SampleError::NoRowKey { tables: missing })
}
}
fn delete_order(
roles: &BTreeMap<String, SampleRole>,
edges: &[ForeignKeyConstraint],
) -> Result<Vec<String>, SampleError> {
let removed: BTreeSet<&str> = roles
.iter()
.filter(|(_, role)| role.is_subsetted())
.map(|(name, _)| name.as_str())
.collect();
let mut parents: BTreeMap<&str, BTreeSet<&str>> =
removed.iter().map(|n| (*n, BTreeSet::new())).collect();
let mut incoming: BTreeMap<&str, BTreeSet<&str>> =
removed.iter().map(|n| (*n, BTreeSet::new())).collect();
for edge in edges {
let (child, parent) = (edge.child_table.as_str(), edge.parent_table.as_str());
if child == parent || !removed.contains(child) || !removed.contains(parent) {
continue;
}
parents.entry(child).or_default().insert(parent);
incoming.entry(parent).or_default().insert(child);
}
let mut ready: BTreeSet<&str> = incoming
.iter()
.filter(|(_, children)| children.is_empty())
.map(|(name, _)| *name)
.collect();
let mut remaining = incoming.clone();
let mut ordered: Vec<String> = Vec::with_capacity(roles.len());
while let Some(next) = ready.iter().next().copied() {
ready.remove(next);
ordered.push(next.to_owned());
for parent in &parents[next] {
let children = remaining.get_mut(parent).expect("edge target is a table");
children.remove(next);
if children.is_empty() {
ready.insert(parent);
}
}
remaining.remove(next);
}
if ordered.len() == removed.len() {
ordered.extend(
roles
.iter()
.filter(|(_, role)| !role.is_subsetted())
.map(|(name, _)| name.clone()),
);
return Ok(ordered);
}
let mut residue: BTreeSet<&str> = remaining.keys().copied().collect();
loop {
let sinks: Vec<&str> = residue
.iter()
.filter(|name| !parents[**name].iter().any(|p| residue.contains(p)))
.copied()
.collect();
if sinks.is_empty() {
break;
}
for sink in sinks {
residue.remove(sink);
}
}
Err(SampleError::ForeignKeyCycle {
tables: sorted_unique(residue.into_iter().map(str::to_owned)),
})
}
fn sorted_unique(items: impl Iterator<Item = String>) -> Vec<String> {
items.collect::<BTreeSet<_>>().into_iter().collect()
}
fn key_expr(alias: &str, key: &[String]) -> String {
if let [single] = key {
return format!("{alias}.{}", quote_ident(single));
}
let parts: Vec<String> = key
.iter()
.map(|column| format!("quote_nullable({alias}.{})", quote_ident(column)))
.collect();
format!("({})", parts.join(" || ',' || "))
}
const ITERATION_LIMIT_MESSAGE: &str = "the sample selection did not settle; the closure walk did \
not reach a fixpoint";
pub fn dollar_tag(body: &str) -> String {
let mut tag = String::from("$autumn_walk$");
while body.contains(&tag) {
tag.insert(tag.len() - 1, '_');
}
tag
}
pub fn comment_safe(name: &str) -> String {
name.chars()
.map(|c| {
if c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | '.' | ' ') {
c
} else {
'_'
}
})
.collect()
}
fn join_on(edge: &ForeignKeyConstraint, child: &str, parent: &str) -> String {
edge.child_columns
.iter()
.zip(&edge.parent_columns)
.map(|(c, p)| format!("{parent}.{} = {child}.{}", quote_ident(p), quote_ident(c)))
.collect::<Vec<_>>()
.join(" AND ")
}
fn frontier_of(set: &str) -> String {
format!("{set}_front")
}
fn next_of(set: &str) -> String {
format!("{set}_next")
}
fn ascends_into(parent: &SampleTable) -> bool {
parent.role != SampleRole::AlwaysInclude
}
const fn descends_between(parent: &SampleTable, child: &SampleTable) -> bool {
parent.role.descends() && child.role.descends()
}
impl SamplePlan {
fn table(&self, name: &str) -> &SampleTable {
self.tables
.iter()
.find(|t| t.table == name)
.expect("every edge names a table in the plan")
}
fn keyed_tables(&self) -> impl Iterator<Item = &SampleTable> {
self.tables
.iter()
.filter(|t| t.role != SampleRole::NeverInclude)
}
fn descend_reachable(&self) -> BTreeSet<&str> {
let mut reached: BTreeSet<&str> = self
.tables
.iter()
.filter(|t| t.role.descends_from_seed())
.map(|t| t.table.as_str())
.collect();
let mut grew = true;
while grew {
grew = false;
for edge in &self.walk_edges {
let child = self.table(&edge.child_table);
let parent = self.table(&edge.parent_table);
if reached.contains(parent.table.as_str()) && descends_between(parent, child) {
grew |= reached.insert(child.table.as_str());
}
}
}
reached
}
fn walk_sets(&self) -> (BTreeSet<String>, BTreeSet<String>) {
let reachable = self.descend_reachable();
let (mut read, mut written) = (BTreeSet::new(), BTreeSet::new());
for edge in &self.walk_edges {
let child = self.table(&edge.child_table);
let parent = self.table(&edge.parent_table);
if ascends_into(parent) {
read.insert(child.keep.clone());
written.insert(parent.keep.clone());
}
if reachable.contains(parent.table.as_str()) && descends_between(parent, child) {
read.insert(parent.descend.clone());
written.insert(child.keep.clone());
written.insert(child.descend.clone());
}
}
(read, written)
}
fn frontier_sets(&self) -> Vec<String> {
self.walk_sets().0.into_iter().collect()
}
fn next_sets(&self) -> Vec<String> {
let (read, written) = self.walk_sets();
read.intersection(&written).cloned().collect()
}
fn insert_into(next_sets: &[String], tag: &str, set: &str, select: &str) -> String {
if next_sets.iter().any(|f| f == set) {
format!(
"{tag}WITH added AS (INSERT INTO {set_ident} (k) {select} RETURNING k), \
frontier AS (INSERT INTO {frontier} (k) SELECT k FROM added) \
INSERT INTO {next} (k) SELECT k FROM added",
set_ident = quote_ident(set),
frontier = quote_ident(&frontier_of(set)),
next = quote_ident(&next_of(set)),
)
} else {
format!("{tag}INSERT INTO {} (k) {select}", quote_ident(set))
}
}
#[must_use]
pub fn setup_statements(&self) -> Vec<String> {
self.keyed_tables()
.flat_map(|table| {
[&table.keep, &table.descend].map(|set| {
format!(
"CREATE TEMPORARY TABLE {} ON COMMIT DROP AS \
SELECT {} AS k FROM {} AS t WITH NO DATA",
quote_ident(set),
key_expr("t", &table.key),
qualified(&table.table),
)
})
})
.chain(
self.frontier_sets()
.into_iter()
.map(|set| (frontier_of(&set), set))
.chain(self.next_sets().into_iter().map(|set| (next_of(&set), set)))
.map(|(name, set)| {
format!(
"CREATE TEMPORARY TABLE {} ON COMMIT DROP AS \
SELECT k FROM {} WITH NO DATA",
quote_ident(&name),
quote_ident(&set),
)
}),
)
.collect()
}
#[must_use]
pub fn index_statements(&self) -> Vec<String> {
let mut out = Vec::new();
for table in self.keyed_tables() {
for set in [&table.keep, &table.descend] {
out.push(format!("CREATE INDEX ON {}(k)", quote_ident(set)));
out.push(format!("ANALYZE {}", quote_ident(set)));
}
}
for set in self.frontier_sets() {
out.push(format!("ANALYZE {}", quote_ident(&frontier_of(&set))));
}
out
}
#[must_use]
pub fn analyze_statements(&self) -> Vec<String> {
self.keyed_tables()
.flat_map(|table| [table.keep.clone(), table.descend.clone()])
.chain(self.frontier_sets().iter().map(|set| frontier_of(set)))
.map(|set| format!("ANALYZE {}", quote_ident(&set)))
.collect()
}
#[must_use]
pub fn rotate_statements(&self) -> Vec<String> {
let next_sets = self.next_sets();
let mut out = Vec::new();
for set in self.frontier_sets() {
let frontier = quote_ident(&frontier_of(&set));
out.push(format!("TRUNCATE {frontier}"));
if next_sets.contains(&set) {
let next = quote_ident(&next_of(&set));
out.push(format!("INSERT INTO {frontier} (k) SELECT k FROM {next}"));
out.push(format!("TRUNCATE {next}"));
}
}
out
}
#[must_use]
pub fn end_of_pass_statements(&self) -> Vec<String> {
let mut out = self.rotate_statements();
out.extend(self.analyze_statements());
out
}
#[must_use]
pub fn seed_statements(&self, counts: &BTreeMap<String, i64>) -> Vec<String> {
let seed = quote_literal(&self.seed.to_string());
let mut out = Vec::new();
for table in &self.tables {
let key = key_expr("t", &table.key);
match table.role {
SampleRole::Root(amount) => {
let rows = amount.rows(counts.get(&table.table).copied().unwrap_or(0));
out.push(format!(
"INSERT INTO {} (k) SELECT k FROM (\
SELECT {key} AS k FROM {} AS t \
ORDER BY md5({seed} || '|' || ({key})::text), ({key})::text \
LIMIT {rows}) AS chosen",
quote_ident(&table.keep),
qualified(&table.table),
));
out.push(format!(
"INSERT INTO {} (k) SELECT k FROM {}",
quote_ident(&table.descend),
quote_ident(&table.keep),
));
}
SampleRole::AlwaysInclude => out.push(format!(
"INSERT INTO {} (k) SELECT {key} FROM {} AS t",
quote_ident(&table.keep),
qualified(&table.table),
)),
SampleRole::NeverInclude | SampleRole::Related => {}
}
}
for set in self.frontier_sets() {
out.push(format!(
"INSERT INTO {} (k) SELECT k FROM {}",
quote_ident(&frontier_of(&set)),
quote_ident(&set),
));
}
out
}
#[must_use]
pub fn walk_statements(&self) -> Vec<String> {
let next_sets = self.next_sets();
let reachable = self.descend_reachable();
let mut out = Vec::new();
for edge in &self.walk_edges {
let child = self.table(&edge.child_table);
let parent = self.table(&edge.parent_table);
let child_key = key_expr("c", &child.key);
let parent_key = key_expr("p", &parent.key);
let on = join_on(edge, "c", "p");
let tag = format!("/* {} */ ", comment_safe(&edge.name));
if ascends_into(parent) {
let select = format!(
"SELECT DISTINCT {parent_key} \
FROM {parent_table} AS p \
JOIN {child_table} AS c ON {on} \
JOIN {child_frontier} AS ck ON ck.k = {child_key} \
WHERE NOT EXISTS (SELECT 1 FROM {keep} AS existing \
WHERE existing.k = {parent_key})",
keep = quote_ident(&parent.keep),
parent_table = qualified(&parent.table),
child_table = qualified(&child.table),
child_frontier = quote_ident(&frontier_of(&child.keep)),
);
out.push(Self::insert_into(&next_sets, &tag, &parent.keep, &select));
}
if reachable.contains(parent.table.as_str()) && descends_between(parent, child) {
for set in [&child.keep, &child.descend] {
let select = format!(
"SELECT DISTINCT {child_key} \
FROM {child_table} AS c \
JOIN {parent_table} AS p ON {on} \
JOIN {parent_frontier} AS pd ON pd.k = {parent_key} \
WHERE NOT EXISTS (SELECT 1 FROM {set_ident} AS existing \
WHERE existing.k = {child_key})",
set_ident = quote_ident(set),
child_table = qualified(&child.table),
parent_table = qualified(&parent.table),
parent_frontier = quote_ident(&frontier_of(&parent.descend)),
);
out.push(Self::insert_into(&next_sets, &tag, set, &select));
}
}
}
out
}
#[must_use]
pub fn walk_loop_statement(&self) -> String {
let mut body = vec![
"DECLARE".to_owned(),
" moved bigint;".to_owned(),
" total bigint;".to_owned(),
" passes integer := 0;".to_owned(),
"BEGIN".to_owned(),
" LOOP".to_owned(),
" total := 0;".to_owned(),
];
for statement in self.walk_statements() {
body.push(format!(" EXECUTE {};", quote_literal(&statement)));
body.push(" GET DIAGNOSTICS moved = ROW_COUNT;".to_owned());
body.push(" total := total + moved;".to_owned());
}
for statement in self.end_of_pass_statements() {
body.push(format!(" {statement};"));
}
body.push(" passes := passes + 1;".to_owned());
body.push(" EXIT WHEN total = 0;".to_owned());
body.push(format!(" IF passes >= {MAX_PASSES} THEN"));
body.push(format!(
" RAISE EXCEPTION {};",
quote_literal(ITERATION_LIMIT_MESSAGE),
));
body.push(" END IF;".to_owned());
body.push(" END LOOP;".to_owned());
body.push("END".to_owned());
let body = body.join("\n");
let tag = dollar_tag(&body);
format!("DO {tag}\n{body}\n{tag}")
}
#[must_use]
pub fn delete_statements(&self) -> Vec<String> {
self.tables
.iter()
.filter_map(|table| match table.role {
_ if !table.role.is_subsetted() => None,
SampleRole::AlwaysInclude => None,
SampleRole::NeverInclude => {
Some(format!("DELETE FROM {}", qualified(&table.table)))
}
SampleRole::Root(_) | SampleRole::Related => Some(format!(
"DELETE FROM {} AS t WHERE NOT EXISTS \
(SELECT 1 FROM {} AS k WHERE k.k = {})",
qualified(&table.table),
quote_ident(&table.keep),
key_expr("t", &table.key),
)),
})
.collect()
}
#[must_use]
pub fn subsetted_tables(&self) -> Vec<&str> {
self.tables
.iter()
.filter(|t| t.role.is_subsetted())
.map(|t| t.table.as_str())
.collect()
}
#[must_use]
pub fn emptied_tables(&self) -> Vec<(&str, String)> {
self.tables
.iter()
.filter(|t| t.role == SampleRole::NeverInclude)
.map(|t| {
(
t.table.as_str(),
format!("DELETE FROM {}", qualified(&t.table)),
)
})
.collect()
}
#[must_use]
pub fn locked_tables(&self) -> Vec<&str> {
let mut tables: Vec<&str> = self
.tables
.iter()
.map(|t| t.table.as_str())
.chain(
self.verify_edges
.iter()
.flat_map(|edge| [edge.child_table.as_str(), edge.parent_table.as_str()]),
)
.collect();
tables.sort_unstable();
tables.dedup();
tables
}
#[must_use]
pub fn integrity_statements(&self) -> Vec<(String, String)> {
self.verify_edges
.iter()
.map(|edge| {
let on = join_on(edge, "c", "p");
let populated = edge
.child_columns
.iter()
.map(|c| format!("c.{} IS NOT NULL", quote_ident(c)))
.collect::<Vec<_>>()
.join(" AND ");
let missing = format!("p.{} IS NULL", quote_ident(&edge.parent_columns[0]));
let mixed_null = if edge.match_full && edge.child_columns.len() > 1 {
let any_null = edge
.child_columns
.iter()
.map(|c| format!("c.{} IS NULL", quote_ident(c)))
.collect::<Vec<_>>()
.join(" OR ");
Some(format!(
"({any_null}) AND ({populated_any})",
populated_any = edge
.child_columns
.iter()
.map(|c| format!("c.{} IS NOT NULL", quote_ident(c)))
.collect::<Vec<_>>()
.join(" OR ")
))
} else {
None
};
(
format!(
"{} ({} -> {})",
comment_safe(&edge.name),
comment_safe(&edge.child_table),
comment_safe(&edge.parent_table),
),
format!(
"SELECT count(*) AS n FROM {} AS c \
LEFT JOIN {} AS p ON {on} \
WHERE ({populated} AND {missing}){extra}",
qualified(&edge.child_table),
qualified(&edge.parent_table),
extra = mixed_null
.as_ref()
.map_or_else(String::new, |m| format!(" OR ({m})")),
),
)
})
.collect()
}
}
#[derive(diesel::QueryableByName)]
struct CountRow {
#[diesel(sql_type = diesel::sql_types::BigInt)]
n: i64,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SampleCount {
pub table: String,
pub role: &'static str,
pub before: i64,
pub after: i64,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SampleOutcome {
pub counts: Vec<SampleCount>,
pub passes: usize,
pub verified: usize,
pub size_before: i64,
}
pub fn data_size(
conn: &mut PgConnection,
plan: &SamplePlan,
also: &[String],
) -> Result<i64, diesel::result::Error> {
let mut measured: Vec<&str> = plan
.tables
.iter()
.map(|t| t.table.as_str())
.chain(also.iter().map(String::as_str))
.collect();
measured.sort_unstable();
measured.dedup();
let names = measured
.into_iter()
.map(quote_literal)
.collect::<Vec<_>>()
.join(", ");
if names.is_empty() {
return Ok(0);
}
let rows: Vec<CountRow> = sql_query(format!(
"WITH RECURSIVE named AS ( \
SELECT c.oid FROM pg_class c \
JOIN pg_namespace ns ON ns.oid = c.relnamespace AND ns.nspname = 'public' \
WHERE c.relname IN ({names}) \
), tree AS ( \
SELECT oid FROM named \
UNION \
SELECT i.inhrelid FROM pg_inherits i JOIN tree t ON t.oid = i.inhparent \
) \
SELECT coalesce(sum(pg_total_relation_size(oid)), 0)::bigint AS n FROM tree"
))
.load(conn)?;
Ok(rows.first().map_or(0, |r| r.n))
}
pub fn source_counts(
conn: &mut PgConnection,
plan: &SamplePlan,
) -> Result<BTreeMap<String, i64>, diesel::result::Error> {
let mut counts = BTreeMap::new();
for table in &plan.tables {
counts.insert(table.table.clone(), count_rows(conn, &table.table)?);
}
Ok(counts)
}
fn count_rows(conn: &mut PgConnection, table: &str) -> Result<i64, diesel::result::Error> {
let rows: Vec<CountRow> =
sql_query(format!("SELECT count(*) AS n FROM {}", qualified(table))).load(conn)?;
Ok(rows.first().map_or(0, |r| r.n))
}
#[derive(Debug)]
pub enum SampleFailure {
Refused(SampleError),
Db(diesel::result::Error),
}
impl From<diesel::result::Error> for SampleFailure {
fn from(e: diesel::result::Error) -> Self {
Self::Db(e)
}
}
pub fn apply(
conn: &mut PgConnection,
plan: &SamplePlan,
also_measured: &[String],
) -> Result<SampleOutcome, SampleFailure> {
let size_before = data_size(conn, plan, also_measured)?;
let before = source_counts(conn, plan)?;
for statement in plan.setup_statements() {
sql_query(statement).execute(conn)?;
}
for statement in plan.seed_statements(&before) {
sql_query(statement).execute(conn)?;
}
for statement in plan.index_statements() {
sql_query(statement).execute(conn)?;
}
let walk = plan.walk_statements();
let end_of_pass = plan.end_of_pass_statements();
let mut passes = 0;
loop {
let mut selected = 0;
for statement in &walk {
selected += sql_query(statement).execute(conn)?;
}
for statement in &end_of_pass {
sql_query(statement).execute(conn)?;
}
passes += 1;
if selected == 0 {
break;
}
if passes >= MAX_PASSES {
return Err(SampleFailure::Refused(SampleError::IterationLimit));
}
}
for statement in plan.delete_statements() {
sql_query(statement).execute(conn)?;
}
let mut violations = Vec::new();
let checks = plan.integrity_statements();
for (label, sql) in &checks {
let orphans = sql_query(sql)
.load::<CountRow>(conn)?
.first()
.map_or(0, |r| r.n);
if orphans > 0 {
violations.push(format!("{label}: {orphans} unresolved reference(s)"));
}
}
if !violations.is_empty() {
violations.sort();
return Err(SampleFailure::Refused(SampleError::IntegrityViolation {
violations,
}));
}
let mut counts = Vec::with_capacity(plan.tables.len());
for table in &plan.tables {
counts.push(SampleCount {
role: table.role.as_str(),
before: before[&table.table],
after: count_rows(conn, &table.table)?,
table: table.table.clone(),
});
}
Ok(SampleOutcome {
counts,
passes,
verified: checks.len(),
size_before,
})
}
#[cfg(test)]
mod tests {
use super::*;
fn table(name: &str, key: &[&str]) -> (String, Vec<String>) {
(
name.to_owned(),
key.iter().map(|k| (*k).to_owned()).collect(),
)
}
fn fk(
name: &str,
child: &str,
child_col: &str,
parent: &str,
parent_col: &str,
) -> ForeignKeyConstraint {
ForeignKeyConstraint {
name: name.to_owned(),
child_table: child.to_owned(),
child_columns: vec![child_col.to_owned()],
parent_table: parent.to_owned(),
parent_columns: vec![parent_col.to_owned()],
match_full: false,
}
}
fn root(table: &str, amount: SampleAmount) -> SampleSpec {
SampleSpec {
table: table.to_owned(),
amount,
}
}
fn schema() -> (Vec<(String, Vec<String>)>, Vec<ForeignKeyConstraint>) {
(
vec![
table("users", &["id"]),
table("comments", &["id"]),
table("countries", &["id"]),
table("audit_logs", &["id"]),
],
vec![
fk("comments_user_fk", "comments", "user_id", "users", "id"),
fk("users_country_fk", "users", "country_id", "countries", "id"),
],
)
}
fn plan_of(
roots: &[SampleSpec],
rules: &SampleRules,
tables: &[(String, Vec<String>)],
foreign_keys: &[ForeignKeyConstraint],
) -> Result<SamplePlan, SampleError> {
build_plan(&SampleInputs {
roots,
seed: 7,
rules,
tables,
foreign_keys,
framework_tables: &BTreeSet::new(),
purged: &BTreeSet::new(),
partitions: &BTreeMap::new(),
})
}
fn fixture_rules() -> SampleRules {
SampleRules {
always_include: vec!["countries".to_owned()],
never_include: vec!["audit_logs".to_owned()],
}
}
fn role_of(plan: &SamplePlan, table: &str) -> SampleRole {
plan.tables
.iter()
.find(|t| t.table == table)
.unwrap_or_else(|| panic!("{table} is not in the plan"))
.role
}
fn joined(statements: &[String]) -> String {
statements.join("\n")
}
#[test]
fn spec_parses_a_percentage() {
assert_eq!(
parse_spec("users=1%").unwrap(),
root("users", SampleAmount::Percent(1.0))
);
assert_eq!(
parse_spec("users=2.5%").unwrap(),
root("users", SampleAmount::Percent(2.5))
);
}
#[test]
fn spec_parses_an_absolute_count() {
assert_eq!(
parse_spec("users=500").unwrap(),
root("users", SampleAmount::Count(500))
);
}
#[test]
fn spec_splits_a_table_name_containing_the_delimiter() {
assert_eq!(
parse_spec("events=2026=500").unwrap(),
root("events=2026", SampleAmount::Count(500))
);
assert_eq!(
parse_spec("events=2026=1%").unwrap(),
root("events=2026", SampleAmount::Percent(1.0))
);
assert_eq!(
parse_spec("users=500").unwrap(),
root("users", SampleAmount::Count(500))
);
}
#[test]
fn spec_keeps_leading_and_trailing_space_in_a_table_name() {
assert_eq!(
parse_spec(" users=5").unwrap(),
root(" users", SampleAmount::Count(5))
);
assert_eq!(
parse_spec("users =5").unwrap(),
root("users ", SampleAmount::Count(5))
);
assert_eq!(
parse_spec("users= 5 ").unwrap(),
root("users", SampleAmount::Count(5))
);
assert_eq!(
parse_spec("users= 2.5% ").unwrap(),
root("users", SampleAmount::Percent(2.5))
);
}
#[test]
fn spec_rejects_malformed_amounts() {
for bad in [
"users", "=1%", "users=", "users=0", "users=0%", "users=101%", "users=-5",
"users=abc",
"users=1.5", ] {
assert!(
matches!(parse_spec(bad), Err(SampleError::InvalidSpec { .. })),
"{bad:?} should be rejected"
);
}
}
#[test]
fn a_percentage_rounds_up_to_at_least_one_row() {
assert_eq!(SampleAmount::Percent(1.0).rows(10), 1);
assert_eq!(SampleAmount::Percent(1.0).rows(1000), 10);
assert_eq!(SampleAmount::Percent(100.0).rows(1000), 1000);
assert_eq!(SampleAmount::Percent(50.0).rows(0), 0);
assert_eq!(SampleAmount::Percent(5e-324).rows(1), 1);
assert_eq!(SampleAmount::Percent(5e-324).rows(10), 1);
assert_eq!(SampleAmount::Percent(5e-324).rows(200), 1);
assert_eq!(SampleAmount::Percent(5e-324).rows(0), 0);
}
#[test]
fn a_count_is_capped_at_the_source_row_count() {
assert_eq!(SampleAmount::Count(500).rows(1000), 500);
assert_eq!(SampleAmount::Count(5000).rows(1000), 1000);
}
#[test]
fn plan_assigns_root_always_never_and_derived_roles() {
let (tables, keys) = schema();
let plan = plan_of(
&[root("users", SampleAmount::Percent(1.0))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap();
assert_eq!(
role_of(&plan, "users"),
SampleRole::Root(SampleAmount::Percent(1.0))
);
assert_eq!(role_of(&plan, "comments"), SampleRole::Related);
assert_eq!(role_of(&plan, "countries"), SampleRole::AlwaysInclude);
assert_eq!(role_of(&plan, "audit_logs"), SampleRole::NeverInclude);
}
#[test]
fn plan_refuses_a_table_no_root_can_reach() {
let (tables, keys) = schema();
let err = plan_of(
&[root("users", SampleAmount::Percent(1.0))],
&SampleRules {
always_include: vec!["countries".to_owned()],
never_include: Vec::new(),
},
&tables,
&keys,
)
.unwrap_err();
assert_eq!(
err,
SampleError::UncoveredTables {
tables: vec!["audit_logs".to_owned()]
}
);
}
#[test]
fn plan_covers_a_table_reachable_only_through_a_lookup_table() {
let mut tables = schema().0;
tables.push(table("regions", &["id"]));
let mut keys = schema().1;
keys.push(fk(
"countries_region_fk",
"countries",
"region_id",
"regions",
"id",
));
let plan = plan_of(
&[root("users", SampleAmount::Percent(1.0))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap();
assert_eq!(role_of(&plan, "regions"), SampleRole::Related);
}
#[test]
fn plan_refuses_a_table_hanging_off_a_table_the_walk_only_ascended_into() {
let (mut tables, mut keys) = schema();
tables.push(table("orgs", &["id"]));
tables.push(table("org_settings", &["id"]));
keys.push(fk("users_org_fk", "users", "org_id", "orgs", "id"));
keys.push(fk(
"org_settings_org_fk",
"org_settings",
"org_id",
"orgs",
"id",
));
let err = plan_of(
&[root("users", SampleAmount::Percent(1.0))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap_err();
assert_eq!(
err,
SampleError::UncoveredTables {
tables: vec!["org_settings".to_owned()]
}
);
}
#[test]
fn plan_covers_a_table_below_a_root_through_two_hops() {
let (mut tables, mut keys) = schema();
tables.push(table("comment_votes", &["id"]));
keys.push(fk(
"comment_votes_comment_fk",
"comment_votes",
"comment_id",
"comments",
"id",
));
let plan = plan_of(
&[root("users", SampleAmount::Percent(1.0))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap();
assert_eq!(role_of(&plan, "comment_votes"), SampleRole::Related);
}
#[test]
fn plan_refuses_a_reference_into_a_never_include_table() {
let (mut tables, mut keys) = schema();
tables.push(table("notes", &["id"]));
keys.push(fk("notes_user_fk", "notes", "user_id", "users", "id"));
keys.push(fk(
"notes_audit_fk",
"notes",
"audit_id",
"audit_logs",
"id",
));
let err = plan_of(
&[root("users", SampleAmount::Percent(1.0))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap_err();
assert_eq!(
err,
SampleError::NeverIncludeReferenced {
edges: vec!["notes -> audit_logs (notes_audit_fk)".to_owned()]
}
);
}
#[test]
fn plan_refuses_an_unknown_root() {
let (tables, keys) = schema();
let err = plan_of(
&[root("nope", SampleAmount::Count(10))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap_err();
assert_eq!(
err,
SampleError::UnknownRoot {
tables: vec!["\"nope\"".to_owned()]
},
"quoted, so a name whose only oddity is invisible reads as one"
);
}
#[test]
fn plan_suggests_the_trimmed_name_for_a_space_padded_root() {
let (tables, keys) = schema();
let err = plan_of(
&[root(" users", SampleAmount::Count(10))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap_err();
let SampleError::UnknownRoot { tables: reported } = err else {
panic!("a space-padded root must be unknown, not silently trimmed");
};
assert_eq!(reported.len(), 1);
assert!(
reported[0].contains("\" users\"") && reported[0].contains("did you mean \"users\""),
"the message must show the space and name the table it would have hit: {}",
reported[0]
);
let err = plan_of(
&[root(" nope ", SampleAmount::Count(10))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap_err();
let SampleError::UnknownRoot { tables: reported } = err else {
panic!("unknown is unknown");
};
assert!(
!reported[0].contains("did you mean"),
"no suggestion where trimming would not have matched: {}",
reported[0]
);
}
#[test]
fn plan_refuses_a_duplicate_root() {
let (tables, keys) = schema();
let err = plan_of(
&[
root("users", SampleAmount::Count(10)),
root("users", SampleAmount::Count(20)),
],
&fixture_rules(),
&tables,
&keys,
)
.unwrap_err();
assert_eq!(
err,
SampleError::DuplicateRoot {
table: "users".to_owned()
}
);
}
#[test]
fn plan_refuses_a_root_that_is_also_never_included() {
let (tables, keys) = schema();
let err = plan_of(
&[root("users", SampleAmount::Count(10))],
&SampleRules {
always_include: vec!["countries".to_owned()],
never_include: vec!["users".to_owned(), "audit_logs".to_owned()],
},
&tables,
&keys,
)
.unwrap_err();
assert_eq!(
err,
SampleError::RootNeverIncluded {
tables: vec!["users".to_owned()]
}
);
}
#[test]
fn plan_refuses_a_root_that_is_also_copied_whole() {
let (tables, keys) = schema();
let err = plan_of(
&[root("countries", SampleAmount::Count(10))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap_err();
assert_eq!(
err,
SampleError::RootAlwaysIncluded {
tables: vec!["countries".to_owned()]
}
);
}
#[test]
fn plan_refuses_a_table_declared_both_always_and_never() {
let (tables, keys) = schema();
let err = plan_of(
&[root("users", SampleAmount::Count(10))],
&SampleRules {
always_include: vec!["countries".to_owned(), "audit_logs".to_owned()],
never_include: vec!["audit_logs".to_owned()],
},
&tables,
&keys,
)
.unwrap_err();
assert_eq!(
err,
SampleError::RuleContradiction {
tables: vec!["audit_logs".to_owned()]
}
);
}
#[test]
fn plan_refuses_a_stale_sample_rule() {
let (tables, keys) = schema();
let err = plan_of(
&[root("users", SampleAmount::Count(10))],
&SampleRules {
always_include: vec!["countries".to_owned(), "gone".to_owned()],
never_include: vec!["audit_logs".to_owned()],
},
&tables,
&keys,
)
.unwrap_err();
assert_eq!(
err,
SampleError::StaleRule {
tables: vec!["gone".to_owned()]
}
);
}
#[test]
fn plan_refuses_a_framework_table_in_the_sample_rules() {
let (tables, keys) = schema();
let err = build_plan(&SampleInputs {
roots: &[root("users", SampleAmount::Count(10))],
seed: 0,
rules: &SampleRules {
always_include: vec!["countries".to_owned()],
never_include: vec!["audit_logs".to_owned(), "autumn_jobs".to_owned()],
},
tables: &tables,
foreign_keys: &keys,
framework_tables: &BTreeSet::from(["autumn_jobs".to_owned()]),
purged: &BTreeSet::new(),
partitions: &BTreeMap::new(),
})
.unwrap_err();
assert_eq!(
err,
SampleError::FrameworkRule {
tables: vec!["autumn_jobs".to_owned()]
}
);
}
#[test]
fn plan_refuses_a_table_without_a_primary_key() {
let (mut tables, mut keys) = schema();
tables.push(table("logins", &[]));
keys.push(fk("logins_user_fk", "logins", "user_id", "users", "id"));
let err = plan_of(
&[root("users", SampleAmount::Count(10))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap_err();
assert_eq!(
err,
SampleError::NoRowKey {
tables: vec!["logins".to_owned()]
}
);
}
#[test]
fn plan_ignores_a_missing_primary_key_on_a_never_include_table() {
let (mut tables, keys) = schema();
tables.push(table("raw_events", &[]));
let plan = plan_of(
&[root("users", SampleAmount::Count(10))],
&SampleRules {
always_include: vec!["countries".to_owned()],
never_include: vec!["audit_logs".to_owned(), "raw_events".to_owned()],
},
&tables,
&keys,
)
.unwrap();
assert_eq!(role_of(&plan, "raw_events"), SampleRole::NeverInclude);
}
#[test]
fn plan_refuses_a_reference_cycle_between_tables() {
let (mut tables, mut keys) = schema();
tables.push(table("orgs", &["id"]));
keys.push(fk("users_org_fk", "users", "org_id", "orgs", "id"));
keys.push(fk("orgs_owner_fk", "orgs", "owner_id", "users", "id"));
let err = plan_of(
&[root("users", SampleAmount::Count(10))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap_err();
assert_eq!(
err,
SampleError::ForeignKeyCycle {
tables: vec!["orgs".to_owned(), "users".to_owned()]
}
);
}
#[test]
fn a_cycle_is_broken_by_copying_one_of_its_tables_whole() {
let (mut tables, mut keys) = schema();
tables.push(table("orgs", &["id"]));
keys.push(fk("users_org_fk", "users", "org_id", "orgs", "id"));
keys.push(fk("orgs_owner_fk", "orgs", "owner_id", "users", "id"));
let plan = plan_of(
&[root("users", SampleAmount::Count(10))],
&SampleRules {
always_include: vec!["countries".to_owned(), "orgs".to_owned()],
never_include: vec!["audit_logs".to_owned()],
},
&tables,
&keys,
)
.unwrap();
assert_eq!(role_of(&plan, "orgs"), SampleRole::AlwaysInclude);
assert!(
!joined(&plan.delete_statements()).contains("\"orgs\""),
"a full-copy table keeps every row, so it is never deleted from"
);
}
#[test]
fn plan_allows_a_self_referencing_table() {
let (tables, mut keys) = schema();
keys.push(fk("users_manager_fk", "users", "manager_id", "users", "id"));
let plan = plan_of(
&[root("users", SampleAmount::Count(10))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap();
assert!(
plan.walk_edges.iter().any(|e| e.name == "users_manager_fk"),
"a self-reference is still walked, so managers are pulled in"
);
}
#[test]
fn plan_refuses_a_table_outside_the_sample_that_references_a_sampled_one() {
let (tables, mut keys) = schema();
keys.push(fk("jobs_user_fk", "autumn_jobs", "user_id", "users", "id"));
let err = build_plan(&SampleInputs {
roots: &[root("users", SampleAmount::Count(10))],
seed: 0,
rules: &fixture_rules(),
tables: &tables,
foreign_keys: &keys,
framework_tables: &BTreeSet::from(["autumn_jobs".to_owned()]),
purged: &BTreeSet::new(),
partitions: &BTreeMap::new(),
})
.unwrap_err();
assert_eq!(
err,
SampleError::OutsideTableReferencesSampled {
edges: vec!["autumn_jobs -> users (jobs_user_fk)".to_owned()]
}
);
}
#[test]
fn plan_accepts_a_purged_framework_table_referencing_a_sampled_one() {
let (tables, mut keys) = schema();
keys.push(fk("jobs_user_fk", "autumn_jobs", "user_id", "users", "id"));
let plan = build_plan(&SampleInputs {
roots: &[root("users", SampleAmount::Count(10))],
seed: 0,
rules: &fixture_rules(),
tables: &tables,
foreign_keys: &keys,
framework_tables: &BTreeSet::from(["autumn_jobs".to_owned()]),
purged: &BTreeSet::from(["autumn_jobs".to_owned()]),
partitions: &BTreeMap::new(),
})
.unwrap();
assert!(
plan.walk_edges
.iter()
.all(|e| e.child_table != "autumn_jobs"),
"a framework table is never part of the walk"
);
}
#[test]
fn without_a_root_the_app_tables_are_uncovered() {
let (tables, keys) = schema();
let err = plan_of(&[], &fixture_rules(), &tables, &keys).unwrap_err();
assert_eq!(
err,
SampleError::UncoveredTables {
tables: vec!["comments".to_owned(), "users".to_owned()]
}
);
}
#[test]
fn a_purge_waits_for_the_sample_when_an_emptied_table_references_it() {
let (tables, mut keys) = schema();
keys.push(fk(
"audit_logs_job_fk",
"audit_logs",
"job_id",
"autumn_jobs",
"id",
));
let plan = build_plan(&SampleInputs {
roots: &[root("users", SampleAmount::Count(10))],
seed: 7,
rules: &fixture_rules(),
tables: &tables,
foreign_keys: &keys,
framework_tables: &BTreeSet::from(["autumn_jobs".to_owned()]),
purged: &BTreeSet::from(["autumn_jobs".to_owned()]),
partitions: &BTreeMap::new(),
})
.unwrap();
assert_eq!(
plan.purge_after,
BTreeSet::from(["autumn_jobs".to_owned()]),
"the purge its emptied child references must run after the sample"
);
}
#[test]
fn a_deferred_purge_drags_the_purges_it_references_with_it() {
let (tables, mut keys) = schema();
keys.push(fk(
"audit_logs_job_fk",
"audit_logs",
"job_id",
"autumn_jobs",
"id",
));
keys.push(fk(
"jobs_tracking_fk",
"autumn_jobs",
"tracking_id",
"autumn_job_tracking",
"id",
));
let framework =
BTreeSet::from(["autumn_jobs".to_owned(), "autumn_job_tracking".to_owned()]);
let plan = build_plan(&SampleInputs {
roots: &[root("users", SampleAmount::Count(10))],
seed: 7,
rules: &fixture_rules(),
tables: &tables,
foreign_keys: &keys,
framework_tables: &framework,
purged: &framework,
partitions: &BTreeMap::new(),
})
.unwrap();
assert_eq!(
plan.purge_after,
BTreeSet::from(["autumn_jobs".to_owned(), "autumn_job_tracking".to_owned()]),
"the referenced purge must be deferred alongside the one that needs it"
);
}
#[test]
fn a_propagated_deferral_that_contradicts_is_still_refused() {
let (tables, mut keys) = schema();
keys.push(fk(
"audit_logs_job_fk",
"audit_logs",
"job_id",
"autumn_jobs",
"id",
));
keys.push(fk(
"jobs_tracking_fk",
"autumn_jobs",
"tracking_id",
"autumn_job_tracking",
"id",
));
keys.push(fk(
"tracking_user_fk",
"autumn_job_tracking",
"user_id",
"users",
"id",
));
let framework =
BTreeSet::from(["autumn_jobs".to_owned(), "autumn_job_tracking".to_owned()]);
let err = build_plan(&SampleInputs {
roots: &[root("users", SampleAmount::Count(10))],
seed: 7,
rules: &fixture_rules(),
tables: &tables,
foreign_keys: &keys,
framework_tables: &framework,
purged: &framework,
partitions: &BTreeMap::new(),
})
.unwrap_err();
let SampleError::PurgeOrderContradiction { tables } = err else {
panic!("expected a purge-order contradiction, got {err:?}");
};
assert!(
tables.iter().any(|t| t.starts_with("autumn_job_tracking:")),
"the propagated deferral must be checked too: {tables:?}"
);
}
#[test]
fn a_hundred_percent_root_removes_nothing_so_an_outside_reference_is_fine() {
let (tables, mut keys) = schema();
keys.push(fk("jobs_user_fk", "autumn_jobs", "user_id", "users", "id"));
let plan = build_plan(&SampleInputs {
roots: &[root("users", SampleAmount::Percent(100.0))],
seed: 7,
rules: &fixture_rules(),
tables: &tables,
foreign_keys: &keys,
framework_tables: &BTreeSet::from(["autumn_jobs".to_owned()]),
purged: &BTreeSet::new(),
partitions: &BTreeMap::new(),
})
.unwrap();
assert_eq!(
role_of(&plan, "users"),
SampleRole::Root(SampleAmount::Percent(100.0))
);
}
#[test]
fn a_partial_root_still_refuses_the_same_outside_reference() {
let (tables, mut keys) = schema();
keys.push(fk("jobs_user_fk", "autumn_jobs", "user_id", "users", "id"));
let err = build_plan(&SampleInputs {
roots: &[root("users", SampleAmount::Percent(99.0))],
seed: 7,
rules: &fixture_rules(),
tables: &tables,
foreign_keys: &keys,
framework_tables: &BTreeSet::from(["autumn_jobs".to_owned()]),
purged: &BTreeSet::new(),
partitions: &BTreeMap::new(),
})
.unwrap_err();
assert!(
matches!(err, SampleError::OutsideTableReferencesSampled { .. }),
"a partial root must still be treated as removing rows: {err:?}"
);
}
#[test]
fn a_hundred_percent_root_still_descends_into_its_children() {
let (tables, keys) = schema();
let plan = build_plan(&SampleInputs {
roots: &[root("users", SampleAmount::Percent(100.0))],
seed: 7,
rules: &fixture_rules(),
tables: &tables,
foreign_keys: &keys,
framework_tables: &BTreeSet::new(),
purged: &BTreeSet::new(),
partitions: &BTreeMap::new(),
})
.unwrap();
assert_eq!(role_of(&plan, "comments"), SampleRole::Related);
}
#[test]
fn a_purge_needed_both_before_and_after_the_sample_is_refused() {
let (tables, mut keys) = schema();
keys.push(fk(
"audit_logs_job_fk",
"audit_logs",
"job_id",
"autumn_jobs",
"id",
));
keys.push(fk("jobs_user_fk", "autumn_jobs", "user_id", "users", "id"));
let err = build_plan(&SampleInputs {
roots: &[root("users", SampleAmount::Count(10))],
seed: 7,
rules: &fixture_rules(),
tables: &tables,
foreign_keys: &keys,
framework_tables: &BTreeSet::from(["autumn_jobs".to_owned()]),
purged: &BTreeSet::from(["autumn_jobs".to_owned()]),
partitions: &BTreeMap::new(),
})
.unwrap_err();
let SampleError::PurgeOrderContradiction { tables } = err else {
panic!("expected a purge-order contradiction, got {err:?}");
};
assert_eq!(tables.len(), 1);
assert!(tables[0].starts_with("autumn_jobs:"), "{tables:?}");
assert!(
tables[0].contains("jobs_user_fk") && tables[0].contains("audit_logs_job_fk"),
"the diagnostic must name both sides of the contradiction: {tables:?}"
);
}
#[test]
fn a_purge_that_only_references_a_sampled_table_still_runs_first() {
let (tables, mut keys) = schema();
keys.push(fk("jobs_user_fk", "autumn_jobs", "user_id", "users", "id"));
let plan = build_plan(&SampleInputs {
roots: &[root("users", SampleAmount::Count(10))],
seed: 7,
rules: &fixture_rules(),
tables: &tables,
foreign_keys: &keys,
framework_tables: &BTreeSet::from(["autumn_jobs".to_owned()]),
purged: &BTreeSet::from(["autumn_jobs".to_owned()]),
partitions: &BTreeMap::new(),
})
.unwrap();
assert!(plan.purge_after.is_empty());
}
#[test]
fn a_purge_a_retained_table_references_is_refused() {
let (tables, mut keys) = schema();
keys.push(fk(
"comments_job_fk",
"comments",
"job_id",
"autumn_jobs",
"id",
));
let err = build_plan(&SampleInputs {
roots: &[root("users", SampleAmount::Count(10))],
seed: 7,
rules: &fixture_rules(),
tables: &tables,
foreign_keys: &keys,
framework_tables: &BTreeSet::from(["autumn_jobs".to_owned()]),
purged: &BTreeSet::from(["autumn_jobs".to_owned()]),
partitions: &BTreeMap::new(),
})
.unwrap_err();
let SampleError::RetainedReferencesPurged { edges } = err else {
panic!("expected a retained-into-purged refusal, got {err:?}");
};
assert_eq!(edges.len(), 1);
assert!(edges[0].contains("comments_job_fk"), "{edges:?}");
}
#[test]
fn a_purge_nothing_sampled_references_still_runs_first() {
let (tables, keys) = schema();
let plan = build_plan(&SampleInputs {
roots: &[root("users", SampleAmount::Count(10))],
seed: 7,
rules: &fixture_rules(),
tables: &tables,
foreign_keys: &keys,
framework_tables: &BTreeSet::from(["autumn_jobs".to_owned()]),
purged: &BTreeSet::from(["autumn_jobs".to_owned()]),
partitions: &BTreeMap::new(),
})
.unwrap();
assert!(plan.purge_after.is_empty());
}
#[test]
fn a_partition_local_foreign_key_is_refused_rather_than_dropped() {
let (tables, mut keys) = schema();
keys.push(fk(
"comments_2026_local_fk",
"comments_2026",
"user_id",
"users",
"id",
));
let err = build_plan(&SampleInputs {
roots: &[root("users", SampleAmount::Count(10))],
seed: 7,
rules: &fixture_rules(),
tables: &tables,
foreign_keys: &keys,
framework_tables: &BTreeSet::new(),
purged: &BTreeSet::new(),
partitions: &BTreeMap::from([("comments_2026".to_owned(), "comments".to_owned())]),
})
.unwrap_err();
let SampleError::PartitionLocalForeignKey { edges } = err else {
panic!("expected a partition-local refusal, got {err:?}");
};
assert_eq!(edges.len(), 1);
assert!(edges[0].contains("comments_2026_local_fk"), "{edges:?}");
}
#[test]
fn a_partition_with_no_local_foreign_key_plans_through_its_parent() {
let (tables, keys) = schema();
let plan = build_plan(&SampleInputs {
roots: &[root("users", SampleAmount::Count(10))],
seed: 7,
rules: &fixture_rules(),
tables: &tables,
foreign_keys: &keys,
framework_tables: &BTreeSet::new(),
purged: &BTreeSet::new(),
partitions: &BTreeMap::from([("comments_2026".to_owned(), "comments".to_owned())]),
})
.unwrap();
assert!(
plan.walk_edges
.iter()
.all(|e| e.child_table != "comments_2026"),
"a partition's cloned constraint must not be walked"
);
assert!(
plan.verify_edges
.iter()
.all(|e| e.child_table != "comments_2026"),
"nor verified twice"
);
}
#[test]
fn deletes_run_children_before_parents() {
let (tables, keys) = schema();
let plan = plan_of(
&[root("users", SampleAmount::Count(10))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap();
let order: Vec<&str> = plan.tables.iter().map(|t| t.table.as_str()).collect();
let at = |name: &str| order.iter().position(|t| *t == name).unwrap();
assert!(
at("comments") < at("users"),
"a child must be emptied before its parent: {order:?}"
);
assert!(
at("users") < at("countries"),
"the lookup table's parents come last: {order:?}"
);
}
#[test]
fn root_seeding_is_deterministic_bounded_and_seeded() {
let (tables, keys) = schema();
let plan = plan_of(
&[root("users", SampleAmount::Percent(1.0))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap();
let sql = joined(&plan.seed_statements(&BTreeMap::from([
("users".to_owned(), 1000_i64),
("countries".to_owned(), 12),
])));
assert!(sql.contains("md5('7' || '|'"), "seeded order key: {sql}");
assert!(sql.contains("ORDER BY"), "{sql}");
assert!(sql.contains("LIMIT 10"), "1% of 1000 rows: {sql}");
assert!(
sql.contains("FROM \"public\".\"countries\""),
"always-include tables are seeded in full: {sql}"
);
}
#[test]
fn a_different_seed_changes_the_order_key() {
let (tables, keys) = schema();
let counts = BTreeMap::from([("users".to_owned(), 100_i64), ("countries".to_owned(), 1)]);
let with_seed = |seed: u64| {
build_plan(&SampleInputs {
roots: &[root("users", SampleAmount::Count(5))],
seed,
rules: &fixture_rules(),
tables: &tables,
foreign_keys: &keys,
framework_tables: &BTreeSet::new(),
purged: &BTreeSet::new(),
partitions: &BTreeMap::new(),
})
.unwrap()
.seed_statements(&counts)
};
assert_ne!(joined(&with_seed(1)), joined(&with_seed(2)));
}
#[test]
fn a_never_include_table_is_never_seeded() {
let (tables, keys) = schema();
let plan = plan_of(
&[root("users", SampleAmount::Count(10))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap();
let sql = joined(&plan.seed_statements(&BTreeMap::new()));
assert!(
!sql.contains("audit_logs"),
"an excluded table selects no rows at all: {sql}"
);
let deletes = joined(&plan.delete_statements());
assert!(
deletes.contains("DELETE FROM \"public\".\"audit_logs\""),
"an excluded table is emptied: {deletes}"
);
}
#[test]
fn an_always_include_table_is_never_deleted_from() {
let (tables, keys) = schema();
let plan = plan_of(
&[root("users", SampleAmount::Count(10))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap();
let deletes = joined(&plan.delete_statements());
assert!(
!deletes.contains("\"countries\""),
"a full-copy table keeps every row: {deletes}"
);
}
#[test]
fn statements_are_schema_qualified() {
let (tables, keys) = schema();
let plan = plan_of(
&[root("users", SampleAmount::Count(10))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap();
let all = format!(
"{}\n{}\n{}",
joined(&plan.seed_statements(&BTreeMap::from([("users".to_owned(), 10_i64)]))),
joined(&plan.walk_statements()),
joined(&plan.delete_statements()),
);
for (name, _) in &tables {
let quoted = quote_ident(name);
assert_eq!(
all.matches("ed).count(),
all.matches(&format!("\"public\".{quoted}")).count(),
"every reference to {name} must be public-qualified: {all}"
);
}
}
#[test]
fn an_integrity_label_cannot_break_out_of_a_sql_comment() {
let (tables, _) = schema();
let hostile = "comments_user_fk\nCOMMIT; SELECT 'INJECTED'; --";
let keys = vec![
fk(hostile, "comments", "user_id", "users", "id"),
fk("users_country_fk", "users", "country_id", "countries", "id"),
];
let plan = plan_of(
&[root("users", SampleAmount::Count(10))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap();
let labels: Vec<String> = plan
.integrity_statements()
.into_iter()
.map(|(label, _)| label)
.collect();
assert!(
!labels.is_empty(),
"the fixture must produce a label to sanitize"
);
for label in &labels {
assert!(
!label.contains('\n') && !label.contains('\r'),
"a label reaching a `--` comment must stay on one line: {label:?}"
);
}
assert!(
labels.iter().any(|l| l.starts_with("comments_user_fk")),
"and must still identify the constraint: {labels:?}"
);
}
#[test]
fn the_printed_walk_is_a_loop_that_repeats_every_statement() {
let (tables, keys) = schema();
let plan = plan_of(
&[root("users", SampleAmount::Count(10))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap();
let walk = plan.walk_statements();
let analyze = plan.analyze_statements();
let loop_sql = plan.walk_loop_statement();
assert!(
!walk.is_empty(),
"the fixture must have a walk to loop over"
);
for statement in &walk {
assert!(
loop_sql.contains(&format!(" EXECUTE {};", quote_literal(statement))),
"the loop must run {statement} through EXECUTE:\n{loop_sql}"
);
}
for statement in &analyze {
assert!(
loop_sql.contains(statement),
"the loop must re-analyze per pass, like `apply`: {statement}"
);
}
assert_eq!(
loop_sql
.matches("GET DIAGNOSTICS moved = ROW_COUNT;")
.count(),
walk.len(),
"each statement's own row count must be captured:\n{loop_sql}"
);
assert!(
loop_sql.contains("EXIT WHEN total = 0;"),
"the loop must end on a pass that selects nothing:\n{loop_sql}"
);
assert!(
loop_sql.contains(&format!("IF passes >= {MAX_PASSES} THEN")),
"the printed loop must carry the same ceiling `apply` does:\n{loop_sql}"
);
}
#[test]
fn the_walk_loop_is_dollar_quoted_past_any_identifier() {
assert_eq!(dollar_tag("INSERT INTO x"), "$autumn_walk$");
assert_eq!(
dollar_tag(r#"INSERT INTO "a$autumn_walk$b""#),
"$autumn_walk_$"
);
assert_eq!(
dollar_tag(r#""a$autumn_walk$b" "c$autumn_walk_$d""#),
"$autumn_walk__$"
);
let body = r#"x "a$autumn_walk$b""#;
assert!(
!body.contains(&dollar_tag(body)),
"the chosen tag must not appear in the body it quotes"
);
}
#[test]
fn the_walk_ascends_into_the_parent_and_descends_into_the_child() {
let (tables, keys) = schema();
let plan = plan_of(
&[root("users", SampleAmount::Count(10))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap();
let set_of = |table: &str| {
let t = plan.tables.iter().find(|t| t.table == table).unwrap();
(t.keep.clone(), t.descend.clone())
};
let (users_keep, users_descend) = set_of("users");
let (comments_keep, comments_descend) = set_of("comments");
let walk = plan.walk_statements();
assert!(
walk.iter().any(|s| s.starts_with(&format!(
"/* comments_user_fk */ INSERT INTO {}",
quote_ident(&users_keep)
)) && s.contains("e_ident(&frontier_of(&comments_keep)))),
"the ascend must fill users' keep set from comments': {walk:?}"
);
for target in [&comments_keep, &comments_descend] {
assert!(
walk.iter().any(|s| s.starts_with("/* comments_user_fk */ ")
&& s.contains(&format!(
"INSERT INTO {} (k) SELECT DISTINCT",
quote_ident(target)
))
&& s.contains("e_ident(&frontier_of(&users_descend)))),
"the descend must fill {target} from users' descend set: {walk:?}"
);
}
assert!(
!walk.iter().any(|s| s.contains(&format!(
"INSERT INTO {} (k) SELECT DISTINCT c.",
quote_ident(&users_descend)
))),
"ascent must never make a row descend-eligible: {walk:?}"
);
}
#[test]
fn the_walk_never_descends_out_of_an_always_include_table() {
let (tables, keys) = schema();
let plan = plan_of(
&[root("users", SampleAmount::Count(10))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap();
let countries = plan
.tables
.iter()
.find(|t| t.table == "countries")
.unwrap()
.descend
.clone();
let sql = joined(&plan.walk_statements());
assert!(
!sql.contains(&countries),
"no statement may read or write the lookup table's descend set: {sql}"
);
}
#[test]
fn a_composite_primary_key_becomes_an_unambiguous_row_key() {
let tables = vec![
table("users", &["id"]),
table("tags", &["id"]),
table("user_tags", &["user_id", "tag_id"]),
];
let keys = vec![
fk("user_tags_user_fk", "user_tags", "user_id", "users", "id"),
fk("user_tags_tag_fk", "user_tags", "tag_id", "tags", "id"),
];
let plan = plan_of(
&[root("users", SampleAmount::Count(10))],
&SampleRules::default(),
&tables,
&keys,
)
.unwrap();
let sql = joined(&plan.delete_statements());
assert!(
sql.contains("quote_nullable"),
"a composite key must be rendered unambiguously: {sql}"
);
}
#[test]
fn a_composite_foreign_key_joins_on_every_component() {
let tables = vec![
table("orders", &["tenant_id", "id"]),
table("order_lines", &["id"]),
];
let keys = vec![ForeignKeyConstraint {
name: "order_lines_order_fk".to_owned(),
child_table: "order_lines".to_owned(),
child_columns: vec!["tenant_id".to_owned(), "order_id".to_owned()],
parent_table: "orders".to_owned(),
parent_columns: vec!["tenant_id".to_owned(), "id".to_owned()],
match_full: false,
}];
let plan = plan_of(
&[root("orders", SampleAmount::Count(10))],
&SampleRules::default(),
&tables,
&keys,
)
.unwrap();
let sql = joined(&plan.walk_statements());
assert!(
sql.contains("p.\"tenant_id\" = c.\"tenant_id\" AND p.\"id\" = c.\"order_id\""),
"the join must pair every component in key order: {sql}"
);
}
#[test]
fn integrity_checks_cover_every_foreign_key() {
let (tables, keys) = schema();
let plan = plan_of(
&[root("users", SampleAmount::Count(10))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap();
let checks = plan.integrity_statements();
assert_eq!(checks.len(), 2, "one check per foreign key");
for (_, sql) in plan.integrity_statements() {
assert!(sql.contains("count(*)"), "{sql}");
assert!(
sql.contains("IS NULL"),
"an orphan is a missing parent: {sql}"
);
}
}
#[test]
fn a_full_copy_child_of_an_outside_parent_is_still_verified() {
let (tables, mut keys) = schema();
keys.push(fk(
"countries_job_fk",
"countries",
"job_id",
"autumn_jobs",
"id",
));
let plan = build_plan(&SampleInputs {
roots: &[root("users", SampleAmount::Count(10))],
seed: 7,
rules: &fixture_rules(),
tables: &tables,
foreign_keys: &keys,
framework_tables: &BTreeSet::from(["autumn_jobs".to_owned()]),
purged: &BTreeSet::new(),
partitions: &BTreeMap::new(),
})
.unwrap();
assert!(
plan.verify_edges
.iter()
.any(|e| e.name == "countries_job_fk"),
"a full-copy child's outside reference must still be re-counted"
);
assert!(
plan.walk_edges.iter().all(|e| e.name != "countries_job_fk"),
"but it must not enter the walk"
);
}
#[test]
fn an_excluded_partition_leaf_still_defers_the_purge_it_references() {
let (tables, mut keys) = schema();
keys.push(fk(
"audit_p0_job_fk",
"audit_logs_p0",
"job_id",
"autumn_jobs",
"id",
));
let partitions = BTreeMap::from([("audit_logs_p0".to_owned(), "audit_logs".to_owned())]);
let purged = BTreeSet::from(["autumn_jobs".to_owned()]);
let plan = build_plan(&SampleInputs {
roots: &[root("users", SampleAmount::Count(10))],
seed: 7,
rules: &fixture_rules(),
tables: &tables,
foreign_keys: &keys,
framework_tables: &purged,
purged: &purged,
partitions: &partitions,
})
.expect("an excluded leaf's key into a purged table must not refuse the run");
assert!(
plan.purge_after.contains("autumn_jobs"),
"the purge must wait until the sample has emptied the leaf: {:?}",
plan.purge_after
);
assert!(
plan.walk_edges.iter().all(|e| e.name != "audit_p0_job_fk"),
"and the edge still has nothing to say to the walk"
);
}
#[test]
fn an_outgoing_key_from_an_excluded_partition_leaf_is_not_refused() {
let (tables, mut keys) = schema();
keys.push(fk(
"leaf_lookup_fk",
"audit_logs_p0",
"country_id",
"countries",
"id",
));
let partitions = BTreeMap::from([("audit_logs_p0".to_owned(), "audit_logs".to_owned())]);
let plan = build_plan(&SampleInputs {
roots: &[root("users", SampleAmount::Count(10))],
seed: 7,
rules: &fixture_rules(),
tables: &tables,
foreign_keys: &keys,
framework_tables: &BTreeSet::new(),
purged: &BTreeSet::new(),
partitions: &partitions,
})
.expect("an excluded leaf's outgoing key must not be refused");
assert!(
plan.walk_edges.iter().all(|e| e.name != "leaf_lookup_fk"),
"and it must not enter the walk either"
);
let mut kept = fixture_rules();
kept.never_include.clear();
let err = build_plan(&SampleInputs {
roots: &[root("users", SampleAmount::Count(10))],
seed: 7,
rules: &kept,
tables: &tables,
foreign_keys: &keys,
framework_tables: &BTreeSet::new(),
purged: &BTreeSet::new(),
partitions: &partitions,
})
.unwrap_err();
assert!(
matches!(err, SampleError::PartitionLocalForeignKey { .. }),
"a leaf whose rows survive must still be refused: {err}"
);
}
#[test]
fn an_exact_100_percent_root_is_not_deleted_from() {
let (tables, keys) = schema();
let plan = plan_of(
&[root("users", SampleAmount::Percent(100.0))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap();
let deletes = plan.delete_statements();
assert!(
!deletes.iter().any(|s| s.contains(r#""public"."users""#)),
"a root that removes nothing must not be deleted from: {deletes:?}"
);
assert!(
deletes.iter().any(|s| s.contains(r#""public"."comments""#)),
"but a genuinely subsetted table must be: {deletes:?}"
);
}
#[test]
fn a_key_whose_sides_disagree_on_arity_is_refused() {
let (tables, _) = schema();
let mut lopsided = fk("comments_user_fk", "comments", "user_id", "users", "id");
lopsided.child_columns.push("body".to_owned());
let err = plan_of(
&[root("users", SampleAmount::Count(10))],
&fixture_rules(),
&tables,
&[lopsided],
)
.unwrap_err();
let message = err.to_string();
assert!(
matches!(err, SampleError::KeyArityMismatch { .. }),
"a lopsided key must be refused, not zipped short: {message}"
);
assert!(
message.contains("comments_user_fk") && message.contains("2 vs 1"),
"and the refusal must name the key and both counts: {message}"
);
}
#[test]
fn an_outside_child_of_a_full_copy_parent_is_still_verified() {
let (tables, mut keys) = schema();
keys.push(fk(
"job_country_fk",
"autumn_jobs",
"country_id",
"countries",
"id",
));
let plan = build_plan(&SampleInputs {
roots: &[root("users", SampleAmount::Count(10))],
seed: 7,
rules: &fixture_rules(),
tables: &tables,
foreign_keys: &keys,
framework_tables: &BTreeSet::from(["autumn_jobs".to_owned()]),
purged: &BTreeSet::new(),
partitions: &BTreeMap::new(),
})
.unwrap();
assert!(
plan.verify_edges.iter().any(|e| e.name == "job_country_fk"),
"an outside child of a full-copy parent must be re-counted: {:?}",
plan.verify_edges
.iter()
.map(|e| &e.name)
.collect::<Vec<_>>()
);
assert!(
plan.walk_edges.iter().all(|e| e.name != "job_country_fk"),
"but it must not enter the walk — nothing outside the universe is sampled"
);
}
#[test]
fn the_recounts_own_endpoints_are_locked_and_rls_checked() {
let (tables, mut keys) = schema();
keys.push(fk(
"job_country_fk",
"autumn_jobs",
"country_id",
"countries",
"id",
));
let plan = build_plan(&SampleInputs {
roots: &[root("users", SampleAmount::Count(10))],
seed: 7,
rules: &fixture_rules(),
tables: &tables,
foreign_keys: &keys,
framework_tables: &BTreeSet::from(["autumn_jobs".to_owned()]),
purged: &BTreeSet::new(),
partitions: &BTreeMap::new(),
})
.unwrap();
let locked = plan.locked_tables();
assert!(
locked.contains(&"autumn_jobs"),
"an outside endpoint of the re-count must be locked and RLS-checked: {locked:?}"
);
for table in &plan.tables {
assert!(
locked.contains(&table.table.as_str()),
"{} must stay locked: {locked:?}",
table.table
);
}
let mut sorted = locked.clone();
sorted.sort_unstable();
sorted.dedup();
assert_eq!(sorted, locked, "the list must be sorted and deduplicated");
}
#[test]
fn a_purged_child_of_a_full_copy_parent_needs_no_verification() {
let (tables, mut keys) = schema();
keys.push(fk(
"job_country_fk",
"autumn_jobs",
"country_id",
"countries",
"id",
));
let plan = build_plan(&SampleInputs {
roots: &[root("users", SampleAmount::Count(10))],
seed: 7,
rules: &fixture_rules(),
tables: &tables,
foreign_keys: &keys,
framework_tables: &BTreeSet::from(["autumn_jobs".to_owned()]),
purged: &BTreeSet::from(["autumn_jobs".to_owned()]),
partitions: &BTreeMap::new(),
})
.unwrap();
assert!(
plan.verify_edges.iter().all(|e| e.name != "job_country_fk"),
"a purged child's references need no re-count"
);
}
#[test]
fn an_emptied_child_of_an_outside_parent_needs_no_verification() {
let (tables, mut keys) = schema();
keys.push(fk(
"audit_job_fk",
"audit_logs",
"job_id",
"autumn_jobs",
"id",
));
let plan = build_plan(&SampleInputs {
roots: &[root("users", SampleAmount::Count(10))],
seed: 7,
rules: &fixture_rules(),
tables: &tables,
foreign_keys: &keys,
framework_tables: &BTreeSet::from(["autumn_jobs".to_owned()]),
purged: &BTreeSet::new(),
partitions: &BTreeMap::new(),
})
.unwrap();
assert!(
plan.verify_edges.iter().all(|e| e.name != "audit_job_fk"),
"an emptied table's references need no re-count"
);
}
#[test]
fn a_match_full_composite_key_also_counts_partially_null_tuples() {
let (tables, mut keys) = schema();
keys[0].child_columns = vec!["user_id".to_owned(), "tenant_id".to_owned()];
keys[0].parent_columns = vec!["id".to_owned(), "tenant_id".to_owned()];
keys[0].match_full = true;
keys[0].name = "comments_user_full_fk".to_owned();
let plan = plan_of(
&[root("users", SampleAmount::Count(10))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap();
let (_, sql) = plan
.integrity_statements()
.into_iter()
.find(|(label, _)| label.starts_with("comments_user_full_fk"))
.expect("the composite key must be checked");
assert!(
sql.contains(r#""user_id" IS NULL"#) && sql.contains(r#""tenant_id" IS NULL"#),
"a partially-NULL tuple must be counted too: {sql}"
);
}
#[test]
fn a_match_simple_composite_key_ignores_partially_null_tuples() {
let (tables, mut keys) = schema();
keys[0].child_columns = vec!["user_id".to_owned(), "tenant_id".to_owned()];
keys[0].parent_columns = vec!["id".to_owned(), "tenant_id".to_owned()];
keys[0].name = "comments_user_simple_fk".to_owned();
let plan = plan_of(
&[root("users", SampleAmount::Count(10))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap();
let (_, sql) = plan
.integrity_statements()
.into_iter()
.find(|(label, _)| label.starts_with("comments_user_simple_fk"))
.expect("the composite key must be checked");
assert!(
!sql.contains("IS NULL) AND ("),
"MATCH SIMPLE must not gain the mixed-NULL arm: {sql}"
);
}
#[test]
fn an_excluded_table_without_a_primary_key_builds_no_set() {
let (mut tables, keys) = schema();
tables.push(table("request_logs", &[]));
let plan = plan_of(
&[root("users", SampleAmount::Count(10))],
&SampleRules {
always_include: vec!["countries".to_owned()],
never_include: vec!["audit_logs".to_owned(), "request_logs".to_owned()],
},
&tables,
&keys,
)
.unwrap();
let setup = joined(&plan.setup_statements());
assert!(
!setup.contains("request_logs") && !setup.contains("audit_logs"),
"an excluded table gets no keep set: {setup}"
);
assert!(
!setup.contains("SELECT AS k") && !setup.contains("SELECT () AS k"),
"no statement may carry an empty row key: {setup}"
);
assert!(
joined(&plan.delete_statements()).contains("DELETE FROM \"public\".\"request_logs\""),
"it is still emptied"
);
}
#[test]
fn a_constraint_name_cannot_escape_its_sql_comment() {
let (tables, mut keys) = schema();
keys.push(fk(
"*/WITH z AS(INSERT INTO loot SELECT 1 RETURNING 1)/*",
"comments",
"user_id",
"users",
"id",
));
let plan = plan_of(
&[root("users", SampleAmount::Count(10))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap();
for statement in plan.walk_statements() {
let Some((tag, _)) = statement
.strip_prefix("/* ")
.and_then(|rest| rest.split_once(" */ "))
else {
panic!("every walk statement opens with a tag: {statement}");
};
assert!(
!tag.contains('*') && !tag.contains('/'),
"the tag can only be closed by the one that follows it: {tag:?}"
);
}
assert_eq!(
comment_safe("*/DROP TABLE users;--"),
"__DROP TABLE users_--"
);
}
#[test]
fn sets_are_indexed_and_analysed_after_the_roots_are_seeded() {
let (tables, keys) = schema();
let plan = plan_of(
&[root("users", SampleAmount::Count(10))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap();
let index = joined(&plan.index_statements());
assert!(
index.contains("CREATE INDEX ON") && index.contains("ANALYZE "),
"{index}"
);
assert!(
!joined(&plan.setup_statements()).contains("CREATE INDEX"),
"the index must not be built before the rows arrive"
);
assert!(
plan.analyze_statements()
.iter()
.all(|s| s.starts_with("ANALYZE ")),
"each pass re-analyses the sets it just grew"
);
}
#[test]
fn setup_creates_a_keep_and_descend_set_per_table() {
let (tables, keys) = schema();
let plan = plan_of(
&[root("users", SampleAmount::Count(10))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap();
let sql = joined(&plan.setup_statements());
let keyed = plan
.tables
.iter()
.filter(|t| t.role != SampleRole::NeverInclude)
.count();
assert_eq!(
sql.matches("CREATE TEMPORARY TABLE").count(),
keyed * 2 + plan.frontier_sets().len() + plan.next_sets().len(),
"{sql}"
);
assert!(sql.contains("ON COMMIT DROP"), "{sql}");
}
fn self_referential_plan() -> SamplePlan {
let (tables, mut keys) = schema();
keys.push(fk("users_manager_fk", "users", "manager_id", "users", "id"));
plan_of(
&[root("users", SampleAmount::Count(10))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap()
}
fn sets_of(plan: &SamplePlan, table: &str) -> (String, String) {
let t = plan.tables.iter().find(|t| t.table == table).unwrap();
(t.keep.clone(), t.descend.clone())
}
#[test]
fn the_walk_joins_the_frontier_not_the_full_set() {
let (tables, keys) = schema();
let plan = plan_of(
&[root("users", SampleAmount::Count(10))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap();
let (comments_keep, _) = sets_of(&plan, "comments");
let (_, users_descend) = sets_of(&plan, "users");
let sql = joined(&plan.walk_statements());
assert!(
sql.contains(&format!(
"JOIN {} AS ck",
quote_ident(&frontier_of(&comments_keep))
)),
"the ascend must read the child's keep frontier: {sql}"
);
assert!(
sql.contains(&format!(
"JOIN {} AS pd",
quote_ident(&frontier_of(&users_descend))
)),
"the descend must read the parent's descend frontier: {sql}"
);
for full in [&comments_keep, &users_descend] {
assert!(
!sql.contains(&format!("JOIN {} AS", quote_ident(full))),
"no pass may join the full set {full}: {sql}"
);
}
}
#[test]
fn only_the_sets_a_pass_reads_get_a_frontier() {
let (tables, keys) = schema();
let plan = plan_of(
&[root("users", SampleAmount::Count(10))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap();
let (comments_keep, _) = sets_of(&plan, "comments");
let (_, users_descend) = sets_of(&plan, "users");
let mut expected = vec![comments_keep.clone(), users_descend.clone()];
expected.sort();
assert_eq!(plan.frontier_sets(), expected);
assert_eq!(plan.next_sets(), vec![comments_keep.clone()]);
let setup = joined(&plan.setup_statements());
let keyed = plan
.tables
.iter()
.filter(|t| t.role != SampleRole::NeverInclude)
.count();
assert_eq!(
setup.matches("CREATE TEMPORARY TABLE").count(),
keyed * 2 + 3,
"a keep and descend set per table, two frontiers, one next set: {setup}"
);
let created = |name: &str| {
setup.contains(&format!(
"CREATE TEMPORARY TABLE {} ON COMMIT DROP",
quote_ident(name)
))
};
assert!(created(&frontier_of(&comments_keep)), "{setup}");
assert!(created(&next_of(&comments_keep)), "{setup}");
assert!(created(&frontier_of(&users_descend)), "{setup}");
assert!(!created(&next_of(&users_descend)), "{setup}");
}
#[test]
fn every_insert_into_a_read_set_also_fills_its_frontier_and_next() {
let plan = self_referential_plan();
let walk = plan.walk_statements();
let frontiers = plan.frontier_sets();
assert_eq!(frontiers.len(), 3, "users' two sets and comments' keep set");
for set in &frontiers {
let into = format!("INSERT INTO {} (k)", quote_ident(set));
let writers: Vec<&String> = walk.iter().filter(|s| s.contains(&into)).collect();
assert!(!writers.is_empty(), "the fixture must write {set}");
for statement in writers {
for name in [frontier_of(set), next_of(set)] {
assert!(
statement.contains(&format!(
"INSERT INTO {} (k) SELECT k FROM added",
quote_ident(&name)
)),
"an insert into {set} must also fill {name}: {statement}"
);
}
}
}
}
#[test]
fn an_insert_into_an_unread_set_writes_only_that_set() {
let (tables, keys) = schema();
let plan = plan_of(
&[root("users", SampleAmount::Count(10))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap();
let (users_keep, _) = sets_of(&plan, "users");
let walk = plan.walk_statements();
let ascend = walk
.iter()
.find(|s| s.contains(&format!("INSERT INTO {} (k)", quote_ident(&users_keep))))
.expect("the fixture ascends into users");
assert!(
!ascend.contains("WITH added AS"),
"users' keep set has no frontier, so nothing else is written: {ascend}"
);
}
#[test]
fn the_frontier_starts_as_the_seeded_set() {
let plan = self_referential_plan();
let seed = plan.seed_statements(&BTreeMap::new());
let frontier_names: Vec<String> = plan
.frontier_sets()
.iter()
.map(|set| quote_ident(&frontier_of(set)))
.collect();
let last_root_seed = seed
.iter()
.rposition(|s| !frontier_names.iter().any(|f| s.contains(f.as_str())))
.expect("roots are seeded");
for set in plan.frontier_sets() {
let copy = format!(
"INSERT INTO {} (k) SELECT k FROM {}",
quote_ident(&frontier_of(&set)),
quote_ident(&set)
);
let at = seed
.iter()
.position(|s| *s == copy)
.unwrap_or_else(|| panic!("{copy} is missing: {seed:?}"));
assert!(at > last_root_seed, "copy only after the seeds: {seed:?}");
}
let index = joined(&plan.index_statements());
for set in plan.frontier_sets() {
assert!(
index.contains(&format!("ANALYZE {}", quote_ident(&frontier_of(&set)))),
"the first pass must plan against the seeded frontier: {index}"
);
}
}
#[test]
fn each_pass_moves_what_it_added_into_the_frontier() {
let plan = self_referential_plan();
let rotate = plan.rotate_statements();
assert_eq!(
plan.next_sets(),
plan.frontier_sets(),
"every frontier of the fixture also grows during the walk"
);
let mut expected = Vec::new();
for set in plan.frontier_sets() {
let frontier = quote_ident(&frontier_of(&set));
let next = quote_ident(&next_of(&set));
expected.push(format!("TRUNCATE {frontier}"));
expected.push(format!("INSERT INTO {frontier} (k) SELECT k FROM {next}"));
expected.push(format!("TRUNCATE {next}"));
}
assert!(!expected.is_empty(), "the fixture must have a frontier");
assert_eq!(rotate, expected);
let analyze = plan.analyze_statements();
for set in plan.frontier_sets() {
assert!(
analyze.contains(&format!("ANALYZE {}", quote_ident(&frontier_of(&set)))),
"each pass must plan against the new frontier: {analyze:?}"
);
}
}
#[test]
fn the_printed_loop_moves_the_frontier_between_the_walk_and_the_analyze() {
let plan = self_referential_plan();
let loop_sql = plan.walk_loop_statement();
let walk = plan.walk_statements();
let rotate = plan.rotate_statements();
let last_walk = walk
.iter()
.map(|s| loop_sql.find(s.as_str()).expect("walk is in the loop"))
.max()
.unwrap();
let first_analyze = plan
.analyze_statements()
.iter()
.map(|s| {
loop_sql
.rfind(&format!(" {s};"))
.expect("analyze is in the loop")
})
.min()
.unwrap();
assert!(!rotate.is_empty());
for statement in &rotate {
let at = loop_sql
.find(&format!(" {statement};"))
.unwrap_or_else(|| panic!("the loop must carry {statement}:\n{loop_sql}"));
assert!(
at > last_walk && at < first_analyze,
"{statement} must run after the walk and before the analyze:\n{loop_sql}"
);
}
assert_eq!(
loop_sql
.matches("GET DIAGNOSTICS moved = ROW_COUNT;")
.count(),
walk.len(),
"{loop_sql}"
);
}
#[test]
fn apply_and_the_printed_loop_share_one_end_of_pass_list() {
let plan = self_referential_plan();
let mut expected = plan.rotate_statements();
expected.extend(plan.analyze_statements());
assert_eq!(plan.end_of_pass_statements(), expected);
let loop_sql = plan.walk_loop_statement();
let body: String = expected
.iter()
.fold(String::new(), |body, s| body + " " + s + ";\n");
assert!(
loop_sql.contains(&body),
"the loop must run the end-of-pass list in order:\n{loop_sql}"
);
}
#[test]
fn a_full_copy_child_seeds_its_whole_keep_set_into_the_frontier() {
let (mut tables, mut keys) = schema();
tables.push(table("user_flags", &["id"]));
keys.push(fk(
"user_flags_user_fk",
"user_flags",
"user_id",
"users",
"id",
));
let rules = SampleRules {
always_include: vec!["countries".to_owned(), "user_flags".to_owned()],
never_include: vec!["audit_logs".to_owned()],
};
let plan = plan_of(
&[root("users", SampleAmount::Count(10))],
&rules,
&tables,
&keys,
)
.unwrap();
let (flags_keep, flags_descend) = sets_of(&plan, "user_flags");
assert!(plan.frontier_sets().contains(&flags_keep));
assert!(
!plan.frontier_sets().contains(&flags_descend),
"nothing descends out of a full-copy table"
);
let seed = plan.seed_statements(&BTreeMap::new());
let fill = seed
.iter()
.position(|s| {
s.starts_with(&format!(
"INSERT INTO {} (k) SELECT",
quote_ident(&flags_keep)
))
})
.expect("the full copy is seeded");
let copy = seed
.iter()
.position(|s| {
*s == format!(
"INSERT INTO {} (k) SELECT k FROM {}",
quote_ident(&frontier_of(&flags_keep)),
quote_ident(&flags_keep)
)
})
.expect("the frontier copies the full keep set");
assert!(copy > fill, "{seed:?}");
assert!(
joined(&plan.walk_statements()).contains(&format!(
"JOIN {} AS ck",
quote_ident(&frontier_of(&flags_keep))
)),
"the ascend out of the full copy reads its frontier"
);
}
#[test]
fn a_frontier_only_the_seed_fills_is_emptied_after_the_first_pass() {
let (tables, keys) = schema();
let plan = plan_of(
&[root("users", SampleAmount::Count(10))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap();
let (_, users_descend) = sets_of(&plan, "users");
let frontier = quote_ident(&frontier_of(&users_descend));
let rotate = plan.rotate_statements();
assert!(
rotate.contains(&format!("TRUNCATE {frontier}")),
"{rotate:?}"
);
assert!(
!rotate
.iter()
.any(|s| s.starts_with(&format!("INSERT INTO {frontier}"))),
"nothing refills it: {rotate:?}"
);
}
#[test]
fn the_walk_does_not_descend_from_a_parent_no_root_reaches() {
let (mut tables, mut keys) = schema();
tables.push(table("orgs", &["id"]));
keys.push(fk("users_org_fk", "users", "org_id", "orgs", "id"));
let plan = plan_of(
&[root("users", SampleAmount::Count(10))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap();
let (orgs_keep, orgs_descend) = sets_of(&plan, "orgs");
let walk = plan.walk_statements();
assert!(
walk.iter()
.any(|s| s.contains(&format!("INSERT INTO {} (k)", quote_ident(&orgs_keep)))),
"users still ascend into orgs: {walk:?}"
);
assert!(
!walk
.iter()
.any(|s| s.contains("e_ident(&frontier_of(&orgs_descend)))),
"no statement descends from orgs: {walk:?}"
);
assert!(!plan.frontier_sets().contains(&orgs_descend));
}
#[test]
fn a_descend_set_reached_through_a_related_table_still_walks() {
let (mut tables, mut keys) = schema();
tables.push(table("replies", &["id"]));
keys.push(fk(
"replies_comment_fk",
"replies",
"comment_id",
"comments",
"id",
));
let plan = plan_of(
&[root("users", SampleAmount::Count(10))],
&fixture_rules(),
&tables,
&keys,
)
.unwrap();
let (_, comments_descend) = sets_of(&plan, "comments");
assert!(plan.next_sets().contains(&comments_descend));
assert!(
joined(&plan.walk_statements()).contains(&format!(
"JOIN {} AS pd",
quote_ident(&frontier_of(&comments_descend))
)),
"replies descend from comments' frontier"
);
}
}