use std::collections::HashMap;
use std::fmt;
use super::{
OpsError, control_plane, control_plane_dsn_env, control_plane_error, open_control_plane,
};
use crate::backends::control_plane::postgres::Adoption;
use crate::backends::control_plane::schema::{self, SchemaStatus};
use crate::config::Config;
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum State {
Current { version: i32 },
Pending { pending: Vec<(i32, &'static str)> },
Applied { applied: Vec<(i32, &'static str)> },
Adopted {
adopted: Vec<(i32, &'static str)>,
pending: Vec<(i32, &'static str)>,
},
Refused { reason: String },
}
impl State {
fn from_status(status: &SchemaStatus) -> Self {
match status {
SchemaStatus::Current { version } => Self::Current { version: *version },
SchemaStatus::Absent | SchemaStatus::Behind { .. } => Self::Pending {
pending: named(&schema::pending(status)),
},
refused => Self::Refused {
reason: refused.to_string(),
},
}
}
pub fn is_ok(&self) -> bool {
!matches!(self, Self::Refused { .. })
}
pub fn is_settled(&self) -> bool {
match self {
Self::Pending { .. } | Self::Refused { .. } => false,
Self::Adopted { pending, .. } => pending.is_empty(),
Self::Current { .. } | Self::Applied { .. } => true,
}
}
}
fn named(versions: &[i32]) -> Vec<(i32, &'static str)> {
versions
.iter()
.filter_map(|version| {
schema::MIGRATIONS
.iter()
.find(|migration| migration.version == *version)
.map(|migration| (migration.version, migration.name))
})
.collect()
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Report {
NoControlPlane,
ControlPlane {
dsn_env: String,
state: State,
},
}
impl Report {
pub fn state(&self) -> Option<&State> {
match self {
Self::NoControlPlane => None,
Self::ControlPlane { state, .. } => Some(state),
}
}
pub fn is_ok(&self) -> bool {
self.state().is_none_or(State::is_ok)
}
pub fn is_settled(&self) -> bool {
self.state().is_none_or(State::is_settled)
}
}
impl fmt::Display for Report {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let Self::ControlPlane { dsn_env, state } = self else {
return write!(
f,
"stateless mode: no control plane is configured, so there is no schema to migrate"
);
};
write!(f, "control plane (${dsn_env}): ")?;
match state {
State::Current { version } => write!(f, "schema v{version} is current"),
State::Pending { pending } => {
write!(
f,
"{} migration(s) pending: {}",
pending.len(),
list(pending)
)
}
State::Applied { applied } => {
write!(
f,
"applied {} migration(s): {}",
applied.len(),
list(applied)
)
}
State::Adopted { adopted, pending } => {
write!(
f,
"adopted {} migration(s) as already applied: {}",
adopted.len(),
list(adopted)
)?;
if pending.is_empty() {
return write!(f, "; the schema is now current");
}
write!(
f,
"; {} migration(s) still pending: {} (run `axond migrate apply`)",
pending.len(),
list(pending)
)
}
State::Refused { reason } => write!(f, "refused: {reason}"),
}
}
}
fn list(migrations: &[(i32, &'static str)]) -> String {
migrations
.iter()
.map(|(version, name)| format!("v{version} {name}"))
.collect::<Vec<_>>()
.join(", ")
}
pub async fn status(config: &Config, env: &HashMap<String, String>) -> Result<Report, OpsError> {
let Some(control_plane) = control_plane(config) else {
return Ok(Report::NoControlPlane);
};
let dsn_env = control_plane_dsn_env(control_plane);
let store = open_control_plane(control_plane, env).await?;
let status = store.schema_status().await.map_err(control_plane_error)?;
Ok(Report::ControlPlane {
dsn_env,
state: State::from_status(&status),
})
}
pub async fn apply(config: &Config, env: &HashMap<String, String>) -> Result<Report, OpsError> {
let Some(control_plane) = control_plane(config) else {
return Ok(Report::NoControlPlane);
};
let dsn_env = control_plane_dsn_env(control_plane);
let store = open_control_plane(control_plane, env).await?;
let applied = store
.apply_migrations()
.await
.map_err(control_plane_error)?;
let state = if applied.is_empty() {
State::Current {
version: schema::required_version(),
}
} else {
State::Applied {
applied: named(&applied),
}
};
Ok(Report::ControlPlane { dsn_env, state })
}
pub async fn adopt(config: &Config, env: &HashMap<String, String>) -> Result<Report, OpsError> {
let Some(control_plane) = control_plane(config) else {
return Ok(Report::NoControlPlane);
};
let dsn_env = control_plane_dsn_env(control_plane);
let store = open_control_plane(control_plane, env).await?;
let state = match store.adopt_ledger().await.map_err(control_plane_error)? {
Adoption::Recorded { versions, status } => State::Adopted {
adopted: named(&versions),
pending: named(&schema::pending(&status)),
},
Adoption::AlreadyRecorded { status } => State::from_status(&status),
};
Ok(Report::ControlPlane { dsn_env, state })
}
#[cfg(test)]
mod tests {
use super::*;
use std::str::FromStr;
use crate::desired_state::Checksum;
use crate::ops::tests::{stateful_toml, stateless_toml};
fn state(status: &SchemaStatus) -> State {
State::from_status(status)
}
#[tokio::test]
async fn a_stateless_install_has_nothing_to_migrate_and_needs_no_postgres() {
let config = Config::from_toml_str(stateless_toml()).expect("valid stateless config");
let env = HashMap::new();
for report in [
status(&config, &env).await.expect("status"),
apply(&config, &env).await.expect("apply"),
adopt(&config, &env).await.expect("adopt"),
] {
assert_eq!(report, Report::NoControlPlane);
assert!(report.is_ok() && report.is_settled(), "{report}");
assert!(report.to_string().contains("no control plane"), "{report}");
}
}
#[tokio::test]
async fn an_unset_reference_fails_before_connecting_and_names_the_variable() {
let config = Config::from_toml_str(stateful_toml()).expect("valid stateful config");
let env = HashMap::new();
for error in [
status(&config, &env)
.await
.expect_err("no DSN to connect with"),
apply(&config, &env)
.await
.expect_err("no DSN to connect with"),
] {
assert_eq!(
error,
OpsError::MissingDsn {
target: crate::ops::CONTROL_PLANE.to_owned(),
dsn_env: "GW_CONTROL_PLANE_DSN".to_owned(),
}
);
assert!(!error.is_retryable(), "exporting a variable is not a retry");
}
}
#[test]
fn an_absent_schema_is_pending_every_shipped_migration() {
let State::Pending { pending } = state(&SchemaStatus::Absent) else {
panic!("a fresh install has migrations to apply");
};
assert_eq!(
pending,
schema::MIGRATIONS
.iter()
.map(|migration| (migration.version, migration.name))
.collect::<Vec<_>>()
);
let state = State::Pending { pending };
assert!(state.is_ok(), "pending is not a failure to report");
assert!(
!state.is_settled(),
"a deployment gate must not pass while a migration is outstanding"
);
}
#[test]
fn an_already_migrated_schema_is_current_and_has_nothing_pending() {
let status = SchemaStatus::Current {
version: schema::required_version(),
};
let state = state(&status);
assert_eq!(
state,
State::Current {
version: schema::required_version()
}
);
assert!(state.is_ok() && state.is_settled());
assert!(schema::pending(&status).is_empty());
}
#[test]
fn a_future_schema_is_refused_and_says_a_newer_build_owns_it() {
let state = state(&SchemaStatus::Ahead {
applied: 99,
required: schema::required_version(),
});
let State::Refused { reason } = &state else {
panic!("a schema a newer build wrote is not one this build may migrate: {state:?}");
};
assert!(reason.contains("newer gateway"), "{reason}");
assert!(!state.is_ok() && !state.is_settled());
}
#[test]
fn drift_is_refused_and_names_the_version_that_was_edited() {
let state = state(&SchemaStatus::Drifted {
version: 1,
expected: schema::MIGRATIONS[0].checksum(),
found: Checksum::of(b"edited in place"),
});
let State::Refused { reason } = &state else {
panic!("an edited applied migration is not migratable: {state:?}");
};
assert!(reason.contains("v1"), "{reason}");
assert!(reason.contains("edited in place"), "{reason}");
}
#[test]
fn a_hole_in_the_history_and_a_renamed_migration_are_refused_separately() {
let incomplete = state(&SchemaStatus::Incomplete {
applied: 3,
missing: vec![2],
});
let State::Refused { reason } = &incomplete else {
panic!("an incomplete prefix is not a history this build can extend");
};
assert!(reason.contains("missing v2"), "{reason}");
let renamed = state(&SchemaStatus::Renamed {
version: 1,
expected: schema::MIGRATIONS[0].name,
found: "control_plane_0001_initial_patched".to_owned(),
});
let State::Refused { reason } = &renamed else {
panic!("a renamed migration is not the one this build ships");
};
assert!(
reason.contains("control_plane_0001_initial_patched"),
"{reason}"
);
assert_ne!(incomplete, renamed, "the two refusals are distinguishable");
}
#[test]
fn a_ledger_this_build_did_not_write_is_refused_rather_than_migrated() {
let state = state(&SchemaStatus::Malformed {
message: "column `checksum` does not exist".to_owned(),
});
assert!(!state.is_ok(), "{state:?}");
let State::Refused { reason } = &state else {
panic!("a foreign ledger is a refusal");
};
assert!(reason.contains("checksum"), "{reason}");
}
#[test]
fn reports_print_the_reference_and_never_a_dsn() {
let reports = [
Report::NoControlPlane,
Report::ControlPlane {
dsn_env: "GW_CONTROL_PLANE_DSN".to_owned(),
state: State::Pending {
pending: named(&[1]),
},
},
Report::ControlPlane {
dsn_env: "GW_CONTROL_PLANE_DSN".to_owned(),
state: State::Applied {
applied: named(&[1]),
},
},
Report::ControlPlane {
dsn_env: "GW_CONTROL_PLANE_DSN".to_owned(),
state: State::Current { version: 1 },
},
];
for report in reports {
let rendered = report.to_string();
assert!(!rendered.contains("postgres://"), "{rendered}");
assert!(!rendered.contains("hunter2"), "{rendered}");
}
}
async fn fixture() -> Option<Fixture> {
let dsn = crate::test_services::postgres_dsn()?;
let schema = format!(
"cp_ops_{}",
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.expect("clock")
.as_nanos()
);
let client = client(&dsn).await;
client
.batch_execute(&format!("CREATE SCHEMA {schema}"))
.await
.expect("create the test schema");
let config = Config::from_toml_str(&format!(
"mode = \"stateful\"\n\
[control_plane]\n\
dsn_env = \"GW_CONTROL_PLANE_DSN\"\n\
schema = \"{schema}\"\n\
[secret_store]\n\
kek_env = \"GW_KEK\"\n\
[[admin_breakglass]]\n\
env = \"GW_BREAKGLASS\"\n"
))
.expect("valid stateful config");
let env = HashMap::from([("GW_CONTROL_PLANE_DSN".to_owned(), dsn.clone())]);
Some(Fixture {
config,
env,
schema,
dsn,
})
}
struct Fixture {
config: Config,
env: HashMap<String, String>,
schema: String,
dsn: String,
}
struct Cleanup {
dsn: String,
sql: String,
}
impl Drop for Cleanup {
fn drop(&mut self) {
let (dsn, sql) = (self.dsn.clone(), self.sql.clone());
std::thread::spawn(move || {
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("a runtime to clean up on")
.block_on(async move {
let _ = client(&dsn).await.batch_execute(&sql).await;
});
})
.join()
.expect("clean up what the test created");
}
}
impl Fixture {
async fn observe(&self) -> tokio_postgres::Client {
let client = client(&self.dsn).await;
client
.batch_execute(&format!("SET search_path TO {}", self.schema))
.await
.expect("set the test search path");
client
}
async fn ledger_exists(&self) -> bool {
self.observe()
.await
.query_one(
"SELECT to_regclass($1)::text",
&[&format!("{}.axond_cp_schema_migration", self.schema)],
)
.await
.expect("probe the ledger")
.get::<_, Option<String>>(0)
.is_some()
}
async fn ledger(&self) -> Vec<(i32, String, String)> {
self.observe()
.await
.query(
"SELECT version, name, checksum FROM axond_cp_schema_migration ORDER BY \
version",
&[],
)
.await
.expect("read the ledger")
.iter()
.map(|row| (row.get(0), row.get(1), row.get(2)))
.collect()
}
async fn hand_applied(&self) {
self.hand_applied_through(schema::MIGRATIONS.len()).await;
}
async fn hand_applied_through(&self, versions: usize) {
let client = self.observe().await;
for migration in schema::MIGRATIONS.iter().take(versions) {
client
.batch_execute(migration.sql)
.await
.expect("apply the shipped DDL the way an operator would");
}
assert!(
self.ledger().await.is_empty(),
"applying the shipped DDL by hand must not record anything: that is the whole \
problem adoption exists for"
);
}
async fn relation_exists(&self, relation: &str) -> bool {
self.observe()
.await
.query_one(
"SELECT to_regclass($1)::text",
&[&format!("{}.{relation}", self.schema)],
)
.await
.expect("probe a relation")
.get::<_, Option<String>>(0)
.is_some()
}
}
async fn client(dsn: &str) -> tokio_postgres::Client {
let (client, connection) = tokio_postgres::Config::from_str(dsn)
.expect("test dsn")
.connect(crate::usage::tls_connector())
.await
.expect("connect to the test database");
tokio::spawn(async move {
let _ = connection.await;
});
client
}
#[tokio::test]
async fn status_reports_a_fresh_database_without_creating_anything_in_it() {
let Some(fixture) = fixture().await else {
return;
};
let report = status(&fixture.config, &fixture.env)
.await
.expect("a reachable database has a status");
assert!(
matches!(report.state(), Some(State::Pending { .. })),
"{report}"
);
assert!(!report.is_settled(), "a fresh install has an apply to run");
assert!(
!fixture.ledger_exists().await,
"`migrate status` must not create the ledger it reads"
);
}
#[tokio::test]
async fn a_second_apply_is_current_rather_than_a_second_migration() {
let Some(fixture) = fixture().await else {
return;
};
let first = apply(&fixture.config, &fixture.env).await.expect("apply");
assert_eq!(
first.state(),
Some(&State::Applied {
applied: named(
&schema::MIGRATIONS
.iter()
.map(|m| m.version)
.collect::<Vec<_>>()
),
}),
"{first}"
);
let ledger = fixture.ledger().await;
assert_eq!(ledger.len(), schema::MIGRATIONS.len());
let second = apply(&fixture.config, &fixture.env)
.await
.expect("a second apply is a no-op, not a failure");
assert_eq!(
second.state(),
Some(&State::Current {
version: schema::required_version()
}),
"{second}"
);
assert_eq!(
fixture.ledger().await,
ledger,
"a repeated apply must not record a migration twice"
);
let status = status(&fixture.config, &fixture.env).await.expect("status");
assert!(status.is_ok() && status.is_settled(), "{status}");
}
#[tokio::test]
async fn a_current_database_is_not_migrated_again() {
let Some(fixture) = fixture().await else {
return;
};
apply(&fixture.config, &fixture.env)
.await
.expect("the first apply migrates");
fixture
.observe()
.await
.batch_execute("DROP TABLE axond_cp_idempotency CASCADE")
.await
.expect("drop a table the migration creates");
let second = apply(&fixture.config, &fixture.env)
.await
.expect("a current database is a no-op, not a failure");
assert_eq!(
second.state(),
Some(&State::Current {
version: schema::required_version()
}),
"{second}"
);
let recreated = fixture
.observe()
.await
.query_one(
"SELECT to_regclass($1)::text",
&[&format!("{}.axond_cp_idempotency", fixture.schema)],
)
.await
.expect("probe the dropped table")
.get::<_, Option<String>>(0)
.is_some();
assert!(
!recreated,
"applying to a current schema re-executed the shipped migration SQL"
);
}
#[tokio::test]
async fn an_empty_ledger_is_refused_rather_than_migrated_from_zero() {
let Some(fixture) = fixture().await else {
return;
};
fixture
.observe()
.await
.batch_execute(
"CREATE TABLE axond_cp_schema_migration (
version integer PRIMARY KEY,
name text NOT NULL,
checksum text NOT NULL,
applied_at timestamptz NOT NULL DEFAULT now()
)",
)
.await
.expect("create an empty ledger");
let reported = status(&fixture.config, &fixture.env)
.await
.expect("an empty ledger has a status");
let Some(State::Refused { reason }) = reported.state() else {
panic!("an empty ledger is not something to migrate from zero: {reported}");
};
assert!(
reason.contains("records no migrations")
&& reason.contains("axond migrate adopt")
&& reason.contains("drop the empty"),
"the refusal names both ways out of an empty ledger: {reason}"
);
let error = apply(&fixture.config, &fixture.env)
.await
.expect_err("apply must refuse an empty ledger");
assert!(
matches!(error, OpsError::Refused { .. }) && !error.is_retryable(),
"an operator decision, not an outage: {error:?}"
);
assert!(
fixture.ledger().await.is_empty(),
"a refused apply must not record a migration"
);
let created = fixture
.observe()
.await
.query_one(
"SELECT to_regclass($1)::text",
&[&format!("{}.axond_cp_blob", fixture.schema)],
)
.await
.expect("probe a table the migration would create")
.get::<_, Option<String>>(0)
.is_some();
assert!(
!created,
"a refused apply executed the shipped migration SQL anyway"
);
let error = adopt(&fixture.config, &fixture.env)
.await
.expect_err("there is no baseline to adopt when no object is present");
assert!(
matches!(error, OpsError::Refused { .. }) && !error.is_retryable(),
"an operator decision, not an outage: {error:?}"
);
assert!(
error.to_string().contains("drop the empty")
&& error.to_string().contains("axond migrate apply"),
"the refusal names the way forward for an unapplied database: {error}"
);
assert!(
error
.to_string()
.contains(&format!("schema `{}`", fixture.schema)),
"and names where it looked, because the ledger can answer from one schema on a \
search path while the objects are sought in another: {error}"
);
assert!(
fixture.ledger().await.is_empty(),
"a refused adoption must not record a baseline"
);
assert!(
!fixture.relation_exists("axond_cp_blob").await,
"adoption must never execute migration SQL"
);
let client = fixture.observe().await;
for migration in schema::MIGRATIONS.iter() {
client
.execute(
"INSERT INTO axond_cp_schema_migration (version, name, checksum) VALUES ($1, \
$2, $3)",
&[
&migration.version,
&migration.name,
&migration.checksum().to_string(),
],
)
.await
.expect("record the baseline the DDL corresponds to");
}
let adopted = apply(&fixture.config, &fixture.env)
.await
.expect("a recorded baseline is current");
assert_eq!(
adopted.state(),
Some(&State::Current {
version: schema::required_version()
}),
"{adopted}"
);
}
#[tokio::test]
async fn a_hand_applied_schema_missing_its_seed_row_is_refused_rather_than_adopted() {
let Some(fixture) = fixture().await else {
return;
};
fixture.hand_applied().await;
fixture
.observe()
.await
.batch_execute("DELETE FROM axond_cp_head")
.await
.expect("undo the seed the shipped file ends with");
let error = adopt(&fixture.config, &fixture.env)
.await
.expect_err("a migration that did not finish is not a baseline");
assert!(
matches!(error, OpsError::Refused { .. }) && !error.is_retryable(),
"an operator decision, not an outage: {error:?}"
);
assert!(
error.to_string().contains("only partly applied")
&& error
.to_string()
.contains("`axond_cp_head` has no seeded row"),
"the refusal names the repair, and the table is present so it must not claim \
otherwise: {error}"
);
assert!(
fixture.ledger().await.is_empty(),
"a refused adoption must not record a baseline"
);
}
#[tokio::test]
async fn a_role_that_cannot_read_the_evidence_refuses_rather_than_advising_a_retry() {
let Some(fixture) = fixture().await else {
return;
};
fixture.hand_applied().await;
let role = format!("{}_probe", fixture.schema);
let client = client(&fixture.dsn).await;
if client
.batch_execute(&format!(
"CREATE ROLE {role} LOGIN PASSWORD 'adopt-probe';
GRANT USAGE ON SCHEMA {} TO {role};
GRANT SELECT, INSERT ON {}.axond_cp_schema_migration TO {role}",
fixture.schema, fixture.schema
))
.await
.is_err()
{
return;
}
let _role_cleanup = Cleanup {
dsn: fixture.dsn.clone(),
sql: format!(
"REVOKE ALL ON ALL TABLES IN SCHEMA {} FROM {role};
REVOKE ALL ON SCHEMA {} FROM {role};
DROP ROLE {role}",
fixture.schema, fixture.schema
),
};
for table in ["axond_cp_blob", "axond_cp_head"] {
let granted: bool = client
.query_one(
"SELECT has_table_privilege($1, $2, 'SELECT')",
&[&role, &format!("{}.{table}", fixture.schema)],
)
.await
.expect("ask what the role may read")
.get(0);
assert!(!granted, "{table} must not be readable by {role}");
}
let Some((scheme, rest)) = fixture.dsn.split_once("://") else {
panic!("a DSN with a scheme");
};
let host = rest.split_once('@').map_or(rest, |(_, host)| host);
let env = HashMap::from([(
"GW_CONTROL_PLANE_DSN".to_owned(),
format!("{scheme}://{role}:adopt-probe@{host}"),
)]);
let error = adopt(&fixture.config, &env)
.await
.expect_err("evidence that cannot be read is not evidence");
assert!(
matches!(error, OpsError::Refused { .. }),
"the server rejected the read, which is a grant to make: {error:?}"
);
assert!(
!error.is_retryable(),
"a rollout gate must stop rather than loop: {error}"
);
assert!(
error.to_string().contains("42501") && error.to_string().contains("no retry clears it"),
"the refusal names the SQLSTATE and says a retry will not help: {error}"
);
assert!(
error.to_string().contains("axond_cp_head"),
"the seed probe runs only once its table is confirmed, so naming it is also the proof \
that the relation probes answered for a role with no read on those tables: {error}"
);
assert!(
fixture.ledger().await.is_empty(),
"a refused adoption must not record a baseline"
);
client
.batch_execute(&format!(
"GRANT SELECT ON ALL TABLES IN SCHEMA {} TO {role};
REVOKE INSERT ON {}.axond_cp_schema_migration FROM {role}",
fixture.schema, fixture.schema
))
.await
.expect("let the role read the evidence but not record it");
let error = adopt(&fixture.config, &env)
.await
.expect_err("a baseline that cannot be written is not recorded");
assert!(
matches!(error, OpsError::Refused { .. }) && !error.is_retryable(),
"a rejected write is a grant to make, not an outage: {error:?}"
);
assert!(
error.to_string().contains("recording the adopted baseline"),
"adoption runs no migration, so the refusal must not name one: {error}"
);
assert!(
fixture.ledger().await.is_empty(),
"a refused adoption must not record a baseline"
);
client
.batch_execute(&format!(
"REVOKE SELECT ON {}.axond_cp_schema_migration FROM {role}",
fixture.schema
))
.await
.expect("take the ledger read away as well");
let error = adopt(&fixture.config, &env)
.await
.expect_err("a ledger that cannot be read is not an empty ledger");
assert!(
matches!(error, OpsError::Refused { .. }) && !error.is_retryable(),
"a rejected read is an operator decision at every step of adoption: {error:?}"
);
assert!(
fixture.ledger().await.is_empty(),
"a refused adoption must not record a baseline"
);
}
#[tokio::test]
async fn objects_in_another_schema_on_the_path_are_not_evidence_of_an_applied_baseline() {
let Some(fixture) = fixture().await else {
return;
};
let neighbour = format!("{}_neighbour", fixture.schema);
let client = client(&fixture.dsn).await;
client
.batch_execute(&format!(
"CREATE SCHEMA {neighbour}; SET search_path TO {neighbour}"
))
.await
.expect("create the neighbouring schema");
let _neighbour_cleanup = Cleanup {
dsn: fixture.dsn.clone(),
sql: format!("DROP SCHEMA {neighbour} CASCADE"),
};
for migration in schema::MIGRATIONS.iter() {
client
.batch_execute(migration.sql)
.await
.expect("apply the shipped DDL into the neighbour");
}
fixture
.observe()
.await
.batch_execute(
"CREATE TABLE axond_cp_schema_migration (
version integer PRIMARY KEY,
name text NOT NULL,
checksum text NOT NULL,
applied_at timestamptz NOT NULL DEFAULT now()
)",
)
.await
.expect("create an empty ledger");
let config = Config::from_toml_str(
"mode = \"stateful\"\n\
[control_plane]\n\
dsn_env = \"GW_CONTROL_PLANE_DSN\"\n\
[secret_store]\n\
kek_env = \"GW_KEK\"\n\
[[admin_breakglass]]\n\
env = \"GW_BREAKGLASS\"\n",
)
.expect("valid stateful config without a schema of its own");
let separator = if fixture.dsn.contains('?') { '&' } else { '?' };
let env = HashMap::from([(
"GW_CONTROL_PLANE_DSN".to_owned(),
format!(
"{}{separator}options=-c%20search_path%3D{},{neighbour}",
fixture.dsn, fixture.schema
),
)]);
let error = adopt(&config, &env)
.await
.expect_err("a neighbour's tables are not this schema's baseline");
assert!(
matches!(error, OpsError::Refused { .. }) && !error.is_retryable(),
"an operator decision, not an outage: {error:?}"
);
assert!(
error.to_string().contains("drop the empty"),
"the refusal is the one for a database where nothing was applied: {error}"
);
assert!(
fixture.ledger().await.is_empty(),
"a baseline was recorded for objects that live in another schema"
);
}
#[tokio::test]
async fn a_missing_schema_refuses_the_apply_rather_than_advising_a_retry() {
let Some(mut fixture) = fixture().await else {
return;
};
let missing = format!("{}_absent", fixture.schema);
fixture.config = Config::from_toml_str(&format!(
"mode = \"stateful\"\n\
[control_plane]\n\
dsn_env = \"GW_CONTROL_PLANE_DSN\"\n\
schema = \"{missing}\"\n\
[secret_store]\n\
kek_env = \"GW_KEK\"\n\
[[admin_breakglass]]\n\
env = \"GW_BREAKGLASS\"\n"
))
.expect("valid stateful config");
let error = apply(&fixture.config, &fixture.env)
.await
.expect_err("a schema that does not exist cannot be migrated");
assert!(
matches!(error, OpsError::Refused { .. }),
"the server rejected the DDL, which is an operator's to fix: {error:?}"
);
assert!(
!error.is_retryable(),
"a rollout gate must stop rather than loop: {error}"
);
assert!(
error.to_string().contains("schema exists"),
"the refusal names what to check: {error}"
);
}
#[tokio::test]
async fn concurrent_applies_migrate_the_database_once() {
let Some(fixture) = fixture().await else {
return;
};
let (left, right) = tokio::join!(
apply(&fixture.config, &fixture.env),
apply(&fixture.config, &fixture.env)
);
let states = [
left.expect("the first apply").state().cloned(),
right.expect("the second apply").state().cloned(),
];
assert_eq!(
states
.iter()
.filter(|state| matches!(state, Some(State::Applied { .. })))
.count(),
1,
"exactly one of two concurrent applies migrates: {states:?}"
);
assert!(
states
.iter()
.any(|state| matches!(state, Some(State::Current { .. }))),
"the apply that lost the race finds the schema current: {states:?}"
);
assert_eq!(
fixture.ledger().await.len(),
schema::MIGRATIONS.len(),
"each migration is recorded once however many applies ran"
);
}
#[tokio::test]
async fn a_future_ledger_is_reported_by_status_and_refused_by_apply() {
let Some(fixture) = fixture().await else {
return;
};
apply(&fixture.config, &fixture.env)
.await
.expect("migrate to current first");
fixture
.observe()
.await
.execute(
"INSERT INTO axond_cp_schema_migration (version, name, checksum) VALUES ($1, $2, \
$3)",
&[
&999_i32,
&"control_plane_0999_from_the_future",
&Checksum::of(b"a newer build wrote this").to_string(),
],
)
.await
.expect("record a future migration");
let report = status(&fixture.config, &fixture.env)
.await
.expect("a future schema is a state to report, not a failure to read");
let Some(State::Refused { reason }) = report.state() else {
panic!("a future ledger is refused: {report}");
};
assert!(reason.contains("newer gateway"), "{reason}");
assert!(!report.is_ok(), "the CLI exits non-zero on this");
let error = apply(&fixture.config, &fixture.env)
.await
.expect_err("a future ledger must not be migrated");
assert!(
matches!(error, OpsError::Refused { .. }) && !error.is_retryable(),
"{error}"
);
}
#[tokio::test]
async fn a_drifted_ledger_is_refused_by_both_commands() {
let Some(fixture) = fixture().await else {
return;
};
apply(&fixture.config, &fixture.env)
.await
.expect("migrate to current first");
fixture
.observe()
.await
.execute(
"UPDATE axond_cp_schema_migration SET checksum = $1 WHERE version = 1",
&[&Checksum::of(b"edited in place").to_string()],
)
.await
.expect("edit the recorded checksum");
let report = status(&fixture.config, &fixture.env).await.expect("status");
let Some(State::Refused { reason }) = report.state() else {
panic!("drift is refused: {report}");
};
assert!(reason.contains("edited in place"), "{reason}");
assert!(
apply(&fixture.config, &fixture.env)
.await
.is_err_and(|error| matches!(error, OpsError::Refused { .. })),
"drift is not something an apply resolves"
);
}
#[tokio::test]
async fn a_foreign_ledger_is_refused_rather_than_treated_as_absent() {
let Some(fixture) = fixture().await else {
return;
};
fixture
.observe()
.await
.batch_execute("CREATE TABLE axond_cp_schema_migration (id int primary key)")
.await
.expect("take the ledger's name");
let report = status(&fixture.config, &fixture.env).await.expect("status");
let Some(State::Refused { reason }) = report.state() else {
panic!("a foreign table under the ledger's name is refused: {report}");
};
assert!(
reason.contains("is not the one this build writes"),
"{reason}"
);
assert!(
apply(&fixture.config, &fixture.env).await.is_err(),
"an apply must not write into a table it cannot account for"
);
}
#[tokio::test]
async fn a_ledger_shaped_table_with_other_column_types_is_refused_not_a_panic() {
let Some(fixture) = fixture().await else {
return;
};
fixture
.observe()
.await
.batch_execute(
"CREATE TABLE axond_cp_schema_migration \
(version text primary key, name text, checksum bytea)",
)
.await
.expect("take the ledger's name with other types");
fixture
.observe()
.await
.batch_execute(
"INSERT INTO axond_cp_schema_migration VALUES ('one', 'whatever', '\\x00')",
)
.await
.expect("give it a row to decode");
let report = status(&fixture.config, &fixture.env)
.await
.expect("a decode disagreement is a status, not an error");
let Some(State::Refused { reason }) = report.state() else {
panic!("a ledger this build cannot read is refused: {report}");
};
assert!(
reason.contains("is not the one this build writes"),
"{reason}"
);
assert!(
apply(&fixture.config, &fixture.env).await.is_err(),
"an apply must not write into a table it cannot account for"
);
}
#[tokio::test]
async fn a_transient_ledger_read_failure_stays_retryable() {
let Some(fixture) = fixture().await else {
return;
};
let raise = |code: &str| {
format!(
"CREATE FUNCTION ledger_{code}() RETURNS TABLE(version integer, name text, \
checksum text) AS $$ BEGIN RAISE EXCEPTION 'simulated' USING ERRCODE = \
'{code}'; END $$ LANGUAGE plpgsql;\n\
CREATE VIEW axond_cp_schema_migration AS SELECT * FROM ledger_{code}();"
)
};
fixture
.observe()
.await
.batch_execute(&raise("40001"))
.await
.expect("stand in for a serialization failure");
let error = status(&fixture.config, &fixture.env)
.await
.expect_err("a serialization failure is an outage, not a verdict");
assert!(
error.is_retryable(),
"a transient server error must stay retryable: {error}"
);
let client = fixture.observe().await;
client
.batch_execute("DROP VIEW axond_cp_schema_migration")
.await
.expect("drop the stand-in");
client
.batch_execute(&raise("42703"))
.await
.expect("stand in for an undefined column");
let report = status(&fixture.config, &fixture.env)
.await
.expect("a schema disagreement is a status, not an error");
assert!(
matches!(report.state(), Some(State::Refused { .. })),
"{report}"
);
}
#[tokio::test]
async fn an_unreachable_database_is_retryable_and_never_echoes_the_dsn() {
let config = Config::from_toml_str(
"mode = \"stateful\"\n\
[control_plane]\n\
dsn_env = \"GW_CONTROL_PLANE_DSN\"\n\
connect_timeout_ms = 500\n\
[secret_store]\n\
kek_env = \"GW_KEK\"\n\
[[admin_breakglass]]\n\
env = \"GW_BREAKGLASS\"\n",
)
.expect("valid stateful config");
let env = HashMap::from([(
"GW_CONTROL_PLANE_DSN".to_owned(),
"postgres://axond:hunter2@127.0.0.1:1/axond".to_owned(),
)]);
for error in [
status(&config, &env).await.expect_err("nothing answers"),
apply(&config, &env).await.expect_err("nothing answers"),
adopt(&config, &env).await.expect_err("nothing answers"),
] {
assert!(error.is_retryable(), "{error}");
let rendered = error.to_string();
assert!(!rendered.contains("hunter2"), "{rendered}");
assert!(!rendered.contains("postgres://"), "{rendered}");
}
}
#[test]
fn an_applied_report_names_the_files_that_ran() {
let report = Report::ControlPlane {
dsn_env: "GW_CONTROL_PLANE_DSN".to_owned(),
state: State::Applied {
applied: named(&[1]),
},
};
let rendered = report.to_string();
assert!(
rendered.contains("v1 control_plane_0001_initial"),
"{rendered}"
);
assert!(report.is_ok() && report.is_settled(), "{rendered}");
}
#[test]
fn an_adopted_report_names_the_baseline_and_what_is_still_pending() {
let whole = Report::ControlPlane {
dsn_env: "GW_CONTROL_PLANE_DSN".to_owned(),
state: State::Adopted {
adopted: named(&[1]),
pending: Vec::new(),
},
};
let rendered = whole.to_string();
assert!(
rendered.contains("adopted 1 migration(s) as already applied")
&& rendered.contains("v1 control_plane_0001_initial")
&& rendered.contains("now current"),
"{rendered}"
);
assert!(whole.is_ok() && whole.is_settled(), "{rendered}");
let partial = Report::ControlPlane {
dsn_env: "GW_CONTROL_PLANE_DSN".to_owned(),
state: State::Adopted {
adopted: named(&[1]),
pending: named(&[1]),
},
};
assert!(partial.is_ok() && !partial.is_settled(), "{partial}");
assert!(
partial.to_string().contains("axond migrate apply"),
"{partial}"
);
}
#[tokio::test]
async fn a_hand_applied_schema_is_adopted_as_the_baseline_its_objects_prove() {
let Some(fixture) = fixture().await else {
return;
};
fixture.hand_applied().await;
let refused = status(&fixture.config, &fixture.env)
.await
.expect("an unrecorded schema has a status");
assert!(
matches!(refused.state(), Some(State::Refused { .. })),
"an unrecorded schema is refused until it is adopted: {refused}"
);
let report = adopt(&fixture.config, &fixture.env)
.await
.expect("the objects the shipped DDL declares are all present");
assert_eq!(
report.state(),
Some(&State::Adopted {
adopted: named(
&schema::MIGRATIONS
.iter()
.map(|migration| migration.version)
.collect::<Vec<_>>()
),
pending: Vec::new(),
}),
"{report}"
);
assert_eq!(
fixture.ledger().await,
schema::MIGRATIONS
.iter()
.map(|migration| (
migration.version,
migration.name.to_owned(),
migration.checksum().to_string()
))
.collect::<Vec<_>>(),
"an adopted baseline is the ledger an apply would have written"
);
let settled = status(&fixture.config, &fixture.env)
.await
.expect("an adopted schema has a status");
assert_eq!(
settled.state(),
Some(&State::Current {
version: schema::required_version()
}),
"{settled}"
);
fixture
.observe()
.await
.batch_execute("DROP TABLE axond_cp_idempotency CASCADE")
.await
.expect("drop a table the migration creates");
let applied = apply(&fixture.config, &fixture.env)
.await
.expect("an adopted schema is current, so an apply is a no-op");
assert_eq!(
applied.state(),
Some(&State::Current {
version: schema::required_version()
}),
"{applied}"
);
assert!(
!fixture.relation_exists("axond_cp_idempotency").await,
"applying after an adoption replayed the shipped migration SQL"
);
}
#[tokio::test]
async fn a_schema_hand_applied_only_as_far_as_v1_adopts_v1_and_leaves_v2_pending() {
let Some(fixture) = fixture().await else {
return;
};
fixture.hand_applied_through(1).await;
assert!(
!fixture.relation_exists("axond_cp_tenant").await,
"v2's objects must not be there: this is the pre-tenancy manual state"
);
let rest: Vec<i32> = schema::MIGRATIONS
.iter()
.map(|migration| migration.version)
.filter(|version| *version > 1)
.collect();
let report = adopt(&fixture.config, &fixture.env)
.await
.expect("v1's objects are all present, so v1 is adoptable");
assert_eq!(
report.state(),
Some(&State::Adopted {
adopted: named(&[1]),
pending: named(&rest),
}),
"{report}"
);
assert!(
report.is_ok() && !report.is_settled(),
"a baseline with a migration still pending is not a schema to serve: {report}"
);
assert_eq!(
fixture.ledger().await,
vec![(
1,
schema::MIGRATIONS[0].name.to_owned(),
schema::MIGRATIONS[0].checksum().to_string()
)],
"only the version the objects account for may be recorded"
);
let applied = apply(&fixture.config, &fixture.env)
.await
.expect("an adopted prefix is behind, so an apply runs the rest");
assert_eq!(
applied.state(),
Some(&State::Applied {
applied: named(&rest)
}),
"{applied}"
);
assert!(
fixture.relation_exists("axond_cp_tenant").await,
"the apply that followed the adoption has to have run v2"
);
}
#[tokio::test]
async fn a_second_adopt_reports_the_recorded_history_rather_than_writing_again() {
let Some(fixture) = fixture().await else {
return;
};
fixture.hand_applied().await;
adopt(&fixture.config, &fixture.env)
.await
.expect("the first adoption records the baseline");
let first = fixture.ledger().await;
let second = adopt(&fixture.config, &fixture.env)
.await
.expect("a recorded history is not a refusal");
assert_eq!(
second.state(),
Some(&State::Current {
version: schema::required_version()
}),
"a second adoption reports the history it found: {second}"
);
assert_eq!(
fixture.ledger().await,
first,
"a second adoption rewrote the ledger it should have left alone"
);
}
#[tokio::test]
async fn adopting_a_migrated_database_reports_it_and_records_nothing() {
let Some(fixture) = fixture().await else {
return;
};
apply(&fixture.config, &fixture.env)
.await
.expect("migrate normally");
let recorded = fixture.ledger().await;
let report = adopt(&fixture.config, &fixture.env)
.await
.expect("a migrated database is current, not adoptable");
assert_eq!(
report.state(),
Some(&State::Current {
version: schema::required_version()
}),
"{report}"
);
assert_eq!(fixture.ledger().await, recorded, "{report}");
}
#[tokio::test]
async fn a_partly_applied_schema_is_refused_without_recording_anything() {
let Some(fixture) = fixture().await else {
return;
};
fixture.hand_applied().await;
fixture
.observe()
.await
.batch_execute("DROP TABLE axond_cp_head CASCADE")
.await
.expect("leave the hand-applied schema incomplete");
let error = adopt(&fixture.config, &fixture.env)
.await
.expect_err("an incomplete schema has no baseline");
assert!(
matches!(error, OpsError::Refused { .. }) && !error.is_retryable(),
"an operator decision, not an outage: {error:?}"
);
let rendered = error.to_string();
assert!(
rendered.contains("only partly applied") && rendered.contains("axond_cp_head"),
"the refusal names the object that is missing: {rendered}"
);
assert!(
fixture.ledger().await.is_empty(),
"a refused adoption must record no version at all, not the ones it got through"
);
assert!(
!fixture.relation_exists("axond_cp_head").await,
"adoption executed DDL to repair what it should have refused"
);
let report = status(&fixture.config, &fixture.env)
.await
.expect("status still reads");
assert!(
matches!(report.state(), Some(State::Refused { .. })),
"{report}"
);
assert!(
apply(&fixture.config, &fixture.env)
.await
.expect_err("apply still refuses an unrecorded schema")
.to_string()
.contains("records no migrations")
);
}
#[tokio::test]
async fn adoption_refuses_every_schema_that_is_not_an_empty_ledger() {
let Some(fixture) = fixture().await else {
return;
};
let error = adopt(&fixture.config, &fixture.env)
.await
.expect_err("an absent schema is not adoptable");
assert!(
matches!(error, OpsError::Refused { .. }) && !error.is_retryable(),
"{error:?}"
);
assert!(
error.to_string().contains("existing but empty"),
"the refusal says what adoption is for: {error}"
);
assert!(
!fixture.ledger_exists().await,
"a refused adoption created the ledger it refused to reconcile"
);
apply(&fixture.config, &fixture.env)
.await
.expect("migrate to current first");
fixture
.observe()
.await
.execute(
"UPDATE axond_cp_schema_migration SET checksum = $1 WHERE version = $2",
&[&Checksum::of(b"an edited migration").to_string(), &1_i32],
)
.await
.expect("drift the recorded checksum");
let error = adopt(&fixture.config, &fixture.env)
.await
.expect_err("a drifted history is not adoptable");
assert!(
matches!(error, OpsError::Refused { .. }) && !error.is_retryable(),
"{error:?}"
);
assert_eq!(
fixture.ledger().await.first().map(|row| row.2.clone()),
Some(Checksum::of(b"an edited migration").to_string()),
"a refused adoption rewrote a recorded checksum"
);
}
#[tokio::test]
async fn a_tenancy_effect_undone_by_hand_is_refused_and_named() {
let Some(fixture) = fixture().await else {
return;
};
fixture.hand_applied().await;
for (undo, named, redo) in [
(
"ALTER TABLE axond_cp_head NO FORCE ROW LEVEL SECURITY",
"forced row level security on `axond_cp_head` is not enabled",
"ALTER TABLE axond_cp_head FORCE ROW LEVEL SECURITY",
),
(
"ALTER TABLE axond_cp_tenant DISABLE ROW LEVEL SECURITY",
"row level security on `axond_cp_tenant` is not enabled",
"ALTER TABLE axond_cp_tenant ENABLE ROW LEVEL SECURITY",
),
(
"DROP POLICY axond_cp_blob_isolation ON axond_cp_blob",
"`axond_cp_blob`'s `axond_cp_blob_isolation` policy is not present",
"CREATE POLICY axond_cp_blob_isolation ON axond_cp_blob USING (true)",
),
(
"ALTER TABLE axond_cp_mutation DROP CONSTRAINT \
axond_cp_mutation_actor_attribution",
"`axond_cp_mutation`'s `axond_cp_mutation_actor_attribution` constraint is not \
present",
"ALTER TABLE axond_cp_mutation ADD CONSTRAINT \
axond_cp_mutation_actor_attribution CHECK (true)",
),
(
"ALTER TABLE axond_cp_audit_event ADD CONSTRAINT \
axond_cp_audit_event_actor_kind_check CHECK (true)",
"`axond_cp_audit_event`'s `axond_cp_audit_event_actor_kind_check` constraint is \
still present",
"ALTER TABLE axond_cp_audit_event DROP CONSTRAINT \
axond_cp_audit_event_actor_kind_check",
),
(
"ALTER TABLE axond_cp_audit_event DROP COLUMN actor_principal_id",
"`axond_cp_audit_event`'s `actor_principal_id` column is not present",
"ALTER TABLE axond_cp_audit_event \
ADD COLUMN actor_principal_id text NULL, \
ADD CONSTRAINT axond_cp_audit_event_actor_attribution CHECK (true)",
),
] {
fixture
.observe()
.await
.batch_execute(undo)
.await
.unwrap_or_else(|error| panic!("undo one tenancy effect ({undo}): {error}"));
let error = adopt(&fixture.config, &fixture.env)
.await
.expect_err("a schema missing one of v2's effects has no baseline");
assert!(
matches!(error, OpsError::Refused { .. }) && !error.is_retryable(),
"an operator decision, not an outage: {error:?}"
);
let rendered = error.to_string();
assert!(
rendered.contains("only partly applied") && rendered.contains(named),
"the refusal has to name what is wrong ({named}): {rendered}"
);
assert!(
fixture.ledger().await.is_empty(),
"a refused adoption recorded a version anyway: {undo}"
);
fixture
.observe()
.await
.batch_execute(redo)
.await
.unwrap_or_else(|error| panic!("put the tenancy effect back ({redo}): {error}"));
}
let report = adopt(&fixture.config, &fixture.env)
.await
.expect("a fully hand-applied v1+v2 schema is adoptable");
assert_eq!(
report.state(),
Some(&State::Adopted {
adopted: named(
&schema::MIGRATIONS
.iter()
.map(|migration| migration.version)
.collect::<Vec<_>>()
),
pending: Vec::new(),
}),
"{report}"
);
}
#[tokio::test]
async fn concurrent_adoptions_record_the_baseline_once() {
let Some(fixture) = fixture().await else {
return;
};
fixture.hand_applied().await;
let (left, right) = tokio::join!(
adopt(&fixture.config, &fixture.env),
adopt(&fixture.config, &fixture.env)
);
let states = [
left.expect("the first adoption").state().cloned(),
right.expect("the second adoption").state().cloned(),
];
assert_eq!(
states
.iter()
.filter(|state| matches!(state, Some(State::Adopted { .. })))
.count(),
1,
"exactly one of two concurrent adoptions records a baseline: {states:?}"
);
assert!(
states
.iter()
.any(|state| matches!(state, Some(State::Current { .. }))),
"the adoption that lost the race finds a recorded history: {states:?}"
);
assert_eq!(
fixture.ledger().await.len(),
schema::MIGRATIONS.len(),
"each migration is recorded once however many adoptions ran"
);
}
}