mod bake;
pub mod exec;
mod fleet;
pub mod media;
pub mod proxy;
use std::net::{TcpStream, ToSocketAddrs};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;
use autumn_web::alerts::{Alert, AlertChannel, AlertCondition};
use autumn_web::config::{AutumnConfig, DeployConfig, Env};
use proxy::ProxyController;
use serde::Deserialize;
const SSH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
const MIGRATIONS_DIR: &str = "migrations";
const DEPLOY_HOST_MISSING_DETAIL: &str = "no target host configured";
const DEPLOY_HOST_MISSING_HINT: &str = "Set `[deploy] host` in autumn.toml to your server's SSH-reachable address \
(or `[deploy] hosts = [\"<address>\", …]` to deploy a fleet)";
pub const BAKE_ROLLED_BACK: &str = "The host was rolled back to the previous release.";
pub const BAKE_ROLLBACK_FAILED: &str = "The automatic rollback did not finish, so the host can \
still run the new release. Run `autumn deploy rollback`.";
pub const BAKE_ROLLBACK_REFUSED: &str = "The automatic rollback was not safe, so the host still \
runs the new release. Follow the recovery steps above.";
pub const BAKE_FROZEN: &str = "--no-rollback is set, so the new release still serves. Run \
`autumn deploy rollback` to roll back.";
#[derive(Debug, thiserror::Error)]
pub enum DeployError {
#[error("failed to load configuration: {0}")]
Config(String),
#[error("{0}")]
UnsupportedPlatform(String),
#[error(
"preflight failed: {0} check(s) did not pass — resolve the issues above and re-run \
`autumn deploy check`"
)]
PreflightFailed(usize),
#[error("release binary not found at {0} — run `autumn build --embed` first")]
BinaryMissing(String),
#[error("deploy execution failed: {0}")]
Exec(String),
#[error(
"the bake failed on {host}: {reason}. {result}{}",
.schema_note.map_or_else(String::new, |note| format!(" Note: {note}"))
)]
BakeFailed {
host: String,
reason: String,
result: &'static str,
schema_note: Option<&'static str>,
},
#[error("{0}")]
FleetHalted(Box<FleetHalt>),
#[error(
"fleet rollback incomplete: {not_rolled_back} of {total} host(s) were not rolled back \
— see the per-host table above. A host with no previous release recorded cannot be \
rolled back; deploy the intended release to it explicitly, or retry one host with \
`autumn deploy rollback --only <host>`"
)]
FleetRollbackFailed {
not_rolled_back: usize,
total: usize,
},
#[error(
"fleet drift detected — the status table above names each host and reason \
(`--strict` exits non-zero on drift so it can be alerted on)"
)]
DriftDetected,
#[error(
"fleet maintenance {verb} incomplete: {failed} of {total} host(s) failed — the \
table above names the hosts that DID change, so they can be reversed by hand \
if the fleet must stay consistent"
)]
FleetMaintenanceFailed {
verb: &'static str,
failed: usize,
total: usize,
},
}
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
#[error(
"fleet rollout halted at {failed_host} during `{failed_step}` — the remaining hosts were \
not touched, and any migration that already ran was NOT rolled back (automatic \
compensation restores binaries only)"
)]
pub struct FleetHalt {
pub failed_host: String,
pub failed_step: &'static str,
pub rolled_back: Vec<String>,
pub torn_down: Vec<String>,
pub still_on_new: Vec<String>,
pub degraded: Vec<(String, &'static str)>,
pub route_removal_failed: Vec<(String, &'static str)>,
pub manual: Vec<(String, &'static str)>,
}
impl FleetHalt {
#[must_use]
pub fn risk(&self) -> Option<&'static str> {
(self.failed_step == fleet::DRAIN_OLD_LABEL
&& self.still_on_new.contains(&self.failed_host))
.then_some(fleet::OLD_SLOT_MAY_RUN_NOTE)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DeployAction {
Check,
Plan,
Rollback,
Up,
Status,
MaintenanceOn,
MaintenanceOff,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct MaintenanceOnArgs {
pub message: Option<String>,
pub allow_ips: Vec<String>,
pub readonly: bool,
pub bypass_header: Option<(String, String)>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct DeployOptions {
pub only: Vec<String>,
pub no_rollback: bool,
pub json: bool,
pub strict: bool,
pub maintenance: Option<MaintenanceOnArgs>,
pub bake_secs: Option<u64>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ResolvedDeployConfig {
pub host: Option<String>,
pub user: String,
pub ssh_port: u16,
pub app_name: String,
pub app_dir: String,
pub service_name: String,
pub readiness_timeout_secs: u64,
pub keep_releases: u32,
pub profile: String,
pub tls_enabled: bool,
pub tls_host: Option<String>,
pub install_proxy: bool,
pub sqlite_data: SqliteDataPlacement,
pub release_payloads: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub enum SqliteDataPlacement {
#[default]
None,
Relative(String),
Persistent(String),
}
#[allow(clippy::redundant_pub_crate)]
pub(crate) fn canonicalize_deploy_profile(raw: &str) -> String {
let trimmed = raw.trim();
if trimmed.is_empty() {
return "prod".to_owned();
}
if trimmed.eq_ignore_ascii_case("production") {
return "prod".to_owned();
}
if trimmed.eq_ignore_ascii_case("development") {
return "dev".to_owned();
}
if trimmed.eq_ignore_ascii_case("prod") {
return "prod".to_owned();
}
if trimmed.eq_ignore_ascii_case("dev") {
return "dev".to_owned();
}
trimmed.to_owned()
}
#[allow(clippy::redundant_pub_crate)]
pub(crate) fn trimmed_deploy_profile(raw: &str) -> String {
let trimmed = raw.trim();
if trimmed.is_empty() {
return "prod".to_owned();
}
trimmed.to_owned()
}
impl ResolvedDeployConfig {
pub fn resolve(cfg: &DeployConfig, project_name: &str) -> Result<Self, String> {
let non_blank = |s: &Option<String>| {
s.as_ref()
.map(|v| v.trim().to_owned())
.filter(|v| !v.is_empty())
};
let app_name = non_blank(&cfg.app_name).unwrap_or_else(|| project_name.to_owned());
let app_dir = non_blank(&cfg.app_dir).unwrap_or_else(|| format!("/srv/autumn/{app_name}"));
let service_name = non_blank(&cfg.service_name).unwrap_or_else(|| app_name.clone());
let tls_host = if cfg.tls.enabled {
let host = non_blank(&cfg.tls.host).ok_or_else(|| {
"[deploy.tls] requires a non-empty `host` when enabled: set \
`[deploy.tls] host = \"<public-hostname>\"` in autumn.toml to the DNS \
name the TLS certificate should be issued for"
.to_owned()
})?;
Some(host)
} else {
None
};
Ok(Self {
host: non_blank(&cfg.host),
user: cfg.user.clone(),
ssh_port: cfg.ssh_port,
app_name,
app_dir,
service_name,
readiness_timeout_secs: cfg.readiness_timeout_secs,
keep_releases: cfg.keep_releases,
profile: trimmed_deploy_profile(&cfg.profile),
tls_enabled: cfg.tls.enabled,
tls_host,
install_proxy: cfg.install_proxy,
sqlite_data: SqliteDataPlacement::None,
release_payloads: Vec::new(),
})
}
#[must_use]
pub fn with_sqlite_data_placement(mut self, placement: SqliteDataPlacement) -> Self {
self.sqlite_data = placement;
self
}
#[must_use]
pub fn with_release_payloads(mut self, payloads: Vec<String>) -> Self {
self.release_payloads = payloads;
self
}
#[must_use]
pub fn relative_sqlite_data_file(&self) -> Option<&str> {
match &self.sqlite_data {
SqliteDataPlacement::Relative(rel) => Some(rel),
SqliteDataPlacement::None | SqliteDataPlacement::Persistent(_) => None,
}
}
#[must_use]
pub fn persistent_sqlite_data_file(&self) -> Option<&str> {
match &self.sqlite_data {
SqliteDataPlacement::Persistent(path) => Some(path),
SqliteDataPlacement::None | SqliteDataPlacement::Relative(_) => None,
}
}
#[must_use]
pub fn shared_dir(&self) -> String {
format!("{}/shared", self.app_dir)
}
#[must_use]
pub fn env_file(&self) -> String {
format!("{}/autumn.env", self.shared_dir())
}
#[must_use]
pub fn releases_dir(&self) -> String {
format!("{}/releases", self.app_dir)
}
#[must_use]
pub fn current_symlink(&self) -> String {
format!("{}/current", self.app_dir)
}
#[must_use]
pub fn maintenance_flag_file(&self) -> String {
format!(
"{}/{}",
self.shared_dir(),
autumn_web::maintenance::MAINTENANCE_FLAG_BASENAME
)
}
#[must_use]
pub fn shared_data_dir(&self) -> String {
format!("{}/data", self.shared_dir())
}
#[must_use]
pub fn shared_sqlite_data_file(&self) -> Option<String> {
self.relative_sqlite_data_file()
.map(|rel| format!("{}/{rel}", self.shared_data_dir()))
}
#[must_use]
pub fn sqlite_data_marker_file(&self) -> String {
format!("{}/sqlite-data-adopted", self.shared_dir())
}
}
#[allow(clippy::redundant_pub_crate)]
pub(crate) fn deploy_host_list(cfg: &DeployConfig) -> Result<Vec<String>, String> {
let host = cfg
.host
.as_deref()
.map(str::trim)
.filter(|host| !host.is_empty());
if host.is_some() && !cfg.hosts.is_empty() {
return Err(
"`[deploy] host` and `[deploy] hosts` are mutually exclusive: keep the \
single-server `[deploy] host = \"<address>\"` or the fleet list \
`[deploy] hosts = [\"<address>\", …]` in autumn.toml, not both (#1621)"
.to_owned(),
);
}
if cfg.hosts.is_empty() {
return Ok(host.map(str::to_owned).into_iter().collect());
}
let mut hosts: Vec<String> = Vec::with_capacity(cfg.hosts.len());
for (index, entry) in cfg.hosts.iter().enumerate() {
let trimmed = entry.trim();
if trimmed.is_empty() {
return Err(format!(
"`[deploy] hosts` entry {index} is blank: every fleet entry must be an \
SSH-reachable hostname or IP (#1621)"
));
}
if hosts.iter().any(|seen| seen == trimmed) {
return Err(format!(
"`[deploy] hosts` lists `{trimmed}` more than once: each fleet host must \
appear exactly once — deploying the same server twice corrupts its \
previous-release chain (#1621)"
));
}
hosts.push(trimmed.to_owned());
}
Ok(hosts)
}
fn select_hosts(configured: &[String], only: &[String]) -> Result<Vec<String>, String> {
if only.is_empty() {
return Ok(configured.to_vec());
}
for wanted in only {
let wanted = wanted.trim();
if !configured.iter().any(|host| host == wanted) {
let known = if configured.is_empty() {
"none — set `[deploy] hosts` in autumn.toml first".to_owned()
} else {
configured.join(", ")
};
return Err(format!(
"`--only {wanted}` names a host that is not in this project's deploy \
configuration. Configured hosts: {known}. `--only` selects from that list \
verbatim — it never resolves or guesses a name, so a typo can't deploy the \
wrong machine (#1621)"
));
}
}
Ok(configured
.iter()
.filter(|host| only.iter().any(|wanted| wanted.trim() == host.as_str()))
.cloned()
.collect())
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ResolvedFleet {
pub hosts: Vec<ResolvedDeployConfig>,
}
impl ResolvedFleet {
pub fn resolve(cfg: &DeployConfig, project_name: &str) -> Result<Self, String> {
let addresses = deploy_host_list(cfg)?;
if addresses.is_empty() {
return Err(format!(
"{DEPLOY_HOST_MISSING_DETAIL}: {DEPLOY_HOST_MISSING_HINT}"
));
}
let shared = ResolvedDeployConfig::resolve(cfg, project_name)?;
Ok(Self::from_targets(&shared, &addresses))
}
#[must_use]
pub fn from_targets(shared: &ResolvedDeployConfig, addresses: &[String]) -> Self {
if addresses.len() <= 1 {
return Self {
hosts: vec![ResolvedDeployConfig {
host: addresses.first().cloned().or_else(|| shared.host.clone()),
..shared.clone()
}],
};
}
Self {
hosts: addresses
.iter()
.map(|host| ResolvedDeployConfig {
host: Some(host.clone()),
..shared.clone()
})
.collect(),
}
}
#[must_use]
pub const fn is_single(&self) -> bool {
self.hosts.len() == 1
}
}
const fn is_single_host_deploy(fleet: &ResolvedFleet, configured_host_count: usize) -> bool {
fleet.is_single() && configured_host_count <= 1
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DeployStep {
pub label: &'static str,
pub description: String,
}
impl DeployStep {
fn new(label: &'static str, description: impl Into<String>) -> Self {
Self {
label,
description: description.into(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PreflightCheck {
pub name: &'static str,
pub scope: Option<String>,
pub passed: bool,
pub deferred: bool,
pub detail: String,
pub hint: Option<&'static str>,
}
impl PreflightCheck {
fn pass(name: &'static str, detail: impl Into<String>) -> Self {
Self {
name,
scope: None,
passed: true,
deferred: false,
detail: detail.into(),
hint: None,
}
}
fn fail(name: &'static str, detail: impl Into<String>, hint: &'static str) -> Self {
Self {
name,
scope: None,
passed: false,
deferred: false,
detail: detail.into(),
hint: Some(hint),
}
}
#[must_use]
fn scoped(mut self, scope: Option<String>) -> Self {
self.scope = scope;
self
}
#[must_use]
pub const fn blocking(&self) -> bool {
!self.passed && !self.deferred
}
}
#[allow(clippy::redundant_pub_crate)]
#[cfg_attr(not(test), allow(dead_code))]
pub(crate) const PREFLIGHT_GRADERS: [&str; 4] = [
"ssh_reachability",
"signing_secret",
"database_url",
"migrate_check",
];
#[allow(clippy::redundant_pub_crate)]
pub(crate) const DOCTOR_PREFLIGHT_GRADERS: [&str; 4] = [
"deploy_ssh_reachability",
"deploy_signing_secret",
"deploy_database_url",
"deploy_migrate_check",
];
#[must_use]
pub fn grade_deploy_host_present(host: Option<&str>) -> PreflightCheck {
match host.map(str::trim).filter(|h| !h.is_empty()) {
Some(_) => PreflightCheck::pass("deploy_host", "deploy target host is configured"),
None => PreflightCheck::fail(
"deploy_host",
DEPLOY_HOST_MISSING_DETAIL,
DEPLOY_HOST_MISSING_HINT,
),
}
}
#[must_use]
pub fn grade_deploy_hosts_present(hosts: &[String]) -> PreflightCheck {
match hosts {
[] => grade_deploy_host_present(None),
[only] => grade_deploy_host_present(Some(only)),
many => PreflightCheck::pass(
"deploy_host",
format!(
"{} deploy target hosts configured, in rollout order: {}",
many.len(),
many.join(", ")
),
),
}
}
#[must_use]
pub fn grade_fleet_ssh_reachability(
hosts: &[String],
ssh_port: u16,
timeout: Duration,
) -> PreflightCheck {
match hosts {
[] => grade_ssh_reachability(None, ssh_port, timeout),
[only] => grade_ssh_reachability(Some(only.as_str()), ssh_port, timeout),
many => {
let graded: Vec<(&str, PreflightCheck)> = many
.iter()
.map(|host| {
(
host.as_str(),
grade_ssh_reachability(Some(host.as_str()), ssh_port, timeout),
)
})
.collect();
let unreachable: Vec<&str> = graded
.iter()
.filter(|(_, check)| !check.passed)
.map(|(host, _)| *host)
.collect();
if unreachable.is_empty() {
PreflightCheck::pass(
"ssh_reachability",
format!(
"SSH port reachable on all {} fleet hosts: {}",
many.len(),
many.join(", ")
),
)
} else {
PreflightCheck::fail(
"ssh_reachability",
format!(
"cannot reach {} of {} fleet hosts on port {ssh_port}: {}",
unreachable.len(),
many.len(),
unreachable.join(", ")
),
"Confirm the server is up, the SSH port is open, and any firewall allows your IP",
)
}
}
}
}
#[must_use]
pub fn grade_ssh_reachability(
host: Option<&str>,
ssh_port: u16,
timeout: Duration,
) -> PreflightCheck {
let Some(host) = host.map(str::trim).filter(|h| !h.is_empty()) else {
return PreflightCheck::fail(
"ssh_reachability",
DEPLOY_HOST_MISSING_DETAIL,
DEPLOY_HOST_MISSING_HINT,
);
};
let addrs = match (host, ssh_port).to_socket_addrs() {
Ok(addrs) => addrs.collect::<Vec<_>>(),
Err(e) => {
return PreflightCheck::fail(
"ssh_reachability",
format!("could not resolve {host}:{ssh_port}: {e}"),
"Check that `[deploy] host` is a resolvable hostname or a valid IP address",
);
}
};
if addrs.is_empty() {
return PreflightCheck::fail(
"ssh_reachability",
format!("{host}:{ssh_port} resolved to no addresses"),
"Check that `[deploy] host` is a resolvable hostname or a valid IP address",
);
}
let mut last_err = None;
for addr in &addrs {
match TcpStream::connect_timeout(addr, timeout) {
Ok(_) => {
return PreflightCheck::pass(
"ssh_reachability",
format!("SSH port reachable at {host}:{ssh_port}"),
);
}
Err(e) => last_err = Some(e),
}
}
let detail = last_err.map_or_else(
|| format!("cannot reach {host}:{ssh_port}"),
|e| format!("cannot reach {host}:{ssh_port}: {e}"),
);
PreflightCheck::fail(
"ssh_reachability",
detail,
"Confirm the server is up, the SSH port is open, and any firewall allows your IP",
)
}
#[must_use]
pub fn grade_signing_secret(
secret: Option<&str>,
previous_secrets: &[String],
is_production: bool,
) -> PreflightCheck {
let Some(value) = secret.map(str::trim).filter(|s| !s.is_empty()) else {
return PreflightCheck::fail(
"signing_secret",
"no signing secret configured",
"Set AUTUMN_SECURITY__SIGNING_SECRET (generate with `openssl rand -hex 32`)",
);
};
if is_production {
use autumn_web::security::{SigningSecretError, validate_signing_secret};
if let Err(error) = validate_signing_secret(Some(value), true) {
let detail = match error {
SigningSecretError::MissingInProduction => {
"signing secret is required in production".to_owned()
}
SigningSecretError::TooShort { actual, required } => format!(
"signing secret is too short ({actual} bytes, minimum {required}) for production"
),
SigningSecretError::KnownWeakValue(_) => {
"signing secret is a known demo/template value not allowed in production"
.to_owned()
}
};
return PreflightCheck::fail(
"signing_secret",
detail,
"Set a strong AUTUMN_SECURITY__SIGNING_SECRET (generate with `openssl rand -hex 32`)",
);
}
for (index, previous) in previous_secrets.iter().enumerate() {
if let Err(error) = validate_signing_secret(Some(previous.as_str()), true) {
let position = index + 1;
let detail = match error {
SigningSecretError::MissingInProduction => format!(
"previous (rotation) signing secret #{position} is empty and not allowed in production"
),
SigningSecretError::TooShort { actual, required } => format!(
"previous (rotation) signing secret #{position} is too short ({actual} bytes, minimum {required}) for production"
),
SigningSecretError::KnownWeakValue(_) => format!(
"previous (rotation) signing secret #{position} is a known demo/template value not allowed in production"
),
};
return PreflightCheck::fail(
"signing_secret",
detail,
"Remove or rotate out the weak entry in security.signing_secret.previous_secrets (each must be as strong as the current secret)",
);
}
}
}
PreflightCheck::pass("signing_secret", "signing secret is configured")
}
#[must_use]
pub fn grade_database_url(
url: Option<&str>,
migrations_dir: &Path,
database_configured: bool,
db_backed_runtime: bool,
) -> PreflightCheck {
match url.map(str::trim).filter(|u| !u.is_empty()) {
Some(_) => PreflightCheck::pass("database_url", "database URL is configured"),
None if !migrations_dir.exists() && !database_configured && !db_backed_runtime => {
PreflightCheck::pass(
"database_url",
"no database configured (nothing to connect to)",
)
}
None => PreflightCheck::fail(
"database_url",
"no writable database URL: this app is database-backed (migrations, a \
`[database]` section, or a Postgres-backed runtime feature such as \
`jobs.backend`/`scheduler.backend`) and needs a primary/control \
(`database.primary_url`) or shard-primary URL; `database.replica_url` \
alone is not a writable target",
"Set database.primary_url (or database.url) in autumn.toml, or AUTUMN_DATABASE__URL",
),
}
}
#[must_use]
pub fn grade_migrate_check(migrations_dir: &Path) -> PreflightCheck {
grade_migrate_check_for(
crate::generate::detect_backend_offline(Path::new("."), None),
migrations_dir,
)
}
#[must_use]
pub fn grade_migrate_check_for(
backend: autumn_web::config::DatabaseBackend,
migrations_dir: &Path,
) -> PreflightCheck {
if !migrations_dir.exists() {
return PreflightCheck::pass(
"migrate_check",
"no migrations directory (nothing to check)",
);
}
match crate::migrate::check_migrations_in_dir_for(backend, migrations_dir) {
Ok(reports) => {
let unsafe_names: Vec<&str> = reports
.iter()
.filter(|r| crate::migrate::safety::has_unsafe_findings(&r.up))
.map(|r| r.name.as_str())
.collect();
if unsafe_names.is_empty() {
PreflightCheck::pass(
"migrate_check",
format!("{} migration(s) safe for a rolling deploy", reports.len()),
)
} else {
PreflightCheck::fail(
"migrate_check",
format!(
"migration(s) unsafe for a rolling deploy: {}",
unsafe_names.join(", ")
),
"Run `autumn migrate check` and apply the expand/contract pattern before deploying",
)
}
}
Err(e) => PreflightCheck::fail(
"migrate_check",
format!("could not scan migrations: {e}"),
"Run `autumn migrate check` to see the full report",
),
}
}
#[must_use]
pub fn render_systemd_unit(cfg: &ResolvedDeployConfig) -> String {
format!(
"[Unit]\n\
Description=Autumn application: {app}\n\
After=network-online.target\n\
Wants=network-online.target\n\
\n\
[Service]\n\
Type=simple\n\
User={user}\n\
WorkingDirectory={current}\n\
EnvironmentFile={env_file}\n\
Environment=AUTUMN_MANIFEST_DIR={manifest_dir}\n\
ExecStart={current}/{app}\n\
Restart=on-failure\n\
RestartSec=2\n\
\n\
[Install]\n\
WantedBy=multi-user.target\n",
app = cfg.app_name,
user = cfg.user,
current = cfg.current_symlink(),
env_file = cfg.env_file(),
manifest_dir = cfg.current_symlink(),
)
}
#[must_use]
pub fn render_app_unit(
cfg: &ResolvedDeployConfig,
release_dir: &str,
app_port: u16,
slot: &str,
) -> String {
format!(
"[Unit]\n\
Description=Autumn application: {app} ({slot})\n\
After=network-online.target\n\
Wants=network-online.target\n\
\n\
[Service]\n\
Type=simple\n\
User={user}\n\
WorkingDirectory={release_dir}\n\
EnvironmentFile={env_file}\n\
Environment=AUTUMN_MANIFEST_DIR={manifest_dir}\n\
Environment={maintenance_env}={maintenance_flag}\n\
Environment=AUTUMN_SERVER__HOST=127.0.0.1\n\
Environment=AUTUMN_SERVER__PORT={app_port}\n\
ExecStart={release_dir}/{app}\n\
Restart=on-failure\n\
RestartSec=2\n\
\n\
[Install]\n\
WantedBy=multi-user.target\n",
app = cfg.app_name,
user = cfg.user,
env_file = cfg.env_file(),
manifest_dir = release_dir,
maintenance_env = autumn_web::maintenance::MAINTENANCE_FLAG_FILE_ENV,
maintenance_flag = cfg.maintenance_flag_file(),
)
}
#[must_use]
pub fn build_deploy_plan(cfg: &ResolvedDeployConfig) -> Vec<DeployStep> {
let mut steps = vec![
DeployStep::new(
"build",
"Build the embedded single-binary release locally (`autumn build --embed`)",
),
DeployStep::new(
"prepare-host",
"Ensure the target has a working kamal-proxy — the FIRST thing that touches the \
server. A host that already has one is untouched; a host that has none gets the \
pinned build installed, which also installs (and leaves behind) any of `curl` \
and a container runtime it is missing, and pulls the pinned proxy image from \
Docker Hub. Decline with `[deploy] install_proxy = false`",
),
DeployStep::new(
"upload",
format!(
"Upload the binary to a new timestamped release dir under {}",
cfg.releases_dir()
),
),
DeployStep::new(
"migrate",
"Run pending migrations BEFORE the new release takes traffic — on a redeploy an \
abort here leaves the current version serving; on a first deploy the release is \
never started",
),
DeployStep::new(
"start-candidate",
"Start the new release as a candidate on a SEPARATE listener (a distinct \
port/socket from the live service) — it does NOT bind the live \
`server.port`, so the old release keeps serving traffic uninterrupted",
),
DeployStep::new(
"readiness-gate",
format!(
"Poll the candidate's /ready on its separate listener within {}s — roll back on \
timeout",
cfg.readiness_timeout_secs
),
),
DeployStep::new(
"cutover",
format!(
"Hand live traffic over to the candidate — reverse-proxy upstream swap or systemd \
socket-activation handoff (mechanism finalized in the execution slice) — then \
promote it by pointing the {} symlink at the new release",
cfg.current_symlink(),
),
),
DeployStep::new(
"drain",
"Drain and stop the previous release once traffic has moved to the candidate",
),
DeployStep::new(
"prune",
format!(
"Prune old releases, retaining the most recent {} — always keeping the releases \
the current symlink and previous-release marker point at (rollback targets)",
cfg.keep_releases
),
),
];
insert_sqlite_data_steps(cfg, &mut steps);
steps
}
fn insert_sqlite_data_steps(cfg: &ResolvedDeployConfig, steps: &mut Vec<DeployStep>) {
let data = if let Some(shared) = cfg.shared_sqlite_data_file() {
let rel = cfg.relative_sqlite_data_file().unwrap_or_default();
DeployStep::new(
"link-data",
format!(
"Link the release's {rel} at the persistent {shared}. Refuses when {current}/{rel} \
is a real file or links elsewhere (a one-time adoption the refusal explains), or \
when {shared} is gone but {marker} records that it existed",
current = cfg.current_symlink(),
marker = cfg.sqlite_data_marker_file(),
),
)
} else if let Some(path) = cfg.persistent_sqlite_data_file() {
DeployStep::new(
"check-data-dir",
format!(
"Check on the host that {path} resolves outside {releases}, and create its \
directory when it is in {data}",
releases = cfg.releases_dir(),
data = cfg.shared_data_dir(),
),
)
} else {
return;
};
let at = |label: &str, steps: &[DeployStep]| steps.iter().position(|s| s.label == label);
if let Some(upload) = at("upload", steps) {
steps.insert(upload, data);
}
if let Some(migrate) = at("migrate", steps) {
steps.insert(
migrate.saturating_add(1),
DeployStep::new(
"record-data-adopted",
format!(
"Record {} once the database exists in {}, so a later missing volume is \
refused, not replaced with an empty database",
cfg.sqlite_data_marker_file(),
cfg.shared_data_dir()
),
),
);
}
}
#[must_use]
pub fn build_rollback_plan(cfg: &ResolvedDeployConfig) -> Vec<DeployStep> {
vec![
DeployStep::new(
"write-target-unit",
format!(
"Re-render the previous release's {} slot unit from the persisted \
marker (its dir + port) so the restart never relaunches a slot unit \
an earlier failed redeploy left pointing at a removed candidate",
cfg.service_name
),
),
DeployStep::new(
"daemon-reload",
"Reload systemd so the restart below loads the freshly re-rendered unit",
),
DeployStep::new(
"restart-previous",
format!(
"Restart the previous release's {} slot unit so it is healthy \
before the proxy flips traffic back to it",
cfg.service_name
),
),
DeployStep::new(
"proxy-flip",
"Flip the reverse-proxy upstream back to the previous release \
(health-gated on /ready before traffic moves)",
),
DeployStep::new(
"record-previous",
"Record the release being rolled back FROM (its dir + former-live slot) \
as the new previous-release marker so a subsequent rollback returns to it",
),
DeployStep::new(
"repoint",
format!(
"Point the {} symlink back at the previous release",
cfg.current_symlink()
),
),
DeployStep::new(
"record-live-slot",
"Record the previous slot as the live slot so the next deploy's mode \
detection sees it serving",
),
DeployStep::new(
"readiness-gate",
format!(
"Re-probe /ready within {}s to confirm the rollback is healthy",
cfg.readiness_timeout_secs
),
),
DeployStep::new(
"drain-rolled-back-slot",
"Disable the slot that was live before the rollback (the one traffic \
moved away from) so the next deploy sees it genuinely idle and starts \
the new binary fresh",
),
]
}
const DEPLOY_TIER_TWO_POLICY_COMMAND: &str = "autumn deploy up / rollback / status / maintenance";
const fn reaches_a_remote_host(action: DeployAction) -> bool {
match action {
DeployAction::Check | DeployAction::Plan => false,
DeployAction::Up
| DeployAction::Rollback
| DeployAction::Status
| DeployAction::MaintenanceOn
| DeployAction::MaintenanceOff => true,
}
}
fn windows_deploy_refusal(os: &str, action: DeployAction) -> Option<String> {
(os == "windows" && reaches_a_remote_host(action))
.then(|| crate::platform::tier_two_windows_error(DEPLOY_TIER_TWO_POLICY_COMMAND))
}
pub fn run(action: DeployAction, options: &DeployOptions) -> Result<(), DeployError> {
if let Some(refusal) = windows_deploy_refusal(std::env::consts::OS, action) {
return Err(DeployError::UnsupportedPlatform(refusal));
}
let ambient_config = AutumnConfig::load_lenient_unknown_roots()
.map_err(|e| DeployError::Config(e.to_string()))?;
let deploy_cfg = ambient_config.deploy.unwrap_or_default();
let host_list = deploy_host_list(&deploy_cfg).map_err(DeployError::Config)?;
let targets = select_hosts(&host_list, &options.only).map_err(DeployError::Config)?;
for line in fleet::only_selection_warning_lines(&host_list, &targets) {
eprintln!("{line}");
}
let resolved = ResolvedDeployConfig::resolve(&deploy_cfg, &resolve_project_name())
.map_err(DeployError::Config)?;
match action {
DeployAction::Plan => {
let (media_cfg, _ffmpeg_bin) = load_media_host_config(&resolved)?;
let (planned, note) = match load_runtime_config(&resolved.profile) {
Ok(config) => (
with_sqlite_data_contract(&resolved, &config),
sqlite_plan_note(&config, &resolved),
),
Err(e) => (
resolved.clone(),
Some(format!(
"the `{}` profile config did not load, so SQLite data steps are not \
shown: {e}",
resolved.profile
)),
),
};
print_plan(&planned, &host_list, &media_cfg, note.as_deref());
Ok(())
}
DeployAction::Check => run_check(
&load_runtime_config(&resolved.profile)?,
&resolved,
&targets,
host_list.len(),
),
DeployAction::Rollback => run_rollback(
&load_runtime_config(&resolved.profile)?,
&resolved,
&targets,
host_list.len(),
),
DeployAction::Status => {
let fleet = ResolvedFleet::resolve(&deploy_cfg, &resolve_project_name())
.map_err(DeployError::Config)?;
let port = status_public_port(&resolved)?;
run_status(
&port,
&resolved.app_name,
&resolved.profile,
&fleet,
options,
)
}
DeployAction::MaintenanceOn | DeployAction::MaintenanceOff => {
let fleet = ResolvedFleet::resolve(&deploy_cfg, &resolve_project_name())
.map_err(DeployError::Config)?;
let port = status_public_port(&resolved)?;
run_fleet_maintenance_with(
&fleet,
&port,
&resolved.profile,
action,
options.maintenance.as_ref(),
|cfg| {
exec::SshTarget::from_resolved(cfg)
.map(exec::SshExecutor::new)
.ok_or_else(|| DeployError::Config(DEPLOY_HOST_MISSING_DETAIL.to_owned()))
},
)
}
DeployAction::Up => {
let (media_cfg, ffmpeg_bin) = load_media_host_config(&resolved)?;
run_up(
&load_runtime_config(&resolved.profile)?,
&resolved,
&targets,
host_list.len(),
&media_cfg,
&ffmpeg_bin,
options,
)
}
}
}
fn load_media_host_config(
resolved: &ResolvedDeployConfig,
) -> Result<(media::MediaMtxHostConfig, String), DeployError> {
load_media_host_config_in(&manifest_project_dirs(), &resolved.profile)
}
fn load_media_host_config_in(
dirs: &[PathBuf],
profile_raw: &str,
) -> Result<(media::MediaMtxHostConfig, String), DeployError> {
let mut merged = toml::Value::Table(toml::map::Map::new());
let base_toml: Option<toml::Value> = match first_dir_with_file(dirs, "autumn.toml") {
Some(base_path) => match std::fs::read_to_string(&base_path) {
Ok(base_str) => {
let base: toml::Value = toml::from_str(&base_str).map_err(|e| {
DeployError::Config(format!("invalid config in {}: {e}", base_path.display()))
})?;
deep_merge_toml(&mut merged, base.clone());
Some(base)
}
Err(_) => None,
},
None => None,
};
let canonical = canonicalize_deploy_profile(profile_raw);
if let Some(base) = &base_toml {
for name in profile_inline_lookup_names(&canonical) {
if let Some(section) = profile_section_from_base_toml(base, name) {
deep_merge_toml(&mut merged, section);
}
}
}
for name in autumn_web::config::profile_override_file_lookup_names(&canonical, profile_raw) {
let basename = format!("autumn-{name}.toml");
if let Some(path) = first_dir_with_file(dirs, &basename) {
let overlay_str = std::fs::read_to_string(&path).map_err(|e| {
DeployError::Config(format!("could not read {}: {e}", path.display()))
})?;
let overlay: toml::Value = toml::from_str(&overlay_str).map_err(|e| {
DeployError::Config(format!("invalid config in {}: {e}", path.display()))
})?;
deep_merge_toml(&mut merged, overlay);
break;
}
}
media::media_host_config_from_value(merged).map_err(|e| {
DeployError::Config(format!(
"invalid [media] config for deploy profile `{profile_raw}`: {e}"
))
})
}
fn profile_inline_lookup_names(canonical: &str) -> Vec<&str> {
match canonical {
"prod" => vec!["production", "prod"],
"dev" => vec!["development", "dev"],
other => vec![other],
}
}
fn profile_section_from_base_toml(base: &toml::Value, profile: &str) -> Option<toml::Value> {
base.get("profile")
.and_then(toml::Value::as_table)
.and_then(|profiles| profiles.get(profile))
.and_then(toml::Value::as_table)
.map(|table| toml::Value::Table(table.clone()))
}
fn deep_merge_toml(base: &mut toml::Value, overlay: toml::Value) {
deep_merge_toml_depth(base, overlay, 0);
}
fn deep_merge_toml_depth(base: &mut toml::Value, overlay: toml::Value, depth: usize) {
const MAX_MERGE_DEPTH: usize = 16;
if depth > MAX_MERGE_DEPTH {
return;
}
let toml::Value::Table(overlay_table) = overlay else {
return;
};
let Some(base_table) = base.as_table_mut() else {
return;
};
for (key, overlay_val) in overlay_table {
let is_recursive_merge =
overlay_val.is_table() && base_table.get(&key).is_some_and(toml::Value::is_table);
if is_recursive_merge {
if let Some(base_val) = base_table.get_mut(&key) {
deep_merge_toml_depth(base_val, overlay_val, depth + 1);
}
} else {
base_table.insert(key, overlay_val);
}
}
}
struct ForcedProfileEnv<E: Env> {
profile: String,
inner: E,
}
impl<E: Env> Env for ForcedProfileEnv<E> {
fn var(&self, key: &str) -> Result<String, std::env::VarError> {
if key == "AUTUMN_ENV" {
return Ok(self.profile.clone());
}
if key == "AUTUMN_DOTENV" {
return self
.inner
.var("AUTUMN_DOTENV")
.or_else(|_| Ok("1".to_owned()));
}
self.inner.var(key)
}
}
fn load_runtime_config(profile: &str) -> Result<AutumnConfig, DeployError> {
let forced = deploy_profile_env_overlay(profile)?;
AutumnConfig::load_with_env_lenient_unknown_roots(&forced)
.map_err(|e| DeployError::Config(e.to_string()))
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct StatusPort {
port: u16,
degraded: Option<String>,
}
fn status_public_port(resolved: &ResolvedDeployConfig) -> Result<StatusPort, DeployError> {
let loaded = load_runtime_config(&resolved.profile);
status_public_port_with(&manifest_project_dirs(), &resolved.profile, loaded)
}
fn status_public_port_with(
dirs: &[PathBuf],
profile_raw: &str,
loaded: Result<AutumnConfig, DeployError>,
) -> Result<StatusPort, DeployError> {
match loaded {
Ok(config) => Ok(StatusPort {
port: config.server.port,
degraded: None,
}),
Err(err) => {
let overlay = deploy_profile_env_overlay(profile_raw)?;
Ok(StatusPort {
port: declared_server_port_in(dirs, profile_raw, &overlay)
.unwrap_or_else(|| AutumnConfig::default().server.port),
degraded: Some(err.to_string()),
})
}
}
}
fn declared_server_port_in(dirs: &[PathBuf], profile_raw: &str, env: &dyn Env) -> Option<u16> {
if let Ok(raw) = env.var("AUTUMN_SERVER__PORT")
&& let Ok(port) = raw.trim().parse::<u16>()
{
return Some(port);
}
let mut merged = toml::Value::Table(toml::map::Map::new());
let base_toml: Option<toml::Value> = first_dir_with_file(dirs, "autumn.toml")
.and_then(|path| std::fs::read_to_string(path).ok())
.and_then(|text| toml::from_str::<toml::Value>(&text).ok());
let canonical = canonicalize_deploy_profile(profile_raw);
if let Some(base) = &base_toml {
deep_merge_toml(&mut merged, base.clone());
for name in profile_inline_lookup_names(&canonical) {
if let Some(section) = profile_section_from_base_toml(base, name) {
deep_merge_toml(&mut merged, section);
}
}
}
for name in autumn_web::config::profile_override_file_lookup_names(&canonical, profile_raw) {
let Some(path) = first_dir_with_file(dirs, &format!("autumn-{name}.toml")) else {
continue;
};
if let Ok(text) = std::fs::read_to_string(&path)
&& let Ok(overlay) = toml::from_str::<toml::Value>(&text)
{
deep_merge_toml(&mut merged, overlay);
}
break;
}
merged
.get("server")
.and_then(|server| server.get("port"))
.and_then(toml::Value::as_integer)
.and_then(|port| u16::try_from(port).ok())
}
#[allow(clippy::redundant_pub_crate)]
pub(crate) fn deploy_profile_env_overlay(profile_raw: &str) -> Result<impl Env, DeployError> {
use autumn_web::config::OsEnv;
let gating = ForcedProfileEnv {
profile: profile_raw.to_owned(),
inner: OsEnv,
};
let dotenv_profile = canonicalize_deploy_profile(profile_raw);
let inner = autumn_web::dotenv::os_env_with_dotenv_for_profile_using(&gating, &dotenv_profile)
.map_err(|e| DeployError::Config(e.to_string()))?;
Ok(ForcedProfileEnv {
profile: profile_raw.to_owned(),
inner,
})
}
fn resolve_project_name() -> String {
let cwd = std::env::current_dir().unwrap_or_else(|_| PathBuf::from("."));
crate::release::read_project_name(&cwd)
.ok()
.or_else(|| {
cwd.file_name()
.map(|n| n.to_string_lossy().into_owned())
.filter(|n| !n.is_empty())
})
.unwrap_or_else(|| "app".to_owned())
}
fn resolve_writable_db_url(db: &autumn_web::config::DatabaseConfig) -> Option<&str> {
db.effective_primary_url()
.or_else(|| db.shards.first().map(|shard| shard.primary_url.as_str()))
}
fn sqlite_data_placement(
config: &AutumnConfig,
resolved: &ResolvedDeployConfig,
) -> SqliteDataPlacement {
match classify_sqlite_data_file(resolve_writable_db_url(&config.database), resolved) {
SqliteDataFile::Relative(rel) => SqliteDataPlacement::Relative(rel),
SqliteDataFile::Persistent(path) => SqliteDataPlacement::Persistent(path),
SqliteDataFile::NotSqlite | SqliteDataFile::Refused(_) => SqliteDataPlacement::None,
}
}
fn with_sqlite_data_contract(
resolved: &ResolvedDeployConfig,
config: &AutumnConfig,
) -> ResolvedDeployConfig {
resolved
.clone()
.with_release_payloads(release_payload_basenames(config, resolved))
.with_sqlite_data_placement(sqlite_data_placement(config, resolved))
}
fn sqlite_plan_note(config: &AutumnConfig, resolved: &ResolvedDeployConfig) -> Option<String> {
match classify_sqlite_data_file(resolve_writable_db_url(&config.database), resolved) {
SqliteDataFile::Refused(reason) => Some(reason),
SqliteDataFile::NotSqlite | SqliteDataFile::Relative(_) | SqliteDataFile::Persistent(_) => {
None
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum SqliteDataFile {
NotSqlite,
Relative(String),
Persistent(String),
Refused(String),
}
#[must_use]
pub fn classify_sqlite_data_file(url: Option<&str>, cfg: &ResolvedDeployConfig) -> SqliteDataFile {
let Some(url) = url.map(str::trim).filter(|u| !u.is_empty()) else {
return SqliteDataFile::NotSqlite;
};
if autumn_web::config::DatabaseBackend::detect(url)
!= Some(autumn_web::config::DatabaseBackend::Sqlite)
{
return SqliteDataFile::NotSqlite;
}
let Some(path) = autumn_web::replication::database_file(url) else {
return SqliteDataFile::Refused(
"the configured SQLite database is IN-MEMORY, so it cannot survive a deploy \
(or a restart). Point `[database] url` at a file, e.g. \
sqlite:///srv/autumn/<app>/shared/data/app.db"
.to_owned(),
);
};
let Some(raw) = path.to_str().map(str::to_owned) else {
return SqliteDataFile::Refused(format!(
"the configured SQLite database path ({}) is not valid UTF-8, so a deploy \
cannot address it. Rename the file.",
path.display()
));
};
let text = lexically_normalized(&raw);
let app_dir = lexically_normalized(&cfg.app_dir);
if !app_dir.starts_with('/') {
return SqliteDataFile::Refused(format!(
"the deploy's app directory ({}) is not an absolute path, so where the \
SQLite database {text} would live cannot be decided. Set `[deploy] \
app_dir` to an absolute path.",
cfg.app_dir
));
}
if text.starts_with('/') {
let shared_data = lexically_normalized(&cfg.shared_data_dir());
if within(&text, &app_dir) && !within(&text, &shared_data) {
return SqliteDataFile::Refused(format!(
"the configured SQLite database {text} lives inside the deploy's own app \
directory ({app_dir}), where only {} is yours: `releases/` is replaced on \
every deploy and deleted by release retention, `current` is a symlink into \
it, and the rest of `shared/` holds deploy state files that would \
overwrite the database. Move the file to {}, or outside {app_dir} \
altogether.",
cfg.shared_data_dir(),
cfg.shared_data_dir()
));
}
return SqliteDataFile::Persistent(text);
}
if text.is_empty() || text == ".." || text.starts_with("../") {
return SqliteDataFile::Refused(format!(
"the configured SQLite database path {text:?} does not name a file inside the \
release directory it is resolved against, so a deploy cannot keep it durable. \
Use a plain relative name (sqlite://app.db) or an absolute path."
));
}
if let Some(payload) = collides_with_release_payload(&text, cfg) {
return SqliteDataFile::Refused(format!(
"the configured SQLite database {text} is also a file this deploy uploads \
into the release directory ({payload}), which would overwrite the database \
on the first deploy. Rename the database, or move it under a \
subdirectory (sqlite://data/app.db)."
));
}
SqliteDataFile::Relative(text)
}
fn collides_with_release_payload(text: &str, cfg: &ResolvedDeployConfig) -> Option<String> {
let text = text.split('/').next().filter(|c| !c.is_empty())?;
if text == cfg.app_name {
return Some(format!("the {} binary", cfg.app_name));
}
let is_manifest = text == "autumn.toml"
|| (text.starts_with("autumn-")
&& text.len() > "autumn-.toml".len()
&& text.rsplit_once('.').is_some_and(|(_, ext)| ext == "toml"));
if is_manifest {
return Some(format!("the config manifest {text}"));
}
cfg.release_payloads
.iter()
.find(|payload| payload.as_str() == text)
.map(|payload| format!("the uploaded file {payload}"))
}
fn lexically_normalized(path: &str) -> String {
let absolute = path.starts_with('/');
let mut names: Vec<&str> = Vec::new();
for segment in path.split('/') {
match segment {
"" | "." => {}
".." => {
if names.last().is_some_and(|name| *name != "..") {
names.pop();
} else if !absolute {
names.push("..");
}
}
name => names.push(name),
}
}
let joined = names.join("/");
if absolute {
format!("/{joined}")
} else {
joined
}
}
fn within(path: &str, root: &str) -> bool {
let root = root.trim_end_matches('/');
path == root || path.starts_with(&format!("{root}/"))
}
#[must_use]
pub fn grade_sqlite_data_persistence(
url: Option<&str>,
cfg: &ResolvedDeployConfig,
) -> Option<PreflightCheck> {
match classify_sqlite_data_file(url, cfg) {
SqliteDataFile::NotSqlite => None,
SqliteDataFile::Relative(rel) => Some(PreflightCheck::pass(
"sqlite_data_file",
format!(
"SQLite data file kept in {}/{rel} across releases",
cfg.shared_data_dir()
),
)),
SqliteDataFile::Persistent(path) => Some(PreflightCheck::pass(
"sqlite_data_file",
format!("SQLite data file {path} is outside the releases directory"),
)),
SqliteDataFile::Refused(detail) => Some(PreflightCheck::fail(
"sqlite_data_file",
detail,
"Set `[database] url` to a path a deploy can keep: a plain relative name \
(sqlite://app.db, kept under the app's shared/data dir) or an absolute path \
outside the releases directory",
)),
}
}
fn collect_preflight(
config: &AutumnConfig,
resolved: &ResolvedDeployConfig,
) -> Vec<PreflightCheck> {
let mut checks = vec![grade_ssh_reachability(
resolved.host.as_deref(),
resolved.ssh_port,
SSH_PROBE_TIMEOUT,
)];
checks.extend(collect_project_preflight(config, resolved));
checks
}
fn collect_project_preflight(
config: &AutumnConfig,
resolved: &ResolvedDeployConfig,
) -> Vec<PreflightCheck> {
let database_configured = config.database.url.is_some()
|| config.database.primary_url.is_some()
|| config.database.replica_url.is_some()
|| config.database.has_shards();
let mut checks = vec![
grade_signing_secret(
config.security.signing_secret.secret.as_deref(),
&config.security.signing_secret.previous_secrets,
is_production_profile(Some(&canonicalize_deploy_profile(&resolved.profile))),
),
grade_database_url(
resolve_writable_db_url(&config.database),
Path::new(MIGRATIONS_DIR),
database_configured,
requires_database_pool(config),
),
grade_migrate_check_for(
resolve_writable_db_url(&config.database)
.and_then(autumn_web::config::DatabaseBackend::detect)
.unwrap_or(autumn_web::config::DatabaseBackend::Postgres),
Path::new(MIGRATIONS_DIR),
),
];
checks.extend(grade_sqlite_data_persistence(
resolve_writable_db_url(&config.database),
resolved,
));
checks
}
fn collect_fleet_preflight(
config: &AutumnConfig,
fleet: &ResolvedFleet,
configured_host_count: usize,
) -> Vec<PreflightCheck> {
let Some(shared) = fleet.hosts.first() else {
return vec![grade_ssh_reachability(None, 0, SSH_PROBE_TIMEOUT)];
};
if is_single_host_deploy(fleet, configured_host_count) {
return collect_preflight(config, shared);
}
let mut checks: Vec<PreflightCheck> = fleet
.hosts
.iter()
.map(|cfg| {
grade_ssh_reachability(cfg.host.as_deref(), cfg.ssh_port, SSH_PROBE_TIMEOUT)
.scoped(cfg.host.clone())
})
.collect();
checks.extend(collect_project_preflight(config, shared));
checks
}
#[must_use]
#[allow(clippy::redundant_pub_crate)]
pub(crate) fn is_production_profile(profile: Option<&str>) -> bool {
matches!(profile, Some("prod" | "production"))
}
#[must_use]
fn requires_database_pool(config: &AutumnConfig) -> bool {
matches!(config.jobs.backend.as_str(), "postgres" | "sqlite")
|| matches!(
config.scheduler.backend,
autumn_web::config::SchedulerBackend::Postgres
| autumn_web::config::SchedulerBackend::Sqlite
)
}
fn preflight_label(check: &PreflightCheck) -> String {
check.scope.as_ref().map_or_else(
|| check.name.to_owned(),
|scope| format!("{} ({scope})", check.name),
)
}
fn report_preflight(checks: &[PreflightCheck]) -> usize {
let mut failed = 0_usize;
for check in checks {
let label = preflight_label(check);
if check.passed {
eprintln!("\u{2705} {label}: {}", check.detail);
} else if check.deferred {
eprintln!("\u{26A0}\u{FE0F} {label}: {}", check.detail);
if let Some(hint) = check.hint {
eprintln!(" \u{2192} {hint}");
}
} else {
failed += 1;
eprintln!("\u{274C} {label}: {}", check.detail);
if let Some(hint) = check.hint {
eprintln!(" \u{2192} {hint}");
}
}
}
eprintln!();
failed
}
fn refuse_if_preflight_failed(checks: &[PreflightCheck]) -> Result<(), DeployError> {
let failed = report_preflight(checks);
if failed > 0 {
return Err(DeployError::PreflightFailed(failed));
}
Ok(())
}
fn run_check(
config: &AutumnConfig,
resolved: &ResolvedDeployConfig,
hosts: &[String],
configured_host_count: usize,
) -> Result<(), DeployError> {
eprintln!("\u{1F342} autumn deploy check\n");
let fleet = ResolvedFleet::from_targets(resolved, hosts);
let checks = collect_fleet_preflight(config, &fleet, configured_host_count);
let failed = report_preflight(&checks);
eprintln!(
"{}",
manifest_preflight_notice(&locate_manifest_uploads(resolved))
);
if failed == 0 {
eprintln!("\u{2705} All {} preflight check(s) passed.", checks.len());
Ok(())
} else {
eprintln!(
"\u{274C} {failed} of {} preflight check(s) failed.",
checks.len()
);
Err(DeployError::PreflightFailed(failed))
}
}
fn print_plan(
resolved: &ResolvedDeployConfig,
hosts: &[String],
media_cfg: &media::MediaMtxHostConfig,
sqlite_note: Option<&str>,
) {
println!("\u{1F342} autumn deploy plan (dry-run)\n");
println!("systemd unit ({}.service):\n", resolved.service_name);
println!("{}", render_systemd_unit(resolved));
println!("Deploy steps (zero-downtime):");
for (i, step) in build_deploy_plan(resolved).iter().enumerate() {
println!(" {}. [{}] {}", i + 1, step.label, step.description);
}
if let Some(note) = sqlite_note {
println!("\n SQLite data file: {note}");
}
if hosts.len() > 1 {
for line in fleet::fleet_plan_lines(hosts) {
println!("{line}");
}
}
if media_cfg.enabled {
print_media_plan(media_cfg);
}
}
fn print_media_plan(media_cfg: &media::MediaMtxHostConfig) {
let controller = media::MediaMtxController::new(media_cfg.clone());
println!(
"\nMediaMTX host provisioning (media unit {}.service):\n",
media_cfg.unit_name
);
println!("{}", media::render_mediamtx_unit(media_cfg));
if media_cfg.install_binary {
println!(
"Before cutover: [media-install-binary] installs MediaMTX {} at {} if nothing \
is there (pinned image digest).",
media::MEDIAMTX_KNOWN_GOOD_VERSION,
media_cfg.binary_path,
);
}
println!("MediaMTX provisioning steps:");
for (i, op) in controller.ensure_installed_ops().iter().enumerate() {
println!(" {}. [{}]", i + 1, op.label());
}
println!(
"\nMediaMTX doctor checks at `deploy up`: {}, {}, {}, {}, {}, {}",
media::CHECK_MEDIAMTX_PORTS_DISTINCT,
media::CHECK_MEDIAMTX_PORTS_MATCH_BASES,
media::CHECK_FFMPEG_PREFLIGHT,
media::CHECK_MEDIAMTX_BINARY,
media::CHECK_RECORDINGS_DIR_WRITABLE,
media::CHECK_MEDIAMTX_PORTS_AVAILABLE,
);
println!(
"CSP: the app must allow these MediaMTX origins in connect-src/media-src \
(frame-src for WebRTC): {}",
media_cfg.required_csp_origins().join(", "),
);
}
fn check_media_host_preflight(
media_cfg: &media::MediaMtxHostConfig,
ffmpeg_bin: &str,
executor: &impl exec::DeployExecutor,
) -> Result<(), DeployError> {
if !media_cfg.enabled {
return Ok(());
}
eprintln!("\u{1F3A5} media host preflight (autumn-media)\n");
let checks = media::collect_media_doctor_checks(executor, media_cfg, ffmpeg_bin);
let failed = report_preflight(&checks);
if failed > 0 {
return Err(DeployError::PreflightFailed(failed));
}
Ok(())
}
fn provision_media_host(
media_cfg: &media::MediaMtxHostConfig,
executor: &impl exec::DeployExecutor,
) -> Result<(), DeployError> {
if !media_cfg.enabled {
return Ok(());
}
eprintln!("\u{1F3A5} provisioning MediaMTX host (autumn-media)\n");
let controller = media::MediaMtxController::new(media_cfg.clone());
exec::run_ops(&controller.ensure_installed_ops(), executor)
.map_err(|e| DeployError::Exec(e.to_string()))?;
eprintln!(
"\u{2705} MediaMTX provisioned ({}.service)",
media_cfg.unit_name
);
Ok(())
}
fn media_binary_install_ops(media_cfg: &media::MediaMtxHostConfig) -> Vec<exec::DeployOp> {
media::MediaMtxController::new(media_cfg.clone()).binary_install_ops()
}
fn build_env_file(config: &AutumnConfig, resolved: &ResolvedDeployConfig) -> exec::Secret {
let mut body = String::new();
body.push_str("AUTUMN_ENV=");
body.push_str(&resolved.profile);
body.push('\n');
if let Some(secret) = config
.security
.signing_secret
.secret
.as_deref()
.map(str::trim)
.filter(|s| !s.is_empty())
{
body.push_str("AUTUMN_SECURITY__SIGNING_SECRET=");
body.push_str(secret);
body.push('\n');
}
if let Some(url) = resolve_writable_db_url(&config.database) {
body.push_str("AUTUMN_DATABASE__URL=");
body.push_str(url);
body.push('\n');
}
exec::Secret::new(body)
}
fn resolve_release_binary(resolved: &ResolvedDeployConfig) -> Result<PathBuf, DeployError> {
let path = PathBuf::from("target")
.join("release")
.join(&resolved.app_name);
if path.exists() {
Ok(path)
} else {
Err(DeployError::BinaryMissing(path.display().to_string()))
}
}
fn default_release_id() -> String {
chrono::Utc::now().format("%Y%m%dT%H%M%SZ").to_string()
}
fn manifest_project_dirs() -> Vec<PathBuf> {
let cwd = std::env::current_dir().unwrap_or_else(|_| PathBuf::from("."));
if let Ok(dir) = std::env::var("AUTUMN_MANIFEST_DIR")
&& !dir.trim().is_empty()
{
let manifest_dir = PathBuf::from(dir);
if manifest_dir == cwd {
return vec![cwd];
}
return vec![manifest_dir, cwd];
}
vec![cwd]
}
fn first_dir_with_file(dirs: &[PathBuf], filename: &str) -> Option<PathBuf> {
dirs.iter().map(|d| d.join(filename)).find(|p| p.is_file())
}
fn manifest_uploads_in(dirs: &[PathBuf], profile: &str) -> Vec<exec::ManifestUpload> {
let mut uploads = Vec::new();
if let Some(base) = first_dir_with_file(dirs, "autumn.toml") {
uploads.push(exec::ManifestUpload {
local: base,
remote_basename: "autumn.toml".to_owned(),
});
}
let normalized =
autumn_web::config::normalize_profile_name(profile).unwrap_or_else(|| "prod".to_owned());
for name in autumn_web::config::profile_override_file_lookup_names(&normalized, profile) {
let basename = format!("autumn-{name}.toml");
if let Some(local) = first_dir_with_file(dirs, &basename) {
uploads.push(exec::ManifestUpload {
local,
remote_basename: basename,
});
break;
}
}
if let Some(configured) = configured_capacity_contract(dirs, profile) {
if Path::new(&configured)
.parent()
.is_some_and(|p| p != Path::new(""))
{
eprintln!(
"\u{26A0}\u{FE0F} warning: [server] capacity_contract is '{configured}', which is \
not beside autumn.toml.\n The release directory is flat, so this file is NOT \
uploaded and the deployed app will\n fall back to unlimited admission. Move the \
contract next to autumn.toml, or ship it\n yourself as part of your release."
);
} else if let Some(local) = first_dir_with_file(dirs, &configured) {
uploads.push(exec::ManifestUpload {
local,
remote_basename: configured,
});
}
}
uploads
}
fn configured_capacity_contract(dirs: &[PathBuf], profile: &str) -> Option<String> {
let mut merged = first_dir_with_file(dirs, "autumn.toml")
.and_then(|path| std::fs::read_to_string(path).ok())
.and_then(|text| toml::from_str::<toml::Value>(&text).ok())
.unwrap_or_else(|| toml::Value::Table(toml::map::Map::new()));
let canonical =
autumn_web::config::normalize_profile_name(profile).unwrap_or_else(|| "prod".to_owned());
for name in profile_inline_lookup_names(&canonical) {
if let Some(section) = profile_section_from_base_toml(&merged.clone(), name) {
deep_merge_toml(&mut merged, section);
}
}
for name in autumn_web::config::profile_override_file_lookup_names(&canonical, profile) {
if let Some(local) = first_dir_with_file(dirs, &format!("autumn-{name}.toml")) {
if let Ok(text) = std::fs::read_to_string(&local)
&& let Ok(overlay) = toml::from_str::<toml::Value>(&text)
{
deep_merge_toml(&mut merged, overlay);
}
break;
}
}
merged
.get("server")
.and_then(|server| server.get("capacity_contract"))
.and_then(toml::Value::as_str)
.filter(|value| !value.is_empty())
.map(str::to_owned)
}
fn locate_manifest_uploads(resolved: &ResolvedDeployConfig) -> Vec<exec::ManifestUpload> {
manifest_uploads_in(&manifest_project_dirs(), &resolved.profile)
}
fn release_payload_basenames(
_config: &AutumnConfig,
resolved: &ResolvedDeployConfig,
) -> Vec<String> {
let mut names = vec![resolved.app_name.clone()];
names.extend(
locate_manifest_uploads(resolved)
.into_iter()
.map(|upload| upload.remote_basename),
);
names
}
fn manifest_preflight_notice(uploads: &[exec::ManifestUpload]) -> String {
if uploads.is_empty() {
return "\u{26A0}\u{FE0F} warning: no autumn.toml found in the project directory; the \
deployed app will run built-in defaults for all non-secret settings (secrets \
still come from the env file). Add an autumn.toml to control the deployed \
configuration."
.to_owned();
}
let names: Vec<&str> = uploads.iter().map(|u| u.remote_basename.as_str()).collect();
format!(
"Uploading project config to the release dir: {}",
names.join(", ")
)
}
const PROXY_HTTPS_PORT: u16 = 443;
fn validate_public_port(public_port: u16) -> Result<(), String> {
let blue_port = exec::slot_app_port(public_port, exec::SLOT_BLUE);
let green_port = exec::slot_app_port(public_port, exec::SLOT_GREEN);
let collision = if public_port == PROXY_HTTPS_PORT {
Some("the proxy's HTTP listener")
} else if blue_port == PROXY_HTTPS_PORT {
Some("the blue app slot")
} else if green_port == PROXY_HTTPS_PORT {
Some("the green app slot")
} else {
None
};
if let Some(which) = collision {
return Err(format!(
"deploy public/HTTP port {public_port} is invalid — {which} would land on \
{PROXY_HTTPS_PORT}, which kamal-proxy reserves for its HTTPS listener \
(blue/green app slots bind `public+1`/`public+2`); use 80 (the default) or \
another port outside {}\u{2013}{PROXY_HTTPS_PORT}",
PROXY_HTTPS_PORT - 2,
));
}
Ok(())
}
fn resolve_public_port_move(
installed: &exec::InstalledProxyPort,
new_public_port: u16,
) -> Result<Option<exec::PublicPortMove>, String> {
match installed {
exec::InstalledProxyPort::Absent => Ok(None),
exec::InstalledProxyPort::Port(current) if *current == new_public_port => Ok(None),
exec::InstalledProxyPort::Port(old_port) => Ok(Some(exec::PublicPortMove {
old_port: *old_port,
new_port: new_public_port,
})),
exec::InstalledProxyPort::Unreadable => Err(format!(
"Cannot verify the installed kamal-proxy's HTTP port before a redeploy \
(its systemd unit is present but its `--http-port` could not be read), so \
a concurrent server.port change (config requests {new_public_port}) cannot \
be ruled out and is refused to avoid stranding public traffic mid-cutover. \
Re-provision the host (or repair the kamal-proxy unit) and retry."
)),
}
}
fn refuse_unprovable_proxy_options(marker: &exec::ProxyOptionsMarker) -> Result<(), String> {
match marker {
exec::ProxyOptionsMarker::Absent | exec::ProxyOptionsMarker::Options(_) => Ok(()),
exec::ProxyOptionsMarker::Unreadable => Err(
"Cannot verify the last-deployed proxy TLS/host options before a redeploy \
(the shared/proxy-options marker is present but unreadable), so a concurrent \
deploy.tls.host change cannot be preserved across the one-time reboot-durability \
restart and is refused to avoid stranding the live release behind the wrong \
TLS/host on a rollback. Redeploy with deploy.tls unchanged to repair the marker, \
then change the host in a separate deploy. Tracked in #2074."
.to_owned(),
),
}
}
fn refuse_sqlite_fleet(configured_hosts: usize, writable_db_sqlite: bool) -> Result<(), String> {
if configured_hosts > 1 && writable_db_sqlite {
return Err(
"\u{2717} A multi-host deploy cannot run on SQLite. `[deploy] hosts` lists more \
than one server and the configured `[database]` URL is a sqlite:// target, but \
every host receives the SAME database URL — which means N independent database \
FILES, one per machine: no shared schema, no advisory lock to serialize \
migrations, and each host serving a different dataset behind your load \
balancer. Point `[database] url` at a Postgres server every host can reach, or \
deploy a single host with `[deploy] host`. Tracking: #1621."
.to_owned(),
);
}
Ok(())
}
fn refuse_media_fleet(configured_hosts: usize, media_enabled: bool) -> Result<(), String> {
if configured_hosts > 1 && media_enabled {
return Err(
"\u{2717} A multi-host deploy cannot provision MediaMTX. `[deploy] hosts` lists \
more than one server and `[media.mediamtx] enabled = true`, but host media \
provisioning has no teardown or rollback path: fanning it out would leave one \
MediaMTX daemon per host on identical ports, with divergent recording sets and \
nothing to undo them with when the app rollout is rolled back. Deploy media on \
a single host with `[deploy] host`, or set `[media.mediamtx] enabled = false` \
and provision the media host separately. Tracking: #1621."
.to_owned(),
);
}
Ok(())
}
fn refuse_tls_fleet(configured_hosts: usize, tls_enabled: bool) -> Result<(), String> {
if configured_hosts > 1 && tls_enabled {
return Err(
"\u{2717} A multi-host deploy cannot terminate TLS per host. `[deploy] hosts` \
lists more than one server and `[deploy.tls] enabled = true`, but the \
deploy-managed proxy runs on EVERY host: each one would request a certificate \
for the same `[deploy.tls] host` from behind your load balancer, only one can \
answer a given ACME challenge, and the rest burn Let's Encrypt \
failed-validation and duplicate-certificate rate limits. Terminate TLS at the \
load balancer that fronts the fleet and set `[deploy.tls] enabled = false`. \
Tracking: #1621."
.to_owned(),
);
}
Ok(())
}
const FLEET_AUTO_MIGRATE_WARNING: &str = "`[database] auto_migrate` is on and this deploy targets more than one host: every \
host applies migrations at boot, so the hosts RACE each other during the rollout \
and a checksum mismatch exits the process under `Restart=on-failure` — a crash \
loop mid-rollout. Turn it off for fleets and let the rollout's single `migrate` \
step own the schema (#1621).";
fn fleet_auto_migrate_warning(
configured_hosts: usize,
auto_migrate: Option<bool>,
auto_migrate_in_production: bool,
) -> Option<&'static str> {
let enabled = auto_migrate.unwrap_or(auto_migrate_in_production);
(configured_hosts > 1 && enabled).then_some(FLEET_AUTO_MIGRATE_WARNING)
}
const FLEET_SHARED_BACKEND_WARNING_TAIL: &str = "each host then runs its OWN copy, so work enqueued on one host is only ever run by \
that host, whatever is still queued dies when the next rollout drains that host's \
old slot, `unique`/`concurrency` limits stop being enforced fleet-wide, and every \
scheduled task fires once PER HOST. Move them to a shared backend (`postgres` or \
`redis`) before you rely on background work across the fleet \u{2014} see \
docs/guide/fleet-deploys.md (#1621).";
fn fleet_shared_backend_warning(
configured_hosts: usize,
jobs_backend: &str,
scheduler_in_process: bool,
) -> Option<String> {
if configured_hosts <= 1 {
return None;
}
let jobs_local = jobs_backend.trim().eq_ignore_ascii_case("local");
let keys = match (jobs_local, scheduler_in_process) {
(true, true) => {
"`[jobs] backend = \"local\"` (an in-process queue) and `[scheduler] \
backend = \"in_process\"` (a per-process timer) are in effect"
}
(true, false) => "`[jobs] backend = \"local\"` (an in-process queue) is in effect",
(false, true) => {
"`[scheduler] backend = \"in_process\"` (a per-process timer) is in effect"
}
(false, false) => return None,
};
Some(format!(
"{keys} and this deploy targets more than one host \u{2014} they are PER HOST: \
{FLEET_SHARED_BACKEND_WARNING_TAIL}"
))
}
fn writable_db_is_sqlite(db: &autumn_web::config::DatabaseConfig) -> bool {
resolve_writable_db_url(db).is_some_and(|url| {
matches!(
autumn_web::config::DatabaseBackend::detect(url),
Some(autumn_web::config::DatabaseBackend::Sqlite)
)
})
}
fn run_up(
config: &AutumnConfig,
resolved: &ResolvedDeployConfig,
hosts: &[String],
configured_host_count: usize,
media_cfg: &media::MediaMtxHostConfig,
ffmpeg_bin: &str,
options: &DeployOptions,
) -> Result<(), DeployError> {
eprintln!("\u{1F342} autumn deploy up\n");
let resolved = &with_sqlite_data_contract(resolved, config);
let fleet = ResolvedFleet::from_targets(resolved, hosts);
let checks = collect_fleet_preflight(config, &fleet, configured_host_count);
refuse_if_preflight_failed(&checks)?;
let binary = resolve_release_binary(resolved)?;
let env_file = build_env_file(config, resolved);
let manifests = locate_manifest_uploads(resolved);
eprintln!("{}", manifest_preflight_notice(&manifests));
let release_id = default_release_id();
let public_port = config.server.port;
validate_public_port(public_port).map_err(DeployError::Config)?;
let proxy = proxy::KamalProxyController::new(resolved.readiness_timeout_secs)
.with_tls_host(resolved.tls_host.clone());
let bake_policy = resolve_bake_policy(config, options.bake_secs)?;
if let Some(policy) = &bake_policy {
eprintln!("{}", policy.describe());
for warning in &policy.warnings {
eprintln!("\u{26A0}\u{FE0F} {warning}");
}
eprintln!();
}
let metrics_path = bake::metrics_path(&config.actuator.prefix);
let bake_host = bake::host_header(&config.security.trusted_hosts.hosts);
run_up_with(
&FleetUpInput {
app_name: &resolved.app_name,
profile: &resolved.profile,
fleet: &fleet,
proxy: &proxy,
checks: &checks,
env_file: &env_file,
binary: &binary,
manifests: &manifests,
release_id: &release_id,
public_port,
media_cfg,
ffmpeg_bin,
configured_host_count,
db: FleetDatabaseFacts {
sqlite: writable_db_is_sqlite(&config.database),
auto_migrate: config.database.auto_migrate,
auto_migrate_in_production: config.database.auto_migrate_in_production,
},
jobs_backend: &config.jobs.backend,
scheduler_in_process: matches!(
config.scheduler.backend,
autumn_web::config::SchedulerBackend::InProcess
),
auto_rollback: !options.no_rollback,
bake: bake_policy.as_ref(),
metrics_path: &metrics_path,
bake_host: bake_host.as_deref(),
},
|cfg| {
exec::SshTarget::from_resolved(cfg)
.map(exec::SshExecutor::new)
.ok_or_else(|| DeployError::Config(DEPLOY_HOST_MISSING_DETAIL.to_owned()))
},
)
}
fn resolve_bake_policy(
config: &AutumnConfig,
flag: Option<u64>,
) -> Result<Option<bake::BakePolicy>, DeployError> {
let bake_config = config
.deploy
.as_ref()
.map(|deploy| deploy.bake.clone())
.unwrap_or_default();
if flag.unwrap_or(bake_config.duration_secs) == 0 {
return Ok(None);
}
let slos =
autumn_web::slo::validate(&config.slo).map_err(|e| DeployError::Config(e.to_string()))?;
let prod = matches!(config.profile.as_deref(), Some("prod" | "production"));
let hosts = &config.security.trusted_hosts.hosts;
if prod && bake::host_header(hosts).is_none() && !hosts.iter().any(|h| h.trim() == "*") {
return Err(DeployError::Config(
"the bake needs a trusted host in the prod profile: add your domain to \
[security.trusted_hosts] hosts, or turn the bake off"
.to_owned(),
));
}
bake::resolve_policy(&bake_config, &slos, flag).map_err(DeployError::Config)
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
struct FleetDatabaseFacts {
sqlite: bool,
auto_migrate: Option<bool>,
auto_migrate_in_production: bool,
}
struct FleetUpInput<'a, P: ProxyController> {
app_name: &'a str,
profile: &'a str,
fleet: &'a ResolvedFleet,
proxy: &'a P,
checks: &'a [PreflightCheck],
env_file: &'a exec::Secret,
binary: &'a Path,
manifests: &'a [exec::ManifestUpload],
release_id: &'a str,
public_port: u16,
media_cfg: &'a media::MediaMtxHostConfig,
ffmpeg_bin: &'a str,
configured_host_count: usize,
db: FleetDatabaseFacts,
jobs_backend: &'a str,
scheduler_in_process: bool,
auto_rollback: bool,
bake: Option<&'a bake::BakePolicy>,
metrics_path: &'a str,
bake_host: Option<&'a str>,
}
struct HostProbeState {
mode: fleet::HostMode,
slots: exec::SlotPlan,
repair: bool,
reregister_options: exec::ProxyServiceOptions,
banner: String,
needs_proxy_install: bool,
port_move: Option<exec::PublicPortMove>,
}
const fn provisioning(cfg: &ResolvedDeployConfig) -> exec::ProxyProvisioning {
if cfg.install_proxy {
exec::ProxyProvisioning::Auto
} else {
exec::ProxyProvisioning::Disabled
}
}
fn probe_host_for_up<E, P>(
cfg: &ResolvedDeployConfig,
input: &FleetUpInput<'_, P>,
executor: &E,
) -> Result<HostProbeState, DeployError>
where
E: exec::DeployExecutor,
P: ProxyController,
{
let host = cfg.host.as_deref().unwrap_or_default();
let single = is_single_host_deploy(input.fleet, input.configured_host_count);
let scoped = |message: String| {
if single {
message
} else {
format!("host {host}: {message}")
}
};
let needs_proxy_install = matches!(
exec::assess_proxy_readiness(input.proxy, executor, provisioning(cfg))
.map_err(|e| DeployError::Exec(scoped(e.to_string())))?,
exec::ProxyReadiness::NeedsInstall
);
let probe = exec::probe_deploy_state(cfg, executor)
.map_err(|e| DeployError::Exec(scoped(e.to_string())))?;
match exec::probe_release_dir(cfg, input.release_id, executor)
.map_err(|e| DeployError::Exec(scoped(e.to_string())))?
{
exec::ReleaseDirState::Absent => {}
exec::ReleaseDirState::Present => {
return Err(DeployError::Config(scoped(format!(
"release directory {}/{} already exists — a previous run of this release \
id already wrote into it, and re-uploading would put the NEW binary in \
the directory `shared/previous-release` points at (so a rollback would \
roll FORWARD). Wait a second and re-run `autumn deploy up`, or remove \
that directory if you are certain it is stale. Tracked in #1621.",
cfg.releases_dir(),
input.release_id,
))));
}
exec::ReleaseDirState::Unreadable => {
return Err(DeployError::Config(scoped(format!(
"cannot verify whether release directory {}/{} already exists (the probe \
returned neither sentinel), so a release-id collision cannot be ruled \
out and the deploy is refused rather than risk overwriting the release \
`shared/previous-release` points at. Tracked in #1621.",
cfg.releases_dir(),
input.release_id,
))));
}
}
match probe.mode {
exec::DeployMode::First => Ok(HostProbeState {
mode: fleet::HostMode::from_deploy_mode(&probe.mode),
slots: exec::SlotPlan::first(input.public_port),
repair: false,
reregister_options: input.proxy.proxy_service_options(),
banner: "first deploy".to_owned(),
needs_proxy_install,
port_move: None,
}),
exec::DeployMode::Redeploy { live_slot } => {
let port_move =
resolve_public_port_move(&probe.installed_proxy_port, input.public_port)
.map_err(|message| DeployError::Config(scoped(message)))?;
let effective_public_port = port_move.map_or(input.public_port, |m| m.old_port);
refuse_unprovable_proxy_options(&probe.last_proxy_options)
.map_err(|message| DeployError::Config(scoped(message)))?;
let reregister_options = match &probe.last_proxy_options {
exec::ProxyOptionsMarker::Options(old) => old.clone(),
_ => input.proxy.proxy_service_options(),
};
let reconcile = exec::reconcile_live_slot(
live_slot,
&probe.proxy_list,
&cfg.service_name,
effective_public_port,
);
if let Some(warn) = &reconcile.warn {
eprintln!("\u{26A0}\u{FE0F} {}", scoped(warn.clone()));
}
let slots = exec::SlotPlan::redeploy(effective_public_port, reconcile.live_slot);
let mut banner = format!(
"zero-downtime redeploy ({} \u{2192} {})",
slots.live_slot, slots.candidate_slot
);
if let Some(mv) = port_move {
banner = format!(
"{banner}, then moving server.port {} \u{2192} {}",
mv.old_port, mv.new_port
);
}
Ok(HostProbeState {
mode: fleet::HostMode::from_deploy_mode(&probe.mode),
slots,
repair: reconcile.repair,
reregister_options,
banner,
needs_proxy_install,
port_move,
})
}
}
}
#[allow(clippy::too_many_lines)]
fn run_up_with<E, P, F>(input: &FleetUpInput<'_, P>, make_executor: F) -> Result<(), DeployError>
where
E: exec::DeployExecutor,
P: ProxyController,
F: Fn(&ResolvedDeployConfig) -> Result<E, DeployError>,
{
let fleet = input.fleet;
let single = is_single_host_deploy(fleet, input.configured_host_count);
let total = fleet.hosts.len();
refuse_sqlite_fleet(input.configured_host_count, input.db.sqlite)
.map_err(DeployError::Config)?;
refuse_media_fleet(input.configured_host_count, input.media_cfg.enabled)
.map_err(DeployError::Config)?;
let shared = fleet
.hosts
.first()
.ok_or_else(|| DeployError::Config(DEPLOY_HOST_MISSING_DETAIL.to_owned()))?;
refuse_tls_fleet(input.configured_host_count, shared.tls_enabled)
.map_err(DeployError::Config)?;
if let Some(warning) = fleet_auto_migrate_warning(
input.configured_host_count,
input.db.auto_migrate,
input.db.auto_migrate_in_production,
) {
eprintln!("\u{26A0}\u{FE0F} {warning}\n");
}
if let Some(warning) = fleet_shared_backend_warning(
input.configured_host_count,
input.jobs_backend,
input.scheduler_in_process,
) {
eprintln!("\u{26A0}\u{FE0F} {warning}\n");
}
let executors = fleet
.hosts
.iter()
.map(&make_executor)
.collect::<Result<Vec<E>, DeployError>>()?;
check_media_host_preflight(input.media_cfg, input.ffmpeg_bin, &executors[0])?;
let probes = fleet
.hosts
.iter()
.zip(&executors)
.map(|(cfg, executor)| probe_host_for_up(cfg, input, executor))
.collect::<Result<Vec<HostProbeState>, DeployError>>()?;
let modes: Vec<fleet::HostMode> = probes.iter().map(|probe| probe.mode).collect();
let plan = fleet::plan_fleet(fleet, &modes).map_err(|e| DeployError::Config(e.to_string()))?;
if !single {
for line in fleet::fleet_rollout_lines(&plan, input.release_id) {
eprintln!("{line}");
}
eprintln!();
}
let mut outcomes = vec![fleet::HostOutcome::Untouched; total];
let mut degraded: Vec<(String, &'static str)> = Vec::new();
let mut halt: Option<(String, &'static str)> = None;
for (index, host_plan) in plan.hosts.iter().enumerate() {
let cfg = &fleet.hosts[index];
let state = &probes[index];
let executor = &executors[index];
let release_dir = format!("{}/{}", cfg.releases_dir(), input.release_id);
let unit = render_app_unit(
cfg,
&release_dir,
state.slots.candidate_port,
state.slots.candidate_slot,
);
let mut ops = fleet::host_ops(
host_plan,
&fleet::HostOpsInput {
cfg,
proxy: input.proxy,
unit: &unit,
env_file: input.env_file,
binary_local: input.binary,
manifests: input.manifests,
release_id: input.release_id,
slots: &state.slots,
reregister_options: &state.reregister_options,
},
);
if state.repair {
ops.insert(
0,
exec::live_slot_marker_repair_op(
cfg,
state.slots.live_slot,
state.slots.public_port,
),
);
}
ops.splice(0..0, media_binary_install_ops(input.media_cfg));
if state.needs_proxy_install {
eprintln!(
"{}host preparation: {}",
if single {
String::new()
} else {
format!("host {}: ", host_plan.host)
},
input.proxy.binary_install_note(),
);
let install = input.proxy.binary_install_ops().unwrap_or_default();
ops.splice(0..0, install);
}
let teardown = match host_plan.mode {
fleet::HostMode::First => {
exec::first_deploy_teardown_ops(cfg, input.release_id, &state.slots)
}
fleet::HostMode::Redeploy => {
exec::candidate_teardown_ops(cfg, input.release_id, &state.slots)
}
};
if single {
eprintln!(
"Deploying release {} to {} ({})\u{2026}\n",
input.release_id, host_plan.host, state.banner,
);
} else {
eprintln!(
"[{}/{total} {}] deploying release {} ({})\u{2026}",
index + 1,
host_plan.host,
input.release_id,
state.banner,
);
}
let result = match host_plan.mode {
fleet::HostMode::First => {
exec::execute_first_deploy(input.checks, &ops, &teardown, executor)
}
fleet::HostMode::Redeploy => {
exec::execute_redeploy(input.checks, &ops, &teardown, executor)
}
};
match result {
Ok(()) => {
if let Some(mv) = state.port_move {
let options = input.proxy.proxy_service_options();
let rebind_ops = exec::public_port_rebind_ops(
cfg,
input.proxy,
input.release_id,
state.slots.candidate_port,
&options,
mv.new_port,
);
let rollback_ops = exec::public_port_rebind_ops(
cfg,
input.proxy,
input.release_id,
state.slots.candidate_port,
&options,
mv.old_port,
);
if let Err(rebind_err) = exec::execute_public_port_rebind(
&rebind_ops,
&rollback_ops,
mv.old_port,
mv.new_port,
executor,
) {
if single {
if matches!(rebind_err, exec::PublicPortRebindError::RolledBack { .. })
&& let BakeStep::Fail(bake_error) = bake_host(
input,
index,
single,
host_plan,
cfg,
&state.slots,
executor,
)
{
eprintln!("\u{26A0}\u{FE0F} {rebind_err}");
return Err(bake_error);
}
return Err(DeployError::Exec(rebind_err.to_string()));
}
match rebind_err {
exec::PublicPortRebindError::RolledBack { .. } => {
outcomes[index] = fleet::HostOutcome::Degraded {
label: "public-port-rebind",
};
degraded.push((host_plan.host.clone(), "public-port-rebind"));
eprintln!(
"\u{26A0}\u{FE0F} [{}/{total} {}] serving {} \u{2014} \
{rebind_err}\n",
index + 1,
host_plan.host,
input.release_id,
);
if matches!(
bake_host(
input,
index,
single,
host_plan,
cfg,
&state.slots,
executor
),
BakeStep::Halt | BakeStep::Fail(_)
) {
outcomes[index] = fleet::HostOutcome::LiveOnNew {
failed_step: bake::BAKE_LABEL,
};
halt = Some((host_plan.host.clone(), bake::BAKE_LABEL));
break;
}
continue;
}
exec::PublicPortRebindError::RollbackFailed { .. } => {
outcomes[index] = fleet::HostOutcome::Manual {
reason: fleet::MANUAL_PUBLIC_PORT_UNKNOWN,
};
eprintln!(
"\n\u{274C} rollout halted at {} (`public-port-rebind`) \
\u{2014} the remaining hosts were not touched \u{2014} \
{rebind_err}\n",
host_plan.host,
);
halt = Some((host_plan.host.clone(), "public-port-rebind"));
break;
}
}
}
}
match bake_host(input, index, single, host_plan, cfg, &state.slots, executor) {
BakeStep::Continue => {}
BakeStep::Fail(error) => return Err(error),
BakeStep::Halt => {
outcomes[index] = fleet::HostOutcome::LiveOnNew {
failed_step: bake::BAKE_LABEL,
};
halt = Some((host_plan.host.clone(), bake::BAKE_LABEL));
break;
}
}
outcomes[index] = fleet::HostOutcome::Serving;
if !single {
if host_plan.migrate == exec::MigrateStep::Run {
eprintln!("\u{26A0}\u{FE0F} {}", fleet::FLEET_SCHEMA_MOVED_NOTE);
}
eprintln!(
"\u{2705} [{}/{total} {}] serving {}\n",
index + 1,
host_plan.host,
input.release_id,
);
}
}
Err(err) => {
outcomes[index] = fleet::classify_host_outcome(&err);
if single {
if matches!(outcomes[index], fleet::HostOutcome::Degraded { .. })
&& let BakeStep::Fail(bake_error) =
bake_host(input, index, single, host_plan, cfg, &state.slots, executor)
{
eprintln!("\u{26A0}\u{FE0F} {err}");
return Err(bake_error);
}
let message = fleet::single_host_schema_note(&plan, &outcomes).map_or_else(
|| err.to_string(),
|note| format!("{err}\n\u{26A0}\u{FE0F} {note}"),
);
return Err(DeployError::Exec(message));
}
let failed_step = fleet::failed_step_label(&err);
if matches!(
outcomes[index],
fleet::HostOutcome::Degraded { label } if label == fleet::DRAIN_OLD_LABEL
) {
let prefix = format!("[{}/{total} {}]", index + 1, host_plan.host);
let old_slot = match exec::retry_drain_old(cfg, state.slots.live_slot, executor)
{
Ok(old_slot) => old_slot,
Err(retry_err) => {
eprintln!("\u{26A0}\u{FE0F} {prefix} the retry failed: {retry_err}");
exec::OldSlotState::Unreadable
}
};
outcomes[index] = fleet::drain_old_outcome(old_slot);
if old_slot == exec::OldSlotState::Stopped {
eprintln!(
"\u{26A0}\u{FE0F} {prefix} `{}` failed, but the retry stopped and \
disabled the old slot.",
fleet::DRAIN_OLD_LABEL,
);
} else {
eprintln!(
"\u{26A0}\u{FE0F} {prefix} serving {} \u{2014} but `{}` failed \
AFTER the cutover: {}.",
input.release_id,
fleet::DRAIN_OLD_LABEL,
fleet::OLD_SLOT_MAY_RUN_NOTE,
);
}
}
if let fleet::HostOutcome::Degraded { label } = outcomes[index] {
degraded.push((host_plan.host.clone(), label));
let refusal = if label == "record-proxy-options" {
", which will refuse it"
} else {
""
};
eprintln!(
"\u{26A0}\u{FE0F} [{}/{total} {}] serving {} \u{2014} but `{label}` \
failed AFTER the cutover. Traffic is healthy, so the rollout \
continues; repair this host (a redeploy does) before the next \
deploy{refusal}.\n",
index + 1,
host_plan.host,
input.release_id,
);
if matches!(
bake_host(input, index, single, host_plan, cfg, &state.slots, executor),
BakeStep::Halt | BakeStep::Fail(_)
) {
outcomes[index] = fleet::HostOutcome::LiveOnNew {
failed_step: bake::BAKE_LABEL,
};
halt = Some((host_plan.host.clone(), bake::BAKE_LABEL));
break;
}
continue;
}
eprintln!(
"\n\u{274C} rollout halted at {} (`{failed_step}`) \u{2014} the remaining \
hosts were not touched.\n",
host_plan.host,
);
halt = Some((host_plan.host.clone(), failed_step));
break;
}
}
}
if let Some((failed_host, failed_step)) = halt {
if input.auto_rollback {
compensate_fleet(input, &plan, &probes, &executors, &mut outcomes);
}
for line in fleet::fleet_summary_lines(&plan, &outcomes, input.release_id) {
eprintln!("{line}");
}
let error = fleet_halted(&plan, &outcomes, °raded, failed_host, failed_step);
if let DeployError::FleetHalted(ref halt) = error {
emit_fleet_halted_alert(halt, input.app_name, input.profile);
}
return Err(error);
}
{
let executor = &executors[0];
provision_media_host(input.media_cfg, executor)?;
}
if single {
eprintln!("\n\u{2705} Deploy complete. Roll back with `autumn deploy rollback`.");
} else {
for line in fleet::fleet_summary_lines(&plan, &outcomes, input.release_id) {
eprintln!("{line}");
}
eprintln!(
"{}",
fleet::fleet_completion_line(
input.release_id,
total,
input.configured_host_count,
degraded.len(),
)
);
}
Ok(())
}
enum BakeStep {
Continue,
Halt,
Fail(DeployError),
}
fn bake_host<E, P>(
input: &FleetUpInput<'_, P>,
index: usize,
single: bool,
host_plan: &fleet::HostPlan,
cfg: &ResolvedDeployConfig,
slots: &exec::SlotPlan,
executor: &E,
) -> BakeStep
where
E: exec::DeployExecutor,
P: ProxyController,
{
let Some(policy) = input.bake else {
return BakeStep::Continue;
};
let prefix = if single {
String::new()
} else {
format!(
"[{}/{} {}] ",
index + 1,
input.fleet.hosts.len(),
host_plan.host
)
};
if matches!(host_plan.mode, fleet::HostMode::First) {
eprintln!("{prefix}bake skipped: a first deploy has no previous release to roll back to.");
return BakeStep::Continue;
}
let unit = exec::slot_unit_name(&cfg.service_name, slots.candidate_slot);
let target = bake::BakeTarget {
port: slots.candidate_port,
metrics_path: input.metrics_path,
host_header: input.bake_host,
unit: &unit,
};
let breach = match bake::run(policy, &target, executor, &mut |line| {
eprintln!("{prefix}{line}");
}) {
bake::BakeOutcome::Passed { responses, judged } => {
if judged {
eprintln!("{prefix}bake passed ({responses} responses).");
} else {
eprintln!(
"\u{26A0}\u{FE0F} {prefix}bake passed with no verdict: {responses} \
responses is fewer than {}.",
policy.min_requests
);
}
return BakeStep::Continue;
}
bake::BakeOutcome::Breached(breach) => breach,
};
eprintln!("\n\u{274C} {prefix}bake failed: {breach}\n");
if !single {
return BakeStep::Halt;
}
let result = if input.auto_rollback {
match compensate_rollback(cfg, input, slots.public_port, executor) {
fleet::HostOutcome::CompensatedRollback => BAKE_ROLLED_BACK,
fleet::HostOutcome::Manual { .. } => BAKE_ROLLBACK_REFUSED,
_ => BAKE_ROLLBACK_FAILED,
}
} else {
BAKE_FROZEN
};
let schema_note = (result == BAKE_ROLLED_BACK && host_plan.migrate == exec::MigrateStep::Run)
.then_some(fleet::SINGLE_HOST_SCHEMA_AHEAD_NOTE);
if result != BAKE_ROLLED_BACK
&& let Err(error) = provision_media_host(input.media_cfg, executor)
{
eprintln!("\u{26A0}\u{FE0F} {error}");
}
BakeStep::Fail(DeployError::BakeFailed {
host: host_plan.host.clone(),
reason: breach.to_string(),
result,
schema_note,
})
}
fn compensate_fleet<E, P>(
input: &FleetUpInput<'_, P>,
plan: &fleet::FleetPlan,
probes: &[HostProbeState],
executors: &[E],
outcomes: &mut [fleet::HostOutcome],
) where
E: exec::DeployExecutor,
P: ProxyController,
{
let modes: Vec<fleet::HostMode> = plan.hosts.iter().map(|host| host.mode).collect();
let actions = fleet::fleet_rollback_set(outcomes, &modes);
if actions.is_empty() {
return;
}
eprintln!(
"\u{21A9}\u{FE0F} Compensating {} host(s) in reverse rollout order \u{2014} BINARIES \
only; a migration that already ran is NOT rolled back.\n",
actions.len(),
);
for action in actions {
let index = match action {
fleet::RollbackAction::Rollback(index)
| fleet::RollbackAction::Teardown(index)
| fleet::RollbackAction::Manual(index, _) => index,
};
let cfg = &input.fleet.hosts[index];
let host = plan.hosts[index].host.as_str();
outcomes[index] = match action {
fleet::RollbackAction::Rollback(_) => {
eprintln!(
"\u{21A9}\u{FE0F} [{host}] rolling back to the previous release\u{2026}"
);
compensate_rollback(
cfg,
input,
probes[index].slots.public_port,
&executors[index],
)
}
fleet::RollbackAction::Teardown(_) => {
eprintln!(
"\u{21A9}\u{FE0F} [{host}] removing this run's first deploy \
(no previous release to return to)\u{2026}"
);
compensate_teardown(cfg, input, &probes[index].slots, &executors[index])
}
fleet::RollbackAction::Manual(_, reason) => {
for line in fleet::manual_recovery_lines(cfg, reason) {
eprintln!("{line}");
}
if matches!(outcomes[index], fleet::HostOutcome::AmbiguousMarkers) {
fleet::HostOutcome::AmbiguousMarkers
} else {
fleet::HostOutcome::Manual { reason }
}
}
};
eprintln!();
}
}
fn compensate_rollback<E, P>(
cfg: &ResolvedDeployConfig,
input: &FleetUpInput<'_, P>,
public_port: u16,
executor: &E,
) -> fleet::HostOutcome
where
E: exec::DeployExecutor,
P: ProxyController,
{
let target = match exec::resolve_rollback_target(cfg, public_port, executor) {
Ok(target) => target,
Err(exec::DeployExecError::NoPreviousRelease) => {
return manual_outcome(cfg, fleet::MANUAL_NO_PREVIOUS_RELEASE);
}
Err(_) => {
return manual_outcome(cfg, fleet::MANUAL_ROLLBACK_TARGET_UNPROVABLE);
}
};
match exec::probe_rollback_target_dir(&target.release_dir, executor) {
Ok(exec::ReleaseDirState::Present) => {}
Ok(exec::ReleaseDirState::Absent) => {
return manual_outcome(cfg, fleet::MANUAL_ROLLBACK_TARGET_MISSING);
}
Ok(exec::ReleaseDirState::Unreadable) | Err(_) => {
return manual_outcome(cfg, fleet::MANUAL_ROLLBACK_TARGET_UNPROVABLE);
}
}
let ops = exec::rollback_ops(cfg, input.proxy, &target);
let teardown = exec::rollback_teardown_ops(cfg, &target);
match exec::execute_rollback(input.checks, &ops, &teardown, executor) {
Ok(()) => fleet::HostOutcome::CompensatedRollback,
Err(err) => {
let failed_step = fleet::failed_step_label(&err);
eprintln!(
"\u{274C} [{}] the compensating rollback FAILED at `{failed_step}` \u{2014} this \
host is still on {}. The remaining hosts are still compensated.",
cfg.host.as_deref().unwrap_or_default(),
input.release_id,
);
fleet::HostOutcome::CompensationFailed { failed_step }
}
}
}
fn compensate_teardown<E, P>(
cfg: &ResolvedDeployConfig,
input: &FleetUpInput<'_, P>,
slots: &exec::SlotPlan,
executor: &E,
) -> fleet::HostOutcome
where
E: exec::DeployExecutor,
P: ProxyController,
{
let app_ops = exec::first_deploy_teardown_ops(cfg, input.release_id, slots);
if let Err(err) = exec::run_ops(&app_ops, executor) {
let failed_step = fleet::failed_step_label(&err);
eprintln!(
"\u{274C} [{}] removing the first deploy FAILED at `{failed_step}` \u{2014} this \
host is still on {}. The remaining hosts are still compensated.",
cfg.host.as_deref().unwrap_or_default(),
input.release_id,
);
return fleet::HostOutcome::CompensationFailed { failed_step };
}
let deregister = input.proxy.deregister_op(&cfg.service_name);
match exec::run_ops(&[deregister], executor) {
Ok(()) => fleet::HostOutcome::CompensatedTeardown,
Err(err) => {
let failed_step = fleet::failed_step_label(&err);
eprintln!(
"\u{26A0}\u{FE0F} [{}] removed the first deploy, but its proxy route removal \
FAILED at `{failed_step}` \u{2014} its public port may still answer 502 until it \
is redeployed or the route is removed by hand.",
cfg.host.as_deref().unwrap_or_default(),
);
fleet::HostOutcome::CompensatedTeardownRouteFailed { failed_step }
}
}
}
fn manual_outcome(cfg: &ResolvedDeployConfig, reason: &'static str) -> fleet::HostOutcome {
for line in fleet::manual_recovery_lines(cfg, reason) {
eprintln!("{line}");
}
fleet::HostOutcome::Manual { reason }
}
fn alert_channels_for(
alerts: &autumn_web::alerts::AlertConfig,
http: &autumn_web::config::HttpConfig,
) -> Vec<Arc<dyn AlertChannel>> {
if !alerts.is_active() {
return Vec::new();
}
let client = autumn_web::http::Client::from_config(&http.client);
crate::alert::configured_http_channels(alerts, &client)
}
fn declared_alert_config_in(
dirs: &[PathBuf],
profile_raw: &str,
) -> Option<(
autumn_web::alerts::AlertConfig,
autumn_web::config::HttpConfig,
)> {
#[derive(Deserialize, Default)]
#[serde(default)]
struct AlertsHttpTomlRoot {
alerts: autumn_web::alerts::AlertConfig,
http: autumn_web::config::HttpConfig,
}
let mut merged = toml::Value::Table(toml::map::Map::new());
let base_toml: Option<toml::Value> = first_dir_with_file(dirs, "autumn.toml")
.and_then(|path| std::fs::read_to_string(path).ok())
.and_then(|text| toml::from_str::<toml::Value>(&text).ok());
let canonical = canonicalize_deploy_profile(profile_raw);
if let Some(base) = &base_toml {
deep_merge_toml(&mut merged, base.clone());
for name in profile_inline_lookup_names(&canonical) {
if let Some(section) = profile_section_from_base_toml(base, name) {
deep_merge_toml(&mut merged, section);
}
}
}
for name in autumn_web::config::profile_override_file_lookup_names(&canonical, profile_raw) {
let Some(path) = first_dir_with_file(dirs, &format!("autumn-{name}.toml")) else {
continue;
};
let text = std::fs::read_to_string(&path).ok()?;
let overlay: toml::Value = toml::from_str(&text).ok()?;
deep_merge_toml(&mut merged, overlay);
break;
}
if merged_alerts_or_http_has_unknown_key(&merged) {
return None;
}
let root: AlertsHttpTomlRoot = merged.try_into().ok()?;
Some((root.alerts, root.http))
}
fn merged_alerts_or_http_has_unknown_key(merged: &toml::Value) -> bool {
let Ok(toml_str) = toml::to_string(merged) else {
return false;
};
let schema = AutumnConfig::get_schema_keys();
AutumnConfig::validate_toml(&toml_str, &schema)
.iter()
.any(|(path, _)| {
path == "alerts"
|| path.starts_with("alerts.")
|| path == "http"
|| path.starts_with("http.")
})
}
fn deploy_alert_channels_for_profile(profile: &str) -> Vec<Arc<dyn AlertChannel>> {
if let Ok(config) = load_runtime_config(profile) {
return alert_channels_for(&config.alerts, &config.http);
}
let Some((alerts, http)) = declared_alert_config_in(&manifest_project_dirs(), profile) else {
eprintln!(" \u{26A0} alert skipped: could not load configuration for profile {profile}");
return Vec::new();
};
let (alerts, http) = match deploy_profile_env_overlay(profile) {
Ok(env) => apply_alert_env_overrides(alerts, http, &env),
Err(_) => apply_alert_env_overrides(alerts, http, &autumn_web::config::OsEnv),
};
alert_channels_for(&alerts, &http)
}
fn apply_alert_env_overrides(
alerts: autumn_web::alerts::AlertConfig,
http: autumn_web::config::HttpConfig,
env: &dyn Env,
) -> (
autumn_web::alerts::AlertConfig,
autumn_web::config::HttpConfig,
) {
let mut scratch = AutumnConfig {
alerts: Box::new(alerts),
http,
..AutumnConfig::default()
};
scratch.apply_env_overrides_with_env(env);
(*scratch.alerts, scratch.http)
}
fn deliver_alert(channels: &[Arc<dyn AlertChannel>], alert: &Alert) {
if channels.is_empty() {
return;
}
let runtime = match tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
{
Ok(rt) => rt,
Err(e) => {
eprintln!(" \u{26A0} alert not sent: could not start async runtime: {e}");
return;
}
};
runtime.block_on(async {
for channel in channels {
if let Err(error) = channel.deliver(alert).await {
eprintln!(
" \u{26A0} alert not delivered via {}: {error}",
channel.name()
);
}
}
});
}
fn join_hosts(hosts: &[String]) -> String {
hosts.join(", ")
}
fn join_host_reasons(pairs: &[(String, &'static str)]) -> String {
pairs
.iter()
.map(|(host, reason)| format!("{host}: {reason}"))
.collect::<Vec<_>>()
.join("; ")
}
fn build_fleet_halted_alert(halt: &FleetHalt, app_name: &str, profile: &str) -> Alert {
let alert = Alert::trigger(
AlertCondition::ScheduledTaskFailure,
format!("scheduled_task_failure:deploy-fleet-halted:{app_name}:{profile}"),
)
.title("Fleet rollout halted")
.summary(halt.to_string())
.where_to_look("autumn deploy status")
.detail("app_name", app_name)
.detail("profile", profile)
.detail("failed_host", halt.failed_host.clone())
.detail("failed_step", halt.failed_step)
.detail("rolled_back", join_hosts(&halt.rolled_back))
.detail("torn_down", join_hosts(&halt.torn_down))
.detail("still_on_new", join_hosts(&halt.still_on_new))
.detail("degraded", join_host_reasons(&halt.degraded))
.detail(
"route_removal_failed",
join_host_reasons(&halt.route_removal_failed),
)
.detail("manual", join_host_reasons(&halt.manual));
match halt.risk() {
Some(risk) => alert.detail("risk", risk),
None => alert,
}
.build()
}
fn emit_fleet_halted_alert(halt: &FleetHalt, app_name: &str, profile: &str) {
let channels = deploy_alert_channels_for_profile(profile);
deliver_alert(
&channels,
&build_fleet_halted_alert(halt, app_name, profile),
);
}
fn drift_alert_summary(report: &fleet::DriftReport) -> String {
let mut parts = Vec::new();
if report.version_drift {
parts.push(format!(
"{} known release(s) are live at once",
report.known_releases().len()
));
}
if !report.state_drift.is_empty() {
parts.push(format!(
"{} host(s) have state drift",
report.state_drift.len()
));
}
parts.join("; ")
}
fn build_drift_alert(report: &fleet::DriftReport, app_name: &str, profile: &str) -> Alert {
Alert::trigger(
AlertCondition::ScheduledTaskFailure,
format!("scheduled_task_failure:deploy-drift:{app_name}:{profile}"),
)
.title("Fleet drift detected")
.summary(drift_alert_summary(report))
.where_to_look("autumn deploy status")
.detail("app_name", app_name)
.detail("profile", profile)
.detail("version_drift", report.version_drift.to_string())
.detail("state_drift", join_host_reasons(&report.state_drift))
.build()
}
fn emit_drift_alert(report: &fleet::DriftReport, app_name: &str, profile: &str) {
let channels = deploy_alert_channels_for_profile(profile);
deliver_alert(&channels, &build_drift_alert(report, app_name, profile));
}
fn fleet_halted(
plan: &fleet::FleetPlan,
outcomes: &[fleet::HostOutcome],
degraded: &[(String, &'static str)],
failed_host: String,
failed_step: &'static str,
) -> DeployError {
let named = |want: fn(&fleet::HostOutcome) -> bool| -> Vec<String> {
plan.hosts
.iter()
.zip(outcomes)
.filter(|(_, outcome)| want(outcome))
.map(|(host, _)| host.host.clone())
.collect()
};
DeployError::FleetHalted(Box::new(FleetHalt {
failed_host,
failed_step,
rolled_back: named(|o| {
matches!(
o,
fleet::HostOutcome::RolledBack { .. } | fleet::HostOutcome::CompensatedRollback
)
}),
torn_down: named(|o| {
matches!(
o,
fleet::HostOutcome::TornDown { .. }
| fleet::HostOutcome::CompensatedTeardown
| fleet::HostOutcome::CompensatedTeardownRouteFailed { .. }
)
}),
still_on_new: named(fleet::HostOutcome::on_new_release),
degraded: degraded.to_vec(),
route_removal_failed: plan
.hosts
.iter()
.zip(outcomes)
.filter_map(|(host, outcome)| match outcome {
fleet::HostOutcome::CompensatedTeardownRouteFailed { failed_step } => {
Some((host.host.clone(), *failed_step))
}
_ => None,
})
.collect(),
manual: plan
.hosts
.iter()
.zip(outcomes)
.filter_map(|(host, outcome)| match outcome {
fleet::HostOutcome::AmbiguousMarkers => {
Some((host.host.clone(), fleet::MANUAL_AMBIGUOUS_MARKERS))
}
fleet::HostOutcome::Manual { reason } => Some((host.host.clone(), *reason)),
_ => None,
})
.collect(),
}))
}
fn run_rollback(
config: &AutumnConfig,
resolved: &ResolvedDeployConfig,
hosts: &[String],
configured_host_count: usize,
) -> Result<(), DeployError> {
eprintln!("\u{1F342} autumn deploy rollback\n");
let resolved = &with_sqlite_data_contract(resolved, config);
let fleet = ResolvedFleet::from_targets(resolved, hosts);
let fleet_rollback = !fleet.is_single();
let checks = collect_fleet_preflight(config, &fleet, configured_host_count);
let failed = report_preflight(&checks);
if blocking_rollback_failures(&checks, fleet_rollback) > 0 {
return Err(DeployError::PreflightFailed(failed));
}
if failed > 0 {
eprintln!(
"\u{26A0}\u{FE0F} {failed} host(s) above did not answer the reachability probe. A \
ROLLBACK continues without them \u{2014} stopping would leave the hosts that ARE \
reachable on the release you are leaving \u{2014} but each is reported as NOT \
rolled back and this command still exits non-zero.\n"
);
}
eprintln!("Rollback steps:");
for (i, step) in build_rollback_plan(resolved).iter().enumerate() {
eprintln!(" {}. [{}] {}", i + 1, step.label, step.description);
}
eprintln!();
let public_port = config.server.port;
let proxy = proxy::KamalProxyController::new(resolved.readiness_timeout_secs)
.with_tls_host(resolved.tls_host.clone());
if fleet_rollback {
return run_fleet_rollback_with(&checks, &fleet, &proxy, public_port, |cfg| {
exec::SshTarget::from_resolved(cfg)
.map(exec::SshExecutor::new)
.ok_or_else(|| DeployError::Config(DEPLOY_HOST_MISSING_DETAIL.to_owned()))
});
}
let single = fleet
.hosts
.first()
.ok_or_else(|| DeployError::Config(DEPLOY_HOST_MISSING_DETAIL.to_owned()))?;
let target = exec::SshTarget::from_resolved(single)
.ok_or_else(|| DeployError::Config(DEPLOY_HOST_MISSING_DETAIL.to_owned()))?;
let executor = exec::SshExecutor::new(target);
let rollback_target = exec::resolve_rollback_target(single, public_port, &executor)
.map_err(|e| DeployError::Exec(e.to_string()))?;
eprintln!(
"Rolling back {} to {} ({})\u{2026}\n",
single.host.as_deref().unwrap_or_default(),
rollback_target.release_dir,
rollback_target.slot,
);
let ops = exec::rollback_ops(single, &proxy, &rollback_target);
let teardown = exec::rollback_teardown_ops(single, &rollback_target);
exec::execute_rollback(&checks, &ops, &teardown, &executor)
.map_err(|e| DeployError::Exec(e.to_string()))?;
eprintln!("\n\u{2705} Rollback complete.");
Ok(())
}
const SSH_REACHABILITY_CHECK: &str = "ssh_reachability";
fn is_per_host_reachability(check: &PreflightCheck) -> bool {
check.name == SSH_REACHABILITY_CHECK && check.scope.is_some()
}
fn blocking_rollback_failures(checks: &[PreflightCheck], fleet_rollback: bool) -> usize {
checks
.iter()
.filter(|check| check.blocking() && !(fleet_rollback && is_per_host_reachability(check)))
.count()
}
fn run_fleet_rollback_with<E, P, F>(
checks: &[PreflightCheck],
fleet: &ResolvedFleet,
proxy: &P,
public_port: u16,
make_executor: F,
) -> Result<(), DeployError>
where
E: exec::DeployExecutor,
P: ProxyController,
F: Fn(&ResolvedDeployConfig) -> Result<E, DeployError>,
{
let total = fleet.hosts.len();
let executors = fleet
.hosts
.iter()
.map(&make_executor)
.collect::<Result<Vec<E>, DeployError>>()?;
let names: Vec<String> = fleet
.hosts
.iter()
.map(|cfg| cfg.host.clone().unwrap_or_default())
.collect();
eprintln!(
"Rolling {total} hosts back to their previous release, ONE AT A TIME, NEWEST FIRST \
(the reverse of `[deploy] hosts` order):\n"
);
let mut outcomes = vec![fleet::RollbackOutcome::NoPreviousRelease; total];
for index in (0..total).rev() {
let cfg = &fleet.hosts[index];
let host = names[index].as_str();
eprintln!("[{}/{total} {host}] rolling back\u{2026}", index + 1);
let host_checks: Vec<PreflightCheck> = checks
.iter()
.filter(|check| {
!is_per_host_reachability(check) || check.scope.as_deref() == Some(host)
})
.cloned()
.collect();
outcomes[index] = if host_checks.iter().any(PreflightCheck::blocking) {
eprintln!(
"\u{274C} [{host}] did not answer the preflight reachability probe \u{2014} \
this host was NOT rolled back. The remaining hosts still are."
);
fleet::RollbackOutcome::Failed {
failed_step: SSH_REACHABILITY_CHECK,
}
} else {
rollback_one_host(
&host_checks,
cfg,
proxy,
public_port,
host,
&executors[index],
)
};
eprintln!();
}
for line in fleet::fleet_rollback_summary_lines(&names, &outcomes) {
eprintln!("{line}");
}
let rolled_back = outcomes
.iter()
.filter(|outcome| matches!(outcome, fleet::RollbackOutcome::RolledBack))
.count();
if rolled_back < total {
return Err(DeployError::FleetRollbackFailed {
not_rolled_back: total - rolled_back,
total,
});
}
eprintln!("\n\u{2705} Fleet rollback complete \u{2014} all {total} hosts restored.");
Ok(())
}
fn rollback_one_host<E, P>(
checks: &[PreflightCheck],
cfg: &ResolvedDeployConfig,
proxy: &P,
public_port: u16,
host: &str,
executor: &E,
) -> fleet::RollbackOutcome
where
E: exec::DeployExecutor,
P: ProxyController,
{
let target = match exec::resolve_rollback_target(cfg, public_port, executor) {
Ok(target) => target,
Err(exec::DeployExecError::NoPreviousRelease) => {
eprintln!(
"\u{26A0}\u{FE0F} [{host}] no previous release recorded \u{2014} nothing to \
roll back to on this host; it keeps serving what it is serving now."
);
return fleet::RollbackOutcome::NoPreviousRelease;
}
Err(err) => {
let failed_step = fleet::failed_step_label(&err);
eprintln!(
"\u{274C} [{host}] could not resolve the previous release (`{failed_step}`) \
\u{2014} this host was NOT rolled back. The remaining hosts still are."
);
return fleet::RollbackOutcome::Failed { failed_step };
}
};
eprintln!(
" \u{2192} {host} returns to {} ({})",
target.release_dir, target.slot,
);
let ops = exec::rollback_ops(cfg, proxy, &target);
let teardown = exec::rollback_teardown_ops(cfg, &target);
match exec::execute_rollback(checks, &ops, &teardown, executor) {
Ok(()) => fleet::RollbackOutcome::RolledBack,
Err(err) => {
let failed_step = fleet::failed_step_label(&err);
eprintln!(
"\u{274C} [{host}] the rollback FAILED at `{failed_step}` \u{2014} this host \
was not restored. The remaining hosts still are."
);
fleet::RollbackOutcome::Failed { failed_step }
}
}
}
fn collect_fleet_status<E, F>(
fleet: &ResolvedFleet,
public_port: u16,
make_executor: F,
) -> Result<Vec<fleet::HostStatus>, DeployError>
where
E: exec::DeployExecutor,
F: Fn(&ResolvedDeployConfig) -> Result<E, DeployError>,
{
let mut statuses = Vec::with_capacity(fleet.hosts.len());
for cfg in &fleet.hosts {
let executor = make_executor(cfg)?;
let status = exec::probe_host_status(cfg, public_port, &executor).map_or_else(
|_| fleet::HostStatus::unreachable(cfg.host.clone().unwrap_or_default()),
|probe| fleet::HostStatus::from_probe(cfg, public_port, &probe),
);
statuses.push(status);
}
Ok(statuses)
}
fn fleet_status_json(
statuses: &[fleet::HostStatus],
report: &fleet::DriftReport,
) -> serde_json::Value {
let hosts: Vec<serde_json::Value> = statuses
.iter()
.map(|status| {
let mode = if status.reachable {
match status.mode {
Some(fleet::HostMode::First) => "not deployed",
Some(fleet::HostMode::Redeploy) => "deployed",
None => "unknown",
}
} else {
"unreachable"
};
serde_json::json!({
"host": status.host,
"reachable": status.reachable,
"mode": mode,
"release": match &status.release {
fleet::ReleaseId::Known(id) => serde_json::Value::String(id.clone()),
fleet::ReleaseId::Unknown => serde_json::Value::Null,
},
"live_slot": status.live_slot,
"ready": status.ready_code,
"maintenance": match status.maintenance {
exec::MaintenanceStatus::On => serde_json::Value::Bool(true),
exec::MaintenanceStatus::Off => serde_json::Value::Bool(false),
exec::MaintenanceStatus::Unknown => serde_json::Value::Null,
},
"proxy_port": match status.installed_proxy_port {
exec::InstalledProxyPort::Port(port) => {
serde_json::Value::Number(port.into())
}
_ => serde_json::Value::Null,
},
"last_deploy": status.last_deploy.as_ref().map_or(
serde_json::Value::Null,
|last| serde_json::json!({ "result": last.result, "at": last.at }),
),
"drift": report
.state_drift
.iter()
.filter(|(host, _)| *host == status.host)
.map(|(_, reason)| *reason)
.collect::<Vec<_>>(),
})
})
.collect();
serde_json::json!({
"hosts": hosts,
"version_drift": report.version_drift,
"state_drift": report
.state_drift
.iter()
.map(|(host, reason)| serde_json::json!({ "host": host, "reason": reason }))
.collect::<Vec<_>>(),
"drifted": report.drifted(),
})
}
fn warn_degraded_port(port: &StatusPort, profile: &str, continues: &str) {
let Some(reason) = &port.degraded else {
return;
};
eprintln!(
"\u{26A0}\u{FE0F} this project's configuration does not validate under the \
`{profile}` deploy profile: {reason}"
);
eprintln!("{continues}\n");
}
fn run_status(
port: &StatusPort,
app_name: &str,
profile: &str,
fleet: &ResolvedFleet,
options: &DeployOptions,
) -> Result<(), DeployError> {
if !options.json {
eprintln!("\u{1F342} autumn deploy status\n");
}
warn_degraded_port(
port,
profile,
&format!(
" `deploy status` is read-only, so it continues against the DECLARED \
`[server] port = {}` for that profile (read without validation). \
`autumn deploy check`/`up` still refuse to run until the config is fixed.",
port.port
),
);
let statuses = collect_fleet_status(fleet, port.port, |cfg| {
exec::SshTarget::from_resolved(cfg)
.map(exec::SshExecutor::new)
.ok_or_else(|| DeployError::Config(DEPLOY_HOST_MISSING_DETAIL.to_owned()))
})?;
let report = fleet::fleet_drift(&statuses);
if options.json {
println!(
"{}",
serde_json::to_string_pretty(&fleet_status_json(&statuses, &report))
.expect("a serde_json::Value always serializes")
);
} else {
for line in fleet::fleet_status_lines(&statuses, &report) {
println!("{line}");
}
}
if options.strict && report.drifted() {
emit_drift_alert(&report, app_name, profile);
return Err(DeployError::DriftDetected);
}
Ok(())
}
fn maintenance_flag_body(args: &MaintenanceOnArgs) -> String {
let config = autumn_web::maintenance::MaintenanceConfig {
message: args.message.clone(),
allow_ips: args.allow_ips.clone(),
readonly: args.readonly,
bypass_header: args.bypass_header.clone(),
};
serde_json::to_string_pretty(&config).expect("MaintenanceConfig always serializes")
}
fn run_fleet_maintenance_with<E, F>(
fleet: &ResolvedFleet,
port: &StatusPort,
profile: &str,
action: DeployAction,
args: Option<&MaintenanceOnArgs>,
make_executor: F,
) -> Result<(), DeployError>
where
E: exec::DeployExecutor,
F: Fn(&ResolvedDeployConfig) -> Result<E, DeployError>,
{
let on = matches!(action, DeployAction::MaintenanceOn);
let verb = if on { "on" } else { "off" };
eprintln!("\u{1F342} autumn deploy maintenance {verb}\n");
warn_degraded_port(
port,
profile,
&format!(
" `deploy maintenance` continues against the DECLARED `[server] port = {}` \
for that profile (read without validation); it uses that port only to identify \
which slot unit each host is running. `autumn deploy check`/`up` still refuse \
to run until the config is fixed.",
port.port
),
);
let total = fleet.hosts.len();
let executors = fleet
.hosts
.iter()
.map(&make_executor)
.collect::<Result<Vec<E>, DeployError>>()?;
let names: Vec<String> = fleet
.hosts
.iter()
.map(|cfg| cfg.host.clone().unwrap_or_default())
.collect();
let default_args = MaintenanceOnArgs::default();
let body = maintenance_flag_body(args.unwrap_or(&default_args));
let mut outcomes = Vec::with_capacity(total);
for (index, cfg) in fleet.hosts.iter().enumerate() {
eprintln!(
"[{}/{total} {}] maintenance {verb}\u{2026}",
index + 1,
names[index],
);
outcomes.push(maintenance_one_host(
cfg,
port.port,
on,
&body,
&executors[index],
));
eprintln!();
}
for line in fleet::fleet_maintenance_summary_lines(&names, &outcomes, on) {
eprintln!("{line}");
}
let failed = outcomes.iter().filter(|outcome| outcome.failed()).count();
if failed > 0 {
return Err(DeployError::FleetMaintenanceFailed {
verb: if on { "on" } else { "off" },
failed,
total,
});
}
Ok(())
}
enum LiveFlagTarget {
NoRelease,
Path(String),
Unproved,
}
fn maintenance_one_host<E: exec::DeployExecutor>(
cfg: &ResolvedDeployConfig,
public_port: u16,
on: bool,
body: &str,
executor: &E,
) -> fleet::MaintenanceOutcome {
let Ok(probe) = exec::probe_deploy_state(cfg, executor) else {
return fleet::MaintenanceOutcome::Failed {
failed_step: "detect-current",
};
};
let target = match probe.reconcile(cfg, public_port) {
None => LiveFlagTarget::NoRelease,
Some(reconcile) => {
match exec::probe_live_maintenance_flag(cfg, reconcile.live_slot, executor) {
Ok(Some(flag)) => LiveFlagTarget::Path(flag.path),
Ok(None) | Err(_) => LiveFlagTarget::Unproved,
}
}
};
let shared = cfg.maintenance_flag_file();
let live_path = match &target {
LiveFlagTarget::Path(path) if *path != shared => Some(path.clone()),
_ => None,
};
let mut ops: Vec<exec::DeployOp> = Vec::new();
let shared_index = if on {
if let Some(parent) = remote_parent_dir(&shared) {
ops.push(exec::DeployOp::Run(exec::RemoteCommand::new(
"maintenance-prepare-shared-flag-dir",
format!("mkdir -p {}", exec::shell_quote(parent)),
)));
}
let shared_index = ops.len();
ops.push(exec::DeployOp::WriteFile {
label: "maintenance-write-shared",
contents: exec::FileContents::Plain(body.to_owned()),
remote_path: shared,
mode: Some(0o600),
});
if let Some(path) = &live_path {
if let Some(parent) = remote_parent_dir(path) {
ops.push(exec::DeployOp::Run(exec::RemoteCommand::new(
"maintenance-prepare-live-flag-dir",
format!("mkdir -p {}", exec::shell_quote(parent)),
)));
}
ops.push(exec::DeployOp::WriteFile {
label: "maintenance-write-live-flag",
contents: exec::FileContents::Plain(body.to_owned()),
remote_path: path.clone(),
mode: Some(0o600),
});
}
shared_index
} else {
let mut paths = exec::shell_quote(&shared);
if let Some(path) = &live_path {
paths.push(' ');
paths.push_str(&exec::shell_quote(path));
}
let shared_index = ops.len();
ops.push(exec::DeployOp::Run(exec::RemoteCommand::new(
"maintenance-clear",
format!("rm -f {paths}"),
)));
shared_index
};
for (index, op) in ops.iter().enumerate() {
if exec::run_ops(std::slice::from_ref(op), executor).is_err() {
let failed_step = op.label();
return if index <= shared_index {
fleet::MaintenanceOutcome::Failed { failed_step }
} else {
fleet::MaintenanceOutcome::LiveUnitUnchanged { failed_step }
};
}
}
match target {
LiveFlagTarget::NoRelease => fleet::MaintenanceOutcome::AppliedSharedOnly,
LiveFlagTarget::Path(_) => fleet::MaintenanceOutcome::Applied,
LiveFlagTarget::Unproved => fleet::MaintenanceOutcome::LiveUnitUnchanged {
failed_step: "detect-maintenance-flag",
},
}
}
fn remote_parent_dir(path: &str) -> Option<&str> {
let (parent, _) = path.rsplit_once('/')?;
(!parent.is_empty()).then_some(parent)
}
#[cfg(test)]
mod tests {
const ALL_DEPLOY_ACTIONS: [DeployAction; 7] = [
DeployAction::Check,
DeployAction::Plan,
DeployAction::Rollback,
DeployAction::Up,
DeployAction::Status,
DeployAction::MaintenanceOn,
DeployAction::MaintenanceOff,
];
const REMOTE_ACTIONS: [DeployAction; 5] = [
DeployAction::Up,
DeployAction::Rollback,
DeployAction::Status,
DeployAction::MaintenanceOn,
DeployAction::MaintenanceOff,
];
const LOCAL_ACTIONS: [DeployAction; 2] = [DeployAction::Check, DeployAction::Plan];
#[test]
fn windows_deploys_are_refused_with_the_policy_message() {
for action in REMOTE_ACTIONS {
let refusal =
windows_deploy_refusal("windows", action).expect("windows must be refused");
assert!(refusal.contains("Tier 2 (WSL2)"), "{action:?}: {refusal}");
assert!(
refusal.contains(crate::platform::POLICY_DOC_URL),
"{action:?}: {refusal}"
);
}
}
#[test]
fn the_locally_verifiable_spine_runs_natively_on_windows() {
for action in LOCAL_ACTIONS {
assert!(
windows_deploy_refusal("windows", action).is_none(),
"{action:?} is local-only and must run natively on Windows"
);
}
}
#[test]
fn unix_deploys_are_not_refused() {
for action in REMOTE_ACTIONS.iter().chain(LOCAL_ACTIONS.iter()).copied() {
assert!(
windows_deploy_refusal("linux", action).is_none(),
"{action:?}"
);
assert!(
windows_deploy_refusal("macos", action).is_none(),
"{action:?}"
);
}
}
#[test]
fn every_deploy_action_is_classified() {
let classified = REMOTE_ACTIONS.len() + LOCAL_ACTIONS.len();
assert_eq!(
classified,
ALL_DEPLOY_ACTIONS.len(),
"every DeployAction must be classified as remote or local"
);
for action in ALL_DEPLOY_ACTIONS {
assert!(
REMOTE_ACTIONS.contains(&action) || LOCAL_ACTIONS.contains(&action),
"{action:?} is not classified"
);
}
}
#[test]
fn the_refusal_is_an_error_so_it_exits_non_zero() {
let err = DeployError::UnsupportedPlatform(
windows_deploy_refusal("windows", DeployAction::Up).unwrap(),
);
assert!(err.to_string().contains("Tier 2 (WSL2)"));
}
#[test]
fn the_refusal_does_not_masquerade_as_a_config_failure() {
let err = DeployError::UnsupportedPlatform(
windows_deploy_refusal("windows", DeployAction::Up).unwrap(),
);
assert!(
!err.to_string().contains("failed to load configuration"),
"{err}"
);
}
use super::*;
fn resolved_defaults() -> ResolvedDeployConfig {
ResolvedDeployConfig::resolve(&DeployConfig::default(), "myapp")
.expect("deploy config resolves")
}
#[test]
fn public_port_443_is_rejected_as_a_proxy_https_collision() {
let err = validate_public_port(443).expect_err("port 443 must be rejected");
assert!(
err.contains("443")
&& err.contains("HTTPS listener")
&& err.contains("proxy's HTTP listener"),
"collision message must name 443/the HTTPS listener/the proxy HTTP port, got: {err}",
);
assert!(validate_public_port(80).is_ok());
assert!(validate_public_port(8080).is_ok());
}
#[test]
fn public_ports_whose_slots_reach_443_are_rejected() {
let green = validate_public_port(441).expect_err("441 (green slot=443) rejected");
assert!(
green.contains("green app slot") && green.contains("443"),
"441 message must name the green slot landing on 443, got: {green}",
);
let blue = validate_public_port(442).expect_err("442 (blue slot=443) rejected");
assert!(
blue.contains("blue app slot") && blue.contains("443"),
"442 message must name the blue slot landing on 443, got: {blue}",
);
assert!(validate_public_port(443).is_err());
assert!(validate_public_port(440).is_ok());
assert!(validate_public_port(444).is_ok());
assert!(validate_public_port(8080).is_ok());
}
#[test]
fn tls_does_not_gate_the_public_port() {
assert!(validate_public_port(80).is_ok());
assert!(validate_public_port(3000).is_ok());
assert!(validate_public_port(8080).is_ok());
}
#[test]
fn redeploy_detects_a_concurrent_server_port_change() {
let mv = resolve_public_port_move(&exec::InstalledProxyPort::Port(80), 8080)
.expect("a changed server.port must resolve, not error")
.expect("a changed server.port must be detected as a move");
assert_eq!(mv.old_port, 80);
assert_eq!(mv.new_port, 8080);
}
#[test]
fn redeploy_allows_an_unchanged_server_port() {
assert_eq!(
resolve_public_port_move(&exec::InstalledProxyPort::Port(80), 80),
Ok(None),
"an unchanged-port redeploy must resolve to no move",
);
assert_eq!(
resolve_public_port_move(&exec::InstalledProxyPort::Port(3000), 3000),
Ok(None),
"an unchanged non-default port must also resolve to no move",
);
}
#[test]
fn redeploy_fails_closed_on_an_unreadable_installed_port() {
let err = resolve_public_port_move(&exec::InstalledProxyPort::Unreadable, 80)
.expect_err("an unreadable installed proxy port must fail closed");
assert!(
err.contains("could not be read"),
"the fail-closed message must explain the unreadable unit: {err}",
);
}
#[test]
fn redeploy_allows_when_no_proxy_unit_is_installed() {
assert_eq!(
resolve_public_port_move(&exec::InstalledProxyPort::Absent, 8080),
Ok(None),
"an absent installed proxy unit must resolve to no move",
);
}
#[test]
fn redeploy_moves_the_public_port_after_the_cutover_drains_the_old_release() {
let fleet = fleet_of(&["web-a"]);
let probe = "redeploy:blue\t81\n\
---autumn-kamal-proxy-list---\n\
---autumn-kamal-proxy-unit---\n--http-port 80\n"
.to_owned();
let recorder = fleet::test_support::FleetRecorder::new()
.script("web-a", "proxy-compat-probe", compatible_deploy_help())
.script("web-a", "detect-current", probe)
.script("web-a", "probe-release-dir", "absent");
let fixture = FleetFixture::new();
run_up_with(&fixture.input(&fleet), |cfg| Ok(recorder.executor(cfg)))
.expect("a redeploy with a changed server.port succeeds (Option C)");
let labels = recorder.run_labels_for("web-a");
let drain = labels
.iter()
.position(|l| *l == "drain-old")
.expect("the cutover drains the old release");
let restarts: Vec<usize> = labels
.iter()
.enumerate()
.filter(|(_, l)| **l == "proxy-restart-if-changed")
.map(|(i, _)| i)
.collect();
assert_eq!(
restarts.len(),
2,
"one restart-if-changed from the cutover's own durability refresh, one \
from the phase-4 public-port rebind: {labels:?}"
);
assert!(
restarts[0] < drain,
"the durability-refresh restart-if-changed runs at the HEAD of the \
cutover, before the old release ever drains: {labels:?}"
);
assert!(
restarts[1] > drain,
"the public-port move must run strictly AFTER the old release drains: \
{labels:?}"
);
let restart_shells: Vec<String> = recorder
.calls_for("web-a")
.iter()
.filter_map(|c| match c {
exec::test_support::RecordedCall::Run { label, shell }
if *label == "proxy-restart-if-changed" =>
{
Some(shell.clone())
}
_ => None,
})
.collect();
assert_eq!(restart_shells.len(), 2);
assert!(
restart_shells[1].contains("--target '127.0.0.1:82'"),
"phase 4 re-registers the candidate's OWN loopback port: {}",
restart_shells[1],
);
}
#[test]
fn a_public_port_rebind_failure_degrades_the_host_and_continues_the_rollout() {
let fleet = fleet_of(&["web-a", "web-b"]);
let probe_a = "redeploy:blue\t81\n\
---autumn-kamal-proxy-list---\n\
---autumn-kamal-proxy-unit---\n--http-port 80\n"
.to_owned();
let recorder = fleet::test_support::FleetRecorder::new()
.script("web-a", "proxy-compat-probe", compatible_deploy_help())
.script("web-a", "detect-current", probe_a)
.script("web-a", "probe-release-dir", "absent")
.fail_on_occurrence("web-a", "proxy-restart-if-changed", 2);
let recorder = script_redeploy(recorder, "web-b");
let fixture = FleetFixture::new();
run_up_with(&fixture.input(&fleet), |cfg| Ok(recorder.executor(cfg)))
.expect("a rolled-back port move degrades a host but never halts the rollout");
let labels_a = recorder.run_labels_for("web-a");
assert_eq!(
labels_a
.iter()
.filter(|l| **l == "proxy-restart-if-changed")
.count(),
3,
"one no-op from the cutover's own durability refresh, one FAILING phase-4 \
forward attempt, one SUCCEEDING phase-4 rollback attempt: {labels_a:?}"
);
assert!(
recorder.run_labels_for("web-b").contains(&"drain-old"),
"web-b must still be deployed after web-a degrades: {:?}",
recorder.run_labels_for("web-b")
);
}
#[test]
fn a_public_port_rebind_double_failure_halts_the_rollout_as_manual() {
let fleet = fleet_of(&["web-a", "web-b"]);
let probe_a = "redeploy:blue\t81\n\
---autumn-kamal-proxy-list---\n\
---autumn-kamal-proxy-unit---\n--http-port 80\n"
.to_owned();
let recorder = fleet::test_support::FleetRecorder::new()
.script("web-a", "proxy-compat-probe", compatible_deploy_help())
.script("web-a", "detect-current", probe_a)
.script("web-a", "probe-release-dir", "absent")
.fail_on_occurrence("web-a", "proxy-restart-if-changed", 2)
.fail_on_occurrence("web-a", "proxy-restart-if-changed", 3);
let recorder = script_redeploy(recorder, "web-b");
let fixture = FleetFixture::new();
let err = run_up_with(&fixture.input(&fleet), |cfg| Ok(recorder.executor(cfg)))
.expect_err("a doubly-failed port move must halt the rollout");
let halt = fleet_halt_of(&err);
assert_eq!(halt.failed_host, "web-a");
assert_eq!(halt.failed_step, "public-port-rebind");
assert_eq!(
halt.manual,
vec![("web-a".to_owned(), fleet::MANUAL_PUBLIC_PORT_UNKNOWN)],
"web-a must be reported Manual (needs a human), never Degraded, which \
would falsely claim traffic is fine"
);
assert!(
halt.degraded.is_empty(),
"web-a's unknown public bind must not be counted as a healthy degradation: \
{:?}",
halt.degraded,
);
assert!(
recorder
.run_labels_for("web-b")
.iter()
.all(|l| READ_ONLY_PROBES.contains(l)),
"web-b must not be mutated once the rollout halts at web-a: {:?}",
recorder.run_labels_for("web-b")
);
}
#[test]
fn redeploy_fails_closed_on_an_unreadable_proxy_options_marker() {
let err = refuse_unprovable_proxy_options(&exec::ProxyOptionsMarker::Unreadable)
.expect_err("an unreadable proxy-options marker must fail closed");
assert!(
err.contains("proxy-options") && err.contains("unreadable"),
"the fail-closed message must name the unreadable proxy-options marker: {err}",
);
assert!(
err.contains("deploy.tls unchanged") && err.contains("#2074"),
"the message must spell out the two-deploy repair and reference #2074: {err}",
);
}
#[test]
fn redeploy_allows_an_absent_proxy_options_marker() {
assert!(
refuse_unprovable_proxy_options(&exec::ProxyOptionsMarker::Absent).is_ok(),
"an absent proxy-options marker must not trigger the refuse",
);
}
#[test]
fn redeploy_allows_a_readable_proxy_options_marker() {
let unchanged = exec::ProxyOptionsMarker::Options(exec::ProxyServiceOptions {
tls: true,
host: Some("app.example.com".to_owned()),
});
assert!(
refuse_unprovable_proxy_options(&unchanged).is_ok(),
"an unchanged readable marker must be allowed (preserve is a no-op)",
);
let changed = exec::ProxyOptionsMarker::Options(exec::ProxyServiceOptions {
tls: false,
host: None,
});
assert!(
refuse_unprovable_proxy_options(&changed).is_ok(),
"a readable marker whose options differ from the new config is still allowed \
(the old options are preserved, not refused)",
);
}
#[test]
fn resolve_fills_default_chain_from_project_name() {
let resolved = resolved_defaults();
assert_eq!(resolved.app_name, "myapp");
assert_eq!(resolved.app_dir, "/srv/autumn/myapp");
assert_eq!(resolved.service_name, "myapp");
assert_eq!(resolved.user, "root");
assert_eq!(resolved.ssh_port, 22);
assert_eq!(resolved.readiness_timeout_secs, 60);
assert_eq!(resolved.keep_releases, 3);
assert_eq!(resolved.host, None);
assert_eq!(resolved.profile, "prod");
assert!(!resolved.tls_enabled);
assert_eq!(resolved.tls_host, None);
}
#[test]
fn tls_absent_table_leaves_tls_disabled() {
let resolved = resolved_defaults();
assert!(!resolved.tls_enabled);
assert_eq!(resolved.tls_host, None);
}
#[test]
fn tls_enabled_with_host_resolves_tls_host() {
let cfg = DeployConfig {
tls: autumn_web::config::DeployTlsConfig {
enabled: true,
host: Some("app.example.com".to_owned()),
},
..DeployConfig::default()
};
let resolved =
ResolvedDeployConfig::resolve(&cfg, "myapp").expect("enabled + host resolves");
assert!(resolved.tls_enabled);
assert_eq!(resolved.tls_host.as_deref(), Some("app.example.com"));
}
#[test]
fn tls_enabled_without_host_is_a_resolve_error() {
for host in [None, Some(String::new()), Some(" ".to_owned())] {
let cfg = DeployConfig {
tls: autumn_web::config::DeployTlsConfig {
enabled: true,
host,
},
..DeployConfig::default()
};
let err = ResolvedDeployConfig::resolve(&cfg, "myapp")
.expect_err("enabled TLS without a host must be rejected");
assert!(
err.contains("[deploy.tls]") && err.contains("host"),
"error should name the section and the missing key, got: {err}",
);
}
}
#[test]
fn host_preparation_is_on_by_default_and_declinable() {
assert!(
resolved_defaults().install_proxy,
"a bare `[deploy]` table must prepare the host"
);
assert_eq!(
provisioning(&resolved_defaults()),
exec::ProxyProvisioning::Auto
);
let declined = ResolvedDeployConfig::resolve(
&DeployConfig {
install_proxy: false,
..DeployConfig::default()
},
"myapp",
)
.expect("declining host prep resolves");
assert!(!declined.install_proxy);
assert_eq!(provisioning(&declined), exec::ProxyProvisioning::Disabled);
}
fn fleet_cfg(hosts: &[&str]) -> DeployConfig {
DeployConfig {
hosts: hosts.iter().map(|h| (*h).to_owned()).collect(),
..DeployConfig::default()
}
}
#[test]
fn fleet_resolve_keeps_declaration_order_as_the_rollout_order() {
let fleet = ResolvedFleet::resolve(&fleet_cfg(&["web-3", "web-1", "web-2"]), "myapp")
.expect("a well-formed fleet resolves");
let hosts: Vec<Option<&str>> = fleet.hosts.iter().map(|h| h.host.as_deref()).collect();
assert_eq!(
hosts,
vec![Some("web-3"), Some("web-1"), Some("web-2")],
"fleet resolution must preserve declaration order, got: {hosts:?}"
);
assert!(
!fleet.is_single(),
"a 3-host fleet must not report as single-host, got: {:?}",
fleet.hosts.len()
);
}
#[test]
fn fleet_resolve_rejects_host_and_hosts_set_together_naming_both_keys() {
let cfg = DeployConfig {
host: Some("203.0.113.10".to_owned()),
hosts: vec!["web-1.example.com".to_owned()],
..DeployConfig::default()
};
let err = ResolvedFleet::resolve(&cfg, "myapp")
.expect_err("host + hosts together must be rejected");
assert!(
err.contains("[deploy] host") && err.contains("[deploy] hosts"),
"the mutual-exclusion error must name BOTH keys, got: {err}"
);
assert!(
err.contains("#1621"),
"fail-closed refusals cite the tracking issue, got: {err}"
);
}
#[test]
fn fleet_resolve_rejects_a_blank_hosts_entry_naming_its_index() {
for (index, hosts) in [
(0_usize, vec![String::new(), "web-2".to_owned()]),
(1, vec!["web-1".to_owned(), " ".to_owned()]),
(
2,
vec!["web-1".to_owned(), "web-2".to_owned(), "\t".to_owned()],
),
] {
let cfg = DeployConfig {
hosts,
..DeployConfig::default()
};
let err = ResolvedFleet::resolve(&cfg, "myapp")
.expect_err("a blank hosts entry must be rejected");
assert!(
err.contains("[deploy] hosts") && err.contains(&format!("{index}")),
"the blank-entry error must name the key and the 0-based index {index}, got: {err}"
);
}
}
#[test]
fn fleet_resolve_rejects_duplicate_hosts_after_trimming_naming_the_value() {
let cfg = fleet_cfg(&["web-1.example.com", "web-2", " web-1.example.com "]);
let err = ResolvedFleet::resolve(&cfg, "myapp")
.expect_err("a duplicate hosts entry must be rejected");
assert!(
err.contains("web-1.example.com"),
"the duplicate error must name the repeated value, got: {err}"
);
assert!(
err.contains("[deploy] hosts"),
"the duplicate error must name the config key, got: {err}"
);
}
#[test]
fn fleet_resolve_without_any_host_keeps_the_deploy_host_message() {
let err = ResolvedFleet::resolve(&DeployConfig::default(), "myapp")
.expect_err("no host and no hosts must be rejected");
assert!(
err.contains("[deploy] host"),
"the missing-target error must keep the historical `[deploy] host` \
substring, got: {err}"
);
assert!(
err.contains("hosts"),
"the missing-target error must also offer the fleet spelling, got: {err}"
);
let blank = DeployConfig {
host: Some(" ".to_owned()),
..DeployConfig::default()
};
let blank_err = ResolvedFleet::resolve(&blank, "myapp")
.expect_err("a blank host must be rejected like an absent one");
assert!(
blank_err.contains("[deploy] host"),
"a blank host must report the missing-target message, got: {blank_err}"
);
}
#[test]
fn single_entry_hosts_resolves_to_the_same_view_as_host() {
let single = DeployConfig {
host: Some("203.0.113.10".to_owned()),
app_name: Some("shop".to_owned()),
user: "deploy".to_owned(),
ssh_port: 2222,
readiness_timeout_secs: 90,
keep_releases: 5,
profile: "staging".to_owned(),
..DeployConfig::default()
};
let fleet_cfg = DeployConfig {
host: None,
hosts: vec!["203.0.113.10".to_owned()],
..single.clone()
};
let expected =
ResolvedDeployConfig::resolve(&single, "myapp").expect("single host resolves");
let fleet =
ResolvedFleet::resolve(&fleet_cfg, "myapp").expect("single-entry fleet resolves");
assert!(
fleet.is_single(),
"a one-entry hosts list must report as single-host, got {} hosts",
fleet.hosts.len()
);
assert_eq!(
fleet.hosts[0], expected,
"a one-entry `hosts` list must resolve to exactly the `host` view; \
fleet: {:?}, single: {expected:?}",
fleet.hosts[0]
);
}
fn preflight_config() -> AutumnConfig {
AutumnConfig::default()
}
fn deploy_cfg() -> ResolvedDeployConfig {
ResolvedDeployConfig::resolve(
&DeployConfig {
host: Some("203.0.113.10".to_owned()),
..DeployConfig::default()
},
"myapp",
)
.expect("deploy config resolves")
}
#[test]
fn classify_sqlite_data_file_separates_relative_absolute_and_not_sqlite() {
let cfg = deploy_cfg();
for relative in ["sqlite://app.db", "sqlite://./app.db", "sqlite:data/app.db"] {
assert!(
matches!(
classify_sqlite_data_file(Some(relative), &cfg),
SqliteDataFile::Relative(_)
),
"{relative}"
);
}
for spelling in ["sqlite://./app.db", "sqlite://app.db"] {
assert_eq!(
classify_sqlite_data_file(Some(spelling), &cfg),
SqliteDataFile::Relative("app.db".to_owned()),
"{spelling}"
);
}
for spelling in [
"sqlite://data/app.db",
"sqlite://data//app.db",
"sqlite://data/./app.db",
"sqlite://data/nested/../app.db",
] {
assert_eq!(
classify_sqlite_data_file(Some(spelling), &cfg),
SqliteDataFile::Relative("data/app.db".to_owned()),
"{spelling}"
);
}
assert_eq!(
classify_sqlite_data_file(Some("file:app%20data.db?mode=rwc"), &cfg),
SqliteDataFile::Relative("app data.db".to_owned())
);
assert_eq!(
classify_sqlite_data_file(Some("sqlite:///var/lib/myapp/app.db"), &cfg),
SqliteDataFile::Persistent("/var/lib/myapp/app.db".to_owned())
);
assert_eq!(
classify_sqlite_data_file(Some("sqlite:///srv/autumn/myapp/shared/data/app.db"), &cfg),
SqliteDataFile::Persistent("/srv/autumn/myapp/shared/data/app.db".to_owned())
);
for other in [Some("postgres://u:p@h/app"), Some(" "), None] {
assert_eq!(
classify_sqlite_data_file(other, &cfg),
SqliteDataFile::NotSqlite,
"{other:?}"
);
}
}
#[test]
fn path_grading_uses_posix_rules_on_every_host() {
assert_eq!(
lexically_normalized("/srv/autumn/myapp/shared/../releases/r1/app.db"),
"/srv/autumn/myapp/releases/r1/app.db"
);
assert_eq!(
lexically_normalized("./data//nested/../app.db"),
"data/app.db"
);
assert_eq!(
lexically_normalized("/a/../.."),
"/",
"an absolute path cannot climb above the root"
);
assert_eq!(
lexically_normalized("/../srv/autumn/myapp"),
"/srv/autumn/myapp",
"a leading `..` on an absolute path is dropped, as POSIX does"
);
assert_eq!(lexically_normalized("../app.db"), "../app.db");
assert_eq!(lexically_normalized("."), "");
assert!(
lexically_normalized("/var/lib/app.db").starts_with('/'),
"a POSIX absolute path must stay absolute"
);
}
#[test]
fn classify_sqlite_data_file_refuses_what_a_deploy_cannot_keep() {
let cfg = deploy_cfg();
let memory = classify_sqlite_data_file(Some("sqlite::memory:"), &cfg);
let SqliteDataFile::Refused(detail) = memory else {
panic!("an in-memory database must be refused, got {memory:?}");
};
assert!(detail.contains("IN-MEMORY"), "{detail}");
for inside in [
"sqlite:///srv/autumn/myapp/releases/20260714T120000Z/app.db",
"sqlite:///srv/autumn/myapp/current/app.db",
"sqlite:///srv/autumn/myapp/app.db",
] {
let got = classify_sqlite_data_file(Some(inside), &cfg);
let SqliteDataFile::Refused(detail) = got else {
panic!("{inside} must be refused, got {got:?}");
};
assert!(
detail.contains("/srv/autumn/myapp") && detail.contains("shared/data"),
"the refusal must name the problem and the fix: {detail}"
);
}
assert!(matches!(
classify_sqlite_data_file(Some("sqlite:///srv/autumn/myapp-staging/app.db"), &cfg),
SqliteDataFile::Persistent(_)
));
let sneaky = classify_sqlite_data_file(
Some("sqlite:///srv/autumn/myapp/shared/../releases/r1/app.db"),
&cfg,
);
let SqliteDataFile::Refused(detail) = sneaky else {
panic!("a path that resolves into releases/ must be refused, got {sneaky:?}");
};
assert!(
detail.contains("/srv/autumn/myapp/releases/r1/app.db"),
"the refusal must name the RESOLVED path: {detail}"
);
assert_eq!(
classify_sqlite_data_file(
Some("sqlite:///srv/autumn/myapp/../shared-data/app.db"),
&cfg
),
SqliteDataFile::Persistent("/srv/autumn/shared-data/app.db".to_owned())
);
assert_eq!(
classify_sqlite_data_file(Some("sqlite:///../../app.db"), &cfg),
SqliteDataFile::Persistent("/app.db".to_owned())
);
for refused in [
"sqlite::memory:",
"sqlite:///srv/autumn/myapp/current/app.db",
"sqlite://..",
] {
if let SqliteDataFile::Refused(detail) = classify_sqlite_data_file(Some(refused), &cfg)
{
assert!(!detail.contains(" "), "{refused}: {detail}");
}
}
for bad in [
"sqlite://../../app.db",
"sqlite://.",
"sqlite://./",
"sqlite://data/../..",
] {
let got = classify_sqlite_data_file(Some(bad), &cfg);
assert!(
matches!(got, SqliteDataFile::Refused(_)),
"{bad} must be refused, got {got:?}"
);
}
}
#[test]
fn an_app_dir_that_starts_above_root_still_grades_containment() {
let aliased = ResolvedDeployConfig::resolve(
&DeployConfig {
host: Some("203.0.113.10".to_owned()),
app_dir: Some("/../srv/autumn/myapp".to_owned()),
..DeployConfig::default()
},
"myapp",
)
.expect("resolves");
let inside = classify_sqlite_data_file(
Some("sqlite:///srv/autumn/myapp/releases/r1/app.db"),
&aliased,
);
assert!(
matches!(inside, SqliteDataFile::Refused(_)),
"a database in the releases dir must be refused, got: {inside:?}"
);
assert!(
matches!(
classify_sqlite_data_file(
Some("sqlite:///srv/autumn/myapp/shared/data/app.db"),
&aliased,
),
SqliteDataFile::Persistent(_)
),
"the shared dir must stay durable through the aliased app dir"
);
assert!(
matches!(
classify_sqlite_data_file(Some("sqlite:///var/lib/app.db"), &aliased),
SqliteDataFile::Persistent(_)
),
"a path outside the app dir is left alone"
);
}
#[test]
fn an_ungradable_app_dir_refuses_rather_than_claiming_durability() {
let relative = ResolvedDeployConfig::resolve(
&DeployConfig {
host: Some("203.0.113.10".to_owned()),
app_dir: Some("srv/autumn/myapp".to_owned()),
..DeployConfig::default()
},
"myapp",
)
.expect("resolves");
let graded = classify_sqlite_data_file(
Some("sqlite:///srv/autumn/myapp/releases/r1/app.db"),
&relative,
);
assert!(
matches!(graded, SqliteDataFile::Refused(_)),
"a non-absolute app dir must refuse, never grade Persistent: {graded:?}"
);
}
#[test]
fn classify_sqlite_data_file_normalizes_the_app_dir_as_well() {
let aliased = ResolvedDeployConfig::resolve(
&DeployConfig {
host: Some("203.0.113.10".to_owned()),
app_dir: Some("/srv/autumn/tmp/../myapp/".to_owned()),
..DeployConfig::default()
},
"myapp",
)
.expect("resolves");
let inside = classify_sqlite_data_file(
Some("sqlite:///srv/autumn/myapp/releases/r1/app.db"),
&aliased,
);
let SqliteDataFile::Refused(detail) = inside else {
panic!("a database in the resolved releases dir must be refused, got {inside:?}");
};
assert!(
detail.contains("/srv/autumn/myapp"),
"the refusal must name the RESOLVED app dir: {detail}"
);
assert!(matches!(
classify_sqlite_data_file(
Some("sqlite:///srv/autumn/myapp/shared/data/app.db"),
&aliased
),
SqliteDataFile::Persistent(_)
));
assert!(matches!(
classify_sqlite_data_file(Some("sqlite:///var/lib/myapp/app.db"), &aliased),
SqliteDataFile::Persistent(_)
));
}
#[test]
fn classify_sqlite_data_file_refuses_a_collision_with_a_release_payload() {
let cfg = deploy_cfg().with_release_payloads(vec!["prod.lock".to_owned()]);
for (url, expected) in [
("sqlite://myapp", "myapp binary"),
("sqlite://autumn.toml", "autumn.toml"),
("sqlite://autumn-prod.toml", "autumn-prod.toml"),
("sqlite://autumn-Production.toml", "autumn-Production.toml"),
("sqlite://prod.lock", "prod.lock"),
] {
let got = classify_sqlite_data_file(Some(url), &cfg);
let SqliteDataFile::Refused(detail) = got else {
panic!("{url} collides with a release payload but was graded {got:?}");
};
assert!(
detail.contains(expected),
"{url}: the refusal must name what it collides with: {detail}"
);
}
assert!(matches!(
classify_sqlite_data_file(Some("sqlite://./autumn.toml"), &cfg),
SqliteDataFile::Refused(_)
));
for safe in [
"sqlite://app.db",
"sqlite://data/myapp",
"sqlite://data/autumn.toml",
"sqlite://autumn.db",
"sqlite://autumn-prod.db",
"sqlite://autumn-.toml",
] {
assert!(
matches!(
classify_sqlite_data_file(Some(safe), &cfg),
SqliteDataFile::Relative(_)
),
"{safe} cannot collide with a release payload and must be kept"
);
}
}
#[test]
fn classify_sqlite_data_file_refuses_deploy_owned_paths_under_shared() {
let cfg = deploy_cfg();
for owned in [
"sqlite:///srv/autumn/myapp/shared/autumn.env",
"sqlite:///srv/autumn/myapp/shared/live-slot",
"sqlite:///srv/autumn/myapp/shared/previous-release",
"sqlite:///srv/autumn/myapp/shared/proxy-options",
"sqlite:///srv/autumn/myapp/shared/last-deploy",
"sqlite:///srv/autumn/myapp/shared/sqlite-data-adopted",
"sqlite:///srv/autumn/myapp/shared/some-future-state-file",
"sqlite:///srv/autumn/myapp/shared/app.db",
] {
let got = classify_sqlite_data_file(Some(owned), &cfg);
let SqliteDataFile::Refused(detail) = got else {
panic!("{owned} is a deploy-owned path but graded {got:?}");
};
assert!(
detail.contains("/srv/autumn/myapp/shared/data"),
"{owned}: the refusal must name where the database may live: {detail}"
);
}
for ours in [
"sqlite:///srv/autumn/myapp/shared/data/app.db",
"sqlite:///srv/autumn/myapp/shared/data/nested/app.db",
] {
assert!(
matches!(
classify_sqlite_data_file(Some(ours), &cfg),
SqliteDataFile::Persistent(_)
),
"{ours} must still be kept"
);
}
assert!(matches!(
classify_sqlite_data_file(Some("sqlite:///var/lib/myapp/app.db"), &cfg),
SqliteDataFile::Persistent(_)
));
}
#[test]
fn classify_sqlite_data_file_refuses_a_payload_as_a_leading_component() {
let cfg = deploy_cfg().with_release_payloads(vec!["prod.lock".to_owned()]);
for url in [
"sqlite://myapp/myapp",
"sqlite://myapp/app.db",
"sqlite://myapp/nested/app.db",
"sqlite://autumn.toml/app.db",
"sqlite://autumn-prod.toml/app.db",
"sqlite://prod.lock/app.db",
"sqlite://myapp",
"sqlite://./myapp/app.db",
] {
assert!(
matches!(
classify_sqlite_data_file(Some(url), &cfg),
SqliteDataFile::Refused(_)
),
"{url}: a release payload as the leading component is written through"
);
}
for safe in [
"sqlite://data/app.db",
"sqlite://data/myapp",
"sqlite://data/autumn.toml",
"sqlite://myapp.db",
] {
assert!(
matches!(
classify_sqlite_data_file(Some(safe), &cfg),
SqliteDataFile::Relative(_)
),
"{safe} cannot collide and must be kept"
);
}
}
#[test]
fn classify_sqlite_data_file_refuses_a_relative_app_dir_on_both_branches() {
let relative_dir = ResolvedDeployConfig::resolve(
&DeployConfig {
host: Some("203.0.113.10".to_owned()),
app_dir: Some("myapp".to_owned()),
..DeployConfig::default()
},
"myapp",
)
.expect("resolves");
for url in [
"sqlite:///var/lib/myapp/app.db",
"sqlite://app.db",
"sqlite://data/app.db",
] {
let got = classify_sqlite_data_file(Some(url), &relative_dir);
let SqliteDataFile::Refused(detail) = got else {
panic!("a relative app_dir makes {url} undecidable, but it graded {got:?}");
};
assert!(
detail.contains("myapp") && detail.contains("absolute"),
"{url}: the refusal must name the app dir and the fix: {detail}"
);
}
assert!(matches!(
classify_sqlite_data_file(Some("postgres://localhost/app"), &relative_dir),
SqliteDataFile::NotSqlite
));
}
#[cfg(unix)]
#[test]
fn classify_sqlite_data_file_refuses_a_non_utf8_path() {
let got = classify_sqlite_data_file(Some("file:app%FF.db"), &deploy_cfg());
let SqliteDataFile::Refused(detail) = got else {
panic!("a non-UTF-8 path must be refused, got {got:?}");
};
assert!(detail.contains("not valid UTF-8"), "{detail}");
}
#[test]
fn sqlite_data_persistence_grader_is_emitted_only_for_a_sqlite_app() {
let cfg = deploy_cfg();
assert!(grade_sqlite_data_persistence(Some("postgres://h/app"), &cfg).is_none());
assert!(grade_sqlite_data_persistence(None, &cfg).is_none());
let pass = grade_sqlite_data_persistence(Some("sqlite://app.db"), &cfg)
.expect("a SQLite app is graded");
assert_eq!(pass.name, "sqlite_data_file");
assert!(pass.passed, "a relative path is kept, not refused");
assert!(
pass.detail.contains("/srv/autumn/myapp/shared/data/app.db"),
"the pass must name where the file is kept: {}",
pass.detail
);
for refused in [
"sqlite::memory:",
"sqlite:///srv/autumn/myapp/current/app.db",
] {
let fail =
grade_sqlite_data_persistence(Some(refused), &cfg).expect("a SQLite app is graded");
assert!(!fail.passed, "{refused} must fail preflight");
}
}
#[test]
fn preflight_scope_is_none_for_one_host_and_the_host_for_a_fleet() {
let config = preflight_config();
let one = ResolvedFleet::resolve(&fleet_cfg(&["web-1.example.test"]), "myapp")
.expect("one-host fleet resolves");
let checks = collect_fleet_preflight(&config, &one, 1);
assert!(
checks.iter().all(|check| check.scope.is_none()),
"a single-host fleet must carry no scope (AC-1): {:?}",
checks
.iter()
.map(|c| (c.name, &c.scope))
.collect::<Vec<_>>()
);
let many = ResolvedFleet::resolve(
&fleet_cfg(&["web-1.example.test", "web-2.example.test"]),
"myapp",
)
.expect("two-host fleet resolves");
let checks = collect_fleet_preflight(&config, &many, 2);
let scoped: Vec<(&str, Option<&str>)> = checks
.iter()
.map(|check| (check.name, check.scope.as_deref()))
.collect();
assert_eq!(
scoped
.iter()
.filter(|(name, _)| *name == "ssh_reachability")
.map(|(_, scope)| *scope)
.collect::<Vec<_>>(),
vec![Some("web-1.example.test"), Some("web-2.example.test")],
"each per-host grader must name its host, in rollout order: {scoped:?}"
);
assert!(
scoped
.iter()
.filter(|(name, _)| *name != "ssh_reachability")
.all(|(_, scope)| scope.is_none()),
"project-wide graders must stay unscoped: {scoped:?}"
);
}
#[test]
fn an_empty_fleet_still_fails_the_preflight_gate() {
let empty = ResolvedFleet { hosts: vec![] };
let checks = collect_fleet_preflight(&preflight_config(), &empty, 0);
assert!(
checks
.iter()
.any(|check| check.blocking() && check.detail.contains(DEPLOY_HOST_MISSING_DETAIL)),
"a fleet with no target must fail the preflight gate, not return no checks: {:?}",
checks
.iter()
.map(|c| (c.name, c.passed, &c.detail))
.collect::<Vec<_>>()
);
}
#[test]
fn no_fleet_preflight_grader_returns_deferred() {
let config = preflight_config();
for hosts in [
vec!["web-1.example.test"],
vec!["web-1.example.test", "web-2.example.test"],
vec![
"web-1.example.test",
"web-2.example.test",
"web-3.example.test",
],
] {
let fleet =
ResolvedFleet::resolve(&fleet_cfg(&hosts), "myapp").expect("fleet resolves");
for check in collect_fleet_preflight(&config, &fleet, hosts.len()) {
assert!(
!check.deferred,
"fleet grader `{}` must never defer (scope {:?}): {}",
check.name, check.scope, check.detail
);
}
}
}
#[test]
fn preflight_report_label_appends_the_scope_only_when_present() {
let plain = PreflightCheck::pass("ssh_reachability", "reachable");
assert_eq!(preflight_label(&plain), "ssh_reachability");
let scoped = plain.scoped(Some("web-2.example.test".to_owned()));
assert_eq!(
preflight_label(&scoped),
"ssh_reachability (web-2.example.test)"
);
assert_eq!(
scoped.name, "ssh_reachability",
"the stable identifier must not absorb the host"
);
}
#[test]
fn doctor_deploy_check_names_are_the_shared_grader_names_prefixed() {
assert_eq!(
PREFLIGHT_GRADERS.len(),
DOCTOR_PREFLIGHT_GRADERS.len(),
"the two lists must stay the same length"
);
for (grader, doctor_name) in PREFLIGHT_GRADERS.iter().zip(DOCTOR_PREFLIGHT_GRADERS) {
assert_eq!(
doctor_name,
format!("deploy_{grader}"),
"doctor's name for `{grader}` must be the prefixed grader name"
);
}
let fleet = ResolvedFleet::resolve(
&fleet_cfg(&["web-1.example.test", "web-2.example.test"]),
"myapp",
)
.expect("fleet resolves");
let emitted: Vec<&str> = collect_fleet_preflight(&preflight_config(), &fleet, 2)
.iter()
.map(|check| check.name)
.collect();
assert_eq!(
emitted,
vec![
"ssh_reachability",
"ssh_reachability",
"signing_secret",
"database_url",
"migrate_check"
],
"the fleet preflight emits the per-host grader once per host, then the \
project graders once"
);
let mut distinct = emitted.clone();
distinct.sort_unstable();
distinct.dedup();
let mut expected = PREFLIGHT_GRADERS.to_vec();
expected.sort_unstable();
assert_eq!(distinct, expected);
}
#[test]
fn fleet_resolve_trims_each_host_like_the_single_host_path() {
let padded = fleet_cfg(&[" 203.0.113.10 "]);
let fleet = ResolvedFleet::resolve(&padded, "myapp").expect("padded fleet resolves");
let expected = ResolvedDeployConfig::resolve(
&DeployConfig {
host: Some(" 203.0.113.10 ".to_owned()),
..DeployConfig::default()
},
"myapp",
)
.expect("padded single host resolves");
assert_eq!(
fleet.hosts[0], expected,
"each fleet entry must be trimmed exactly like the scalar host, got: {:?}",
fleet.hosts[0]
);
}
#[test]
fn fleet_resolve_shares_the_single_host_defaults_and_tls_validation() {
let fleet = ResolvedFleet::resolve(&fleet_cfg(&["web-1", "web-2"]), "myapp")
.expect("a well-formed fleet resolves");
for host in &fleet.hosts {
assert_eq!(host.app_name, "myapp", "got: {host:?}");
assert_eq!(host.app_dir, "/srv/autumn/myapp", "got: {host:?}");
assert_eq!(host.service_name, "myapp", "got: {host:?}");
assert_eq!(host.profile, "prod", "got: {host:?}");
}
let cfg = DeployConfig {
hosts: vec!["web-1".to_owned(), "web-2".to_owned()],
tls: autumn_web::config::DeployTlsConfig {
enabled: true,
host: None,
},
..DeployConfig::default()
};
let err = ResolvedFleet::resolve(&cfg, "myapp")
.expect_err("enabled TLS without a host must be rejected for a fleet too");
assert!(
err.contains("[deploy.tls]") && err.contains("host"),
"the fleet must reuse the single-host TLS rejection, got: {err}"
);
}
#[test]
fn fleet_of_one_renders_todays_unit() {
let single = DeployConfig {
host: Some("203.0.113.10".to_owned()),
app_name: Some("shop".to_owned()),
user: "deploy".to_owned(),
..DeployConfig::default()
};
let fleet_spelling = DeployConfig {
host: None,
hosts: vec!["203.0.113.10".to_owned()],
..single.clone()
};
let today = ResolvedDeployConfig::resolve(&single, "shop").expect("single host resolves");
let fleet =
ResolvedFleet::resolve(&fleet_spelling, "shop").expect("single-entry fleet resolves");
let release_dir = "/srv/autumn/shop/releases/20240101120000";
let today_unit = render_app_unit(&today, release_dir, 3001, "blue");
let fleet_unit = render_app_unit(&fleet.hosts[0], release_dir, 3001, "blue");
assert_eq!(
fleet_unit, today_unit,
"a one-entry fleet must render today's slot unit verbatim;\nfleet:\n{fleet_unit}\ntoday:\n{today_unit}"
);
let today_service = render_systemd_unit(&today);
let fleet_service = render_systemd_unit(&fleet.hosts[0]);
assert_eq!(
fleet_service, today_service,
"a one-entry fleet must render today's service unit verbatim;\nfleet:\n{fleet_service}\ntoday:\n{today_service}"
);
}
#[test]
fn build_env_file_sets_production_profile_by_default() {
let config = AutumnConfig::default();
let resolved = ResolvedDeployConfig::resolve(&DeployConfig::default(), "myapp")
.expect("deploy config resolves");
let env_file = build_env_file(&config, &resolved);
assert!(
env_file.expose().lines().any(|l| l == "AUTUMN_ENV=prod"),
"env file should pin AUTUMN_ENV=prod by default, got: {:?}",
env_file.expose()
);
}
#[test]
fn deploy_status_reads_its_probe_port_under_the_deploy_profile_without_validating() {
use autumn_web::config::MockEnv;
let dir = tempfile::TempDir::new().expect("temp project dir");
std::fs::write(
dir.path().join("autumn.toml"),
"[server]\n\
port = 8080\n\
\n\
[profile.prod.server]\n\
port = 9090\n",
)
.expect("write autumn.toml");
let dirs = vec![dir.path().to_path_buf()];
assert_eq!(
declared_server_port_in(&dirs, "prod", &MockEnv::new()),
Some(9090),
);
assert_eq!(
declared_server_port_in(&dirs, "production", &MockEnv::new()),
Some(9090),
);
assert_eq!(
declared_server_port_in(&dirs, "staging", &MockEnv::new()),
Some(8080),
);
assert_eq!(
declared_server_port_in(
&dirs,
"prod",
&MockEnv::new().with("AUTUMN_SERVER__PORT", "7070")
),
Some(7070),
);
std::fs::write(
dir.path().join("autumn-prod.toml"),
"[server]\nport = 9191\n",
)
.expect("write autumn-prod.toml");
assert_eq!(
declared_server_port_in(&dirs, "prod", &MockEnv::new()),
Some(9191),
);
let empty = tempfile::TempDir::new().expect("temp dir");
assert_eq!(
declared_server_port_in(&[empty.path().to_path_buf()], "prod", &MockEnv::new()),
None,
);
}
#[test]
fn deploy_status_degrades_instead_of_aborting_on_an_unrelated_config_error() {
let dir = tempfile::TempDir::new().expect("temp project dir");
std::fs::write(
dir.path().join("autumn.toml"),
"[server]\nport = 8080\n\n[profile.prod.server]\nport = 9090\n",
)
.expect("write autumn.toml");
let dirs = vec![dir.path().to_path_buf()];
let degraded = status_public_port_with(
&dirs,
"prod",
Err(DeployError::Config(
"scheduler.max_concurrency must be greater than 0".to_owned(),
)),
)
.expect("a read-only status must not abort on an unrelated config error");
assert_eq!(degraded.port, 9090, "the PROFILE's declared port is used");
assert!(
degraded
.degraded
.as_deref()
.is_some_and(|reason| reason.contains("scheduler.max_concurrency")),
"the degradation must be reported, never silent: {:?}",
degraded.degraded
);
let ok = status_public_port_with(
&dirs,
"prod",
Ok(AutumnConfig {
server: autumn_web::config::ServerConfig {
port: 4321,
..autumn_web::config::ServerConfig::default()
},
..AutumnConfig::default()
}),
)
.expect("a valid config resolves");
assert_eq!(
ok,
StatusPort {
port: 4321,
degraded: None,
},
);
}
#[test]
fn build_env_file_honors_deploy_profile_override() {
let config = AutumnConfig::default();
let cfg = DeployConfig {
profile: "staging".to_owned(),
..DeployConfig::default()
};
let resolved =
ResolvedDeployConfig::resolve(&cfg, "myapp").expect("deploy config resolves");
let env_file = build_env_file(&config, &resolved);
assert!(
env_file.expose().lines().any(|l| l == "AUTUMN_ENV=staging"),
"env file should honor the [deploy] profile override, got: {:?}",
env_file.expose()
);
}
#[test]
fn runtime_config_loads_profile_scoped_prod_values_for_env_file_and_grading() {
use autumn_web::config::MockEnv;
let dir = tempfile::TempDir::new().expect("temp project dir");
let prod_secret = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
let prod_db_url = "postgres://prod_user:prod_pw@proddb.internal/app";
std::fs::write(
dir.path().join("autumn.toml"),
format!(
"[deploy]\n\
host = \"deploy.example.test\"\n\
\n\
[database]\n\
primary_url = \"postgres://dev:dev@localhost/devapp\"\n\
\n\
[security.signing_secret]\n\
secret = \"changeme\"\n\
\n\
[profile.prod.database]\n\
primary_url = \"{prod_db_url}\"\n\
\n\
[profile.prod.security.signing_secret]\n\
secret = \"{prod_secret}\"\n"
),
)
.expect("write autumn.toml");
let resolved =
ResolvedDeployConfig::resolve(&DeployConfig::default(), "demoapp").expect("resolves");
assert_eq!(resolved.profile, "prod");
let forced = ForcedProfileEnv {
profile: resolved.profile.clone(),
inner: MockEnv::new().with("AUTUMN_MANIFEST_DIR", dir.path().to_str().unwrap()),
};
let runtime_config =
AutumnConfig::load_with_env(&forced).expect("load config under prod profile");
let env_file = build_env_file(&runtime_config, &resolved);
let body = env_file.expose();
assert!(
body.lines().any(|l| l == "AUTUMN_ENV=prod"),
"env file must pin AUTUMN_ENV to the deploy profile, got: {body:?}"
);
assert!(
body.lines()
.any(|l| l == format!("AUTUMN_SECURITY__SIGNING_SECRET={prod_secret}")),
"env file must carry the PROD signing secret, got: {body:?}"
);
assert!(
body.lines()
.any(|l| l == format!("AUTUMN_DATABASE__URL={prod_db_url}")),
"env file must carry the PROD database URL, got: {body:?}"
);
assert!(
!body.contains("changeme"),
"env file must not ship the dev/base signing secret"
);
assert!(
!body.contains("devapp"),
"env file must not ship the dev/base database URL"
);
let signing = collect_preflight(&runtime_config, &resolved)
.into_iter()
.find(|c| c.name == "signing_secret")
.expect("preflight includes a signing_secret check");
assert!(
signing.passed,
"the strong PROD signing secret must pass preflight: {}",
signing.detail
);
}
#[test]
fn forced_profile_env_overrides_only_autumn_env() {
use autumn_web::config::MockEnv;
let forced = ForcedProfileEnv {
profile: "prod".to_owned(),
inner: MockEnv::new()
.with("AUTUMN_ENV", "dev")
.with("SOME_OTHER_KEY", "kept"),
};
assert_eq!(forced.var("AUTUMN_ENV").as_deref(), Ok("prod"));
assert_eq!(forced.var("SOME_OTHER_KEY").as_deref(), Ok("kept"));
assert!(forced.var("UNSET_KEY").is_err());
}
#[test]
fn forced_profile_env_honors_explicit_autumn_dotenv() {
use autumn_web::config::MockEnv;
let off = ForcedProfileEnv {
profile: "prod".to_owned(),
inner: MockEnv::new().with("AUTUMN_DOTENV", "0"),
};
assert_eq!(off.var("AUTUMN_DOTENV").as_deref(), Ok("0"));
assert_eq!(off.var("AUTUMN_ENV").as_deref(), Ok("prod"));
let unset = ForcedProfileEnv {
profile: "prod".to_owned(),
inner: MockEnv::new(),
};
assert_eq!(unset.var("AUTUMN_DOTENV").as_deref(), Ok("1"));
let on = ForcedProfileEnv {
profile: "prod".to_owned(),
inner: MockEnv::new().with("AUTUMN_DOTENV", "true"),
};
assert_eq!(on.var("AUTUMN_DOTENV").as_deref(), Ok("true"));
}
#[test]
fn preflight_grades_signing_secret_against_resolved_deploy_profile() {
let find_signing = |checks: &[PreflightCheck]| {
checks
.iter()
.find(|c| c.name == "signing_secret")
.expect("preflight includes a signing_secret check")
.clone()
};
let mut config = AutumnConfig::default();
config.security.signing_secret.secret = Some("changeme".to_owned());
assert!(
!is_production_profile(config.profile.as_deref()),
"guard: the local runtime profile is non-production in this test"
);
let prod_resolved =
ResolvedDeployConfig::resolve(&DeployConfig::default(), "myapp").expect("resolves");
assert_eq!(prod_resolved.profile, "prod");
let prod_check = find_signing(&collect_preflight(&config, &prod_resolved));
assert!(
!prod_check.passed,
"weak secret must fail preflight under the default (prod) deploy \
profile: {}",
prod_check.detail
);
assert!(!prod_check.detail.contains("changeme"));
let staging_resolved = ResolvedDeployConfig::resolve(
&DeployConfig {
profile: "staging".to_owned(),
..DeployConfig::default()
},
"myapp",
)
.expect("deploy config resolves");
let staging_check = find_signing(&collect_preflight(&config, &staging_resolved));
assert!(
staging_check.passed,
"weak secret should pass preflight under a non-production deploy \
profile: {}",
staging_check.detail
);
}
#[test]
fn resolve_preserves_raw_profile_alias_while_grading_normalized() {
let find_signing = |checks: &[PreflightCheck]| {
checks
.iter()
.find(|c| c.name == "signing_secret")
.expect("preflight includes a signing_secret check")
.clone()
};
let mut config = AutumnConfig::default();
config.security.signing_secret.secret = Some("changeme".to_owned());
for (raw, expected) in [
("PROD", "PROD"),
("Production", "Production"),
(" production ", "production"),
] {
let resolved = ResolvedDeployConfig::resolve(
&DeployConfig {
profile: raw.to_owned(),
..DeployConfig::default()
},
"myapp",
)
.expect("deploy config resolves");
assert_eq!(
resolved.profile, expected,
"profile {raw:?} should preserve the trimmed raw spelling"
);
let expected_env = format!("AUTUMN_ENV={expected}");
assert!(
build_env_file(&config, &resolved)
.expose()
.lines()
.any(|l| l == expected_env),
"env file should pin {expected_env:?} for raw profile {raw:?}"
);
assert!(
is_production_profile(Some(&canonicalize_deploy_profile(&resolved.profile))),
"normalized profile must grade as production for raw profile {raw:?}"
);
let check = find_signing(&collect_preflight(&config, &resolved));
assert!(
!check.passed,
"weak secret must fail preflight for production spelling {raw:?}: {}",
check.detail
);
assert!(!check.detail.contains("changeme"));
}
let staging = ResolvedDeployConfig::resolve(
&DeployConfig {
profile: "staging".to_owned(),
..DeployConfig::default()
},
"myapp",
)
.expect("deploy config resolves");
assert_eq!(staging.profile, "staging");
assert!(!is_production_profile(Some(&staging.profile)));
let staging_check = find_signing(&collect_preflight(&config, &staging));
assert!(
staging_check.passed,
"weak secret should pass preflight under the custom `staging` profile: {}",
staging_check.detail
);
}
#[test]
fn resolve_honors_explicit_overrides() {
let cfg = DeployConfig {
host: Some("203.0.113.10".to_owned()),
app_name: Some("web".to_owned()),
app_dir: Some("/opt/web".to_owned()),
service_name: Some("web-svc".to_owned()),
..DeployConfig::default()
};
let resolved =
ResolvedDeployConfig::resolve(&cfg, "ignored").expect("deploy config resolves");
assert_eq!(resolved.app_name, "web");
assert_eq!(resolved.app_dir, "/opt/web");
assert_eq!(resolved.service_name, "web-svc");
assert_eq!(resolved.host.as_deref(), Some("203.0.113.10"));
}
#[test]
fn resolve_blank_overrides_fall_back_to_defaults() {
let cfg = DeployConfig {
app_name: Some(" ".to_owned()),
app_dir: Some(String::new()),
..DeployConfig::default()
};
let resolved =
ResolvedDeployConfig::resolve(&cfg, "fallback").expect("deploy config resolves");
assert_eq!(resolved.app_name, "fallback");
assert_eq!(resolved.app_dir, "/srv/autumn/fallback");
}
#[test]
fn systemd_unit_contains_service_paths_and_env_file() {
let cfg = ResolvedDeployConfig::resolve(
&DeployConfig {
app_name: Some("shop".to_owned()),
service_name: Some("shop-web".to_owned()),
user: "deploy".to_owned(),
..DeployConfig::default()
},
"shop",
)
.expect("deploy config resolves");
let unit = render_systemd_unit(&cfg);
assert!(unit.contains("Description=Autumn application: shop"));
assert!(unit.contains("User=deploy"));
assert!(unit.contains("WorkingDirectory=/srv/autumn/shop/current"));
assert!(unit.contains("ExecStart=/srv/autumn/shop/current/shop"));
assert!(!unit.contains("autumn serve --release"));
assert!(unit.contains("Restart=on-failure"));
assert!(unit.contains("WantedBy=multi-user.target"));
assert!(unit.contains("EnvironmentFile=/srv/autumn/shop/shared/autumn.env"));
assert!(!unit.contains("Environment=SECRET"));
assert!(unit.contains("Environment=AUTUMN_MANIFEST_DIR=/srv/autumn/shop/current"));
}
#[test]
fn app_unit_points_the_maintenance_flag_at_the_shared_dir() {
let cfg = ResolvedDeployConfig::resolve(
&DeployConfig {
app_name: Some("shop".to_owned()),
..DeployConfig::default()
},
"shop",
)
.expect("deploy config resolves");
let unit = render_app_unit(&cfg, "/srv/autumn/shop/releases/r1", 3001, "blue");
assert!(
unit.contains(
"Environment=AUTUMN_MAINTENANCE_FLAG_FILE=/srv/autumn/shop/shared/autumn-maintenance.json"
),
"slot unit must point the maintenance flag at the shared dir: {unit}"
);
assert!(
!unit.contains("AUTUMN_MAINTENANCE_FLAG_FILE=/srv/autumn/shop/releases"),
"the maintenance flag must not be release-scoped: {unit}"
);
assert!(unit.contains(&format!(
"Environment=AUTUMN_MAINTENANCE_FLAG_FILE={}",
cfg.maintenance_flag_file()
)));
assert!(unit.contains(autumn_web::maintenance::MAINTENANCE_FLAG_FILE_ENV));
assert!(
cfg.maintenance_flag_file()
.ends_with(autumn_web::maintenance::MAINTENANCE_FLAG_BASENAME)
);
}
#[test]
fn app_unit_sets_manifest_dir_to_release_for_uploaded_config() {
let cfg = ResolvedDeployConfig::resolve(
&DeployConfig {
app_name: Some("shop".to_owned()),
..DeployConfig::default()
},
"shop",
)
.expect("deploy config resolves");
let release_dir = "/srv/autumn/shop/releases/r1";
let unit = render_app_unit(&cfg, release_dir, 3001, "blue");
assert!(
unit.contains(&format!("Environment=AUTUMN_MANIFEST_DIR={release_dir}")),
"slot unit must set AUTUMN_MANIFEST_DIR to the release dir: {unit}"
);
assert!(unit.contains(&format!("WorkingDirectory={release_dir}")));
assert!(unit.contains("EnvironmentFile=/srv/autumn/shop/shared/autumn.env"));
}
#[test]
fn render_systemd_unit_sets_manifest_dir_to_current_symlink() {
let cfg = ResolvedDeployConfig::resolve(
&DeployConfig {
app_name: Some("shop".to_owned()),
..DeployConfig::default()
},
"shop",
)
.expect("deploy config resolves");
let unit = render_systemd_unit(&cfg);
let current = cfg.current_symlink();
assert!(
unit.contains(&format!("Environment=AUTUMN_MANIFEST_DIR={current}")),
"render_systemd_unit must set AUTUMN_MANIFEST_DIR to the current symlink: {unit}"
);
assert!(unit.contains(&format!("WorkingDirectory={current}")));
}
#[test]
fn manifest_uploads_base_only_when_no_profile_sibling() {
let dir = tempfile::tempdir().expect("tempdir");
std::fs::write(dir.path().join("autumn.toml"), "[server]\nport = 8080\n")
.expect("write autumn.toml");
let dirs = vec![dir.path().to_path_buf()];
let uploads = manifest_uploads_in(&dirs, "prod");
assert_eq!(uploads.len(), 1, "only autumn.toml is present");
assert_eq!(uploads[0].remote_basename, "autumn.toml");
assert_eq!(uploads[0].local, dir.path().join("autumn.toml"));
}
#[test]
fn manifest_uploads_include_the_configured_capacity_contract() {
let dir = tempfile::tempdir().expect("tempdir");
std::fs::write(
dir.path().join("autumn.toml"),
"[server]\ncapacity_contract = \"capacity.lock\"\n",
)
.expect("write base");
std::fs::write(dir.path().join("capacity.lock"), "version = 1\n").expect("write contract");
let dirs = vec![dir.path().to_path_buf()];
let uploads = manifest_uploads_in(&dirs, "prod");
let names: Vec<&str> = uploads.iter().map(|u| u.remote_basename.as_str()).collect();
assert_eq!(names, vec!["autumn.toml", "capacity.lock"]);
}
#[test]
fn manifest_uploads_skip_a_capacity_contract_that_is_not_configured() {
let dir = tempfile::tempdir().expect("tempdir");
std::fs::write(dir.path().join("autumn.toml"), "[server]\n").expect("write base");
std::fs::write(dir.path().join("capacity.lock"), "version = 1\n").expect("write contract");
let dirs = vec![dir.path().to_path_buf()];
let uploads = manifest_uploads_in(&dirs, "prod");
let names: Vec<&str> = uploads.iter().map(|u| u.remote_basename.as_str()).collect();
assert_eq!(names, vec!["autumn.toml"]);
}
#[test]
fn manifest_uploads_read_the_contract_from_an_inline_profile_overlay() {
let dir = tempfile::tempdir().expect("tempdir");
std::fs::write(
dir.path().join("autumn.toml"),
"[server]\n\n[profile.prod.server]\ncapacity_contract = \"capacity.lock\"\n",
)
.expect("write base");
std::fs::write(dir.path().join("capacity.lock"), "version = 1\n").expect("write contract");
let dirs = vec![dir.path().to_path_buf()];
let uploads = manifest_uploads_in(&dirs, "prod");
let names: Vec<&str> = uploads.iter().map(|u| u.remote_basename.as_str()).collect();
assert_eq!(names, vec!["autumn.toml", "capacity.lock"]);
}
#[test]
fn manifest_uploads_read_the_contract_from_a_profile_only_manifest() {
let dir = tempfile::tempdir().expect("tempdir");
std::fs::write(
dir.path().join("autumn-prod.toml"),
"[server]\ncapacity_contract = \"capacity.lock\"\n",
)
.expect("write sibling");
std::fs::write(dir.path().join("capacity.lock"), "version = 1\n").expect("write contract");
let dirs = vec![dir.path().to_path_buf()];
let uploads = manifest_uploads_in(&dirs, "prod");
let names: Vec<&str> = uploads.iter().map(|u| u.remote_basename.as_str()).collect();
assert_eq!(names, vec!["autumn-prod.toml", "capacity.lock"]);
}
#[test]
fn manifest_uploads_refuse_to_flatten_a_nested_contract_path() {
let dir = tempfile::tempdir().expect("tempdir");
std::fs::write(
dir.path().join("autumn.toml"),
"[server]\ncapacity_contract = \"contracts/prod.lock\"\n",
)
.expect("write base");
std::fs::create_dir_all(dir.path().join("contracts")).expect("mkdir");
std::fs::write(dir.path().join("contracts/prod.lock"), "version = 1\n")
.expect("write contract");
let dirs = vec![dir.path().to_path_buf()];
let uploads = manifest_uploads_in(&dirs, "prod");
let names: Vec<&str> = uploads.iter().map(|u| u.remote_basename.as_str()).collect();
assert_eq!(names, vec!["autumn.toml"]);
}
#[test]
fn manifest_uploads_honour_a_profile_sibling_clearing_the_contract() {
let dir = tempfile::tempdir().expect("tempdir");
std::fs::write(
dir.path().join("autumn.toml"),
"[server]\ncapacity_contract = \"capacity.lock\"\n",
)
.expect("write base");
std::fs::write(
dir.path().join("autumn-prod.toml"),
"[server]\ncapacity_contract = \"\"\n",
)
.expect("write sibling");
std::fs::write(dir.path().join("capacity.lock"), "version = 1\n").expect("write contract");
let dirs = vec![dir.path().to_path_buf()];
let uploads = manifest_uploads_in(&dirs, "prod");
let names: Vec<&str> = uploads.iter().map(|u| u.remote_basename.as_str()).collect();
assert_eq!(names, vec!["autumn.toml", "autumn-prod.toml"]);
}
#[test]
fn manifest_uploads_include_profile_sibling_when_present() {
let dir = tempfile::tempdir().expect("tempdir");
std::fs::write(dir.path().join("autumn.toml"), "[server]\n").expect("write base");
std::fs::write(dir.path().join("autumn-prod.toml"), "[server]\n").expect("write sibling");
let dirs = vec![dir.path().to_path_buf()];
let uploads = manifest_uploads_in(&dirs, "prod");
let names: Vec<&str> = uploads.iter().map(|u| u.remote_basename.as_str()).collect();
assert_eq!(names, vec!["autumn.toml", "autumn-prod.toml"]);
}
#[test]
fn manifest_uploads_normalize_production_to_canonical_sibling() {
let dir = tempfile::tempdir().expect("tempdir");
std::fs::write(dir.path().join("autumn.toml"), "[server]\n").expect("write base");
std::fs::write(dir.path().join("autumn-production.toml"), "[server]\n")
.expect("write sibling");
let dirs = vec![dir.path().to_path_buf()];
let uploads = manifest_uploads_in(&dirs, "Production");
let names: Vec<&str> = uploads.iter().map(|u| u.remote_basename.as_str()).collect();
assert_eq!(names, vec!["autumn.toml", "autumn-production.toml"]);
assert!(
!uploads
.iter()
.any(|u| u.remote_basename == "autumn-Production.toml"),
"must not upload the raw-cased spelling"
);
}
#[test]
fn manifest_uploads_check_canonical_profile_sibling() {
let dir = tempfile::tempdir().expect("tempdir");
std::fs::write(dir.path().join("autumn.toml"), "[server]\n").expect("write base");
std::fs::write(dir.path().join("autumn-prod.toml"), "[server]\n").expect("write sibling");
let dirs = vec![dir.path().to_path_buf()];
let uploads = manifest_uploads_in(&dirs, "production");
assert!(
uploads
.iter()
.any(|u| u.remote_basename == "autumn-prod.toml"),
"canonical prod sibling is uploaded for raw profile `production`"
);
}
#[test]
fn manifest_uploads_raw_prod_prefers_prod_sibling_first() {
let dir = tempfile::tempdir().expect("tempdir");
std::fs::write(dir.path().join("autumn.toml"), "[server]\n").expect("write base");
std::fs::write(dir.path().join("autumn-prod.toml"), "[server]\n").expect("write sibling");
let dirs = vec![dir.path().to_path_buf()];
let uploads = manifest_uploads_in(&dirs, "prod");
let names: Vec<&str> = uploads.iter().map(|u| u.remote_basename.as_str()).collect();
assert_eq!(names, vec!["autumn.toml", "autumn-prod.toml"]);
}
#[test]
fn manifest_uploads_custom_profile_uses_verbatim_sibling() {
let dir = tempfile::tempdir().expect("tempdir");
std::fs::write(dir.path().join("autumn.toml"), "[server]\n").expect("write base");
std::fs::write(dir.path().join("autumn-staging.toml"), "[server]\n")
.expect("write sibling");
let dirs = vec![dir.path().to_path_buf()];
let uploads = manifest_uploads_in(&dirs, "staging");
let names: Vec<&str> = uploads.iter().map(|u| u.remote_basename.as_str()).collect();
assert_eq!(names, vec!["autumn.toml", "autumn-staging.toml"]);
}
#[test]
fn manifest_uploads_first_lookup_name_wins_when_both_present() {
let dir = tempfile::tempdir().expect("tempdir");
std::fs::write(dir.path().join("autumn.toml"), "[server]\n").expect("write base");
std::fs::write(dir.path().join("autumn-production.toml"), "[server]\n")
.expect("write production");
std::fs::write(dir.path().join("autumn-prod.toml"), "[server]\n").expect("write prod");
let dirs = vec![dir.path().to_path_buf()];
let uploads = manifest_uploads_in(&dirs, "production");
let siblings: Vec<&str> = uploads
.iter()
.map(|u| u.remote_basename.as_str())
.filter(|n| n.starts_with("autumn-"))
.collect();
assert_eq!(
siblings,
vec!["autumn-production.toml"],
"only the first-lookup-name sibling is uploaded, matching the runtime"
);
}
#[test]
fn manifest_uploads_base_falls_back_to_cwd_dir() {
let manifest_dir = tempfile::tempdir().expect("manifest dir");
let cwd = tempfile::tempdir().expect("cwd dir");
std::fs::write(cwd.path().join("autumn.toml"), "[server]\n").expect("write base");
std::fs::write(cwd.path().join("autumn-prod.toml"), "[server]\n").expect("write sibling");
let dirs = vec![manifest_dir.path().to_path_buf(), cwd.path().to_path_buf()];
let uploads = manifest_uploads_in(&dirs, "prod");
assert_eq!(
uploads.len(),
2,
"base + sibling picked up from the CWD fallback"
);
assert_eq!(uploads[0].remote_basename, "autumn.toml");
assert_eq!(uploads[0].local, cwd.path().join("autumn.toml"));
assert_eq!(uploads[1].remote_basename, "autumn-prod.toml");
assert_eq!(uploads[1].local, cwd.path().join("autumn-prod.toml"));
}
#[test]
fn manifest_uploads_primary_dir_wins_over_fallback() {
let manifest_dir = tempfile::tempdir().expect("manifest dir");
let cwd = tempfile::tempdir().expect("cwd dir");
std::fs::write(manifest_dir.path().join("autumn.toml"), "[server]\n").expect("primary");
std::fs::write(cwd.path().join("autumn.toml"), "[server]\n").expect("fallback");
let dirs = vec![manifest_dir.path().to_path_buf(), cwd.path().to_path_buf()];
let uploads = manifest_uploads_in(&dirs, "prod");
assert_eq!(uploads[0].local, manifest_dir.path().join("autumn.toml"));
}
#[test]
fn manifest_uploads_empty_when_no_manifest() {
let dir = tempfile::tempdir().expect("tempdir");
let dirs = vec![dir.path().to_path_buf()];
let uploads = manifest_uploads_in(&dirs, "prod");
assert!(uploads.is_empty(), "no autumn.toml → nothing to upload");
}
#[test]
fn media_config_no_manifest_is_disabled_default() {
let dir = tempfile::tempdir().expect("tempdir");
let dirs = vec![dir.path().to_path_buf()];
let (cfg, bin) = load_media_host_config_in(&dirs, "prod").expect("loads");
assert!(!cfg.enabled, "no autumn.toml → controller disabled");
assert_eq!(bin, media::DEFAULT_FFMPEG_BIN);
}
#[test]
fn media_config_honors_inline_profile_section() {
let dir = tempfile::tempdir().expect("tempdir");
std::fs::write(
dir.path().join("autumn.toml"),
"[media.ffmpeg]\nbin = \"/usr/bin/ffmpeg\"\n\
[profile.prod.media.mediamtx]\nenabled = true\napi_port = 19997\n",
)
.expect("write base");
let dirs = vec![dir.path().to_path_buf()];
let (prod, _) = load_media_host_config_in(&dirs, "prod").expect("loads prod");
assert!(prod.enabled, "prod profile enables media");
assert_eq!(prod.api_port, 19997);
let (dev, _) = load_media_host_config_in(&dirs, "dev").expect("loads dev");
assert!(
!dev.enabled,
"dev has no profile media section → base default"
);
}
#[test]
fn media_config_honors_profile_override_file() {
let dir = tempfile::tempdir().expect("tempdir");
std::fs::write(dir.path().join("autumn.toml"), "[server]\nport = 3000\n")
.expect("write base");
std::fs::write(
dir.path().join("autumn-prod.toml"),
"[media.mediamtx]\nenabled = true\nrtmp_port = 11935\n",
)
.expect("write prod override");
let dirs = vec![dir.path().to_path_buf()];
let (prod, _) = load_media_host_config_in(&dirs, "prod").expect("loads prod");
assert!(prod.enabled, "autumn-prod.toml enables media");
assert_eq!(prod.rtmp_port, 11935);
}
#[test]
fn media_config_honors_profile_override_file_without_base_autumn_toml() {
let dir = tempfile::tempdir().expect("tempdir");
std::fs::write(
dir.path().join("autumn-prod.toml"),
"[media.mediamtx]\nenabled = true\nrtmp_port = 11935\n",
)
.expect("write prod override");
let dirs = vec![dir.path().to_path_buf()];
let (prod, _) = load_media_host_config_in(&dirs, "prod").expect("loads prod");
assert!(
prod.enabled,
"profile-only media config is provisioned with no base autumn.toml"
);
assert_eq!(prod.rtmp_port, 11935);
let (dev, _) = load_media_host_config_in(&dirs, "dev").expect("loads dev");
assert!(!dev.enabled, "dev has no override file → disabled default");
}
#[test]
fn media_config_profile_layer_overrides_base_value() {
let dir = tempfile::tempdir().expect("tempdir");
std::fs::write(
dir.path().join("autumn.toml"),
"[media.mediamtx]\nenabled = true\napi_port = 9997\n",
)
.expect("write base");
std::fs::write(
dir.path().join("autumn-prod.toml"),
"[media.mediamtx]\napi_port = 29997\n",
)
.expect("write prod override");
let dirs = vec![dir.path().to_path_buf()];
let (prod, _) = load_media_host_config_in(&dirs, "prod").expect("loads prod");
assert!(
prod.enabled,
"base enablement is preserved under deep merge"
);
assert_eq!(
prod.api_port, 29997,
"profile override wins over the base value"
);
}
#[test]
fn media_config_base_only_when_profile_has_no_media_layer() {
let dir = tempfile::tempdir().expect("tempdir");
std::fs::write(
dir.path().join("autumn.toml"),
"[media.mediamtx]\nenabled = true\napi_port = 9001\n",
)
.expect("write base");
let dirs = vec![dir.path().to_path_buf()];
let (prod, _) = load_media_host_config_in(&dirs, "prod").expect("loads prod");
assert!(prod.enabled);
assert_eq!(prod.api_port, 9001);
}
#[test]
fn media_config_fails_closed_on_invalid_value_in_active_profile_layer() {
let dir = tempfile::tempdir().expect("tempdir");
std::fs::write(
dir.path().join("autumn.toml"),
"[media.mediamtx]\nenabled = true\n",
)
.expect("write base");
std::fs::write(
dir.path().join("autumn-prod.toml"),
"[media.mediamtx]\napi_port = \"nope\"\n",
)
.expect("write bad prod override");
let dirs = vec![dir.path().to_path_buf()];
let err = load_media_host_config_in(&dirs, "prod").expect_err("must fail closed");
assert!(
matches!(err, DeployError::Config(_)),
"invalid profile-layer media config aborts the deploy: {err:?}"
);
}
#[test]
fn manifest_notice_warns_loudly_when_no_manifest() {
let notice = manifest_preflight_notice(&[]);
assert!(
notice.contains("no autumn.toml") && notice.contains("built-in defaults"),
"no-manifest notice must loudly warn about silent defaults: {notice}"
);
}
#[test]
fn manifest_notice_confirms_uploaded_files() {
let uploads = vec![
exec::ManifestUpload {
local: PathBuf::from("/x/autumn.toml"),
remote_basename: "autumn.toml".to_owned(),
},
exec::ManifestUpload {
local: PathBuf::from("/x/autumn-prod.toml"),
remote_basename: "autumn-prod.toml".to_owned(),
},
];
let notice = manifest_preflight_notice(&uploads);
assert!(notice.contains("autumn.toml"));
assert!(notice.contains("autumn-prod.toml"));
assert!(!notice.contains("no autumn.toml"));
}
#[test]
fn deploy_plan_renders_the_sqlite_data_steps() {
let labels = |url: &str| -> Vec<&'static str> {
let mut config = AutumnConfig::default();
config.database.url = Some(url.to_owned());
let cfg = with_sqlite_data_contract(&resolved_defaults(), &config);
build_deploy_plan(&cfg).iter().map(|s| s.label).collect()
};
let position = |labels: &[&str], label: &str| {
labels
.iter()
.position(|l| *l == label)
.unwrap_or_else(|| panic!("no {label} step: {labels:?}"))
};
let relative = labels("sqlite://app.db");
assert!(position(&relative, "link-data") < position(&relative, "upload"));
assert!(position(&relative, "migrate") < position(&relative, "record-data-adopted"));
let absolute = labels("sqlite:///var/lib/myapp/app.db");
assert!(position(&absolute, "check-data-dir") < position(&absolute, "upload"));
assert!(!absolute.contains(&"link-data"), "{absolute:?}");
let postgres = labels("postgres://db.example.com/app");
for label in ["link-data", "check-data-dir", "record-data-adopted"] {
assert!(!postgres.contains(&label), "{postgres:?}");
}
}
#[test]
fn deploy_plan_names_a_refused_sqlite_database() {
let mut config = AutumnConfig::default();
config.database.url = Some("sqlite::memory:".to_owned());
let note = sqlite_plan_note(&config, &resolved_defaults()).expect("a note");
assert!(note.contains("IN-MEMORY"), "{note}");
config.database.url = Some("sqlite://app.db".to_owned());
assert_eq!(sqlite_plan_note(&config, &resolved_defaults()), None);
}
#[test]
fn deploy_plan_runs_migrations_before_cutover_with_bounded_readiness_gate() {
let resolved = resolved_defaults();
let plan = build_deploy_plan(&resolved);
let labels: Vec<&str> = plan.iter().map(|s| s.label).collect();
let migrate = labels
.iter()
.position(|&l| l == "migrate")
.expect("migrate step");
let candidate = labels
.iter()
.position(|&l| l == "start-candidate")
.expect("start-candidate step");
let readiness = labels
.iter()
.position(|&l| l == "readiness-gate")
.expect("readiness-gate step");
let cutover = labels
.iter()
.position(|&l| l == "cutover")
.expect("cutover step");
let drain = labels
.iter()
.position(|&l| l == "drain")
.expect("drain step");
assert!(
migrate < readiness,
"migrations must precede the readiness gate"
);
assert!(
migrate < cutover,
"migrations must precede cutover (a failed migration leaves the old version serving)"
);
assert!(readiness < cutover, "readiness gate must precede cutover");
let candidate_desc = &plan[candidate].description;
let candidate_lc = candidate_desc.to_lowercase();
assert!(
candidate_lc.contains("separate") || candidate_lc.contains("distinct"),
"candidate must start on a separate/distinct listener: {candidate_desc}"
);
assert!(
candidate_lc.contains("does not bind") || candidate_lc.contains("not bind"),
"candidate step must state it does NOT bind the live port: {candidate_desc}"
);
assert!(
readiness < cutover,
"handoff must follow the readiness gate"
);
assert!(
cutover < drain,
"traffic handoff must precede draining the old release"
);
let cutover_desc = &plan[cutover].description.to_lowercase();
assert!(
cutover_desc.contains("hand") || cutover_desc.contains("traffic"),
"cutover must describe a traffic handoff: {}",
plan[cutover].description
);
let gate = &plan[readiness].description;
assert!(
gate.contains("60s"),
"readiness gate should be bounded: {gate}"
);
assert!(
gate.to_lowercase().contains("roll back"),
"gate should roll back on timeout: {gate}"
);
assert_eq!(plan.last().expect("non-empty plan").label, "prune");
assert!(plan.last().unwrap().description.contains('3'));
}
#[test]
fn rollback_plan_matches_rollback_ops_sequence() {
let resolved = resolved_defaults();
let plan = build_rollback_plan(&resolved);
let labels: Vec<&str> = plan.iter().map(|s| s.label).collect();
assert_eq!(
labels,
vec![
"write-target-unit",
"daemon-reload",
"restart-previous",
"proxy-flip",
"record-previous",
"repoint",
"record-live-slot",
"readiness-gate",
"drain-rolled-back-slot",
],
"printed rollback plan must match rollback_ops' execution order"
);
let pos = |l: &str| labels.iter().position(|&x| x == l).expect("step present");
assert!(
pos("write-target-unit") < pos("daemon-reload")
&& pos("daemon-reload") < pos("restart-previous"),
"the target unit is re-rendered and daemon-reloaded before the restart"
);
assert!(
pos("restart-previous") < pos("proxy-flip"),
"previous unit must be up before the health-gated flip"
);
assert!(
pos("proxy-flip") < pos("record-previous"),
"flip must precede recording the previous-release marker"
);
assert!(
pos("record-previous") < pos("repoint"),
"the previous-release marker is recorded before `current` moves off it"
);
assert!(
pos("proxy-flip") < pos("repoint"),
"flip must precede the current-symlink repoint"
);
assert!(
pos("restart-previous") < pos("readiness-gate"),
"restart must precede the /ready re-probe"
);
assert!(
pos("readiness-gate") < pos("drain-rolled-back-slot"),
"the former-live slot is drained last, after the /ready re-probe"
);
}
#[test]
fn fleet_plan_matches_fleet_ops_sequence() {
let fleet = ResolvedFleet::resolve(&fleet_cfg(&["web-a", "web-b", "web-c"]), "myapp")
.expect("a well-formed fleet resolves");
let modes = [
fleet::HostMode::Redeploy,
fleet::HostMode::Redeploy,
fleet::HostMode::Redeploy,
];
let plan = fleet::plan_fleet(&fleet, &modes).expect("a well-formed fleet plans");
let hosts: Vec<String> = fleet
.hosts
.iter()
.map(|h| h.host.clone().unwrap_or_default())
.collect();
let planned: Vec<String> = plan.hosts.iter().map(|h| h.host.clone()).collect();
assert_eq!(
planned, hosts,
"the executable fleet plan must keep declaration order"
);
let rendered = fleet::fleet_plan_lines(&hosts).join("\n");
let mut previous = 0usize;
for host in &hosts {
let at = rendered
.find(host.as_str())
.unwrap_or_else(|| panic!("{host} must be named in the plan:\n{rendered}"));
assert!(
at >= previous,
"the printed fleet plan must list hosts in rollout order:\n{rendered}"
);
previous = at;
}
let flat = fleet::test_support::fleet_op_labels(&fleet, &plan);
assert_eq!(
flat.iter().filter(|l| **l == "migrate").count(),
1,
"the plan promises one migration; the ops must schedule one: {flat:?}"
);
assert_eq!(
rendered
.matches(fleet::FLEET_MIGRATE_PLACEMENT_NOTE)
.count(),
1,
"the migrate placement is one fleet-wide note, not a per-host line:\n{rendered}"
);
let migrate = flat
.iter()
.position(|l| *l == "migrate")
.expect("the fleet migrates");
let boundary = flat
.iter()
.position(|l| *l == "proxy-flip" || *l == "proxy-route")
.expect("the fleet cuts over");
assert!(
migrate < boundary,
"the plan's `[migrate] < [cutover]` claim must hold in the real ops: \
migrate at {migrate}, boundary at {boundary}, labels: {flat:?}"
);
let step_labels: Vec<&str> = build_deploy_plan(&fleet.hosts[0])
.iter()
.map(|s| s.label)
.collect();
let (executed, neutral): (Vec<&str>, Vec<&str>) =
step_labels.iter().partition(|l| flat.contains(l));
assert_eq!(
executed,
vec!["migrate", "start-candidate", "readiness-gate", "prune"],
"these printed steps must name real executed ops: {step_labels:?} vs {flat:?}"
);
assert_eq!(
neutral,
vec!["build", "prepare-host", "upload", "cutover", "drain"],
"only the documented mechanism-neutral plan labels may be absent from \
the ops: {step_labels:?} vs {flat:?}"
);
assert_eq!(
step_labels.last().copied(),
Some("prune"),
"the printed plan ends with prune"
);
assert_eq!(
flat.last().copied(),
Some("prune"),
"the fleet's last host ends with prune: {flat:?}"
);
}
#[test]
fn deploy_host_present_grader_is_offline_and_flags_missing_host() {
let ok = grade_deploy_host_present(Some("203.0.113.10"));
assert!(ok.passed);
assert_eq!(ok.name, "deploy_host");
let missing = grade_deploy_host_present(None);
assert!(!missing.passed, "missing host must fail offline");
assert_eq!(missing.name, "deploy_host");
assert_eq!(missing.detail, DEPLOY_HOST_MISSING_DETAIL);
assert!(missing.hint.unwrap().contains("[deploy] host"));
let blank = grade_deploy_host_present(Some(" "));
assert!(!blank.passed, "blank host must fail offline");
assert_eq!(blank.detail, DEPLOY_HOST_MISSING_DETAIL);
let probe = grade_ssh_reachability(None, 22, Duration::from_millis(50));
assert_eq!(probe.detail, missing.detail);
assert_eq!(probe.hint, missing.hint);
}
#[test]
fn ssh_reachability_fails_fast_without_host() {
let check = grade_ssh_reachability(None, 22, Duration::from_millis(50));
assert!(!check.passed);
assert_eq!(check.name, "ssh_reachability");
assert!(check.hint.unwrap().contains("[deploy] host"));
let blank = grade_ssh_reachability(Some(" "), 22, Duration::from_millis(50));
assert!(!blank.passed);
}
#[test]
fn ssh_reachability_fails_when_all_addresses_unreachable() {
let check = grade_ssh_reachability(Some("127.0.0.1"), 1, Duration::from_millis(100));
assert!(!check.passed, "closed port should fail: {}", check.detail);
assert_eq!(check.name, "ssh_reachability");
assert!(check.detail.contains("cannot reach"));
}
#[test]
fn signing_secret_grader_never_echoes_value() {
let present = grade_signing_secret(Some("super-secret-value"), &[], false);
assert!(present.passed);
assert!(!present.detail.contains("super-secret-value"));
let missing = grade_signing_secret(None, &[], false);
assert!(!missing.passed);
assert!(!grade_signing_secret(Some(" "), &[], false).passed);
}
#[test]
fn signing_secret_grader_non_production_accepts_weak_secret() {
assert!(grade_signing_secret(Some("changeme"), &[], false).passed);
assert!(grade_signing_secret(Some("short"), &[], false).passed);
assert!(grade_signing_secret(Some("changeme"), &["also-weak".to_owned()], false).passed);
}
#[test]
fn signing_secret_grader_production_rejects_weak_secret() {
let demo = grade_signing_secret(Some("changeme"), &[], true);
assert!(!demo.passed, "demo value must fail in production");
assert!(demo.hint.is_some());
assert!(!demo.detail.contains("changeme"));
let short = grade_signing_secret(Some("too-short"), &[], true);
assert!(!short.passed, "too-short secret must fail in production");
assert!(!short.detail.contains("too-short"));
}
#[test]
fn signing_secret_grader_production_accepts_strong_secret() {
let strong = "a".repeat(64);
let check = grade_signing_secret(Some(&strong), &[], true);
assert!(
check.passed,
"strong production secret should pass: {}",
check.detail
);
assert!(!check.detail.contains(&strong));
}
#[test]
fn signing_secret_grader_production_missing_fails() {
let check = grade_signing_secret(None, &[], true);
assert!(!check.passed);
assert!(check.hint.is_some());
}
#[test]
fn signing_secret_grader_production_rejects_weak_previous_secret() {
let strong = "a".repeat(64);
let check = grade_signing_secret(Some(&strong), &["changeme".to_owned()], true);
assert!(
!check.passed,
"weak previous_secrets entry must fail in production"
);
assert!(check.hint.is_some());
assert!(!check.detail.contains("changeme"));
assert!(
check.detail.contains("previous"),
"detail should name the rotation secret: {}",
check.detail
);
}
#[test]
fn signing_secret_grader_production_accepts_strong_previous_secrets() {
let strong = "a".repeat(64);
let previous = vec!["b".repeat(64), "c".repeat(64)];
let check = grade_signing_secret(Some(&strong), &previous, true);
assert!(
check.passed,
"strong current + strong previous should pass: {}",
check.detail
);
}
#[test]
fn signing_secret_grader_non_production_ignores_weak_previous_secret() {
let check = grade_signing_secret(Some("dev-secret"), &["changeme".to_owned()], false);
assert!(check.passed);
}
#[test]
fn database_url_grader_never_echoes_value() {
let absent = Path::new("autumn-deploy-no-such-migrations-dir-echo-guard");
let present = grade_database_url(Some("postgres://user:pw@host/db"), absent, true, false);
assert!(present.passed);
assert!(!present.detail.contains("pw"));
assert!(!grade_database_url(None, absent, true, false).passed);
}
#[test]
fn database_url_grader_passes_for_db_free_app() {
let tmp = tempfile::TempDir::new().unwrap();
let absent_migrations = tmp.path().join("migrations");
assert!(!absent_migrations.exists());
let check = grade_database_url(None, &absent_migrations, false, false);
assert!(
check.passed,
"DB-free app should pass the DB-URL preflight: {}",
check.detail
);
}
#[test]
fn database_url_grader_fails_for_db_backed_runtime_without_url() {
let tmp = tempfile::TempDir::new().unwrap();
let absent_migrations = tmp.path().join("migrations");
assert!(!absent_migrations.exists());
let check = grade_database_url(None, &absent_migrations, false, true);
assert!(
!check.passed,
"DB-backed runtime feature without a URL must fail preflight"
);
assert!(
check.hint.is_some(),
"the failure should carry an actionable remediation hint"
);
}
#[test]
fn database_url_grader_passes_for_db_backed_runtime_with_url() {
let tmp = tempfile::TempDir::new().unwrap();
let absent_migrations = tmp.path().join("migrations");
assert!(!absent_migrations.exists());
let check = grade_database_url(
Some("postgres://user:pw@host/db"),
&absent_migrations,
false,
true,
);
assert!(
check.passed,
"DB-backed runtime feature with a URL should pass: {}",
check.detail
);
assert!(!check.detail.contains("pw"));
}
#[test]
fn database_url_grader_fails_for_db_backed_app_without_url() {
let tmp = tempfile::TempDir::new().unwrap();
let migrations = tmp.path().join("migrations");
std::fs::create_dir(&migrations).unwrap();
let check = grade_database_url(None, &migrations, false, false);
assert!(
!check.passed,
"DB-backed app without a URL should fail preflight"
);
assert!(
check.hint.is_some(),
"the failure should carry an actionable remediation hint"
);
}
#[test]
fn shard_only_config_resolves_a_usable_db_url() {
use autumn_web::config::{DatabaseConfig, ShardConfig};
let db = DatabaseConfig {
shards: vec![ShardConfig {
name: "shard0".to_owned(),
primary_url: "postgres://user:pw@shard0/app".to_owned(),
..Default::default()
}],
..Default::default()
};
assert!(db.effective_primary_url().is_none());
let url = resolve_writable_db_url(&db);
assert_eq!(url, Some("postgres://user:pw@shard0/app"));
let tmp = tempfile::TempDir::new().unwrap();
let migrations = tmp.path().join("migrations");
std::fs::create_dir(&migrations).unwrap();
let check = grade_database_url(url, &migrations, true, false);
assert!(
check.passed,
"shard-only app with migrations should pass the DB-URL preflight: {}",
check.detail
);
assert!(!check.detail.contains("pw"));
}
#[test]
fn replica_only_config_resolves_no_writable_url_and_fails_with_migrations() {
use autumn_web::config::DatabaseConfig;
let db = DatabaseConfig {
replica_url: Some("postgres://user:pw@replica/app".to_owned()),
..Default::default()
};
assert!(db.effective_primary_url().is_none());
let url = resolve_writable_db_url(&db);
assert_eq!(
url, None,
"replica_url alone is not a writable migration target"
);
let tmp = tempfile::TempDir::new().unwrap();
let migrations = tmp.path().join("migrations");
std::fs::create_dir(&migrations).unwrap();
let check = grade_database_url(url, &migrations, true, false);
assert!(
!check.passed,
"replica-only app with migrations must fail the DB-URL preflight"
);
assert!(
check.hint.is_some(),
"the failure should carry an actionable remediation hint"
);
assert!(!check.detail.contains("pw"));
}
#[test]
fn migrate_check_passes_when_no_migrations_dir() {
let dir = std::env::temp_dir().join("autumn-deploy-no-such-migrations-dir-xyz");
let check = grade_migrate_check(&dir);
assert!(
check.passed,
"absent migrations dir should pass: {}",
check.detail
);
}
#[test]
fn is_production_profile_matches_runtime_rule() {
assert!(is_production_profile(Some("prod")));
assert!(is_production_profile(Some("production")));
assert!(!is_production_profile(Some("dev")));
assert!(!is_production_profile(Some("staging")));
assert!(!is_production_profile(None));
}
#[test]
fn requires_database_pool_detects_postgres_backends() {
use autumn_web::config::{AutumnConfig, SchedulerBackend};
let mut config = AutumnConfig::default();
assert!(!requires_database_pool(&config));
config.jobs.backend = "postgres".to_owned();
assert!(requires_database_pool(&config));
let mut config = AutumnConfig::default();
config.scheduler.backend = SchedulerBackend::Postgres;
assert!(requires_database_pool(&config));
let mut config = AutumnConfig::default();
config.jobs.backend = "redis".to_owned();
assert!(!requires_database_pool(&config));
}
#[test]
fn rollback_dispatch_does_not_load_media_config() {
let src = include_str!("deploy.rs");
let needle = ["load_media_host_config", "(&resolved)?"].concat();
let call_sites = src.matches(needle.as_str()).count();
assert_eq!(
call_sites, 2,
"expected exactly two fallible load-media call sites (Plan + Up), found {call_sites}",
);
let run_body = src
.split("pub fn run(action: DeployAction)")
.nth(1)
.expect("run() present");
let rollback_arm = run_body
.split("DeployAction::Rollback =>")
.nth(1)
.expect("Rollback arm present");
let rollback_line = rollback_arm
.split("DeployAction::Up")
.next()
.expect("Up arm follows Rollback");
assert!(
!rollback_line.contains("load_media_host_config"),
"the Rollback arm must not load the media config",
);
let check_arm = run_body
.split("DeployAction::Check =>")
.nth(1)
.and_then(|s| s.split("DeployAction::Rollback").next())
.expect("Check arm present");
assert!(
!check_arm.contains("load_media_host_config"),
"the Check arm must not load the media config",
);
}
#[test]
fn media_provisioning_is_deferred_past_app_cutover_in_up() {
let src = include_str!("deploy.rs");
let up_body = src
.split("fn run_up(")
.nth(1)
.and_then(|s| s.split("\nfn run_rollback(").next())
.expect("run_up body present");
let preflight_at = up_body
.find("check_media_host_preflight(input.media_cfg")
.expect("preflight call present on the up path");
let commit_at = up_body
.find("for (index, host_plan) in plan.hosts.iter().enumerate()")
.expect("per-host execution loop present on the up path");
let provision_at = up_body
.find("provision_media_host(input.media_cfg, executor)?;")
.expect("deferred provisioning call present on the up path");
assert!(
preflight_at < commit_at,
"media preflight must precede the per-host execution loop",
);
assert!(
commit_at < provision_at,
"MediaMTX provisioning must run only AFTER every host's cutover commits — so a \
rolled-back or halted rollout (which returns from the loop's halt path) never \
reaches it",
);
let preflight_fn = src
.split("fn check_media_host_preflight(")
.nth(1)
.and_then(|s| s.split("\nfn provision_media_host(").next())
.expect("check_media_host_preflight present");
assert!(
preflight_fn.contains("collect_media_doctor_checks"),
"preflight must run the doctor checks",
);
assert!(
!preflight_fn.contains("ensure_installed_ops"),
"preflight must NOT drive the mutating controller ops (write/restart)",
);
let after_commit = &up_body[commit_at..];
assert!(
after_commit.contains("provision_media_host(input.media_cfg, executor)?;"),
"provisioning must sit on the post-commit path (unreachable when a host's \
deploy errors out and the rollout halts)",
);
}
#[test]
fn media_binary_install_runs_before_cutover_in_up() {
let src = include_str!("deploy.rs");
let up_body = src
.split("fn run_up(")
.nth(1)
.and_then(|s| s.split("\nfn run_rollback(").next())
.expect("run_up body present");
let loop_at = up_body
.find("for (index, host_plan) in plan.hosts.iter().enumerate()")
.expect("per-host execution loop present");
let install_at = up_body
.find("ops.splice(0..0, media_binary_install_ops(input.media_cfg));")
.expect("the media binary install is spliced into the host's ops");
let proxy_install_at = up_body
.find("ops.splice(0..0, install);")
.expect("the proxy install splice is present");
let provision_at = up_body
.find("provision_media_host(input.media_cfg, executor)?;")
.expect("deferred provisioning call present");
assert!(loop_at < install_at, "install runs inside the host's turn");
assert!(
install_at < proxy_install_at,
"spliced before the proxy install, so it runs right after it"
);
assert!(install_at < provision_at);
}
const READ_ONLY_PROBES: [&str; 3] =
["proxy-compat-probe", "detect-current", "probe-release-dir"];
const REDEPLOY_RUN_LABELS: [&str; 13] = [
"proxy-snapshot-unit",
"proxy-install",
"proxy-restart-if-changed",
"prepare-dirs",
"daemon-reload",
"start-candidate",
"migrate",
"readiness-gate",
"proxy-flip",
"commit-markers",
"record-proxy-options",
"drain-old",
"prune",
];
const FLEET_ROLLBACK_RUN_LABELS: [&str; 7] = [
"resolve-previous",
"daemon-reload",
"restart-previous",
"proxy-flip",
"commit-markers",
"readiness-gate",
"drain-rolled-back-slot",
];
const FLEET_RELEASE_ID: &str = "20260714T120000Z";
const FLEET_PUBLIC_PORT: u16 = 3000;
fn fleet_halt_of(err: &DeployError) -> &FleetHalt {
match err {
DeployError::FleetHalted(halt) => halt,
other => panic!("expected a fleet halt, got: {other:?}"),
}
}
fn fleet_of(hosts: &[&str]) -> ResolvedFleet {
ResolvedFleet::resolve(
&DeployConfig {
hosts: hosts.iter().map(|h| (*h).to_owned()).collect(),
..DeployConfig::default()
},
"myapp",
)
.expect("a well-formed fleet resolves")
}
fn redeploy_probe() -> String {
"redeploy:blue\t3001\n---autumn-kamal-proxy-list---\n".to_owned()
}
fn first_deploy_probe() -> String {
"first\n---autumn-kamal-proxy-list---\n".to_owned()
}
fn unreadable_proxy_options_probe() -> String {
"redeploy:blue\t3001\n\
---autumn-kamal-proxy-list---\n\
---autumn-kamal-proxy-unit---\n\
--http-port 3000\n\
---autumn-kamal-proxy-options---\n\
garbage\n"
.to_owned()
}
fn compatible_deploy_help() -> &'static str {
"Usage:\n kamal-proxy deploy SERVICE [flags]\n\nFlags:\n \
--target host:port\n --health-check-path string\n --host strings\n \
--tls\n --deploy-timeout duration\n --drain-timeout duration\n \
--force\n\
---autumn-kamal-proxy-remove-help---\
Usage:\n kamal-proxy remove SERVICE [flags]\n"
}
fn fleet_manifests() -> Vec<exec::ManifestUpload> {
vec![exec::ManifestUpload {
local: PathBuf::from("/local/autumn.toml"),
remote_basename: "autumn.toml".to_owned(),
}]
}
fn script_redeploy(
recorder: fleet::test_support::FleetRecorder,
host: &str,
) -> fleet::test_support::FleetRecorder {
recorder
.script(host, "proxy-compat-probe", compatible_deploy_help())
.script(host, "detect-current", redeploy_probe())
.script(host, "probe-release-dir", "absent")
}
fn script_first_deploy(
recorder: fleet::test_support::FleetRecorder,
host: &str,
) -> fleet::test_support::FleetRecorder {
recorder
.script(host, "proxy-compat-probe", compatible_deploy_help())
.script(host, "detect-current", first_deploy_probe())
.script(host, "probe-release-dir", "absent")
}
fn script_bare_first_deploy(
recorder: fleet::test_support::FleetRecorder,
host: &str,
) -> fleet::test_support::FleetRecorder {
recorder
.script(host, "proxy-compat-probe", "")
.script(host, "detect-current", first_deploy_probe())
.script(host, "probe-release-dir", "absent")
}
#[test]
fn a_bare_host_installs_the_proxy_as_the_first_op_of_its_own_turn() {
let fleet = fleet_of(&["web-a", "web-b"]);
let mut recorder = fleet::test_support::FleetRecorder::new();
recorder = script_bare_first_deploy(recorder, "web-a");
recorder = script_bare_first_deploy(recorder, "web-b");
let fixture = FleetFixture::new();
run_up_with(&fixture.input(&fleet), |cfg| Ok(recorder.executor(cfg)))
.expect("a bare fleet deploys after preparing each host");
for host in ["web-a", "web-b"] {
let labels = recorder.run_labels_for(host);
let install = labels
.iter()
.position(|l| *l == "install-proxy")
.unwrap_or_else(|| panic!("host {host} must be prepared: {labels:?}"));
let first_mutating = recorder
.first_mutating(host, &READ_ONLY_PROBES)
.expect("a deployed host mutates something");
assert_eq!(
labels[install..install + 2].to_vec(),
vec!["install-proxy", "proxy-install"],
"the install must precede the proxy unit it supervises on {host}: {labels:?}"
);
assert!(
recorder
.positions_of("install-proxy")
.contains(&first_mutating),
"host preparation must be the FIRST thing that touches {host} \
(its first mutating call was at tape position {first_mutating})"
);
}
let first_install = *recorder
.positions_of("install-proxy")
.first()
.expect("the fleet prepares a host");
for label in READ_ONLY_PROBES {
let last_probe = *recorder
.positions_of(label)
.last()
.unwrap_or_else(|| panic!("every host runs {label}"));
assert!(
last_probe < first_install,
"the probe phase must finish before anything is installed: {label} at \
{last_probe}, first install at {first_install}"
);
}
}
#[test]
fn a_bare_redeploy_host_is_prepared_before_its_marker_repair() {
let fleet = fleet_of(&["web-a"]);
let drifted = "redeploy:blue\t3001\n\
---autumn-kamal-proxy-list---\n\
Service Host Target State TLS\n\
myapp example.com 127.0.0.1:3002 running no\n\
---autumn-kamal-proxy-unit---\n--http-port 3000\n"
.to_owned();
let recorder = fleet::test_support::FleetRecorder::new()
.script("web-a", "proxy-compat-probe", "")
.script("web-a", "detect-current", drifted)
.script("web-a", "probe-release-dir", "absent");
let fixture = FleetFixture::new();
run_up_with(&fixture.input(&fleet), |cfg| Ok(recorder.executor(cfg)))
.expect("a bare, drifted host deploys after being prepared");
let labels = recorder.run_labels_for("web-a");
let install = labels
.iter()
.position(|l| *l == "install-proxy")
.expect("a bare host is prepared");
let repair = labels
.iter()
.position(|l| *l == exec::LIVE_SLOT_REPAIR_LABEL)
.expect("a drifted marker is repaired");
assert!(
install < repair,
"host preparation must precede the marker repair: {labels:?}"
);
let first_mutating = recorder
.first_mutating("web-a", &READ_ONLY_PROBES)
.expect("a deployed host mutates something");
assert!(
recorder
.positions_of("install-proxy")
.contains(&first_mutating),
"host preparation is still the FIRST thing that touches the host"
);
}
#[test]
fn declining_host_preparation_refuses_a_bare_host_without_touching_it() {
let mut cfg = fleet_cfg(&["web-a"]);
cfg.install_proxy = false;
let fleet = ResolvedFleet::resolve(&cfg, "myapp").expect("a one-host fleet resolves");
let recorder = fleet::test_support::FleetRecorder::new()
.script("web-a", "proxy-compat-probe", "")
.script("web-a", "detect-current", first_deploy_probe())
.script("web-a", "probe-release-dir", "absent");
let fixture = FleetFixture::new();
let err = run_up_with(&fixture.input(&fleet), |cfg| Ok(recorder.executor(cfg)))
.expect_err("a bare host with host prep declined must refuse");
let message = err.to_string();
assert!(
message.contains("install_proxy"),
"the refusal must name the setting in force: {message}"
);
assert_eq!(
recorder.run_labels_for("web-a"),
vec!["proxy-compat-probe"],
"a declined host must be left untouched"
);
}
#[test]
fn a_failed_host_preparation_halts_the_rollout_and_touches_no_later_host() {
let fleet = fleet_of(&["web-a", "web-b"]);
let mut recorder = fleet::test_support::FleetRecorder::new();
recorder = script_bare_first_deploy(recorder, "web-a");
recorder = script_bare_first_deploy(recorder, "web-b");
recorder = recorder.fail("web-a", "install-proxy");
let fixture = FleetFixture::new();
let err = run_up_with(&fixture.input(&fleet), |cfg| Ok(recorder.executor(cfg)))
.expect_err("a failed host preparation must fail the rollout");
assert!(
err.to_string().contains("install-proxy"),
"the failing step is named: {err}"
);
let web_a = recorder.run_labels_for("web-a");
let (probes, rest) = web_a.split_at(READ_ONLY_PROBES.len());
assert_eq!(probes, READ_ONLY_PROBES);
assert_eq!(rest[0], "install-proxy");
assert!(
rest[1..].iter().all(|l| l.starts_with("teardown-")),
"only compensation may follow a failed host preparation: {web_a:?}"
);
assert_eq!(
recorder.run_labels_for("web-b"),
READ_ONLY_PROBES.to_vec(),
"a halt must leave every later host untouched"
);
}
#[test]
fn a_host_that_already_has_the_proxy_is_never_prepared() {
let fleet = fleet_of(&["web-a", "web-b"]);
let mut recorder = fleet::test_support::FleetRecorder::new();
recorder = script_first_deploy(recorder, "web-a");
recorder = script_redeploy(recorder, "web-b");
let fixture = FleetFixture::new();
run_up_with(&fixture.input(&fleet), |cfg| Ok(recorder.executor(cfg)))
.expect("an equipped fleet deploys");
assert!(
recorder.positions_of("install-proxy").is_empty(),
"an equipped host must never be re-prepared"
);
}
const PREVIOUS_RELEASE_DIR: &str = "/srv/autumn/myapp/releases/20260101T000000Z";
fn script_compensation(
recorder: fleet::test_support::FleetRecorder,
host: &str,
target_dir: &'static str,
) -> fleet::test_support::FleetRecorder {
recorder
.script(
host,
"resolve-previous",
format!("prev:{PREVIOUS_RELEASE_DIR}\tblue\t3001"),
)
.script(host, "probe-rollback-target", target_dir)
}
const COMPENSATION_RUN_LABELS: [&str; 8] = [
"resolve-previous",
"probe-rollback-target",
"daemon-reload",
"restart-previous",
"proxy-flip",
"commit-markers",
"readiness-gate",
"drain-rolled-back-slot",
];
fn redeploy_then_compensated(migrated: bool) -> Vec<&'static str> {
READ_ONLY_PROBES
.iter()
.copied()
.chain(
REDEPLOY_RUN_LABELS
.iter()
.copied()
.filter(|label| migrated || *label != "migrate"),
)
.chain(COMPENSATION_RUN_LABELS)
.collect()
}
struct FleetFixture {
env_file: exec::Secret,
manifests: Vec<exec::ManifestUpload>,
proxy: proxy::KamalProxyController,
media_cfg: media::MediaMtxHostConfig,
binary: PathBuf,
}
impl FleetFixture {
fn new() -> Self {
Self {
env_file: exec::Secret::new("AUTUMN_SECURITY__SIGNING_SECRET=topsecret\n"),
manifests: fleet_manifests(),
proxy: proxy::KamalProxyController::new(60),
media_cfg: media::MediaMtxHostConfig::default(),
binary: PathBuf::from("/local/target/release/myapp"),
}
}
fn input<'a>(
&'a self,
fleet: &'a ResolvedFleet,
) -> FleetUpInput<'a, proxy::KamalProxyController> {
FleetUpInput {
profile: "prod",
app_name: "myapp",
fleet,
proxy: &self.proxy,
checks: &[],
env_file: &self.env_file,
binary: &self.binary,
manifests: &self.manifests,
release_id: FLEET_RELEASE_ID,
public_port: FLEET_PUBLIC_PORT,
media_cfg: &self.media_cfg,
ffmpeg_bin: "ffmpeg",
configured_host_count: fleet.hosts.len(),
db: FleetDatabaseFacts::default(),
jobs_backend: "postgres",
scheduler_in_process: false,
auto_rollback: true,
bake: None,
metrics_path: "/actuator/metrics",
bake_host: Some("app.example.com"),
}
}
fn input_frozen<'a>(
&'a self,
fleet: &'a ResolvedFleet,
) -> FleetUpInput<'a, proxy::KamalProxyController> {
FleetUpInput {
profile: "prod",
app_name: "myapp",
auto_rollback: false,
..self.input(fleet)
}
}
fn todays_calls(
&self,
cfg: &ResolvedDeployConfig,
mode: fleet::HostMode,
migrate: exec::MigrateStep,
) -> Vec<exec::test_support::RecordedCall> {
let slots = match mode {
fleet::HostMode::First => exec::SlotPlan::first(FLEET_PUBLIC_PORT),
fleet::HostMode::Redeploy => {
exec::SlotPlan::redeploy(FLEET_PUBLIC_PORT, exec::SLOT_BLUE)
}
};
let release_dir = format!("{}/{FLEET_RELEASE_ID}", cfg.releases_dir());
let unit = render_app_unit(
cfg,
&release_dir,
slots.candidate_port,
slots.candidate_slot,
);
let ops = match mode {
fleet::HostMode::First => exec::first_deploy_ops(
cfg,
&self.proxy,
&unit,
self.env_file.clone(),
&self.binary,
&self.manifests,
FLEET_RELEASE_ID,
&slots,
migrate,
),
fleet::HostMode::Redeploy => exec::cutover_ops(
cfg,
&self.proxy,
&unit,
self.env_file.clone(),
&self.binary,
&self.manifests,
FLEET_RELEASE_ID,
&slots,
&self.proxy.proxy_service_options(),
migrate,
),
};
let recorder = exec::test_support::RecordingExecutor::new();
exec::run_ops(&ops, &recorder).expect("the recording fake never fails");
recorder.calls()
}
}
#[test]
fn fleet_halted_carries_only_host_names_and_static_labels() {
let err = fleet_halted(
&fleet::FleetPlan {
hosts: vec![
fleet::HostPlan {
host: "web-a".to_owned(),
mode: fleet::HostMode::Redeploy,
migrate: exec::MigrateStep::Run,
},
fleet::HostPlan {
host: "web-b".to_owned(),
mode: fleet::HostMode::Redeploy,
migrate: exec::MigrateStep::Skip,
},
],
},
&[
fleet::HostOutcome::Degraded { label: "prune" },
fleet::HostOutcome::AmbiguousMarkers,
],
&[("web-a".to_owned(), "prune")],
"web-b".to_owned(),
"migrate",
);
let rendered = format!("{err}\n{err:?}");
assert!(
rendered.contains("web-b") && rendered.contains("migrate"),
"the halt must name the failing host and step: {rendered}"
);
assert!(
rendered.contains("prune") && rendered.contains("web-a"),
"the degraded log must name the host and the housekeeping label: {rendered}"
);
assert!(
rendered.contains(fleet::MANUAL_AMBIGUOUS_MARKERS),
"an ambiguous-marker host must be reported as needing a human: {rendered}"
);
assert!(
rendered.contains("NOT rolled back"),
"the halt must state that the schema did not come back with the binaries: \
{rendered}"
);
for secret in [
"postgres://",
"topsecret",
"AUTUMN_SECURITY__SIGNING_SECRET",
"systemd-run",
] {
assert!(
!rendered.contains(secret),
"FleetHalted must never carry `{secret}`: {rendered}"
);
}
}
#[derive(Default)]
struct CapturingChannel {
received: Arc<std::sync::Mutex<Vec<Alert>>>,
}
impl AlertChannel for CapturingChannel {
fn name(&self) -> &'static str {
"capturing"
}
fn deliver<'a>(&'a self, alert: &'a Alert) -> autumn_web::alerts::AlertDeliveryFuture<'a> {
let received = Arc::clone(&self.received);
let cloned = alert.clone();
Box::pin(async move {
received.lock().expect("lock").push(cloned);
Ok(())
})
}
}
fn sample_halt() -> FleetHalt {
FleetHalt {
failed_host: "web-b".to_owned(),
failed_step: "migrate",
rolled_back: vec!["web-a".to_owned()],
torn_down: vec![],
still_on_new: vec![],
degraded: vec![("web-a".to_owned(), "prune")],
route_removal_failed: vec![],
manual: vec![("web-c".to_owned(), fleet::MANUAL_AMBIGUOUS_MARKERS)],
}
}
#[test]
fn alert_channels_for_respects_the_enabled_master_switch() {
let alerts = autumn_web::alerts::AlertConfig {
enabled: false,
pagerduty_routing_key: Some("R0123".to_owned()),
..Default::default()
};
let http = autumn_web::config::HttpConfig::default();
assert!(
alert_channels_for(&alerts, &http).is_empty(),
"a disabled [alerts] config must build no channel"
);
}
#[test]
fn alert_channels_for_builds_channels_when_active() {
let alerts = autumn_web::alerts::AlertConfig {
enabled: true,
pagerduty_routing_key: Some("R0123".to_owned()),
..Default::default()
};
let http = autumn_web::config::HttpConfig::default();
assert_eq!(
alert_channels_for(&alerts, &http).len(),
1,
"an enabled [alerts] config with a destination must build a channel"
);
}
#[test]
fn deploy_alert_channels_for_profile_is_empty_without_a_configured_destination() {
assert!(deploy_alert_channels_for_profile("prod").is_empty());
}
#[test]
fn declared_alert_config_in_reads_the_target_profile_section() {
let dir = tempfile::TempDir::new().expect("temp project dir");
std::fs::write(
dir.path().join("autumn.toml"),
"[deploy]\nhost = \"deploy.example.test\"\n\n\
[profile.prod.alerts]\npagerduty_routing_key = \"R0123\"\n",
)
.expect("write autumn.toml");
let (alerts, _http) = declared_alert_config_in(&[dir.path().to_path_buf()], "prod")
.expect("the merged [alerts] subtree deserializes");
assert_eq!(alerts.pagerduty_routing_key.as_deref(), Some("R0123"));
}
#[test]
fn declared_alert_config_in_ignores_an_unrelated_bad_section() {
let dir = tempfile::TempDir::new().expect("temp project dir");
std::fs::write(
dir.path().join("autumn.toml"),
"[deploy]\nhost = \"deploy.example.test\"\n\n\
[scheduler]\nbackend = 12345\n\n\
[alerts]\npagerduty_routing_key = \"R0123\"\n",
)
.expect("write autumn.toml");
let (alerts, _http) = declared_alert_config_in(&[dir.path().to_path_buf()], "prod")
.expect("a bad [scheduler] value must not break the [alerts] read");
assert_eq!(alerts.pagerduty_routing_key.as_deref(), Some("R0123"));
}
#[test]
fn declared_alert_config_in_fails_closed_on_an_unreadable_profile_override() {
let dir = tempfile::TempDir::new().expect("temp project dir");
std::fs::write(
dir.path().join("autumn.toml"),
"[deploy]\nhost = \"deploy.example.test\"\n\n\
[alerts]\npagerduty_routing_key = \"base-key\"\n",
)
.expect("write autumn.toml");
std::fs::write(dir.path().join("autumn-prod.toml"), "not valid toml")
.expect("write autumn-prod.toml");
assert!(
declared_alert_config_in(&[dir.path().to_path_buf()], "prod").is_none(),
"a malformed profile override file must fail closed, not fall back to base config"
);
}
#[test]
fn declared_alert_config_in_fails_closed_on_an_unknown_alerts_key() {
let dir = tempfile::TempDir::new().expect("temp project dir");
std::fs::write(
dir.path().join("autumn.toml"),
"[deploy]\nhost = \"deploy.example.test\"\n\n\
[alerts]\nenabeld = false\npagerduty_routing_key = \"R0123\"\n",
)
.expect("write autumn.toml");
assert!(
declared_alert_config_in(&[dir.path().to_path_buf()], "prod").is_none(),
"a typo'd [alerts] key must fail closed, not silently default enabled back to true"
);
}
#[test]
fn apply_alert_env_overrides_reads_autumn_alerts_env_vars() {
use autumn_web::config::MockEnv;
let alerts = autumn_web::alerts::AlertConfig::default();
let http = autumn_web::config::HttpConfig::default();
let env = MockEnv::new().with("AUTUMN_ALERTS__PAGERDUTY_ROUTING_KEY", "R9999");
let (alerts, _http) = apply_alert_env_overrides(alerts, http, &env);
assert_eq!(alerts.pagerduty_routing_key.as_deref(), Some("R9999"));
}
#[test]
fn apply_alert_env_overrides_can_disable_a_toml_configured_destination() {
use autumn_web::config::MockEnv;
let alerts = autumn_web::alerts::AlertConfig {
pagerduty_routing_key: Some("R0123".to_owned()),
..Default::default()
};
let http = autumn_web::config::HttpConfig::default();
let env = MockEnv::new().with("AUTUMN_ALERTS__ENABLED", "false");
let (alerts, _http) = apply_alert_env_overrides(alerts, http, &env);
assert!(!alerts.is_active());
}
#[test]
fn build_fleet_halted_alert_is_scheduled_task_failure() {
let alert = build_fleet_halted_alert(&sample_halt(), "myapp", "production");
assert_eq!(alert.condition, AlertCondition::ScheduledTaskFailure);
assert_eq!(alert.title, "Fleet rollout halted");
assert_eq!(
alert.dedup_key,
"scheduled_task_failure:deploy-fleet-halted:myapp:production"
);
assert!(
alert.summary.contains("web-b") && alert.summary.contains("migrate"),
"the alert must name the failing host and step: {}",
alert.summary
);
assert_eq!(
alert.where_to_look, "autumn deploy status",
"review finding on #2267: must not point at the /actuator/tasks default"
);
}
#[test]
fn build_fleet_halted_alert_is_scoped_to_profile_for_dedup() {
let staging = build_fleet_halted_alert(&sample_halt(), "myapp", "staging");
let production = build_fleet_halted_alert(&sample_halt(), "myapp", "production");
assert_ne!(staging.dedup_key, production.dedup_key);
}
#[test]
fn build_fleet_halted_alert_is_scoped_to_app_name_for_dedup() {
let app_one = build_fleet_halted_alert(&sample_halt(), "app-one", "production");
let app_two = build_fleet_halted_alert(&sample_halt(), "app-two", "production");
assert_ne!(app_one.dedup_key, app_two.dedup_key);
}
#[test]
fn build_fleet_halted_alert_carries_only_host_names_and_static_labels() {
let alert = build_fleet_halted_alert(&sample_halt(), "myapp", "production");
let rendered = format!("{alert:?}");
for secret in [
"postgres://",
"topsecret",
"AUTUMN_SECURITY__SIGNING_SECRET",
"systemd-run",
] {
assert!(
!rendered.contains(secret),
"a fleet-halted alert must never carry `{secret}`: {rendered}"
);
}
}
#[test]
fn deliver_fleet_halted_alert_reaches_configured_channel() {
let capture = Arc::new(std::sync::Mutex::new(Vec::new()));
let channel = Arc::new(CapturingChannel {
received: Arc::clone(&capture),
});
let channels: Vec<Arc<dyn AlertChannel>> = vec![channel];
deliver_alert(
&channels,
&build_fleet_halted_alert(&sample_halt(), "myapp", "production"),
);
let delivered = capture.lock().expect("lock").clone();
assert_eq!(delivered.len(), 1, "exactly one alert must be delivered");
assert_eq!(delivered[0].condition, AlertCondition::ScheduledTaskFailure);
}
#[test]
fn deliver_alert_is_noop_without_channels() {
deliver_alert(
&[],
&build_fleet_halted_alert(&sample_halt(), "myapp", "production"),
);
}
#[test]
fn build_drift_alert_is_scheduled_task_failure_scoped_to_profile() {
let report = fleet::DriftReport {
releases: vec![
("web-a".to_owned(), fleet::ReleaseId::Known("r1".to_owned())),
("web-b".to_owned(), fleet::ReleaseId::Known("r2".to_owned())),
],
version_drift: true,
state_drift: vec![("web-b".to_owned(), fleet::DRIFT_HOST_NOT_DEPLOYED)],
};
let alert = build_drift_alert(&report, "myapp", "production");
assert_eq!(alert.condition, AlertCondition::ScheduledTaskFailure);
assert_eq!(alert.title, "Fleet drift detected");
assert_eq!(
alert.dedup_key,
"scheduled_task_failure:deploy-drift:myapp:production"
);
assert!(
alert.summary.contains("release"),
"version drift must be named in the summary: {}",
alert.summary
);
assert_eq!(
alert.where_to_look, "autumn deploy status",
"review finding on #2267: must not point at the /actuator/tasks default"
);
}
#[test]
fn build_drift_alert_is_scoped_to_profile_for_dedup() {
let report = fleet::DriftReport {
releases: vec![],
version_drift: false,
state_drift: vec![("web-a".to_owned(), fleet::DRIFT_LIVE_SLOT_MARKER)],
};
let staging = build_drift_alert(&report, "myapp", "staging");
let production = build_drift_alert(&report, "myapp", "production");
assert_ne!(staging.dedup_key, production.dedup_key);
}
#[test]
fn build_drift_alert_is_scoped_to_app_name_for_dedup() {
let report = fleet::DriftReport {
releases: vec![],
version_drift: false,
state_drift: vec![("web-a".to_owned(), fleet::DRIFT_LIVE_SLOT_MARKER)],
};
let app_one = build_drift_alert(&report, "app-one", "production");
let app_two = build_drift_alert(&report, "app-two", "production");
assert_ne!(app_one.dedup_key, app_two.dedup_key);
}
#[test]
fn deliver_drift_alert_reaches_configured_channel() {
let capture = Arc::new(std::sync::Mutex::new(Vec::new()));
let channel = Arc::new(CapturingChannel {
received: Arc::clone(&capture),
});
let channels: Vec<Arc<dyn AlertChannel>> = vec![channel];
let report = fleet::DriftReport {
releases: vec![],
version_drift: false,
state_drift: vec![("web-a".to_owned(), fleet::DRIFT_LIVE_SLOT_MARKER)],
};
deliver_alert(&channels, &build_drift_alert(&report, "myapp", "staging"));
let delivered = capture.lock().expect("lock").clone();
assert_eq!(delivered.len(), 1);
assert_eq!(
delivered[0].dedup_key,
"scheduled_task_failure:deploy-drift:myapp:staging"
);
}
#[test]
fn fleet_of_one_matches_todays_single_host_sequence() {
let fleet = fleet_of(&["203.0.113.10"]);
let recorder = script_redeploy(fleet::test_support::FleetRecorder::new(), "203.0.113.10");
let fixture = FleetFixture::new();
run_up_with(&fixture.input(&fleet), |cfg| Ok(recorder.executor(cfg)))
.expect("a scripted one-host fleet deploys cleanly");
let expected_labels: Vec<&'static str> = READ_ONLY_PROBES
.iter()
.copied()
.chain(REDEPLOY_RUN_LABELS)
.collect();
assert_eq!(
recorder.run_labels_for("203.0.113.10"),
expected_labels,
"a one-host fleet must run today's exact zero-downtime sequence"
);
let observed = recorder.calls_for("203.0.113.10");
let (probes, mutating) = observed.split_at(READ_ONLY_PROBES.len());
assert_eq!(
probes
.iter()
.filter_map(exec::test_support::RecordedCall::run_label)
.collect::<Vec<_>>(),
READ_ONLY_PROBES.to_vec(),
"the rollout prologue must be read-only probes only"
);
assert_eq!(
mutating,
fixture
.todays_calls(
&fleet.hosts[0],
fleet::HostMode::Redeploy,
exec::MigrateStep::Run
)
.as_slice(),
"a one-host fleet's ops must be byte-identical to today's single-host ops"
);
}
#[test]
fn fleet_of_one_first_deploy_matches_todays_single_host_sequence() {
let fleet = fleet_of(&["203.0.113.10"]);
let recorder = fleet::test_support::FleetRecorder::new()
.script(
"203.0.113.10",
"proxy-compat-probe",
compatible_deploy_help(),
)
.script("203.0.113.10", "detect-current", first_deploy_probe())
.script("203.0.113.10", "probe-release-dir", "absent");
let fixture = FleetFixture::new();
run_up_with(&fixture.input(&fleet), |cfg| Ok(recorder.executor(cfg)))
.expect("a scripted one-host first deploy runs cleanly");
let observed = recorder.calls_for("203.0.113.10");
let (_, mutating) = observed.split_at(READ_ONLY_PROBES.len());
assert_eq!(
mutating,
fixture
.todays_calls(
&fleet.hosts[0],
fleet::HostMode::First,
exec::MigrateStep::Run
)
.as_slice(),
"a one-host first deploy must be byte-identical to today's single-host first deploy"
);
assert!(
recorder.run_labels_for("203.0.113.10").contains(&"migrate"),
"the first-deploy path migrates before the release takes traffic (#1607)"
);
}
#[test]
fn rolling_order_replaces_hosts_strictly_in_sequence() {
let hosts = ["web-a", "web-b", "web-c"];
let fleet = fleet_of(&hosts);
let mut recorder = fleet::test_support::FleetRecorder::new();
for host in hosts {
recorder = script_redeploy(recorder, host);
}
let fixture = FleetFixture::new();
run_up_with(&fixture.input(&fleet), |cfg| Ok(recorder.executor(cfg)))
.expect("a scripted three-host fleet rolls out cleanly");
for (index, host) in hosts.iter().enumerate() {
let expected: Vec<&'static str> = READ_ONLY_PROBES
.iter()
.copied()
.chain(
REDEPLOY_RUN_LABELS
.iter()
.copied()
.filter(|l| index == 0 || *l != "migrate"),
)
.collect();
assert_eq!(
recorder.run_labels_for(host),
expected,
"{host} must run the full per-host sequence ({}the migration)",
if index == 0 { "with " } else { "without " }
);
}
for pair in hosts.windows(2) {
let (previous, next) = (pair[0], pair[1]);
let flip = recorder
.index_of(previous, "proxy-flip")
.unwrap_or_else(|| panic!("{previous} must cut over"));
let first_mutation = recorder
.first_mutating(next, &READ_ONLY_PROBES)
.unwrap_or_else(|| panic!("{next} must be deployed"));
assert!(
flip < first_mutation,
"{next} must not be touched until {previous} has cut over: \
{previous}'s proxy-flip at {flip}, {next}'s first mutation at {first_mutation}"
);
}
}
#[test]
fn unreadable_marker_on_any_host_aborts_before_any_mutation() {
let fleet = fleet_of(&["web-a", "web-b", "web-c"]);
let mut recorder = fleet::test_support::FleetRecorder::new();
for host in ["web-a", "web-b"] {
recorder = script_redeploy(recorder, host);
}
recorder = recorder
.script("web-c", "proxy-compat-probe", compatible_deploy_help())
.script("web-c", "detect-current", unreadable_proxy_options_probe())
.script("web-c", "probe-release-dir", "absent");
let fixture = FleetFixture::new();
let err = run_up_with(&fixture.input(&fleet), |cfg| Ok(recorder.executor(cfg)))
.expect_err("an unprovable proxy-options marker must refuse the rollout");
let message = err.to_string();
assert!(
message.contains("web-c"),
"the refusal must name the offending HOST, got: {message}"
);
assert!(
message.contains("proxy-options") && message.contains("#2074"),
"the refusal must keep the single-host #2074 message, got: {message}"
);
let mutating = recorder.mutating(&READ_ONLY_PROBES);
assert!(
mutating.is_empty(),
"a refused rollout must mutate NOTHING anywhere in the fleet, got: {mutating:?}"
);
}
#[test]
fn existing_release_dir_on_any_host_refuses_the_rollout() {
let fleet = fleet_of(&["web-a", "web-b", "web-c"]);
let mut recorder = fleet::test_support::FleetRecorder::new();
for host in ["web-a", "web-c"] {
recorder = script_redeploy(recorder, host);
}
recorder = recorder
.script("web-b", "proxy-compat-probe", compatible_deploy_help())
.script("web-b", "detect-current", redeploy_probe())
.script("web-b", "probe-release-dir", "present");
let fixture = FleetFixture::new();
let err = run_up_with(&fixture.input(&fleet), |cfg| Ok(recorder.executor(cfg)))
.expect_err("an existing release dir must refuse the rollout");
let message = err.to_string();
assert!(
message.contains("web-b") && message.contains(FLEET_RELEASE_ID),
"the refusal must name the host and the colliding release, got: {message}"
);
let mutating = recorder.mutating(&READ_ONLY_PROBES);
assert!(
mutating.is_empty(),
"a refused rollout must mutate NOTHING anywhere in the fleet, got: {mutating:?}"
);
}
#[test]
fn a_failed_host_halts_the_rollout_and_leaves_later_hosts_untouched() {
let hosts = ["web-a", "web-b", "web-c"];
let fleet = fleet_of(&hosts);
let mut recorder = fleet::test_support::FleetRecorder::new();
for host in hosts {
recorder = script_redeploy(recorder, host);
}
recorder = recorder.fail("web-b", "readiness-gate");
let fixture = FleetFixture::new();
let err = run_up_with(&fixture.input_frozen(&fleet), |cfg| {
Ok(recorder.executor(cfg))
})
.expect_err("a mid-rollout failure must halt the rollout");
let halt = fleet_halt_of(&err);
assert!(
halt.degraded.is_empty() && halt.manual.is_empty(),
"a clean pre-boundary halt needs no degraded/manual rows: {:?} / {:?}",
halt.degraded,
halt.manual
);
assert_eq!(
halt.failed_host, "web-b",
"the halt must name the failing host"
);
assert_eq!(
halt.failed_step, "readiness-gate",
"the halt must name the failing step"
);
assert_eq!(
halt.rolled_back,
vec!["web-b".to_owned()],
"a pre-boundary failure rolls the failing host's candidate back"
);
assert!(
halt.torn_down.is_empty(),
"a redeploy host is rolled back, not torn down: {:?}",
halt.torn_down
);
assert_eq!(
halt.still_on_new,
vec!["web-a".to_owned()],
"the already-cut-over hosts must be named so they can be compensated"
);
assert!(
recorder.run_labels_for("web-a").contains(&"prune"),
"web-a must have completed its cutover"
);
let web_b = recorder.run_labels_for("web-b");
assert!(
web_b.contains(&"readiness-gate")
&& !web_b.contains(&"proxy-flip")
&& web_b.contains(&"teardown-candidate-unit")
&& web_b.contains(&"teardown-candidate-dir"),
"web-b must fail at the gate, never flip, and tear its candidate down: {web_b:?}"
);
let web_c: Vec<_> = recorder
.mutating(&READ_ONLY_PROBES)
.into_iter()
.filter(|(host, _)| host == "web-c")
.collect();
assert!(
web_c.is_empty(),
"the rollout must halt: web-c must be mutated in no way, got: {web_c:?}"
);
}
#[test]
fn a_pre_boundary_failure_rolls_the_already_cut_over_hosts_back() {
let hosts = ["web-a", "web-b", "web-c"];
let fleet = fleet_of(&hosts);
let mut recorder = fleet::test_support::FleetRecorder::new();
for host in hosts {
recorder = script_redeploy(recorder, host);
}
recorder =
script_compensation(recorder, "web-a", "present").fail("web-b", "readiness-gate");
let fixture = FleetFixture::new();
let err = run_up_with(&fixture.input(&fleet), |cfg| Ok(recorder.executor(cfg)))
.expect_err("a mid-rollout failure must halt the rollout");
assert_eq!(
recorder.run_labels_for("web-a"),
redeploy_then_compensated(true),
"web-a must be rolled back with the on-demand rollback primitives"
);
assert!(
recorder
.mutating(&READ_ONLY_PROBES)
.iter()
.all(|(host, _)| host != "web-c"),
"web-c was never reached, so compensation must not touch it either"
);
let halt = fleet_halt_of(&err);
assert_eq!(
halt.rolled_back,
vec!["web-a".to_owned(), "web-b".to_owned()],
"both the compensated host and the self-torn-down host are back on their \
previous release"
);
assert!(
halt.still_on_new.is_empty() && halt.torn_down.is_empty() && halt.manual.is_empty(),
"the fleet converged: nothing left forward, nothing manual — got {:?} / {:?} / {:?}",
halt.still_on_new,
halt.torn_down,
halt.manual
);
}
#[test]
fn a_post_boundary_functional_failure_compensates_newest_first_including_itself() {
let hosts = ["web-a", "web-b", "web-c"];
let fleet = fleet_of(&hosts);
let mut recorder = fleet::test_support::FleetRecorder::new();
for host in hosts {
recorder = script_redeploy(recorder, host);
}
recorder = script_compensation(recorder, "web-a", "present");
recorder =
script_compensation(recorder, "web-b", "present").transport_fail("web-b", "drain-old");
let fixture = FleetFixture::new();
let err = run_up_with(&fixture.input(&fleet), |cfg| Ok(recorder.executor(cfg)))
.expect_err("a post-boundary failure must halt the rollout");
let web_b = recorder
.index_of("web-b", "restart-previous")
.expect("web-b must be compensated — it is live on the new release");
let web_a = recorder
.index_of("web-a", "restart-previous")
.expect("web-a must be compensated");
assert!(
web_b < web_a,
"compensation must run in REVERSE rollout order: web-b's rollback at \
{web_b}, web-a's at {web_a}"
);
assert!(
recorder.index_of("web-c", "restart-previous").is_none(),
"web-c never cut over, so it must not be rolled back"
);
let halt = fleet_halt_of(&err);
assert_eq!(halt.failed_host, "web-b");
assert_eq!(
halt.failed_step, "ssh-transport",
"a dropped transport attributes to the transport, never to a fabricated \
step label"
);
assert_eq!(
halt.rolled_back,
vec!["web-a".to_owned(), "web-b".to_owned()],
"the post-boundary host is compensated TOO — it is on the new release"
);
assert!(
halt.still_on_new.is_empty(),
"the fleet must converge on one version: {:?}",
halt.still_on_new
);
}
#[test]
fn a_completed_first_deploy_is_compensated_by_teardown_not_rollback() {
let fleet = fleet_of(&["web-a", "web-b"]);
let mut recorder = fleet::test_support::FleetRecorder::new();
recorder = script_first_deploy(recorder, "web-a");
recorder = script_redeploy(recorder, "web-b").fail("web-b", "readiness-gate");
let fixture = FleetFixture::new();
let err = run_up_with(&fixture.input(&fleet), |cfg| Ok(recorder.executor(cfg)))
.expect_err("a mid-rollout failure must halt the rollout");
let web_a = recorder.run_labels_for("web-a");
for teardown in [
"teardown-candidate-unit",
"teardown-candidate-dir",
"teardown-current-symlink",
"teardown-slot-markers",
"proxy-deregister",
] {
assert!(
web_a.contains(&teardown),
"a compensated first deploy must run `{teardown}`: {web_a:?}"
);
}
assert!(
!web_a.contains(&"resolve-previous") && !web_a.contains(&"restart-previous"),
"a first-deploy host has no previous release, so it must never be sent \
through the rollback primitives: {web_a:?}"
);
let halt = fleet_halt_of(&err);
assert_eq!(
halt.torn_down,
vec!["web-a".to_owned()],
"the compensated first deploy is reported as torn down, not rolled back"
);
assert_eq!(
halt.rolled_back,
vec!["web-b".to_owned()],
"the failing redeploy host's own candidate teardown still stands"
);
assert!(
halt.still_on_new.is_empty(),
"nothing may be left on the new release"
);
}
#[test]
fn a_completed_first_deploy_compensation_removes_the_proxy_route() {
let fleet = fleet_of(&["web-a", "web-b"]);
let mut recorder = fleet::test_support::FleetRecorder::new();
recorder = script_first_deploy(recorder, "web-a");
recorder = script_redeploy(recorder, "web-b").fail("web-b", "readiness-gate");
let fixture = FleetFixture::new();
run_up_with(&fixture.input(&fleet), |cfg| Ok(recorder.executor(cfg)))
.expect_err("a mid-rollout failure must halt the rollout");
let calls = recorder.calls_for("web-a");
let labels: Vec<&str> = calls
.iter()
.filter_map(|call| match call {
exec::test_support::RecordedCall::Run { label, .. } => Some(*label),
exec::test_support::RecordedCall::Upload { .. } => None,
})
.collect();
let deregister_at = labels
.iter()
.position(|l| *l == "proxy-deregister")
.expect("the compensated first deploy must deregister the proxy route");
let last_deploy_at = labels
.iter()
.position(|l| *l == "teardown-last-deploy")
.expect("the teardown must still record its result");
assert!(
last_deploy_at < deregister_at,
"the app teardown, marker write included, must fully finish BEFORE the \
separate route-removal step starts: {labels:?}"
);
let shell = calls
.iter()
.find_map(|call| match call {
exec::test_support::RecordedCall::Run { label, shell }
if *label == "proxy-deregister" =>
{
Some(shell.as_str())
}
_ => None,
})
.expect("proxy-deregister ran");
assert_eq!(shell, "env -u XDG_RUNTIME_DIR kamal-proxy remove 'myapp'");
}
#[test]
fn a_failed_deregister_reports_its_own_outcome_not_a_generic_compensation_failure() {
let fleet = fleet_of(&["web-a", "web-b"]);
let mut recorder = fleet::test_support::FleetRecorder::new();
recorder = script_first_deploy(recorder, "web-a").fail("web-a", "proxy-deregister");
recorder = script_redeploy(recorder, "web-b").fail("web-b", "readiness-gate");
let fixture = FleetFixture::new();
let err = run_up_with(&fixture.input(&fleet), |cfg| Ok(recorder.executor(cfg)))
.expect_err("a mid-rollout failure must halt the rollout");
let halt = fleet_halt_of(&err);
assert_eq!(
halt.torn_down,
vec!["web-a".to_owned()],
"the app is gone, so this host is torn down, not still forward"
);
assert!(
!halt.still_on_new.contains(&"web-a".to_owned()),
"a failed deregister must never be told to `rollback` a host with \
nothing installed: {:?}",
halt.still_on_new
);
assert!(
!halt.manual.iter().any(|(host, _)| host == "web-a"),
"this is not a declined-automatically case: {:?}",
halt.manual
);
assert_eq!(
halt.route_removal_failed,
vec![("web-a".to_owned(), "proxy-deregister")],
"the route-removal failure must be preserved in a dedicated field: {:?}",
halt.route_removal_failed
);
let calls = recorder.calls_for("web-a");
let last_deploy_writes: Vec<&str> = calls
.iter()
.filter_map(|call| match call {
exec::test_support::RecordedCall::Run { label, shell }
if *label == "teardown-last-deploy" =>
{
Some(shell.as_str())
}
_ => None,
})
.collect();
let last = *last_deploy_writes
.last()
.expect("the marker must still be recorded despite the deregister failure");
assert!(
last.contains("'torn down'") && !last.contains("'deployed'"),
"a fully torn-down host must not report a successful deploy: {last}"
);
}
#[test]
fn a_dropped_transport_on_deregister_still_reads_as_torn_down() {
let fleet = fleet_of(&["web-a", "web-b"]);
let mut recorder = fleet::test_support::FleetRecorder::new();
recorder =
script_first_deploy(recorder, "web-a").transport_fail("web-a", "proxy-deregister");
recorder = script_redeploy(recorder, "web-b").fail("web-b", "readiness-gate");
let fixture = FleetFixture::new();
let err = run_up_with(&fixture.input(&fleet), |cfg| Ok(recorder.executor(cfg)))
.expect_err("a mid-rollout failure must halt the rollout");
let halt = fleet_halt_of(&err);
assert_eq!(
halt.torn_down,
vec!["web-a".to_owned()],
"a dropped transport on the route step alone must not read as still forward"
);
assert!(
!halt.still_on_new.contains(&"web-a".to_owned()),
"must never suggest `rollback` a host with nothing installed: {:?}",
halt.still_on_new
);
}
#[test]
fn a_compensated_first_deploy_stops_reporting_a_successful_last_deploy() {
let fleet = fleet_of(&["web-a", "web-b"]);
let mut recorder = fleet::test_support::FleetRecorder::new();
recorder = script_first_deploy(recorder, "web-a");
recorder = script_redeploy(recorder, "web-b").fail("web-b", "readiness-gate");
let fixture = FleetFixture::new();
run_up_with(&fixture.input(&fleet), |cfg| Ok(recorder.executor(cfg)))
.expect_err("a mid-rollout failure must halt the rollout");
let calls = recorder.calls_for("web-a");
let last_deploy_writes: Vec<(&'static str, &str)> = calls
.iter()
.filter_map(|call| match call {
exec::test_support::RecordedCall::Run { label, shell }
if shell.contains("shared/last-deploy") =>
{
Some((*label, shell.as_str()))
}
_ => None,
})
.collect();
let (last_label, last_shell) = *last_deploy_writes
.last()
.expect("the compensated host writes the last-deploy marker at least once");
assert_eq!(
last_label, "teardown-last-deploy",
"the TEARDOWN must have the last word on this host's last-deploy marker, \
got: {last_deploy_writes:?}"
);
assert!(
last_shell.contains("'torn down'") && !last_shell.contains("'deployed'"),
"a torn-down host must not report a successful deploy: {last_shell}"
);
assert!(
last_deploy_writes
.iter()
.any(|(label, shell)| *label == "record-proxy-options"
&& shell.contains("'deployed'")),
"the first deploy must have recorded `deployed` first: {last_deploy_writes:?}"
);
}
#[test]
fn a_failed_compensation_does_not_stop_the_earlier_hosts() {
let hosts = ["web-a", "web-b", "web-c"];
let fleet = fleet_of(&hosts);
let mut recorder = fleet::test_support::FleetRecorder::new();
for host in hosts {
recorder = script_redeploy(recorder, host);
}
recorder = script_compensation(recorder, "web-a", "present");
recorder =
script_compensation(recorder, "web-b", "present").fail("web-b", "restart-previous");
recorder =
script_compensation(recorder, "web-c", "present").transport_fail("web-c", "drain-old");
let fixture = FleetFixture::new();
let err = run_up_with(&fixture.input(&fleet), |cfg| Ok(recorder.executor(cfg)))
.expect_err("a post-boundary failure must halt the rollout");
let web_b = recorder.run_labels_for("web-b");
assert!(
web_b.contains(&"restart-previous")
&& !web_b
.iter()
.skip_while(|l| **l != "restart-previous")
.any(|l| *l == "commit-markers")
&& web_b.contains(&"teardown-rollback-slot"),
"web-b's rollback must fail before its flip and disable the slot it \
restarted: {web_b:?}"
);
let web_a = recorder
.index_of("web-a", "restart-previous")
.expect("web-a must still be compensated after web-b's rollback failed");
let web_b_attempt = recorder
.index_of("web-b", "restart-previous")
.expect("web-b's rollback was attempted");
assert!(
web_b_attempt < web_a,
"the failed rollback must not stop the earlier host: web-b at \
{web_b_attempt}, web-a at {web_a}"
);
let halt = fleet_halt_of(&err);
assert_eq!(
halt.rolled_back,
vec!["web-a".to_owned(), "web-c".to_owned()],
"every host whose compensation SUCCEEDED is reported as rolled back"
);
assert_eq!(
halt.still_on_new,
vec!["web-b".to_owned()],
"a host whose compensation failed is still on the new release and must be \
named as such — reporting the attempt instead of the result is how an \
operator believes a mixed fleet converged"
);
}
#[test]
fn a_missing_rollback_target_dir_declines_the_rollback() {
let fleet = fleet_of(&["web-a", "web-b"]);
let mut recorder = fleet::test_support::FleetRecorder::new();
for host in ["web-a", "web-b"] {
recorder = script_redeploy(recorder, host);
}
recorder = script_compensation(recorder, "web-a", "absent").fail("web-b", "readiness-gate");
let fixture = FleetFixture::new();
let err = run_up_with(&fixture.input(&fleet), |cfg| Ok(recorder.executor(cfg)))
.expect_err("a mid-rollout failure must halt the rollout");
let web_a = recorder.run_labels_for("web-a");
assert!(
web_a.contains(&"probe-rollback-target"),
"the precheck must actually run: {web_a:?}"
);
assert!(
!web_a.contains(&"restart-previous") && !web_a.contains(&"drain-rolled-back-slot"),
"a declined rollback must run NO rollback ops: {web_a:?}"
);
assert_eq!(
web_a.iter().filter(|label| **label == "proxy-flip").count(),
1,
"the declined host must keep exactly its cutover flip — zero further \
flips: {web_a:?}"
);
let halt = fleet_halt_of(&err);
assert_eq!(
halt.manual,
vec![("web-a".to_owned(), fleet::MANUAL_ROLLBACK_TARGET_MISSING)],
"the declined host must be reported with its reason"
);
assert_eq!(
halt.still_on_new,
vec!["web-a".to_owned()],
"a host that was NOT rolled back is still on the new release"
);
assert_eq!(
halt.rolled_back,
vec!["web-b".to_owned()],
"the failing host's own teardown still stands"
);
}
#[test]
fn a_commit_markers_failure_is_never_auto_rolled_back() {
let fleet = fleet_of(&["web-a", "web-b"]);
let mut recorder = fleet::test_support::FleetRecorder::new();
for host in ["web-a", "web-b"] {
recorder = script_redeploy(recorder, host);
}
recorder =
script_compensation(recorder, "web-a", "present").fail("web-b", "commit-markers");
let fixture = FleetFixture::new();
let err = run_up_with(&fixture.input(&fleet), |cfg| Ok(recorder.executor(cfg)))
.expect_err("an ambiguous-marker failure must halt the rollout");
let web_b = recorder.run_labels_for("web-b");
for rollback_op in [
"resolve-previous",
"probe-rollback-target",
"restart-previous",
"drain-rolled-back-slot",
] {
assert!(
!web_b.contains(&rollback_op),
"an ambiguous-marker host must see ZERO rollback-flavoured ops, got \
`{rollback_op}`: {web_b:?}"
);
}
assert_eq!(
recorder.run_labels_for("web-a"),
redeploy_then_compensated(true),
"the earlier host is still compensated"
);
let halt = fleet_halt_of(&err);
assert_eq!(halt.failed_step, "commit-markers");
assert_eq!(
halt.manual,
vec![("web-b".to_owned(), fleet::MANUAL_AMBIGUOUS_MARKERS)],
"the ambiguous host must be reported as needing a human"
);
assert_eq!(
halt.still_on_new,
vec!["web-b".to_owned()],
"it was not rolled back, so it is still on the new release"
);
assert_eq!(halt.rolled_back, vec!["web-a".to_owned()]);
}
#[test]
fn a_transport_failure_at_commit_markers_is_never_auto_rolled_back() {
let fleet = fleet_of(&["web-a", "web-b"]);
let mut recorder = fleet::test_support::FleetRecorder::new();
for host in ["web-a", "web-b"] {
recorder = script_redeploy(recorder, host);
}
recorder = script_compensation(recorder, "web-a", "present")
.transport_fail("web-b", "commit-markers");
let fixture = FleetFixture::new();
let err = run_up_with(&fixture.input(&fleet), |cfg| Ok(recorder.executor(cfg)))
.expect_err("an ambiguous-marker failure must halt the rollout");
let web_b = recorder.run_labels_for("web-b");
for rollback_op in [
"resolve-previous",
"probe-rollback-target",
"restart-previous",
"drain-rolled-back-slot",
] {
assert!(
!web_b.contains(&rollback_op),
"a host whose `commit-markers` failed must see ZERO rollback-flavoured \
ops whatever the error shape, got `{rollback_op}`: {web_b:?}"
);
}
assert_eq!(
recorder.run_labels_for("web-a"),
redeploy_then_compensated(true),
"the earlier host is still compensated"
);
let halt = fleet_halt_of(&err);
assert_eq!(
halt.manual,
vec![("web-b".to_owned(), fleet::MANUAL_AMBIGUOUS_MARKERS)],
"the host with an unprovable marker triple must be reported as needing a human"
);
assert_eq!(
halt.still_on_new,
vec!["web-b".to_owned()],
"it was not rolled back, so it is still on the new release"
);
assert_eq!(halt.rolled_back, vec!["web-a".to_owned()]);
}
#[test]
fn a_housekeeping_failure_degrades_the_host_and_continues_the_rollout() {
let hosts = ["web-a", "web-b", "web-c"];
let fleet = fleet_of(&hosts);
let mut recorder = fleet::test_support::FleetRecorder::new();
for host in hosts {
recorder = script_redeploy(recorder, host);
}
recorder = recorder.fail("web-b", "prune");
let fixture = FleetFixture::new();
run_up_with(&fixture.input(&fleet), |cfg| Ok(recorder.executor(cfg)))
.expect("a post-cutover housekeeping failure must not fail the rollout");
assert_eq!(
recorder.run_labels_for("web-c"),
READ_ONLY_PROBES
.iter()
.copied()
.chain(
REDEPLOY_RUN_LABELS
.iter()
.copied()
.filter(|label| *label != "migrate")
)
.collect::<Vec<_>>(),
"the rollout must CONTINUE past a degraded host"
);
let web_b = recorder.run_labels_for("web-b");
assert!(
web_b.contains(&"prune")
&& !web_b.contains(&"teardown-candidate-unit")
&& !web_b.contains(&"restart-previous"),
"a degraded host is live and healthy: nothing may be torn down or rolled \
back on it: {web_b:?}"
);
assert!(
recorder.positions_of("resolve-previous").is_empty(),
"a rollout that only degraded must compensate NOTHING anywhere"
);
}
enum DrainRetry {
Stdout(&'static str),
CommandFails,
TransportFails,
}
fn drain_old_failure_run(
retry: &DrainRetry,
frozen: bool,
) -> (fleet::test_support::FleetRecorder, Result<(), DeployError>) {
let hosts = ["web-a", "web-b", "web-c"];
let fleet = fleet_of(&hosts);
let mut recorder = fleet::test_support::FleetRecorder::new();
for host in hosts {
recorder = script_compensation(script_redeploy(recorder, host), host, "present");
}
recorder = recorder.fail("web-b", "drain-old");
recorder = match retry {
DrainRetry::Stdout(stdout) => recorder.script("web-b", "drain-old-retry", *stdout),
DrainRetry::CommandFails => recorder.fail("web-b", "drain-old-retry"),
DrainRetry::TransportFails => recorder.transport_fail("web-b", "drain-old-retry"),
};
let fixture = FleetFixture::new();
let input = if frozen {
fixture.input_frozen(&fleet)
} else {
fixture.input(&fleet)
};
let result = run_up_with(&input, |cfg| Ok(recorder.executor(cfg)));
(recorder, result)
}
fn assert_one_retry_of_the_drained_unit(recorder: &fleet::test_support::FleetRecorder) {
let web_b = recorder.run_labels_for("web-b");
assert_eq!(
web_b.iter().filter(|l| **l == "drain-old-retry").count(),
1,
"retry one time only: {web_b:?}"
);
assert!(
web_b
.windows(2)
.any(|w| w == ["drain-old", "drain-old-retry"]),
"the retry runs right after the failed drain: {web_b:?}"
);
let shell_of = |label: &str| {
recorder
.calls_for("web-b")
.into_iter()
.find_map(|call| match call {
exec::test_support::RecordedCall::Run { label: l, shell } if l == label => {
Some(shell)
}
_ => None,
})
.expect("the op ran")
};
let drained = shell_of("drain-old");
let unit = drained.rsplit(' ').next().expect("drain-old names a unit");
assert!(
shell_of("drain-old-retry").contains(&format!("unit='{unit}'")),
"the retry must target the old slot `{unit}`, never the new one"
);
}
#[test]
fn a_drain_old_failure_whose_old_slot_may_still_run_halts_the_rollout() {
for retry in [
DrainRetry::Stdout("not-stopped"),
DrainRetry::Stdout("bash: -c: line 0"),
DrainRetry::CommandFails,
DrainRetry::TransportFails,
] {
let (recorder, result) = drain_old_failure_run(&retry, false);
let err = result.expect_err("an old slot that may still run must halt the rollout");
assert_one_retry_of_the_drained_unit(&recorder);
assert!(
!recorder.run_labels_for("web-b").contains(&"prune"),
"nothing more of the deploy runs on the halted host"
);
assert_eq!(
recorder.run_labels_for("web-c"),
READ_ONLY_PROBES.to_vec(),
"the host after the halt must not be touched"
);
let halt = fleet_halt_of(&err);
assert_eq!(halt.failed_host, "web-b");
assert_eq!(halt.failed_step, "drain-old");
assert_eq!(
halt.rolled_back,
vec!["web-a".to_owned(), "web-b".to_owned()],
"both cut-over hosts are compensated, so each runs ONE slot again"
);
assert!(halt.still_on_new.is_empty(), "{:?}", halt.still_on_new);
assert!(
halt.degraded.is_empty(),
"a live old slot is not housekeeping debris: {:?}",
halt.degraded
);
assert_eq!(
halt.risk(),
None,
"compensation left one slot running, so the risk is gone"
);
}
}
#[test]
fn a_frozen_drain_old_halt_leaves_the_hosts_and_names_the_risk() {
let (recorder, result) = drain_old_failure_run(&DrainRetry::Stdout("not-stopped"), true);
let err = result.expect_err("a frozen halt is still a failure");
assert!(
recorder.positions_of("resolve-previous").is_empty(),
"a frozen halt compensates nothing"
);
let halt = fleet_halt_of(&err);
assert_eq!(halt.failed_step, "drain-old");
assert!(halt.rolled_back.is_empty(), "{:?}", halt.rolled_back);
assert_eq!(
halt.still_on_new,
vec!["web-a".to_owned(), "web-b".to_owned()]
);
assert_eq!(halt.risk(), Some(fleet::OLD_SLOT_MAY_RUN_NOTE));
let alert = build_fleet_halted_alert(halt, "myapp", "prod");
assert_eq!(
alert.details.get("risk").map(String::as_str),
Some(fleet::OLD_SLOT_MAY_RUN_NOTE),
"the alert must name the risk, not only the step"
);
}
#[test]
fn a_drain_old_failure_whose_old_slot_is_proven_stopped_still_degrades() {
let (recorder, result) = drain_old_failure_run(&DrainRetry::Stdout("stopped"), false);
result.expect("a proven-stopped old slot must not fail the rollout");
assert_one_retry_of_the_drained_unit(&recorder);
assert!(
!recorder
.run_labels_for("web-b")
.contains(&"restart-previous"),
"the host stays on the new release"
);
assert_eq!(
recorder.run_labels_for("web-c"),
READ_ONLY_PROBES
.iter()
.copied()
.chain(
REDEPLOY_RUN_LABELS
.iter()
.copied()
.filter(|label| *label != "migrate")
)
.collect::<Vec<_>>(),
"the rollout must continue past the degraded host"
);
assert!(
recorder.positions_of("resolve-previous").is_empty(),
"nothing is compensated"
);
}
#[test]
fn a_single_host_drain_old_failure_keeps_todays_error_and_does_not_retry() {
let fleet = fleet_of(&["203.0.113.10"]);
let recorder = script_redeploy(fleet::test_support::FleetRecorder::new(), "203.0.113.10")
.fail("203.0.113.10", "drain-old");
let fixture = FleetFixture::new();
let err = run_up_with(&fixture.input(&fleet), |cfg| Ok(recorder.executor(cfg)))
.expect_err("a single-host drain failure must fail the deploy");
assert!(matches!(err, DeployError::Exec(_)), "{err:?}");
assert_eq!(
recorder.run_labels_for("203.0.113.10").last(),
Some(&"drain-old"),
"N = 1 stops at the failed drain and does not retry"
);
}
#[test]
fn only_a_drain_old_halt_carries_a_risk() {
assert_eq!(sample_halt().risk(), None, "a `migrate` halt adds no risk");
let alert = build_fleet_halted_alert(&sample_halt(), "myapp", "production");
assert!(
!alert.details.contains_key("risk"),
"other halts keep today's alert details"
);
}
#[test]
fn compensation_never_touches_the_schema() {
let hosts = ["web-a", "web-b"];
let fleet = fleet_of(&hosts);
let mut recorder = fleet::test_support::FleetRecorder::new();
for host in hosts {
recorder = script_redeploy(recorder, host);
}
recorder = script_compensation(recorder, "web-a", "present");
recorder =
script_compensation(recorder, "web-b", "present").transport_fail("web-b", "drain-old");
let fixture = FleetFixture::new();
let err = run_up_with(&fixture.input(&fleet), |cfg| Ok(recorder.executor(cfg)))
.expect_err("a post-boundary failure must halt the rollout");
let migrations = recorder.positions_of("migrate");
assert_eq!(
migrations.len(),
1,
"the fleet migrates once and compensation adds none: {migrations:?}"
);
let compensation_started = recorder
.positions_of("resolve-previous")
.first()
.copied()
.expect("compensation ran");
assert!(
migrations[0] < compensation_started,
"the only migration must predate compensation: migrate at {}, compensation \
from {compensation_started}",
migrations[0]
);
assert!(
err.to_string().contains("NOT rolled back"),
"the halt must state that the schema did not come back with the binaries: {err}"
);
}
fn single_host_exec_message(err: &DeployError) -> &str {
match err {
DeployError::Exec(message) => message,
other => panic!("N = 1 must never produce fleet vocabulary, got: {other:?}"),
}
}
fn pre_cutover_error(mode: fleet::HostMode, failed_step: &'static str) -> String {
let source = Box::new(exec::DeployExecError::CommandFailed {
label: failed_step,
message: "scripted failure".to_owned(),
});
match mode {
fleet::HostMode::Redeploy => exec::DeployExecError::CandidateRolledBack {
failed_step,
source,
},
fleet::HostMode::First => exec::DeployExecError::FirstDeployTornDown {
failed_step,
source,
},
}
.to_string()
}
fn failed_single_host_deploy(
mode: fleet::HostMode,
failed_step: &'static str,
) -> (String, Vec<&'static str>) {
let host = "203.0.113.10";
let recorder = fleet::test_support::FleetRecorder::new();
let recorder = match mode {
fleet::HostMode::Redeploy => script_redeploy(recorder, host),
fleet::HostMode::First => script_first_deploy(recorder, host),
}
.fail(host, failed_step);
let fleet = fleet_of(&[host]);
let fixture = FleetFixture::new();
let err = run_up_with(&fixture.input(&fleet), |cfg| Ok(recorder.executor(cfg)))
.expect_err("the scripted failure must fail the deploy");
(
single_host_exec_message(&err).to_owned(),
recorder.run_labels_for(host),
)
}
#[test]
fn a_single_host_redeploy_rolled_back_after_migrate_names_the_schema() {
let (message, labels) =
failed_single_host_deploy(fleet::HostMode::Redeploy, "readiness-gate");
assert_eq!(
message,
format!(
"{}\n\u{26A0}\u{FE0F} {}",
pre_cutover_error(fleet::HostMode::Redeploy, "readiness-gate"),
fleet::SINGLE_HOST_SCHEMA_AHEAD_NOTE
),
"the executor error stays first; the schema note follows it"
);
assert!(
labels.contains(&"migrate") && !labels.contains(&"proxy-flip"),
"the deploy must migrate, then stop at the gate: {labels:?}"
);
}
#[test]
fn a_single_host_marker_repair_failure_keeps_todays_error() {
let host = "203.0.113.10";
let drifted = "redeploy:blue\t3001\n\
---autumn-kamal-proxy-list---\n\
Service Host Target State TLS\n\
myapp example.com 127.0.0.1:3002 running no\n\
---autumn-kamal-proxy-unit---\n--http-port 3000\n"
.to_owned();
let recorder = fleet::test_support::FleetRecorder::new()
.script(host, "proxy-compat-probe", compatible_deploy_help())
.script(host, "detect-current", drifted)
.script(host, "probe-release-dir", "absent")
.fail_on_occurrence(host, exec::LIVE_SLOT_REPAIR_LABEL, 1);
let fleet = fleet_of(&[host]);
let fixture = FleetFixture::new();
let err = run_up_with(&fixture.input(&fleet), |cfg| Ok(recorder.executor(cfg)))
.expect_err("the failed repair must fail the deploy");
let labels = recorder.run_labels_for(host);
assert!(
!labels.contains(&"migrate"),
"the repair runs before `migrate`: {labels:?}"
);
assert_eq!(
single_host_exec_message(&err),
pre_cutover_error(fleet::HostMode::Redeploy, exec::LIVE_SLOT_REPAIR_LABEL),
);
}
#[test]
fn a_single_host_first_deploy_torn_down_after_migrate_names_the_schema() {
let (message, labels) = failed_single_host_deploy(fleet::HostMode::First, "readiness-gate");
assert_eq!(
message,
format!(
"{}\n\u{26A0}\u{FE0F} {}",
pre_cutover_error(fleet::HostMode::First, "readiness-gate"),
fleet::SINGLE_HOST_FIRST_DEPLOY_SCHEMA_NOTE
),
);
assert!(
labels.contains(&"migrate"),
"the first deploy must migrate before the gate: {labels:?}"
);
}
#[test]
fn a_single_host_failure_before_migrate_keeps_todays_error() {
for mode in [fleet::HostMode::Redeploy, fleet::HostMode::First] {
let (message, labels) = failed_single_host_deploy(mode, "daemon-reload");
assert_eq!(
message,
pre_cutover_error(mode, "daemon-reload"),
"{mode:?}: a failure before `migrate` keeps today's error verbatim"
);
assert!(
!labels.contains(&"migrate"),
"{mode:?}: the scripted failure must land before `migrate`: {labels:?}"
);
}
}
#[test]
fn a_single_host_failure_keeps_todays_error_and_compensates_nothing() {
let fleet = fleet_of(&["203.0.113.10"]);
let recorder = script_redeploy(fleet::test_support::FleetRecorder::new(), "203.0.113.10")
.fail("203.0.113.10", "prune");
let fixture = FleetFixture::new();
let err = run_up_with(&fixture.input(&fleet), |cfg| Ok(recorder.executor(cfg)))
.expect_err("a single-host post-boundary failure must still fail the deploy");
match &err {
DeployError::Exec(message) => assert_eq!(
*message,
exec::DeployExecError::CommandFailed {
label: "prune",
message: "scripted failure".to_owned(),
}
.to_string(),
"the single-host path must surface the raw executor error verbatim"
),
other => panic!("N = 1 must never produce fleet vocabulary, got: {other:?}"),
}
assert_eq!(
recorder.run_labels_for("203.0.113.10"),
READ_ONLY_PROBES
.iter()
.copied()
.chain(REDEPLOY_RUN_LABELS)
.collect::<Vec<_>>(),
"the one host runs today's exact sequence and nothing more — no \
compensation, no extra probe"
);
}
fn tls_fleet_of(hosts: &[&str]) -> ResolvedFleet {
ResolvedFleet::resolve(
&DeployConfig {
hosts: hosts.iter().map(|h| (*h).to_owned()).collect(),
tls: autumn_web::config::DeployTlsConfig {
enabled: true,
host: Some("shop.example.com".to_owned()),
},
..DeployConfig::default()
},
"myapp",
)
.expect("a well-formed TLS fleet resolves")
}
#[test]
fn the_fleet_refusals_name_their_config_key_and_spare_a_single_host() {
let sqlite = refuse_sqlite_fleet(3, true).expect_err("a sqlite fleet must be refused");
assert!(
sqlite.contains("sqlite") && sqlite.contains("#1621"),
"the sqlite refusal must name the backend and the issue: {sqlite}"
);
assert!(
sqlite.contains("[database]"),
"the sqlite refusal must name the config section to change: {sqlite}"
);
let media = refuse_media_fleet(3, true).expect_err("a media fleet must be refused");
assert!(
media.contains("[media.mediamtx]") && media.contains("#1621"),
"the media refusal must name the config key and the issue: {media}"
);
let tls = refuse_tls_fleet(3, true).expect_err("a TLS fleet must be refused");
assert!(
tls.contains("[deploy.tls]") && tls.contains("#1621"),
"the TLS refusal must name the config key and the issue: {tls}"
);
assert!(
tls.contains("load balancer"),
"the TLS refusal must point at LB-terminated TLS: {tls}"
);
assert!(refuse_sqlite_fleet(1, true).is_ok());
assert!(refuse_media_fleet(1, true).is_ok());
assert!(refuse_tls_fleet(1, true).is_ok());
assert!(refuse_sqlite_fleet(3, false).is_ok());
assert!(refuse_media_fleet(3, false).is_ok());
assert!(refuse_tls_fleet(3, false).is_ok());
}
#[test]
fn writable_db_is_sqlite_reads_the_url_the_deploy_would_ship() {
use autumn_web::config::DatabaseConfig;
let sqlite = DatabaseConfig {
url: Some("sqlite://app.db".to_owned()),
..DatabaseConfig::default()
};
assert!(writable_db_is_sqlite(&sqlite));
let postgres = DatabaseConfig {
url: Some("postgres://user:pw@db.internal/app".to_owned()),
..DatabaseConfig::default()
};
assert!(!writable_db_is_sqlite(&postgres));
assert!(!writable_db_is_sqlite(&DatabaseConfig::default()));
}
#[test]
fn a_sqlite_fleet_is_refused_before_any_remote_command() {
let fleet = fleet_of(&["web-a", "web-b", "web-c"]);
let recorder = fleet::test_support::FleetRecorder::new();
let fixture = FleetFixture::new();
let input = FleetUpInput {
profile: "prod",
app_name: "myapp",
db: FleetDatabaseFacts {
sqlite: true,
..FleetDatabaseFacts::default()
},
..fixture.input(&fleet)
};
let err = run_up_with(&input, |cfg| Ok(recorder.executor(cfg)))
.expect_err("a sqlite fleet must be refused");
match &err {
DeployError::Config(message) => assert!(
message.contains("sqlite") && message.contains("#1621"),
"the refusal must name the backend and the issue: {message}"
),
other => panic!("expected a config refusal, got: {other:?}"),
}
assert!(
recorder.mutating(&[]).is_empty(),
"a refused fleet must run NO remote command at all, probes included: {:?}",
recorder.mutating(&[])
);
}
#[test]
fn a_media_fleet_is_refused_before_any_remote_command() {
let fleet = fleet_of(&["web-a", "web-b"]);
let recorder = fleet::test_support::FleetRecorder::new();
let mut fixture = FleetFixture::new();
fixture.media_cfg.enabled = true;
let input = fixture.input(&fleet);
let err = run_up_with(&input, |cfg| Ok(recorder.executor(cfg)))
.expect_err("a media-enabled fleet must be refused");
match &err {
DeployError::Config(message) => assert!(
message.contains("[media.mediamtx]") && message.contains("#1621"),
"the refusal must name the config key and the issue: {message}"
),
other => panic!("expected a config refusal, got: {other:?}"),
}
assert!(
recorder.mutating(&[]).is_empty(),
"a refused fleet must run NO remote command at all, probes included: {:?}",
recorder.mutating(&[])
);
}
#[test]
fn a_tls_fleet_is_refused_before_any_remote_command() {
let fleet = tls_fleet_of(&["web-a", "web-b"]);
let recorder = fleet::test_support::FleetRecorder::new();
let fixture = FleetFixture::new();
let err = run_up_with(&fixture.input(&fleet), |cfg| Ok(recorder.executor(cfg)))
.expect_err("a TLS-enabled fleet must be refused");
match &err {
DeployError::Config(message) => assert!(
message.contains("[deploy.tls]") && message.contains("load balancer"),
"the refusal must name the config key and point at LB-terminated TLS: \
{message}"
),
other => panic!("expected a config refusal, got: {other:?}"),
}
assert!(
recorder.mutating(&[]).is_empty(),
"a refused fleet must run NO remote command at all, probes included: {:?}",
recorder.mutating(&[])
);
}
#[test]
fn up_refuses_the_whole_fleet_when_one_host_fails_preflight() {
let fleet = fleet_of(&["web-a", "web-b", "web-c"]);
let checks = vec![
PreflightCheck::pass("signing_secret", "ok"),
PreflightCheck::fail("ssh_reachability", "unreachable", "check the network")
.scoped(Some("web-b".to_owned())),
];
let recorder = fleet::test_support::FleetRecorder::new();
let fixture = FleetFixture::new();
let executor_builds = std::cell::Cell::new(0_usize);
let err = refuse_if_preflight_failed(&checks)
.and_then(|()| {
run_up_with(&fixture.input(&fleet), |cfg| {
executor_builds.set(executor_builds.get() + 1);
Ok(recorder.executor(cfg))
})
})
.expect_err("a failing host must refuse the whole rollout");
assert!(
matches!(err, DeployError::PreflightFailed(1)),
"expected PreflightFailed(1), got {err:?}"
);
assert_eq!(
executor_builds.get(),
0,
"no executor may be built for any host when preflight fails"
);
assert!(
recorder.mutating(&[]).is_empty(),
"a refused fleet must run NO remote command at all, probes included: {:?}",
recorder.mutating(&[])
);
}
#[test]
fn up_calls_the_preflight_gate_before_handing_off_to_run_up_with() {
let src = include_str!("deploy.rs");
let up_body = src
.split("fn run_up(")
.nth(1)
.and_then(|s| s.split("\nfn run_rollback(").next())
.expect("run_up body present");
let gate_at = up_body
.find("refuse_if_preflight_failed(&checks)")
.expect("the preflight gate call present on the up path");
let handoff_at = up_body
.find("run_up_with(")
.expect("the run_up_with hand-off present on the up path");
assert!(
gate_at < handoff_at,
"run_up must call the preflight gate BEFORE handing off to run_up_with",
);
}
#[test]
fn the_refusals_grade_the_configured_topology_not_the_narrowed_rollout() {
let fleet = fleet_of(&["web-b"]);
let recorder = fleet::test_support::FleetRecorder::new();
let fixture = FleetFixture::new();
let input = FleetUpInput {
profile: "prod",
app_name: "myapp",
configured_host_count: 3,
db: FleetDatabaseFacts {
sqlite: true,
..FleetDatabaseFacts::default()
},
..fixture.input(&fleet)
};
let err = run_up_with(&input, |cfg| Ok(recorder.executor(cfg)))
.expect_err("`--only` must not defeat a topology refusal");
assert!(
matches!(&err, DeployError::Config(m) if m.contains("sqlite")),
"expected the sqlite refusal, got: {err:?}"
);
}
#[test]
fn auto_migrate_warns_for_a_fleet_and_stays_silent_for_one_host() {
let warning = fleet_auto_migrate_warning(3, Some(true), false)
.expect("a fleet with auto_migrate on must be warned");
assert!(
warning.contains("auto_migrate") && warning.contains("#1621"),
"the warning must name the key and the issue: {warning}"
);
assert!(
warning.contains("boot") && warning.contains("crash loop"),
"the warning must name the mechanism (boot races) and the symptom: {warning}"
);
assert!(fleet_auto_migrate_warning(3, None, true).is_some());
assert!(
fleet_auto_migrate_warning(3, Some(false), true).is_none(),
"an explicit `auto_migrate = false` wins over the alias"
);
assert!(fleet_auto_migrate_warning(1, Some(true), true).is_none());
assert!(fleet_auto_migrate_warning(3, None, false).is_none());
}
#[test]
fn per_host_job_and_scheduler_backends_warn_for_a_fleet_and_stay_silent_for_one_host() {
let warning = fleet_shared_backend_warning(3, "local", true)
.expect("a fleet on the in-process defaults must be warned");
assert!(
warning.contains("[jobs] backend")
&& warning.contains("[scheduler] backend")
&& warning.contains("#1621"),
"the warning must name both keys and the issue: {warning}"
);
assert!(
warning.to_lowercase().contains("per host") && warning.contains("drain"),
"the warning must name the mechanism (a queue per host, drained away): {warning}"
);
let jobs_only = fleet_shared_backend_warning(2, "local", false)
.expect("a per-host job queue alone must warn");
assert!(
jobs_only.contains("[jobs] backend") && !jobs_only.contains("[scheduler] backend"),
"a durable scheduler must not be named as a problem: {jobs_only}"
);
let scheduler_only = fleet_shared_backend_warning(2, "postgres", true)
.expect("a per-host scheduler alone must warn");
assert!(
scheduler_only.contains("[scheduler] backend")
&& !scheduler_only.contains("[jobs] backend"),
"a durable job backend must not be named as a problem: {scheduler_only}"
);
assert!(fleet_shared_backend_warning(1, "local", true).is_none());
assert!(fleet_shared_backend_warning(5, "postgres", false).is_none());
assert!(fleet_shared_backend_warning(5, " Redis ", false).is_none());
}
#[test]
fn only_selects_in_declaration_order_and_rejects_an_unknown_host() {
let configured: Vec<String> = ["web-a", "web-b", "web-c"]
.iter()
.map(|h| (*h).to_owned())
.collect();
assert_eq!(
select_hosts(&configured, &[]).expect("an empty selection is the whole fleet"),
configured,
"no `--only` must select the whole fleet, in order"
);
assert_eq!(
select_hosts(&configured, &["web-c".to_owned(), "web-a".to_owned()])
.expect("a valid selection resolves"),
vec!["web-a".to_owned(), "web-c".to_owned()],
"the selection must keep DECLARATION order, not flag order"
);
assert_eq!(
select_hosts(&configured, &["web-b".to_owned(), "web-b".to_owned()])
.expect("a repeated name is not an error"),
vec!["web-b".to_owned()],
"a repeated `--only` selects that host once"
);
let err = select_hosts(&configured, &["web-9".to_owned()])
.expect_err("an unknown host must be refused");
assert!(
err.contains("web-9") && err.contains("--only"),
"the error must quote the unmatched value and the flag: {err}"
);
for host in &configured {
assert!(
err.contains(host.as_str()),
"the error must list the configured hosts so the operator can fix the \
typo: {err}"
);
}
}
#[test]
fn only_restricts_the_rollout_and_moves_the_migration_with_it() {
let selected = select_hosts(
&["web-a", "web-b", "web-c"]
.iter()
.map(|h| (*h).to_owned())
.collect::<Vec<_>>(),
&["web-b".to_owned()],
)
.expect("web-b is in the configured fleet");
let fleet = ResolvedFleet::from_targets(&resolved_defaults(), &selected);
let recorder = script_redeploy(fleet::test_support::FleetRecorder::new(), "web-b");
let fixture = FleetFixture::new();
let input = FleetUpInput {
profile: "prod",
app_name: "myapp",
configured_host_count: 3,
..fixture.input(&fleet)
};
run_up_with(&input, |cfg| Ok(recorder.executor(cfg)))
.expect("a narrowed rollout deploys the selected host");
assert_eq!(
recorder.run_labels_for("web-b"),
READ_ONLY_PROBES
.iter()
.copied()
.chain(REDEPLOY_RUN_LABELS)
.collect::<Vec<_>>(),
"the selected host runs the full redeploy, migration included"
);
for skipped in ["web-a", "web-c"] {
assert!(
recorder.calls_for(skipped).is_empty(),
"`--only` must not touch {skipped} at all — not even a read-only probe"
);
}
}
#[test]
fn a_narrowed_rollout_keeps_fleet_semantics_rather_than_the_single_host_path() {
let fleet = fleet_of(&["web-b"]);
let recorder = script_redeploy(fleet::test_support::FleetRecorder::new(), "web-b")
.fail("web-b", "readiness-gate");
let fixture = FleetFixture::new();
let input = FleetUpInput {
profile: "prod",
app_name: "myapp",
configured_host_count: 3,
..fixture.input(&fleet)
};
let err = run_up_with(&input, |cfg| Ok(recorder.executor(cfg)))
.expect_err("the selected host's failure must fail the deploy");
let halt = fleet_halt_of(&err);
assert_eq!(halt.failed_host, "web-b");
assert_eq!(halt.failed_step, "readiness-gate");
}
#[test]
fn a_narrowed_probe_refusal_still_names_its_host() {
let narrowed = fleet_of(&["web-b"]);
let recorder = fleet::test_support::FleetRecorder::new()
.script("web-b", "proxy-compat-probe", compatible_deploy_help())
.script("web-b", "detect-current", redeploy_probe())
.script("web-b", "probe-release-dir", "present");
let fixture = FleetFixture::new();
let input = FleetUpInput {
profile: "prod",
app_name: "myapp",
configured_host_count: 3,
..fixture.input(&narrowed)
};
let narrowed_message = run_up_with(&input, |cfg| Ok(recorder.executor(cfg)))
.expect_err("an existing release dir must refuse the rollout")
.to_string();
assert!(
narrowed_message.contains("host web-b: release directory"),
"a narrowed run keeps fleet attribution on probe refusals, got: \
{narrowed_message}"
);
let single = fleet_of(&["web-b"]);
let single_recorder = fleet::test_support::FleetRecorder::new()
.script("web-b", "proxy-compat-probe", compatible_deploy_help())
.script("web-b", "detect-current", redeploy_probe())
.script("web-b", "probe-release-dir", "present");
let single_message = run_up_with(&fixture.input(&single), |cfg| {
Ok(single_recorder.executor(cfg))
})
.expect_err("an existing release dir must refuse the rollout")
.to_string();
assert!(
!single_message.contains("host web-b: "),
"a genuine single-host refusal must keep the unattributed pre-#1621 \
text, got: {single_message}"
);
assert_eq!(
narrowed_message.replace("host web-b: ", ""),
single_message,
"the two must differ ONLY by the attribution prefix"
);
}
#[test]
fn a_narrowed_preflight_still_scopes_its_reachability_row() {
let config = preflight_config();
let narrowed = ResolvedFleet::resolve(&fleet_cfg(&["web-2.example.test"]), "myapp")
.expect("a narrowed fleet resolves");
let scoped = collect_fleet_preflight(&config, &narrowed, 3);
assert_eq!(
scoped
.iter()
.filter(|check| check.name == SSH_REACHABILITY_CHECK)
.map(|check| check.scope.as_deref())
.collect::<Vec<_>>(),
vec![Some("web-2.example.test")],
"a narrowed run must keep host attribution: {:?}",
scoped
.iter()
.map(|c| (c.name, &c.scope))
.collect::<Vec<_>>()
);
let genuine = collect_fleet_preflight(&config, &narrowed, 1);
assert!(
genuine.iter().all(|check| check.scope.is_none()),
"a genuine one-host config must stay unscoped (AC-1): {:?}",
genuine
.iter()
.map(|c| (c.name, &c.scope))
.collect::<Vec<_>>()
);
}
#[test]
fn no_rollback_freezes_the_fleet_instead_of_compensating() {
let hosts = ["web-a", "web-b", "web-c"];
let fleet = fleet_of(&hosts);
let mut recorder = fleet::test_support::FleetRecorder::new();
for host in hosts {
recorder = script_redeploy(recorder, host);
}
recorder = recorder.transport_fail("web-b", "drain-old");
let fixture = FleetFixture::new();
let err = run_up_with(&fixture.input_frozen(&fleet), |cfg| {
Ok(recorder.executor(cfg))
})
.expect_err("a post-boundary failure must fail the deploy");
let halt = fleet_halt_of(&err);
assert_eq!(
halt.still_on_new,
vec!["web-a".to_owned(), "web-b".to_owned()],
"a frozen fleet reports every host left on the new release"
);
assert!(
halt.rolled_back.is_empty() && halt.torn_down.is_empty(),
"freezing compensates NOTHING: {:?} / {:?}",
halt.rolled_back,
halt.torn_down
);
assert_eq!(
recorder
.positions_of("proxy-flip")
.iter()
.filter(|_| true)
.count(),
2,
"exactly two flips happened (web-a's and web-b's cutovers) — no host was \
flipped back"
);
assert!(
!recorder
.run_labels_for("web-a")
.contains(&"restart-previous"),
"a frozen rollout must not roll web-a back"
);
assert!(
include_str!("deploy.rs").contains("auto_rollback: !options.no_rollback"),
"`--no-rollback` must map to the `auto_rollback` driver seam"
);
}
fn script_previous_release(
recorder: fleet::test_support::FleetRecorder,
host: &str,
) -> fleet::test_support::FleetRecorder {
recorder.script(
host,
"resolve-previous",
format!("prev:{PREVIOUS_RELEASE_DIR}\tblue\t3001"),
)
}
fn drive_fleet_rollback(
fleet: &ResolvedFleet,
recorder: &fleet::test_support::FleetRecorder,
) -> Result<(), DeployError> {
let proxy = proxy::KamalProxyController::new(60);
run_fleet_rollback_with(&[], fleet, &proxy, FLEET_PUBLIC_PORT, |cfg| {
Ok(recorder.executor(cfg))
})
}
fn status_probe(release: &str) -> String {
format!(
"redeploy:blue\t3001\n\
---autumn-kamal-proxy-list---\n\
Service Host Target State TLS\n\
myapp example.com 127.0.0.1:3001 running no\n\
---autumn-kamal-proxy-unit---\n--http-port 3000\n\
---autumn-kamal-proxy-options---\n0\t\n\
---autumn-current-release---\n/srv/autumn/myapp/releases/{release}"
)
}
const STATUS_SHARED_FLAG: &str = "/srv/autumn/myapp/shared/autumn-maintenance.json";
fn script_status(
recorder: fleet::test_support::FleetRecorder,
host: &str,
release: &str,
ready: &str,
maintenance: bool,
) -> fleet::test_support::FleetRecorder {
recorder
.script(host, "detect-current", status_probe(release))
.script(
host,
"probe-host-status",
format!(
"{ready}\n---autumn-host-status---\n{flag}\n---autumn-host-status---\n\
deployed\t2026-07-14T12:00:03Z\n---autumn-host-status---\n\
{STATUS_SHARED_FLAG}\n{flag}",
flag = if maintenance { "maintenance-on" } else { "" }
),
)
}
fn script_status_last_deploy(
recorder: fleet::test_support::FleetRecorder,
host: &str,
release: &str,
last_deploy: &str,
) -> fleet::test_support::FleetRecorder {
recorder
.script(host, "detect-current", status_probe(release))
.script(
host,
"probe-host-status",
format!(
"200\n---autumn-host-status---\n\n---autumn-host-status---\n{last_deploy}\n\
---autumn-host-status---\n{STATUS_SHARED_FLAG}\n"
),
)
}
fn drive_status(
fleet: &ResolvedFleet,
recorder: &fleet::test_support::FleetRecorder,
) -> Result<Vec<fleet::HostStatus>, DeployError> {
collect_fleet_status(fleet, FLEET_PUBLIC_PORT, |cfg| Ok(recorder.executor(cfg)))
}
#[test]
fn fleet_status_reports_every_host_read_only_and_in_declaration_order() {
let hosts = ["web-a", "web-b", "web-c"];
let fleet = fleet_of(&hosts);
let mut recorder = fleet::test_support::FleetRecorder::new();
for host in hosts {
recorder = script_status(recorder, host, "20260714T120000Z", "200", false);
}
let statuses =
drive_status(&fleet, &recorder).expect("a read-only status never fails the command");
assert_eq!(
statuses.iter().map(|s| s.host.as_str()).collect::<Vec<_>>(),
hosts.to_vec(),
"status must keep `[deploy] hosts` order"
);
for host in hosts {
assert_eq!(
recorder.run_labels_for(host),
vec!["detect-current", "probe-host-status"],
"{host}: status must run ONLY the two read-only probes"
);
}
assert!(
recorder
.mutating(&["detect-current", "probe-host-status"])
.is_empty(),
"`deploy status` must never mutate a host"
);
let report = fleet::fleet_drift(&statuses);
assert!(!report.drifted(), "a converged fleet reports no drift");
}
#[test]
fn fleet_status_reports_an_unreachable_host_as_a_row_not_an_abort() {
let hosts = ["web-a", "web-b", "web-c"];
let fleet = fleet_of(&hosts);
let mut recorder = fleet::test_support::FleetRecorder::new();
recorder = script_status(recorder, "web-a", "r1", "200", false);
recorder = script_status(recorder, "web-b", "r1", "200", false)
.transport_fail("web-b", "detect-current");
recorder = script_status(recorder, "web-c", "r1", "200", false);
let statuses = drive_status(&fleet, &recorder).expect("status reports");
assert_eq!(statuses.len(), 3, "every host gets a row");
assert!(!statuses[1].reachable, "host 2 is reported unreachable");
assert!(statuses[0].reachable && statuses[2].reachable);
let report = fleet::fleet_drift(&statuses);
assert!(
!report.version_drift && report.state_drift.is_empty(),
"an unreachable host must not manufacture drift: {report:?}"
);
assert_eq!(report.unknown_releases(), vec!["web-b"]);
}
#[test]
fn fleet_status_surfaces_version_drift_across_mixed_releases() {
let hosts = ["web-a", "web-b"];
let fleet = fleet_of(&hosts);
let mut recorder = fleet::test_support::FleetRecorder::new();
recorder = script_status(recorder, "web-a", "20260714T120000Z", "200", false);
recorder = script_status(recorder, "web-b", "20260714T130000Z", "200", true);
let statuses = drive_status(&fleet, &recorder).expect("status reports");
let report = fleet::fleet_drift(&statuses);
assert!(report.version_drift, "different releases are drift");
assert_eq!(
statuses[1].maintenance,
exec::MaintenanceStatus::On,
"the flag file is read per host"
);
assert!(
statuses[1].ready_code == Some(200),
"maintenance does NOT change readiness — the two are orthogonal"
);
let json = fleet_status_json(&statuses, &report);
assert_eq!(json["version_drift"], serde_json::json!(true));
assert_eq!(
json["hosts"][0]["release"],
serde_json::json!("20260714T120000Z")
);
assert_eq!(json["hosts"][1]["maintenance"], serde_json::json!(true));
assert_eq!(json["hosts"][1]["ready"], serde_json::json!(200));
}
#[test]
fn status_reports_the_last_deploy_result_per_host() {
let fleet = fleet_of(&["web-a", "web-b", "web-c"]);
let mut recorder = fleet::test_support::FleetRecorder::new();
recorder =
script_status_last_deploy(recorder, "web-a", "r1", "rolled back\t2026-07-14T12:31:10Z");
recorder =
script_status_last_deploy(recorder, "web-b", "r1", "deployed\t2026-07-14T12:00:03Z");
recorder = script_status_last_deploy(recorder, "web-c", "r1", "");
let statuses = drive_status(&fleet, &recorder).expect("status reports");
assert_eq!(
statuses[0].last_deploy.as_ref().map(|l| l.result.as_str()),
Some("rolled back"),
);
assert_eq!(
statuses[0]
.last_deploy
.as_ref()
.and_then(|l| l.at.as_deref()),
Some("2026-07-14T12:31:10Z"),
);
assert_eq!(
statuses[1].last_deploy.as_ref().map(|l| l.result.as_str()),
Some("deployed"),
);
assert_eq!(statuses[2].last_deploy, None);
let report = fleet::fleet_drift(&statuses);
assert!(!report.drifted(), "a past rollback is not fleet drift");
let rendered = fleet::fleet_status_lines(&statuses, &report).join("\n");
assert!(
rendered.contains("last deploy: rolled back 2026-07-14T12:31:10Z")
&& rendered.contains("last deploy: deployed 2026-07-14T12:00:03Z")
&& rendered.contains("last deploy: ?"),
"every host's last-deploy cell must be rendered:\n{rendered}"
);
assert!(
rendered.contains(fleet::LAST_DEPLOY_SCOPE_NOTE),
"the status table must state what `last deploy` does and does not know:\n{rendered}"
);
let json = fleet_status_json(&statuses, &report);
assert_eq!(
json["hosts"][0]["last_deploy"],
serde_json::json!({ "result": "rolled back", "at": "2026-07-14T12:31:10Z" }),
);
assert_eq!(json["hosts"][2]["last_deploy"], serde_json::Value::Null);
}
#[test]
fn status_never_renders_a_green_marker_for_an_unready_host() {
let fleet = fleet_of(&["web-a", "web-b", "web-c"]);
let mut recorder = fleet::test_support::FleetRecorder::new();
recorder = script_status(recorder, "web-a", "r1", "200", false);
recorder = script_status(recorder, "web-b", "r1", "503", false);
recorder = script_status(recorder, "web-c", "r1", "000", false);
let statuses = drive_status(&fleet, &recorder).expect("status reports");
let report = fleet::fleet_drift(&statuses);
assert!(!report.drifted(), "{:?}", report.state_drift);
let rendered = fleet::fleet_status_lines(&statuses, &report).join("\n");
let row = |host: &str| {
rendered
.lines()
.find(|line| line.contains(host))
.expect("every host has a row")
};
assert!(row("web-a").starts_with(" \u{2705}"), "{rendered}");
for host in ["web-b", "web-c"] {
assert!(row(host).starts_with(" \u{26A0}"), "{rendered}");
}
}
#[test]
fn fleet_status_json_shape_is_stable_and_carries_no_secret() {
let fleet = fleet_of(&["web-a", "web-b"]);
let mut recorder = fleet::test_support::FleetRecorder::new();
recorder = script_status(recorder, "web-a", "r1", "200", false);
recorder = script_status(recorder, "web-b", "r1", "503", false);
let statuses = drive_status(&fleet, &recorder).expect("status reports");
let json = fleet_status_json(&statuses, &fleet::fleet_drift(&statuses));
let host = &json["hosts"][0];
for field in [
"host",
"reachable",
"mode",
"release",
"live_slot",
"ready",
"maintenance",
"proxy_port",
"last_deploy",
"drift",
] {
assert!(
host.get(field).is_some(),
"the documented per-host field `{field}` must be present: {json}"
);
}
for field in ["hosts", "version_drift", "state_drift", "drifted"] {
assert!(
json.get(field).is_some(),
"the documented top-level field `{field}` must be present: {json}"
);
}
assert_eq!(host["host"], serde_json::json!("web-a"));
assert_eq!(host["mode"], serde_json::json!("deployed"));
assert_eq!(json["hosts"][1]["ready"], serde_json::json!(503));
let rendered = json.to_string();
for secret in ["AUTUMN_SECURITY__SIGNING_SECRET", "postgres://", "curl"] {
assert!(
!rendered.contains(secret),
"the status JSON must never carry `{secret}`: {rendered}"
);
}
}
const MAINTENANCE_SHARED_PATH: &str = "/srv/autumn/myapp/shared/autumn-maintenance.json";
const MAINTENANCE_LEGACY_PATH: &str =
"/srv/autumn/myapp/releases/r1/tmp/autumn-maintenance.json";
const MAINTENANCE_LIVE_UNIT_PATH: &str =
"/srv/autumn/myapp/releases/r2/tmp/autumn-maintenance.json";
fn drive_maintenance(
fleet: &ResolvedFleet,
recorder: &fleet::test_support::FleetRecorder,
verb: DeployAction,
args: Option<&MaintenanceOnArgs>,
) -> Result<(), DeployError> {
let port = StatusPort {
port: FLEET_PUBLIC_PORT,
degraded: None,
};
run_fleet_maintenance_with(fleet, &port, "prod", verb, args, |cfg| {
Ok(recorder.executor(cfg))
})
}
fn script_maintenance(
recorder: fleet::test_support::FleetRecorder,
host: &str,
release: &str,
unit_flag: &str,
) -> fleet::test_support::FleetRecorder {
recorder
.script(host, "detect-current", status_probe(release))
.script(
host,
"detect-maintenance-flag",
if unit_flag.is_empty() {
String::new()
} else {
format!("{unit_flag}\n")
},
)
}
fn upload_paths(recorder: &fleet::test_support::FleetRecorder, host: &str) -> Vec<String> {
recorder
.calls_for(host)
.iter()
.filter_map(|call| match call {
exec::test_support::RecordedCall::Upload { remote_path, .. } => {
Some(remote_path.clone())
}
exec::test_support::RecordedCall::Run { .. } => None,
})
.collect()
}
#[test]
fn fleet_maintenance_on_writes_both_flag_paths_on_every_host() {
let hosts = ["web-a", "web-b"];
let fleet = fleet_of(&hosts);
let mut recorder = fleet::test_support::FleetRecorder::new();
for host in hosts {
recorder = script_maintenance(recorder, host, "r1", MAINTENANCE_LEGACY_PATH);
}
let args = MaintenanceOnArgs {
message: Some("Upgrading the database".to_owned()),
readonly: true,
..MaintenanceOnArgs::default()
};
drive_maintenance(&fleet, &recorder, DeployAction::MaintenanceOn, Some(&args))
.expect("every host accepts the write");
for host in hosts {
let paths = upload_paths(&recorder, host);
assert!(
paths.iter().any(|p| p == MAINTENANCE_SHARED_PATH),
"{host} must get the shared (authoritative) flag: {paths:?}"
);
assert!(
paths.iter().any(|p| p == MAINTENANCE_LEGACY_PATH),
"{host} must also get the release-scoped flag for pre-#1621 units: {paths:?}"
);
assert!(
recorder
.run_labels_for(host)
.contains(&"maintenance-prepare-live-flag-dir"),
"{host} must create the dir holding the running unit's flag before \
writing into it"
);
}
}
#[test]
fn fleet_maintenance_flag_is_written_0600_and_never_echoes_the_bypass_token() {
let hosts = ["web-a", "web-b"];
let fleet = fleet_of(&hosts);
let mut recorder = fleet::test_support::FleetRecorder::new();
for host in hosts {
recorder = script_maintenance(recorder, host, "r1", MAINTENANCE_LEGACY_PATH);
}
let args = MaintenanceOnArgs {
message: Some("Upgrading the database".to_owned()),
allow_ips: vec!["10.0.0.0/8".to_owned()],
readonly: true,
bypass_header: Some(("X-Dev-Bypass".to_owned(), "s3cr3t-bypass".to_owned())),
};
drive_maintenance(&fleet, &recorder, DeployAction::MaintenanceOn, Some(&args))
.expect("every host accepts the write");
for host in hosts {
let uploads: Vec<(String, Option<u32>)> = recorder
.calls_for(host)
.iter()
.filter_map(|call| match call {
exec::test_support::RecordedCall::Upload { remote_path, mode } => {
Some((remote_path.clone(), *mode))
}
exec::test_support::RecordedCall::Run { .. } => None,
})
.collect();
assert_eq!(
uploads.len(),
2,
"{host} writes the shared and the release-scoped flag: {uploads:?}"
);
for (path, mode) in &uploads {
assert_eq!(
*mode,
Some(0o600),
"{host}: {path} carries the bypass token, so it must not be world-readable"
);
}
}
let calls_debug = format!("{:#?}", recorder.calls_for("web-a"));
assert!(
!calls_debug.contains("s3cr3t-bypass"),
"the bypass token leaked into the recorded executor calls: {calls_debug}"
);
}
#[test]
fn fleet_maintenance_on_writes_shared_only_when_no_release_is_promoted() {
let fleet = fleet_of(&["web-a"]);
let recorder = fleet::test_support::FleetRecorder::new().script(
"web-a",
"detect-current",
"first\n---autumn-kamal-proxy-list---\n",
);
drive_maintenance(
&fleet,
&recorder,
DeployAction::MaintenanceOn,
Some(&MaintenanceOnArgs::default()),
)
.expect("a shared-only write is a success, not a failure");
let paths = upload_paths(&recorder, "web-a");
assert_eq!(
paths,
vec![MAINTENANCE_SHARED_PATH.to_owned()],
"only the shared flag can be written without a resolvable release"
);
assert!(
!recorder
.run_labels_for("web-a")
.contains(&"maintenance-prepare-live-flag-dir"),
"no running unit means no second flag dir to prepare"
);
}
#[test]
fn fleet_maintenance_on_prepares_the_shared_flag_dir_before_uploading() {
let fleet = fleet_of(&["web-a"]);
let recorder = fleet::test_support::FleetRecorder::new().script(
"web-a",
"detect-current",
"first\n---autumn-kamal-proxy-list---\n",
);
drive_maintenance(
&fleet,
&recorder,
DeployAction::MaintenanceOn,
Some(&MaintenanceOnArgs::default()),
)
.expect("a shared-only write is a success, not a failure");
let calls = recorder.calls_for("web-a");
let mkdir_pos = calls
.iter()
.position(|call| {
matches!(
call,
exec::test_support::RecordedCall::Run { label, shell }
if *label == "maintenance-prepare-shared-flag-dir"
&& shell.contains("mkdir -p")
&& shell.contains("/srv/autumn/myapp/shared")
)
})
.expect("the shared flag's parent must be created before the upload");
let upload_pos = calls
.iter()
.position(|call| {
matches!(
call,
exec::test_support::RecordedCall::Upload { remote_path, .. }
if remote_path == MAINTENANCE_SHARED_PATH
)
})
.expect("the shared flag is uploaded");
assert!(
mkdir_pos < upload_pos,
"the shared dir is created BEFORE the shared flag is written"
);
assert_eq!(
upload_paths(&recorder, "web-a"),
vec![MAINTENANCE_SHARED_PATH.to_owned()],
"only the shared flag can be written without a resolvable release"
);
}
#[test]
fn fleet_maintenance_writes_the_flag_the_running_unit_polls_not_the_symlink_target() {
let fleet = fleet_of(&["web-a"]);
let recorder = script_maintenance(
fleet::test_support::FleetRecorder::new(),
"web-a",
"r1",
MAINTENANCE_LIVE_UNIT_PATH,
);
drive_maintenance(
&fleet,
&recorder,
DeployAction::MaintenanceOn,
Some(&MaintenanceOnArgs::default()),
)
.expect("both writes land");
assert_eq!(
upload_paths(&recorder, "web-a"),
vec![
MAINTENANCE_SHARED_PATH.to_owned(),
MAINTENANCE_LIVE_UNIT_PATH.to_owned(),
],
"the shared flag first (A2), then the file the RUNNING unit polls — and \
never the stale `current` target"
);
}
#[test]
fn fleet_maintenance_on_fails_closed_when_the_running_units_flag_is_unproved() {
let fleet = fleet_of(&["web-a"]);
let recorder =
script_maintenance(fleet::test_support::FleetRecorder::new(), "web-a", "r1", "");
let err = drive_maintenance(
&fleet,
&recorder,
DeployAction::MaintenanceOn,
Some(&MaintenanceOnArgs::default()),
)
.expect_err("an unproved live-unit flag must not exit 0");
assert!(
err.to_string().contains("1 of 1"),
"the host counts as failed in the aggregate: {err}"
);
assert_eq!(
upload_paths(&recorder, "web-a"),
vec![MAINTENANCE_SHARED_PATH.to_owned()],
"the shared flag is still written first; nothing is GUESSED for the rest"
);
}
#[test]
fn fleet_maintenance_off_never_claims_to_clear_a_flag_it_cannot_locate() {
let fleet = fleet_of(&["web-a"]);
let recorder =
script_maintenance(fleet::test_support::FleetRecorder::new(), "web-a", "r1", "");
drive_maintenance(&fleet, &recorder, DeployAction::MaintenanceOff, None)
.expect_err("an unproved live-unit flag must not exit 0");
let shell = recorder
.calls_for("web-a")
.iter()
.find_map(|call| match call {
exec::test_support::RecordedCall::Run { label, shell }
if *label == "maintenance-clear" =>
{
Some(shell.clone())
}
_ => None,
})
.expect("the shared flag is still cleared");
assert!(
shell.contains(MAINTENANCE_SHARED_PATH),
"the shared flag is still removed: {shell}"
);
assert!(
!shell.contains(MAINTENANCE_LEGACY_PATH),
"a path derived from `current` must not be removed on a hunch: {shell}"
);
}
#[test]
fn fleet_maintenance_row_says_the_running_units_flag_was_not_changed() {
let rendered = fleet::fleet_maintenance_summary_lines(
&["web-a".to_owned()],
&[fleet::MaintenanceOutcome::LiveUnitUnchanged {
failed_step: "detect-maintenance-flag",
}],
true,
)
.join("\n");
assert!(
rendered.contains("detect-maintenance-flag"),
"the row names the step that could not be proved:\n{rendered}"
);
assert!(
rendered.contains("shared flag") && rendered.contains("RUNNING"),
"the row must distinguish the shared flag from the running unit's:\n{rendered}"
);
}
#[test]
fn fleet_maintenance_serializes_the_framework_wire_format() {
let args = MaintenanceOnArgs {
message: Some("Upgrading".to_owned()),
allow_ips: vec!["10.0.0.0/8".to_owned()],
readonly: true,
bypass_header: Some(("X-Bypass".to_owned(), "tok".to_owned())),
};
let body = maintenance_flag_body(&args);
let parsed: autumn_web::maintenance::MaintenanceConfig =
serde_json::from_str(&body).expect("the body must be a MaintenanceConfig");
assert_eq!(parsed.message.as_deref(), Some("Upgrading"));
assert!(parsed.readonly);
assert_eq!(parsed.allow_ips, vec!["10.0.0.0/8"]);
assert_eq!(
parsed
.bypass_header
.as_ref()
.map(|(n, v)| (n.as_str(), v.as_str())),
Some(("X-Bypass", "tok"))
);
}
#[test]
fn fleet_maintenance_off_removes_both_flag_paths() {
let hosts = ["web-a", "web-b"];
let fleet = fleet_of(&hosts);
let mut recorder = fleet::test_support::FleetRecorder::new();
for host in hosts {
recorder = script_maintenance(recorder, host, "r1", MAINTENANCE_LEGACY_PATH);
}
drive_maintenance(&fleet, &recorder, DeployAction::MaintenanceOff, None)
.expect("every host clears cleanly");
for host in hosts {
let shell = recorder
.calls_for(host)
.iter()
.find_map(|call| match call {
exec::test_support::RecordedCall::Run { label, shell }
if *label == "maintenance-clear" =>
{
Some(shell.clone())
}
_ => None,
})
.unwrap_or_else(|| panic!("{host} must run the clear op"));
assert!(
shell.contains(MAINTENANCE_SHARED_PATH) && shell.contains(MAINTENANCE_LEGACY_PATH),
"{host} must clear BOTH paths: {shell}"
);
assert!(
shell.contains("rm -f"),
"clearing an absent flag must not be an error: {shell}"
);
assert!(
upload_paths(&recorder, host).is_empty(),
"`off` writes nothing"
);
}
}
#[test]
fn fleet_maintenance_attempts_every_host_and_reports_the_ones_that_failed() {
let hosts = ["web-a", "web-b", "web-c"];
let fleet = fleet_of(&hosts);
let mut recorder = fleet::test_support::FleetRecorder::new();
for host in hosts {
recorder = script_maintenance(recorder, host, "r1", MAINTENANCE_LEGACY_PATH);
}
recorder = recorder.fail_upload("web-b", MAINTENANCE_SHARED_PATH);
let err = drive_maintenance(
&fleet,
&recorder,
DeployAction::MaintenanceOn,
Some(&MaintenanceOnArgs::default()),
)
.expect_err("a failed host must exit non-zero");
let message = err.to_string();
assert!(
message.contains("1 of 3"),
"the error must count the failures against the fleet: {message}"
);
for host in ["web-a", "web-c"] {
assert!(
upload_paths(&recorder, host)
.iter()
.any(|p| p == MAINTENANCE_SHARED_PATH),
"{host} must still be attempted (and changed) after web-b failed"
);
}
assert!(
upload_paths(&recorder, "web-b")
.iter()
.any(|p| p == MAINTENANCE_SHARED_PATH),
"the failing host is attempted too, not skipped"
);
for noise in ["scripted upload failure", "rm -f", "printf"] {
assert!(
!message.contains(noise),
"the aggregate error must carry no remote detail: {message}"
);
}
}
#[test]
fn fleet_maintenance_rows_name_every_host_and_the_orthogonality_note() {
let rendered = fleet::fleet_maintenance_summary_lines(
&["web-a".to_owned(), "web-b".to_owned(), "web-c".to_owned()],
&[
fleet::MaintenanceOutcome::Applied,
fleet::MaintenanceOutcome::Failed {
failed_step: "maintenance-write-shared",
},
fleet::MaintenanceOutcome::AppliedSharedOnly,
],
true,
)
.join("\n");
for host in ["web-a", "web-b", "web-c"] {
assert!(rendered.contains(host), "every host is named:\n{rendered}");
}
assert!(
rendered.contains(fleet::MAINTENANCE_DOES_NOT_DRAIN_NOTE),
"the orthogonality note must be printed:\n{rendered}"
);
assert!(
rendered.contains("maintenance-write-shared"),
"a failure names its step label:\n{rendered}"
);
assert!(
rendered.contains("Changed anyway: web-a, web-c"),
"the hosts that DID change must be named so they can be reversed:\n{rendered}"
);
}
#[test]
fn fleet_rollback_rolls_back_every_host_newest_first() {
let hosts = ["web-a", "web-b", "web-c"];
let fleet = fleet_of(&hosts);
let mut recorder = fleet::test_support::FleetRecorder::new();
for host in hosts {
recorder = script_previous_release(recorder, host);
}
drive_fleet_rollback(&fleet, &recorder).expect("every host rolls back cleanly");
for host in hosts {
assert_eq!(
recorder.run_labels_for(host),
FLEET_ROLLBACK_RUN_LABELS.to_vec(),
"{host} must run the UNCHANGED on-demand rollback sequence"
);
}
let flips: Vec<usize> = hosts
.iter()
.map(|host| {
recorder
.index_of(host, "proxy-flip")
.unwrap_or_else(|| panic!("{host} must flip back"))
})
.collect();
assert!(
flips[2] < flips[1] && flips[1] < flips[0],
"hosts must roll back newest-first (web-c, web-b, web-a), got positions {flips:?}"
);
}
#[test]
fn fleet_rollback_rolls_the_reachable_hosts_back_past_a_dead_peer() {
let hosts = ["web-a", "web-b", "web-c"];
let fleet = fleet_of(&hosts);
let mut recorder = fleet::test_support::FleetRecorder::new();
for host in ["web-a", "web-c"] {
recorder = script_previous_release(recorder, host);
}
let checks = vec![
PreflightCheck::fail(
"ssh_reachability",
"cannot reach web-b:22",
"check the host",
)
.scoped(Some("web-b".to_owned())),
PreflightCheck::pass("migrate_check", "ok"),
];
let proxy = proxy::KamalProxyController::new(60);
let err = run_fleet_rollback_with(&checks, &fleet, &proxy, FLEET_PUBLIC_PORT, |cfg| {
Ok(recorder.executor(cfg))
})
.expect_err("a fleet that did not fully converge must exit non-zero");
assert!(
matches!(
&err,
DeployError::FleetRollbackFailed {
not_rolled_back: 1,
total: 3
}
),
"the unreachable host is reported as NOT rolled back, got: {err:?}"
);
for host in ["web-a", "web-c"] {
assert_eq!(
recorder.run_labels_for(host),
FLEET_ROLLBACK_RUN_LABELS.to_vec(),
"{host} is reachable and must be rolled back despite web-b being dead"
);
}
assert!(
recorder.run_labels_for("web-b").is_empty(),
"an unreachable host must not be touched at all: {:?}",
recorder.run_labels_for("web-b")
);
}
#[test]
fn a_rollback_still_hard_gates_on_a_project_wide_preflight_failure() {
let reachable_fleet = vec![
PreflightCheck::fail(
"ssh_reachability",
"cannot reach web-b:22",
"check the host",
)
.scoped(Some("web-b".to_owned())),
];
assert_eq!(
blocking_rollback_failures(&reachable_fleet, true),
0,
"a per-host reachability failure must not block the reachable hosts"
);
let mut project_wide = reachable_fleet;
project_wide.push(PreflightCheck::fail(
"migrate_check",
"migrations do not compile",
"fix them",
));
assert_eq!(
blocking_rollback_failures(&project_wide, true),
1,
"a project-wide grader still aborts the whole command"
);
let single = vec![PreflightCheck::fail(
"ssh_reachability",
"cannot reach web-a:22",
"check the host",
)];
assert_eq!(
blocking_rollback_failures(&single, false),
1,
"a single-host rollback must keep aborting before touching the server"
);
let narrowed = vec![
PreflightCheck::fail(
"ssh_reachability",
"cannot reach web-b:22",
"check the host",
)
.scoped(Some("web-b".to_owned())),
];
assert_eq!(
blocking_rollback_failures(&narrowed, false),
1,
"a narrowed single-host rollback must still abort before touching the server"
);
}
#[test]
fn fleet_rollback_skips_a_host_with_no_previous_release_and_rolls_the_rest_back() {
let hosts = ["web-a", "web-b", "web-c"];
let fleet = fleet_of(&hosts);
let mut recorder = fleet::test_support::FleetRecorder::new();
for host in ["web-a", "web-c"] {
recorder = script_previous_release(recorder, host);
}
recorder = recorder.script("web-b", "resolve-previous", "none");
let err = drive_fleet_rollback(&fleet, &recorder)
.expect_err("a fleet that did not fully converge must exit non-zero");
assert!(
matches!(
&err,
DeployError::FleetRollbackFailed {
not_rolled_back: 1,
total: 3
}
),
"exactly one host must be reported as un-rolled-back, got: {err:?}"
);
assert!(
err.to_string().contains("no previous release"),
"the error must explain the skipped case, not just count it: {err}"
);
assert_eq!(
recorder.run_labels_for("web-b"),
vec!["resolve-previous"],
"a host with no previous release must be probed and then left ALONE"
);
for host in ["web-a", "web-c"] {
assert_eq!(
recorder.run_labels_for(host),
FLEET_ROLLBACK_RUN_LABELS.to_vec(),
"{host} must still roll back — a skip is not a halt"
);
}
}
#[test]
fn fleet_rollback_continues_past_a_failed_host_and_exits_non_zero() {
let hosts = ["web-a", "web-b", "web-c"];
let fleet = fleet_of(&hosts);
let mut recorder = fleet::test_support::FleetRecorder::new();
for host in hosts {
recorder = script_previous_release(recorder, host);
}
recorder = recorder.fail("web-c", "readiness-gate");
let err = drive_fleet_rollback(&fleet, &recorder)
.expect_err("a failed host must fail the fleet rollback");
match &err {
DeployError::FleetRollbackFailed {
not_rolled_back,
total,
} => {
assert_eq!((*not_rolled_back, *total), (1, 3));
}
other => panic!("expected a fleet-rollback failure, got: {other:?}"),
}
for host in ["web-a", "web-b"] {
assert!(
recorder.run_labels_for(host).contains(&"proxy-flip"),
"{host} must still be rolled back after web-c failed"
);
}
let rendered = format!("{err}\n{err:?}");
for secret in [
"postgres://",
"topsecret",
"AUTUMN_SECURITY__SIGNING_SECRET",
] {
assert!(
!rendered.contains(secret),
"the fleet-rollback error must never carry `{secret}`: {rendered}"
);
}
}
#[test]
fn a_fleet_rollback_that_rolled_nothing_back_is_an_error() {
let hosts = ["web-a", "web-b"];
let fleet = fleet_of(&hosts);
let mut recorder = fleet::test_support::FleetRecorder::new();
for host in hosts {
recorder = recorder.script(host, "resolve-previous", "none");
}
let err = drive_fleet_rollback(&fleet, &recorder)
.expect_err("a rollback that rolled nothing back must not report success");
assert!(
matches!(
&err,
DeployError::FleetRollbackFailed {
not_rolled_back: 2,
total: 2
}
),
"expected all-hosts-unrolled, got: {err:?}"
);
}
#[test]
fn operator_facing_fleet_error_text_carries_no_line_continuation_gutter() {
assert_eq!(
DeployError::DriftDetected.to_string(),
"fleet drift detected \u{2014} the status table above names each host and reason \
(`--strict` exits non-zero on drift so it can be alerted on)",
);
assert_eq!(
DeployError::FleetMaintenanceFailed {
verb: "on",
failed: 1,
total: 3,
}
.to_string(),
"fleet maintenance on incomplete: 1 of 3 host(s) failed \u{2014} the table above \
names the hosts that DID change, so they can be reversed by hand if the fleet \
must stay consistent",
);
for rendered in [
DeployError::DriftDetected.to_string(),
DeployError::FleetMaintenanceFailed {
verb: "off",
failed: 2,
total: 5,
}
.to_string(),
] {
assert!(
!rendered.contains(" "),
"a run of spaces mid-sentence means a `\\` continuation was dropped: {rendered}",
);
}
}
fn bake_policy() -> bake::BakePolicy {
bake::BakePolicy {
duration_secs: 20,
interval_secs: 10,
min_requests: 20,
max_error_ppm: 14_400,
latency: Vec::new(),
source: "test".to_owned(),
warnings: Vec::new(),
}
}
fn bake_metrics(ok: u64, errors: u64) -> String {
format!(
r#"{{"http":{{"latency_ms":{{"p50":1,"p95":2,"p99":3}},
"by_status":{{"2xx":{ok},"3xx":0,"4xx":0,"5xx":{errors}}}}}}}
{}
0
"#,
bake::RESTARTS_MARKER
)
}
fn script_bake(
mut recorder: fleet::test_support::FleetRecorder,
host: &str,
samples: &[(u64, u64)],
) -> fleet::test_support::FleetRecorder {
for (index, (ok, errors)) in samples.iter().enumerate() {
recorder = recorder.script_occurrence(
host,
bake::SAMPLE_LABEL,
index + 1,
bake_metrics(*ok, *errors),
);
}
recorder
}
const HEALTHY_BAKE: [(u64, u64); 3] = [(10, 0), (110, 0), (210, 1)];
const SPIKED_BAKE: [(u64, u64); 3] = [(10, 0), (110, 0), (170, 40)];
#[test]
fn an_error_spike_during_the_bake_rolls_a_single_host_back() {
let fleet = fleet_of(&["web-a"]);
let recorder = script_redeploy(fleet::test_support::FleetRecorder::new(), "web-a");
let recorder = script_compensation(recorder, "web-a", "present");
let recorder = script_bake(recorder, "web-a", &SPIKED_BAKE);
let fixture = FleetFixture::new();
let policy = bake_policy();
let input = FleetUpInput {
bake: Some(&policy),
..fixture.input(&fleet)
};
let err = run_up_with(&input, |cfg| Ok(recorder.executor(cfg)))
.expect_err("a breached bake must fail the deploy");
let DeployError::BakeFailed {
host,
reason,
result,
schema_note,
} = &err
else {
panic!("expected BakeFailed, got {err:?}");
};
assert_eq!(host, "web-a");
assert!(reason.contains("40 of 198 responses were 5xx"), "{reason}");
assert_eq!(*result, BAKE_ROLLED_BACK);
assert_eq!(*schema_note, Some(fleet::SINGLE_HOST_SCHEMA_AHEAD_NOTE));
assert!(err.to_string().contains("was NOT rolled back"), "{err}");
let flip = recorder.index_of("web-a", "proxy-flip").expect("cut over");
let first_sample = recorder
.index_of("web-a", bake::SAMPLE_LABEL)
.expect("the bake sampled");
let restart = recorder
.index_of("web-a", "restart-previous")
.expect("the host must be rolled back");
assert!(flip < first_sample && first_sample < restart);
let labels = recorder.run_labels_for("web-a");
assert_eq!(
labels.iter().filter(|l| **l == bake::SAMPLE_LABEL).count(),
3,
"the bake stops at the breach: {labels:?}"
);
}
#[test]
fn a_bake_rollback_during_a_port_move_derives_the_legacy_port_from_the_old_port() {
let fleet = fleet_of(&["web-a"]);
let probe = "redeploy:blue\t81\n\
---autumn-kamal-proxy-list---\n\
---autumn-kamal-proxy-unit---\n--http-port 80\n"
.to_owned();
let recorder = fleet::test_support::FleetRecorder::new()
.script("web-a", "proxy-compat-probe", compatible_deploy_help())
.script("web-a", "detect-current", probe)
.script("web-a", "probe-release-dir", "absent")
.script(
"web-a",
"resolve-previous",
format!("prev:{PREVIOUS_RELEASE_DIR}\tblue"),
)
.script("web-a", "probe-rollback-target", "present");
let recorder = script_bake(recorder, "web-a", &SPIKED_BAKE);
let fixture = FleetFixture::new();
let policy = bake_policy();
let input = FleetUpInput {
bake: Some(&policy),
..fixture.input(&fleet)
};
let err = run_up_with(&input, |cfg| Ok(recorder.executor(cfg)))
.expect_err("a breached bake must fail the deploy");
let DeployError::BakeFailed { result, .. } = &err else {
panic!("expected BakeFailed, got {err:?}");
};
assert_eq!(*result, BAKE_ROLLED_BACK);
let restart = recorder
.index_of("web-a", "restart-previous")
.expect("rolled back");
let rollback: Vec<String> = recorder.calls_for("web-a")[restart..]
.iter()
.filter_map(|call| match call {
exec::test_support::RecordedCall::Run { shell, .. } => Some(shell.clone()),
exec::test_support::RecordedCall::Upload { .. } => None,
})
.collect();
assert!(
rollback.iter().any(|shell| shell.contains(":81")),
"the rollback must target blue on the old port: {rollback:#?}"
);
assert!(
rollback.iter().all(|shell| !shell.contains(":3001")),
"the rollback must not derive the port from the new server.port: {rollback:#?}"
);
}
#[test]
fn a_healthy_bake_completes_the_deploy() {
let fleet = fleet_of(&["web-a"]);
let recorder = script_redeploy(fleet::test_support::FleetRecorder::new(), "web-a");
let recorder = script_bake(recorder, "web-a", &HEALTHY_BAKE);
let fixture = FleetFixture::new();
let policy = bake_policy();
let input = FleetUpInput {
bake: Some(&policy),
..fixture.input(&fleet)
};
run_up_with(&input, |cfg| Ok(recorder.executor(cfg))).expect("a healthy bake passes");
assert!(recorder.index_of("web-a", "restart-previous").is_none());
let samples = recorder
.run_labels_for("web-a")
.into_iter()
.filter(|l| *l == bake::SAMPLE_LABEL)
.count();
assert_eq!(samples, 3);
}
#[test]
fn the_bake_samples_the_new_release_on_its_slot_port() {
let fleet = fleet_of(&["web-a"]);
let recorder = script_redeploy(fleet::test_support::FleetRecorder::new(), "web-a");
let recorder = script_bake(recorder, "web-a", &HEALTHY_BAKE);
let fixture = FleetFixture::new();
let policy = bake_policy();
let input = FleetUpInput {
bake: Some(&policy),
metrics_path: "/ops/metrics",
..fixture.input(&fleet)
};
run_up_with(&input, |cfg| Ok(recorder.executor(cfg))).expect("passes");
let shell = recorder
.calls_for("web-a")
.into_iter()
.find_map(|call| match call {
exec::test_support::RecordedCall::Run { label, shell }
if label == bake::SAMPLE_LABEL =>
{
Some(shell)
}
_ => None,
})
.expect("a sample");
assert!(
shell.contains("'http://127.0.0.1:3002/ops/metrics'"),
"{shell}"
);
assert!(shell.contains("-H 'Host: app.example.com'"), "{shell}");
assert!(shell.contains("'myapp-green.service'"), "{shell}");
}
#[test]
fn a_bake_breach_halts_the_fleet_and_rolls_back_every_host_on_the_new_release() {
let hosts = ["web-a", "web-b", "web-c"];
let fleet = fleet_of(&hosts);
let mut recorder = fleet::test_support::FleetRecorder::new();
for host in hosts {
recorder = script_redeploy(recorder, host);
}
recorder = script_compensation(recorder, "web-a", "present");
recorder = script_compensation(recorder, "web-b", "present");
recorder = script_bake(recorder, "web-a", &HEALTHY_BAKE);
recorder = script_bake(recorder, "web-b", &SPIKED_BAKE);
let fixture = FleetFixture::new();
let policy = bake_policy();
let input = FleetUpInput {
bake: Some(&policy),
..fixture.input(&fleet)
};
let err = run_up_with(&input, |cfg| Ok(recorder.executor(cfg)))
.expect_err("a breached bake must halt the rollout");
let halt = fleet_halt_of(&err);
assert_eq!(halt.failed_host, "web-b");
assert_eq!(halt.failed_step, bake::BAKE_LABEL);
assert_eq!(
halt.rolled_back,
vec!["web-a".to_owned(), "web-b".to_owned()]
);
assert!(halt.still_on_new.is_empty(), "{:?}", halt.still_on_new);
let web_b = recorder
.index_of("web-b", "restart-previous")
.expect("web-b");
let web_a = recorder
.index_of("web-a", "restart-previous")
.expect("web-a");
assert!(web_b < web_a, "newest first");
assert!(
recorder
.run_labels_for("web-c")
.iter()
.all(|l| !l.starts_with("start-")),
"web-c must not be touched"
);
}
#[test]
fn no_rollback_leaves_a_failed_bake_live_and_says_so() {
let fleet = fleet_of(&["web-a"]);
let recorder = script_redeploy(fleet::test_support::FleetRecorder::new(), "web-a");
let recorder = script_bake(recorder, "web-a", &SPIKED_BAKE);
let fixture = FleetFixture::new();
let policy = bake_policy();
let input = FleetUpInput {
bake: Some(&policy),
..fixture.input_frozen(&fleet)
};
let err = run_up_with(&input, |cfg| Ok(recorder.executor(cfg))).expect_err("fails");
assert!(
matches!(err, DeployError::BakeFailed { result, .. } if result == BAKE_FROZEN),
"{err:?}"
);
assert!(recorder.index_of("web-a", "restart-previous").is_none());
}
#[test]
fn a_failed_automatic_rollback_after_the_bake_is_reported() {
let fleet = fleet_of(&["web-a"]);
let recorder = script_redeploy(fleet::test_support::FleetRecorder::new(), "web-a");
let recorder =
script_compensation(recorder, "web-a", "present").fail("web-a", "restart-previous");
let recorder = script_bake(recorder, "web-a", &SPIKED_BAKE);
let fixture = FleetFixture::new();
let policy = bake_policy();
let input = FleetUpInput {
bake: Some(&policy),
..fixture.input(&fleet)
};
let err = run_up_with(&input, |cfg| Ok(recorder.executor(cfg))).expect_err("fails");
assert!(
matches!(err, DeployError::BakeFailed { result, schema_note: None, .. } if result == BAKE_ROLLBACK_FAILED),
"{err:?}"
);
}
#[test]
fn an_unsafe_rollback_after_the_bake_is_refused_and_reported() {
let fleet = fleet_of(&["web-a"]);
let recorder = script_redeploy(fleet::test_support::FleetRecorder::new(), "web-a");
let recorder = script_compensation(recorder, "web-a", "absent");
let recorder = script_bake(recorder, "web-a", &SPIKED_BAKE);
let fixture = FleetFixture::new();
let policy = bake_policy();
let input = FleetUpInput {
bake: Some(&policy),
..fixture.input(&fleet)
};
let err = run_up_with(&input, |cfg| Ok(recorder.executor(cfg))).expect_err("fails");
assert!(
matches!(err, DeployError::BakeFailed { result, .. } if result == BAKE_ROLLBACK_REFUSED),
"{err:?}"
);
assert!(recorder.index_of("web-a", "restart-previous").is_none());
}
#[test]
fn no_rollback_freezes_a_fleet_whose_bake_failed() {
let hosts = ["web-a", "web-b", "web-c"];
let fleet = fleet_of(&hosts);
let mut recorder = fleet::test_support::FleetRecorder::new();
for host in hosts {
recorder = script_redeploy(recorder, host);
}
recorder = script_bake(recorder, "web-a", &HEALTHY_BAKE);
recorder = script_bake(recorder, "web-b", &SPIKED_BAKE);
let fixture = FleetFixture::new();
let policy = bake_policy();
let input = FleetUpInput {
bake: Some(&policy),
..fixture.input_frozen(&fleet)
};
let err = run_up_with(&input, |cfg| Ok(recorder.executor(cfg))).expect_err("halts");
let halt = fleet_halt_of(&err);
assert_eq!(halt.failed_host, "web-b");
assert_eq!(halt.failed_step, bake::BAKE_LABEL);
assert!(halt.rolled_back.is_empty(), "{:?}", halt.rolled_back);
assert_eq!(
halt.still_on_new,
vec!["web-a".to_owned(), "web-b".to_owned()]
);
assert!(recorder.positions_of("restart-previous").is_empty());
}
#[test]
fn a_host_degraded_by_a_port_rebind_still_bakes() {
let fleet = fleet_of(&["web-a", "web-b"]);
let probe_a = "redeploy:blue\t81\n\
---autumn-kamal-proxy-list---\n\
---autumn-kamal-proxy-unit---\n--http-port 80\n"
.to_owned();
let recorder = fleet::test_support::FleetRecorder::new()
.script("web-a", "proxy-compat-probe", compatible_deploy_help())
.script("web-a", "detect-current", probe_a)
.script("web-a", "probe-release-dir", "absent")
.fail_on_occurrence("web-a", "proxy-restart-if-changed", 2);
let recorder = script_compensation(recorder, "web-a", "present");
let recorder = script_bake(recorder, "web-a", &SPIKED_BAKE);
let recorder = script_redeploy(recorder, "web-b");
let fixture = FleetFixture::new();
let policy = bake_policy();
let input = FleetUpInput {
bake: Some(&policy),
..fixture.input(&fleet)
};
let err = run_up_with(&input, |cfg| Ok(recorder.executor(cfg))).expect_err("halts");
let halt = fleet_halt_of(&err);
assert_eq!(halt.failed_host, "web-a");
assert_eq!(halt.failed_step, bake::BAKE_LABEL);
assert!(recorder.index_of("web-a", bake::SAMPLE_LABEL).is_some());
assert!(
recorder
.run_labels_for("web-b")
.iter()
.all(|l| !l.starts_with("start-"))
);
}
#[test]
fn a_degraded_host_still_bakes() {
let hosts = ["web-a", "web-b"];
let fleet = fleet_of(&hosts);
let mut recorder = fleet::test_support::FleetRecorder::new();
for host in hosts {
recorder = script_redeploy(recorder, host);
}
recorder =
script_compensation(recorder, "web-a", "present").fail("web-a", "record-proxy-options");
recorder = script_bake(recorder, "web-a", &SPIKED_BAKE);
let fixture = FleetFixture::new();
let policy = bake_policy();
let input = FleetUpInput {
bake: Some(&policy),
..fixture.input(&fleet)
};
let err = run_up_with(&input, |cfg| Ok(recorder.executor(cfg))).expect_err("halts");
let halt = fleet_halt_of(&err);
assert_eq!(halt.failed_host, "web-a");
assert_eq!(halt.failed_step, bake::BAKE_LABEL);
assert_eq!(halt.rolled_back, vec!["web-a".to_owned()]);
assert!(
halt.degraded
.iter()
.any(|(h, l)| h == "web-a" && *l == "record-proxy-options"),
"the debris is still reported: {:?}",
halt.degraded
);
assert!(
recorder
.run_labels_for("web-b")
.iter()
.all(|l| !l.starts_with("start-"))
);
}
#[test]
fn a_first_deploy_skips_the_bake() {
let fleet = fleet_of(&["web-a"]);
let recorder = script_first_deploy(fleet::test_support::FleetRecorder::new(), "web-a");
let fixture = FleetFixture::new();
let policy = bake_policy();
let input = FleetUpInput {
bake: Some(&policy),
..fixture.input(&fleet)
};
run_up_with(&input, |cfg| Ok(recorder.executor(cfg))).expect("first deploy passes");
assert!(recorder.index_of("web-a", bake::SAMPLE_LABEL).is_none());
}
#[test]
fn the_bake_in_prod_needs_a_trusted_host() {
let mut config = AutumnConfig {
profile: Some("prod".to_owned()),
..AutumnConfig::default()
};
assert!(matches!(
resolve_bake_policy(&config, Some(30)),
Err(DeployError::Config(message)) if message.contains("[security.trusted_hosts]")
));
assert!(resolve_bake_policy(&config, None).expect("off").is_none());
config.security.trusted_hosts.hosts = vec!["app.example.com".to_owned()];
assert!(
resolve_bake_policy(&config, Some(30))
.expect("ok")
.is_some()
);
config.security.trusted_hosts.hosts = vec!["*".to_owned()];
assert!(
resolve_bake_policy(&config, Some(30))
.expect("ok")
.is_some()
);
}
#[test]
fn a_degraded_single_host_bakes_and_reports_the_bake() {
let fleet = fleet_of(&["web-a"]);
let recorder = script_redeploy(fleet::test_support::FleetRecorder::new(), "web-a")
.fail("web-a", "record-proxy-options");
let recorder = script_compensation(recorder, "web-a", "present");
let recorder = script_bake(recorder, "web-a", &SPIKED_BAKE);
let fixture = FleetFixture::new();
let policy = bake_policy();
let input = FleetUpInput {
bake: Some(&policy),
..fixture.input(&fleet)
};
let err = run_up_with(&input, |cfg| Ok(recorder.executor(cfg))).expect_err("fails");
assert!(
matches!(err, DeployError::BakeFailed { result, .. } if result == BAKE_ROLLED_BACK),
"{err:?}"
);
assert!(recorder.index_of("web-a", "restart-previous").is_some());
}
#[test]
fn a_degraded_single_host_with_a_good_bake_keeps_the_housekeeping_error() {
let fleet = fleet_of(&["web-a"]);
let recorder = script_redeploy(fleet::test_support::FleetRecorder::new(), "web-a")
.fail("web-a", "record-proxy-options");
let recorder = script_bake(recorder, "web-a", &HEALTHY_BAKE);
let fixture = FleetFixture::new();
let policy = bake_policy();
let input = FleetUpInput {
bake: Some(&policy),
..fixture.input(&fleet)
};
let err = run_up_with(&input, |cfg| Ok(recorder.executor(cfg))).expect_err("fails");
assert!(matches!(err, DeployError::Exec(_)), "{err:?}");
assert!(recorder.index_of("web-a", bake::SAMPLE_LABEL).is_some());
assert!(recorder.index_of("web-a", "restart-previous").is_none());
}
#[test]
fn the_bake_policy_comes_from_config_slos_and_the_flag() {
let mut config = AutumnConfig::default();
assert!(resolve_bake_policy(&config, None).expect("ok").is_none());
let policy = resolve_bake_policy(&config, Some(30))
.expect("ok")
.expect("on");
assert_eq!(policy.duration_secs, 30);
assert_eq!(policy.max_error_ppm, bake::DEFAULT_MAX_ERROR_PPM);
config.slo = vec![autumn_web::slo::SloConfig {
name: "availability".to_owned(),
objective: 99.9,
sli: autumn_web::slo::SliKind::Availability,
route: None,
threshold_ms: None,
description: None,
}];
let policy = resolve_bake_policy(&config, Some(30))
.expect("ok")
.expect("on");
assert_eq!(policy.max_error_ppm, 14_400);
config.slo[0].objective = 100.0;
assert!(resolve_bake_policy(&config, None).expect("off").is_none());
assert!(matches!(
resolve_bake_policy(&config, Some(30)),
Err(DeployError::Config(message)) if message.contains("[[slo]] availability")
));
}
}