use std::path::{Path, PathBuf};
use serde::Serialize;
use std::time::Duration;
use crate::actions::deploy::{
Declared, DeployPlanResult, PlanOptions, VerifyOptions, deploy_plan, load_target,
};
use crate::client::{DeployApi, GatewayApi};
use crate::deploy::apply::gates::{
GateFlags, GateOutcome, Redundancy, evaluate_gates, validate_drift_reason, validate_plan_sha256,
};
use crate::deploy::apply::hold::{HoldModel, SystemClock};
use crate::deploy::apply::host::ApplyHost;
use crate::deploy::apply::journal::RecoveryAction;
use crate::deploy::apply::preflight::DesignersPolicy;
use crate::deploy::apply::record::RunRecord;
use crate::deploy::apply::run_store::RunId;
use crate::deploy::apply::{
ApplyContext, ApplyOptions, BACKUP_UNSYNCED_SLUG, InterruptedRun, OutcomeKind,
ROLLED_BACK_SLUG, RollbackCause, RollbackChecked, RollbackContext, RollbackKind,
RollbackOptions, check_rollback, restore_set, run_apply, run_rollback,
};
use crate::deploy::apply::{ApplyObserver, ApplyOptions as EngineOptions};
use crate::deploy::pair::{BACKUP_POLL_S, BackupNode, BackupPolicy, BackupReport};
use crate::deploy::plan::PlanSummary;
use crate::deploy::target::Target;
use crate::deploy::transport::TreeReader;
use crate::deploy::transport::apply_host_for_node;
use crate::error::CoreError;
use crate::poll::PollConfig;
#[derive(Debug, Clone, Default)]
pub struct PrepareOptions {
pub base: Option<PathBuf>,
pub gates: GateFlags,
}
#[derive(Debug, Clone, PartialEq)]
pub enum ApplyPrepared {
Noop(DeployApplyResult),
Ready {
plan: DeployPlanResult,
prose: String,
},
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct DeployApplyResult {
pub env: String,
pub gateway: String,
pub mode: String,
pub node: String,
pub transport: String,
pub run_id: Option<String>,
pub run_dir: Option<String>,
pub outcome: OutcomeKind,
pub ok: bool,
pub slug: Option<&'static str>,
pub cause: Option<RollbackCause>,
pub plan_sha256: String,
pub render_sha256: String,
pub summary: PlanSummary,
pub written: Vec<String>,
pub deleted: Vec<String>,
pub warnings: Vec<String>,
pub excluded_tag_providers: Vec<String>,
pub gwbk: Option<String>,
pub skip_gwbk: bool,
pub restart_pending: Vec<String>,
pub restart_pending_new: Vec<String>,
pub backup: Option<Box<BackupReport>>,
}
impl DeployApplyResult {
pub fn failure_exit_code(&self) -> Option<u8> {
matches!(
self.outcome,
OutcomeKind::RolledBack | OutcomeKind::AppliedBackupUnsynced
)
.then_some(6)
}
}
fn base_result(plan: &DeployPlanResult, outcome: OutcomeKind) -> DeployApplyResult {
DeployApplyResult {
env: plan.env.clone(),
gateway: plan.gateway.clone(),
mode: plan.mode.clone(),
node: plan.node.clone(),
transport: plan.transport.clone(),
run_id: None,
run_dir: None,
outcome,
ok: !matches!(
outcome,
OutcomeKind::RolledBack | OutcomeKind::AppliedBackupUnsynced
),
slug: match outcome {
OutcomeKind::RolledBack => Some(ROLLED_BACK_SLUG),
OutcomeKind::AppliedBackupUnsynced => Some(BACKUP_UNSYNCED_SLUG),
OutcomeKind::Applied | OutcomeKind::Noop => None,
},
cause: None,
plan_sha256: plan.plan_sha256.clone(),
render_sha256: plan.render_sha256.clone(),
summary: plan.summary.clone(),
written: Vec::new(),
deleted: Vec::new(),
warnings: plan.warnings.clone(),
excluded_tag_providers: plan.excluded_tag_providers.clone(),
gwbk: None,
skip_gwbk: false,
restart_pending: Vec::new(),
restart_pending_new: Vec::new(),
backup: None,
}
}
pub async fn deploy_apply_prepare(
declared: &Declared,
reader: &dyn TreeReader,
opts: &PrepareOptions,
) -> Result<ApplyPrepared, CoreError> {
if let Some(hash) = &opts.gates.plan_sha256 {
validate_plan_sha256(hash)?;
}
if let Some(reason) = &opts.gates.allow_drift {
validate_drift_reason(reason)?;
}
let plan = deploy_plan(
declared,
reader,
&PlanOptions {
base: opts.base.clone(),
diff: false,
},
)
.await?;
match evaluate_gates(&plan, &opts.gates, Redundancy::of(&declared.target))? {
GateOutcome::Noop => Ok(ApplyPrepared::Noop(base_result(&plan, OutcomeKind::Noop))),
GateOutcome::Ready { prose } => Ok(ApplyPrepared::Ready { plan, prose }),
}
}
pub async fn deploy_apply_execute(
prepared: ApplyPrepared,
ctx: &ApplyContext<'_>,
opts: &ApplyOptions,
) -> Result<DeployApplyResult, CoreError> {
let plan = match prepared {
ApplyPrepared::Noop(result) => return Ok(result),
ApplyPrepared::Ready { plan, .. } => plan,
};
let outcome = run_apply(ctx, &plan, opts).await?;
let mut result = base_result(&plan, outcome.kind);
result.run_id = outcome.run_id;
result.run_dir = outcome.run_dir;
result.cause = outcome.cause;
result.written = outcome.written;
result.deleted = outcome.deleted;
result.warnings.extend(outcome.warnings);
result.gwbk = outcome.gwbk;
result.skip_gwbk = outcome.skip_gwbk;
result.restart_pending = outcome.restart_pending;
result.restart_pending_new = outcome.restart_pending_new;
result.backup = outcome.backup.map(Box::new);
Ok(result)
}
pub struct ApplySurfaces<'a> {
pub deploy: &'a dyn DeployApi,
pub backup: Option<&'a BackupNode<'a>>,
pub gateway: &'a dyn GatewayApi,
}
#[derive(Debug, Clone)]
pub struct ApplyRunOptions {
pub base: Option<PathBuf>,
pub gates: GateFlags,
pub project: String,
pub designers: DesignersPolicy,
pub skip_gwbk: bool,
pub backup_timeout: Duration,
}
const SCAN_POLL_INTERVAL: Duration = Duration::from_millis(500);
const SCAN_WAIT: Duration = Duration::from_secs(120);
fn scan_poll(subject: &str) -> PollConfig {
PollConfig {
subject: subject.to_string(),
interval: SCAN_POLL_INTERVAL,
max: Duration::from_secs(2),
deadline: SCAN_WAIT,
}
}
pub async fn deploy_apply_run(
prepared: ApplyPrepared,
declared: &Declared,
repo: &Path,
reader: &dyn TreeReader,
surfaces: &ApplySurfaces<'_>,
opts: &ApplyRunOptions,
observer: &dyn ApplyObserver,
) -> Result<DeployApplyResult, CoreError> {
deploy_apply_run_in(
prepared, declared, repo, repo, reader, surfaces, opts, observer,
)
.await
}
#[allow(clippy::too_many_arguments)]
pub async fn deploy_apply_run_in(
prepared: ApplyPrepared,
declared: &Declared,
repo: &Path,
targets: &Path,
reader: &dyn TreeReader,
surfaces: &ApplySurfaces<'_>,
opts: &ApplyRunOptions,
observer: &dyn ApplyObserver,
) -> Result<DeployApplyResult, CoreError> {
if let ApplyPrepared::Noop(result) = prepared {
return Ok(result);
}
let host = apply_host_for_node(
targets,
&declared.target_file,
"nodes.master",
&declared.target.nodes.master,
&declared.target.gateway,
&declared.target.mode,
)?;
let ctx = ApplyContext {
declared,
repo,
reader,
host: host.as_ref(),
deploy: surfaces.deploy,
gateway: surfaces.gateway,
backup: surfaces.backup,
clock: &SystemClock,
observer,
};
let engine = EngineOptions {
base: opts.base.clone(),
gates: opts.gates.clone(),
project: opts.project.clone(),
hold: HoldModel::MEASURED,
poll: scan_poll("config scan after apply"),
verify: VerifyOptions::default(),
designers: opts.designers,
skip_gwbk: opts.skip_gwbk,
backup: BackupPolicy {
budget: opts.backup_timeout,
interval: Duration::from_secs(BACKUP_POLL_S),
},
};
deploy_apply_execute(prepared, &ctx, &engine).await
}
#[derive(Debug, Clone, Copy, Default)]
pub struct RollbackPrepareOptions {
pub discard_live_changes: bool,
}
#[derive(Debug, Clone)]
pub struct RollbackPrepared {
pub run_id: String,
pub record: Option<RunRecord>,
pub interrupted: Option<InterruptedRun>,
pub prose: String,
pub live_changed: Vec<String>,
pub target: Target,
pub target_file: String,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct DeployRollbackResult {
pub env: String,
pub gateway: String,
pub mode: String,
pub run_id: String,
pub target_run_id: String,
pub run_dir: String,
pub outcome: RollbackKind,
pub ok: bool,
pub restored: Vec<String>,
pub deleted: Vec<String>,
pub discarded_live_changes: Vec<String>,
pub warnings: Vec<String>,
pub restart_pending: Vec<String>,
pub restart_pending_new: Vec<String>,
pub backup: Option<crate::deploy::pair::BackupReport>,
}
fn rollback_prose(run_id: &str, target: &Target, checked: &RollbackChecked) -> String {
let record = checked.record.as_ref();
let (restored, deleted) = record.map(restore_set).unwrap_or_default();
let ledger = record
.and_then(|r| {
r.touched
.iter()
.find(|e| e.path == crate::deploy::apply::fingerprint::LEDGER_KEY)
})
.map(|e| {
if e.pre_sha256.is_some() {
", and restore the ledger"
} else {
", and remove the ledger"
}
})
.unwrap_or("");
let mut prose = match (&checked.interrupted, record) {
(Some(found), _) if found.action == RecoveryAction::ClearWithoutRestore => format!(
"Clear run {run_id} on gateway {} (mode {}), which was interrupted at phase {} \
before any file moved and whose touched files are as they were before the run: \
remove its journal, partial snapshot and staging. Nothing is restored.",
target.gateway,
target.mode,
found.journal.phase.as_str(),
),
(Some(found), Some(record)) => format!(
"Recover run {run_id} on gateway {} (mode {}), which was interrupted at phase {}: \
restore {} file(s) from the on-box snapshot and delete {} file(s) the run \
created{ledger}, then remove the journal. The run was applying plan_sha256 {}.",
target.gateway,
target.mode,
found.journal.phase.as_str(),
restored.len(),
deleted.len(),
record.plan_sha256,
),
(Some(found), None) => format!(
"Recover run {run_id} on gateway {} (mode {}), which was interrupted at phase {} \
and whose record is missing: restore the touched files from the on-box snapshot, \
then remove the journal.",
target.gateway,
target.mode,
found.journal.phase.as_str(),
),
(None, Some(record)) => format!(
"Roll back run {run_id} on gateway {} (mode {}): restore {} file(s) from the on-box \
snapshot and delete {} file(s) the run created{ledger}. The run applied \
plan_sha256 {}.",
target.gateway,
target.mode,
restored.len(),
deleted.len(),
record.plan_sha256,
),
(None, None) => format!("Roll back run {run_id} on gateway {}.", target.gateway),
};
if !checked.live_changed.is_empty() {
prose.push_str(&format!(
" {} file(s) changed after the run and will be overwritten: {}.",
checked.live_changed.len(),
checked
.live_changed
.iter()
.take(crate::error::APPLY_LIST_SHOWN)
.cloned()
.collect::<Vec<_>>()
.join(", ")
));
}
prose
}
pub async fn deploy_rollback_prepare(
repo: &Path,
env: &str,
run_id_text: &str,
reader: &dyn TreeReader,
host: &dyn ApplyHost,
opts: &RollbackPrepareOptions,
) -> Result<RollbackPrepared, CoreError> {
deploy_rollback_prepare_in(repo, repo, env, run_id_text, reader, host, opts).await
}
pub async fn deploy_rollback_prepare_in(
repo: &Path,
targets: &Path,
env: &str,
run_id_text: &str,
reader: &dyn TreeReader,
host: &dyn ApplyHost,
opts: &RollbackPrepareOptions,
) -> Result<RollbackPrepared, CoreError> {
let run_id = RunId::parse(run_id_text)?;
let (target_file, target) = load_target(targets, env)?;
let checked = check_rollback(
env,
repo,
&target,
reader,
host,
&run_id,
opts.discard_live_changes,
)
.await?;
Ok(RollbackPrepared {
prose: rollback_prose(run_id.as_str(), &target, &checked),
run_id: run_id.as_str().to_string(),
record: checked.record,
interrupted: checked.interrupted,
live_changed: checked.live_changed,
target,
target_file,
})
}
pub async fn deploy_rollback_execute(
prepared: RollbackPrepared,
ctx: &RollbackContext<'_>,
opts: &RollbackOptions,
) -> Result<DeployRollbackResult, CoreError> {
if prepared.run_id != opts.run_id.as_str() {
return Err(CoreError::Internal(
"the prepared rollback is for another run than the options name".to_string(),
));
}
let outcome = run_rollback(ctx, opts).await?;
Ok(DeployRollbackResult {
env: ctx.env.to_string(),
gateway: ctx.target.gateway.clone(),
mode: ctx.target.mode.clone(),
run_id: outcome.run_id,
target_run_id: outcome.target_run_id,
run_dir: outcome.run_dir,
outcome: outcome.kind,
ok: true,
restored: outcome.restored,
deleted: outcome.deleted,
discarded_live_changes: outcome.discarded_live_changes,
warnings: outcome.warnings,
restart_pending: outcome.restart_pending,
restart_pending_new: outcome.restart_pending_new,
backup: outcome.backup,
})
}
pub async fn deploy_rollback_prepare_for_env(
repo: &Path,
env: &str,
run_id_text: &str,
reader: &dyn TreeReader,
opts: &RollbackPrepareOptions,
) -> Result<RollbackPrepared, CoreError> {
deploy_rollback_prepare_for_env_in(repo, repo, env, run_id_text, reader, opts).await
}
pub async fn deploy_rollback_prepare_for_env_in(
repo: &Path,
targets: &Path,
env: &str,
run_id_text: &str,
reader: &dyn TreeReader,
opts: &RollbackPrepareOptions,
) -> Result<RollbackPrepared, CoreError> {
RunId::parse(run_id_text)?;
let (target_file, target) = load_target(targets, env)?;
let host = apply_host_for_node(
targets,
&target_file,
"nodes.master",
&target.nodes.master,
&target.gateway,
&target.mode,
)?;
deploy_rollback_prepare_in(repo, targets, env, run_id_text, reader, host.as_ref(), opts).await
}
#[derive(Debug, Clone, Copy, Default)]
pub struct RollbackRunOptions {
pub discard_live_changes: bool,
pub designers: DesignersPolicy,
pub backup: crate::deploy::pair::BackupPolicy,
}
pub async fn deploy_rollback_run(
prepared: RollbackPrepared,
env: &str,
repo: &Path,
reader: &dyn TreeReader,
surfaces: &ApplySurfaces<'_>,
opts: &RollbackRunOptions,
observer: &dyn ApplyObserver,
) -> Result<DeployRollbackResult, CoreError> {
deploy_rollback_run_in(prepared, env, repo, repo, reader, surfaces, opts, observer).await
}
#[allow(clippy::too_many_arguments)]
pub async fn deploy_rollback_run_in(
prepared: RollbackPrepared,
env: &str,
repo: &Path,
targets: &Path,
reader: &dyn TreeReader,
surfaces: &ApplySurfaces<'_>,
opts: &RollbackRunOptions,
observer: &dyn ApplyObserver,
) -> Result<DeployRollbackResult, CoreError> {
let host = apply_host_for_node(
targets,
&prepared.target_file,
"nodes.master",
&prepared.target.nodes.master,
&prepared.target.gateway,
&prepared.target.mode,
)?;
let run_id = RunId::parse(&prepared.run_id)?;
let target = prepared.target.clone();
let target_file = prepared.target_file.clone();
let ctx = RollbackContext {
env,
repo,
target_file: &target_file,
target: &target,
reader,
host: host.as_ref(),
deploy: surfaces.deploy,
gateway: surfaces.gateway,
backup: surfaces.backup,
clock: &SystemClock,
observer,
};
let engine = RollbackOptions {
run_id,
discard_live_changes: opts.discard_live_changes,
designers: opts.designers,
hold: HoldModel::MEASURED,
poll: scan_poll("config scan after rollback"),
backup: opts.backup,
};
deploy_rollback_execute(prepared, &ctx, &engine).await
}