use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};
use serde_json::json;
use crate::agentgraph::{self, Interrupted, TurnAddress};
use crate::driver;
use crate::engine::CANCEL_INPUT;
use crate::error::{Error, Result, EXIT_REFUSED, EXIT_SUCCESS};
use crate::event::{Envelope, Phase, Source};
use crate::journal::{self, Journal, StopTeardown};
use crate::ledger::{self, DispatchRecord, RunPaths};
use crate::sys;
use crate::views::{RunView, Survey};
const MARKER: &str = "shutting-down.json";
const POLL: Duration = Duration::from_millis(50);
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub(crate) enum Answer {
NoTurn,
Failed,
Delivered,
NotAsked,
}
impl Answer {
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::Delivered => "delivered",
Self::NoTurn => "no-turn",
Self::Failed => "failed",
Self::NotAsked => "not-asked",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ShutdownScope {
Run(String),
Mine,
Host,
}
impl ShutdownScope {
pub(crate) const fn as_str(&self) -> &'static str {
match self {
Self::Run(_) => "run",
Self::Mine => "mine",
Self::Host => "host",
}
}
const fn keeps_the_ownership_rule(&self) -> bool {
!matches!(self, Self::Host)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ShutdownRequest {
pub scope: ShutdownScope,
pub session: String,
pub grace: Duration,
pub force: bool,
}
impl ShutdownRequest {
fn asks(&self) -> bool {
!self.force && !self.grace.is_zero()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DispatchEnding {
Graceful,
Killed,
StillRunning,
}
impl DispatchEnding {
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::Graceful => "graceful",
Self::Killed => "killed",
Self::StillRunning => "still-running",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Preserved {
Pushed,
AlreadyOnOrigin,
NoRemote,
Refused,
}
impl Preserved {
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::Pushed => "pushed",
Self::AlreadyOnOrigin => "already-on-origin",
Self::NoRemote => "no-remote",
Self::Refused => "refused",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct DispatchStopped {
pub node: String,
pub pid: u32,
pub interrupt: String,
pub detail: String,
pub ended: DispatchEnding,
pub waited: Duration,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BranchPreserved {
pub identity: String,
pub branch: String,
pub outcome: Preserved,
pub remote: Option<String>,
pub commit: Option<String>,
pub detail: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RunShutdown {
pub run: String,
pub owner: String,
pub forced_over_owner: bool,
pub dispatches: Vec<DispatchStopped>,
pub teardown: StopTeardown,
pub branches: Vec<BranchPreserved>,
}
impl RunShutdown {
fn clean(&self) -> bool {
matches!(
self.teardown,
StopTeardown::Signalled | StopTeardown::NothingToStop | StopTeardown::Elsewhere
) && self.dispatches.iter().all(|stopped| match stopped.ended {
DispatchEnding::Graceful => true,
DispatchEnding::Killed => stopped.interrupt == Answer::NotAsked.as_str(),
DispatchEnding::StillRunning => false,
}) && self
.branches
.iter()
.all(|branch| branch.outcome != Preserved::Refused)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Shutdown {
pub root: PathBuf,
pub scope: ShutdownScope,
pub grace: Duration,
pub forced: bool,
pub runs: Vec<RunShutdown>,
pub not_pushed: Vec<(String, String)>,
pub not_pushed_unread: Option<String>,
}
impl Shutdown {
pub fn exit_code(&self) -> i32 {
if self.runs.iter().all(RunShutdown::clean) {
EXIT_SUCCESS
} else {
EXIT_REFUSED
}
}
}
pub(crate) fn begun(paths: &RunPaths) -> bool {
marker(paths).try_exists().unwrap_or(true)
}
pub(crate) fn adopted(paths: &RunPaths) -> Result<()> {
let path = marker(paths);
match std::fs::remove_file(&path) {
Ok(()) => Ok(()),
Err(why) if why.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(source) => Err(Error::Ledger { path, source }),
}
}
fn marker(paths: &RunPaths) -> PathBuf {
paths.dir.join(MARKER)
}
pub fn shutdown(root: &Path, request: ShutdownRequest) -> Result<Shutdown> {
if Instant::now().checked_add(request.grace).is_none() {
return Err(Error::Invalid(format!(
"a grace of {}s is further away than this host's clock can count to; nothing \
was signalled — name a grace in seconds this host can wait out",
request.grace.as_secs()
)));
}
let selected = select(root, &request)?;
let mut runs = Vec::new();
let mut pushed: BTreeSet<(String, String)> = BTreeSet::new();
for view in &selected {
let one = shut_one_down(root, view, &request);
for branch in &one.branches {
if branch.outcome == Preserved::Pushed {
pushed.insert((branch.identity.clone(), branch.branch.clone()));
}
}
runs.push(one);
}
let (not_pushed, not_pushed_unread) = elsewhere_on_this_host(&pushed);
let forced = !request.asks();
Ok(Shutdown {
root: root.to_path_buf(),
scope: request.scope,
grace: request.grace,
forced,
runs,
not_pushed,
not_pushed_unread,
})
}
fn select(root: &Path, request: &ShutdownRequest) -> Result<Vec<RunView>> {
match &request.scope {
ShutdownScope::Run(run) => {
let view = RunView::open(&crate::verbs::resolved(root, run)?)?;
if !view.launch.owned_by(&request.session) {
return Err(Error::NotOwned {
run: view.paths.run.clone(),
owner: view.launch.owner_label(&request.session),
});
}
Ok(vec![view])
}
ShutdownScope::Mine => Ok(surveyed(root)
.into_iter()
.filter(|view| {
let mine = view.launch.owned_by(&request.session);
if !mine {
eprintln!(
"onepipeline: run '{}' belongs to {}; `--mine` shuts down only this \
session's runs, so it was left running and nothing was signalled for it",
view.paths.run,
view.launch.owner_label(&request.session)
);
}
mine
})
.collect()),
ShutdownScope::Host => Ok(surveyed(root)),
}
}
fn surveyed(root: &Path) -> Vec<RunView> {
let survey = Survey::of(root);
for refused in &survey.skipped {
eprintln!(
"onepipeline: run root '{}' could not be read ({}); this shutdown signalled nothing \
in it and pushed none of its branches",
crate::views::one_line(&refused.path.display().to_string()),
crate::views::one_line(&refused.reason)
);
}
survey.views
}
fn shut_one_down(root: &Path, view: &RunView, request: &ShutdownRequest) -> RunShutdown {
let paths = &view.paths;
let owner = view.launch.owner_label(&request.session);
let forced_over_owner =
!request.scope.keeps_the_ownership_rule() && !view.launch.owned_by(&request.session);
if forced_over_owner {
eprintln!(
"onepipeline: run '{}' belongs to {owner}; shutting this host down includes it",
paths.run
);
}
let held = hold_the_run(paths);
let asks = request.asks() && held.is_ok();
if let Err(why) = &held {
eprintln!(
"onepipeline: run '{}': the hold that stops it dispatching could not be written — \
{why}; so that its driver starts nothing new, nothing of it is asked or waited \
for and it goes straight to the teardown",
paths.run
);
}
let live = live_dispatches(paths).unwrap_or_else(|why| {
eprintln!("onepipeline: {why}");
Vec::new()
});
let watched = in_flight(view, live);
let from = match std::fs::metadata(paths.journal()) {
Ok(held) => held.len(),
Err(why) if why.kind() == std::io::ErrorKind::NotFound => 0,
Err(why) => {
eprintln!(
"onepipeline: run '{}': where its journal stood could not be read — {why}; \
nothing it records during the wait is read, so each dispatch is known to \
have ended by its own process alone",
paths.run
);
u64::MAX
}
};
let mut journal = Journal::open(paths);
let addresses = addresses_by_node(&view.events);
let mut asked: Vec<(Watched, String, String)> = Vec::new();
for dispatch in watched {
let (word, detail) = if !request.asks() {
(
Answer::NotAsked,
"nothing was asked of this dispatch: the shutdown was forced, so it went \
straight to the teardown and whatever its turn had not committed is gone"
.to_string(),
)
} else if !asks {
(
Answer::NotAsked,
"nothing was asked of this dispatch: the hold that stops its run dispatching \
could not be written, and waiting out the grace without it would have let the \
run's driver start new work, so it went straight to the teardown and whatever \
its turn had not committed is gone"
.to_string(),
)
} else if dispatch.in_the_driver {
(
Answer::NoTurn,
format!(
"it was inside a publication of its own, which has no turn to interrupt, \
so there was nothing to ask; it is killed in {}s if the publication has \
not settled by then",
request.grace.as_secs()
),
)
} else {
ask_it_to_stop(
&mut journal,
&dispatch.record.node,
addresses
.get(&dispatch.record.node)
.map_or(&[][..], Vec::as_slice),
request.grace,
)
};
asked.push((dispatch, word.as_str().to_string(), detail));
}
let asked_at = Instant::now();
let waited = wait_for_them(paths, from, &asked, asked_at, asks, request.grace);
let teardown = tear_the_run_down(paths, view);
let now = RunView::open(paths)
.map_err(|why| {
eprintln!(
"onepipeline: run '{}' could not be read again after its teardown — {why}; \
what a publication had reached is reported as it stood when the shutdown began",
paths.run
);
})
.ok();
let dispatches: Vec<DispatchStopped> = asked
.into_iter()
.map(|(dispatch, interrupt, detail)| {
let record = dispatch.record;
let publishing = now
.as_ref()
.is_some_and(|now| publishing_phase(now, &record.node).is_some());
let (how, waited) = match (
waited.ended.get(&Waited::key(&record)),
waited.exited.get(&Waited::key(&record)),
) {
(Some(after), _) => (DispatchEnding::Graceful, *after),
(None, Some(after)) if !publishing => (DispatchEnding::Graceful, *after),
_ if sys::claim_on(record.pid, &record.started).is_over() => {
(DispatchEnding::Killed, asked_at.elapsed())
}
_ => (DispatchEnding::StillRunning, asked_at.elapsed()),
};
DispatchStopped {
detail: with_the_publication_state(
now.as_ref().unwrap_or(view),
&record.node,
how,
detail,
),
node: record.node,
pid: record.pid,
interrupt,
ended: how,
waited,
}
})
.collect();
for stopped in &dispatches {
let written = journal.emit(
journal::PipelineKind::DispatchStopped,
journal::labels(&paths.run, Some(&stopped.node)),
journal::payload(&[
("pid", json!(stopped.pid)),
("interrupt", json!(stopped.interrupt)),
("detail", json!(stopped.detail)),
("ended", json!(stopped.ended.as_str())),
(
"waited_ms",
json!(u64::try_from(stopped.waited.as_millis()).unwrap_or(u64::MAX)),
),
]),
);
unrecorded(&paths.run, "a dispatch-stopped", written);
}
let branches = preserve_every_branch(view, now.as_ref());
let shutdown = RunShutdown {
run: paths.run.clone(),
owner,
forced_over_owner,
dispatches,
teardown,
branches,
};
let written = journal.emit(
journal::PipelineKind::HostShutdown,
journal::labels(&paths.run, None),
journal::payload(&[
("scope", json!(request.scope.as_str())),
("owner", json!(shutdown.owner)),
("forced", json!(!request.asks())),
("grace_seconds", json!(request.grace.as_secs())),
(
"dispatches",
json!(u32::try_from(shutdown.dispatches.len()).unwrap_or(u32::MAX)),
),
(
"graceful",
json!(counted(&shutdown, DispatchEnding::Graceful)),
),
("killed", json!(counted(&shutdown, DispatchEnding::Killed))),
(journal::STOP_TEARDOWN, json!(shutdown.teardown.word())),
("root", json!(root.display().to_string())),
(
"branches",
json!(shutdown
.branches
.iter()
.map(|branch| json!({
"identity": branch.identity,
"branch": branch.branch,
"result": branch.outcome.as_str(),
"remote": branch.remote,
"commit": branch.commit,
"detail": branch.detail,
}))
.collect::<Vec<_>>()),
),
]),
);
unrecorded(&paths.run, "the host-shutdown", written);
shutdown
}
fn unrecorded(run: &str, what: &str, written: Result<()>) {
if let Err(why) = written {
eprintln!(
"onepipeline: run '{run}': {what} record could not be written to its journal — {why}"
);
}
}
fn counted(shutdown: &RunShutdown, ending: DispatchEnding) -> u32 {
u32::try_from(
shutdown
.dispatches
.iter()
.filter(|stopped| stopped.ended == ending)
.count(),
)
.unwrap_or(u32::MAX)
}
fn hold_the_run(paths: &RunPaths) -> Result<()> {
let path = marker(paths);
std::fs::write(
&path,
json!({"at": sys::now_rfc3339(), "pid": sys::pid()}).to_string(),
)
.map_err(|source| Error::Ledger { path, source })
}
fn live_dispatches(paths: &RunPaths) -> Result<Vec<DispatchRecord>> {
let here = sys::hostname();
Ok(ledger::dispatches_of(paths)
.map_err(|why| {
Error::Refused(format!(
"run '{}': this build cannot establish what it is running — {why}; nothing \
of it is asked to stop or waited for, and its branches are preserved anyway",
paths.run
))
})?
.into_iter()
.filter(|record| record.host == here)
.filter(|record| {
matches!(
sys::claim_on(record.pid, &record.started),
sys::Claim::Proved
)
})
.collect())
}
struct Watched {
record: DispatchRecord,
in_the_driver: bool,
}
fn in_flight(view: &RunView, live: Vec<DispatchRecord>) -> Vec<Watched> {
let mut watched: Vec<Watched> = live
.into_iter()
.map(|record| Watched {
record,
in_the_driver: false,
})
.collect();
let Some(driver) = live_driver(view) else {
return watched;
};
for (node, status) in view.state.statuses() {
if status != crate::graph::NodeStatus::Running
|| watched.iter().any(|dispatch| dispatch.record.node == node)
|| publishing_phase(view, &node).is_none()
{
continue;
}
watched.push(Watched {
record: DispatchRecord {
node,
..driver.clone()
},
in_the_driver: true,
});
}
watched
}
fn live_driver(view: &RunView) -> Option<DispatchRecord> {
let launch = &view.launch;
let pid = launch.driver_pid()?.get();
let started = launch.driver_stamp()?;
(launch.recorded_host() == Some(sys::hostname().as_str())
&& matches!(sys::claim_on(pid, started), sys::Claim::Proved))
.then(|| DispatchRecord {
node: String::new(),
pid,
host: sys::hostname(),
dispatched_at: String::new(),
started: started.to_string(),
})
}
fn ask_it_to_stop(
journal: &mut Journal,
node: &str,
addresses: &[TurnAddress],
grace: Duration,
) -> (Answer, String) {
if addresses.is_empty() {
return (
Answer::NoTurn,
format!(
"nothing of this dispatch has named a turn to interrupt, so there was nothing \
to ask; it is killed in {}s if it has not exited by then",
grace.as_secs()
),
);
}
let mut word = Answer::NoTurn;
let mut answers = Vec::new();
for address in addresses {
let interrupt = agentgraph::interrupt(address, CANCEL_INPUT);
for mut event in interrupt.events {
if event.labels.node.is_none() {
event.labels.node = Some(node.to_string());
}
if let Err(why) = journal.relay(&event) {
eprintln!(
"onepipeline: node '{node}': the record of its interrupt could not be \
written to the run's journal — {why}"
);
}
}
let (rank, answer) = match &interrupt.outcome {
Interrupted::Delivered => (
Answer::Delivered,
"the running turn took the redirection".into(),
),
Interrupted::Failed(why) => (Answer::Failed, format!("the lever failed ({why})")),
Interrupted::NoTurn(why) => (Answer::NoTurn, format!("no turn to redirect ({why})")),
};
word = word.max(rank);
answers.push(format!("{}: {answer}", address.member()));
}
(
word,
format!(
"asked the {} turn(s) this dispatch had named to stop, commit, and end — {}. It is \
killed in {}s if it has not exited by then",
addresses.len(),
answers.join("; "),
grace.as_secs()
),
)
}
fn wait_for_them(
paths: &RunPaths,
from: u64,
asked: &[(Watched, String, String)],
asked_at: Instant,
asks: bool,
grace: Duration,
) -> Waited {
let mut waited = Waited::default();
if !asks {
return waited;
}
let deadline = asked_at
.checked_add(grace)
.unwrap_or_else(|| asked_at + Duration::from_secs(u64::from(u32::MAX)));
let driven = asked.iter().any(|(dispatch, ..)| dispatch.in_the_driver)
|| RunView::open(paths)
.ok()
.as_ref()
.and_then(live_driver)
.is_some();
loop {
let left_flight = if driven {
left_flight_since(paths, from)
} else {
BTreeSet::new()
};
let mut standing = false;
for (dispatch, ..) in asked {
let record = &dispatch.record;
if waited.ended.contains_key(&Waited::key(record)) {
continue;
}
let process_over =
dispatch.in_the_driver || sys::claim_on(record.pid, &record.started).is_over();
if process_over && !dispatch.in_the_driver {
waited
.exited
.entry(Waited::key(record))
.or_insert_with(|| asked_at.elapsed());
}
if process_over && (!driven || left_flight.contains(&record.node)) {
waited.ended.insert(Waited::key(record), asked_at.elapsed());
} else {
standing = true;
}
}
if !standing || Instant::now() >= deadline {
return waited;
}
std::thread::sleep(POLL);
}
}
#[derive(Default)]
struct Waited {
ended: BTreeMap<(String, u32), Duration>,
exited: BTreeMap<(String, u32), Duration>,
}
impl Waited {
fn key(record: &DispatchRecord) -> (String, u32) {
(record.node.clone(), record.pid)
}
}
fn left_flight_since(paths: &RunPaths, from: u64) -> BTreeSet<String> {
ledger::read_envelope_lines(&paths.journal(), from)
.into_iter()
.filter_map(|line| line.envelope)
.filter(|event| {
matches!(
journal::PipelineKind::from_wire(&event.kind),
Some(journal::PipelineKind::NodeSettled | journal::PipelineKind::NodeRequeued)
)
})
.filter_map(|event| event.labels.node)
.collect()
}
fn tear_the_run_down(paths: &RunPaths, view: &RunView) -> StopTeardown {
match driver::terminate(paths, &view.launch) {
Ok(teardown) => driver::established(teardown),
Err(why) => {
eprintln!(
"onepipeline: run '{}': this build cannot establish what it is running — {why}; \
nothing was signalled, and the branches below are preserved anyway",
paths.run
);
StopTeardown::NotAttempted
}
}
}
fn addresses_by_node(events: &[Envelope]) -> BTreeMap<String, Vec<TurnAddress>> {
let mut by_node: BTreeMap<String, Vec<TurnAddress>> = BTreeMap::new();
for event in events {
if event.source != Source::Agentgraph {
continue;
}
let (Some(node), Some(run), Some(member)) = (
event.labels.node.as_deref(),
event.labels.run_id.as_deref(),
event.labels.member.as_deref(),
) else {
continue;
};
let Some(address) = TurnAddress::of(run, member) else {
continue;
};
let named = by_node.entry(node.to_string()).or_default();
if !named.contains(&address) {
named.push(address);
}
}
by_node
}
fn with_the_publication_state(
view: &RunView,
node: &str,
ending: DispatchEnding,
detail: String,
) -> String {
if ending == DispatchEnding::Graceful {
return detail;
}
let Some(phase) = publishing_phase(view, node) else {
return detail;
};
let state = match view.state.change_urls.get(node) {
Some(url) => format!(
"its change request is open at {url}, pushed but with no merge-path verdict on \
this attempt"
),
None => match view.state.sessions.get(node) {
Some(session) => format!(
"its work is on branch {}, and nothing has merged it — the publication was \
stopped before its merge path answered",
session.branch()
),
None => "nothing has merged what it was publishing — the publication was stopped \
before its merge path answered"
.to_string(),
},
};
format!(
"{detail}. It was inside a publication of its own when the deadline reaped it (phase \
{phase}): {state}. `onepipeline adopt {}` re-dispatches the node pinned to that \
branch, which takes the publication up from where it stopped",
view.paths.run
)
}
fn publishing_phase(view: &RunView, node: &str) -> Option<Phase> {
view.events
.iter()
.rev()
.filter(|event| event.source == Source::Vcs)
.find(|event| event.labels.node.as_deref() == Some(node))
.and_then(|event| event.dimensions.phase)
.filter(|phase| !matches!(phase, Phase::Development))
}
fn preserve_every_branch(view: &RunView, now: Option<&RunView>) -> Vec<BranchPreserved> {
let mut preserved = Vec::new();
let mut offered = branches_of(view);
for pair in now.map(branches_of).unwrap_or_default() {
if !offered.contains(&pair) {
offered.push(pair);
}
}
for (repo, branch) in offered {
let request = onevcs::PreserveRequest {
repo: repo.clone(),
branch: branch.clone(),
};
preserved.push(match onevcs::preserve(&request) {
Ok(done) => BranchPreserved {
identity: done.identity,
branch: done.branch,
outcome: match done.outcome {
onevcs::Preservation::Pushed => Preserved::Pushed,
onevcs::Preservation::AlreadyOnOrigin => Preserved::AlreadyOnOrigin,
onevcs::Preservation::NoRemote => Preserved::NoRemote,
},
remote: done.remote,
commit: done.commit,
detail: format!("preserved from {}", done.from.display()),
},
Err(why) => BranchPreserved {
identity: repo,
branch,
outcome: Preserved::Refused,
remote: None,
commit: None,
detail: why.to_string(),
},
});
}
preserved
}
fn branches_of(view: &RunView) -> Vec<(String, String)> {
let statuses = view.state.statuses();
let finished_landing = |node: &str| {
view.state.landings.get(node) == Some(&crate::graph::Landing::Landed)
&& statuses.get(node) != Some(&crate::graph::NodeStatus::Running)
};
let repo_of = |node: &str| {
view.state
.graph
.get(node)
.and_then(|node| node.repo.clone())
};
let named = view
.state
.sessions
.iter()
.chain(view.state.abandoned.iter())
.map(|(node, session)| (node, session.branch().as_str().to_string()))
.chain(
view.state
.branches
.iter()
.map(|(node, branch)| (node, branch.clone())),
);
let mut found: Vec<(String, String)> = Vec::new();
for (node, branch) in named {
if finished_landing(node) {
continue;
}
if let Some(repo) = repo_of(node) {
let pair = (repo, branch);
if !found.contains(&pair) {
found.push(pair);
}
}
}
found
}
fn elsewhere_on_this_host(
pushed: &BTreeSet<(String, String)>,
) -> (Vec<(String, String)>, Option<String>) {
match onevcs::recoverable(&onevcs::Scope::All) {
Ok(rows) => (
rows.into_iter()
.map(|row| (row.identity, row.branch.branch))
.filter(|pair| !pushed.contains(pair))
.collect(),
None,
),
Err(why) => (Vec::new(), Some(why.to_string())),
}
}
const UNPROVEN: &str = "on its origin unproven — the branch reached its origin without that \
repository's own hook or merge path having run, and nothing can merge \
it without a publication that does run one";
pub fn render_shutdown(shutdown: &Shutdown) -> String {
let mut out = format!(
"shutdown scope {} grace {}s{} runs root {}\n",
shutdown.scope.as_str(),
shutdown.grace.as_secs(),
if shutdown.forced { " forced" } else { "" },
shutdown.root.display()
);
if shutdown.runs.is_empty() {
out.push_str(" no runs selected: nothing was signalled\n");
}
for run in &shutdown.runs {
out.push_str(&format!(
" {} owner {} selected by --{}{}\n",
run.run,
run.owner,
shutdown.scope.as_str(),
if run.forced_over_owner {
", which is another session's run"
} else {
""
}
));
if run.dispatches.is_empty() {
out.push_str(" no live dispatch to interrupt\n");
}
for stopped in &run.dispatches {
out.push_str(&format!(
" {} (pid {}): interrupt {} — {}; ended {}, after {}\n",
stopped.node,
stopped.pid,
stopped.interrupt,
stopped.detail,
stopped.ended.as_str(),
crate::telemetry::duration(
u64::try_from(stopped.waited.as_millis()).unwrap_or(u64::MAX)
)
));
}
out.push_str(&format!(" teardown: {}\n", teardown_said(run)));
if run.branches.is_empty() {
out.push_str(" no branch this run's records name\n");
}
for branch in &run.branches {
out.push_str(&format!(" {}\n", branch_said(branch)));
}
}
out.push_str(&last_section(shutdown));
out
}
fn teardown_said(run: &RunShutdown) -> String {
let said = crate::verbs::Stopped {
run: run.run.clone(),
owner: run.owner.clone(),
forced: run.forced_over_owner,
teardown: run.teardown,
clean: false,
}
.refusal()
.unwrap_or_else(|| {
format!(
"{} — every process this run named was reached",
run.teardown.word()
)
});
let survivors: Vec<String> = run
.dispatches
.iter()
.filter(|stopped| stopped.ended == DispatchEnding::StillRunning)
.map(|stopped| format!("{} (pid {})", stopped.node, stopped.pid))
.collect();
if survivors.is_empty() {
said
} else {
format!("{said}. Still running: {}", survivors.join(", "))
}
}
fn branch_said(branch: &BranchPreserved) -> String {
let what = match branch.outcome {
Preserved::Pushed => format!(
"{UNPROVEN}{}",
branch
.remote
.as_deref()
.map_or(String::new(), |remote| format!(" ({remote})"))
),
Preserved::AlreadyOnOrigin => {
"already on its origin at this commit; nothing was pushed".to_string()
}
Preserved::NoRemote => {
"this identity has no origin to push to, so nothing outside this host carries it"
.to_string()
}
Preserved::Refused => format!("could not be preserved: {}", branch.detail),
};
format!(
"{}@{}: {what}{}",
branch.identity,
branch.branch,
branch
.commit
.as_deref()
.map_or(String::new(), |commit| format!(" [{commit}]"))
)
}
fn last_section(shutdown: &Shutdown) -> String {
if let Some(why) = &shutdown.not_pushed_unread {
return format!(
" other unpublished branches on this host: not read — {why}. This is not a count \
of zero: there may be work here nothing outside this machine carries\n"
);
}
if shutdown.not_pushed.is_empty() {
return " other unpublished branches on this host: none\n".to_string();
}
let mut out = format!(
" other unpublished branches on this host, which this shutdown did not push ({}):\n",
shutdown.not_pushed.len()
);
for (identity, branch) in &shutdown.not_pushed {
out.push_str(&format!(" {identity}@{branch}\n"));
}
out
}