use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::{LazyLock, Mutex};
use crate::util::UnwrapPoison;
const ERROR_LOG_NAME: &str = "error.log";
pub(crate) const NO_STORE: &str = "store: none (this block names no store)";
pub(crate) const NO_DB_PATH: &str = "db path: none (this block names neither)";
const ARTIFACT_STATE_UNAVAILABLE: &str = "artifact state: not obtainable (no -wal size measured)";
pub(crate) fn artifact_state_line(wal_bytes: Option<u64>) -> String {
match wal_bytes {
Some(bytes) => format!("artifact state: wal_size={bytes}"),
None => ARTIFACT_STATE_UNAVAILABLE.to_string(),
}
}
pub(crate) const ENVIRONMENT_CAUSE: &str = "cause: environment — not store damage";
#[derive(Debug)]
pub(crate) enum FailureKind {
StartUpRefusal,
StartUpFailure,
CheckpointFailure,
ExitCheckpointFailure,
RuntimeIntegrityFailure,
ShrinkRefused,
}
impl FailureKind {
fn header(&self) -> &'static str {
match self {
Self::StartUpRefusal => "MahBot start-up refusal — ",
Self::StartUpFailure => "MahBot start-up failure — ",
Self::CheckpointFailure => "MahBot checkpoint failure — ",
Self::ExitCheckpointFailure => "MahBot exit checkpoint failure — ",
Self::RuntimeIntegrityFailure => "MahBot runtime integrity failure — ",
Self::ShrinkRefused => "MahBot store shrink refused — ",
}
}
}
#[derive(Debug)]
pub(crate) struct FailureReport {
kind: FailureKind,
store: Option<&'static str>,
db_path: Option<PathBuf>,
step: Option<String>,
reason: Option<String>,
environment: bool,
extra: Vec<String>,
}
impl FailureReport {
#[must_use]
pub(crate) fn new(kind: FailureKind) -> Self {
Self {
kind,
store: None,
db_path: None,
step: None,
reason: None,
environment: false,
extra: Vec::new(),
}
}
#[must_use]
pub(crate) fn store(mut self, store: &'static str) -> Self {
self.store = Some(store);
self
}
#[must_use]
pub(crate) fn db_path(mut self, db_path: PathBuf) -> Self {
self.db_path = Some(db_path);
self
}
#[must_use]
pub(crate) fn step(mut self, step: &str) -> Self {
self.step = Some(step.to_string());
self
}
#[must_use]
pub(crate) fn reason(mut self, reason: impl Into<String>) -> Self {
self.reason = Some(reason.into());
self
}
#[must_use]
pub(crate) fn environment(mut self, environment: bool) -> Self {
self.environment = environment;
self
}
#[must_use]
pub(crate) fn extra(mut self, line: impl Into<String>) -> Self {
self.extra.push(line.into());
self
}
#[must_use]
pub(crate) fn render(&self) -> String {
use std::fmt::Write;
let mut body = String::new();
let _ = writeln!(
body,
"{}{}",
self.kind.header(),
chrono::Utc::now().to_rfc3339()
);
match self.store {
Some(store) => {
let _ = writeln!(body, "store: {store}");
}
None => {
let _ = writeln!(body, "{NO_STORE}");
}
}
match &self.db_path {
Some(db_path) => {
let _ = writeln!(body, "db path: {}", db_path.display());
}
None => {
let _ = writeln!(body, "{NO_DB_PATH}");
}
}
if let Some(step) = &self.step {
let _ = writeln!(body, "step: {step}");
}
if self.environment {
let _ = writeln!(body, "{ENVIRONMENT_CAUSE}");
}
if let Some(reason) = &self.reason {
let _ = writeln!(body, "reason: {reason}");
}
for line in &self.extra {
let _ = writeln!(body, "{line}");
}
body
}
}
pub(crate) struct RoundCounter(LazyLock<Mutex<HashMap<String, u64>>>);
impl RoundCounter {
pub(crate) const fn new() -> Self {
Self(LazyLock::new(|| Mutex::new(HashMap::new())))
}
pub(crate) fn prior_rounds(&self, key: &str) -> u64 {
let mut rounds = self.0.lock().unwrap_poison();
let count = rounds.entry(key.to_string()).or_insert(0);
*count += 1;
*count - 1
}
}
fn condition_lines(block: &str) -> String {
let mut kept: Vec<&str> = Vec::new();
for line in block.lines() {
if line.starts_with("MahBot ") {
let head = match line.rsplit_once(' ') {
Some((head, stamp)) if stamp.contains('T') => head,
_ => line,
};
kept.push(head);
} else if line.starts_with("store: ")
|| line.starts_with("db path: ")
|| line.starts_with("step: ")
|| line.starts_with("cause: ")
{
kept.push(line);
}
}
kept.join("\n")
}
fn condition_is_recorded(path: &Path, block: &str) -> bool {
let Ok(existing) = std::fs::read_to_string(path) else {
return false;
};
let condition = condition_lines(block);
existing
.split("\n\n")
.any(|held| condition_lines(held) == condition)
}
fn append_failure_record(root: &Path, report: &str) -> std::io::Result<std::path::PathBuf> {
use std::io::Write;
std::fs::create_dir_all(root)?;
let path = root.join(ERROR_LOG_NAME);
let mut file = std::fs::OpenOptions::new()
.create(true)
.append(true)
.open(&path)?;
file.write_all(format!("{report}\n").as_bytes())?;
Ok(path)
}
#[derive(Debug)]
pub(crate) enum Filed {
Appended(PathBuf),
Held(PathBuf),
}
pub(crate) fn record(root: Option<&Path>, block: &str) -> Option<Filed> {
let Some(root) = root else {
crate::boot::timestamped_stderr(&unfiled_block(
block,
&format!(
"the storage root is unresolvable — this failure could not be filed in \
{ERROR_LOG_NAME}"
),
));
return None;
};
let path = root.join(ERROR_LOG_NAME);
if condition_is_recorded(&path, block) {
return Some(Filed::Held(path));
}
match append_failure_record(root, block) {
Ok(path) => Some(Filed::Appended(path)),
Err(e) => {
crate::boot::timestamped_stderr(&unfiled_block(
block,
&format!(
"the failure could not be filed in {}: {e:#}",
root.join(ERROR_LOG_NAME).display(),
),
));
None
}
}
}
pub(crate) fn recorded_pointer(what: &str, filed: Option<Filed>) -> Option<String> {
filed.map(|filed| {
let (verb, path) = match filed {
Filed::Appended(path) => ("recorded", path),
Filed::Held(path) => ("already recorded", path),
};
format!("{what} {verb} in {}", path.display())
})
}
pub(crate) fn record_and_point(root: Option<&Path>, what: &str, report: &FailureReport) {
let filed = record(root, &report.render());
if let Some(pointer) = recorded_pointer(what, filed) {
tracing::info!("{pointer}");
}
}
fn unfiled_block(block: &str, reason: &str) -> String {
format!("{block}\nnote: {reason}")
}
pub(crate) fn start_up_report(
kind: FailureKind,
store: Option<(&'static str, PathBuf)>,
reason: String,
environment: bool,
) -> FailureReport {
let report = FailureReport::new(kind)
.reason(reason)
.environment(environment);
match store {
Some((name, db_path)) => report.store(name).db_path(db_path),
None => report,
}
}
pub(crate) fn record_startup_refusal(
root: &Path,
store: &'static str,
db_path: &Path,
reason: &str,
environment: bool,
) -> Option<Filed> {
let report = start_up_report(
FailureKind::StartUpRefusal,
Some((store, db_path.to_path_buf())),
reason.to_string(),
environment,
);
record(Some(root), &report.render())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn render_emits_every_field_in_order() {
let report = FailureReport::new(FailureKind::RuntimeIntegrityFailure)
.store("core")
.db_path(PathBuf::from("/tmp/core.db"))
.step("boot::open_stores")
.environment(true)
.reason("the check could not run")
.extra("artifact state: wal_size=0")
.render();
let lines: Vec<&str> = report.lines().collect();
let (header, timestamp) = lines[0]
.split_once(" — ")
.expect("the block's first line must be a header and an em-dashed timestamp");
assert_eq!(
header, "MahBot runtime integrity failure",
"the header must name the failure kind: {report}"
);
assert!(
!timestamp.is_empty() && timestamp.starts_with("20"),
"the header must carry an RFC 3339 UTC timestamp: {report}"
);
assert_eq!(lines[1], "store: core", "got: {report}");
assert_eq!(lines[2], "db path: /tmp/core.db", "got: {report}");
assert_eq!(
lines[3], "step: boot::open_stores",
"the failing step must follow the db path: {report}"
);
assert_eq!(
lines[4], ENVIRONMENT_CAUSE,
"an environment-caused failure must say so: {report}"
);
assert_eq!(lines[5], "reason: the check could not run", "got: {report}");
assert_eq!(
lines[6], "artifact state: wal_size=0",
"extras must follow verbatim: {report}"
);
let bare = FailureReport::new(FailureKind::StartUpFailure).render();
assert!(
bare.contains(NO_STORE) && bare.contains(NO_DB_PATH),
"a block that names no store must say so, never omit the lines: {bare}"
);
assert!(
!bare.contains(ENVIRONMENT_CAUSE)
&& !bare.contains("reason:")
&& !bare.contains("step:"),
"absent optional fields must not be rendered: {bare}"
);
}
#[test]
fn startup_refusal_block_is_written_with_its_environment_note() {
let tmp = tempfile::TempDir::new().expect("temp dir for test");
let db_path = tmp.path().join("db/core.db");
record_startup_refusal(tmp.path(), "core", &db_path, "permission denied", true);
let body = std::fs::read_to_string(tmp.path().join(ERROR_LOG_NAME)).expect("error.log");
assert!(
body.starts_with("MahBot start-up refusal — ")
&& body.contains("store: core\n")
&& body.contains(&format!("db path: {}\n", db_path.display()))
&& body.contains(&format!("{ENVIRONMENT_CAUSE}\n"))
&& body.contains("reason: permission denied\n"),
"got: {body}"
);
}
#[test]
fn a_repeated_launch_failure_is_filed_once() {
let tmp = tempfile::TempDir::new().expect("temp dir for test");
let blocks = || {
std::fs::read_to_string(tmp.path().join(ERROR_LOG_NAME))
.unwrap_or_default()
.matches("MahBot start-up failure")
.count()
};
let filed = |report: FailureReport| {
let before = blocks();
assert!(
record(Some(tmp.path()), &report.render()).is_some(),
"the write must land in the record"
);
blocks() > before
};
let startup = |reason: &str, environment: bool| {
start_up_report(
FailureKind::StartUpFailure,
None,
reason.to_string(),
environment,
)
.step("providers::init_global")
};
assert!(
filed(startup("no provider credential", false)),
"the first failure is filed"
);
assert!(
!filed(startup("the provider credential is missing", false)),
"the same condition worded differently is not filed again"
);
let held = recorded_pointer(
"start-up failure",
record(
Some(tmp.path()),
&startup("no provider credential", false).render(),
),
);
assert!(
held.as_deref()
.is_some_and(|p| p.starts_with("start-up failure already recorded in ")),
"a condition the record holds must be pointed at as such: {held:?}"
);
assert!(
!filed(
startup("the provider credential is missing", false)
.extra("artifact state: wal_size=9")
),
"an extra line is not identity either"
);
assert!(
filed(
startup("the provider credential is missing", false).step("config::load_or_init")
),
"a different step is a condition of its own"
);
assert!(
filed(startup("no provider credential", true)),
"a different cause class is a condition of its own"
);
assert!(
filed(
FailureReport::new(FailureKind::StartUpFailure)
.store("core")
.db_path(PathBuf::from("/tmp/db/core.db"))
.reason("no provider credential")
),
"a different artifact is a condition of its own"
);
assert!(
filed(
FailureReport::new(FailureKind::StartUpFailure)
.store("core")
.db_path(PathBuf::from("/tmp/db/logs.db"))
.reason("no provider credential")
),
"another db path of one store is a condition of its own"
);
let body = std::fs::read_to_string(tmp.path().join(ERROR_LOG_NAME)).expect("error.log");
assert_eq!(blocks(), 5, "five conditions, five blocks: {body}");
}
#[test]
fn record_files_the_block_with_the_file_terminator() {
let tmp = tempfile::TempDir::new().expect("temp dir for test");
let block = format!("MahBot start-up failure — probe\n{NO_STORE}\n");
record(Some(tmp.path()), &block);
let body = std::fs::read_to_string(tmp.path().join(ERROR_LOG_NAME)).expect("error.log");
assert_eq!(
body,
format!("{block}\n"),
"the block and the file's blank-line terminator must be written: {body:?}"
);
}
#[test]
fn record_creates_a_missing_root_directory() {
let tmp = tempfile::TempDir::new().expect("temp dir for test");
let root = tmp.path().join("not-created-yet");
let filed = record(
Some(&root),
&format!("MahBot start-up failure — probe\n{NO_STORE}\n"),
);
assert!(
matches!(filed, Some(Filed::Appended(ref path)) if path == &root.join(ERROR_LOG_NAME)),
"the block must be appended under the missing root: {filed:?}"
);
let body = std::fs::read_to_string(root.join(ERROR_LOG_NAME)).expect("error.log");
assert!(
body.contains("MahBot start-up failure — probe"),
"got: {body:?}"
);
}
#[test]
fn unfiled_block_carries_the_block_and_the_reason() {
let text = unfiled_block(
"MahBot start-up failure — probe\nstore: core\n",
"the storage root is unresolvable",
);
assert!(
text.starts_with("MahBot start-up failure — probe\nstore: core\n"),
"the whole block must reach stderr: {text:?}"
);
assert!(
text.contains("note: the storage root is unresolvable"),
"the fallback must say why it is on stderr: {text:?}"
);
}
}