aion-cli 0.31.0

//! `aion server status`: which layers am I actually running?
//!
//! Three layers, each observed independently and each absence its own
//! honest state (no pid file ≠ server down ≠ HTTP unreachable ≠ not ready
//! — no layer's silence is ever reported as another layer's health):
//!
//! 1. **The record**: the pid file's incarnation, verified against the live
//!    process table.
//! 2. **The build**: `GET /build` on the address the record says the server
//!    bound (or the configured address when no record exists, with the
//!    source named; a BOOTING record has bound nothing, and the honest line
//!    is that the doors are not open yet — "unreachable" would name a fault
//!    where there is only a boot in progress) — asserted on the BODY, never the status code, because
//!    the ops-console SPA catch-all answers `200 text/html` for any unknown
//!    route on an image without the endpoint.
//! 3. **Readiness**: `GET /health/ready`, which answers by status code with
//!    an empty body — so a 200 carrying a body is the SPA shell on an image
//!    without the route. Whether that shell is a FOREIGN service or this
//!    server without the route mounted is decided by the build layer's own
//!    evidence: if `/build` just answered with a valid identity, the address
//!    is this aion server and the honest line is "does not mount
//!    /health/ready" — never an accusation that someone else took the port.
//!
//! Exit codes, so a script can ask "is the server up?": 0 when the RECORDED
//! incarnation is RUNNING, verified, and SERVING; 1 when it is not (gone,
//! mismatched, still BOOTING, DRAINING, or no record — each face printed);
//! 2 when the verb itself could not do its job (unresolvable home, unreadable
//! pid file). Booting and draining are exit 1 for the same reason a gone
//! server is: the question is "can I send work at it", and the answer is no —
//! but the printed state says which of the three it is, so an operator (or a
//! wait loop) can tell "not yet" from "not any more" from "not at all". A configuration that
//! cannot load is NOT exit 2: it feeds only the side layers (the store line,
//! and the probe address when no record exists), so the failure is stated as
//! its own layer's fact and the predicate still answers 0 or 1. The verdict
//! is about the recorded incarnation — the server the control verbs can
//! address — not about whether ANY server answers on some address: a server
//! that booted unclaimed answers HTTP but exits 1 here, with the finding
//! printed. `aion server stop` codes the no-record case differently (0,
//! idempotent success) because stopping is an action whose goal state
//! already holds, while status is a predicate whose answer is "no".

use std::net::SocketAddr;
use std::path::PathBuf;
use std::process::ExitCode;

use aion_server::config::{CliOverrides, ServerConfig, aion_home};
use aion_server::control::{IncarnationProbe, IncarnationState, incarnation, outcome, pid_file};
use clap::Args;
use serde::Deserialize;

use crate::console::PROBE_STEP_TIMEOUT;

/// Arguments for `aion server status`.
#[derive(Args, Clone, Debug)]
pub struct StatusArgs {
    /// Path to the TOML server configuration file. Used to resolve the store
    /// path and, when no pid file exists, the address to probe.
    #[arg(long)]
    config: Option<PathBuf>,
}

/// What `GET /build` answered when the body parsed as a build identity.
#[derive(Debug, Deserialize)]
struct BuildBody {
    version: String,
    commit: String,
    dirty: String,
    built_at: String,
}

/// Run `aion server status`.
pub async fn run(args: &StatusArgs) -> ExitCode {
    let home = match aion_home() {
        Ok(home) => home.path,
        Err(error) => {
            eprintln!("aion server status: could not resolve the Aion home: {error}");
            return ExitCode::from(2);
        }
    };
    let overrides = CliOverrides {
        config_path: args.config.clone(),
        ..CliOverrides::default()
    };
    // The predicate this verb answers — "is the recorded server up?" — needs
    // only the pid record and the process table. Configuration feeds two
    // side layers (the store line, and the probe address when no record
    // exists), so a broken config file must not silence the answer at
    // exactly the moment an operator most needs it: state the failure as
    // its own layer's fact and keep answering what is answerable.
    let config = match ServerConfig::load(&overrides) {
        Ok(config) => Some(config),
        Err(error) => {
            eprintln!(
                "configuration: could not load ({error}) — the store line and the \
                 no-record probe address are unavailable; the record and process \
                 layers below still answer"
            );
            None
        }
    };
    println!("home: {}", home.display());
    render_store_layer(config.as_ref());

    // Layer 1: the record. An unreadable file is a verb failure, not a
    // reportable server state — but the HTTP layers still answer from the
    // configured address so the operator gets every observable fact.
    let mut verb_failed = false;
    let record = match pid_file::read(&home) {
        Ok(record) => record,
        Err(error) => {
            eprintln!("pid file: {error}");
            verb_failed = true;
            None
        }
    };
    let mut running_verified = false;
    let http_address = if let Some(record) = &record {
        running_verified = render_record_layer(&home, record);
        if let Some(address) = record.http_address {
            println!("probing the address the record says the server bound: {address}");
            Some(address)
        } else {
            // A booting incarnation has bound nothing yet. Probing the
            // CONFIGURED address here would be a category error: an answer
            // would come from some OTHER server, and silence would print as
            // "unreachable" — a fault, when what is actually happening is a
            // boot doing exactly what it should.
            println!(
                "the doors are not open yet: this incarnation has bound no listener, \
                 so there is nothing to probe. That is the expected state while a \
                 server boots, not a fault"
            );
            None
        }
    } else {
        if !verb_failed {
            println!(
                "pid file: none at {} — no server has claimed this home (or the \
                 running server predates the pid file)",
                pid_file::pid_file_path(&home).display()
            );
        }
        if let Some(config) = &config {
            let address = config.server.listen_address;
            println!("probing the CONFIGURED address (no recorded bind): {address}");
            Some(address)
        } else {
            // No recorded bind and no readable config: there is no address
            // to probe, and inventing one would answer a question nobody
            // asked. The record layer's verdict stands.
            println!(
                "no recorded bind and no readable configuration — the listener \
                 layers cannot be probed"
            );
            None
        }
    };

    // Layers 2 and 3: the server's own listener. The build layer's verdict
    // — "this address answered with an aion build identity" — feeds the
    // readiness layer's diagnosis of a 200-with-body.
    let mut identity_confirmed = false;
    if let Some(http_address) = http_address {
        identity_confirmed = render_build_layer(http_address).await;
        render_ready_layer(http_address, identity_confirmed).await;
    }
    if let Some(http_address) = http_address
        && identity_confirmed
        && !running_verified
    {
        // The two layers disagree in the one direction worth a warning: an
        // aion server IS answering on this address, but it is not the
        // verified recorded incarnation — a server that booted UNCLAIMED
        // over another claimant's record, or one predating the pid file.
        // The exit code stays honest to the verb's question (the RECORDED
        // server), and this line stops the operator reading "not running"
        // as "nothing is listening".
        println!(
            "note: an aion server IS answering on {http_address}, but it is not a \
             verified recorded incarnation of this home — a server booted unclaimed \
             (another claimant held the record), or one predating the pid file. The \
             control verbs address only the recorded server"
        );
    }
    if verb_failed {
        ExitCode::from(2)
    } else if running_verified {
        ExitCode::SUCCESS
    } else {
        ExitCode::FAILURE
    }
}

/// The store line is always stated, and the BACKEND decides what it says:
/// config resolution fills `data_dir` unconditionally (`<AION_HOME>/data`
/// when the file names none), so the directory's presence proves nothing
/// about durability — a memory-backend server dies with its state no matter
/// what path the config carries. Same truth `aion launch` prints.
fn render_store_layer(config: Option<&ServerConfig>) {
    match config.map(|config| &config.store) {
        Some(store) => match store.backend {
            aion_server::config::StoreBackend::Haematite => match &store.data_dir {
                Some(data_dir) => println!("store: {data_dir}"),
                None => println!("store: haematite (data directory resolved by the server)"),
            },
            aion_server::config::StoreBackend::Memory => {
                println!("store: in-memory — state does not survive a stop");
            }
        },
        None => println!("store: unknown — the configuration could not be read"),
    }
}

/// Render the pid-file record and its incarnation verdict, including the
/// death note's account when the recorded process is gone. Returns whether
/// the recorded incarnation is RUNNING and verified — the fact the verb's
/// exit code answers with.
fn render_record_layer(home: &std::path::Path, record: &pid_file::PidRecord) -> bool {
    println!(
        "pid file: pid {} — version {}, commit {}, {}",
        record.pid,
        record.version,
        record.commit,
        bound_addresses_line(record)
    );
    match incarnation::probe(record) {
        IncarnationProbe::Verified { .. } => {
            let uptime_secs = record.running_for_secs();
            println!("process: RUNNING, incarnation verified (started {uptime_secs}s ago)");
            println!("state: {}", state_line(record));
            // Only a SERVING incarnation answers this verb's predicate with
            // yes. A booting one is not up yet and a draining one is on its
            // way out; a script that keyed on either as "ready" would send
            // work at a server that cannot take it.
            return record.state == IncarnationState::Serving;
        }
        IncarnationProbe::ProcessGone => {
            println!(
                "process: GONE — the recorded pid is not in the process table; the \
                 server died without removing its record"
            );
            match outcome::read_fate(home, record.pid) {
                Ok(fate) => crate::server_stop::render_fate(record.pid, &fate),
                Err(error) => println!("death note: {error}"),
            }
        }
        IncarnationProbe::DifferentIncarnation {
            live_started_at_unix_secs,
            exe,
        } => {
            let exe_note = exe.map_or_else(String::new, |path| {
                format!(" (running `{}`)", path.display())
            });
            println!(
                "process: MISMATCH — pid {} is alive but started at unix second {}, \
                 not the recorded {}{exe_note}; the pid was recycled onto a different \
                 process after the recorded server died",
                record.pid, live_started_at_unix_secs, record.started_at_unix_secs
            );
        }
    }
    false
}

/// The record's address clause: what it bound, or that it has bound nothing.
///
/// A booting incarnation genuinely has no addresses, and saying so is the
/// point — a rendered `0.0.0.0:0` or an inherited configured address would
/// both be inventions.
fn bound_addresses_line(record: &pid_file::PidRecord) -> String {
    match (record.http_address, record.grpc_address) {
        (Some(http), Some(grpc)) => format!("bound http {http} / grpc {grpc}"),
        (Some(http), None) => format!("bound http {http} / grpc not yet bound"),
        (None, Some(grpc)) => format!("http not yet bound / bound grpc {grpc}"),
        (None, None) => "no listener bound yet".to_owned(),
    }
}

/// The state line: which of the three states the incarnation recorded, and
/// for a boot, exactly how far it has got and whether it is still moving.
///
/// `last progress Ns ago` is the load-bearing half. A stage token alone
/// cannot distinguish a store that is chewing through shard 40 of 64 from
/// one that stopped at shard 17 an hour ago, and those two want opposite
/// decisions from whoever is reading.
fn state_line(record: &pid_file::PidRecord) -> String {
    let label = record.state.label();
    match record.state {
        IncarnationState::Serving => label.to_owned(),
        IncarnationState::Booting | IncarnationState::Draining => {
            let reached = record.stage_line().map_or_else(
                || " (no stage reported yet".to_owned(),
                |line| format!(" (stage: {line}"),
            );
            let progress = record
                .stage_age_secs()
                .map_or_else(String::new, |age| format!(", last progress {age}s ago"));
            format!(
                "{label}{reached}, started {}s ago{progress})",
                record.running_for_secs()
            )
        }
    }
}

/// Render the `/build` layer, asserted on the body. Returns whether the
/// address answered with a valid aion build identity — the evidence the
/// readiness layer uses to tell "this server without the route" apart from
/// "someone else's service on this address".
async fn render_build_layer(address: SocketAddr) -> bool {
    let url = format!("http://{address}/build");
    match probe(&url).await {
        Err(error) => {
            println!("build: unreachable — {error}");
            false
        }
        Ok((status, body)) => {
            if status == reqwest::StatusCode::UNAUTHORIZED
                || status == reqwest::StatusCode::FORBIDDEN
            {
                println!(
                    "build: the endpoint refused this unauthenticated probe ({status}); \
                     the listener is up but auth is enabled"
                );
                return false;
            }
            if let Ok(build) = serde_json::from_str::<BuildBody>(&body) {
                let dirty = if build.dirty == "true" { "-dirty" } else { "" };
                println!(
                    "build: {}+{}{} built {}",
                    build.version, build.commit, dirty, build.built_at
                );
                true
            } else {
                println!(
                    "build: the listener answered {status} but the body is not a build \
                     identity — an app shell or a different service is on this address \
                     (an image without /build answers 200 here; the body is the truth)"
                );
                false
            }
        }
    }
}

/// Render the `/health/ready` layer. The endpoint answers by status code
/// with an EMPTY body, so a 200 carrying a body is the SPA shell answering
/// for an unmounted route — and whether that shell belongs to THIS server or
/// to a foreign service is decided by `identity_confirmed`, the build
/// layer's own verdict for the same address moments earlier.
async fn render_ready_layer(address: SocketAddr, identity_confirmed: bool) {
    let url = format!("http://{address}/health/ready");
    match probe(&url).await {
        Err(error) => println!("ready: unreachable — {error}"),
        Ok((status, body)) => {
            if status == reqwest::StatusCode::OK && body.is_empty() {
                println!("ready: YES — the store answered the readiness probe");
            } else if status == reqwest::StatusCode::SERVICE_UNAVAILABLE {
                println!(
                    "ready: NO — the server is up but its runtime or store did not \
                     answer the readiness probe"
                );
            } else if status == reqwest::StatusCode::OK && identity_confirmed {
                println!(
                    "ready: this server does not mount /health/ready (the SPA shell \
                     answered for the route); the address IS this aion server — its \
                     /build just answered with a valid identity — but no health state \
                     was commissioned"
                );
            } else if status == reqwest::StatusCode::OK {
                println!(
                    "ready: the listener answered 200 with a body, but /health/ready \
                     answers with an empty one — an app shell or a different service \
                     is on this address"
                );
            } else if status == reqwest::StatusCode::UNAUTHORIZED
                || status == reqwest::StatusCode::FORBIDDEN
            {
                // The same face the build layer names for these codes: the
                // listener is up, auth is on, and this probe is
                // unauthenticated — not an unexpected answer, and not a
                // readiness verdict either way.
                println!(
                    "ready: the endpoint refused this unauthenticated probe ({status}); \
                     the listener is up but auth is enabled, so readiness cannot be \
                     probed unauthenticated"
                );
            } else {
                println!("ready: unexpected answer {status}");
            }
        }
    }
}

/// One bounded GET, returning status and body text.
async fn probe(url: &str) -> Result<(reqwest::StatusCode, String), String> {
    let client = reqwest::Client::builder()
        .timeout(PROBE_STEP_TIMEOUT)
        .build()
        .map_err(|error| error.to_string())?;
    let response = client
        .get(url)
        .send()
        .await
        .map_err(|error| error.to_string())?;
    let status = response.status();
    let body = response.text().await.map_err(|error| error.to_string())?;
    Ok((status, body))
}

#[cfg(test)]
mod tests {
    use super::{bound_addresses_line, state_line};
    use aion_server::control::{IncarnationState, PidRecord};

    fn record(state: IncarnationState) -> PidRecord {
        PidRecord {
            pid: 8941,
            started_at_unix_secs: aion_server::control::pid_file::now_unix_secs()
                .saturating_sub(93),
            binary_sha256: "0".repeat(64),
            version: "0.25.1".to_owned(),
            commit: "test".to_owned(),
            state,
            http_address: None,
            grpc_address: None,
            intended_http_address: None,
            intended_grpc_address: None,
            stage: Some("wal-recovery".to_owned()),
            stage_detail: Some("materializing shard 17 of 64".to_owned()),
            stage_seq: 17,
            stage_updated_at_unix_secs: aion_server::control::pid_file::now_unix_secs()
                .saturating_sub(2),
            drain_timeout_seconds: 0,
        }
    }

    /// 🔴 The line the outage needed. A booting server must report WHERE it
    /// is and WHEN it last moved: the stage alone cannot tell a store that is
    /// working from one that stopped an hour ago, and those two want opposite
    /// decisions from whoever is reading.
    #[test]
    fn a_booting_record_renders_its_stage_and_its_last_progress() {
        let line = state_line(&record(IncarnationState::Booting));
        assert!(line.starts_with("BOOTING"), "{line}");
        assert!(line.contains("wal-recovery"), "{line}");
        assert!(line.contains("materializing shard 17 of 64"), "{line}");
        assert!(line.contains("started 93s ago"), "{line}");
        assert!(line.contains("last progress"), "{line}");
    }

    /// A draining server is described the same way: it is mid-something, and
    /// an operator waiting on it needs the same two facts.
    #[test]
    fn a_draining_record_is_rendered_as_mid_something() {
        let line = state_line(&record(IncarnationState::Draining));
        assert!(line.starts_with("DRAINING"), "{line}");
        assert!(line.contains("started 93s ago"), "{line}");
    }

    /// A serving server is not mid-anything: one word, no stage clause.
    #[test]
    fn a_serving_record_renders_as_one_word() {
        assert_eq!(state_line(&record(IncarnationState::Serving)), "SERVING");
    }

    /// A record with no addresses says so rather than rendering an invented
    /// one. `0.0.0.0:0`, or the CONFIGURED address, would both be claims the
    /// record does not make — and the second would send a probe at whatever
    /// else happens to hold that port.
    #[test]
    fn an_unbound_record_names_the_absence_rather_than_inventing_an_address()
    -> Result<(), std::net::AddrParseError> {
        let mut unbound = record(IncarnationState::Booting);
        assert_eq!(bound_addresses_line(&unbound), "no listener bound yet");

        unbound.http_address = Some("127.0.0.1:8080".parse()?);
        assert_eq!(
            bound_addresses_line(&unbound),
            "bound http 127.0.0.1:8080 / grpc not yet bound"
        );

        unbound.grpc_address = Some("127.0.0.1:50051".parse()?);
        assert_eq!(
            bound_addresses_line(&unbound),
            "bound http 127.0.0.1:8080 / grpc 127.0.0.1:50051"
        );
        Ok(())
    }
}