aion_server/control/

claim.rs

//! The birth claim: taking the home's pid file BEFORE the store opens.
//!
//! The claim runs at stage zero of the boot — after the death note is armed,
//! before configuration is consumed, before the store is opened, before
//! anything that can take minutes. From that instant the home has a record,
//! so `aion server status` can describe the boot, `aion server stop` can stop
//! it, and a second `aion server` on the same home is REFUSED instead of
//! stacking silently behind the store's writer lock.
//!
//! What the claim does with what it finds is the whole design:
//!
//! | found | verdict |
//! |---|---|
//! | nothing | claim |
//! | a dead incarnation's record | claim, reporting the debris |
//! | a reused pid | claim, leaving the stranger alone |
//! | unparseable content | claim, reporting it |
//! | a LIVE incarnation whose addresses collide | **refuse the boot** |
//! | a LIVE incarnation on other addresses | boot UNCLAIMED (legal multi-server home) |
//! | a LIVE incarnation that is DRAINING | **succeed it** |
//!
//! Collision is decided against the addresses this boot INTENDS to bind
//! (resolved config), because at birth nothing is bound yet. A booting
//! sibling with no addresses recorded is ALWAYS a collision: it has not
//! chosen its doors yet, both boots read the same config, and the only
//! honest reading of two boots racing on one home is that they are the same
//! server twice.

use std::net::SocketAddr;
use std::path::{Path, PathBuf};

use tracing::{info, warn};

use super::guard::PidFileGuard;
use super::incarnation;
use super::pid_file::{
    IncarnationState, PidRecord, StaleReconciliation, lock_pid_mutation, pid_file_error,
    pid_file_path, write_record_atomically,
};
use crate::error::ServerError;

/// The addresses a boot intends to bind, resolved from its own configuration
/// before any listener exists. The birth claim compares a live holder's
/// RECORDED addresses against these to decide whether the two servers
/// collide.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct IntendedAddresses {
    /// The HTTP/ops-console address this boot will bind.
    pub http: SocketAddr,
    /// The gRPC address this boot will bind.
    pub grpc: SocketAddr,
}

/// The refusal: this Aion home is already held by a LIVE server incarnation
/// whose doors collide with the ones this boot intends to open.
///
/// This is the stacking killer. Before it existed, a second `aion server` on
/// a home whose first server was still replaying WAL saw an empty-looking
/// home (no record until bind, no listener to probe), started anyway, and
/// blocked without a word inside `Database::open` on the store's writer lock.
/// Four servers stacked that way on 2026-08-26 and nothing in the estate
/// could see them.
#[derive(Clone, Debug, thiserror::Error)]
#[error("{}", render_refusal(.home, .holder))]
pub struct HomeAlreadyClaimed {
    /// The Aion home both servers want.
    pub home: PathBuf,
    /// The live incarnation that already holds it — carried whole so a
    /// caller can render whatever the operator needs without re-reading a
    /// file that may have moved on.
    pub holder: PidRecord,
}

/// The refusal's operator-facing sentence: who holds the home, what it is
/// doing right now, and the three ways out.
fn render_refusal(home: &Path, holder: &PidRecord) -> String {
    let stage = match holder.stage_line() {
        Some(stage) => format!(", stage: {stage}"),
        None => String::new(),
    };
    format!(
        "refusing to start: the Aion home `{home}` is already held by a LIVE server — \
         pid {pid}, version {version}, state {state}{stage}, started {age}s ago. Two \
         servers on one home do not share it: the second blocks inside the store's \
         writer lock, silently, until the first exits. Watch that server with `aion \
         server status`; stop it with `aion server stop`; or give this server its own \
         home by setting AION_HOME",
        home = home.display(),
        pid = holder.pid,
        version = holder.version,
        state = holder.state.label(),
        age = holder.running_for_secs(),
    )
}

/// Claim the home's pid file at BIRTH, before the store is opened.
///
/// `record` is this incarnation's identity with `state = Booting` and no
/// addresses; `intended` is what this boot is about to bind, used only to
/// decide collision against a live holder.
///
/// # Errors
///
/// Returns [`ServerError::HomeAlreadyClaimed`] when a live incarnation
/// already holds the home on colliding addresses — the boot must not
/// continue. Returns [`ServerError::PidFile`] when the `run/` directory or
/// the file cannot be created, written, or renamed into place, or when the
/// mutation lock cannot be taken. A stale file that cannot be REASONED about
/// (unreadable) is reconciled and reported, never an error: refusing to boot
/// over a corrupt leftover would turn a crash's debris into an outage.
pub fn claim_at_birth(
    home: &Path,
    record: &PidRecord,
    intended: IntendedAddresses,
) -> Result<PidFileGuard, ServerError> {
    let path = pid_file_path(home);
    let run_dir = path.parent().ok_or_else(|| {
        pid_file_error(format!(
            "pid file path `{}` has no parent directory",
            path.display()
        ))
    })?;
    std::fs::create_dir_all(run_dir).map_err(|io_error| {
        pid_file_error(format!(
            "could not create run directory `{}`: {io_error}",
            run_dir.display()
        ))
    })?;
    // The read (reconcile) and the rename are one critical section: two
    // servers booting on one home reach this lock at the same instant, and
    // the serialization is exactly what makes the second one's reconcile see
    // the first one's record. Without it both would find an empty home.
    let mutation_lock = lock_pid_mutation(&path)?;
    let reconciliation = reconcile_existing(&path, intended);
    match &reconciliation {
        StaleReconciliation::LiveIncarnationElsewhere(existing) => {
            // The two servers want different doors, so neither is in the
            // other's way. Overwriting the record here would leave a running
            // server no verb can address, and refusing would break every
            // legitimate multi-server home (concurrent test servers, fleet
            // boxes on one default home). So this boot RUNS UNCLAIMED: the
            // first claimant keeps the record and the verbs, this incarnation
            // serves without either, and the warning names the consequence.
            warn!(
                path = %path.display(),
                recorded_pid = existing.pid,
                recorded_state = existing.state.label(),
                recorded_http_address = ?existing.http_address,
                this_http_address = %intended.http,
                "a live server already holds this home's pid file on different \
                 addresses; this incarnation boots UNCLAIMED — `aion server \
                 stop`/`status` will address the recorded server, not this one, and \
                 this boot reports no stages. Give each server its own AION_HOME to \
                 make both addressable"
            );
            drop(mutation_lock);
            return Ok(PidFileGuard::unclaimed(
                path,
                record.clone(),
                reconciliation,
            ));
        }
        StaleReconciliation::Collision(existing) => {
            let refusal = HomeAlreadyClaimed {
                home: home.to_path_buf(),
                holder: existing.clone(),
            };
            warn!(
                path = %path.display(),
                holder_pid = existing.pid,
                holder_state = existing.state.label(),
                holder_stage = ?existing.stage,
                "refusing to boot: this home is held by a live server on colliding \
                 addresses"
            );
            drop(mutation_lock);
            return Err(ServerError::HomeAlreadyClaimed {
                refusal: Box::new(refusal),
            });
        }
        other => report_reconciliation(&path, other),
    }
    write_record_atomically(&path, record)?;
    drop(mutation_lock);
    info!(
        path = %path.display(),
        pid = record.pid,
        started_at_unix_secs = record.started_at_unix_secs,
        state = record.state.label(),
        "pid file written at birth; this incarnation has claimed the home"
    );
    Ok(PidFileGuard::claimed(path, record.clone(), reconciliation))
}

/// Classify what is already at `path` without touching it.
///
/// The one face the caller must never see outside this module's own decision
/// table is [`StaleReconciliation::Collision`]: it is not a reconciliation at
/// all but a refusal, and [`claim_at_birth`] converts it into one.
fn reconcile_existing(path: &Path, intended: IntendedAddresses) -> StaleReconciliation {
    let content = match std::fs::read_to_string(path) {
        Ok(content) => content,
        Err(io_error) if io_error.kind() == std::io::ErrorKind::NotFound => {
            return StaleReconciliation::NonePresent;
        }
        Err(io_error) => {
            return StaleReconciliation::Unreadable {
                reason: format!("could not read the existing file: {io_error}"),
            };
        }
    };
    let record = match serde_json::from_str::<PidRecord>(&content) {
        Ok(record) => record,
        Err(parse_error) => {
            return StaleReconciliation::Unreadable {
                reason: format!("existing content does not parse as a pid record: {parse_error}"),
            };
        }
    };
    match incarnation::probe(&record) {
        incarnation::IncarnationProbe::ProcessGone => StaleReconciliation::DeadIncarnation(record),
        incarnation::IncarnationProbe::DifferentIncarnation { .. } => {
            StaleReconciliation::ReusedPid(record)
        }
        incarnation::IncarnationProbe::Verified { .. } => match record.state {
            // A drainer is on its way out and has already stopped serving new
            // work: succeeding it is the whole point of `aion server restart`.
            // The successor waits out the store writer lock the drainer still
            // holds — by design, and the stage report says so.
            IncarnationState::Draining => StaleReconciliation::SucceededDrainer(record),
            IncarnationState::Booting | IncarnationState::Serving => {
                if collides(&record, intended) {
                    StaleReconciliation::Collision(record)
                } else {
                    StaleReconciliation::LiveIncarnationElsewhere(record)
                }
            }
        },
    }
}

/// Whether a live holder's recorded doors collide with the ones this boot
/// intends to open.
///
/// The holder's BOUND addresses are the strongest fact and win when present.
/// A holder still booting has bound nothing — but its record carries the
/// addresses its configuration said it WILL bind, written at birth, and
/// comparing intention against intention is exactly as decisive: two
/// explicit configs naming different doors are two servers, and refusing the
/// second would break every legitimate multi-server home the moment their
/// boots overlap (measured: battery run 0b066959, two e2e servers with
/// distinct stores and ports, the second refused mid-`store-open`). A record
/// with NEITHER pair was written by a build that predates the intended
/// addresses; nothing is comparable, and the only honest reading left is a
/// collision — two boots on one home, indistinguishable from the same
/// server twice.
fn collides(holder: &PidRecord, intended: IntendedAddresses) -> bool {
    let holder_doors = match (holder.http_address, holder.grpc_address) {
        (None, None) => (holder.intended_http_address, holder.intended_grpc_address),
        bound => bound,
    };
    match holder_doors {
        (None, None) => true,
        (http, grpc) => {
            http.is_some_and(|address| address == intended.http || address == intended.grpc)
                || grpc.is_some_and(|address| address == intended.http || address == intended.grpc)
        }
    }
}

/// Say what the reconciliation found, at the severity it deserves.
///
/// The two faces that return before reaching here — the unclaimed boot and
/// the refusal — report themselves inside [`claim_at_birth`], where each has
/// a consequence to name that this reporter does not know about.
fn report_reconciliation(path: &Path, reconciliation: &StaleReconciliation) {
    match reconciliation {
        StaleReconciliation::NonePresent
        | StaleReconciliation::LiveIncarnationElsewhere(_)
        | StaleReconciliation::Collision(_) => {}
        StaleReconciliation::DeadIncarnation(record) => {
            warn!(
                path = %path.display(),
                stale_pid = record.pid,
                stale_started_at_unix_secs = record.started_at_unix_secs,
                stale_version = %record.version,
                stale_state = record.state.label(),
                "stale pid file: the recorded server (pid gone) died without removing \
                 its record; replacing it with this incarnation's"
            );
        }
        StaleReconciliation::ReusedPid(record) => {
            warn!(
                path = %path.display(),
                stale_pid = record.pid,
                stale_started_at_unix_secs = record.started_at_unix_secs,
                "stale pid file: the recorded pid is alive but belongs to a different \
                 process (pid reused after the recorded server died); replacing the \
                 record and leaving that process untouched"
            );
        }
        StaleReconciliation::SucceededDrainer(record) => {
            info!(
                path = %path.display(),
                draining_pid = record.pid,
                draining_version = %record.version,
                "succeeding a DRAINING server on this home: it has seen its \
                 termination signal and is on its way out, so this incarnation takes \
                 the record. Its own guard leaves this claim alone at exit (the \
                 compare is by incarnation identity). This boot waits out the store \
                 writer lock the drainer still holds — that wait is the handover, not \
                 a fault"
            );
        }
        StaleReconciliation::Unreadable { reason } => {
            warn!(
                path = %path.display(),
                %reason,
                "pid file present but unreadable as a record (hand-written or \
                 corrupt); replacing it with this incarnation's"
            );
        }
    }
}

#[cfg(test)]
#[path = "claim_tests.rs"]
mod tests;