reddb_server/runtime/

impl_replication_commit.rs

use super::*;

impl RedDBRuntime {
    /// PLAN.md Phase 11.4 — owned snapshot of every registered
    /// replica's state on this primary. Returns empty vec on
    /// non-primary instances or when no replicas are registered yet.
    pub fn primary_replica_snapshots(&self) -> Vec<crate::replication::primary::ReplicaState> {
        self.inner
            .db
            .replication
            .as_ref()
            .map(|repl| repl.replica_snapshots())
            .unwrap_or_default()
    }

    /// Issue #839 — the primary's current logical-WAL head LSN, used as
    /// the reference point for per-replica lag. `0` on non-primary
    /// instances or before the logical spool has any records.
    pub fn primary_logical_head_lsn(&self) -> u64 {
        self.inner
            .db
            .replication
            .as_ref()
            .map(|repl| repl.current_logical_lsn())
            .unwrap_or(0)
    }

    /// Issue #839 — count of pulls that forced a full re-bootstrap since
    /// process start. The primary operator alert signal; always `0` on a
    /// non-primary instance.
    pub fn replication_full_resync_count(&self) -> u64 {
        self.inner
            .db
            .replication
            .as_ref()
            .map(|repl| repl.full_resync_count())
            .unwrap_or(0)
    }

    /// Issue #839 — count of pulls served as a partial (incremental)
    /// resync since process start. Always `0` on a non-primary instance.
    pub fn replication_partial_resync_count(&self) -> u64 {
        self.inner
            .db
            .replication
            .as_ref()
            .map(|repl| repl.partial_resync_count())
            .unwrap_or(0)
    }

    /// Issue #1243 (PRD #1237 Phase B) — count of primary↔replica
    /// reconnects observed by this node's replica loop since process
    /// start. A reconnect is a link that was healthy, dropped (the pull
    /// loop fell back to `connecting`), and recovered. The initial connect
    /// is **not** counted. Always `0` on a node whose replica loop never
    /// ran (e.g. a standalone or primary instance). In-memory only: resets
    /// to `0` on process restart, like the resync counters above.
    pub fn replication_reconnects_count(&self) -> u64 {
        self.inner.replica_link_metrics.reconnects_total()
    }

    /// Issue #1243 — this node's persisted replica identity, read-only.
    /// Unlike [`node_id`](Self::node_id) / `resolve_replica_id` this never
    /// generates or persists a new id; it returns the empty string when one
    /// has not been assigned yet. Used as the bounded `replica_id`
    /// dimension on `reddb_replication_reconnects_total`.
    pub fn replication_replica_id(&self) -> String {
        self.config_string("red.replication.replica_id", "")
    }

    /// Issue #1243 — drive the reconnect tracker from a replica health
    /// state string. Production code reaches this through the replica
    /// loop's health-persist chokepoint; exposed on the runtime so an
    /// integration test can drive a deterministic link drop/restore
    /// without standing up a full gRPC link on a memory-constrained host.
    pub fn observe_replica_link_state(&self, state: &str) {
        self.inner.replica_link_metrics.observe_state(state);
    }

    pub fn enforce_primary_replica_retention_limits(
        &self,
    ) -> Vec<(String, reddb_file::ReplicationSlotInvalidationCause)> {
        self.inner
            .db
            .replication
            .as_ref()
            .map(|repl| repl.enforce_retention_limits(crate::utils::now_unix_millis() as u128))
            .unwrap_or_default()
    }

    /// Issue #839 — this node's stable identity, surfaced as the leader
    /// identity in `/replication/status` when the node is the primary.
    /// Reuses the same persisted id a replica advertises to the primary,
    /// so a cluster has one stable name per node regardless of role.
    pub fn node_id(&self) -> String {
        self.resolve_replica_id()
    }

    /// Issue #826 — re-evaluate write-admission flow control from the
    /// live primary replica registry and return the resulting throttle
    /// state. Computes the max lag across in-quorum replicas (async
    /// read-replicas excluded) against the primary's current LSN and
    /// engages/releases the `WriteGate` throttle accordingly.
    ///
    /// No-op (returns `false`) on non-primary instances or when flow
    /// control is disabled (soft target `0`). Cheap enough to call on
    /// the replica-ack path and from `/metrics` scrapes so the throttle
    /// tracks lag without a dedicated background loop.
    pub fn refresh_replication_flow_control(&self) -> bool {
        let flow = self.inner.write_gate.flow_control();
        if !flow.is_enabled() {
            return false;
        }
        let Some(repl) = self.inner.db.replication.as_ref() else {
            return false;
        };
        let primary_lsn = repl.current_logical_lsn();
        let replicas = repl.replica_snapshots();
        flow.observe(&replicas, primary_lsn)
    }

    /// PLAN.md Phase 11.4 — active commit policy. Reads
    /// `RED_PRIMARY_COMMIT_POLICY` once at runtime construction;
    /// future env reloads will need a reload endpoint. Default is
    /// `Local` — current behavior, no replica blocking.
    pub fn commit_policy(&self) -> crate::replication::CommitPolicy {
        crate::replication::CommitPolicy::from_env()
    }

    /// Issue #1001 — resolve the *effective* commit policy for one collection by
    /// combining the cluster default ([`commit_policy`](Self::commit_policy)),
    /// the collection's declared override, the collection data model, and the
    /// deployment's HA intent (`RED_CLUSTER_HA_INTENT`).
    ///
    /// Both write admission and failover eligibility call this so they read the
    /// same decision: a durable model (transactional/queue/audit/config/vault)
    /// may not silently use local-only acknowledgement under declared HA intent
    /// — that returns [`CommitPolicyViolation`] and the caller must fail closed.
    /// Explicitly ephemeral/cache-like collections may opt into local commit
    /// with the documented failover-eligibility data-loss window.
    pub fn resolve_commit_policy(
        &self,
        model: crate::cluster::CollectionDataModel,
        collection_override: Option<crate::replication::CommitPolicy>,
    ) -> Result<crate::cluster::CommitPolicyResolution, crate::cluster::CommitPolicyViolation> {
        crate::cluster::resolve_commit_policy(
            self.commit_policy(),
            collection_override,
            model,
            crate::cluster::HaIntent::from_env(),
        )
    }

    pub fn resolve_request_commit_policy(
        &self,
        request_override: Option<crate::replication::CommitPolicy>,
    ) -> Result<crate::cluster::CommitPolicyResolution, crate::cluster::CommitPolicyViolation> {
        let floor = crate::cluster::CommitPolicyResolution {
            effective: self.commit_policy(),
            source: crate::cluster::ResolutionSource::ClusterDefault,
            guardrail: crate::cluster::GuardrailDisposition::NotApplicable,
        };
        crate::cluster::resolve_request_commit_policy(floor, request_override)
    }

    pub fn primary_replica_durability(&self) -> reddb_file::ReplicationDurability {
        Self::primary_replica_durability_for_policy(self.commit_policy())
    }

    pub(crate) fn primary_replica_durability_for_policy(
        policy: crate::replication::CommitPolicy,
    ) -> reddb_file::ReplicationDurability {
        match policy {
            crate::replication::CommitPolicy::AckN(n) if n > 0 => {
                reddb_file::ReplicationDurability::RemoteFlush {
                    quorum: u16::try_from(n).unwrap_or(u16::MAX),
                }
            }
            _ => reddb_file::ReplicationDurability::Async,
        }
    }

    /// PLAN.md Phase 11.5 — accessor for replica-side apply error
    /// counters (gap / divergence / apply / decode / apply_miss). Returned
    /// snapshot is consistent across the counters; the labels match
    /// `reddb_replica_apply_errors_total{kind}`. Issue #814 adds the
    /// `apply_miss` kind for deletes against a missing target.
    pub fn replica_apply_error_counts(
        &self,
    ) -> [(crate::replication::logical::ApplyErrorKind, u64); 6] {
        self.inner.replica_apply_metrics.snapshot()
    }

    /// Issue #1242 — accessor for WAL apply throughput counters.
    /// Returns `(bytes_applied_total, records_applied_total)` as a
    /// monotonic snapshot since process start. Used to export
    /// `reddb_replication_apply_bytes_total` and
    /// `reddb_replication_apply_records_total` via `/metrics`.
    pub fn replica_apply_throughput_counts(&self) -> (u64, u64) {
        self.inner.replica_apply_metrics.snapshot_throughput()
    }

    /// PLAN.md Phase 11.4 — observability snapshot of every
    /// replica's durable LSN as known to the commit waiter. Empty
    /// vec on non-primary instances or when no replica has acked.
    pub fn commit_waiter_snapshot(&self) -> Vec<(String, u64)> {
        self.inner
            .db
            .replication
            .as_ref()
            .map(|repl| repl.commit_waiter.snapshot())
            .unwrap_or_default()
    }

    /// PLAN.md Phase 11.4 — `(reached, timed_out, not_required, last_micros)`
    /// counters for /metrics. Always-zero on non-primary instances.
    pub fn commit_waiter_metrics_snapshot(&self) -> (u64, u64, u64, u64) {
        self.inner
            .db
            .replication
            .as_ref()
            .map(|repl| repl.commit_waiter.metrics_snapshot())
            .unwrap_or((0, 0, 0, 0))
    }

    /// Named commit watermark: highest LSN durable on the active
    /// synchronous commit quorum. Returns 0 when the active policy does
    /// not require replica durability.
    pub fn commit_watermark(&self) -> u64 {
        match self.primary_replica_durability() {
            reddb_file::ReplicationDurability::RemoteWrite { quorum }
            | reddb_file::ReplicationDurability::RemoteFlush { quorum }
            | reddb_file::ReplicationDurability::RemoteApply { quorum }
                if quorum > 0 =>
            {
                self.inner
                    .db
                    .replication
                    .as_ref()
                    .map(|repl| repl.commit_waiter.commit_watermark(u32::from(quorum)))
                    .unwrap_or(0)
            }
            _ if matches!(
                self.commit_policy(),
                crate::replication::CommitPolicy::Quorum
            ) =>
            {
                self.inner
                    .db
                    .quorum
                    .as_ref()
                    .map(|q| q.commit_watermark())
                    .unwrap_or(0)
            }
            _ => 0,
        }
    }

    /// PLAN.md Phase 11.4 — block until at least `count` replicas
    /// have durably applied through `target_lsn`, or `timeout`
    /// elapses. Returns the `AwaitOutcome` so the caller can decide
    /// whether to surface a timeout error to the client or continue
    /// (the policy mapping lives in the commit dispatcher).
    ///
    /// Used by the `ack_n` commit policy once the operator flips
    /// `RED_PRIMARY_COMMIT_POLICY` away from `local`.
    pub fn await_replica_acks(
        &self,
        target_lsn: u64,
        count: u32,
        timeout: std::time::Duration,
    ) -> crate::replication::AwaitOutcome {
        match &self.inner.db.replication {
            Some(repl) => repl.commit_waiter.await_acks(target_lsn, count, timeout),
            None => {
                // No replication configured: policy must be `Local`.
                // Treat as immediate `NotRequired` so callers don't
                // block on a degenerate setup.
                crate::replication::AwaitOutcome::NotRequired
            }
        }
    }

    /// Enforce the requested durability. A missed synchronous threshold is
    /// an uncertain replica outcome after local commit, never success and
    /// never a claim that the source transaction rolled back.
    pub fn enforce_commit_policy(
        &self,
        post_lsn: u64,
    ) -> RedDBResult<crate::replication::AwaitOutcome> {
        let policy = crate::replication::CommitPolicy::from_env_strict()
            .map_err(RedDBError::InvalidConfig)?;
        self.enforce_commit_policy_for_policy(post_lsn, policy)
    }

    pub fn enforce_commit_policy_for_request(
        &self,
        post_lsn: u64,
        request_override: Option<crate::replication::CommitPolicy>,
    ) -> RedDBResult<crate::replication::AwaitOutcome> {
        let policy = self.validated_request_commit_policy(request_override)?;
        self.enforce_commit_policy_for_policy(post_lsn, policy)
    }

    pub(super) fn enforce_commit_policy_for_policy(
        &self,
        post_lsn: u64,
        policy: crate::replication::CommitPolicy,
    ) -> RedDBResult<crate::replication::AwaitOutcome> {
        self.validate_commit_policy(policy)?;
        if post_lsn == 0 {
            return Ok(crate::replication::AwaitOutcome::NotRequired);
        }
        if matches!(policy, crate::replication::CommitPolicy::Quorum) {
            return match self.inner.db.wait_for_replication_quorum(post_lsn) {
                Ok(()) => Ok(crate::replication::AwaitOutcome::Reached(0)),
                Err(err) => {
                    tracing::warn!(
                        target: "reddb::commit",
                        post_lsn,
                        error = %err,
                        "quorum: timed out waiting for commit watermark"
                    );
                    Err(RedDBError::CommitUncertain {
                        lsn: post_lsn,
                        detail: err.to_string(),
                    })
                }
            };
        }

        let n = match policy {
            crate::replication::CommitPolicy::AckN(n) if n > 0 => n,
            _ => return Ok(crate::replication::AwaitOutcome::NotRequired),
        };
        let timeout_ms = std::env::var("RED_REPLICATION_ACK_TIMEOUT_MS")
            .ok()
            .and_then(|v| v.parse::<u64>().ok())
            .unwrap_or(5_000);
        let outcome =
            self.await_replica_acks(post_lsn, n, std::time::Duration::from_millis(timeout_ms));
        {
            use crate::runtime::control_events::{EventKind, Outcome, Sensitivity};
            let (event_outcome, fields) = match &outcome {
                crate::replication::AwaitOutcome::Reached(count) => (
                    Outcome::Allowed,
                    vec![
                        (
                            "post_lsn".to_string(),
                            Sensitivity::raw(post_lsn.to_string()),
                        ),
                        ("required".to_string(), Sensitivity::raw(n.to_string())),
                        ("observed".to_string(), Sensitivity::raw(count.to_string())),
                        (
                            "timeout_ms".to_string(),
                            Sensitivity::raw(timeout_ms.to_string()),
                        ),
                    ],
                ),
                crate::replication::AwaitOutcome::TimedOut { observed, required } => (
                    Outcome::Error,
                    vec![
                        (
                            "post_lsn".to_string(),
                            Sensitivity::raw(post_lsn.to_string()),
                        ),
                        (
                            "required".to_string(),
                            Sensitivity::raw(required.to_string()),
                        ),
                        (
                            "observed".to_string(),
                            Sensitivity::raw(observed.to_string()),
                        ),
                        (
                            "timeout_ms".to_string(),
                            Sensitivity::raw(timeout_ms.to_string()),
                        ),
                    ],
                ),
                crate::replication::AwaitOutcome::NotRequired => (Outcome::Allowed, Vec::new()),
            };
            if !fields.is_empty() {
                self.emit_control_event(
                    EventKind::ReplicationSafety,
                    event_outcome,
                    "replication_commit_policy",
                    Some(format!("replication:lsn:{post_lsn}")),
                    None,
                    fields,
                ).map_err(|error| {
                    if matches!(outcome, crate::replication::AwaitOutcome::TimedOut { .. }) {
                        RedDBError::CommitUncertain {
                            lsn: post_lsn,
                            detail: format!("replica ACK threshold missed; recording the outcome also failed: {error}"),
                        }
                    } else { error }
                })?;
            }
        }
        if let crate::replication::AwaitOutcome::TimedOut { observed, required } = &outcome {
            tracing::warn!(
                target: "reddb::commit",
                post_lsn,
                observed = *observed,
                required = *required,
                timeout_ms,
                "ack_n: timed out waiting for replicas"
            );
            return Err(RedDBError::CommitUncertain {
                lsn: post_lsn,
                detail: format!("observed={observed} required={required}"),
            });
        }
        Ok(outcome)
    }

    pub(crate) fn validate_commit_policy(
        &self,
        policy: crate::replication::CommitPolicy,
    ) -> RedDBResult<()> {
        use crate::replication::CommitPolicy;
        match policy {
            CommitPolicy::Local | CommitPolicy::AckN(0) => Ok(()),
            CommitPolicy::RemoteWal => Err(RedDBError::InvalidConfig(
                "remote_wal commit acknowledgement is not implemented".into(),
            )),
            CommitPolicy::AckN(n) if n > u32::from(u16::MAX) => Err(RedDBError::InvalidConfig(
                "ack_n exceeds the supported replica quorum size".into(),
            )),
            _ if self.inner.db.replication.is_none() => Err(RedDBError::InvalidConfig(
                "synchronous commit policy requires a replication primary".into(),
            )),
            CommitPolicy::Quorum if self.inner.db.options().replication.quorum.is_async() => {
                Err(RedDBError::InvalidConfig(
                    "quorum commit policy requires a synchronous quorum configuration".into(),
                ))
            }
            _ => Ok(()),
        }
    }

    pub(crate) fn validated_request_commit_policy(
        &self,
        request: Option<crate::replication::CommitPolicy>,
    ) -> RedDBResult<crate::replication::CommitPolicy> {
        let configured = crate::replication::CommitPolicy::from_env_strict()
            .map_err(RedDBError::InvalidConfig)?;
        if let Some(request) = request {
            reject_incomparable_commit_policies(configured, request)?;
        }
        let floor = crate::cluster::CommitPolicyResolution {
            effective: configured,
            source: crate::cluster::ResolutionSource::ClusterDefault,
            guardrail: crate::cluster::GuardrailDisposition::NotApplicable,
        };
        let effective = crate::cluster::resolve_request_commit_policy(floor, request)
            .map_err(commit_policy_violation_error)?
            .effective;
        self.validate_commit_policy(effective)?;
        Ok(effective)
    }
}

pub(crate) fn commit_policy_violation_error(
    violation: crate::cluster::CommitPolicyViolation,
) -> RedDBError {
    let code = violation.code();
    let detail = violation.message();
    let mut validation = crate::json::Map::new();
    validation.insert(
        "code".to_string(),
        crate::json::Value::String(code.to_string()),
    );
    validation.insert(
        "message".to_string(),
        crate::json::Value::String(detail.clone()),
    );
    RedDBError::Validation {
        message: format!("{code}: {detail}"),
        validation: crate::json::Value::Object(validation),
    }
}

// Quorum membership/threshold and an absolute ACK count are different
// requirements. Until both can be retained, reject a switch between them.
pub(super) fn reject_incomparable_commit_policies(
    prior: crate::replication::CommitPolicy,
    next: crate::replication::CommitPolicy,
) -> RedDBResult<()> {
    use crate::replication::CommitPolicy;
    if matches!((prior, next), (CommitPolicy::AckN(n), CommitPolicy::Quorum) | (CommitPolicy::Quorum, CommitPolicy::AckN(n)) if n > 0)
    {
        return Err(RedDBError::InvalidConfig(
            "cannot mix AckN and Quorum commit requirements".into(),
        ));
    }
    Ok(())
}