tencrypt_core/

lib.rs

use std::fs::OpenOptions;
use std::io::Write;
use std::path::Path;

use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use uuid::Uuid;

#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum CertState {
    Requested,
    PolicyValidated,
    DnsChallengePrepared,
    DnsChallengePropagating,
    Issuing,
    Issued,
    RenewalDue,
    Renewing,
    Failed,
    Revoked,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct IssuanceRequest {
    pub request_id: String,
    pub hostname: String,
    pub actor: String,
    pub requested_at: DateTime<Utc>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StateTransition {
    pub from: CertState,
    pub to: CertState,
    pub reason: String,
    pub at: DateTime<Utc>,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct IssuanceReport {
    pub request_id: String,
    pub hostname: String,
    pub actor: String,
    pub status: String,
    pub started_at: DateTime<Utc>,
    pub completed_at: DateTime<Utc>,
    pub transitions: Vec<StateTransition>,
}

#[derive(Debug, thiserror::Error)]
pub enum RequestValidationError {
    #[error("hostname must contain at least one dot")]
    InvalidHostname,
    #[error("hostname must be a valid RFC 1123-style DNS name")]
    InvalidHostnameFormat,
}

pub fn new_request(hostname: String, actor: String) -> IssuanceRequest {
    IssuanceRequest {
        request_id: Uuid::new_v4().to_string(),
        hostname,
        actor,
        requested_at: Utc::now(),
    }
}

pub fn validate_hostname(hostname: &str) -> Result<(), RequestValidationError> {
    if !hostname.contains('.') {
        return Err(RequestValidationError::InvalidHostname);
    }

    if hostname.is_empty() || hostname.len() > 253 {
        return Err(RequestValidationError::InvalidHostnameFormat);
    }

    for label in hostname.split('.') {
        if label.is_empty() || label.len() > 63 {
            return Err(RequestValidationError::InvalidHostnameFormat);
        }

        let bytes = label.as_bytes();
        if bytes.first() == Some(&b'-') || bytes.last() == Some(&b'-') {
            return Err(RequestValidationError::InvalidHostnameFormat);
        }

        if !bytes
            .iter()
            .all(|b| b.is_ascii_alphanumeric() || *b == b'-')
        {
            return Err(RequestValidationError::InvalidHostnameFormat);
        }
    }

    Ok(())
}

pub fn validate_request(req: &IssuanceRequest) -> Result<(), RequestValidationError> {
    validate_hostname(&req.hostname)
}

pub fn dry_run_issue(req: &IssuanceRequest) -> Result<IssuanceReport, RequestValidationError> {
    validate_request(req)?;

    let started = Utc::now();
    let transitions = vec![
        StateTransition {
            from: CertState::Requested,
            to: CertState::PolicyValidated,
            reason: "policy checks passed".to_string(),
            at: Utc::now(),
        },
        StateTransition {
            from: CertState::PolicyValidated,
            to: CertState::DnsChallengePrepared,
            reason: "acme dns challenge prepared".to_string(),
            at: Utc::now(),
        },
        StateTransition {
            from: CertState::DnsChallengePrepared,
            to: CertState::DnsChallengePropagating,
            reason: "waiting for dns propagation".to_string(),
            at: Utc::now(),
        },
        StateTransition {
            from: CertState::DnsChallengePropagating,
            to: CertState::Issuing,
            reason: "acme order started".to_string(),
            at: Utc::now(),
        },
        StateTransition {
            from: CertState::Issuing,
            to: CertState::Issued,
            reason: "certificate issued (dry run)".to_string(),
            at: Utc::now(),
        },
    ];

    Ok(IssuanceReport {
        request_id: req.request_id.clone(),
        hostname: req.hostname.clone(),
        actor: req.actor.clone(),
        status: "success".to_string(),
        started_at: started,
        completed_at: Utc::now(),
        transitions,
    })
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct AuditRecord {
    pub id: String,
    pub timestamp: DateTime<Utc>,
    pub actor: String,
    pub operation: String,
    pub resource: String,
    pub status: String,
    pub message: String,
}

pub fn write_audit_jsonl(path: &Path, report: &IssuanceReport) -> anyhow::Result<()> {
    let mut file = OpenOptions::new().create(true).append(true).open(path)?;
    let rec = AuditRecord {
        id: Uuid::new_v4().to_string(),
        timestamp: Utc::now(),
        actor: report.actor.clone(),
        operation: "dry_run_issue".to_string(),
        resource: report.hostname.clone(),
        status: report.status.clone(),
        message: format!("request_id={}", report.request_id),
    };
    let line = serde_json::to_string(&rec)?;
    writeln!(file, "{line}")?;
    Ok(())
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CloudEvent {
    pub specversion: String,
    pub id: String,
    pub source: String,
    #[serde(rename = "type")]
    pub event_type: String,
    pub time: DateTime<Utc>,
    pub subject: String,
    pub datacontenttype: String,
    pub data: serde_json::Value,
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TraefikStaticConfigInput {
    pub email: String,
    pub resolver_name: String,
    pub cert_storage_file: String,
    pub cloudflare_token_env: String,
    pub log_level: String,
}

impl Default for TraefikStaticConfigInput {
    fn default() -> Self {
        Self {
            email: "ops@example.com".to_string(),
            resolver_name: "cloudflare".to_string(),
            cert_storage_file: "/var/traefik/certs/acme.json".to_string(),
            cloudflare_token_env: "CF_DNS_API_TOKEN".to_string(),
            log_level: "INFO".to_string(),
        }
    }
}

#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TraefikRouterConfig {
    pub service_name: String,
    pub hostname: String,
    pub service_port: u16,
    pub resolver_name: String,
}

impl TraefikRouterConfig {
    pub fn router_base_name(&self) -> String {
        self.service_name.replace('_', "-")
    }
}

pub fn render_traefik_static_yaml(input: &TraefikStaticConfigInput) -> String {
    format!(
        "global:\n  checkNewVersion: false\n  sendAnonymousUsage: false\n\nlog:\n  level: {log_level}\n\napi:\n  dashboard: true\n\nentryPoints:\n  web:\n    address: \":80\"\n    http:\n      redirections:\n        entryPoint:\n          to: websecure\n          scheme: https\n          permanent: true\n  websecure:\n    address: \":443\"\n\nproviders:\n  docker:\n    endpoint: \"unix:///var/run/docker.sock\"\n    exposedByDefault: false\n\ncertificatesResolvers:\n  {resolver_name}:\n    acme:\n      email: \"{email}\"\n      storage: \"{storage}\"\n      dnsChallenge:\n        provider: cloudflare\n        resolvers:\n          - \"1.1.1.1:53\"\n          - \"8.8.8.8:53\"\n\n# Required environment variable at runtime:\n# {token_env}=<cloudflare_dns_api_token>\n",
        log_level = input.log_level,
        resolver_name = input.resolver_name,
        email = input.email,
        storage = input.cert_storage_file,
        token_env = input.cloudflare_token_env,
    )
}

pub fn generate_router_labels(config: &TraefikRouterConfig) -> Vec<String> {
    let base = config.router_base_name();
    vec![
        "traefik.enable=true".to_string(),
        format!(
            "traefik.http.routers.{base}-http.rule=Host(`{host}`)",
            host = config.hostname
        ),
        format!("traefik.http.routers.{base}-http.entrypoints=web"),
        format!(
            "traefik.http.routers.{base}-https.rule=Host(`{host}`)",
            host = config.hostname
        ),
        format!("traefik.http.routers.{base}-https.entrypoints=websecure"),
        format!("traefik.http.routers.{base}-https.tls=true"),
        format!(
            "traefik.http.routers.{base}-https.tls.certresolver={resolver}",
            resolver = config.resolver_name
        ),
        format!(
            "traefik.http.services.{base}.loadbalancer.server.port={port}",
            port = config.service_port
        ),
    ]
}

pub fn cert_state_names() -> Vec<&'static str> {
    vec![
        "requested",
        "policy_validated",
        "dns_challenge_prepared",
        "dns_challenge_propagating",
        "issuing",
        "issued",
        "renewal_due",
        "renewing",
        "failed",
        "revoked",
    ]
}

pub fn events_from_report(report: &IssuanceReport) -> Vec<CloudEvent> {
    report
        .transitions
        .iter()
        .map(|t| CloudEvent {
            specversion: "1.0".to_string(),
            id: Uuid::new_v4().to_string(),
            source: "com.tencrypt.issuer".to_string(),
            event_type: format!(
                "com.tencrypt.certificate.{}.v1",
                state_name(t.to).replace('_', "-")
            ),
            time: t.at,
            subject: report.hostname.clone(),
            datacontenttype: "application/json".to_string(),
            data: serde_json::json!({
                "request_id": report.request_id,
                "hostname": report.hostname,
                "actor": report.actor,
                "from": state_name(t.from),
                "to": state_name(t.to),
                "reason": t.reason,
            }),
        })
        .collect()
}

pub fn write_events_jsonl(path: &Path, report: &IssuanceReport) -> anyhow::Result<()> {
    let mut file = OpenOptions::new().create(true).append(true).open(path)?;
    for event in events_from_report(report) {
        let line = serde_json::to_string(&event)?;
        writeln!(file, "{line}")?;
    }
    Ok(())
}

/// Parse a `CertState` from its snake_case name string.
pub fn cert_state_from_str(s: &str) -> Option<CertState> {
    match s {
        "requested" => Some(CertState::Requested),
        "policy_validated" => Some(CertState::PolicyValidated),
        "dns_challenge_prepared" => Some(CertState::DnsChallengePrepared),
        "dns_challenge_propagating" => Some(CertState::DnsChallengePropagating),
        "issuing" => Some(CertState::Issuing),
        "issued" => Some(CertState::Issued),
        "renewal_due" => Some(CertState::RenewalDue),
        "renewing" => Some(CertState::Renewing),
        "failed" => Some(CertState::Failed),
        "revoked" => Some(CertState::Revoked),
        _ => None,
    }
}

// ── M3: Idempotent Reconcile Loop ──────────────────────────────────────────

/// The action the reconciler should take given the current certificate state.
/// `reconcile_step` is a pure function: same input always produces the same output.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ReconcileDecision {
    /// Advance to the next lifecycle step.
    Proceed,
    /// Pause and re-check after a propagation delay.
    Wait,
    /// Certificate is in a terminal state — nothing to do.
    Done,
    /// Certificate failed; caller should apply backoff then retry or abandon.
    RetryOrAbandon,
}

/// Pure reconcile function: given current state, return the required action.
/// Makes no I/O, mutates nothing, has no side effects.
pub fn reconcile_step(state: CertState) -> ReconcileDecision {
    match state {
        CertState::Requested => ReconcileDecision::Proceed,
        CertState::PolicyValidated => ReconcileDecision::Proceed,
        CertState::DnsChallengePrepared => ReconcileDecision::Proceed,
        CertState::DnsChallengePropagating => ReconcileDecision::Wait,
        CertState::Issuing => ReconcileDecision::Proceed,
        CertState::Issued => ReconcileDecision::Done,
        CertState::RenewalDue => ReconcileDecision::Proceed,
        CertState::Renewing => ReconcileDecision::Proceed,
        CertState::Failed => ReconcileDecision::RetryOrAbandon,
        CertState::Revoked => ReconcileDecision::Done,
    }
}

// ── M3: Retry / Backoff Policy ─────────────────────────────────────────────

use std::time::Duration;

/// Configuration for exponential backoff without jitter.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BackoffConfig {
    /// Maximum number of retry attempts before giving up.
    pub max_retries: u32,
    /// Base delay in milliseconds for the first retry.
    pub base_delay_ms: u64,
    /// Maximum delay cap in milliseconds after exponential growth.
    pub max_delay_ms: u64,
}

impl Default for BackoffConfig {
    fn default() -> Self {
        Self {
            max_retries: 5,
            base_delay_ms: 1_000,
            max_delay_ms: 60_000,
        }
    }
}

/// Tracks how many attempts have been made so far.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct BackoffState {
    /// Zero-based attempt counter. `0` means first retry has not yet occurred.
    pub attempt: u32,
}

/// Returns the delay before the next attempt, or `None` if retries are exhausted.
/// Delay doubles each attempt and is capped at `config.max_delay_ms`.
pub fn next_backoff_delay(config: &BackoffConfig, state: &BackoffState) -> Option<Duration> {
    if state.attempt >= config.max_retries {
        return None;
    }
    let multiplier = 1u64.checked_shl(state.attempt).unwrap_or(u64::MAX);
    let delay_ms = config
        .base_delay_ms
        .saturating_mul(multiplier)
        .min(config.max_delay_ms);
    Some(Duration::from_millis(delay_ms))
}

/// Jittered variant of [`next_backoff_delay`].
///
/// The caller provides `jitter_ms` (e.g. computed from a PRNG) so this
/// function remains pure and deterministic in tests (pass `0` for no jitter).
/// The jitter is added to the base exponential delay *before* the cap is
/// applied.
pub fn next_backoff_delay_jittered(
    config: &BackoffConfig,
    state: &BackoffState,
    jitter_ms: u64,
) -> Option<Duration> {
    if state.attempt >= config.max_retries {
        return None;
    }
    let multiplier = 1u64.checked_shl(state.attempt).unwrap_or(u64::MAX);
    let base_ms = config.base_delay_ms.saturating_mul(multiplier);
    let delay_ms = base_ms.saturating_add(jitter_ms).min(config.max_delay_ms);
    Some(Duration::from_millis(delay_ms))
}

// ── AAA-2: Per-domain policy overrides ────────────────────────────────────

/// Per-domain backoff policy override.
/// `hostname_pattern` is an exact hostname match for AAA-1/2; wildcard
/// support is deferred to AAA-3.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DomainPolicy {
    pub hostname_pattern: String,
    pub backoff: Option<BackoffConfig>,
}

/// Resolve the effective backoff config for a hostname.
///
/// Returns the first matching `DomainPolicy.backoff` when `hostname_pattern`
/// equals `hostname`, otherwise returns `default`.
pub fn resolve_backoff<'a>(
    hostname: &str,
    policies: &'a [DomainPolicy],
    default: &'a BackoffConfig,
) -> &'a BackoffConfig {
    policies
        .iter()
        .find(|p| p.hostname_pattern == hostname)
        .and_then(|p| p.backoff.as_ref())
        .unwrap_or(default)
}

// ── M3: SLO and Metrics Baseline ───────────────────────────────────────────

/// In-process certificate operation counters (single-threaded).
/// Serialize to JSON for periodic log-line metrics export.
#[derive(Debug, Default, Clone, Serialize, Deserialize)]
pub struct CertMetrics {
    /// Total certificates successfully issued since process start.
    pub issued_total: u64,
    /// Total certificate issuances that failed since process start.
    pub failed_total: u64,
    /// Issuances currently in progress.
    pub in_flight: u64,
}

impl CertMetrics {
    /// Call when a new issuance request begins.
    pub fn record_start(&mut self) {
        self.in_flight += 1;
    }

    /// Call when an issuance completes successfully.
    pub fn record_success(&mut self) {
        self.issued_total += 1;
        self.in_flight = self.in_flight.saturating_sub(1);
    }

    /// Call when an issuance fails terminally (after all retries).
    pub fn record_failure(&mut self) {
        self.failed_total += 1;
        self.in_flight = self.in_flight.saturating_sub(1);
    }
}

// ── Traefik ACME JSON polling ──────────────────────────────────────────────

/// Minimal deserialization of Traefik's `acme.json`.
/// Structure: `{ "<resolver>": { "Certificates": [ { "domain": { "main": "...", "sans": [...] } } ] } }`
#[derive(Debug, Deserialize)]
struct AcmeFile {
    #[serde(flatten)]
    resolvers: std::collections::HashMap<String, AcmeResolver>,
}

#[derive(Debug, Deserialize)]
struct AcmeResolver {
    #[serde(rename = "Certificates")]
    certificates: Option<Vec<AcmeCert>>,
}

#[derive(Debug, Deserialize)]
struct AcmeCert {
    domain: AcmeDomain,
}

#[derive(Debug, Deserialize)]
struct AcmeDomain {
    main: String,
    #[serde(default)]
    sans: Vec<String>,
}

/// Returns `true` if `acme.json` at `path` contains a certificate entry whose
/// `domain.main` or `domain.sans` matches `hostname` under the given `resolver`.
///
/// Returns `false` (not an error) when:
/// - The file does not exist yet (Traefik hasn't started the challenge)
/// - The resolver section is absent
/// - The certificates array is empty
///
/// Returns `Err` only for genuine I/O or JSON parse failures.
pub fn acme_cert_present(path: &Path, resolver: &str, hostname: &str) -> anyhow::Result<bool> {
    let raw = match std::fs::read_to_string(path) {
        Ok(s) => s,
        Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(false),
        Err(e) => return Err(e.into()),
    };
    if raw.trim().is_empty() {
        return Ok(false);
    }
    let acme: AcmeFile = serde_json::from_str(&raw)?;
    let Some(res) = acme.resolvers.get(resolver) else {
        return Ok(false);
    };
    let certs = res.certificates.as_deref().unwrap_or(&[]);
    Ok(certs
        .iter()
        .any(|c| c.domain.main == hostname || c.domain.sans.iter().any(|s| s == hostname)))
}

// ── Cell model: CertRecord + StateStore ───────────────────────────────────

/// A single certificate tracked by the reconciler.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CertRecord {
    pub hostname: String,
    pub state: CertState,
    /// Retry attempt counter, reset to 0 on successful state advance.
    pub attempt: u32,
    pub last_updated: DateTime<Utc>,
}

impl CertRecord {
    pub fn new(hostname: String) -> Self {
        Self {
            hostname,
            state: CertState::Requested,
            attempt: 0,
            last_updated: Utc::now(),
        }
    }
}

/// File-backed collection of `CertRecord`s.
/// Single-writer assumption — concurrent cell runs must not share the same file.
#[derive(Debug, Default, Serialize, Deserialize)]
pub struct StateStore {
    pub certs: Vec<CertRecord>,
}

impl StateStore {
    /// Load from a JSON file. Returns an empty store when the file does not exist.
    pub fn load(path: &Path) -> anyhow::Result<Self> {
        match std::fs::read_to_string(path) {
            Ok(s) => Ok(serde_json::from_str(&s)?),
            Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(Self::default()),
            Err(e) => Err(e.into()),
        }
    }

    /// Write to a JSON file atomically (write to `.tmp`, then rename).
    pub fn save(&self, path: &Path) -> anyhow::Result<()> {
        if let Some(parent) = path.parent() {
            std::fs::create_dir_all(parent)?;
        }
        let tmp = path.with_extension("json.tmp");
        std::fs::write(&tmp, serde_json::to_string_pretty(self)?)?;
        std::fs::rename(&tmp, path)?;
        Ok(())
    }
}

/// The next state in the forward-progress pipeline for `Proceed` decisions.
/// Returns `None` for terminal or wait states.
pub fn next_state(current: CertState) -> Option<CertState> {
    match current {
        CertState::Requested => Some(CertState::PolicyValidated),
        CertState::PolicyValidated => Some(CertState::DnsChallengePrepared),
        CertState::DnsChallengePrepared => Some(CertState::DnsChallengePropagating),
        CertState::DnsChallengePropagating => Some(CertState::Issuing),
        CertState::Issuing => Some(CertState::Issued),
        CertState::RenewalDue => Some(CertState::Renewing),
        CertState::Renewing => Some(CertState::Issued),
        // Terminal / wait — no automatic next state
        CertState::Issued | CertState::Failed | CertState::Revoked => None,
    }
}

/// Advance a `CertRecord` by one reconcile step using the given backoff config.
///
/// Returns a `StateTransition` when the state changes, `None` when no change
/// is made (Done, Wait, or backoff not yet exhausted without a retry-triggering
/// condition).
///
/// Mutates `record` in place — caller must save the `StateStore` afterward.
pub fn advance_record(record: &mut CertRecord, config: &BackoffConfig) -> Option<StateTransition> {
    match reconcile_step(record.state) {
        ReconcileDecision::Done => None,
        ReconcileDecision::Wait => None,
        ReconcileDecision::Proceed => {
            let to = next_state(record.state)?;
            let from = record.state;
            record.state = to;
            record.attempt = 0;
            record.last_updated = Utc::now();
            Some(StateTransition {
                from,
                to,
                reason: "reconcile: proceed".to_string(),
                at: record.last_updated,
            })
        }
        ReconcileDecision::RetryOrAbandon => {
            let bs = BackoffState {
                attempt: record.attempt,
            };
            if next_backoff_delay(config, &bs).is_some() {
                record.attempt += 1;
                record.last_updated = Utc::now();
                // No state change — caller schedules retry after backoff delay
                None
            } else {
                // Retries exhausted — transition to Failed
                let from = record.state;
                record.state = CertState::Failed;
                record.last_updated = Utc::now();
                Some(StateTransition {
                    from,
                    to: CertState::Failed,
                    reason: "reconcile: retries exhausted".to_string(),
                    at: record.last_updated,
                })
            }
        }
    }
}

fn state_name(state: CertState) -> &'static str {
    match state {
        CertState::Requested => "requested",
        CertState::PolicyValidated => "policy_validated",
        CertState::DnsChallengePrepared => "dns_challenge_prepared",
        CertState::DnsChallengePropagating => "dns_challenge_propagating",
        CertState::Issuing => "issuing",
        CertState::Issued => "issued",
        CertState::RenewalDue => "renewal_due",
        CertState::Renewing => "renewing",
        CertState::Failed => "failed",
        CertState::Revoked => "revoked",
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn dry_run_transitions_to_issued() {
        let req = IssuanceRequest {
            request_id: "req-1".to_string(),
            hostname: "app.example.com".to_string(),
            actor: "test".to_string(),
            requested_at: Utc::now(),
        };

        let report = dry_run_issue(&req).expect("dry run should pass");
        let last = report.transitions.last().expect("has transitions");
        assert_eq!(last.to, CertState::Issued);
    }

    #[test]
    fn invalid_hostname_fails_validation() {
        let req = IssuanceRequest {
            request_id: "req-2".to_string(),
            hostname: "localhost".to_string(),
            actor: "test".to_string(),
            requested_at: Utc::now(),
        };

        let result = validate_request(&req);
        assert!(result.is_err());
    }

    #[test]
    fn hostname_with_invalid_characters_fails_validation() {
        let req = IssuanceRequest {
            request_id: "req-3".to_string(),
            hostname: "bad host.example.com".to_string(),
            actor: "test".to_string(),
            requested_at: Utc::now(),
        };

        let result = validate_request(&req);
        assert!(matches!(
            result,
            Err(RequestValidationError::InvalidHostnameFormat)
        ));
    }

    #[test]
    fn hostname_with_leading_dash_label_fails_validation() {
        let req = IssuanceRequest {
            request_id: "req-4".to_string(),
            hostname: "-bad.example.com".to_string(),
            actor: "test".to_string(),
            requested_at: Utc::now(),
        };

        let result = validate_request(&req);
        assert!(matches!(
            result,
            Err(RequestValidationError::InvalidHostnameFormat)
        ));
    }

    #[test]
    fn hostname_with_valid_dns_labels_passes_validation() {
        let req = IssuanceRequest {
            request_id: "req-5".to_string(),
            hostname: "api-1.example.com".to_string(),
            actor: "test".to_string(),
            requested_at: Utc::now(),
        };

        assert!(validate_request(&req).is_ok());
    }

    #[test]
    fn static_config_contains_cloudflare_acme_resolver() {
        let cfg = TraefikStaticConfigInput {
            email: "ops@acme.test".to_string(),
            resolver_name: "cloudflare".to_string(),
            cert_storage_file: "/var/traefik/certs/acme.json".to_string(),
            cloudflare_token_env: "CF_DNS_API_TOKEN".to_string(),
            log_level: "DEBUG".to_string(),
        };
        let rendered = render_traefik_static_yaml(&cfg);
        assert!(rendered.contains("certificatesResolvers:"));
        assert!(rendered.contains("provider: cloudflare"));
        assert!(rendered.contains("CF_DNS_API_TOKEN"));
    }

    #[test]
    fn generated_labels_include_https_certresolver() {
        let cfg = TraefikRouterConfig {
            service_name: "issuer-api".to_string(),
            hostname: "issuer.example.com".to_string(),
            service_port: 8080,
            resolver_name: "cloudflare".to_string(),
        };
        let labels = generate_router_labels(&cfg);
        assert!(labels
            .iter()
            .any(|x| x == "traefik.http.routers.issuer-api-https.tls.certresolver=cloudflare"));
        assert!(labels
            .iter()
            .any(|x| { x == "traefik.http.services.issuer-api.loadbalancer.server.port=8080" }));
    }

    // ── M3 reconcile tests ──────────────────────────────────────────────────

    #[test]
    fn reconcile_requested_returns_proceed() {
        assert_eq!(
            reconcile_step(CertState::Requested),
            ReconcileDecision::Proceed
        );
    }

    #[test]
    fn reconcile_policy_validated_returns_proceed() {
        assert_eq!(
            reconcile_step(CertState::PolicyValidated),
            ReconcileDecision::Proceed
        );
    }

    #[test]
    fn reconcile_dns_challenge_prepared_returns_proceed() {
        assert_eq!(
            reconcile_step(CertState::DnsChallengePrepared),
            ReconcileDecision::Proceed
        );
    }

    #[test]
    fn reconcile_dns_challenge_propagating_returns_wait() {
        assert_eq!(
            reconcile_step(CertState::DnsChallengePropagating),
            ReconcileDecision::Wait
        );
    }

    #[test]
    fn reconcile_issuing_returns_proceed() {
        assert_eq!(
            reconcile_step(CertState::Issuing),
            ReconcileDecision::Proceed
        );
    }

    #[test]
    fn reconcile_issued_returns_done() {
        assert_eq!(reconcile_step(CertState::Issued), ReconcileDecision::Done);
    }

    #[test]
    fn reconcile_renewal_due_returns_proceed() {
        assert_eq!(
            reconcile_step(CertState::RenewalDue),
            ReconcileDecision::Proceed
        );
    }

    #[test]
    fn reconcile_failed_returns_retry_or_abandon() {
        assert_eq!(
            reconcile_step(CertState::Failed),
            ReconcileDecision::RetryOrAbandon
        );
    }

    #[test]
    fn reconcile_revoked_returns_done() {
        assert_eq!(reconcile_step(CertState::Revoked), ReconcileDecision::Done);
    }

    // ── M3 backoff tests ────────────────────────────────────────────────────

    #[test]
    fn backoff_attempt_0_returns_base_delay() {
        let cfg = BackoffConfig {
            max_retries: 3,
            base_delay_ms: 1_000,
            max_delay_ms: 60_000,
        };
        let state = BackoffState { attempt: 0 };
        assert_eq!(
            next_backoff_delay(&cfg, &state),
            Some(Duration::from_millis(1_000))
        );
    }

    #[test]
    fn backoff_attempt_1_doubles_delay() {
        let cfg = BackoffConfig {
            max_retries: 3,
            base_delay_ms: 1_000,
            max_delay_ms: 60_000,
        };
        let state = BackoffState { attempt: 1 };
        assert_eq!(
            next_backoff_delay(&cfg, &state),
            Some(Duration::from_millis(2_000))
        );
    }

    #[test]
    fn backoff_caps_at_max_delay() {
        let cfg = BackoffConfig {
            max_retries: 10,
            base_delay_ms: 1_000,
            max_delay_ms: 5_000,
        };
        let state = BackoffState { attempt: 5 };
        assert_eq!(
            next_backoff_delay(&cfg, &state),
            Some(Duration::from_millis(5_000))
        );
    }

    #[test]
    fn backoff_returns_none_after_max_retries() {
        let cfg = BackoffConfig {
            max_retries: 3,
            base_delay_ms: 1_000,
            max_delay_ms: 60_000,
        };
        let state = BackoffState { attempt: 3 };
        assert_eq!(next_backoff_delay(&cfg, &state), None);
    }

    // ── M3 metrics tests ────────────────────────────────────────────────────

    #[test]
    fn metrics_record_start_increments_in_flight() {
        let mut m = CertMetrics::default();
        m.record_start();
        assert_eq!(m.in_flight, 1);
    }

    #[test]
    fn metrics_record_success_increments_issued_and_decrements_in_flight() {
        let mut m = CertMetrics::default();
        m.record_start();
        m.record_success();
        assert_eq!(m.issued_total, 1);
        assert_eq!(m.in_flight, 0);
    }

    #[test]
    fn metrics_record_failure_increments_failed_and_decrements_in_flight() {
        let mut m = CertMetrics::default();
        m.record_start();
        m.record_failure();
        assert_eq!(m.failed_total, 1);
        assert_eq!(m.in_flight, 0);
    }

    #[test]
    fn metrics_in_flight_does_not_underflow() {
        let mut m = CertMetrics::default();
        m.record_success(); // no prior start — saturating_sub must not panic
        assert_eq!(m.in_flight, 0);
    }

    #[test]
    fn metrics_snapshot_serializes_to_json() {
        let mut m = CertMetrics::default();
        m.record_start();
        m.record_success();
        let json = serde_json::to_string(&m).expect("serialization must succeed");
        assert!(json.contains("issued_total"));
        assert!(json.contains("in_flight"));
    }

    #[test]
    fn state_store_save_creates_missing_parent_directories() {
        let unique = format!("tencrypt-test-{}", Uuid::new_v4());
        let path = std::env::temp_dir()
            .join(unique)
            .join("nested")
            .join("certs.json");

        let store = StateStore {
            certs: vec![CertRecord::new("app.example.com".to_string())],
        };

        store
            .save(&path)
            .expect("save should create parent directories");
        let reloaded = StateStore::load(&path).expect("load should succeed");
        assert_eq!(reloaded.certs.len(), 1);

        std::fs::remove_dir_all(
            path.parent()
                .and_then(|p| p.parent())
                .expect("test directory layout should exist"),
        )
        .expect("temporary directory cleanup should succeed");
    }

    // ── AAA-2 jittered backoff tests ────────────────────────────────────────

    #[test]
    fn jittered_backoff_zero_jitter_matches_plain() {
        let cfg = BackoffConfig {
            max_retries: 5,
            base_delay_ms: 1_000,
            max_delay_ms: 60_000,
        };
        let state = BackoffState { attempt: 0 };
        assert_eq!(
            next_backoff_delay_jittered(&cfg, &state, 0),
            next_backoff_delay(&cfg, &state)
        );
    }

    #[test]
    fn jittered_backoff_adds_jitter_ms() {
        let cfg = BackoffConfig {
            max_retries: 5,
            base_delay_ms: 1_000,
            max_delay_ms: 60_000,
        };
        let state = BackoffState { attempt: 0 };
        assert_eq!(
            next_backoff_delay_jittered(&cfg, &state, 250),
            Some(Duration::from_millis(1_250))
        );
    }

    #[test]
    fn jittered_backoff_caps_at_max_delay() {
        let cfg = BackoffConfig {
            max_retries: 5,
            base_delay_ms: 1_000,
            max_delay_ms: 3_000,
        };
        let state = BackoffState { attempt: 2 }; // 4_000ms base
        assert_eq!(
            next_backoff_delay_jittered(&cfg, &state, 500),
            Some(Duration::from_millis(3_000))
        );
    }

    #[test]
    fn jittered_backoff_returns_none_when_exhausted() {
        let cfg = BackoffConfig::default();
        let state = BackoffState {
            attempt: cfg.max_retries,
        };
        assert_eq!(next_backoff_delay_jittered(&cfg, &state, 100), None);
    }

    // ── AAA-2 domain policy tests ────────────────────────────────────────────

    #[test]
    fn resolve_backoff_returns_default_when_no_policies() {
        let default = BackoffConfig::default();
        let result = resolve_backoff("foo.example.com", &[], &default);
        assert_eq!(result.max_retries, default.max_retries);
    }

    #[test]
    fn resolve_backoff_returns_override_for_matching_hostname() {
        let default = BackoffConfig::default();
        let override_cfg = BackoffConfig {
            max_retries: 2,
            base_delay_ms: 500,
            max_delay_ms: 5_000,
        };
        let policies = vec![DomainPolicy {
            hostname_pattern: "strict.example.com".to_string(),
            backoff: Some(override_cfg.clone()),
        }];
        let result = resolve_backoff("strict.example.com", &policies, &default);
        assert_eq!(result.max_retries, 2);
        assert_eq!(result.base_delay_ms, 500);
    }

    #[test]
    fn resolve_backoff_skips_none_override() {
        let default = BackoffConfig::default();
        let policies = vec![DomainPolicy {
            hostname_pattern: "opt-out.example.com".to_string(),
            backoff: None,
        }];
        let result = resolve_backoff("opt-out.example.com", &policies, &default);
        assert_eq!(result.max_retries, default.max_retries);
    }

    // ── Cell model tests ────────────────────────────────────────────────────

    #[test]
    fn state_store_load_returns_empty_when_file_missing() {
        let store = StateStore::load(Path::new("/tmp/tencrypt-nonexistent-state.json"))
            .expect("missing file should return empty store");
        assert!(store.certs.is_empty());
    }

    #[test]
    fn state_store_roundtrip() {
        let dir = std::env::temp_dir();
        let path = dir.join("tencrypt-test-state.json");
        let mut store = StateStore::default();
        store
            .certs
            .push(CertRecord::new("rt.example.com".to_string()));
        store.save(&path).expect("save should succeed");
        let loaded = StateStore::load(&path).expect("load should succeed");
        assert_eq!(loaded.certs.len(), 1);
        assert_eq!(loaded.certs[0].hostname, "rt.example.com");
        drop(loaded);
        std::fs::remove_file(&path).ok();
    }

    #[test]
    fn advance_record_proceeds_from_requested_to_policy_validated() {
        let mut record = CertRecord::new("adv.example.com".to_string());
        let cfg = BackoffConfig::default();
        let t = advance_record(&mut record, &cfg).expect("should produce a transition");
        assert_eq!(t.from, CertState::Requested);
        assert_eq!(t.to, CertState::PolicyValidated);
        assert_eq!(record.state, CertState::PolicyValidated);
        assert_eq!(record.attempt, 0);
    }

    #[test]
    fn advance_record_returns_none_for_issued() {
        let mut record = CertRecord {
            hostname: "done.example.com".to_string(),
            state: CertState::Issued,
            attempt: 0,
            last_updated: Utc::now(),
        };
        let cfg = BackoffConfig::default();
        assert!(advance_record(&mut record, &cfg).is_none());
        assert_eq!(record.state, CertState::Issued);
    }

    #[test]
    fn advance_record_returns_none_for_wait() {
        let mut record = CertRecord {
            hostname: "wait.example.com".to_string(),
            state: CertState::DnsChallengePropagating,
            attempt: 0,
            last_updated: Utc::now(),
        };
        let cfg = BackoffConfig::default();
        assert!(advance_record(&mut record, &cfg).is_none());
        assert_eq!(record.state, CertState::DnsChallengePropagating);
    }

    #[test]
    fn advance_record_increments_attempt_on_retry_or_abandon() {
        let mut record = CertRecord {
            hostname: "fail.example.com".to_string(),
            state: CertState::Failed,
            attempt: 0,
            last_updated: Utc::now(),
        };
        let cfg = BackoffConfig {
            max_retries: 3,
            base_delay_ms: 100,
            max_delay_ms: 1_000,
        };
        // First call: retries remain, increment attempt, no transition
        let t = advance_record(&mut record, &cfg);
        assert!(t.is_none());
        assert_eq!(record.attempt, 1);
        assert_eq!(record.state, CertState::Failed);
    }

    #[test]
    fn advance_record_transitions_to_failed_when_retries_exhausted() {
        let mut record = CertRecord {
            hostname: "exhaust.example.com".to_string(),
            state: CertState::Failed,
            attempt: 3,
            last_updated: Utc::now(),
        };
        let cfg = BackoffConfig {
            max_retries: 3,
            base_delay_ms: 100,
            max_delay_ms: 1_000,
        };
        let t = advance_record(&mut record, &cfg).expect("should transition to Failed");
        assert_eq!(t.from, CertState::Failed);
        assert_eq!(t.to, CertState::Failed);
    }
}