ignition-core 1.6.8

//! Unit tests of the pure backup decisions (24-05 Task 2).

use std::collections::{BTreeMap, BTreeSet};
use std::time::Duration;

use super::*;
use crate::client::deploy_api::SecretNameListing;
use crate::deploy::layout::RelPath;
use crate::deploy::transport::FileBlob;
use crate::deploy::tree::{RenderedTree, sha256_hex};
use crate::deploy::verify::{Expected, ExpectedResource, Remote};

const CANARY: &str = "PAIR-CANARY-TEXT";

fn signals(connected: bool, pending: bool, before: Option<i64>, now: Option<i64>) -> BackupSignals {
    BackupSignals {
        peer_connected: connected,
        sync_pending: pending,
        last_sync_before: before,
        last_sync_now: now,
        project_state: Some("Good".to_string()),
    }
}

fn check(id: BackupCheckId, status: CheckStatus, detail: &str) -> BackupCheck {
    BackupCheck {
        id,
        status,
        detail: detail.to_string(),
        findings: Vec::new(),
    }
}

fn content(
    status_mode: CheckStatus,
    status_rb: CheckStatus,
    status_files: CheckStatus,
) -> Vec<BackupCheck> {
    vec![
        check(BackupCheckId::Mode, status_mode, "mode"),
        check(BackupCheckId::ReadBack, status_rb, "read back"),
        check(BackupCheckId::Files, status_files, "files"),
    ]
}

/// A tick on which the backup has converged.
fn good() -> TickOutcome {
    TickOutcome {
        signals: Ok(signals(true, false, Some(100), Some(200))),
        exhausted: false,
        content: Some(content(
            CheckStatus::Pass,
            CheckStatus::Pass,
            CheckStatus::Pass,
        )),
        files_checked: 4,
        stable_ticks: 0,
    }
}

fn policy() -> BackupPolicy {
    BackupPolicy {
        budget: Duration::from_secs(10),
        interval: Duration::from_secs(5),
    }
}

// ---- wait_step ---------------------------------------------------------------

#[test]
fn wait_step_budget_is_inclusive() {
    let mut open = good();
    open.content = None;
    let policy = policy();
    // Elapsed equal to the budget has passed.
    assert_eq!(
        wait_step(&open, Duration::from_secs(10), &policy),
        PairWaitStep::TimedOut
    );
    // One millisecond less sleeps for that millisecond.
    assert_eq!(
        wait_step(&open, Duration::from_millis(9_999), &policy),
        PairWaitStep::Sleep(Duration::from_millis(1))
    );
    // A converged tick is Converged even at the budget.
    assert_eq!(
        wait_step(&good(), Duration::from_secs(10), &policy),
        PairWaitStep::Converged
    );
    // The sync-exhausted signal ends the wait at any time (D-10).
    let mut exhausted = open.clone();
    exhausted.exhausted = true;
    assert_eq!(
        wait_step(&exhausted, Duration::ZERO, &policy),
        PairWaitStep::Exhausted
    );
    assert_eq!(
        wait_step(&exhausted, Duration::from_secs(10), &policy),
        PairWaitStep::Exhausted
    );
}

#[test]
fn wait_step_sleep_is_clamped() {
    let mut open = good();
    open.content = None;
    // 2 s left of a 10 s budget, interval 5 s: sleep 2 s.
    assert_eq!(
        wait_step(&open, Duration::from_secs(8), &policy()),
        PairWaitStep::Sleep(Duration::from_secs(2))
    );
    // Plenty left: the interval.
    assert_eq!(
        wait_step(&open, Duration::ZERO, &policy()),
        PairWaitStep::Sleep(Duration::from_secs(5))
    );
    // An interval of zero sleeps 1 ms, never spins.
    let zero = BackupPolicy {
        budget: Duration::from_secs(10),
        interval: Duration::ZERO,
    };
    assert_eq!(
        wait_step(&open, Duration::from_secs(1), &zero),
        PairWaitStep::Sleep(Duration::from_millis(1))
    );
}

#[test]
fn the_default_policy_is_the_published_one() {
    let policy = BackupPolicy::default();
    assert_eq!(policy.budget, Duration::from_secs(BACKUP_TIMEOUT_DEFAULT_S));
    assert_eq!(policy.interval, Duration::from_secs(BACKUP_POLL_S));
    assert_eq!((BACKUP_TIMEOUT_DEFAULT_S, BACKUP_POLL_S), (300, 5));
}

// ---- converged ---------------------------------------------------------------

#[test]
fn converged_needs_every_clause() {
    // K1 a: the timestamp advance is a clause.
    assert_eq!(SYNC_PREDICATE, SyncPredicate::TimestampAdvanced);
    let p = SYNC_PREDICATE;
    assert!(converged(&good(), p));

    let mut t = good();
    t.signals = Ok(signals(false, false, Some(100), Some(200)));
    assert!(!converged(&t, p), "peer not connected");

    let mut t = good();
    t.signals = Ok(signals(true, true, Some(100), Some(200)));
    assert!(!converged(&t, p), "sync still pending");

    let mut t = good();
    t.signals = Ok(signals(true, false, Some(200), Some(200)));
    assert!(!converged(&t, p), "timestamp did not advance");

    let mut t = good();
    t.signals = Ok(signals(true, false, None, None));
    assert!(!converged(&t, p), "never synced is not an advance");

    for (mode, rb, files) in [
        (CheckStatus::Fail, CheckStatus::Pass, CheckStatus::Pass),
        (CheckStatus::Pass, CheckStatus::Fail, CheckStatus::Pass),
        (CheckStatus::Pass, CheckStatus::Pass, CheckStatus::Fail),
    ] {
        let mut t = good();
        t.content = Some(content(mode, rb, files));
        assert!(!converged(&t, p), "{mode:?} {rb:?} {files:?}");
    }

    let mut t = good();
    t.content = None;
    assert!(!converged(&t, p), "content not read yet");

    let mut t = good();
    t.signals = Err("network_error".to_string());
    assert!(!converged(&t, p), "status unreadable");

    // A warn (a REST-only backup's skipped file check) does not block.
    let mut t = good();
    t.content = Some(content(
        CheckStatus::Pass,
        CheckStatus::Pass,
        CheckStatus::Warn,
    ));
    assert!(converged(&t, p));
}

/// debug deploy-tag-redundancy-sync (T6b, T6c): after a master restart a
/// wedged pair reports `projectState` `Good` on both nodes while the backup's
/// `config-resource` sync stays stuck and its files differ. `projectState` is
/// recorded, never a condition (V9): a `Good` backup whose files check fails
/// is not converged, in the apply's wait and in `deploy verify` alike, and the
/// reason is `content_differs`.
#[test]
fn project_state_good_never_outweighs_a_failed_files_check() {
    let mut tick = good();
    tick.content = Some(content(
        CheckStatus::Pass,
        CheckStatus::Pass,
        CheckStatus::Fail,
    ));
    for state in ["Good", "SyncError"] {
        if let Ok(s) = tick.signals.as_mut() {
            s.project_state = Some(state.to_string());
        }
        assert!(!converged(&tick, SYNC_PREDICATE), "{state}");
        assert_eq!(
            classify(&tick),
            Some(UnsyncedReason::ContentDiffers),
            "{state}"
        );
        let applied = report(&tick, Some(100), 10, 10, None);
        assert!(!applied.converged, "{state}");
        assert_eq!(applied.project_state.as_deref(), Some(state));
        let verified = verify_report(&tick);
        assert!(!verified.converged, "{state}");
        assert_eq!(
            verified.reason,
            Some(UnsyncedReason::ContentDiffers),
            "{state}"
        );
        assert_eq!(verified.project_state.as_deref(), Some(state), "{state}");
    }
}

#[test]
fn a_stable_tick_count_is_the_other_predicate() {
    let mut t = good();
    // StableTicks ignores the timestamp and counts ticks (K1 b, not chosen).
    t.signals = Ok(signals(true, false, None, None));
    t.stable_ticks = 2;
    assert!(converged(&t, SyncPredicate::StableTicks(2)));
    t.stable_ticks = 1;
    assert!(!converged(&t, SyncPredicate::StableTicks(2)));
}

// ---- classify ----------------------------------------------------------------

#[test]
fn classify_precedence() {
    assert_eq!(classify(&good()), None);

    // unreadable status
    let mut t = good();
    t.signals = Err("network_error".to_string());
    t.exhausted = true;
    assert_eq!(classify(&t), Some(UnsyncedReason::Unreachable));

    // readable status, a file read that could not be answered
    let mut t = good();
    t.content = Some(vec![
        check(BackupCheckId::Mode, CheckStatus::Pass, "mode"),
        check(BackupCheckId::ReadBack, CheckStatus::Pass, "read back"),
        check(
            BackupCheckId::Files,
            CheckStatus::Fail,
            "unreachable: transport_failed",
        ),
    ]);
    assert_eq!(classify(&t), Some(UnsyncedReason::Unreachable));

    // exhausted beats disconnected
    let mut t = good();
    t.signals = Ok(signals(false, false, Some(100), Some(100)));
    t.exhausted = true;
    assert_eq!(classify(&t), Some(UnsyncedReason::SyncExhausted));

    // disconnected beats differing content
    let mut t = good();
    t.signals = Ok(signals(false, false, Some(100), Some(200)));
    t.content = Some(content(
        CheckStatus::Pass,
        CheckStatus::Pass,
        CheckStatus::Fail,
    ));
    assert_eq!(classify(&t), Some(UnsyncedReason::NotSyncing));

    // pending and a timestamp that did not move are both not syncing
    let mut t = good();
    t.signals = Ok(signals(true, true, Some(100), Some(200)));
    assert_eq!(classify(&t), Some(UnsyncedReason::NotSyncing));
    let mut t = good();
    t.signals = Ok(signals(true, false, Some(100), Some(100)));
    assert_eq!(classify(&t), Some(UnsyncedReason::NotSyncing));

    // linked, advanced, content differing
    let mut t = good();
    t.content = Some(content(
        CheckStatus::Pass,
        CheckStatus::Fail,
        CheckStatus::Pass,
    ));
    assert_eq!(classify(&t), Some(UnsyncedReason::ContentDiffers));
}

#[test]
fn the_report_follows_the_tick() {
    let mut t = good();
    t.content = Some(content(
        CheckStatus::Pass,
        CheckStatus::Pass,
        CheckStatus::Fail,
    ));
    let r = report(
        &t,
        Some(100),
        300,
        45,
        Some(ResyncRecord {
            ok: true,
            code: None,
        }),
    );
    assert!(!r.converged);
    assert_eq!(r.reason, Some(UnsyncedReason::ContentDiffers));
    assert_eq!((r.budget_s, r.waited_s, r.files_checked), (300, 45, 4));
    assert_eq!(
        (r.last_sync_before, r.last_sync_after),
        (Some(100), Some(200))
    );
    assert_eq!(r.project_state.as_deref(), Some("Good"));
    let ids: Vec<BackupCheckId> = r.checks.iter().map(|c| c.id).collect();
    assert_eq!(
        ids,
        [
            BackupCheckId::Redundancy,
            BackupCheckId::Mode,
            BackupCheckId::ReadBack,
            BackupCheckId::Files
        ]
    );

    let ok = report(&good(), Some(100), 300, 5, None);
    assert!(ok.converged);
    assert_eq!(ok.reason, None);

    // An unreadable status keeps the earlier sample and fails the redundancy check.
    let mut t = good();
    t.signals = Err("network_error".to_string());
    t.content = None;
    let r = report(&t, Some(100), 300, 300, None);
    assert_eq!(r.reason, Some(UnsyncedReason::Unreachable));
    assert_eq!((r.last_sync_before, r.last_sync_after), (Some(100), None));
    assert_eq!(r.checks.len(), 1);
    assert_eq!(r.checks[0].status, CheckStatus::Fail);
}

#[test]
fn verify_report_has_its_own_verdict() {
    // A backup that never synced (-1) and has no earlier sample is settled
    // for verify: the redundancy check is linked and not pending only.
    let mut t = good();
    t.signals = Ok(signals(true, false, None, None));
    let r = verify_report(&t);
    assert!(r.converged, "{r:?}");
    assert_eq!(r.reason, None);
    assert_eq!((r.budget_s, r.waited_s), (0, 0));
    assert!(r.resync.is_none());
    assert_eq!(r.last_sync_before, None);
    assert_eq!(r.last_sync_after, None);
    assert_eq!(r.checks[0].id, BackupCheckId::Redundancy);
    assert_eq!(r.checks[0].status, CheckStatus::Pass);
    assert_eq!(r.checks[0].detail, "peerConnected, no sync pending");

    // A warn (a REST-only backup's file check) does not fail it.
    let mut t = good();
    t.content = Some(content(
        CheckStatus::Pass,
        CheckStatus::Pass,
        CheckStatus::Warn,
    ));
    assert!(verify_report(&t).converged);

    // Not linked, or a sync still pending: not_syncing.
    for (connected, pending) in [(false, false), (true, true)] {
        let mut t = good();
        t.signals = Ok(signals(connected, pending, Some(1), Some(2)));
        let r = verify_report(&t);
        assert!(!r.converged);
        assert_eq!(r.reason, Some(UnsyncedReason::NotSyncing));
    }

    // A failed content check on a settled backup: content_differs.
    let mut t = good();
    t.content = Some(content(
        CheckStatus::Pass,
        CheckStatus::Fail,
        CheckStatus::Pass,
    ));
    assert_eq!(
        verify_report(&t).reason,
        Some(UnsyncedReason::ContentDiffers)
    );

    // A read that could not be answered outranks the rest.
    let mut t = good();
    t.signals = Ok(signals(false, false, None, None));
    t.content = Some(vec![unreachable_check(
        BackupCheckId::Files,
        "network_error",
    )]);
    assert_eq!(verify_report(&t).reason, Some(UnsyncedReason::Unreachable));
    let mut t = good();
    t.signals = Err("network_error".to_string());
    assert_eq!(verify_report(&t).reason, Some(UnsyncedReason::Unreachable));

    // No content evaluated is never converged.
    let mut t = good();
    t.content = None;
    assert!(!verify_report(&t).converged);
}

// ---- content checks ----------------------------------------------------------

fn expected(mode: &str) -> Expected {
    Expected {
        mode: mode.to_string(),
        resources: vec![ExpectedResource::exact(
            "ignition",
            "tag-provider",
            Some("default"),
            serde_json::json!({"a": 1}),
        )],
        not_checked: Vec::new(),
        declared_ids: Vec::new(),
        tag_units: Vec::new(),
        probes: Vec::new(),
    }
}

#[test]
fn the_mode_and_read_back_checks_reuse_the_verify_evaluation() {
    let mode = mode_backup_check(&expected("prod"), Some("prod"));
    assert_eq!(
        (mode.id, mode.status),
        (BackupCheckId::Mode, CheckStatus::Pass)
    );
    let mode = mode_backup_check(&expected("prod"), None);
    assert_eq!(mode.status, CheckStatus::Fail);

    let mut warnings = Vec::new();
    let mut seen = BTreeMap::new();
    seen.insert(
        "ignition/tag-provider/default".to_string(),
        Remote::Config(serde_json::json!({"a": 1})),
    );
    let (rb, checked) = read_back_backup_check(&expected("prod"), &seen, &mut warnings);
    assert_eq!(
        (rb.id, rb.status, checked),
        (BackupCheckId::ReadBack, CheckStatus::Pass, 1)
    );
    seen.insert(
        "ignition/tag-provider/default".to_string(),
        Remote::Config(serde_json::json!({"a": 2})),
    );
    let (rb, _) = read_back_backup_check(&expected("prod"), &seen, &mut warnings);
    assert_eq!(rb.status, CheckStatus::Fail);
    assert_eq!(rb.findings.len(), 1);
}

#[test]
fn an_unanswered_read_is_an_unreachable_check() {
    let c = unreachable_check(BackupCheckId::Files, "transport_failed");
    assert_eq!((c.id, c.status), (BackupCheckId::Files, CheckStatus::Fail));
    assert_eq!(c.detail, "unreachable: transport_failed");
}

// ---- compare_files -----------------------------------------------------------

fn path(s: &str) -> RelPath {
    RelPath::new(s).expect("a path")
}

fn blob(bytes: &[u8]) -> FileBlob {
    FileBlob {
        bytes: bytes.to_vec(),
        sha256: sha256_hex(bytes),
    }
}

fn files(entries: &[(&str, &[u8])]) -> BTreeMap<RelPath, FileBlob> {
    entries.iter().map(|(p, b)| (path(p), blob(b))).collect()
}

fn set(paths: &[&str]) -> BTreeSet<RelPath> {
    paths.iter().map(|p| path(p)).collect()
}

#[test]
fn compare_files_bytes() {
    let master = files(&[
        ("m/t/a/config.json", b"{\"x\":1}"),
        ("m/t/b/config.json", b"{\"y\":2}"),
        ("m/t/c/config.json", b"line\n"),
        ("m/t/d/config.json", b"{}"),
    ]);
    let present = set(&[
        "m/t/a/config.json",
        "m/t/b/config.json",
        "m/t/c/config.json",
        "m/t/d/config.json",
    ]);

    // identical: clean, `checked` is the present-set size
    let same = compare_files(&master, &master.clone(), &present, &BTreeSet::new());
    assert_eq!(same.checked, 4);
    assert!(same.differing.is_empty() && same.missing.is_empty() && same.unexpected.is_empty());
    assert_eq!(files_check(&same).status, CheckStatus::Pass);

    // one byte differing, one file missing on the backup, one delete still there
    let backup = files(&[
        ("m/t/a/config.json", b"{\"x\":2}"),
        ("m/t/c/config.json", b"line\n"),
        ("m/t/d/config.json", b"{}"),
        ("m/t/gone/config.json", b"{}"),
    ]);
    let cmp = compare_files(&master, &backup, &present, &set(&["m/t/gone/config.json"]));
    assert_eq!(cmp.checked, 4);
    assert_eq!(cmp.differing, ["m/t/a/config.json"]);
    assert_eq!(cmp.missing, ["m/t/b/config.json"]);
    assert_eq!(cmp.unexpected, ["m/t/gone/config.json"]);
    let check = files_check(&cmp);
    assert_eq!(check.status, CheckStatus::Fail);
    let subjects: Vec<&str> = check.findings.iter().map(|f| f.subject.as_str()).collect();
    assert_eq!(
        subjects,
        [
            "m/t/a/config.json",
            "m/t/b/config.json",
            "m/t/gone/config.json"
        ]
    );

    // CRLF against LF differs: no newline normalization (edge DPAIR-03 encoding)
    let crlf = files(&[("m/t/c/config.json", b"line\r\n")]);
    let cmp = compare_files(
        &files(&[("m/t/c/config.json", b"line\n")]),
        &crlf,
        &set(&["m/t/c/config.json"]),
        &BTreeSet::new(),
    );
    assert_eq!(cmp.differing, ["m/t/c/config.json"]);

    // paths come out byte-sorted whatever the map order
    let master = files(&[("z/a", b"1"), ("a/z", b"1"), ("m/m", b"1")]);
    let backup = files(&[("z/a", b"2"), ("a/z", b"2"), ("m/m", b"2")]);
    let cmp = compare_files(
        &master,
        &backup,
        &set(&["z/a", "a/z", "m/m"]),
        &BTreeSet::new(),
    );
    assert_eq!(cmp.differing, ["a/z", "m/m", "z/a"]);
}

#[test]
fn compare_files_lists_a_bounded_number_of_paths() {
    let many: Vec<String> = (0..25)
        .map(|i| format!("m/t/u{i:02}/config.json"))
        .collect();
    let master: BTreeMap<RelPath, FileBlob> = many.iter().map(|p| (path(p), blob(b"1"))).collect();
    let cmp = compare_files(
        &master,
        &BTreeMap::new(),
        &many.iter().map(|p| path(p)).collect(),
        &BTreeSet::new(),
    );
    assert_eq!(cmp.missing.len(), 25);
    let check = files_check(&cmp);
    assert!(check.findings.len() <= crate::error::APPLY_LIST_SHOWN);
    assert!(check.detail.contains("25"), "{}", check.detail);
}

#[test]
fn only_resource_json_is_compared_as_a_document() {
    // K2 b (revised): the `resource.json` row, with exactly the two
    // modification-stamp members left out, and the `unary-resource.json`
    // row of quick 261102-cef (P8), with none left out. No other file.
    assert_eq!(
        FILE_NORMALIZATION,
        &[
            (
                "resource.json",
                &[
                    "attributes.lastModification",
                    "attributes.lastModificationSignature"
                ][..]
            ),
            ("unary-resource.json", &[][..]),
        ]
    );

    let compact = br#"{"uuid":"u","lastModificationSignature":"s","attributes":{"a":1,"b":[1,2]}}"#;
    let reordered =
        br#"{"lastModificationSignature":"s","uuid":"u","attributes":{"b":[1,2],"a":1}}"#;
    let pretty = b"{\n  \"uuid\": \"u\",\n  \"lastModificationSignature\": \"s\",\n  \"attributes\": {\n    \"a\": 1,\n    \"b\": [1, 2]\n  }\n}\n";
    let changed = br#"{"uuid":"u","lastModificationSignature":"T","attributes":{"a":1,"b":[1,2]}}"#;
    let p = "ignition/tag-provider/default/resource.json";
    let one = |master: &[u8], backup: &[u8]| {
        compare_files(
            &files(&[(p, master)]),
            &files(&[(p, backup)]),
            &set(&[p]),
            &BTreeSet::new(),
        )
    };

    assert!(one(compact, reordered).differing.is_empty(), "reordered");
    assert!(one(compact, pretty).differing.is_empty(), "pretty-printed");
    assert_eq!(one(compact, changed).differing, [p], "a changed value");
    assert_eq!(one(compact, b"not json").differing, [p], "not a document");
    assert_eq!(one(b"not json", b"not json ").differing, [p]);
    // a member missing is a different document, not an excluded member
    assert_eq!(one(compact, br#"{"uuid":"u"}"#).differing, [p]);
    // the top-level `lastModificationSignature` is not one of the two
    // excluded members (those live under `attributes`)
    assert_eq!(one(compact, changed).differing, [p], "top-level signature");

    // every other file stays byte-equal, even when the JSON is equal
    let c = "ignition/tag-provider/default/config.json";
    let cmp = compare_files(
        &files(&[(c, br#"{"a":1,"b":2}"#)]),
        &files(&[(c, br#"{"b":2,"a":1}"#)]),
        &set(&[c]),
        &BTreeSet::new(),
    );
    assert_eq!(cmp.differing, [c], "config.json is byte-equal only");

    // only the file NAME is a row: a directory named resource.json is not
    let d = "ignition/tag-provider/resource.json/config.json";
    let cmp = compare_files(
        &files(&[(d, br#"{"a":1,"b":2}"#)]),
        &files(&[(d, br#"{"b":2,"a":1}"#)]),
        &set(&[d]),
        &BTreeSet::new(),
    );
    assert_eq!(cmp.differing, [d]);
}

/// quick 261102-cef (P8, debug Evidence C1): the backup pretty-prints the
/// marker of a tag folder it received in `core`; the parsed document is what
/// counts, and every member does.
#[test]
fn core_key_files_check_unary_parsed() {
    let p = "ignition/tag-definition/default/Deploy/unary-resource.json";
    let compact = br#"{"scope":"G","version":1,"restricted":false,"overridable":true,"files":["tags.json"],"attributes":{"config":{}}}"#;
    let pretty = serde_json::to_vec_pretty(
        &serde_json::from_slice::<serde_json::Value>(compact).expect("json"),
    )
    .expect("pretty");
    let changed = br#"{"scope":"G","version":1,"restricted":false,"overridable":true,"files":["tags.json","udts.json"],"attributes":{"config":{}}}"#;
    let one = |master: &[u8], backup: &[u8]| {
        compare_files(
            &files(&[(p, master)]),
            &files(&[(p, backup)]),
            &set(&[p]),
            &BTreeSet::new(),
        )
    };
    assert!(one(compact, &pretty).differing.is_empty(), "pretty-printed");
    assert_eq!(one(compact, changed).differing, [p], "a changed member");
    assert_eq!(one(compact, b"not json").differing, [p], "not a document");
    // `tags.json` stays byte-equal
    let t = "ignition/tag-definition/default/Deploy/tags.json";
    let cmp = compare_files(
        &files(&[(t, br#"[{"a":1}]"#)]),
        &files(&[(t, b"[\n  {\n    \"a\": 1\n  }\n]")]),
        &set(&[t]),
        &BTreeSet::new(),
    );
    assert_eq!(cmp.differing, [t]);
}

const STAMPED: &[u8] = br#"{"scope":"G","version":1,"attributes":{"uuid":"u","enabled":true,"lastModification":{"actor":"admin","timestamp":"2026-10-01T00:00:00Z"},"lastModificationSignature":"stale"}}"#;

fn one_resource(master: &[u8], backup: &[u8]) -> FileComparison {
    let p = "ignition/tag-provider/default/resource.json";
    compare_files(
        &files(&[(p, master)]),
        &files(&[(p, backup)]),
        &set(&[p]),
        &BTreeSet::new(),
    )
}

#[test]
fn resource_json_ignores_only_the_two_modification_stamp_members() {
    // the backup re-saved the unit: a new actor, timestamp and signature
    let resaved = br#"{"scope":"G","version":1,"attributes":{"uuid":"u","enabled":true,"lastModification":{"actor":"external","timestamp":"2026-10-21T10:00:00Z"},"lastModificationSignature":"fresh"}}"#;
    assert!(one_resource(STAMPED, resaved).differing.is_empty());

    // one side carries no stamp at all
    let bare = br#"{"scope":"G","version":1,"attributes":{"uuid":"u","enabled":true}}"#;
    assert!(one_resource(STAMPED, bare).differing.is_empty());
    assert!(one_resource(bare, STAMPED).differing.is_empty());

    // stamps differ AND the layout differs: still the same document
    let pretty = b"{\n \"attributes\": {\"enabled\": true, \"uuid\": \"u\",\n \"lastModificationSignature\": \"fresh\"},\n \"version\": 1, \"scope\": \"G\"}\n";
    assert!(one_resource(STAMPED, pretty).differing.is_empty());
}

#[test]
fn resource_json_still_differs_on_every_other_member() {
    let p = "ignition/tag-provider/default/resource.json";
    let enabled = br#"{"scope":"G","version":1,"attributes":{"uuid":"u","enabled":false,"lastModification":{"actor":"admin","timestamp":"2026-10-01T00:00:00Z"},"lastModificationSignature":"stale"}}"#;
    assert_eq!(one_resource(STAMPED, enabled).differing, [p], "enabled");
    let uuid = br#"{"scope":"G","version":1,"attributes":{"uuid":"v","enabled":true}}"#;
    assert_eq!(one_resource(STAMPED, uuid).differing, [p], "uuid");
    let top = br#"{"scope":"G","version":2,"attributes":{"uuid":"u","enabled":true}}"#;
    assert_eq!(
        one_resource(STAMPED, top).differing,
        [p],
        "top-level member"
    );
    let extra = br#"{"scope":"G","version":1,"attributes":{"uuid":"u","enabled":true,"other":1}}"#;
    assert_eq!(one_resource(STAMPED, extra).differing, [p], "extra member");
    // a stamp-like member nested elsewhere is not excluded
    let nested = br#"{"scope":"G","version":1,"attributes":{"uuid":"u","enabled":true,"x":{"lastModification":1}}}"#;
    let nested2 = br#"{"scope":"G","version":1,"attributes":{"uuid":"u","enabled":true,"x":{"lastModification":2}}}"#;
    assert_eq!(one_resource(nested, nested2).differing, [p], "nested");
    let top_stamp = br#"{"lastModification":1,"attributes":{}}"#;
    let top_stamp2 = br#"{"lastModification":2,"attributes":{}}"#;
    assert_eq!(
        one_resource(top_stamp, top_stamp2).differing,
        [p],
        "top-level lastModification is not attributes.lastModification"
    );
}

#[test]
fn resource_json_without_an_object_attributes_is_compared_as_before() {
    let p = "ignition/tag-provider/default/resource.json";
    // no attributes at all
    assert!(
        one_resource(br#"{"a":1}"#, br#"{"a":1}"#)
            .differing
            .is_empty()
    );
    assert_eq!(one_resource(br#"{"a":1}"#, br#"{"a":2}"#).differing, [p]);
    // attributes of a non-object type: no panic, whole-document equality
    for bad in ["null", "[]", "[1,2]", "\"s\"", "7", "true"] {
        let a = format!(r#"{{"attributes":{bad}}}"#);
        let b = format!(r#"{{"attributes":{bad} }}"#);
        assert!(
            one_resource(a.as_bytes(), b.as_bytes())
                .differing
                .is_empty()
        );
        let c = br#"{"attributes":{"enabled":true}}"#;
        assert_eq!(one_resource(a.as_bytes(), c).differing, [p], "{bad}");
    }
    // the document itself is not an object
    assert!(one_resource(b"[1]", b"[ 1 ]").differing.is_empty());
    assert_eq!(one_resource(b"[1]", b"[2]").differing, [p]);
}

#[test]
fn other_files_do_not_ignore_the_modification_stamp() {
    let c = "ignition/tag-provider/default/config.json";
    let cmp = compare_files(
        &files(&[(c, STAMPED)]),
        &files(&[(
            c,
            br#"{"scope":"G","version":1,"attributes":{"uuid":"u","enabled":true}}"#,
        )]),
        &set(&[c]),
        &BTreeSet::new(),
    );
    assert_eq!(cmp.differing, [c]);
}

// ---- unreadable units (quick-261029-ms3) ----------------------------------------

/// A node read with `paths` (mode collection) unreadable at 262 characters.
fn unreadable_on(paths: &[&str]) -> UnreadableUnits {
    let observed = ObservedFiles {
        unreadable: paths
            .iter()
            .map(|p| crate::deploy::transport::Unreadable {
                collection: "mode".to_string(),
                path: path(p),
                length: 262,
            })
            .collect(),
        ..ObservedFiles::default()
    };
    unreadable_units(&observed)
}

const DEEP_TAGS: &str = "ignition/tag-definition/default/Deep/tags.json";
const DEEP_UNIT: &str = "ignition/tag-definition/default/Deep";

#[test]
fn unreadable_rendered_unit_on_the_backup_is_red_never_missing() {
    let master = files(&[(DEEP_TAGS, b"[]")]);
    let backup = files(&[]);
    let present = set(&[DEEP_TAGS]);
    let mut cmp = compare_files(&master, &backup, &present, &BTreeSet::new());
    assert_eq!(cmp.missing, [DEEP_TAGS], "before: read as absent");
    let none = UnreadableUnits::new();
    exempt_unreadable(
        &mut cmp,
        &present,
        &BTreeSet::new(),
        &[("master", &none), ("backup", &unreadable_on(&[DEEP_TAGS]))],
    );
    assert!(cmp.missing.is_empty(), "never absent: {cmp:?}");
    assert_eq!(
        cmp.unreadable,
        [("backup".to_string(), DEEP_UNIT.to_string())]
    );
    let check = files_check(&cmp);
    assert_eq!(check.status, CheckStatus::Fail);
    assert_eq!(check.findings.len(), 1, "{:?}", check.findings);
    assert_eq!(check.findings[0].code, "unreadable");
    assert_eq!(check.findings[0].subject, DEEP_UNIT);
    assert!(check.findings[0].detail.contains("backup"), "{check:?}");
}

#[test]
fn unreadable_deleted_path_on_the_backup_is_red_never_gone() {
    // The apply deleted the unit; the backup still holds it at a path its
    // sftp server cannot read. Before: no `file_unexpected`, a false green.
    let gone = "ignition/tag-definition/default/Gone/tags.json";
    let master = files(&[]);
    let backup = files(&[]);
    let absent = set(&[gone]);
    let mut cmp = compare_files(&master, &backup, &BTreeSet::new(), &absent);
    assert_eq!(files_check(&cmp).status, CheckStatus::Pass, "before");
    exempt_unreadable(
        &mut cmp,
        &BTreeSet::new(),
        &absent,
        &[
            ("master", &UnreadableUnits::new()),
            ("backup", &unreadable_on(&[gone])),
        ],
    );
    let check = files_check(&cmp);
    assert_eq!(check.status, CheckStatus::Fail, "{check:?}");
    assert_eq!(check.findings[0].code, "unreadable");
}

#[test]
fn unreadable_rendered_unit_on_the_master_is_red_never_differing() {
    let backup = files(&[(DEEP_TAGS, b"[]")]);
    let present = set(&[DEEP_TAGS]);
    let mut cmp = compare_files(&files(&[]), &backup, &present, &BTreeSet::new());
    assert_eq!(cmp.differing, [DEEP_TAGS], "before");
    exempt_unreadable(
        &mut cmp,
        &present,
        &BTreeSet::new(),
        &[
            ("master", &unreadable_on(&[DEEP_TAGS])),
            ("backup", &UnreadableUnits::new()),
        ],
    );
    assert!(cmp.differing.is_empty());
    assert_eq!(
        cmp.unreadable,
        [("master".to_string(), DEEP_UNIT.to_string())]
    );
    assert_eq!(files_check(&cmp).status, CheckStatus::Fail);
}

#[test]
fn unreadable_unit_outside_both_sets_is_a_warning_only() {
    let other = "ignition/tag-definition/MQTT/Edge/tags.json";
    let master = files(&[(DEEP_TAGS, b"[]")]);
    let present = set(&[DEEP_TAGS]);
    let mut cmp = compare_files(&master, &master.clone(), &present, &BTreeSet::new());
    exempt_unreadable(
        &mut cmp,
        &present,
        &BTreeSet::new(),
        &[
            ("master", &UnreadableUnits::new()),
            ("backup", &unreadable_on(&[other])),
        ],
    );
    assert!(cmp.unreadable.is_empty());
    assert_eq!(
        cmp.unreadable_unchecked,
        [(
            "backup".to_string(),
            "ignition/tag-definition/MQTT/Edge".to_string()
        )]
    );
    let check = files_check(&cmp);
    assert_eq!(check.status, CheckStatus::Warn, "{check:?}");
    assert!(check.findings.is_empty());
    assert!(
        check.detail.contains("ignition/tag-definition/MQTT/Edge"),
        "{}",
        check.detail
    );
    assert_eq!(
        unreadable_warnings(&cmp),
        [
            "backup: ignition/tag-definition/MQTT/Edge is unreadable on the host and is neither rendered nor deleted; it was not compared"
        ]
    );
}

#[test]
fn unreadable_nothing_leaves_the_comparison_as_it_was() {
    let master = files(&[(DEEP_TAGS, b"[]")]);
    let present = set(&[DEEP_TAGS]);
    let before = compare_files(&master, &files(&[]), &present, &BTreeSet::new());
    let mut after = before.clone();
    let none = UnreadableUnits::new();
    exempt_unreadable(
        &mut after,
        &present,
        &BTreeSet::new(),
        &[("master", &none), ("backup", &none)],
    );
    assert_eq!(after, before);
    assert_eq!(files_check(&after), files_check(&before));
}

#[test]
fn a_rest_only_backup_skips_the_file_check_with_a_warning() {
    let c = files_skipped_check();
    assert_eq!((c.id, c.status), (BackupCheckId::Files, CheckStatus::Warn));
    assert!(c.detail.contains("nodes.backup"), "{}", c.detail);
    assert!(!c.detail.contains("D-0"), "no planning ids in user text");
}

// ---- secrets -----------------------------------------------------------------

fn refs(pairs: &[(&str, &str)]) -> BTreeSet<(String, String)> {
    pairs
        .iter()
        .map(|(p, n)| ((*p).to_string(), (*n).to_string()))
        .collect()
}

fn names(list: &[&str]) -> SecretNameListing {
    SecretNameListing::Listed(list.iter().map(|n| (*n).to_string()).collect())
}

fn listings(entries: Vec<(&str, SecretNameListing)>) -> BTreeMap<String, SecretNameListing> {
    entries
        .into_iter()
        .map(|(p, l)| (p.to_string(), l))
        .collect()
}

fn missing(report: &SecretNodeReport) -> Vec<(&str, &str)> {
    report
        .missing
        .iter()
        .map(|m| (m.provider.as_str(), m.name.as_str()))
        .collect()
}

#[test]
fn check_secret_names_cases() {
    let none = BTreeSet::new();
    let referenced = refs(&[("vault", "b"), ("file", "z"), ("file", "a"), ("vault", "a")]);

    // all present
    let all = listings(vec![
        ("file", names(&["a", "z"])),
        ("vault", names(&["a", "b"])),
    ]);
    let r = check_secret_names("master", &referenced, &all, &none, SecretCheckMode::Verify);
    assert_eq!(r.status, CheckStatus::Pass);
    assert_eq!((r.providers_checked, r.node.as_str()), (2, "master"));
    assert!(r.missing.is_empty() && r.unverifiable.is_empty());

    // missing names are sorted by provider then name
    let some = listings(vec![("file", names(&["a"])), ("vault", names(&[]))]);
    let r = check_secret_names("backup", &referenced, &some, &none, SecretCheckMode::Verify);
    assert_eq!(r.status, CheckStatus::Fail);
    assert_eq!(missing(&r), [("file", "z"), ("vault", "a"), ("vault", "b")]);

    // K7: a provider that cannot list is unverifiable, a warn when nothing is missing
    let cannot = listings(vec![
        ("file", names(&["a", "z"])),
        ("vault", SecretNameListing::CannotList),
    ]);
    let r = check_secret_names(
        "master",
        &referenced,
        &cannot,
        &none,
        SecretCheckMode::Verify,
    );
    assert_eq!(r.status, CheckStatus::Warn);
    assert_eq!(r.unverifiable, ["vault"]);
    assert!(r.missing.is_empty());
    // ... and a failure when something else is missing
    let both = listings(vec![
        ("file", names(&["a"])),
        ("vault", SecretNameListing::CannotList),
    ]);
    let r = check_secret_names("master", &referenced, &both, &none, SecretCheckMode::Verify);
    assert_eq!(r.status, CheckStatus::Fail);
    assert_eq!(
        (missing(&r), r.unverifiable.as_slice()),
        (vec![("file", "z")], ["vault".to_string()].as_slice())
    );

    // K8: a provider the render declares that answers 404 is skipped in a preflight ...
    let declared: BTreeSet<String> = ["vault".to_string()].into();
    let absent = listings(vec![
        ("file", names(&["a", "z"])),
        ("vault", SecretNameListing::NoProvider),
    ]);
    let r = check_secret_names(
        "backup",
        &referenced,
        &absent,
        &declared,
        SecretCheckMode::Preflight,
    );
    assert!(r.missing.is_empty(), "{:?}", r.missing);
    assert_eq!(r.skipped_providers, ["vault"]);
    assert_eq!(r.status, CheckStatus::Pass);
    // ... but verify requires it
    let r = check_secret_names(
        "backup",
        &referenced,
        &absent,
        &declared,
        SecretCheckMode::Verify,
    );
    assert_eq!(missing(&r), [("vault", "a"), ("vault", "b")]);
    assert!(r.skipped_providers.is_empty());
    assert_eq!(r.status, CheckStatus::Fail);
    // a provider the render does NOT declare that is absent is missing in a preflight
    let r = check_secret_names(
        "backup",
        &referenced,
        &absent,
        &none,
        SecretCheckMode::Preflight,
    );
    assert_eq!(missing(&r), [("vault", "a"), ("vault", "b")]);
    // a provider with no listing at all reads as absent
    let r = check_secret_names(
        "backup",
        &refs(&[("ghost", "x")]),
        &BTreeMap::new(),
        &none,
        SecretCheckMode::Verify,
    );
    assert_eq!(missing(&r), [("ghost", "x")]);

    // no reference at all: pass, nothing checked
    let r = check_secret_names(
        "master",
        &BTreeSet::new(),
        &BTreeMap::new(),
        &none,
        SecretCheckMode::Verify,
    );
    assert_eq!((r.status, r.providers_checked), (CheckStatus::Pass, 0));

    // a provider listing {"secrets":[]} makes every name referenced in it missing
    let empty = listings(vec![("file", names(&[]))]);
    let r = check_secret_names(
        "master",
        &refs(&[("file", "a"), ("file", "b")]),
        &empty,
        &none,
        SecretCheckMode::Verify,
    );
    assert_eq!(missing(&r), [("file", "a"), ("file", "b")]);
    assert_eq!(r.providers_checked, 1);

    // K9: exact, case-sensitive
    let cased = listings(vec![("file", names(&["name"]))]);
    let r = check_secret_names(
        "master",
        &refs(&[("file", "Name")]),
        &cased,
        &none,
        SecretCheckMode::Verify,
    );
    assert_eq!(missing(&r), [("file", "Name")]);
}

#[test]
fn secret_reports_list_the_master_before_the_backup() {
    let none = BTreeSet::new();
    let a = check_secret_names(
        "backup",
        &BTreeSet::new(),
        &BTreeMap::new(),
        &none,
        SecretCheckMode::Verify,
    );
    let b = check_secret_names(
        "master",
        &BTreeSet::new(),
        &BTreeMap::new(),
        &none,
        SecretCheckMode::Verify,
    );
    let report = SecretNamesReport::ordered(vec![a, b]);
    let nodes: Vec<&str> = report.nodes.iter().map(|n| n.node.as_str()).collect();
    assert_eq!(nodes, ["master", "backup"]);
}

#[test]
fn a_secret_report_is_names_only_in_the_published_shape() {
    let none = BTreeSet::new();
    let some = listings(vec![
        ("file", names(&["a"])),
        ("vault", SecretNameListing::CannotList),
    ]);
    let r = check_secret_names(
        "master",
        &refs(&[("file", "z"), ("vault", "q")]),
        &some,
        &none,
        SecretCheckMode::Verify,
    );
    let value = serde_json::to_value(&r).expect("serializes");
    let mut keys: Vec<&str> = value
        .as_object()
        .expect("object")
        .keys()
        .map(String::as_str)
        .collect();
    keys.sort_unstable();
    // V7: skipped_providers is omitted when empty
    assert_eq!(
        keys,
        [
            "missing",
            "node",
            "providers_checked",
            "status",
            "unverifiable"
        ]
    );
    assert_eq!(
        value["missing"][0],
        serde_json::json!({"provider": "file", "name": "z"})
    );
    assert_eq!(value["unverifiable"], serde_json::json!(["vault"]));
    assert_eq!(value["status"], "fail");
}

fn tree(entries: &[(&str, &str)]) -> RenderedTree {
    RenderedTree {
        files: entries
            .iter()
            .map(|(p, b)| (path(p), b.as_bytes().to_vec()))
            .collect(),
    }
}

#[test]
fn collect_referenced_secrets_and_declared_providers() {
    let reference = |p: &str, n: &str| {
        format!(r#"{{"type":"Referenced","data":{{"providerName":"{p}","secretName":"{n}"}}}}"#)
    };
    let t = tree(&[
        (
            "ignition/database-connection/main/config.json",
            &format!(r#"{{"password":{}}}"#, reference("file", "db-pass")),
        ),
        (
            "ignition/smtp/mail/config.json",
            &format!(
                r#"{{"a":[{},{}]}}"#,
                reference("vault", "k"),
                reference("file", "db-pass")
            ),
        ),
        (
            "ignition/tag-provider/default/tags.json",
            &format!(
                r#"{{"tags":[{{"p":{}}}]}}"#,
                reference("file", "tag-secret")
            ),
        ),
        (
            "ignition/secret-provider/file/config.json",
            r#"{"profile":{"type":"INTERNAL"}}"#,
        ),
        ("ignition/secret-provider/file/resource.json", "{}"),
        ("ignition/secret-provider/vault/config.json", "{}"),
        ("ignition/secret-provider/vault/secret", "not json at all"),
        ("ignition/secret-provider", "{}"),
        ("ignition/other/x/notes.txt", "Referenced providerName"),
        ("ignition/other/x/broken.json", "{not json"),
        (
            "ignition/other/x/lookalike.json",
            r#"{"type":"referenced","data":{"providerName":"p","secretName":"n"}}"#,
        ),
    ]);
    assert_eq!(
        collect_referenced_secrets(&t),
        refs(&[("file", "db-pass"), ("file", "tag-secret"), ("vault", "k")])
    );
    let providers: Vec<String> = declared_secret_providers(&t).into_iter().collect();
    assert_eq!(providers, ["file", "vault"]);
}

#[test]
fn secrets_defined_by_render_reads_names_from_declared_providers_only() {
    // WR-04: a name the render itself adds to a provider it declares is
    // defined by the deploy. Names only; a value in the entry is not read.
    let t = tree(&[
        (
            "ignition/secret-provider/file/config.json",
            r#"{"profile":{"type":"file"},"settings":{"files":{"New":{"filePath":"/tmp/New","fileType":"CLEARTEXT"},"b":{}}}}"#,
        ),
        (
            "ignition/secret-provider/internal/config.json",
            r#"{"profile":{"type":"internal"},"settings":{"secrets":{"k":{"ciphertext":"PAIR-CANARY-TEXT"}}}}"#,
        ),
        // not JSON, no settings, wrong member types: nothing defined
        ("ignition/secret-provider/broken/config.json", "{not json"),
        ("ignition/secret-provider/empty/config.json", "{}"),
        (
            "ignition/secret-provider/odd/config.json",
            r#"{"settings":{"files":["x"],"secrets":"y"}}"#,
        ),
        // outside a provider's own config.json
        (
            "ignition/secret-provider/file/resource.json",
            r#"{"settings":{"files":{"Sneaky":{}}}}"#,
        ),
        (
            "ignition/other/x/config.json",
            r#"{"settings":{"files":{"Elsewhere":{}}}}"#,
        ),
    ]);
    assert_eq!(
        secrets_defined_by_render(&t),
        refs(&[("file", "New"), ("file", "b"), ("internal", "k")])
    );
    assert!(secrets_defined_by_render(&tree(&[])).is_empty());
}

// ---- gateway text ------------------------------------------------------------

#[test]
fn report_carries_no_gateway_text() {
    // A status whose members carry gateway text: a `lastSyncError`, an unknown
    // member and a hostile `projectState`.
    let status: RedundancyStatusWire = serde_json::from_value(serde_json::json!({
        "role": "Backup", "projectState": format!("Good\n{CANARY}{}", "x".repeat(200)),
        "activityLevel": "Cold", "peerConnected": true, "syncPending": false,
        "uptime": -1, "lastSyncTimestamp": 200,
        "lastSyncError": CANARY, "somethingNew": {"message": CANARY}
    }))
    .expect("parses");
    let signals = BackupSignals::from_status(Some(100), &status);
    // `projectState` is kept only when it is a short status word.
    assert_eq!(signals.project_state, None);
    let mut ordinary = status.clone();
    ordinary.project_state = Some("Out of Date".to_string());
    assert_eq!(
        BackupSignals::from_status(Some(100), &ordinary)
            .project_state
            .as_deref(),
        Some("Out of Date")
    );
    let signals = BackupSignals {
        project_state: Some("Good".to_string()),
        ..signals
    };

    // The log line that signals exhaustion carries the canary: the matcher
    // answers a bool and nothing else.
    let line = format!(
        "[config-resource] State sync request failed. Sync operation has failed 5 time(s), which exceeds the max attempt count. No further synchronization attempts will be made. redundant-provider=config-resource {CANARY}"
    );
    assert!(is_sync_exhausted(&line));

    let tick = TickOutcome {
        signals: Ok(signals),
        exhausted: is_sync_exhausted(&line),
        content: Some(vec![unreachable_check(
            BackupCheckId::Files,
            "transport_failed",
        )]),
        files_checked: 0,
        stable_ticks: 0,
    };
    let rendered = serde_json::to_string(&report(&tick, Some(100), 300, 10, None)).expect("json");
    // the ordinary status word is kept
    assert!(rendered.contains("Good"), "{rendered}");
    assert!(!rendered.contains(CANARY), "{rendered}");

    // the secret-name report holds names and providers only
    let r = check_secret_names(
        "master",
        &refs(&[("file", "a")]),
        &listings(vec![("file", names(&[]))]),
        &BTreeSet::new(),
        SecretCheckMode::Verify,
    );
    assert!(
        !serde_json::to_string(&SecretNamesReport { nodes: vec![r] })
            .expect("json")
            .contains(CANARY)
    );
}

// ---- is_sync_exhausted -------------------------------------------------------

#[test]
fn is_sync_exhausted_matches_the_spike_line() {
    // The verbatim line of the spike's results/RIG.md (24-RESEARCH, 24-02 plan).
    let spike = "E [R.S.SyncManager] [config-resource] State sync request failed. Sync operation has failed 5 time(s), which exceeds the max attempt count. No further synchronization attempts will be made. redundant-provider=config-resource";
    assert!(is_sync_exhausted(spike));
    // the message alone, as the logs route returns it
    assert!(is_sync_exhausted(
        "State sync request failed. Sync operation has failed 5 time(s), which exceeds the max attempt count. No further synchronization attempts will be made."
    ));
    // the other lines of the same incident do not match
    assert!(!is_sync_exhausted(
        "Active deployment mode differs from peer node!"
    ));
    assert!(!is_sync_exhausted(
        "[config-resource] After applying diff, new local system snapshot did not match remote system snapshot"
    ));
    assert!(!is_sync_exhausted(""));
}