subc-daemon 0.48.3

Embeddable subc daemon: bootstrap, module supervision, and opaque-byte splice routing.
Documentation
use super::*;

#[tokio::test]
async fn park_ownership_approved_release_failure_is_latched() {
    approved_park_release_failure(false).await;
}

#[tokio::test]
async fn park_ownership_deferred_approved_release_failure_is_latched() {
    approved_park_release_failure(true).await;
}

async fn approved_park_release_failure(defer_first: bool) {
    let fixture = legacy().await;
    entry(&fixture.rig, "protocol", json!("none"));
    let first = fixture.rig.stamp(&fixture.owner);
    fixture
        .handler
        .reconcile_exit_for_test(fixture.owner.clone(), first)
        .await
        .unwrap();
    fixture.module.set_enabled(false).await.unwrap();
    entry(&fixture.rig, "drain_timeout_ms", json!(0));
    let store = fixture
        .rig
        .path
        .join("store")
        .join(crate::config_store::STORE_FILE);
    if !defer_first {
        std::fs::write(&store, b"tampered").unwrap();
        assert!(fixture.owner.check_recovery().unwrap());
    }
    let generation = approve(
        &fixture.rig,
        &fixture.owner,
        if defer_first {
            PlanKind::Change
        } else {
            PlanKind::Stamp
        },
    );
    let approved = fixture
        .owner
        .authority_exit_configuration(generation)
        .unwrap()
        .unwrap()
        .3;
    let gate = fixture.owner.module_gate("a").unwrap();
    let held = if defer_first {
        Some(gate.lock_owned().await)
    } else {
        None
    };
    fixture
        .handler
        .reconcile_exit_for_test(fixture.owner.clone(), generation)
        .await
        .unwrap();
    if defer_first {
        assert_eq!(fixture.module.state().unwrap(), ModuleState::Restarting);
        std::fs::write(&store, b"tampered").unwrap();
        assert!(fixture.owner.check_recovery().unwrap());
        drop(held);
        until(|| fixture.module.state().unwrap() == ModuleState::Stopped).await;
        let recovery = approve(&fixture.rig, &fixture.owner, PlanKind::Stamp);
        fixture
            .handler
            .reconcile_exit_for_test(fixture.owner.clone(), recovery)
            .await
            .unwrap();
    } else {
        drop(held);
    }
    assert_eq!(fixture.module.state().unwrap(), ModuleState::Stopped);
    let (daemon, saved) = fixture.module.config_hold_origin_for_test();
    assert!(!daemon);
    assert_eq!(saved, Some(approved.identity()));
    fixture.owner.fail_delivery_spawns_for_module_for_test("a");
    let failure = fixture.module.release_config_hold().await.unwrap_err();
    assert!(failure.to_string().contains("delivery_spawn_exhausted"));
    assert_eq!(fixture.module.state().unwrap(), ModuleState::Failed);
    assert!(
        fixture
            .module
            .has_approved_start_failure(&approved.identity()),
        "a failed release must retain the approval that owned the parked attempt"
    );
    assert_eq!(fixture.module.retry_deadline_for_test(), None);
    fixture.owner.clear_delivery_spawn_failure_for_test();
    let next = approve(&fixture.rig, &fixture.owner, PlanKind::Change);
    let next_approved = fixture
        .owner
        .authority_exit_configuration(next)
        .unwrap()
        .unwrap()
        .3;
    assert!(!next_approved.changes_entry("a"));
    fixture
        .handler
        .reconcile_exit_for_test(fixture.owner.clone(), next)
        .await
        .unwrap();
    let after = fixture.module.status().unwrap();
    let latched = fixture
        .module
        .has_approved_start_failure(&next_approved.identity());
    drop(fixture.primary);
    fixture.module.stop().await.unwrap();
    assert!(after.process_alive, "{after:?}");
    assert!(!latched);
}

#[tokio::test]
async fn park_ownership_daemon_attempt_has_no_approval() {
    let fixture = gated_claim_fixture().await;
    entry(&fixture.rig, "drain_timeout_ms", json!(0));
    let generation = approve(&fixture.rig, &fixture.owner, PlanKind::Change);
    let approved = fixture
        .owner
        .authority_exit_configuration(generation)
        .unwrap()
        .unwrap()
        .3;
    fixture
        .module
        .reconcile_approved_configuration(approved)
        .await
        .unwrap();
    fixture
        .owner
        .complete_authority_exit_for_test(generation)
        .unwrap();
    std::fs::write(
        fixture
            .rig
            .path
            .join("store")
            .join(crate::config_store::STORE_FILE),
        b"tampered",
    )
    .unwrap();
    assert!(fixture.owner.check_recovery().unwrap());
    fixture
        .module
        .config_delivery_failed(fixture.primary.connection_id(&fixture.handler), true)
        .await
        .unwrap();
    until(|| fixture.module.state().unwrap() == ModuleState::Stopped).await;
    let origin = fixture.module.config_hold_origin_for_test();
    drop(fixture.primary);
    fixture.module.stop().await.unwrap();
    assert!(
        origin.0,
        "a delivery-failure recovery belongs to the daemon"
    );
    assert!(origin.1.is_none(), "it cannot borrow the latest approval");
}

async fn without_advancing_time<T>(future: impl std::future::Future<Output = T>) -> T {
    let started = std::time::Instant::now();
    tokio::pin!(future);
    loop {
        tokio::select! {
            biased;
            result = &mut future => return result,
            _ = tokio::task::yield_now() => {
                assert!(started.elapsed() < Duration::from_secs(10), "real process or blocking work did not complete");
            }
        }
    }
}

#[tokio::test]
async fn delivery_failure_release_retry_is_timed_from_its_own_failure() {
    let fixture = gated_claim_fixture().await;
    entry(&fixture.rig, "drain_timeout_ms", json!(0));
    let configured = approve(&fixture.rig, &fixture.owner, PlanKind::Change);
    let approved = fixture
        .owner
        .authority_exit_configuration(configured)
        .unwrap()
        .unwrap()
        .3;
    fixture
        .module
        .reconcile_approved_configuration(approved)
        .await
        .unwrap();
    fixture
        .owner
        .complete_authority_exit_for_test(configured)
        .unwrap();
    let connection = fixture.primary.connection_id(&fixture.handler);
    std::fs::write(
        fixture
            .rig
            .path
            .join("store")
            .join(crate::config_store::STORE_FILE),
        b"tampered",
    )
    .unwrap();
    assert!(fixture.owner.check_recovery().unwrap());
    fixture
        .module
        .config_delivery_failed(connection, true)
        .await
        .unwrap();
    until(|| fixture.module.state().unwrap() == ModuleState::Stopped).await;
    assert!(!fixture.module.status().unwrap().process_alive);
    tokio::time::pause();
    tokio::time::advance(Duration::from_secs(3600)).await;
    let generation = approve(&fixture.rig, &fixture.owner, PlanKind::Unstamp);
    fixture
        .owner
        .complete_authority_exit_for_test(generation)
        .unwrap();
    let mut missing = spec(&fixture.rig);
    missing.program = fixture.rig.path.join("missing-program");
    without_advancing_time(
        fixture
            .module
            .update_configuration(missing, Default::default(), None),
    )
    .await
    .unwrap();
    let failed_at = tokio::time::Instant::now();
    assert!(without_advancing_time(fixture.module.release_config_hold())
        .await
        .is_err());
    assert_eq!(tokio::time::Instant::now(), failed_at);
    let deadline = fixture.module.retry_deadline_for_test();
    let after_release = fixture.module.status().unwrap();
    without_advancing_time(fixture.module.config_exit_hold())
        .await
        .unwrap();
    let after_barrier = fixture.module.status().unwrap();
    drop(fixture.primary);
    fixture.module.stop().await.unwrap();
    assert_eq!(
        deadline,
        Some(failed_at + Duration::from_secs(600)),
        "a long park cannot backdate the release failure or skip a retry tier"
    );
    assert_eq!(after_release.state, ModuleState::Failed);
    assert_eq!(
        after_barrier.spawn_generation,
        after_release.spawn_generation
    );
}

#[tokio::test]
async fn terminal_config_exit_failure_cannot_be_revived_by_a_hold_release() {
    let fixture = gated_claim_fixture().await;
    entry(&fixture.rig, "drain_timeout_ms", json!(0));
    let configured = approve(&fixture.rig, &fixture.owner, PlanKind::Change);
    let approved = fixture
        .owner
        .authority_exit_configuration(configured)
        .unwrap()
        .unwrap()
        .3;
    fixture
        .module
        .reconcile_approved_configuration(approved)
        .await
        .unwrap();
    fixture
        .owner
        .complete_authority_exit_for_test(configured)
        .unwrap();
    std::fs::write(
        fixture
            .rig
            .path
            .join("store")
            .join(crate::config_store::STORE_FILE),
        b"tampered",
    )
    .unwrap();
    assert!(fixture.owner.check_recovery().unwrap());
    fixture
        .module
        .config_delivery_failed(fixture.primary.connection_id(&fixture.handler), true)
        .await
        .unwrap();
    until(|| fixture.module.state().unwrap() == ModuleState::Stopped).await;
    let reason = crate::supervise::ModuleStopReason {
        code: "config_exit_failed".into(),
        error_kind: "injected_terminal_failure".into(),
    };
    assert!(fixture.module.config_hold_origin_for_test().0);
    fixture
        .module
        .config_exit_failed(
            reason.clone(),
            fixture.owner.terminal_exit_approval_identity().unwrap(),
        )
        .unwrap();
    assert_eq!(fixture.module.config_hold_origin_for_test(), (false, None));
    let generation = approve(&fixture.rig, &fixture.owner, PlanKind::Unstamp);
    fixture
        .owner
        .complete_authority_exit_for_test(generation)
        .unwrap();
    let released = fixture.module.release_config_hold().await;
    let after = fixture.module.status().unwrap();
    drop(fixture.primary);
    fixture.module.stop().await.unwrap();
    assert!(
        !released.unwrap(),
        "a terminal configuration-exit failure is no longer parked"
    );
    assert_eq!(after.state, ModuleState::Stopped);
    assert_eq!(after.stop_reason, Some(reason));
    assert!(!after.process_alive);
}