shep 0.22.0

use super::*;

/// Wall-clock tests, skipped by every CI job but the serial `slow` one
///
/// The `restart` cases fork and exec a dog's binary inside a budget. A
/// probe that runs out of budget answers unknown, and unknown is silent,
/// so contention turns their subject into thin air.
///
/// The node case needs a real node to start and exit inside the budget.
mod slow {
    /// A probe budget no contention can exhaust
    ///
    /// Not a claim about how long a probe should take: far enough above
    /// this suite's contention that timing stops being a variable.
    const PROBE_BUDGET: Duration = Duration::from_secs(30);

    use super::*;

    /// A shepherd that restarts whatever it is asked to and lists an
    /// empty flock afterwards
    ///
    /// Enough for `restart` to reach the end, so a test asserting an
    /// empty stderr is asserting about the dog check.
    #[cfg(unix)]
    fn answering_a_restart(request: &Request) -> Response {
        match request {
            Request::Restart { .. } => Response::Restarted {
                accepted: Vec::new(),
                refused: Vec::new(),
            },
            _ => Response::Flock(Vec::new()),
        }
    }

    /// The ordering is read off stderr rather than off a clock: the
    /// shepherd refuses the restart, so its refusal lands on the same
    /// stream as the warning and the two offsets say which ran first.
    // Unix only because of the fixture: `adopted_dog` writes a
    // `#!/bin/sh` script, so on Windows the probe answers unknown and the
    // three `restarts_in_silence` tests would pass vacuously.
    #[cfg(unix)]
    #[tokio::test]
    async fn a_dog_whose_disk_binary_cannot_connect_is_warned_about_before_the_restart() {
        let dir = tempfile::tempdir().unwrap();
        let paths = adopted_dog(dir.path(), "log-rotate", &below_floor_answer());
        let sock = shep_client::testing::control_address(dir.path());
        let (client, _daemon) =
            fake_client_replying_err(&sock, RpcErrorCode::NotFound, "no such sheep").await;

        let mut out = Vec::new();
        let mut err = Vec::new();
        {
            let mut streams = Streams::bare(&mut out, &mut err, Format::Table);
            let args = SelectorArgs {
                selectors: vec!["log-rotate".into()],
            };
            let _ = restart_within(&client, &mut streams, Some(&paths), &args, PROBE_BUDGET).await;
        }

        let text = String::from_utf8(err).unwrap();
        let warning = text
            .find("dog_binary_skew")
            .unwrap_or_else(|| panic!("the restart must warn first: {text}"));
        let refusal = text
            .find("no such sheep")
            .unwrap_or_else(|| panic!("the restart must still be attempted: {text}"));
        assert!(
            warning < refusal,
            "the warning must reach the operator before the restart does: {text}"
        );
    }

    /// The operator knows which of the two fixes they meant, so the
    /// message names both and tells them what the restart just did.
    #[cfg(unix)]
    #[tokio::test]
    async fn the_restart_warning_names_both_numbers_and_both_ways_out() {
        let dir = tempfile::tempdir().unwrap();
        let paths = adopted_dog(dir.path(), "log-rotate", &below_floor_answer());
        let sock = shep_client::testing::control_address(dir.path());
        let (client, _envelopes) = fake_client_answering(&sock, answering_a_restart).await;

        let mut out = Vec::new();
        let mut err = Vec::new();
        {
            let mut streams = Streams::bare(&mut out, &mut err, Format::Table);
            let args = SelectorArgs {
                selectors: vec!["log-rotate".into()],
            };
            let _ = restart_within(&client, &mut streams, Some(&paths), &args, PROBE_BUDGET).await;
        }

        let text = String::from_utf8(err).unwrap();
        let disk = shep_core::protocol::MIN_SUPPORTED
            .saturating_sub(1)
            .to_string();
        let floor = shep_core::protocol::MIN_SUPPORTED.to_string();
        assert!(
            text.contains(&disk) && text.contains(&floor),
            "the warning names both numbers: {text}"
        );
        assert!(
            text.contains("Run a shep that accepts protocol") && text.contains("reinstall the dog"),
            "the warning names both ways out, and picks neither: {text}"
        );
        assert!(
            text.contains("log-rotate"),
            "the warning names the dog it is about: {text}"
        );
    }

    /// Every restart of every working flock is in this case, so a line
    /// here is a line an operator learns to skip.
    #[cfg(unix)]
    #[tokio::test]
    async fn a_dog_whose_disk_binary_is_current_restarts_in_silence() {
        let dir = tempfile::tempdir().unwrap();
        let answer = format!(
            "echo 'shep-log-rotate 0.1.3'\necho 'shep-protocol: {}'",
            shep_client::PROTOCOL_VERSION
        );
        let paths = adopted_dog(dir.path(), "log-rotate", &answer);
        let sock = shep_client::testing::control_address(dir.path());
        let (client, mut envelopes) = fake_client_answering(&sock, answering_a_restart).await;

        let mut out = Vec::new();
        let mut err = Vec::new();
        {
            let mut streams = Streams::bare(&mut out, &mut err, Format::Table);
            let args = SelectorArgs {
                selectors: vec!["log-rotate".into()],
            };
            let _ = restart_within(&client, &mut streams, Some(&paths), &args, PROBE_BUDGET).await;
        }

        assert_eq!(
            String::from_utf8(err).unwrap(),
            "",
            "a healthy dog restarts with nothing said about it"
        );
        assert_eq!(
            tokio::time::timeout(Duration::from_secs(5), envelopes.recv())
                .await
                .expect("restart must reach the wire; it hung instead of sending a request")
                .unwrap()
                .body,
            Request::Restart {
                selector: SelectorSpec::Name("log-rotate".into()),
            },
            "and it is still restarted"
        );
    }

    /// `docs/dogs.md` promises that not answering is never held against
    /// a dog. Unknown is not stale: it is the state `adopt` records.
    #[cfg(unix)]
    #[tokio::test]
    async fn a_dog_that_answers_nothing_restarts_in_silence() {
        let dir = tempfile::tempdir().unwrap();
        let paths = adopted_dog(
            dir.path(),
            "log-rotate",
            "echo 'shep-log-rotate does not understand --version' >&2\nexit 2",
        );
        let sock = shep_client::testing::control_address(dir.path());
        let (client, mut envelopes) = fake_client_answering(&sock, answering_a_restart).await;

        let mut out = Vec::new();
        let mut err = Vec::new();
        {
            let mut streams = Streams::bare(&mut out, &mut err, Format::Table);
            let args = SelectorArgs {
                selectors: vec!["log-rotate".into()],
            };
            let _ = restart_within(&client, &mut streams, Some(&paths), &args, PROBE_BUDGET).await;
        }

        assert_eq!(
            String::from_utf8(err).unwrap(),
            "",
            "a dog that does not answer is not a dog that is stale"
        );
        assert_eq!(
            tokio::time::timeout(Duration::from_secs(5), envelopes.recv())
                .await
                .expect("restart must reach the wire; it hung instead of sending a request")
                .unwrap()
                .body,
            Request::Restart {
                selector: SelectorSpec::Name("log-rotate".into()),
            },
            "and it is still restarted"
        );
    }

    /// The second shape of unknown, reaching a different line from the
    /// dog that answers nothing at all: there is an answer here, it just
    /// does not say.
    #[cfg(unix)]
    #[tokio::test]
    async fn a_dog_that_names_no_protocol_restarts_in_silence() {
        let dir = tempfile::tempdir().unwrap();
        let paths = adopted_dog(dir.path(), "log-rotate", "echo 'shep-log-rotate 0.1.3'");
        let sock = shep_client::testing::control_address(dir.path());
        let (client, mut envelopes) = fake_client_answering(&sock, answering_a_restart).await;

        let mut out = Vec::new();
        let mut err = Vec::new();
        {
            let mut streams = Streams::bare(&mut out, &mut err, Format::Table);
            let args = SelectorArgs {
                selectors: vec!["log-rotate".into()],
            };
            let _ = restart_within(&client, &mut streams, Some(&paths), &args, PROBE_BUDGET).await;
        }

        assert_eq!(
            String::from_utf8(err).unwrap(),
            "",
            "an unstated protocol is unknown, and unknown is not stale"
        );
        assert_eq!(
            tokio::time::timeout(Duration::from_secs(5), envelopes.recv())
                .await
                .expect("restart must reach the wire; it hung instead of sending a request")
                .unwrap()
                .body,
            Request::Restart {
                selector: SelectorSpec::Name("log-rotate".into()),
            },
            "and it is still restarted"
        );
    }

    /// A built-in dog's binary is the shepherd's own, so there is no
    /// second thing to drift. The config here has a stale adopted dog in
    /// it too, so the silence is about this name rather than about an
    /// empty `adopted_dogs`.
    #[cfg(unix)]
    #[tokio::test]
    async fn a_built_in_dog_is_never_asked_what_its_binary_speaks() {
        let dir = tempfile::tempdir().unwrap();
        let paths = adopted_dog(dir.path(), "log-rotate", &below_floor_answer());
        crate::commands::shep_toml::ShepToml::edit(&paths.daemon_config, |cfg| {
            cfg.enable_dog("metrics").unwrap();
        })
        .unwrap();
        let sock = shep_client::testing::control_address(dir.path());
        let (client, mut envelopes) = fake_client_answering(&sock, answering_a_restart).await;

        let mut out = Vec::new();
        let mut err = Vec::new();
        {
            let mut streams = Streams::bare(&mut out, &mut err, Format::Table);
            let args = SelectorArgs {
                selectors: vec!["metrics".into()],
            };
            let _ = restart_within(&client, &mut streams, Some(&paths), &args, PROBE_BUDGET).await;
        }

        assert_eq!(
            String::from_utf8(err).unwrap(),
            "",
            "a built-in dog has no binary of its own to be stale"
        );
        assert_eq!(
            tokio::time::timeout(Duration::from_secs(5), envelopes.recv())
                .await
                .expect("restart must reach the wire; it hung instead of sending a request")
                .unwrap()
                .body,
            Request::Restart {
                selector: SelectorSpec::Name("metrics".into()),
            },
            "and it is still restarted"
        );
    }

    /// A module that exports its config and leaves a detached node
    /// holding the pipes shep reads. The holder outlives `budget`
    /// threefold, or the reads would finish and the case pass for the
    /// wrong reason.
    fn held_pipe_module(budget: Duration) -> String {
        format!(
            "require('child_process')\
                 .spawn(process.execPath, ['-e', 'setTimeout(()=>{{}},{})'], \
                 {{ detached: true, stdio: 'inherit' }})\
                 .unref(); \
                 module.exports = {{ app: [] }};",
            (budget * 3).as_millis()
        )
    }

    /// node itself exits here: `detached` plus `unref` takes the child
    /// off node's event loop, and `stdio: inherit` hands it the pipes
    /// shep is reading, so only the reads run out of budget.
    #[test]
    fn a_js_flockfile_leaving_a_process_on_the_pipe_says_that_instead() {
        if !node_available() {
            return;
        }
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("flock.js");

        // A ladder, not one number: `run_bounded` waits for node and then
        // spends the rest on the held pipe, so the budget is this case's
        // runtime. A `Killed` verdict is read as a slow machine and retried
        // with more room; the last rung falls through.
        const BUDGETS: [Duration; 3] = [
            Duration::from_secs(5),
            Duration::from_secs(20),
            Duration::from_secs(80),
        ];

        for (attempt, budget) in BUDGETS.iter().copied().enumerate() {
            std::fs::write(&path, held_pipe_module(budget)).unwrap();
            let err = evaluate_js_flockfile(&path, budget).unwrap_err();
            let message = err.to_string();

            // node never got as far as exiting, so there was nothing
            // holding a pipe yet to say anything about. Give it more
            // room. The last rung falls through instead, so a machine
            // that cannot do it in 80s fails with the message it earned.
            if message.contains("still running") && attempt + 1 < BUDGETS.len() {
                continue;
            }

            assert_eq!(target_exit_code(&err), ExitCode::InvalidConfig);
            assert!(
                message.contains("left behind still holds the output"),
                "got: {message}"
            );
            assert!(
                !message.contains("killed"),
                "node exited on its own, so nothing was killed: {message}"
            );
            return;
        }
    }
}

/// Runs `shep restart kelpie` against a shepherd whose `kelpie` dog was spawned
/// with `spawned_env` while `shep.toml` now holds `entry_env`, handing back
/// the stderr and the requests the shepherd saw.
async fn restart_kelpie(entry_env: &str, spawned_env: &[(&str, &str)]) -> (String, Vec<Request>) {
    use crate::commands::dogs::entry_drift_support::{home_and_spawned, shepherd_running};

    let dir = tempfile::tempdir().unwrap();
    let (paths, spawned) = home_and_spawned(dir.path(), entry_env, spawned_env);
    let restarted = shep_core::protocol::ProcessInfo::builder(
        1,
        "kelpie",
        shep_core::status::ProcStatus::Online,
    )
    .dog(Some(shep_core::protocol::DogSource::Adopted {
        path: spawned.script.clone(),
        channel: false,
        grants: None,
    }))
    .build();
    let base = shepherd_running(spawned, shep_core::status::ProcStatus::Online);
    let sock = shep_client::testing::control_address(dir.path());
    let (client, mut envelopes) = fake_client_answering(&sock, move |req| match req {
        Request::Restart { .. } => Response::Restarted {
            accepted: vec![restarted.clone()],
            refused: Vec::new(),
        },
        other => base(other),
    })
    .await;

    let mut out = Vec::new();
    let mut err = Vec::new();
    {
        let mut streams = Streams::bare(&mut out, &mut err, Format::Table);
        let args = SelectorArgs {
            selectors: vec!["kelpie".into()],
        };
        let _ = restart_within(
            &client,
            &mut streams,
            Some(&paths),
            &args,
            Duration::from_millis(1),
        )
        .await;
    }
    let sent = std::iter::from_fn(|| envelopes.try_recv().ok())
        .map(|envelope| envelope.body)
        .collect();
    (String::from_utf8(err).unwrap(), sent)
}

/// The restart goes ahead and then says so: the shepherd respawns from the
/// spec it holds, so the notice is the only word the operator gets that the
/// dog came back without the key.
#[tokio::test]
async fn a_restart_of_a_dog_whose_entry_changed_restarts_and_says_it_kept_the_old_one() {
    let (text, sent) = restart_kelpie("PADDOCK_KEY = \"{{secret:paddock-key}}\"", &[]).await;

    assert!(
        text.contains("`kelpie`")
            && text.contains("shep disable kelpie")
            && text.contains("shep enable kelpie"),
        "the dog and the way to apply its entry must be named: {text}"
    );
    assert!(
        !text.contains("paddock-key"),
        "a value, or a reference standing for one, must never be printed: {text}"
    );
    assert!(
        sent.iter()
            .any(|req| matches!(req, Request::Restart { .. })),
        "a notice is not a refusal: {sent:?}"
    );
}

#[tokio::test]
async fn a_restart_of_a_dog_whose_entry_is_unchanged_says_nothing_about_it() {
    let (text, sent) = restart_kelpie(
        "PADDOCK_KEY = \"{{secret:paddock-key}}\"",
        &[("PADDOCK_KEY", "{{secret:paddock-key}}")],
    )
    .await;

    assert_eq!(text, "");
    assert!(
        sent.iter()
            .any(|req| matches!(req, Request::Restart { .. }))
    );
}