wsb 0.258.6441

//! What a supervisor is handed, held to what the daemon actually is.
//!
//! Everything here works on the RENDER rather than on a machine. Installing a
//! service touches the operator's real systemd manager and their real login
//! agents, so a test that installed one would leave a service behind on
//! whichever machine ran the suite. What can be tested without doing that is
//! the part that goes wrong silently: a definition that names something other
//! than the daemon, a path the supervisor cannot parse, a switch that does not
//! switch.

use super::*;

fn spawn() -> Spawn {
    Spawn {
        listen: crate::daemon::LOOPBACK,
        // Exact, so nothing probes for a free port inside a test.
        http_port: crate::daemon::Port::Exact(3123),
        parent_pid: None,
        debug: false,
        peer_port: 7717,
    }
}

fn record_for(workspace: &str) -> Record {
    let root = PathBuf::from(workspace);
    Record {
        executable: PathBuf::from("/home/op/.cargo/bin/wsb-daemon"),
        arguments: crate::daemon::daemon_arguments(&root, &spawn())
            .expect("an exact port resolves without probing"),
        workspace_root: root,
        log: PathBuf::from("/home/op/.wsb/daemon.log"),
        path: "/home/op/.cargo/bin:/usr/bin".to_string(),
        machine_dir: PathBuf::from("/home/op/ws/prj/.state/hosts/h0123456789ab"),
        prj: Some(PathBuf::from("/home/op/ws/prj")),
        autostart: false,
    }
}

/// TEST11208: the service starts the daemon with the arguments the daemon is
/// started with.
///
/// The whole claim of this module is that a supervisor-started daemon and an
/// on-demand one are the same daemon. Nothing enforces that but this: the two
/// argument lists come from one function, and a definition built from a
/// hand-written copy of it would go on working until the day a flag was added
/// to one and not the other — at which point the daemon a machine starts at
/// login quietly differs from the one a terminal starts, in a way no error
/// ever mentions.
#[test]
fn test11208_the_unit_starts_the_daemon_with_the_arguments_the_daemon_takes() {
    let record = record_for("/home/op/ws/machinefabric");
    let unit = render(Supervisor::Systemd, &record);

    let exec = unit
        .lines()
        .find_map(|line| line.strip_prefix("ExecStart="))
        .expect("the unit must say what to start");

    let expected = crate::daemon::daemon_arguments(
        &PathBuf::from("/home/op/ws/machinefabric"),
        &spawn(),
    )
    .expect("an exact port resolves without probing");

    assert_eq!(
        exec,
        format!("/home/op/.cargo/bin/wsb-daemon {}", expected.join(" ")),
        "the unit's ExecStart is not the daemon's own argument vector"
    );
    // And the same vector reaches launchd, element by element rather than as
    // one string, which is what a plist wants.
    let plist = render(Supervisor::Launchd, &record);
    for argument in &expected {
        assert!(
            plist.contains(&format!("<string>{argument}</string>")),
            "the plist does not pass {argument} to the daemon"
        );
    }
}

/// TEST11209: a workspace under a path with a space still starts.
///
/// systemd splits `ExecStart` on whitespace and has its own quoting. An
/// unquoted `--workspace /home/op/My Work/machinefabric` is not a unit that
/// starts the daemon against the wrong directory — it is a unit that refuses
/// to load, on a machine where every other wsb command works, with a message
/// about an argument nobody wrote.
#[test]
fn test11209_a_path_with_a_space_is_quoted_for_the_supervisor() {
    let record = record_for("/home/op/My Work/machinefabric");
    let unit = render(Supervisor::Systemd, &record);
    let exec = unit
        .lines()
        .find_map(|line| line.strip_prefix("ExecStart="))
        .expect("the unit must say what to start");

    assert!(
        exec.contains("\"/home/op/My Work/machinefabric\""),
        "the workspace path is not quoted: {exec}"
    );
    // Quoting is for the words that need it. A unit with every word in quotes
    // is unreadable, and reading it is how an operator checks what is running.
    assert!(
        exec.contains(" --workspace "),
        "flags that need no quoting were quoted anyway: {exec}"
    );

    // launchd takes an array, so the same path needs no quoting there — and
    // quoting it would put the quotes IN the argument.
    let plist = render(Supervisor::Launchd, &record);
    assert!(
        plist.contains("<string>/home/op/My Work/machinefabric</string>"),
        "the plist mangled a path that needs nothing done to it"
    );
}

/// TEST11210: autostart is the one thing enable and disable change.
///
/// Both supervisors express it differently and both are rendered from the same
/// field, so this is the check that the field reaches each of them — and, just
/// as much, that nothing ELSE moves when it does. A disable that also changed
/// the port or the executable would be a command that did more than it said.
#[test]
fn test11210_autostart_is_the_only_difference_between_enabled_and_disabled() {
    let mut off = record_for("/home/op/ws/machinefabric");
    off.autostart = false;
    let mut on = off.clone();
    on.autostart = true;

    let plist_off = render(Supervisor::Launchd, &off);
    let plist_on = render(Supervisor::Launchd, &on);
    assert!(
        plist_off.contains("<key>RunAtLoad</key>\n  <false/>"),
        "a disabled agent still says it starts at load"
    );
    assert!(
        plist_on.contains("<key>RunAtLoad</key>\n  <true/>"),
        "an enabled agent does not say it starts at load"
    );
    // Only the RunAtLoad line is normalised — `<false/>` also appears under
    // KeepAlive, and blanking every one of them would make this pass while
    // enabling quietly changed the restart policy too.
    assert_eq!(
        plist_off.replace("<key>RunAtLoad</key>\n  <false/>", "<key>RunAtLoad</key>\n  <TOGGLE/>"),
        plist_on.replace("<key>RunAtLoad</key>\n  <true/>", "<key>RunAtLoad</key>\n  <TOGGLE/>"),
        "enabling changed something other than whether it starts at login"
    );

    // systemd keeps the switch in the manager rather than in the file, so the
    // unit is the same either way. That is not an oversight to be fixed by
    // rendering something different — `systemctl enable` is what enables — and
    // a unit that differed would make `daemon-reload` see a changed file for a
    // change that is not in it.
    assert_eq!(
        render(Supervisor::Systemd, &off),
        render(Supervisor::Systemd, &on),
        "the systemd unit encodes autostart, which belongs to the manager"
    );
}

/// TEST11211: the plist survives an ampersand.
///
/// A `PATH` picks up whatever directories a machine has, and a workspace lives
/// wherever somebody put it. One `&` in either makes the plist malformed XML;
/// launchd then refuses to load an agent for a reason that has nothing to do
/// with the daemon.
#[test]
fn test11211_the_plist_escapes_what_xml_cannot_carry() {
    let mut record = record_for("/home/op/R&D/machinefabric");
    record.path = "/opt/A&B/bin:/usr/bin".to_string();

    let plist = render(Supervisor::Launchd, &record);
    assert!(
        plist.contains("<string>/home/op/R&amp;D/machinefabric</string>"),
        "an ampersand in the workspace path reached the plist raw"
    );
    assert!(
        plist.contains("<string>/opt/A&amp;B/bin:/usr/bin</string>"),
        "an ampersand in PATH reached the plist raw"
    );
    assert!(
        !plist.contains("R&D"),
        "the plist is not well-formed XML"
    );
}

/// TEST11212: a definition that no longer matches the record is drift.
///
/// The state this exists for: wsb is rebuilt somewhere else, or installed to a
/// different prefix. The unit still names the executable that used to be
/// there, the supervisor keeps failing to start it, and every symptom is about
/// a missing file rather than about a service that needs reinstalling. Saying
/// "this is not what your record renders" is what turns that into one step.
#[test]
fn test11212_a_definition_that_is_not_the_render_reads_as_drift() {
    let record = record_for("/home/op/ws/machinefabric");
    let current = render(Supervisor::Systemd, &record);

    assert!(
        !drifted(Supervisor::Systemd, &record, Some(&current)),
        "an untouched definition reads as drifted"
    );
    let moved = current.replace("/home/op/.cargo/bin", "/usr/local/bin");
    assert!(
        drifted(Supervisor::Systemd, &record, Some(&moved)),
        "a unit naming an executable that is not the recorded one reads as current"
    );
    // Absent is a half-install, not drift. `installed` is what reports that,
    // and calling it drift sends an operator to reinstall for the wrong reason.
    assert!(
        !drifted(Supervisor::Systemd, &record, None),
        "a missing definition was reported as drift rather than as absent"
    );
}

/// TEST11213: the supervisor writes where the daemon already writes.
///
/// `wsb daemon log` reads one file. A service that logged to the journal
/// instead would give an operator two places to look and no way to know which
/// held the run they were asking about — and the one they would try first is
/// the one the command names.
#[test]
fn test11213_a_service_writes_to_the_log_the_daemon_command_reads() {
    let record = record_for("/home/op/ws/machinefabric");
    let log = record.log.display().to_string();

    let unit = render(Supervisor::Systemd, &record);
    assert!(
        unit.contains(&format!("StandardOutput=append:{log}")),
        "the unit does not append the daemon's output to its log"
    );
    assert!(
        unit.contains(&format!("StandardError=append:{log}")),
        "the unit does not append the daemon's errors to its log"
    );

    let plist = render(Supervisor::Launchd, &record);
    assert!(
        plist.contains("<key>StandardOutPath</key>"),
        "the agent does not say where its output goes"
    );
    assert_eq!(
        plist.matches(&format!("<string>{log}</string>")).count(),
        2,
        "the agent does not send both streams to the daemon's log"
    );
}

/// TEST11214: `PATH` is carried into the definition.
///
/// Not decoration. A systemd user service inherits the manager's environment
/// and a launchd agent inherits launchd's; neither is a login shell's. Without
/// this the daemon starts, serves the workbench, and then answers "command not
/// found" to every build — on a machine where the same build succeeds from a
/// terminal, which is the hardest kind of failure to place.
#[test]
fn test11214_the_definition_carries_the_path_the_daemon_needs() {
    let record = record_for("/home/op/ws/machinefabric");

    assert!(
        render(Supervisor::Systemd, &record)
            .contains(&format!("Environment=PATH={}", record.path)),
        "the unit does not give the daemon a PATH"
    );
    let plist = render(Supervisor::Launchd, &record);
    assert!(
        plist.contains("<key>PATH</key>") && plist.contains(&format!("<string>{}</string>", record.path)),
        "the agent does not give the daemon a PATH"
    );
}

/// TEST11215: a restart the supervisor did not ask for is not one it performs.
///
/// `wsb daemon stop` is a deliberate act — the store the daemon holds goes
/// with it — and a supervisor configured to restart on ANY exit would bring
/// the daemon straight back, making the command impossible to obey. Both
/// definitions have to say "on failure" and only that.
#[test]
fn test11215_the_supervisor_restarts_a_failure_and_not_a_deliberate_stop() {
    let record = record_for("/home/op/ws/machinefabric");

    let unit = render(Supervisor::Systemd, &record);
    assert!(
        unit.contains("Restart=on-failure"),
        "the unit does not restart the daemon when it fails"
    );
    assert!(
        !unit.contains("Restart=always"),
        "the unit would restart a daemon that was deliberately stopped"
    );

    let plist = render(Supervisor::Launchd, &record);
    assert!(
        plist.contains("<key>SuccessfulExit</key>\n    <false/>"),
        "the agent would restart a daemon that exited cleanly"
    );
}

/// TEST20053: a service's daemon keeps its state where the install found it.
///
/// A supervisor does not start a login shell, so a daemon started by one had
/// no `PRJ`, worked its machine directory out again, and kept a second, empty
/// machine of its own in `~/.wsb`: a new identity, no pairings, minutes spent
/// reading every agent session into a database nobody looked at, and every
/// `wsb` command refusing because two machine directories existed. Both
/// definitions hand it the directory the install resolved, and `PRJ`.
#[test]
fn test20053_the_service_keeps_its_state_where_the_install_found_it() {
    let record = record_for("/home/op/ws/prj/machinefabric/main-worktree");

    let unit = render(Supervisor::Systemd, &record);
    assert!(
        unit.contains("Environment=WSB_MACHINE_DIR=/home/op/ws/prj/.state/hosts/h0123456789ab\n"),
        "the unit does not say where the daemon's state is: {unit}"
    );
    assert!(unit.contains("Environment=PRJ=/home/op/ws/prj\n"), "the unit does not carry PRJ: {unit}");

    let plist = render(Supervisor::Launchd, &record);
    assert!(
        plist.contains("<key>WSB_MACHINE_DIR</key>\n    <string>/home/op/ws/prj/.state/hosts/h0123456789ab</string>"),
        "the agent does not say where the daemon's state is: {plist}"
    );
    assert!(
        plist.contains("<key>PRJ</key>\n    <string>/home/op/ws/prj</string>"),
        "the agent does not carry PRJ: {plist}"
    );

    // An install made without PRJ hands on none, rather than an empty one.
    let without = Record { prj: None, ..record };
    assert!(!render(Supervisor::Systemd, &without).contains("PRJ="));
    assert!(!render(Supervisor::Launchd, &without).contains("<key>PRJ</key>"));
}

/// TEST20054: a reinstall keeps whether the service starts at login, over a
/// record an older wsb wrote.
///
/// That record cannot be read as this wsb's — it lacks what this one adds —
/// and an install that refused it, or read it as "not enabled", would leave a
/// machine either stuck or no longer starting its daemon at login.
#[test]
fn test20054_a_reinstall_keeps_autostart_over_an_older_record() {
    let dir = tempfile::tempdir().expect("a directory");
    std::fs::write(
        record_path(dir.path()),
        r#"{"executable":"/x/wsb-daemon","arguments":[],"workspace_root":"/w","log":"/l","path":"/usr/bin","autostart":true}"#,
    )
    .expect("an older record");

    assert!(record(dir.path()).is_err(), "an older record read as this wsb's");
    assert!(installed_autostart(dir.path()).expect("its switch reads"), "a reinstall would turn autostart off");
    assert!(!installed_autostart(&dir.path().join("none")).expect("no record"), "no service is not an enabled one");
}

/// A supervisor that does what launchd does around a bootout, and remembers
/// what it was asked.
///
/// A booted-out job stays listed while the daemon shuts down — for `lingers`
/// questions here — and a bootstrap of a label that is still listed fails the
/// way launchd fails it.
struct StandIn {
    calls: Vec<String>,
    loaded: bool,
    lingers: u32,
    leaving: Option<u32>,
}

impl StandIn {
    fn running(lingers: u32) -> Self {
        Self { calls: Vec::new(), loaded: true, lingers, leaving: None }
    }

    fn verbs(&self) -> Vec<&str> {
        self.calls.iter().map(String::as_str).collect()
    }
}

impl Control for StandIn {
    fn systemctl(&mut self, arguments: &[&str]) -> Result<String, String> {
        self.calls.push(format!("systemctl {}", arguments.join(" ")));
        Ok(String::new())
    }

    fn launchctl(&mut self, arguments: &[String]) -> Result<String, String> {
        let verb = arguments[0].as_str();
        self.calls.push(verb.to_string());
        match verb {
            "bootout" if self.loaded => {
                self.leaving = Some(self.lingers);
                Ok(String::new())
            }
            "bootout" => Err("Boot-out failed: 3: No such process".to_string()),
            "bootstrap" if self.loaded => Err("Bootstrap failed: 5: Input/output error".to_string()),
            "bootstrap" => {
                self.loaded = true;
                Ok(String::new())
            }
            "kickstart" if self.loaded => Ok(String::new()),
            other => Err(format!("the stand-in was asked to {other} with the job in no state for it")),
        }
    }

    fn launchd_loaded(&mut self, _target: &str) -> Result<bool, String> {
        self.calls.push("print".to_string());
        match self.leaving {
            Some(0) => {
                self.leaving = None;
                self.loaded = false;
            }
            Some(left) => self.leaving = Some(left - 1),
            None => {}
        }
        Ok(self.loaded)
    }
}

/// TEST20056: an install over a running launchd service loads the new
/// definition once launchd has let go of the old one.
///
/// `bootout` returns while the daemon is still shutting down. A bootstrap sent
/// straight after it reached a launchd that still held the label, and every
/// reinstall over a running service failed with "Bootstrap failed: 5".
#[test]
fn test20056_an_install_over_a_running_launchd_job_waits_for_it_to_go() {
    let dir = tempfile::tempdir().expect("a directory");
    let definition = dir.path().join(format!("{LABEL}.plist"));
    let record = Record { autostart: true, ..record_for("/home/op/ws/prj/machinefabric/main-worktree") };
    let mut launchd = StandIn::running(3);

    apply_with(&mut launchd, Supervisor::Launchd, &record, &definition, Duration::from_secs(5))
        .expect("the new definition is loaded once the old job has gone");

    assert_eq!(
        launchd.verbs(),
        ["bootout", "print", "print", "print", "print", "bootstrap", "kickstart"],
        "the job was not bootstrapped once, after launchd stopped listing the old one"
    );
    assert_eq!(
        std::fs::read_to_string(&definition).expect("the plist"),
        render(Supervisor::Launchd, &record)
    );
}

/// TEST20057: a launchd job that never leaves is a refusal naming it.
///
/// Not a bootstrap sent anyway, which fails with an I/O error that names
/// neither the job nor what to do about it.
#[test]
fn test20057_a_launchd_job_that_does_not_go_is_named() {
    let dir = tempfile::tempdir().expect("a directory");
    let definition = dir.path().join(format!("{LABEL}.plist"));
    let record = Record { autostart: true, ..record_for("/home/op/ws/prj/machinefabric/main-worktree") };
    let mut launchd = StandIn::running(u32::MAX);

    let refused = apply_with(&mut launchd, Supervisor::Launchd, &record, &definition, Duration::from_millis(300))
        .expect_err("a job launchd keeps holding cannot be replaced");

    assert!(refused.contains(LABEL), "the refusal does not name the job: {refused}");
    assert!(refused.contains("launchctl print"), "the refusal does not say how to look: {refused}");
    assert!(
        !launchd.verbs().contains(&"bootstrap"),
        "a bootstrap was sent over a job launchd still held: {:?}",
        launchd.verbs()
    );
}

/// TEST20058: a running systemd unit is restarted onto a definition that
/// changed, and only then.
///
/// `daemon-reload` reads the new unit and leaves the running daemon as it was
/// started, so a reinstall that moved the executable or the environment
/// reported success and changed nothing until the next login. A definition
/// that did not change is not a reason to end the daemon — `disable` never
/// changes the unit, and says the running daemon keeps running.
#[test]
fn test20058_a_running_systemd_unit_is_restarted_onto_a_changed_definition() {
    let dir = tempfile::tempdir().expect("a directory");
    let definition = dir.path().join(UNIT);
    let enabled = Record { autostart: true, ..record_for("/home/op/ws/prj/machinefabric/main-worktree") };
    std::fs::write(&definition, render(Supervisor::Systemd, &record_for("/home/op/ws/elsewhere")))
        .expect("an older unit");

    let mut systemd = StandIn::running(0);
    apply_with(&mut systemd, Supervisor::Systemd, &enabled, &definition, Duration::from_secs(5))
        .expect("a changed definition is applied");
    assert_eq!(
        systemd.verbs(),
        [
            "systemctl daemon-reload",
            &format!("systemctl try-restart {UNIT}"),
            &format!("systemctl enable --now {UNIT}"),
        ],
        "a running unit was left on the definition it was started from"
    );

    let mut systemd = StandIn::running(0);
    apply_with(&mut systemd, Supervisor::Systemd, &enabled, &definition, Duration::from_secs(5))
        .expect("the same definition is applied");
    assert_eq!(
        systemd.verbs(),
        ["systemctl daemon-reload", &format!("systemctl enable --now {UNIT}")],
        "an unchanged definition restarted the daemon"
    );

    let mut systemd = StandIn::running(0);
    let disabled = Record { autostart: false, ..enabled };
    apply_with(&mut systemd, Supervisor::Systemd, &disabled, &definition, Duration::from_secs(5))
        .expect("disabling is applied");
    assert_eq!(
        systemd.verbs(),
        ["systemctl daemon-reload", &format!("systemctl disable {UNIT}")],
        "disabling ended the running daemon"
    );
}