bynk_driver/
dev_run.rs

1//! #2138 (end-to-end testing track §3.5): `bynk test --against dev`, the
2//! `system` suites run against a local `bynk dev` the runner starts itself.
3//!
4//! For each distinct `env` the `system` cases run with (one group), this:
5//!
6//! 1. starts `bynk dev --ready-json` (local `wrangler dev`/workerd only, never
7//!    `-- --remote`) with the group's `env` and a fixed test value for every
8//!    other secret the Workers declare, passed as `-- --var NAME:VALUE`;
9//! 2. waits for the readiness file, whose URLs must all be loopback;
10//! 3. runs the group's cases against those URLs (this binary again, with
11//!    [`crate::remote::DEV_URLS_ENV`] and [`crate::remote::DEV_ENV_ENV`]),
12//!    signing with the same secret values;
13//! 4. stops `bynk dev` by its own pid (it stops its wranglers and their
14//!    `workerd`s), then its process group if anything is left.
15//!
16//! Groups run in turn; their results are reported together.
17
18use std::collections::{BTreeMap, BTreeSet};
19use std::path::{Path, PathBuf};
20use std::process::{Child, Command, ExitCode, Stdio};
21use std::time::{Duration, Instant};
22
23use bynk_emit::project::{BuildTarget, RemoteTests};
24
25use crate::remote::{DEV, DEV_ENV_ENV, DEV_SECRETS_ENV, DEV_URLS_ENV, is_loopback_http};
26use crate::test_json::TestRun;
27
28/// The value every declared secret the case doesn't set takes, both in the
29/// Workers (`-- --var`) and in the runner that signs for them.
30pub const TEST_SECRET: &str = "__bynk_test_secret";
31
32/// How long to wait for a `bynk dev` to answer before giving up.
33const READY_TIMEOUT: Duration = Duration::from_secs(180);
34
35/// How long a stopped `bynk dev` gets to stop its Workers before its process
36/// group is killed.
37const STOP_GRACE: Duration = Duration::from_secs(30);
38
39/// What the orchestration needs from the `test` invocation.
40#[derive(Debug, Clone)]
41pub struct DevRunArgs {
42    pub input: PathBuf,
43    pub output: PathBuf,
44    pub json: bool,
45    pub seed: Option<String>,
46    pub case: Option<String>,
47    pub skip_unsupported: bool,
48    /// `--settle`, forwarded to each group's run.
49    pub settle: Option<String>,
50    /// The `run.id` every group's run shares.
51    pub run_id: String,
52    /// The `bynk` binary that starts `bynk dev`.
53    pub bynk: PathBuf,
54}
55
56/// Run every `system` suite against a local `bynk dev`, one per distinct
57/// `env`.
58pub fn run(program: &str, args: &DevRunArgs) -> ExitCode {
59    let fail = |message: String| {
60        if args.json {
61            print!("{}", TestRun::runtime_error(message, None).render());
62        } else {
63            eprintln!("{program} test: {message}");
64        }
65        ExitCode::FAILURE
66    };
67    let plan = match plan(args) {
68        Ok(plan) => plan,
69        Err(Planned::Failed(code)) => return code,
70        Err(Planned::Message(m)) => return fail(m),
71    };
72    if plan.groups.is_empty() {
73        if args.json {
74            print!(
75                "{}",
76                TestRun::empty()
77                    .with_remote(format!("remote:{DEV}"), &[])
78                    .render()
79            );
80        } else {
81            eprintln!("{program} test: no `system` suites to run against `bynk dev`");
82        }
83        return ExitCode::SUCCESS;
84    }
85    let mut docs: Vec<serde_json::Value> = Vec::new();
86    let mut all_ok = true;
87    for (i, group) in plan.groups.iter().enumerate() {
88        let vars = dev_vars(group, &plan.secrets);
89        if !args.json {
90            eprintln!(
91                "{program} test: starting `bynk dev`{}",
92                if group.is_empty() {
93                    String::new()
94                } else {
95                    format!(
96                        " with env {}",
97                        group
98                            .iter()
99                            .map(|(k, v)| format!("{k}={v:?}"))
100                            .collect::<Vec<_>>()
101                            .join(", ")
102                    )
103                }
104            );
105        }
106        let ready = args.output.join(format!(".bynk-dev-ready-{i}.json"));
107        let log = args.output.join(format!(".bynk-dev-{i}.log"));
108        let mut dev = match Dev::start(&args.bynk, &args.input, &ready, &log, &vars) {
109            Ok(dev) => dev,
110            Err(e) => return fail(format!("could not start `bynk dev`: {e}")),
111        };
112        let urls = match dev.wait_ready(&ready) {
113            Ok(urls) => urls,
114            Err(e) => {
115                drop(dev);
116                return fail(format!("{e} (its output is in {})", log.display()));
117            }
118        };
119        let outcome = run_group(args, group, &urls, &vars);
120        dev.stop();
121        match outcome {
122            Ok((ok, doc)) => {
123                all_ok &= ok;
124                docs.extend(doc);
125            }
126            Err(e) => return fail(e),
127        }
128    }
129    if args.json {
130        print!("{}", merge(&docs).render());
131    }
132    if all_ok {
133        ExitCode::SUCCESS
134    } else {
135        ExitCode::FAILURE
136    }
137}
138
139/// The groups to run, and the secrets the Workers declare.
140struct Plan {
141    groups: Vec<BTreeMap<String, String>>,
142    secrets: BTreeSet<String>,
143}
144
145enum Planned {
146    /// A compile failure, already reported.
147    Failed(ExitCode),
148    Message(String),
149}
150
151/// Compile the project as a `--against dev` build once: refuse what a real
152/// runtime can't honour (as the group runs would), and read the distinct
153/// `env` maps of its `system` cases and the secrets its Workers declare.
154fn plan(args: &DevRunArgs) -> Result<Plan, Planned> {
155    let options = crate::try_project_options(&args.input)
156        .map_err(|e| Planned::Message(e.to_string()))?
157        .contracts(true)
158        .remote_tests(Some(RemoteTests {
159            skip_unsupported: args.skip_unsupported,
160            dev: true,
161            ..RemoteTests::default()
162        }))
163        .target(BuildTarget::Workers);
164    let out = match bynk_emit::project::compile_project(&options) {
165        Ok(out) => out,
166        Err(failure) => {
167            if args.json {
168                print!(
169                    "{}",
170                    TestRun::compile_error(crate::project_failure_short_lines(&failure)).render()
171                );
172            } else {
173                crate::print_project_failure(&failure);
174            }
175            return Err(Planned::Failed(ExitCode::FAILURE));
176        }
177    };
178    let mut groups: Vec<BTreeMap<String, String>> = Vec::new();
179    for env in out.discovered.iter().flat_map(|s| s.envs.iter()) {
180        if !groups.contains(env) {
181            groups.push(env.clone());
182        }
183    }
184    let mut secrets = BTreeSet::new();
185    for (path, doc) in &out.artefacts.docs {
186        if path.file_name().is_some_and(|n| n == "bynk-secrets.json") {
187            let manifest: serde_json::Value = serde_json::from_str(&doc.text()).unwrap_or_default();
188            for key in ["declared", "read"] {
189                for name in manifest[key].as_array().into_iter().flatten() {
190                    if let Some(name) = name.as_str() {
191                        secrets.insert(name.to_string());
192                    }
193                }
194            }
195        }
196    }
197    Ok(Plan { groups, secrets })
198}
199
200/// The variables one `bynk dev` serves: the group's `env`, then each declared
201/// secret the group doesn't set at the fixed test value.
202pub fn dev_vars(
203    group: &BTreeMap<String, String>,
204    secrets: &BTreeSet<String>,
205) -> BTreeMap<String, String> {
206    let mut vars = group.clone();
207    for s in secrets {
208        vars.entry(s.clone())
209            .or_insert_with(|| TEST_SECRET.to_string());
210    }
211    vars
212}
213
214/// Run one group's cases against `urls`: this binary again, as a run
215/// against `dev` with the URLs and the group's `env`, and the group's
216/// variables in its environment so the drivers sign with them. Returns
217/// whether it passed and, under `--format json`, its document.
218fn run_group(
219    args: &DevRunArgs,
220    group: &BTreeMap<String, String>,
221    urls: &BTreeMap<String, String>,
222    vars: &BTreeMap<String, String>,
223) -> Result<(bool, Option<serde_json::Value>), String> {
224    let me = std::env::current_exe().map_err(|e| format!("could not find this binary: {e}"))?;
225    let mut cmd = Command::new(me);
226    cmd.arg("test")
227        .arg(&args.input)
228        .arg("--output")
229        .arg(&args.output);
230    cmd.arg("--format")
231        .arg(if args.json { "json" } else { "rich" });
232    cmd.args(["--against", DEV]);
233    if let Some(seed) = &args.seed {
234        cmd.args(["--seed", seed]);
235    }
236    if let Some(case) = &args.case {
237        cmd.args(["--case", case]);
238    }
239    if args.skip_unsupported {
240        cmd.arg("--skip-unsupported");
241    }
242    if let Some(settle) = &args.settle {
243        cmd.args(["--settle", settle]);
244    }
245    cmd.env_remove(crate::remote::DEV_COMMAND_ENV);
246    // One `run.id` for every group of this run, through the orchestration's
247    // own variable: the group's runner always sets the harness's.
248    cmd.env(crate::remote::DEV_RUN_ID_ENV, &args.run_id);
249    cmd.env(
250        DEV_URLS_ENV,
251        serde_json::to_string(urls).expect("a string map serialises"),
252    );
253    cmd.env(
254        DEV_ENV_ENV,
255        serde_json::to_string(group).expect("a string map serialises"),
256    );
257    // One JSON variable, never a variable per name: an `env` entry or secret
258    // named `PATH` or `NODE_OPTIONS` must not change the runner itself.
259    cmd.env(
260        DEV_SECRETS_ENV,
261        serde_json::to_string(vars).expect("a string map serialises"),
262    );
263    if args.json {
264        let out = cmd
265            .stderr(Stdio::inherit())
266            .output()
267            .map_err(|e| format!("could not run the suites: {e}"))?;
268        let doc = serde_json::from_slice(&out.stdout)
269            .map_err(|e| format!("the run printed no JSON document: {e}"))?;
270        Ok((out.status.success(), Some(doc)))
271    } else {
272        let status = cmd
273            .status()
274            .map_err(|e| format!("could not run the suites: {e}"))?;
275        Ok((status.success(), None))
276    }
277}
278
279/// Fold the groups' documents into one: counts summed, suites of the same
280/// name merged, the first error kept, `transport: "remote:dev"`.
281fn merge(docs: &[serde_json::Value]) -> TestRun {
282    use crate::test_json::{Case, Location, Suite, TestError};
283    let count = |d: &serde_json::Value, k: &str| d[k].as_u64().unwrap_or(0) as u32;
284    let mut run = TestRun::empty();
285    run.transport = Some(format!("remote:{DEV}"));
286    let suites = run.suites.get_or_insert_with(Vec::new);
287    for doc in docs {
288        run.passed += count(doc, "passed");
289        run.failed += count(doc, "failed");
290        run.skipped += count(doc, "skipped");
291        for s in doc["suites"].as_array().into_iter().flatten() {
292            let name = s["name"].as_str().unwrap_or_default().to_string();
293            let cases = s["cases"].as_array().into_iter().flatten().map(|c| Case {
294                name: c["name"].as_str().unwrap_or_default().to_string(),
295                outcome: c["outcome"].as_str().unwrap_or_default().to_string(),
296                message: c["message"].as_str().map(str::to_string),
297                location: c["location"].as_object().map(|l| Location {
298                    path: l["path"].as_str().unwrap_or_default().to_string(),
299                    line: l["line"].as_u64().unwrap_or(0) as u32,
300                    col: l["col"].as_u64().unwrap_or(0) as u32,
301                }),
302                retries: c["retries"].as_u64().map(|n| n as u32),
303            });
304            match suites.iter_mut().find(|x| x.name == name) {
305                Some(existing) => {
306                    // Every group's build reports the same skipped cases;
307                    // list each once.
308                    for case in cases {
309                        let repeat = case.outcome == "skipped"
310                            && existing
311                                .cases
312                                .iter()
313                                .any(|c| c.outcome == "skipped" && c.name == case.name);
314                        if repeat {
315                            run.skipped = run.skipped.saturating_sub(1);
316                        } else {
317                            existing.cases.push(case);
318                        }
319                    }
320                }
321                None => suites.push(Suite {
322                    name,
323                    kind: s["kind"].as_str().unwrap_or("integration").to_string(),
324                    cases: cases.collect(),
325                }),
326            }
327        }
328        if run.error.is_none()
329            && let Some(e) = doc["error"].as_object()
330        {
331            run.error = Some(TestError {
332                kind: e["kind"].as_str().unwrap_or("runtime").to_string(),
333                message: e
334                    .get("message")
335                    .and_then(|m| m.as_str())
336                    .map(str::to_string),
337                diagnostics: e
338                    .get("diagnostics")
339                    .and_then(|d| d.as_array())
340                    .into_iter()
341                    .flatten()
342                    .filter_map(|d| d.as_str().map(str::to_string))
343                    .collect(),
344                stderr: e.get("stderr").and_then(|m| m.as_str()).map(str::to_string),
345            });
346        }
347    }
348    run
349}
350
351/// Remove a readiness file left by an earlier run, so it is never read as
352/// this session being ready. A missing file is fine.
353pub fn remove_stale(ready: &Path) -> std::io::Result<()> {
354    match std::fs::remove_file(ready) {
355        Err(e) if e.kind() != std::io::ErrorKind::NotFound => Err(e),
356        _ => Ok(()),
357    }
358}
359
360/// Two distinct localhost ports, free when asked: the base of the HTTP and
361/// the inspector allocations `bynk dev` makes.
362fn free_ports() -> std::io::Result<(u16, u16)> {
363    let a = std::net::TcpListener::bind("127.0.0.1:0")?;
364    let b = std::net::TcpListener::bind("127.0.0.1:0")?;
365    Ok((a.local_addr()?.port(), b.local_addr()?.port()))
366}
367
368/// A running `bynk dev`, stopped on drop.
369struct Dev {
370    child: Child,
371    stopped: bool,
372}
373
374impl Dev {
375    fn start(
376        bynk: &Path,
377        input: &Path,
378        ready: &Path,
379        log: &Path,
380        vars: &BTreeMap<String, String>,
381    ) -> std::io::Result<Dev> {
382        if let Some(dir) = ready.parent() {
383            std::fs::create_dir_all(dir)?;
384        }
385        // A file left by an earlier run that was killed must never read as
386        // this session being ready.
387        crate::dev_run::remove_stale(ready)?;
388        let (port, inspector) = free_ports()?;
389        let log_file = std::fs::File::create(log)?;
390        let mut cmd = Command::new(bynk);
391        cmd.arg("dev")
392            .arg(input)
393            .arg("--base-port")
394            .arg(port.to_string())
395            .arg("--inspect-port")
396            .arg(inspector.to_string())
397            .arg("--ready-json")
398            .arg(ready)
399            .arg("--");
400        // Only variables are passed through: never `--remote`.
401        for (k, v) in vars {
402            cmd.arg("--var").arg(format!("{k}:{v}"));
403        }
404        #[cfg(unix)]
405        std::os::unix::process::CommandExt::process_group(&mut cmd, 0);
406        let child = cmd
407            .stdin(Stdio::null())
408            .stdout(log_file.try_clone()?)
409            .stderr(log_file)
410            .spawn()?;
411        Ok(Dev {
412            child,
413            stopped: false,
414        })
415    }
416
417    /// Wait for the readiness file and read its URLs, which must all be
418    /// loopback.
419    fn wait_ready(&mut self, ready: &Path) -> Result<BTreeMap<String, String>, String> {
420        let deadline = Instant::now() + READY_TIMEOUT;
421        loop {
422            if let Ok(text) = std::fs::read_to_string(ready)
423                && let Ok(urls) = serde_json::from_str::<BTreeMap<String, String>>(&text)
424            {
425                if let Some(bad) = urls.values().find(|u| !is_loopback_http(u)) {
426                    return Err(format!("`bynk dev` reported `{bad}`, which isn't loopback"));
427                }
428                return Ok(urls);
429            }
430            if let Ok(Some(status)) = self.child.try_wait() {
431                return Err(format!("`bynk dev` exited before it was ready ({status})"));
432            }
433            if Instant::now() >= deadline {
434                return Err(format!(
435                    "`bynk dev` wasn't ready within {}s",
436                    READY_TIMEOUT.as_secs()
437                ));
438            }
439            std::thread::sleep(Duration::from_millis(250));
440        }
441    }
442
443    /// Stop `bynk dev`: SIGTERM to its own pid, so it stops its Workers; then,
444    /// always, its whole process group, so a wrangler or `workerd` it leaves
445    /// behind (because it exited first, or didn't stop them in time) never
446    /// leaks. Runs once; a second call (the drop after an explicit stop) does
447    /// nothing.
448    fn stop(&mut self) {
449        if self.stopped {
450            return;
451        }
452        self.stopped = true;
453        #[cfg(unix)]
454        {
455            let pid = self.child.id().to_string();
456            if !matches!(self.child.try_wait(), Ok(Some(_))) {
457                let _ = Command::new("kill").args(["-s", "TERM", &pid]).status();
458                let deadline = Instant::now() + STOP_GRACE;
459                while Instant::now() < deadline {
460                    if matches!(self.child.try_wait(), Ok(Some(_))) {
461                        break;
462                    }
463                    std::thread::sleep(Duration::from_millis(100));
464                }
465            }
466            // `bynk dev` leads its own process group, so `-<pid>` names
467            // exactly what it started and nothing else on the machine.
468            let _ = Command::new("kill")
469                .args(["-s", "KILL", "--", &format!("-{pid}")])
470                .stderr(Stdio::null())
471                .status();
472        }
473        let _ = self.child.kill();
474        let _ = self.child.wait();
475    }
476}
477
478impl Drop for Dev {
479    fn drop(&mut self) {
480        self.stop();
481    }
482}
483
484#[cfg(test)]
485mod tests {
486    use super::*;
487
488    #[test]
489    fn a_group_s_env_wins_and_other_secrets_get_the_test_value() {
490        let group = BTreeMap::from([
491            ("PAYMENTS".to_string(), "fake".to_string()),
492            ("AUTH_SECRET".to_string(), "s3".to_string()),
493        ]);
494        let secrets = BTreeSet::from(["AUTH_SECRET".to_string(), "HOOK_SECRET".to_string()]);
495        let vars = dev_vars(&group, &secrets);
496        assert_eq!(vars["PAYMENTS"], "fake");
497        assert_eq!(vars["AUTH_SECRET"], "s3");
498        assert_eq!(vars["HOOK_SECRET"], TEST_SECRET);
499    }
500
501    #[test]
502    fn a_stale_ready_file_is_removed_before_a_session_starts() {
503        let dir = std::env::temp_dir().join(format!("bynk-dev-stale-{}", std::process::id()));
504        std::fs::create_dir_all(&dir).unwrap();
505        let ready = dir.join(".bynk-dev-ready-0.json");
506        // An earlier, killed run's file, pointing somewhere bogus.
507        std::fs::write(&ready, r#"{"svc":"http://localhost:1"}"#).unwrap();
508        // `Dev::start` removes it before spawning, even when the spawn then
509        // fails (here: no such binary), so it can never read as ready.
510        let started = Dev::start(
511            &dir.join("no-such-bynk"),
512            &dir,
513            &ready,
514            &dir.join("dev.log"),
515            &BTreeMap::new(),
516        );
517        assert!(started.is_err());
518        assert!(!ready.exists(), "the stale file is gone");
519        remove_stale(&ready).expect("removing an absent file is fine");
520        let _ = std::fs::remove_dir_all(&dir);
521    }
522
523    #[test]
524    fn group_documents_merge_into_one() {
525        let a = serde_json::json!({"passed": 1, "failed": 0, "suites": [{"name": "s", "kind": "integration", "cases": [{"name": "a", "outcome": "pass"}]}], "transport": "remote:dev"});
526        let b = serde_json::json!({"passed": 0, "failed": 1, "suites": [{"name": "s", "kind": "integration", "cases": [{"name": "b", "outcome": "fail", "message": "m"}]}], "transport": "remote:dev"});
527        let run = merge(&[a, b]);
528        assert_eq!((run.passed, run.failed), (1, 1));
529        let suites = run.suites.expect("suites");
530        assert_eq!(suites.len(), 1);
531        assert_eq!(suites[0].cases.len(), 2);
532        assert_eq!(run.transport.as_deref(), Some("remote:dev"));
533    }
534}
bynk_driver/dev_run.rs

bynk_driver/
dev_run.rs
