yosh_plugin_manager/
scenario.rs

1//! Declarative scenarios for `yosh plugin test`. One TOML file per
2//! scenario; each scenario is a sequence of `step` entries, each step
3//! is one exec / hook invocation plus an `expect` block.
4
5use std::collections::BTreeMap;
6use std::path::PathBuf;
7
8use serde::Deserialize;
9
10#[derive(Debug, Deserialize)]
11pub struct Scenario {
12    pub plugin: PathBuf,
13    #[serde(default)]
14    pub description: String,
15    #[serde(default)]
16    pub env: EnvConfig,
17    #[serde(default)]
18    pub files: BTreeMap<String, String>,
19    #[serde(rename = "step", default)]
20    pub steps: Vec<Step>,
21}
22
23#[derive(Debug, Default, Deserialize)]
24pub struct EnvConfig {
25    #[serde(default)]
26    pub caps: Vec<String>,
27    #[serde(default)]
28    pub vars: BTreeMap<String, String>,
29    #[serde(default)]
30    pub exported: Vec<String>,
31    #[serde(default)]
32    pub cwd: String,
33    #[serde(default)]
34    pub allow_exec: Vec<String>,
35    #[serde(default)]
36    pub sandbox_root: String,
37    #[serde(default = "default_timeout_ms")]
38    pub timeout_ms: u64,
39}
40
41fn default_timeout_ms() -> u64 {
42    5000
43}
44
45#[derive(Debug, Deserialize)]
46#[serde(tag = "call", rename_all = "lowercase")]
47pub enum Step {
48    Exec {
49        args: Vec<String>,
50        #[serde(default)]
51        expect: Expect,
52    },
53    Hook {
54        name: HookName,
55        args: Vec<toml::Value>,
56        #[serde(default)]
57        expect: Expect,
58    },
59}
60
61#[derive(Debug, Deserialize, PartialEq, Eq, Clone, Copy)]
62#[serde(rename_all = "kebab-case")]
63pub enum HookName {
64    PreExec,
65    PostExec,
66    OnCd,
67    PrePrompt,
68}
69
70#[derive(Debug, Default, Deserialize)]
71#[serde(deny_unknown_fields)]
72pub struct Expect {
73    pub exit: Option<i32>,
74    pub stdout: Option<String>,
75    pub stderr: Option<String>,
76    pub stdout_contains: Option<String>,
77    pub stderr_contains: Option<String>,
78    pub stdout_regex: Option<String>,
79    pub stderr_regex: Option<String>,
80    pub vars_set: Option<BTreeMap<String, String>>,
81    pub vars_export: Option<BTreeMap<String, String>>,
82    pub files_write: Option<BTreeMap<String, FileExpect>>,
83    pub exec_called: Option<Vec<ExecCallExpect>>,
84    pub trap: Option<bool>,
85}
86
87// Note: `denied: bool` (listed in spec §5 as a future expect key) is
88// intentionally not implemented here. Observing capability-denied
89// errors from the harness requires plumbing a counter through every
90// host import (each `Err(Denied)` increments). Deferred — for now,
91// authors detect denial via `stdout_regex` on guest-side error
92// handling or via specific `exit` codes the guest returns on
93// `Err(ErrorCode::Denied)`.
94
95#[derive(Debug, Deserialize)]
96#[serde(untagged)]
97pub enum FileExpect {
98    Bytes(String),
99    Struct {
100        #[serde(default)]
101        len: Option<usize>,
102        #[serde(default)]
103        bytes_eq: Option<String>,
104    },
105}
106
107#[derive(Debug, Deserialize)]
108pub struct ExecCallExpect {
109    pub program: String,
110    #[serde(default)]
111    pub args: Vec<String>,
112    pub exit: Option<i32>,
113}
114
115pub fn parse(path: &std::path::Path) -> Result<Scenario, String> {
116    let s = std::fs::read_to_string(path).map_err(|e| format!("read {}: {}", path.display(), e))?;
117    let parsed: Scenario =
118        toml::from_str(&s).map_err(|e| format!("parse {}: {}", path.display(), e))?;
119    Ok(parsed)
120}
121
122use crate::runner::{HookCall, RunOutcome, invoke_exec, invoke_hook, load_plugin};
123use crate::test_host::TestState;
124use yosh_plugin_api::pattern::CommandPattern;
125use yosh_plugin_api::{capabilities_to_bitflags, parse_capability};
126
127#[derive(Debug)]
128pub enum StepResult {
129    Pass,
130    Fail(String),
131}
132
133pub fn run_scenario(path: &std::path::Path) -> Vec<StepResult> {
134    let scenario = match parse(path) {
135        Ok(s) => s,
136        Err(e) => return vec![StepResult::Fail(format!("parse error: {}", e))],
137    };
138
139    let wasm_path = path
140        .parent()
141        .map(|p| p.join(&scenario.plugin))
142        .unwrap_or(scenario.plugin.clone());
143    let mut results = Vec::new();
144
145    for (idx, step) in scenario.steps.iter().enumerate() {
146        let state = build_state(&scenario);
147        let timeout = std::time::Duration::from_millis(scenario.env.timeout_ms);
148        let loaded = match load_plugin(&wasm_path, state, timeout) {
149            Ok(l) => l,
150            Err(e) => {
151                results.push(StepResult::Fail(format!("step {}: load: {}", idx + 1, e)));
152                continue;
153            }
154        };
155
156        let (outcome, expect) = match step {
157            Step::Exec { args, expect } => {
158                if args.is_empty() {
159                    results.push(StepResult::Fail(format!(
160                        "step {}: exec needs at least 1 arg",
161                        idx + 1
162                    )));
163                    continue;
164                }
165                let (cmd, rest) = (&args[0], &args[1..]);
166                (invoke_exec(loaded, cmd, rest), expect)
167            }
168            Step::Hook { name, args, expect } => {
169                let call = match build_hook_call(*name, args) {
170                    Ok(c) => c,
171                    Err(e) => {
172                        results.push(StepResult::Fail(format!(
173                            "step {}: hook args: {}",
174                            idx + 1,
175                            e
176                        )));
177                        continue;
178                    }
179                };
180                (invoke_hook(loaded, call), expect)
181            }
182        };
183
184        results.push(evaluate(idx + 1, &outcome, expect));
185    }
186
187    if results.is_empty() {
188        results.push(StepResult::Pass);
189    }
190    results
191}
192
193fn build_state(scenario: &Scenario) -> TestState {
194    let mut state = TestState::default();
195    let parsed_caps: Vec<_> = scenario
196        .env
197        .caps
198        .iter()
199        .filter_map(|s| parse_capability(s))
200        .collect();
201    state.caps = capabilities_to_bitflags(&parsed_caps);
202    for (k, v) in &scenario.env.vars {
203        state.vars.insert(k.clone(), v.clone());
204    }
205    for k in &scenario.env.exported {
206        state.exported.insert(k.clone());
207    }
208    if !scenario.env.cwd.is_empty() {
209        state.cwd = scenario.env.cwd.clone().into();
210    }
211    state.allow_exec = scenario
212        .env
213        .allow_exec
214        .iter()
215        .filter_map(|p| CommandPattern::parse(p).ok())
216        .collect();
217    if !scenario.env.sandbox_root.is_empty() {
218        state.sandbox_root = Some(std::path::PathBuf::from(&scenario.env.sandbox_root));
219    } else {
220        for (k, v) in &scenario.files {
221            state
222                .files
223                .insert(std::path::PathBuf::from(k), v.as_bytes().to_vec());
224        }
225    }
226    state
227}
228
229fn build_hook_call(name: HookName, args: &[toml::Value]) -> Result<HookCall, String> {
230    fn s(v: &toml::Value) -> Result<String, String> {
231        v.as_str()
232            .map(|s| s.to_string())
233            .ok_or_else(|| "expected string".into())
234    }
235    fn i(v: &toml::Value) -> Result<i32, String> {
236        v.as_integer()
237            .map(|i| i as i32)
238            .ok_or_else(|| "expected integer".into())
239    }
240    match name {
241        HookName::PreExec => Ok(HookCall::PreExec {
242            command_line: s(args.first().ok_or("missing arg")?)?,
243        }),
244        HookName::PostExec => {
245            let cl = s(args.first().ok_or("missing command_line")?)?;
246            let ec = i(args.get(1).ok_or("missing exit_code")?)?;
247            Ok(HookCall::PostExec {
248                command_line: cl,
249                exit_code: ec,
250            })
251        }
252        HookName::OnCd => {
253            let old = s(args.first().ok_or("missing old")?)?;
254            let new = s(args.get(1).ok_or("missing new")?)?;
255            Ok(HookCall::OnCd { old, new })
256        }
257        HookName::PrePrompt => Ok(HookCall::PrePrompt),
258    }
259}
260
261fn evaluate(step_idx: usize, o: &RunOutcome, e: &Expect) -> StepResult {
262    macro_rules! fail {
263        ($($t:tt)*) => {{ return StepResult::Fail(format!("step {}: {}", step_idx, format_args!($($t)*))) }};
264    }
265
266    if let Some(want) = e.exit {
267        match o.exit_code {
268            Some(got) if got == want => {}
269            Some(got) => fail!("exit: want {}, got {}", want, got),
270            None => fail!("exit: want {}, got (no exit code — hook?)", want),
271        }
272    }
273
274    let stdout_str = String::from_utf8_lossy(&o.stdout);
275    let stderr_str = String::from_utf8_lossy(&o.stderr);
276
277    if let Some(want) = &e.stdout
278        && stdout_str != *want
279    {
280        fail!("stdout mismatch: want {:?}, got {:?}", want, stdout_str);
281    }
282    if let Some(want) = &e.stderr
283        && stderr_str != *want
284    {
285        fail!("stderr mismatch: want {:?}, got {:?}", want, stderr_str);
286    }
287    if let Some(sub) = &e.stdout_contains
288        && !stdout_str.contains(sub.as_str())
289    {
290        fail!("stdout_contains {:?} not found in {:?}", sub, stdout_str);
291    }
292    if let Some(sub) = &e.stderr_contains
293        && !stderr_str.contains(sub.as_str())
294    {
295        fail!("stderr_contains {:?} not found in {:?}", sub, stderr_str);
296    }
297    if let Some(re) = &e.stdout_regex {
298        let rx = regex::Regex::new(re).map_err(|err| err.to_string());
299        match rx {
300            Ok(rx) if !rx.is_match(&stdout_str) => {
301                fail!("stdout_regex {:?} did not match {:?}", re, stdout_str)
302            }
303            Err(err) => fail!("stdout_regex invalid: {}", err),
304            _ => {}
305        }
306    }
307    if let Some(re) = &e.stderr_regex {
308        let rx = regex::Regex::new(re).map_err(|err| err.to_string());
309        match rx {
310            Ok(rx) if !rx.is_match(&stderr_str) => {
311                fail!("stderr_regex {:?} did not match {:?}", re, stderr_str)
312            }
313            Err(err) => fail!("stderr_regex invalid: {}", err),
314            _ => {}
315        }
316    }
317
318    if let Some(want) = &e.vars_set {
319        let got: BTreeMap<String, String> = o.set_log.iter().cloned().collect();
320        if got != *want {
321            fail!("vars_set: want {:?}, got {:?}", want, got);
322        }
323    }
324    if let Some(want) = &e.vars_export {
325        let got: BTreeMap<String, String> = o.export_log.iter().cloned().collect();
326        if got != *want {
327            fail!("vars_export: want {:?}, got {:?}", want, got);
328        }
329    }
330
331    if let Some(want) = &e.files_write {
332        let got: BTreeMap<String, usize> = o
333            .write_log
334            .iter()
335            .map(|(p, n)| (p.display().to_string(), *n))
336            .collect();
337        for (path, expectation) in want {
338            match expectation {
339                FileExpect::Bytes(b) => {
340                    let want_len = b.len();
341                    match got.get(path) {
342                        Some(actual) if *actual == want_len => {}
343                        Some(actual) => fail!(
344                            "files_write[{}] len: want {}, got {}",
345                            path,
346                            want_len,
347                            actual
348                        ),
349                        None => fail!("files_write[{}] not written", path),
350                    }
351                }
352                FileExpect::Struct { len, bytes_eq } => {
353                    if let Some(l) = len {
354                        match got.get(path) {
355                            Some(actual) if *actual == *l => {}
356                            Some(actual) => {
357                                fail!("files_write[{}] len: want {}, got {}", path, l, actual)
358                            }
359                            None => fail!("files_write[{}] not written", path),
360                        }
361                    }
362                    if let Some(b) = bytes_eq {
363                        let want_len = b.len();
364                        match got.get(path) {
365                            Some(actual) if *actual == want_len => {}
366                            Some(actual) => fail!(
367                                "files_write[{}] bytes_eq len: want {}, got {}",
368                                path,
369                                want_len,
370                                actual
371                            ),
372                            None => fail!("files_write[{}] not written", path),
373                        }
374                    }
375                }
376            }
377        }
378    }
379
380    if let Some(want_seq) = &e.exec_called {
381        if want_seq.len() != o.exec_log.len() {
382            fail!(
383                "exec_called: want {} calls, got {}",
384                want_seq.len(),
385                o.exec_log.len()
386            );
387        }
388        for (i, (w, g)) in want_seq.iter().zip(o.exec_log.iter()).enumerate() {
389            if w.program != g.program {
390                fail!(
391                    "exec_called[{}].program: want {}, got {}",
392                    i,
393                    w.program,
394                    g.program
395                );
396            }
397            if w.args != g.args {
398                fail!(
399                    "exec_called[{}].args: want {:?}, got {:?}",
400                    i,
401                    w.args,
402                    g.args
403                );
404            }
405            if let Some(exit) = w.exit
406                && exit != g.exit_code
407            {
408                fail!(
409                    "exec_called[{}].exit: want {}, got {}",
410                    i,
411                    exit,
412                    g.exit_code
413                );
414            }
415        }
416    }
417
418    if let Some(want) = e.trap {
419        let got = o.error_kind == Some("trap");
420        if got != want {
421            fail!("trap: want {}, got {}", want, got);
422        }
423    }
424
425    StepResult::Pass
426}
427
428#[cfg(test)]
429mod evaluator_tests {
430    use super::*;
431    use crate::runner::RunOutcome;
432
433    fn outcome_with(exit: Option<i32>, stdout: &[u8]) -> RunOutcome {
434        RunOutcome {
435            exit_code: exit,
436            stdout: stdout.to_vec(),
437            stderr: Vec::new(),
438            set_log: Vec::new(),
439            export_log: Vec::new(),
440            write_log: Vec::new(),
441            exec_log: Vec::new(),
442            error: None,
443            error_kind: None,
444        }
445    }
446
447    #[test]
448    fn expect_exit_match_passes() {
449        let o = outcome_with(Some(0), b"");
450        let e = Expect {
451            exit: Some(0),
452            ..Default::default()
453        };
454        assert!(matches!(evaluate(1, &o, &e), StepResult::Pass));
455    }
456
457    #[test]
458    fn expect_exit_mismatch_fails() {
459        let o = outcome_with(Some(2), b"");
460        let e = Expect {
461            exit: Some(0),
462            ..Default::default()
463        };
464        match evaluate(1, &o, &e) {
465            StepResult::Fail(s) => assert!(s.contains("exit")),
466            _ => panic!("expected fail"),
467        }
468    }
469
470    #[test]
471    fn expect_stdout_contains_works() {
472        let o = outcome_with(Some(0), b"hello world\n");
473        let e = Expect {
474            stdout_contains: Some("world".into()),
475            ..Default::default()
476        };
477        assert!(matches!(evaluate(1, &o, &e), StepResult::Pass));
478    }
479
480    #[test]
481    fn run_dir_collects_toml_files() {
482        let tmp = tempfile::tempdir().unwrap();
483        let a = tmp.path().join("a.toml");
484        std::fs::write(
485            &a,
486            r#"
487            plugin = "missing.wasm"
488            [[step]]
489            call = "exec"
490            args = ["x"]
491        "#,
492        )
493        .unwrap();
494        let reports = run_dir(tmp.path(), None);
495        assert_eq!(reports.len(), 1);
496        assert!(!reports[0].passed()); // wasm missing
497    }
498
499    #[test]
500    fn format_summary_json_has_summary_line() {
501        let reports = vec![];
502        let s = format_summary_json(&reports);
503        assert!(s.contains("\"summary\""));
504    }
505}
506
507#[derive(Debug)]
508pub struct ScenarioReport {
509    pub file: std::path::PathBuf,
510    pub steps: Vec<StepResult>,
511}
512
513impl ScenarioReport {
514    pub fn passed(&self) -> bool {
515        self.steps.iter().all(|r| matches!(r, StepResult::Pass))
516    }
517}
518
519pub fn run_dir(path: &std::path::Path, filter: Option<&str>) -> Vec<ScenarioReport> {
520    let mut reports = Vec::new();
521    let filter_rx = filter.and_then(|f| match regex::Regex::new(f) {
522        Ok(rx) => Some(rx),
523        Err(e) => {
524            eprintln!(
525                "yosh-plugin: ignoring invalid --filter regex {:?}: {}",
526                f, e
527            );
528            None
529        }
530    });
531
532    fn walk(dir: &std::path::Path, out: &mut Vec<std::path::PathBuf>) {
533        let Ok(rd) = std::fs::read_dir(dir) else {
534            return;
535        };
536        for entry in rd.flatten() {
537            let p = entry.path();
538            if p.is_dir() {
539                walk(&p, out);
540            } else if p.extension().and_then(|s| s.to_str()) == Some("toml") {
541                out.push(p);
542            }
543        }
544    }
545
546    let mut paths = Vec::new();
547    if path.is_dir() {
548        walk(path, &mut paths);
549    } else if path.exists() {
550        paths.push(path.to_path_buf());
551    }
552    paths.sort();
553
554    for p in paths {
555        if let Some(rx) = &filter_rx
556            && !rx.is_match(&p.to_string_lossy())
557        {
558            continue;
559        }
560        let results = run_scenario(&p);
561        reports.push(ScenarioReport {
562            file: p,
563            steps: results,
564        });
565    }
566    reports
567}
568
569pub fn format_summary_human(reports: &[ScenarioReport]) -> String {
570    use std::fmt::Write as _;
571    let mut out = String::new();
572    let _ = writeln!(out, "running {} scenarios", reports.len());
573    let mut passed = 0;
574    let mut failed = 0;
575    for r in reports {
576        if r.passed() {
577            passed += 1;
578            let _ = writeln!(out, "  \u{2713} {}", r.file.display());
579        } else {
580            failed += 1;
581            let _ = writeln!(out, "  \u{2717} {}", r.file.display());
582            for s in &r.steps {
583                if let StepResult::Fail(msg) = s {
584                    let _ = writeln!(out, "      {}", msg);
585                }
586            }
587        }
588    }
589    let _ = writeln!(out, "{} passed, {} failed", passed, failed);
590    out
591}
592
593pub fn format_summary_json(reports: &[ScenarioReport]) -> String {
594    use std::fmt::Write as _;
595    let mut out = String::new();
596    let mut passed = 0;
597    let mut failed = 0;
598    for r in reports {
599        if r.passed() {
600            passed += 1;
601            let _ = writeln!(
602                out,
603                "{}",
604                serde_json::json!({
605                    "file": r.file.display().to_string(),
606                    "status": "pass",
607                    "steps": r.steps.len()
608                })
609            );
610        } else {
611            failed += 1;
612            let reason = r
613                .steps
614                .iter()
615                .find_map(|s| match s {
616                    StepResult::Fail(m) => Some(m.clone()),
617                    _ => None,
618                })
619                .unwrap_or_default();
620            let _ = writeln!(
621                out,
622                "{}",
623                serde_json::json!({
624                    "file": r.file.display().to_string(),
625                    "status": "fail",
626                    "reason": reason
627                })
628            );
629        }
630    }
631    let _ = writeln!(
632        out,
633        "{}",
634        serde_json::json!({
635            "summary": { "passed": passed, "failed": failed, "total": reports.len() }
636        })
637    );
638    out
639}
640
641#[cfg(test)]
642mod tests {
643    use super::*;
644
645    fn parse_str(s: &str) -> Result<Scenario, String> {
646        toml::from_str(s).map_err(|e| e.to_string())
647    }
648
649    #[test]
650    fn minimal_scenario_parses() {
651        let sc = parse_str(
652            r#"
653            plugin = "a.wasm"
654            [[step]]
655            call = "exec"
656            args = ["echo", "hi"]
657        "#,
658        )
659        .unwrap();
660        assert_eq!(sc.plugin.to_str().unwrap(), "a.wasm");
661        assert_eq!(sc.steps.len(), 1);
662        match &sc.steps[0] {
663            Step::Exec { args, .. } => {
664                assert_eq!(args, &vec!["echo".to_string(), "hi".to_string()])
665            }
666            _ => panic!("expected exec step"),
667        }
668    }
669
670    #[test]
671    fn unknown_expect_key_rejected() {
672        let err = parse_str(
673            r#"
674            plugin = "a.wasm"
675            [[step]]
676            call = "exec"
677            args = ["x"]
678            [step.expect]
679            mystery = "boom"
680        "#,
681        )
682        .unwrap_err();
683        assert!(err.contains("mystery") || err.contains("unknown field"));
684    }
685
686    #[test]
687    fn hook_step_parses() {
688        let sc = parse_str(
689            r#"
690            plugin = "a.wasm"
691            [[step]]
692            call = "hook"
693            name = "on-cd"
694            args = ["/old", "/new"]
695        "#,
696        )
697        .unwrap();
698        match &sc.steps[0] {
699            Step::Hook { name, args, .. } => {
700                assert_eq!(*name, HookName::OnCd);
701                assert_eq!(args.len(), 2);
702            }
703            _ => panic!("expected hook step"),
704        }
705    }
706}
yosh_plugin_manager/scenario.rs

yosh_plugin_manager/
scenario.rs
