rightkit_qa/
suite.rs

1//! A QA run: select scenarios by tier, take the run lock, create the isolated
2//! workspace, run scenarios sequentially, sweep orphans, and write hashed evidence.
3use crate::audio::SystemMute;
4use crate::config::{validate_tier_list, QaConfig};
5use crate::engine::{build_engine, EngineTarget};
6use crate::evidence::{Bundle, CheckStatus, RunStatus};
7use crate::lock::{self, LockOptions};
8use crate::process::{sweep_registries, Tracker};
9use crate::scenario::{
10    discover, run_step, tier_selected, unmet_requirement, Ctx, Scenario, StepRecord,
11};
12use crate::util::{err, now_iso, platform, Result};
13use crate::workspace;
14use serde_json::{json, Value};
15use std::path::{Path, PathBuf};
16use std::time::{Duration, Instant};
17
18#[derive(Debug, Clone, Default)]
19pub struct SuiteOptions {
20    pub tier: Option<String>,
21    pub only: Option<String>,
22    pub engine: Option<PathBuf>,
23    pub ui_binary: Option<PathBuf>,
24    pub build_engine: bool,
25    pub build_ui: bool,
26    pub keep_data: bool,
27    /// Explicit scenario files (override discovery).
28    pub files: Vec<PathBuf>,
29}
30
31#[derive(Debug, Clone)]
32pub struct Outcome {
33    pub name: String,
34    pub file: PathBuf,
35    pub tier: String,
36    pub status: CheckStatus,
37    pub reason: Option<String>,
38    pub ms: u128,
39    pub steps: Vec<StepRecord>,
40}
41
42impl Outcome {
43    fn to_value(&self) -> Value {
44        json!({
45            "name": self.name, "file": self.file.to_string_lossy(), "tier": self.tier,
46            "status": format!("{:?}", self.status).to_lowercase(), "reason": self.reason, "ms": self.ms,
47            "steps": self.steps.iter().map(|s| json!({"index": s.index, "kind": s.kind, "name": s.name, "ok": s.ok, "ms": s.ms, "error": s.error})).collect::<Vec<_>>(),
48        })
49    }
50}
51
52#[derive(Debug)]
53pub struct SuiteReport {
54    pub exit_code: i32,
55    pub run_id: String,
56    pub run_dir: PathBuf,
57    pub evidence_path: PathBuf,
58    pub results_path: PathBuf,
59    pub outcomes: Vec<Outcome>,
60}
61
62pub fn select(cfg: &QaConfig, opts: &SuiteOptions) -> Result<Vec<Scenario>> {
63    let tier = opts
64        .tier
65        .clone()
66        .unwrap_or_else(|| cfg.default_tier.clone());
67    validate_tier_list(&tier)?;
68    let files = if opts.files.is_empty() {
69        discover(&cfg.scenario_dirs)?
70    } else {
71        opts.files.clone()
72    };
73    let mut out = vec![];
74    for f in files {
75        let s = Scenario::load(&f)?;
76        let matches_only = opts
77            .only
78            .as_ref()
79            .map(|o| s.name.contains(o.as_str()) || f.to_string_lossy().contains(o.as_str()))
80            .unwrap_or(true);
81        if matches_only && (!opts.files.is_empty() || tier_selected(&s.tier, &tier)) {
82            out.push(s);
83        }
84    }
85    Ok(out)
86}
87
88fn engine_target(
89    cfg: &QaConfig,
90    opts: &SuiteOptions,
91    tracker: &Tracker,
92    tier: &str,
93) -> Result<Option<EngineTarget>> {
94    let mut env: Vec<(String, String)> = cfg
95        .env
96        .iter()
97        .chain(cfg.engine.env.iter())
98        .map(|(k, v)| (k.clone(), v.clone()))
99        .collect();
100    env.push(("RIGHTKIT_QA_TIER".into(), tier.into()));
101    env.push(("RIGHTKIT_QA_MUTE".into(), "1".into()));
102    let binary = if let Some(p) = &opts.engine {
103        Some(p.clone())
104    } else if let Ok(p) = std::env::var("RIGHTKIT_QA_ENGINE") {
105        Some(PathBuf::from(p))
106    } else if opts.build_engine {
107        if cfg.engine.build.is_empty() {
108            return err("--build needs [engine].build in the config");
109        }
110        let pkg = cfg.engine.package.clone().ok_or_else(|| {
111            crate::util::Error("--build needs [engine].package (the cargo target name)".into())
112        })?;
113        let build_env: Vec<(String, String)> = cfg
114            .env
115            .iter()
116            .map(|(k, v)| (k.clone(), v.clone()))
117            .collect();
118        Some(build_engine(
119            &cfg.app_root,
120            &cfg.engine.build,
121            &pkg,
122            &build_env,
123            tracker,
124        )?)
125    } else {
126        cfg.engine_binary()?
127    };
128    Ok(binary.map(|binary| EngineTarget {
129        binary,
130        env,
131        data_dir_env: cfg.engine.data_dir_env.clone(),
132    }))
133}
134
135fn ui_binary(cfg: &QaConfig, opts: &SuiteOptions, tracker: &Tracker) -> Result<Option<PathBuf>> {
136    if let Some(p) = &opts.ui_binary {
137        return Ok(Some(p.clone()));
138    }
139    if let Ok(p) = std::env::var("RIGHTKIT_QA_UI_BINARY") {
140        return Ok(Some(PathBuf::from(p)));
141    }
142    if opts.build_ui {
143        if cfg.ui.build.is_empty() {
144            return err("--build-ui needs [ui].build in the config");
145        }
146        let pkg = cfg.ui.package.clone().ok_or_else(|| {
147            crate::util::Error("--build-ui needs [ui].package (the cargo target name)".into())
148        })?;
149        let env: Vec<(String, String)> = cfg
150            .env
151            .iter()
152            .map(|(k, v)| (k.clone(), v.clone()))
153            .collect();
154        return Ok(Some(build_engine(
155            &cfg.app_root,
156            &cfg.ui.build,
157            &pkg,
158            &env,
159            tracker,
160        )?));
161    }
162    cfg.ui_binary()
163}
164
165pub fn run_suite(cfg: &QaConfig, opts: &SuiteOptions) -> Result<SuiteReport> {
166    run_suite_at(cfg, opts, None)
167}
168
169/// Test/embedding entry point with an explicit managed root.
170pub fn run_suite_in_managed_root(
171    cfg: &QaConfig,
172    opts: &SuiteOptions,
173    managed_root: &Path,
174) -> Result<SuiteReport> {
175    run_suite_at(cfg, opts, Some(managed_root))
176}
177
178fn run_suite_at(
179    cfg: &QaConfig,
180    opts: &SuiteOptions,
181    managed_override: Option<&Path>,
182) -> Result<SuiteReport> {
183    let platform = platform()?;
184    let tier = opts
185        .tier
186        .clone()
187        .unwrap_or_else(|| cfg.default_tier.clone());
188    let scenarios = select(cfg, opts)?;
189    if scenarios.is_empty() {
190        return err(format!(
191            "no scenarios matched tier '{tier}'{}",
192            opts.only
193                .as_ref()
194                .map(|o| format!(" --only {o}"))
195                .unwrap_or_default()
196        ));
197    }
198    let managed_root = managed_override
199        .map(Path::to_path_buf)
200        .unwrap_or_else(|| crate::storage::managed_root(None));
201    let managed_app_root = crate::storage::prepare_app_root(&managed_root, &cfg.app)?;
202    let lock_opts = LockOptions {
203        timeout: Duration::from_secs(cfg.lock.timeout_s.unwrap_or(120)),
204        stale: Duration::from_secs(cfg.lock.stale_s.unwrap_or(60)),
205        ..Default::default()
206    };
207    let mut ownership = lock::acquire(
208        &managed_app_root.join("native.lock"),
209        &format!("{}:{tier}", cfg.app),
210        &lock_opts,
211    )?;
212    let run_id = crate::util::new_id();
213    // Earlier crashed runs: kill survivors by recorded pid + command line, then drop their dirs.
214    let swept = sweep_registries(&managed_app_root, &run_id);
215    let ws = workspace::create(&managed_root, Some(&run_id), &cfg.app)?;
216    let tracker = Tracker::with_registry(ws.root.join("pids.json"));
217    let mut bundle = Bundle::new(&cfg.app, platform, &run_id)?;
218    let _mute = cfg.audio.system_mute.then(SystemMute::engage);
219
220    let needs_engine = scenarios
221        .iter()
222        .any(|s| s.target == "engine" && unmet_requirement(&s.requires, &cfg.app_root).is_none());
223    let needs_ui = scenarios.iter().any(|s| s.target == "ui");
224    let engine = engine_target(cfg, opts, &tracker, &tier)?;
225    let ui = if needs_ui {
226        ui_binary(cfg, opts, &tracker)?
227    } else {
228        None
229    };
230    if needs_engine && engine.is_none() {
231        ownership.release();
232        return err("selected scenarios need the engine but no binary is configured (use --engine, RIGHTKIT_QA_ENGINE, [engine].binary, or --build)");
233    }
234
235    let identity = match run_identity(
236        cfg,
237        &scenarios,
238        engine.as_ref().map(|e| e.binary.as_path()),
239        ui.as_deref(),
240        needs_engine,
241        false,
242    ) {
243        Ok(i) => i,
244        Err(e) => {
245            ownership.release();
246            return Err(e);
247        }
248    };
249    bundle.set_identity(identity)?;
250    println!(
251        "[qa] app={} platform={platform} tier={tier} scenarios={} run={run_id}",
252        cfg.app,
253        scenarios.len()
254    );
255    if let Some(e) = &engine {
256        println!("[qa] engine={}", e.binary.display());
257    }
258    if !swept.is_empty() {
259        println!("[qa] swept orphan pids from earlier runs: {swept:?}");
260    }
261
262    let mut outcomes = vec![];
263    for sc in &scenarios {
264        let started = Instant::now();
265        let name = sc.name.clone();
266        if let Some(reason) = unmet_requirement(&sc.requires, &cfg.app_root) {
267            if sc.optional {
268                println!("[qa] SKIP  {name}: {reason} (optional)");
269                bundle.add_check(
270                    &name,
271                    CheckStatus::Skipped,
272                    json!({"reason": reason, "optional": true}),
273                    &[],
274                )?;
275                outcomes.push(Outcome {
276                    name,
277                    file: sc.file.clone(),
278                    tier: sc.tier.clone(),
279                    status: CheckStatus::Skipped,
280                    reason: Some(reason),
281                    ms: 0,
282                    steps: vec![],
283                });
284            } else {
285                let reason = format!("mandatory scenario could not run: {reason} (mark it `optional = true` only if skipping is acceptable)");
286                println!("[qa] FAIL  {name}: {reason}");
287                bundle.add_check(
288                    &name,
289                    CheckStatus::Failed,
290                    json!({"error": reason, "mandatory": true}),
291                    &[],
292                )?;
293                outcomes.push(Outcome {
294                    name,
295                    file: sc.file.clone(),
296                    tier: sc.tier.clone(),
297                    status: CheckStatus::Failed,
298                    reason: Some(reason),
299                    ms: 0,
300                    steps: vec![],
301                });
302            }
303            continue;
304        }
305        if sc.target == "ui" && ui.is_none() {
306            let reason =
307                "no UI binary for this platform ([ui].binaries / --ui-binary / --build-ui)"
308                    .to_string();
309            println!("[qa] FAIL  {name}: {reason}");
310            bundle.add_check(&name, CheckStatus::Failed, json!({"error": reason}), &[])?;
311            outcomes.push(Outcome {
312                name,
313                file: sc.file.clone(),
314                tier: sc.tier.clone(),
315                status: CheckStatus::Failed,
316                reason: Some(reason),
317                ms: 0,
318                steps: vec![],
319            });
320            continue;
321        }
322        let mut ctx = Ctx::new(cfg, engine.clone(), ui.clone(), &ws, &tracker, &name)?;
323        let mut steps = vec![];
324        let mut failure: Option<String> = None;
325        let limit = Duration::from_secs(sc.timeout_s.unwrap_or(600));
326        for (i, step) in sc.steps.iter().enumerate() {
327            let t = Instant::now();
328            if started.elapsed() > limit {
329                failure = Some(format!(
330                    "scenario exceeded its {}s budget before step {}",
331                    limit.as_secs(),
332                    i + 1
333                ));
334                break;
335            }
336            let label = step.label.clone().unwrap_or_else(|| step.kind.clone());
337            match run_step(&mut ctx, step) {
338                Ok(_) => steps.push(StepRecord {
339                    index: i + 1,
340                    kind: step.kind.clone(),
341                    name: label,
342                    ok: true,
343                    ms: t.elapsed().as_millis(),
344                    error: None,
345                }),
346                Err(e) => {
347                    steps.push(StepRecord {
348                        index: i + 1,
349                        kind: step.kind.clone(),
350                        name: label,
351                        ok: false,
352                        ms: t.elapsed().as_millis(),
353                        error: Some(e.0.clone()),
354                    });
355                    failure = Some(e.0);
356                    break;
357                }
358            }
359        }
360        if failure.is_some() {
361            if let Some((c, _)) = &ctx.ui {
362                let p = ctx.log_dir.join("failure.png");
363                if c.screenshot_to(&p).is_ok() {
364                    ctx.artifacts.push((format!("{name}/failure.png"), p));
365                }
366            }
367        }
368        let problems = ctx.teardown();
369        if failure.is_none() && !problems.is_empty() {
370            failure = Some(problems.join("; "));
371        }
372        // Hash every artifact the scenario produced, then bind the check to them.
373        let mut names = vec![];
374        let mut artifact_errors = vec![];
375        for (n, path) in &ctx.artifacts {
376            if names.contains(n) {
377                continue;
378            }
379            match ws
380                .preserve_artifact(path)
381                .and_then(|kept| bundle.add_artifact(n, &kept))
382            {
383                Ok(_) => names.push(n.clone()),
384                Err(e) => artifact_errors.push(e.0),
385            }
386        }
387        if !artifact_errors.is_empty() {
388            let msg = format!(
389                "required evidence could not be recorded: {}",
390                artifact_errors.join("; ")
391            );
392            failure = Some(match failure.take() {
393                Some(f) => format!("{f}; {msg}"),
394                None => msg,
395            });
396        }
397        let status = if failure.is_none() {
398            CheckStatus::Passed
399        } else {
400            CheckStatus::Failed
401        };
402        let details = json!({"file": sc.file.to_string_lossy(), "tier": sc.tier, "steps": steps.len(), "error": failure});
403        bundle.add_check(&name, status, details, &names)?;
404        println!(
405            "[qa] {}  {name} ({} ms){}",
406            if failure.is_none() { "PASS" } else { "FAIL" },
407            started.elapsed().as_millis(),
408            failure
409                .as_ref()
410                .map(|f| format!("\n       {f}"))
411                .unwrap_or_default()
412        );
413        outcomes.push(Outcome {
414            name,
415            file: sc.file.clone(),
416            tier: sc.tier.clone(),
417            status,
418            reason: failure,
419            ms: started.elapsed().as_millis(),
420            steps,
421        });
422    }
423
424    tracker.kill_all();
425    let failed = outcomes
426        .iter()
427        .filter(|o| o.status == CheckStatus::Failed)
428        .count();
429    let passed = outcomes
430        .iter()
431        .filter(|o| o.status == CheckStatus::Passed)
432        .count();
433    // Skips are only ever optional scenarios; a run whose scenarios were all skipped proves nothing.
434    bundle.finish(if failed == 0 && passed > 0 {
435        RunStatus::Passed
436    } else {
437        RunStatus::Failed
438    });
439    let evidence_path = ws.evidence_dir.join("evidence.json");
440    let results_path = ws.evidence_dir.join("results.json");
441    std::fs::write(
442        &results_path,
443        serde_json::to_vec_pretty(&json!({
444            "app": cfg.app, "platform": platform, "runId": run_id, "tier": tier, "finishedAt": now_iso(),
445            "passed": passed, "failed": failed, "skipped": outcomes.len() - passed - failed,
446            "scenarios": outcomes.iter().map(Outcome::to_value).collect::<Vec<_>>(),
447        }))?,
448    )?;
449    bundle.write(&evidence_path)?;
450    let _ = std::fs::write(
451        ws.root.parent().unwrap_or(&ws.root).join("latest.json"),
452        serde_json::to_vec_pretty(
453            &json!({"runId": run_id, "evidence": evidence_path.to_string_lossy(), "results": results_path.to_string_lossy()}),
454        )?,
455    );
456    ws.finish()?;
457    if !opts.keep_data {
458        let report = workspace::dispose(&ws, true, 5, 200);
459        for (p, why) in report.retained {
460            println!("[qa] WARN could not remove {}: {why}", p.display());
461        }
462    }
463    println!(
464        "[qa] {passed} passed, {failed} failed, {} skipped; evidence {}",
465        outcomes.len() - passed - failed,
466        evidence_path.display()
467    );
468    ownership.release();
469    Ok(SuiteReport {
470        exit_code: if failed == 0 && passed > 0 { 0 } else { 1 },
471        run_id,
472        run_dir: ws.root.clone(),
473        evidence_path,
474        results_path,
475        outcomes,
476    })
477}
478
479fn file_sha(path: &std::path::Path) -> Result<String> {
480    let bytes = std::fs::read(path).map_err(|e| {
481        crate::util::Error(format!(
482            "cannot hash {} for the evidence identity: {e}",
483            path.display()
484        ))
485    })?;
486    Ok(crate::util::sha256_hex(&bytes))
487}
488
489fn git_revision(root: &std::path::Path) -> Value {
490    let git = |args: &[&str]| {
491        let mut git = std::process::Command::new("git");
492        git.arg("-C").arg(root).args(args);
493        #[cfg(windows)]
494        {
495            use std::os::windows::process::CommandExt;
496            git.creation_flags(0x0800_0000); // CREATE_NO_WINDOW: never flash a console from a GUI host
497        }
498        git.output()
499            .ok()
500            .filter(|o| o.status.success())
501            .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string())
502    };
503    match git(&["rev-parse", "HEAD"]) {
504        Some(head) => {
505            json!({"head": head, "dirty": git(&["status", "--porcelain"]).map(|s| !s.is_empty())})
506        }
507        None => Value::Null,
508    }
509}
510
511/// What this run exercises: config, scenario files, engine/UI binaries and the source revision,
512/// all by digest, so a receipt cannot be reused for different bytes.
513fn run_identity(
514    cfg: &QaConfig,
515    scenarios: &[Scenario],
516    engine: Option<&std::path::Path>,
517    ui: Option<&std::path::Path>,
518    needs_engine: bool,
519    needs_ui: bool,
520) -> Result<Value> {
521    let mut scenario_ids = vec![];
522    for sc in scenarios {
523        scenario_ids.push(json!({"name": sc.name, "file": sc.file.to_string_lossy(), "sha256": file_sha(&sc.file)?}));
524    }
525    let binary = |p: Option<&std::path::Path>, what: &str, needed: bool| -> Result<Value> {
526        match p {
527            Some(p) => {
528                let target = if p.is_dir() { None } else { Some(p) };
529                match target {
530                    Some(t) => Ok(json!({"path": t.to_string_lossy(), "sha256": file_sha(t)?})),
531                    None => {
532                        // A `.app` bundle: bind to every executable inside Contents/MacOS.
533                        let mut exes = vec![];
534                        if let Ok(rd) = std::fs::read_dir(p.join("Contents").join("MacOS")) {
535                            for e in rd.flatten() {
536                                exes.push(json!({"path": e.path().to_string_lossy(), "sha256": file_sha(&e.path())?}));
537                            }
538                        }
539                        if exes.is_empty() {
540                            return err(format!(
541                                "{} has no executable to bind into the evidence identity",
542                                p.display()
543                            ));
544                        }
545                        Ok(json!({"path": p.to_string_lossy(), "executables": exes}))
546                    }
547                }
548            }
549            None if needed => err(format!(
550                "evidence identity needs the {what} binary but none resolved"
551            )),
552            None => Ok(Value::Null),
553        }
554    };
555    Ok(json!({
556        "config": {"path": cfg.config_path.to_string_lossy(), "sha256": file_sha(&cfg.config_path)?},
557        "scenarios": scenario_ids,
558        "engine": binary(engine, "engine", needs_engine)?,
559        "ui": binary(ui, "UI", needs_ui)?,
560        "source": git_revision(&cfg.app_root),
561    }))
562}
rightkit_qa/suite.rs

rightkit_qa/
suite.rs
