rightkit_qa/
suite.rs

1//! A QA run: select scenarios by tier, take the run lock, create the isolated
2//! workspace, run scenarios sequentially, sweep orphans, and write hashed evidence.
3use crate::audio::SystemMute;
4use crate::config::{validate_tier_list, QaConfig};
5use crate::engine::{build_engine, EngineTarget};
6use crate::evidence::{Bundle, CheckStatus, RunStatus};
7use crate::lock::{self, LockOptions};
8use crate::process::{sweep_registries, Tracker};
9use crate::scenario::{
10    discover, run_step, tier_selected, unmet_requirement, Ctx, Scenario, StepRecord,
11};
12use crate::util::{err, now_iso, platform, Result};
13use crate::workspace;
14use serde_json::{json, Value};
15use std::path::PathBuf;
16use std::time::{Duration, Instant};
17
18#[derive(Debug, Clone, Default)]
19pub struct SuiteOptions {
20    pub tier: Option<String>,
21    pub only: Option<String>,
22    pub engine: Option<PathBuf>,
23    pub ui_binary: Option<PathBuf>,
24    pub build_engine: bool,
25    pub build_ui: bool,
26    pub keep_data: bool,
27    /// Explicit scenario files (override discovery).
28    pub files: Vec<PathBuf>,
29}
30
31#[derive(Debug, Clone)]
32pub struct Outcome {
33    pub name: String,
34    pub file: PathBuf,
35    pub tier: String,
36    pub status: CheckStatus,
37    pub reason: Option<String>,
38    pub ms: u128,
39    pub steps: Vec<StepRecord>,
40}
41
42impl Outcome {
43    fn to_value(&self) -> Value {
44        json!({
45            "name": self.name, "file": self.file.to_string_lossy(), "tier": self.tier,
46            "status": format!("{:?}", self.status).to_lowercase(), "reason": self.reason, "ms": self.ms,
47            "steps": self.steps.iter().map(|s| json!({"index": s.index, "kind": s.kind, "name": s.name, "ok": s.ok, "ms": s.ms, "error": s.error})).collect::<Vec<_>>(),
48        })
49    }
50}
51
52#[derive(Debug)]
53pub struct SuiteReport {
54    pub exit_code: i32,
55    pub run_id: String,
56    pub run_dir: PathBuf,
57    pub evidence_path: PathBuf,
58    pub results_path: PathBuf,
59    pub outcomes: Vec<Outcome>,
60}
61
62pub fn select(cfg: &QaConfig, opts: &SuiteOptions) -> Result<Vec<Scenario>> {
63    let tier = opts
64        .tier
65        .clone()
66        .unwrap_or_else(|| cfg.default_tier.clone());
67    validate_tier_list(&tier)?;
68    let files = if opts.files.is_empty() {
69        discover(&cfg.scenario_dirs)?
70    } else {
71        opts.files.clone()
72    };
73    let mut out = vec![];
74    for f in files {
75        let s = Scenario::load(&f)?;
76        let matches_only = opts
77            .only
78            .as_ref()
79            .map(|o| s.name.contains(o.as_str()) || f.to_string_lossy().contains(o.as_str()))
80            .unwrap_or(true);
81        if matches_only && (!opts.files.is_empty() || tier_selected(&s.tier, &tier)) {
82            out.push(s);
83        }
84    }
85    Ok(out)
86}
87
88fn engine_target(
89    cfg: &QaConfig,
90    opts: &SuiteOptions,
91    tracker: &Tracker,
92    tier: &str,
93) -> Result<Option<EngineTarget>> {
94    let mut env: Vec<(String, String)> = cfg
95        .env
96        .iter()
97        .chain(cfg.engine.env.iter())
98        .map(|(k, v)| (k.clone(), v.clone()))
99        .collect();
100    env.push(("RIGHTKIT_QA_TIER".into(), tier.into()));
101    env.push(("RIGHTKIT_QA_MUTE".into(), "1".into()));
102    let binary = if let Some(p) = &opts.engine {
103        Some(p.clone())
104    } else if let Ok(p) = std::env::var("RIGHTKIT_QA_ENGINE") {
105        Some(PathBuf::from(p))
106    } else if opts.build_engine {
107        if cfg.engine.build.is_empty() {
108            return err("--build needs [engine].build in the config");
109        }
110        let pkg = cfg.engine.package.clone().ok_or_else(|| {
111            crate::util::Error("--build needs [engine].package (the cargo target name)".into())
112        })?;
113        let build_env: Vec<(String, String)> = cfg
114            .env
115            .iter()
116            .map(|(k, v)| (k.clone(), v.clone()))
117            .collect();
118        Some(build_engine(
119            &cfg.app_root,
120            &cfg.engine.build,
121            &pkg,
122            &build_env,
123            tracker,
124        )?)
125    } else {
126        cfg.engine_binary()?
127    };
128    Ok(binary.map(|binary| EngineTarget {
129        binary,
130        env,
131        data_dir_env: cfg.engine.data_dir_env.clone(),
132    }))
133}
134
135fn ui_binary(cfg: &QaConfig, opts: &SuiteOptions, tracker: &Tracker) -> Result<Option<PathBuf>> {
136    if let Some(p) = &opts.ui_binary {
137        return Ok(Some(p.clone()));
138    }
139    if let Ok(p) = std::env::var("RIGHTKIT_QA_UI_BINARY") {
140        return Ok(Some(PathBuf::from(p)));
141    }
142    if opts.build_ui {
143        if cfg.ui.build.is_empty() {
144            return err("--build-ui needs [ui].build in the config");
145        }
146        let pkg = cfg.ui.package.clone().ok_or_else(|| {
147            crate::util::Error("--build-ui needs [ui].package (the cargo target name)".into())
148        })?;
149        let env: Vec<(String, String)> = cfg
150            .env
151            .iter()
152            .map(|(k, v)| (k.clone(), v.clone()))
153            .collect();
154        return Ok(Some(build_engine(
155            &cfg.app_root,
156            &cfg.ui.build,
157            &pkg,
158            &env,
159            tracker,
160        )?));
161    }
162    cfg.ui_binary()
163}
164
165pub fn run_suite(cfg: &QaConfig, opts: &SuiteOptions) -> Result<SuiteReport> {
166    let platform = platform()?;
167    let tier = opts
168        .tier
169        .clone()
170        .unwrap_or_else(|| cfg.default_tier.clone());
171    let scenarios = select(cfg, opts)?;
172    if scenarios.is_empty() {
173        return err(format!(
174            "no scenarios matched tier '{tier}'{}",
175            opts.only
176                .as_ref()
177                .map(|o| format!(" --only {o}"))
178                .unwrap_or_default()
179        ));
180    }
181    std::fs::create_dir_all(&cfg.evidence_root)?;
182    let lock_opts = LockOptions {
183        timeout: Duration::from_secs(cfg.lock.timeout_s.unwrap_or(120)),
184        stale: Duration::from_secs(cfg.lock.stale_s.unwrap_or(60)),
185        ..Default::default()
186    };
187    let mut ownership = lock::acquire(
188        &cfg.evidence_root.join("native.lock"),
189        &format!("{}:{tier}", cfg.app),
190        &lock_opts,
191    )?;
192    let run_id = crate::util::new_id();
193    // Earlier crashed runs: kill survivors by recorded pid + command line, then drop their dirs.
194    let swept = sweep_registries(&cfg.evidence_root.join("runs"), &run_id);
195    workspace::prune_abandoned(
196        &cfg.evidence_root,
197        Duration::from_secs(6 * 3600),
198        std::slice::from_ref(&run_id),
199    );
200    workspace::prune_abandoned_homes(
201        &cfg.app,
202        Duration::from_secs(6 * 3600),
203        std::slice::from_ref(&run_id),
204    );
205    let ws = workspace::create(&cfg.evidence_root, Some(&run_id), &cfg.app)?;
206    let tracker = Tracker::with_registry(ws.root.join("pids.json"));
207    let mut bundle = Bundle::new(&cfg.app, platform, &run_id)?;
208    let _mute = cfg.audio.system_mute.then(SystemMute::engage);
209
210    let needs_engine = scenarios
211        .iter()
212        .any(|s| s.target == "engine" && unmet_requirement(&s.requires, &cfg.app_root).is_none());
213    let needs_ui = scenarios.iter().any(|s| s.target == "ui");
214    let engine = engine_target(cfg, opts, &tracker, &tier)?;
215    let ui = if needs_ui {
216        ui_binary(cfg, opts, &tracker)?
217    } else {
218        None
219    };
220    if needs_engine && engine.is_none() {
221        ownership.release();
222        return err("selected scenarios need the engine but no binary is configured (use --engine, RIGHTKIT_QA_ENGINE, [engine].binary, or --build)");
223    }
224
225    let identity = match run_identity(
226        cfg,
227        &scenarios,
228        engine.as_ref().map(|e| e.binary.as_path()),
229        ui.as_deref(),
230        needs_engine,
231        false,
232    ) {
233        Ok(i) => i,
234        Err(e) => {
235            ownership.release();
236            return Err(e);
237        }
238    };
239    bundle.set_identity(identity)?;
240    println!(
241        "[qa] app={} platform={platform} tier={tier} scenarios={} run={run_id}",
242        cfg.app,
243        scenarios.len()
244    );
245    if let Some(e) = &engine {
246        println!("[qa] engine={}", e.binary.display());
247    }
248    if !swept.is_empty() {
249        println!("[qa] swept orphan pids from earlier runs: {swept:?}");
250    }
251
252    let mut outcomes = vec![];
253    for sc in &scenarios {
254        let started = Instant::now();
255        let name = sc.name.clone();
256        if let Some(reason) = unmet_requirement(&sc.requires, &cfg.app_root) {
257            if sc.optional {
258                println!("[qa] SKIP  {name}: {reason} (optional)");
259                bundle.add_check(
260                    &name,
261                    CheckStatus::Skipped,
262                    json!({"reason": reason, "optional": true}),
263                    &[],
264                )?;
265                outcomes.push(Outcome {
266                    name,
267                    file: sc.file.clone(),
268                    tier: sc.tier.clone(),
269                    status: CheckStatus::Skipped,
270                    reason: Some(reason),
271                    ms: 0,
272                    steps: vec![],
273                });
274            } else {
275                let reason = format!("mandatory scenario could not run: {reason} (mark it `optional = true` only if skipping is acceptable)");
276                println!("[qa] FAIL  {name}: {reason}");
277                bundle.add_check(
278                    &name,
279                    CheckStatus::Failed,
280                    json!({"error": reason, "mandatory": true}),
281                    &[],
282                )?;
283                outcomes.push(Outcome {
284                    name,
285                    file: sc.file.clone(),
286                    tier: sc.tier.clone(),
287                    status: CheckStatus::Failed,
288                    reason: Some(reason),
289                    ms: 0,
290                    steps: vec![],
291                });
292            }
293            continue;
294        }
295        if sc.target == "ui" && ui.is_none() {
296            let reason =
297                "no UI binary for this platform ([ui].binaries / --ui-binary / --build-ui)"
298                    .to_string();
299            println!("[qa] FAIL  {name}: {reason}");
300            bundle.add_check(&name, CheckStatus::Failed, json!({"error": reason}), &[])?;
301            outcomes.push(Outcome {
302                name,
303                file: sc.file.clone(),
304                tier: sc.tier.clone(),
305                status: CheckStatus::Failed,
306                reason: Some(reason),
307                ms: 0,
308                steps: vec![],
309            });
310            continue;
311        }
312        let mut ctx = Ctx::new(cfg, engine.clone(), ui.clone(), &ws, &tracker, &name)?;
313        let mut steps = vec![];
314        let mut failure: Option<String> = None;
315        let limit = Duration::from_secs(sc.timeout_s.unwrap_or(600));
316        for (i, step) in sc.steps.iter().enumerate() {
317            let t = Instant::now();
318            if started.elapsed() > limit {
319                failure = Some(format!(
320                    "scenario exceeded its {}s budget before step {}",
321                    limit.as_secs(),
322                    i + 1
323                ));
324                break;
325            }
326            let label = step.label.clone().unwrap_or_else(|| step.kind.clone());
327            match run_step(&mut ctx, step) {
328                Ok(_) => steps.push(StepRecord {
329                    index: i + 1,
330                    kind: step.kind.clone(),
331                    name: label,
332                    ok: true,
333                    ms: t.elapsed().as_millis(),
334                    error: None,
335                }),
336                Err(e) => {
337                    steps.push(StepRecord {
338                        index: i + 1,
339                        kind: step.kind.clone(),
340                        name: label,
341                        ok: false,
342                        ms: t.elapsed().as_millis(),
343                        error: Some(e.0.clone()),
344                    });
345                    failure = Some(e.0);
346                    break;
347                }
348            }
349        }
350        if failure.is_some() {
351            if let Some((c, _)) = &ctx.ui {
352                let p = ctx.log_dir.join("failure.png");
353                if c.screenshot_to(&p).is_ok() {
354                    ctx.artifacts.push((format!("{name}/failure.png"), p));
355                }
356            }
357        }
358        let problems = ctx.teardown();
359        if failure.is_none() && !problems.is_empty() {
360            failure = Some(problems.join("; "));
361        }
362        // Hash every artifact the scenario produced, then bind the check to them.
363        let mut names = vec![];
364        let mut artifact_errors = vec![];
365        for (n, path) in &ctx.artifacts {
366            if names.contains(n) {
367                continue;
368            }
369            match bundle.add_artifact(n, path) {
370                Ok(_) => names.push(n.clone()),
371                Err(e) => artifact_errors.push(e.0),
372            }
373        }
374        if !artifact_errors.is_empty() {
375            let msg = format!(
376                "required evidence could not be recorded: {}",
377                artifact_errors.join("; ")
378            );
379            failure = Some(match failure.take() {
380                Some(f) => format!("{f}; {msg}"),
381                None => msg,
382            });
383        }
384        let status = if failure.is_none() {
385            CheckStatus::Passed
386        } else {
387            CheckStatus::Failed
388        };
389        let details = json!({"file": sc.file.to_string_lossy(), "tier": sc.tier, "steps": steps.len(), "error": failure});
390        bundle.add_check(&name, status, details, &names)?;
391        println!(
392            "[qa] {}  {name} ({} ms){}",
393            if failure.is_none() { "PASS" } else { "FAIL" },
394            started.elapsed().as_millis(),
395            failure
396                .as_ref()
397                .map(|f| format!("\n       {f}"))
398                .unwrap_or_default()
399        );
400        outcomes.push(Outcome {
401            name,
402            file: sc.file.clone(),
403            tier: sc.tier.clone(),
404            status,
405            reason: failure,
406            ms: started.elapsed().as_millis(),
407            steps,
408        });
409    }
410
411    tracker.kill_all();
412    let failed = outcomes
413        .iter()
414        .filter(|o| o.status == CheckStatus::Failed)
415        .count();
416    let passed = outcomes
417        .iter()
418        .filter(|o| o.status == CheckStatus::Passed)
419        .count();
420    // Skips are only ever optional scenarios; a run whose scenarios were all skipped proves nothing.
421    bundle.finish(if failed == 0 && passed > 0 {
422        RunStatus::Passed
423    } else {
424        RunStatus::Failed
425    });
426    let evidence_path = ws.evidence_dir.join("evidence.json");
427    let results_path = ws.evidence_dir.join("results.json");
428    std::fs::write(
429        &results_path,
430        serde_json::to_vec_pretty(&json!({
431            "app": cfg.app, "platform": platform, "runId": run_id, "tier": tier, "finishedAt": now_iso(),
432            "passed": passed, "failed": failed, "skipped": outcomes.len() - passed - failed,
433            "scenarios": outcomes.iter().map(Outcome::to_value).collect::<Vec<_>>(),
434        }))?,
435    )?;
436    bundle.write(&evidence_path)?;
437    let _ = std::fs::write(
438        cfg.evidence_root.join("latest.json"),
439        serde_json::to_vec_pretty(
440            &json!({"runId": run_id, "evidence": evidence_path.to_string_lossy(), "results": results_path.to_string_lossy()}),
441        )?,
442    );
443    if !opts.keep_data {
444        let report = workspace::dispose(&ws, true, 5, 200);
445        for (p, why) in report.retained {
446            println!("[qa] WARN could not remove {}: {why}", p.display());
447        }
448    }
449    println!(
450        "[qa] {passed} passed, {failed} failed, {} skipped; evidence {}",
451        outcomes.len() - passed - failed,
452        evidence_path.display()
453    );
454    ownership.release();
455    Ok(SuiteReport {
456        exit_code: if failed == 0 && passed > 0 { 0 } else { 1 },
457        run_id,
458        run_dir: ws.root.clone(),
459        evidence_path,
460        results_path,
461        outcomes,
462    })
463}
464
465fn file_sha(path: &std::path::Path) -> Result<String> {
466    let bytes = std::fs::read(path).map_err(|e| {
467        crate::util::Error(format!(
468            "cannot hash {} for the evidence identity: {e}",
469            path.display()
470        ))
471    })?;
472    Ok(crate::util::sha256_hex(&bytes))
473}
474
475fn git_revision(root: &std::path::Path) -> Value {
476    let git = |args: &[&str]| {
477        let mut git = std::process::Command::new("git");
478        git.arg("-C").arg(root).args(args);
479        #[cfg(windows)]
480        {
481            use std::os::windows::process::CommandExt;
482            git.creation_flags(0x0800_0000); // CREATE_NO_WINDOW: never flash a console from a GUI host
483        }
484        git.output()
485            .ok()
486            .filter(|o| o.status.success())
487            .map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string())
488    };
489    match git(&["rev-parse", "HEAD"]) {
490        Some(head) => {
491            json!({"head": head, "dirty": git(&["status", "--porcelain"]).map(|s| !s.is_empty())})
492        }
493        None => Value::Null,
494    }
495}
496
497/// What this run exercises: config, scenario files, engine/UI binaries and the source revision,
498/// all by digest, so a receipt cannot be reused for different bytes.
499fn run_identity(
500    cfg: &QaConfig,
501    scenarios: &[Scenario],
502    engine: Option<&std::path::Path>,
503    ui: Option<&std::path::Path>,
504    needs_engine: bool,
505    needs_ui: bool,
506) -> Result<Value> {
507    let mut scenario_ids = vec![];
508    for sc in scenarios {
509        scenario_ids.push(json!({"name": sc.name, "file": sc.file.to_string_lossy(), "sha256": file_sha(&sc.file)?}));
510    }
511    let binary = |p: Option<&std::path::Path>, what: &str, needed: bool| -> Result<Value> {
512        match p {
513            Some(p) => {
514                let target = if p.is_dir() { None } else { Some(p) };
515                match target {
516                    Some(t) => Ok(json!({"path": t.to_string_lossy(), "sha256": file_sha(t)?})),
517                    None => {
518                        // A `.app` bundle: bind to every executable inside Contents/MacOS.
519                        let mut exes = vec![];
520                        if let Ok(rd) = std::fs::read_dir(p.join("Contents").join("MacOS")) {
521                            for e in rd.flatten() {
522                                exes.push(json!({"path": e.path().to_string_lossy(), "sha256": file_sha(&e.path())?}));
523                            }
524                        }
525                        if exes.is_empty() {
526                            return err(format!(
527                                "{} has no executable to bind into the evidence identity",
528                                p.display()
529                            ));
530                        }
531                        Ok(json!({"path": p.to_string_lossy(), "executables": exes}))
532                    }
533                }
534            }
535            None if needed => err(format!(
536                "evidence identity needs the {what} binary but none resolved"
537            )),
538            None => Ok(Value::Null),
539        }
540    };
541    Ok(json!({
542        "config": {"path": cfg.config_path.to_string_lossy(), "sha256": file_sha(&cfg.config_path)?},
543        "scenarios": scenario_ids,
544        "engine": binary(engine, "engine", needs_engine)?,
545        "ui": binary(ui, "UI", needs_ui)?,
546        "source": git_revision(&cfg.app_root),
547    }))
548}
rightkit_qa/suite.rs

rightkit_qa/
suite.rs
