pub mod verifier;
pub mod scenario;
pub mod multi;
use std::collections::BTreeMap;
use std::fs;
use std::io;
use std::os::unix::fs as unix_fs;
use std::path::{Path, PathBuf};
use chrono::{DateTime, SecondsFormat, Utc};
use serde_json::{Value, json};
pub use verifier::{
C1_COVER, C2_DEFERRED, PERMANENT_SKIP, VerifyError, VerifyReport, grep_pub_fn, verify_cover,
verify_cover_or_panic,
};
#[derive(Clone, Debug, PartialEq, Eq, Default)]
pub struct Narration {
pub screen: String,
pub target: String,
pub why: String,
pub action: String,
}
impl Narration {
pub fn new(
screen: impl Into<String>,
target: impl Into<String>,
why: impl Into<String>,
action: impl Into<String>,
) -> Self {
Self {
screen: screen.into(),
target: target.into(),
why: why.into(),
action: action.into(),
}
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum CapabilityStatus {
Passed {
narration: Narration,
evidence: String,
},
Failed {
narration: Narration,
failure_kind: String,
message: String,
evidence_path: Option<String>,
},
Skipped {
narration: Narration,
reason: String,
},
}
impl CapabilityStatus {
fn status_str(&self) -> &'static str {
match self {
CapabilityStatus::Passed { .. } => "passed",
CapabilityStatus::Failed { .. } => "failed",
CapabilityStatus::Skipped { .. } => "skipped",
}
}
fn narration(&self) -> &Narration {
match self {
CapabilityStatus::Passed { narration, .. }
| CapabilityStatus::Failed { narration, .. }
| CapabilityStatus::Skipped { narration, .. } => narration,
}
}
}
pub struct Harness {
run_dir: PathBuf,
capability_status: BTreeMap<String, CapabilityStatus>,
selector_spec_status: BTreeMap<String, CapabilityStatus>,
maestro_parity_status: BTreeMap<String, CapabilityStatus>,
v2_e2e_status: BTreeMap<String, CapabilityStatus>,
smix_commit: String,
insight_commit: Option<String>,
udid: String,
started_at: DateTime<Utc>,
capsule_reconcile: Option<CapsuleReconcileSummary>,
}
#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
pub struct CapsuleReconcileSummary {
pub issued_count: usize,
pub focus_change_count: usize,
pub attributed_count: usize,
pub unattributed_count: usize,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub unattributed_events: Vec<smix_runner_client::RecordedEvent>,
}
impl CapsuleReconcileSummary {
pub fn from_reconciliation(r: &crate::CapsuleReconciliation) -> Self {
Self {
issued_count: r.issued_count,
focus_change_count: r.focus_change_count,
attributed_count: r.attributed_count,
unattributed_count: r.unattributed_count,
unattributed_events: r.unattributed_events.clone(),
}
}
}
impl Harness {
pub fn new(parent_dir: &Path, udid: String) -> io::Result<Self> {
let started_at = Utc::now();
let dir_name = started_at
.to_rfc3339_opts(SecondsFormat::Secs, true)
.replace(':', "-");
let run_dir = parent_dir.join(dir_name);
Self::new_with_run_dir(run_dir, udid, started_at)
}
pub fn new_with_run_dir(
run_dir: PathBuf,
udid: String,
started_at: DateTime<Utc>,
) -> io::Result<Self> {
fs::create_dir_all(&run_dir)?;
fs::create_dir_all(run_dir.join("screenshots"))?;
let smix_commit = git_head_sha(&run_dir).unwrap_or_default();
let insight_commit = git_head_sha(Path::new(&insight_repo_path()));
Ok(Self {
run_dir,
capability_status: BTreeMap::new(),
selector_spec_status: BTreeMap::new(),
maestro_parity_status: BTreeMap::new(),
v2_e2e_status: BTreeMap::new(),
smix_commit,
insight_commit,
udid,
started_at,
capsule_reconcile: None,
})
}
pub fn set_capsule_reconcile(&mut self, summary: CapsuleReconcileSummary) {
self.capsule_reconcile = Some(summary);
}
pub fn run_dir(&self) -> &Path {
&self.run_dir
}
pub fn record_pass(&mut self, name: &str, narration: Narration, evidence: &str) {
print_narration_block(name, "PASS", &narration, evidence);
self.capability_status.insert(
name.to_string(),
CapabilityStatus::Passed {
narration,
evidence: evidence.to_string(),
},
);
}
pub fn record_fail(
&mut self,
name: &str,
narration: Narration,
failure_kind: &str,
message: &str,
evidence_path: Option<String>,
) {
print_narration_block(
name,
"FAIL",
&narration,
&format!("{failure_kind} — {message}"),
);
self.capability_status.insert(
name.to_string(),
CapabilityStatus::Failed {
narration,
failure_kind: failure_kind.to_string(),
message: message.to_string(),
evidence_path,
},
);
}
pub fn record_skip(&mut self, name: &str, narration: Narration, reason: &str) {
print_narration_block(name, "SKIP", &narration, reason);
self.capability_status.insert(
name.to_string(),
CapabilityStatus::Skipped {
narration,
reason: reason.to_string(),
},
);
}
pub fn record_selector_spec_pass(&mut self, name: &str, narration: Narration, evidence: &str) {
print_narration_block(name, "PASS", &narration, evidence);
self.selector_spec_status.insert(
name.to_string(),
CapabilityStatus::Passed {
narration,
evidence: evidence.to_string(),
},
);
}
pub fn record_selector_spec_fail(
&mut self,
name: &str,
narration: Narration,
failure_kind: &str,
message: &str,
evidence_path: Option<String>,
) {
print_narration_block(
name,
"FAIL",
&narration,
&format!("{failure_kind} — {message}"),
);
self.selector_spec_status.insert(
name.to_string(),
CapabilityStatus::Failed {
narration,
failure_kind: failure_kind.to_string(),
message: message.to_string(),
evidence_path,
},
);
}
pub fn record_maestro_parity_pass(&mut self, name: &str, narration: Narration, evidence: &str) {
print_narration_block(name, "PASS", &narration, evidence);
self.maestro_parity_status.insert(
name.to_string(),
CapabilityStatus::Passed {
narration,
evidence: evidence.to_string(),
},
);
}
pub fn record_maestro_parity_fail(
&mut self,
name: &str,
narration: Narration,
failure_kind: &str,
message: &str,
evidence_path: Option<String>,
) {
print_narration_block(
name,
"FAIL",
&narration,
&format!("{failure_kind} — {message}"),
);
self.maestro_parity_status.insert(
name.to_string(),
CapabilityStatus::Failed {
narration,
failure_kind: failure_kind.to_string(),
message: message.to_string(),
evidence_path,
},
);
}
pub fn record_maestro_parity_skip(&mut self, name: &str, narration: Narration, reason: &str) {
print_narration_block(name, "SKIP", &narration, reason);
self.maestro_parity_status.insert(
name.to_string(),
CapabilityStatus::Skipped {
narration,
reason: reason.to_string(),
},
);
}
pub fn maestro_parity_status(&self) -> &BTreeMap<String, CapabilityStatus> {
&self.maestro_parity_status
}
pub fn record_v2_e2e_pass(&mut self, name: &str, narration: Narration, evidence: &str) {
print_narration_block(name, "PASS", &narration, evidence);
self.v2_e2e_status.insert(
name.to_string(),
CapabilityStatus::Passed {
narration,
evidence: evidence.to_string(),
},
);
}
pub fn record_v2_e2e_fail(
&mut self,
name: &str,
narration: Narration,
failure_kind: &str,
message: &str,
evidence_path: Option<String>,
) {
print_narration_block(
name,
"FAIL",
&narration,
&format!("{failure_kind} — {message}"),
);
self.v2_e2e_status.insert(
name.to_string(),
CapabilityStatus::Failed {
narration,
failure_kind: failure_kind.to_string(),
message: message.to_string(),
evidence_path,
},
);
}
pub fn record_v2_e2e_skip(&mut self, name: &str, narration: Narration, reason: &str) {
print_narration_block(name, "SKIP", &narration, reason);
self.v2_e2e_status.insert(
name.to_string(),
CapabilityStatus::Skipped {
narration,
reason: reason.to_string(),
},
);
}
pub fn v2_e2e_status(&self) -> &BTreeMap<String, CapabilityStatus> {
&self.v2_e2e_status
}
pub fn record_selector_spec_skip(&mut self, name: &str, narration: Narration, reason: &str) {
print_narration_block(name, "SKIP", &narration, reason);
self.selector_spec_status.insert(
name.to_string(),
CapabilityStatus::Skipped {
narration,
reason: reason.to_string(),
},
);
}
pub fn write_result_json(&self, verify_report: &VerifyReport) -> io::Result<PathBuf> {
let path = self.run_dir.join("result.json");
let capabilities: Vec<Value> = self
.capability_status
.iter()
.map(|(name, status)| capability_to_json(name, status))
.collect();
let covered_count = self
.capability_status
.values()
.filter(|s| matches!(s, CapabilityStatus::Passed { .. }))
.count();
let selector_spec_segments: Vec<Value> = self
.selector_spec_status
.iter()
.map(|(name, status)| capability_to_json(name, status))
.collect();
let maestro_parity: Vec<Value> = self
.maestro_parity_status
.iter()
.map(|(name, status)| capability_to_json(name, status))
.collect();
let maestro_parity_cover_count = self
.maestro_parity_status
.values()
.filter(|s| matches!(s, CapabilityStatus::Passed { .. }))
.count();
let v2_e2e: Vec<Value> = self
.v2_e2e_status
.iter()
.map(|(name, status)| capability_to_json(name, status))
.collect();
let v2_e2e_cover_count = self
.v2_e2e_status
.values()
.filter(|s| matches!(s, CapabilityStatus::Passed { .. }))
.count();
let json = json!({
"timestamp": self.started_at.to_rfc3339_opts(SecondsFormat::Secs, true),
"smix_commit": self.smix_commit,
"insight_commit": self.insight_commit,
"udid": self.udid,
"capabilities": capabilities,
"selector_spec_segments": selector_spec_segments,
"maestro_parity": maestro_parity,
"maestro_parity_cover_count": maestro_parity_cover_count,
"v2_e2e": v2_e2e,
"v2_e2e_cover_count": v2_e2e_cover_count,
"capsule_reconcile": self.capsule_reconcile,
"coverage_pct": verify_report.coverage_pct,
"sdk_pub_fn_count": verify_report.sdk_pub_fn_count,
"c1_cover_count": verify_report.c1_cover_count,
"c2_deferred_count": verify_report.c2_deferred_count,
"permanent_skip_count": verify_report.permanent_skip_count,
"covered_count": covered_count,
});
fs::write(&path, serde_json::to_string_pretty(&json).unwrap())?;
Ok(path)
}
pub fn write_report_md(&self, verify_report: &VerifyReport) -> io::Result<PathBuf> {
let path = self.run_dir.join("report.md");
let mut md = String::new();
md.push_str("# selftest report\n\n");
md.push_str(&format!(
"- timestamp: `{}`\n",
self.started_at.to_rfc3339_opts(SecondsFormat::Secs, true)
));
md.push_str(&format!("- udid: `{}`\n", self.udid));
md.push_str(&format!("- smix_commit: `{}`\n", self.smix_commit));
if let Some(ic) = &self.insight_commit {
md.push_str(&format!("- insight_commit: `{ic}`\n"));
}
md.push_str(&format!(
"- coverage: **{:.1}%** ({}/{} sdk pub fn covered by C1 selftest, C2 deferred {}, permanent skip {})\n\n",
verify_report.coverage_pct,
verify_report.c1_cover_count,
verify_report.sdk_pub_fn_count,
verify_report.c2_deferred_count,
verify_report.permanent_skip_count,
));
md.push_str("## capability matrix\n\n");
render_status_matrix(&mut md, &self.capability_status);
if !self.selector_spec_status.is_empty() {
md.push_str("## selector / failure spec drill\n\n");
render_status_matrix(&mut md, &self.selector_spec_status);
}
fs::write(&path, md)?;
Ok(path)
}
pub fn update_latest_symlink(&self) -> io::Result<()> {
let parent = self.run_dir.parent().expect("run_dir has a parent");
let latest = parent.join("latest");
let tmp = parent.join("latest.tmp");
let _ = fs::remove_file(&tmp);
let target = self.run_dir.file_name().expect("run_dir has a name");
unix_fs::symlink(target, &tmp)?;
fs::rename(tmp, latest)?;
Ok(())
}
pub fn finish(&self) -> i32 {
let (cap_passed, cap_failed, cap_skipped) = tally(&self.capability_status);
let (spec_passed, spec_failed, spec_skipped) = tally(&self.selector_spec_status);
eprintln!(
"selftest narration roll-up: capability {}P/{}F/{}S (total {}) + selector_spec {}P/{}F/{}S (total {})",
cap_passed,
cap_failed,
cap_skipped,
self.capability_status.len(),
spec_passed,
spec_failed,
spec_skipped,
self.selector_spec_status.len(),
);
if cap_failed + cap_skipped + spec_failed + spec_skipped > 0 {
2
} else {
0
}
}
pub fn capability_status(&self) -> &BTreeMap<String, CapabilityStatus> {
&self.capability_status
}
pub fn selector_spec_status(&self) -> &BTreeMap<String, CapabilityStatus> {
&self.selector_spec_status
}
}
fn tally(map: &BTreeMap<String, CapabilityStatus>) -> (u32, u32, u32) {
let mut passed = 0u32;
let mut failed = 0u32;
let mut skipped = 0u32;
for s in map.values() {
match s {
CapabilityStatus::Passed { .. } => passed += 1,
CapabilityStatus::Failed { .. } => failed += 1,
CapabilityStatus::Skipped { .. } => skipped += 1,
}
}
(passed, failed, skipped)
}
fn render_status_matrix(md: &mut String, statuses: &BTreeMap<String, CapabilityStatus>) {
for (name, status) in statuses {
let status_label = match status {
CapabilityStatus::Passed { .. } => "PASS",
CapabilityStatus::Failed { .. } => "FAIL",
CapabilityStatus::Skipped { .. } => "SKIP",
};
let evidence_line = match status {
CapabilityStatus::Passed { evidence, .. } => evidence.clone(),
CapabilityStatus::Failed {
failure_kind,
message,
evidence_path,
..
} => {
let mut line = format!("{failure_kind} — {message}");
if let Some(p) = evidence_path {
line.push_str(&format!(" (evidence: `{p}`)"));
}
line
}
CapabilityStatus::Skipped { reason, .. } => reason.clone(),
};
let n = status.narration();
md.push_str(&format!("### `{name}` [{status_label}]\n"));
md.push_str(&format!("- **screen:** {}\n", n.screen));
md.push_str(&format!("- **target:** {}\n", n.target));
md.push_str(&format!("- **why:** {}\n", n.why));
md.push_str(&format!("- **action:** `{}`\n", n.action));
md.push_str(&format!("- **evidence:** {evidence_line}\n\n"));
}
}
fn print_narration_block(name: &str, status: &str, narration: &Narration, tail: &str) {
eprintln!("--- selftest segment: {name} [{status}] ---");
eprintln!(" screen: {}", narration.screen);
eprintln!(" target: {}", narration.target);
eprintln!(" why: {}", narration.why);
eprintln!(" action: {}", narration.action);
eprintln!(" {}: {tail}", status.to_lowercase());
}
fn capability_to_json(name: &str, status: &CapabilityStatus) -> Value {
let n = status.narration();
match status {
CapabilityStatus::Passed { evidence, .. } => json!({
"name": name,
"status": status.status_str(),
"screen": n.screen,
"target": n.target,
"why": n.why,
"action": n.action,
"evidence": evidence,
"failure_kind": Value::Null,
}),
CapabilityStatus::Failed {
failure_kind,
message,
evidence_path,
..
} => json!({
"name": name,
"status": status.status_str(),
"screen": n.screen,
"target": n.target,
"why": n.why,
"action": n.action,
"failure_kind": failure_kind,
"message": message,
"evidence_path": evidence_path,
}),
CapabilityStatus::Skipped { reason, .. } => json!({
"name": name,
"status": status.status_str(),
"screen": n.screen,
"target": n.target,
"why": n.why,
"action": n.action,
"reason": reason,
}),
}
}
fn git_head_sha(repo: &Path) -> Option<String> {
let out = std::process::Command::new("git")
.args(["rev-parse", "HEAD"])
.current_dir(repo)
.output()
.ok()?;
if !out.status.success() {
return None;
}
let sha = String::from_utf8(out.stdout).ok()?.trim().to_string();
if sha.is_empty() { None } else { Some(sha) }
}
fn insight_repo_path() -> String {
std::env::var("INSIGHT_REPO_PATH").unwrap_or_else(|_| {
let home = std::env::var("HOME").unwrap_or_default();
format!("{home}/workspace/qualcomm/insight")
})
}