use std::collections::{BTreeMap, BTreeSet};
use std::fmt;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::{Duration, Instant};
use async_trait::async_trait;
use serde::Deserialize;
use serde_json::{json, Value};
use super::expect::check_expectation;
use crate::core::{Error, Result};
#[derive(Debug, Clone, Deserialize)]
#[non_exhaustive]
pub struct Scenario {
pub id: String,
pub category: String,
#[serde(default)]
pub description: String,
#[serde(default)]
pub requires: Vec<String>,
#[serde(default, rename = "batchOptions")]
pub batch_options: Option<Value>,
pub steps: Vec<Step>,
}
#[derive(Debug, Clone, Deserialize)]
#[non_exhaustive]
pub struct Step {
pub op: String,
#[serde(default)]
pub args: Value,
#[serde(default)]
pub expect: Option<Value>,
#[serde(default)]
pub save: Option<String>,
#[serde(default)]
pub requires: Vec<String>,
}
pub fn load_scenarios(dir: impl AsRef<Path>) -> Result<Vec<Scenario>> {
let dir = dir.as_ref();
let entries = std::fs::read_dir(dir).map_err(|e| {
Error::config(format!(
"conformance directory {} cannot be read",
dir.display()
))
.with_source(e)
})?;
let mut scenarios = Vec::new();
for entry in entries.flatten() {
let path = entry.path();
if path.extension().and_then(|e| e.to_str()) != Some("json") {
continue;
}
let text = std::fs::read_to_string(&path).map_err(|e| {
Error::config(format!("scenario {} cannot be read", path.display())).with_source(e)
})?;
let scenario: Scenario = serde_json::from_str(&text)
.map_err(|e| Error::config(format!("scenario {} is invalid: {e}", path.display())))?;
scenarios.push(scenario);
}
scenarios.sort_by(|a, b| a.id.cmp(&b.id));
Ok(scenarios)
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct Vars(BTreeMap<String, String>);
impl Vars {
pub fn new() -> Self {
Vars::default()
}
pub fn get(&self, name: &str) -> Option<&str> {
self.0.get(name).map(String::as_str)
}
pub fn set(&mut self, name: impl Into<String>, value: impl Into<String>) {
self.0.insert(name.into(), value.into());
}
pub fn substitute(&self, text: &str) -> String {
let mut out = String::with_capacity(text.len());
let mut rest = text;
while let Some(start) = rest.find("${") {
out.push_str(&rest[..start]);
let after = &rest[start + 2..];
match after.find('}') {
Some(end) => {
let name = &after[..end];
match self.0.get(name) {
Some(v) => out.push_str(v),
None => out.push_str(&rest[start..start + 3 + end]),
}
rest = &after[end + 1..];
}
None => {
out.push_str(&rest[start..]);
rest = "";
}
}
}
out.push_str(rest);
out
}
pub fn substitute_value(&self, value: &Value) -> Value {
match value {
Value::String(s) => Value::String(self.substitute(s)),
Value::Array(items) => {
Value::Array(items.iter().map(|v| self.substitute_value(v)).collect())
}
Value::Object(map) => Value::Object(
map.iter()
.map(|(k, v)| (self.substitute(k), self.substitute_value(v)))
.collect(),
),
other => other.clone(),
}
}
}
#[derive(Debug, Clone, PartialEq)]
pub struct StepOutput {
pub value: Value,
pub display: String,
pub reveal: Option<String>,
pub count: Option<usize>,
}
impl StepOutput {
pub fn none() -> Self {
StepOutput::json(Value::Null)
}
pub fn json(value: Value) -> Self {
let display = value.to_string();
StepOutput {
value,
display,
reveal: None,
count: None,
}
}
pub fn with_display(mut self, display: impl Into<String>) -> Self {
self.display = display.into();
self
}
pub fn with_reveal(mut self, reveal: impl Into<String>) -> Self {
self.reveal = Some(reveal.into());
self
}
pub fn with_count(mut self, count: usize) -> Self {
self.count = Some(count);
self
}
}
#[async_trait]
pub trait CategoryHandler: Send + Sync {
fn category(&self) -> &str;
async fn open(&self, scenario: &Scenario, vars: &Vars) -> Result<Box<dyn ScenarioSession>>;
}
#[async_trait]
pub trait ScenarioSession: Send {
async fn run(&mut self, op: &str, args: &Value, vars: &Vars) -> Result<StepOutput>;
async fn close(&mut self) {}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ScenarioStatus {
Passed,
Failed(String),
Skipped(String),
}
#[derive(Debug, Clone)]
pub struct ScenarioResult {
pub id: String,
pub category: String,
pub status: ScenarioStatus,
pub skipped_steps: Vec<String>,
pub duration: Duration,
}
#[derive(Debug, Clone, Default)]
pub struct ConformanceReport {
pub results: Vec<ScenarioResult>,
}
impl ConformanceReport {
fn count(&self, f: impl Fn(&ScenarioStatus) -> bool) -> usize {
self.results.iter().filter(|r| f(&r.status)).count()
}
pub fn passed(&self) -> usize {
self.count(|s| matches!(s, ScenarioStatus::Passed))
}
pub fn failed(&self) -> usize {
self.count(|s| matches!(s, ScenarioStatus::Failed(_)))
}
pub fn skipped(&self) -> usize {
self.count(|s| matches!(s, ScenarioStatus::Skipped(_)))
}
pub fn is_success(&self) -> bool {
self.failed() == 0
}
pub fn result(&self, id: &str) -> Option<&ScenarioResult> {
self.results.iter().find(|r| r.id == id)
}
pub fn to_json(&self) -> Value {
let results: Vec<Value> = self
.results
.iter()
.map(|r| {
let (status, reason) = match &r.status {
ScenarioStatus::Passed => ("passed", None),
ScenarioStatus::Failed(m) => ("failed", Some(m.clone())),
ScenarioStatus::Skipped(m) => ("skipped", Some(m.clone())),
};
json!({
"id": r.id, "category": r.category, "status": status, "reason": reason,
"skippedSteps": r.skipped_steps, "durationMs": r.duration.as_millis() as u64,
})
})
.collect();
json!({
"passed": self.passed(), "failed": self.failed(), "skipped": self.skipped(),
"results": results,
})
}
pub fn assert_success(&self) {
assert!(self.is_success(), "conformance failed:\n{self}");
}
}
impl fmt::Display for ConformanceReport {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
writeln!(
f,
"conformance: {} passed, {} failed, {} skipped",
self.passed(),
self.failed(),
self.skipped()
)?;
for r in &self.results {
match &r.status {
ScenarioStatus::Passed => writeln!(f, " PASS {}", r.id)?,
ScenarioStatus::Failed(m) => writeln!(f, " FAIL {}: {m}", r.id)?,
ScenarioStatus::Skipped(m) => writeln!(f, " SKIP {}: {m}", r.id)?,
}
for s in &r.skipped_steps {
writeln!(f, " skipped step {s}")?;
}
}
Ok(())
}
}
pub const DEFAULT_PREFIX: &str = "grow-conformance/${run}/";
#[derive(Clone)]
pub struct ConformanceRunner {
dir: PathBuf,
capabilities: BTreeSet<String>,
prefix: String,
vars: Vars,
handlers: Vec<Arc<dyn CategoryHandler>>,
only: Option<BTreeSet<String>>,
}
impl fmt::Debug for ConformanceRunner {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ConformanceRunner")
.field("dir", &self.dir)
.field("capabilities", &self.capabilities)
.field(
"handlers",
&self
.handlers
.iter()
.map(|h| h.category().to_owned())
.collect::<Vec<_>>(),
)
.finish_non_exhaustive()
}
}
impl ConformanceRunner {
pub fn new(dir: impl Into<PathBuf>) -> Self {
ConformanceRunner {
dir: dir.into(),
capabilities: BTreeSet::new(),
prefix: DEFAULT_PREFIX.to_owned(),
vars: Vars::new(),
handlers: Vec::new(),
only: None,
}
}
pub fn capabilities<I, S>(mut self, caps: I) -> Self
where
I: IntoIterator<Item = S>,
S: Into<String>,
{
self.capabilities.extend(caps.into_iter().map(Into::into));
self
}
pub fn prefix(mut self, prefix: impl Into<String>) -> Self {
self.prefix = prefix.into();
self
}
pub fn var(mut self, name: impl Into<String>, value: impl Into<String>) -> Self {
self.vars.set(name, value);
self
}
pub fn handler(mut self, handler: impl CategoryHandler + 'static) -> Self {
self.handlers.push(Arc::new(handler));
self
}
pub fn only<I, S>(mut self, ids: I) -> Self
where
I: IntoIterator<Item = S>,
S: Into<String>,
{
self.only = Some(ids.into_iter().map(Into::into).collect());
self
}
pub async fn run(&self) -> Result<ConformanceReport> {
let scenarios = load_scenarios(&self.dir)?;
Ok(self.run_scenarios(&scenarios).await)
}
pub async fn run_scenarios(&self, scenarios: &[Scenario]) -> ConformanceReport {
let mut report = ConformanceReport::default();
for scenario in scenarios {
if self
.only
.as_ref()
.is_some_and(|ids| !ids.contains(&scenario.id))
{
continue;
}
let Some(handler) = self
.handlers
.iter()
.find(|h| h.category() == scenario.category)
else {
continue;
};
report
.results
.push(self.run_one(handler.as_ref(), scenario).await);
}
report
}
fn missing(&self, requires: &[String]) -> Vec<String> {
requires
.iter()
.filter(|r| !self.capabilities.contains(*r))
.cloned()
.collect()
}
async fn run_one(&self, handler: &dyn CategoryHandler, scenario: &Scenario) -> ScenarioResult {
let started = Instant::now();
let mut result = ScenarioResult {
id: scenario.id.clone(),
category: scenario.category.clone(),
status: ScenarioStatus::Passed,
skipped_steps: Vec::new(),
duration: Duration::ZERO,
};
let missing = self.missing(&scenario.requires);
if !missing.is_empty() {
result.status =
ScenarioStatus::Skipped(format!("missing capabilities: {}", missing.join(", ")));
return result;
}
let mut vars = self.vars.clone();
vars.set("run", run_id());
let prefix = vars.substitute(&self.prefix);
vars.set("prefix", prefix);
let mut session = match handler.open(scenario, &vars).await {
Ok(s) => s,
Err(e) => {
result.status =
ScenarioStatus::Failed(format!("setup failed: {} ({})", e, e.code()));
result.duration = started.elapsed();
return result;
}
};
for (i, step) in scenario.steps.iter().enumerate() {
let missing = self.missing(&step.requires);
if !missing.is_empty() {
result.skipped_steps.push(format!(
"{i}:{} (missing: {})",
step.op,
missing.join(", ")
));
continue;
}
let args = vars.substitute_value(&step.args);
let outcome = session.run(&step.op, &args, &vars).await;
let checked = match &step.expect {
Some(expect) => check_expectation(&vars.substitute_value(expect), &outcome),
None => match &outcome {
Ok(_) => Ok(()),
Err(e) => Err(format!("unexpected error {}: {}", e.code(), e)),
},
};
if let Err(msg) = checked {
result.status = ScenarioStatus::Failed(format!("step {i} ({}): {msg}", step.op));
break;
}
if let (Some(name), Ok(out)) = (&step.save, &outcome) {
let saved = match &out.value {
Value::String(s) => s.clone(),
other => other.to_string(),
};
vars.set(name.clone(), saved);
}
}
session.close().await;
result.duration = started.elapsed();
result
}
}
pub fn run_id() -> String {
const ALPHABET: &[u8] = b"abcdefghijklmnopqrstuvwxyz0123456789";
(0..8)
.map(|_| ALPHABET[rand::random_range(0..ALPHABET.len())] as char)
.collect()
}