use serde::{Deserialize, Serialize};
use crate::actions::Via;
use crate::next::NextStep;
use crate::verify::{HeldOut, RedClass, Verdict};
pub const DEFAULT_TRIES: u32 = 3;
pub const MAX_TRIES: u32 = 5;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
pub struct Statement {
pub id: String,
pub text: String,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
pub struct Triad {
pub current: Statement,
pub expected: Vec<Statement>,
pub unchanged: Vec<Statement>,
pub digest: String,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TriadError {
Empty,
NoExpected,
}
impl TriadError {
#[must_use]
pub const fn code(self) -> &'static str {
match self {
Self::Empty => "BUG-EMPTY",
Self::NoExpected => "BUG-NO-EXPECTED",
}
}
#[must_use]
pub const fn message(self) -> &'static str {
match self {
Self::Empty => {
"the symptom and every statement need words: `speccoder bug \"<what happens>\" --expected \"<what should>\"`"
}
Self::NoExpected => {
"say what should happen instead: `--expected \"…\"` (one per behaviour)"
}
}
}
}
#[must_use]
pub fn normalise(text: &str) -> String {
text.split_whitespace().collect::<Vec<_>>().join(" ")
}
impl Triad {
pub fn new(
current: &str,
expected: &[String],
unchanged: &[String],
hash: impl Fn(&[u8]) -> String,
) -> Result<Self, TriadError> {
let current = normalise(current);
if current.is_empty() {
return Err(TriadError::Empty);
}
let expected: Vec<String> = expected.iter().map(|e| normalise(e)).collect();
let unchanged: Vec<String> = unchanged.iter().map(|u| normalise(u)).collect();
if expected.iter().chain(&unchanged).any(String::is_empty) {
return Err(TriadError::Empty);
}
if expected.is_empty() {
return Err(TriadError::NoExpected);
}
let statements = |prefix: char, texts: Vec<String>| -> Vec<Statement> {
texts
.into_iter()
.enumerate()
.map(|(i, text)| Statement {
id: format!("{prefix}{}", i + 1),
text,
})
.collect()
};
let mut t = Self {
current: Statement {
id: "C".to_owned(),
text: current,
},
expected: statements('E', expected),
unchanged: statements('U', unchanged),
digest: String::new(),
};
t.digest = format!("sha256:{}", hash(t.canonical().as_bytes()));
Ok(t)
}
#[must_use]
pub fn canonical(&self) -> String {
self.statements()
.map(|s| format!("{}\t{}\n", s.id, s.text))
.collect()
}
pub fn statements(&self) -> impl Iterator<Item = &Statement> {
std::iter::once(&self.current)
.chain(&self.expected)
.chain(&self.unchanged)
}
#[must_use]
pub fn statement(&self, id: &str) -> Option<&Statement> {
self.statements().find(|s| s.id == id)
}
#[must_use]
pub fn cited(&self) -> Vec<String> {
self.expected
.iter()
.chain(&self.unchanged)
.map(|s| s.id.clone())
.collect()
}
#[must_use]
pub fn labelled(&self) -> String {
let mut out = format!("Current: {}\n", self.current.text);
for e in &self.expected {
out.push_str(&format!("Expected: {}\n", e.text));
}
for u in &self.unchanged {
out.push_str(&format!("Unchanged: {}\n", u.text));
}
out
}
#[must_use]
pub fn title(&self) -> String {
let first = self
.current
.text
.split(['.', '!', '?'])
.find(|s| !s.trim().is_empty())
.unwrap_or(&self.current.text)
.trim();
let words: Vec<&str> = first.split_whitespace().take(8).collect();
let mut t = words.join(" ");
if first.split_whitespace().count() > 8 {
t.push('…');
}
t
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum HomeKind {
NativeSpeckitBug,
NativeKiroBugfix,
StandingSpeckit,
StandingOpenspec,
}
impl HomeKind {
#[must_use]
pub const fn native(self) -> bool {
matches!(self, Self::NativeSpeckitBug | Self::NativeKiroBugfix)
}
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::NativeSpeckitBug => "native_speckit_bug",
Self::NativeKiroBugfix => "native_kiro_bugfix",
Self::StandingSpeckit => "standing_speckit",
Self::StandingOpenspec => "standing_openspec",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
pub struct BugHome {
pub kind: HomeKind,
pub location: String,
pub reason: String,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
#[serde(rename_all = "snake_case")]
#[schemars(rename = "BugStage")]
pub enum Stage {
Record,
Reproduce,
Confirm,
Unchanged,
Fix,
Ship,
Done,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
#[serde(rename_all = "snake_case")]
#[schemars(rename = "BugOutcome")]
pub enum Outcome {
Recorded,
AwaitingConfirmation,
CannotReproduce,
AlreadySatisfied,
UnchangedNotHolding,
Fixed,
Failed,
}
impl Outcome {
#[must_use]
pub const fn as_str(self) -> &'static str {
match self {
Self::Recorded => "recorded",
Self::AwaitingConfirmation => "awaiting_confirmation",
Self::CannotReproduce => "cannot_reproduce",
Self::AlreadySatisfied => "already_satisfied",
Self::UnchangedNotHolding => "unchanged_not_holding",
Self::Fixed => "fixed",
Self::Failed => "failed",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
#[serde(rename_all = "snake_case")]
#[schemars(rename = "ReproductionSource")]
pub enum Source {
Author,
Person,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
#[serde(rename_all = "snake_case")]
#[schemars(rename = "FinalState")]
pub enum Final {
Passed,
Failed,
NotRun,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
#[serde(rename_all = "snake_case")]
#[schemars(rename = "RevertOutcome")]
pub enum Revert {
Rejected,
Survived,
Unviable,
NotRun,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
pub struct Reproduction {
pub statement: String,
pub file: String,
pub test: String,
pub source: Source,
pub baseline: Vec<RedClass>,
pub digest: String,
pub argv: Vec<String>,
#[serde(rename = "final")]
pub final_: Final,
pub revert: Revert,
}
impl Reproduction {
#[must_use]
pub fn reproduces(&self) -> bool {
self.statement.starts_with('E')
}
#[must_use]
pub fn red(&self) -> bool {
self.baseline.len() >= 2 && self.baseline.iter().all(|c| *c == RedClass::Assertion)
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
pub struct HeldOutVariant {
pub statement: String,
pub baseline: RedClass,
#[serde(rename = "final")]
pub final_: Final,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
pub struct TryStatement {
pub statement: String,
pub class: RedClass,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub excerpt: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
pub struct ReproduceTry {
pub n: u32,
pub per_statement: Vec<TryStatement>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub discarded: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub run_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub usage: Option<serde_json::Value>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub rejection: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Reproduce {
Reproduced {
n: u32,
},
AlreadySatisfied,
Retry,
CannotReproduce {
reproduced: Vec<String>,
satisfied: Vec<String>,
not_demonstrated: Vec<String>,
},
}
fn class_in<'t>(t: &'t ReproduceTry, statement: &str) -> Option<&'t RedClass> {
t.per_statement
.iter()
.find(|s| s.statement == statement)
.map(|s| &s.class)
}
#[must_use]
pub fn decide_reproduce(tries: &[ReproduceTry], expected: &[String], max: u32) -> Reproduce {
if let Some(t) = tries.last().filter(|t| {
t.discarded.is_none()
&& !expected.is_empty()
&& expected
.iter()
.all(|e| class_in(t, e) == Some(&RedClass::Assertion))
}) {
return Reproduce::Reproduced { n: t.n };
}
let made = u32::try_from(tries.len()).unwrap_or(u32::MAX);
if made < max.max(1) {
return Reproduce::Retry;
}
let all_passed = !tries.is_empty()
&& tries.iter().all(|t| {
t.discarded.is_none()
&& expected
.iter()
.all(|e| class_in(t, e) == Some(&RedClass::Passed))
});
if all_passed {
return Reproduce::AlreadySatisfied;
}
let mut reproduced = Vec::new();
let mut satisfied = Vec::new();
let mut not_demonstrated = Vec::new();
for e in expected {
let classes: Vec<&RedClass> = tries
.iter()
.filter(|t| t.discarded.is_none())
.filter_map(|t| class_in(t, e))
.collect();
if classes.contains(&&RedClass::Assertion) {
reproduced.push(e.clone());
} else if !classes.is_empty() && classes.iter().all(|c| **c == RedClass::Passed) {
satisfied.push(e.clone());
} else {
not_demonstrated.push(e.clone());
}
}
Reproduce::CannotReproduce {
reproduced,
satisfied,
not_demonstrated,
}
}
#[must_use]
pub fn rejected_try(reason: &str) -> String {
format!("rejected by the person: {reason}")
}
pub const MAX_SLUG: usize = 64;
#[must_use]
pub fn valid_slug(slug: &str) -> bool {
!slug.is_empty()
&& slug.len() <= MAX_SLUG
&& slug
.bytes()
.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-')
&& slug
.bytes()
.next()
.is_some_and(|b| b.is_ascii_lowercase() || b.is_ascii_digit())
}
#[must_use]
pub fn bug_class(first: RedClass, second: Option<RedClass>) -> RedClass {
match (first, second) {
(RedClass::Assertion, Some(RedClass::Assertion)) => RedClass::Assertion,
(RedClass::Assertion, _) => RedClass::Flaky,
(c, _) => c,
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum Basis {
Shown,
NamedTest,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
pub struct Confirmation {
pub digest: String,
pub triad_digest: String,
pub by: String,
pub via: Via,
pub basis: Basis,
pub at: u64,
}
impl Confirmation {
#[must_use]
pub fn current(&self, triad_digest: &str) -> bool {
self.triad_digest == triad_digest
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
pub struct Rejection {
pub digest: String,
pub reason: String,
pub by: String,
pub via: Via,
pub at: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum PassToPassState {
Selected,
NoneSelected,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum MemberFinal {
Passed,
Failed,
Missing,
NotRun,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
#[schemars(rename = "PassToPassMember")]
pub struct Member {
pub test: String,
pub file: String,
pub rule: String,
#[serde(rename = "final")]
pub final_: MemberFinal,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
pub struct PassToPass {
pub state: PassToPassState,
pub members: Vec<Member>,
}
impl PassToPass {
#[must_use]
pub fn held(&self) -> bool {
self.members.iter().all(|m| m.final_ == MemberFinal::Passed)
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
pub struct RevertResult {
pub outcome: Revert,
pub rejected_by: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
pub struct SpecialCase {
pub statement: String,
pub digest: String,
pub rejected: bool,
pub regenerated: bool,
#[serde(default)]
pub ruled: bool,
}
#[derive(Debug, Clone, Copy)]
pub struct Discrimination<'a> {
pub reproductions: &'a [Reproduction],
pub revert: &'a RevertResult,
pub special_case: &'a [SpecialCase],
pub special_case_error: Option<&'a str>,
}
#[must_use]
pub fn bug_verdict(
base: Verdict,
d: &Discrimination<'_>,
p2p: &PassToPass,
held_out: &HeldOut,
) -> Verdict {
let mut reasons = Vec::new();
for r in d.reproductions {
if r.final_ != Final::Passed {
reasons.push(format!(
"the reproduction of {} does not pass on the fixed tree",
r.statement
));
}
}
if d.revert.rejected_by.is_empty() {
reasons.push(match d.revert.outcome {
Revert::Unviable => "the reproduction does not discriminate the fix: the revert does not build, and no reproduction rejects it by an assertion".to_owned(),
_ => "the reproduction does not discriminate the fix: with the change undone the reproductions still pass".to_owned(),
});
}
let mut reproducing: Vec<&str> = d
.reproductions
.iter()
.filter(|r| r.reproduces())
.map(|r| r.statement.as_str())
.collect();
reproducing.dedup();
for st in reproducing {
if !d.special_case.iter().any(|s| s.statement == st) {
reasons.push(format!(
"no special-case candidate for {st} ({}): the reproduction's strength is not shown",
d.special_case_error
.unwrap_or("the special-case author wrote none")
));
}
}
for s in d.special_case {
if !s.rejected && !s.ruled {
reasons.push(format!(
"reproduction too weak: a fix that special-cases {}'s input passes the reproductions and the held-out variants",
s.statement
));
}
}
for m in &p2p.members {
match m.final_ {
MemberFinal::Passed => {}
MemberFinal::Missing => {
reasons.push(format!("pass-to-pass: {} is missing", m.test));
}
MemberFinal::Failed | MemberFinal::NotRun => {
reasons.push(format!("pass-to-pass: {} does not pass", m.test));
}
}
}
match held_out {
HeldOut::Passed { .. } | HeldOut::Off => {}
HeldOut::Absent { reason } => reasons.push(format!(
"no held-out variant was kept ({reason}): the fix cannot be checked on other inputs"
)),
HeldOut::Failed { scenarios } => reasons.push(format!(
"held-out variant failed for {}",
scenarios.join(", ")
)),
}
match base {
Verdict::Failed {
reasons: mut before,
} => {
for r in reasons {
if !before.contains(&r) {
before.push(r);
}
}
Verdict::Failed { reasons: before }
}
v if reasons.is_empty() => v,
_ => Verdict::Failed { reasons },
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
#[schemars(rename = "AuthorClaim")]
pub struct Claim {
pub text: String,
pub by: String,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
#[schemars(rename = "StatementsShown")]
pub struct Statements {
pub reproduced: Vec<String>,
pub not_reproduced: Vec<String>,
pub satisfied: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
pub struct Handoff {
pub outcome: Outcome,
pub tries: Vec<ReproduceTry>,
pub statements: Statements,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cost: Option<serde_json::Value>,
pub elapsed_ms: u64,
pub next: Vec<NextStep>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub claim: Option<Claim>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum CandidateOrigin {
Patch,
Pr,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
pub struct CandidateRefusal {
pub code: String,
pub paths: Vec<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
pub struct Candidate {
pub origin: CandidateOrigin,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub path: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub number: Option<u64>,
pub digest: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub head_sha: Option<String>,
pub applied: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub refused: Option<CandidateRefusal>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
pub struct IssueRef {
pub number: u64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub repo: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub context_digest: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
pub struct PreFrozenRef {
pub manifest: String,
pub approval: String,
pub wording: String,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
pub struct AcceptanceRef {
pub source: crate::acceptance::AcceptanceSource,
pub pre_frozen: PreFrozenRef,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, schemars::JsonSchema)]
pub struct BugReport {
pub home: BugHome,
pub unit: String,
pub requirement: String,
pub task: String,
pub title: String,
pub triad: Triad,
pub stage: Stage,
pub outcome: Outcome,
#[serde(default)]
pub verdict: Option<Verdict>,
#[serde(default)]
pub code: Option<String>,
#[serde(default)]
pub reproductions: Vec<Reproduction>,
#[serde(default)]
pub held_out: Vec<HeldOutVariant>,
#[serde(default)]
pub held_out_sealed: Option<crate::acceptance::SealedRef>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub held_out_reason: Option<String>,
#[serde(default)]
pub confirm_digest: Option<String>,
#[serde(default)]
pub confirmation: Option<Confirmation>,
#[serde(default)]
pub rejections: Vec<Rejection>,
#[serde(default)]
pub acceptance: Option<AcceptanceRef>,
#[serde(default)]
pub pass_to_pass: Option<PassToPass>,
#[serde(default)]
pub revert: Option<RevertResult>,
#[serde(default)]
pub special_case: Vec<SpecialCase>,
#[serde(default)]
pub tries: Vec<ReproduceTry>,
pub max_tries: u32,
#[serde(default)]
pub handoff: Option<Handoff>,
#[serde(default)]
pub issue: Option<IssueRef>,
#[serde(default)]
pub candidate: Option<Candidate>,
#[serde(default)]
pub recorded_commit: Option<String>,
#[serde(default)]
pub commit: Option<String>,
#[serde(default)]
pub run_id: Option<String>,
}
impl BugReport {
#[must_use]
pub fn expected_ids(&self) -> Vec<String> {
self.triad.expected.iter().map(|s| s.id.clone()).collect()
}
#[must_use]
pub fn key(&self) -> String {
format!("{}/{}", self.unit, self.task)
}
}
#[cfg(test)]
mod tests {
use proptest::prelude::*;
use super::*;
fn hash(b: &[u8]) -> String {
format!(
"{:016x}",
b.iter().fold(0xcbf2_9ce4_8422_2325_u64, |h, x| {
(h ^ u64::from(*x)).wrapping_mul(0x0100_0000_01b3)
})
)
}
fn strings(v: &[&str]) -> Vec<String> {
v.iter().map(|s| (*s).to_owned()).collect()
}
#[test]
fn a_triad_numbers_its_statements_and_refuses_empty_words() {
let t = Triad::new(
" saving an empty title crashes ",
&strings(&["an empty title is refused", "the form keeps its text"]),
&strings(&["a title is saved"]),
hash,
)
.unwrap_or_else(|e| panic!("{e:?}"));
assert_eq!(t.current.text, "saving an empty title crashes");
assert_eq!(t.cited(), ["E1", "E2", "U1"]);
assert!(t.digest.starts_with("sha256:"));
assert_eq!(
t.labelled(),
"Current: saving an empty title crashes\nExpected: an empty title is refused\nExpected: the form keeps its text\nUnchanged: a title is saved\n"
);
assert_eq!(t.title(), "saving an empty title crashes");
assert_eq!(
Triad::new(" ", &strings(&["x"]), &[], hash),
Err(TriadError::Empty)
);
assert_eq!(Triad::new("x", &[], &[], hash), Err(TriadError::NoExpected));
assert_eq!(
Triad::new("x", &strings(&[" "]), &[], hash),
Err(TriadError::Empty)
);
assert_eq!(TriadError::NoExpected.code(), "BUG-NO-EXPECTED");
}
fn try_of(n: u32, classes: &[(&str, RedClass)]) -> ReproduceTry {
ReproduceTry {
n,
per_statement: classes
.iter()
.map(|(s, c)| TryStatement {
statement: (*s).to_owned(),
class: *c,
excerpt: None,
})
.collect(),
discarded: None,
run_id: None,
usage: None,
rejection: None,
}
}
fn red_class() -> impl Strategy<Value = RedClass> {
prop_oneof![
Just(RedClass::Assertion),
Just(RedClass::Absent),
Just(RedClass::BuildError),
Just(RedClass::CollectionError),
Just(RedClass::Timeout),
Just(RedClass::NoTests),
Just(RedClass::Flaky),
Just(RedClass::Unclassified),
Just(RedClass::Passed),
]
}
proptest! {
#[test]
fn reproduce_is_decided_by_classes_alone(
classes in proptest::collection::vec(proptest::collection::vec(red_class(), 2), 1..6),
max in 1u32..6,
) {
let expected = strings(&["E1", "E2"]);
let tries: Vec<ReproduceTry> = classes
.iter()
.enumerate()
.map(|(i, c)| try_of(u32::try_from(i).unwrap_or(0) + 1, &[("E1", c[0]), ("E2", c[1])]))
.collect();
let all_red = tries.last().is_some_and(|t| t.per_statement.iter().all(|s| s.class == RedClass::Assertion));
match decide_reproduce(&tries, &expected, max) {
Reproduce::Reproduced { n } => {
prop_assert!(all_red);
prop_assert_eq!(n as usize, tries.len());
let t = &tries[(n - 1) as usize];
prop_assert!(t.per_statement.iter().all(|s| s.class == RedClass::Assertion));
}
Reproduce::AlreadySatisfied => {
prop_assert!(!all_red);
prop_assert!(tries.len() >= max as usize);
prop_assert!(tries.iter().all(|t| t.per_statement.iter().all(|s| s.class == RedClass::Passed)));
}
Reproduce::Retry => {
prop_assert!(!all_red);
prop_assert!(tries.len() < max as usize);
}
Reproduce::CannotReproduce { reproduced, satisfied, not_demonstrated } => {
prop_assert!(!all_red);
prop_assert!(tries.len() >= max as usize);
let mut named: Vec<String> = reproduced.iter().chain(&satisfied).chain(¬_demonstrated).cloned().collect();
named.sort();
prop_assert_eq!(named, expected.clone());
}
}
}
#[test]
fn a_bug_is_red_only_by_two_assertions(a in red_class(), b in red_class()) {
let c = bug_class(a, Some(b));
prop_assert_eq!(c == RedClass::Assertion, a == RedClass::Assertion && b == RedClass::Assertion);
}
#[test]
fn verified_needs_every_bug_rule(
finals in proptest::collection::vec(prop_oneof![Just(Final::Passed), Just(Final::Failed), Just(Final::NotRun)], 1..4),
rejected_by in proptest::collection::vec("E[1-3]", 0..3),
special in proptest::collection::vec((any::<bool>(), any::<bool>()), 0..3),
members in proptest::collection::vec(prop_oneof![Just(MemberFinal::Passed), Just(MemberFinal::Failed), Just(MemberFinal::Missing)], 0..4),
held in prop_oneof![
Just(HeldOut::Passed { files: 1 }),
Just(HeldOut::Failed { scenarios: vec!["E1".to_owned()] }),
Just(HeldOut::Absent { reason: "all_discarded".to_owned() }),
Just(HeldOut::Off),
],
) {
let reproductions: Vec<Reproduction> = finals.iter().enumerate().map(|(i, f)| Reproduction {
statement: format!("E{}", i + 1),
file: format!("tests/repro_{i}.rs"),
test: "t".to_owned(),
source: Source::Author,
baseline: vec![RedClass::Assertion, RedClass::Assertion],
digest: "sha256:x".to_owned(),
argv: Vec::new(),
final_: *f,
revert: Revert::NotRun,
}).collect();
let revert = RevertResult { outcome: if rejected_by.is_empty() { Revert::Survived } else { Revert::Rejected }, rejected_by: rejected_by.clone() };
let special_case: Vec<SpecialCase> = special.iter().enumerate().map(|(i, (rejected, ruled))| SpecialCase {
statement: format!("E{}", i + 1), digest: "sha256:s".to_owned(), rejected: *rejected, regenerated: false, ruled: *ruled,
}).collect();
let p2p = PassToPass {
state: if members.is_empty() { PassToPassState::NoneSelected } else { PassToPassState::Selected },
members: members.iter().map(|m| Member { test: "held_out_secret_name".to_owned(), file: "tests/a.rs".to_owned(), rule: "same_file".to_owned(), final_: *m }).collect(),
};
let d = Discrimination { reproductions: &reproductions, revert: &revert, special_case: &special_case, special_case_error: None };
let v = bug_verdict(Verdict::Verified, &d, &p2p, &held);
let ok = finals.iter().all(|f| *f == Final::Passed)
&& !rejected_by.is_empty()
&& (1..=finals.len()).all(|i| special_case.iter().any(|s| s.statement == format!("E{i}")))
&& special.iter().all(|(r, ruled)| *r || *ruled)
&& members.iter().all(|m| *m == MemberFinal::Passed)
&& matches!(held, HeldOut::Passed { .. } | HeldOut::Off);
prop_assert_eq!(v == Verdict::Verified, ok);
if let Verdict::Failed { reasons } = &v {
prop_assert!(reasons.iter().all(|r| !r.contains("held_out_") || r.starts_with("pass-to-pass")));
}
let failed = bug_verdict(Verdict::Failed { reasons: vec!["check `x` failed".to_owned()] }, &d, &p2p, &held);
prop_assert!(matches!(failed, Verdict::Failed { .. }), "a failed verdict stays failed");
}
}
#[test]
fn absent_and_flaky_are_not_a_bugs_red() {
assert_eq!(
bug_class(RedClass::Absent, Some(RedClass::Absent)),
RedClass::Absent
);
assert_eq!(
bug_class(RedClass::Assertion, Some(RedClass::Passed)),
RedClass::Flaky
);
assert_eq!(bug_class(RedClass::Assertion, None), RedClass::Flaky);
let tries = [try_of(1, &[("E1", RedClass::Absent)])];
assert!(matches!(
decide_reproduce(&tries, &strings(&["E1"]), 1),
Reproduce::CannotReproduce { .. }
));
}
#[test]
fn a_rejected_try_is_discarded_and_the_tries_stay_bounded() {
let e1 = strings(&["E1"]);
let mut rejected = try_of(1, &[("E1", RedClass::Assertion)]);
rejected.discarded = Some(rejected_try("it tests the wrong field"));
assert_eq!(
decide_reproduce(std::slice::from_ref(&rejected), &e1, 3),
Reproduce::Retry
);
let again = try_of(2, &[("E1", RedClass::Assertion)]);
assert_eq!(
decide_reproduce(&[rejected.clone(), again], &e1, 3),
Reproduce::Reproduced { n: 2 }
);
let all: Vec<ReproduceTry> = (1..=3)
.map(|n| ReproduceTry {
n,
..rejected.clone()
})
.collect();
assert_eq!(
decide_reproduce(&all, &e1, 3),
Reproduce::CannotReproduce {
reproduced: Vec::new(),
satisfied: Vec::new(),
not_demonstrated: e1.clone(),
}
);
let tries = [
try_of(1, &[("E1", RedClass::Assertion)]),
try_of(2, &[("E1", RedClass::BuildError)]),
];
assert_eq!(decide_reproduce(&tries, &e1, 3), Reproduce::Retry);
}
#[test]
fn a_missing_special_case_withholds_verified() {
let reproductions = vec![Reproduction {
statement: "E1".to_owned(),
file: "tests/repro.rs".to_owned(),
test: "t".to_owned(),
source: Source::Author,
baseline: vec![RedClass::Assertion, RedClass::Assertion],
digest: "sha256:x".to_owned(),
argv: Vec::new(),
final_: Final::Passed,
revert: Revert::Rejected,
}];
let revert = RevertResult {
outcome: Revert::Rejected,
rejected_by: strings(&["E1"]),
};
let p2p = PassToPass {
state: PassToPassState::NoneSelected,
members: Vec::new(),
};
let d = Discrimination {
reproductions: &reproductions,
revert: &revert,
special_case: &[],
special_case_error: Some("the author call failed: timeout"),
};
let Verdict::Failed { reasons } =
bug_verdict(Verdict::Verified, &d, &p2p, &HeldOut::Passed { files: 1 })
else {
panic!("verified without a special case");
};
assert!(
reasons
.iter()
.any(|r| r.contains("E1") && r.contains("the author call failed: timeout")),
"{reasons:?}"
);
}
#[test]
fn a_slug_is_never_a_path() {
for ok in ["empty-title-crash", "a", "0-x", "saving-an-empty-title-2"] {
assert!(valid_slug(ok), "{ok}");
}
for bad in [
"",
"..",
"../x",
"a/b",
"a\\b",
"-x",
"A",
"a b",
"a.b",
"é",
&"a".repeat(MAX_SLUG + 1),
] {
assert!(!valid_slug(bad), "{bad}");
}
}
#[test]
fn mixed_tries_name_what_was_satisfied() {
let tries = [
try_of(1, &[("E1", RedClass::Assertion), ("E2", RedClass::Passed)]),
try_of(2, &[("E1", RedClass::BuildError), ("E2", RedClass::Passed)]),
];
assert_eq!(
decide_reproduce(&tries, &strings(&["E1", "E2"]), 2),
Reproduce::CannotReproduce {
reproduced: strings(&["E1"]),
satisfied: strings(&["E2"]),
not_demonstrated: Vec::new(),
}
);
let passed = [
try_of(1, &[("E1", RedClass::Passed)]),
try_of(2, &[("E1", RedClass::Passed)]),
];
assert_eq!(
decide_reproduce(&passed, &strings(&["E1"]), 2),
Reproduce::AlreadySatisfied
);
assert_eq!(
decide_reproduce(&passed[..1], &strings(&["E1"]), 2),
Reproduce::Retry
);
}
}