#![allow(dead_code, clippy::all)]
#![allow(non_snake_case, unused_imports)]
use std::path::{Path, PathBuf};
use std::time::Duration;
use chrono::Utc;
use ought_analysis::types::*;
use ought_analysis::{audit, bisect, blame, survey};
use ought_gen::generator::{GeneratedTest, Language};
use ought_run::runner::Runner;
use ought_run::{RunResult, TestDetails, TestResult, TestStatus};
use ought_spec::types::*;
use ought_spec::SpecGraph;
struct StubRunner;
impl Runner for StubRunner {
fn run(&self, _: &[GeneratedTest], _: &Path) -> anyhow::Result<RunResult> {
Ok(RunResult {
results: vec![],
total_duration: Duration::ZERO,
})
}
fn is_available(&self) -> bool {
true
}
fn name(&self) -> &str {
"stub"
}
}
#[test]
fn test_survey_survey_result_can_be_constructed() {
let result = SurveyResult {
uncovered: vec![],
};
assert!(result.uncovered.is_empty());
}
#[test]
fn test_survey_uncovered_behavior_has_expected_fields() {
let behavior = UncoveredBehavior {
file: PathBuf::from("src/api.rs"),
line: 42,
description: "create_user has no clause".to_string(),
suggested_clause: "MUST create a user with the given name".to_string(),
suggested_keyword: Keyword::Must,
suggested_spec: PathBuf::from("ought/api.ought.md"),
};
assert_eq!(behavior.file, PathBuf::from("src/api.rs"));
assert_eq!(behavior.line, 42);
assert!(!behavior.description.is_empty());
assert!(!behavior.suggested_clause.is_empty());
assert_eq!(behavior.suggested_keyword, Keyword::Must);
assert_eq!(behavior.suggested_spec, PathBuf::from("ought/api.ought.md"));
}
#[test]
fn test_survey_must_output_a_list_of_uncovered_behaviors_with_file_and_line_refe() {
let behaviors = vec![
UncoveredBehavior {
file: PathBuf::from("src/api.rs"),
line: 1,
description: "create_user has no clause".to_string(),
suggested_clause: "MUST create a user with the given name".to_string(),
suggested_keyword: Keyword::Must,
suggested_spec: PathBuf::from("ought/api.ought.md"),
},
UncoveredBehavior {
file: PathBuf::from("src/api.rs"),
line: 2,
description: "delete_user has no clause".to_string(),
suggested_clause: "MUST delete the user with the given id".to_string(),
suggested_keyword: Keyword::Must,
suggested_spec: PathBuf::from("ought/api.ought.md"),
},
];
let result = SurveyResult {
uncovered: behaviors,
};
assert!(!result.uncovered.is_empty(), "survey must output at least one uncovered behavior");
for b in &result.uncovered {
assert!(
b.file != PathBuf::from(""),
"each uncovered behavior must include a non-empty file path"
);
assert!(
b.line > 0,
"each uncovered behavior must include a positive line reference; got line {}",
b.line
);
}
}
#[test]
fn test_survey_must_suggest_concrete_clause_text_with_appropriate_keyword_for_ea() {
const BEHAVIORAL_KEYWORDS: &[Keyword] = &[
Keyword::Must,
Keyword::MustNot,
Keyword::Should,
Keyword::ShouldNot,
Keyword::May,
Keyword::Wont,
Keyword::MustAlways,
Keyword::MustBy,
];
let behaviors = vec![
UncoveredBehavior {
file: PathBuf::from("src/service.rs"),
line: 1,
description: "process_payment uncovered".to_string(),
suggested_clause: "MUST process the payment and return success status".to_string(),
suggested_keyword: Keyword::Must,
suggested_spec: PathBuf::from("ought/svc.ought.md"),
},
UncoveredBehavior {
file: PathBuf::from("src/service.rs"),
line: 1,
description: "payment error path".to_string(),
suggested_clause: "SHOULD return false when payment fails".to_string(),
suggested_keyword: Keyword::Should,
suggested_spec: PathBuf::from("ought/svc.ought.md"),
},
];
for b in &behaviors {
assert!(
!b.suggested_clause.is_empty(),
"suggested_clause must be non-empty for every uncovered behavior"
);
assert!(
BEHAVIORAL_KEYWORDS.contains(&b.suggested_keyword),
"suggested_keyword {:?} must be a deontic behavioral keyword, not structural",
b.suggested_keyword
);
}
}
#[test]
fn test_survey_should_group_suggestions_by_the_spec_file_they_would_belong_to() {
let behaviors = vec![
UncoveredBehavior {
file: PathBuf::from("src/lib.rs"),
line: 1,
description: "auth_login uncovered".to_string(),
suggested_clause: "MUST authenticate the user on login".to_string(),
suggested_keyword: Keyword::Must,
suggested_spec: PathBuf::from("auth.ought.md"),
},
UncoveredBehavior {
file: PathBuf::from("src/lib.rs"),
line: 2,
description: "auth_logout uncovered".to_string(),
suggested_clause: "MUST invalidate session on logout".to_string(),
suggested_keyword: Keyword::Must,
suggested_spec: PathBuf::from("auth.ought.md"),
},
UncoveredBehavior {
file: PathBuf::from("src/lib.rs"),
line: 3,
description: "billing_charge uncovered".to_string(),
suggested_clause: "MUST charge the correct amount".to_string(),
suggested_keyword: Keyword::Must,
suggested_spec: PathBuf::from("billing.ought.md"),
},
];
let result = SurveyResult {
uncovered: behaviors,
};
assert_eq!(result.uncovered.len(), 3, "all three behaviors must be reported");
let mut seen_specs: Vec<PathBuf> = Vec::new();
let mut last_spec: Option<PathBuf> = None;
for b in &result.uncovered {
if last_spec.as_ref() != Some(&b.suggested_spec) {
assert!(
!seen_specs.contains(&b.suggested_spec),
"behaviors for {:?} are not grouped -- they appear interleaved with other spec files",
b.suggested_spec
);
if let Some(prev) = last_spec.take() {
seen_specs.push(prev);
}
last_spec = Some(b.suggested_spec.clone());
}
}
}
#[test]
fn test_survey_should_offer_to_append_suggested_clauses_to_the_relevant_spec_file() {
let behavior = UncoveredBehavior {
file: PathBuf::from("src/router.rs"),
line: 1,
description: "route function uncovered".to_string(),
suggested_clause: "MUST dispatch requests to the correct handler".to_string(),
suggested_keyword: Keyword::Must,
suggested_spec: PathBuf::from("specs/router.ought.md"),
};
assert!(
behavior.suggested_spec != PathBuf::from(""),
"every uncovered behavior must include a suggested_spec path for append-offer"
);
}
#[test]
fn test_survey_should_rank_uncovered_behaviors_by_risk_public_api_internal_helper() {
let public_behavior = UncoveredBehavior {
file: PathBuf::from("src/lib.rs"),
line: 1,
description: "public_process (public API)".to_string(),
suggested_clause: "MUST process data and return result".to_string(),
suggested_keyword: Keyword::Must,
suggested_spec: PathBuf::from("ought/svc.ought.md"),
};
let private_behavior = UncoveredBehavior {
file: PathBuf::from("src/lib.rs"),
line: 2,
description: "private_validate (internal helper)".to_string(),
suggested_clause: "SHOULD validate non-empty data".to_string(),
suggested_keyword: Keyword::Should,
suggested_spec: PathBuf::from("ought/svc.ought.md"),
};
let result = SurveyResult {
uncovered: vec![public_behavior, private_behavior],
};
assert_eq!(result.uncovered.len(), 2);
let first = &result.uncovered[0];
let second = &result.uncovered[1];
assert!(
first.description.contains("public"),
"public API behavior must rank first; got description: {:?}",
first.description
);
assert!(
second.description.contains("private"),
"internal helper must rank second; got description: {:?}",
second.description
);
}
#[test]
fn test_survey_wont_auto_add_clauses_without_user_confirmation() {
let result = SurveyResult {
uncovered: vec![UncoveredBehavior {
file: PathBuf::from("src/lib.rs"),
line: 1,
description: "uncovered_fn has no clause".to_string(),
suggested_clause: "MUST implement uncovered_fn".to_string(),
suggested_keyword: Keyword::Must,
suggested_spec: PathBuf::from("specs/svc.ought.md"),
}],
};
assert!(
!result.uncovered.is_empty(),
"survey must still return suggestions even though it does not write them"
);
}
#[test]
fn test_survey_must_read_source_files_from_the_given_path_or_project_source_root() {
let tmp = std::env::temp_dir().join("ought_test_survey_source");
let _ = std::fs::remove_dir_all(&tmp);
std::fs::create_dir_all(&tmp).unwrap();
std::fs::write(
tmp.join("example.rs"),
"pub fn uncovered_function() {\n // does something\n}\n",
)
.unwrap();
let empty_dir = std::env::temp_dir().join("ought_test_survey_empty_specs");
let _ = std::fs::remove_dir_all(&empty_dir);
std::fs::create_dir_all(&empty_dir).unwrap();
let specs = SpecGraph::from_roots(&[empty_dir.clone()]).unwrap();
let result = survey::survey(&specs, &[tmp.clone()]).unwrap();
assert!(
!result.uncovered.is_empty(),
"survey must discover uncovered behaviors from source files"
);
assert!(
result.uncovered.iter().any(|u| u.description.contains("uncovered_function")),
"survey must find 'uncovered_function' in the source file"
);
let _ = std::fs::remove_dir_all(&tmp);
let _ = std::fs::remove_dir_all(&empty_dir);
}
#[test]
fn test_survey_must_read_all_existing_spec_files_to_know_what_is_already_covered() {
let tmp = std::env::temp_dir().join("ought_test_survey_covered");
let _ = std::fs::remove_dir_all(&tmp);
std::fs::create_dir_all(&tmp).unwrap();
std::fs::write(
tmp.join("example.rs"),
"pub fn create_user() {\n // creates a user\n}\npub fn delete_user() {\n // deletes a user\n}\n",
)
.unwrap();
let spec_dir = std::env::temp_dir().join("ought_test_survey_covered_specs");
let _ = std::fs::remove_dir_all(&spec_dir);
std::fs::create_dir_all(&spec_dir).unwrap();
std::fs::write(
spec_dir.join("api.ought.md"),
"# API\n\n## Users\n\n- **MUST** create_user and return the new user id\n",
)
.unwrap();
let specs = SpecGraph::from_roots(&[spec_dir.clone()]).unwrap();
let result = survey::survey(&specs, &[tmp.clone()]).unwrap();
let has_create = result.uncovered.iter().any(|u| u.description.contains("create_user"));
assert!(
!has_create,
"survey must not report 'create_user' as uncovered when a clause mentions it"
);
let has_delete = result.uncovered.iter().any(|u| u.description.contains("delete_user"));
assert!(
has_delete,
"survey must report 'delete_user' as uncovered since no clause mentions it"
);
let _ = std::fs::remove_dir_all(&tmp);
let _ = std::fs::remove_dir_all(&spec_dir);
}
#[test]
fn test_survey_must_use_the_llm_to_identify_public_behaviors_apis_and_logic_bran() {
let tmp = std::env::temp_dir().join("ought_test_survey_public");
let _ = std::fs::remove_dir_all(&tmp);
std::fs::create_dir_all(&tmp).unwrap();
std::fs::write(
tmp.join("service.rs"),
"pub fn public_api_method() {}\nfn private_helper() {}\n",
)
.unwrap();
let spec_dir = std::env::temp_dir().join("ought_test_survey_public_specs");
let _ = std::fs::remove_dir_all(&spec_dir);
std::fs::create_dir_all(&spec_dir).unwrap();
let specs = SpecGraph::from_roots(&[spec_dir.clone()]).unwrap();
let result = survey::survey(&specs, &[tmp.clone()]).unwrap();
let public_found = result.uncovered.iter().any(|u| u.description.contains("public_api_method"));
assert!(
public_found,
"survey must identify public function signatures from source files"
);
let _ = std::fs::remove_dir_all(&tmp);
let _ = std::fs::remove_dir_all(&spec_dir);
}
#[test]
fn test_audit_audit_result_can_be_constructed() {
let result = AuditResult {
findings: vec![],
};
assert!(result.findings.is_empty());
}
#[test]
fn test_audit_audit_finding_has_expected_fields() {
let finding = AuditFinding {
kind: AuditFindingKind::Contradiction,
description: "Two clauses cannot both hold".to_string(),
clauses: vec![
ClauseId("auth::login::must_a".to_string()),
ClauseId("auth::login::must_b".to_string()),
],
suggestion: Some("Resolve the contradiction by choosing one".to_string()),
confidence: Some(0.92),
};
assert_eq!(finding.kind, AuditFindingKind::Contradiction);
assert!(!finding.description.is_empty());
assert_eq!(finding.clauses.len(), 2);
assert!(finding.suggestion.is_some());
assert!(finding.confidence.is_some());
}
#[test]
fn test_audit_must_categorize_findings_as_contradiction_gap_ambiguity_or_redund() {
let findings = vec![
AuditFinding {
kind: AuditFindingKind::Contradiction,
description: "Two clauses cannot both hold".to_string(),
clauses: vec![ClauseId("auth::login::must_a".to_string())],
suggestion: None,
confidence: None,
},
AuditFinding {
kind: AuditFindingKind::Gap,
description: "Missing fallback clause".to_string(),
clauses: vec![ClauseId("auth::login::must_a".to_string())],
suggestion: None,
confidence: None,
},
AuditFinding {
kind: AuditFindingKind::Ambiguity,
description: "Clause text is unclear about timing".to_string(),
clauses: vec![ClauseId("auth::login::must_c".to_string())],
suggestion: None,
confidence: None,
},
AuditFinding {
kind: AuditFindingKind::Redundancy,
description: "Two clauses express the same obligation".to_string(),
clauses: vec![ClauseId("auth::login::must_a".to_string())],
suggestion: None,
confidence: None,
},
];
let kinds: Vec<AuditFindingKind> = findings.iter().map(|f| f.kind).collect();
assert!(kinds.contains(&AuditFindingKind::Contradiction));
assert!(kinds.contains(&AuditFindingKind::Gap));
assert!(kinds.contains(&AuditFindingKind::Ambiguity));
assert!(kinds.contains(&AuditFindingKind::Redundancy));
}
#[test]
fn test_audit_must_reference_the_specific_clauses_involved_in_each_finding_file() {
let finding = AuditFinding {
kind: AuditFindingKind::Contradiction,
description: "Conflicting HTTP status obligations".to_string(),
clauses: vec![
ClauseId("api::responses::must_return_200".to_string()),
ClauseId("api::responses::must_return_201".to_string()),
],
suggestion: None,
confidence: None,
};
assert!(
!finding.clauses.is_empty(),
"each finding must reference at least one specific clause"
);
for clause_id in &finding.clauses {
assert!(!clause_id.0.is_empty(), "each referenced clause ID must be non-empty");
}
}
#[test]
fn test_audit_must_detect_must_by_deadline_conflicts_e_g_an_operation_with_a_10() {
let finding = AuditFinding {
kind: AuditFindingKind::Contradiction,
description: "checkout MUST BY 100ms but calls payment MUST BY 200ms -- sub-operation deadline exceeds parent deadline".to_string(),
clauses: vec![
ClauseId("checkout::process::must_by_100ms_complete_the_checkout".to_string()),
ClauseId("payment::charge::must_by_200ms_charge_the_payment".to_string()),
],
suggestion: Some("Reduce the payment deadline below 100ms or increase the checkout deadline".to_string()),
confidence: Some(0.95),
};
assert_eq!(finding.kind, AuditFindingKind::Contradiction);
assert!(
finding.description.contains("100ms") || finding.description.contains("deadline"),
"finding must describe deadline conflict"
);
}
#[test]
fn test_audit_must_detect_must_always_invariant_conflicts_e_g_two_invariants_th() {
let finding = AuditFinding {
kind: AuditFindingKind::Contradiction,
description: "MUST ALWAYS maintain exactly one active session conflicts with MUST ALWAYS allow multiple concurrent sessions".to_string(),
clauses: vec![
ClauseId("auth::session::must_always_maintain_exactly_one_active_session".to_string()),
ClauseId("auth::session::must_always_support_multiple_concurrent_sessions".to_string()),
],
suggestion: Some("Reconcile session invariants by choosing a single-session or multi-session model".to_string()),
confidence: Some(0.98),
};
assert_eq!(finding.kind, AuditFindingKind::Contradiction);
assert!(
finding.description.contains("MUST ALWAYS") || finding.description.contains("invariant"),
"finding must describe invariant conflict"
);
}
#[test]
fn test_audit_should_detect_given_blocks_with_overlapping_conditions_that_impose() {
let finding = AuditFinding {
kind: AuditFindingKind::Contradiction,
description: "GIVEN user is authenticated (MUST allow write) overlaps with GIVEN user is a guest (MUST NOT allow write)".to_string(),
clauses: vec![
ClauseId("api::access::given_authenticated_must_allow_full_read_write".to_string()),
ClauseId("api::access::given_guest_must_not_allow_write".to_string()),
],
suggestion: Some("Make GIVEN conditions mutually exclusive or add explicit precedence rules".to_string()),
confidence: Some(0.80),
};
assert!(
finding.kind == AuditFindingKind::Contradiction || finding.kind == AuditFindingKind::Ambiguity,
"overlapping GIVEN conditions should be Contradiction or Ambiguity"
);
assert!(
finding.description.contains("GIVEN") || finding.description.contains("overlap"),
"description must reference the overlapping conditions"
);
}
#[test]
fn test_audit_should_detect_must_obligations_that_lack_otherwise_fallbacks_where() {
let finding = AuditFinding {
kind: AuditFindingKind::Gap,
description: "MUST fetch configuration from the remote server has no OTHERWISE fallback".to_string(),
clauses: vec![ClauseId("config::remote::must_fetch_configuration".to_string())],
suggestion: Some("Add an OTHERWISE clause specifying fallback behavior".to_string()),
confidence: Some(0.87),
};
assert_eq!(finding.kind, AuditFindingKind::Gap);
assert!(
finding.description.contains("OTHERWISE") || finding.description.contains("fallback"),
"finding must reference missing OTHERWISE"
);
}
#[test]
fn test_audit_should_suggest_resolutions_for_each_finding() {
let findings = vec![
AuditFinding {
kind: AuditFindingKind::Contradiction,
description: "Conflicting HTTP status codes".to_string(),
clauses: vec![ClauseId("api::must_return_200".to_string())],
suggestion: Some("Use 200 for reads and 201 for creation".to_string()),
confidence: None,
},
AuditFinding {
kind: AuditFindingKind::Gap,
description: "No clause for database unavailability".to_string(),
clauses: vec![ClauseId("data::must_persist_records".to_string())],
suggestion: Some("Add an OTHERWISE clause for database unreachable".to_string()),
confidence: None,
},
];
for finding in &findings {
assert!(
finding
.suggestion
.as_deref()
.map(|s| !s.is_empty())
.unwrap_or(false),
"audit should include a non-empty resolution suggestion for each finding; finding: {:?}",
finding.description
);
}
}
#[test]
fn test_audit_may_assign_a_confidence_score_to_each_finding() {
let findings = vec![
AuditFinding {
kind: AuditFindingKind::Contradiction,
description: "Conflicting auth obligations".to_string(),
clauses: vec![ClauseId("auth::must_a".to_string())],
suggestion: None,
confidence: Some(0.92),
},
AuditFinding {
kind: AuditFindingKind::Gap,
description: "Missing rate-limit clause".to_string(),
clauses: vec![ClauseId("api::ratelimit::must_throttle".to_string())],
suggestion: None,
confidence: Some(0.65),
},
AuditFinding {
kind: AuditFindingKind::Ambiguity,
description: "Vague timing requirement".to_string(),
clauses: vec![ClauseId("svc::must_respond".to_string())],
suggestion: None,
confidence: None, },
];
for finding in &findings {
if let Some(confidence) = finding.confidence {
assert!(
(0.0..=1.0).contains(&confidence),
"confidence score must be in the range [0.0, 1.0] when assigned; got {} for finding: {:?}",
confidence,
finding.description
);
}
}
}
#[test]
fn test_audit_must_use_the_llm_to_identify_gaps_areas_where_related_clauses_exi() {
let spec_dir = std::env::temp_dir().join("ought_test_audit_gaps");
let _ = std::fs::remove_dir_all(&spec_dir);
std::fs::create_dir_all(&spec_dir).unwrap();
std::fs::write(
spec_dir.join("api.ought.md"),
"# API\n\n## Remote\n\n- **MUST** fetch configuration from the remote server\n",
)
.unwrap();
let specs = SpecGraph::from_roots(&[spec_dir.clone()]).unwrap();
let result = audit::audit(&specs).unwrap();
let has_gap = result.findings.iter().any(|f| f.kind == AuditFindingKind::Gap);
assert!(
has_gap,
"audit must identify gaps such as missing OTHERWISE on network-dependent MUST clauses"
);
let _ = std::fs::remove_dir_all(&spec_dir);
}
#[test]
fn test_audit_must_use_the_llm_to_identify_contradictions_between_clauses_acros() {
let spec_dir = std::env::temp_dir().join("ought_test_audit_contradictions");
let _ = std::fs::remove_dir_all(&spec_dir);
std::fs::create_dir_all(&spec_dir).unwrap();
std::fs::write(
spec_dir.join("auth.ought.md"),
"# Auth\n\n## Sessions\n\n- **MUST** allow only one active session per user\n- **MUST** support multiple concurrent sessions per user\n",
)
.unwrap();
let specs = SpecGraph::from_roots(&[spec_dir.clone()]).unwrap();
let result = audit::audit(&specs);
assert!(result.is_ok(), "audit must not panic on valid specs");
let _ = std::fs::remove_dir_all(&spec_dir);
}
#[test]
fn test_audit_must_read_all_spec_files_and_their_cross_references() {
let spec_dir = std::env::temp_dir().join("ought_test_audit_cross_refs");
let _ = std::fs::remove_dir_all(&spec_dir);
std::fs::create_dir_all(&spec_dir).unwrap();
std::fs::write(
spec_dir.join("a.ought.md"),
"# Module A\n\n## Core\n\n- **MUST** do thing A\n",
)
.unwrap();
std::fs::write(
spec_dir.join("b.ought.md"),
"# Module B\n\n## Core\n\n- **MUST** do thing B\n",
)
.unwrap();
let specs = SpecGraph::from_roots(&[spec_dir.clone()]).unwrap();
let result = audit::audit(&specs).unwrap();
assert!(
result.findings.is_empty() || !result.findings.is_empty(),
"audit must return a valid AuditResult regardless of finding count"
);
let _ = std::fs::remove_dir_all(&spec_dir);
}
#[test]
fn test_audit_should_read_relevant_source_code_to_ground_the_analysis_in_implemen() {
let spec_dir = std::env::temp_dir().join("ought_test_audit_source_code");
let _ = std::fs::remove_dir_all(&spec_dir);
std::fs::create_dir_all(&spec_dir).unwrap();
std::fs::write(
spec_dir.join("svc.ought.md"),
"# Service\n\nsource: src/\n\n## API\n\n- **MUST** return 200 for valid requests\n",
)
.unwrap();
let specs = SpecGraph::from_roots(&[spec_dir.clone()]).unwrap();
let result = audit::audit(&specs);
assert!(result.is_ok(), "audit must not panic when source paths are referenced in specs");
let _ = std::fs::remove_dir_all(&spec_dir);
}
#[test]
fn test_blame_blame_result_can_be_constructed() {
let result = BlameResult {
clause_id: ClauseId("auth::login::must_return_401".to_string()),
last_passed: None,
first_failed: None,
likely_commit: None,
narrative: "The clause has never passed.".to_string(),
suggested_fix: None,
};
assert_eq!(result.clause_id, ClauseId("auth::login::must_return_401".to_string()));
assert!(result.last_passed.is_none());
assert!(result.first_failed.is_none());
assert!(result.likely_commit.is_none());
assert!(!result.narrative.is_empty());
assert!(result.suggested_fix.is_none());
}
#[test]
fn test_blame_commit_info_has_expected_fields() {
let commit = CommitInfo {
hash: "abc123def456".to_string(),
message: "refactor: simplify auth responses".to_string(),
author: "Jane Developer <jane@example.com>".to_string(),
date: Utc::now(),
};
assert!(!commit.hash.is_empty());
assert!(!commit.message.is_empty());
assert!(!commit.author.is_empty());
assert!(commit.date.timestamp() > 0);
}
#[test]
fn test_blame_must_accept_a_clause_identifier_e_g_auth_login_must_return_401() {
let clause_id = ClauseId("auth::login::must_return_401".to_string());
let result = BlameResult {
clause_id: clause_id.clone(),
last_passed: None,
first_failed: None,
likely_commit: None,
narrative: "stub".to_string(),
suggested_fix: None,
};
assert_eq!(
result.clause_id, clause_id,
"blame result must carry the same clause_id that was passed in"
);
}
#[test]
fn test_blame_must_output_a_narrative_explanation_of_what_broke_and_why() {
let result = BlameResult {
clause_id: ClauseId("auth::login::must_return_401".to_string()),
last_passed: Some(Utc::now()),
first_failed: Some(Utc::now()),
likely_commit: Some(CommitInfo {
hash: "abc123".to_string(),
message: "refactor: auth responses".to_string(),
author: "Dev <dev@example.com>".to_string(),
date: Utc::now(),
}),
narrative: "The auth handler was refactored to return 200 instead of 401 for invalid credentials.".to_string(),
suggested_fix: None,
};
assert!(
!result.narrative.is_empty(),
"blame must output a non-empty narrative explanation of what broke and why"
);
}
#[test]
fn test_blame_must_output_the_timeline_last_passing_run_first_failure_relevant() {
let now = Utc::now();
let result = BlameResult {
clause_id: ClauseId("auth::login::must_return_401".to_string()),
last_passed: Some(now),
first_failed: Some(now),
likely_commit: Some(CommitInfo {
hash: "abc123def".to_string(),
message: "refactor: auth responses".to_string(),
author: "Dev <dev@example.com>".to_string(),
date: now,
}),
narrative: "Timeline: last passed before breaking commit, first failed after.".to_string(),
suggested_fix: None,
};
assert!(result.last_passed.is_some(), "blame must output the last passing run timestamp");
assert!(result.first_failed.is_some(), "blame must output the first failure timestamp");
assert!(
result.likely_commit.is_some(),
"blame must output the relevant commits in the timeline"
);
}
#[test]
fn test_blame_should_name_the_author_of_the_likely_responsible_commit() {
let commit = CommitInfo {
hash: "deadbeef1234".to_string(),
message: "refactor: simplify auth responses".to_string(),
author: "Jane Developer <jane@example.com>".to_string(),
date: Utc::now(),
};
assert!(
!commit.author.is_empty(),
"blame should name the author of the likely-responsible commit; got empty author"
);
}
#[test]
fn test_blame_should_identify_the_specific_commit_and_file_change_most_likely_res() {
let result = BlameResult {
clause_id: ClauseId("auth::login::must_return_401".to_string()),
last_passed: None,
first_failed: None,
likely_commit: Some(CommitInfo {
hash: "abc123def456".to_string(),
message: "refactor: simplify auth error responses".to_string(),
author: "Jane Developer <jane@example.com>".to_string(),
date: Utc::now(),
}),
narrative: "Commit abc123def456 is most likely responsible.".to_string(),
suggested_fix: None,
};
assert!(
result.likely_commit.is_some(),
"blame should identify the specific commit most likely responsible for the failure"
);
let commit = result.likely_commit.unwrap();
assert!(
!commit.hash.is_empty(),
"blame should populate the commit hash of the likely-responsible change; got empty hash"
);
}
#[test]
fn test_blame_should_suggest_a_fix_when_the_cause_is_clear() {
let result = BlameResult {
clause_id: ClauseId("auth::login::must_return_401".to_string()),
last_passed: None,
first_failed: None,
likely_commit: None,
narrative: "The test broke because the authentication handler was changed.".to_string(),
suggested_fix: Some(
"Restore the 401 status code in src/auth.rs line 42".to_string(),
),
};
assert!(
result.suggested_fix.is_some(),
"blame should suggest a fix when the cause is clear; got None"
);
let fix = result.suggested_fix.unwrap();
assert!(!fix.is_empty(), "suggested_fix must be a non-empty string describing the fix");
}
#[test]
fn test_blame_must_not_require_a_running_llm_if_the_clause_has_never_passed_just_re() {
let result = BlameResult {
clause_id: ClauseId("auth::login::must_return_401".to_string()),
last_passed: None,
first_failed: None,
likely_commit: None,
narrative: "This clause has never passed.".to_string(),
suggested_fix: None,
};
assert!(
result.last_passed.is_none(),
"last_passed must be None when the clause has never passed"
);
assert!(
result.narrative.to_lowercase().contains("never passed"),
"blame must report that the clause has never passed in the narrative; got: {:?}",
result.narrative
);
}
#[test]
fn test_blame_must_use_git_history_to_find_when_the_clause_last_passed_and_what() {
let clause_id = ClauseId("auth::login::must_return_401".to_string());
let spec_dir = std::env::temp_dir().join("ought_test_blame_git_history");
let _ = std::fs::remove_dir_all(&spec_dir);
std::fs::create_dir_all(&spec_dir).unwrap();
let specs = SpecGraph::from_roots(&[spec_dir.clone()]).unwrap();
let run_result = RunResult {
results: vec![TestResult {
clause_id: clause_id.clone(),
status: TestStatus::Failed,
message: Some("Expected 401 but got 200".to_string()),
duration: Duration::from_millis(50),
details: TestDetails {
failure_message: Some("assertion failed: status == 401".to_string()),
..Default::default()
},
}],
total_duration: Duration::from_millis(50),
};
let result = blame::blame(&clause_id, &specs, &run_result).unwrap();
assert_eq!(result.clause_id, clause_id);
assert!(!result.narrative.is_empty(), "blame must produce a narrative");
assert!(
result.narrative.contains("failing") || result.narrative.contains("Failed"),
"narrative must mention the failure; got: {:?}",
result.narrative
);
let _ = std::fs::remove_dir_all(&spec_dir);
}
#[test]
fn test_blame_must_use_the_llm_to_correlate_the_source_diff_with_the_failure_an() {
let clause_id = ClauseId("svc::process::must_succeed".to_string());
let spec_dir = std::env::temp_dir().join("ought_test_blame_llm_correlate");
let _ = std::fs::remove_dir_all(&spec_dir);
std::fs::create_dir_all(&spec_dir).unwrap();
let specs = SpecGraph::from_roots(&[spec_dir.clone()]).unwrap();
let run_result = RunResult {
results: vec![TestResult {
clause_id: clause_id.clone(),
status: TestStatus::Failed,
message: Some("process returned error".to_string()),
duration: Duration::from_millis(10),
details: TestDetails::default(),
}],
total_duration: Duration::from_millis(10),
};
let result = blame::blame(&clause_id, &specs, &run_result);
assert!(result.is_ok(), "blame must not panic");
let _ = std::fs::remove_dir_all(&spec_dir);
}
#[test]
fn test_blame_must_retrieve_the_clause_its_generated_test_and_the_failure_outpu() {
let clause_id = ClauseId("data::store::must_persist".to_string());
let spec_dir = std::env::temp_dir().join("ought_test_blame_retrieve");
let _ = std::fs::remove_dir_all(&spec_dir);
std::fs::create_dir_all(&spec_dir).unwrap();
let specs = SpecGraph::from_roots(&[spec_dir.clone()]).unwrap();
let run_result = RunResult {
results: vec![TestResult {
clause_id: clause_id.clone(),
status: TestStatus::Failed,
message: Some("data was not persisted".to_string()),
duration: Duration::from_millis(100),
details: TestDetails {
failure_message: Some("assertion failed: db.contains(key)".to_string()),
..Default::default()
},
}],
total_duration: Duration::from_millis(100),
};
let result = blame::blame(&clause_id, &specs, &run_result).unwrap();
assert!(
result.narrative.contains("assertion failed") || result.narrative.contains("not persisted"),
"blame narrative must include the failure output; got: {:?}",
result.narrative
);
let _ = std::fs::remove_dir_all(&spec_dir);
}
#[test]
fn test_bisect_bisect_result_can_be_constructed() {
let result = BisectResult {
clause_id: ClauseId("auth::login::must_return_401".to_string()),
breaking_commit: CommitInfo {
hash: "abc123".to_string(),
message: "breaking change".to_string(),
author: "Dev <dev@example.com>".to_string(),
date: Utc::now(),
},
diff_summary: "Modified src/auth.rs: changed status from 401 to 200".to_string(),
};
assert_eq!(
result.clause_id,
ClauseId("auth::login::must_return_401".to_string())
);
assert!(!result.breaking_commit.hash.is_empty());
assert!(!result.breaking_commit.message.is_empty());
assert!(!result.breaking_commit.author.is_empty());
assert!(!result.diff_summary.is_empty());
}
#[test]
fn test_bisect_bisect_options_can_be_constructed() {
let options_default = bisect::BisectOptions {
range: None,
regenerate: false,
};
assert!(options_default.range.is_none());
assert!(!options_default.regenerate);
let options_with_range = bisect::BisectOptions {
range: Some("abc123..def456".to_string()),
regenerate: true,
};
assert_eq!(options_with_range.range.as_deref(), Some("abc123..def456"));
assert!(options_with_range.regenerate);
}
#[test]
fn test_bisect_must_accept_a_clause_identifier() {
let clause_id = ClauseId("auth::login::must_return_401".to_string());
let result = BisectResult {
clause_id: clause_id.clone(),
breaking_commit: CommitInfo {
hash: "abc123".to_string(),
message: "breaking change".to_string(),
author: "Dev <dev@example.com>".to_string(),
date: Utc::now(),
},
diff_summary: "changed auth.rs".to_string(),
};
assert_eq!(
result.clause_id, clause_id,
"bisect result must carry back the same clause_id that was passed in"
);
}
#[test]
fn test_bisect_must_show_the_commit_message_author_date_and_diff_summary_for_the() {
let now = Utc::now();
let result = BisectResult {
clause_id: ClauseId("auth::login::must_return_401".to_string()),
breaking_commit: CommitInfo {
hash: "deadbeef".to_string(),
message: "refactor: simplify auth -- always return 200".to_string(),
author: "Bob Refactorer <bob@example.com>".to_string(),
date: now,
},
diff_summary: "Modified src/auth.rs: changed status code from 401 to 200".to_string(),
};
let commit = &result.breaking_commit;
assert!(
!commit.message.is_empty(),
"bisect must populate the breaking commit message; got empty string"
);
assert!(
commit.message.contains("simplify auth") || commit.message.contains("200"),
"breaking commit message must match the actual commit; got: {:?}",
commit.message
);
assert!(
!commit.author.is_empty(),
"bisect must populate the breaking commit author; got empty string"
);
assert!(
commit.author.contains("Bob") || commit.author.contains("bob@example.com"),
"breaking commit author must match the committer; got: {:?}",
commit.author
);
assert!(
commit.date.timestamp() > 0,
"bisect must populate a non-zero date for the breaking commit; got: {:?}",
commit.date
);
assert!(
!result.diff_summary.is_empty(),
"bisect must populate diff_summary describing what changed; got empty string"
);
assert!(
result.diff_summary.contains("auth.rs") || result.diff_summary.contains("status"),
"diff summary must reference the changed file; got: {:?}",
result.diff_summary
);
}
#[test]
fn test_bisect_must_report_the_first_commit_where_the_clause_fails() {
let result = BisectResult {
clause_id: ClauseId("auth::login::must_return_401".to_string()),
breaking_commit: CommitInfo {
hash: "commit_3_hash".to_string(),
message: "commit 3".to_string(),
author: "Test Runner <test@example.com>".to_string(),
date: Utc::now(),
},
diff_summary: "Changed status.txt from pass to fail".to_string(),
};
assert!(
result.breaking_commit.message.contains("commit 3"),
"bisect must narrow to the first failing commit; got: {:?}",
result.breaking_commit.message
);
}
#[test]
fn test_bisect_should_support_range_from_to_to_limit_the_search_space() {
let options = bisect::BisectOptions {
range: Some("abc123..def456".to_string()),
regenerate: false,
};
assert!(
options.range.is_some(),
"BisectOptions must support a range field to limit the search space"
);
let range = options.range.unwrap();
assert!(
range.contains(".."),
"range should be in the format from..to; got: {:?}",
range
);
}
#[test]
fn test_bisect_should_use_the_generated_test_from_the_current_manifest_not_regener() {
let options = bisect::BisectOptions {
range: None,
regenerate: false,
};
assert!(
!options.regenerate,
"without --regenerate, bisect should reuse the manifest test"
);
}
#[test]
#[ignore = "requires a dedicated test git repository to verify working tree restoration"]
fn test_bisect_must_always_restore_the_working_tree_to_its_original_state_after_complet() {
}
#[test]
#[ignore = "requires a dedicated test git repository with known failing commits"]
fn test_bisect_must_perform_a_git_bisect_style_binary_search_checkout_commit_gen() {
}
#[test]
#[ignore = "requires a dedicated test git repository to verify branch restoration"]
fn test_bisect_must_restore_the_working_tree_to_the_original_branch() {
}
#[test]
#[ignore = "bisect progress saving (--continue) is not yet implemented"]
fn test_bisect_should_save_progress_so_ought_bisect_continue_can_resume() {
}
#[test]
#[ignore = "bisect result caching is not yet implemented"]
fn test_bisect_should_cache_test_results_per_commit_to_avoid_redundant_runs() {
}
#[test]
fn test_traits_runner_trait_can_be_implemented() {
let runner = StubRunner;
assert!(runner.is_available());
assert_eq!(runner.name(), "stub");
let result = runner.run(&[], Path::new("/tmp"));
assert!(result.is_ok(), "mock runner must succeed");
let run_result = result.unwrap();
assert!(run_result.results.is_empty());
assert_eq!(run_result.total_duration, Duration::ZERO);
}
#[test]
fn test_traits_runner_is_object_safe() {
let runner: Box<dyn Runner> = Box::new(StubRunner);
assert!(runner.is_available());
assert_eq!(runner.name(), "stub");
}
#[test]
fn test_signatures_survey_function_exists() {
let _fn_ptr: fn(&SpecGraph, &[PathBuf]) -> anyhow::Result<SurveyResult> =
survey::survey;
}
#[test]
fn test_signatures_audit_function_exists() {
let _fn_ptr: fn(&SpecGraph) -> anyhow::Result<AuditResult> = audit::audit;
}
#[test]
fn test_signatures_blame_function_exists() {
let _fn_ptr: fn(
&ClauseId,
&SpecGraph,
&RunResult,
) -> anyhow::Result<BlameResult> = blame::blame;
}
#[test]
fn test_signatures_bisect_function_exists() {
let _fn_ptr: fn(
&ClauseId,
&SpecGraph,
&dyn Runner,
&bisect::BisectOptions,
) -> anyhow::Result<BisectResult> = bisect::bisect;
}