use converge_provider_api::{
ChatMessage, ChatRequest, ChatResponse, ChatRole, DynChatBackend, ResponseFormat,
};
use regex::Regex;
use std::path::Path;
use std::sync::Arc;
use crate::truths::{TruthGovernance, parse_truth_document};
pub fn preprocess_truths(content: &str) -> String {
let re = Regex::new(r"(?m)^(\s*)Truth:").unwrap();
re.replace_all(content, "${1}Feature:").to_string()
}
#[derive(Debug, Clone)]
pub struct ValidationConfig {
pub check_business_sense: bool,
pub check_compilability: bool,
pub check_conventions: bool,
pub min_confidence: f64,
}
impl Default for ValidationConfig {
fn default() -> Self {
Self {
check_business_sense: true,
check_compilability: true,
check_conventions: true,
min_confidence: 0.7,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ValidationIssue {
pub location: String,
pub category: IssueCategory,
pub severity: Severity,
pub message: String,
pub suggestion: Option<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum IssueCategory {
BusinessSense,
Compilability,
Convention,
Syntax,
NotRelatedError,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum Severity {
Info,
Warning,
Error,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ScenarioMeta {
pub name: String,
pub kind: Option<ScenarioKind>,
pub invariant_class: Option<InvariantClassTag>,
pub id: Option<String>,
pub provider: Option<String>,
pub is_test: bool,
pub raw_tags: Vec<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum ScenarioKind {
Invariant,
Validation,
Suggestor,
EndToEnd,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum InvariantClassTag {
Structural,
Semantic,
Acceptance,
}
pub fn extract_scenario_meta(name: &str, tags: &[String]) -> ScenarioMeta {
let mut kind = None;
let mut invariant_class = None;
let mut id = None;
let mut provider = None;
let mut is_test = false;
for raw_tag in tags {
let tag = raw_tag.strip_prefix('@').unwrap_or(raw_tag);
match tag {
"invariant" => kind = Some(ScenarioKind::Invariant),
"validation" => kind = Some(ScenarioKind::Validation),
"agent" => kind = Some(ScenarioKind::Suggestor),
"e2e" => kind = Some(ScenarioKind::EndToEnd),
"structural" => invariant_class = Some(InvariantClassTag::Structural),
"semantic" => invariant_class = Some(InvariantClassTag::Semantic),
"acceptance" => invariant_class = Some(InvariantClassTag::Acceptance),
"llm" => provider = Some("llm".to_string()),
"test" => is_test = true,
t if t.starts_with("id:") => {
id = Some(t.strip_prefix("id:").unwrap_or("").to_string());
}
_ => {} }
}
ScenarioMeta {
name: name.to_string(),
kind,
invariant_class,
id,
provider,
is_test,
raw_tags: tags.to_vec(),
}
}
pub fn extract_all_metas(content: &str) -> Result<Vec<ScenarioMeta>, ValidationError> {
let document = parse_truth_document(content)?;
let feature = gherkin::Feature::parse(&document.gherkin, gherkin::GherkinEnv::default())
.map_err(|e| ValidationError::ParseError(format!("{e}")))?;
Ok(feature
.scenarios
.iter()
.map(|s| extract_scenario_meta(&s.name, &s.tags))
.collect())
}
#[derive(Debug, Clone, PartialEq)]
pub struct SpecValidation {
pub is_valid: bool,
pub file_path: String,
pub scenario_count: usize,
pub issues: Vec<ValidationIssue>,
pub confidence: f64,
pub scenario_metas: Vec<ScenarioMeta>,
pub governance: TruthGovernance,
}
impl SpecValidation {
#[must_use]
pub fn has_errors(&self) -> bool {
self.issues.iter().any(|i| i.severity == Severity::Error)
}
#[must_use]
pub fn has_warnings(&self) -> bool {
self.issues.iter().any(|i| i.severity == Severity::Warning)
}
#[must_use]
pub fn summary(&self) -> String {
let errors = self
.issues
.iter()
.filter(|i| i.severity == Severity::Error)
.count();
let warnings = self
.issues
.iter()
.filter(|i| i.severity == Severity::Warning)
.count();
if self.is_valid {
format!(
"✓ {} validated ({} scenarios, {} warnings)",
self.file_path, self.scenario_count, warnings
)
} else {
format!(
"✗ {} invalid ({} errors, {} warnings)",
self.file_path, errors, warnings
)
}
}
}
pub struct GherkinValidator {
backend: Arc<dyn DynChatBackend>,
config: ValidationConfig,
}
impl std::fmt::Debug for GherkinValidator {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("GherkinValidator")
.field("config", &self.config)
.finish_non_exhaustive()
}
}
impl GherkinValidator {
#[must_use]
pub fn new(backend: Arc<dyn DynChatBackend>, config: ValidationConfig) -> Self {
Self { backend, config }
}
pub async fn validate(
&self,
content: &str,
file_name: &str,
) -> Result<SpecValidation, ValidationError> {
let document = parse_truth_document(content)?;
let feature = gherkin::Feature::parse(&document.gherkin, gherkin::GherkinEnv::default())
.map_err(|e| ValidationError::ParseError(format!("{e}")))?;
let mut issues = Vec::new();
let scenario_count = feature.scenarios.len();
for scenario in &feature.scenarios {
let scenario_issues = self.validate_scenario(&feature, scenario).await?;
issues.extend(scenario_issues);
}
let feature_issues = self.validate_feature(&feature, &document.governance);
issues.extend(feature_issues);
let scenario_metas: Vec<ScenarioMeta> = feature
.scenarios
.iter()
.map(|s| extract_scenario_meta(&s.name, &s.tags))
.collect();
let has_errors = issues.iter().any(|i| i.severity == Severity::Error);
let confidence = if issues.is_empty() { 1.0 } else { 0.7 };
Ok(SpecValidation {
is_valid: !has_errors,
file_path: file_name.to_string(),
scenario_count,
issues,
confidence,
scenario_metas,
governance: document.governance,
})
}
pub async fn validate_file(
&self,
path: impl AsRef<Path>,
) -> Result<SpecValidation, ValidationError> {
let path = path.as_ref();
let content = tokio::fs::read_to_string(path)
.await
.map_err(|e| ValidationError::IoError(format!("{e}")))?;
let file_name = path
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("unknown");
self.validate(&content, file_name).await
}
async fn validate_scenario(
&self,
feature: &gherkin::Feature,
scenario: &gherkin::Scenario,
) -> Result<Vec<ValidationIssue>, ValidationError> {
let mut issues = Vec::new();
if self.config.check_business_sense {
match self.check_business_sense(feature, scenario).await {
Ok(Some(issue)) => issues.push(issue),
Ok(None) => {} Err(e) => {
return Err(e);
}
}
}
if self.config.check_compilability {
match self.check_compilability(feature, scenario).await {
Ok(Some(issue)) => issues.push(issue),
Ok(None) => {} Err(e) => {
return Err(e);
}
}
}
if self.config.check_conventions {
issues.extend(self.check_conventions(scenario));
}
Ok(issues)
}
#[allow(clippy::unused_self)]
fn validate_feature(
&self,
feature: &gherkin::Feature,
governance: &TruthGovernance,
) -> Vec<ValidationIssue> {
let mut issues = Vec::new();
if feature.description.is_none() {
issues.push(ValidationIssue {
location: "Feature".to_string(),
category: IssueCategory::Convention,
severity: Severity::Warning,
message: "Feature lacks a description".to_string(),
suggestion: Some("Add a description explaining the business purpose".to_string()),
});
}
if feature.scenarios.is_empty() {
issues.push(ValidationIssue {
location: "Feature".to_string(),
category: IssueCategory::Convention,
severity: Severity::Error,
message: "Feature has no scenarios".to_string(),
suggestion: Some("Add at least one scenario".to_string()),
});
}
if governance.intent.is_none() {
issues.push(ValidationIssue {
location: "Truth".to_string(),
category: IssueCategory::Convention,
severity: Severity::Warning,
message: "Truth lacks an Intent block".to_string(),
suggestion: Some(
"Add an Intent block with at least Outcome or Goal for governance context"
.to_string(),
),
});
}
let mentions_approval = feature.scenarios.iter().any(|scenario| {
scenario.name.to_ascii_lowercase().contains("approval")
|| scenario.steps.iter().any(|step| {
let text = step.value.to_ascii_lowercase();
text.contains("approval") || text.contains("approved")
})
});
if mentions_approval && governance.authority.is_none() {
issues.push(ValidationIssue {
location: "Truth".to_string(),
category: IssueCategory::Convention,
severity: Severity::Warning,
message: "Approval semantics appear in scenarios without an Authority block"
.to_string(),
suggestion: Some(
"Add an Authority block to make approval and authorization explicit"
.to_string(),
),
});
}
issues
}
async fn check_business_sense(
&self,
feature: &gherkin::Feature,
scenario: &gherkin::Scenario,
) -> Result<Option<ValidationIssue>, ValidationError> {
let prompt = format!(
r"You are a business analyst validating Gherkin specifications for a multi-agent AI system called Converge.
Feature: {}
Scenario: {}
Steps:
{}
Evaluate if this scenario makes business sense:
1. Is the precondition (Given) realistic and well-defined?
2. Is the action (When) meaningful and testable?
3. Is the expected outcome (Then) measurable and valuable?
Respond with ONLY one of:
- VALID: if the scenario makes business sense
- INVALID: <reason> if it doesn't make sense
- UNCLEAR: <question> if more context is needed",
feature.name,
scenario.name,
format_steps(&scenario.steps)
);
let system_prompt = "You are a strict business requirements validator. Be concise.";
let response = chat(&self.backend, system_prompt, &prompt, Some(200), Some(0.3)).await?;
let content = response.content.trim();
if content.starts_with("INVALID:") {
let reason = content.strip_prefix("INVALID:").unwrap_or("").trim();
Ok(Some(ValidationIssue {
location: format!("Scenario: {}", scenario.name),
category: IssueCategory::BusinessSense,
severity: Severity::Error,
message: reason.to_string(),
suggestion: None,
}))
} else if content.starts_with("UNCLEAR:") {
let question = content.strip_prefix("UNCLEAR:").unwrap_or("").trim();
Ok(Some(ValidationIssue {
location: format!("Scenario: {}", scenario.name),
category: IssueCategory::BusinessSense,
severity: Severity::Warning,
message: format!("Ambiguous: {question}"),
suggestion: Some("Clarify the scenario requirements".to_string()),
}))
} else {
Ok(None) }
}
async fn check_compilability(
&self,
feature: &gherkin::Feature,
scenario: &gherkin::Scenario,
) -> Result<Option<ValidationIssue>, ValidationError> {
let prompt = format!(
r"You are a Rust developer checking if a Gherkin scenario can be compiled to a runtime invariant.
In Converge, invariants are Rust structs implementing:
```rust
trait Invariant {{
fn name(&self) -> &str;
fn class(&self) -> InvariantClass; // Structural, Semantic, or Acceptance
fn check(&self, ctx: &Context) -> InvariantResult;
}}
```
The Context has typed facts in categories: Seeds, Hypotheses, Strategies, Constraints, Signals, Competitors, Evaluations.
Feature: {}
Scenario: {}
Steps:
{}
Can this scenario be implemented as a Converge Invariant?
Respond with ONLY one of:
- COMPILABLE: <invariant_class> - brief description of implementation
- NOT_COMPILABLE: <reason why it cannot be a runtime check>
- NEEDS_REFACTOR: <suggestion to make it compilable>",
feature.name,
scenario.name,
format_steps(&scenario.steps)
);
let system_prompt =
"You are a Rust expert. Be precise about what can be checked at runtime.";
let response = chat(&self.backend, system_prompt, &prompt, Some(200), Some(0.3)).await?;
let content = response.content.trim();
if content.starts_with("NOT_COMPILABLE:") {
let reason = content.strip_prefix("NOT_COMPILABLE:").unwrap_or("").trim();
Ok(Some(ValidationIssue {
location: format!("Scenario: {}", scenario.name),
category: IssueCategory::Compilability,
severity: Severity::Error,
message: format!("Cannot compile to invariant: {reason}"),
suggestion: None,
}))
} else if content.starts_with("NEEDS_REFACTOR:") {
let suggestion = content.strip_prefix("NEEDS_REFACTOR:").unwrap_or("").trim();
Ok(Some(ValidationIssue {
location: format!("Scenario: {}", scenario.name),
category: IssueCategory::Compilability,
severity: Severity::Warning,
message: "Scenario needs refactoring to be compilable".to_string(),
suggestion: Some(suggestion.to_string()),
}))
} else {
Ok(None) }
}
#[allow(clippy::unused_self)]
fn check_conventions(&self, scenario: &gherkin::Scenario) -> Vec<ValidationIssue> {
let mut issues = Vec::new();
if scenario.name.is_empty() {
issues.push(ValidationIssue {
location: "Scenario".to_string(),
category: IssueCategory::Convention,
severity: Severity::Error,
message: "Scenario has no name".to_string(),
suggestion: Some("Add a descriptive name".to_string()),
});
}
let has_given = scenario
.steps
.iter()
.any(|s| matches!(s.ty, gherkin::StepType::Given));
let has_when = scenario
.steps
.iter()
.any(|s| matches!(s.ty, gherkin::StepType::When));
let has_then = scenario
.steps
.iter()
.any(|s| matches!(s.ty, gherkin::StepType::Then));
if !has_given && !has_when {
issues.push(ValidationIssue {
location: format!("Scenario: {}", scenario.name),
category: IssueCategory::Convention,
severity: Severity::Warning,
message: "Scenario lacks Given or When steps".to_string(),
suggestion: Some("Add preconditions (Given) or actions (When)".to_string()),
});
}
if !has_then {
issues.push(ValidationIssue {
location: format!("Scenario: {}", scenario.name),
category: IssueCategory::Convention,
severity: Severity::Error,
message: "Scenario lacks Then steps (expected outcomes)".to_string(),
suggestion: Some(
"Add at least one Then step defining the expected outcome".to_string(),
),
});
}
for step in &scenario.steps {
if step.value.contains("should") && matches!(step.ty, gherkin::StepType::Then) {
} else if step.value.contains("must") || step.value.contains("always") {
} else if step.value.contains("might") || step.value.contains("maybe") {
issues.push(ValidationIssue {
location: format!("Step: {}", step.value),
category: IssueCategory::Convention,
severity: Severity::Warning,
message: "Uncertain language in step ('might', 'maybe')".to_string(),
suggestion: Some("Use definite language for testable assertions".to_string()),
});
}
}
issues
}
}
pub struct SpecGenerator {
backend: Arc<dyn DynChatBackend>,
}
impl std::fmt::Debug for SpecGenerator {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("SpecGenerator").finish_non_exhaustive()
}
}
impl SpecGenerator {
#[must_use]
pub fn new(backend: Arc<dyn DynChatBackend>) -> Self {
Self { backend }
}
pub async fn generate_from_text(&self, text: &str) -> Result<String, ValidationError> {
let prompt = format!(
r"You are a requirements engineer for a multi-agent AI system called Converge.
Convert the following free text into a valid Gherkin/Truth specification.
Free Text:
{text}
Rules for generation:
1. Use Converge Truth syntax (`Truth:` instead of `Feature:`).
2. Include a concise business description immediately after the Truth header.
3. Ensure at least one scenario is generated.
4. Each scenario must have Given/When/Then steps.
5. Use definite language (avoid 'might', 'maybe').
6. Focus on testable business outcomes.
Return ONLY the Gherkin content, no explanation or preamble.
Example Format:
Truth: <name>
<description line 1>
<description line 2>
Scenario: <name>
Given <state>
When <action>
Then <outcome>"
);
let system_prompt =
"You are an expert Gherkin spec writer. Respond with ONLY the specification.";
let response = chat(&self.backend, system_prompt, &prompt, Some(1000), Some(0.3)).await?;
Ok(response.content.trim().to_string())
}
}
async fn chat(
backend: &Arc<dyn DynChatBackend>,
system: &str,
user_prompt: &str,
max_tokens: Option<u32>,
temperature: Option<f32>,
) -> Result<ChatResponse, ValidationError> {
let request = ChatRequest {
messages: vec![ChatMessage {
role: ChatRole::User,
content: user_prompt.to_string(),
tool_calls: Vec::new(),
tool_call_id: None,
}],
system: Some(system.to_string()),
tools: Vec::new(),
response_format: ResponseFormat::default(),
max_tokens,
temperature,
stop_sequences: Vec::new(),
model: None,
};
backend.chat(request).await.map_err(|e| {
ValidationError::LlmError(format!("NOT_RELATED_ERROR: LLM API call failed: {e}"))
})
}
fn format_steps(steps: &[gherkin::Step]) -> String {
steps
.iter()
.map(|s| format!("{:?} {}", s.keyword, s.value))
.collect::<Vec<_>>()
.join("\n")
}
#[derive(Debug, Clone)]
pub enum ValidationError {
ParseError(String),
IoError(String),
LlmError(String),
}
impl std::fmt::Display for ValidationError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::ParseError(msg) => write!(f, "Parse error: {msg}"),
Self::IoError(msg) => write!(f, "IO error: {msg}"),
Self::LlmError(msg) => write!(f, "LLM error: {msg}"),
}
}
}
impl std::error::Error for ValidationError {}
#[cfg(test)]
mod tests {
use super::*;
use crate::mock_llm::StaticChatBackend;
fn mock_valid_backend() -> Arc<dyn DynChatBackend> {
Arc::new(StaticChatBackend::queued([
"VALID",
"COMPILABLE: Acceptance - check strategy count",
]))
}
#[test]
fn preprocess_converts_truth_to_feature() {
let input = "Truth: Get paid for delivered work\n Scenario: Invoice";
let output = preprocess_truths(input);
assert!(output.starts_with("Feature:"));
assert!(output.contains("Scenario: Invoice"));
}
#[test]
fn preprocess_preserves_feature_keyword() {
let input = "Feature: Standard Gherkin\n Scenario: Test";
let output = preprocess_truths(input);
assert_eq!(input, output);
}
#[test]
fn validation_config_default() {
let config = ValidationConfig::default();
assert!(config.check_conventions);
assert!(config.check_business_sense);
assert!(config.check_compilability);
assert!((config.min_confidence - 0.7).abs() < f64::EPSILON);
}
#[test]
fn validation_config_custom() {
let config = ValidationConfig {
check_business_sense: false,
min_confidence: 0.9,
..ValidationConfig::default()
};
assert!(!config.check_business_sense);
assert!((config.min_confidence - 0.9).abs() < f64::EPSILON);
assert!(config.check_conventions);
}
#[tokio::test]
async fn validates_truth_syntax() {
let content = r"
Truth: Get paid for delivered work
Scenario: Invoice and collect
Given work is marked as delivered
When the system converges
Then invoice is issued
";
let validator = GherkinValidator::new(mock_valid_backend(), ValidationConfig::default());
let result = validator.validate(content, "money.truth").await.unwrap();
assert_eq!(result.scenario_count, 1);
}
#[tokio::test]
async fn validates_truth_with_governance_blocks() {
let content = r#"
Truth: Governed release
Intent:
Outcome: "Ship safely"
Authority:
Actor: release_manager
Requires Approval: security_owner
Evidence:
Requires: test_report
Scenario: Approval is required before release
Given a release candidate exists
When the system converges
Then deployment MUST NOT occur without approval
"#;
let validator = GherkinValidator::new(mock_valid_backend(), ValidationConfig::default());
let result = validator.validate(content, "release.truths").await.unwrap();
assert_eq!(result.scenario_count, 1);
assert_eq!(
result.governance.authority.unwrap().requires_approval,
vec!["security_owner".to_string()]
);
assert!(result.governance.intent.is_some());
}
#[tokio::test]
async fn validates_simple_feature() {
let content = r"
Feature: Growth Strategy Validation
Scenario: Multiple strategies required
When the system converges
Then at least two distinct growth strategies exist
";
let validator = GherkinValidator::new(mock_valid_backend(), ValidationConfig::default());
let result = validator.validate(content, "test.feature").await.unwrap();
assert_eq!(result.scenario_count, 1);
}
#[tokio::test]
async fn detects_missing_then() {
let content = r"
Feature: Bad Spec
Scenario: No assertions
Given some precondition
When something happens
";
let validator = GherkinValidator::new(
mock_valid_backend(),
ValidationConfig {
check_business_sense: false,
check_compilability: false,
check_conventions: true,
min_confidence: 0.7,
},
);
let result = validator.validate(content, "bad.feature").await.unwrap();
assert!(result.has_errors());
assert!(
result
.issues
.iter()
.any(|i| i.category == IssueCategory::Convention && i.message.contains("Then"))
);
}
#[tokio::test]
async fn detects_uncertain_language() {
let content = r"
Feature: Uncertain Spec
Scenario: Maybe works
When something happens
Then it might succeed
";
let validator = GherkinValidator::new(
mock_valid_backend(),
ValidationConfig {
check_business_sense: false,
check_compilability: false,
check_conventions: true,
min_confidence: 0.7,
},
);
let result = validator
.validate(content, "uncertain.feature")
.await
.unwrap();
assert!(result.has_warnings());
assert!(result.issues.iter().any(|i| i.message.contains("might")));
}
#[tokio::test]
async fn handles_llm_invalid_response() {
let backend: Arc<dyn DynChatBackend> = Arc::new(StaticChatBackend::queued([
"INVALID: The scenario describes an untestable state",
"COMPILABLE: Acceptance",
]));
let content = r"
Feature: Test
Scenario: Bad business logic
When magic happens
Then everything is perfect forever
";
let validator = GherkinValidator::new(backend, ValidationConfig::default());
let result = validator.validate(content, "test.feature").await.unwrap();
assert!(
result.issues.iter().any(
|i| i.category == IssueCategory::BusinessSense && i.severity == Severity::Error
)
);
}
#[tokio::test]
async fn generates_spec_from_text() {
let mock_spec = "Truth: Test\n Scenario: Test\n Given X\n Then Y";
let backend: Arc<dyn DynChatBackend> = Arc::new(StaticChatBackend::queued([mock_spec]));
let generator = SpecGenerator::new(backend);
let result = generator
.generate_from_text("Make a test spec")
.await
.unwrap();
assert_eq!(result, mock_spec);
}
#[test]
fn extract_invariant_structural_tags() {
let tags = vec![
"invariant".to_string(),
"structural".to_string(),
"id:brand_safety".to_string(),
];
let meta = extract_scenario_meta("Strategies must not contain brand-unsafe terms", &tags);
assert_eq!(meta.kind, Some(ScenarioKind::Invariant));
assert_eq!(meta.invariant_class, Some(InvariantClassTag::Structural));
assert_eq!(meta.id.as_deref(), Some("brand_safety"));
assert!(!meta.is_test);
assert!(meta.provider.is_none());
}
#[test]
fn extract_invariant_acceptance_tags() {
let tags = vec![
"invariant".to_string(),
"acceptance".to_string(),
"id:require_multiple_strategies".to_string(),
];
let meta = extract_scenario_meta("At least 2 strategies must exist", &tags);
assert_eq!(meta.kind, Some(ScenarioKind::Invariant));
assert_eq!(meta.invariant_class, Some(InvariantClassTag::Acceptance));
assert_eq!(meta.id.as_deref(), Some("require_multiple_strategies"));
}
#[test]
fn extract_invariant_semantic_tags() {
let tags = vec![
"invariant".to_string(),
"semantic".to_string(),
"id:require_evaluation_rationale".to_string(),
];
let meta = extract_scenario_meta("Evaluations must include score", &tags);
assert_eq!(meta.kind, Some(ScenarioKind::Invariant));
assert_eq!(meta.invariant_class, Some(InvariantClassTag::Semantic));
assert_eq!(meta.id.as_deref(), Some("require_evaluation_rationale"));
}
#[test]
fn extract_validation_tags() {
let tags = vec![
"validation".to_string(),
"id:confidence_threshold".to_string(),
];
let meta = extract_scenario_meta("Proposals must meet confidence threshold", &tags);
assert_eq!(meta.kind, Some(ScenarioKind::Validation));
assert!(meta.invariant_class.is_none());
assert_eq!(meta.id.as_deref(), Some("confidence_threshold"));
}
#[test]
fn extract_agent_llm_tags() {
let tags = vec![
"agent".to_string(),
"llm".to_string(),
"id:market_signal".to_string(),
];
let meta = extract_scenario_meta("Market Signal agent proposes Signals", &tags);
assert_eq!(meta.kind, Some(ScenarioKind::Suggestor));
assert_eq!(meta.provider.as_deref(), Some("llm"));
assert_eq!(meta.id.as_deref(), Some("market_signal"));
}
#[test]
fn extract_e2e_test_tags() {
let tags = vec!["e2e".to_string(), "test".to_string()];
let meta = extract_scenario_meta("Pack converges from Seeds", &tags);
assert_eq!(meta.kind, Some(ScenarioKind::EndToEnd));
assert!(meta.is_test);
assert!(meta.id.is_none());
}
#[test]
fn extract_with_at_prefix() {
let tags = vec![
"@invariant".to_string(),
"@structural".to_string(),
"@id:brand_safety".to_string(),
];
let meta = extract_scenario_meta("Test with @ prefix", &tags);
assert_eq!(meta.kind, Some(ScenarioKind::Invariant));
assert_eq!(meta.invariant_class, Some(InvariantClassTag::Structural));
assert_eq!(meta.id.as_deref(), Some("brand_safety"));
}
#[test]
fn extract_no_tags() {
let meta = extract_scenario_meta("Untagged scenario", &[]);
assert!(meta.kind.is_none());
assert!(meta.invariant_class.is_none());
assert!(meta.id.is_none());
assert!(!meta.is_test);
}
#[test]
fn extract_unknown_tags_preserved() {
let tags = vec![
"custom".to_string(),
"invariant".to_string(),
"id:test".to_string(),
];
let meta = extract_scenario_meta("With custom tag", &tags);
assert_eq!(meta.raw_tags.len(), 3);
assert_eq!(meta.kind, Some(ScenarioKind::Invariant));
}
#[test]
fn extract_all_metas_from_truth_file() {
let content = r#"
Truth: Growth Strategy Pack
Multi-agent growth strategy analysis.
@invariant @structural @id:brand_safety
Scenario: Strategies must not contain brand-unsafe terms
Given any fact under key "Strategies"
Then it must not contain forbidden terms
@invariant @acceptance @id:require_multiple_strategies
Scenario: At least 2 strategies must exist at convergence
Given the engine halts with reason "Converged"
Then the Context key "Strategies" contains at least 2 facts
@agent @llm @id:market_signal
Scenario: Market Signal agent proposes Signals from Seeds
Given the Context contains facts under key "Seeds"
When agent "market_signal" executes
Then it proposes facts under key "Signals"
@e2e @test
Scenario: Pack converges from Seeds to evaluated Strategies
Given seed facts are present
When the pack runs to convergence
Then all invariants pass
"#;
let metas = extract_all_metas(content).unwrap();
assert_eq!(metas.len(), 4);
assert_eq!(metas[0].kind, Some(ScenarioKind::Invariant));
assert_eq!(
metas[0].invariant_class,
Some(InvariantClassTag::Structural)
);
assert_eq!(metas[0].id.as_deref(), Some("brand_safety"));
assert_eq!(metas[1].kind, Some(ScenarioKind::Invariant));
assert_eq!(
metas[1].invariant_class,
Some(InvariantClassTag::Acceptance)
);
assert_eq!(metas[1].id.as_deref(), Some("require_multiple_strategies"));
assert_eq!(metas[2].kind, Some(ScenarioKind::Suggestor));
assert_eq!(metas[2].provider.as_deref(), Some("llm"));
assert_eq!(metas[3].kind, Some(ScenarioKind::EndToEnd));
assert!(metas[3].is_test);
}
#[tokio::test]
async fn validator_populates_scenario_metas() {
let content = r"
Truth: Test
@invariant @structural @id:test_inv
Scenario: Test invariant
Given precondition
When action occurs
Then outcome is verified
";
let validator = GherkinValidator::new(mock_valid_backend(), ValidationConfig::default());
let result = validator.validate(content, "test.truth").await.unwrap();
assert_eq!(result.scenario_metas.len(), 1);
assert_eq!(result.scenario_metas[0].kind, Some(ScenarioKind::Invariant));
assert_eq!(result.scenario_metas[0].id.as_deref(), Some("test_inv"));
}
#[test]
fn extract_meta_invariant_without_class() {
let tags = vec!["invariant".to_string(), "id:no_class".to_string()];
let meta = extract_scenario_meta("Invariant without class", &tags);
assert_eq!(meta.kind, Some(ScenarioKind::Invariant));
assert!(meta.invariant_class.is_none()); }
#[test]
fn extract_meta_class_without_kind() {
let tags = vec!["structural".to_string()];
let meta = extract_scenario_meta("Orphan class", &tags);
assert!(meta.kind.is_none());
assert_eq!(meta.invariant_class, Some(InvariantClassTag::Structural));
}
#[test]
fn extract_meta_empty_id() {
let tags = vec!["invariant".to_string(), "id:".to_string()];
let meta = extract_scenario_meta("Empty id", &tags);
assert_eq!(meta.id.as_deref(), Some(""));
}
#[test]
fn extract_all_metas_parse_error() {
let bad = "This is not valid Gherkin at all";
let result = extract_all_metas(bad);
assert!(result.is_err());
}
#[tokio::test]
async fn valid_spec_produces_no_errors() {
let content = r"
Truth: Reliable Invoicing
Ensure invoices are generated promptly.
Scenario: Invoice generated on delivery
Given work is marked as delivered
When the billing cycle runs
Then an invoice must be created
";
let validator = GherkinValidator::new(mock_valid_backend(), ValidationConfig::default());
let result = validator.validate(content, "invoice.truths").await.unwrap();
assert!(result.is_valid);
assert!(!result.has_errors());
assert_eq!(result.scenario_count, 1);
}
#[tokio::test]
async fn conventions_only_mode_flags_missing_then() {
let content = r"
Feature: Incomplete
Scenario: No outcome defined
Given a precondition
When an action occurs
";
let config = ValidationConfig {
check_business_sense: false,
check_compilability: false,
check_conventions: true,
min_confidence: 0.7,
};
let backend: Arc<dyn DynChatBackend> = Arc::new(StaticChatBackend::constant("VALID"));
let validator = GherkinValidator::new(backend, config);
let result = validator
.validate(content, "no_then.feature")
.await
.unwrap();
assert!(!result.is_valid);
assert!(result.has_errors());
let then_issue = result
.issues
.iter()
.find(|i| i.message.contains("Then"))
.unwrap();
assert_eq!(then_issue.severity, Severity::Error);
assert_eq!(then_issue.category, IssueCategory::Convention);
}
#[tokio::test]
async fn llm_unclear_response_produces_warning() {
let backend: Arc<dyn DynChatBackend> = Arc::new(StaticChatBackend::queued([
"UNCLEAR: What does 'delivered' mean in this context?",
"COMPILABLE: Structural",
]));
let content = r"
Feature: Ambiguous
Scenario: Vague delivery check
Given work is delivered
When system runs
Then it should be invoiced
";
let validator = GherkinValidator::new(backend, ValidationConfig::default());
let result = validator
.validate(content, "ambiguous.feature")
.await
.unwrap();
let unclear_issue = result
.issues
.iter()
.find(|i| i.category == IssueCategory::BusinessSense)
.unwrap();
assert_eq!(unclear_issue.severity, Severity::Warning);
assert!(unclear_issue.message.contains("Ambiguous"));
assert!(unclear_issue.suggestion.is_some());
}
#[tokio::test]
async fn llm_not_compilable_response_produces_error() {
let backend: Arc<dyn DynChatBackend> = Arc::new(StaticChatBackend::queued([
"VALID",
"NOT_COMPILABLE: Requires external API call at runtime",
]));
let content = r"
Feature: External Check
Scenario: Requires network
Given an external API is available
When the system checks status
Then the response must be 200
";
let validator = GherkinValidator::new(backend, ValidationConfig::default());
let result = validator
.validate(content, "external.feature")
.await
.unwrap();
let compile_issue = result
.issues
.iter()
.find(|i| i.category == IssueCategory::Compilability)
.unwrap();
assert_eq!(compile_issue.severity, Severity::Error);
assert!(compile_issue.message.contains("Cannot compile"));
}
#[tokio::test]
async fn llm_needs_refactor_response_produces_warning() {
let backend: Arc<dyn DynChatBackend> = Arc::new(StaticChatBackend::queued([
"VALID",
"NEEDS_REFACTOR: Split into two scenarios for testability",
]));
let content = r"
Feature: Refactor Needed
Scenario: Complex check
Given multiple preconditions hold
When the system converges
Then all outcomes must be satisfied
";
let validator = GherkinValidator::new(backend, ValidationConfig::default());
let result = validator
.validate(content, "refactor.feature")
.await
.unwrap();
let refactor_issue = result
.issues
.iter()
.find(|i| i.category == IssueCategory::Compilability)
.unwrap();
assert_eq!(refactor_issue.severity, Severity::Warning);
assert!(refactor_issue.suggestion.is_some());
}
#[tokio::test]
async fn empty_feature_produces_error() {
let backend: Arc<dyn DynChatBackend> = Arc::new(StaticChatBackend::constant("VALID"));
let content = r"
Feature: Empty
";
let config = ValidationConfig {
check_business_sense: false,
check_compilability: false,
check_conventions: true,
min_confidence: 0.7,
};
let validator = GherkinValidator::new(backend, config);
let result = validator.validate(content, "empty.feature").await.unwrap();
assert!(!result.is_valid);
let empty_issue = result
.issues
.iter()
.find(|i| i.message.contains("no scenarios"))
.unwrap();
assert_eq!(empty_issue.severity, Severity::Error);
}
#[tokio::test]
async fn missing_description_produces_warning() {
let backend: Arc<dyn DynChatBackend> = Arc::new(StaticChatBackend::queued([
"VALID",
"COMPILABLE: Structural",
]));
let content = r"
Feature: No Description
Scenario: Basic
Given X
When Y
Then Z
";
let validator = GherkinValidator::new(backend, ValidationConfig::default());
let result = validator.validate(content, "nodesc.feature").await.unwrap();
assert!(result.has_warnings());
assert!(
result
.issues
.iter()
.any(|i| i.message.contains("lacks a description"))
);
}
#[tokio::test]
async fn approval_without_authority_produces_warning() {
let backend: Arc<dyn DynChatBackend> = Arc::new(StaticChatBackend::queued([
"VALID",
"COMPILABLE: Acceptance",
]));
let content = r"
Truth: Release Process
Manages release approvals.
Scenario: Approval required
Given a release candidate exists
When approval is requested
Then deployment must be approved
";
let validator = GherkinValidator::new(backend, ValidationConfig::default());
let result = validator.validate(content, "release.truths").await.unwrap();
assert!(
result
.issues
.iter()
.any(|i| i.message.contains("Authority block"))
);
}
#[tokio::test]
async fn missing_intent_produces_warning() {
let backend: Arc<dyn DynChatBackend> = Arc::new(StaticChatBackend::queued([
"VALID",
"COMPILABLE: Structural",
]));
let content = r"
Truth: No Intent
Description only.
Scenario: Something happens
Given precondition
When action
Then outcome
";
let validator = GherkinValidator::new(backend, ValidationConfig::default());
let result = validator
.validate(content, "nointent.truths")
.await
.unwrap();
assert!(
result
.issues
.iter()
.any(|i| i.message.contains("Intent block"))
);
}
#[tokio::test]
async fn validate_file_nonexistent_returns_io_error() {
let backend: Arc<dyn DynChatBackend> = Arc::new(StaticChatBackend::constant("VALID"));
let validator = GherkinValidator::new(backend, ValidationConfig::default());
let result = validator.validate_file("/nonexistent/path.truths").await;
assert!(result.is_err());
let err = result.unwrap_err();
assert!(matches!(err, ValidationError::IoError(_)));
}
#[tokio::test]
async fn parse_error_on_invalid_gherkin() {
let backend: Arc<dyn DynChatBackend> = Arc::new(StaticChatBackend::constant("VALID"));
let validator = GherkinValidator::new(backend, ValidationConfig::default());
let result = validator
.validate("not valid gherkin content", "bad.feature")
.await;
assert!(result.is_err());
assert!(matches!(
result.unwrap_err(),
ValidationError::ParseError(_)
));
}
#[tokio::test]
async fn generate_from_text_returns_trimmed_content() {
let expected = "Truth: Payment\n Scenario: Pay on delivery\n Given work done\n Then invoice sent";
let backend: Arc<dyn DynChatBackend> =
Arc::new(StaticChatBackend::constant(format!(" {expected} \n")));
let generator = SpecGenerator::new(backend);
let result = generator
.generate_from_text("We need to pay for delivered work")
.await
.unwrap();
assert_eq!(result, expected);
}
#[tokio::test]
async fn generate_from_empty_text_still_calls_backend() {
let backend: Arc<dyn DynChatBackend> = Arc::new(StaticChatBackend::constant(
"Truth: Empty\n Scenario: Noop\n Then nothing",
));
let generator = SpecGenerator::new(backend);
let result = generator.generate_from_text("").await.unwrap();
assert!(result.starts_with("Truth:"));
}
#[test]
fn spec_validation_summary_valid() {
let v = SpecValidation {
is_valid: true,
file_path: "test.truths".to_string(),
scenario_count: 3,
issues: vec![ValidationIssue {
location: "Feature".to_string(),
category: IssueCategory::Convention,
severity: Severity::Warning,
message: "some warning".to_string(),
suggestion: None,
}],
confidence: 0.9,
scenario_metas: vec![],
governance: TruthGovernance::default(),
};
let summary = v.summary();
assert!(summary.contains("3 scenarios"));
assert!(summary.contains("1 warnings"));
assert!(summary.contains("test.truths"));
}
#[test]
fn spec_validation_summary_invalid() {
let v = SpecValidation {
is_valid: false,
file_path: "bad.feature".to_string(),
scenario_count: 1,
issues: vec![
ValidationIssue {
location: "Scenario".to_string(),
category: IssueCategory::Syntax,
severity: Severity::Error,
message: "broken".to_string(),
suggestion: None,
},
ValidationIssue {
location: "Feature".to_string(),
category: IssueCategory::Convention,
severity: Severity::Warning,
message: "meh".to_string(),
suggestion: None,
},
],
confidence: 0.5,
scenario_metas: vec![],
governance: TruthGovernance::default(),
};
let summary = v.summary();
assert!(summary.contains("1 errors"));
assert!(summary.contains("1 warnings"));
assert!(summary.contains("bad.feature"));
}
#[test]
fn validation_error_display() {
let parse = ValidationError::ParseError("bad syntax".into());
assert_eq!(format!("{parse}"), "Parse error: bad syntax");
let io = ValidationError::IoError("not found".into());
assert_eq!(format!("{io}"), "IO error: not found");
let llm = ValidationError::LlmError("timeout".into());
assert_eq!(format!("{llm}"), "LLM error: timeout");
}
#[test]
fn validator_debug_format() {
let backend: Arc<dyn DynChatBackend> = Arc::new(StaticChatBackend::constant("VALID"));
let validator = GherkinValidator::new(backend, ValidationConfig::default());
let debug = format!("{validator:?}");
assert!(debug.contains("GherkinValidator"));
assert!(debug.contains("config"));
}
#[test]
fn spec_generator_debug_format() {
let backend: Arc<dyn DynChatBackend> = Arc::new(StaticChatBackend::constant(""));
let generator = SpecGenerator::new(backend);
let debug = format!("{generator:?}");
assert!(debug.contains("SpecGenerator"));
}
#[tokio::test]
async fn maybe_language_flagged() {
let content = r"
Feature: Uncertain
Scenario: Perhaps works
Given something
When maybe it runs
Then it should work
";
let config = ValidationConfig {
check_business_sense: false,
check_compilability: false,
check_conventions: true,
min_confidence: 0.7,
};
let backend: Arc<dyn DynChatBackend> = Arc::new(StaticChatBackend::constant("VALID"));
let validator = GherkinValidator::new(backend, config);
let result = validator.validate(content, "maybe.feature").await.unwrap();
assert!(result.issues.iter().any(|i| i.message.contains("maybe")));
}
#[tokio::test]
async fn scenario_without_name_flagged() {
let content = "Feature: Test\n Scenario:\n Given X\n Then Y\n";
let config = ValidationConfig {
check_business_sense: false,
check_compilability: false,
check_conventions: true,
min_confidence: 0.7,
};
let backend: Arc<dyn DynChatBackend> = Arc::new(StaticChatBackend::constant("VALID"));
let validator = GherkinValidator::new(backend, config);
let result = validator.validate(content, "noname.feature").await.unwrap();
assert!(result.issues.iter().any(|i| i.message.contains("no name")));
}
mod property_tests {
use super::*;
use proptest::prelude::*;
proptest! {
#[test]
fn preprocess_never_crashes(s in "\\PC*") {
let _ = preprocess_truths(&s);
}
#[test]
fn truth_to_feature_conversion(s in ".*Truth:.*") {
let _output = preprocess_truths(&s);
}
#[test]
fn idempotency_of_feature(s in ".*Feature:.*") {
if !s.contains("Truth:") {
let output = preprocess_truths(&s);
assert_eq!(s, output);
}
}
#[test]
fn extract_meta_never_crashes(
name in "\\PC{0,100}",
tags in proptest::collection::vec("[a-z:_@]{1,30}", 0..10)
) {
let _ = extract_scenario_meta(&name, &tags);
}
#[test]
fn extract_meta_preserves_all_raw_tags(
tags in proptest::collection::vec("[a-z]{1,10}", 0..5)
) {
let meta = extract_scenario_meta("test", &tags);
assert_eq!(meta.raw_tags.len(), tags.len());
}
#[test]
fn extract_meta_id_always_from_id_prefix(
suffix in "[a-z_]{1,20}"
) {
let tags = vec![format!("id:{suffix}")];
let meta = extract_scenario_meta("test", &tags);
assert_eq!(meta.id.as_deref(), Some(suffix.as_str()));
}
}
}
}