converge_optimization/packs/
testing.rs

1//! Test harness for pack scenarios
2
3use converge_pack::Pack;
4use converge_pack::gate::{GateDecision, GateResult, ProblemSpec};
5
6/// Test scenario for a pack
7#[derive(Debug, Clone)]
8pub struct TestScenario {
9    /// Scenario name
10    pub name: String,
11    /// Description of what's being tested
12    pub description: String,
13    /// Problem specification for this scenario
14    pub spec: ProblemSpec,
15    /// Expected outcome
16    pub expected: ExpectedOutcome,
17}
18
19impl TestScenario {
20    /// Create a new test scenario
21    pub fn new(
22        name: impl Into<String>,
23        description: impl Into<String>,
24        spec: ProblemSpec,
25        expected: ExpectedOutcome,
26    ) -> Self {
27        Self {
28            name: name.into(),
29            description: description.into(),
30            spec,
31            expected,
32        }
33    }
34
35    /// Create a feasibility test (expects solution)
36    pub fn feasible(
37        name: impl Into<String>,
38        description: impl Into<String>,
39        spec: ProblemSpec,
40        min_confidence: f64,
41    ) -> Self {
42        Self::new(
43            name,
44            description,
45            spec,
46            ExpectedOutcome::Feasible {
47                min_confidence,
48                required_invariants: Vec::new(),
49            },
50        )
51    }
52
53    /// Create an infeasibility test (expects no solution)
54    pub fn infeasible(
55        name: impl Into<String>,
56        description: impl Into<String>,
57        spec: ProblemSpec,
58        expected_violations: Vec<String>,
59    ) -> Self {
60        Self::new(
61            name,
62            description,
63            spec,
64            ExpectedOutcome::Infeasible {
65                expected_violations,
66            },
67        )
68    }
69}
70
71/// Expected outcome of a test scenario
72#[derive(Debug, Clone)]
73pub enum ExpectedOutcome {
74    /// Should find a feasible solution
75    Feasible {
76        /// Minimum confidence score expected
77        min_confidence: f64,
78        /// Invariants that must pass
79        required_invariants: Vec<String>,
80    },
81    /// Should report infeasible
82    Infeasible {
83        /// Expected constraint violations
84        expected_violations: Vec<String>,
85    },
86    /// Should match specific gate decision
87    GateDecision {
88        /// Expected decision
89        decision: GateDecision,
90    },
91    /// Should produce deterministic output
92    Deterministic {
93        /// Expected output hash or value
94        expected_output: serde_json::Value,
95    },
96}
97
98/// Result of running a test scenario
99#[derive(Debug)]
100pub struct ScenarioResult {
101    /// Scenario name
102    pub name: String,
103    /// Whether the test passed
104    pub passed: bool,
105    /// Error message if failed
106    pub error: Option<String>,
107    /// Actual confidence (if applicable)
108    pub actual_confidence: Option<f64>,
109    /// Actual gate decision (if applicable)
110    pub actual_decision: Option<GateDecision>,
111    /// Duration in milliseconds
112    pub duration_ms: f64,
113}
114
115impl ScenarioResult {
116    /// Create a passing result
117    pub fn pass(name: impl Into<String>, duration_ms: f64) -> Self {
118        Self {
119            name: name.into(),
120            passed: true,
121            error: None,
122            actual_confidence: None,
123            actual_decision: None,
124            duration_ms,
125        }
126    }
127
128    /// Create a failing result
129    pub fn fail(name: impl Into<String>, error: impl Into<String>, duration_ms: f64) -> Self {
130        Self {
131            name: name.into(),
132            passed: false,
133            error: Some(error.into()),
134            actual_confidence: None,
135            actual_decision: None,
136            duration_ms,
137        }
138    }
139
140    /// Add actual confidence to result
141    pub fn with_confidence(mut self, confidence: f64) -> Self {
142        self.actual_confidence = Some(confidence);
143        self
144    }
145
146    /// Add actual decision to result
147    pub fn with_decision(mut self, decision: GateDecision) -> Self {
148        self.actual_decision = Some(decision);
149        self
150    }
151}
152
153/// Run a single test scenario against a pack
154pub fn run_scenario(pack: &dyn Pack, scenario: &TestScenario) -> ScenarioResult {
155    let start = std::time::Instant::now();
156
157    // Solve the problem
158    let solve_result = match pack.solve(&scenario.spec) {
159        Ok(result) => result,
160        Err(e) => {
161            // Check if we expected infeasibility
162            if let ExpectedOutcome::Infeasible { .. } = &scenario.expected {
163                return ScenarioResult::pass(
164                    &scenario.name,
165                    start.elapsed().as_secs_f64() * 1000.0,
166                );
167            }
168            return ScenarioResult::fail(
169                &scenario.name,
170                format!("Solve failed: {}", e),
171                start.elapsed().as_secs_f64() * 1000.0,
172            );
173        }
174    };
175
176    // Check invariants
177    let invariant_results = match pack.check_invariants(&solve_result.plan) {
178        Ok(results) => results,
179        Err(e) => {
180            return ScenarioResult::fail(
181                &scenario.name,
182                format!("Invariant check failed: {}", e),
183                start.elapsed().as_secs_f64() * 1000.0,
184            );
185        }
186    };
187
188    // Evaluate gate
189    let gate = pack.evaluate_gate(&solve_result.plan, &invariant_results);
190    let duration_ms = start.elapsed().as_secs_f64() * 1000.0;
191
192    // Check against expected outcome
193    match &scenario.expected {
194        ExpectedOutcome::Feasible {
195            min_confidence,
196            required_invariants,
197        } => {
198            if solve_result.plan.confidence() < *min_confidence {
199                return ScenarioResult::fail(
200                    &scenario.name,
201                    format!(
202                        "Confidence too low: {} < {}",
203                        solve_result.plan.confidence(),
204                        min_confidence
205                    ),
206                    duration_ms,
207                )
208                .with_confidence(solve_result.plan.confidence());
209            }
210
211            // Check required invariants passed
212            for required in required_invariants {
213                let result = invariant_results.iter().find(|r| &r.invariant == required);
214                match result {
215                    Some(r) if !r.passed => {
216                        return ScenarioResult::fail(
217                            &scenario.name,
218                            format!("Required invariant '{}' failed", required),
219                            duration_ms,
220                        );
221                    }
222                    None => {
223                        return ScenarioResult::fail(
224                            &scenario.name,
225                            format!("Required invariant '{}' not found", required),
226                            duration_ms,
227                        );
228                    }
229                    _ => {}
230                }
231            }
232
233            ScenarioResult::pass(&scenario.name, duration_ms)
234                .with_confidence(solve_result.plan.confidence())
235                .with_decision(gate.decision)
236        }
237
238        ExpectedOutcome::Infeasible {
239            expected_violations,
240        } => {
241            // Should not have found a feasible solution
242            if solve_result.is_feasible() && gate.is_promoted() {
243                return ScenarioResult::fail(
244                    &scenario.name,
245                    "Expected infeasible but found solution",
246                    duration_ms,
247                );
248            }
249
250            // Check for expected violations
251            for expected in expected_violations {
252                let has_violation = invariant_results
253                    .iter()
254                    .any(|r| !r.passed && r.invariant == *expected);
255                if !has_violation {
256                    return ScenarioResult::fail(
257                        &scenario.name,
258                        format!("Expected violation '{}' not found", expected),
259                        duration_ms,
260                    );
261                }
262            }
263
264            ScenarioResult::pass(&scenario.name, duration_ms).with_decision(gate.decision)
265        }
266
267        ExpectedOutcome::GateDecision { decision } => {
268            if gate.decision != *decision {
269                return ScenarioResult::fail(
270                    &scenario.name,
271                    format!("Expected {:?} but got {:?}", decision, gate.decision),
272                    duration_ms,
273                )
274                .with_decision(gate.decision);
275            }
276            ScenarioResult::pass(&scenario.name, duration_ms).with_decision(gate.decision)
277        }
278
279        ExpectedOutcome::Deterministic { expected_output } => {
280            if solve_result.plan.plan != *expected_output {
281                return ScenarioResult::fail(
282                    &scenario.name,
283                    format!(
284                        "Output mismatch: expected {:?}, got {:?}",
285                        expected_output, solve_result.plan.plan
286                    ),
287                    duration_ms,
288                );
289            }
290            ScenarioResult::pass(&scenario.name, duration_ms)
291        }
292    }
293}
294
295/// Run all scenarios for a pack
296pub fn run_all_scenarios(pack: &dyn Pack, scenarios: &[TestScenario]) -> Vec<ScenarioResult> {
297    scenarios.iter().map(|s| run_scenario(pack, s)).collect()
298}
299
300/// Summary of scenario results
301#[derive(Debug)]
302pub struct ScenarioSummary {
303    /// Total scenarios run
304    pub total: usize,
305    /// Scenarios passed
306    pub passed: usize,
307    /// Scenarios failed
308    pub failed: usize,
309    /// Total duration in milliseconds
310    pub total_duration_ms: f64,
311}
312
313impl ScenarioSummary {
314    /// Create from results
315    pub fn from_results(results: &[ScenarioResult]) -> Self {
316        let passed = results.iter().filter(|r| r.passed).count();
317        let total_duration_ms: f64 = results.iter().map(|r| r.duration_ms).sum();
318
319        Self {
320            total: results.len(),
321            passed,
322            failed: results.len() - passed,
323            total_duration_ms,
324        }
325    }
326
327    /// Check if all scenarios passed
328    pub fn all_passed(&self) -> bool {
329        self.failed == 0
330    }
331}
332
333/// Helper to create a minimal problem spec for testing
334pub fn test_problem_spec(_pack_name: &str, inputs: serde_json::Value) -> GateResult<ProblemSpec> {
335    use converge_pack::gate::ObjectiveSpec;
336
337    ProblemSpec::builder(format!("test-{}", test_id()), "test-tenant")
338        .objective(ObjectiveSpec::maximize("score"))
339        .inputs_raw(inputs)
340        .seed(42) // Fixed seed for determinism
341        .build()
342}
343
344fn test_id() -> String {
345    static COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
346    let id = COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
347    format!("{id:032x}")
348}
349
350#[cfg(test)]
351mod tests {
352    use super::*;
353
354    #[test]
355    fn test_scenario_result() {
356        let pass = ScenarioResult::pass("test", 10.0);
357        assert!(pass.passed);
358
359        let fail = ScenarioResult::fail("test", "error", 10.0);
360        assert!(!fail.passed);
361        assert_eq!(fail.error, Some("error".to_string()));
362    }
363
364    #[test]
365    fn test_scenario_summary() {
366        let results = vec![
367            ScenarioResult::pass("test1", 10.0),
368            ScenarioResult::pass("test2", 20.0),
369            ScenarioResult::fail("test3", "error", 5.0),
370        ];
371
372        let summary = ScenarioSummary::from_results(&results);
373        assert_eq!(summary.total, 3);
374        assert_eq!(summary.passed, 2);
375        assert_eq!(summary.failed, 1);
376        assert!(!summary.all_passed());
377    }
378
379    #[test]
380    fn test_test_problem_spec() {
381        let spec = test_problem_spec("test-pack", serde_json::json!({"key": "value"})).unwrap();
382
383        assert!(spec.problem_id.starts_with("test-"));
384        assert_eq!(spec.tenant_scope, "test-tenant");
385        assert_eq!(spec.seed(), 42);
386    }
387}
converge_optimization/packs/testing.rs

converge_optimization/packs/
testing.rs
