icebox/ai/
validator.rs

1use crate::core::framework::SharedFramework;
2use crate::core::safety::{now_secs, RiskLevel};
3
4use crate::ai::agent::Planner;
5use crate::ai::orchestrator::{CampaignReport, Orchestrator};
6
7#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
8pub struct ValidationReport {
9    pub ran_at: u64,
10    pub policy_version: u64,
11    pub campaign: CampaignReport,
12}
13
14#[derive(Debug, Clone, serde::Serialize)]
15pub struct ValidationDiff {
16    pub policy_version_a: u64,
17    pub policy_version_b: u64,
18    pub jobs_delta: i64,
19    pub evidence_delta: i64,
20    pub decisions_delta: i64,
21    pub traces_delta: i64,
22    pub target_count: usize,
23}
24
25pub async fn run_validation<F>(
26    fw: SharedFramework,
27    targets: &[String],
28    max_risk: RiskLevel,
29    planner_factory: F,
30) -> ValidationReport
31where
32    F: Fn() -> Box<dyn Planner> + Send + Sync,
33{
34    let mut orch = Orchestrator::new(fw.clone(), max_risk);
35    orch.set_approved(true);
36    let campaign = orch.run(targets, planner_factory).await;
37
38    let fw = fw.lock().await;
39    ValidationReport {
40        ran_at: now_secs(),
41        policy_version: fw.executor.policy_set.version,
42        campaign,
43    }
44}
45
46pub fn diff(a: &ValidationReport, b: &ValidationReport) -> ValidationDiff {
47    ValidationDiff {
48        policy_version_a: a.policy_version,
49        policy_version_b: b.policy_version,
50        jobs_delta: b.campaign.total_jobs as i64 - a.campaign.total_jobs as i64,
51        evidence_delta: b.campaign.total_evidence as i64 - a.campaign.total_evidence as i64,
52        decisions_delta: b.campaign.total_decisions as i64 - a.campaign.total_decisions as i64,
53        traces_delta: b.campaign.total_traces as i64 - a.campaign.total_traces as i64,
54        target_count: b.campaign.targets.len(),
55    }
56}
57
58#[cfg(test)]
59mod tests {
60    use super::*;
61    use crate::ai::agent::{Action, AnalysisOutput, ReportOutput};
62    use crate::core::executor::ModuleExecutor;
63    use crate::core::framework::new_shared_framework;
64    use crate::core::safety::{Charter, ScopeManager};
65
66    pub struct MockPlanner {
67        pub actions: Vec<Action>,
68    }
69
70    impl MockPlanner {
71        pub fn new() -> Self {
72            let mut options = std::collections::HashMap::new();
73            options.insert("host".to_string(), String::new());
74            options.insert("ports".to_string(), "1-1024".to_string());
75            MockPlanner {
76                actions: vec![Action {
77                    module: "tcp_port_scanner".to_string(),
78                    options,
79                    target: String::new(),
80                    priority: 50,
81                    reason: "scripted recon".to_string(),
82                }],
83            }
84        }
85    }
86
87    #[async_trait::async_trait]
88    impl Planner for MockPlanner {
89        async fn analyze(&self, _ctx: &str) -> anyhow::Result<AnalysisOutput> {
90            Ok(AnalysisOutput {
91                summary: "scripted analysis".to_string(),
92                vulnerabilities: vec![],
93                recommended_modules: vec!["tcp_port_scanner".to_string()],
94            })
95        }
96        async fn plan(&self, _ctx: &str) -> anyhow::Result<Vec<Action>> {
97            Ok(self.actions.clone())
98        }
99        async fn summarize(&self, _ctx: &str) -> anyhow::Result<ReportOutput> {
100            Ok(ReportOutput {
101                title: "scripted".to_string(),
102                summary: "scripted campaign".to_string(),
103                findings: vec![],
104                actions_taken: vec![],
105                recommendations: vec![],
106            })
107        }
108    }
109
110    #[tokio::test]
111    async fn validation_captures_policy_version_and_totals() {
112        let exec = ModuleExecutor::new(
113            Charter::accept("eval", vec!["auth".into()]),
114            ScopeManager::new(vec!["127.0.0.1".into()]),
115            RiskLevel::Critical,
116        );
117        let fw = new_shared_framework(exec);
118        let targets = vec!["127.0.0.1".to_string()];
119        let report = run_validation(fw, &targets, RiskLevel::Critical, || {
120            Box::new(MockPlanner::new())
121        })
122        .await;
123        assert_eq!(report.policy_version, 1);
124        assert!(report.campaign.total_jobs > 0);
125    }
126
127    #[test]
128    fn diff_computes_deltas() {
129        let base = ValidationReport {
130            ran_at: 1,
131            policy_version: 1,
132            campaign: CampaignReport {
133                targets: vec!["10.0.0.1".into()],
134                summaries: vec!["ok".into()],
135                ok: 1,
136                failed: 0,
137                total_jobs: 3,
138                total_sessions: 1,
139                total_decisions: 2,
140                total_evidence: 2,
141                total_traces: 4,
142            },
143        };
144        let next = ValidationReport {
145            ran_at: 2,
146            policy_version: 2,
147            campaign: CampaignReport {
148                targets: vec!["10.0.0.1".into()],
149                summaries: vec!["ok".into()],
150                ok: 1,
151                failed: 0,
152                total_jobs: 5,
153                total_sessions: 1,
154                total_decisions: 3,
155                total_evidence: 4,
156                total_traces: 6,
157            },
158        };
159        let d = diff(&base, &next);
160        assert_eq!(d.policy_version_a, 1);
161        assert_eq!(d.policy_version_b, 2);
162        assert_eq!(d.jobs_delta, 2);
163        assert_eq!(d.evidence_delta, 2);
164        assert_eq!(d.traces_delta, 2);
165    }
166}
icebox/ai/validator.rs

icebox/ai/
validator.rs
