agentforge_parser/
validator.rs

1use agentforge_core::{AgentFile, LintError, LintSeverity};
2
3/// Result of validating an agent file.
4#[derive(Debug)]
5pub struct ValidationResult {
6    pub errors: Vec<LintError>,
7    pub warnings: Vec<LintError>,
8}
9
10impl ValidationResult {
11    pub fn is_valid(&self) -> bool {
12        self.errors.is_empty()
13    }
14
15    pub fn all_issues(&self) -> Vec<&LintError> {
16        self.errors.iter().chain(self.warnings.iter()).collect()
17    }
18}
19
20/// Validate a parsed `AgentFile` and return lint errors and warnings.
21pub fn validate_agent_file(agent: &AgentFile) -> ValidationResult {
22    let mut errors = Vec::new();
23    let mut warnings = Vec::new();
24
25    // Required fields
26    if agent.name.is_empty() {
27        errors.push(lint_error("name", "Agent name must not be empty"));
28    }
29
30    if agent.system_prompt.trim().is_empty() {
31        errors.push(lint_error(
32            "system_prompt",
33            "System prompt must not be empty",
34        ));
35    }
36
37    if agent.model.model_id.is_empty() {
38        errors.push(lint_error("model.model_id", "Model ID must not be empty"));
39    }
40
41    // Temperature validation
42    if let Some(temp) = agent.model.temperature {
43        if !(0.0..=2.0).contains(&temp) {
44            errors.push(lint_error(
45                "model.temperature",
46                &format!("Temperature {temp} is out of valid range [0.0, 2.0]"),
47            ));
48        }
49    }
50
51    // Tool validation
52    let tool_names: std::collections::HashSet<&str> =
53        agent.tools.iter().map(|t| t.name.as_str()).collect();
54
55    if tool_names.len() != agent.tools.len() {
56        errors.push(lint_error("tools", "Duplicate tool names detected"));
57    }
58
59    for tool in &agent.tools {
60        if tool.name.is_empty() {
61            errors.push(lint_error("tools[].name", "Tool name must not be empty"));
62        }
63        if tool.description.is_empty() {
64            warnings.push(lint_warning(
65                &format!("tools[{}].description", tool.name),
66                "Tool has no description — this reduces scoring accuracy",
67            ));
68        }
69        // Validate tool parameters is a valid JSON Schema object
70        if tool.parameters.get("type").is_none() {
71            warnings.push(lint_warning(
72                &format!("tools[{}].parameters", tool.name),
73                "Tool parameters should have a 'type' field",
74            ));
75        }
76    }
77
78    // Output schema validation
79    if let Some(schema) = &agent.output_schema {
80        if schema.get("type").is_none() && schema.get("$ref").is_none() {
81            warnings.push(lint_warning(
82                "output_schema",
83                "Output schema should specify a 'type' field",
84            ));
85        }
86    } else {
87        warnings.push(lint_warning(
88            "output_schema",
89            "No output schema defined — output schema compliance scoring will be skipped",
90        ));
91    }
92
93    // Eval hints validation
94    if let Some(hints) = &agent.eval_hints {
95        if let Some(threshold) = hints.pass_threshold {
96            if !(0.0..=1.0).contains(&threshold) {
97                errors.push(lint_error(
98                    "eval_hints.pass_threshold",
99                    &format!("pass_threshold {threshold} must be between 0.0 and 1.0"),
100                ));
101            }
102        }
103        if let Some(count) = hints.scenario_count {
104            if count == 0 {
105                errors.push(lint_error(
106                    "eval_hints.scenario_count",
107                    "scenario_count must be > 0",
108                ));
109            }
110            if count > 2000 {
111                warnings.push(lint_warning(
112                    "eval_hints.scenario_count",
113                    &format!("scenario_count {count} exceeds recommended max of 2000"),
114                ));
115            }
116        }
117
118        // Check that critical_tools reference actual defined tools
119        for critical_tool in &hints.critical_tools {
120            if !tool_names.contains(critical_tool.as_str()) {
121                warnings.push(lint_warning(
122                    "eval_hints.critical_tools",
123                    &format!("Critical tool '{critical_tool}' is not defined in tools[]"),
124                ));
125            }
126        }
127    }
128
129    // Constraints validation
130    if agent.constraints.is_empty() {
131        warnings.push(lint_warning(
132            "constraints",
133            "No constraints defined — instruction adherence scoring will be limited",
134        ));
135    }
136
137    ValidationResult { errors, warnings }
138}
139
140fn lint_error(field: &str, message: &str) -> LintError {
141    LintError {
142        field: field.to_string(),
143        message: message.to_string(),
144        severity: LintSeverity::Error,
145    }
146}
147
148fn lint_warning(field: &str, message: &str) -> LintError {
149    LintError {
150        field: field.to_string(),
151        message: message.to_string(),
152        severity: LintSeverity::Warning,
153    }
154}
155
156#[cfg(test)]
157mod tests {
158    use super::*;
159    use agentforge_core::{AgentFile, EvalHints, ModelConfig, ModelProvider, ToolDefinition};
160
161    fn make_valid_agent() -> AgentFile {
162        AgentFile {
163            agentforge_schema_version: "1".to_string(),
164            name: "test-agent".to_string(),
165            version: "1.0.0".to_string(),
166            model: ModelConfig {
167                provider: ModelProvider::Openai,
168                model_id: "gpt-4o".to_string(),
169                temperature: Some(0.2),
170                max_tokens: Some(2048),
171                top_p: None,
172            },
173            system_prompt: "You are a helpful assistant.".to_string(),
174            tools: vec![ToolDefinition {
175                name: "search".to_string(),
176                description: "Search the web".to_string(),
177                parameters: serde_json::json!({"type": "object", "properties": {}}),
178            }],
179            output_schema: Some(serde_json::json!({"type": "object"})),
180            constraints: vec!["Always be polite.".to_string()],
181            eval_hints: Some(EvalHints {
182                domain: Some("general".to_string()),
183                typical_turns: Some(3),
184                critical_tools: vec!["search".to_string()],
185                pass_threshold: Some(0.85),
186                scenario_count: Some(100),
187            }),
188            metadata: None,
189        }
190    }
191
192    #[test]
193    fn valid_agent_passes() {
194        let agent = make_valid_agent();
195        let result = validate_agent_file(&agent);
196        assert!(result.is_valid(), "Errors: {:?}", result.errors);
197    }
198
199    #[test]
200    fn empty_name_is_error() {
201        let mut agent = make_valid_agent();
202        agent.name = "".to_string();
203        let result = validate_agent_file(&agent);
204        assert!(!result.is_valid());
205        assert!(result.errors.iter().any(|e| e.field == "name"));
206    }
207
208    #[test]
209    fn invalid_temperature_is_error() {
210        let mut agent = make_valid_agent();
211        agent.model.temperature = Some(3.0);
212        let result = validate_agent_file(&agent);
213        assert!(!result.is_valid());
214    }
215
216    #[test]
217    fn undefined_critical_tool_is_warning() {
218        let mut agent = make_valid_agent();
219        if let Some(hints) = agent.eval_hints.as_mut() {
220            hints.critical_tools = vec!["nonexistent_tool".to_string()];
221        }
222        let result = validate_agent_file(&agent);
223        assert!(result.is_valid()); // It's a warning, not an error
224        assert!(!result.warnings.is_empty());
225    }
226
227    #[test]
228    fn missing_output_schema_is_warning() {
229        let mut agent = make_valid_agent();
230        agent.output_schema = None;
231        let result = validate_agent_file(&agent);
232        assert!(result.is_valid()); // Warning, not error
233        assert!(result.warnings.iter().any(|w| w.field == "output_schema"));
234    }
235
236    #[test]
237    fn duplicate_tool_names_is_error() {
238        let mut agent = make_valid_agent();
239        agent.tools.push(agent.tools[0].clone());
240        let result = validate_agent_file(&agent);
241        assert!(!result.is_valid());
242    }
243}
agentforge_parser/validator.rs

agentforge_parser/
validator.rs
