bob_runtime/
output_validation.rs

1//! # Output Validation
2//!
3//! Validates LLM responses against a JSON Schema for structured output support.
4//!
5//! When a request specifies an `output_schema`, the scheduler validates the
6//! final LLM response against this schema. On failure, it re-prompts the LLM
7//! with the validation error, up to `max_output_retries` times.
8
9use serde_json::Value;
10
11/// Validate a JSON value against a JSON Schema.
12///
13/// Returns `Ok(())` if valid, or `Err(message)` with a human-readable
14/// validation error description.
15pub fn validate_output(value: &Value, schema: &Value) -> Result<(), String> {
16    let compiled =
17        jsonschema::Validator::new(schema).map_err(|e| format!("invalid schema: {e}"))?;
18    if compiled.is_valid(value) {
19        Ok(())
20    } else {
21        let errors: Vec<String> = compiled.iter_errors(value).map(|e| format!("{e}")).collect();
22        Err(format!("output validation failed: {}", errors.join("; ")))
23    }
24}
25
26/// Validate a JSON string against a JSON Schema.
27///
28/// Attempts to parse the string as JSON first, then validates against the schema.
29pub fn validate_output_str(content: &str, schema: &Value) -> Result<Value, String> {
30    let value: Value = serde_json::from_str(content).map_err(|e| format!("invalid JSON: {e}"))?;
31    validate_output(&value, schema)?;
32    Ok(value)
33}
34
35/// Build a re-prompt message for a validation failure.
36pub fn validation_error_prompt(content: &str, error: &str) -> String {
37    format!(
38        "Your previous response did not match the required schema.\n\
39         Validation error: {error}\n\
40         Your response was: {content}\n\
41         Please respond with a valid JSON object matching the required schema."
42    )
43}
44
45#[cfg(test)]
46mod tests {
47    use serde_json::json;
48
49    use super::*;
50
51    fn person_schema() -> Value {
52        json!({
53            "type": "object",
54            "properties": {
55                "name": {"type": "string"},
56                "age": {"type": "integer"}
57            },
58            "required": ["name", "age"]
59        })
60    }
61
62    #[test]
63    fn valid_output_passes() {
64        let value = json!({"name": "Alice", "age": 30});
65        assert!(validate_output(&value, &person_schema()).is_ok());
66    }
67
68    #[test]
69    fn missing_required_field_fails() {
70        let value = json!({"name": "Alice"});
71        let result = validate_output(&value, &person_schema());
72        assert!(result.is_err());
73        assert!(result.unwrap_err().contains("age"));
74    }
75
76    #[test]
77    fn wrong_type_fails() {
78        let value = json!({"name": "Alice", "age": "thirty"});
79        assert!(validate_output(&value, &person_schema()).is_err());
80    }
81
82    #[test]
83    fn validate_output_str_parses_and_validates() {
84        let content = r#"{"name": "Bob", "age": 25}"#;
85        let result = validate_output_str(content, &person_schema());
86        assert!(result.is_ok());
87    }
88
89    #[test]
90    fn validate_output_str_rejects_invalid_json() {
91        let content = "not json";
92        let result = validate_output_str(content, &person_schema());
93        assert!(result.is_err());
94        assert!(result.unwrap_err().contains("invalid JSON"));
95    }
96
97    #[test]
98    fn validation_error_prompt_contains_details() {
99        let prompt = validation_error_prompt("bad content", "missing field 'age'");
100        assert!(prompt.contains("bad content"));
101        assert!(prompt.contains("missing field 'age'"));
102        assert!(prompt.contains("required schema"));
103    }
104}
bob_runtime/output_validation.rs

bob_runtime/
output_validation.rs
