leviath_scripting/
output_validator.rs

1//! Script-backed validators for an agent's final output.
2//!
3//! A blueprint can name any format it likes, and most of them are formats this
4//! codebase has never heard of. The built-in checks cover a handful that can be
5//! parsed with a crate we already ship; everything else would mean owning that
6//! format's parser and schema language, which is a cost the output system
7//! deliberately does not take on.
8//!
9//! So the knowledge lives with the person who has it. An agent that produces
10//! a2ui, a house report format, or a dialect of CSV nobody else uses ships a
11//! `.rhai` file beside its blueprint that says what "valid" means:
12//!
13//! ```rhai
14//! // @validator a2ui
15//! fn validate(content) {
16//!     let doc = parse_json(content);
17//!     if doc.root == () { return "the document has no `root` node"; }
18//!     ()   // fine
19//! }
20//! ```
21//!
22//! One function, one contract: **return `()` when the answer is fine, or a
23//! string saying what is wrong**. The string goes back to the agent as the same
24//! `[error]` refusal a schema failure produces, and it tries again. Rhai passes
25//! by value, so a return value is the only thing a script can say - the same
26//! shape the region hooks use.
27//!
28//! Execution runs on a fresh hardened engine per call: no filesystem, no
29//! network, no `eval`, operation-bounded. A validator that throws, loops, or
30//! returns something that is neither `()` nor a string is an authoring error,
31//! reported as such rather than being allowed to fail every submission.
32
33use rhai::{AST, Dynamic, Engine, Scope};
34
35/// Operation budget for a validator: a pure data check over one document, the
36/// same policy the region hooks get rather than the far larger budget the
37/// IO-driving script tools and providers need.
38const VALIDATOR_MAX_OPERATIONS: u64 = 100_000;
39
40/// A compiled output validator, ready to call.
41///
42/// Compiled once when the agent spawns, so a broken script is a spawn error
43/// rather than a surprise at the end of a long run - the worst possible moment
44/// to discover the agent cannot hand back its work.
45#[derive(Debug, Clone)]
46pub struct OutputValidator {
47    /// The script path as written in the blueprint, for error context.
48    pub path: String,
49    ast: AST,
50}
51
52/// Build the hardened engine every validator call runs on.
53fn build_engine() -> Engine {
54    let mut engine = Engine::new();
55    crate::harden(&mut engine, VALIDATOR_MAX_OPERATIONS);
56    crate::functions::register_functions(&mut engine);
57    crate::types::register_types(&mut engine);
58    engine
59}
60
61/// Compile an output validator and check its shape.
62///
63/// `validate(content)` must exist and take exactly one parameter. A script that
64/// defines nothing, or defines it with the wrong arity, is refused here rather
65/// than silently never running.
66pub fn compile(path: &str, source: &str) -> crate::Result<OutputValidator> {
67    let engine = build_engine();
68    let ast = engine
69        .compile(source)
70        .map_err(|e| crate::Error::CompilationFailed(format!("{path}: {e}")))?;
71
72    let arity = ast
73        .iter_functions()
74        .find(|f| f.name == "validate")
75        .map(|f| f.params.len());
76    match arity {
77        Some(1) => Ok(OutputValidator {
78            path: path.to_string(),
79            ast,
80        }),
81        Some(n) => Err(crate::Error::ValidationFailed(format!(
82            "{path}: fn validate must take exactly one parameter (content), found {n}"
83        ))),
84        None => Err(crate::Error::ValidationFailed(format!(
85            "{path}: script must define fn validate(content)"
86        ))),
87    }
88}
89
90/// What a validator said about one submission.
91#[derive(Debug, Clone, PartialEq, Eq)]
92pub enum Verdict {
93    /// The answer is fine.
94    Valid,
95    /// The answer is not, for this reason. Goes back to the agent verbatim.
96    Invalid(String),
97    /// The validator itself is broken: it threw, ran out of operations, or
98    /// returned something that is neither `()` nor a string.
99    ///
100    /// Kept apart from [`Invalid`](Self::Invalid) on purpose. A broken validator
101    /// must not read as "every answer is wrong", which would burn the agent's
102    /// whole retry budget on a script bug and end the run with nothing.
103    Unusable(String),
104}
105
106/// Run `validator` over `content`.
107pub fn validate(validator: &OutputValidator, content: &str) -> Verdict {
108    let engine = build_engine();
109    let result: Result<Dynamic, _> = engine.call_fn(
110        &mut Scope::new(),
111        &validator.ast,
112        "validate",
113        (content.to_string(),),
114    );
115    let value = match result {
116        Ok(v) => v,
117        Err(e) => {
118            return Verdict::Unusable(format!("{}: validate: {e}", validator.path));
119        }
120    };
121    if value.is_unit() {
122        return Verdict::Valid;
123    }
124    match value.into_string() {
125        // A script may also say "fine" by returning an empty string, which is
126        // easy to write by accident and unambiguous in meaning.
127        Ok(reason) if reason.trim().is_empty() => Verdict::Valid,
128        Ok(reason) => Verdict::Invalid(reason),
129        Err(actual) => Verdict::Unusable(format!(
130            "{}: validate must return () or a string, got {actual}",
131            validator.path
132        )),
133    }
134}
135
136#[cfg(test)]
137mod tests;
leviath_scripting/output_validator.rs

leviath_scripting/
output_validator.rs
