sim_lib_forge/
verify.rs

1use std::collections::{BTreeMap, BTreeSet};
2
3use sim_codec_bridge::{BridgeBook, BridgePacket, BridgeVotePayload, content_id_string};
4use sim_kernel::{ContentId, Cx, Error, Expr, Result, Symbol};
5use sim_lib_bridge::{effective_caps, rx_check};
6use sim_value::{access::field, build::entry};
7
8use crate::{CompiledIntent, lift::content_id_for_expr};
9
10/// Semantic verifier registered for a compiled intent.
11#[derive(Clone, Debug, PartialEq, Eq)]
12pub enum Verifier {
13    /// Deterministic predicate evaluated over the decoded answer expression.
14    Assertion {
15        /// Predicate expression. Supported predicates are `forge/equals`,
16        /// `forge/number-between`, and `forge/field-number-between`.
17        predicate: Expr,
18    },
19    /// Judge vote represented as a checked BRIDGE packet.
20    Judge {
21        /// Seat that authored the judge packet.
22        seat: String,
23        /// Checked BRIDGE packet carrying one or more `bridge/Vote` parts.
24        packet: Box<BridgePacket>,
25        /// Optional parent packet used for BRIDGE reply legality.
26        reply_to: Option<Box<BridgePacket>>,
27        /// Vote target that must reach quorum.
28        target: String,
29        /// Minimum positive votes required for this verifier to pass.
30        min_votes: u32,
31    },
32    /// Retrieval or ground-truth check against cited evidence.
33    Evidence {
34        /// `Given` ids, URIs, or content-id strings that must exist in the
35        /// verifier catalog.
36        cites: Vec<String>,
37    },
38}
39
40/// Oracle backing a concrete verification probe.
41#[derive(Clone, Debug, PartialEq, Eq)]
42pub enum ProbeOracle {
43    /// Expected decoded answer for the probe case.
44    Expected(Expr),
45    /// Content id of evidence that supplies the expected answer.
46    Evidence(ContentId),
47}
48
49/// Concrete case that proves a compiled intent against required verifiers.
50#[derive(Clone, Debug, PartialEq, Eq)]
51pub struct VerifyProbe {
52    /// Concrete arguments for the known case.
53    pub args: Expr,
54    /// Expected answer or evidence reference for the known case.
55    pub oracle: ProbeOracle,
56    /// Verifier ids that must pass on this probe.
57    pub verifier_ids: Vec<Symbol>,
58}
59
60impl VerifyProbe {
61    /// Returns the content id of this probe record.
62    pub fn content_id(&self) -> Result<ContentId> {
63        content_id_for_expr(&self.to_expr())
64    }
65
66    fn to_expr(&self) -> Expr {
67        Expr::Map(vec![
68            entry("args", self.args.clone()),
69            entry("oracle", self.oracle.to_expr()),
70            entry(
71                "verifiers",
72                Expr::Vector(
73                    self.verifier_ids
74                        .iter()
75                        .cloned()
76                        .map(Expr::Symbol)
77                        .collect(),
78                ),
79            ),
80        ])
81    }
82}
83
84impl ProbeOracle {
85    fn to_expr(&self) -> Expr {
86        match self {
87            Self::Expected(expected) => Expr::Map(vec![
88                entry("kind", Expr::Symbol(Symbol::qualified("forge", "Expected"))),
89                entry("answer", expected.clone()),
90            ]),
91            Self::Evidence(id) => Expr::Map(vec![
92                entry("kind", Expr::Symbol(Symbol::qualified("forge", "Evidence"))),
93                entry("content-id", Expr::String(content_id_string(id))),
94            ]),
95        }
96    }
97}
98
99/// One failed semantic verifier.
100#[derive(Clone, Debug, PartialEq, Eq)]
101pub struct VerifyFailure {
102    /// Verifier or probe id that failed.
103    pub id: Symbol,
104    /// Human-readable reason for the failure.
105    pub reason: String,
106}
107
108/// Semantic verification result for one answer or probe set.
109#[derive(Clone, Debug, Default, PartialEq, Eq)]
110pub struct VerifyReport {
111    /// Verifier ids that passed.
112    pub passed: Vec<Symbol>,
113    /// Verifier or probe failures.
114    pub failed: Vec<VerifyFailure>,
115}
116
117impl VerifyReport {
118    /// Returns true when every required verifier passed.
119    pub fn accepted(&self) -> bool {
120        self.failed.is_empty()
121    }
122
123    fn pass(&mut self, id: Symbol) {
124        self.passed.push(id);
125    }
126
127    fn fail(&mut self, id: Symbol, reason: impl Into<String>) {
128        self.failed.push(VerifyFailure {
129            id,
130            reason: reason.into(),
131        });
132    }
133
134    fn extend(&mut self, mut other: VerifyReport) {
135        self.passed.append(&mut other.passed);
136        self.failed.append(&mut other.failed);
137    }
138}
139
140/// Local catalog binding verifier ids, probes, and cited ground truth.
141#[derive(Clone, Debug, Default)]
142pub struct VerifyCatalog {
143    verifiers: BTreeMap<Symbol, Verifier>,
144    probes: BTreeMap<ContentId, VerifyProbe>,
145    intent_probes: BTreeMap<Symbol, BTreeSet<ContentId>>,
146    evidence: BTreeMap<String, Expr>,
147}
148
149impl VerifyCatalog {
150    /// Builds an empty verifier catalog.
151    pub fn new() -> Self {
152        Self::default()
153    }
154
155    /// Registers or replaces a verifier by stable id.
156    pub fn register_verifier(&mut self, id: Symbol, verifier: Verifier) {
157        self.verifiers.insert(id, verifier);
158    }
159
160    /// Returns a verifier by id.
161    pub fn verifier(&self, id: &Symbol) -> Option<&Verifier> {
162        self.verifiers.get(id)
163    }
164
165    /// Registers a probe for an intent name and returns its content id.
166    pub fn register_probe(&mut self, intent: Symbol, probe: VerifyProbe) -> Result<ContentId> {
167        let id = probe.content_id()?;
168        self.insert_probe(intent, id.clone(), probe);
169        Ok(id)
170    }
171
172    /// Inserts a probe with an explicit content id.
173    pub fn insert_probe(&mut self, intent: Symbol, id: ContentId, probe: VerifyProbe) {
174        self.probes.insert(id.clone(), probe);
175        self.intent_probes.entry(intent).or_default().insert(id);
176    }
177
178    /// Returns probe ids attached to `intent`.
179    pub fn probe_ids_for(&self, intent: &Symbol) -> Vec<ContentId> {
180        self.intent_probes
181            .get(intent)
182            .into_iter()
183            .flat_map(|ids| ids.iter().cloned())
184            .collect()
185    }
186
187    /// Stores cited evidence by `Given` id, URI, or content-id string.
188    pub fn insert_evidence(&mut self, cite: impl Into<String>, value: Expr) {
189        self.evidence.insert(cite.into(), value);
190    }
191
192    /// Stores cited evidence under its canonical content-id string.
193    pub fn insert_content_evidence(&mut self, id: &ContentId, value: Expr) {
194        self.insert_evidence(content_id_string(id), value);
195    }
196
197    /// Runs the intent's required verifiers against a decoded answer.
198    pub fn verify_answer(
199        &self,
200        cx: &mut Cx,
201        intent: &CompiledIntent,
202        answer: &Expr,
203    ) -> Result<VerifyReport> {
204        let mut report = VerifyReport::default();
205        for id in &intent.verifiers {
206            match self.verifiers.get(id) {
207                Some(verifier) => match self.run_verifier(cx, verifier, answer) {
208                    Ok(()) => report.pass(id.clone()),
209                    Err(reason) => report.fail(id.clone(), reason),
210                },
211                None => report.fail(id.clone(), "semantic verifier is not registered"),
212            }
213        }
214        Ok(report)
215    }
216
217    /// Runs a concrete probe using only the verifiers named by the probe.
218    pub fn verify_probe(
219        &self,
220        cx: &mut Cx,
221        intent: &CompiledIntent,
222        probe: &VerifyProbe,
223    ) -> Result<VerifyReport> {
224        let answer = self.answer_for_oracle(&probe.oracle)?;
225        let mut scoped = intent.clone();
226        scoped.verifiers = probe.verifier_ids.clone();
227        self.verify_answer(cx, &scoped, &answer)
228    }
229
230    /// Runs every probe named by the intent and fails closed when proof is
231    /// missing or incomplete.
232    pub fn verify_intent_probes(
233        &self,
234        cx: &mut Cx,
235        intent: &CompiledIntent,
236    ) -> Result<VerifyReport> {
237        let mut report = VerifyReport::default();
238        if intent.verifiers.is_empty() {
239            report.fail(
240                Symbol::qualified("forge", "verifier"),
241                "intent declares no semantic verifiers",
242            );
243            return Ok(report);
244        }
245        if intent.probes.is_empty() {
246            report.fail(
247                Symbol::qualified("forge", "probe"),
248                "intent declares no verification probes",
249            );
250            return Ok(report);
251        }
252
253        let covered = self.covered_verifiers(intent);
254        for required in &intent.verifiers {
255            if !covered.contains(required) {
256                report.fail(
257                    required.clone(),
258                    "no verification probe requires this verifier",
259                );
260            }
261        }
262
263        for id in &intent.probes {
264            match self.probes.get(id) {
265                Some(probe) => report.extend(self.verify_probe(cx, intent, probe)?),
266                None => report.fail(
267                    Symbol::qualified("forge", "probe"),
268                    format!(
269                        "verification probe {} is not registered",
270                        content_id_string(id)
271                    ),
272                ),
273            }
274        }
275        Ok(report)
276    }
277
278    fn covered_verifiers(&self, intent: &CompiledIntent) -> BTreeSet<Symbol> {
279        intent
280            .probes
281            .iter()
282            .filter_map(|id| self.probes.get(id))
283            .flat_map(|probe| probe.verifier_ids.iter().cloned())
284            .collect()
285    }
286
287    fn answer_for_oracle(&self, oracle: &ProbeOracle) -> Result<Expr> {
288        match oracle {
289            ProbeOracle::Expected(answer) => Ok(answer.clone()),
290            ProbeOracle::Evidence(id) => self
291                .evidence
292                .get(&content_id_string(id))
293                .cloned()
294                .ok_or_else(|| {
295                    Error::Eval(format!(
296                        "probe evidence {} is absent",
297                        content_id_string(id)
298                    ))
299                }),
300        }
301    }
302
303    fn run_verifier(
304        &self,
305        cx: &mut Cx,
306        verifier: &Verifier,
307        answer: &Expr,
308    ) -> std::result::Result<(), String> {
309        match verifier {
310            Verifier::Assertion { predicate } => {
311                let narrowed = sim_kernel::CapabilitySet::new();
312                cx.with_capabilities(narrowed, |_| check_assertion(predicate, answer))
313                    .map_err(|err| err.to_string())
314                    .and_then(|result| result)
315            }
316            Verifier::Judge {
317                seat,
318                packet,
319                reply_to,
320                target,
321                min_votes,
322            } => {
323                let narrowed = effective_caps(cx, packet).map_err(|err| err.to_string())?;
324                cx.with_capabilities(narrowed, |scoped| {
325                    check_judge(
326                        scoped,
327                        seat,
328                        packet,
329                        reply_to.as_deref(),
330                        target,
331                        *min_votes,
332                    )
333                })
334                .map_err(|err| err.to_string())
335                .and_then(|result| result)
336            }
337            Verifier::Evidence { cites } => {
338                let narrowed = sim_kernel::CapabilitySet::new();
339                cx.with_capabilities(narrowed, |_| Ok(self.check_evidence(cites, answer)))
340                    .map_err(|err| err.to_string())
341                    .and_then(|result| result)
342            }
343        }
344    }
345
346    fn check_evidence(&self, cites: &[String], answer: &Expr) -> std::result::Result<(), String> {
347        if cites.is_empty() {
348            return Err("evidence verifier must cite at least one source".to_owned());
349        }
350        let mut matched = false;
351        for cite in cites {
352            let Some(evidence) = self.evidence.get(cite) else {
353                return Err(format!("cited evidence {cite} is absent"));
354            };
355            matched |= answer.canonical_eq(evidence);
356        }
357        if matched {
358            Ok(())
359        } else {
360            Err("answer does not match cited ground truth".to_owned())
361        }
362    }
363}
364
365/// Runs semantic verification with an empty catalog.
366///
367/// Callers that need registered verifier definitions should use
368/// [`VerifyCatalog::verify_answer`].
369pub fn verify_answer(cx: &mut Cx, intent: &CompiledIntent, answer: &Expr) -> Result<VerifyReport> {
370    VerifyCatalog::new().verify_answer(cx, intent, answer)
371}
372
373fn check_assertion(predicate: &Expr, answer: &Expr) -> Result<std::result::Result<(), String>> {
374    let kind = match field(predicate, "predicate") {
375        Some(Expr::Symbol(symbol)) => symbol,
376        _ => {
377            return Ok(Err(
378                "assertion predicate must name a predicate symbol".to_owned()
379            ));
380        }
381    };
382
383    if *kind == Symbol::qualified("forge", "equals") {
384        let Some(expected) = field(predicate, "expected") else {
385            return Ok(Err("forge/equals predicate is missing expected".to_owned()));
386        };
387        return if answer.canonical_eq(expected) {
388            Ok(Ok(()))
389        } else {
390            Ok(Err("answer did not equal expected expression".to_owned()))
391        };
392    }
393
394    if *kind == Symbol::qualified("forge", "number-between") {
395        return check_number_between(predicate, answer);
396    }
397
398    if *kind == Symbol::qualified("forge", "field-number-between") {
399        let field_name = match field(predicate, "field") {
400            Some(Expr::String(name)) => name.as_str(),
401            Some(Expr::Symbol(symbol)) if symbol.namespace.is_none() => symbol.name.as_ref(),
402            _ => {
403                return Ok(Err(
404                    "field-number-between predicate is missing field".to_owned()
405                ));
406            }
407        };
408        let Some(value) = field(answer, field_name) else {
409            return Ok(Err(format!("answer is missing field {field_name}")));
410        };
411        return check_number_between(predicate, value);
412    }
413
414    Ok(Err(format!("unsupported assertion predicate {kind}")))
415}
416
417fn check_number_between(predicate: &Expr, value: &Expr) -> Result<std::result::Result<(), String>> {
418    let Some(value) = integer_value(value)? else {
419        return Ok(Err("answer value is not an integer number".to_owned()));
420    };
421    let min = optional_i64(predicate, "min")?;
422    let max = optional_i64(predicate, "max")?;
423    if let Some(min) = min
424        && value < min
425    {
426        return Ok(Err(format!("answer value {value} is below minimum {min}")));
427    }
428    if let Some(max) = max
429        && value > max
430    {
431        return Ok(Err(format!("answer value {value} is above maximum {max}")));
432    }
433    Ok(Ok(()))
434}
435
436fn optional_i64(expr: &Expr, name: &str) -> Result<Option<i64>> {
437    match field(expr, name) {
438        Some(value) => integer_value(value)?
439            .map(Some)
440            .ok_or_else(|| Error::Eval(format!("{name} must be an integer number when present"))),
441        None => Ok(None),
442    }
443}
444
445fn integer_value(expr: &Expr) -> Result<Option<i64>> {
446    match expr {
447        Expr::Number(number) => number
448            .canonical
449            .parse::<i64>()
450            .map(Some)
451            .map_err(|_| Error::Eval(format!("{} is not an integer", number.canonical))),
452        _ => Ok(None),
453    }
454}
455
456fn check_judge(
457    cx: &mut Cx,
458    seat: &str,
459    packet: &BridgePacket,
460    reply_to: Option<&BridgePacket>,
461    target: &str,
462    min_votes: u32,
463) -> Result<std::result::Result<(), String>> {
464    if min_votes == 0 {
465        return Ok(Err("judge quorum must require at least one vote".to_owned()));
466    }
467    if packet.header.from != seat {
468        return Ok(Err(format!(
469            "judge packet came from {}, expected {seat}",
470            packet.header.from
471        )));
472    }
473    let report = rx_check(cx, &BridgeBook::standard(), packet, reply_to)?;
474    if !report.accepted() {
475        return Ok(Err(format!(
476            "judge packet failed BRIDGE rx_check: {:?}",
477            report.obligations
478        )));
479    }
480
481    let mut votes = 0u32;
482    for part in &packet.body {
483        if part.kind != Symbol::qualified("bridge", "Vote") {
484            continue;
485        }
486        let vote = BridgeVotePayload::from_expr(&part.payload)?;
487        if vote.target == target && vote.scores.iter().any(|score| score.value > 0) {
488            votes = votes.saturating_add(1);
489        }
490    }
491
492    if votes >= min_votes {
493        Ok(Ok(()))
494    } else {
495        Ok(Err(format!(
496            "judge quorum for {target} has {votes} vote(s), needs {min_votes}"
497        )))
498    }
499}
sim_lib_forge/verify.rs

sim_lib_forge/
verify.rs
