areev_loop/
llm.rs

1//! Optional LLM enrichment (proposal §9).
2//!
3//! The engine's deterministic output stays a pure function of `(store, params,
4//! now)`. This layer is strictly **additive**: with a backend attached the
5//! pipeline gains two optional stages —
6//!
7//! ```text
8//! ANALYZE (deterministic) → DISCOVER (LLM) → ENRICH (LLM) → VALIDATE+DEDUP → STORE
9//! ```
10//!
11//! and with no backend those stages are the identity function, so the no-LLM
12//! path is byte-for-byte the deterministic path. The LLM can only:
13//!   - **DISCOVER**: propose *new* draft recommendations, which enter through
14//!     the ordinary candidate/dedup/store path stamped `origin = llm` — so they
15//!     can **never auto-apply** and never target prompt/host surfaces; and
16//!   - **ENRICH**: add a whitelisted `guidance` note to a deterministic
17//!     recommendation. The engine-templated summary is always kept; the model
18//!     never rewrites it.
19//!
20//! Trust floor (enforced by the engine, not the backend): responses are parsed
21//! to a fixed schema (unknown fields dropped, strings capped), DISCOVER drafts
22//! must cite evidence hashes present in the bundle, instructions never
23//! interleave with evidence, and a failed/timed-out/garbled call drops the LLM
24//! contribution for the run rather than failing it.
25//!
26//! `CommandLlm` mirrors the shipped `CommandEmbed`: whitespace-split argv (no
27//! shell), one process per call, a JSON request on stdin and a JSON response on
28//! stdout, and a construction-time probe that fails loud.
29
30use crate::error::{Error, Result};
31use serde::{Deserialize, Serialize};
32use std::io::Write;
33use std::process::{Command, Stdio};
34
35/// Caps that bound what a single LLM contribution can inject (defense in depth;
36/// the engine enforces them after parsing).
37pub const MAX_LLM_DRAFTS: usize = 8;
38pub const MAX_GUIDANCE_LEN: usize = 600;
39pub const MAX_SUMMARY_LEN: usize = 200;
40
41/// A backend that answers one JSON request with one JSON response. Object-safe
42/// so the engine can hold a `Box<dyn LlmBackend>`.
43pub trait LlmBackend: Send + Sync {
44    /// Model identifier, stamped as provenance on `origin = llm` grains.
45    fn model(&self) -> &str;
46    /// Run one request. `request` is a JSON string; the returned text is
47    /// expected to be JSON and is validated by the caller.
48    fn complete(&self, request: &str) -> Result<String>;
49}
50
51/// Boxed backends forward — lets decorators wrap `Box<dyn LlmBackend>`
52/// without knowing the concrete type.
53impl<T: LlmBackend + ?Sized> LlmBackend for Box<T> {
54    fn model(&self) -> &str {
55        (**self).model()
56    }
57    fn complete(&self, request: &str) -> Result<String> {
58        (**self).complete(request)
59    }
60}
61
62// ---- wire schema (request) -------------------------------------------------
63
64/// One deterministic finding, handed to DISCOVER as context (never as an
65/// instruction — see `LlmRequest`).
66#[derive(Debug, Clone, Serialize)]
67pub struct FindingBrief {
68    pub analyzer: String,
69    pub summary: String,
70    pub target: String,
71    pub severity: String,
72}
73
74/// One evidence grain, provenance-tagged.
75#[derive(Debug, Clone, Serialize)]
76pub struct EvidenceItem {
77    pub hash: String,
78    pub grain_type: String,
79    pub text: String,
80}
81
82/// The request envelope. `op` selects the stage; `instructions` is a fixed
83/// engine string kept in its own field so it never interleaves with evidence.
84#[derive(Debug, Clone, Serialize)]
85pub struct LlmRequest<'a> {
86    #[serde(rename = "loop")]
87    pub loop_proto: u8,
88    pub op: &'a str,
89    pub instructions: &'a str,
90    #[serde(skip_serializing_if = "Vec::is_empty")]
91    pub findings: Vec<FindingBrief>,
92    #[serde(skip_serializing_if = "Vec::is_empty")]
93    pub evidence: Vec<EvidenceItem>,
94    /// The operator's recent decisions — what they reject/approve — so the
95    /// model learns this reviewer's taste. (Bounded by the engine.)
96    #[serde(skip_serializing_if = "Vec::is_empty")]
97    pub rejected: Vec<String>,
98    #[serde(skip_serializing_if = "Vec::is_empty")]
99    pub approved: Vec<String>,
100}
101
102// ---- wire schema (response) ------------------------------------------------
103
104/// One DISCOVER draft as returned by the model. Unknown fields are dropped by
105/// serde; the engine further validates (cite-check, caps, target class,
106/// grounding, and independent verification before it is ever stored).
107#[derive(Debug, Clone, Deserialize, Default)]
108#[serde(default)]
109pub struct LlmDraft {
110    pub summary: String,
111    pub target: String,
112    pub guidance: String,
113    pub evidence: Vec<String>,
114    /// The model's self-reported confidence 0.0–1.0 that this finding is both
115    /// correct and materially useful (§5.1). Missing/garbled → 0.0 (rejected by
116    /// the confidence floor), a safe default.
117    pub confidence: f64,
118}
119
120/// The DISCOVER response.
121#[derive(Debug, Clone, Deserialize, Default)]
122#[serde(default)]
123pub struct DiscoverResponse {
124    pub recommendations: Vec<LlmDraft>,
125}
126
127/// The ENRICH response: guidance keyed by target_ref of a deterministic rec.
128#[derive(Debug, Clone, Deserialize, Default)]
129#[serde(default)]
130pub struct EnrichResponse {
131    /// `[{ "target": "...", "guidance": "..." }]`
132    pub notes: Vec<EnrichNote>,
133}
134
135#[derive(Debug, Clone, Deserialize, Default)]
136#[serde(default)]
137pub struct EnrichNote {
138    pub target: String,
139    pub guidance: String,
140}
141
142// ---- verifier stages (§5.2 GROUND, §5.3 VERIFY) ----------------------------
143
144/// GROUND request: for each candidate draft, does its cited evidence actually
145/// *entail* the claim? Decompose-then-entail is asked of the model here; a
146/// stronger deployment can swap a dedicated entailment checker behind the same
147/// shape. Kept a separate op/call from DISCOVER (proposer ≠ grounder).
148#[derive(Debug, Clone, Serialize)]
149pub struct GroundRequest<'a> {
150    #[serde(rename = "loop")]
151    pub loop_proto: u8,
152    pub op: &'a str, // "ground"
153    pub instructions: &'a str,
154    pub claims: Vec<GroundItem>,
155}
156
157#[derive(Debug, Clone, Serialize)]
158pub struct GroundItem {
159    pub id: usize,
160    pub claim: String,
161    pub evidence: Vec<EvidenceItem>,
162}
163
164#[derive(Debug, Clone, Deserialize, Default)]
165#[serde(default)]
166pub struct GroundResponse {
167    pub results: Vec<GroundResult>,
168}
169
170#[derive(Debug, Clone, Deserialize, Default)]
171#[serde(default)]
172pub struct GroundResult {
173    pub id: usize,
174    pub supported: bool,
175    pub reason: String,
176}
177
178/// VERIFY request: an **independent** adversarial pass (a separate call from the
179/// proposer — the anti-Goodhart rule) that tries to refute each grounded draft
180/// on novelty / reality / out-of-context grounds and returns keep/kill + a
181/// calibrated confidence. Deterministic findings are passed as context so the
182/// verifier can reject drafts that merely restate them.
183#[derive(Debug, Clone, Serialize)]
184pub struct VerifyRequest<'a> {
185    #[serde(rename = "loop")]
186    pub loop_proto: u8,
187    pub op: &'a str, // "verify"
188    pub instructions: &'a str,
189    pub findings: Vec<VerifyItem>,
190}
191
192#[derive(Debug, Clone, Serialize)]
193pub struct VerifyItem {
194    pub id: usize,
195    pub summary: String,
196    pub target: String,
197    pub evidence: Vec<EvidenceItem>,
198}
199
200#[derive(Debug, Clone, Deserialize, Default)]
201#[serde(default)]
202pub struct VerifyResponse {
203    pub results: Vec<VerifyResult>,
204}
205
206#[derive(Debug, Clone, Deserialize, Default)]
207#[serde(default)]
208pub struct VerifyResult {
209    pub id: usize,
210    pub keep: bool,
211    pub confidence: f64,
212    pub reason: String,
213}
214
215/// The probe response.
216#[derive(Debug, Clone, Deserialize, Default)]
217#[serde(default)]
218struct ProbeResponse {
219    model: String,
220}
221
222/// A subprocess LLM backend. One process per call; argv is whitespace-split
223/// with no shell (identical rules to `CommandEmbed`).
224pub struct CommandLlm {
225    argv: Vec<String>,
226    model: String,
227}
228
229impl CommandLlm {
230    /// Construct and probe. The probe (`{"loop":1,"op":"probe"}`) must return
231    /// JSON with a `model` (or one is supplied), so a misconfigured command
232    /// fails at construction, not mid-run.
233    pub fn new(cmd: &str, model: Option<&str>) -> Result<Self> {
234        let argv: Vec<String> = cmd.split_whitespace().map(str::to_string).collect();
235        if argv.is_empty() {
236            return Err(Error::LlmBackend("--llm-cmd is empty".into()));
237        }
238        let mut me = CommandLlm {
239            argv,
240            model: model.unwrap_or("").to_string(),
241        };
242        let probe = me.run(r#"{"loop":1,"op":"probe"}"#)?;
243        let parsed: ProbeResponse = serde_json::from_str(probe.trim()).map_err(|e| {
244            Error::LlmBackend(format!("--llm-cmd probe did not return JSON with a model: {e}"))
245        })?;
246        if me.model.is_empty() {
247            me.model = if parsed.model.is_empty() {
248                "unspecified".to_string()
249            } else {
250                parsed.model
251            };
252        }
253        Ok(me)
254    }
255
256    fn run(&self, request: &str) -> Result<String> {
257        let mut child = Command::new(&self.argv[0])
258            .args(&self.argv[1..])
259            .stdin(Stdio::piped())
260            .stdout(Stdio::piped())
261            .stderr(Stdio::inherit())
262            .spawn()
263            .map_err(|e| Error::LlmBackend(format!("spawn --llm-cmd {:?}: {e}", self.argv[0])))?;
264        {
265            let mut stdin = child.stdin.take().expect("stdin piped");
266            stdin
267                .write_all(request.as_bytes())
268                .map_err(|e| Error::LlmBackend(format!("write to --llm-cmd: {e}")))?;
269        }
270        let out = child
271            .wait_with_output()
272            .map_err(|e| Error::LlmBackend(format!("--llm-cmd wait: {e}")))?;
273        if !out.status.success() {
274            return Err(Error::LlmBackend(format!(
275                "--llm-cmd exited with {}",
276                out.status
277            )));
278        }
279        String::from_utf8(out.stdout)
280            .map_err(|e| Error::LlmBackend(format!("--llm-cmd stdout not UTF-8: {e}")))
281    }
282}
283
284impl LlmBackend for CommandLlm {
285    fn model(&self) -> &str {
286        &self.model
287    }
288    fn complete(&self, request: &str) -> Result<String> {
289        self.run(request)
290    }
291}
292
293/// Parse a DISCOVER response, dropping anything malformed. Never errors on
294/// model garbage — a bad response yields no drafts.
295pub fn parse_discover(raw: &str) -> DiscoverResponse {
296    serde_json::from_str(raw.trim()).unwrap_or_default()
297}
298
299/// Parse an ENRICH response, dropping anything malformed.
300pub fn parse_enrich(raw: &str) -> EnrichResponse {
301    serde_json::from_str(raw.trim()).unwrap_or_default()
302}
303
304/// Parse a GROUND response; garbage → no results (⇒ every draft is treated as
305/// ungrounded and dropped, the safe default).
306pub fn parse_ground(raw: &str) -> GroundResponse {
307    serde_json::from_str(raw.trim()).unwrap_or_default()
308}
309
310/// Parse a VERIFY response; garbage → no results (⇒ every draft is dropped).
311pub fn parse_verify(raw: &str) -> VerifyResponse {
312    serde_json::from_str(raw.trim()).unwrap_or_default()
313}
314
315/// Truncate to a char cap without splitting a UTF-8 boundary.
316pub fn cap(s: &str, max: usize) -> String {
317    if s.chars().count() <= max {
318        s.to_string()
319    } else {
320        s.chars().take(max).collect()
321    }
322}
323
324#[cfg(test)]
325mod tests {
326    use super::*;
327
328    #[test]
329    fn parse_discover_drops_garbage() {
330        assert!(parse_discover("not json").recommendations.is_empty());
331        let r = parse_discover(r#"{"recommendations":[{"summary":"s","target":"entity:x/y","evidence":["h1"],"junk":1}]}"#);
332        assert_eq!(r.recommendations.len(), 1);
333        assert_eq!(r.recommendations[0].summary, "s");
334        assert_eq!(r.recommendations[0].evidence, vec!["h1"]);
335    }
336
337    #[test]
338    fn parse_enrich_reads_notes() {
339        let r = parse_enrich(r#"{"notes":[{"target":"entity:a/b","guidance":"g"}]}"#);
340        assert_eq!(r.notes.len(), 1);
341        assert_eq!(r.notes[0].guidance, "g");
342    }
343
344    #[test]
345    fn cap_respects_char_boundaries() {
346        assert_eq!(cap("hello", 3), "hel");
347        assert_eq!(cap("héllo", 2), "hé");
348        assert_eq!(cap("hi", 5), "hi");
349    }
350}
areev_loop/llm.rs

areev_loop/
llm.rs
