areev_loop/
llm.rs

1//! Optional LLM enrichment (proposal §9).
2//!
3//! The engine's deterministic output stays a pure function of `(store, params,
4//! now)`. This layer is strictly **additive**: with a backend attached the
5//! pipeline gains two optional stages —
6//!
7//! ```text
8//! ANALYZE (deterministic) → DISCOVER (LLM) → ENRICH (LLM) → VALIDATE+DEDUP → STORE
9//! ```
10//!
11//! and with no backend those stages are the identity function, so the no-LLM
12//! path is byte-for-byte the deterministic path. The LLM can only:
13//!   - **DISCOVER**: propose *new* draft recommendations, which enter through
14//!     the ordinary candidate/dedup/store path stamped `origin = llm` — so they
15//!     can **never auto-apply** and never target prompt/host surfaces; and
16//!   - **ENRICH**: add a whitelisted `guidance` note to a deterministic
17//!     recommendation. The engine-templated summary is always kept; the model
18//!     never rewrites it.
19//!
20//! Trust floor (enforced by the engine, not the backend): responses are parsed
21//! to a fixed schema (unknown fields dropped, strings capped), DISCOVER drafts
22//! must cite evidence hashes present in the bundle, instructions never
23//! interleave with evidence, and a failed/timed-out/garbled call drops the LLM
24//! contribution for the run rather than failing it.
25//!
26//! `CommandLlm` mirrors the shipped `CommandEmbed`: whitespace-split argv (no
27//! shell), one process per call, a JSON request on stdin and a JSON response on
28//! stdout, and a construction-time probe that fails loud.
29
30use crate::error::{Error, Result};
31use serde::{Deserialize, Serialize};
32use std::io::Write;
33use std::process::{Command, Stdio};
34
35/// Caps that bound what a single LLM contribution can inject (defense in depth;
36/// the engine enforces them after parsing).
37pub const MAX_LLM_DRAFTS: usize = 8;
38pub const MAX_GUIDANCE_LEN: usize = 600;
39pub const MAX_SUMMARY_LEN: usize = 200;
40
41/// A backend that answers one JSON request with one JSON response. Object-safe
42/// so the engine can hold a `Box<dyn LlmBackend>`.
43pub trait LlmBackend: Send + Sync {
44    /// Model identifier, stamped as provenance on `origin = llm` grains.
45    fn model(&self) -> &str;
46    /// Run one request. `request` is a JSON string; the returned text is
47    /// expected to be JSON and is validated by the caller.
48    fn complete(&self, request: &str) -> Result<String>;
49}
50
51// ---- wire schema (request) -------------------------------------------------
52
53/// One deterministic finding, handed to DISCOVER as context (never as an
54/// instruction — see `LlmRequest`).
55#[derive(Debug, Clone, Serialize)]
56pub struct FindingBrief {
57    pub analyzer: String,
58    pub summary: String,
59    pub target: String,
60    pub severity: String,
61}
62
63/// One evidence grain, provenance-tagged.
64#[derive(Debug, Clone, Serialize)]
65pub struct EvidenceItem {
66    pub hash: String,
67    pub grain_type: String,
68    pub text: String,
69}
70
71/// The request envelope. `op` selects the stage; `instructions` is a fixed
72/// engine string kept in its own field so it never interleaves with evidence.
73#[derive(Debug, Clone, Serialize)]
74pub struct LlmRequest<'a> {
75    #[serde(rename = "loop")]
76    pub loop_proto: u8,
77    pub op: &'a str,
78    pub instructions: &'a str,
79    #[serde(skip_serializing_if = "Vec::is_empty")]
80    pub findings: Vec<FindingBrief>,
81    #[serde(skip_serializing_if = "Vec::is_empty")]
82    pub evidence: Vec<EvidenceItem>,
83    /// The operator's recent decisions — what they reject/approve — so the
84    /// model learns this reviewer's taste. (Bounded by the engine.)
85    #[serde(skip_serializing_if = "Vec::is_empty")]
86    pub rejected: Vec<String>,
87    #[serde(skip_serializing_if = "Vec::is_empty")]
88    pub approved: Vec<String>,
89}
90
91// ---- wire schema (response) ------------------------------------------------
92
93/// One DISCOVER draft as returned by the model. Unknown fields are dropped by
94/// serde; the engine further validates (cite-check, caps, target class,
95/// grounding, and independent verification before it is ever stored).
96#[derive(Debug, Clone, Deserialize, Default)]
97#[serde(default)]
98pub struct LlmDraft {
99    pub summary: String,
100    pub target: String,
101    pub guidance: String,
102    pub evidence: Vec<String>,
103    /// The model's self-reported confidence 0.0–1.0 that this finding is both
104    /// correct and materially useful (§5.1). Missing/garbled → 0.0 (rejected by
105    /// the confidence floor), a safe default.
106    pub confidence: f64,
107}
108
109/// The DISCOVER response.
110#[derive(Debug, Clone, Deserialize, Default)]
111#[serde(default)]
112pub struct DiscoverResponse {
113    pub recommendations: Vec<LlmDraft>,
114}
115
116/// The ENRICH response: guidance keyed by target_ref of a deterministic rec.
117#[derive(Debug, Clone, Deserialize, Default)]
118#[serde(default)]
119pub struct EnrichResponse {
120    /// `[{ "target": "...", "guidance": "..." }]`
121    pub notes: Vec<EnrichNote>,
122}
123
124#[derive(Debug, Clone, Deserialize, Default)]
125#[serde(default)]
126pub struct EnrichNote {
127    pub target: String,
128    pub guidance: String,
129}
130
131// ---- verifier stages (§5.2 GROUND, §5.3 VERIFY) ----------------------------
132
133/// GROUND request: for each candidate draft, does its cited evidence actually
134/// *entail* the claim? Decompose-then-entail is asked of the model here; a
135/// stronger deployment can swap a dedicated entailment checker behind the same
136/// shape. Kept a separate op/call from DISCOVER (proposer ≠ grounder).
137#[derive(Debug, Clone, Serialize)]
138pub struct GroundRequest<'a> {
139    #[serde(rename = "loop")]
140    pub loop_proto: u8,
141    pub op: &'a str, // "ground"
142    pub instructions: &'a str,
143    pub claims: Vec<GroundItem>,
144}
145
146#[derive(Debug, Clone, Serialize)]
147pub struct GroundItem {
148    pub id: usize,
149    pub claim: String,
150    pub evidence: Vec<EvidenceItem>,
151}
152
153#[derive(Debug, Clone, Deserialize, Default)]
154#[serde(default)]
155pub struct GroundResponse {
156    pub results: Vec<GroundResult>,
157}
158
159#[derive(Debug, Clone, Deserialize, Default)]
160#[serde(default)]
161pub struct GroundResult {
162    pub id: usize,
163    pub supported: bool,
164    pub reason: String,
165}
166
167/// VERIFY request: an **independent** adversarial pass (a separate call from the
168/// proposer — the anti-Goodhart rule) that tries to refute each grounded draft
169/// on novelty / reality / out-of-context grounds and returns keep/kill + a
170/// calibrated confidence. Deterministic findings are passed as context so the
171/// verifier can reject drafts that merely restate them.
172#[derive(Debug, Clone, Serialize)]
173pub struct VerifyRequest<'a> {
174    #[serde(rename = "loop")]
175    pub loop_proto: u8,
176    pub op: &'a str, // "verify"
177    pub instructions: &'a str,
178    pub findings: Vec<VerifyItem>,
179}
180
181#[derive(Debug, Clone, Serialize)]
182pub struct VerifyItem {
183    pub id: usize,
184    pub summary: String,
185    pub target: String,
186    pub evidence: Vec<EvidenceItem>,
187}
188
189#[derive(Debug, Clone, Deserialize, Default)]
190#[serde(default)]
191pub struct VerifyResponse {
192    pub results: Vec<VerifyResult>,
193}
194
195#[derive(Debug, Clone, Deserialize, Default)]
196#[serde(default)]
197pub struct VerifyResult {
198    pub id: usize,
199    pub keep: bool,
200    pub confidence: f64,
201    pub reason: String,
202}
203
204/// The probe response.
205#[derive(Debug, Clone, Deserialize, Default)]
206#[serde(default)]
207struct ProbeResponse {
208    model: String,
209}
210
211/// A subprocess LLM backend. One process per call; argv is whitespace-split
212/// with no shell (identical rules to `CommandEmbed`).
213pub struct CommandLlm {
214    argv: Vec<String>,
215    model: String,
216}
217
218impl CommandLlm {
219    /// Construct and probe. The probe (`{"loop":1,"op":"probe"}`) must return
220    /// JSON with a `model` (or one is supplied), so a misconfigured command
221    /// fails at construction, not mid-run.
222    pub fn new(cmd: &str, model: Option<&str>) -> Result<Self> {
223        let argv: Vec<String> = cmd.split_whitespace().map(str::to_string).collect();
224        if argv.is_empty() {
225            return Err(Error::LlmBackend("--llm-cmd is empty".into()));
226        }
227        let mut me = CommandLlm {
228            argv,
229            model: model.unwrap_or("").to_string(),
230        };
231        let probe = me.run(r#"{"loop":1,"op":"probe"}"#)?;
232        let parsed: ProbeResponse = serde_json::from_str(probe.trim()).map_err(|e| {
233            Error::LlmBackend(format!("--llm-cmd probe did not return JSON with a model: {e}"))
234        })?;
235        if me.model.is_empty() {
236            me.model = if parsed.model.is_empty() {
237                "unspecified".to_string()
238            } else {
239                parsed.model
240            };
241        }
242        Ok(me)
243    }
244
245    fn run(&self, request: &str) -> Result<String> {
246        let mut child = Command::new(&self.argv[0])
247            .args(&self.argv[1..])
248            .stdin(Stdio::piped())
249            .stdout(Stdio::piped())
250            .stderr(Stdio::inherit())
251            .spawn()
252            .map_err(|e| Error::LlmBackend(format!("spawn --llm-cmd {:?}: {e}", self.argv[0])))?;
253        {
254            let mut stdin = child.stdin.take().expect("stdin piped");
255            stdin
256                .write_all(request.as_bytes())
257                .map_err(|e| Error::LlmBackend(format!("write to --llm-cmd: {e}")))?;
258        }
259        let out = child
260            .wait_with_output()
261            .map_err(|e| Error::LlmBackend(format!("--llm-cmd wait: {e}")))?;
262        if !out.status.success() {
263            return Err(Error::LlmBackend(format!(
264                "--llm-cmd exited with {}",
265                out.status
266            )));
267        }
268        String::from_utf8(out.stdout)
269            .map_err(|e| Error::LlmBackend(format!("--llm-cmd stdout not UTF-8: {e}")))
270    }
271}
272
273impl LlmBackend for CommandLlm {
274    fn model(&self) -> &str {
275        &self.model
276    }
277    fn complete(&self, request: &str) -> Result<String> {
278        self.run(request)
279    }
280}
281
282/// Parse a DISCOVER response, dropping anything malformed. Never errors on
283/// model garbage — a bad response yields no drafts.
284pub fn parse_discover(raw: &str) -> DiscoverResponse {
285    serde_json::from_str(raw.trim()).unwrap_or_default()
286}
287
288/// Parse an ENRICH response, dropping anything malformed.
289pub fn parse_enrich(raw: &str) -> EnrichResponse {
290    serde_json::from_str(raw.trim()).unwrap_or_default()
291}
292
293/// Parse a GROUND response; garbage → no results (⇒ every draft is treated as
294/// ungrounded and dropped, the safe default).
295pub fn parse_ground(raw: &str) -> GroundResponse {
296    serde_json::from_str(raw.trim()).unwrap_or_default()
297}
298
299/// Parse a VERIFY response; garbage → no results (⇒ every draft is dropped).
300pub fn parse_verify(raw: &str) -> VerifyResponse {
301    serde_json::from_str(raw.trim()).unwrap_or_default()
302}
303
304/// Truncate to a char cap without splitting a UTF-8 boundary.
305pub fn cap(s: &str, max: usize) -> String {
306    if s.chars().count() <= max {
307        s.to_string()
308    } else {
309        s.chars().take(max).collect()
310    }
311}
312
313#[cfg(test)]
314mod tests {
315    use super::*;
316
317    #[test]
318    fn parse_discover_drops_garbage() {
319        assert!(parse_discover("not json").recommendations.is_empty());
320        let r = parse_discover(r#"{"recommendations":[{"summary":"s","target":"entity:x/y","evidence":["h1"],"junk":1}]}"#);
321        assert_eq!(r.recommendations.len(), 1);
322        assert_eq!(r.recommendations[0].summary, "s");
323        assert_eq!(r.recommendations[0].evidence, vec!["h1"]);
324    }
325
326    #[test]
327    fn parse_enrich_reads_notes() {
328        let r = parse_enrich(r#"{"notes":[{"target":"entity:a/b","guidance":"g"}]}"#);
329        assert_eq!(r.notes.len(), 1);
330        assert_eq!(r.notes[0].guidance, "g");
331    }
332
333    #[test]
334    fn cap_respects_char_boundaries() {
335        assert_eq!(cap("hello", 3), "hel");
336        assert_eq!(cap("héllo", 2), "hé");
337        assert_eq!(cap("hi", 5), "hi");
338    }
339}
areev_loop/llm.rs

areev_loop/
llm.rs
