areev_loop/
llm.rs

1//! Optional LLM enrichment (proposal §9).
2//!
3//! The engine's deterministic output stays a pure function of `(store, params,
4//! now)`. This layer is strictly **additive**: with a backend attached the
5//! pipeline gains two optional stages —
6//!
7//! ```text
8//! ANALYZE (deterministic) → DISCOVER (LLM) → ENRICH (LLM) → VALIDATE+DEDUP → STORE
9//! ```
10//!
11//! and with no backend those stages are the identity function, so the no-LLM
12//! path is byte-for-byte the deterministic path. The LLM can only:
13//!   - **DISCOVER**: propose *new* draft recommendations, which enter through
14//!     the ordinary candidate/dedup/store path stamped `origin = llm` — so they
15//!     can **never auto-apply** and never target prompt/host surfaces; and
16//!   - **ENRICH**: add a whitelisted `guidance` note to a deterministic
17//!     recommendation. The engine-templated summary is always kept; the model
18//!     never rewrites it.
19//!
20//! Trust floor (enforced by the engine, not the backend): responses are parsed
21//! to a fixed schema (unknown fields dropped, strings capped), DISCOVER drafts
22//! must cite evidence hashes present in the bundle, instructions never
23//! interleave with evidence, and a failed/timed-out/garbled call drops the LLM
24//! contribution for the run rather than failing it.
25//!
26//! `CommandLlm` mirrors the shipped `CommandEmbed`: whitespace-split argv (no
27//! shell), one process per call, a JSON request on stdin and a JSON response on
28//! stdout, and a construction-time probe that fails loud.
29
30use crate::error::{Error, Result};
31use serde::{Deserialize, Serialize};
32
33/// Caps that bound what a single LLM contribution can inject (defense in depth;
34/// the engine enforces them after parsing).
35pub const MAX_LLM_DRAFTS: usize = 8;
36pub const MAX_GUIDANCE_LEN: usize = 600;
37pub const MAX_SUMMARY_LEN: usize = 200;
38
39/// A backend that answers one JSON request with one JSON response. Object-safe
40/// so the engine can hold a `Box<dyn LlmBackend>`.
41pub trait LlmBackend: Send + Sync {
42    /// Model identifier, stamped as provenance on `origin = llm` grains.
43    fn model(&self) -> &str;
44    /// Run one request. `request` is a JSON string; the returned text is
45    /// expected to be JSON and is validated by the caller.
46    fn complete(&self, request: &str) -> Result<String>;
47}
48
49/// Boxed backends forward — lets decorators wrap `Box<dyn LlmBackend>`
50/// without knowing the concrete type.
51impl<T: LlmBackend + ?Sized> LlmBackend for Box<T> {
52    fn model(&self) -> &str {
53        (**self).model()
54    }
55    fn complete(&self, request: &str) -> Result<String> {
56        (**self).complete(request)
57    }
58}
59
60// ---- wire schema (request) -------------------------------------------------
61
62/// One deterministic finding, handed to DISCOVER as context (never as an
63/// instruction — see `LlmRequest`).
64#[derive(Debug, Clone, Serialize)]
65pub struct FindingBrief {
66    pub analyzer: String,
67    pub summary: String,
68    pub target: String,
69    pub severity: String,
70}
71
72/// One evidence grain, provenance-tagged.
73#[derive(Debug, Clone, Serialize)]
74pub struct EvidenceItem {
75    pub hash: String,
76    pub grain_type: String,
77    pub text: String,
78}
79
80/// The request envelope. `op` selects the stage; `instructions` is a fixed
81/// engine string kept in its own field so it never interleaves with evidence.
82#[derive(Debug, Clone, Serialize)]
83pub struct LlmRequest<'a> {
84    #[serde(rename = "loop")]
85    pub loop_proto: u8,
86    pub op: &'a str,
87    pub instructions: &'a str,
88    #[serde(skip_serializing_if = "Vec::is_empty")]
89    pub findings: Vec<FindingBrief>,
90    #[serde(skip_serializing_if = "Vec::is_empty")]
91    pub evidence: Vec<EvidenceItem>,
92    /// The operator's recent decisions — what they reject/approve — so the
93    /// model learns this reviewer's taste. (Bounded by the engine.)
94    #[serde(skip_serializing_if = "Vec::is_empty")]
95    pub rejected: Vec<String>,
96    #[serde(skip_serializing_if = "Vec::is_empty")]
97    pub approved: Vec<String>,
98}
99
100// ---- wire schema (response) ------------------------------------------------
101
102/// One DISCOVER draft as returned by the model. Unknown fields are dropped by
103/// serde; the engine further validates (cite-check, caps, target class,
104/// grounding, and independent verification before it is ever stored).
105#[derive(Debug, Clone, Deserialize, Default)]
106#[serde(default)]
107pub struct LlmDraft {
108    pub summary: String,
109    pub target: String,
110    pub guidance: String,
111    pub evidence: Vec<String>,
112    /// The model's self-reported confidence 0.0–1.0 that this finding is both
113    /// correct and materially useful (§5.1). Missing/garbled → 0.0 (rejected by
114    /// the confidence floor), a safe default.
115    pub confidence: f64,
116}
117
118/// The DISCOVER response.
119#[derive(Debug, Clone, Deserialize, Default)]
120#[serde(default)]
121pub struct DiscoverResponse {
122    pub recommendations: Vec<LlmDraft>,
123}
124
125/// The ENRICH response: guidance keyed by target_ref of a deterministic rec.
126#[derive(Debug, Clone, Deserialize, Default)]
127#[serde(default)]
128pub struct EnrichResponse {
129    /// `[{ "target": "...", "guidance": "..." }]`
130    pub notes: Vec<EnrichNote>,
131}
132
133#[derive(Debug, Clone, Deserialize, Default)]
134#[serde(default)]
135pub struct EnrichNote {
136    pub target: String,
137    pub guidance: String,
138}
139
140// ---- verifier stages (§5.2 GROUND, §5.3 VERIFY) ----------------------------
141
142/// GROUND request: for each candidate draft, does its cited evidence actually
143/// *entail* the claim? Decompose-then-entail is asked of the model here; a
144/// stronger deployment can swap a dedicated entailment checker behind the same
145/// shape. Kept a separate op/call from DISCOVER (proposer ≠ grounder).
146#[derive(Debug, Clone, Serialize)]
147pub struct GroundRequest<'a> {
148    #[serde(rename = "loop")]
149    pub loop_proto: u8,
150    pub op: &'a str, // "ground"
151    pub instructions: &'a str,
152    pub claims: Vec<GroundItem>,
153}
154
155#[derive(Debug, Clone, Serialize)]
156pub struct GroundItem {
157    pub id: usize,
158    pub claim: String,
159    pub evidence: Vec<EvidenceItem>,
160}
161
162#[derive(Debug, Clone, Deserialize, Default)]
163#[serde(default)]
164pub struct GroundResponse {
165    pub results: Vec<GroundResult>,
166}
167
168#[derive(Debug, Clone, Deserialize, Default)]
169#[serde(default)]
170pub struct GroundResult {
171    pub id: usize,
172    pub supported: bool,
173    pub reason: String,
174}
175
176/// VERIFY request: an **independent** adversarial pass (a separate call from the
177/// proposer — the anti-Goodhart rule) that tries to refute each grounded draft
178/// on novelty / reality / out-of-context grounds and returns keep/kill + a
179/// calibrated confidence. Deterministic findings are passed as context so the
180/// verifier can reject drafts that merely restate them.
181#[derive(Debug, Clone, Serialize)]
182pub struct VerifyRequest<'a> {
183    #[serde(rename = "loop")]
184    pub loop_proto: u8,
185    pub op: &'a str, // "verify"
186    pub instructions: &'a str,
187    pub findings: Vec<VerifyItem>,
188}
189
190#[derive(Debug, Clone, Serialize)]
191pub struct VerifyItem {
192    pub id: usize,
193    pub summary: String,
194    pub target: String,
195    pub evidence: Vec<EvidenceItem>,
196}
197
198#[derive(Debug, Clone, Deserialize, Default)]
199#[serde(default)]
200pub struct VerifyResponse {
201    pub results: Vec<VerifyResult>,
202}
203
204#[derive(Debug, Clone, Deserialize, Default)]
205#[serde(default)]
206pub struct VerifyResult {
207    pub id: usize,
208    pub keep: bool,
209    pub confidence: f64,
210    pub reason: String,
211}
212
213/// The probe response.
214#[derive(Debug, Clone, Deserialize, Default)]
215#[serde(default)]
216struct ProbeResponse {
217    model: String,
218}
219
220/// A subprocess LLM backend. One process per call; argv is whitespace-split
221/// with no shell (identical rules to `CommandEmbed`).
222pub struct CommandLlm {
223    argv: Vec<String>,
224    model: String,
225}
226
227impl CommandLlm {
228    /// Construct and probe. The probe (`{"loop":1,"op":"probe"}`) must return
229    /// JSON with a `model` (or one is supplied), so a misconfigured command
230    /// fails at construction, not mid-run.
231    pub fn new(cmd: &str, model: Option<&str>) -> Result<Self> {
232        let argv: Vec<String> = cmd.split_whitespace().map(str::to_string).collect();
233        if argv.is_empty() {
234            return Err(Error::LlmBackend("--llm-cmd is empty".into()));
235        }
236        let mut me = CommandLlm {
237            argv,
238            model: model.unwrap_or("").to_string(),
239        };
240        let probe = me.run(r#"{"loop":1,"op":"probe"}"#)?;
241        let parsed: ProbeResponse = serde_json::from_str(probe.trim()).map_err(|e| {
242            Error::LlmBackend(format!("--llm-cmd probe did not return JSON with a model: {e}"))
243        })?;
244        if me.model.is_empty() {
245            me.model = if parsed.model.is_empty() {
246                "unspecified".to_string()
247            } else {
248                parsed.model
249            };
250        }
251        Ok(me)
252    }
253
254    fn run(&self, request: &str) -> Result<String> {
255        let out = crate::proc::run_argv(&self.argv, request, Some(crate::proc::DEFAULT_TIMEOUT))
256            .map_err(|e| Error::LlmBackend(format!("spawn --llm-cmd {:?}: {e}", self.argv[0])))?;
257        if let Some(why) = out.failure("--llm-cmd") {
258            return Err(Error::LlmBackend(why));
259        }
260        String::from_utf8(out.stdout)
261            .map_err(|e| Error::LlmBackend(format!("--llm-cmd stdout not UTF-8: {e}")))
262    }
263}
264
265impl LlmBackend for CommandLlm {
266    fn model(&self) -> &str {
267        &self.model
268    }
269    fn complete(&self, request: &str) -> Result<String> {
270        self.run(request)
271    }
272}
273
274/// Parse a DISCOVER response, dropping anything malformed. Never errors on
275/// model garbage — a bad response yields no drafts.
276pub fn parse_discover(raw: &str) -> DiscoverResponse {
277    serde_json::from_str(raw.trim()).unwrap_or_default()
278}
279
280/// Parse an ENRICH response, dropping anything malformed.
281pub fn parse_enrich(raw: &str) -> EnrichResponse {
282    serde_json::from_str(raw.trim()).unwrap_or_default()
283}
284
285/// Parse a GROUND response; garbage → no results (⇒ every draft is treated as
286/// ungrounded and dropped, the safe default).
287pub fn parse_ground(raw: &str) -> GroundResponse {
288    serde_json::from_str(raw.trim()).unwrap_or_default()
289}
290
291/// Parse a VERIFY response; garbage → no results (⇒ every draft is dropped).
292pub fn parse_verify(raw: &str) -> VerifyResponse {
293    serde_json::from_str(raw.trim()).unwrap_or_default()
294}
295
296/// Truncate to a char cap without splitting a UTF-8 boundary.
297pub fn cap(s: &str, max: usize) -> String {
298    if s.chars().count() <= max {
299        s.to_string()
300    } else {
301        s.chars().take(max).collect()
302    }
303}
304
305#[cfg(test)]
306mod tests {
307    use super::*;
308
309    #[test]
310    fn parse_discover_drops_garbage() {
311        assert!(parse_discover("not json").recommendations.is_empty());
312        let r = parse_discover(r#"{"recommendations":[{"summary":"s","target":"entity:x/y","evidence":["h1"],"junk":1}]}"#);
313        assert_eq!(r.recommendations.len(), 1);
314        assert_eq!(r.recommendations[0].summary, "s");
315        assert_eq!(r.recommendations[0].evidence, vec!["h1"]);
316    }
317
318    #[test]
319    fn parse_enrich_reads_notes() {
320        let r = parse_enrich(r#"{"notes":[{"target":"entity:a/b","guidance":"g"}]}"#);
321        assert_eq!(r.notes.len(), 1);
322        assert_eq!(r.notes[0].guidance, "g");
323    }
324
325    #[test]
326    fn cap_respects_char_boundaries() {
327        assert_eq!(cap("hello", 3), "hel");
328        assert_eq!(cap("héllo", 2), "hé");
329        assert_eq!(cap("hi", 5), "hi");
330    }
331}
areev_loop/llm.rs

areev_loop/
llm.rs
