recall_wire/
evaluations.rs

1//! `/v1/evaluations`: reports on what memory holds, made by the worker.
2//!
3//! A run is asked for with [`EvaluationRequest`] and becomes an
4//! [`KIND_EVALUATE`](crate::jobs::KIND_EVALUATE) job, which `recall-worker`
5//! claims like a merge. The claim carries the files to look at; the worker
6//! runs its checks ([`KINDS`]) and posts back an
7//! [`EvaluateResult`](crate::jobs::EvaluateResult): `findings` and
8//! `details`.
9//!
10//! **The split is the point.** A [`Finding`] holds only enums, a file's
11//! identity, line numbers and the files it relates to: never a word of a
12//! note. The server refuses a result whose finding carries any other key
13//! ([`check_finding`]), holds an `id` of any other shape, or names a file
14//! it does not store, so nothing a worker writes into a finding can be note
15//! text. Everything that quotes a note, the excerpt, the reasoning and a
16//! suggested edit, goes in `details` ([`Details`]), which only
17//! `GET /v1/evaluations/{id}` returns, and only to the operator or an admin
18//! device.
19//!
20//! `details` is stored as plain JSON for now. The design seals it with the
21//! content key (`docs/design/part5-plan.md`, PR 6), and that is still the
22//! intended end state: encryption was parked on 2026-09-25, and when it
23//! comes back only how `details` is stored changes, never what a finding
24//! may hold.
25//!
26//! Nothing here changes memory. A suggested edit is applied only when the
27//! owner runs `recall eval apply`, which writes it to the local file and
28//! pushes it like any other edit.
29
30use std::collections::BTreeMap;
31
32use serde::{Deserialize, Serialize};
33use serde_json::Value;
34
35use crate::validate::{validate_file_path, validate_project_key};
36
37/// `POST` asks for a run; `GET` lists runs, newest first. Admin only.
38pub const EVALUATIONS_PATH: &str = "/v1/evaluations";
39
40/// `GET`: one run, with its findings and details. Admin only.
41pub fn evaluation_path(id: &str) -> String {
42    format!("{EVALUATIONS_PATH}/{id}")
43}
44
45/// The most projects one request may name.
46pub const MAX_PROJECTS: usize = 100;
47
48/// The most findings one result may carry. A worker that finds more keeps
49/// the first this many and says so in `details`.
50pub const MAX_FINDINGS: usize = 1000;
51
52/// The most files one finding may name as related.
53pub const MAX_RELATED: usize = 10;
54
55/// A file that is in memory, where it should not be: a key, a token.
56pub const KIND_SECRET: &str = "secret";
57/// Two notes that say opposite things. The only check that asks `claude`,
58/// and it runs only when the request asks for it.
59pub const KIND_CONTRADICTION: &str = "contradiction";
60/// A `MEMORY.md` line linking to a file that is not there.
61pub const KIND_DEAD_LINK: &str = "dead_link";
62/// A project file that says it is about the user (`type: user`), which
63/// belongs in the global scope: what `recall promote` is for.
64pub const KIND_WRONG_SCOPE: &str = "wrong_scope";
65/// The same paragraph in two files, or two scopes.
66pub const KIND_DUPLICATE: &str = "duplicate";
67/// A file unchanged for a long while that names a path or a command, for
68/// the owner to confirm is still true.
69pub const KIND_STALE: &str = "stale";
70
71/// Every kind of finding, most urgent first: the order a report lists them
72/// in.
73pub const KINDS: [&str; 6] = [
74    KIND_SECRET,
75    KIND_CONTRADICTION,
76    KIND_DEAD_LINK,
77    KIND_WRONG_SCOPE,
78    KIND_DUPLICATE,
79    KIND_STALE,
80];
81
82/// A finding that can wait.
83pub const SEVERITY_LOW: &str = "low";
84/// A finding worth fixing soon.
85pub const SEVERITY_MEDIUM: &str = "medium";
86/// A finding to fix now: a secret.
87pub const SEVERITY_HIGH: &str = "high";
88
89/// Every severity, lowest first.
90pub const SEVERITIES: [&str; 3] = [SEVERITY_LOW, SEVERITY_MEDIUM, SEVERITY_HIGH];
91
92/// Waiting for a worker to claim it.
93pub const STATE_QUEUED: &str = "queued";
94/// A worker holds it.
95pub const STATE_RUNNING: &str = "running";
96/// Its report is in.
97pub const STATE_DONE: &str = "done";
98/// Out of attempts: `error` says why.
99pub const STATE_FAILED: &str = "failed";
100
101/// How a global scope's key starts (`global:eko`). An evaluation always
102/// reads the global scopes beside the projects it was asked for.
103pub const GLOBAL_PREFIX: &str = "global:";
104
105/// How a machine scope's key starts (`machine:mbp`).
106pub const MACHINE_PREFIX: &str = "machine:";
107
108/// Body of `POST /v1/evaluations`. Both members may be left out: `{}` asks
109/// for every project, without the contradiction check.
110#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
111pub struct EvaluationRequest {
112    /// The project keys to look at; empty for every project. The global
113    /// scopes are always read beside them, since a duplicate, a dead
114    /// `global/` link or a contradiction may lie between a project and
115    /// them.
116    #[serde(default)]
117    pub projects: Vec<String>,
118    /// Whether to run the contradiction check, one `claude -p` per project,
119    /// which spends the owner's Claude usage. Off unless asked for.
120    #[serde(default)]
121    pub contradictions: bool,
122}
123
124/// `POST /v1/evaluations`'s answer.
125#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
126pub struct EvaluationCreated {
127    /// `eval_…`.
128    pub id: String,
129    /// [`STATE_QUEUED`].
130    pub state: String,
131    /// The `evaluate` job the worker will claim.
132    pub job: String,
133}
134
135/// One run, as `GET /v1/evaluations` lists it: counts, never findings or
136/// details.
137#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
138pub struct EvaluationSummary {
139    /// `eval_…`.
140    pub id: String,
141    /// [`STATE_QUEUED`], [`STATE_RUNNING`], [`STATE_DONE`] or
142    /// [`STATE_FAILED`].
143    pub state: String,
144    /// When it was asked for.
145    pub created_at: String,
146    /// When its report came in, or it failed; `null` until then.
147    pub finished_at: Option<String>,
148    /// How many findings of each kind, by kind; only kinds it found.
149    #[serde(default)]
150    pub counts: BTreeMap<String, u64>,
151    /// The projects asked for; empty for every project.
152    #[serde(default)]
153    pub projects: Vec<String>,
154    /// Whether the contradiction check was asked for.
155    #[serde(default)]
156    pub contradictions: bool,
157    /// Why it failed, or the last error of an attempt that will be retried;
158    /// `null` otherwise.
159    #[serde(default)]
160    pub error: Option<String>,
161}
162
163/// `GET /v1/evaluations`.
164#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
165pub struct EvaluationList {
166    /// At most 200, newest first.
167    pub evaluations: Vec<EvaluationSummary>,
168}
169
170/// `GET /v1/evaluations/{id}`: one run, its findings and its details.
171#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
172pub struct Evaluation {
173    /// `eval_…`.
174    pub id: String,
175    /// As in [`EvaluationSummary::state`].
176    pub state: String,
177    /// When it was asked for.
178    pub created_at: String,
179    /// When its report came in, or it failed; `null` until then.
180    pub finished_at: Option<String>,
181    /// What it found: empty until it is done.
182    #[serde(default)]
183    pub findings: Vec<Finding>,
184    /// The excerpts, the reasoning and a suggested edit per finding, as a
185    /// [`Details`]; `null` until it is done, and always `null` to the admin
186    /// page's passkey session, which never holds note text.
187    #[serde(default)]
188    pub details: Option<Value>,
189    /// The projects asked for; empty for every project.
190    #[serde(default)]
191    pub projects: Vec<String>,
192    /// Whether the contradiction check was asked for.
193    #[serde(default)]
194    pub contradictions: bool,
195    /// As in [`EvaluationSummary::error`].
196    #[serde(default)]
197    pub error: Option<String>,
198}
199
200/// One finding: what kind, how bad, where. Nothing else, ever: see the
201/// module docs and [`check_finding`].
202#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
203pub struct Finding {
204    /// `f1`, `f2`, …: unique within its report, and how `details` and
205    /// `recall eval apply` name it.
206    pub id: String,
207    /// One of [`KINDS`].
208    pub kind: String,
209    /// One of [`SEVERITIES`].
210    pub severity: String,
211    /// The project the file belongs to.
212    pub project_key: String,
213    /// The file.
214    pub file_path: String,
215    /// The first and last line it concerns, counting from 1.
216    pub lines: [u32; 2],
217    /// Other files it concerns: where a duplicate's first copy is, or what
218    /// a note contradicts.
219    #[serde(default)]
220    pub related: Vec<FileRef>,
221}
222
223/// A file, by its project and path.
224#[derive(Debug, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
225pub struct FileRef {
226    /// The project.
227    pub project_key: String,
228    /// The file.
229    pub file_path: String,
230}
231
232/// The keys a finding has, in order, and the only ones it may.
233pub const FINDING_KEYS: [&str; 7] = [
234    "id",
235    "kind",
236    "severity",
237    "project_key",
238    "file_path",
239    "lines",
240    "related",
241];
242
243/// The keys a related file has, and the only ones it may.
244pub const FILE_REF_KEYS: [&str; 2] = ["project_key", "file_path"];
245
246/// Whether `id` is a finding id: `f` and a number from 1, at most six
247/// digits.
248pub fn is_finding_id(id: &str) -> bool {
249    let Some(digits) = id.strip_prefix('f') else {
250        return false;
251    };
252    (1..=6).contains(&digits.len())
253        && digits.bytes().all(|b| b.is_ascii_digit())
254        && !digits.starts_with('0')
255}
256
257/// Reads one finding as the server receives it, refusing anything but the
258/// shape [`Finding`] has: exactly [`FINDING_KEYS`], an id [`is_finding_id`]
259/// accepts, a kind and a severity from their lists, two line numbers from
260/// 1 in order, a valid project key and path, and at most [`MAX_RELATED`]
261/// related files, each exactly [`FILE_REF_KEYS`].
262///
263/// This is what keeps note text out of the part of a report the API can
264/// read: a free-text member has nowhere to go. Whether each file named is
265/// one the server holds is the server's own check, after this.
266pub fn check_finding(value: &Value) -> Result<Finding, String> {
267    let Some(map) = value.as_object() else {
268        return Err("a finding is not an object".to_string());
269    };
270    let id = map
271        .get("id")
272        .and_then(Value::as_str)
273        .unwrap_or("(no id)")
274        .to_string();
275    exact_keys(map, &FINDING_KEYS, &format!("finding {id}"))?;
276    let finding: Finding =
277        serde_json::from_value(value.clone()).map_err(|e| format!("finding {id}: {e}"))?;
278    if !is_finding_id(&finding.id) {
279        return Err(format!("finding {id}: an id is f and a number, such as f1"));
280    }
281    if !KINDS.contains(&finding.kind.as_str()) {
282        return Err(format!("finding {id}: no kind {:?} exists", finding.kind));
283    }
284    if !SEVERITIES.contains(&finding.severity.as_str()) {
285        return Err(format!(
286            "finding {id}: no severity {:?} exists",
287            finding.severity
288        ));
289    }
290    let [first, last] = finding.lines;
291    if first == 0 || last < first {
292        return Err(format!(
293            "finding {id}: lines are two line numbers from 1, the first no later than the last"
294        ));
295    }
296    check_file(&finding.project_key, &finding.file_path)
297        .map_err(|e| format!("finding {id}: {e}"))?;
298    let related = map["related"]
299        .as_array()
300        .ok_or_else(|| format!("finding {id}: related is not a list"))?;
301    if related.len() > MAX_RELATED {
302        return Err(format!("finding {id}: at most {MAX_RELATED} related files"));
303    }
304    for r in related {
305        let Some(r) = r.as_object() else {
306            return Err(format!("finding {id}: a related file is not an object"));
307        };
308        exact_keys(r, &FILE_REF_KEYS, &format!("finding {id}'s related file"))?;
309    }
310    for r in &finding.related {
311        check_file(&r.project_key, &r.file_path).map_err(|e| format!("finding {id}: {e}"))?;
312    }
313    Ok(finding)
314}
315
316fn check_file(project_key: &str, file_path: &str) -> Result<(), String> {
317    validate_project_key(project_key).map_err(|e| e.to_string())?;
318    validate_file_path(file_path).map_err(|e| e.to_string())?;
319    Ok(())
320}
321
322fn exact_keys(
323    map: &serde_json::Map<String, Value>,
324    keys: &[&str],
325    what: &str,
326) -> Result<(), String> {
327    if let Some(extra) = map.keys().find(|k| !keys.contains(&k.as_str())) {
328        return Err(format!(
329            "{what} has a key {extra:?}; it may have only {}",
330            keys.join(", ")
331        ));
332    }
333    if let Some(missing) = keys.iter().find(|k| !map.contains_key(**k)) {
334        return Err(format!("{what} has no {missing}"));
335    }
336    Ok(())
337}
338
339/// What `details` holds: everything about a finding that quotes a note.
340/// The server stores it as it came and never reads it; the worker writes
341/// it and `recall eval` reads it.
342#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
343pub struct Details {
344    /// By finding id.
345    #[serde(default)]
346    pub findings: BTreeMap<String, FindingDetail>,
347    /// What was not checked, and why: a contradiction check the CLI could
348    /// not run, a project too large for one call, findings past
349    /// [`MAX_FINDINGS`].
350    #[serde(default)]
351    pub skipped: Vec<Skipped>,
352}
353
354/// The note text behind one finding.
355#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
356pub struct FindingDetail {
357    /// The lines concerned, as they are in the file. A secret is masked
358    /// here: a report is not another place for a key to be kept.
359    #[serde(default)]
360    pub excerpt: String,
361    /// Why this is a finding, and what to do about it.
362    #[serde(default)]
363    pub reasoning: String,
364    /// An edit that would resolve it, when there is one to suggest.
365    #[serde(default)]
366    pub suggested_edit: Option<SuggestedEdit>,
367}
368
369/// An edit `recall eval apply` can make: lines `lines` of one file,
370/// replaced with `replacement`, only if the file is still the version the
371/// evaluation read.
372#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
373pub struct SuggestedEdit {
374    /// The project.
375    pub project_key: String,
376    /// The file.
377    pub file_path: String,
378    /// [`content_sha256`](crate::content_sha256) of the version the
379    /// evaluation read: applying refuses any other.
380    pub base_sha256: String,
381    /// The first and last line replaced, counting from 1.
382    pub lines: [u32; 2],
383    /// What they become; empty removes them. Each line of it ends with a
384    /// newline, as the lines it replaces did.
385    pub replacement: String,
386}
387
388impl SuggestedEdit {
389    /// `content` with this edit made, or why it cannot be: the content is
390    /// not the version the edit was made against, or has fewer lines.
391    pub fn apply_to(&self, content: &str) -> Result<String, String> {
392        if crate::content_sha256(content) != self.base_sha256 {
393            return Err(format!(
394                "{} has changed since the report read it",
395                self.file_path
396            ));
397        }
398        let lines: Vec<&str> = content.split_inclusive('\n').collect();
399        let [first, last] = self.lines;
400        if first == 0 || last < first || last as usize > lines.len() {
401            return Err(format!("{} has no lines {first} to {last}", self.file_path));
402        }
403        let mut out = String::with_capacity(content.len() + self.replacement.len());
404        for line in &lines[..first as usize - 1] {
405            out.push_str(line);
406        }
407        out.push_str(&self.replacement);
408        for line in &lines[last as usize..] {
409            out.push_str(line);
410        }
411        Ok(out)
412    }
413}
414
415/// Something a run did not check.
416#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
417pub struct Skipped {
418    /// The check, one of [`KINDS`].
419    pub check: String,
420    /// The project it was not run for; empty for all of them.
421    #[serde(default)]
422    pub project_key: String,
423    /// Why.
424    pub reason: String,
425}
426
427#[cfg(test)]
428mod tests {
429    use super::*;
430    use serde_json::json;
431
432    fn finding() -> Value {
433        json!({"id": "f1", "kind": "secret", "severity": "high", "project_key": "acme/app",
434               "file_path": "topics/deploy.md", "lines": [12, 12], "related": []})
435    }
436
437    #[test]
438    fn a_finding_of_the_documented_shape_is_read() {
439        let f = check_finding(&finding()).unwrap();
440        assert_eq!((f.id.as_str(), f.lines), ("f1", [12, 12]));
441        assert_eq!(
442            serde_json::to_value(&f).unwrap(),
443            finding(),
444            "the keys, in order"
445        );
446    }
447
448    /// The whole point: no member a note could be written into.
449    #[test]
450    fn a_finding_with_any_other_key_is_refused() {
451        for key in ["excerpt", "reasoning", "note", "Id"] {
452            let mut f = finding();
453            f[key] = json!("tokens live in 1Password");
454            let err = check_finding(&f).unwrap_err();
455            assert!(err.contains(&format!("{key:?}")), "{err}");
456        }
457        let mut f = finding();
458        f["related"] = json!([{"project_key": "global:eko", "file_path": "tools.md", "why": "x"}]);
459        assert!(check_finding(&f).unwrap_err().contains("\"why\""));
460        let mut f = finding();
461        f.as_object_mut().unwrap().remove("related");
462        assert!(check_finding(&f).unwrap_err().contains("has no related"));
463    }
464
465    #[test]
466    fn every_member_is_held_to_its_shape() {
467        let cases: [(&str, Value); 9] = [
468            ("id", json!("finding one")),
469            ("id", json!("f0")),
470            ("id", json!("f1234567")),
471            ("kind", json!("typo")),
472            ("severity", json!("urgent")),
473            ("lines", json!([0, 1])),
474            ("lines", json!([5, 4])),
475            ("file_path", json!("../etc/passwd")),
476            (
477                "related",
478                json!(vec![json!({"project_key": "a/b", "file_path": "c.md"}); 11]),
479            ),
480        ];
481        for (key, value) in cases {
482            let mut f = finding();
483            f[key] = value.clone();
484            assert!(check_finding(&f).is_err(), "{key} = {value} was accepted");
485        }
486        assert!(is_finding_id("f999999"));
487    }
488
489    #[test]
490    fn a_request_defaults_to_every_project_without_contradictions() {
491        let req: EvaluationRequest = serde_json::from_str("{}").unwrap();
492        assert_eq!(req, EvaluationRequest::default());
493        assert!(!req.contradictions);
494    }
495
496    #[test]
497    fn an_edit_applies_only_to_the_version_it_was_made_against() {
498        let content = "# Deploy\n- token: abc\n- use make deploy\n";
499        let edit = SuggestedEdit {
500            project_key: "acme/app".into(),
501            file_path: "deploy.md".into(),
502            base_sha256: crate::content_sha256(content),
503            lines: [2, 2],
504            replacement: "- token: [removed]\n".into(),
505        };
506        assert_eq!(
507            edit.apply_to(content).unwrap(),
508            "# Deploy\n- token: [removed]\n- use make deploy\n"
509        );
510        let removal = SuggestedEdit {
511            replacement: String::new(),
512            lines: [2, 3],
513            ..edit.clone()
514        };
515        assert_eq!(removal.apply_to(content).unwrap(), "# Deploy\n");
516        assert!(edit.apply_to("# Deploy\n").unwrap_err().contains("changed"));
517        let beyond = SuggestedEdit {
518            lines: [4, 4],
519            ..edit
520        };
521        assert!(beyond.apply_to(content).unwrap_err().contains("no lines"));
522    }
523
524    #[test]
525    fn a_path() {
526        assert_eq!(evaluation_path("eval_a"), "/v1/evaluations/eval_a");
527    }
528}
recall_wire/evaluations.rs

recall_wire/
evaluations.rs
