Skip to main content

kernel_apis/memory/
dream.rs

1//! Dreaming over a curating backend (memorantado; `supports("claims")`): re-checking kept claims, and the pages
2//! dreaming writes (the startup brief and topic pages) that a backend without a model cannot write itself.
3//!
4//! [`MemoryService::dream`]: the claims of each scope not confirmed for `days` are checked in plain code first: a
5//! claim whose quoted files still hold their quotes is confirmed (with the files as evidence), with no model call. A
6//! claim whose quote is gone goes to the judge (`judge.decide`: supported, stale or contradicted; anything it is not
7//! sure of is left for review): supported is revised (the claim kept with the quotes that still hold), stale and
8//! contradicted are retracted. A claim kept in the window and not looked at yet that shares its situation with an
9//! older claim of the same scope is asked about too: contradicting claims are flagged as a conflict for a person or
10//! a later dream, and once settled the newer claim is confirmed so it is not asked again. Nothing changed, nothing
11//! asked: the judge is called only for those two cases.
12
13use super::{ErrorCode, MemoryError, MemoryService, Quote, Result, quoted_path, staleness};
14use kernel::Signal;
15use serde_json::{Map, Value, json};
16use std::path::{Path, PathBuf};
17use std::sync::Arc;
18
19/// `judge.decide(state, questions, kind)` as the session asks it: the verdict's JSON.
20pub type Decide = Arc<dyn Fn(&str, &Value, &str, &Signal) -> std::result::Result<Value, String> + Send + Sync>;
21
22/// The startup brief's budget: it opens every context, so it stays short.
23pub const BRIEF_MAX_LINES: usize = 200;
24pub const BRIEF_MAX_CHARS: usize = 4000;
25/// A topic page's budget.
26pub const TOPIC_MAX_LINES: usize = 120;
27pub const TOPIC_MAX_CHARS: usize = 6000;
28/// Claims of a scope one dream reads, and checks at most.
29const LIST_LIMIT: u64 = 200;
30const DEFAULT_LIMIT: usize = 50;
31const DEFAULT_DAYS: i64 = 7;
32const DAY_MS: i64 = 86_400_000;
33/// The most of a changed file the judge is shown.
34const FILE_CHARS: usize = 6000;
35
36fn status_question() -> Value {
37    json!({"status": {
38        "type": "choice",
39        "instructions": "A coding agent kept the memory claim below because the quoted file text backed it. That \
40            text is no longer in the file. Judge the claim against the file as it is now.",
41        "criteria": {
42            "supported": "The current file still backs the claim: the text moved or was reworded.",
43            "stale": "The claim no longer applies: what it describes is gone or changed.",
44            "contradicted": "The current file says the opposite of the claim.",
45        },
46    }})
47}
48
49fn contradiction_question() -> Value {
50    json!({"contradict": {
51        "type": "yesno",
52        "instructions": "Do these two memory claims contradict each other, so that an agent cannot follow both?",
53    }})
54}
55
56/// The answer to `name` when the verdict settled it (judge::Verdict `settled`), else `None` (review, abstained).
57fn settled<'a>(verdict: &'a Value, name: &str) -> Option<&'a Value> {
58    (verdict["settled"] == true).then(|| &verdict["answers"][name]["value"])
59}
60
61fn quotes_of(metadata: &Value) -> Vec<Quote> {
62    let parsed: Vec<Value> = metadata["ultron_quotes"]
63        .as_str()
64        .and_then(|text| serde_json::from_str(text).ok())
65        .unwrap_or_default();
66    parsed
67        .iter()
68        .filter_map(|q| {
69            Some(Quote {
70                path: quoted_path(q["path"].as_str()?)?.to_string(),
71                quote: q["quote"].as_str()?.to_string(),
72            })
73        })
74        .collect()
75}
76
77/// When a claim was last confirmed, else made (ISO strings compare in time order).
78fn confirmed_at(claim: &Value) -> &str {
79    claim["lastConfirmedAt"]
80        .as_str()
81        .or(claim["createdAt"].as_str())
82        .unwrap_or("")
83}
84
85fn situation(claim: &Value) -> Option<&str> {
86    claim["metadata"]["ultron_situation"]
87        .as_str()
88        .map(jsrt::string::trim)
89        .filter(|s| !s.is_empty())
90}
91
92/// Whether `text` fits a page's budget: the reason when it does not.
93pub fn page_budget(text: &str, topic: bool) -> std::result::Result<(), String> {
94    let (lines, chars) = if topic {
95        (TOPIC_MAX_LINES, TOPIC_MAX_CHARS)
96    } else {
97        (BRIEF_MAX_LINES, BRIEF_MAX_CHARS)
98    };
99    let what = if topic { "a topic page" } else { "the brief" };
100    if jsrt::string::trim(text).is_empty() {
101        return Err(format!("{what} is empty"));
102    }
103    if text.lines().count() > lines || text.chars().count() > chars {
104        return Err(format!("{what} is over its budget ({lines} lines, {chars} characters)"));
105    }
106    Ok(())
107}
108
109impl MemoryService {
110    /// Have dreaming's verdicts asked of the judge (the session's `judge.decide`).
111    pub fn with_judge(mut self, decide: Decide) -> Self {
112        self.judge = Some(decide);
113        self
114    }
115
116    fn claims_backend(&self) -> Result<()> {
117        if self.backend.supports("claims") {
118            Ok(())
119        } else {
120            Err(MemoryError::code(ErrorCode::UnsupportedApi))
121        }
122    }
123
124    fn backend_call<T>(&self, outcome: std::result::Result<T, String>) -> Result<T> {
125        outcome.map_err(|error| MemoryError::of(&error))
126    }
127
128    /// The files under `root` that `quotes` cite, as they are now, for a confirmation's evidence.
129    fn evidence_of(&self, root: &Path, quotes: &[Quote]) -> Value {
130        Value::Array(self.quoted_files_in(root, quotes))
131    }
132
133    /// The project a dream or page names (`project`: its directory), else the session's: its tags, and the
134    /// directory its quotes are checked in.
135    fn project_scope(&self, payload: &Map<String, Value>) -> Result<(Vec<String>, Option<PathBuf>)> {
136        match payload.get("project").and_then(Value::as_str) {
137            Some(path) if Path::new(path).is_absolute() => {
138                let tags = vec![format!("ultron:project:{}", super::directory_identity(path))];
139                Ok((tags, Some(PathBuf::from(path))))
140            }
141            Some(_) => Err(MemoryError::code(ErrorCode::InvalidInput)),
142            None => Ok((self.tags("project")?, self.root.clone())),
143        }
144    }
145
146    /// The scopes a dream re-checks: the project's (with its directory) and the global one (whose quotes name
147    /// files of whatever project they came from, so they are not checked).
148    fn dream_scopes(&self, payload: &Map<String, Value>) -> Result<Vec<(Vec<String>, Option<PathBuf>)>> {
149        let mut scopes = Vec::new();
150        if let Ok((tags, root)) = self.project_scope(payload) {
151            scopes.push((tags, root));
152        } else if payload.contains_key("project") {
153            return Err(MemoryError::code(ErrorCode::InvalidInput));
154        }
155        if let Ok(tags) = self.tags("global") {
156            scopes.push((tags, None));
157        }
158        Ok(scopes)
159    }
160
161    /// The identity a page of `scope` is named by: the project's (`project`, else the session's) or the global one.
162    fn page_identity(&self, scope: &str, payload: &Map<String, Value>) -> Result<String> {
163        let tag = match scope {
164            "project" => self.project_scope(payload)?.0.into_iter().next(),
165            _ => self.tags(scope)?.into_iter().next(),
166        };
167        tag.and_then(|tag| super::tag_identity(&tag))
168            .ok_or(MemoryError::code(ErrorCode::UnsupportedScope))
169    }
170
171    /// `memory.dream({days?, limit?})`: see the module docs. `{confirmed, revised, retracted, conflicts, review,
172    /// unchecked, judged}`; ids are the backend's claim ids.
173    pub fn dream(&self, payload: &Map<String, Value>, signal: &Signal) -> Result<Value> {
174        self.claims_backend()?;
175        let days = payload.get("days").and_then(Value::as_i64).unwrap_or(DEFAULT_DAYS);
176        let limit = payload
177            .get("limit")
178            .and_then(Value::as_u64)
179            .map_or(DEFAULT_LIMIT, |n| n as usize);
180        let cutoff = jsrt::date::iso_string((self.clock)() - days * DAY_MS);
181        let (mut confirmed, mut revised, mut retracted, mut conflicts, mut review) =
182            (Vec::new(), Vec::new(), Vec::new(), Vec::new(), Vec::new());
183        let mut unchecked = 0;
184        let mut changed: Vec<(Value, Vec<Quote>, String, PathBuf)> = Vec::new();
185        let mut pairs: Vec<(Value, Value)> = Vec::new();
186        for (tags, root) in self.dream_scopes(payload)? {
187            let request = json!({
188                "tags": tags, "tagsMatch": "exact", "status": "active", "sort": "confirmed", "limit": LIST_LIMIT,
189            });
190            let listed = self.backend_call(self.backend.list(&request, signal))?;
191            let claims: Vec<Value> = listed["claims"].as_array().cloned().unwrap_or_default();
192            for claim in claims.iter().filter(|c| confirmed_at(c) < cutoff.as_str()).take(limit) {
193                let quotes = quotes_of(&claim["metadata"]);
194                // A global claim quotes files of whatever project it came from: nothing here to check it against.
195                let (Some(root), false) = (root.as_deref(), quotes.is_empty()) else {
196                    unchecked += 1;
197                    continue;
198                };
199                let id = claim["id"].as_str().unwrap_or_default();
200                match staleness(root, &claim["metadata"]) {
201                    None => {
202                        let evidence = json!({"evidence": self.evidence_of(root, &quotes), "tags": tags});
203                        self.backend_call(self.backend.confirm(id, &evidence, signal))?;
204                        confirmed.push(Value::from(id));
205                    }
206                    Some(why) => changed.push((claim.clone(), quotes, why, root.to_path_buf())),
207                }
208            }
209            // A claim kept in the window and not yet looked at, beside an older one of the same situation.
210            for newer in claims
211                .iter()
212                .filter(|c| c["createdAt"].as_str() >= Some(cutoff.as_str()) && c["lastConfirmedAt"].is_null())
213            {
214                for older in &claims {
215                    if older["createdAt"].as_str() < newer["createdAt"].as_str()
216                        && older["id"] != newer["id"]
217                        && situation(older).is_some()
218                        && situation(older) == situation(newer)
219                    {
220                        pairs.push((newer.clone(), older.clone()));
221                    }
222                }
223            }
224        }
225        let judge = match &self.judge {
226            Some(judge) => judge.clone(),
227            None => {
228                for (claim, _, why, _) in &changed {
229                    review.push(json!({"id": claim["id"], "why": why}));
230                }
231                for (newer, older) in &pairs {
232                    review.push(json!({"id": newer["id"], "why": format!("may contradict {}", older["id"])}));
233                }
234                return Ok(json!({
235                    "confirmed": confirmed, "revised": revised, "retracted": retracted, "conflicts": conflicts,
236                    "review": review, "unchecked": unchecked, "judged": 0,
237                }));
238            }
239        };
240        let mut judged = 0;
241        for (claim, quotes, why, root) in changed {
242            let id = claim["id"].as_str().unwrap_or_default().to_string();
243            let mut state = format!("Memory claim: {}\n", claim["text"].as_str().unwrap_or_default());
244            let mut held = Vec::new();
245            for quote in &quotes {
246                let current = super::read_quoted(&root.join(&quote.path));
247                let holds = current
248                    .as_deref()
249                    .is_some_and(|text| super::squeezed(text).contains(&super::squeezed(&quote.quote)));
250                state.push_str(&format!("\nQuoted from {}: \"{}\"\n", quote.path, quote.quote));
251                match (&current, holds) {
252                    (_, true) => {
253                        state.push_str("The file still holds this text.\n");
254                        held.push(quote.clone());
255                    }
256                    (Some(text), false) => {
257                        let shown: String = text.chars().take(FILE_CHARS).collect();
258                        state.push_str(&format!("The file now reads:\n{shown}\n"));
259                    }
260                    (None, false) => state.push_str("The file is gone.\n"),
261                }
262            }
263            judged += 1;
264            let verdict = judge(&state, &status_question(), "memory.evidence", signal)
265                .map_err(|_| MemoryError::code(ErrorCode::OperationError))?;
266            match settled(&verdict, "status").and_then(Value::as_str) {
267                Some("supported") => {
268                    let mut metadata = claim["metadata"].clone();
269                    match held.is_empty() {
270                        true => {
271                            if let Some(map) = metadata.as_object_mut() {
272                                map.remove("ultron_quotes");
273                            }
274                        }
275                        false => {
276                            let kept: Vec<Value> =
277                                held.iter().map(|q| json!({"path": q.path, "quote": q.quote})).collect();
278                            metadata["ultron_quotes"] = Value::Array(kept).to_string().into();
279                            // The host checked what remains: only it marks a claim verified.
280                            metadata["ultron_evidence_class"] = "verified".into();
281                        }
282                    }
283                    let request = json!({
284                        "text": claim["text"], "metadata": metadata, "evidence": self.evidence_of(&root, &held),
285                        "tags": claim["tags"],
286                    });
287                    let new = self.backend_call(self.backend.supersede(&id, &request, signal))?;
288                    revised.push(json!({"id": id, "by": new["id"]}));
289                }
290                Some(answer @ ("stale" | "contradicted")) => {
291                    let reason = format!("dream: {answer} ({why})");
292                    let request = json!({"state": "invalidated", "reason": reason});
293                    self.backend_call(self.backend.curate(&id, &request, signal))?;
294                    retracted.push(json!({"id": id, "reason": reason}));
295                }
296                _ => review.push(json!({"id": id, "why": why, "verdict": verdict["rung"]})),
297            }
298        }
299        let mut looked_at: Vec<Value> = Vec::new();
300        for (newer, older) in pairs {
301            let state = format!(
302                "Both claims apply when: {}\n\nClaim A (kept recently): {}\n\nClaim B (kept earlier): {}",
303                situation(&newer).unwrap_or_default(),
304                newer["text"].as_str().unwrap_or_default(),
305                older["text"].as_str().unwrap_or_default(),
306            );
307            judged += 1;
308            let verdict = judge(&state, &contradiction_question(), "memory.conflict", signal)
309                .map_err(|_| MemoryError::code(ErrorCode::OperationError))?;
310            match settled(&verdict, "contradict") {
311                Some(Value::Bool(true)) => {
312                    let request = json!({
313                        "id": newer["id"], "conflictingId": older["id"],
314                        "reason": format!("dream: the claims contradict when {}", situation(&newer).unwrap_or_default()),
315                    });
316                    self.backend_call(self.backend.conflict(&request, signal))?;
317                    conflicts.push(json!({"id": newer["id"], "conflictingId": older["id"]}));
318                }
319                Some(_) => {}
320                None => {
321                    review.push(json!({"id": newer["id"], "why": format!("may contradict {}", older["id"])}));
322                    continue;
323                }
324            }
325            // Settled: the newer claim is marked looked at (confirmed), so the next dream does not ask again.
326            if !looked_at.contains(&newer["id"]) {
327                let id = newer["id"].as_str().unwrap_or_default();
328                self.backend_call(self.backend.confirm(id, &json!({}), signal))?;
329                looked_at.push(newer["id"].clone());
330            }
331        }
332        Ok(json!({
333            "confirmed": confirmed, "revised": revised, "retracted": retracted, "conflicts": conflicts,
334            "review": review, "unchecked": unchecked, "judged": judged,
335        }))
336    }
337
338    /// The id of the brief of `scope` (`project`: the project brief, `global`: the user's brief), or of its topic
339    /// page `topic`, for the scope's `identity`.
340    pub(super) fn page_id(scope: &str, identity: &str, topic: Option<&str>) -> String {
341        match (scope, topic) {
342            (_, Some(topic)) => format!("ultron-topic-{scope}-{identity}-{topic}"),
343            ("global", None) => format!("ultron-user-{identity}"),
344            _ => format!("ultron-project-{identity}"),
345        }
346    }
347
348    fn pages_tag(scope: &str, identity: &str) -> String {
349        format!("ultron:pages:{scope}:{identity}")
350    }
351
352    /// `memory.brief(text, scope, topic?)` (dreaming): writes the startup brief of `scope` (what opens each context
353    /// as the project's or the user's brief), or its topic page `topic`, replacing the one before. Within
354    /// [`page_budget`]; a text holding a secret is refused. `{id, scope, topic}`.
355    pub fn write_page(&self, payload: &Map<String, Value>, signal: &Signal) -> Result<Value> {
356        self.claims_backend()?;
357        let text = payload.get("text").and_then(Value::as_str).unwrap_or_default();
358        let scope = super::scope_of(payload.get("scope").unwrap_or(&json!("project")))?;
359        let topic = payload.get("topic").and_then(Value::as_str);
360        if page_budget(text, topic.is_some()).is_err()
361            || topic.is_some_and(|t| !crate::memory_records::is_kebab(t))
362            || scope == "session"
363        {
364            return Err(MemoryError::code(ErrorCode::InvalidInput));
365        }
366        let check = json!({"action": "propose", "text": text, "scope": scope});
367        let decision = super::gate_decision(
368            &(self.gate)(&check, signal).map_err(|_| MemoryError::code(ErrorCode::OperationError))?,
369            false,
370        )?;
371        if decision["action"] != "keep" {
372            return Err(MemoryError::code(ErrorCode::InvalidInput));
373        }
374        let identity = self.page_identity(scope, payload)?;
375        let id = Self::page_id(scope, &identity, topic);
376        let mut metadata =
377            json!({"ultron_source": "dream", "ultron_page": if topic.is_some() { "topic" } else { "brief" }});
378        if let Some(topic) = topic {
379            metadata["ultron_topic"] = topic.into();
380        }
381        let request = json!({
382            "items": [{"content": text, "document_id": id, "tags": [Self::pages_tag(scope, &identity)],
383                "update_mode": "replace", "metadata": metadata}],
384            "async": false,
385        });
386        self.backend_call(self.backend.retain(&request, signal))?;
387        Ok(json!({"id": id, "scope": scope, "topic": topic}))
388    }
389
390    /// `memory.topic(topic?, scope)`: the topic page `topic` of `scope` (`{topic, content, refreshedAt}`, null when
391    /// there is none), or without a topic the names of the scope's topic pages.
392    pub fn topic(&self, payload: &Map<String, Value>, signal: &Signal) -> Result<Value> {
393        self.claims_backend()?;
394        let scope = super::scope_of(payload.get("scope").unwrap_or(&json!("project")))?;
395        let identity = self.page_identity(scope, payload)?;
396        match payload.get("topic").and_then(Value::as_str) {
397            Some(topic) => {
398                let id = Self::page_id(scope, &identity, Some(topic));
399                let page = self.backend_call(self.backend.mental_model(&id, signal))?;
400                Ok(match page["content"].as_str() {
401                    Some(content) => {
402                        json!({"topic": topic, "content": content, "refreshedAt": page["last_refreshed_at"]})
403                    }
404                    None => Value::Null,
405                })
406            }
407            None => {
408                let request = json!({"tags": [Self::pages_tag(scope, &identity)], "tagsMatch": "exact", "status": "active", "limit": LIST_LIMIT});
409                let listed = self.backend_call(self.backend.list(&request, signal))?;
410                let topics: Vec<Value> = listed["claims"]
411                    .as_array()
412                    .into_iter()
413                    .flatten()
414                    .filter_map(|c| c["metadata"]["ultron_topic"].as_str().map(Value::from))
415                    .collect();
416                Ok(Value::Array(topics))
417            }
418        }
419    }
420}