Skip to main content

kernel_apis/
memory_records.rs

1//! Typed memory records (idea from Mirror's `memory_record.rs`, fitted to Ultron's journal + Hindsight memory).
2//!
3//! `memory.propose` may carry a `type` (preference, fact, procedure, pitfall, reference), a `situation` (one short
4//! line: when a future session should use the record), a stable `key` (a kebab-case handle: proposing the same key
5//! again replaces that record instead of adding another) and `pathGlobs` (repository-relative file patterns the
6//! record applies to). The text still lives only in Hindsight and the session journal still holds hashes only; the
7//! typed metadata goes to a [`RecordIndex`] every session of the machine shares, so a later session can find a
8//! record by key, point at it when a matching file changes ([`crate::memory_pointers`]) and read it by id.
9//!
10//! Bounds (bytes of UTF-8): content per type ([`content_cap`]): pitfall 1 KiB, preference 2 KiB, fact and reference
11//! 4 KiB, procedure 8 KiB; situation 200 bytes on one line; key 64 bytes; at most 16 globs of at most 256 bytes;
12//! source 1 KiB. Records without these fields are untyped and behave exactly as before.
13
14use crate::memory::{ErrorCode, MemoryError};
15use kernel::store::{DocumentStore, MemoryDocumentStore};
16
17use fancy_regex::Regex;
18use serde_json::{Map, Value, json};
19use std::path::PathBuf;
20use std::sync::{Arc, Mutex};
21
22pub const RECORD_TYPES: [&str; 5] = ["preference", "fact", "procedure", "pitfall", "reference"];
23pub const MAX_SITUATION_BYTES: usize = 200;
24pub const MAX_KEY_BYTES: usize = 64;
25pub const MAX_GLOBS: usize = 16;
26pub const MAX_GLOB_BYTES: usize = 256;
27pub const MAX_SOURCE_BYTES: usize = 1024;
28
29/// The most content (UTF-8 bytes) a record of `kind` may hold.
30pub fn content_cap(kind: &str) -> usize {
31    match kind {
32        "pitfall" => 1024,
33        "preference" => 2048,
34        "procedure" => 8192,
35        _ => 4096,
36    }
37}
38
39/// The typed part of a record.
40#[derive(Debug, Clone, PartialEq, Eq)]
41pub struct Typed {
42    pub kind: String,
43    pub key: Option<String>,
44    pub situation: String,
45    pub path_globs: Vec<String>,
46    /// Where an imported record came from (`ultron memory import`); never set from the REPL.
47    pub source: Option<String>,
48}
49
50impl Typed {
51    pub fn to_json(&self) -> Value {
52        let mut out = Map::new();
53        out.insert("type".into(), self.kind.clone().into());
54        if let Some(key) = &self.key {
55            out.insert("key".into(), key.clone().into());
56        }
57        out.insert("situation".into(), self.situation.clone().into());
58        out.insert("pathGlobs".into(), json!(self.path_globs));
59        if let Some(source) = &self.source {
60            out.insert("source".into(), source.clone().into());
61        }
62        Value::Object(out)
63    }
64
65    /// Every typed text the sensitive check must see besides the content.
66    pub fn screened(&self) -> String {
67        let mut parts = vec![self.situation.clone()];
68        parts.extend(self.key.clone());
69        parts.extend(self.path_globs.iter().cloned());
70        parts.join("\n")
71    }
72}
73
74const TYPED_FIELDS: [&str; 5] = ["type", "key", "situation", "pathGlobs", "source"];
75
76fn invalid() -> MemoryError {
77    MemoryError::code(ErrorCode::InvalidInput)
78}
79
80fn has_control(text: &str) -> bool {
81    text.chars().any(char::is_control)
82}
83
84/// A kebab-case slug: lowercase ASCII letters and digits joined by single dashes.
85pub fn is_kebab(key: &str) -> bool {
86    !key.is_empty()
87        && key.len() <= MAX_KEY_BYTES
88        && !key.starts_with('-')
89        && !key.ends_with('-')
90        && !key.contains("--")
91        && key
92            .bytes()
93            .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-')
94}
95
96/// The typed fields of a propose payload: `None` for an untyped record; INVALID_INPUT when a field is malformed,
97/// `type` or `situation` is missing, or `text` is over the type's cap.
98pub fn typed_fields(payload: &Map<String, Value>, text: &str) -> Result<Option<Typed>, MemoryError> {
99    if !TYPED_FIELDS.iter().any(|field| payload.contains_key(*field)) {
100        return Ok(None);
101    }
102    let kind = payload
103        .get("type")
104        .and_then(Value::as_str)
105        .filter(|kind| RECORD_TYPES.contains(kind))
106        .ok_or_else(invalid)?
107        .to_string();
108    let situation = payload.get("situation").and_then(Value::as_str).ok_or_else(invalid)?;
109    let situation = situation.trim();
110    if situation.is_empty() || situation.len() > MAX_SITUATION_BYTES || has_control(situation) {
111        return Err(invalid());
112    }
113    let key = match payload.get("key") {
114        None => None,
115        Some(Value::String(key)) if is_kebab(key) => Some(key.clone()),
116        Some(_) => return Err(invalid()),
117    };
118    let path_globs = match payload.get("pathGlobs") {
119        None => Vec::new(),
120        Some(Value::Array(items)) if items.len() <= MAX_GLOBS => {
121            let mut globs: Vec<String> = Vec::new();
122            for item in items {
123                let glob = item.as_str().ok_or_else(invalid)?;
124                Glob::new(glob)?;
125                if !globs.iter().any(|seen| seen == glob) {
126                    globs.push(glob.to_string());
127                }
128            }
129            globs
130        }
131        Some(_) => return Err(invalid()),
132    };
133    let source = match payload.get("source") {
134        None => None,
135        Some(Value::String(source)) if source.len() <= MAX_SOURCE_BYTES && !has_control(source) => Some(source.clone()),
136        Some(_) => return Err(invalid()),
137    };
138    if text.len() > content_cap(&kind) {
139        return Err(invalid());
140    }
141    Ok(Some(Typed {
142        kind,
143        key,
144        situation: situation.to_string(),
145        path_globs,
146        source,
147    }))
148}
149
150/// A repository-relative file pattern: `*` and `?` stay within one path segment, `**` as a whole segment spans any
151/// number of segments (also none), `[abc]` / `[!abc]` match one character of a set, `{a,b}` either alternative.
152#[derive(Debug, Clone)]
153pub struct Glob {
154    regex: Regex,
155}
156
157impl Glob {
158    /// INVALID_INPUT for an empty, absolute or escaping (`..`) pattern, `//`, an unbalanced `[`/`{` or one over
159    /// 256 bytes.
160    pub fn new(pattern: &str) -> Result<Glob, MemoryError> {
161        if pattern.is_empty()
162            || pattern.len() > MAX_GLOB_BYTES
163            || pattern.starts_with('/')
164            || pattern.contains("//")
165            || pattern.contains('\\')
166            || has_control(pattern)
167            || pattern.split('/').any(|segment| segment == "..")
168        {
169            return Err(invalid());
170        }
171        let source = translate(pattern).ok_or_else(invalid)?;
172        let regex = Regex::new(&source).map_err(|_| invalid())?;
173        Ok(Glob { regex })
174    }
175
176    /// Whether the repository-relative `path` (with `/`) matches.
177    pub fn matches(&self, path: &str) -> bool {
178        self.regex.is_match(path).unwrap_or(false)
179    }
180}
181
182/// Whether `path` matches `pattern`; an invalid pattern matches nothing.
183pub fn glob_matches(pattern: &str, path: &str) -> bool {
184    Glob::new(pattern).is_ok_and(|glob| glob.matches(path))
185}
186
187fn escape(ch: char, out: &mut String) {
188    if "\\.+*?()|[]{}^$#&-~".contains(ch) {
189        out.push('\\');
190    }
191    out.push(ch);
192}
193
194fn translate(pattern: &str) -> Option<String> {
195    let chars: Vec<char> = pattern.chars().collect();
196    let mut out = String::from("^");
197    let mut index = 0;
198    let mut in_brace = false;
199    while index < chars.len() {
200        let ch = chars[index];
201        match ch {
202            '*' if chars.get(index + 1) == Some(&'*') => {
203                let starts = index == 0 || chars[index - 1] == '/';
204                let next = chars.get(index + 2);
205                if !starts || !matches!(next, None | Some('/')) {
206                    return None;
207                }
208                if next == Some(&'/') {
209                    // `**/`: any number of whole segments, also none.
210                    out.push_str("(?:[^/]+/)*");
211                    index += 3;
212                } else if index == 0 {
213                    out.push_str(".*");
214                    index += 2;
215                } else {
216                    // `dir/**`: everything below dir.
217                    out.pop();
218                    out.push_str("(?:/.*)?");
219                    index += 2;
220                }
221                continue;
222            }
223            '*' => out.push_str("[^/]*"),
224            '?' => out.push_str("[^/]"),
225            '[' => {
226                let end = chars[index + 1..].iter().position(|c| *c == ']')? + index + 1;
227                let mut set: Vec<char> = chars[index + 1..end].to_vec();
228                let negated = set.first() == Some(&'!');
229                if negated {
230                    set.remove(0);
231                }
232                if set.is_empty() || set.contains(&'/') || set.contains(&'[') {
233                    return None;
234                }
235                out.push('[');
236                if negated {
237                    out.push('^');
238                    out.push('/');
239                }
240                for c in set {
241                    if c != '-' && "\\^&~".contains(c) {
242                        out.push('\\');
243                    }
244                    out.push(c);
245                }
246                out.push(']');
247                index = end + 1;
248                continue;
249            }
250            ']' => return None,
251            '{' if !in_brace => {
252                in_brace = true;
253                out.push_str("(?:");
254            }
255            ',' if in_brace => out.push('|'),
256            '}' if in_brace => {
257                in_brace = false;
258                out.push(')');
259            }
260            '{' | '}' => return None,
261            other => escape(other, &mut out),
262        }
263        index += 1;
264    }
265    if in_brace {
266        return None;
267    }
268    out.push('$');
269    Some(out)
270}
271
272/// One indexed record: its Hindsight document id, typed fields, scope and the identity tags it was stored with.
273#[derive(Debug, Clone, PartialEq, Eq)]
274pub struct IndexEntry {
275    pub id: String,
276    pub typed: Typed,
277    pub scope: String,
278    pub tags: Vec<String>,
279    pub text_hash: String,
280    pub updated_at: String,
281}
282
283impl IndexEntry {
284    fn to_json(&self) -> Value {
285        let mut out = Map::new();
286        out.insert("id".into(), self.id.clone().into());
287        if let Value::Object(typed) = self.typed.to_json() {
288            out.extend(typed);
289        }
290        out.insert("scope".into(), self.scope.clone().into());
291        out.insert("tags".into(), json!(self.tags));
292        out.insert("textHash".into(), self.text_hash.clone().into());
293        out.insert("updatedAt".into(), self.updated_at.clone().into());
294        Value::Object(out)
295    }
296
297    fn from_json(value: &Value) -> Option<IndexEntry> {
298        let map = value.as_object()?;
299        let text = |key: &str| map.get(key).and_then(Value::as_str).map(str::to_string);
300        let strings = |key: &str| -> Option<Vec<String>> {
301            map.get(key)?
302                .as_array()?
303                .iter()
304                .map(|item| item.as_str().map(str::to_string))
305                .collect()
306        };
307        Some(IndexEntry {
308            id: text("id")?,
309            typed: Typed {
310                kind: text("type").filter(|kind| RECORD_TYPES.contains(&kind.as_str()))?,
311                key: text("key"),
312                situation: text("situation")?,
313                path_globs: strings("pathGlobs")?,
314                source: text("source"),
315            },
316            scope: text("scope")?,
317            tags: strings("tags")?,
318            text_hash: text("textHash")?,
319            updated_at: text("updatedAt").unwrap_or_default(),
320        })
321    }
322
323    /// Whether a session whose scope filters are `filters` sees this record (same scope, same identity tags).
324    pub fn visible(&self, filters: &Map<String, Value>) -> bool {
325        let Some(tags) = filters.get(&self.scope).and_then(Value::as_array) else {
326            return false;
327        };
328        tags.len() == self.tags.len()
329            && tags
330                .iter()
331                .all(|tag| tag.as_str().is_some_and(|tag| self.tags.iter().any(|own| own == tag)))
332    }
333}
334
335/// The typed records of every session on the machine: `{"version": 1, "records": [...]}` in one document. Writes
336/// read, change and replace the document while holding an exclusive lock on `<path>.lock` (file-backed), so two
337/// sessions never lose each other's records.
338pub struct RecordIndex {
339    store: Arc<dyn DocumentStore>,
340    lock_path: Option<PathBuf>,
341    local: Mutex<()>,
342}
343
344impl Default for RecordIndex {
345    fn default() -> Self {
346        RecordIndex::in_memory()
347    }
348}
349
350impl RecordIndex {
351    pub fn in_memory() -> RecordIndex {
352        RecordIndex {
353            store: Arc::new(MemoryDocumentStore::default()),
354            lock_path: None,
355            local: Mutex::new(()),
356        }
357    }
358
359    /// The index file every session of an agent directory shares.
360    pub fn file(path: impl Into<PathBuf>) -> RecordIndex {
361        let path = path.into();
362        let lock_path = PathBuf::from(format!("{}.lock", path.display()));
363        RecordIndex {
364            store: Arc::new(kernel::store::FileDocumentStore::new(path)),
365            lock_path: Some(lock_path),
366            local: Mutex::new(()),
367        }
368    }
369
370    /// `<agent dir>/memory/records.json`.
371    pub fn for_agent_dir(agent_dir: &std::path::Path) -> RecordIndex {
372        RecordIndex::file(agent_dir.join("memory").join("records.json"))
373    }
374
375    fn parse(document: Option<Value>) -> Result<Vec<IndexEntry>, String> {
376        let Some(document) = document else {
377            return Ok(Vec::new());
378        };
379        if document["version"] != 1 {
380            return Err("unsupported memory record index".into());
381        }
382        Ok(document["records"]
383            .as_array()
384            .into_iter()
385            .flatten()
386            .filter_map(IndexEntry::from_json)
387            .collect())
388    }
389
390    /// Every indexed record, in the order they were first indexed.
391    pub fn entries(&self) -> Result<Vec<IndexEntry>, String> {
392        Self::parse(self.store.read()?)
393    }
394
395    /// The record with document id `id`.
396    pub fn entry(&self, id: &str) -> Result<Option<IndexEntry>, String> {
397        Ok(self.entries()?.into_iter().find(|entry| entry.id == id))
398    }
399
400    /// The visible record holding `key` in `scope`.
401    pub fn by_key(&self, key: &str, scope: &str, filters: &Map<String, Value>) -> Result<Option<IndexEntry>, String> {
402        Ok(self
403            .entries()?
404            .into_iter()
405            .find(|entry| entry.scope == scope && entry.typed.key.as_deref() == Some(key) && entry.visible(filters)))
406    }
407
408    fn change(&self, change: impl FnOnce(&mut Vec<IndexEntry>)) -> Result<(), String> {
409        let _local = jsrt::sync::lock(&self.local);
410        let _file = match &self.lock_path {
411            Some(path) => {
412                if let Some(parent) = path.parent() {
413                    std::fs::create_dir_all(parent).map_err(|e| e.to_string())?;
414                }
415                let file = std::fs::OpenOptions::new()
416                    .create(true)
417                    .truncate(false)
418                    .write(true)
419                    .open(path)
420                    .map_err(|e| e.to_string())?;
421                file.lock().map_err(|e| e.to_string())?;
422                Some(file)
423            }
424            None => None,
425        };
426        let mut entries = Self::parse(self.store.read()?)?;
427        change(&mut entries);
428        let records: Vec<Value> = entries.iter().map(IndexEntry::to_json).collect();
429        self.store.write(&json!({"version": 1, "records": records}))
430    }
431
432    /// Add `entry`, replacing the record with its id (it keeps its place).
433    pub fn upsert(&self, entry: IndexEntry) -> Result<(), String> {
434        self.change(|entries| match entries.iter_mut().find(|e| e.id == entry.id) {
435            Some(existing) => *existing = entry,
436            None => entries.push(entry),
437        })
438    }
439
440    pub fn remove(&self, id: &str) -> Result<(), String> {
441        self.change(|entries| entries.retain(|entry| entry.id != id))
442    }
443}