Skip to main content

rig_memory/
format.rs

1//! The saved format, [`FORMAT`]: one JSON-lines log per agent, named
2//! by the agent's id, plus the session's own ([`SESSION`]). Each line is a
3//! [`Stamp`] and one record: a `header` first, then `state` (a saved value
4//! by key; the latest wins, `null` removes it), `retired`, or one
5//! [`Change`](rig_core::conversation::Change) of the agent's conversation
6//! (`push`, `splice`, `halt`, `aside`, `reset`) with who made it (`by`).
7//! The header's `lineage` is the log's own
8//! [`Lineage`](rig_core::conversation::Lineage), chosen when it was
9//! started, so a conversation saved to it can be told from one that only
10//! has as many changes. Its `rewrites` say who rewrote the log in place
11//! ([`rewrite`](crate::rewrite)) and when; each rewrite gives the log a
12//! fresh lineage.
13//!
14//! A record keeps when each entry was made: a new entry was made when its
15//! record says (`t`), an entry put back as it was says so itself (`made`),
16//! and a reset's state carries each entry's.
17//!
18//! ```json
19//! {"seq":0,"n":0,"t":1791779409181,"type":"header","format":"rig-session/4","log":"cd9a…","lineage":8125…}
20//! {"seq":1,"n":1,"t":1791779409181,"type":"state","key":"my_app::Mode","value":"Plan"}
21//! {"seq":2,"n":2,"t":1791779409182,"type":"push","entries":[{"message":{…},"id":0}]}
22//! ```
23
24use std::fmt;
25use std::io;
26
27use rig_core::conversation::{
28    Aside, AsideId, Author, Change, ChangeRecord, Conversation, Entry, Halt, Lineage, MessageId,
29};
30use serde::{Deserialize, Serialize};
31use serde_json::Value;
32
33/// The saved format a log's header names: the one format this rig reads.
34pub const FORMAT: &str = "rig-session/4";
35
36/// The log of the session's own saved values, beside the agents' logs.
37pub const SESSION: &str = "session";
38
39/// Where a record is in a session: its place among the session's records
40/// in the order they were made (`n`, what [`as_of`](crate::as_of) cuts
41/// by), its place in its log (`seq`, dense from
42/// the header's 0), and when it was written (`t`, milliseconds since the
43/// Unix epoch).
44#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
45pub struct Stamp {
46    /// Its place among the session's records: it increases along every
47    /// log, and across the logs one [`Writer`](crate::Writer) or
48    /// [`Session::save`](crate::Session::save) wrote.
49    pub n: u64,
50    /// Its place in its log.
51    pub seq: u64,
52    /// When it was written.
53    pub t: u64,
54}
55
56/// Why a session cannot be read or saved. Reading is strict: a session
57/// with any record it cannot read is refused whole, and nothing is written.
58#[derive(Debug, thiserror::Error)]
59pub enum SessionError {
60    /// The store failed.
61    #[error(transparent)]
62    Io(#[from] io::Error),
63    /// The directory holds the single-file layout older than `rig-session/2` (a
64    /// top-level `*.jsonl`); it is left as it is.
65    #[error("this session was made by another version of rig; start a new session")]
66    OldLayout,
67    /// A log's header names a format other than [`FORMAT`], or none; it is
68    /// left as it is.
69    #[error(
70        "this session was made by another version of rig; start a new session (the log `{log}` \
71         is in format {}, not {FORMAT})",
72        .found.as_deref().unwrap_or("(none)")
73    )]
74    Format {
75        /// The log.
76        log: String,
77        /// The format its header names.
78        found: Option<String>,
79    },
80    /// A record cannot be read, or its change does not apply.
81    #[error("the log `{log}` cannot be read at line {line}: {why}")]
82    Record {
83        /// The log.
84        log: String,
85        /// The line, from 1.
86        line: usize,
87        /// Why.
88        why: String,
89    },
90    /// The log moved on since the conversation saved to it was read: saving
91    /// would lose the changes made there.
92    #[error("the log `{log}` changed since it was read")]
93    Changed {
94        /// The log.
95        log: String,
96    },
97}
98
99impl SessionError {
100    /// Line `line` of the log `log` cannot be read, for `why`.
101    pub(crate) fn record(log: &str, line: usize, why: String) -> Self {
102        let log = log.to_owned();
103        Self::Record { log, line, why }
104    }
105}
106
107impl From<SessionError> for io::Error {
108    /// The store's own error back; any other as
109    /// [`io::ErrorKind::InvalidData`], so `?` works in code that returns
110    /// [`io::Result`].
111    fn from(error: SessionError) -> Self {
112        match error {
113            SessionError::Io(error) => error,
114            error => io::Error::new(io::ErrorKind::InvalidData, error),
115        }
116    }
117}
118
119/// What reading a session repaired, or found it must work around.
120#[derive(Clone, Debug, PartialEq, Eq)]
121pub enum Repair {
122    /// The log ended in a line a crash cut short. The log is read up to
123    /// `kept` bytes; the `cut` bytes after were set aside in the named file
124    /// `set_aside` when the log was opened for writing (`None` when it was
125    /// only read).
126    TornTail {
127        /// The log.
128        log: String,
129        /// The bytes read.
130        kept: u64,
131        /// The bytes cut.
132        cut: u64,
133        /// The file they were set aside in.
134        set_aside: Option<String>,
135    },
136    /// An image's blob cannot be read, so the image has no data.
137    LostImage {
138        /// The log.
139        log: String,
140        /// Why.
141        why: String,
142    },
143}
144
145impl fmt::Display for Repair {
146    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
147        match self {
148            Self::TornTail {
149                log,
150                cut,
151                set_aside,
152                ..
153            } => {
154                write!(f, "the log `{log}` ended in a line cut short ({cut} bytes)")?;
155                match set_aside {
156                    Some(file) => write!(f, ", set aside in {file}"),
157                    None => Ok(()),
158                }
159            }
160            Self::LostImage { log, why } => write!(f, "an image of the log `{log}` is lost: {why}"),
161        }
162    }
163}
164
165/// The first record of a log.
166#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
167pub(crate) struct Header {
168    #[serde(default, skip_serializing_if = "Option::is_none")]
169    pub format: Option<String>,
170    #[serde(default)]
171    pub log: String,
172    /// The log of the agent that spawned this one.
173    #[serde(default, skip_serializing_if = "Option::is_none")]
174    pub parent: Option<String>,
175    /// The log's own lineage, which every conversation loaded from it or
176    /// saved to it carries.
177    #[serde(default)]
178    pub lineage: Lineage,
179    /// Who rewrote the log in place, and when, oldest first.
180    #[serde(default, skip_serializing_if = "Vec::is_empty")]
181    pub rewrites: Vec<LogRewrite>,
182}
183
184/// A rewrite of a log in place ([`rewrite`](crate::rewrite)): when, and by
185/// whom, such as the plugin that redacted it.
186#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
187pub struct LogRewrite {
188    /// When, in milliseconds since the Unix epoch.
189    pub t: u64,
190    /// Who.
191    pub by: Author,
192}
193
194impl Header {
195    /// The header of a log just started, with a fresh lineage.
196    pub fn new(log: &str, parent: Option<&str>) -> Self {
197        Self {
198            format: Some(FORMAT.to_owned()),
199            log: log.to_owned(),
200            parent: parent.map(str::to_owned),
201            lineage: Lineage::fresh(),
202            rewrites: Vec::new(),
203        }
204    }
205}
206
207/// A record that is no change of the conversation.
208#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
209#[serde(tag = "type", rename_all = "snake_case")]
210pub(crate) enum Meta {
211    Header(Header),
212    State {
213        key: String,
214        #[serde(default)]
215        value: Option<Value>,
216    },
217    Retired,
218}
219
220/// A record as read.
221pub(crate) enum Body {
222    Meta(Meta),
223    /// Its `at` is the stamp's `t`.
224    Change(ChangeRecord),
225}
226
227/// The fields every line starts with.
228#[derive(Deserialize)]
229struct Head<'a> {
230    seq: u64,
231    n: u64,
232    t: u64,
233    #[serde(rename = "type", borrow)]
234    kind: std::borrow::Cow<'a, str>,
235}
236
237/// The record types that are changes of the conversation.
238const CHANGES: [&str; 5] = ["push", "splice", "halt", "aside", "reset"];
239
240/// A line's stamp, and whether it is a change of the conversation, read
241/// without the rest of it.
242pub(crate) fn stamp_of(line: &[u8]) -> Result<(Stamp, bool), String> {
243    let head: Head<'_> = serde_json::from_slice(line).map_err(|error| error.to_string())?;
244    Ok((head.stamp(), CHANGES.contains(&&*head.kind)))
245}
246
247impl Head<'_> {
248    fn stamp(&self) -> Stamp {
249        Stamp {
250            n: self.n,
251            seq: self.seq,
252            t: self.t,
253        }
254    }
255}
256
257/// A line read in one pass: its stamp and type, and, for a change, who
258/// made it and every field a change has, of which the type says which it
259/// uses. A change read whole without its stamp would be buffered twice:
260/// once for the fields around it, once for its type.
261#[derive(Deserialize)]
262struct Read<'a> {
263    seq: u64,
264    n: u64,
265    t: u64,
266    #[serde(rename = "type", borrow)]
267    kind: std::borrow::Cow<'a, str>,
268    #[serde(default)]
269    by: Option<Author>,
270    entries: Option<Vec<Entry>>,
271    from: Option<usize>,
272    to: Option<usize>,
273    removed: Option<Vec<MessageId>>,
274    with: Option<Vec<Entry>>,
275    #[serde(default)]
276    halt: Option<Halt>,
277    id: Option<AsideId>,
278    #[serde(default)]
279    aside: Option<Aside>,
280    state: Option<Box<Conversation>>,
281}
282
283impl Read<'_> {
284    /// The change the line holds.
285    fn change(self) -> Result<Change, String> {
286        fn given<T>(field: Option<T>, name: &str) -> Result<T, String> {
287            field.ok_or_else(|| format!("missing field `{name}`"))
288        }
289        Ok(match &*self.kind {
290            "push" => Change::Push {
291                entries: given(self.entries, "entries")?,
292            },
293            "splice" => Change::Splice {
294                from: given(self.from, "from")?,
295                to: given(self.to, "to")?,
296                removed: given(self.removed, "removed")?,
297                with: given(self.with, "with")?,
298            },
299            "halt" => Change::Halt { halt: self.halt },
300            "aside" => Change::Aside {
301                id: given(self.id, "id")?,
302                aside: self.aside,
303            },
304            "reset" => Change::Reset {
305                state: given(self.state, "state")?,
306            },
307            other => return Err(format!("unknown record type {other:?}")),
308        })
309    }
310}
311
312/// Reads one line.
313pub(crate) fn parse(line: &[u8]) -> Result<(Stamp, Body), String> {
314    let mut read: Read<'_> = serde_json::from_slice(line).map_err(|error| error.to_string())?;
315    let stamp = Stamp {
316        n: read.n,
317        seq: read.seq,
318        t: read.t,
319    };
320    let body = match &*read.kind {
321        "header" | "state" | "retired" => {
322            Body::Meta(serde_json::from_slice(line).map_err(|error| error.to_string())?)
323        }
324        _ => {
325            let by = read.by.take();
326            Body::Change(ChangeRecord {
327                change: read.change()?,
328                by,
329                at: Some(stamp.t),
330            })
331        }
332    };
333    Ok((stamp, body))
334}
335
336/// A line as written.
337#[derive(Serialize)]
338struct Line<'a> {
339    seq: u64,
340    n: u64,
341    t: u64,
342    #[serde(skip_serializing_if = "Option::is_none")]
343    by: Option<&'a Author>,
344    #[serde(flatten)]
345    body: Out<'a>,
346}
347
348#[derive(Serialize)]
349#[serde(untagged)]
350enum Out<'a> {
351    Meta(&'a Meta),
352    Change(&'a Change),
353}
354
355/// Writes a meta record line, newline included, to `out`.
356pub(crate) fn write_meta(out: &mut Vec<u8>, stamp: Stamp, meta: &Meta) -> serde_json::Result<()> {
357    write(out, stamp, None, Out::Meta(meta))
358}
359
360/// Writes a change line, newline included, to `out`.
361pub(crate) fn write_change(
362    out: &mut Vec<u8>,
363    stamp: Stamp,
364    by: Option<&Author>,
365    change: &Change,
366) -> serde_json::Result<()> {
367    write(out, stamp, by, Out::Change(change))
368}
369
370/// Writes a line, newline included, to `out`.
371fn write(
372    out: &mut Vec<u8>,
373    stamp: Stamp,
374    by: Option<&Author>,
375    body: Out<'_>,
376) -> serde_json::Result<()> {
377    let line = Line {
378        seq: stamp.seq,
379        n: stamp.n,
380        t: stamp.t,
381        by,
382        body,
383    };
384    serde_json::to_writer(&mut *out, &line)?;
385    out.push(b'\n');
386    Ok(())
387}
388
389/// The complete lines of a log as stored. A torn tail a crash left (bytes
390/// after the last newline, or a last line that is not JSON) is not one.
391#[derive(Clone, Debug, Default)]
392pub(crate) struct LogLines {
393    bytes: Vec<u8>,
394    /// Where each complete line, newline included, ends.
395    ends: Vec<usize>,
396}
397
398impl LogLines {
399    pub fn parse(bytes: Vec<u8>) -> Self {
400        let mut ends: Vec<usize> = memchr::memchr_iter(b'\n', &bytes)
401            .map(|at| at + 1)
402            .collect();
403        let last_start = ends
404            .len()
405            .checked_sub(2)
406            .and_then(|at| ends.get(at))
407            .copied();
408        let last = ends
409            .last()
410            .and_then(|end| bytes.get(last_start.unwrap_or(0)..end.saturating_sub(1)));
411        if last.is_some_and(|line| {
412            !line.is_empty() && serde_json::from_slice::<serde::de::IgnoredAny>(line).is_err()
413        }) {
414            ends.pop();
415        }
416        Self { bytes, ends }
417    }
418
419    /// Each complete line that is not blank, without its newline, with its
420    /// number from 1 and the byte offset just past it.
421    pub fn iter(&self) -> impl Iterator<Item = (usize, u64, &[u8])> {
422        let starts = std::iter::once(0).chain(self.ends.iter().copied());
423        starts
424            .zip(self.ends.iter().copied())
425            .enumerate()
426            .filter_map(|(at, (start, end))| {
427                let line = self.bytes.get(start..end.saturating_sub(1))?;
428                (!line.is_empty()).then_some((at + 1, end as u64, line))
429            })
430    }
431
432    /// The bytes the complete lines cover.
433    pub fn len(&self) -> u64 {
434        self.ends.last().map_or(0, |end| *end as u64)
435    }
436
437    /// The bytes after the complete lines: a torn tail.
438    pub fn tail(&self) -> &[u8] {
439        let len = usize::try_from(self.len()).unwrap_or(usize::MAX);
440        self.bytes.get(len..).unwrap_or_default()
441    }
442
443    /// The log's first `len` bytes, no further than its complete lines.
444    pub fn prefix(&self, len: u64) -> &[u8] {
445        let len = usize::try_from(len.min(self.len())).unwrap_or(0);
446        self.bytes.get(..len).unwrap_or_default()
447    }
448}
449
450#[cfg(test)]
451mod tests;