rig-memory 0.46.0

Conversations over time for the Rig agent framework: recent messages, compaction, and saving and resuming sessions.
Documentation
//! Bringing a session saved in an earlier format to [`FORMAT`]: a
//! [`Chain`] of steps, each taking the whole session (its logs and files)
//! from one format to the next, so a change of the format adds one step.
//! Opening a session refuses another format ([`SessionError::Format`]),
//! and only then does a caller run [`Chain::migrate`], so a session already
//! in the format gets no migration pass. The chain rig ships, [`SHIPPED`],
//! has no step yet: every format before [`FORMAT`] was never released, so
//! each is refused.
//!
//! A migration checks the migrated session whole before it writes, keeps a
//! copy of the session as it was under [`KEPT`] through the same store,
//! then replaces what changed. It takes no lock: the caller holds the
//! session, as the launcher's lock holds a running one.
//!
//! ```
//! use rig_memory::{MemoryStore, migration};
//!
//! let store = MemoryStore::default();
//! migration::SHIPPED.migrate(&store, &|_, _| Ok(()))?;
//! # Ok::<(), rig_memory::SessionError>(())
//! ```

use std::collections::BTreeMap;
use std::io;

use serde_json::Value;

use crate::format::{Header, LogLines, Meta, SESSION};
use crate::session::note_meta;
use crate::{FORMAT, META, MemoryStore, Repair, Session, SessionError, SessionMeta, Store};

/// Where a migration keeps the session as it was, through the session's
/// own store: `rig/kept/<format>/logs/<log>.jsonl` and
/// `rig/kept/<format>/files/<file>`. A migration never reads these as the
/// session's own files.
pub const KEPT: &str = "rig/kept/";

/// Checks a saved value, by its key, as the session's reader would read
/// it: an error when its type is known and the value does not decode to
/// it. A key whose type is unknown, such as one of a plugin that is not
/// installed, is kept as it is and must pass.
pub type Verify<'a> = dyn Fn(&str, &Value) -> io::Result<()> + 'a;

/// The steps from each earlier format a session can be brought from, in
/// the order they apply.
pub struct Chain {
    steps: &'static [Step],
}

/// One step, from the session format `from` to `to`, over the whole
/// session.
pub(crate) struct Step {
    pub from: &'static str,
    pub to: &'static str,
    pub apply: fn(&mut Snapshot) -> Result<(), SessionError>,
    /// A session in `from`, generated by the test code.
    #[cfg(test)]
    pub fixture: fn() -> Result<MemoryStore, SessionError>,
}

/// The chain rig ships. Every change of [`FORMAT`] adds a step to it, from
/// the format before, with a generated fixture.
pub const SHIPPED: Chain = Chain { steps: &[] };

/// The first format rig released, which the chain starts from.
#[cfg(test)]
const FIRST: &str = "rig-session/4";

/// A session's logs and files, by name, as stored; the files under
/// [`KEPT`] left out.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub(crate) struct Snapshot {
    pub logs: BTreeMap<String, Vec<u8>>,
    pub files: BTreeMap<String, Vec<u8>>,
}

impl Snapshot {
    pub(crate) fn read(store: &dyn Store) -> io::Result<Self> {
        let mut snapshot = Self::default();
        for name in store.logs()? {
            let bytes = store.read_log(&name)?;
            snapshot.logs.insert(name, bytes);
        }
        for name in store.files()? {
            if !name.starts_with(KEPT) {
                let bytes = store.read_file(&name)?;
                snapshot.files.insert(name, bytes);
            }
        }
        Ok(snapshot)
    }

    /// Writes it into the empty `store`.
    pub(crate) fn write_to(&self, store: &dyn Store) -> Result<(), SessionError> {
        for (name, bytes) in &self.logs {
            store.replace_log(name, 0, bytes).map_err(io::Error::from)?;
        }
        for (name, bytes) in &self.files {
            store.write_file(name, bytes)?;
        }
        Ok(())
    }

    /// The format its logs' headers name, with a log that names it; `None`
    /// when no log has a line. The logs of one session name one format.
    fn format(&self) -> Result<Option<(String, Option<String>)>, SessionError> {
        let mut named: Option<(String, Option<String>)> = None;
        for (log, bytes) in &self.logs {
            let lines = LogLines::parse(bytes.clone());
            let Some((_, _, first)) = lines.iter().next() else {
                continue;
            };
            let found = serde_json::from_slice::<Header>(first)
                .ok()
                .and_then(|header| header.format);
            match &named {
                Some((_, format)) if *format != found => {
                    let log = log.clone();
                    return Err(SessionError::Format { log, found });
                }
                Some(_) => {}
                None => named = Some((log.clone(), found)),
            }
        }
        Ok(named)
    }
}

impl Chain {
    /// Brings the session in `store` to [`FORMAT`] through this chain's
    /// steps, when it is in an earlier format one of them starts from;
    /// a session already in [`FORMAT`], or with no complete log line, is
    /// left alone.
    /// The migrated session is checked whole before anything is written:
    /// every record reads and its change applies, every image blob a
    /// message names loads, and each saved value a step changed (the newest
    /// of its key, as a read gives it, and the listed ones of [`META`])
    /// decodes when `verify` knows its type. A value no step changed is
    /// left to the restore, which skips one that no longer decodes. Then the
    /// session as it was is copied under [`KEPT`] and read back, and only
    /// then is each changed file and log replaced. Logs, values and files
    /// a step does not know are kept byte for byte.
    ///
    /// # Errors
    ///
    /// [`SessionError::Format`] when no step starts from the session's
    /// format, its logs name different formats, or a log names none;
    /// [`io::ErrorKind::AlreadyExists`] when the store holds a kept copy
    /// from this format already, of a migration that did not finish; a
    /// step's or a check's error; or the store failing. Each but the last
    /// is returned before anything is written. [`SessionError::Changed`]
    /// when a log changed while it was migrated.
    pub fn migrate(&self, store: &dyn Store, verify: &Verify<'_>) -> Result<(), SessionError> {
        let original = Snapshot::read(store)?;
        let Some((log, found)) = original.format()? else {
            return Ok(());
        };
        let from = match found.as_deref() {
            Some(FORMAT) => return Ok(()),
            Some(from) => from,
            None => return Err(SessionError::Format { log, found }),
        };
        let kept = format!("{KEPT}{from}/");
        if store.files()?.iter().any(|name| name.starts_with(&kept)) {
            return Err(io::Error::new(
                io::ErrorKind::AlreadyExists,
                format!(
                    "a migration from {from} did not finish; the session as it was is \
                     under `{kept}`: put it back and remove it, or remove it, to migrate \
                     again"
                ),
            )
            .into());
        }
        let mut migrated = original.clone();
        let mut at = from;
        while at != FORMAT {
            let Some(step) = self.steps.iter().find(|step| step.from == at) else {
                return Err(SessionError::Format { log, found });
            };
            (step.apply)(&mut migrated)?;
            at = step.to;
        }
        check(&original, &migrated, verify)?;
        keep(store, &original, &kept)?;
        replace(store, &original, &migrated)?;
        if Snapshot::read(store)? != migrated {
            return Err(invalid("the migrated session reads back otherwise").into());
        }
        Ok(())
    }
}

fn invalid(why: impl Into<String>) -> io::Error {
    io::Error::new(io::ErrorKind::InvalidData, why.into())
}

/// Checks `migrated`, the session `original` migrated, as
/// [`Chain::migrate`] says.
fn check(
    original: &Snapshot,
    migrated: &Snapshot,
    verify: &Verify<'_>,
) -> Result<(), SessionError> {
    if let Some(log) = original
        .logs
        .keys()
        .find(|log| !migrated.logs.contains_key(*log))
    {
        return Err(invalid(format!("a migration step removed the log `{log}`")).into());
    }
    let store = MemoryStore::default();
    migrated.write_to(&store)?;
    let session = Session::read(&store)?;
    if let Some(lost) = session
        .repairs
        .iter()
        .find(|repair| matches!(repair, Repair::LostImage { .. }))
    {
        return Err(invalid(format!("the migrated session does not read whole: {lost}")).into());
    }
    let listed = |files: &BTreeMap<String, Vec<u8>>| {
        let meta = files.get(META).map(|bytes| serde_json::from_slice(bytes));
        meta.transpose()
            .map(|meta: Option<SessionMeta>| meta.unwrap_or_default().listed)
    };
    let listed_before = listed(&original.files).unwrap_or_default();
    let listed_after = listed(&migrated.files).map_err(invalid_json)?;
    let before = saved(original);
    let logs = session
        .agents
        .iter()
        .map(|agent| (agent.id.as_str(), &agent.saved));
    let logs = logs.chain([(SESSION, &session.saved)]);
    let values = logs.flat_map(|(log, values)| changed(before.get(log), values));
    let listed = changed(Some(&listed_before), &listed_after);
    for (key, value) in values.chain(listed) {
        verify(key, value)?;
    }
    Ok(())
}

/// The values of `after` that are not in `before` as they are.
fn changed<'a>(
    before: Option<&'a BTreeMap<String, Value>>,
    after: &'a BTreeMap<String, Value>,
) -> impl Iterator<Item = (&'a String, &'a Value)> {
    let kept =
        move |key: &String, value: &Value| before.and_then(|before| before.get(key)) == Some(value);
    after.iter().filter(move |&(key, value)| !kept(key, value))
}

/// The newest saved value of each key of each log of `original`: what a
/// step may have changed. Its `state` records are read as plain JSON (a
/// `type` of `state`, a `key` and a `value`), so a step that changes the
/// rest of a record's shape still finds the values it left as they were.
fn saved(original: &Snapshot) -> BTreeMap<&str, BTreeMap<String, Value>> {
    let fold = |bytes: &Vec<u8>| {
        let (mut state, mut retired) = (BTreeMap::new(), false);
        for (_, _, record) in LogLines::parse(bytes.clone()).iter().skip(1) {
            let Ok(record) = serde_json::from_slice::<Value>(record) else {
                continue;
            };
            let field = |name| record.get(name);
            if field("type").and_then(Value::as_str) == Some("state")
                && let Some(key) = field("key").and_then(Value::as_str)
            {
                let (key, value) = (key.to_owned(), field("value").cloned());
                note_meta(Meta::State { key, value }, &mut state, &mut retired);
            }
        }
        state
    };
    let logs = original.logs.iter();
    logs.map(|(log, bytes)| (log.as_str(), fold(bytes)))
        .collect()
}

fn invalid_json(error: serde_json::Error) -> io::Error {
    invalid(error.to_string())
}

/// Copies `original` into `store` under `kept`, and reads the copy back.
fn keep(store: &dyn Store, original: &Snapshot, kept: &str) -> io::Result<()> {
    let logs = original
        .logs
        .iter()
        .map(|(name, bytes)| (format!("{kept}logs/{name}.jsonl"), bytes));
    let files = original
        .files
        .iter()
        .map(|(name, bytes)| (format!("{kept}files/{name}"), bytes));
    let copies: Vec<_> = logs.chain(files).collect();
    for (name, bytes) in &copies {
        store.write_file(name, bytes)?;
    }
    for (name, bytes) in &copies {
        if store.read_file(name)? != **bytes {
            return Err(invalid(format!(
                "the kept copy `{name}` reads back otherwise"
            )));
        }
    }
    Ok(())
}

/// Replaces in `store` what changed from `original` to `migrated`: the
/// files first, then the logs, each only if it is as `original` read it.
fn replace(
    store: &dyn Store,
    original: &Snapshot,
    migrated: &Snapshot,
) -> Result<(), SessionError> {
    for name in original.files.keys() {
        if !migrated.files.contains_key(name) {
            store.remove_file(name)?;
        }
    }
    for (name, bytes) in &migrated.files {
        if original.files.get(name) != Some(bytes) {
            store.write_file(name, bytes)?;
        }
    }
    for (name, bytes) in &migrated.logs {
        let was = original.logs.get(name);
        if was == Some(bytes) {
            continue;
        }
        let len = was.map_or(0, |was| was.len() as u64);
        match store.replace_log(name, len, bytes) {
            Err(crate::LogWriteError::Changed) => {
                return Err(SessionError::Changed { log: name.clone() });
            }
            replaced => replaced.map_err(io::Error::from)?,
        }
    }
    Ok(())
}

#[cfg(any(test, feature = "test-utils"))]
pub mod testing;

#[cfg(test)]
mod tests;