Skip to main content

rig_memory/
migration.rs

1//! Bringing a session saved in an earlier format to [`FORMAT`]: a
2//! [`Chain`] of steps, each taking the whole session (its logs and files)
3//! from one format to the next, so a change of the format adds one step.
4//! Opening a session refuses another format ([`SessionError::Format`]),
5//! and only then does a caller run [`Chain::migrate`], so a session already
6//! in the format gets no migration pass. The chain rig ships, [`SHIPPED`],
7//! has no step yet: every format before [`FORMAT`] was never released, so
8//! each is refused.
9//!
10//! A migration checks the migrated session whole before it writes, keeps a
11//! copy of the session as it was under [`KEPT`] through the same store,
12//! then replaces what changed. It takes no lock: the caller holds the
13//! session, as the launcher's lock holds a running one.
14//!
15//! ```
16//! use rig_memory::{MemoryStore, migration};
17//!
18//! let store = MemoryStore::default();
19//! migration::SHIPPED.migrate(&store, &|_, _| Ok(()))?;
20//! # Ok::<(), rig_memory::SessionError>(())
21//! ```
22
23use std::collections::BTreeMap;
24use std::io;
25
26use serde_json::Value;
27
28use crate::format::{Header, LogLines, Meta, SESSION};
29use crate::session::note_meta;
30use crate::{FORMAT, META, MemoryStore, Repair, Session, SessionError, SessionMeta, Store};
31
32/// Where a migration keeps the session as it was, through the session's
33/// own store: `rig/kept/<format>/logs/<log>.jsonl` and
34/// `rig/kept/<format>/files/<file>`. A migration never reads these as the
35/// session's own files.
36pub const KEPT: &str = "rig/kept/";
37
38/// Checks a saved value, by its key, as the session's reader would read
39/// it: an error when its type is known and the value does not decode to
40/// it. A key whose type is unknown, such as one of a plugin that is not
41/// installed, is kept as it is and must pass.
42pub type Verify<'a> = dyn Fn(&str, &Value) -> io::Result<()> + 'a;
43
44/// The steps from each earlier format a session can be brought from, in
45/// the order they apply.
46pub struct Chain {
47    steps: &'static [Step],
48}
49
50/// One step, from the session format `from` to `to`, over the whole
51/// session.
52pub(crate) struct Step {
53    pub from: &'static str,
54    pub to: &'static str,
55    pub apply: fn(&mut Snapshot) -> Result<(), SessionError>,
56    /// A session in `from`, generated by the test code.
57    #[cfg(test)]
58    pub fixture: fn() -> Result<MemoryStore, SessionError>,
59}
60
61/// The chain rig ships. Every change of [`FORMAT`] adds a step to it, from
62/// the format before, with a generated fixture.
63pub const SHIPPED: Chain = Chain { steps: &[] };
64
65/// The first format rig released, which the chain starts from.
66#[cfg(test)]
67const FIRST: &str = "rig-session/4";
68
69/// A session's logs and files, by name, as stored; the files under
70/// [`KEPT`] left out.
71#[derive(Clone, Debug, Default, PartialEq, Eq)]
72pub(crate) struct Snapshot {
73    pub logs: BTreeMap<String, Vec<u8>>,
74    pub files: BTreeMap<String, Vec<u8>>,
75}
76
77impl Snapshot {
78    pub(crate) fn read(store: &dyn Store) -> io::Result<Self> {
79        let mut snapshot = Self::default();
80        for name in store.logs()? {
81            let bytes = store.read_log(&name)?;
82            snapshot.logs.insert(name, bytes);
83        }
84        for name in store.files()? {
85            if !name.starts_with(KEPT) {
86                let bytes = store.read_file(&name)?;
87                snapshot.files.insert(name, bytes);
88            }
89        }
90        Ok(snapshot)
91    }
92
93    /// Writes it into the empty `store`.
94    pub(crate) fn write_to(&self, store: &dyn Store) -> Result<(), SessionError> {
95        for (name, bytes) in &self.logs {
96            store.replace_log(name, 0, bytes).map_err(io::Error::from)?;
97        }
98        for (name, bytes) in &self.files {
99            store.write_file(name, bytes)?;
100        }
101        Ok(())
102    }
103
104    /// The format its logs' headers name, with a log that names it; `None`
105    /// when no log has a line. The logs of one session name one format.
106    fn format(&self) -> Result<Option<(String, Option<String>)>, SessionError> {
107        let mut named: Option<(String, Option<String>)> = None;
108        for (log, bytes) in &self.logs {
109            let lines = LogLines::parse(bytes.clone());
110            let Some((_, _, first)) = lines.iter().next() else {
111                continue;
112            };
113            let found = serde_json::from_slice::<Header>(first)
114                .ok()
115                .and_then(|header| header.format);
116            match &named {
117                Some((_, format)) if *format != found => {
118                    let log = log.clone();
119                    return Err(SessionError::Format { log, found });
120                }
121                Some(_) => {}
122                None => named = Some((log.clone(), found)),
123            }
124        }
125        Ok(named)
126    }
127}
128
129impl Chain {
130    /// Brings the session in `store` to [`FORMAT`] through this chain's
131    /// steps, when it is in an earlier format one of them starts from;
132    /// a session already in [`FORMAT`], or with no complete log line, is
133    /// left alone.
134    /// The migrated session is checked whole before anything is written:
135    /// every record reads and its change applies, every image blob a
136    /// message names loads, and each saved value a step changed (the newest
137    /// of its key, as a read gives it, and the listed ones of [`META`])
138    /// decodes when `verify` knows its type. A value no step changed is
139    /// left to the restore, which skips one that no longer decodes. Then the
140    /// session as it was is copied under [`KEPT`] and read back, and only
141    /// then is each changed file and log replaced. Logs, values and files
142    /// a step does not know are kept byte for byte.
143    ///
144    /// # Errors
145    ///
146    /// [`SessionError::Format`] when no step starts from the session's
147    /// format, its logs name different formats, or a log names none;
148    /// [`io::ErrorKind::AlreadyExists`] when the store holds a kept copy
149    /// from this format already, of a migration that did not finish; a
150    /// step's or a check's error; or the store failing. Each but the last
151    /// is returned before anything is written. [`SessionError::Changed`]
152    /// when a log changed while it was migrated.
153    pub fn migrate(&self, store: &dyn Store, verify: &Verify<'_>) -> Result<(), SessionError> {
154        let original = Snapshot::read(store)?;
155        let Some((log, found)) = original.format()? else {
156            return Ok(());
157        };
158        let from = match found.as_deref() {
159            Some(FORMAT) => return Ok(()),
160            Some(from) => from,
161            None => return Err(SessionError::Format { log, found }),
162        };
163        let kept = format!("{KEPT}{from}/");
164        if store.files()?.iter().any(|name| name.starts_with(&kept)) {
165            return Err(io::Error::new(
166                io::ErrorKind::AlreadyExists,
167                format!(
168                    "a migration from {from} did not finish; the session as it was is \
169                     under `{kept}`: put it back and remove it, or remove it, to migrate \
170                     again"
171                ),
172            )
173            .into());
174        }
175        let mut migrated = original.clone();
176        let mut at = from;
177        while at != FORMAT {
178            let Some(step) = self.steps.iter().find(|step| step.from == at) else {
179                return Err(SessionError::Format { log, found });
180            };
181            (step.apply)(&mut migrated)?;
182            at = step.to;
183        }
184        check(&original, &migrated, verify)?;
185        keep(store, &original, &kept)?;
186        replace(store, &original, &migrated)?;
187        if Snapshot::read(store)? != migrated {
188            return Err(invalid("the migrated session reads back otherwise").into());
189        }
190        Ok(())
191    }
192}
193
194fn invalid(why: impl Into<String>) -> io::Error {
195    io::Error::new(io::ErrorKind::InvalidData, why.into())
196}
197
198/// Checks `migrated`, the session `original` migrated, as
199/// [`Chain::migrate`] says.
200fn check(
201    original: &Snapshot,
202    migrated: &Snapshot,
203    verify: &Verify<'_>,
204) -> Result<(), SessionError> {
205    if let Some(log) = original
206        .logs
207        .keys()
208        .find(|log| !migrated.logs.contains_key(*log))
209    {
210        return Err(invalid(format!("a migration step removed the log `{log}`")).into());
211    }
212    let store = MemoryStore::default();
213    migrated.write_to(&store)?;
214    let session = Session::read(&store)?;
215    if let Some(lost) = session
216        .repairs
217        .iter()
218        .find(|repair| matches!(repair, Repair::LostImage { .. }))
219    {
220        return Err(invalid(format!("the migrated session does not read whole: {lost}")).into());
221    }
222    let listed = |files: &BTreeMap<String, Vec<u8>>| {
223        let meta = files.get(META).map(|bytes| serde_json::from_slice(bytes));
224        meta.transpose()
225            .map(|meta: Option<SessionMeta>| meta.unwrap_or_default().listed)
226    };
227    let listed_before = listed(&original.files).unwrap_or_default();
228    let listed_after = listed(&migrated.files).map_err(invalid_json)?;
229    let before = saved(original);
230    let logs = session
231        .agents
232        .iter()
233        .map(|agent| (agent.id.as_str(), &agent.saved));
234    let logs = logs.chain([(SESSION, &session.saved)]);
235    let values = logs.flat_map(|(log, values)| changed(before.get(log), values));
236    let listed = changed(Some(&listed_before), &listed_after);
237    for (key, value) in values.chain(listed) {
238        verify(key, value)?;
239    }
240    Ok(())
241}
242
243/// The values of `after` that are not in `before` as they are.
244fn changed<'a>(
245    before: Option<&'a BTreeMap<String, Value>>,
246    after: &'a BTreeMap<String, Value>,
247) -> impl Iterator<Item = (&'a String, &'a Value)> {
248    let kept =
249        move |key: &String, value: &Value| before.and_then(|before| before.get(key)) == Some(value);
250    after.iter().filter(move |&(key, value)| !kept(key, value))
251}
252
253/// The newest saved value of each key of each log of `original`: what a
254/// step may have changed. Its `state` records are read as plain JSON (a
255/// `type` of `state`, a `key` and a `value`), so a step that changes the
256/// rest of a record's shape still finds the values it left as they were.
257fn saved(original: &Snapshot) -> BTreeMap<&str, BTreeMap<String, Value>> {
258    let fold = |bytes: &Vec<u8>| {
259        let (mut state, mut retired) = (BTreeMap::new(), false);
260        for (_, _, record) in LogLines::parse(bytes.clone()).iter().skip(1) {
261            let Ok(record) = serde_json::from_slice::<Value>(record) else {
262                continue;
263            };
264            let field = |name| record.get(name);
265            if field("type").and_then(Value::as_str) == Some("state")
266                && let Some(key) = field("key").and_then(Value::as_str)
267            {
268                let (key, value) = (key.to_owned(), field("value").cloned());
269                note_meta(Meta::State { key, value }, &mut state, &mut retired);
270            }
271        }
272        state
273    };
274    let logs = original.logs.iter();
275    logs.map(|(log, bytes)| (log.as_str(), fold(bytes)))
276        .collect()
277}
278
279fn invalid_json(error: serde_json::Error) -> io::Error {
280    invalid(error.to_string())
281}
282
283/// Copies `original` into `store` under `kept`, and reads the copy back.
284fn keep(store: &dyn Store, original: &Snapshot, kept: &str) -> io::Result<()> {
285    let logs = original
286        .logs
287        .iter()
288        .map(|(name, bytes)| (format!("{kept}logs/{name}.jsonl"), bytes));
289    let files = original
290        .files
291        .iter()
292        .map(|(name, bytes)| (format!("{kept}files/{name}"), bytes));
293    let copies: Vec<_> = logs.chain(files).collect();
294    for (name, bytes) in &copies {
295        store.write_file(name, bytes)?;
296    }
297    for (name, bytes) in &copies {
298        if store.read_file(name)? != **bytes {
299            return Err(invalid(format!(
300                "the kept copy `{name}` reads back otherwise"
301            )));
302        }
303    }
304    Ok(())
305}
306
307/// Replaces in `store` what changed from `original` to `migrated`: the
308/// files first, then the logs, each only if it is as `original` read it.
309fn replace(
310    store: &dyn Store,
311    original: &Snapshot,
312    migrated: &Snapshot,
313) -> Result<(), SessionError> {
314    for name in original.files.keys() {
315        if !migrated.files.contains_key(name) {
316            store.remove_file(name)?;
317        }
318    }
319    for (name, bytes) in &migrated.files {
320        if original.files.get(name) != Some(bytes) {
321            store.write_file(name, bytes)?;
322        }
323    }
324    for (name, bytes) in &migrated.logs {
325        let was = original.logs.get(name);
326        if was == Some(bytes) {
327            continue;
328        }
329        let len = was.map_or(0, |was| was.len() as u64);
330        match store.replace_log(name, len, bytes) {
331            Err(crate::LogWriteError::Changed) => {
332                return Err(SessionError::Changed { log: name.clone() });
333            }
334            replaced => replaced.map_err(io::Error::from)?,
335        }
336    }
337    Ok(())
338}
339
340#[cfg(any(test, feature = "test-utils"))]
341pub mod testing;
342
343#[cfg(test)]
344mod tests;