Skip to main content

rig_memory/
store.rs

1//! Where a session is kept: [`Files`] by name, and a [`Store`] that also
2//! holds each log of the session. The blobs its messages name are its
3//! files `blobs/<name>`, named by their content, which a directory store
4//! keeps under `files/blobs/`. [`AsyncStore`] is the same storage reached
5//! asynchronously, such as a database, an object store or IndexedDB; every
6//! [`Store`] is one. A log write names the length the writer read, and the
7//! store writes only if the log is still that long, checking and writing
8//! in one step ([`LogWriteError::Changed`]).
9
10use std::collections::BTreeMap;
11use std::future::{Future, ready};
12use std::io;
13use std::pin::pin;
14use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
15use std::task::{Context, Poll, Waker};
16
17use rig_core::wasm_compat::WasmCompatSend;
18
19use crate::SessionError;
20
21/// Why a log write wrote nothing.
22#[derive(Debug, thiserror::Error)]
23pub enum LogWriteError {
24    /// The log is not the `len` bytes the writer read: another writer
25    /// changed it. Nothing was written.
26    #[error("the log changed since it was read")]
27    Changed,
28    /// The store failed.
29    #[error(transparent)]
30    Io(#[from] io::Error),
31}
32
33impl From<LogWriteError> for io::Error {
34    /// The store's own error back; [`LogWriteError::Changed`] as an
35    /// [`io::ErrorKind::Other`] saying so.
36    fn from(error: LogWriteError) -> Self {
37        match error {
38            LogWriteError::Changed => io::Error::other(LogWriteError::Changed.to_string()),
39            LogWriteError::Io(error) => error,
40        }
41    }
42}
43
44/// A place that keeps files by name, such as a session's plugin data or
45/// effect log. A name is relative and `/`-separated ([`check_name`]).
46/// Every method takes `&self`, so one value is shared, between threads on
47/// every target: a host keeps it as `Arc<dyn Files>` or `Arc<dyn Store>`
48/// in state its systems share, such as a Bevy resource, which is `Send`
49/// and `Sync` on wasm too; `Arc<dyn Store>` and `Box<dyn Store>` are
50/// themselves a [`Store`], so an [`AsyncStore`] too. A store that is not
51/// `Send` and `Sync`, such as one over a JavaScript promise, is an
52/// [`AsyncStore`].
53pub trait Files: Send + Sync + 'static {
54    /// Replaces the file `name` with `bytes`, atomically: a reader sees the
55    /// old content or the new.
56    fn write_file(&self, name: &str, bytes: &[u8]) -> io::Result<()>;
57
58    /// Adds `bytes` at the end of the file `name`, which it creates when
59    /// there is none.
60    fn append_file(&self, name: &str, bytes: &[u8]) -> io::Result<()>;
61
62    /// The file `name`; [`io::ErrorKind::NotFound`] when there is none.
63    fn read_file(&self, name: &str) -> io::Result<Vec<u8>>;
64
65    /// Removes the file `name`; removing one that is not there succeeds.
66    fn remove_file(&self, name: &str) -> io::Result<()>;
67
68    /// The names of the files, in no particular order.
69    fn files(&self) -> io::Result<Vec<String>>;
70}
71
72/// Where a session is kept: its [`Files`], with the content-addressed
73/// blobs its messages name under `blobs/`, and a log per agent plus the
74/// session's own, appended to and read whole. Log names are one part of a
75/// name ([`check_name`] without `/`).
76pub trait Store: Files {
77    /// The names of the logs, in no particular order.
78    fn logs(&self) -> io::Result<Vec<String>>;
79
80    /// The whole log `name`; [`io::ErrorKind::NotFound`] when there is none.
81    fn read_log(&self, name: &str) -> io::Result<Vec<u8>>;
82
83    /// Appends `bytes` to the log `name` if it is `len` bytes long now (0
84    /// when there is none, which it then starts); else
85    /// [`LogWriteError::Changed`], writing nothing. The check and the
86    /// append are one step for every writer of the store, other processes
87    /// included, so of two writers that read the log at the same length
88    /// one appends and the other is refused. A log that shrank and grew
89    /// back to exactly `len` bytes since it was read is not told apart;
90    /// rig shrinks a log only by setting a torn last line aside, or by
91    /// [`rewrite`](crate::rewrite) and [`Self::destroy`], which are for a
92    /// stopped session.
93    fn append_log(&self, name: &str, len: u64, bytes: &[u8]) -> Result<(), LogWriteError>;
94
95    /// Replaces the log `name` with `bytes`, under the condition
96    /// [`Self::append_log`] has, atomically and durably: a reader, and a
97    /// crash or power loss, leave the old log or the new, never a part of
98    /// either or an empty file; an append after it goes after the new
99    /// bytes. Starts the log when there is none.
100    fn replace_log(&self, name: &str, len: u64, bytes: &[u8]) -> Result<(), LogWriteError>;
101
102    /// Removes everything the store holds: logs and files.
103    /// Idempotent: calling it again after a crash midway finishes the job.
104    fn destroy(&self) -> io::Result<()>;
105}
106
107/// A session's storage reached asynchronously: the part of [`Store`] that
108/// [`load_async`](crate::load_async) and [`save_async`](crate::save_async)
109/// use, for a store behind a network or a JavaScript promise. Every
110/// [`Store`] is one, its calls done when made: a local store pays no thread
111/// hop and no allocation for it. A type that implements [`Store`] gets this
112/// from it and cannot have its own (the two impls would overlap): wrap it
113/// in a type of your own instead.
114///
115/// Each method keeps [`Store`]'s contract. A name is checked by
116/// [`check_name`] (a log name is one part) and may come from a log's
117/// content, so a store refuses one that does not pass. The two log writes
118/// check the log's length and write in one step, such as a conditional
119/// `UPDATE … WHERE octet_length(bytes) = $len` or one IndexedDB
120/// `readwrite` transaction, never a read followed by a write. An append
121/// adds its bytes whole, or at worst leaves a cut last line, which the next
122/// save sets aside: [`save_async`](crate::save_async) writes one turn with
123/// one append, so a cancelled or failed save never stores part of one.
124/// [`io::ErrorKind::NotFound`] means there is no such log or file.
125///
126/// Its futures are `Send` natively, and on browser wasm neither they nor
127/// the store need be. A holder that shares the store, such as an agent,
128/// asks for `WasmCompatSend + WasmCompatSync` itself. An async store that
129/// is shared is `Clone` over an inner `Arc`: `Arc<S>` is an `AsyncStore`
130/// only when `S` is a [`Store`].
131pub trait AsyncStore {
132    /// The whole log `name`.
133    fn read_log_async(
134        &self,
135        name: &str,
136    ) -> impl Future<Output = io::Result<Vec<u8>>> + WasmCompatSend;
137
138    /// Appends `bytes` to the log `name` if it is `len` bytes long now
139    /// ([`Store::append_log`]).
140    fn append_log_async(
141        &self,
142        name: &str,
143        len: u64,
144        bytes: &[u8],
145    ) -> impl Future<Output = Result<(), LogWriteError>> + WasmCompatSend;
146
147    /// Replaces the log `name` with `bytes` if it is `len` bytes long now,
148    /// atomically and durably ([`Store::replace_log`]).
149    fn replace_log_async(
150        &self,
151        name: &str,
152        len: u64,
153        bytes: &[u8],
154    ) -> impl Future<Output = Result<(), LogWriteError>> + WasmCompatSend;
155
156    /// The file `name`.
157    fn read_file_async(
158        &self,
159        name: &str,
160    ) -> impl Future<Output = io::Result<Vec<u8>>> + WasmCompatSend;
161
162    /// Replaces the file `name` with `bytes`, atomically.
163    fn write_file_async(
164        &self,
165        name: &str,
166        bytes: &[u8],
167    ) -> impl Future<Output = io::Result<()>> + WasmCompatSend;
168}
169
170/// A local store's calls, done when made.
171impl<S: Store + ?Sized> AsyncStore for S {
172    fn read_log_async(
173        &self,
174        name: &str,
175    ) -> impl Future<Output = io::Result<Vec<u8>>> + WasmCompatSend {
176        ready(self.read_log(name))
177    }
178
179    fn append_log_async(
180        &self,
181        name: &str,
182        len: u64,
183        bytes: &[u8],
184    ) -> impl Future<Output = Result<(), LogWriteError>> + WasmCompatSend {
185        ready(self.append_log(name, len, bytes))
186    }
187
188    fn replace_log_async(
189        &self,
190        name: &str,
191        len: u64,
192        bytes: &[u8],
193    ) -> impl Future<Output = Result<(), LogWriteError>> + WasmCompatSend {
194        ready(self.replace_log(name, len, bytes))
195    }
196
197    fn read_file_async(
198        &self,
199        name: &str,
200    ) -> impl Future<Output = io::Result<Vec<u8>>> + WasmCompatSend {
201        ready(self.read_file(name))
202    }
203
204    fn write_file_async(
205        &self,
206        name: &str,
207        bytes: &[u8],
208    ) -> impl Future<Output = io::Result<()>> + WasmCompatSend {
209        ready(self.write_file(name, bytes))
210    }
211}
212
213/// The store a pointer points at: `Arc<dyn Store>` and `Box<dyn Store>`
214/// are stores, so [`AsyncStore`]s too.
215macro_rules! forward {
216    ($($pointer:ident),*) => {$(
217        impl<S: Files + ?Sized> Files for $pointer<S> {
218            fn write_file(&self, name: &str, bytes: &[u8]) -> io::Result<()> {
219                (**self).write_file(name, bytes)
220            }
221
222            fn append_file(&self, name: &str, bytes: &[u8]) -> io::Result<()> {
223                (**self).append_file(name, bytes)
224            }
225
226            fn read_file(&self, name: &str) -> io::Result<Vec<u8>> {
227                (**self).read_file(name)
228            }
229
230            fn remove_file(&self, name: &str) -> io::Result<()> {
231                (**self).remove_file(name)
232            }
233
234            fn files(&self) -> io::Result<Vec<String>> {
235                (**self).files()
236            }
237        }
238
239        impl<S: Store + ?Sized> Store for $pointer<S> {
240            fn logs(&self) -> io::Result<Vec<String>> {
241                (**self).logs()
242            }
243
244            fn read_log(&self, name: &str) -> io::Result<Vec<u8>> {
245                (**self).read_log(name)
246            }
247
248            fn append_log(&self, name: &str, len: u64, bytes: &[u8]) -> Result<(), LogWriteError> {
249                (**self).append_log(name, len, bytes)
250            }
251
252            fn replace_log(&self, name: &str, len: u64, bytes: &[u8]) -> Result<(), LogWriteError> {
253                (**self).replace_log(name, len, bytes)
254            }
255
256            fn destroy(&self) -> io::Result<()> {
257                (**self).destroy()
258            }
259        }
260    )*};
261}
262
263forward!(Arc, Box);
264
265/// The output of `work`, polled once. A [`Store`]'s calls through
266/// [`AsyncStore`] are done when made, so its saves finish on the first
267/// poll: the synchronous API runs the one asynchronous implementation.
268pub(crate) fn now<T>(
269    work: impl Future<Output = Result<T, SessionError>>,
270) -> Result<T, SessionError> {
271    match pin!(work).poll(&mut Context::from_waker(Waker::noop())) {
272        Poll::Ready(output) => output,
273        Poll::Pending => Err(io::Error::other("a local store's call did not finish").into()),
274    }
275}
276
277/// Refuses `name` unless it is relative and `/`-separated, with no empty,
278/// `.` or `..` part and no `\`, so it names a place inside the store
279/// wherever the store keeps it.
280///
281/// ```
282/// use rig_memory::check_name;
283///
284/// assert!(check_name("plugins/notes.json").is_ok());
285/// assert!(check_name("../elsewhere").is_err());
286/// assert!(check_name("/etc/passwd").is_err());
287/// ```
288///
289/// # Errors
290///
291/// [`io::ErrorKind::InvalidInput`], naming it.
292pub fn check_name(name: &str) -> io::Result<()> {
293    let bad = |part: &str| matches!(part, "" | "." | "..") || part.contains('\\');
294    if name.split('/').any(bad) {
295        let why = format!("{name:?} is no name in a session");
296        return Err(io::Error::new(io::ErrorKind::InvalidInput, why));
297    }
298    Ok(())
299}
300
301/// Refuses `name` as a log name: [`check_name`], in one part.
302///
303/// ```
304/// use rig_memory::check_log_name;
305///
306/// assert!(check_log_name("main").is_ok());
307/// assert!(check_log_name("agents/main").is_err());
308/// ```
309///
310/// # Errors
311///
312/// [`io::ErrorKind::InvalidInput`], naming it.
313pub fn check_log_name(name: &str) -> io::Result<()> {
314    check_name(name)?;
315    if name.contains('/') {
316        let why = format!("{name:?} is no log name");
317        return Err(io::Error::new(io::ErrorKind::InvalidInput, why));
318    }
319    Ok(())
320}
321
322/// The longest name, in bytes, [`check_new_log_name`] allows: with its
323/// file's extension it fits the 255 bytes every disk allows a file name.
324pub const MAX_NEW_LOG_NAME: usize = 200;
325
326/// Refuses `name` for a new log: [`check_log_name`], a name longer than
327/// [`MAX_NEW_LOG_NAME`] bytes, or [`SESSION`](crate::SESSION) in any letter
328/// case ([`same_log_name`]), which names the session's own log on a disk
329/// that ignores case. A log that exists keeps its name: stores read and
330/// write it under [`check_log_name`].
331///
332/// ```
333/// use rig_memory::check_new_log_name;
334///
335/// assert!(check_new_log_name("helper").is_ok());
336/// assert!(check_new_log_name("Session").is_err());
337/// assert!(check_new_log_name(&"x".repeat(300)).is_err());
338/// ```
339///
340/// # Errors
341///
342/// [`io::ErrorKind::InvalidInput`], naming it.
343pub fn check_new_log_name(name: &str) -> io::Result<()> {
344    check_log_name(name)?;
345    let why = if name.len() > MAX_NEW_LOG_NAME {
346        format!("{name:?} is longer than {MAX_NEW_LOG_NAME} bytes")
347    } else if same_log_name(name, crate::SESSION) {
348        format!("{name:?} is the session's own log")
349    } else {
350        return Ok(());
351    };
352    Err(io::Error::new(io::ErrorKind::InvalidInput, why))
353}
354
355/// Whether the logs `a` and `b` may be one file on a disk that ignores
356/// case: their names differ only in letter case, as Unicode's lower- or
357/// uppercase mapping sees it, so the final and medial sigma (`ς`, `σ`)
358/// match too. A file system's own folding can still differ.
359///
360/// ```
361/// use rig_memory::same_log_name;
362///
363/// assert!(same_log_name("Helper", "helper"));
364/// assert!(same_log_name("ς", "σ"));
365/// assert!(!same_log_name("helper", "helpers"));
366/// ```
367pub fn same_log_name(a: &str, b: &str) -> bool {
368    a.to_lowercase() == b.to_lowercase() || a.to_uppercase() == b.to_uppercase()
369}
370
371/// A session kept in memory, such as for tests or where there is no file
372/// system. Cloning shares the contents.
373#[derive(Clone, Default)]
374pub struct MemoryStore(Arc<Mutex<Memory>>);
375
376#[derive(Default)]
377struct Memory {
378    logs: BTreeMap<String, Vec<u8>>,
379    files: BTreeMap<String, Vec<u8>>,
380}
381
382impl MemoryStore {
383    fn memory(&self) -> MutexGuard<'_, Memory> {
384        self.0.lock().unwrap_or_else(PoisonError::into_inner)
385    }
386}
387
388fn not_found() -> io::Error {
389    io::ErrorKind::NotFound.into()
390}
391
392impl Files for MemoryStore {
393    fn write_file(&self, name: &str, bytes: &[u8]) -> io::Result<()> {
394        check_name(name)?;
395        self.memory().files.insert(name.to_owned(), bytes.to_vec());
396        Ok(())
397    }
398
399    fn append_file(&self, name: &str, bytes: &[u8]) -> io::Result<()> {
400        check_name(name)?;
401        let mut memory = self.memory();
402        let file = memory.files.entry(name.to_owned()).or_default();
403        file.extend_from_slice(bytes);
404        Ok(())
405    }
406
407    fn read_file(&self, name: &str) -> io::Result<Vec<u8>> {
408        check_name(name)?;
409        self.memory().files.get(name).cloned().ok_or_else(not_found)
410    }
411
412    fn remove_file(&self, name: &str) -> io::Result<()> {
413        check_name(name)?;
414        self.memory().files.remove(name);
415        Ok(())
416    }
417
418    fn files(&self) -> io::Result<Vec<String>> {
419        Ok(self.memory().files.keys().cloned().collect())
420    }
421}
422
423impl Store for MemoryStore {
424    fn logs(&self) -> io::Result<Vec<String>> {
425        Ok(self.memory().logs.keys().cloned().collect())
426    }
427
428    fn read_log(&self, name: &str) -> io::Result<Vec<u8>> {
429        check_log_name(name)?;
430        self.memory().logs.get(name).cloned().ok_or_else(not_found)
431    }
432
433    fn append_log(&self, name: &str, len: u64, bytes: &[u8]) -> Result<(), LogWriteError> {
434        check_log_name(name)?;
435        let mut memory = self.memory();
436        if memory.logs.get(name).map_or(0, Vec::len) as u64 != len {
437            return Err(LogWriteError::Changed);
438        }
439        let log = memory.logs.entry(name.to_owned()).or_default();
440        log.extend_from_slice(bytes);
441        Ok(())
442    }
443
444    fn replace_log(&self, name: &str, len: u64, bytes: &[u8]) -> Result<(), LogWriteError> {
445        check_log_name(name)?;
446        let mut memory = self.memory();
447        if memory.logs.get(name).map_or(0, Vec::len) as u64 != len {
448            return Err(LogWriteError::Changed);
449        }
450        memory.logs.insert(name.to_owned(), bytes.to_vec());
451        Ok(())
452    }
453
454    fn destroy(&self) -> io::Result<()> {
455        *self.memory() = Memory::default();
456        Ok(())
457    }
458}
459
460/// Where plugins' files are in a session's store.
461pub const PLUGINS: &str = "plugins/";
462
463/// The files of a session's store under [`PLUGINS`], by their names there:
464/// what a plugin keeps for the session.
465pub struct PluginFiles(pub Arc<dyn Files>);
466
467impl Files for PluginFiles {
468    fn write_file(&self, name: &str, bytes: &[u8]) -> io::Result<()> {
469        self.0.write_file(&format!("{PLUGINS}{name}"), bytes)
470    }
471
472    fn append_file(&self, name: &str, bytes: &[u8]) -> io::Result<()> {
473        self.0.append_file(&format!("{PLUGINS}{name}"), bytes)
474    }
475
476    fn read_file(&self, name: &str) -> io::Result<Vec<u8>> {
477        self.0.read_file(&format!("{PLUGINS}{name}"))
478    }
479
480    fn remove_file(&self, name: &str) -> io::Result<()> {
481        self.0.remove_file(&format!("{PLUGINS}{name}"))
482    }
483
484    fn files(&self) -> io::Result<Vec<String>> {
485        let files = self.0.files()?.into_iter();
486        let within = files.filter_map(|name| Some(name.strip_prefix(PLUGINS)?.to_owned()));
487        Ok(within.collect())
488    }
489}