Skip to main content

rig_memory/
store.rs

1//! Where a session is kept: [`Files`] by name, and a [`Store`] that also
2//! holds each log of the session. The blobs its messages name are its
3//! files `blobs/<name>`, named by their content, which a directory store
4//! keeps under `files/blobs/`. [`AsyncStore`] is the same storage reached
5//! asynchronously, such as a database, an object store or IndexedDB; every
6//! [`Store`] is one. A log write names the length the writer read, and the
7//! store writes only if the log is still that long, checking and writing
8//! in one step ([`LogWriteError::Changed`]).
9
10use std::collections::BTreeMap;
11use std::future::{Future, ready};
12use std::io;
13use std::pin::pin;
14use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
15use std::task::{Context, Poll, Waker};
16
17use rig_core::wasm_compat::WasmCompatSend;
18
19use crate::SessionError;
20
21/// Why a log write wrote nothing.
22#[derive(Debug, thiserror::Error)]
23pub enum LogWriteError {
24    /// The log is not the `len` bytes the writer read: another writer
25    /// changed it. Nothing was written.
26    #[error("the log changed since it was read")]
27    Changed,
28    /// The store failed.
29    #[error(transparent)]
30    Io(#[from] io::Error),
31}
32
33impl From<LogWriteError> for io::Error {
34    /// The store's own error back; [`LogWriteError::Changed`] as an
35    /// [`io::ErrorKind::Other`] saying so.
36    fn from(error: LogWriteError) -> Self {
37        match error {
38            LogWriteError::Changed => io::Error::other(LogWriteError::Changed.to_string()),
39            LogWriteError::Io(error) => error,
40        }
41    }
42}
43
44/// A place that keeps files by name, such as a session's plugin data or
45/// effect log. A name is relative and `/`-separated ([`check_name`]).
46/// Every method takes `&self`, so one value is shared, between threads on
47/// every target: a host keeps it as `Arc<dyn Files>` or `Arc<dyn Store>`
48/// in state its systems share, such as a Bevy resource, which is `Send`
49/// and `Sync` on wasm too; `Arc<dyn Store>` and `Box<dyn Store>` are
50/// themselves a [`Store`], so an [`AsyncStore`] too. A store that is not
51/// `Send` and `Sync`, such as one over a JavaScript promise, is an
52/// [`AsyncStore`].
53pub trait Files: Send + Sync + 'static {
54    /// Replaces the file `name` with `bytes`, atomically: a reader sees the
55    /// old content or the new.
56    fn write_file(&self, name: &str, bytes: &[u8]) -> io::Result<()>;
57
58    /// Adds `bytes` at the end of the file `name`, which it creates when
59    /// there is none.
60    fn append_file(&self, name: &str, bytes: &[u8]) -> io::Result<()>;
61
62    /// The file `name`; [`io::ErrorKind::NotFound`] when there is none.
63    fn read_file(&self, name: &str) -> io::Result<Vec<u8>>;
64
65    /// Removes the file `name`; removing one that is not there succeeds.
66    fn remove_file(&self, name: &str) -> io::Result<()>;
67
68    /// The names of the files, in no particular order.
69    fn files(&self) -> io::Result<Vec<String>>;
70}
71
72/// Where a session is kept: its [`Files`], with the content-addressed
73/// blobs its messages name under `blobs/`, and a log per agent plus the
74/// session's own, appended to and read whole. Log names are one part of a
75/// name ([`check_name`] without `/`). The files under
76/// [`migration::KEPT`](crate::migration::KEPT) are the copies a migration
77/// kept of the session as it was.
78pub trait Store: Files {
79    /// The names of the logs, in no particular order.
80    fn logs(&self) -> io::Result<Vec<String>>;
81
82    /// The whole log `name`; [`io::ErrorKind::NotFound`] when there is none.
83    fn read_log(&self, name: &str) -> io::Result<Vec<u8>>;
84
85    /// Appends `bytes` to the log `name` if it is `len` bytes long now (0
86    /// when there is none, which it then starts); else
87    /// [`LogWriteError::Changed`], writing nothing. The check and the
88    /// append are one step for every writer of the store, other processes
89    /// included, so of two writers that read the log at the same length
90    /// one appends and the other is refused. A log that shrank and grew
91    /// back to exactly `len` bytes since it was read is not told apart;
92    /// rig shrinks a log only by setting a torn last line aside, or by
93    /// [`rewrite`](crate::rewrite) and [`Self::destroy`], which are for a
94    /// stopped session.
95    fn append_log(&self, name: &str, len: u64, bytes: &[u8]) -> Result<(), LogWriteError>;
96
97    /// Replaces the log `name` with `bytes`, under the condition
98    /// [`Self::append_log`] has, atomically and durably: a reader, and a
99    /// crash or power loss, leave the old log or the new, never a part of
100    /// either or an empty file; an append after it goes after the new
101    /// bytes. Starts the log when there is none.
102    fn replace_log(&self, name: &str, len: u64, bytes: &[u8]) -> Result<(), LogWriteError>;
103
104    /// Removes everything the store holds: logs and files.
105    /// Idempotent: calling it again after a crash midway finishes the job.
106    fn destroy(&self) -> io::Result<()>;
107}
108
109/// A session's storage reached asynchronously: the part of [`Store`] that
110/// [`load_async`](crate::load_async) and [`save_async`](crate::save_async)
111/// use, for a store behind a network or a JavaScript promise. Every
112/// [`Store`] is one, its calls done when made: a local store pays no thread
113/// hop and no allocation for it. A type that implements [`Store`] gets this
114/// from it and cannot have its own (the two impls would overlap): wrap it
115/// in a type of your own instead.
116///
117/// Each method keeps [`Store`]'s contract. A name is checked by
118/// [`check_name`] (a log name is one part) and may come from a log's
119/// content, so a store refuses one that does not pass. The two log writes
120/// check the log's length and write in one step, such as a conditional
121/// `UPDATE … WHERE octet_length(bytes) = $len` or one IndexedDB
122/// `readwrite` transaction, never a read followed by a write. An append
123/// adds its bytes whole, or at worst leaves a cut last line, which the next
124/// save sets aside: [`save_async`](crate::save_async) writes one turn with
125/// one append, so a cancelled or failed save never stores part of one.
126/// [`io::ErrorKind::NotFound`] means there is no such log or file.
127///
128/// Its futures are `Send` natively, and on browser wasm neither they nor
129/// the store need be. A holder that shares the store, such as an agent,
130/// asks for `WasmCompatSend + WasmCompatSync` itself. An async store that
131/// is shared is `Clone` over an inner `Arc`: `Arc<S>` is an `AsyncStore`
132/// only when `S` is a [`Store`].
133pub trait AsyncStore {
134    /// The whole log `name`.
135    fn read_log_async(
136        &self,
137        name: &str,
138    ) -> impl Future<Output = io::Result<Vec<u8>>> + WasmCompatSend;
139
140    /// Appends `bytes` to the log `name` if it is `len` bytes long now
141    /// ([`Store::append_log`]).
142    fn append_log_async(
143        &self,
144        name: &str,
145        len: u64,
146        bytes: &[u8],
147    ) -> impl Future<Output = Result<(), LogWriteError>> + WasmCompatSend;
148
149    /// Replaces the log `name` with `bytes` if it is `len` bytes long now,
150    /// atomically and durably ([`Store::replace_log`]).
151    fn replace_log_async(
152        &self,
153        name: &str,
154        len: u64,
155        bytes: &[u8],
156    ) -> impl Future<Output = Result<(), LogWriteError>> + WasmCompatSend;
157
158    /// The file `name`.
159    fn read_file_async(
160        &self,
161        name: &str,
162    ) -> impl Future<Output = io::Result<Vec<u8>>> + WasmCompatSend;
163
164    /// Replaces the file `name` with `bytes`, atomically.
165    fn write_file_async(
166        &self,
167        name: &str,
168        bytes: &[u8],
169    ) -> impl Future<Output = io::Result<()>> + WasmCompatSend;
170}
171
172/// A local store's calls, done when made.
173impl<S: Store + ?Sized> AsyncStore for S {
174    fn read_log_async(
175        &self,
176        name: &str,
177    ) -> impl Future<Output = io::Result<Vec<u8>>> + WasmCompatSend {
178        ready(self.read_log(name))
179    }
180
181    fn append_log_async(
182        &self,
183        name: &str,
184        len: u64,
185        bytes: &[u8],
186    ) -> impl Future<Output = Result<(), LogWriteError>> + WasmCompatSend {
187        ready(self.append_log(name, len, bytes))
188    }
189
190    fn replace_log_async(
191        &self,
192        name: &str,
193        len: u64,
194        bytes: &[u8],
195    ) -> impl Future<Output = Result<(), LogWriteError>> + WasmCompatSend {
196        ready(self.replace_log(name, len, bytes))
197    }
198
199    fn read_file_async(
200        &self,
201        name: &str,
202    ) -> impl Future<Output = io::Result<Vec<u8>>> + WasmCompatSend {
203        ready(self.read_file(name))
204    }
205
206    fn write_file_async(
207        &self,
208        name: &str,
209        bytes: &[u8],
210    ) -> impl Future<Output = io::Result<()>> + WasmCompatSend {
211        ready(self.write_file(name, bytes))
212    }
213}
214
215/// The store a pointer points at: `Arc<dyn Store>` and `Box<dyn Store>`
216/// are stores, so [`AsyncStore`]s too.
217macro_rules! forward {
218    ($($pointer:ident),*) => {$(
219        impl<S: Files + ?Sized> Files for $pointer<S> {
220            fn write_file(&self, name: &str, bytes: &[u8]) -> io::Result<()> {
221                (**self).write_file(name, bytes)
222            }
223
224            fn append_file(&self, name: &str, bytes: &[u8]) -> io::Result<()> {
225                (**self).append_file(name, bytes)
226            }
227
228            fn read_file(&self, name: &str) -> io::Result<Vec<u8>> {
229                (**self).read_file(name)
230            }
231
232            fn remove_file(&self, name: &str) -> io::Result<()> {
233                (**self).remove_file(name)
234            }
235
236            fn files(&self) -> io::Result<Vec<String>> {
237                (**self).files()
238            }
239        }
240
241        impl<S: Store + ?Sized> Store for $pointer<S> {
242            fn logs(&self) -> io::Result<Vec<String>> {
243                (**self).logs()
244            }
245
246            fn read_log(&self, name: &str) -> io::Result<Vec<u8>> {
247                (**self).read_log(name)
248            }
249
250            fn append_log(&self, name: &str, len: u64, bytes: &[u8]) -> Result<(), LogWriteError> {
251                (**self).append_log(name, len, bytes)
252            }
253
254            fn replace_log(&self, name: &str, len: u64, bytes: &[u8]) -> Result<(), LogWriteError> {
255                (**self).replace_log(name, len, bytes)
256            }
257
258            fn destroy(&self) -> io::Result<()> {
259                (**self).destroy()
260            }
261        }
262    )*};
263}
264
265forward!(Arc, Box);
266
267/// The output of `work`, polled once. A [`Store`]'s calls through
268/// [`AsyncStore`] are done when made, so its saves finish on the first
269/// poll: the synchronous API runs the one asynchronous implementation.
270pub(crate) fn now<T>(
271    work: impl Future<Output = Result<T, SessionError>>,
272) -> Result<T, SessionError> {
273    match pin!(work).poll(&mut Context::from_waker(Waker::noop())) {
274        Poll::Ready(output) => output,
275        Poll::Pending => Err(io::Error::other("a local store's call did not finish").into()),
276    }
277}
278
279/// Refuses `name` unless it is relative and `/`-separated, with no empty,
280/// `.` or `..` part and no `\`, so it names a place inside the store
281/// wherever the store keeps it.
282///
283/// ```
284/// use rig_memory::check_name;
285///
286/// assert!(check_name("plugins/notes.json").is_ok());
287/// assert!(check_name("../elsewhere").is_err());
288/// assert!(check_name("/etc/passwd").is_err());
289/// ```
290///
291/// # Errors
292///
293/// [`io::ErrorKind::InvalidInput`], naming it.
294pub fn check_name(name: &str) -> io::Result<()> {
295    let bad = |part: &str| matches!(part, "" | "." | "..") || part.contains('\\');
296    if name.split('/').any(bad) {
297        let why = format!("{name:?} is no name in a session");
298        return Err(io::Error::new(io::ErrorKind::InvalidInput, why));
299    }
300    Ok(())
301}
302
303/// Refuses `name` as a log name: [`check_name`], in one part.
304///
305/// ```
306/// use rig_memory::check_log_name;
307///
308/// assert!(check_log_name("main").is_ok());
309/// assert!(check_log_name("agents/main").is_err());
310/// ```
311///
312/// # Errors
313///
314/// [`io::ErrorKind::InvalidInput`], naming it.
315pub fn check_log_name(name: &str) -> io::Result<()> {
316    check_name(name)?;
317    if name.contains('/') {
318        let why = format!("{name:?} is no log name");
319        return Err(io::Error::new(io::ErrorKind::InvalidInput, why));
320    }
321    Ok(())
322}
323
324/// A session kept in memory, such as for tests or where there is no file
325/// system. Cloning shares the contents.
326#[derive(Clone, Default)]
327pub struct MemoryStore(Arc<Mutex<Memory>>);
328
329#[derive(Default)]
330struct Memory {
331    logs: BTreeMap<String, Vec<u8>>,
332    files: BTreeMap<String, Vec<u8>>,
333}
334
335impl MemoryStore {
336    fn memory(&self) -> MutexGuard<'_, Memory> {
337        self.0.lock().unwrap_or_else(PoisonError::into_inner)
338    }
339}
340
341fn not_found() -> io::Error {
342    io::ErrorKind::NotFound.into()
343}
344
345impl Files for MemoryStore {
346    fn write_file(&self, name: &str, bytes: &[u8]) -> io::Result<()> {
347        check_name(name)?;
348        self.memory().files.insert(name.to_owned(), bytes.to_vec());
349        Ok(())
350    }
351
352    fn append_file(&self, name: &str, bytes: &[u8]) -> io::Result<()> {
353        check_name(name)?;
354        let mut memory = self.memory();
355        let file = memory.files.entry(name.to_owned()).or_default();
356        file.extend_from_slice(bytes);
357        Ok(())
358    }
359
360    fn read_file(&self, name: &str) -> io::Result<Vec<u8>> {
361        check_name(name)?;
362        self.memory().files.get(name).cloned().ok_or_else(not_found)
363    }
364
365    fn remove_file(&self, name: &str) -> io::Result<()> {
366        check_name(name)?;
367        self.memory().files.remove(name);
368        Ok(())
369    }
370
371    fn files(&self) -> io::Result<Vec<String>> {
372        Ok(self.memory().files.keys().cloned().collect())
373    }
374}
375
376impl Store for MemoryStore {
377    fn logs(&self) -> io::Result<Vec<String>> {
378        Ok(self.memory().logs.keys().cloned().collect())
379    }
380
381    fn read_log(&self, name: &str) -> io::Result<Vec<u8>> {
382        check_log_name(name)?;
383        self.memory().logs.get(name).cloned().ok_or_else(not_found)
384    }
385
386    fn append_log(&self, name: &str, len: u64, bytes: &[u8]) -> Result<(), LogWriteError> {
387        check_log_name(name)?;
388        let mut memory = self.memory();
389        if memory.logs.get(name).map_or(0, Vec::len) as u64 != len {
390            return Err(LogWriteError::Changed);
391        }
392        let log = memory.logs.entry(name.to_owned()).or_default();
393        log.extend_from_slice(bytes);
394        Ok(())
395    }
396
397    fn replace_log(&self, name: &str, len: u64, bytes: &[u8]) -> Result<(), LogWriteError> {
398        check_log_name(name)?;
399        let mut memory = self.memory();
400        if memory.logs.get(name).map_or(0, Vec::len) as u64 != len {
401            return Err(LogWriteError::Changed);
402        }
403        memory.logs.insert(name.to_owned(), bytes.to_vec());
404        Ok(())
405    }
406
407    fn destroy(&self) -> io::Result<()> {
408        *self.memory() = Memory::default();
409        Ok(())
410    }
411}
412
413/// Where plugins' files are in a session's store.
414pub const PLUGINS: &str = "plugins/";
415
416/// The files of a session's store under [`PLUGINS`], by their names there:
417/// what a plugin keeps for the session.
418pub struct PluginFiles(pub Arc<dyn Files>);
419
420impl Files for PluginFiles {
421    fn write_file(&self, name: &str, bytes: &[u8]) -> io::Result<()> {
422        self.0.write_file(&format!("{PLUGINS}{name}"), bytes)
423    }
424
425    fn append_file(&self, name: &str, bytes: &[u8]) -> io::Result<()> {
426        self.0.append_file(&format!("{PLUGINS}{name}"), bytes)
427    }
428
429    fn read_file(&self, name: &str) -> io::Result<Vec<u8>> {
430        self.0.read_file(&format!("{PLUGINS}{name}"))
431    }
432
433    fn remove_file(&self, name: &str) -> io::Result<()> {
434        self.0.remove_file(&format!("{PLUGINS}{name}"))
435    }
436
437    fn files(&self) -> io::Result<Vec<String>> {
438        let files = self.0.files()?.into_iter();
439        let within = files.filter_map(|name| Some(name.strip_prefix(PLUGINS)?.to_owned()));
440        Ok(within.collect())
441    }
442}