rig_memory/store.rs
1//! Where a session is kept: [`Files`] by name, and a [`Store`] that also
2//! holds each log of the session. The blobs its messages name are its
3//! files `blobs/<name>`, named by their content, which a directory store
4//! keeps under `files/blobs/`. [`AsyncStore`] is the same storage reached
5//! asynchronously, such as a database, an object store or IndexedDB; every
6//! [`Store`] is one. A log write names the length the writer read, and the
7//! store writes only if the log is still that long, checking and writing
8//! in one step ([`LogWriteError::Changed`]).
9
10use std::collections::BTreeMap;
11use std::future::{Future, ready};
12use std::io;
13use std::pin::pin;
14use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
15use std::task::{Context, Poll, Waker};
16
17use rig_core::wasm_compat::WasmCompatSend;
18
19use crate::SessionError;
20
21/// Why a log write wrote nothing.
22#[derive(Debug, thiserror::Error)]
23pub enum LogWriteError {
24 /// The log is not the `len` bytes the writer read: another writer
25 /// changed it. Nothing was written.
26 #[error("the log changed since it was read")]
27 Changed,
28 /// The store failed.
29 #[error(transparent)]
30 Io(#[from] io::Error),
31}
32
33impl From<LogWriteError> for io::Error {
34 /// The store's own error back; [`LogWriteError::Changed`] as an
35 /// [`io::ErrorKind::Other`] saying so.
36 fn from(error: LogWriteError) -> Self {
37 match error {
38 LogWriteError::Changed => io::Error::other(LogWriteError::Changed.to_string()),
39 LogWriteError::Io(error) => error,
40 }
41 }
42}
43
44/// A place that keeps files by name, such as a session's plugin data or
45/// effect log. A name is relative and `/`-separated ([`check_name`]).
46/// Every method takes `&self`, so one value is shared, between threads on
47/// every target: a host keeps it as `Arc<dyn Files>` or `Arc<dyn Store>`
48/// in state its systems share, such as a Bevy resource, which is `Send`
49/// and `Sync` on wasm too; `Arc<dyn Store>` and `Box<dyn Store>` are
50/// themselves a [`Store`], so an [`AsyncStore`] too. A store that is not
51/// `Send` and `Sync`, such as one over a JavaScript promise, is an
52/// [`AsyncStore`].
53pub trait Files: Send + Sync + 'static {
54 /// Replaces the file `name` with `bytes`, atomically: a reader sees the
55 /// old content or the new.
56 fn write_file(&self, name: &str, bytes: &[u8]) -> io::Result<()>;
57
58 /// Adds `bytes` at the end of the file `name`, which it creates when
59 /// there is none.
60 fn append_file(&self, name: &str, bytes: &[u8]) -> io::Result<()>;
61
62 /// The file `name`; [`io::ErrorKind::NotFound`] when there is none.
63 fn read_file(&self, name: &str) -> io::Result<Vec<u8>>;
64
65 /// Removes the file `name`; removing one that is not there succeeds.
66 fn remove_file(&self, name: &str) -> io::Result<()>;
67
68 /// The names of the files, in no particular order.
69 fn files(&self) -> io::Result<Vec<String>>;
70}
71
72/// Where a session is kept: its [`Files`], with the content-addressed
73/// blobs its messages name under `blobs/`, and a log per agent plus the
74/// session's own, appended to and read whole. Log names are one part of a
75/// name ([`check_name`] without `/`).
76pub trait Store: Files {
77 /// The names of the logs, in no particular order.
78 fn logs(&self) -> io::Result<Vec<String>>;
79
80 /// The whole log `name`; [`io::ErrorKind::NotFound`] when there is none.
81 fn read_log(&self, name: &str) -> io::Result<Vec<u8>>;
82
83 /// Appends `bytes` to the log `name` if it is `len` bytes long now (0
84 /// when there is none, which it then starts); else
85 /// [`LogWriteError::Changed`], writing nothing. The check and the
86 /// append are one step for every writer of the store, other processes
87 /// included, so of two writers that read the log at the same length
88 /// one appends and the other is refused. A log that shrank and grew
89 /// back to exactly `len` bytes since it was read is not told apart;
90 /// rig shrinks a log only by setting a torn last line aside, or by
91 /// [`rewrite`](crate::rewrite) and [`Self::destroy`], which are for a
92 /// stopped session.
93 fn append_log(&self, name: &str, len: u64, bytes: &[u8]) -> Result<(), LogWriteError>;
94
95 /// Replaces the log `name` with `bytes`, under the condition
96 /// [`Self::append_log`] has, atomically and durably: a reader, and a
97 /// crash or power loss, leave the old log or the new, never a part of
98 /// either or an empty file; an append after it goes after the new
99 /// bytes. Starts the log when there is none.
100 fn replace_log(&self, name: &str, len: u64, bytes: &[u8]) -> Result<(), LogWriteError>;
101
102 /// Removes everything the store holds: logs and files.
103 /// Idempotent: calling it again after a crash midway finishes the job.
104 fn destroy(&self) -> io::Result<()>;
105}
106
107/// A session's storage reached asynchronously: the part of [`Store`] that
108/// [`load_async`](crate::load_async) and [`save_async`](crate::save_async)
109/// use, for a store behind a network or a JavaScript promise. Every
110/// [`Store`] is one, its calls done when made: a local store pays no thread
111/// hop and no allocation for it. A type that implements [`Store`] gets this
112/// from it and cannot have its own (the two impls would overlap): wrap it
113/// in a type of your own instead.
114///
115/// Each method keeps [`Store`]'s contract. A name is checked by
116/// [`check_name`] (a log name is one part) and may come from a log's
117/// content, so a store refuses one that does not pass. The two log writes
118/// check the log's length and write in one step, such as a conditional
119/// `UPDATE … WHERE octet_length(bytes) = $len` or one IndexedDB
120/// `readwrite` transaction, never a read followed by a write. An append
121/// adds its bytes whole, or at worst leaves a cut last line, which the next
122/// save sets aside: [`save_async`](crate::save_async) writes one turn with
123/// one append, so a cancelled or failed save never stores part of one.
124/// [`io::ErrorKind::NotFound`] means there is no such log or file.
125///
126/// Its futures are `Send` natively, and on browser wasm neither they nor
127/// the store need be. A holder that shares the store, such as an agent,
128/// asks for `WasmCompatSend + WasmCompatSync` itself. An async store that
129/// is shared is `Clone` over an inner `Arc`: `Arc<S>` is an `AsyncStore`
130/// only when `S` is a [`Store`].
131pub trait AsyncStore {
132 /// The whole log `name`.
133 fn read_log_async(
134 &self,
135 name: &str,
136 ) -> impl Future<Output = io::Result<Vec<u8>>> + WasmCompatSend;
137
138 /// Appends `bytes` to the log `name` if it is `len` bytes long now
139 /// ([`Store::append_log`]).
140 fn append_log_async(
141 &self,
142 name: &str,
143 len: u64,
144 bytes: &[u8],
145 ) -> impl Future<Output = Result<(), LogWriteError>> + WasmCompatSend;
146
147 /// Replaces the log `name` with `bytes` if it is `len` bytes long now,
148 /// atomically and durably ([`Store::replace_log`]).
149 fn replace_log_async(
150 &self,
151 name: &str,
152 len: u64,
153 bytes: &[u8],
154 ) -> impl Future<Output = Result<(), LogWriteError>> + WasmCompatSend;
155
156 /// The file `name`.
157 fn read_file_async(
158 &self,
159 name: &str,
160 ) -> impl Future<Output = io::Result<Vec<u8>>> + WasmCompatSend;
161
162 /// Replaces the file `name` with `bytes`, atomically.
163 fn write_file_async(
164 &self,
165 name: &str,
166 bytes: &[u8],
167 ) -> impl Future<Output = io::Result<()>> + WasmCompatSend;
168}
169
170/// A local store's calls, done when made.
171impl<S: Store + ?Sized> AsyncStore for S {
172 fn read_log_async(
173 &self,
174 name: &str,
175 ) -> impl Future<Output = io::Result<Vec<u8>>> + WasmCompatSend {
176 ready(self.read_log(name))
177 }
178
179 fn append_log_async(
180 &self,
181 name: &str,
182 len: u64,
183 bytes: &[u8],
184 ) -> impl Future<Output = Result<(), LogWriteError>> + WasmCompatSend {
185 ready(self.append_log(name, len, bytes))
186 }
187
188 fn replace_log_async(
189 &self,
190 name: &str,
191 len: u64,
192 bytes: &[u8],
193 ) -> impl Future<Output = Result<(), LogWriteError>> + WasmCompatSend {
194 ready(self.replace_log(name, len, bytes))
195 }
196
197 fn read_file_async(
198 &self,
199 name: &str,
200 ) -> impl Future<Output = io::Result<Vec<u8>>> + WasmCompatSend {
201 ready(self.read_file(name))
202 }
203
204 fn write_file_async(
205 &self,
206 name: &str,
207 bytes: &[u8],
208 ) -> impl Future<Output = io::Result<()>> + WasmCompatSend {
209 ready(self.write_file(name, bytes))
210 }
211}
212
213/// The store a pointer points at: `Arc<dyn Store>` and `Box<dyn Store>`
214/// are stores, so [`AsyncStore`]s too.
215macro_rules! forward {
216 ($($pointer:ident),*) => {$(
217 impl<S: Files + ?Sized> Files for $pointer<S> {
218 fn write_file(&self, name: &str, bytes: &[u8]) -> io::Result<()> {
219 (**self).write_file(name, bytes)
220 }
221
222 fn append_file(&self, name: &str, bytes: &[u8]) -> io::Result<()> {
223 (**self).append_file(name, bytes)
224 }
225
226 fn read_file(&self, name: &str) -> io::Result<Vec<u8>> {
227 (**self).read_file(name)
228 }
229
230 fn remove_file(&self, name: &str) -> io::Result<()> {
231 (**self).remove_file(name)
232 }
233
234 fn files(&self) -> io::Result<Vec<String>> {
235 (**self).files()
236 }
237 }
238
239 impl<S: Store + ?Sized> Store for $pointer<S> {
240 fn logs(&self) -> io::Result<Vec<String>> {
241 (**self).logs()
242 }
243
244 fn read_log(&self, name: &str) -> io::Result<Vec<u8>> {
245 (**self).read_log(name)
246 }
247
248 fn append_log(&self, name: &str, len: u64, bytes: &[u8]) -> Result<(), LogWriteError> {
249 (**self).append_log(name, len, bytes)
250 }
251
252 fn replace_log(&self, name: &str, len: u64, bytes: &[u8]) -> Result<(), LogWriteError> {
253 (**self).replace_log(name, len, bytes)
254 }
255
256 fn destroy(&self) -> io::Result<()> {
257 (**self).destroy()
258 }
259 }
260 )*};
261}
262
263forward!(Arc, Box);
264
265/// The output of `work`, polled once. A [`Store`]'s calls through
266/// [`AsyncStore`] are done when made, so its saves finish on the first
267/// poll: the synchronous API runs the one asynchronous implementation.
268pub(crate) fn now<T>(
269 work: impl Future<Output = Result<T, SessionError>>,
270) -> Result<T, SessionError> {
271 match pin!(work).poll(&mut Context::from_waker(Waker::noop())) {
272 Poll::Ready(output) => output,
273 Poll::Pending => Err(io::Error::other("a local store's call did not finish").into()),
274 }
275}
276
277/// Refuses `name` unless it is relative and `/`-separated, with no empty,
278/// `.` or `..` part and no `\`, so it names a place inside the store
279/// wherever the store keeps it.
280///
281/// ```
282/// use rig_memory::check_name;
283///
284/// assert!(check_name("plugins/notes.json").is_ok());
285/// assert!(check_name("../elsewhere").is_err());
286/// assert!(check_name("/etc/passwd").is_err());
287/// ```
288///
289/// # Errors
290///
291/// [`io::ErrorKind::InvalidInput`], naming it.
292pub fn check_name(name: &str) -> io::Result<()> {
293 let bad = |part: &str| matches!(part, "" | "." | "..") || part.contains('\\');
294 if name.split('/').any(bad) {
295 let why = format!("{name:?} is no name in a session");
296 return Err(io::Error::new(io::ErrorKind::InvalidInput, why));
297 }
298 Ok(())
299}
300
301/// Refuses `name` as a log name: [`check_name`], in one part.
302///
303/// ```
304/// use rig_memory::check_log_name;
305///
306/// assert!(check_log_name("main").is_ok());
307/// assert!(check_log_name("agents/main").is_err());
308/// ```
309///
310/// # Errors
311///
312/// [`io::ErrorKind::InvalidInput`], naming it.
313pub fn check_log_name(name: &str) -> io::Result<()> {
314 check_name(name)?;
315 if name.contains('/') {
316 let why = format!("{name:?} is no log name");
317 return Err(io::Error::new(io::ErrorKind::InvalidInput, why));
318 }
319 Ok(())
320}
321
322/// The longest name, in bytes, [`check_new_log_name`] allows: with its
323/// file's extension it fits the 255 bytes every disk allows a file name.
324pub const MAX_NEW_LOG_NAME: usize = 200;
325
326/// Refuses `name` for a new log: [`check_log_name`], a name longer than
327/// [`MAX_NEW_LOG_NAME`] bytes, or [`SESSION`](crate::SESSION) in any letter
328/// case ([`same_log_name`]), which names the session's own log on a disk
329/// that ignores case. A log that exists keeps its name: stores read and
330/// write it under [`check_log_name`].
331///
332/// ```
333/// use rig_memory::check_new_log_name;
334///
335/// assert!(check_new_log_name("helper").is_ok());
336/// assert!(check_new_log_name("Session").is_err());
337/// assert!(check_new_log_name(&"x".repeat(300)).is_err());
338/// ```
339///
340/// # Errors
341///
342/// [`io::ErrorKind::InvalidInput`], naming it.
343pub fn check_new_log_name(name: &str) -> io::Result<()> {
344 check_log_name(name)?;
345 let why = if name.len() > MAX_NEW_LOG_NAME {
346 format!("{name:?} is longer than {MAX_NEW_LOG_NAME} bytes")
347 } else if same_log_name(name, crate::SESSION) {
348 format!("{name:?} is the session's own log")
349 } else {
350 return Ok(());
351 };
352 Err(io::Error::new(io::ErrorKind::InvalidInput, why))
353}
354
355/// Whether the logs `a` and `b` may be one file on a disk that ignores
356/// case: their names differ only in letter case, as Unicode's lower- or
357/// uppercase mapping sees it, so the final and medial sigma (`ς`, `σ`)
358/// match too. A file system's own folding can still differ.
359///
360/// ```
361/// use rig_memory::same_log_name;
362///
363/// assert!(same_log_name("Helper", "helper"));
364/// assert!(same_log_name("ς", "σ"));
365/// assert!(!same_log_name("helper", "helpers"));
366/// ```
367pub fn same_log_name(a: &str, b: &str) -> bool {
368 a.to_lowercase() == b.to_lowercase() || a.to_uppercase() == b.to_uppercase()
369}
370
371/// A session kept in memory, such as for tests or where there is no file
372/// system. Cloning shares the contents.
373#[derive(Clone, Default)]
374pub struct MemoryStore(Arc<Mutex<Memory>>);
375
376#[derive(Default)]
377struct Memory {
378 logs: BTreeMap<String, Vec<u8>>,
379 files: BTreeMap<String, Vec<u8>>,
380}
381
382impl MemoryStore {
383 fn memory(&self) -> MutexGuard<'_, Memory> {
384 self.0.lock().unwrap_or_else(PoisonError::into_inner)
385 }
386}
387
388fn not_found() -> io::Error {
389 io::ErrorKind::NotFound.into()
390}
391
392impl Files for MemoryStore {
393 fn write_file(&self, name: &str, bytes: &[u8]) -> io::Result<()> {
394 check_name(name)?;
395 self.memory().files.insert(name.to_owned(), bytes.to_vec());
396 Ok(())
397 }
398
399 fn append_file(&self, name: &str, bytes: &[u8]) -> io::Result<()> {
400 check_name(name)?;
401 let mut memory = self.memory();
402 let file = memory.files.entry(name.to_owned()).or_default();
403 file.extend_from_slice(bytes);
404 Ok(())
405 }
406
407 fn read_file(&self, name: &str) -> io::Result<Vec<u8>> {
408 check_name(name)?;
409 self.memory().files.get(name).cloned().ok_or_else(not_found)
410 }
411
412 fn remove_file(&self, name: &str) -> io::Result<()> {
413 check_name(name)?;
414 self.memory().files.remove(name);
415 Ok(())
416 }
417
418 fn files(&self) -> io::Result<Vec<String>> {
419 Ok(self.memory().files.keys().cloned().collect())
420 }
421}
422
423impl Store for MemoryStore {
424 fn logs(&self) -> io::Result<Vec<String>> {
425 Ok(self.memory().logs.keys().cloned().collect())
426 }
427
428 fn read_log(&self, name: &str) -> io::Result<Vec<u8>> {
429 check_log_name(name)?;
430 self.memory().logs.get(name).cloned().ok_or_else(not_found)
431 }
432
433 fn append_log(&self, name: &str, len: u64, bytes: &[u8]) -> Result<(), LogWriteError> {
434 check_log_name(name)?;
435 let mut memory = self.memory();
436 if memory.logs.get(name).map_or(0, Vec::len) as u64 != len {
437 return Err(LogWriteError::Changed);
438 }
439 let log = memory.logs.entry(name.to_owned()).or_default();
440 log.extend_from_slice(bytes);
441 Ok(())
442 }
443
444 fn replace_log(&self, name: &str, len: u64, bytes: &[u8]) -> Result<(), LogWriteError> {
445 check_log_name(name)?;
446 let mut memory = self.memory();
447 if memory.logs.get(name).map_or(0, Vec::len) as u64 != len {
448 return Err(LogWriteError::Changed);
449 }
450 memory.logs.insert(name.to_owned(), bytes.to_vec());
451 Ok(())
452 }
453
454 fn destroy(&self) -> io::Result<()> {
455 *self.memory() = Memory::default();
456 Ok(())
457 }
458}
459
460/// Where plugins' files are in a session's store.
461pub const PLUGINS: &str = "plugins/";
462
463/// The files of a session's store under [`PLUGINS`], by their names there:
464/// what a plugin keeps for the session.
465pub struct PluginFiles(pub Arc<dyn Files>);
466
467impl Files for PluginFiles {
468 fn write_file(&self, name: &str, bytes: &[u8]) -> io::Result<()> {
469 self.0.write_file(&format!("{PLUGINS}{name}"), bytes)
470 }
471
472 fn append_file(&self, name: &str, bytes: &[u8]) -> io::Result<()> {
473 self.0.append_file(&format!("{PLUGINS}{name}"), bytes)
474 }
475
476 fn read_file(&self, name: &str) -> io::Result<Vec<u8>> {
477 self.0.read_file(&format!("{PLUGINS}{name}"))
478 }
479
480 fn remove_file(&self, name: &str) -> io::Result<()> {
481 self.0.remove_file(&format!("{PLUGINS}{name}"))
482 }
483
484 fn files(&self) -> io::Result<Vec<String>> {
485 let files = self.0.files()?.into_iter();
486 let within = files.filter_map(|name| Some(name.strip_prefix(PLUGINS)?.to_owned()));
487 Ok(within.collect())
488 }
489}