Skip to main content

rig_memory/
dir.rs

1//! A session kept as files in a directory: `logs/<name>.jsonl` and
2//! `files/<name…>`, its blobs under `files/blobs/`, and the lock file
3//! `.rig-memory.lock` every change takes. The store reads only the two
4//! folders, so whatever else the directory holds is left alone.
5
6use std::fs::{self, File, OpenOptions};
7use std::io::{self, Write};
8use std::path::{Path, PathBuf};
9use std::sync::atomic::{AtomicU64, Ordering};
10
11use crate::{Files, LogWriteError, SessionError, Store, check_log_name, check_name};
12
13const LOGS: &str = "logs";
14const FILES: &str = "files";
15
16/// The extension of a log.
17const LOG: &str = ".jsonl";
18
19/// What a file being written is called until it is complete; a name ending
20/// in it is refused.
21const PART: &str = ".rig-part";
22
23/// What a folder is renamed to while [`Store::destroy`] removes it.
24const TRASH: &str = ".rig-trash";
25
26/// The file every change of a [`DirStore`] locks, beside its folders. It
27/// is never removed, so every store open on the directory locks the same
28/// file.
29const LOCK: &str = ".rig-memory.lock";
30
31/// A [`Store`] in the directory it was opened on, whose folders are made
32/// as files are written. Each change locks the directory's lock file, so
33/// stores open on one directory, in this process or another, take turns;
34/// reads take no lock. Cloning opens the same directory.
35#[derive(Clone)]
36pub struct DirStore {
37    dir: PathBuf,
38    files: DirFiles,
39}
40
41/// [`Files`] kept as plain files under a directory, each name its path
42/// there, such as data kept outside any session. Folders are made as files
43/// are written, and nothing is written before.
44#[derive(Clone, Debug, PartialEq, Eq)]
45pub struct DirFiles(PathBuf);
46
47impl DirFiles {
48    /// The files under `dir`, which need not exist yet.
49    pub fn new(dir: impl Into<PathBuf>) -> Self {
50        Self(dir.into())
51    }
52
53    /// The directory itself.
54    pub fn path(&self) -> &Path {
55        &self.0
56    }
57
58    fn file(&self, name: &str) -> io::Result<PathBuf> {
59        check_name(name)?;
60        refuse_part(name)?;
61        Ok(self.0.join(name))
62    }
63
64    /// The path to write `name` to. On a case-insensitive filesystem a
65    /// name such as `0.RIG-PART` is the name of the part file `0.rig-part`,
66    /// so it is written only where a file of exactly that name is kept
67    /// already, which no part file then is.
68    fn file_to_write(&self, name: &str) -> io::Result<PathBuf> {
69        let path = self.file(name)?;
70        let last = name.rsplit('/').next().unwrap_or(name);
71        let digits = last.bytes().take_while(u8::is_ascii_digit).count();
72        let tail = last.as_bytes().get(digits..).unwrap_or_default();
73        if digits > 0 && tail.eq_ignore_ascii_case(PART.as_bytes()) {
74            let kept = path.parent().map_or(Ok(Vec::new()), names)?;
75            if !kept.iter().any(|kept| kept == last) {
76                let why = format!("{name:?} can name a file being written");
77                return Err(io::Error::new(io::ErrorKind::InvalidInput, why));
78            }
79        }
80        Ok(path)
81    }
82}
83
84impl DirStore {
85    /// The session in `dir`, which need not exist yet. Nothing is written.
86    ///
87    /// # Errors
88    ///
89    /// [`SessionError::OldLayout`] when `dir` holds a top-level `*.jsonl`
90    /// file, as a session saved by an older rig does; it is left as it is.
91    /// [`SessionError::Io`] when `dir` cannot be listed.
92    pub fn open(dir: impl Into<PathBuf>) -> Result<Self, SessionError> {
93        let dir = dir.into();
94        let old = names(&dir)?
95            .into_iter()
96            .any(|name| name.ends_with(LOG) && dir.join(&name).is_file());
97        if old {
98            return Err(SessionError::OldLayout);
99        }
100        Ok(Self {
101            files: DirFiles::new(dir.join(FILES)),
102            dir,
103        })
104    }
105
106    /// The directory's lock file, locked until it is dropped.
107    fn lock(&self) -> io::Result<File> {
108        fs::create_dir_all(&self.dir)?;
109        let lock = OpenOptions::new()
110            .create(true)
111            .write(true)
112            .truncate(false)
113            .open(self.dir.join(LOCK))?;
114        lock.lock()?;
115        Ok(lock)
116    }
117
118    fn log(&self, name: &str) -> io::Result<PathBuf> {
119        check_log_name(name)?;
120        Ok(self.dir.join(LOGS).join(format!("{name}{LOG}")))
121    }
122}
123
124fn refuse_part(name: &str) -> io::Result<()> {
125    if name.ends_with(PART) {
126        let why = format!("{name:?} ends as a file being written does");
127        return Err(io::Error::new(io::ErrorKind::InvalidInput, why));
128    }
129    Ok(())
130}
131
132/// The names in the directory `dir`; none when it does not exist.
133fn names(dir: &Path) -> io::Result<Vec<String>> {
134    let entries = match fs::read_dir(dir) {
135        Err(failure) if failure.kind() == io::ErrorKind::NotFound => return Ok(Vec::new()),
136        entries => entries?,
137    };
138    let names = entries.filter_map(|entry| entry.ok()?.file_name().into_string().ok());
139    Ok(names.collect())
140}
141
142/// Removing what is not there succeeds.
143fn absent_is_fine(removed: io::Result<()>) -> io::Result<()> {
144    match removed {
145        Err(failure) if failure.kind() == io::ErrorKind::NotFound => Ok(()),
146        removed => removed,
147    }
148}
149
150/// Writes `bytes` to `path` atomically: to a part file beside it, then
151/// renamed over it.
152fn write_atomic(path: &Path, bytes: &[u8]) -> io::Result<()> {
153    Part::write(path, bytes, false)?.rename(path)
154}
155
156/// As [`write_atomic`], and durably: the part file is synced before the
157/// rename and the folder after it (best effort), so a power loss leaves
158/// the old file or the new, never an empty one.
159fn write_durable(path: &Path, bytes: &[u8]) -> io::Result<()> {
160    Part::write(path, bytes, true)?.rename(path)?;
161    if let Some(parent) = path.parent() {
162        let _ = File::open(parent).and_then(|folder| folder.sync_all());
163    }
164    Ok(())
165}
166
167/// A part file beside the file it replaces, under a short name of its
168/// own (`<n>.rig-part`), so writers never write to or rename each other's
169/// and any name the folder takes can be replaced. Until it is renamed into
170/// place, dropping it removes it, so a write that fails leaves none behind.
171struct Part(Option<PathBuf>);
172
173impl Part {
174    /// Writes `bytes` to a new part file beside `path`, whose folder it
175    /// makes, synced with `sync`.
176    fn write(path: &Path, bytes: &[u8], sync: bool) -> io::Result<Self> {
177        static NEXT: AtomicU64 = AtomicU64::new(0);
178        if let Some(parent) = path.parent() {
179            fs::create_dir_all(parent)?;
180        }
181        let (part, mut file) = loop {
182            let n = NEXT.fetch_add(1, Ordering::Relaxed);
183            let part = path.with_file_name(format!("{n}{PART}"));
184            // A name another process holds, or a crash left, is skipped.
185            match File::create_new(&part) {
186                Ok(file) => break (Self(Some(part)), file),
187                Err(failure) if failure.kind() == io::ErrorKind::AlreadyExists => {}
188                Err(failure) => return Err(failure),
189            }
190        };
191        file.write_all(bytes)?;
192        if sync {
193            file.sync_all()?;
194        }
195        Ok(part)
196    }
197
198    /// Renames the part file over `path`; its name is then free for
199    /// another writer, so it is no longer removed.
200    fn rename(mut self, path: &Path) -> io::Result<()> {
201        if let Some(part) = &self.0 {
202            fs::rename(part, path)?;
203        }
204        self.0 = None;
205        Ok(())
206    }
207}
208
209impl Drop for Part {
210    fn drop(&mut self) {
211        if let Some(part) = &self.0 {
212            let _ = fs::remove_file(part);
213        }
214    }
215}
216
217impl Files for DirFiles {
218    fn write_file(&self, name: &str, bytes: &[u8]) -> io::Result<()> {
219        write_atomic(&self.file_to_write(name)?, bytes)
220    }
221
222    fn append_file(&self, name: &str, bytes: &[u8]) -> io::Result<()> {
223        let path = self.file_to_write(name)?;
224        if let Some(parent) = path.parent() {
225            fs::create_dir_all(parent)?;
226        }
227        let mut file = OpenOptions::new().create(true).append(true).open(path)?;
228        file.write_all(bytes)
229    }
230
231    fn read_file(&self, name: &str) -> io::Result<Vec<u8>> {
232        fs::read(self.file(name)?)
233    }
234
235    fn remove_file(&self, name: &str) -> io::Result<()> {
236        absent_is_fine(fs::remove_file(self.file(name)?))
237    }
238
239    fn files(&self) -> io::Result<Vec<String>> {
240        let mut files = Vec::new();
241        let mut dirs = vec![(self.0.clone(), String::new())];
242        while let Some((dir, prefix)) = dirs.pop() {
243            for name in names(&dir)? {
244                let path = dir.join(&name);
245                if path.is_dir() {
246                    dirs.push((path, format!("{prefix}{name}/")));
247                } else if !name.ends_with(PART) {
248                    files.push(format!("{prefix}{name}"));
249                }
250            }
251        }
252        Ok(files)
253    }
254}
255
256/// Refuses with [`LogWriteError::Changed`] unless the file at `path` is
257/// `len` bytes long, 0 when there is none.
258fn check_len(path: &Path, len: u64) -> Result<(), LogWriteError> {
259    let held = match fs::metadata(path) {
260        Err(failure) if failure.kind() == io::ErrorKind::NotFound => 0,
261        metadata => metadata?.len(),
262    };
263    if held != len {
264        return Err(LogWriteError::Changed);
265    }
266    Ok(())
267}
268
269impl Files for DirStore {
270    fn write_file(&self, name: &str, bytes: &[u8]) -> io::Result<()> {
271        let _lock = self.lock()?;
272        self.files.write_file(name, bytes)
273    }
274
275    fn append_file(&self, name: &str, bytes: &[u8]) -> io::Result<()> {
276        let _lock = self.lock()?;
277        self.files.append_file(name, bytes)
278    }
279
280    fn read_file(&self, name: &str) -> io::Result<Vec<u8>> {
281        self.files.read_file(name)
282    }
283
284    fn remove_file(&self, name: &str) -> io::Result<()> {
285        let _lock = self.lock()?;
286        self.files.remove_file(name)
287    }
288
289    fn files(&self) -> io::Result<Vec<String>> {
290        self.files.files()
291    }
292}
293
294impl Store for DirStore {
295    fn logs(&self) -> io::Result<Vec<String>> {
296        let names = names(&self.dir.join(LOGS))?.into_iter();
297        let logs = names.filter_map(|name| Some(name.strip_suffix(LOG)?.to_owned()));
298        Ok(logs.filter(|name| !name.is_empty()).collect())
299    }
300
301    fn read_log(&self, name: &str) -> io::Result<Vec<u8>> {
302        fs::read(self.log(name)?)
303    }
304
305    fn append_log(&self, name: &str, len: u64, bytes: &[u8]) -> Result<(), LogWriteError> {
306        let path = self.log(name)?;
307        let _lock = self.lock()?;
308        check_len(&path, len)?;
309        if let Some(parent) = path.parent() {
310            fs::create_dir_all(parent)?;
311        }
312        let mut log = OpenOptions::new().create(true).append(true).open(path)?;
313        Ok(log.write_all(bytes)?)
314    }
315
316    /// The new log is written beside, synced and renamed over the old one:
317    /// the one synced write, as it is rare and replaces history.
318    fn replace_log(&self, name: &str, len: u64, bytes: &[u8]) -> Result<(), LogWriteError> {
319        let path = self.log(name)?;
320        let _lock = self.lock()?;
321        check_len(&path, len)?;
322        Ok(write_durable(&path, bytes)?)
323    }
324
325    /// Each folder is renamed to a trash name inside the directory, so it
326    /// is gone at once, then removed; a second call removes what a crash
327    /// left of the first. The lock file stays.
328    fn destroy(&self) -> io::Result<()> {
329        let _lock = self.lock()?;
330        for folder in [LOGS, FILES] {
331            let (path, trash) = (
332                self.dir.join(folder),
333                self.dir.join(format!("{folder}{TRASH}")),
334            );
335            if path.exists() {
336                absent_is_fine(fs::remove_dir_all(&trash))?;
337                fs::rename(&path, &trash)?;
338            }
339            absent_is_fine(fs::remove_dir_all(&trash))?;
340        }
341        Ok(())
342    }
343}
344
345#[cfg(test)]
346mod tests;