Skip to main content

rig_memory/
dir.rs

1//! A session kept as files in a directory: `logs/<name>.jsonl` and
2//! `files/<name…>`, its blobs under `files/blobs/`, and the lock file
3//! `.rig-memory.lock` every change takes. The store reads only the two
4//! folders, so whatever else the directory holds is left alone.
5
6use std::fs::{self, File, OpenOptions};
7use std::io::{self, Write};
8use std::path::{Path, PathBuf};
9
10use crate::{Files, LogWriteError, SessionError, Store, check_log_name, check_name};
11
12const LOGS: &str = "logs";
13const FILES: &str = "files";
14
15/// The extension of a log.
16const LOG: &str = ".jsonl";
17
18/// What a file being written is called until it is complete; a name ending
19/// in it is refused.
20const PART: &str = ".rig-part";
21
22/// What a folder is renamed to while [`Store::destroy`] removes it.
23const TRASH: &str = ".rig-trash";
24
25/// The file every change of a [`DirStore`] locks, beside its folders. It
26/// is never removed, so every store open on the directory locks the same
27/// file.
28const LOCK: &str = ".rig-memory.lock";
29
30/// A [`Store`] in the directory it was opened on, whose folders are made
31/// as files are written. Each change locks the directory's lock file, so
32/// stores open on one directory, in this process or another, take turns;
33/// reads take no lock. Cloning opens the same directory.
34#[derive(Clone)]
35pub struct DirStore {
36    dir: PathBuf,
37    files: DirFiles,
38}
39
40/// [`Files`] kept as plain files under a directory, each name its path
41/// there, such as data kept outside any session. Folders are made as files
42/// are written, and nothing is written before.
43#[derive(Clone, Debug, PartialEq, Eq)]
44pub struct DirFiles(PathBuf);
45
46impl DirFiles {
47    /// The files under `dir`, which need not exist yet.
48    pub fn new(dir: impl Into<PathBuf>) -> Self {
49        Self(dir.into())
50    }
51
52    /// The directory itself.
53    pub fn path(&self) -> &Path {
54        &self.0
55    }
56
57    fn file(&self, name: &str) -> io::Result<PathBuf> {
58        check_name(name)?;
59        refuse_part(name)?;
60        Ok(self.0.join(name))
61    }
62}
63
64impl DirStore {
65    /// The session in `dir`, which need not exist yet. Nothing is written.
66    ///
67    /// # Errors
68    ///
69    /// [`SessionError::OldLayout`] when `dir` holds a top-level `*.jsonl`
70    /// file, as a session saved by an older rig does; it is left as it is.
71    /// [`SessionError::Io`] when `dir` cannot be listed.
72    pub fn open(dir: impl Into<PathBuf>) -> Result<Self, SessionError> {
73        let dir = dir.into();
74        let old = names(&dir)?
75            .into_iter()
76            .any(|name| name.ends_with(LOG) && dir.join(&name).is_file());
77        if old {
78            return Err(SessionError::OldLayout);
79        }
80        Ok(Self {
81            files: DirFiles::new(dir.join(FILES)),
82            dir,
83        })
84    }
85
86    /// The directory's lock file, locked until it is dropped.
87    fn lock(&self) -> io::Result<File> {
88        fs::create_dir_all(&self.dir)?;
89        let lock = OpenOptions::new()
90            .create(true)
91            .write(true)
92            .truncate(false)
93            .open(self.dir.join(LOCK))?;
94        lock.lock()?;
95        Ok(lock)
96    }
97
98    fn log(&self, name: &str) -> io::Result<PathBuf> {
99        check_log_name(name)?;
100        Ok(self.dir.join(LOGS).join(format!("{name}{LOG}")))
101    }
102}
103
104fn refuse_part(name: &str) -> io::Result<()> {
105    if name.ends_with(PART) {
106        let why = format!("{name:?} ends as a file being written does");
107        return Err(io::Error::new(io::ErrorKind::InvalidInput, why));
108    }
109    Ok(())
110}
111
112/// The names in the directory `dir`; none when it does not exist.
113fn names(dir: &Path) -> io::Result<Vec<String>> {
114    let entries = match fs::read_dir(dir) {
115        Err(failure) if failure.kind() == io::ErrorKind::NotFound => return Ok(Vec::new()),
116        entries => entries?,
117    };
118    let names = entries.filter_map(|entry| entry.ok()?.file_name().into_string().ok());
119    Ok(names.collect())
120}
121
122/// Removing what is not there succeeds.
123fn absent_is_fine(removed: io::Result<()>) -> io::Result<()> {
124    match removed {
125        Err(failure) if failure.kind() == io::ErrorKind::NotFound => Ok(()),
126        removed => removed,
127    }
128}
129
130/// Writes `bytes` to `path` atomically: to a part file beside it, then
131/// renamed over it.
132fn write_atomic(path: &Path, bytes: &[u8]) -> io::Result<()> {
133    let part = write_part(path, bytes, false)?;
134    fs::rename(&part, path)
135}
136
137/// As [`write_atomic`], and durably: the part file is synced before the
138/// rename and the folder after it (best effort), so a power loss leaves
139/// the old file or the new, never an empty one.
140fn write_durable(path: &Path, bytes: &[u8]) -> io::Result<()> {
141    let part = write_part(path, bytes, true)?;
142    fs::rename(&part, path)?;
143    if let Some(parent) = path.parent() {
144        let _ = File::open(parent).and_then(|folder| folder.sync_all());
145    }
146    Ok(())
147}
148
149/// Writes `bytes` to the part file beside `path`, whose folder it makes,
150/// synced with `sync`, and returns the part file's path.
151fn write_part(path: &Path, bytes: &[u8], sync: bool) -> io::Result<PathBuf> {
152    if let Some(parent) = path.parent() {
153        fs::create_dir_all(parent)?;
154    }
155    let mut part = path.as_os_str().to_owned();
156    part.push(PART);
157    let part = PathBuf::from(part);
158    let mut file = File::create(&part)?;
159    file.write_all(bytes)?;
160    if sync {
161        file.sync_all()?;
162    }
163    Ok(part)
164}
165
166impl Files for DirFiles {
167    fn write_file(&self, name: &str, bytes: &[u8]) -> io::Result<()> {
168        write_atomic(&self.file(name)?, bytes)
169    }
170
171    fn append_file(&self, name: &str, bytes: &[u8]) -> io::Result<()> {
172        let path = self.file(name)?;
173        if let Some(parent) = path.parent() {
174            fs::create_dir_all(parent)?;
175        }
176        let mut file = OpenOptions::new().create(true).append(true).open(path)?;
177        file.write_all(bytes)
178    }
179
180    fn read_file(&self, name: &str) -> io::Result<Vec<u8>> {
181        fs::read(self.file(name)?)
182    }
183
184    fn remove_file(&self, name: &str) -> io::Result<()> {
185        absent_is_fine(fs::remove_file(self.file(name)?))
186    }
187
188    fn files(&self) -> io::Result<Vec<String>> {
189        let mut files = Vec::new();
190        let mut dirs = vec![(self.0.clone(), String::new())];
191        while let Some((dir, prefix)) = dirs.pop() {
192            for name in names(&dir)? {
193                if name.ends_with(PART) {
194                    continue;
195                }
196                let path = dir.join(&name);
197                let name = format!("{prefix}{name}");
198                if path.is_dir() {
199                    dirs.push((path, format!("{name}/")));
200                } else {
201                    files.push(name);
202                }
203            }
204        }
205        Ok(files)
206    }
207}
208
209/// Refuses with [`LogWriteError::Changed`] unless the file at `path` is
210/// `len` bytes long, 0 when there is none.
211fn check_len(path: &Path, len: u64) -> Result<(), LogWriteError> {
212    let held = match fs::metadata(path) {
213        Err(failure) if failure.kind() == io::ErrorKind::NotFound => 0,
214        metadata => metadata?.len(),
215    };
216    if held != len {
217        return Err(LogWriteError::Changed);
218    }
219    Ok(())
220}
221
222impl Files for DirStore {
223    fn write_file(&self, name: &str, bytes: &[u8]) -> io::Result<()> {
224        let _lock = self.lock()?;
225        self.files.write_file(name, bytes)
226    }
227
228    fn append_file(&self, name: &str, bytes: &[u8]) -> io::Result<()> {
229        let _lock = self.lock()?;
230        self.files.append_file(name, bytes)
231    }
232
233    fn read_file(&self, name: &str) -> io::Result<Vec<u8>> {
234        self.files.read_file(name)
235    }
236
237    fn remove_file(&self, name: &str) -> io::Result<()> {
238        let _lock = self.lock()?;
239        self.files.remove_file(name)
240    }
241
242    fn files(&self) -> io::Result<Vec<String>> {
243        self.files.files()
244    }
245}
246
247impl Store for DirStore {
248    fn logs(&self) -> io::Result<Vec<String>> {
249        let names = names(&self.dir.join(LOGS))?.into_iter();
250        let logs = names.filter_map(|name| Some(name.strip_suffix(LOG)?.to_owned()));
251        Ok(logs.filter(|name| !name.is_empty()).collect())
252    }
253
254    fn read_log(&self, name: &str) -> io::Result<Vec<u8>> {
255        fs::read(self.log(name)?)
256    }
257
258    fn append_log(&self, name: &str, len: u64, bytes: &[u8]) -> Result<(), LogWriteError> {
259        let path = self.log(name)?;
260        let _lock = self.lock()?;
261        check_len(&path, len)?;
262        if let Some(parent) = path.parent() {
263            fs::create_dir_all(parent)?;
264        }
265        let mut log = OpenOptions::new().create(true).append(true).open(path)?;
266        Ok(log.write_all(bytes)?)
267    }
268
269    /// The new log is written beside, synced and renamed over the old one:
270    /// the one synced write, as it is rare and replaces history.
271    fn replace_log(&self, name: &str, len: u64, bytes: &[u8]) -> Result<(), LogWriteError> {
272        let path = self.log(name)?;
273        let _lock = self.lock()?;
274        check_len(&path, len)?;
275        Ok(write_durable(&path, bytes)?)
276    }
277
278    /// Each folder is renamed to a trash name inside the directory, so it
279    /// is gone at once, then removed; a second call removes what a crash
280    /// left of the first. The lock file stays.
281    fn destroy(&self) -> io::Result<()> {
282        let _lock = self.lock()?;
283        for folder in [LOGS, FILES] {
284            let (path, trash) = (
285                self.dir.join(folder),
286                self.dir.join(format!("{folder}{TRASH}")),
287            );
288            if path.exists() {
289                absent_is_fine(fs::remove_dir_all(&trash))?;
290                fs::rename(&path, &trash)?;
291            }
292            absent_is_fine(fs::remove_dir_all(&trash))?;
293        }
294        Ok(())
295    }
296}
297
298#[cfg(test)]
299mod tests;