byob_plugins/
materialize.rs

1//! Bundled plugins on disk: a bundled plugin's files live in the binary, but a backend or a
2//! script tool needs them as files a process can run. [`materialize`] writes them once to a
3//! versioned cache folder, `<root>/<id>-<byob version>-<content hash>/`, and returns it.
4//!
5//! - The folder name changes whenever the files or byob's version do, so a folder that
6//!   exists is complete and current: it is written under a temporary name and renamed into
7//!   place (two byob processes racing both end up with the same folder).
8//! - Its files are made read-only: edits belong in a fork (`keep_plugin`), not here. Python
9//!   may still add `__pycache__`.
10//! - Older folders of the same plugin are removed when a new one is made, once nothing has
11//!   used them for [`PRUNE_AFTER`] (another byob, of another version, may be running from
12//!   one): every use stamps the folder ([`USED`]).
13
14use std::collections::BTreeMap;
15use std::hash::Hasher as _;
16use std::io::ErrorKind;
17use std::path::{Path, PathBuf};
18use std::time::{Duration, SystemTime};
19
20use crate::manifest::{path_inside, plugin_id_ok};
21
22/// What marks a materialized folder (and holds its hash).
23pub const MARKER: &str = ".byob-bundled";
24/// Touched whenever a byob uses the folder.
25pub const USED: &str = ".byob-used";
26/// An older folder unused this long is removed.
27pub const PRUNE_AFTER: Duration = Duration::from_secs(24 * 60 * 60);
28
29/// Most bytes one materialized plugin may hold.
30const MAX_BYTES: usize = 64 * 1024 * 1024;
31
32/// FNV-1a over the files (path, NUL, text, NUL), stable across runs.
33pub fn content_hash(files: &BTreeMap<String, String>) -> u64 {
34    struct Fnv(u64);
35    impl std::hash::Hasher for Fnv {
36        fn finish(&self) -> u64 {
37            self.0
38        }
39        fn write(&mut self, bytes: &[u8]) {
40            for b in bytes {
41                self.0 ^= u64::from(*b);
42                self.0 = self.0.wrapping_mul(0x0100_0000_01b3);
43            }
44        }
45    }
46    let mut h = Fnv(0xcbf2_9ce4_8422_2325);
47    for (path, text) in files {
48        h.write(path.as_bytes());
49        h.write_u8(0);
50        h.write(text.as_bytes());
51        h.write_u8(0);
52    }
53    h.finish()
54}
55
56/// The folder `id`'s files go to under `root` for this byob `version`.
57pub fn folder(root: &Path, id: &str, version: &str, files: &BTreeMap<String, String>) -> PathBuf {
58    let version: String = version.chars().map(|c| if c.is_ascii_alphanumeric() || c == '.' { c } else { '_' }).take(32).collect();
59    root.join(format!("{id}-{version}-{:016x}", content_hash(files)))
60}
61
62/// Write plugin `id`'s `files` under `root` (if not there yet) and return the folder.
63pub fn materialize(root: &Path, id: &str, version: &str, files: &BTreeMap<String, String>) -> Result<PathBuf, String> {
64    if !plugin_id_ok(id) {
65        return Err(format!("{id:?} isn't a plugin id"));
66    }
67    let dir = folder(root, id, version, files);
68    if dir.join(MARKER).is_file() {
69        stamp(&dir);
70        return Ok(dir);
71    }
72    let total: usize = files.values().map(String::len).sum();
73    if total > MAX_BYTES {
74        return Err(format!("plugin {id} is too big to write out ({total} bytes)"));
75    }
76    std::fs::create_dir_all(root).map_err(|e| format!("can't create {}: {e}", root.display()))?;
77    let nanos = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.subsec_nanos());
78    let tmp = root.join(format!(".tmp-{id}-{}-{nanos:x}", std::process::id()));
79    let written = write_all(&tmp, files);
80    if let Err(e) = written {
81        remove_tree(&tmp);
82        return Err(e);
83    }
84    match std::fs::rename(&tmp, &dir) {
85        Ok(()) => {}
86        // Someone else made it first (a rename onto a non-empty folder fails).
87        Err(_) if dir.join(MARKER).is_file() => remove_tree(&tmp),
88        Err(e) => {
89            remove_tree(&tmp);
90            return Err(format!("can't write {}: {e}", dir.display()));
91        }
92    }
93    stamp(&dir);
94    prune(root, id, &dir, PRUNE_AFTER);
95    Ok(dir)
96}
97
98/// Say `dir` is in use (best effort).
99fn stamp(dir: &Path) {
100    if let Err(e) = std::fs::write(dir.join(USED), b"") {
101        log::debug!("couldn't stamp {}: {e}", dir.display());
102    }
103}
104
105/// How long ago a materialized folder was last used.
106fn unused_for(dir: &Path) -> Option<Duration> {
107    let when = std::fs::metadata(dir.join(USED)).or_else(|_| std::fs::metadata(dir.join(MARKER))).and_then(|m| m.modified()).ok()?;
108    Some(SystemTime::now().duration_since(when).unwrap_or_default())
109}
110
111fn write_all(tmp: &Path, files: &BTreeMap<String, String>) -> Result<(), String> {
112    std::fs::create_dir(tmp).map_err(|e| format!("can't create {}: {e}", tmp.display()))?;
113    for (rel, text) in files {
114        if !path_inside(rel) {
115            return Err(format!("{rel:?} isn't a path inside the plugin"));
116        }
117        let path = tmp.join(rel);
118        if let Some(parent) = path.parent() {
119            std::fs::create_dir_all(parent).map_err(|e| format!("can't create {}: {e}", parent.display()))?;
120        }
121        std::fs::write(&path, text).map_err(|e| format!("can't write {}: {e}", path.display()))?;
122        set_readonly(&path, true);
123    }
124    let hash = format!("{:016x}\n", content_hash(files));
125    std::fs::write(tmp.join(MARKER), hash).map_err(|e| format!("can't write {}: {e}", tmp.display()))
126}
127
128fn set_readonly(path: &Path, on: bool) {
129    if let Ok(m) = std::fs::metadata(path) {
130        let mut p = m.permissions();
131        #[allow(clippy::permissions_set_readonly_false)]
132        p.set_readonly(on);
133        let _ = std::fs::set_permissions(path, p);
134    }
135}
136
137/// Remove a folder of ours, files made writable first (Windows won't delete read-only files).
138fn remove_tree(dir: &Path) {
139    fn writable(dir: &Path, depth: usize) {
140        let Ok(rd) = std::fs::read_dir(dir) else { return };
141        for e in rd.flatten() {
142            let p = e.path();
143            match e.file_type() {
144                Ok(t) if t.is_dir() && depth < 16 => writable(&p, depth + 1),
145                Ok(t) if t.is_file() => set_readonly(&p, false),
146                _ => {}
147            }
148        }
149    }
150    writable(dir, 0);
151    match std::fs::remove_dir_all(dir) {
152        Ok(()) => {}
153        Err(e) if e.kind() == ErrorKind::NotFound => {}
154        Err(e) => log::debug!("couldn't remove {}: {e}", dir.display()),
155    }
156}
157
158/// Remove `id`'s other materialized folders (older versions or contents) unused for `after`.
159fn prune(root: &Path, id: &str, keep: &Path, after: Duration) {
160    let Ok(rd) = std::fs::read_dir(root) else { return };
161    let prefix = format!("{id}-");
162    for e in rd.flatten() {
163        let p = e.path();
164        let name = e.file_name().to_string_lossy().into_owned();
165        // `<id>-<version>-<16 hex>`; another plugin's id may start with ours plus a dash, so
166        // check the shape after the prefix (a version, a dash, the hash).
167        let Some(rest) = name.strip_prefix(&prefix) else { continue };
168        let ours = rest.rsplit_once('-').is_some_and(|(v, h)| h.len() == 16 && h.bytes().all(|b| b.is_ascii_hexdigit()) && !v.contains('-'));
169        if ours && p != keep && p.join(MARKER).is_file() && unused_for(&p).is_some_and(|d| d >= after) {
170            remove_tree(&p);
171        }
172    }
173}
174
175#[cfg(test)]
176mod tests {
177    use super::*;
178
179    fn files(text: &str) -> BTreeMap<String, String> {
180        [("byob-plugin.json".to_owned(), "{}".to_owned()), ("lib/backend.py".to_owned(), text.to_owned())].into_iter().collect()
181    }
182
183    #[test]
184    fn writes_once_read_only_and_keyed_by_version_and_content() {
185        let root = tempfile::tempdir().unwrap();
186        let a = materialize(root.path(), "sheets", "0.2.1", &files("print(1)")).unwrap();
187        assert_eq!(std::fs::read_to_string(a.join("lib/backend.py")).unwrap(), "print(1)");
188        assert!(std::fs::metadata(a.join("lib/backend.py")).unwrap().permissions().readonly());
189        // Again: the same folder, untouched.
190        let mtime = std::fs::metadata(a.join(MARKER)).unwrap().modified().unwrap();
191        assert_eq!(materialize(root.path(), "sheets", "0.2.1", &files("print(1)")).unwrap(), a);
192        assert_eq!(std::fs::metadata(a.join(MARKER)).unwrap().modified().unwrap(), mtime);
193        // A new byob version or new contents: a new folder; the old one, unused for long, is pruned.
194        age(&a);
195        let b = materialize(root.path(), "sheets", "0.3.0", &files("print(1)")).unwrap();
196        assert_ne!(a, b);
197        assert!(!a.exists(), "old folder pruned");
198        let c = materialize(root.path(), "sheets", "0.3.0", &files("print(2)")).unwrap();
199        assert_ne!(b, c);
200        assert!(c.file_name().unwrap().to_string_lossy().starts_with("sheets-0.3.0-"));
201        // Another plugin whose id starts with ours keeps its folder.
202        let d = materialize(root.path(), "sheets-extra", "0.3.0", &files("x")).unwrap();
203        age(&d);
204        let _ = materialize(root.path(), "sheets", "0.3.0", &files("print(3)")).unwrap();
205        assert!(d.exists());
206        // No temporary folders left behind.
207        assert!(std::fs::read_dir(root.path()).unwrap().flatten().all(|e| !e.file_name().to_string_lossy().starts_with(".tmp")));
208    }
209
210    /// Make `dir` look unused for longer than [`PRUNE_AFTER`].
211    fn age(dir: &Path) {
212        let f = std::fs::File::options().write(true).open(dir.join(USED)).unwrap();
213        f.set_modified(SystemTime::now() - PRUNE_AFTER - Duration::from_secs(60)).unwrap();
214    }
215
216    #[test]
217    fn a_folder_in_recent_use_is_kept() {
218        let root = tempfile::tempdir().unwrap();
219        // another byob (the older version) runs from this one
220        let a = materialize(root.path(), "sheets", "0.2.1", &files("print(1)")).unwrap();
221        let b = materialize(root.path(), "sheets", "0.3.0", &files("print(1)")).unwrap();
222        assert!(a.join(MARKER).is_file() && b.join(MARKER).is_file());
223        // it uses it again later: still recent
224        assert_eq!(materialize(root.path(), "sheets", "0.2.1", &files("print(1)")).unwrap(), a);
225        let _ = materialize(root.path(), "sheets", "0.3.0", &files("print(2)")).unwrap();
226        assert!(a.join(MARKER).is_file());
227    }
228
229    #[test]
230    fn refuses_paths_outside_and_bad_ids() {
231        let root = tempfile::tempdir().unwrap();
232        let mut bad = files("x");
233        bad.insert("../evil.py".into(), "x".into());
234        assert!(materialize(root.path(), "p", "1", &bad).is_err());
235        assert!(!root.path().parent().unwrap().join("evil.py").exists());
236        assert!(materialize(root.path(), "../p", "1", &files("x")).is_err());
237        assert!(std::fs::read_dir(root.path()).unwrap().next().is_none(), "nothing left behind");
238    }
239}
byob_plugins/materialize.rs

byob_plugins/
materialize.rs
