shep_core/
overrides.rs

1//! `overrides.json`: what an operator has changed since a Flockfile was
2//! loaded.
3//!
4//! A Flockfile arrives from an app's own repository, so a merged pull
5//! request must not silently change a running flock's live config. This
6//! store holds the fields an operator set that the Flockfile does not
7//! declare. A load merges the two: declared keys win, then the override,
8//! then the built-in default.
9//!
10//! Same on-disk shape as [`crate::kv`]: a read-modify-rename under a
11//! [`crate::file_lock`] on a sibling `overrides.json.lock`.
12
13use core::fmt;
14use std::collections::{BTreeMap, BTreeSet};
15use std::path::Path;
16
17use serde::{Deserialize, Serialize};
18
19use crate::file_lock::FileLock;
20
21/// The on-disk format's version.
22///
23/// A store carrying a higher version is refused rather than read or
24/// replaced ([`OverridesError::FutureVersion`]): there is no undo for a
25/// downgrade that overwrites an operator's live edits.
26pub const OVERRIDES_VERSION: u32 = 1;
27
28/// One sheep's overrides: the fields an operator has set that its current
29/// Flockfile does not declare.
30///
31/// `fields` is a flat JSON object rather than a typed `AppConfig`, since a
32/// newer shep may accept fields this one does not know, and reading must
33/// not silently drop them. `declared` and `declared_env` are not overrides
34/// themselves: they are the Flockfile's declared keys, kept so a merge can
35/// tell a key the Flockfile dropped apart from one it never mentioned.
36#[derive(Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
37pub struct AppOverrides {
38    /// Operator-set field values, keyed by the same names `AppConfig`'s
39    /// fields use. May include an `env` object.
40    pub fields: serde_json::Map<String, serde_json::Value>,
41    /// Names of fields the current Flockfile declares.
42    pub declared: BTreeSet<String>,
43    /// Names of `env` keys the current Flockfile declares.
44    pub declared_env: BTreeSet<String>,
45}
46
47/// Redacted: `fields` can hold an `env` map, and this store is where an
48/// operator's secrets live.
49impl fmt::Debug for AppOverrides {
50    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
51        f.debug_struct("AppOverrides")
52            .field("fields", &format_args!("<{} fields>", self.fields.len()))
53            .field("declared", &self.declared)
54            .field("declared_env", &self.declared_env)
55            .finish()
56    }
57}
58
59/// The file's shape: a version and a flat map of sheep name to overrides.
60///
61/// `BTreeMap`, not `HashMap`, so the file writes in key order: two writes
62/// of the same content produce byte-identical files.
63#[derive(Debug, Default, Serialize, Deserialize)]
64struct OverridesFile {
65    version: u32,
66    apps: BTreeMap<String, AppOverrides>,
67}
68
69impl OverridesFile {
70    /// Whether `name` already holds `record`, `None` meaning absent, in a
71    /// current-version store
72    ///
73    /// A rewrite costs two `fsync`s, each a full drive-cache flush on macOS,
74    /// and every load of an unchanged Flockfile asks for one. A missing or
75    /// older store never holds anything: writing it creates or upgrades it.
76    fn holds(&self, name: &str, record: Option<&AppOverrides>) -> bool {
77        self.version == OVERRIDES_VERSION && self.apps.get(name) == record
78    }
79}
80
81/// Error type returned by this module.
82///
83/// `#[non_exhaustive]`: shep-core is published, so a new failure variant
84/// must not break an out-of-tree `match`.
85///
86/// Wraps `io::Error`/`serde_json::Error` directly rather than stringifying
87/// them, matching [`crate::kv::KvError`], so callers keep the underlying
88/// diagnostic through [`core::error::Error::source`]; this type does not
89/// derive `Clone`/`PartialEq`/`Eq` as a result.
90#[non_exhaustive]
91#[derive(Debug)]
92pub enum OverridesError {
93    /// The store could not be read, written, or replaced.
94    Io(std::io::Error),
95    /// The store's JSON could not be parsed.
96    ///
97    /// Refused rather than repaired: this file is an operator's live config
98    /// and a partial read of it would silently drop overrides that are still
99    /// on disk.
100    Decode(serde_json::Error),
101    /// The store on disk is a version this build does not understand; carries
102    /// that version. Nothing was written.
103    FutureVersion(u32),
104}
105
106impl fmt::Display for OverridesError {
107    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
108        match self {
109            Self::Io(err) => write!(f, "overrides store I/O failed: {err}"),
110            Self::Decode(err) => write!(f, "overrides store failed to parse: {err}"),
111            Self::FutureVersion(version) => {
112                write!(
113                    f,
114                    "overrides store is version {version}, newer than this build understands"
115                )
116            }
117        }
118    }
119}
120
121impl core::error::Error for OverridesError {
122    fn source(&self) -> Option<&(dyn core::error::Error + 'static)> {
123        match self {
124            Self::Io(err) => Some(err),
125            Self::Decode(err) => Some(err),
126            Self::FutureVersion(_) => None,
127        }
128    }
129}
130
131impl From<std::io::Error> for OverridesError {
132    fn from(source: std::io::Error) -> Self {
133        Self::Io(source)
134    }
135}
136
137impl From<serde_json::Error> for OverridesError {
138    fn from(source: serde_json::Error) -> Self {
139        Self::Decode(source)
140    }
141}
142
143/// Reads `path` under the lock the caller already holds.
144///
145/// A missing file reads as an empty, current-version store: a fresh
146/// `$SHEP_HOME` has no overrides, and that is the normal state, not a fault.
147/// Any other `io::Error` propagates.
148fn read_file(path: &Path) -> Result<OverridesFile, OverridesError> {
149    let raw = match std::fs::read_to_string(path) {
150        Ok(raw) => raw,
151        Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
152            return Ok(OverridesFile::default());
153        }
154        Err(err) => return Err(OverridesError::Io(err)),
155    };
156    let file: OverridesFile = serde_json::from_str(&raw)?;
157    if file.version > OVERRIDES_VERSION {
158        return Err(OverridesError::FutureVersion(file.version));
159    }
160    Ok(file)
161}
162
163/// Rewrites `path` to hold exactly `file`.
164fn write_file(path: &Path, file: &OverridesFile) -> Result<(), OverridesError> {
165    crate::atomic_file::write_json(path, "overrides", file).map_err(OverridesError::Io)
166}
167
168/// Every sheep's overrides, in name order.
169///
170/// # Errors
171///
172/// - [`OverridesError::Io`]: the store could not be opened or read. A store
173///   that is simply absent is not an error: it reads as empty.
174/// - [`OverridesError::Decode`]: the file is not the JSON this module
175///   writes.
176/// - [`OverridesError::FutureVersion`]: the file's `version` is newer than
177///   [`OVERRIDES_VERSION`]. Nothing is read and nothing is written.
178pub fn all(path: &Path) -> Result<BTreeMap<String, AppOverrides>, OverridesError> {
179    // Taking the lock here too costs one extra `open`, but it orders this
180    // read against a writer's read-modify-rename instead of racing it.
181    let _lock = FileLock::acquire(path)?;
182    Ok(read_file(path)?.apps)
183}
184
185/// One sheep's overrides, or `None` if it has none.
186///
187/// # Errors
188///
189/// [`OverridesError::Io`], [`OverridesError::Decode`] and
190/// [`OverridesError::FutureVersion`], exactly as [`all`] returns them.
191pub fn get(path: &Path, name: &str) -> Result<Option<AppOverrides>, OverridesError> {
192    Ok(all(path)?.remove(name))
193}
194
195/// Stores `value` under `name`, replacing any previous overrides.
196///
197/// Writes nothing when the store already holds exactly `value` there.
198///
199/// # Errors
200///
201/// - [`OverridesError::FutureVersion`]: the store on disk is newer than
202///   this build understands. Nothing is written.
203/// - [`OverridesError::Decode`]: the existing file could not be parsed.
204/// - [`OverridesError::Io`]: the lock, the temp file, the `fsync` or the
205///   `rename` failed.
206pub fn put(path: &Path, name: &str, value: &AppOverrides) -> Result<(), OverridesError> {
207    let _lock = FileLock::acquire(path)?;
208    let mut file = read_file(path)?;
209    if file.holds(name, Some(value)) {
210        return Ok(());
211    }
212    file.version = OVERRIDES_VERSION;
213    file.apps.insert(name.to_string(), value.clone());
214    write_file(path, &file)
215}
216
217/// Removes `name`'s overrides, returning whether it was there.
218///
219/// # Errors
220///
221/// The same set [`put`] returns: `FutureVersion`, `Decode`, `Io`.
222pub fn remove(path: &Path, name: &str) -> Result<bool, OverridesError> {
223    let _lock = FileLock::acquire(path)?;
224    let mut file = read_file(path)?;
225    let was_present = file.apps.remove(name).is_some();
226    if was_present {
227        file.version = OVERRIDES_VERSION;
228        write_file(path, &file)?;
229    }
230    Ok(was_present)
231}
232
233/// Applies several changes at once: `Some` stores, `None` removes.
234///
235/// One lock and one rewrite for the whole batch, atomic: either every
236/// change lands or none does. Names the batch does not mention are left
237/// untouched, and the read and write happen under the same lock, so this
238/// is safe against a concurrent writer touching a different app. An empty
239/// batch takes no lock and writes nothing, and a batch the store already
240/// holds takes the lock and writes nothing.
241///
242/// # Errors
243///
244/// The same set [`put`] returns: `FutureVersion`, `Decode`, `Io`. Nothing
245/// is written on any of them.
246pub fn update(
247    path: &Path,
248    changes: &BTreeMap<String, Option<AppOverrides>>,
249) -> Result<(), OverridesError> {
250    if changes.is_empty() {
251        return Ok(());
252    }
253    let _lock = FileLock::acquire(path)?;
254    let mut file = read_file(path)?;
255    if changes
256        .iter()
257        .all(|(name, change)| file.holds(name, change.as_ref()))
258    {
259        return Ok(());
260    }
261    for (name, change) in changes {
262        match change {
263            Some(value) => {
264                file.apps.insert(name.clone(), value.clone());
265            }
266            None => {
267                file.apps.remove(name);
268            }
269        }
270    }
271    file.version = OVERRIDES_VERSION;
272    write_file(path, &file)
273}
274
275#[cfg(test)]
276mod tests {
277    use super::*;
278
279    #[test]
280    fn update_stores_removes_and_leaves_the_rest_alone() {
281        let dir = tempfile::TempDir::new().unwrap();
282        let path = dir.path().join("overrides.json");
283        let record = |value: u64| AppOverrides {
284            fields: [("max_restarts".to_string(), serde_json::json!(value))]
285                .into_iter()
286                .collect(),
287            ..AppOverrides::default()
288        };
289        put(&path, "web", &record(1)).unwrap();
290        put(&path, "worker", &record(2)).unwrap();
291        put(&path, "bystander", &record(3)).unwrap();
292
293        let changes = BTreeMap::from([
294            ("web".to_string(), Some(record(9))),
295            ("worker".to_string(), None),
296        ]);
297        update(&path, &changes).unwrap();
298
299        let all = all(&path).unwrap();
300        assert_eq!(all.get("web"), Some(&record(9)));
301        assert_eq!(all.get("worker"), None);
302        assert_eq!(all.get("bystander"), Some(&record(3)));
303    }
304
305    #[test]
306    fn an_empty_update_writes_nothing() {
307        let dir = tempfile::TempDir::new().unwrap();
308        let path = dir.path().join("overrides.json");
309        update(&path, &BTreeMap::new()).unwrap();
310        assert!(!path.exists(), "an empty batch created a store");
311    }
312
313    /// Compact on purpose: any rewrite pretty-prints, so surviving bytes
314    /// mean nothing was written.
315    #[test]
316    fn a_batch_the_store_already_holds_leaves_the_file_alone() {
317        let dir = tempfile::TempDir::new().unwrap();
318        let path = dir.path().join("overrides.json");
319        let compact = r#"{"version":1,"apps":{"web":{"fields":{},"declared":["name","script"],"declared_env":[]}}}"#;
320        std::fs::write(&path, compact).unwrap();
321        let web = AppOverrides {
322            declared: ["name", "script"].iter().map(|s| s.to_string()).collect(),
323            ..AppOverrides::default()
324        };
325
326        let unchanged = BTreeMap::from([
327            ("web".to_string(), Some(web.clone())),
328            ("never-stored".to_string(), None),
329        ]);
330        update(&path, &unchanged).unwrap();
331        put(&path, "web", &web).unwrap();
332
333        assert_eq!(std::fs::read_to_string(&path).unwrap(), compact);
334    }
335
336    #[test]
337    fn one_changed_record_rewrites_the_whole_batch() {
338        let dir = tempfile::TempDir::new().unwrap();
339        let path = dir.path().join("overrides.json");
340        let compact = r#"{"version":1,"apps":{"api":{"fields":{},"declared":["script"],"declared_env":[]},"db":{"fields":{},"declared":["script"],"declared_env":[]}}}"#;
341        std::fs::write(&path, compact).unwrap();
342        let declaring = |keys: &[&str]| AppOverrides {
343            declared: keys.iter().map(|s| s.to_string()).collect(),
344            ..AppOverrides::default()
345        };
346
347        let changes = BTreeMap::from([
348            ("api".to_string(), Some(declaring(&["script"]))),
349            ("db".to_string(), Some(declaring(&["cwd", "script"]))),
350        ]);
351        update(&path, &changes).unwrap();
352
353        assert_ne!(std::fs::read_to_string(&path).unwrap(), compact);
354        let stored = all(&path).unwrap();
355        assert_eq!(stored.get("api"), Some(&declaring(&["script"])));
356        assert_eq!(stored.get("db"), Some(&declaring(&["cwd", "script"])));
357    }
358
359    /// The version is part of what a store holds: a record equal to the
360    /// batch's still gets its file upgraded.
361    #[test]
362    fn an_older_store_is_rewritten_even_when_its_records_match() {
363        let dir = tempfile::TempDir::new().unwrap();
364        let path = dir.path().join("overrides.json");
365        std::fs::write(
366            &path,
367            r#"{"version":0,"apps":{"worker":{"fields":{},"declared":[],"declared_env":[]}}}"#,
368        )
369        .unwrap();
370
371        put(&path, "worker", &AppOverrides::default()).unwrap();
372
373        let raw: serde_json::Value =
374            serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap();
375        assert_eq!(raw["version"], serde_json::json!(OVERRIDES_VERSION));
376    }
377
378    #[test]
379    fn put_then_get_round_trips() {
380        let dir = tempfile::TempDir::new().unwrap();
381        let path = dir.path().join("overrides.json");
382        let mut fields = serde_json::Map::new();
383        fields.insert("max_memory".to_string(), serde_json::json!("512M"));
384        let value = AppOverrides {
385            fields,
386            declared: ["name", "script"].iter().map(|s| s.to_string()).collect(),
387            declared_env: BTreeSet::new(),
388        };
389        put(&path, "web", &value).unwrap();
390        assert_eq!(get(&path, "web").unwrap().as_ref(), Some(&value));
391    }
392
393    #[test]
394    fn a_missing_store_reads_as_empty() {
395        let dir = tempfile::TempDir::new().unwrap();
396        assert!(all(&dir.path().join("overrides.json")).unwrap().is_empty());
397    }
398
399    /// Holds env values, same reason `flock.json` has its own owner-only test.
400    #[cfg(unix)]
401    #[test]
402    fn the_store_is_owner_only() {
403        use std::os::unix::fs::PermissionsExt as _;
404        let dir = tempfile::TempDir::new().unwrap();
405        let path = dir.path().join("overrides.json");
406        put(&path, "web", &AppOverrides::default()).unwrap();
407        let mode = std::fs::metadata(&path).unwrap().permissions().mode();
408        assert_eq!(mode & 0o777, 0o600, "mode was {:o}", mode & 0o777);
409    }
410
411    #[test]
412    fn debug_redacts_override_values() {
413        let mut fields = serde_json::Map::new();
414        fields.insert(
415            "env".to_string(),
416            serde_json::json!({"DATABASE_URL": "postgres://hunter2"}),
417        );
418        let value = AppOverrides {
419            fields,
420            ..AppOverrides::default()
421        };
422        let rendered = format!("{value:?}");
423        assert!(!rendered.contains("hunter2"), "leaked: {rendered}");
424        // Exact string pinned so a lazy derive(Debug) refactor fails here,
425        // matching `config::app`'s own `debug_redacts_env_values`.
426        assert_eq!(
427            rendered,
428            "AppOverrides { fields: <1 fields>, declared: {}, declared_env: {} }"
429        );
430    }
431
432    #[test]
433    fn a_future_version_refuses_without_clobbering() {
434        let dir = tempfile::TempDir::new().unwrap();
435        let path = dir.path().join("overrides.json");
436        std::fs::write(&path, r#"{"version":99,"apps":{}}"#).unwrap();
437        assert!(matches!(
438            get(&path, "web"),
439            Err(OverridesError::FutureVersion(99))
440        ));
441        assert_eq!(
442            std::fs::read_to_string(&path).unwrap(),
443            r#"{"version":99,"apps":{}}"#
444        );
445    }
446
447    /// Bounded: each join is under a timeout, so a lock that deadlocks fails
448    /// this test instead of hanging the suite.
449    #[test]
450    fn two_concurrent_writers_lose_nothing() {
451        let dir = tempfile::TempDir::new().unwrap();
452        let path = dir.path().join("overrides.json");
453        const PER_WRITER: usize = 50;
454
455        let (done_tx, done_rx) = std::sync::mpsc::channel();
456        for writer in 0..2 {
457            let path = path.clone();
458            let done_tx = done_tx.clone();
459            std::thread::spawn(move || {
460                for n in 0..PER_WRITER {
461                    put(&path, &format!("w{writer}-{n}"), &AppOverrides::default()).unwrap();
462                }
463                done_tx.send(()).unwrap();
464            });
465        }
466        drop(done_tx);
467        for _ in 0..2 {
468            done_rx
469                .recv_timeout(std::time::Duration::from_secs(60))
470                .expect("a writer did not finish within 60s");
471        }
472
473        assert_eq!(all(&path).unwrap().len(), PER_WRITER * 2);
474    }
475}
shep_core/overrides.rs

shep_core/
overrides.rs
