Skip to main content

ignition_core/deploy/apply/
journal.rs

1//! The on-box apply journal, format v1 (Phase 23, D-04, D-07, D-09).
2//!
3//! Written on the gateway host before the first file moves and removed when
4//! the run is finalized; a journal that is still there means an apply was
5//! interrupted. It is read by a later runner (`rollback --run`, recovery, the
6//! `deploy plan` refusal), so the layout is one-way and strict, in the
7//! manner of the declared-names ledger ([`crate::deploy::ledger::Ledger`]):
8//! at most [`JOURNAL_MAX_BYTES`] (8 MiB), strict UTF-8, no duplicate key,
9//! bounded depth, no unknown field, every field present.
10//!
11//! ```json
12//! {
13//!   "schema_version": 1,
14//!   "run_id": "<[A-Za-z0-9._-]{1,128}>",
15//!   "gateway": "<target gateway>",
16//!   "mode": "<target mode>",
17//!   "phase": "snapshotting | ready | writing | written | restoring",
18//!   "pre_fingerprint": "<64 lowercase hex>",
19//!   "touched": ["<collection path or ledger.json>", "..."]
20//! }
21//! ```
22//!
23//! `touched` is every path apply writes or deletes plus the ledger (key
24//! `ledger.json`), byte-sorted and unique; each path is an apply key
25//! (`apply_key_problem`: a mode-collection path, or a `.core/` path of a
26//! routed tag type, quick 261102-cef) or exactly `ledger.json`. A refusal is
27//! `deploy_run_invalid` naming the location and the problem, never a file
28//! value beyond a listed path, and never a fallback to a default (T-23-11).
29
30use std::collections::BTreeMap;
31
32use serde::de::DeserializeOwned;
33use serde::{Deserialize, Serialize};
34
35use super::fingerprint::{LEDGER_KEY, fingerprint};
36use super::run_store::RunId;
37use crate::deploy::layout::RelPath;
38use crate::deploy::ledger::LedgerRoot;
39use crate::deploy::node::{self, Node};
40use crate::deploy::target::NodeTarget;
41use crate::deploy::transport::{TreeReader, apply_key_problem};
42use crate::error::CoreError;
43
44pub use crate::deploy::ledger::journal_location;
45
46/// File name of the journal.
47pub const JOURNAL_FILE: &str = "journal.json";
48
49/// The largest journal accepted (8 MiB), checked before any parse.
50pub const JOURNAL_MAX_BYTES: usize = 8_388_608;
51
52/// How far an interrupted apply got.
53#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
54#[serde(rename_all = "snake_case")]
55pub enum JournalPhase {
56    /// Pre-images are being copied; no live file has moved.
57    Snapshotting,
58    /// Every pre-image is copied and the staging is complete; no live file
59    /// has moved.
60    Ready,
61    /// Live files are being moved.
62    Writing,
63    /// Every move is done; the run is not finalized.
64    Written,
65    /// A restore is under way.
66    Restoring,
67}
68
69impl JournalPhase {
70    /// Whether no live file can have moved yet (D-10): `snapshotting` and
71    /// `ready`.
72    pub fn before_first_move(self) -> bool {
73        matches!(self, JournalPhase::Snapshotting | JournalPhase::Ready)
74    }
75
76    /// The phase's name as the journal file spells it.
77    pub fn as_str(self) -> &'static str {
78        match self {
79            JournalPhase::Snapshotting => "snapshotting",
80            JournalPhase::Ready => "ready",
81            JournalPhase::Writing => "writing",
82            JournalPhase::Written => "written",
83            JournalPhase::Restoring => "restoring",
84        }
85    }
86}
87
88/// The only schema version this build reads and writes.
89const SCHEMA_VERSION: u32 = 1;
90
91/// The journal's top-level keys, in write order.
92const FIELDS: [&str; 7] = [
93    "schema_version",
94    "run_id",
95    "gateway",
96    "mode",
97    "phase",
98    "pre_fingerprint",
99    "touched",
100];
101
102/// The journal.
103#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
104#[serde(deny_unknown_fields)]
105pub struct Journal {
106    /// Always 1.
107    pub schema_version: u32,
108    /// The run this journal belongs to.
109    pub run_id: String,
110    /// The gateway directory name (the target's `gateway`).
111    pub gateway: String,
112    /// The deployment mode (the target's `mode`).
113    pub mode: String,
114    /// How far the run got.
115    pub phase: JournalPhase,
116    /// The touched-set fingerprint before the run, 64 lowercase hex digits.
117    pub pre_fingerprint: String,
118    /// The touched paths, byte-sorted and unique.
119    pub touched: Vec<String>,
120}
121
122impl Journal {
123    /// Parse and validate `bytes` read from `location`, for the target
124    /// `gateway` and `mode`.
125    pub fn parse(
126        location: &str,
127        bytes: &[u8],
128        gateway: &str,
129        mode: &str,
130    ) -> Result<Journal, CoreError> {
131        let bad = |problem: String| refuse(location, bytes, problem);
132        let journal: Journal = parse_strict(
133            location,
134            bytes,
135            JOURNAL_MAX_BYTES,
136            &FIELDS,
137            &[(
138                "phase",
139                &["snapshotting", "ready", "writing", "written", "restoring"],
140            )],
141        )?
142        .0;
143        check_header(
144            journal.schema_version,
145            &journal.run_id,
146            (&journal.gateway, gateway),
147            (&journal.mode, mode),
148        )
149        .map_err(bad)?;
150        if !is_hex64(&journal.pre_fingerprint) {
151            return Err(bad(
152                "pre_fingerprint is not 64 lowercase hex digits".to_string()
153            ));
154        }
155        check_touched(journal.touched.iter().map(String::as_str)).map_err(bad)?;
156        Ok(journal)
157    }
158
159    /// The canonical bytes: pretty JSON in field order plus one LF.
160    pub fn to_bytes(&self) -> Vec<u8> {
161        canonical_bytes(self)
162    }
163}
164
165/// What recovering an interrupted run does (D-10).
166#[derive(Debug, Clone, Copy, PartialEq, Eq)]
167pub enum RecoveryAction {
168    /// No move began and the touched set is what it was before the run:
169    /// remove the journal, the partial snapshot and the staging, and restore
170    /// nothing.
171    ClearWithoutRestore,
172    /// Put the snapshot back, scan, then remove the journal.
173    Restore,
174}
175
176impl RecoveryAction {
177    /// The action's name in the recovery run's events.
178    pub fn as_str(self) -> &'static str {
179        match self {
180            RecoveryAction::ClearWithoutRestore => "clear",
181            RecoveryAction::Restore => "restore",
182        }
183    }
184}
185
186/// Decide the recovery of `journal` from `live`, the live hash of each path of
187/// its touched list (`None` when absent). A journal whose phase is before the
188/// first move and whose live fingerprint equals its pre-run fingerprint died
189/// before it changed anything; every other journal is restored (D-10). There
190/// is no third answer: nothing discards a journal that might sit over a
191/// partial tree (D-12).
192pub fn recovery_action(
193    journal: &Journal,
194    live: &BTreeMap<String, Option<String>>,
195) -> RecoveryAction {
196    if journal.phase.before_first_move() && fingerprint(live) == journal.pre_fingerprint {
197        RecoveryAction::ClearWithoutRestore
198    } else {
199        RecoveryAction::Restore
200    }
201}
202
203/// The journal on the box of `gateway` and `mode`, read through `reader` and
204/// parsed strictly (DAPL-09, D-09). `None` means no interrupted run. A journal
205/// that cannot be read or parsed is `deploy_run_invalid` naming the location:
206/// plan, apply and rollback never go on as if it were absent.
207pub async fn journal_on_box(
208    reader: &dyn TreeReader,
209    node: &NodeTarget,
210    gateway: &str,
211    mode: &str,
212) -> Result<Option<Journal>, CoreError> {
213    let Some(bytes) = reader.read_journal(gateway, mode).await? else {
214        return Ok(None);
215    };
216    let location = journal_location(LedgerRoot::for_node(node), gateway, mode);
217    Journal::parse(&location, &bytes, gateway, mode).map(Some)
218}
219
220/// The refusal for `journal`: `deploy_journal_present` naming its run, the
221/// location and (through its message) `ign deploy rollback --run <id> --env
222/// <env>`.
223pub fn journal_present(
224    journal: &Journal,
225    env: &str,
226    node: &NodeTarget,
227    gateway: &str,
228    mode: &str,
229) -> CoreError {
230    CoreError::DeployJournalPresent {
231        run_id: journal.run_id.clone(),
232        env: env.to_string(),
233        location: journal_location(LedgerRoot::for_node(node), gateway, mode),
234        recover_with: None,
235    }
236}
237
238/// The canonical bytes of `value`: pretty JSON plus one LF.
239pub(super) fn canonical_bytes(value: &impl Serialize) -> Vec<u8> {
240    let mut bytes = serde_json::to_vec_pretty(value).unwrap_or_default();
241    bytes.push(b'\n');
242    bytes
243}
244
245/// Whether `text` is 64 lowercase hex digits.
246pub(super) fn is_hex64(text: &str) -> bool {
247    text.len() == 64
248        && text
249            .bytes()
250            .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
251}
252
253/// At most 64 printable characters of `text`, for a problem message.
254pub(super) fn shown(text: &str) -> String {
255    text.chars().filter(|c| !c.is_control()).take(64).collect()
256}
257
258/// The refusal for a journal or record at `location`: `deploy_run_invalid`
259/// naming the location and the problem. The run id is the file's own when it
260/// is a valid id (the rule limits it to a safe charset), else `unknown`.
261pub(super) fn refuse(location: &str, bytes: &[u8], problem: String) -> CoreError {
262    let run_id = serde_json::from_slice::<serde_json::Value>(bytes)
263        .ok()
264        .and_then(|v| v.get("run_id")?.as_str().map(str::to_string))
265        .filter(|id| RunId::parse(id).is_ok())
266        .unwrap_or_else(|| "unknown".to_string());
267    CoreError::DeployRunInvalid {
268        run_id,
269        problem: format!("{location}: {problem}"),
270    }
271}
272
273/// The checks every format applies before it deserializes: not empty, within
274/// `cap`, strict UTF-8 without a repeated key or excess depth, a JSON object,
275/// no unknown key, every key of `fields` present, and each of `enums` one of
276/// its allowed strings. Then the typed read. Returns the value and the
277/// order-preserving document for per-element checks.
278pub(super) fn parse_strict<T: DeserializeOwned>(
279    location: &str,
280    bytes: &[u8],
281    cap: usize,
282    fields: &[&str],
283    enums: &[(&str, &[&str])],
284) -> Result<(T, Node), CoreError> {
285    let bad = |problem: String| refuse(location, bytes, problem);
286    if bytes.is_empty() {
287        return Err(bad("the file is empty".to_string()));
288    }
289    if bytes.len() > cap {
290        return Err(bad(format!(
291            "the file is {} bytes, over the {cap} byte cap",
292            bytes.len()
293        )));
294    }
295    let (_, root) = node::classify(bytes).map_err(bad)?;
296    let Node::Obj(members) = &root else {
297        return Err(bad("the top level is not a JSON object".to_string()));
298    };
299    for (key, _) in members {
300        if !fields.contains(&key.as_str()) {
301            return Err(bad(format!("unknown field `{}`", shown(key))));
302        }
303    }
304    for field in fields {
305        if root.get(field).is_none() {
306            return Err(bad(format!("missing field `{field}`")));
307        }
308    }
309    for (field, allowed) in enums {
310        let ok = root
311            .get(field)
312            .and_then(Node::as_str)
313            .is_some_and(|v| allowed.contains(&v));
314        if !ok {
315            return Err(bad(format!(
316                "`{field}` is not one of {}",
317                allowed.join(", ")
318            )));
319        }
320    }
321    let value: T = serde_json::from_slice(bytes).map_err(|e| {
322        bad(format!(
323            "a field has the wrong type (line {} column {})",
324            e.line(),
325            e.column()
326        ))
327    })?;
328    Ok((value, root))
329}
330
331/// The header checks both formats share: the schema version, the run id and
332/// that the file names this target's gateway and mode. The problem text never
333/// echoes the foreign names.
334pub(super) fn check_header(
335    schema_version: u32,
336    run_id: &str,
337    gateway: (&str, &str),
338    mode: (&str, &str),
339) -> Result<(), String> {
340    if schema_version != SCHEMA_VERSION {
341        return Err(format!(
342            "schema_version is not {SCHEMA_VERSION}, the only version this build reads"
343        ));
344    }
345    if RunId::parse(run_id).is_err() {
346        return Err(
347            "run_id is not 1 to 128 characters of [A-Za-z0-9._-] without a leading dot".to_string(),
348        );
349    }
350    if gateway.0 != gateway.1 {
351        return Err("gateway names a different gateway than the target".to_string());
352    }
353    if mode.0 != mode.1 {
354        return Err("mode names a different mode than the target".to_string());
355    }
356    Ok(())
357}
358
359/// The `touched` rules: each path is an apply key (a collection path, or a
360/// `.core/` path of a routed tag type) or exactly `ledger.json`, and the list
361/// is byte-sorted and unique.
362pub(super) fn check_touched<'a>(paths: impl Iterator<Item = &'a str>) -> Result<(), String> {
363    let mut previous: Option<&str> = None;
364    for (index, path) in paths.enumerate() {
365        let listed = shown(path);
366        if path != LEDGER_KEY {
367            let collection = RelPath::new(path)
368                .ok()
369                .is_some_and(|rel| apply_key_problem(&rel).is_none());
370            if !collection {
371                return Err(format!(
372                    "touched[{index}] `{listed}` is neither a collection path nor `{LEDGER_KEY}`"
373                ));
374            }
375        }
376        if let Some(prev) = previous {
377            if path == prev {
378                return Err(format!(
379                    "touched[{index}] `{listed}` is listed twice (duplicate)"
380                ));
381            }
382            if path.as_bytes() < prev.as_bytes() {
383                return Err(format!(
384                    "touched is not byte-sorted: `{listed}` follows `{}`",
385                    shown(prev)
386                ));
387            }
388        }
389        previous = Some(path);
390    }
391    Ok(())
392}
393
394#[cfg(test)]
395mod tests {
396    use super::*;
397    use serde_json::{Value, json};
398
399    const GW: &str = "tracer-gw";
400    const MODE: &str = "dev";
401    const LOC: &str = "~/ign-deploy/tracer-gw/dev/journal.json";
402    const HASH: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
403
404    fn good() -> Journal {
405        Journal {
406            schema_version: 1,
407            run_id: "20261015T142233Z-a1b2c3".to_string(),
408            gateway: GW.to_string(),
409            mode: MODE.to_string(),
410            phase: JournalPhase::Writing,
411            pre_fingerprint: HASH.to_string(),
412            touched: vec![
413                "ledger.json".to_string(),
414                "mod/type/a.json".to_string(),
415                "mod/type/b.json".to_string(),
416            ],
417        }
418    }
419
420    fn with(edit: impl FnOnce(&mut Value)) -> Vec<u8> {
421        let mut value: Value = serde_json::from_slice(&good().to_bytes()).expect("good is JSON");
422        edit(&mut value);
423        serde_json::to_vec(&value).expect("serialize")
424    }
425
426    fn refusal(bytes: &[u8]) -> String {
427        let err = Journal::parse(LOC, bytes, GW, MODE).expect_err("must be refused");
428        assert_eq!(err.code(), "deploy_run_invalid", "{err}");
429        let text = err.to_string();
430        assert!(text.contains(LOC), "the refusal names the location: {text}");
431        text
432    }
433
434    #[test]
435    fn journal_roundtrip() {
436        let journal = good();
437        let bytes = journal.to_bytes();
438        assert!(bytes.ends_with(b"}\n"));
439        let parsed = Journal::parse(LOC, &bytes, GW, MODE).expect("parses");
440        assert_eq!(parsed, journal);
441        assert_eq!(parsed.to_bytes(), bytes, "byte-identical");
442        for phase in [
443            JournalPhase::Snapshotting,
444            JournalPhase::Ready,
445            JournalPhase::Writing,
446            JournalPhase::Written,
447            JournalPhase::Restoring,
448        ] {
449            let j = Journal { phase, ..good() };
450            assert_eq!(
451                Journal::parse(LOC, &j.to_bytes(), GW, MODE).expect("parses"),
452                j
453            );
454        }
455        let empty = Journal {
456            touched: vec![],
457            ..good()
458        };
459        assert_eq!(
460            Journal::parse(LOC, &empty.to_bytes(), GW, MODE).expect("parses"),
461            empty
462        );
463    }
464
465    #[test]
466    fn the_layout_is_pinned() {
467        let text = String::from_utf8(good().to_bytes()).expect("utf-8");
468        let expected = format!(
469            "{{\n  \"schema_version\": 1,\n  \"run_id\": \"20261015T142233Z-a1b2c3\",\n  \"gateway\": \"tracer-gw\",\n  \"mode\": \"dev\",\n  \"phase\": \"writing\",\n  \"pre_fingerprint\": \"{HASH}\",\n  \"touched\": [\n    \"ledger.json\",\n    \"mod/type/a.json\",\n    \"mod/type/b.json\"\n  ]\n}}\n"
470        );
471        assert_eq!(text, expected);
472    }
473
474    #[test]
475    fn phases_before_the_first_move() {
476        assert!(JournalPhase::Snapshotting.before_first_move());
477        assert!(JournalPhase::Ready.before_first_move());
478        assert!(!JournalPhase::Writing.before_first_move());
479        assert!(!JournalPhase::Written.before_first_move());
480        assert!(!JournalPhase::Restoring.before_first_move());
481    }
482
483    fn live(pairs: &[(&str, Option<&str>)]) -> std::collections::BTreeMap<String, Option<String>> {
484        pairs
485            .iter()
486            .map(|(path, hash)| ((*path).to_string(), hash.map(str::to_string)))
487            .collect()
488    }
489
490    #[test]
491    fn recovery_action_clears_only_an_untouched_run() {
492        use super::super::fingerprint::fingerprint;
493        let pre = live(&[("mod/type/a.json", Some(HASH)), ("mod/type/b.json", None)]);
494        let journal = |phase| Journal {
495            phase,
496            pre_fingerprint: fingerprint(&pre),
497            touched: pre.keys().cloned().collect(),
498            ..good()
499        };
500        let other = live(&[
501            ("mod/type/a.json", Some(HASH)),
502            ("mod/type/b.json", Some(HASH)),
503        ]);
504        for (phase, cleared) in [
505            (JournalPhase::Snapshotting, true),
506            (JournalPhase::Ready, true),
507            (JournalPhase::Writing, false),
508            (JournalPhase::Written, false),
509            (JournalPhase::Restoring, false),
510        ] {
511            let j = journal(phase);
512            let same = recovery_action(&j, &pre);
513            let changed = recovery_action(&j, &other);
514            assert_eq!(
515                same,
516                if cleared {
517                    RecoveryAction::ClearWithoutRestore
518                } else {
519                    RecoveryAction::Restore
520                },
521                "{phase:?}, live equals the pre-run state"
522            );
523            assert_eq!(
524                changed,
525                RecoveryAction::Restore,
526                "{phase:?}, live differs from the pre-run state"
527            );
528        }
529    }
530
531    #[test]
532    fn journal_location_follows_the_ledger_root() {
533        use crate::deploy::ledger::LedgerRoot;
534        assert_eq!(
535            journal_location(LedgerRoot::AccountHome, GW, MODE),
536            "~/ign-deploy/tracer-gw/dev/journal.json"
537        );
538        assert_eq!(
539            journal_location(LedgerRoot::DataDir, GW, MODE),
540            "ign-deploy/tracer-gw/dev/journal.json"
541        );
542    }
543
544    #[test]
545    fn refuses_an_empty_file() {
546        assert!(refusal(b"").contains("empty"));
547    }
548
549    #[test]
550    fn refuses_a_file_over_the_cap() {
551        let big = vec![b' '; JOURNAL_MAX_BYTES + 1];
552        assert!(refusal(&big).contains("cap"));
553    }
554
555    #[test]
556    fn refuses_invalid_utf8() {
557        refusal(b"{\"schema_version\": 1, \"run_id\": \"\xff\"}");
558    }
559
560    #[test]
561    fn refuses_a_duplicate_key() {
562        let text = String::from_utf8(good().to_bytes()).expect("utf-8");
563        let dup = text.replacen(
564            "\"mode\": \"dev\"",
565            "\"mode\": \"dev\",\n  \"mode\": \"dev\"",
566            1,
567        );
568        refusal(dup.as_bytes());
569    }
570
571    #[test]
572    fn refuses_an_unknown_field() {
573        let text = refusal(&with(|v| {
574            v["extra"] = json!(1);
575        }));
576        assert!(text.contains("extra"), "{text}");
577    }
578
579    #[test]
580    fn refuses_a_missing_field() {
581        let text = refusal(&with(|v| {
582            v.as_object_mut().expect("object").remove("phase");
583        }));
584        assert!(text.contains("phase"), "{text}");
585    }
586
587    #[test]
588    fn refuses_another_schema_version() {
589        refusal(&with(|v| v["schema_version"] = json!(2)));
590    }
591
592    #[test]
593    fn refuses_another_gateway() {
594        refusal(&with(|v| v["gateway"] = json!("other-gw")));
595    }
596
597    #[test]
598    fn refuses_another_mode() {
599        refusal(&with(|v| v["mode"] = json!("prod")));
600    }
601
602    #[test]
603    fn refuses_a_bad_run_id() {
604        for bad in ["", "../x", ".hidden", "a b"] {
605            refusal(&with(|v| v["run_id"] = json!(bad)));
606        }
607    }
608
609    #[test]
610    fn refuses_an_unknown_phase() {
611        let text = refusal(&with(|v| v["phase"] = json!("halfway")));
612        assert!(text.contains("phase"), "{text}");
613    }
614
615    #[test]
616    fn refuses_unsorted_touched() {
617        let text = refusal(&with(|v| {
618            v["touched"] = json!(["mod/type/b.json", "mod/type/a.json"]);
619        }));
620        assert!(text.contains("sorted"), "{text}");
621    }
622
623    #[test]
624    fn refuses_duplicate_touched() {
625        let text = refusal(&with(|v| {
626            v["touched"] = json!(["mod/type/a.json", "mod/type/a.json"]);
627        }));
628        assert!(text.contains("duplicate"), "{text}");
629    }
630
631    #[test]
632    fn refuses_a_touched_path_that_is_not_a_collection_path() {
633        for bad in [
634            "a/b",
635            "mod/type/.hidden/x",
636            "mod/type/../x",
637            "/abs/x/y",
638            "mod/type/resource.json/../../z",
639            "mod/type/config-mode.json",
640            "ledger.json.bak",
641        ] {
642            refusal(&with(|v| v["touched"] = json!([bad])));
643        }
644    }
645
646    #[test]
647    fn refuses_a_pre_fingerprint_that_is_not_hex() {
648        for bad in [
649            "",
650            "abc",
651            &HASH.to_uppercase(),
652            &format!("{HASH}0"),
653            "g".repeat(64).as_str(),
654        ] {
655            refusal(&with(|v| v["pre_fingerprint"] = json!(bad)));
656        }
657    }
658
659    #[test]
660    fn refuses_a_wrong_typed_field() {
661        refusal(&with(|v| v["touched"] = json!("mod/type/a.json")));
662        refusal(&with(|v| v["schema_version"] = json!("1")));
663    }
664
665    #[test]
666    fn refuses_a_top_level_that_is_not_an_object() {
667        refusal(b"[]");
668    }
669
670    #[test]
671    fn the_refusal_never_echoes_foreign_values() {
672        let text = refusal(&with(|v| v["gateway"] = json!("secret-gateway-name")));
673        assert!(!text.contains("secret-gateway-name"), "{text}");
674    }
675
676    /// quick 261102-cef (P1): a `.core/` key of a routed tag type is a
677    /// touched path; a `.core/` key of any other type is not.
678    #[test]
679    fn core_key_journal_check_touched() {
680        assert_eq!(
681            check_touched(
682                [
683                    ".core/ignition/tag-definition/default/Deploy/tags.json",
684                    "ignition/tag-definition/default/Deploy/tags.json",
685                    LEDGER_KEY,
686                ]
687                .into_iter()
688            ),
689            Ok(())
690        );
691        for bad in [
692            ".core/ignition/tag-provider/default/config.json",
693            ".core/ignition/database-translator/X/config.json",
694            ".core/.core/ignition/tag-definition/a/b",
695        ] {
696            assert!(check_touched([bad].into_iter()).is_err(), "{bad}");
697        }
698    }
699}