Skip to main content

ignition_core/deploy/apply/
journal.rs

1//! The on-box apply journal, format v1 (Phase 23, D-04, D-07, D-09).
2//!
3//! Written on the gateway host before the first file moves and removed when
4//! the run is finalized; a journal that is still there means an apply was
5//! interrupted. It is read by a later runner (`rollback --run`, recovery, the
6//! `deploy plan` refusal), so the layout is one-way and strict, in the
7//! manner of the declared-names ledger ([`crate::deploy::ledger::Ledger`]):
8//! at most [`JOURNAL_MAX_BYTES`] (8 MiB), strict UTF-8, no duplicate key,
9//! bounded depth, no unknown field, every field present.
10//!
11//! ```json
12//! {
13//!   "schema_version": 1,
14//!   "run_id": "<[A-Za-z0-9._-]{1,128}>",
15//!   "gateway": "<target gateway>",
16//!   "mode": "<target mode>",
17//!   "phase": "snapshotting | ready | writing | written | restoring",
18//!   "pre_fingerprint": "<64 lowercase hex>",
19//!   "touched": ["<collection path or ledger.json>", "..."]
20//! }
21//! ```
22//!
23//! `touched` is every path apply writes or deletes plus the ledger (key
24//! `ledger.json`), byte-sorted and unique; each path is an apply key
25//! (`apply_key_problem`: a mode-collection path, or a `.core/` path of a
26//! routed tag type, quick 261102-cef) or exactly `ledger.json`. A refusal is
27//! `deploy_run_invalid` naming the location and the problem, never a file
28//! value beyond a listed path, and never a fallback to a default (T-23-11).
29
30use std::collections::BTreeMap;
31
32use serde::de::DeserializeOwned;
33use serde::{Deserialize, Serialize};
34
35use super::fingerprint::{LEDGER_KEY, fingerprint};
36use super::run_store::RunId;
37use crate::deploy::layout::RelPath;
38use crate::deploy::ledger::LedgerRoot;
39use crate::deploy::node::{self, Node};
40use crate::deploy::target::NodeTarget;
41use crate::deploy::transport::{TreeReader, apply_key_problem};
42use crate::error::CoreError;
43
44pub use crate::deploy::ledger::journal_location;
45
46/// File name of the journal.
47pub const JOURNAL_FILE: &str = "journal.json";
48
49/// The largest journal accepted (8 MiB), checked before any parse.
50pub const JOURNAL_MAX_BYTES: usize = 8_388_608;
51
52/// How far an interrupted apply got.
53#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
54#[serde(rename_all = "snake_case")]
55pub enum JournalPhase {
56    /// Pre-images are being copied; no live file has moved.
57    Snapshotting,
58    /// Every pre-image is copied and the staging is complete; no live file
59    /// has moved.
60    Ready,
61    /// Live files are being moved.
62    Writing,
63    /// Every move is done; the run is not finalized.
64    Written,
65    /// A restore is under way.
66    Restoring,
67}
68
69impl JournalPhase {
70    /// Whether no live file can have moved yet (D-10): `snapshotting` and
71    /// `ready`.
72    pub fn before_first_move(self) -> bool {
73        matches!(self, JournalPhase::Snapshotting | JournalPhase::Ready)
74    }
75
76    /// The phase's name as the journal file spells it.
77    pub fn as_str(self) -> &'static str {
78        match self {
79            JournalPhase::Snapshotting => "snapshotting",
80            JournalPhase::Ready => "ready",
81            JournalPhase::Writing => "writing",
82            JournalPhase::Written => "written",
83            JournalPhase::Restoring => "restoring",
84        }
85    }
86}
87
88/// The only schema version this build reads and writes.
89const SCHEMA_VERSION: u32 = 1;
90
91/// The journal's top-level keys, in write order.
92const FIELDS: [&str; 7] = [
93    "schema_version",
94    "run_id",
95    "gateway",
96    "mode",
97    "phase",
98    "pre_fingerprint",
99    "touched",
100];
101
102/// The journal.
103#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
104#[serde(deny_unknown_fields)]
105pub struct Journal {
106    /// Always 1.
107    pub schema_version: u32,
108    /// The run this journal belongs to.
109    pub run_id: String,
110    /// The gateway directory name (the target's `gateway`).
111    pub gateway: String,
112    /// The deployment mode (the target's `mode`).
113    pub mode: String,
114    /// How far the run got.
115    pub phase: JournalPhase,
116    /// The touched-set fingerprint before the run, 64 lowercase hex digits.
117    pub pre_fingerprint: String,
118    /// The touched paths, byte-sorted and unique.
119    pub touched: Vec<String>,
120}
121
122impl Journal {
123    /// Parse and validate `bytes` read from `location`, for the target
124    /// `gateway` and `mode`.
125    pub fn parse(
126        location: &str,
127        bytes: &[u8],
128        gateway: &str,
129        mode: &str,
130    ) -> Result<Journal, CoreError> {
131        let bad = |problem: String| refuse(location, bytes, problem);
132        let journal: Journal = parse_strict(
133            location,
134            bytes,
135            JOURNAL_MAX_BYTES,
136            &FIELDS,
137            &[(
138                "phase",
139                &["snapshotting", "ready", "writing", "written", "restoring"],
140            )],
141        )?
142        .0;
143        check_header(
144            journal.schema_version,
145            &journal.run_id,
146            (&journal.gateway, gateway),
147            (&journal.mode, mode),
148        )
149        .map_err(bad)?;
150        if !is_hex64(&journal.pre_fingerprint) {
151            return Err(bad(
152                "pre_fingerprint is not 64 lowercase hex digits".to_string()
153            ));
154        }
155        check_touched(journal.touched.iter().map(String::as_str)).map_err(bad)?;
156        Ok(journal)
157    }
158
159    /// The canonical bytes: pretty JSON in field order plus one LF.
160    pub fn to_bytes(&self) -> Vec<u8> {
161        canonical_bytes(self)
162    }
163}
164
165/// What recovering an interrupted run does (D-10).
166#[derive(Debug, Clone, Copy, PartialEq, Eq)]
167pub enum RecoveryAction {
168    /// No move began and the touched set is what it was before the run:
169    /// remove the journal, the partial snapshot and the staging, and restore
170    /// nothing.
171    ClearWithoutRestore,
172    /// Put the snapshot back, scan, then remove the journal.
173    Restore,
174}
175
176impl RecoveryAction {
177    /// The action's name in the recovery run's events.
178    pub fn as_str(self) -> &'static str {
179        match self {
180            RecoveryAction::ClearWithoutRestore => "clear",
181            RecoveryAction::Restore => "restore",
182        }
183    }
184}
185
186/// Decide the recovery of `journal` from `live`, the live hash of each path of
187/// its touched list (`None` when absent). A journal whose phase is before the
188/// first move and whose live fingerprint equals its pre-run fingerprint died
189/// before it changed anything; every other journal is restored (D-10). There
190/// is no third answer: nothing discards a journal that might sit over a
191/// partial tree (D-12).
192pub fn recovery_action(
193    journal: &Journal,
194    live: &BTreeMap<String, Option<String>>,
195) -> RecoveryAction {
196    if journal.phase.before_first_move() && fingerprint(live) == journal.pre_fingerprint {
197        RecoveryAction::ClearWithoutRestore
198    } else {
199        RecoveryAction::Restore
200    }
201}
202
203/// The journal on the box of `gateway` and `mode`, read through `reader` and
204/// parsed strictly (DAPL-09, D-09). `None` means no interrupted run. A journal
205/// that cannot be read or parsed is `deploy_run_invalid` naming the location:
206/// plan, apply and rollback never go on as if it were absent.
207pub async fn journal_on_box(
208    reader: &dyn TreeReader,
209    node: &NodeTarget,
210    gateway: &str,
211    mode: &str,
212) -> Result<Option<Journal>, CoreError> {
213    let Some(bytes) = reader.read_journal(gateway, mode).await? else {
214        return Ok(None);
215    };
216    let location = journal_location(LedgerRoot::for_node(node), gateway, mode);
217    Journal::parse(&location, &bytes, gateway, mode).map(Some)
218}
219
220/// The refusal for `journal`: `deploy_journal_present` naming its run, the
221/// location and (through its message) `ign deploy rollback --run <id> --env
222/// <env>`.
223pub fn journal_present(
224    journal: &Journal,
225    env: &str,
226    node: &NodeTarget,
227    gateway: &str,
228    mode: &str,
229) -> CoreError {
230    CoreError::DeployJournalPresent {
231        run_id: journal.run_id.clone(),
232        env: env.to_string(),
233        location: journal_location(LedgerRoot::for_node(node), gateway, mode),
234    }
235}
236
237/// The canonical bytes of `value`: pretty JSON plus one LF.
238pub(super) fn canonical_bytes(value: &impl Serialize) -> Vec<u8> {
239    let mut bytes = serde_json::to_vec_pretty(value).unwrap_or_default();
240    bytes.push(b'\n');
241    bytes
242}
243
244/// Whether `text` is 64 lowercase hex digits.
245pub(super) fn is_hex64(text: &str) -> bool {
246    text.len() == 64
247        && text
248            .bytes()
249            .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
250}
251
252/// At most 64 printable characters of `text`, for a problem message.
253pub(super) fn shown(text: &str) -> String {
254    text.chars().filter(|c| !c.is_control()).take(64).collect()
255}
256
257/// The refusal for a journal or record at `location`: `deploy_run_invalid`
258/// naming the location and the problem. The run id is the file's own when it
259/// is a valid id (the rule limits it to a safe charset), else `unknown`.
260pub(super) fn refuse(location: &str, bytes: &[u8], problem: String) -> CoreError {
261    let run_id = serde_json::from_slice::<serde_json::Value>(bytes)
262        .ok()
263        .and_then(|v| v.get("run_id")?.as_str().map(str::to_string))
264        .filter(|id| RunId::parse(id).is_ok())
265        .unwrap_or_else(|| "unknown".to_string());
266    CoreError::DeployRunInvalid {
267        run_id,
268        problem: format!("{location}: {problem}"),
269    }
270}
271
272/// The checks every format applies before it deserializes: not empty, within
273/// `cap`, strict UTF-8 without a repeated key or excess depth, a JSON object,
274/// no unknown key, every key of `fields` present, and each of `enums` one of
275/// its allowed strings. Then the typed read. Returns the value and the
276/// order-preserving document for per-element checks.
277pub(super) fn parse_strict<T: DeserializeOwned>(
278    location: &str,
279    bytes: &[u8],
280    cap: usize,
281    fields: &[&str],
282    enums: &[(&str, &[&str])],
283) -> Result<(T, Node), CoreError> {
284    let bad = |problem: String| refuse(location, bytes, problem);
285    if bytes.is_empty() {
286        return Err(bad("the file is empty".to_string()));
287    }
288    if bytes.len() > cap {
289        return Err(bad(format!(
290            "the file is {} bytes, over the {cap} byte cap",
291            bytes.len()
292        )));
293    }
294    let (_, root) = node::classify(bytes).map_err(bad)?;
295    let Node::Obj(members) = &root else {
296        return Err(bad("the top level is not a JSON object".to_string()));
297    };
298    for (key, _) in members {
299        if !fields.contains(&key.as_str()) {
300            return Err(bad(format!("unknown field `{}`", shown(key))));
301        }
302    }
303    for field in fields {
304        if root.get(field).is_none() {
305            return Err(bad(format!("missing field `{field}`")));
306        }
307    }
308    for (field, allowed) in enums {
309        let ok = root
310            .get(field)
311            .and_then(Node::as_str)
312            .is_some_and(|v| allowed.contains(&v));
313        if !ok {
314            return Err(bad(format!(
315                "`{field}` is not one of {}",
316                allowed.join(", ")
317            )));
318        }
319    }
320    let value: T = serde_json::from_slice(bytes).map_err(|e| {
321        bad(format!(
322            "a field has the wrong type (line {} column {})",
323            e.line(),
324            e.column()
325        ))
326    })?;
327    Ok((value, root))
328}
329
330/// The header checks both formats share: the schema version, the run id and
331/// that the file names this target's gateway and mode. The problem text never
332/// echoes the foreign names.
333pub(super) fn check_header(
334    schema_version: u32,
335    run_id: &str,
336    gateway: (&str, &str),
337    mode: (&str, &str),
338) -> Result<(), String> {
339    if schema_version != SCHEMA_VERSION {
340        return Err(format!(
341            "schema_version is not {SCHEMA_VERSION}, the only version this build reads"
342        ));
343    }
344    if RunId::parse(run_id).is_err() {
345        return Err(
346            "run_id is not 1 to 128 characters of [A-Za-z0-9._-] without a leading dot".to_string(),
347        );
348    }
349    if gateway.0 != gateway.1 {
350        return Err("gateway names a different gateway than the target".to_string());
351    }
352    if mode.0 != mode.1 {
353        return Err("mode names a different mode than the target".to_string());
354    }
355    Ok(())
356}
357
358/// The `touched` rules: each path is an apply key (a collection path, or a
359/// `.core/` path of a routed tag type) or exactly `ledger.json`, and the list
360/// is byte-sorted and unique.
361pub(super) fn check_touched<'a>(paths: impl Iterator<Item = &'a str>) -> Result<(), String> {
362    let mut previous: Option<&str> = None;
363    for (index, path) in paths.enumerate() {
364        let listed = shown(path);
365        if path != LEDGER_KEY {
366            let collection = RelPath::new(path)
367                .ok()
368                .is_some_and(|rel| apply_key_problem(&rel).is_none());
369            if !collection {
370                return Err(format!(
371                    "touched[{index}] `{listed}` is neither a collection path nor `{LEDGER_KEY}`"
372                ));
373            }
374        }
375        if let Some(prev) = previous {
376            if path == prev {
377                return Err(format!(
378                    "touched[{index}] `{listed}` is listed twice (duplicate)"
379                ));
380            }
381            if path.as_bytes() < prev.as_bytes() {
382                return Err(format!(
383                    "touched is not byte-sorted: `{listed}` follows `{}`",
384                    shown(prev)
385                ));
386            }
387        }
388        previous = Some(path);
389    }
390    Ok(())
391}
392
393#[cfg(test)]
394mod tests {
395    use super::*;
396    use serde_json::{Value, json};
397
398    const GW: &str = "tracer-gw";
399    const MODE: &str = "dev";
400    const LOC: &str = "~/ign-deploy/tracer-gw/dev/journal.json";
401    const HASH: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
402
403    fn good() -> Journal {
404        Journal {
405            schema_version: 1,
406            run_id: "20261015T142233Z-a1b2c3".to_string(),
407            gateway: GW.to_string(),
408            mode: MODE.to_string(),
409            phase: JournalPhase::Writing,
410            pre_fingerprint: HASH.to_string(),
411            touched: vec![
412                "ledger.json".to_string(),
413                "mod/type/a.json".to_string(),
414                "mod/type/b.json".to_string(),
415            ],
416        }
417    }
418
419    fn with(edit: impl FnOnce(&mut Value)) -> Vec<u8> {
420        let mut value: Value = serde_json::from_slice(&good().to_bytes()).expect("good is JSON");
421        edit(&mut value);
422        serde_json::to_vec(&value).expect("serialize")
423    }
424
425    fn refusal(bytes: &[u8]) -> String {
426        let err = Journal::parse(LOC, bytes, GW, MODE).expect_err("must be refused");
427        assert_eq!(err.code(), "deploy_run_invalid", "{err}");
428        let text = err.to_string();
429        assert!(text.contains(LOC), "the refusal names the location: {text}");
430        text
431    }
432
433    #[test]
434    fn journal_roundtrip() {
435        let journal = good();
436        let bytes = journal.to_bytes();
437        assert!(bytes.ends_with(b"}\n"));
438        let parsed = Journal::parse(LOC, &bytes, GW, MODE).expect("parses");
439        assert_eq!(parsed, journal);
440        assert_eq!(parsed.to_bytes(), bytes, "byte-identical");
441        for phase in [
442            JournalPhase::Snapshotting,
443            JournalPhase::Ready,
444            JournalPhase::Writing,
445            JournalPhase::Written,
446            JournalPhase::Restoring,
447        ] {
448            let j = Journal { phase, ..good() };
449            assert_eq!(
450                Journal::parse(LOC, &j.to_bytes(), GW, MODE).expect("parses"),
451                j
452            );
453        }
454        let empty = Journal {
455            touched: vec![],
456            ..good()
457        };
458        assert_eq!(
459            Journal::parse(LOC, &empty.to_bytes(), GW, MODE).expect("parses"),
460            empty
461        );
462    }
463
464    #[test]
465    fn the_layout_is_pinned() {
466        let text = String::from_utf8(good().to_bytes()).expect("utf-8");
467        let expected = format!(
468            "{{\n  \"schema_version\": 1,\n  \"run_id\": \"20261015T142233Z-a1b2c3\",\n  \"gateway\": \"tracer-gw\",\n  \"mode\": \"dev\",\n  \"phase\": \"writing\",\n  \"pre_fingerprint\": \"{HASH}\",\n  \"touched\": [\n    \"ledger.json\",\n    \"mod/type/a.json\",\n    \"mod/type/b.json\"\n  ]\n}}\n"
469        );
470        assert_eq!(text, expected);
471    }
472
473    #[test]
474    fn phases_before_the_first_move() {
475        assert!(JournalPhase::Snapshotting.before_first_move());
476        assert!(JournalPhase::Ready.before_first_move());
477        assert!(!JournalPhase::Writing.before_first_move());
478        assert!(!JournalPhase::Written.before_first_move());
479        assert!(!JournalPhase::Restoring.before_first_move());
480    }
481
482    fn live(pairs: &[(&str, Option<&str>)]) -> std::collections::BTreeMap<String, Option<String>> {
483        pairs
484            .iter()
485            .map(|(path, hash)| ((*path).to_string(), hash.map(str::to_string)))
486            .collect()
487    }
488
489    #[test]
490    fn recovery_action_clears_only_an_untouched_run() {
491        use super::super::fingerprint::fingerprint;
492        let pre = live(&[("mod/type/a.json", Some(HASH)), ("mod/type/b.json", None)]);
493        let journal = |phase| Journal {
494            phase,
495            pre_fingerprint: fingerprint(&pre),
496            touched: pre.keys().cloned().collect(),
497            ..good()
498        };
499        let other = live(&[
500            ("mod/type/a.json", Some(HASH)),
501            ("mod/type/b.json", Some(HASH)),
502        ]);
503        for (phase, cleared) in [
504            (JournalPhase::Snapshotting, true),
505            (JournalPhase::Ready, true),
506            (JournalPhase::Writing, false),
507            (JournalPhase::Written, false),
508            (JournalPhase::Restoring, false),
509        ] {
510            let j = journal(phase);
511            let same = recovery_action(&j, &pre);
512            let changed = recovery_action(&j, &other);
513            assert_eq!(
514                same,
515                if cleared {
516                    RecoveryAction::ClearWithoutRestore
517                } else {
518                    RecoveryAction::Restore
519                },
520                "{phase:?}, live equals the pre-run state"
521            );
522            assert_eq!(
523                changed,
524                RecoveryAction::Restore,
525                "{phase:?}, live differs from the pre-run state"
526            );
527        }
528    }
529
530    #[test]
531    fn journal_location_follows_the_ledger_root() {
532        use crate::deploy::ledger::LedgerRoot;
533        assert_eq!(
534            journal_location(LedgerRoot::AccountHome, GW, MODE),
535            "~/ign-deploy/tracer-gw/dev/journal.json"
536        );
537        assert_eq!(
538            journal_location(LedgerRoot::DataDir, GW, MODE),
539            "ign-deploy/tracer-gw/dev/journal.json"
540        );
541    }
542
543    #[test]
544    fn refuses_an_empty_file() {
545        assert!(refusal(b"").contains("empty"));
546    }
547
548    #[test]
549    fn refuses_a_file_over_the_cap() {
550        let big = vec![b' '; JOURNAL_MAX_BYTES + 1];
551        assert!(refusal(&big).contains("cap"));
552    }
553
554    #[test]
555    fn refuses_invalid_utf8() {
556        refusal(b"{\"schema_version\": 1, \"run_id\": \"\xff\"}");
557    }
558
559    #[test]
560    fn refuses_a_duplicate_key() {
561        let text = String::from_utf8(good().to_bytes()).expect("utf-8");
562        let dup = text.replacen(
563            "\"mode\": \"dev\"",
564            "\"mode\": \"dev\",\n  \"mode\": \"dev\"",
565            1,
566        );
567        refusal(dup.as_bytes());
568    }
569
570    #[test]
571    fn refuses_an_unknown_field() {
572        let text = refusal(&with(|v| {
573            v["extra"] = json!(1);
574        }));
575        assert!(text.contains("extra"), "{text}");
576    }
577
578    #[test]
579    fn refuses_a_missing_field() {
580        let text = refusal(&with(|v| {
581            v.as_object_mut().expect("object").remove("phase");
582        }));
583        assert!(text.contains("phase"), "{text}");
584    }
585
586    #[test]
587    fn refuses_another_schema_version() {
588        refusal(&with(|v| v["schema_version"] = json!(2)));
589    }
590
591    #[test]
592    fn refuses_another_gateway() {
593        refusal(&with(|v| v["gateway"] = json!("other-gw")));
594    }
595
596    #[test]
597    fn refuses_another_mode() {
598        refusal(&with(|v| v["mode"] = json!("prod")));
599    }
600
601    #[test]
602    fn refuses_a_bad_run_id() {
603        for bad in ["", "../x", ".hidden", "a b"] {
604            refusal(&with(|v| v["run_id"] = json!(bad)));
605        }
606    }
607
608    #[test]
609    fn refuses_an_unknown_phase() {
610        let text = refusal(&with(|v| v["phase"] = json!("halfway")));
611        assert!(text.contains("phase"), "{text}");
612    }
613
614    #[test]
615    fn refuses_unsorted_touched() {
616        let text = refusal(&with(|v| {
617            v["touched"] = json!(["mod/type/b.json", "mod/type/a.json"]);
618        }));
619        assert!(text.contains("sorted"), "{text}");
620    }
621
622    #[test]
623    fn refuses_duplicate_touched() {
624        let text = refusal(&with(|v| {
625            v["touched"] = json!(["mod/type/a.json", "mod/type/a.json"]);
626        }));
627        assert!(text.contains("duplicate"), "{text}");
628    }
629
630    #[test]
631    fn refuses_a_touched_path_that_is_not_a_collection_path() {
632        for bad in [
633            "a/b",
634            "mod/type/.hidden/x",
635            "mod/type/../x",
636            "/abs/x/y",
637            "mod/type/resource.json/../../z",
638            "mod/type/config-mode.json",
639            "ledger.json.bak",
640        ] {
641            refusal(&with(|v| v["touched"] = json!([bad])));
642        }
643    }
644
645    #[test]
646    fn refuses_a_pre_fingerprint_that_is_not_hex() {
647        for bad in [
648            "",
649            "abc",
650            &HASH.to_uppercase(),
651            &format!("{HASH}0"),
652            "g".repeat(64).as_str(),
653        ] {
654            refusal(&with(|v| v["pre_fingerprint"] = json!(bad)));
655        }
656    }
657
658    #[test]
659    fn refuses_a_wrong_typed_field() {
660        refusal(&with(|v| v["touched"] = json!("mod/type/a.json")));
661        refusal(&with(|v| v["schema_version"] = json!("1")));
662    }
663
664    #[test]
665    fn refuses_a_top_level_that_is_not_an_object() {
666        refusal(b"[]");
667    }
668
669    #[test]
670    fn the_refusal_never_echoes_foreign_values() {
671        let text = refusal(&with(|v| v["gateway"] = json!("secret-gateway-name")));
672        assert!(!text.contains("secret-gateway-name"), "{text}");
673    }
674
675    /// quick 261102-cef (P1): a `.core/` key of a routed tag type is a
676    /// touched path; a `.core/` key of any other type is not.
677    #[test]
678    fn core_key_journal_check_touched() {
679        assert_eq!(
680            check_touched(
681                [
682                    ".core/ignition/tag-definition/default/Deploy/tags.json",
683                    "ignition/tag-definition/default/Deploy/tags.json",
684                    LEDGER_KEY,
685                ]
686                .into_iter()
687            ),
688            Ok(())
689        );
690        for bad in [
691            ".core/ignition/tag-provider/default/config.json",
692            ".core/ignition/database-translator/X/config.json",
693            ".core/.core/ignition/tag-definition/a/b",
694        ] {
695            assert!(check_touched([bad].into_iter()).is_err(), "{bad}");
696        }
697    }
698}