ignition_core/deploy/
pull.rs

1//! The pull engine (Phase 22, DPLN-06): a live tree to a declared tree that
2//! renders back to the live effective view.
3//!
4//! Pure: it takes an [`ObservedFiles`] the transport already read and returns
5//! a [`SourceTree`]; it touches no filesystem, network or clock. The action
6//! (`actions::deploy::deploy_pull`) reads the host through a
7//! [`TreeReader`](super::transport::TreeReader) and writes the result.
8//!
9//! ## Layering (D-12)
10//!
11//! Files of the `core` collection go to the base tree (`resources/`,
12//! `tags/`); files of the target's mode collection go to the
13//! `modes/<env>/` overlay, so [`render`]'s overlay reproduces the gateway's
14//! own layering. A unit in both collections lands in both. Bytes are kept
15//! exactly as read, a bootstrapped resource keeps the gateway's uuid (D-12).
16//!
17//! ## The render fixpoint (D-09)
18//!
19//! A pulled tree is only worth writing if it is one the declared pipeline
20//! accepts and reproduces. [`pull`] therefore renders its own tree and
21//! repeats, dropping units, until a round drops nothing (at most
22//! [`PULL_MAX_ROUNDS`] rounds):
23//!
24//! - a unit holding a secret location (Embedded or plaintext) is dropped with
25//!   reason [`REASON_EMBEDDED_SECRET`];
26//! - a unit any other render finding names is dropped with
27//!   `render_refused: <finding message>`; the messages name paths and key
28//!   names, never values;
29//! - once render succeeds, a unit whose rendered files are not exactly its
30//!   effective live files (mode copy when present, else `core`; same file
31//!   set, same bytes) is dropped with [`REASON_NOT_ROUND_TRIPPABLE`]: a
32//!   minted uuid, a dropped `.gitkeep`, a `core` file the mode copy lacks.
33//!
34//! A render error whose findings map to no unit (for example one naming
35//! `manifest.yaml`) is returned as is: the manifest is the operator's to fix.
36//! Because every kept unit renders byte-for-byte to its live copy, `pull`
37//! then `render` then `plan` is empty, and every dropped unit is `unmanaged`
38//! in the plan (the live copy exists, nothing renders it).
39//!
40//! ## Skip reasons (22-03 checkpoint, option-a)
41//!
42//! `embedded_secret`, `not_round_trippable`, `render_refused: <finding>`,
43//! `type_without_rule`, `denied_type`. No reason ever carries a value.
44
45use std::collections::{BTreeMap, BTreeSet};
46
47use serde::Serialize;
48
49use super::component_problem;
50use super::layout::{RelPath, TAG_TYPES, repo_path};
51use super::manifest::Manifest;
52use super::plan::unit_key;
53use super::render::{RenderInput, RenderOutcome, render};
54use super::rules::{is_denied, type_rule};
55use super::target::Target;
56use super::transport::ObservedFiles;
57use super::tree::SourceTree;
58use crate::error::{CoreError, DeployFinding};
59
60/// The most render rounds [`pull`] runs before it gives up.
61pub const PULL_MAX_ROUNDS: usize = 8;
62
63/// Skip reason: the unit holds an Embedded or plaintext secret location.
64pub const REASON_EMBEDDED_SECRET: &str = "embedded_secret";
65/// Skip reason: render does not reproduce the unit's live files.
66pub const REASON_NOT_ROUND_TRIPPABLE: &str = "not_round_trippable";
67/// Skip reason prefix: render refused the unit; the finding follows.
68pub const REASON_RENDER_REFUSED: &str = "render_refused";
69/// Skip reason: a live type with no render rule.
70pub const REASON_TYPE_WITHOUT_RULE: &str = "type_without_rule";
71/// Skip reason: a live type on the deny list.
72pub const REASON_DENIED_TYPE: &str = "denied_type";
73
74/// One live unit (or type) pull did not write, and why.
75#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
76pub struct Skipped {
77    /// The unit key (`<module>/<type>/<name>`, tag folders
78    /// `ignition/<tag type>/<provider>/<folder>`), or `<module>/<type>` for a
79    /// type left out of a generated manifest. The same key `plan` reports.
80    pub path: String,
81    /// One of the reasons in the module doc. Never a value.
82    pub reason: String,
83}
84
85/// What [`pull`] needs.
86#[derive(Debug, Clone, Copy)]
87pub struct PullInput<'a> {
88    /// Gateway directory name under `gateways/`.
89    pub gateway: &'a str,
90    /// The environment (`--env`): names the `modes/<env>/` overlay.
91    pub env: &'a str,
92    /// The parsed target (render lints consult it).
93    pub target: &'a Target,
94    /// The manifest the pulled tree is rendered under.
95    pub manifest: &'a Manifest,
96    /// The live files: `core` and the target's mode collection.
97    pub observed: &'a ObservedFiles,
98}
99
100/// What [`pull`] produced.
101#[derive(Debug, Clone)]
102pub struct PullOutcome {
103    /// The tree to write under `gateways/<gateway>/`, gateway-relative.
104    pub source: SourceTree,
105    /// The render of `source`: what `deploy render` will produce from it.
106    pub rendered: RenderOutcome,
107    /// Every live unit not written, byte-sorted by path.
108    pub skipped: Vec<Skipped>,
109}
110
111fn refused(message: &str) -> String {
112    format!("{REASON_RENDER_REFUSED}: {message}")
113}
114
115/// Quote a scalar for the generated manifest only when a plain scalar could
116/// be read as another YAML type or break the line.
117fn yaml_scalar(value: &str) -> String {
118    let plain = value
119        .chars()
120        .next()
121        .is_some_and(|c| c.is_ascii_alphabetic() || c == '_')
122        && value
123            .chars()
124            .all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '.' | '/' | '-'))
125        && !matches!(
126            value.to_ascii_lowercase().as_str(),
127            "true" | "false" | "null" | "yes" | "no" | "on" | "off" | "y" | "n"
128        );
129    if plain {
130        value.to_string()
131    } else {
132        // a JSON string is a valid YAML double-quoted scalar
133        serde_json::to_string(value).unwrap_or_else(|_| "\"\"".to_string())
134    }
135}
136
137/// The `manifest.yaml` text for a gateway that has none (D-10), and the live
138/// types it leaves out.
139///
140/// `live_types` are the `(module, type)` directories found in `core` and the
141/// mode collection; `tag_providers` the provider names under
142/// `ignition/tag-provider/`. The manifest lists, byte-sorted, every live type
143/// that has a render rule and is not on the deny list, and none of the four
144/// tag types (they are carried by `tag_providers`). Left out, with a reason:
145/// denied types, types without a rule, and names that are not safe path
146/// components. The text parses with `Manifest::parse`.
147pub fn generate_manifest(
148    live_types: &[(String, String)],
149    tag_providers: &[String],
150) -> (String, Vec<Skipped>) {
151    let mut types: BTreeSet<String> = BTreeSet::new();
152    let mut skipped: BTreeMap<String, String> = BTreeMap::new();
153    for (module, ty) in live_types {
154        if module == "ignition" && TAG_TYPES.contains(&ty.as_str()) {
155            continue;
156        }
157        let id = format!("{module}/{ty}");
158        if let Some(why) = component_problem(module).or_else(|| component_problem(ty)) {
159            skipped.insert(id, refused(&format!("the type name {why}")));
160        } else if is_denied(module, ty) {
161            skipped.insert(id, REASON_DENIED_TYPE.to_string());
162        } else if type_rule(module, ty).is_none() {
163            skipped.insert(id, REASON_TYPE_WITHOUT_RULE.to_string());
164        } else {
165            types.insert(id);
166        }
167    }
168    let mut providers: BTreeSet<String> = BTreeSet::new();
169    for name in tag_providers {
170        match component_problem(name) {
171            Some(why) => {
172                skipped.insert(
173                    format!("ignition/tag-provider/{name}"),
174                    refused(&format!("the tag provider name {why}")),
175                );
176            }
177            None => {
178                providers.insert(name.clone());
179            }
180        }
181    }
182
183    let mut yaml = String::from("schema_version: 1\n");
184    if types.is_empty() {
185        yaml.push_str("types: []\n");
186    } else {
187        yaml.push_str("types:\n");
188        for t in &types {
189            yaml.push_str(&format!("  - {}\n", yaml_scalar(t)));
190        }
191    }
192    if providers.is_empty() {
193        yaml.push_str("tag_providers: []\n");
194    } else {
195        yaml.push_str("tag_providers:\n");
196        for p in &providers {
197            yaml.push_str(&format!("  - {}\n", yaml_scalar(p)));
198        }
199    }
200    let skipped = skipped
201        .into_iter()
202        .map(|(path, reason)| Skipped { path, reason })
203        .collect();
204    (yaml, skipped)
205}
206
207/// The live files of one unit, per collection.
208#[derive(Default)]
209struct Unit {
210    core: Vec<RelPath>,
211    mode: Vec<RelPath>,
212}
213
214/// Every source file of a unit (gateway-relative path and live bytes).
215type Placed<'a> = Vec<(RelPath, &'a [u8])>;
216
217fn place<'a>(unit: &Unit, env: &str, observed: &'a ObservedFiles) -> Result<Placed<'a>, String> {
218    let mut placed = Placed::new();
219    for path in &unit.core {
220        placed.push((repo_path(path)?, observed.core[path].bytes.as_slice()));
221    }
222    for path in &unit.mode {
223        let base = repo_path(path)?;
224        placed.push((
225            RelPath::new(&format!("modes/{env}/{base}"))?,
226            observed.mode[path].bytes.as_slice(),
227        ));
228    }
229    Ok(placed)
230}
231
232/// Repo paths (`gateways/<g>/...`) under or at `path`, as unit keys.
233fn units_at<'a>(index: &'a BTreeMap<String, &'a str>, path: &str) -> BTreeSet<&'a str> {
234    let trimmed = path.trim_end_matches('/');
235    let mut found = BTreeSet::new();
236    if let Some(unit) = index.get(trimmed) {
237        found.insert(*unit);
238    }
239    let prefix = format!("{trimmed}/");
240    for (file, unit) in index.range(prefix.clone()..) {
241        if !file.starts_with(&prefix) {
242            break;
243        }
244        found.insert(*unit);
245    }
246    found
247}
248
249/// The unit a secret location (`<file>: <json path> (...)`) belongs to. The
250/// file part may itself contain `: `, so every split point is tried.
251fn unit_of_secret<'a>(index: &'a BTreeMap<String, &'a str>, location: &str) -> Option<&'a str> {
252    location
253        .match_indices(": ")
254        .find_map(|(at, _)| index.get(&location[..at]).copied())
255}
256
257/// Run the pull: assemble, render to a fixpoint, keep what round-trips.
258pub fn pull(input: &PullInput<'_>) -> Result<PullOutcome, CoreError> {
259    let observed = input.observed;
260    let mut units: BTreeMap<String, Unit> = BTreeMap::new();
261    for path in observed.core.keys() {
262        units
263            .entry(unit_key(path, &observed.core))
264            .or_default()
265            .core
266            .push(path.clone());
267    }
268    for path in observed.mode.keys() {
269        units
270            .entry(unit_key(path, &observed.mode))
271            .or_default()
272            .mode
273            .push(path.clone());
274    }
275
276    let prefix = format!("gateways/{}/", input.gateway);
277    let mut skipped: BTreeMap<String, String> = BTreeMap::new();
278    let mut placed: BTreeMap<&str, Placed<'_>> = BTreeMap::new();
279    let mut index: BTreeMap<String, &str> = BTreeMap::new();
280    for (key, unit) in &units {
281        match place(unit, input.env, observed) {
282            Ok(files) => {
283                for (path, _) in &files {
284                    index.insert(format!("{prefix}{path}"), key.as_str());
285                }
286                placed.insert(key.as_str(), files);
287            }
288            Err(why) => {
289                skipped.insert(key.clone(), refused(&why));
290            }
291        }
292    }
293
294    for _round in 0..PULL_MAX_ROUNDS {
295        let mut files = BTreeMap::new();
296        for (key, unit_files) in &placed {
297            if skipped.contains_key(*key) {
298                continue;
299            }
300            for (path, bytes) in unit_files {
301                files.insert(path.clone(), bytes.to_vec());
302            }
303        }
304        let source = SourceTree { files };
305
306        let result = render(&RenderInput {
307            gateway: input.gateway,
308            env: input.env,
309            manifest: input.manifest,
310            target: input.target,
311            source: &source,
312        });
313
314        let mut dropped: BTreeMap<String, String> = BTreeMap::new();
315        match result {
316            Ok(rendered) => {
317                let mismatches = mismatched_units(&units, &placed, &skipped, observed, &rendered)?;
318                if mismatches.is_empty() {
319                    return Ok(PullOutcome {
320                        source,
321                        rendered,
322                        skipped: skipped
323                            .into_iter()
324                            .map(|(path, reason)| Skipped { path, reason })
325                            .collect(),
326                    });
327                }
328                for key in mismatches {
329                    dropped.insert(key, REASON_NOT_ROUND_TRIPPABLE.to_string());
330                }
331            }
332            Err(err) => {
333                let (secrets, findings): (&[String], &[DeployFinding]) = match &err {
334                    CoreError::EmbeddedSecret(d) => (&d.paths, &d.other_findings),
335                    CoreError::DeployTreeInvalid(d) => (&[], &d.findings),
336                    _ => return Err(err),
337                };
338                let mut unmapped = false;
339                for location in secrets {
340                    match unit_of_secret(&index, location) {
341                        Some(key) => {
342                            dropped.insert(key.to_string(), REASON_EMBEDDED_SECRET.to_string());
343                        }
344                        None => unmapped = true,
345                    }
346                }
347                for finding in findings {
348                    let hit = units_at(&index, &finding.file);
349                    if hit.is_empty() {
350                        unmapped = true;
351                    }
352                    for key in hit {
353                        dropped
354                            .entry(key.to_string())
355                            .or_insert_with(|| refused(&finding.message));
356                    }
357                }
358                // a finding about no unit (the manifest, the target) is the
359                // operator's to fix; a round that drops nothing cannot progress
360                if unmapped || dropped.is_empty() {
361                    return Err(err);
362                }
363            }
364        }
365        skipped.extend(dropped);
366    }
367    Err(CoreError::deploy_tree_invalid(
368        format!("gateways/{}", input.gateway),
369        None,
370        None,
371        "pull did not converge",
372    ))
373}
374
375/// The kept units whose rendered files are not exactly their effective live
376/// files. A rendered group that belongs to no kept unit cannot be attributed
377/// and is an error.
378fn mismatched_units(
379    units: &BTreeMap<String, Unit>,
380    placed: &BTreeMap<&str, Placed<'_>>,
381    skipped: &BTreeMap<String, String>,
382    observed: &ObservedFiles,
383    rendered: &RenderOutcome,
384) -> Result<Vec<String>, CoreError> {
385    let mut groups: BTreeMap<String, BTreeMap<&RelPath, &Vec<u8>>> = BTreeMap::new();
386    for (path, bytes) in &rendered.tree.files {
387        groups
388            .entry(unit_key(path, &rendered.tree.files))
389            .or_default()
390            .insert(path, bytes);
391    }
392    let mut mismatched = Vec::new();
393    let mut kept: BTreeSet<&str> = BTreeSet::new();
394    for (key, unit) in units {
395        if skipped.contains_key(key) || !placed.contains_key(key.as_str()) {
396            continue;
397        }
398        kept.insert(key.as_str());
399        let (paths, layer) = if unit.mode.is_empty() {
400            (&unit.core, &observed.core)
401        } else {
402            (&unit.mode, &observed.mode)
403        };
404        let same = groups.get(key).is_some_and(|want| {
405            want.len() == paths.len()
406                && paths
407                    .iter()
408                    .all(|p| want.get(p).is_some_and(|b| b.as_slice() == layer[p].bytes))
409        });
410        if !same {
411            mismatched.push(key.clone());
412        }
413    }
414    if mismatched.is_empty()
415        && let Some(stray) = groups.keys().find(|k| !kept.contains(k.as_str()))
416    {
417        return Err(CoreError::deploy_tree_invalid(
418            "gateways",
419            None,
420            None,
421            format!("pull could not attribute the rendered unit `{stray}` to a live unit"),
422        ));
423    }
424    Ok(mismatched)
425}
ignition_core/deploy/pull.rs

ignition_core/deploy/
pull.rs
