ignition_core/deploy/
pull.rs

1//! The pull engine (Phase 22, DPLN-06): a live tree to a declared tree that
2//! renders back to the live effective view.
3//!
4//! Pure: it takes an [`ObservedFiles`] the transport already read and returns
5//! a [`SourceTree`]; it touches no filesystem, network or clock. The action
6//! (`actions::deploy::deploy_pull`) reads the host through a
7//! [`TreeReader`](super::transport::TreeReader) and writes the result.
8//!
9//! ## Layering (D-12)
10//!
11//! Files of the `core` collection go to the base tree (`resources/`,
12//! `tags/`); files of the target's mode collection go to the
13//! `modes/<env>/` overlay, so [`render`]'s overlay reproduces the gateway's
14//! own layering. A unit in both collections lands in both. Bytes are kept
15//! exactly as read, a bootstrapped resource keeps the gateway's uuid (D-12).
16//!
17//! ## The render fixpoint (D-09)
18//!
19//! A pulled tree is only worth writing if it is one the declared pipeline
20//! accepts and reproduces. [`pull`] therefore renders its own tree and
21//! repeats, dropping units, until a round drops nothing (at most
22//! [`PULL_MAX_ROUNDS`] rounds):
23//!
24//! - a unit holding a secret location (Embedded or plaintext) is dropped with
25//!   reason [`REASON_EMBEDDED_SECRET`];
26//! - a unit any other render finding names is dropped with
27//!   `render_refused: <finding message>`; the messages name paths and key
28//!   names, never values;
29//! - once render succeeds, a unit whose rendered files are not exactly its
30//!   effective live files (mode copy when present, else `core`; same file
31//!   set, same bytes) is dropped with [`REASON_NOT_ROUND_TRIPPABLE`]: a
32//!   minted uuid, a dropped `.gitkeep`, a `core` file the mode copy lacks.
33//!
34//! A render error whose findings map to no unit (for example one naming
35//! `manifest.yaml`) is returned as is: the manifest is the operator's to fix.
36//! Because every kept unit renders byte-for-byte to its live copy, `pull`
37//! then `render` then `plan` is empty, and every dropped unit is `unmanaged`
38//! in the plan (the live copy exists, nothing renders it).
39//!
40//! ## Skip reasons (22-03 checkpoint, option-a)
41//!
42//! `embedded_secret`, `not_round_trippable`, `render_refused: <finding>`,
43//! `type_without_rule`, `denied_type`. No reason ever carries a value.
44
45use std::collections::{BTreeMap, BTreeSet};
46
47use serde::Serialize;
48
49use super::component_problem;
50use super::layout::{RelPath, TAG_TYPES, repo_path};
51use super::manifest::Manifest;
52use super::plan::{unit_key, unit_key_layer, unreadable_unit_keys};
53use super::render::{RenderInput, RenderOutcome, render};
54use super::rules::{is_denied, type_rule};
55use super::target::Target;
56use super::transport::ObservedFiles;
57use super::tree::SourceTree;
58use crate::error::{CoreError, DeployFinding};
59
60/// The most render rounds [`pull`] runs before it gives up.
61pub const PULL_MAX_ROUNDS: usize = 8;
62
63/// Skip reason: the unit holds an Embedded or plaintext secret location.
64pub const REASON_EMBEDDED_SECRET: &str = "embedded_secret";
65/// Skip reason: render does not reproduce the unit's live files.
66pub const REASON_NOT_ROUND_TRIPPABLE: &str = "not_round_trippable";
67/// Skip reason prefix: render refused the unit; the finding follows.
68pub const REASON_RENDER_REFUSED: &str = "render_refused";
69/// Skip reason: a live type with no render rule.
70pub const REASON_TYPE_WITHOUT_RULE: &str = "type_without_rule";
71/// Skip reason: a live type on the deny list.
72pub const REASON_DENIED_TYPE: &str = "denied_type";
73/// Skip reason: the unit has a live path of 240 characters or more
74/// (`LONG_PATH_LIMIT`) on a Windows sftp host, so it could not be read whole.
75/// Nothing of it is written (no partial unit); the next `plan` reports it
76/// under `unreadable`, never as absent.
77pub const REASON_PATH_TOO_LONG: &str = "path_too_long";
78
79/// One live unit (or type) pull did not write, and why.
80#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
81pub struct Skipped {
82    /// The unit key (`<module>/<type>/<name>`, tag folders
83    /// `ignition/<tag type>/<provider>/<folder>`), or `<module>/<type>` for a
84    /// type left out of a generated manifest. The same key `plan` reports.
85    pub path: String,
86    /// One of the reasons in the module doc. Never a value.
87    pub reason: String,
88}
89
90/// What [`pull`] needs.
91#[derive(Debug, Clone, Copy)]
92pub struct PullInput<'a> {
93    /// Gateway directory name under `gateways/`.
94    pub gateway: &'a str,
95    /// The environment (`--env`): names the `modes/<env>/` overlay.
96    pub env: &'a str,
97    /// The parsed target (render lints consult it).
98    pub target: &'a Target,
99    /// The manifest the pulled tree is rendered under.
100    pub manifest: &'a Manifest,
101    /// The live files: `core` and the target's mode collection.
102    pub observed: &'a ObservedFiles,
103}
104
105/// What [`pull`] produced.
106#[derive(Debug, Clone)]
107pub struct PullOutcome {
108    /// The tree to write under `gateways/<gateway>/`, gateway-relative.
109    pub source: SourceTree,
110    /// The render of `source`: what `deploy render` will produce from it.
111    pub rendered: RenderOutcome,
112    /// Every live unit not written, byte-sorted by path.
113    pub skipped: Vec<Skipped>,
114}
115
116fn refused(message: &str) -> String {
117    format!("{REASON_RENDER_REFUSED}: {message}")
118}
119
120/// Quote a scalar for the generated manifest only when a plain scalar could
121/// be read as another YAML type or break the line.
122fn yaml_scalar(value: &str) -> String {
123    let plain = value
124        .chars()
125        .next()
126        .is_some_and(|c| c.is_ascii_alphabetic() || c == '_')
127        && value
128            .chars()
129            .all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '.' | '/' | '-'))
130        && !matches!(
131            value.to_ascii_lowercase().as_str(),
132            "true" | "false" | "null" | "yes" | "no" | "on" | "off" | "y" | "n"
133        );
134    if plain {
135        value.to_string()
136    } else {
137        // a JSON string is a valid YAML double-quoted scalar
138        serde_json::to_string(value).unwrap_or_else(|_| "\"\"".to_string())
139    }
140}
141
142/// The `manifest.yaml` text for a gateway that has none (D-10), and the live
143/// types it leaves out.
144///
145/// `live_types` are the `(module, type)` directories found in `core` and the
146/// mode collection; `tag_providers` the provider names under
147/// `ignition/tag-provider/`. The manifest lists, byte-sorted, every live type
148/// that has a render rule and is not on the deny list, and none of the four
149/// tag types (they are carried by `tag_providers`). Left out, with a reason:
150/// denied types, types without a rule, and names that are not safe path
151/// components. The text parses with `Manifest::parse`.
152pub fn generate_manifest(
153    live_types: &[(String, String)],
154    tag_providers: &[String],
155) -> (String, Vec<Skipped>) {
156    let mut types: BTreeSet<String> = BTreeSet::new();
157    let mut skipped: BTreeMap<String, String> = BTreeMap::new();
158    for (module, ty) in live_types {
159        if module == "ignition" && TAG_TYPES.contains(&ty.as_str()) {
160            continue;
161        }
162        let id = format!("{module}/{ty}");
163        if let Some(why) = component_problem(module).or_else(|| component_problem(ty)) {
164            skipped.insert(id, refused(&format!("the type name {why}")));
165        } else if is_denied(module, ty) {
166            skipped.insert(id, REASON_DENIED_TYPE.to_string());
167        } else if type_rule(module, ty).is_none() {
168            skipped.insert(id, REASON_TYPE_WITHOUT_RULE.to_string());
169        } else {
170            types.insert(id);
171        }
172    }
173    let mut providers: BTreeSet<String> = BTreeSet::new();
174    for name in tag_providers {
175        match component_problem(name) {
176            Some(why) => {
177                skipped.insert(
178                    format!("ignition/tag-provider/{name}"),
179                    refused(&format!("the tag provider name {why}")),
180                );
181            }
182            None => {
183                providers.insert(name.clone());
184            }
185        }
186    }
187
188    let mut yaml = String::from("schema_version: 1\n");
189    if types.is_empty() {
190        yaml.push_str("types: []\n");
191    } else {
192        yaml.push_str("types:\n");
193        for t in &types {
194            yaml.push_str(&format!("  - {}\n", yaml_scalar(t)));
195        }
196    }
197    if providers.is_empty() {
198        yaml.push_str("tag_providers: []\n");
199    } else {
200        yaml.push_str("tag_providers:\n");
201        for p in &providers {
202            yaml.push_str(&format!("  - {}\n", yaml_scalar(p)));
203        }
204    }
205    let skipped = skipped
206        .into_iter()
207        .map(|(path, reason)| Skipped { path, reason })
208        .collect();
209    (yaml, skipped)
210}
211
212/// The live files of one unit, per collection.
213#[derive(Default)]
214struct Unit {
215    core: Vec<RelPath>,
216    mode: Vec<RelPath>,
217}
218
219/// Every source file of a unit (gateway-relative path and live bytes).
220type Placed<'a> = Vec<(RelPath, &'a [u8])>;
221
222fn place<'a>(unit: &Unit, env: &str, observed: &'a ObservedFiles) -> Result<Placed<'a>, String> {
223    let mut placed = Placed::new();
224    for path in &unit.core {
225        placed.push((repo_path(path)?, observed.core[path].bytes.as_slice()));
226    }
227    for path in &unit.mode {
228        let base = repo_path(path)?;
229        placed.push((
230            RelPath::new(&format!("modes/{env}/{base}"))?,
231            observed.mode[path].bytes.as_slice(),
232        ));
233    }
234    Ok(placed)
235}
236
237/// Repo paths (`gateways/<g>/...`) under or at `path`, as unit keys.
238fn units_at<'a>(index: &'a BTreeMap<String, &'a str>, path: &str) -> BTreeSet<&'a str> {
239    let trimmed = path.trim_end_matches('/');
240    let mut found = BTreeSet::new();
241    if let Some(unit) = index.get(trimmed) {
242        found.insert(*unit);
243    }
244    let prefix = format!("{trimmed}/");
245    for (file, unit) in index.range(prefix.clone()..) {
246        if !file.starts_with(&prefix) {
247            break;
248        }
249        found.insert(*unit);
250    }
251    found
252}
253
254/// The unit a secret location (`<file>: <json path> (...)`) belongs to. The
255/// file part may itself contain `: `, so every split point is tried.
256fn unit_of_secret<'a>(index: &'a BTreeMap<String, &'a str>, location: &str) -> Option<&'a str> {
257    location
258        .match_indices(": ")
259        .find_map(|(at, _)| index.get(&location[..at]).copied())
260}
261
262/// Run the pull: assemble, render to a fixpoint, keep what round-trips.
263pub fn pull(input: &PullInput<'_>) -> Result<PullOutcome, CoreError> {
264    let observed = input.observed;
265    let core_layer = unit_key_layer(observed, "core");
266    let mode_layer = unit_key_layer(observed, "mode");
267    let mut units: BTreeMap<String, Unit> = BTreeMap::new();
268    for path in observed.core.keys() {
269        units
270            .entry(unit_key(path, &core_layer))
271            .or_default()
272            .core
273            .push(path.clone());
274    }
275    for path in observed.mode.keys() {
276        units
277            .entry(unit_key(path, &mode_layer))
278            .or_default()
279            .mode
280            .push(path.clone());
281    }
282
283    let prefix = format!("gateways/{}/", input.gateway);
284    let mut skipped: BTreeMap<String, String> = BTreeMap::new();
285    // an unreadable unit is never written in part; the bootstrap continues
286    for key in unreadable_unit_keys(observed).into_keys() {
287        units.remove(&key);
288        skipped.insert(key, REASON_PATH_TOO_LONG.to_string());
289    }
290    let mut placed: BTreeMap<&str, Placed<'_>> = BTreeMap::new();
291    let mut index: BTreeMap<String, &str> = BTreeMap::new();
292    for (key, unit) in &units {
293        match place(unit, input.env, observed) {
294            Ok(files) => {
295                for (path, _) in &files {
296                    index.insert(format!("{prefix}{path}"), key.as_str());
297                }
298                placed.insert(key.as_str(), files);
299            }
300            Err(why) => {
301                skipped.insert(key.clone(), refused(&why));
302            }
303        }
304    }
305
306    for _round in 0..PULL_MAX_ROUNDS {
307        let mut files = BTreeMap::new();
308        for (key, unit_files) in &placed {
309            if skipped.contains_key(*key) {
310                continue;
311            }
312            for (path, bytes) in unit_files {
313                files.insert(path.clone(), bytes.to_vec());
314            }
315        }
316        let source = SourceTree { files };
317
318        let result = render(&RenderInput {
319            gateway: input.gateway,
320            env: input.env,
321            manifest: input.manifest,
322            target: input.target,
323            source: &source,
324        });
325
326        let mut dropped: BTreeMap<String, String> = BTreeMap::new();
327        match result {
328            Ok(rendered) => {
329                let mismatches = mismatched_units(&units, &placed, &skipped, observed, &rendered)?;
330                if mismatches.is_empty() {
331                    return Ok(PullOutcome {
332                        source,
333                        rendered,
334                        skipped: skipped
335                            .into_iter()
336                            .map(|(path, reason)| Skipped { path, reason })
337                            .collect(),
338                    });
339                }
340                for key in mismatches {
341                    dropped.insert(key, REASON_NOT_ROUND_TRIPPABLE.to_string());
342                }
343            }
344            Err(err) => {
345                let (secrets, findings): (&[String], &[DeployFinding]) = match &err {
346                    CoreError::EmbeddedSecret(d) => (&d.paths, &d.other_findings),
347                    CoreError::DeployTreeInvalid(d) => (&[], &d.findings),
348                    _ => return Err(err),
349                };
350                let mut unmapped = false;
351                for location in secrets {
352                    match unit_of_secret(&index, location) {
353                        Some(key) => {
354                            dropped.insert(key.to_string(), REASON_EMBEDDED_SECRET.to_string());
355                        }
356                        None => unmapped = true,
357                    }
358                }
359                for finding in findings {
360                    let hit = units_at(&index, &finding.file);
361                    if hit.is_empty() {
362                        unmapped = true;
363                    }
364                    for key in hit {
365                        dropped
366                            .entry(key.to_string())
367                            .or_insert_with(|| refused(&finding.message));
368                    }
369                }
370                // a finding about no unit (the manifest, the target) is the
371                // operator's to fix; a round that drops nothing cannot progress
372                if unmapped || dropped.is_empty() {
373                    return Err(err);
374                }
375            }
376        }
377        skipped.extend(dropped);
378    }
379    Err(CoreError::deploy_tree_invalid(
380        format!("gateways/{}", input.gateway),
381        None,
382        None,
383        "pull did not converge",
384    ))
385}
386
387/// The kept units whose rendered files are not exactly their effective live
388/// files. A rendered group that belongs to no kept unit cannot be attributed
389/// and is an error.
390fn mismatched_units(
391    units: &BTreeMap<String, Unit>,
392    placed: &BTreeMap<&str, Placed<'_>>,
393    skipped: &BTreeMap<String, String>,
394    observed: &ObservedFiles,
395    rendered: &RenderOutcome,
396) -> Result<Vec<String>, CoreError> {
397    let mut groups: BTreeMap<String, BTreeMap<&RelPath, &Vec<u8>>> = BTreeMap::new();
398    for (path, bytes) in &rendered.tree.files {
399        groups
400            .entry(unit_key(path, &rendered.tree.files))
401            .or_default()
402            .insert(path, bytes);
403    }
404    let mut mismatched = Vec::new();
405    let mut kept: BTreeSet<&str> = BTreeSet::new();
406    for (key, unit) in units {
407        if skipped.contains_key(key) || !placed.contains_key(key.as_str()) {
408            continue;
409        }
410        kept.insert(key.as_str());
411        let (paths, layer) = if unit.mode.is_empty() {
412            (&unit.core, &observed.core)
413        } else {
414            (&unit.mode, &observed.mode)
415        };
416        let same = groups.get(key).is_some_and(|want| {
417            want.len() == paths.len()
418                && paths
419                    .iter()
420                    .all(|p| want.get(p).is_some_and(|b| b.as_slice() == layer[p].bytes))
421        });
422        if !same {
423            mismatched.push(key.clone());
424        }
425    }
426    if mismatched.is_empty()
427        && let Some(stray) = groups.keys().find(|k| !kept.contains(k.as_str()))
428    {
429        return Err(CoreError::deploy_tree_invalid(
430            "gateways",
431            None,
432            None,
433            format!("pull could not attribute the rendered unit `{stray}` to a live unit"),
434        ));
435    }
436    Ok(mismatched)
437}
ignition_core/deploy/pull.rs

ignition_core/deploy/
pull.rs
