isoloom_spec/
defaults.rs

1//! Defaults: settings that aren't the spec's business but a person's or a team's, in layers,
2//! the last one winning:
3//!
4//! 1. built in (the image table in [`crate::images`], each cloud's region in its generator),
5//! 2. the user's file, `$ISOLOOM_HOME/defaults.yml` (`~/.isoloom/defaults.yml`),
6//! 3. the project's file, `isoloom.defaults.yml` next to the spec (committed: the team's),
7//! 4. environment variables `ISOLOOM_<KEY>`, `__` standing for a dot (`ISOLOOM_CLOUD__AWS__REGION`),
8//! 5. the command line, `-s defaults.<key>=<value>`.
9//!
10//! Mappings deep-merge, so a file that sets one region leaves the others alone. The same `-s`
11//! without the `defaults.` prefix overrides the spec itself (`-s machines.web.vm.os=ubuntu@24.04`),
12//! for one run.
13//!
14//! ```yaml
15//! images:                 # the image table (see Images): OS images, the access machine
16//!   os: { debian-12: { vagrant: my-org/debian-12 } }
17//!   access: { docker: kalilinux/kali-rolling, vm: kali }
18//! vagrant:
19//!   provider: libvirt     # `isoloom run vagrant` passes --provider
20//! cloud:
21//!   aws: { region: eu-west-1 }
22//!   azure: { region: westeurope }
23//! ```
24
25use std::path::{Path, PathBuf};
26
27use indexmap::IndexMap;
28use serde::Deserialize;
29use serde_yaml_ng::{Mapping, Value};
30
31use crate::files::GeneratedFile;
32use crate::host::Host;
33use crate::images::Table;
34use crate::{Error, Result};
35
36/// The project-level file, next to `isoloom.yml`.
37pub const PROJECT_FILE: &str = "isoloom.defaults.yml";
38
39/// The typed defaults, once the layers are merged. Unknown keys are errors.
40#[derive(Debug, Clone, Default, PartialEq, Deserialize)]
41#[serde(deny_unknown_fields)]
42pub struct Defaults {
43    /// The image table: OS images and the access machine the runner supplies.
44    #[serde(default)]
45    pub images: Table,
46    /// How `isoloom run vagrant` runs Vagrant.
47    #[serde(default)]
48    pub vagrant: Vagrant,
49    /// Per cloud (`aws`, `azure`, `gcp`, `digitalocean`, `linode`, `oci`).
50    #[serde(default)]
51    pub cloud: IndexMap<String, Cloud>,
52}
53
54/// The defaults' `vagrant:` section.
55#[derive(Debug, Clone, Default, PartialEq, Deserialize)]
56#[serde(deny_unknown_fields)]
57pub struct Vagrant {
58    /// The provider `isoloom run vagrant` asks for (`virtualbox`, `libvirt`, `vmware_desktop`,
59    /// `parallels`, `utm`, `qemu`, `hyperv`, `vmware_esxi`). Default: Vagrant's own choice.
60    #[serde(default)]
61    pub provider: Option<String>,
62}
63
64/// One cloud's entry in the defaults' `cloud:` section.
65#[derive(Debug, Clone, Default, PartialEq, Deserialize)]
66#[serde(deny_unknown_fields)]
67pub struct Cloud {
68    /// The region the generated module defaults to.
69    #[serde(default)]
70    pub region: Option<String>,
71}
72
73/// Each cloud's built-in region, as its generator writes it. A test keeps them equal.
74pub const BUILTIN_REGIONS: &[(&str, &str)] = &[
75    ("aws", "eu-west-3"),
76    ("azure", "swedencentral"),
77    ("gcp", "europe-west9"),
78    ("digitalocean", "fra1"),
79    ("linode", "fr-par"),
80    ("oci", "eu-paris-1"),
81];
82
83/// One layer's contribution: where it came from, and what it set.
84#[derive(Debug, Clone, PartialEq)]
85pub struct Layer {
86    /// Where the layer comes from (a file, `--images`, an `ISOLOOM_*` variable, `-s`), as `isoloom
87    /// defaults` shows it.
88    pub source: String,
89    /// What the layer sets, as YAML.
90    pub value: Value,
91}
92
93/// The merged defaults with, for every leaf, the layer that set it last.
94#[derive(Debug, Clone, PartialEq)]
95pub struct Resolved {
96    /// The merged defaults, typed.
97    pub defaults: Defaults,
98    /// The merged defaults, as YAML (what `isoloom defaults` prints).
99    pub merged: Value,
100    /// Dotted key -> source, for `isoloom defaults`.
101    pub sources: Vec<(String, String)>,
102}
103
104/// The user's defaults file, in Isoloom's home on `host`.
105#[must_use]
106pub fn user_file(host: &Host) -> PathBuf {
107    host.home().join("defaults.yml")
108}
109
110/// What a project's defaults are read from: the project, the `--images` table, its profile, the
111/// command's `-s` and the host it runs on.
112#[derive(Debug, Clone)]
113pub struct Sources<'a> {
114    /// The project's folder.
115    pub project: &'a Path,
116    /// The `--images` table, between the files and `sets`.
117    pub images_file: Option<&'a Path>,
118    /// The profile, between the project's file and `--images`.
119    pub profile: Option<&'a crate::profile::Profile>,
120    /// The command's `-s` (`defaults.cloud.aws.region=eu-west-1`, the prefix optional).
121    pub sets: &'a [String],
122    /// The host: Isoloom's home, and the `ISOLOOM_*` variables.
123    pub host: Host,
124}
125
126/// Reads every layer for a project: the user's file, the project's, the profile, `--images`,
127/// the host's `ISOLOOM_*` variables, then the `-s` overrides.
128///
129/// # Errors
130///
131/// [`Error::Read`] for a file that can't be read (one that doesn't exist is no layer),
132/// [`Error::Layer`] for one that isn't YAML, [`Error::SetShape`] and [`Error::SetKey`] for a
133/// `-s` that isn't `key=value`, and [`Error::Defaults`] when the merged layers hold a key or a
134/// value the defaults don't have.
135pub fn load(from: &Sources) -> Result<Resolved> {
136    let mut layers = Vec::new();
137    for (label, path) in [("user file", user_file(&from.host)), ("project file", from.project.join(PROJECT_FILE))] {
138        if let Some(value) = read_layer(&path)? {
139            layers.push(Layer {
140                source: format!("{label} {}", path.display()),
141                value,
142            });
143        }
144    }
145    if let Some(p) = from.profile {
146        layers.push(p.layer());
147    }
148    if let Some(f) = from.images_file {
149        let text = std::fs::read_to_string(f).map_err(|source| Error::Read { path: f.to_path_buf(), source })?;
150        let table: Value = serde_yaml_ng::from_str(&text).map_err(|source| Error::Layer { path: f.to_path_buf(), source })?;
151        let mut m = Mapping::new();
152        m.insert(Value::String("images".into()), table);
153        layers.push(Layer {
154            source: format!("--images {}", f.display()),
155            value: Value::Mapping(m),
156        });
157    }
158    for (k, v) in from_env(from.host.text_vars().into_iter()) {
159        layers.push(Layer {
160            source: format!("environment ISOLOOM_{}", k.replace('.', "__").to_uppercase()),
161            value: at_path(&k, scalar(&v)),
162        });
163    }
164    for s in from.sets {
165        let (k, v) = parse_set(s)?;
166        let Some(k) = k.strip_prefix("defaults.") else { continue };
167        layers.push(Layer {
168            source: format!("-s {s}"),
169            value: at_path(k, scalar(v)),
170        });
171    }
172    resolve(&layers)
173}
174
175/// A layer's file, as YAML; none when it doesn't exist.
176fn read_layer(path: &Path) -> Result<Option<Value>> {
177    match std::fs::read_to_string(path) {
178        Ok(text) => serde_yaml_ng::from_str(&text).map(Some).map_err(|source| Error::Layer {
179            path: path.to_path_buf(),
180            source,
181        }),
182        Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None),
183        Err(source) => Err(Error::Read {
184            path: path.to_path_buf(),
185            source,
186        }),
187    }
188}
189
190/// Merges layers in order; later ones win leaf by leaf.
191///
192/// # Errors
193///
194/// [`Error::Defaults`] when the merged layers hold a key or a value the defaults don't have.
195pub fn resolve(layers: &[Layer]) -> Result<Resolved> {
196    let mut merged = Value::Mapping(Mapping::new());
197    let mut sources: Vec<(String, String)> = Vec::new();
198    for layer in layers {
199        merge(&mut merged, &layer.value);
200        for leaf in leaves(&layer.value, "") {
201            sources.retain(|(k, _)| *k != leaf);
202            sources.push((leaf, layer.source.clone()));
203        }
204    }
205    let defaults: Defaults = serde_yaml_ng::from_value(merged.clone()).map_err(|source| Error::Defaults {
206        sources: layers.iter().map(|l| l.source.as_str()).collect::<Vec<_>>().join(", "),
207        source,
208    })?;
209    Ok(Resolved { defaults, merged, sources })
210}
211
212/// `ISOLOOM_CLOUD__AWS__REGION=x` -> `("cloud.aws.region", "x")`. Only the defaults' sections
213/// are read (`ISOLOOM_HOME`, `ISOLOOM_DERIVED` and the publish variables mean other things).
214pub fn from_env(vars: impl Iterator<Item = (String, String)>) -> Vec<(String, String)> {
215    let mut out: Vec<(String, String)> = vars
216        .filter_map(|(k, v)| {
217            let rest = k.strip_prefix("ISOLOOM_")?;
218            let key = rest.to_lowercase().replace("__", ".");
219            let section = key.split('.').next().unwrap_or("");
220            ["images", "vagrant", "cloud"].contains(&section).then_some((key, v))
221        })
222        .collect();
223    out.sort();
224    out
225}
226
227/// `key=value` of a `-s`.
228///
229/// # Errors
230///
231/// [`Error::SetShape`] without an `=`, [`Error::SetKey`] when the key isn't a dotted path.
232pub fn parse_set(s: &str) -> Result<(&str, &str)> {
233    let (k, v) = s.split_once('=').ok_or_else(|| Error::SetShape { set: s.to_owned() })?;
234    if k.is_empty() || k.split('.').any(str::is_empty) {
235        return Err(Error::SetKey { set: s.to_owned() });
236    }
237    Ok((k, v))
238}
239
240/// A value as YAML would read it (`3` a number, `true` a boolean, else text).
241#[must_use]
242pub fn scalar(text: &str) -> Value {
243    serde_yaml_ng::from_str(text).unwrap_or_else(|_| Value::String(text.to_owned()))
244}
245
246/// A nested mapping holding `value` at a dotted path.
247pub(crate) fn at_path(path: &str, value: Value) -> Value {
248    path.rsplit('.').fold(value, |inner, key| {
249        let mut m = Mapping::new();
250        m.insert(Value::String(key.to_owned()), inner);
251        Value::Mapping(m)
252    })
253}
254
255/// Deep-merges `from` into `into`: mappings key by key, anything else replaced.
256pub fn merge(into: &mut Value, from: &Value) {
257    match (into, from) {
258        (Value::Mapping(a), Value::Mapping(b)) => {
259            for (k, v) in b {
260                // A key both have merges (two mappings deeply, anything else replaced).
261                match a.get_mut(k) {
262                    Some(existing) => merge(existing, v),
263                    None => {
264                        a.insert(k.clone(), v.clone());
265                    }
266                }
267            }
268        }
269        (into, from) => *into = from.clone(),
270    }
271}
272
273/// Dotted paths of every leaf of a value.
274#[must_use]
275pub fn leaves(v: &Value, prefix: &str) -> Vec<String> {
276    match v {
277        Value::Mapping(m) if !m.is_empty() => m
278            .iter()
279            .flat_map(|(k, c)| {
280                let key = k.as_str().map_or_else(|| format!("{k:?}"), str::to_owned);
281                leaves(c, &if prefix.is_empty() { key } else { format!("{prefix}.{key}") })
282            })
283            .collect(),
284        _ => vec![prefix.to_owned()],
285    }
286}
287
288/// The spec's YAML with `-s key=value` overrides applied (keys without the `defaults.` prefix),
289/// before it is parsed: `machines.web.vm.os=ubuntu@24.04`, `networks.lab.internet=false`.
290///
291/// # Errors
292///
293/// As [`override_spec_with`].
294pub fn override_spec(yaml: &str, sets: &[String]) -> Result<String> {
295    override_spec_with(yaml, None, sets)
296}
297
298/// [`override_spec`] with `base` (a profile's `spec`) deep-merged first.
299///
300/// # Errors
301///
302/// [`Error::Yaml`] when the spec isn't YAML (or can't be written back), and the errors of
303/// [`parse_set`].
304pub fn override_spec_with(yaml: &str, base: Option<&Value>, sets: &[String]) -> Result<String> {
305    let mut doc: Value = serde_yaml_ng::from_str(yaml)?;
306    let mut any = false;
307    if let Some(b) = base.filter(|b| b.as_mapping().is_some_and(|m| !m.is_empty())) {
308        merge(&mut doc, b);
309        any = true;
310    }
311    for s in sets {
312        let (k, v) = parse_set(s)?;
313        if k.starts_with("defaults.") {
314            continue;
315        }
316        merge(&mut doc, &at_path(k, scalar(v)));
317        any = true;
318    }
319    if !any {
320        return Ok(yaml.to_owned());
321    }
322    Ok(serde_yaml_ng::to_string(&doc)?)
323}
324
325/// Applies the defaults that change generated files: each cloud module's region default.
326/// (Images are applied to the spec before generating; `vagrant.provider` at run time.)
327#[must_use]
328pub fn apply_to_files(files: Vec<GeneratedFile>, d: &Defaults) -> Vec<GeneratedFile> {
329    files
330        .into_iter()
331        .map(|mut f| {
332            // .isoloom[-n]/cloud-vm/<cloud>/main.tf, .isoloom[-n]/cloud-docker/<cloud>/main.tf
333            let parts: Vec<&str> = f.path.split('/').collect();
334            if let [_, target, cloud, "main.tf"] = parts.as_slice()
335                && target.starts_with("cloud-")
336                && let Some(region) = d.cloud.get(*cloud).and_then(|c| c.region.as_deref())
337            {
338                f.contents = set_region(&f.contents, region);
339            }
340            f
341        })
342        .collect()
343}
344
345/// The module text with its `variable "region"` default replaced.
346#[must_use]
347pub fn set_region(tf: &str, region: &str) -> String {
348    const VARIABLE: &str = "variable \"region\"";
349    let replaced = tf.split_once(VARIABLE).and_then(|(before, rest)| {
350        let (block, after) = rest.split_once("\n}")?;
351        let (head, from_default) = block.split_once("default")?;
352        let (to_quote, quoted) = from_default.split_once('"')?;
353        let (_, past_quote) = quoted.split_once('"')?;
354        Some(format!("{before}{VARIABLE}{head}default{to_quote}\"{region}\"{past_quote}\n}}{after}"))
355    });
356    replaced.unwrap_or_else(|| tf.to_owned())
357}
358
359/// What the built-in layer holds, for `isoloom defaults --system`: the clouds' regions (the
360/// image table is documented on its own page and read from the code).
361#[must_use]
362pub fn builtin() -> Value {
363    let mut cloud = Mapping::new();
364    for (c, r) in BUILTIN_REGIONS {
365        let mut m = Mapping::new();
366        m.insert(Value::String("region".into()), Value::String((*r).into()));
367        cloud.insert(Value::String((*c).into()), Value::Mapping(m));
368    }
369    let mut top = Mapping::new();
370    top.insert(Value::String("cloud".into()), Value::Mapping(cloud));
371    Value::Mapping(top)
372}
373
374#[cfg(test)]
375mod tests {
376    use std::ffi::OsString;
377
378    use super::*;
379    use crate::testing::folder;
380
381    /// A test's outcome: a failed step is its report.
382    type Outcome = std::result::Result<(), Box<dyn std::error::Error>>;
383
384    /// A host with these environment variables.
385    fn host(vars: &[(&str, &str)]) -> Host {
386        Host::new(vars.iter().map(|(k, v)| (OsString::from(k), OsString::from(v))))
387    }
388
389    /// The region `d` gives `cloud`.
390    fn region<'d>(d: &'d Defaults, cloud: &str) -> Option<&'d str> {
391        d.cloud.get(cloud)?.region.as_deref()
392    }
393
394    #[test]
395    fn the_users_file_is_in_isoloom_home() {
396        assert_eq!(user_file(&host(&[("ISOLOOM_HOME", "/srv/iso")])), PathBuf::from("/srv/iso/defaults.yml"));
397    }
398
399    #[test]
400    fn layers_merge_in_order_and_each_leaf_names_the_last_that_set_it() -> Outcome {
401        let home = folder("defaults-layers-home")?;
402        let project = folder("defaults-layers-project")?;
403        std::fs::write(home.join("defaults.yml"), "cloud: { aws: { region: from-user }, gcp: { region: from-user } }\n")?;
404        std::fs::write(project.join(PROJECT_FILE), "cloud: { aws: { region: from-project } }\n")?;
405        let home_text = home.to_string_lossy().into_owned();
406        let sets = ["defaults.vagrant.provider=libvirt".to_owned(), "machines.web.os=alpine".to_owned()];
407        let resolved = load(&Sources {
408            project: &project,
409            images_file: None,
410            profile: None,
411            sets: &sets,
412            host: host(&[("ISOLOOM_HOME", &home_text), ("ISOLOOM_CLOUD__AZURE__REGION", "from-env")]),
413        })?;
414        let d = &resolved.defaults;
415        assert_eq!(region(d, "aws"), Some("from-project"));
416        assert_eq!(region(d, "gcp"), Some("from-user"), "a file that sets one region leaves the others alone");
417        assert_eq!(region(d, "azure"), Some("from-env"));
418        assert_eq!(d.vagrant.provider.as_deref(), Some("libvirt"));
419        let user = format!("user file {}", home.join("defaults.yml").display());
420        let sources: Vec<(&str, &str)> = resolved.sources.iter().map(|(k, s)| (k.as_str(), s.as_str())).collect();
421        assert_eq!(
422            sources,
423            [
424                ("cloud.gcp.region", user.as_str()),
425                ("cloud.aws.region", &format!("project file {}", project.join(PROJECT_FILE).display())),
426                ("cloud.azure.region", "environment ISOLOOM_CLOUD__AZURE__REGION"),
427                ("vagrant.provider", "-s defaults.vagrant.provider=libvirt"),
428            ],
429            "a `-s` without the `defaults.` prefix is the spec's, not a layer"
430        );
431        Ok(())
432    }
433
434    #[test]
435    fn a_layer_file_that_cant_be_read_is_an_error_unlike_a_missing_one() -> Outcome {
436        let project = folder("defaults-unreadable")?;
437        let sources = |folder| Sources {
438            project: folder,
439            images_file: None,
440            profile: None,
441            sets: &[],
442            host: host(&[("ISOLOOM_HOME", "/nonexistent/isoloom")]),
443        };
444        assert_eq!(load(&sources(&project))?.defaults, Defaults::default(), "no file, no layer");
445        std::fs::create_dir(project.join(PROJECT_FILE))?;
446        assert!(matches!(load(&sources(&project)), Err(Error::Read { path, .. }) if path == project.join(PROJECT_FILE)));
447        Ok(())
448    }
449
450    #[test]
451    fn only_the_defaults_sections_are_read_from_the_environment() {
452        let vars = [
453            ("ISOLOOM_VAGRANT__PROVIDER", "qemu"),
454            ("ISOLOOM_HOME", "/x"),
455            ("PATH", "/bin"),
456            ("ISOLOOM_IMAGES__ACCESS__VM", "kali"),
457        ];
458        assert_eq!(
459            from_env(vars.iter().map(|(k, v)| ((*k).to_owned(), (*v).to_owned()))),
460            [
461                ("images.access.vm".to_owned(), "kali".to_owned()),
462                ("vagrant.provider".to_owned(), "qemu".to_owned())
463            ]
464        );
465    }
466
467    #[test]
468    fn a_set_is_a_dotted_key_then_its_value() {
469        assert!(matches!(parse_set("a.b=c=d"), Ok(("a.b", "c=d"))));
470        assert!(matches!(parse_set("a.b"), Err(Error::SetShape { .. })));
471        for s in ["=1", "a..b=1", ".a=1"] {
472            assert!(matches!(parse_set(s), Err(Error::SetKey { .. })), "{s}");
473        }
474    }
475
476    #[test]
477    fn an_empty_mapping_is_a_leaf_of_its_own() -> Outcome {
478        let v: Value = serde_yaml_ng::from_str("cloud: { aws: {} }\nvagrant: { provider: qemu }\n")?;
479        assert_eq!(leaves(&v, ""), ["cloud.aws", "vagrant.provider"]);
480        Ok(())
481    }
482
483    /// A cloud module whose region defaults to `region`, between two other variables.
484    fn module(region: &str) -> String {
485        format!(
486            "variable \"zone\" {{\n  default = \"a\"\n}}\nvariable \"region\" {{\n  type    = string\n  default = \"{region}\"\n}}\nvariable \"size\" {{\n  default = \"s\"\n}}\n"
487        )
488    }
489
490    #[test]
491    fn a_region_replaces_only_the_region_variables_default() {
492        assert_eq!(set_region(&module("eu-west-3"), "eu-north-1"), module("eu-north-1"));
493        assert_eq!(
494            set_region("variable \"zone\" {}\n", "eu-north-1"),
495            "variable \"zone\" {}\n",
496            "no region, no change"
497        );
498    }
499
500    #[test]
501    fn regions_reach_each_clouds_module_and_nothing_else() {
502        let file = |path: &str| GeneratedFile {
503            path: path.to_owned(),
504            contents: module("old"),
505        };
506        let d = Defaults {
507            cloud: [("aws".to_owned(), Cloud { region: Some("new".into()) })].into_iter().collect(),
508            ..Defaults::default()
509        };
510        let paths = [
511            ".isoloom/cloud-vm/aws/main.tf",
512            ".isoloom-2/cloud-docker/aws/main.tf",
513            ".isoloom/cloud-vm/gcp/main.tf",
514            ".isoloom/cloud-vm/aws/variables.tf",
515            ".isoloom/vagrant/aws/main.tf",
516            ".isoloom/cloud-vm/aws/extra/main.tf",
517        ];
518        let regions: Vec<bool> = apply_to_files(paths.iter().map(|p| file(p)).collect(), &d)
519            .iter()
520            .map(|f| f.contents == module("new"))
521            .collect();
522        assert_eq!(regions, [true, true, false, false, false, false]);
523    }
524
525    #[test]
526    fn the_built_in_layer_holds_every_clouds_region() -> Outcome {
527        let d: Defaults = serde_yaml_ng::from_value(builtin())?;
528        let regions: Vec<(&str, &str)> = d.cloud.iter().filter_map(|(c, r)| Some((c.as_str(), r.region.as_deref()?))).collect();
529        assert_eq!(regions, BUILTIN_REGIONS);
530        Ok(())
531    }
532}
isoloom_spec/defaults.rs

isoloom_spec/
defaults.rs
