jan_cli/
spec_load.rs

1//! YAML spec loading with `include` links and host-OS filtering.
2
3use std::collections::{BTreeMap, HashSet};
4use std::path::{Path, PathBuf};
5
6use anyhow::{bail, Context, Result};
7use serde::Deserialize;
8use std::borrow::Cow;
9
10use crate::inputs::InputDef;
11use crate::remote::{self, FetchOpts};
12use crate::{
13    deserialize_string_or_seq, CommandNode, EnvSpec, ExecSpec, IncludeLink, IncludeLinkKind,
14    IncludeRef, LocalInclude, Metadata, RemoteInclude, RootSpec,
15};
16
17/// Which platform string to use when filtering `os:` lists on commands.
18#[derive(Clone, Debug, PartialEq, Eq)]
19pub struct HostPlatform {
20    /// Normalized id: `linux`, `macos`, `windows`, or another `std::env::consts::OS` value.
21    pub id: Cow<'static, str>,
22}
23
24impl HostPlatform {
25    /// Resolve the process host, honoring `JAN_OS` when set (for tests and overrides).
26    pub fn detect() -> Self {
27        if let Ok(v) = std::env::var("JAN_OS") {
28            let s = v.trim().to_ascii_lowercase();
29            if !s.is_empty() {
30                return Self::from_normalized(&s);
31            }
32        }
33        Self::from_normalized(std::env::consts::OS)
34    }
35
36    fn from_normalized(os: &str) -> Self {
37        let id = match os {
38            "darwin" | "macos" => Cow::Borrowed("macos"),
39            "linux" => Cow::Borrowed("linux"),
40            "windows" => Cow::Borrowed("windows"),
41            other => Cow::Owned(other.to_string()),
42        };
43        Self { id }
44    }
45}
46
47fn normalize_os_token(tok: &str) -> String {
48    match tok.trim().to_ascii_lowercase().as_str() {
49        "darwin" => "macos".to_string(),
50        s => s.to_string(),
51    }
52}
53
54fn node_visible_for_platform(os_list: &[String], platform: &str) -> bool {
55    if os_list.is_empty() {
56        return true;
57    }
58    os_list.iter().any(|o| normalize_os_token(o) == platform)
59}
60
61#[derive(Debug, Deserialize)]
62struct RawRootSpec {
63    metadata: Option<Metadata>,
64    #[serde(default)]
65    include: Vec<IncludeRef>,
66    #[serde(default)]
67    commands: BTreeMap<String, RawCommandNode>,
68}
69
70#[derive(Debug, Deserialize)]
71struct RawCommandNode {
72    #[serde(default)]
73    os: Vec<String>,
74    #[serde(default)]
75    about: String,
76    path: Option<String>,
77    #[serde(default)]
78    dependencies: Vec<String>,
79    #[serde(default)]
80    requires: Vec<String>,
81    #[serde(default)]
82    env: EnvSpec,
83    #[serde(default)]
84    inputs: BTreeMap<String, InputDef>,
85    #[serde(default, deserialize_with = "deserialize_string_or_seq")]
86    cron: Vec<String>,
87    #[serde(default)]
88    packages: crate::PackagesSpec,
89    include: Option<IncludeRef>,
90    #[serde(default)]
91    commands: BTreeMap<String, RawCommandNode>,
92    exec: Option<ExecSpec>,
93}
94
95#[derive(Clone)]
96struct LoadCtx {
97    /// Canonical root of the directory selected by `jan use`.
98    ///
99    /// Local includes are interpreted relative to this directory. Canonicalization
100    /// also prevents symlink escapes.
101    use_root: PathBuf,
102}
103
104impl LoadCtx {
105    fn read_local_bytes(&self, local: &LocalInclude) -> Result<(PathBuf, Vec<u8>)> {
106        let path = resolve_under_use_root(&self.use_root, &local.path)?;
107        let bytes = std::fs::read(&path)
108            .with_context(|| format!("read included file {}", path.display()))?;
109        if let Some(hash) = local
110            .sha256
111            .as_deref()
112            .map(str::trim)
113            .filter(|s| !s.is_empty())
114        {
115            let expected = remote::normalize_sha256(hash)?;
116            let got = remote::sha256_hex(&bytes);
117            if got != expected {
118                bail!(
119                    "SHA256 mismatch for include `{}`: expected {expected}, got {got}",
120                    local.path
121                );
122            }
123        }
124        Ok((path, bytes))
125    }
126
127    fn visit_token(&self, inc: &IncludeRef) -> Result<String> {
128        match inc {
129            IncludeRef::Local(local) => {
130                let p = resolve_under_use_root(&self.use_root, &local.path)?;
131                Ok(p.to_string_lossy().to_string())
132            }
133            IncludeRef::Remote(_) => Ok(inc.cycle_token()),
134        }
135    }
136}
137
138fn fetch_remote_include(r: &RemoteInclude) -> Result<String> {
139    let mut opts = FetchOpts::new();
140    if let Some(ttl) = r.ttl {
141        opts = opts.with_ttl(ttl);
142    }
143    remote::fetch_verified_text(&r.url, &r.sha256, &opts)
144        .with_context(|| format!("fetch remote include {}", r.url))
145}
146
147/// Resolve `rel` under `use_root`, rejecting escapes. Allows files or directories.
148pub(crate) fn resolve_under_use_root_any(use_root: &Path, rel: &str) -> Result<PathBuf> {
149    let rel = rel.trim();
150    if rel.is_empty() {
151        bail!("empty path");
152    }
153    let p = Path::new(rel);
154    if p.is_absolute() {
155        bail!("path must be relative to the jan use root: {rel}");
156    }
157    if p.components()
158        .any(|c| matches!(c, std::path::Component::ParentDir))
159    {
160        bail!("path must not contain `..`: {rel}");
161    }
162
163    let full = use_root.join(p);
164    let resolved = full
165        .canonicalize()
166        .with_context(|| format!("path not found: {}", full.display()))?;
167    if !resolved.starts_with(use_root) {
168        bail!(
169            "path escapes jan use root: {} (root: {})",
170            resolved.display(),
171            use_root.display()
172        );
173    }
174    Ok(resolved)
175}
176
177/// Resolve `rel` under `use_root`, rejecting escapes and non-files.
178pub(crate) fn resolve_under_use_root(use_root: &Path, rel: &str) -> Result<PathBuf> {
179    let resolved = resolve_under_use_root_any(use_root, rel)?;
180    if !resolved.is_file() {
181        bail!("include is not a file: {}", resolved.display());
182    }
183    Ok(resolved)
184}
185
186fn include_link_for(inc: &IncludeRef, kind: IncludeLinkKind) -> IncludeLink {
187    match inc {
188        IncludeRef::Local(local) => IncludeLink {
189            kind,
190            path: Some(local.path.clone()),
191            url: None,
192            sha256: local.sha256.clone(),
193        },
194        IncludeRef::Remote(r) => IncludeLink {
195            kind,
196            path: None,
197            url: Some(r.url.clone()),
198            sha256: Some(r.sha256.clone()),
199        },
200    }
201}
202
203fn apply_wrapper_overlays(raw: &mut RawCommandNode, node: &mut CommandNode) -> Result<()> {
204    node.os = merge_os_filters(&raw.os, &node.os)?;
205    node.about = overlay_about(&raw.about, std::mem::take(&mut node.about));
206    if !raw.cron.is_empty() {
207        node.cron = std::mem::take(&mut raw.cron);
208    }
209    if !raw.env.is_empty() {
210        node.env.merge_from(std::mem::take(&mut raw.env));
211    }
212    for (k, v) in std::mem::take(&mut raw.inputs) {
213        node.inputs.insert(k, v);
214    }
215    if raw.path.is_some() {
216        node.path = raw.path.take();
217    }
218    if !raw.dependencies.is_empty() {
219        node.dependencies = std::mem::take(&mut raw.dependencies);
220    }
221    if !raw.requires.is_empty() {
222        node.requires = std::mem::take(&mut raw.requires);
223    }
224    if !raw.packages.is_empty() {
225        node.packages.merge_from(std::mem::take(&mut raw.packages));
226    }
227    Ok(())
228}
229
230fn merge_os_filters(outer: &[String], inner: &[String]) -> Result<Vec<String>> {
231    match (outer.is_empty(), inner.is_empty()) {
232        (true, true) => Ok(vec![]),
233        (true, false) => Ok(inner.to_vec()),
234        (false, true) => Ok(outer.to_vec()),
235        (false, false) => {
236            let merged: Vec<String> = outer
237                .iter()
238                .filter(|o| {
239                    let n = normalize_os_token(o);
240                    inner.iter().any(|i| normalize_os_token(i) == n)
241                })
242                .cloned()
243                .collect();
244            if merged.is_empty() {
245                bail!(
246                    "conflicting `os:` filters between include wrapper and included file \
247                     (no platform appears in both lists)"
248                );
249            }
250            Ok(merged)
251        }
252    }
253}
254
255fn overlay_about(overlay: &str, base: String) -> String {
256    let o = overlay.trim();
257    if o.is_empty() {
258        base
259    } else {
260        o.to_string()
261    }
262}
263
264fn resolve_raw_command_node(
265    raw: RawCommandNode,
266    ctx: &LoadCtx,
267    visited: &mut HashSet<String>,
268) -> Result<CommandNode> {
269    if raw.include.is_some() && (raw.exec.is_some() || !raw.commands.is_empty()) {
270        bail!("command with `include` cannot also define `exec` or nested `commands` in the same YAML map");
271    }
272
273    let mut raw = raw;
274    if let Some(inc) = raw.include.take() {
275        let token = ctx.visit_token(&inc)?;
276        if !visited.insert(token.clone()) {
277            bail!("include cycle detected at `{token}`");
278        }
279        let label = inc.cycle_token();
280        let mut node = match &inc {
281            IncludeRef::Local(local) if !local.is_yaml() => {
282                if local.path.trim().is_empty() {
283                    bail!("empty include path");
284                }
285                let (_path, _bytes) = ctx.read_local_bytes(local)?;
286                CommandNode {
287                    about: String::new(),
288                    exec: Some(ExecSpec {
289                        argv: local.argv.clone(),
290                        passthrough: local.passthrough,
291                        file: Some(local.path.clone()),
292                        sha256: local.sha256.clone(),
293                        url: None,
294                        ttl: None,
295                    }),
296                    source: Some(include_link_for(&inc, IncludeLinkKind::Script)),
297                    ..Default::default()
298                }
299            }
300            IncludeRef::Local(local) => {
301                if !local.argv.is_empty() || local.passthrough {
302                    bail!(
303                        "include `{label}`: `argv` / `passthrough` are only valid for script files, not YAML"
304                    );
305                }
306                let (_path, bytes) = ctx.read_local_bytes(local)?;
307                let text = String::from_utf8(bytes)
308                    .with_context(|| format!("include `{label}` is not valid UTF-8"))?;
309                let inner: RawCommandNode = serde_yaml::from_str(&text)
310                    .with_context(|| format!("parse include `{label}`"))?;
311                let mut node = resolve_raw_command_node(inner, ctx, visited)?;
312                node.source = Some(include_link_for(&inc, IncludeLinkKind::Yaml));
313                node
314            }
315            IncludeRef::Remote(r) => {
316                let text = fetch_remote_include(r)?;
317                let inner: RawCommandNode = serde_yaml::from_str(&text)
318                    .with_context(|| format!("parse include `{label}`"))?;
319                let mut node = resolve_raw_command_node(inner, ctx, visited)?;
320                node.source = Some(include_link_for(&inc, IncludeLinkKind::Yaml));
321                node
322            }
323        };
324        visited.remove(&token);
325        apply_wrapper_overlays(&mut raw, &mut node)?;
326        return Ok(node);
327    }
328
329    let mut commands = BTreeMap::new();
330    for (name, child) in raw.commands {
331        commands.insert(name, resolve_raw_command_node(child, ctx, visited)?);
332    }
333
334    Ok(CommandNode {
335        os: raw.os,
336        about: raw.about,
337        path: raw.path,
338        dependencies: raw.dependencies,
339        requires: raw.requires,
340        env: raw.env,
341        inputs: raw.inputs,
342        cron: raw.cron,
343        packages: raw.packages,
344        commands,
345        exec: raw.exec,
346        source: None,
347    })
348}
349
350fn merge_root_includes(mut root: RawRootSpec, use_root: &Path) -> Result<RawRootSpec> {
351    let mut merged = BTreeMap::new();
352    for inc in &root.include {
353        let text = match inc {
354            IncludeRef::Local(local) => {
355                if !local.is_yaml() {
356                    bail!(
357                        "root-level include `{}` must be a YAML file (.yaml / .yml)",
358                        local.path
359                    );
360                }
361                if !local.argv.is_empty() || local.passthrough {
362                    bail!(
363                        "root-level include `{}`: `argv` / `passthrough` are not valid here",
364                        local.path
365                    );
366                }
367                let path = resolve_under_use_root(use_root, &local.path)?;
368                let bytes = std::fs::read(&path)
369                    .with_context(|| format!("read root include {}", path.display()))?;
370                if let Some(hash) = local
371                    .sha256
372                    .as_deref()
373                    .map(str::trim)
374                    .filter(|s| !s.is_empty())
375                {
376                    let expected = remote::normalize_sha256(hash)?;
377                    let got = remote::sha256_hex(&bytes);
378                    if got != expected {
379                        bail!(
380                            "SHA256 mismatch for root include `{}`: expected {expected}, got {got}",
381                            local.path
382                        );
383                    }
384                }
385                String::from_utf8(bytes).with_context(|| {
386                    format!("root include {} is not valid UTF-8", path.display())
387                })?
388            }
389            IncludeRef::Remote(r) => fetch_remote_include(r)?,
390        };
391        let label = inc.cycle_token();
392        let fragment: RawRootSpec =
393            serde_yaml::from_str(&text).with_context(|| format!("parse {label}"))?;
394        let mut expanded = merge_root_includes(fragment, use_root)?;
395        merged.append(&mut expanded.commands);
396    }
397    merged.append(&mut root.commands);
398    root.commands = merged;
399    root.include.clear();
400    Ok(root)
401}
402
403fn materialize_root(raw: RawRootSpec, ctx: &LoadCtx) -> Result<RootSpec> {
404    let mut visited = HashSet::new();
405    let mut commands = BTreeMap::new();
406    for (name, node) in raw.commands {
407        commands.insert(name, resolve_raw_command_node(node, ctx, &mut visited)?);
408    }
409    Ok(RootSpec {
410        metadata: raw.metadata,
411        commands,
412    })
413}
414
415fn validate_root(spec: &RootSpec) -> Result<()> {
416    for (name, node) in &spec.commands {
417        node.validate(name)?;
418    }
419    Ok(())
420}
421
422pub fn load_spec_from_path(spec_path: &Path, platform: HostPlatform) -> Result<RootSpec> {
423    let spec_path = spec_path
424        .canonicalize()
425        .with_context(|| format!("canonicalize spec file {}", spec_path.display()))?;
426    let text = std::fs::read_to_string(&spec_path)
427        .with_context(|| format!("read spec file {}", spec_path.display()))?;
428    let raw: RawRootSpec = serde_yaml::from_str(&text).context("parse YAML spec")?;
429    let use_root = spec_path
430        .parent()
431        .unwrap_or_else(|| Path::new("."))
432        .to_path_buf();
433    let raw = merge_root_includes(raw, &use_root)?;
434    let ctx = LoadCtx { use_root };
435    let mut spec = materialize_root(raw, &ctx)?;
436    // Validate structure before OS filtering: filtering can drop nested nodes (e.g. empty
437    // placeholders) and would otherwise hide invalid `exec` + `commands` combinations.
438    validate_root(&spec)?;
439    filter_spec_for_platform(&mut spec, platform.id.as_ref());
440    Ok(spec)
441}
442
443/// Parse an in-memory spec. Every local `include` resolves relative to `use_root`.
444/// Pass `None` only when the document has no root `include` keys.
445pub fn load_spec_from_str(
446    raw: &str,
447    use_root: Option<&Path>,
448    platform: HostPlatform,
449) -> Result<RootSpec> {
450    let raw: RawRootSpec = serde_yaml::from_str(raw).context("parse YAML spec")?;
451    let has_local_root_include = raw
452        .include
453        .iter()
454        .any(|i| matches!(i, IncludeRef::Local(_)));
455    let canonical_root = use_root
456        .map(|root| {
457            root.canonicalize()
458                .with_context(|| format!("canonicalize jan use root {}", root.display()))
459        })
460        .transpose()?;
461    let raw = if let Some(root) = canonical_root.as_deref() {
462        merge_root_includes(raw, root)?
463    } else {
464        if has_local_root_include {
465            bail!("root-level `include` requires a base directory (pass when calling load_spec_from_str)");
466        }
467        // Remote-only root includes can merge without a local base.
468        if !raw.include.is_empty() {
469            merge_root_includes(raw, Path::new("."))?
470        } else {
471            raw
472        }
473    };
474    let ctx = LoadCtx {
475        use_root: canonical_root.unwrap_or_else(|| PathBuf::from(".")),
476    };
477    let mut spec = materialize_root(raw, &ctx)?;
478    validate_root(&spec)?;
479    filter_spec_for_platform(&mut spec, platform.id.as_ref());
480    Ok(spec)
481}
482
483pub fn filter_spec_for_platform(spec: &mut RootSpec, platform_id: &str) {
484    filter_command_map(&mut spec.commands, platform_id);
485}
486
487fn filter_command_map(map: &mut BTreeMap<String, CommandNode>, platform_id: &str) {
488    map.retain(|_, node| {
489        if !node_visible_for_platform(&node.os, platform_id) {
490            return false;
491        }
492        filter_command_map(&mut node.commands, platform_id);
493        if node.exec.is_some() {
494            return true;
495        }
496        !node.commands.is_empty()
497    });
498}
499
500#[cfg(test)]
501mod tests {
502    use super::*;
503    use crate::{ExecSpec, LocalInclude};
504    use std::fs;
505
506    #[test]
507    fn os_filter_drops_linux_only_branch() {
508        let mut spec = RootSpec {
509            metadata: None,
510            commands: BTreeMap::from([(
511                "sys".into(),
512                CommandNode {
513                    os: vec!["linux".into()],
514                    about: "linux".into(),
515                    commands: BTreeMap::from([(
516                        "ports".into(),
517                        CommandNode {
518                            exec: Some(ExecSpec {
519                                argv: vec!["echo".into(), "x".into()],
520                                passthrough: false,
521                                url: None,
522                                file: None,
523                                sha256: None,
524                                ttl: None,
525                            }),
526                            ..Default::default()
527                        },
528                    )]),
529                    ..Default::default()
530                },
531            )]),
532        };
533        filter_spec_for_platform(&mut spec, "macos");
534        assert!(spec.commands.is_empty());
535    }
536
537    #[test]
538    fn nested_include_resolves_from_use_root() {
539        let tmp = tempfile::tempdir().unwrap();
540        let root = tmp.path();
541        fs::create_dir(root.join("sub")).unwrap();
542        fs::write(
543            root.join("leaf.yaml"),
544            "about: root leaf\nexec:\n  argv: [\"echo\", \"root\"]\n",
545        )
546        .unwrap();
547        fs::write(root.join("sub/outer.yaml"), "include: leaf.yaml\n").unwrap();
548        fs::write(
549            root.join("scripts.spec.yaml"),
550            "commands:\n  outer:\n    include: sub/outer.yaml\n",
551        )
552        .unwrap();
553
554        let spec =
555            load_spec_from_path(&root.join("scripts.spec.yaml"), HostPlatform::detect()).unwrap();
556        assert_eq!(
557            spec.commands["outer"].exec.as_ref().unwrap().argv,
558            vec!["echo", "root"]
559        );
560    }
561
562    #[test]
563    fn include_rejects_absolute_and_parent_paths() {
564        let tmp = tempfile::tempdir().unwrap();
565        let root = tmp.path().join("tree");
566        fs::create_dir(&root).unwrap();
567        let outside = tmp.path().join("outside.yaml");
568        fs::write(
569            &outside,
570            "about: outside\nexec:\n  argv: [\"echo\", \"outside\"]\n",
571        )
572        .unwrap();
573
574        for include in [
575            outside.to_string_lossy().into_owned(),
576            "../outside.yaml".to_string(),
577        ] {
578            fs::write(
579                root.join("scripts.spec.yaml"),
580                format!("commands:\n  escaped:\n    include: {include:?}\n"),
581            )
582            .unwrap();
583            let err = load_spec_from_path(&root.join("scripts.spec.yaml"), HostPlatform::detect())
584                .unwrap_err();
585            assert!(
586                err.to_string().contains("must be relative")
587                    || err.to_string().contains("must not contain `..`"),
588                "{err:#}"
589            );
590        }
591    }
592
593    #[cfg(unix)]
594    #[test]
595    fn include_rejects_symlink_escape() {
596        use std::os::unix::fs::symlink;
597
598        let tmp = tempfile::tempdir().unwrap();
599        let root = tmp.path().join("tree");
600        fs::create_dir(&root).unwrap();
601        let outside = tmp.path().join("outside.yaml");
602        fs::write(
603            &outside,
604            "about: outside\nexec:\n  argv: [\"echo\", \"outside\"]\n",
605        )
606        .unwrap();
607        symlink(&outside, root.join("linked.yaml")).unwrap();
608        fs::write(
609            root.join("scripts.spec.yaml"),
610            "commands:\n  escaped:\n    include: linked.yaml\n",
611        )
612        .unwrap();
613
614        let err = load_spec_from_path(&root.join("scripts.spec.yaml"), HostPlatform::detect())
615            .unwrap_err();
616        assert!(err.to_string().contains("escapes jan use root"), "{err:#}");
617    }
618
619    #[test]
620    fn include_ref_deserializes_local_and_remote() {
621        let local: IncludeRef = serde_yaml::from_str("sub/a.yaml").unwrap();
622        assert_eq!(
623            local,
624            IncludeRef::Local(LocalInclude::from_path("sub/a.yaml"))
625        );
626        let local_map: IncludeRef = serde_yaml::from_str(
627            "path: scripts/x.sh\nsha256: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\nargv: [bash]\npassthrough: true\n",
628        )
629        .unwrap();
630        match local_map {
631            IncludeRef::Local(l) => {
632                assert_eq!(l.path, "scripts/x.sh");
633                assert!(l.passthrough);
634                assert_eq!(l.argv, vec!["bash"]);
635                assert!(l.sha256.is_some());
636            }
637            _ => panic!("expected local"),
638        }
639        let remote: IncludeRef = serde_yaml::from_str(
640            "url: https://example.com/a.yaml\nsha256: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n",
641        )
642        .unwrap();
643        assert!(remote.is_remote());
644    }
645
646    #[test]
647    fn exec_remote_requires_sha256() {
648        let node = CommandNode {
649            exec: Some(ExecSpec {
650                argv: vec![],
651                passthrough: false,
652                url: Some("https://example.com/x.sh".into()),
653                file: None,
654                sha256: None,
655                ttl: None,
656            }),
657            ..Default::default()
658        };
659        assert!(node.validate("x").is_err());
660    }
661}
jan_cli/spec_load.rs

jan_cli/
spec_load.rs
