jan_cli/
spec_load.rs

1//! YAML spec loading with `include` links and host-OS filtering.
2
3use std::collections::{BTreeMap, HashSet};
4use std::path::{Path, PathBuf};
5
6use anyhow::{bail, Context, Result};
7use serde::Deserialize;
8use std::borrow::Cow;
9
10use crate::inputs::InputDef;
11use crate::remote::{self, FetchOpts};
12use crate::{
13    deserialize_string_or_seq, CommandNode, EnvSpec, ExecSpec, IncludeLink, IncludeLinkKind,
14    IncludeRef, LocalInclude, Metadata, RemoteInclude, RootSpec,
15};
16
17/// Which platform string to use when filtering `os:` lists on commands.
18#[derive(Clone, Debug, PartialEq, Eq)]
19pub struct HostPlatform {
20    /// Normalized id: `linux`, `macos`, `windows`, or another `std::env::consts::OS` value.
21    pub id: Cow<'static, str>,
22}
23
24impl HostPlatform {
25    /// Resolve the process host, honoring `JAN_OS` when set (for tests and overrides).
26    pub fn detect() -> Self {
27        if let Ok(v) = std::env::var("JAN_OS") {
28            let s = v.trim().to_ascii_lowercase();
29            if !s.is_empty() {
30                return Self::from_normalized(&s);
31            }
32        }
33        Self::from_normalized(std::env::consts::OS)
34    }
35
36    fn from_normalized(os: &str) -> Self {
37        let id = match os {
38            "darwin" | "macos" => Cow::Borrowed("macos"),
39            "linux" => Cow::Borrowed("linux"),
40            "windows" => Cow::Borrowed("windows"),
41            other => Cow::Owned(other.to_string()),
42        };
43        Self { id }
44    }
45}
46
47fn normalize_os_token(tok: &str) -> String {
48    match tok.trim().to_ascii_lowercase().as_str() {
49        "darwin" => "macos".to_string(),
50        s => s.to_string(),
51    }
52}
53
54fn node_visible_for_platform(os_list: &[String], platform: &str) -> bool {
55    if os_list.is_empty() {
56        return true;
57    }
58    os_list.iter().any(|o| normalize_os_token(o) == platform)
59}
60
61#[derive(Debug, Deserialize)]
62struct RawRootSpec {
63    metadata: Option<Metadata>,
64    #[serde(default)]
65    include: Vec<IncludeRef>,
66    #[serde(default)]
67    commands: BTreeMap<String, RawCommandNode>,
68}
69
70#[derive(Debug, Deserialize)]
71struct RawCommandNode {
72    #[serde(default)]
73    os: Vec<String>,
74    #[serde(default)]
75    about: String,
76    path: Option<String>,
77    #[serde(default)]
78    dependencies: Vec<String>,
79    #[serde(default)]
80    requires: Vec<String>,
81    #[serde(default)]
82    env: EnvSpec,
83    #[serde(default)]
84    inputs: BTreeMap<String, InputDef>,
85    #[serde(default, deserialize_with = "deserialize_string_or_seq")]
86    cron: Vec<String>,
87    #[serde(default)]
88    packages: crate::PackagesSpec,
89    #[serde(default)]
90    tests: BTreeMap<String, crate::CommandTest>,
91    include: Option<IncludeRef>,
92    #[serde(default)]
93    commands: BTreeMap<String, RawCommandNode>,
94    exec: Option<ExecSpec>,
95}
96
97#[derive(Clone)]
98struct LoadCtx {
99    /// Canonical root of the directory selected by `jan use`.
100    ///
101    /// Local includes are interpreted relative to this directory. Canonicalization
102    /// also prevents symlink escapes.
103    use_root: PathBuf,
104}
105
106impl LoadCtx {
107    fn read_local_bytes(&self, local: &LocalInclude) -> Result<(PathBuf, Vec<u8>)> {
108        let path = resolve_under_use_root(&self.use_root, &local.path)?;
109        let bytes = std::fs::read(&path)
110            .with_context(|| format!("read included file {}", path.display()))?;
111        if let Some(hash) = local
112            .sha256
113            .as_deref()
114            .map(str::trim)
115            .filter(|s| !s.is_empty())
116        {
117            let expected = remote::normalize_sha256(hash)?;
118            let got = remote::sha256_hex(&bytes);
119            if got != expected {
120                bail!(
121                    "SHA256 mismatch for include `{}`: expected {expected}, got {got}",
122                    local.path
123                );
124            }
125        }
126        Ok((path, bytes))
127    }
128
129    fn visit_token(&self, inc: &IncludeRef) -> Result<String> {
130        match inc {
131            IncludeRef::Local(local) => {
132                let p = resolve_under_use_root(&self.use_root, &local.path)?;
133                Ok(p.to_string_lossy().to_string())
134            }
135            IncludeRef::Remote(_) => Ok(inc.cycle_token()),
136        }
137    }
138}
139
140fn fetch_remote_include(r: &RemoteInclude) -> Result<String> {
141    let mut opts = FetchOpts::new();
142    if let Some(ttl) = r.ttl {
143        opts = opts.with_ttl(ttl);
144    }
145    remote::fetch_verified_text(&r.url, &r.sha256, &opts)
146        .with_context(|| format!("fetch remote include {}", r.url))
147}
148
149/// Resolve `rel` under `use_root`, rejecting escapes. Allows files or directories.
150pub(crate) fn resolve_under_use_root_any(use_root: &Path, rel: &str) -> Result<PathBuf> {
151    let rel = rel.trim();
152    if rel.is_empty() {
153        bail!("empty path");
154    }
155    let p = Path::new(rel);
156    if p.is_absolute() {
157        bail!("path must be relative to the jan use root: {rel}");
158    }
159    if p.components()
160        .any(|c| matches!(c, std::path::Component::ParentDir))
161    {
162        bail!("path must not contain `..`: {rel}");
163    }
164
165    let full = use_root.join(p);
166    let resolved = full
167        .canonicalize()
168        .with_context(|| format!("path not found: {}", full.display()))?;
169    if !resolved.starts_with(use_root) {
170        bail!(
171            "path escapes jan use root: {} (root: {})",
172            resolved.display(),
173            use_root.display()
174        );
175    }
176    Ok(resolved)
177}
178
179/// Resolve `rel` under `use_root`, rejecting escapes and non-files.
180pub(crate) fn resolve_under_use_root(use_root: &Path, rel: &str) -> Result<PathBuf> {
181    let resolved = resolve_under_use_root_any(use_root, rel)?;
182    if !resolved.is_file() {
183        bail!("include is not a file: {}", resolved.display());
184    }
185    Ok(resolved)
186}
187
188fn include_link_for(inc: &IncludeRef, kind: IncludeLinkKind) -> IncludeLink {
189    match inc {
190        IncludeRef::Local(local) => IncludeLink {
191            kind,
192            path: Some(local.path.clone()),
193            url: None,
194            sha256: local.sha256.clone(),
195        },
196        IncludeRef::Remote(r) => IncludeLink {
197            kind,
198            path: None,
199            url: Some(r.url.clone()),
200            sha256: Some(r.sha256.clone()),
201        },
202    }
203}
204
205fn apply_wrapper_overlays(raw: &mut RawCommandNode, node: &mut CommandNode) -> Result<()> {
206    node.os = merge_os_filters(&raw.os, &node.os)?;
207    node.about = overlay_about(&raw.about, std::mem::take(&mut node.about));
208    if !raw.cron.is_empty() {
209        node.cron = std::mem::take(&mut raw.cron);
210    }
211    if !raw.env.is_empty() {
212        node.env.merge_from(std::mem::take(&mut raw.env));
213    }
214    for (k, v) in std::mem::take(&mut raw.inputs) {
215        node.inputs.insert(k, v);
216    }
217    if raw.path.is_some() {
218        node.path = raw.path.take();
219    }
220    if !raw.dependencies.is_empty() {
221        node.dependencies = std::mem::take(&mut raw.dependencies);
222    }
223    if !raw.requires.is_empty() {
224        node.requires = std::mem::take(&mut raw.requires);
225    }
226    if !raw.packages.is_empty() {
227        node.packages.merge_from(std::mem::take(&mut raw.packages));
228    }
229    for (k, v) in std::mem::take(&mut raw.tests) {
230        node.tests.insert(k, v);
231    }
232    Ok(())
233}
234
235fn merge_os_filters(outer: &[String], inner: &[String]) -> Result<Vec<String>> {
236    match (outer.is_empty(), inner.is_empty()) {
237        (true, true) => Ok(vec![]),
238        (true, false) => Ok(inner.to_vec()),
239        (false, true) => Ok(outer.to_vec()),
240        (false, false) => {
241            let merged: Vec<String> = outer
242                .iter()
243                .filter(|o| {
244                    let n = normalize_os_token(o);
245                    inner.iter().any(|i| normalize_os_token(i) == n)
246                })
247                .cloned()
248                .collect();
249            if merged.is_empty() {
250                bail!(
251                    "conflicting `os:` filters between include wrapper and included file \
252                     (no platform appears in both lists)"
253                );
254            }
255            Ok(merged)
256        }
257    }
258}
259
260fn overlay_about(overlay: &str, base: String) -> String {
261    let o = overlay.trim();
262    if o.is_empty() {
263        base
264    } else {
265        o.to_string()
266    }
267}
268
269fn resolve_raw_command_node(
270    raw: RawCommandNode,
271    ctx: &LoadCtx,
272    visited: &mut HashSet<String>,
273) -> Result<CommandNode> {
274    if raw.include.is_some() && (raw.exec.is_some() || !raw.commands.is_empty()) {
275        bail!("command with `include` cannot also define `exec` or nested `commands` in the same YAML map");
276    }
277
278    let mut raw = raw;
279    if let Some(inc) = raw.include.take() {
280        let token = ctx.visit_token(&inc)?;
281        if !visited.insert(token.clone()) {
282            bail!("include cycle detected at `{token}`");
283        }
284        let label = inc.cycle_token();
285        let mut node = match &inc {
286            IncludeRef::Local(local) if !local.is_yaml() => {
287                if local.path.trim().is_empty() {
288                    bail!("empty include path");
289                }
290                let (_path, _bytes) = ctx.read_local_bytes(local)?;
291                CommandNode {
292                    about: String::new(),
293                    exec: Some(ExecSpec {
294                        argv: local.argv.clone(),
295                        passthrough: local.passthrough,
296                        file: Some(local.path.clone()),
297                        sha256: local.sha256.clone(),
298                        ..Default::default()
299                    }),
300                    source: Some(include_link_for(&inc, IncludeLinkKind::Script)),
301                    ..Default::default()
302                }
303            }
304            IncludeRef::Local(local) => {
305                if !local.argv.is_empty() || local.passthrough {
306                    bail!(
307                        "include `{label}`: `argv` / `passthrough` are only valid for script files, not YAML"
308                    );
309                }
310                let (_path, bytes) = ctx.read_local_bytes(local)?;
311                let text = String::from_utf8(bytes)
312                    .with_context(|| format!("include `{label}` is not valid UTF-8"))?;
313                let inner: RawCommandNode = serde_yaml::from_str(&text)
314                    .with_context(|| format!("parse include `{label}`"))?;
315                let mut node = resolve_raw_command_node(inner, ctx, visited)?;
316                node.source = Some(include_link_for(&inc, IncludeLinkKind::Yaml));
317                node
318            }
319            IncludeRef::Remote(r) => {
320                let text = fetch_remote_include(r)?;
321                let inner: RawCommandNode = serde_yaml::from_str(&text)
322                    .with_context(|| format!("parse include `{label}`"))?;
323                let mut node = resolve_raw_command_node(inner, ctx, visited)?;
324                node.source = Some(include_link_for(&inc, IncludeLinkKind::Yaml));
325                node
326            }
327        };
328        visited.remove(&token);
329        apply_wrapper_overlays(&mut raw, &mut node)?;
330        return Ok(node);
331    }
332
333    let mut commands = BTreeMap::new();
334    for (name, child) in raw.commands {
335        commands.insert(name, resolve_raw_command_node(child, ctx, visited)?);
336    }
337
338    Ok(CommandNode {
339        os: raw.os,
340        about: raw.about,
341        path: raw.path,
342        dependencies: raw.dependencies,
343        requires: raw.requires,
344        env: raw.env,
345        inputs: raw.inputs,
346        cron: raw.cron,
347        packages: raw.packages,
348        tests: raw.tests,
349        commands,
350        exec: raw.exec,
351        source: None,
352    })
353}
354
355fn merge_root_includes(mut root: RawRootSpec, use_root: &Path) -> Result<RawRootSpec> {
356    let mut merged = BTreeMap::new();
357    for inc in &root.include {
358        let text = match inc {
359            IncludeRef::Local(local) => {
360                if !local.is_yaml() {
361                    bail!(
362                        "root-level include `{}` must be a YAML file (.yaml / .yml)",
363                        local.path
364                    );
365                }
366                if !local.argv.is_empty() || local.passthrough {
367                    bail!(
368                        "root-level include `{}`: `argv` / `passthrough` are not valid here",
369                        local.path
370                    );
371                }
372                let path = resolve_under_use_root(use_root, &local.path)?;
373                let bytes = std::fs::read(&path)
374                    .with_context(|| format!("read root include {}", path.display()))?;
375                if let Some(hash) = local
376                    .sha256
377                    .as_deref()
378                    .map(str::trim)
379                    .filter(|s| !s.is_empty())
380                {
381                    let expected = remote::normalize_sha256(hash)?;
382                    let got = remote::sha256_hex(&bytes);
383                    if got != expected {
384                        bail!(
385                            "SHA256 mismatch for root include `{}`: expected {expected}, got {got}",
386                            local.path
387                        );
388                    }
389                }
390                String::from_utf8(bytes).with_context(|| {
391                    format!("root include {} is not valid UTF-8", path.display())
392                })?
393            }
394            IncludeRef::Remote(r) => fetch_remote_include(r)?,
395        };
396        let label = inc.cycle_token();
397        let fragment: RawRootSpec =
398            serde_yaml::from_str(&text).with_context(|| format!("parse {label}"))?;
399        let mut expanded = merge_root_includes(fragment, use_root)?;
400        merged.append(&mut expanded.commands);
401    }
402    merged.append(&mut root.commands);
403    root.commands = merged;
404    root.include.clear();
405    Ok(root)
406}
407
408fn materialize_root(raw: RawRootSpec, ctx: &LoadCtx) -> Result<RootSpec> {
409    let mut visited = HashSet::new();
410    let mut commands = BTreeMap::new();
411    for (name, node) in raw.commands {
412        commands.insert(name, resolve_raw_command_node(node, ctx, &mut visited)?);
413    }
414    Ok(RootSpec {
415        metadata: raw.metadata,
416        commands,
417    })
418}
419
420fn validate_root(spec: &RootSpec) -> Result<()> {
421    for (name, node) in &spec.commands {
422        node.validate(name)?;
423    }
424    Ok(())
425}
426
427pub fn load_spec_from_path(spec_path: &Path, platform: HostPlatform) -> Result<RootSpec> {
428    let spec_path = spec_path
429        .canonicalize()
430        .with_context(|| format!("canonicalize spec file {}", spec_path.display()))?;
431    let text = std::fs::read_to_string(&spec_path)
432        .with_context(|| format!("read spec file {}", spec_path.display()))?;
433    let raw: RawRootSpec = serde_yaml::from_str(&text).context("parse YAML spec")?;
434    let use_root = spec_path
435        .parent()
436        .unwrap_or_else(|| Path::new("."))
437        .to_path_buf();
438    let raw = merge_root_includes(raw, &use_root)?;
439    let ctx = LoadCtx { use_root };
440    let mut spec = materialize_root(raw, &ctx)?;
441    // Validate structure before OS filtering: filtering can drop nested nodes (e.g. empty
442    // placeholders) and would otherwise hide invalid `exec` + `commands` combinations.
443    validate_root(&spec)?;
444    filter_spec_for_platform(&mut spec, platform.id.as_ref());
445    Ok(spec)
446}
447
448/// Parse an in-memory spec. Every local `include` resolves relative to `use_root`.
449/// Pass `None` only when the document has no root `include` keys.
450pub fn load_spec_from_str(
451    raw: &str,
452    use_root: Option<&Path>,
453    platform: HostPlatform,
454) -> Result<RootSpec> {
455    let raw: RawRootSpec = serde_yaml::from_str(raw).context("parse YAML spec")?;
456    let has_local_root_include = raw
457        .include
458        .iter()
459        .any(|i| matches!(i, IncludeRef::Local(_)));
460    let canonical_root = use_root
461        .map(|root| {
462            root.canonicalize()
463                .with_context(|| format!("canonicalize jan use root {}", root.display()))
464        })
465        .transpose()?;
466    let raw = if let Some(root) = canonical_root.as_deref() {
467        merge_root_includes(raw, root)?
468    } else {
469        if has_local_root_include {
470            bail!("root-level `include` requires a base directory (pass when calling load_spec_from_str)");
471        }
472        // Remote-only root includes can merge without a local base.
473        if !raw.include.is_empty() {
474            merge_root_includes(raw, Path::new("."))?
475        } else {
476            raw
477        }
478    };
479    let ctx = LoadCtx {
480        use_root: canonical_root.unwrap_or_else(|| PathBuf::from(".")),
481    };
482    let mut spec = materialize_root(raw, &ctx)?;
483    validate_root(&spec)?;
484    filter_spec_for_platform(&mut spec, platform.id.as_ref());
485    Ok(spec)
486}
487
488pub fn filter_spec_for_platform(spec: &mut RootSpec, platform_id: &str) {
489    filter_command_map(&mut spec.commands, platform_id);
490}
491
492fn filter_command_map(map: &mut BTreeMap<String, CommandNode>, platform_id: &str) {
493    map.retain(|_, node| {
494        if !node_visible_for_platform(&node.os, platform_id) {
495            return false;
496        }
497        filter_command_map(&mut node.commands, platform_id);
498        if node.exec.is_some() {
499            return true;
500        }
501        !node.commands.is_empty()
502    });
503}
504
505#[cfg(test)]
506mod tests {
507    use super::*;
508    use crate::{ExecSpec, LocalInclude};
509    use std::fs;
510
511    #[test]
512    fn os_filter_drops_linux_only_branch() {
513        let mut spec = RootSpec {
514            metadata: None,
515            commands: BTreeMap::from([(
516                "sys".into(),
517                CommandNode {
518                    os: vec!["linux".into()],
519                    about: "linux".into(),
520                    commands: BTreeMap::from([(
521                        "ports".into(),
522                        CommandNode {
523                            exec: Some(ExecSpec {
524                                argv: vec!["echo".into(), "x".into()],
525                                ..Default::default()
526                            }),
527                            ..Default::default()
528                        },
529                    )]),
530                    ..Default::default()
531                },
532            )]),
533        };
534        filter_spec_for_platform(&mut spec, "macos");
535        assert!(spec.commands.is_empty());
536    }
537
538    #[test]
539    fn nested_include_resolves_from_use_root() {
540        let tmp = tempfile::tempdir().unwrap();
541        let root = tmp.path();
542        fs::create_dir(root.join("sub")).unwrap();
543        fs::write(
544            root.join("leaf.yaml"),
545            "about: root leaf\nexec:\n  argv: [\"echo\", \"root\"]\n",
546        )
547        .unwrap();
548        fs::write(root.join("sub/outer.yaml"), "include: leaf.yaml\n").unwrap();
549        fs::write(
550            root.join("scripts.spec.yaml"),
551            "commands:\n  outer:\n    include: sub/outer.yaml\n",
552        )
553        .unwrap();
554
555        let spec =
556            load_spec_from_path(&root.join("scripts.spec.yaml"), HostPlatform::detect()).unwrap();
557        assert_eq!(
558            spec.commands["outer"].exec.as_ref().unwrap().argv,
559            vec!["echo", "root"]
560        );
561    }
562
563    #[test]
564    fn include_rejects_absolute_and_parent_paths() {
565        let tmp = tempfile::tempdir().unwrap();
566        let root = tmp.path().join("tree");
567        fs::create_dir(&root).unwrap();
568        let outside = tmp.path().join("outside.yaml");
569        fs::write(
570            &outside,
571            "about: outside\nexec:\n  argv: [\"echo\", \"outside\"]\n",
572        )
573        .unwrap();
574
575        for include in [
576            outside.to_string_lossy().into_owned(),
577            "../outside.yaml".to_string(),
578        ] {
579            fs::write(
580                root.join("scripts.spec.yaml"),
581                format!("commands:\n  escaped:\n    include: {include:?}\n"),
582            )
583            .unwrap();
584            let err = load_spec_from_path(&root.join("scripts.spec.yaml"), HostPlatform::detect())
585                .unwrap_err();
586            assert!(
587                err.to_string().contains("must be relative")
588                    || err.to_string().contains("must not contain `..`"),
589                "{err:#}"
590            );
591        }
592    }
593
594    #[cfg(unix)]
595    #[test]
596    fn include_rejects_symlink_escape() {
597        use std::os::unix::fs::symlink;
598
599        let tmp = tempfile::tempdir().unwrap();
600        let root = tmp.path().join("tree");
601        fs::create_dir(&root).unwrap();
602        let outside = tmp.path().join("outside.yaml");
603        fs::write(
604            &outside,
605            "about: outside\nexec:\n  argv: [\"echo\", \"outside\"]\n",
606        )
607        .unwrap();
608        symlink(&outside, root.join("linked.yaml")).unwrap();
609        fs::write(
610            root.join("scripts.spec.yaml"),
611            "commands:\n  escaped:\n    include: linked.yaml\n",
612        )
613        .unwrap();
614
615        let err = load_spec_from_path(&root.join("scripts.spec.yaml"), HostPlatform::detect())
616            .unwrap_err();
617        assert!(err.to_string().contains("escapes jan use root"), "{err:#}");
618    }
619
620    #[test]
621    fn include_ref_deserializes_local_and_remote() {
622        let local: IncludeRef = serde_yaml::from_str("sub/a.yaml").unwrap();
623        assert_eq!(
624            local,
625            IncludeRef::Local(LocalInclude::from_path("sub/a.yaml"))
626        );
627        let local_map: IncludeRef = serde_yaml::from_str(
628            "path: scripts/x.sh\nsha256: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\nargv: [bash]\npassthrough: true\n",
629        )
630        .unwrap();
631        match local_map {
632            IncludeRef::Local(l) => {
633                assert_eq!(l.path, "scripts/x.sh");
634                assert!(l.passthrough);
635                assert_eq!(l.argv, vec!["bash"]);
636                assert!(l.sha256.is_some());
637            }
638            _ => panic!("expected local"),
639        }
640        let remote: IncludeRef = serde_yaml::from_str(
641            "url: https://example.com/a.yaml\nsha256: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\n",
642        )
643        .unwrap();
644        assert!(remote.is_remote());
645    }
646
647    #[test]
648    fn exec_remote_requires_sha256() {
649        let node = CommandNode {
650            exec: Some(ExecSpec {
651                url: Some("https://example.com/x.sh".into()),
652                ..Default::default()
653            }),
654            ..Default::default()
655        };
656        assert!(node.validate("x").is_err());
657    }
658}
jan_cli/spec_load.rs

jan_cli/
spec_load.rs
