dre_core/
packages.rs

1//! Macro packages: reusable macros from another project (e.g. `dre_utils`), declared under
2//! `packages:` in the project's root `dependencies.yml` or `packages.yml`:
3//!
4//! ```yaml
5//! packages:
6//!   - git: https://github.com/acme/dre-finance-macros.git
7//!     revision: v2.3.0          # tag, branch or commit
8//!   - local: ../shared/macros   # a folder on disk, read in place
9//! ```
10//!
11//! A package is a folder with a `dre_package.yml` (`name:`) and a `macros/` folder. `dre deps`
12//! clones git packages into `dre_deps/packages/<name>` and pins the commit in `dre.lock`. Their
13//! macros are called through the package's name: `{{ dre_utils.star_except(...) }}`.
14
15use std::collections::BTreeMap;
16use std::path::{Path, PathBuf};
17use std::process::Command;
18
19use serde::{Deserialize, Serialize};
20use serde_yaml_ng::Value;
21
22use crate::diag::Diagnostics;
23use crate::lock::{Lock, LockedPackage};
24use crate::plugins::DEPS_DIR;
25use crate::yaml::YamlFile;
26
27/// Root files that may declare `packages:` (and plugins, like any YAML file).
28pub const DEPENDENCY_FILES: &[&str] = &["dependencies.yml", "packages.yml"];
29pub const PACKAGE_FILE: &str = "dre_package.yml";
30const INSTALLED_COMMIT: &str = ".dre_commit";
31
32#[derive(Debug, Clone, PartialEq, Eq)]
33pub enum Source {
34    Git { url: String, revision: String },
35    Local { path: String },
36}
37
38#[derive(Debug, Clone)]
39pub struct Declared {
40    pub source: Source,
41    /// The file declaring it, relative to the project root.
42    pub file: PathBuf,
43}
44
45/// An installed package, ready to use.
46#[derive(Debug, Clone, Serialize)]
47pub struct Package {
48    pub name: String,
49    #[serde(skip)]
50    pub dir: PathBuf,
51    /// Macro files, absolute.
52    #[serde(skip)]
53    pub macros: Vec<PathBuf>,
54}
55
56#[derive(Deserialize)]
57struct Manifest {
58    name: String,
59}
60
61/// Every `packages:` entry in the root dependency files. Problems go to `diags`.
62pub fn declared(root: &Path, diags: &mut Diagnostics) -> Vec<Declared> {
63    let mut out: Vec<Declared> = Vec::new();
64    for f in DEPENDENCY_FILES {
65        let path = root.join(f);
66        if !path.is_file() {
67            continue;
68        }
69        let mut quiet = Diagnostics::default();
70        let Some(yf) = YamlFile::load(&path, PathBuf::from(f), &mut quiet) else {
71            continue; // the project load reports the parse error
72        };
73        let Some(list) = yf.value.get("packages") else {
74            continue;
75        };
76        let Some(list) = list.as_sequence() else {
77            diags.error(
78                "invalid-packages",
79                Some(yf.display.clone()),
80                yf.line_of("packages", None),
81                "`packages` must be a list",
82            );
83            continue;
84        };
85        for entry in list {
86            let line = yf.line_of("packages", None);
87            let get = |k: &str| entry.get(k).and_then(Value::as_str).map(str::to_string);
88            let source = match (get("git"), get("local"), get("package")) {
89                (Some(url), None, None) => match get("revision") {
90                    Some(revision) => Source::Git { url, revision },
91                    None => {
92                        diags.error(
93                            "invalid-packages",
94                            Some(yf.display.clone()),
95                            line,
96                            format!("git package `{url}` needs a `revision` (a tag, branch or commit)"),
97                        );
98                        continue;
99                    }
100                },
101                (None, Some(path), None) => Source::Local { path },
102                (None, None, Some(name)) => {
103                    diags.error(
104                        "invalid-packages",
105                        Some(yf.display.clone()),
106                        line,
107                        format!("`package: {name}`: registry packages aren't available yet; use `git:` or `local:`"),
108                    );
109                    continue;
110                }
111                _ => {
112                    diags.error(
113                        "invalid-packages",
114                        Some(yf.display.clone()),
115                        line,
116                        "each package needs exactly one of `git:` (with `revision:`) or `local:`",
117                    );
118                    continue;
119                }
120            };
121            let same_place = |a: &Source, b: &Source| match (a, b) {
122                (Source::Git { url: x, .. }, Source::Git { url: y, .. }) => x == y,
123                (Source::Local { path: x }, Source::Local { path: y }) => x == y,
124                _ => false,
125            };
126            match out.iter().find(|d| same_place(&d.source, &source)) {
127                Some(d) if d.source == source => {}
128                Some(d) => diags.error(
129                    "conflicting-packages",
130                    Some(yf.display.clone()),
131                    line,
132                    format!(
133                        "this package is also declared in {} with a different revision",
134                        d.file.display()
135                    ),
136                ),
137                None => out.push(Declared {
138                    source,
139                    file: PathBuf::from(f),
140                }),
141            }
142        }
143    }
144    out
145}
146
147fn manifest_name(dir: &Path) -> Result<String, String> {
148    let p = dir.join(PACKAGE_FILE);
149    let text = std::fs::read_to_string(&p).map_err(|_| format!("{} has no {PACKAGE_FILE}", dir.display()))?;
150    let m: Manifest = serde_yaml_ng::from_str(&text).map_err(|e| format!("{}: {e}", p.display()))?;
151    if m.name.is_empty()
152        || !m.name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_')
153        || m.name.starts_with(|c: char| c.is_ascii_digit())
154    {
155        return Err(format!(
156            "{}: package name `{}` must be letters, digits and `_`",
157            p.display(),
158            m.name
159        ));
160    }
161    Ok(m.name)
162}
163
164fn installed_dir(root: &Path, name: &str) -> PathBuf {
165    root.join(DEPS_DIR).join("packages").join(name)
166}
167
168/// Install missing or outdated git packages into `dre_deps/packages/` and pin them in
169/// `dre.lock`. With `install` false, report what's missing instead. Local packages need nothing.
170pub fn sync(root: &Path, install: bool, mut log: impl FnMut(&str)) -> Result<(), Vec<String>> {
171    let mut diags = Diagnostics::default();
172    let decls = declared(root, &mut diags);
173    if diags.has_errors() {
174        return Ok(()); // reported with the project load
175    }
176    let mut lock = Lock::load(root).map_err(|e| vec![e])?;
177    let mut errors = Vec::new();
178    let mut changed = false;
179    for d in &decls {
180        let Source::Git { url, revision } = &d.source else {
181            continue;
182        };
183        let pinned = lock
184            .packages
185            .iter()
186            .find(|(_, l)| &l.git == url)
187            .map(|(n, l)| (n.clone(), l.clone()));
188        // Up to date: the lock matches the declaration and that commit is what's installed.
189        if let Some((name, l)) = &pinned
190            && &l.revision == revision
191            && std::fs::read_to_string(installed_dir(root, name).join(INSTALLED_COMMIT))
192                .is_ok_and(|c| c.trim() == l.commit)
193        {
194            continue;
195        }
196        if !install {
197            errors.push(format!(
198                "the package from {url} ({revision}) isn't installed; run `dre deps`"
199            ));
200            continue;
201        }
202        // Reinstall the locked commit unless the declared revision changed.
203        let checkout = match &pinned {
204            Some((_, l)) if &l.revision == revision => l.commit.clone(),
205            _ => revision.clone(),
206        };
207        match install_git(root, url, &checkout) {
208            Ok((name, commit)) => {
209                log(&format!(
210                    "package `{name}` from {url} at {revision} ({})",
211                    &commit[..commit.len().min(12)]
212                ));
213                if let Some((old, _)) = &pinned
214                    && old != &name
215                {
216                    lock.packages.remove(old);
217                }
218                let entry = LockedPackage {
219                    git: url.clone(),
220                    revision: revision.clone(),
221                    commit,
222                };
223                if lock.packages.get(&name) != Some(&entry) {
224                    lock.packages.insert(name, entry);
225                    changed = true;
226                }
227            }
228            Err(e) => errors.push(format!("package from {url}: {e}")),
229        }
230    }
231    if changed && let Err(e) = lock.save(root) {
232        errors.push(e);
233    }
234    if errors.is_empty() { Ok(()) } else { Err(errors) }
235}
236
237const GIT_MISSING: &str =
238    "git isn't installed (needed for git packages); install git, or use a `local:` package";
239
240fn git_works() -> bool {
241    Command::new("git")
242        .arg("--version")
243        .output()
244        .is_ok_and(|o| o.status.success() && !o.stdout.is_empty())
245}
246
247fn git(args: &[&str], cwd: Option<&Path>) -> Result<String, String> {
248    let mut c = Command::new("git");
249    c.args(args);
250    if let Some(d) = cwd {
251        c.current_dir(d);
252    }
253    let out = c.output().map_err(|e| match e.kind() {
254        std::io::ErrorKind::NotFound => GIT_MISSING.to_string(),
255        _ => format!("can't run git: {e}"),
256    })?;
257    if !out.status.success() {
258        let stderr = String::from_utf8_lossy(&out.stderr).trim().to_string();
259        // A stub git (macOS without the command line tools) fails without saying why.
260        if stderr.is_empty() && !git_works() {
261            return Err(GIT_MISSING.to_string());
262        }
263        let reason = if stderr.is_empty() {
264            format!("exited with {}", out.status)
265        } else {
266            stderr
267        };
268        return Err(format!("git {}: {reason}", args.join(" ")));
269    }
270    Ok(String::from_utf8_lossy(&out.stdout).trim().to_string())
271}
272
273/// Clone `url` at `checkout` into a temporary folder, then copy it (without `.git`) into
274/// `dre_deps/packages/<name>`.
275fn install_git(root: &Path, url: &str, checkout: &str) -> Result<(String, String), String> {
276    let base = root.join(DEPS_DIR).join("packages");
277    std::fs::create_dir_all(&base).map_err(|e| format!("can't create {}: {e}", base.display()))?;
278    let tmp = std::env::temp_dir().join(format!("dre-package-{}-{}", std::process::id(), base_name(url)));
279    remove_dir_force(&tmp);
280    let result = (|| {
281        git(&["clone", "--quiet", url, &tmp.to_string_lossy()], None)?;
282        git(&["checkout", "--quiet", checkout], Some(&tmp))?;
283        let commit = git(&["rev-parse", "HEAD"], Some(&tmp))?;
284        let name = manifest_name(&tmp)?;
285        let dst = installed_dir(root, &name);
286        remove_dir_force(&dst);
287        copy_tree(&tmp, &dst).map_err(|e| format!("can't install into {}: {e}", dst.display()))?;
288        std::fs::write(dst.join(INSTALLED_COMMIT), &commit).map_err(|e| e.to_string())?;
289        Ok((name, commit))
290    })();
291    remove_dir_force(&tmp);
292    result
293}
294
295/// The last path segment of a URL or path, for a readable temp folder name.
296fn base_name(url: &str) -> String {
297    let url = url.trim_end_matches(['/', '\\']);
298    url.strip_suffix(".git")
299        .unwrap_or(url)
300        .rsplit(['/', '\\', ':'])
301        .next()
302        .unwrap_or("package")
303        .chars()
304        .filter(|c| c.is_ascii_alphanumeric() || *c == '_' || *c == '-')
305        .collect()
306}
307
308/// Copy `from` into `to`, leaving out `.git`.
309fn copy_tree(from: &Path, to: &Path) -> std::io::Result<()> {
310    std::fs::create_dir_all(to)?;
311    for e in std::fs::read_dir(from)? {
312        let e = e?;
313        if e.file_name() == ".git" {
314            continue;
315        }
316        let dst = to.join(e.file_name());
317        if e.file_type()?.is_dir() {
318            copy_tree(&e.path(), &dst)?;
319        } else {
320            std::fs::copy(e.path(), dst)?;
321        }
322    }
323    Ok(())
324}
325
326/// Remove a folder even when it holds read-only files, as git's object store does on Windows.
327fn remove_dir_force(dir: &Path) {
328    if !dir.exists() {
329        return;
330    }
331    for e in walkdir::WalkDir::new(dir).into_iter().filter_map(Result::ok) {
332        if let Ok(meta) = e.metadata() {
333            let mut perm = meta.permissions();
334            if perm.readonly() {
335                #[allow(clippy::permissions_set_readonly_false)]
336                perm.set_readonly(false);
337                let _ = std::fs::set_permissions(e.path(), perm);
338            }
339        }
340    }
341    let _ = std::fs::remove_dir_all(dir);
342}
343
344/// Resolve the declared packages to installed folders and their macro files.
345pub fn resolve(root: &Path, decls: &[Declared], diags: &mut Diagnostics) -> Vec<Package> {
346    let lock = Lock::load(root).unwrap_or_default();
347    let mut out: Vec<Package> = Vec::new();
348    for d in decls {
349        let dir = match &d.source {
350            Source::Local { path } => {
351                let p = root.join(path);
352                if !p.is_dir() {
353                    diags.error(
354                        "package-missing",
355                        Some(d.file.clone()),
356                        None,
357                        format!("local package `{path}` doesn't exist"),
358                    );
359                    continue;
360                }
361                p
362            }
363            Source::Git { url, .. } => match lock.packages.iter().find(|(_, l)| &l.git == url) {
364                Some((name, _)) if installed_dir(root, name).is_dir() => installed_dir(root, name),
365                _ => {
366                    diags.error(
367                        "package-missing",
368                        Some(d.file.clone()),
369                        None,
370                        format!("the package from {url} isn't installed; run `dre deps`"),
371                    );
372                    continue;
373                }
374            },
375        };
376        let name = match manifest_name(&dir) {
377            Ok(n) => n,
378            Err(e) => {
379                diags.error("invalid-package", Some(d.file.clone()), None, e);
380                continue;
381            }
382        };
383        if let Some(other) = out.iter().find(|p| p.name == name) {
384            diags.error(
385                "duplicate-package",
386                Some(d.file.clone()),
387                None,
388                format!(
389                    "two packages are named `{name}` ({} and {})",
390                    other.dir.display(),
391                    dir.display()
392                ),
393            );
394            continue;
395        }
396        let mut macros: Vec<PathBuf> = walkdir::WalkDir::new(dir.join("macros"))
397            .sort_by_file_name()
398            .into_iter()
399            .filter_map(Result::ok)
400            .map(|e| e.into_path())
401            .filter(|p| p.extension().is_some_and(|x| x == "sql"))
402            .collect();
403        macros.sort();
404        out.push(Package { name, dir, macros });
405    }
406    out
407}
408
409/// `dispatch:` in `dre_project.yml`: per macro namespace, where `dispatch()` looks and in what
410/// order. Unlisted namespaces search the root project first, then the namespace itself.
411pub type DispatchOrder = BTreeMap<String, Vec<String>>;
dre_core/packages.rs

dre_core/
packages.rs
