dre_core/
packages.rs

1//! Macro packages: reusable macros from another project (e.g. `dre_utils`), declared under
2//! `packages:` in the project's root `dependencies.yml` or `packages.yml`:
3//!
4//! ```yaml
5//! packages:
6//!   - git: https://github.com/acme/dre-finance-macros.git
7//!     revision: v2.3.0          # tag, branch or commit
8//!   - local: ../shared/macros   # a folder on disk, read in place
9//! ```
10//!
11//! A package is a folder with a `dre_package.yml` (`name:`) and a `macros/` folder. `dre deps`
12//! clones git packages into `dre_deps/packages/<name>` and pins the commit in `dre.lock`. Their
13//! macros are called through the package's name: `{{ dre_utils.star_except(...) }}`.
14
15use std::collections::BTreeMap;
16use std::path::{Path, PathBuf};
17use std::process::Command;
18
19use serde::{Deserialize, Serialize};
20
21use crate::codes::Code;
22use crate::config::de::Loose;
23use crate::config::dependencies::DependenciesFile;
24use crate::diag::Diagnostics;
25use crate::lock::{Lock, LockedPackage};
26use crate::plugins::DEPS_DIR;
27use crate::yaml::YamlFile;
28
29/// Root files that may declare `packages:` (and plugins, like any YAML file).
30pub const DEPENDENCY_FILES: &[&str] = &["dependencies.yml", "packages.yml"];
31pub const PACKAGE_FILE: &str = "dre_package.yml";
32const INSTALLED_COMMIT: &str = ".dre_commit";
33
34#[derive(Debug, Clone, PartialEq, Eq)]
35pub enum Source {
36    Git { url: String, revision: String },
37    Local { path: String },
38}
39
40#[derive(Debug, Clone)]
41pub struct Declared {
42    pub source: Source,
43    /// The file declaring it, relative to the project root.
44    pub file: PathBuf,
45}
46
47/// An installed package, ready to use.
48#[derive(Debug, Clone, Serialize)]
49pub struct Package {
50    pub name: String,
51    #[serde(skip)]
52    pub dir: PathBuf,
53    /// Macro files, absolute.
54    #[serde(skip)]
55    pub macros: Vec<PathBuf>,
56}
57
58#[derive(Deserialize)]
59struct Manifest {
60    name: String,
61}
62
63/// Every `packages:` entry in the root dependency files. Problems go to `diags`.
64pub fn declared(root: &Path, diags: &mut Diagnostics) -> Vec<Declared> {
65    let mut out: Vec<Declared> = Vec::new();
66    for f in DEPENDENCY_FILES {
67        let path = root.join(f);
68        if !path.is_file() {
69            continue;
70        }
71        let mut quiet = Diagnostics::default();
72        let Some(yf) = YamlFile::load(&path, PathBuf::from(f), &mut quiet) else {
73            continue; // the project load reports the parse error
74        };
75        let Ok(dependencies) = crate::config::de::from_node::<DependenciesFile>(&yf.node) else {
76            continue; // the project load reports its shape
77        };
78        let Some(packages) = dependencies.packages else {
79            continue;
80        };
81        let line = packages.line();
82        let Loose::Ok(list) = packages.value else {
83            diags.error(
84                Code::InvalidPackages,
85                Some(yf.display.clone()),
86                line,
87                "`packages` must be a list",
88            );
89            continue;
90        };
91        for entry in list {
92            let get = |v: Option<&Loose<String>>| v.and_then(Loose::ok).cloned();
93            let entry = entry.ok();
94            let (git, local, package, revision) = (
95                get(entry.and_then(|e| e.git.as_ref())),
96                get(entry.and_then(|e| e.local.as_ref())),
97                get(entry.and_then(|e| e.package.as_ref())),
98                get(entry.and_then(|e| e.revision.as_ref())),
99            );
100            let source = match (git, local, package) {
101                (Some(url), None, None) => match revision {
102                    Some(revision) => Source::Git { url, revision },
103                    None => {
104                        diags.error(
105                            Code::InvalidPackages,
106                            Some(yf.display.clone()),
107                            line,
108                            format!("git package `{url}` needs a `revision` (a tag, branch or commit)"),
109                        );
110                        continue;
111                    }
112                },
113                (None, Some(path), None) => Source::Local { path },
114                (None, None, Some(name)) => {
115                    diags.error(
116                        Code::InvalidPackages,
117                        Some(yf.display.clone()),
118                        line,
119                        format!("`package: {name}`: registry packages aren't available yet; use `git:` or `local:`"),
120                    );
121                    continue;
122                }
123                _ => {
124                    diags.error(
125                        Code::InvalidPackages,
126                        Some(yf.display.clone()),
127                        line,
128                        "each package needs exactly one of `git:` (with `revision:`) or `local:`",
129                    );
130                    continue;
131                }
132            };
133            let same_place = |a: &Source, b: &Source| match (a, b) {
134                (Source::Git { url: x, .. }, Source::Git { url: y, .. }) => x == y,
135                (Source::Local { path: x }, Source::Local { path: y }) => x == y,
136                _ => false,
137            };
138            match out.iter().find(|d| same_place(&d.source, &source)) {
139                Some(d) if d.source == source => {}
140                Some(d) => diags.error(
141                    Code::ConflictingPackages,
142                    Some(yf.display.clone()),
143                    line,
144                    format!(
145                        "this package is also declared in {} with a different revision",
146                        d.file.display()
147                    ),
148                ),
149                None => out.push(Declared {
150                    source,
151                    file: PathBuf::from(f),
152                }),
153            }
154        }
155    }
156    out
157}
158
159fn manifest_name(dir: &Path) -> Result<String, String> {
160    let p = dir.join(PACKAGE_FILE);
161    let text = std::fs::read_to_string(&p).map_err(|_| format!("{} has no {PACKAGE_FILE}", dir.display()))?;
162    let m: Manifest = serde_saphyr::from_str(&text).map_err(|e| format!("{}: {e}", p.display()))?;
163    if m.name.is_empty()
164        || !m.name.chars().all(|c| c.is_ascii_alphanumeric() || c == '_')
165        || m.name.starts_with(|c: char| c.is_ascii_digit())
166    {
167        return Err(format!(
168            "{}: package name `{}` must be letters, digits and `_`",
169            p.display(),
170            m.name
171        ));
172    }
173    Ok(m.name)
174}
175
176fn installed_dir(root: &Path, name: &str) -> PathBuf {
177    root.join(DEPS_DIR).join("packages").join(name)
178}
179
180/// Install missing or outdated git packages into `dre_deps/packages/` and pin them in
181/// `dre.lock`. With `install` false, report what's missing instead. Local packages need nothing.
182pub fn sync(root: &Path, install: bool, mut log: impl FnMut(&str)) -> Result<(), Vec<String>> {
183    let mut diags = Diagnostics::default();
184    let decls = declared(root, &mut diags);
185    if diags.has_errors() {
186        return Ok(()); // reported with the project load
187    }
188    let mut lock = Lock::load(root).map_err(|e| vec![e])?;
189    let mut errors = Vec::new();
190    let mut changed = false;
191    for d in &decls {
192        let Source::Git { url, revision } = &d.source else {
193            continue;
194        };
195        let pinned = lock
196            .packages
197            .iter()
198            .find(|(_, l)| &l.git == url)
199            .map(|(n, l)| (n.clone(), l.clone()));
200        // Up to date: the lock matches the declaration and that commit is what's installed.
201        if let Some((name, l)) = &pinned
202            && &l.revision == revision
203            && std::fs::read_to_string(installed_dir(root, name).join(INSTALLED_COMMIT))
204                .is_ok_and(|c| c.trim() == l.commit)
205        {
206            continue;
207        }
208        if !install {
209            errors.push(format!(
210                "the package from {url} ({revision}) isn't installed; run `dre deps`"
211            ));
212            continue;
213        }
214        // Reinstall the locked commit unless the declared revision changed.
215        let checkout = match &pinned {
216            Some((_, l)) if &l.revision == revision => l.commit.clone(),
217            _ => revision.clone(),
218        };
219        match install_git(root, url, &checkout) {
220            Ok((name, commit)) => {
221                log(&format!(
222                    "package `{name}` from {url} at {revision} ({})",
223                    &commit[..commit.len().min(12)]
224                ));
225                if let Some((old, _)) = &pinned
226                    && old != &name
227                {
228                    lock.packages.remove(old);
229                }
230                let entry = LockedPackage {
231                    git: url.clone(),
232                    revision: revision.clone(),
233                    commit,
234                };
235                if lock.packages.get(&name) != Some(&entry) {
236                    lock.packages.insert(name, entry);
237                    changed = true;
238                }
239            }
240            Err(e) => errors.push(format!("package from {url}: {e}")),
241        }
242    }
243    if changed && let Err(e) = lock.save(root) {
244        errors.push(e);
245    }
246    if errors.is_empty() { Ok(()) } else { Err(errors) }
247}
248
249const GIT_MISSING: &str =
250    "git isn't installed (needed for git packages); install git, or use a `local:` package";
251
252fn git_works() -> bool {
253    Command::new("git")
254        .arg("--version")
255        .output()
256        .is_ok_and(|o| o.status.success() && !o.stdout.is_empty())
257}
258
259fn git(args: &[&str], cwd: Option<&Path>) -> Result<String, String> {
260    let mut c = Command::new("git");
261    c.args(args);
262    if let Some(d) = cwd {
263        c.current_dir(d);
264    }
265    let out = c.output().map_err(|e| match e.kind() {
266        std::io::ErrorKind::NotFound => GIT_MISSING.to_string(),
267        _ => format!("can't run git: {e}"),
268    })?;
269    if !out.status.success() {
270        let stderr = String::from_utf8_lossy(&out.stderr).trim().to_string();
271        // A stub git (macOS without the command line tools) fails without saying why.
272        if stderr.is_empty() && !git_works() {
273            return Err(GIT_MISSING.to_string());
274        }
275        let reason = if stderr.is_empty() {
276            format!("exited with {}", out.status)
277        } else {
278            stderr
279        };
280        return Err(format!("git {}: {reason}", args.join(" ")));
281    }
282    Ok(String::from_utf8_lossy(&out.stdout).trim().to_string())
283}
284
285/// Clone `url` at `checkout` into a temporary folder, then copy it (without `.git`) into
286/// `dre_deps/packages/<name>`.
287fn install_git(root: &Path, url: &str, checkout: &str) -> Result<(String, String), String> {
288    let base = root.join(DEPS_DIR).join("packages");
289    std::fs::create_dir_all(&base).map_err(|e| format!("can't create {}: {e}", base.display()))?;
290    let tmp = std::env::temp_dir().join(format!("dre-package-{}-{}", std::process::id(), base_name(url)));
291    remove_dir_force(&tmp);
292    let result = (|| {
293        git(&["clone", "--quiet", url, &tmp.to_string_lossy()], None)?;
294        git(&["checkout", "--quiet", checkout], Some(&tmp))?;
295        let commit = git(&["rev-parse", "HEAD"], Some(&tmp))?;
296        let name = manifest_name(&tmp)?;
297        let dst = installed_dir(root, &name);
298        remove_dir_force(&dst);
299        copy_tree(&tmp, &dst).map_err(|e| format!("can't install into {}: {e}", dst.display()))?;
300        crate::fs::write_atomic(&dst.join(INSTALLED_COMMIT), commit.as_bytes()).map_err(|e| e.to_string())?;
301        Ok((name, commit))
302    })();
303    remove_dir_force(&tmp);
304    result
305}
306
307/// The last path segment of a URL or path, for a readable temp folder name.
308fn base_name(url: &str) -> String {
309    let url = url.trim_end_matches(['/', '\\']);
310    url.strip_suffix(".git")
311        .unwrap_or(url)
312        .rsplit(['/', '\\', ':'])
313        .next()
314        .unwrap_or("package")
315        .chars()
316        .filter(|c| c.is_ascii_alphanumeric() || *c == '_' || *c == '-')
317        .collect()
318}
319
320/// Copy `from` into `to`, leaving out `.git`.
321fn copy_tree(from: &Path, to: &Path) -> std::io::Result<()> {
322    std::fs::create_dir_all(to)?;
323    for e in std::fs::read_dir(from)? {
324        let e = e?;
325        if e.file_name() == ".git" {
326            continue;
327        }
328        let dst = to.join(e.file_name());
329        if e.file_type()?.is_dir() {
330            copy_tree(&e.path(), &dst)?;
331        } else {
332            std::fs::copy(e.path(), dst)?;
333        }
334    }
335    Ok(())
336}
337
338/// Remove a folder even when it holds read-only files, as git's object store does on Windows.
339fn remove_dir_force(dir: &Path) {
340    if !dir.exists() {
341        return;
342    }
343    for e in walkdir::WalkDir::new(dir).into_iter().filter_map(Result::ok) {
344        if let Ok(meta) = e.metadata() {
345            let mut perm = meta.permissions();
346            if perm.readonly() {
347                #[allow(clippy::permissions_set_readonly_false)]
348                perm.set_readonly(false);
349                let _ = std::fs::set_permissions(e.path(), perm);
350            }
351        }
352    }
353    let _ = std::fs::remove_dir_all(dir);
354}
355
356/// Resolve the declared packages to installed folders and their macro files.
357pub fn resolve(root: &Path, decls: &[Declared], diags: &mut Diagnostics) -> Vec<Package> {
358    let lock = Lock::load(root).unwrap_or_default();
359    let mut out: Vec<Package> = Vec::new();
360    for d in decls {
361        let dir = match &d.source {
362            Source::Local { path } => {
363                let p = root.join(path);
364                if !p.is_dir() {
365                    diags.error(
366                        Code::PackageMissing,
367                        Some(d.file.clone()),
368                        None,
369                        format!("local package `{path}` doesn't exist"),
370                    );
371                    continue;
372                }
373                p
374            }
375            Source::Git { url, .. } => match lock.packages.iter().find(|(_, l)| &l.git == url) {
376                Some((name, _)) if installed_dir(root, name).is_dir() => installed_dir(root, name),
377                _ => {
378                    diags.error(
379                        Code::PackageMissing,
380                        Some(d.file.clone()),
381                        None,
382                        format!("the package from {url} isn't installed; run `dre deps`"),
383                    );
384                    continue;
385                }
386            },
387        };
388        let name = match manifest_name(&dir) {
389            Ok(n) => n,
390            Err(e) => {
391                diags.error(Code::InvalidPackage, Some(d.file.clone()), None, e);
392                continue;
393            }
394        };
395        if let Some(other) = out.iter().find(|p| p.name == name) {
396            diags.error(
397                Code::DuplicatePackage,
398                Some(d.file.clone()),
399                None,
400                format!(
401                    "two packages are named `{name}` ({} and {})",
402                    other.dir.display(),
403                    dir.display()
404                ),
405            );
406            continue;
407        }
408        let mut macros: Vec<PathBuf> = walkdir::WalkDir::new(dir.join("macros"))
409            .sort_by_file_name()
410            .into_iter()
411            .filter_map(Result::ok)
412            .map(|e| e.into_path())
413            .filter(|p| p.extension().is_some_and(|x| x == "sql"))
414            .collect();
415        macros.sort();
416        out.push(Package { name, dir, macros });
417    }
418    out
419}
420
421/// `dispatch:` in `dre_project.yml`: per macro namespace, where `dispatch()` looks and in what
422/// order. Unlisted namespaces search the root project first, then the namespace itself.
423pub type DispatchOrder = BTreeMap<String, Vec<String>>;
dre_core/packages.rs

dre_core/
packages.rs
