devset_core/
source.rs

1//! Where a layer's profile comes from: a local directory, or a git repository at one commit.
2
3use alloc::borrow::Cow;
4use alloc::sync::Arc;
5use core::fmt;
6use std::collections::HashMap;
7use std::io::{self, Read};
8use std::sync::Mutex;
9
10use camino::{Utf8Component, Utf8Path, Utf8PathBuf};
11use derive_more::{Display, Into};
12use etcetera::BaseStrategy;
13use ignore::WalkBuilder;
14use schemars::{JsonSchema, Schema, SchemaGenerator};
15use serde::{Deserialize, Serialize};
16
17use crate::errors::{Result, SourceError};
18use crate::git;
19use crate::path::RelPath;
20use crate::profile::MANIFEST;
21use crate::tree::Tree;
22
23/// Where a layer's profile comes from; written as [`SourceSpec`].
24#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
25#[serde(try_from = "SourceSpec", into = "SourceSpec")]
26pub enum Source {
27    /// A directory, relative to the target root unless absolute.
28    Dir(Utf8PathBuf),
29    /// A git repository.
30    Git {
31        /// Anything `git fetch` accepts.
32        url: String,
33        /// Which commit.
34        at: GitRef,
35        /// The profile's directory in the repository; its root if `None`.
36        path: Option<RelPath>,
37    },
38}
39
40/// Which commit of a git source.
41#[derive(Clone, Debug, PartialEq, Eq)]
42pub enum GitRef {
43    /// The remote's default branch.
44    Head,
45    /// A branch.
46    Branch(String),
47    /// A tag.
48    Tag(String),
49    /// A commit.
50    Rev(Oid),
51}
52
53/// A full commit id: 40 or 64 hex digits, lowercase.
54#[derive(Clone, PartialEq, Eq, Display, derive_more::Debug, Into, Serialize, Deserialize)]
55#[debug("{_0}")]
56#[into(String)]
57#[serde(try_from = "String", into = "String")]
58pub struct Oid(Box<str>);
59
60/// A source as written in `config.toml`: Cargo's git-dependency fields.
61#[derive(Debug, Default, Serialize, Deserialize, JsonSchema)]
62#[serde(deny_unknown_fields)]
63pub struct SourceSpec {
64    /// Repository URL.
65    #[serde(default, skip_serializing_if = "Option::is_none")]
66    pub git: Option<String>,
67    /// Tag to use; only with `git`.
68    #[serde(default, skip_serializing_if = "Option::is_none")]
69    pub tag: Option<String>,
70    /// Branch to use; only with `git`.
71    #[serde(default, skip_serializing_if = "Option::is_none")]
72    pub branch: Option<String>,
73    /// Full commit id to use; only with `git`.
74    #[serde(default, skip_serializing_if = "Option::is_none")]
75    pub rev: Option<String>,
76    /// The profile's directory: relative to the target, or within the repository with `git`.
77    #[serde(default, skip_serializing_if = "Option::is_none")]
78    pub path: Option<String>,
79}
80
81impl TryFrom<SourceSpec> for Source {
82    type Error = SourceError;
83
84    fn try_from(spec: SourceSpec) -> Result<Self, SourceError> {
85        let SourceSpec { git, tag, branch, rev, path } = spec;
86        let Some(url) = git else {
87            if tag.is_some() || branch.is_some() || rev.is_some() {
88                return Err(SourceError::RefWithoutGit);
89            }
90            let path = path.ok_or(SourceError::NoLocation)?;
91            return dir(&path).map(Self::Dir).ok_or(SourceError::EmptyPath);
92        };
93        // A leading `-` would reach git as an option.
94        if url.is_empty() || url.starts_with('-') {
95            return Err(SourceError::Url { url });
96        }
97        let at = match (tag, branch, rev) {
98            (None, None, None) => GitRef::Head,
99            (Some(tag), None, None) => GitRef::Tag(tag),
100            (None, Some(branch), None) => GitRef::Branch(branch),
101            (None, None, Some(rev)) => GitRef::Rev(Oid::try_from(rev)?),
102            _ => return Err(SourceError::TwoRefs),
103        };
104        let path = path.filter(|p| !p.is_empty()).map(RelPath::try_from).transpose()?;
105        Ok(Self::Git { url, at, path })
106    }
107}
108
109/// `path` without `.` components, so one directory has one spelling; `None` when empty.
110fn dir(path: &str) -> Option<Utf8PathBuf> {
111    if path.is_empty() {
112        return None;
113    }
114    let dir: Utf8PathBuf =
115        Utf8Path::new(path).components().filter(|c| *c != Utf8Component::CurDir).collect();
116    Some(if dir.as_str().is_empty() { Utf8PathBuf::from(".") } else { dir })
117}
118
119impl From<Source> for SourceSpec {
120    fn from(source: Source) -> Self {
121        match source {
122            Source::Dir(path) => Self { path: Some(path.into_string()), ..Self::default() },
123            Source::Git { url, at, path } => {
124                let mut spec =
125                    Self { git: Some(url), path: path.map(String::from), ..Self::default() };
126                match at {
127                    GitRef::Head => {},
128                    GitRef::Branch(branch) => spec.branch = Some(branch),
129                    GitRef::Tag(tag) => spec.tag = Some(tag),
130                    GitRef::Rev(rev) => spec.rev = Some(rev.into()),
131                }
132                spec
133            },
134        }
135    }
136}
137
138impl JsonSchema for Source {
139    fn schema_name() -> Cow<'static, str> {
140        SourceSpec::schema_name()
141    }
142
143    fn json_schema(generator: &mut SchemaGenerator) -> Schema {
144        SourceSpec::json_schema(generator)
145    }
146}
147
148impl Source {
149    /// Whether `other` is this profile at another ref: one repository and path, two refs.
150    pub(crate) fn twin(&self, other: &Self) -> bool {
151        match (self, other) {
152            (
153                Self::Git { url, at, path },
154                Self::Git { url: other_url, at: other_at, path: other_path },
155            ) => url == other_url && path == other_path && at != other_at,
156            (Self::Dir(_) | Self::Git { .. }, Self::Dir(_) | Self::Git { .. }) => false,
157        }
158    }
159}
160
161impl fmt::Display for Source {
162    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
163        match self {
164            Self::Dir(path) => f.write_str(path.as_str()),
165            Self::Git { url, at, path } => {
166                f.write_str(url)?;
167                match at {
168                    GitRef::Head => {},
169                    GitRef::Branch(branch) => write!(f, " branch {branch}")?,
170                    GitRef::Tag(tag) => write!(f, " tag {tag}")?,
171                    GitRef::Rev(rev) => write!(f, " rev {rev}")?,
172                }
173                path.as_ref().map_or(Ok(()), |path| write!(f, " path {path}"))
174            },
175        }
176    }
177}
178
179impl Oid {
180    /// The id as hex.
181    #[must_use]
182    pub fn as_str(&self) -> &str {
183        &self.0
184    }
185
186    /// Its first seven digits, as git abbreviates it.
187    #[must_use]
188    pub fn short(&self) -> &str {
189        self.0.get(..7).unwrap_or(&self.0)
190    }
191}
192
193impl TryFrom<String> for Oid {
194    type Error = SourceError;
195
196    fn try_from(mut id: String) -> Result<Self, SourceError> {
197        if !matches!(id.len(), 40 | 64) || !id.bytes().all(|b| b.is_ascii_hexdigit()) {
198            return Err(SourceError::Rev { rev: id });
199        }
200        id.make_ascii_lowercase();
201        Ok(Self(id.into_boxed_str()))
202    }
203}
204
205/// Remote access for a git layer, reported through [`Cache::on_fetch`].
206#[derive(Clone, Copy, Debug)]
207pub enum Fetch<'a> {
208    /// About to contact this URL.
209    Start(&'a str),
210    /// Finished with it.
211    Done(&'a str),
212}
213
214/// What [`Cache::on_fetch`] installs.
215type Hook = Arc<dyn Fn(Fetch<'_>) + Send + Sync>;
216
217/// Fetched git profiles, shared by every target on the machine.
218///
219/// One process sees one commit per ref: each ref is asked of its remote once.
220#[derive(Clone, derive_more::Debug)]
221pub struct Cache {
222    /// Where it lives.
223    dir: Utf8PathBuf,
224    /// Told about remote access.
225    #[debug(skip)]
226    hook: Option<Hook>,
227    /// Refs resolved so far, by URL and ref name, so one process sees one commit per ref.
228    #[debug(skip)]
229    refs: Arc<Mutex<HashMap<(String, String), Oid>>>,
230    /// Whether git may ask for credentials on the terminal.
231    prompts: bool,
232}
233
234impl Cache {
235    /// The per-user cache: `$XDG_CACHE_HOME/devset`, or the platform's equivalent.
236    ///
237    /// # Errors
238    /// [`Error::Io`](crate::Error::Io), there is no home directory, or its path is not UTF-8.
239    pub fn user() -> Result<Self> {
240        let base = etcetera::choose_base_strategy().map_err(io::Error::other)?;
241        let dir = Utf8PathBuf::try_from(base.cache_dir()).map_err(io::Error::other)?;
242        Ok(Self::at(dir.join("devset")))
243    }
244
245    /// A cache in `dir`.
246    #[must_use]
247    pub fn at(dir: Utf8PathBuf) -> Self {
248        Self { dir, hook: None, refs: Arc::new(Mutex::new(HashMap::new())), prompts: false }
249    }
250
251    /// This cache, calling `hook` around every remote access so a caller can show progress.
252    #[must_use]
253    pub fn on_fetch<F: Fn(Fetch<'_>) + Send + Sync + 'static>(mut self, hook: F) -> Self {
254        self.hook = Some(Arc::new(hook));
255        self
256    }
257
258    /// This cache, letting git ask for credentials on the terminal when `allowed`.
259    ///
260    /// Off unless asked for, so a run with no one to answer fails rather than waits.
261    #[must_use]
262    pub const fn prompting(mut self, allowed: bool) -> Self {
263        self.prompts = allowed;
264        self
265    }
266
267    /// Whether git may ask for credentials on the terminal.
268    pub(crate) const fn prompts(&self) -> bool {
269        self.prompts
270    }
271
272    /// Where the cache lives.
273    #[must_use]
274    pub fn path(&self) -> &Utf8Path {
275        &self.dir
276    }
277
278    /// The commit `name` at `url` was resolved to earlier in this process.
279    pub(crate) fn resolved(&self, url: &str, name: &str) -> Option<Oid> {
280        let refs = self.refs.lock().ok()?;
281        refs.get(&(url.to_owned(), name.to_owned())).cloned()
282    }
283
284    /// Remembers that `name` at `url` is `rev`.
285    pub(crate) fn remember(&self, url: &str, name: &str, rev: &Oid) {
286        if let Ok(mut refs) = self.refs.lock() {
287            refs.insert((url.to_owned(), name.to_owned()), rev.clone());
288        }
289    }
290
291    /// Reports `event` to the hook, if any.
292    pub(crate) fn notify(&self, event: Fetch<'_>) {
293        if let Some(hook) = &self.hook {
294            hook(event);
295        }
296    }
297}
298
299/// A source opened at one revision.
300#[derive(Debug)]
301pub(crate) enum Reader {
302    /// A local directory.
303    Dir(Utf8PathBuf),
304    /// A commit in the cache.
305    Git(git::Commit),
306}
307
308impl Source {
309    /// This source as written from `from`, written from `to` instead: a local directory, or a
310    /// local repository, relative to `to`; unchanged when it is absolute or remote.
311    #[must_use]
312    pub fn rebased(self, from: &Utf8Path, to: &Utf8Path) -> Self {
313        match self {
314            Self::Dir(dir) if dir.is_absolute() => Self::Dir(dir),
315            Self::Dir(dir) => Self::Dir(relative(&from.join(dir), to)),
316            Self::Git { url, at, path } if !remote(&url) && !Utf8Path::new(&url).is_absolute() => {
317                let url = relative(&from.join(&url), to).into_string();
318                Self::Git { url, at, path }
319            },
320            git @ Self::Git { .. } => git,
321        }
322    }
323
324    /// Opens the source: a git source at `pin` if given, else at its ref, fetching as needed.
325    pub(crate) fn open(&self, root: &Utf8Path, pin: Option<&Oid>, cache: &Cache) -> Result<Reader> {
326        match self {
327            Self::Dir(dir) => Ok(Reader::Dir(lexical(&root.join(dir)))),
328            Self::Git { url, at, path } => {
329                git::Commit::open(&locate(url, root), url, at, path.as_ref(), pin, cache)
330                    .map(Reader::Git)
331            },
332        }
333    }
334}
335
336/// `url` as `git` sees it from any directory: a local path is made absolute against `root`.
337fn locate<'a>(url: &'a str, root: &Utf8Path) -> Cow<'a, str> {
338    if remote(url) {
339        Cow::Borrowed(url)
340    } else {
341        Cow::Owned(lexical(&root.join(url)).into_string())
342    }
343}
344
345/// Whether `url` names a remote repository rather than a local path.
346///
347/// Git's rule: `host:path` is scp-like only when the colon precedes any slash.
348fn remote(url: &str) -> bool {
349    url.contains("://")
350        || url.find(':').is_some_and(|colon| url.find('/').is_none_or(|slash| colon < slash))
351}
352
353/// `path` with `.` dropped and each `..` taking back the directory before it, by name alone, as
354/// a shell resolves `cd ../x`; so a path under a directory not yet made still resolves.
355fn lexical(path: &Utf8Path) -> Utf8PathBuf {
356    let mut resolved = Utf8PathBuf::new();
357    for component in path.components() {
358        match component {
359            Utf8Component::CurDir => {},
360            Utf8Component::ParentDir
361                if matches!(resolved.components().next_back(), Some(Utf8Component::Normal(_))) =>
362            {
363                resolved.pop();
364            },
365            Utf8Component::ParentDir
366            | Utf8Component::Prefix(_)
367            | Utf8Component::RootDir
368            | Utf8Component::Normal(_) => resolved.push(component),
369        }
370    }
371    resolved
372}
373
374/// `path` relative to `base`, both resolved by name: `../p` for `/a/p` from `/a/repo`.
375fn relative(path: &Utf8Path, base: &Utf8Path) -> Utf8PathBuf {
376    let (path, base) = (lexical(path), lexical(base));
377    let shared = path.components().zip(base.components()).take_while(|(a, b)| a == b).count();
378    let up = base.components().skip(shared).map(|_| Utf8Component::ParentDir);
379    let down = path.components().skip(shared);
380    let relative: Utf8PathBuf = up.chain(down).collect();
381    if relative.as_str().is_empty() { Utf8PathBuf::from(".") } else { relative }
382}
383
384impl Reader {
385    /// The commit, for a versioned source.
386    pub(crate) const fn rev(&self) -> Option<&Oid> {
387        match self {
388            Self::Dir(_) => None,
389            Self::Git(commit) => Some(commit.rev()),
390        }
391    }
392
393    /// Every directory in the source that holds a `profile.toml`, relative to its root; `""` for
394    /// the root. Hidden and ignored directories are left out, as git leaves out what it ignores.
395    pub(crate) fn manifests(&self) -> Vec<String> {
396        match self {
397            Self::Git(commit) => commit.manifests(),
398            Self::Dir(root) => local_manifests(root),
399        }
400    }
401
402    /// The files at `paths` under `dir` in the profile, keyed by `paths`; absent ones left out.
403    pub(crate) fn read(&self, dir: &str, paths: &[RelPath]) -> Result<Tree> {
404        match self {
405            Self::Dir(root) => read_dir(&root.join(dir), paths),
406            Self::Git(commit) => commit.read(dir, paths),
407        }
408    }
409}
410
411/// [`Reader::manifests`] for a local directory.
412fn local_manifests(root: &Utf8Path) -> Vec<String> {
413    let mut found: Vec<String> = WalkBuilder::new(root)
414        .require_git(false)
415        .git_global(false)
416        .build()
417        .filter_map(Result::ok)
418        .filter(|entry| entry.file_type().is_some_and(|kind| !kind.is_dir()))
419        .filter(|entry| entry.file_name() == MANIFEST)
420        .filter_map(|entry| {
421            let dir = entry.path().parent()?.strip_prefix(root).ok()?;
422            let dir = Utf8Path::from_path(dir)?;
423            Some(dir.components().map(|part| part.as_str()).collect::<Vec<_>>().join("/"))
424        })
425        .collect();
426    found.sort();
427    found
428}
429
430/// [`Reader::read`] for a local directory.
431fn read_dir(dir: &Utf8Path, paths: &[RelPath]) -> Result<Tree> {
432    let mut tree = Tree::default();
433    for path in paths {
434        let file = path.under(dir);
435        match fs_err::symlink_metadata(&file) {
436            Ok(meta) if meta.is_file() => {
437                tree.insert(path.clone(), |buf| {
438                    fs_err::File::open(&file)?.read_to_end(buf)?;
439                    Ok(())
440                })?;
441            },
442            Ok(_) => {
443                return Err(SourceError::NotAFile { file: file.into_string() }.into());
444            },
445            Err(e) if e.kind() == io::ErrorKind::NotFound => {},
446            Err(e) => return Err(e.into()),
447        }
448    }
449    Ok(tree)
450}
451
452#[cfg(test)]
453mod tests {
454    use super::{GitRef, Source, SourceSpec};
455
456    fn parse(toml: &str) -> Result<Source, String> {
457        toml::from_str::<SourceSpec>(toml)
458            .map_err(|e| e.to_string())
459            .and_then(|spec| Source::try_from(spec).map_err(|e| e.to_string()))
460    }
461
462    #[test]
463    fn valid() {
464        assert_eq!(parse(r#"path = "../p""#), Ok(Source::Dir("../p".into())), "local directory");
465        for (written, dir) in [("./../p/", "../p"), ("./p", "p"), (".", "."), ("./", ".")] {
466            let toml = format!("path = \"{written}\"");
467            assert_eq!(parse(&toml), Ok(Source::Dir(dir.into())), "{written:?} has one spelling");
468        }
469        let Ok(Source::Git { url, at, path }) = parse(
470            r#"git = "https://h/r"
471tag = "v1"
472path = "rust""#,
473        ) else {
474            panic!("git source should parse");
475        };
476        assert_eq!(
477            (url.as_str(), at, path.map(String::from)),
478            ("https://h/r", GitRef::Tag("v1".into()), Some("rust".into())),
479            "fields"
480        );
481        let rev = "A".repeat(40);
482        let Ok(Source::Git { at: GitRef::Rev(oid), .. }) =
483            parse(&format!("git = \"u\"\nrev = \"{rev}\""))
484        else {
485            panic!("rev should parse");
486        };
487        assert_eq!(oid.as_str(), "a".repeat(40), "commit ids are lowercased");
488    }
489
490    #[test]
491    fn invalid() {
492        for toml in [
493            "",
494            r#"path = """#,
495            r#"tag = "v1""#,
496            r#"path = "p"
497branch = "b""#,
498            r#"git = "u"
499tag = "v1"
500branch = "b""#,
501            r#"git = "u"
502rev = "abc""#,
503            r#"git = "--upload-pack=x""#,
504            r#"git = "u"
505path = "../x""#,
506            r#"git = "u"
507bogus = 1"#,
508        ] {
509            assert!(parse(toml).is_err(), "should be rejected: {toml:?}");
510        }
511    }
512
513    #[test]
514    fn local_git_paths_are_anchored() {
515        let root = camino::Utf8Path::new("/work/repo");
516        for (url, want) in [
517            ("../p.git", "/work/p.git"),
518            ("/abs/p.git", "/abs/p.git"),
519            ("https://h/p", "https://h/p"),
520            ("file:///p", "file:///p"),
521            ("git@h:org/p", "git@h:org/p"),
522            ("./a:b", "/work/repo/a:b"),
523        ] {
524            assert_eq!(super::locate(url, root), want, "{url}");
525        }
526    }
527
528    #[test]
529    fn sources_are_rebased_by_name() {
530        let (cwd, root) = (camino::Utf8Path::new("/w"), camino::Utf8Path::new("/w/hello"));
531        let dir = Source::Dir("p".into()).rebased(cwd, root);
532        assert_eq!(dir, Source::Dir("../p".into()), "a directory, from the target");
533        let git = |url: &str| Source::Git { url: url.to_owned(), at: GitRef::Head, path: None };
534        assert_eq!(git("p.git").rebased(cwd, root), git("../p.git"), "a local repository");
535        for kept in ["https://h/p", "git@h:org/p", "/abs/p.git"] {
536            assert_eq!(git(kept).rebased(cwd, root), git(kept), "{kept} is left as it is");
537        }
538        assert_eq!(super::relative("/w/hello".into(), root), camino::Utf8Path::new("."), "itself");
539        let absolute = Source::Dir("/abs/p".into()).rebased(cwd, root);
540        assert_eq!(absolute, Source::Dir("/abs/p".into()), "an absolute directory stays so");
541    }
542
543    #[test]
544    fn round_trips() {
545        let toml = "git = \"https://h/r\"\nbranch = \"main\"\npath = \"a/b\"\n";
546        let source = parse(toml).unwrap();
547        assert_eq!(toml::to_string(&source).unwrap(), toml, "serialized as written");
548    }
549}
devset_core/source.rs

devset_core/
source.rs
