devset_core/
source.rs

1//! Where a layer's profile comes from: a local directory, or a git repository at one commit.
2
3use alloc::borrow::Cow;
4use alloc::sync::Arc;
5use core::fmt;
6use std::collections::HashMap;
7use std::io::{self, Read};
8use std::sync::Mutex;
9
10use camino::{Utf8Component, Utf8Path, Utf8PathBuf};
11use derive_more::{Display, Into};
12use etcetera::BaseStrategy;
13use ignore::WalkBuilder;
14use schemars::{JsonSchema, Schema, SchemaGenerator};
15use serde::{Deserialize, Serialize};
16
17use crate::errors::{Result, SourceError};
18use crate::git;
19use crate::path::RelPath;
20use crate::profile::MANIFEST;
21use crate::tree::Tree;
22
23/// Where a layer's profile comes from; written as [`SourceSpec`].
24#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
25#[serde(try_from = "SourceSpec", into = "SourceSpec")]
26pub enum Source {
27    /// A directory, relative to the target root unless absolute.
28    Dir(Utf8PathBuf),
29    /// A git repository.
30    Git {
31        /// Anything `git fetch` accepts.
32        url: String,
33        /// Which commit.
34        at: GitRef,
35        /// The profile's directory in the repository; its root if `None`.
36        path: Option<RelPath>,
37    },
38}
39
40/// Which commit of a git source.
41#[derive(Clone, Debug, PartialEq, Eq)]
42pub enum GitRef {
43    /// The remote's default branch.
44    Head,
45    /// A branch.
46    Branch(String),
47    /// A tag.
48    Tag(String),
49    /// A commit.
50    Rev(Oid),
51}
52
53/// A full commit id: 40 or 64 hex digits, lowercase.
54#[derive(Clone, PartialEq, Eq, Display, derive_more::Debug, Into, Serialize, Deserialize)]
55#[debug("{_0}")]
56#[into(String)]
57#[serde(try_from = "String", into = "String")]
58pub struct Oid(Box<str>);
59
60/// A source as written in `config.toml`: Cargo's git-dependency fields.
61#[derive(Debug, Default, Serialize, Deserialize, JsonSchema)]
62#[serde(deny_unknown_fields)]
63pub struct SourceSpec {
64    /// Repository URL.
65    #[serde(default, skip_serializing_if = "Option::is_none")]
66    pub git: Option<String>,
67    /// Tag to use; only with `git`.
68    #[serde(default, skip_serializing_if = "Option::is_none")]
69    pub tag: Option<String>,
70    /// Branch to use; only with `git`.
71    #[serde(default, skip_serializing_if = "Option::is_none")]
72    pub branch: Option<String>,
73    /// Full commit id to use; only with `git`.
74    #[serde(default, skip_serializing_if = "Option::is_none")]
75    pub rev: Option<String>,
76    /// The profile's directory: relative to the target, or within the repository with `git`.
77    #[serde(default, skip_serializing_if = "Option::is_none")]
78    pub path: Option<String>,
79}
80
81impl TryFrom<SourceSpec> for Source {
82    type Error = SourceError;
83
84    fn try_from(spec: SourceSpec) -> Result<Self, SourceError> {
85        let SourceSpec { git, tag, branch, rev, path } = spec;
86        let Some(url) = git else {
87            if tag.is_some() || branch.is_some() || rev.is_some() {
88                return Err(SourceError::RefWithoutGit);
89            }
90            let path = path.ok_or(SourceError::NoLocation)?;
91            return dir(&path).map(Self::Dir).ok_or(SourceError::EmptyPath);
92        };
93        // A leading `-` would reach git as an option.
94        if url.is_empty() || url.starts_with('-') {
95            return Err(SourceError::Url { url });
96        }
97        let at = match (tag, branch, rev) {
98            (None, None, None) => GitRef::Head,
99            (Some(tag), None, None) => GitRef::Tag(tag),
100            (None, Some(branch), None) => GitRef::Branch(branch),
101            (None, None, Some(rev)) => GitRef::Rev(Oid::try_from(rev)?),
102            _ => return Err(SourceError::TwoRefs),
103        };
104        let path = path.filter(|p| !p.is_empty()).map(RelPath::try_from).transpose()?;
105        Ok(Self::Git { url, at, path })
106    }
107}
108
109/// `path` without `.` components, so one directory has one spelling; `None` when empty.
110fn dir(path: &str) -> Option<Utf8PathBuf> {
111    if path.is_empty() {
112        return None;
113    }
114    let dir: Utf8PathBuf =
115        Utf8Path::new(path).components().filter(|c| *c != Utf8Component::CurDir).collect();
116    Some(if dir.as_str().is_empty() { Utf8PathBuf::from(".") } else { dir })
117}
118
119impl From<Source> for SourceSpec {
120    fn from(source: Source) -> Self {
121        match source {
122            Source::Dir(path) => Self { path: Some(path.into_string()), ..Self::default() },
123            Source::Git { url, at, path } => {
124                let mut spec =
125                    Self { git: Some(url), path: path.map(String::from), ..Self::default() };
126                match at {
127                    GitRef::Head => {},
128                    GitRef::Branch(branch) => spec.branch = Some(branch),
129                    GitRef::Tag(tag) => spec.tag = Some(tag),
130                    GitRef::Rev(rev) => spec.rev = Some(rev.into()),
131                }
132                spec
133            },
134        }
135    }
136}
137
138impl JsonSchema for Source {
139    fn schema_name() -> Cow<'static, str> {
140        SourceSpec::schema_name()
141    }
142
143    fn json_schema(generator: &mut SchemaGenerator) -> Schema {
144        SourceSpec::json_schema(generator)
145    }
146}
147
148impl Source {
149    /// Whether `other` is this profile at another ref: one repository and path, two refs.
150    pub(crate) fn twin(&self, other: &Self) -> bool {
151        match (self, other) {
152            (
153                Self::Git { url, at, path },
154                Self::Git { url: other_url, at: other_at, path: other_path },
155            ) => url == other_url && path == other_path && at != other_at,
156            (Self::Dir(_) | Self::Git { .. }, Self::Dir(_) | Self::Git { .. }) => false,
157        }
158    }
159}
160
161impl fmt::Display for Source {
162    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
163        match self {
164            Self::Dir(path) => f.write_str(path.as_str()),
165            Self::Git { url, at, path } => {
166                f.write_str(url)?;
167                match at {
168                    GitRef::Head => {},
169                    GitRef::Branch(branch) => write!(f, " branch {branch}")?,
170                    GitRef::Tag(tag) => write!(f, " tag {tag}")?,
171                    GitRef::Rev(rev) => write!(f, " rev {rev}")?,
172                }
173                path.as_ref().map_or(Ok(()), |path| write!(f, " path {path}"))
174            },
175        }
176    }
177}
178
179impl Oid {
180    /// The id as hex.
181    #[must_use]
182    pub fn as_str(&self) -> &str {
183        &self.0
184    }
185
186    /// Its first seven digits, as git abbreviates it.
187    #[must_use]
188    pub fn short(&self) -> &str {
189        self.0.get(..7).unwrap_or(&self.0)
190    }
191}
192
193impl TryFrom<String> for Oid {
194    type Error = SourceError;
195
196    fn try_from(mut id: String) -> Result<Self, SourceError> {
197        if !matches!(id.len(), 40 | 64) || !id.bytes().all(|b| b.is_ascii_hexdigit()) {
198            return Err(SourceError::Rev { rev: id });
199        }
200        id.make_ascii_lowercase();
201        Ok(Self(id.into_boxed_str()))
202    }
203}
204
205/// Remote access for a git layer, reported through [`Cache::on_fetch`].
206#[derive(Clone, Copy, Debug)]
207pub enum Fetch<'a> {
208    /// About to contact this URL.
209    Start(&'a str),
210    /// Finished with it.
211    Done(&'a str),
212}
213
214/// What [`Cache::on_fetch`] installs.
215type Hook = Arc<dyn Fn(Fetch<'_>) + Send + Sync>;
216
217/// Fetched git profiles, shared by every target on the machine.
218///
219/// One process sees one commit per ref: each ref is asked of its remote once.
220#[derive(Clone, derive_more::Debug)]
221pub struct Cache {
222    /// Where it lives.
223    dir: Utf8PathBuf,
224    /// Told about remote access.
225    #[debug(skip)]
226    hook: Option<Hook>,
227    /// Refs resolved so far, by URL and ref name, so one process sees one commit per ref.
228    #[debug(skip)]
229    refs: Arc<Mutex<HashMap<(String, String), Oid>>>,
230    /// Whether git may ask for credentials on the terminal.
231    prompts: bool,
232}
233
234impl Cache {
235    /// The per-user cache: `$XDG_CACHE_HOME/devset`, or the platform's equivalent.
236    ///
237    /// # Errors
238    /// [`Error::Io`](crate::Error::Io), there is no home directory, or its path is not UTF-8.
239    pub fn user() -> Result<Self> {
240        let base = etcetera::choose_base_strategy().map_err(io::Error::other)?;
241        let dir = Utf8PathBuf::try_from(base.cache_dir()).map_err(io::Error::other)?;
242        Ok(Self::at(dir.join("devset")))
243    }
244
245    /// A cache in `dir`.
246    #[must_use]
247    pub fn at(dir: Utf8PathBuf) -> Self {
248        Self { dir, hook: None, refs: Arc::new(Mutex::new(HashMap::new())), prompts: false }
249    }
250
251    /// This cache, calling `hook` around every remote access so a caller can show progress.
252    #[must_use]
253    pub fn on_fetch<F: Fn(Fetch<'_>) + Send + Sync + 'static>(mut self, hook: F) -> Self {
254        self.hook = Some(Arc::new(hook));
255        self
256    }
257
258    /// This cache, letting git ask for credentials on the terminal when `allowed`.
259    ///
260    /// Off unless asked for, so a run with no one to answer fails rather than waits.
261    #[must_use]
262    pub const fn prompting(mut self, allowed: bool) -> Self {
263        self.prompts = allowed;
264        self
265    }
266
267    /// Whether git may ask for credentials on the terminal.
268    pub(crate) const fn prompts(&self) -> bool {
269        self.prompts
270    }
271
272    /// Where the cache lives.
273    #[must_use]
274    pub fn path(&self) -> &Utf8Path {
275        &self.dir
276    }
277
278    /// The commit `name` at `url` was resolved to earlier in this process.
279    pub(crate) fn resolved(&self, url: &str, name: &str) -> Option<Oid> {
280        let refs = self.refs.lock().ok()?;
281        refs.get(&(url.to_owned(), name.to_owned())).cloned()
282    }
283
284    /// Remembers that `name` at `url` is `rev`.
285    pub(crate) fn remember(&self, url: &str, name: &str, rev: &Oid) {
286        if let Ok(mut refs) = self.refs.lock() {
287            refs.insert((url.to_owned(), name.to_owned()), rev.clone());
288        }
289    }
290
291    /// Reports `event` to the hook, if any.
292    pub(crate) fn notify(&self, event: Fetch<'_>) {
293        if let Some(hook) = &self.hook {
294            hook(event);
295        }
296    }
297}
298
299/// A source opened at one revision.
300#[derive(Debug)]
301pub(crate) enum Reader {
302    /// A local directory.
303    Dir(Utf8PathBuf),
304    /// A commit in the cache.
305    Git(git::Commit),
306}
307
308impl Source {
309    /// This source as written from `from`, written from `to` instead: a local directory, or a
310    /// local repository, relative to `to`; unchanged when it is absolute or remote.
311    #[must_use]
312    pub fn rebased(self, from: &Utf8Path, to: &Utf8Path) -> Self {
313        match self {
314            Self::Dir(dir) if dir.is_absolute() => Self::Dir(dir),
315            Self::Dir(dir) => Self::Dir(relative(&from.join(dir), to)),
316            Self::Git { url, at, path } if !remote(&url) && !Utf8Path::new(&url).is_absolute() => {
317                let url = relative(&from.join(&url), to).into_string();
318                Self::Git { url, at, path }
319            },
320            git @ Self::Git { .. } => git,
321        }
322    }
323
324    /// The releases newer than the tag this source is pinned to, oldest first: tags that read as
325    /// versions, with a `v` or without, later than it, prereleases left out. None for a source
326    /// not pinned to such a tag.
327    ///
328    /// # Errors
329    /// [`Error::Source`](crate::Error::Source), git cannot list the remote's tags.
330    pub fn newer_tags(&self, root: &Utf8Path, cache: &Cache) -> Result<Vec<String>> {
331        let Self::Git { url, at: GitRef::Tag(tag), .. } = self else { return Ok(Vec::new()) };
332        let Some(have) = version(tag) else { return Ok(Vec::new()) };
333        let mut newer: Vec<(semver::Version, String)> =
334            git::tags(&locate(url, root), url, cache.prompts())?
335                .into_iter()
336                .filter_map(|name| version(&name).map(|version| (version, name)))
337                .filter(|(version, _)| version.pre.is_empty() && *version > have)
338                .collect();
339        newer.sort();
340        Ok(newer.into_iter().map(|(_, name)| name).collect())
341    }
342
343    /// Opens the source: a git source at `pin` if given, else at its ref, fetching as needed.
344    pub(crate) fn open(&self, root: &Utf8Path, pin: Option<&Oid>, cache: &Cache) -> Result<Reader> {
345        match self {
346            Self::Dir(dir) => Ok(Reader::Dir(lexical(&root.join(dir)))),
347            Self::Git { url, at, path } => {
348                git::Commit::open(&locate(url, root), url, at, path.as_ref(), pin, cache)
349                    .map(Reader::Git)
350            },
351        }
352    }
353}
354
355/// `tag` as a version, `v1.2.3` or `1.2.3`; `None` for a tag that is not one.
356fn version(tag: &str) -> Option<semver::Version> {
357    semver::Version::parse(tag.strip_prefix('v').unwrap_or(tag)).ok()
358}
359
360/// `url` as `git` sees it from any directory: a local path is made absolute against `root`.
361fn locate<'a>(url: &'a str, root: &Utf8Path) -> Cow<'a, str> {
362    if remote(url) {
363        Cow::Borrowed(url)
364    } else {
365        Cow::Owned(lexical(&root.join(url)).into_string())
366    }
367}
368
369/// Whether `url` names a remote repository rather than a local path.
370///
371/// Git's rule: `host:path` is scp-like only when the colon precedes any slash.
372fn remote(url: &str) -> bool {
373    url.contains("://")
374        || url.find(':').is_some_and(|colon| url.find('/').is_none_or(|slash| colon < slash))
375}
376
377/// `path` with `.` dropped and each `..` taking back the directory before it, by name alone, as
378/// a shell resolves `cd ../x`; so a path under a directory not yet made still resolves.
379fn lexical(path: &Utf8Path) -> Utf8PathBuf {
380    let mut resolved = Utf8PathBuf::new();
381    for component in path.components() {
382        match component {
383            Utf8Component::CurDir => {},
384            Utf8Component::ParentDir
385                if matches!(resolved.components().next_back(), Some(Utf8Component::Normal(_))) =>
386            {
387                resolved.pop();
388            },
389            Utf8Component::ParentDir
390            | Utf8Component::Prefix(_)
391            | Utf8Component::RootDir
392            | Utf8Component::Normal(_) => resolved.push(component),
393        }
394    }
395    resolved
396}
397
398/// `path` relative to `base`, both resolved by name: `../p` for `/a/p` from `/a/repo`.
399fn relative(path: &Utf8Path, base: &Utf8Path) -> Utf8PathBuf {
400    let (path, base) = (lexical(path), lexical(base));
401    let shared = path.components().zip(base.components()).take_while(|(a, b)| a == b).count();
402    let up = base.components().skip(shared).map(|_| Utf8Component::ParentDir);
403    let down = path.components().skip(shared);
404    let relative: Utf8PathBuf = up.chain(down).collect();
405    if relative.as_str().is_empty() { Utf8PathBuf::from(".") } else { relative }
406}
407
408impl Reader {
409    /// The commit, for a versioned source.
410    pub(crate) const fn rev(&self) -> Option<&Oid> {
411        match self {
412            Self::Dir(_) => None,
413            Self::Git(commit) => Some(commit.rev()),
414        }
415    }
416
417    /// Every directory in the source that holds a `profile.toml`, relative to its root; `""` for
418    /// the root. Hidden and ignored directories are left out, as git leaves out what it ignores.
419    pub(crate) fn manifests(&self) -> Vec<String> {
420        match self {
421            Self::Git(commit) => commit.manifests(),
422            Self::Dir(root) => local_manifests(root),
423        }
424    }
425
426    /// The files at `paths` under `dir` in the profile, keyed by `paths`; absent ones left out.
427    pub(crate) fn read(&self, dir: &str, paths: &[RelPath]) -> Result<Tree> {
428        match self {
429            Self::Dir(root) => read_dir(&root.join(dir), paths),
430            Self::Git(commit) => commit.read(dir, paths),
431        }
432    }
433}
434
435/// [`Reader::manifests`] for a local directory.
436fn local_manifests(root: &Utf8Path) -> Vec<String> {
437    let mut found: Vec<String> = WalkBuilder::new(root)
438        .require_git(false)
439        .git_global(false)
440        .build()
441        .filter_map(Result::ok)
442        .filter(|entry| entry.file_type().is_some_and(|kind| !kind.is_dir()))
443        .filter(|entry| entry.file_name() == MANIFEST)
444        .filter_map(|entry| {
445            let dir = entry.path().parent()?.strip_prefix(root).ok()?;
446            let dir = Utf8Path::from_path(dir)?;
447            Some(dir.components().map(|part| part.as_str()).collect::<Vec<_>>().join("/"))
448        })
449        .collect();
450    found.sort();
451    found
452}
453
454/// [`Reader::read`] for a local directory.
455fn read_dir(dir: &Utf8Path, paths: &[RelPath]) -> Result<Tree> {
456    let mut tree = Tree::default();
457    for path in paths {
458        let file = path.under(dir);
459        match fs_err::symlink_metadata(&file) {
460            Ok(meta) if meta.is_file() => {
461                tree.insert(path.clone(), |buf| {
462                    fs_err::File::open(&file)?.read_to_end(buf)?;
463                    Ok(())
464                })?;
465            },
466            Ok(_) => {
467                return Err(SourceError::NotAFile { file: file.into_string() }.into());
468            },
469            Err(e) if e.kind() == io::ErrorKind::NotFound => {},
470            Err(e) => return Err(e.into()),
471        }
472    }
473    Ok(tree)
474}
475
476#[cfg(test)]
477mod tests {
478    use super::{GitRef, Source, SourceSpec, version};
479
480    #[test]
481    fn a_release_reads_as_a_version() {
482        let read = |tag: &str| version(tag).map(|version| version.to_string());
483        assert_eq!(read("v1.2.3").as_deref(), Some("1.2.3"), "with a v");
484        assert_eq!(read("1.2.3").as_deref(), Some("1.2.3"), "or without");
485        assert_eq!(read("nightly"), None, "a name is no version");
486    }
487
488    fn parse(toml: &str) -> Result<Source, String> {
489        toml::from_str::<SourceSpec>(toml)
490            .map_err(|e| e.to_string())
491            .and_then(|spec| Source::try_from(spec).map_err(|e| e.to_string()))
492    }
493
494    #[test]
495    fn valid() {
496        assert_eq!(parse(r#"path = "../p""#), Ok(Source::Dir("../p".into())), "local directory");
497        for (written, dir) in [("./../p/", "../p"), ("./p", "p"), (".", "."), ("./", ".")] {
498            let toml = format!("path = \"{written}\"");
499            assert_eq!(parse(&toml), Ok(Source::Dir(dir.into())), "{written:?} has one spelling");
500        }
501        let Ok(Source::Git { url, at, path }) = parse(
502            r#"git = "https://h/r"
503tag = "v1"
504path = "rust""#,
505        ) else {
506            panic!("git source should parse");
507        };
508        assert_eq!(
509            (url.as_str(), at, path.map(String::from)),
510            ("https://h/r", GitRef::Tag("v1".into()), Some("rust".into())),
511            "fields"
512        );
513        let rev = "A".repeat(40);
514        let Ok(Source::Git { at: GitRef::Rev(oid), .. }) =
515            parse(&format!("git = \"u\"\nrev = \"{rev}\""))
516        else {
517            panic!("rev should parse");
518        };
519        assert_eq!(oid.as_str(), "a".repeat(40), "commit ids are lowercased");
520    }
521
522    #[test]
523    fn invalid() {
524        for toml in [
525            "",
526            r#"path = """#,
527            r#"tag = "v1""#,
528            r#"path = "p"
529branch = "b""#,
530            r#"git = "u"
531tag = "v1"
532branch = "b""#,
533            r#"git = "u"
534rev = "abc""#,
535            r#"git = "--upload-pack=x""#,
536            r#"git = "u"
537path = "../x""#,
538            r#"git = "u"
539bogus = 1"#,
540        ] {
541            assert!(parse(toml).is_err(), "should be rejected: {toml:?}");
542        }
543    }
544
545    #[test]
546    fn local_git_paths_are_anchored() {
547        let root = camino::Utf8Path::new("/work/repo");
548        for (url, want) in [
549            ("../p.git", "/work/p.git"),
550            ("/abs/p.git", "/abs/p.git"),
551            ("https://h/p", "https://h/p"),
552            ("file:///p", "file:///p"),
553            ("git@h:org/p", "git@h:org/p"),
554            ("./a:b", "/work/repo/a:b"),
555        ] {
556            assert_eq!(super::locate(url, root), want, "{url}");
557        }
558    }
559
560    #[test]
561    fn sources_are_rebased_by_name() {
562        let (cwd, root) = (camino::Utf8Path::new("/w"), camino::Utf8Path::new("/w/hello"));
563        let dir = Source::Dir("p".into()).rebased(cwd, root);
564        assert_eq!(dir, Source::Dir("../p".into()), "a directory, from the target");
565        let git = |url: &str| Source::Git { url: url.to_owned(), at: GitRef::Head, path: None };
566        assert_eq!(git("p.git").rebased(cwd, root), git("../p.git"), "a local repository");
567        for kept in ["https://h/p", "git@h:org/p", "/abs/p.git"] {
568            assert_eq!(git(kept).rebased(cwd, root), git(kept), "{kept} is left as it is");
569        }
570        assert_eq!(super::relative("/w/hello".into(), root), camino::Utf8Path::new("."), "itself");
571        let absolute = Source::Dir("/abs/p".into()).rebased(cwd, root);
572        assert_eq!(absolute, Source::Dir("/abs/p".into()), "an absolute directory stays so");
573    }
574
575    #[test]
576    fn round_trips() {
577        let toml = "git = \"https://h/r\"\nbranch = \"main\"\npath = \"a/b\"\n";
578        let source = parse(toml).unwrap();
579        assert_eq!(toml::to_string(&source).unwrap(), toml, "serialized as written");
580    }
581}
devset_core/source.rs

devset_core/
source.rs
