lex_syntax/
workspace.rs

1//! `lex.toml` manifest parsing and package resolution.
2//!
3//! A `lex.toml` file marks a project root and declares its package
4//! dependencies. Import paths of the form `"pkg-name/module"` are
5//! resolved against this file.
6//!
7//! ## File format
8//!
9//! ```toml
10//! [package]
11//! name = "lex-web"
12//! version = "0.1.0"
13//!
14//! [dependencies]
15//! lex-schema = { path = "../lex-schema" }
16//! # pin to a tag:
17//! lex-schema = { git = "https://github.com/alpibrusl/lex-schema", tag = "v1.2.0" }
18//! # pin to a commit:
19//! lex-schema = { git = "https://github.com/alpibrusl/lex-schema", rev = "abc1234" }
20//! # track a branch:
21//! lex-schema = { git = "https://github.com/alpibrusl/lex-schema", branch = "stable" }
22//! # or from a registry:
23//! lex-schema = { registry = "https://lexhub.alpibru.com", version = "0.9.2" }
24//! ```
25//!
26//! At most one of `tag`, `rev`, `branch` may be set; omitting all three
27//! clones the default branch at HEAD (not reproducible — pin for releases).
28//!
29//! ## Module resolution
30//!
31//! `import "lex-schema/validate" as v` splits into `pkg = "lex-schema"`,
32//! `module = "validate"`. The loader:
33//!
34//! 1. Walks up from the importing file to find the nearest `lex.toml`.
35//! 2. Looks up `lex-schema` in `[dependencies]`.
36//! 3. For `path =`: resolves `{dep_path}/src/validate.lex`; falls back to
37//!    `{dep_path}/validate.lex` if `src/` doesn't exist.
38//! 4. For `git =`: clones the repo into `~/.lex/packages/lex-schema-<ref>/`
39//!    (once; subsequent loads hit the cache), then resolves the same way.
40
41use serde::Deserialize;
42use std::collections::HashMap;
43use std::path::{Path, PathBuf};
44
45// ── Manifest types ────────────────────────────────────────────────────────────
46
47#[derive(Debug, Deserialize)]
48pub struct Manifest {
49    pub package: Option<PackageMeta>,
50    #[serde(default)]
51    pub dependencies: HashMap<String, Dependency>,
52    /// `[store]` — where store-touching commands (`lex branch`,
53    /// `publish`, `merge`, `op`, `store`, ...) keep this project's
54    /// stages, branches, and traces when no `--store` is given (#772).
55    #[serde(default)]
56    pub store: Option<StoreSection>,
57}
58
59/// `[store]` table of `lex.toml`.
60#[derive(Debug, Default, Deserialize)]
61pub struct StoreSection {
62    /// Store root, relative to the manifest's directory (an absolute
63    /// path is used as is). Defaults to `.lex/store` when absent.
64    #[serde(default)]
65    pub path: Option<String>,
66}
67
68#[derive(Debug, Deserialize)]
69pub struct PackageMeta {
70    pub name: String,
71    #[serde(default)]
72    pub version: String,
73    #[serde(default)]
74    pub description: Option<String>,
75    /// Default registry URL for `lex pkg publish` when `--registry` is not supplied.
76    #[serde(default)]
77    pub registry: Option<String>,
78    /// Minimum toolchain this package needs, as `lex = "0.10.15"`.
79    ///
80    /// Packages across the org have declared this for a long time; until
81    /// #803 nothing read it, because serde silently drops unknown fields.
82    /// A dependency that had moved onto a newer stdlib installed without
83    /// complaint into a project pinned older, and the mismatch surfaced
84    /// as `unknown_field` errors naming a function nobody in the
85    /// consuming repo had written.
86    #[serde(default)]
87    pub lex: Option<String>,
88}
89
90/// Parse a `MAJOR.MINOR.PATCH` version into comparable parts.
91///
92/// Deliberately not a semver crate: these are toolchain pins written by
93/// hand in `lex.toml`, always three numeric components, and a
94/// dependency that spells its floor some other way should be skipped
95/// rather than guessed at.
96pub fn parse_version(v: &str) -> Option<(u64, u64, u64)> {
97    let mut it = v.trim().trim_start_matches('v').split('.');
98    let major = it.next()?.parse().ok()?;
99    let minor = it.next()?.parse().ok()?;
100    let patch = it.next()?.parse().ok()?;
101    if it.next().is_some() {
102        return None;
103    }
104    Some((major, minor, patch))
105}
106
107/// Does `running` satisfy a declared `floor`?
108///
109/// `None` when either side is unparseable — the caller should say so
110/// rather than treat "cannot tell" as either satisfied or violated.
111pub fn satisfies_floor(running: &str, floor: &str) -> Option<bool> {
112    Some(parse_version(running)? >= parse_version(floor)?)
113}
114
115#[derive(Debug, Deserialize)]
116#[serde(untagged)]
117pub enum Dependency {
118    Path     { path: String },
119    Git      {
120        git:    String,
121        #[serde(default)]
122        branch: Option<String>,
123        #[serde(default)]
124        tag:    Option<String>,
125        #[serde(default)]
126        rev:    Option<String>,
127    },
128    Registry { registry: String, version: String },
129}
130
131impl Dependency {
132    /// Return an error if more than one of branch/tag/rev is set.
133    pub fn validate(&self) -> Result<(), String> {
134        if let Dependency::Git { branch, tag, rev, .. } = self {
135            let count = [branch, tag, rev].iter().filter(|o| o.is_some()).count();
136            if count > 1 {
137                return Err("at most one of `branch`, `tag`, `rev` may be set on a git dependency".into());
138            }
139        }
140        Ok(())
141    }
142}
143
144impl Manifest {
145    pub fn load(toml_path: &Path) -> Result<Self, String> {
146        let src = std::fs::read_to_string(toml_path)
147            .map_err(|e| format!("reading {}: {e}", toml_path.display()))?;
148        toml::from_str(&src)
149            .map_err(|e| format!("parsing {}: {e}", toml_path.display()))
150    }
151}
152
153// ── Discovery ─────────────────────────────────────────────────────────────────
154
155/// Walk up from `start` (a file or directory) looking for `lex.toml`.
156/// Returns `(toml_path, toml_dir)` for the nearest ancestor that has one.
157pub fn find_manifest(start: &Path) -> Option<(PathBuf, PathBuf)> {
158    let mut dir = if start.is_dir() {
159        start.to_path_buf()
160    } else {
161        start.parent()?.to_path_buf()
162    };
163    loop {
164        let candidate = dir.join("lex.toml");
165        if candidate.exists() {
166            return Some((candidate, dir));
167        }
168        match dir.parent() {
169            Some(p) if p != dir => dir = p.to_path_buf(),
170            _ => return None,
171        }
172    }
173}
174
175// ── Resolution ────────────────────────────────────────────────────────────────
176
177/// Resolve `pkg_name/module_path` to a `.lex` file on disk.
178///
179/// `importer` is the file that contains the import statement; it's used
180/// to locate the nearest `lex.toml`.
181pub fn resolve_package_import(
182    importer: &Path,
183    pkg_name: &str,
184    module_path: &str,
185) -> Result<PathBuf, PackageError> {
186    let (toml_path, toml_dir) = find_manifest(importer).ok_or_else(|| {
187        PackageError::NoManifest {
188            reference: format!("{pkg_name}/{module_path}"),
189            searched_from: importer.display().to_string(),
190        }
191    })?;
192
193    let manifest = Manifest::load(&toml_path)
194        .map_err(|e| PackageError::ManifestParse { path: toml_path.display().to_string(), detail: e })?;
195
196    let dep = manifest.dependencies.get(pkg_name).ok_or_else(|| {
197        PackageError::UnknownPackage {
198            name: pkg_name.to_string(),
199            manifest: toml_path.display().to_string(),
200        }
201    })?;
202
203    let pkg_root = match dep {
204        Dependency::Path { path } => {
205            let raw = toml_dir.join(path);
206            raw.canonicalize().map_err(|e| PackageError::Io {
207                path: raw.display().to_string(),
208                detail: e.to_string(),
209            })?
210        }
211        Dependency::Git { git, branch, tag, rev } => {
212            dep.validate().map_err(|e| PackageError::ManifestParse {
213                path: toml_path.display().to_string(),
214                detail: e,
215            })?;
216            let git_ref = GitRef::from(branch.as_deref(), tag.as_deref(), rev.as_deref());
217            git_ensure_cached(pkg_name, git, &git_ref)?
218        }
219        Dependency::Registry { registry, version } => {
220            registry_ensure_cached(pkg_name, registry, version)?
221        }
222    };
223
224    find_module_file(&pkg_root, module_path).ok_or_else(|| PackageError::ModuleNotFound {
225        pkg: pkg_name.to_string(),
226        module: module_path.to_string(),
227        pkg_root: pkg_root.display().to_string(),
228    })
229}
230
231/// Look for `{module_path}.lex` inside a package root, checking `src/`
232/// first then the root itself.
233fn find_module_file(pkg_root: &Path, module_path: &str) -> Option<PathBuf> {
234    let rel = PathBuf::from(module_path).with_extension("lex");
235    let in_src = pkg_root.join("src").join(&rel);
236    if in_src.exists() {
237        return Some(in_src);
238    }
239    let at_root = pkg_root.join(&rel);
240    if at_root.exists() {
241        return Some(at_root);
242    }
243    None
244}
245
246// ── Git cache ─────────────────────────────────────────────────────────────────
247
248/// Parsed ref from a git dependency declaration.
249#[derive(Debug)]
250enum GitRef<'a> {
251    Branch(&'a str),
252    Tag(&'a str),
253    Rev(&'a str),
254    DefaultBranch,
255}
256
257impl<'a> GitRef<'a> {
258    fn from(branch: Option<&'a str>, tag: Option<&'a str>, rev: Option<&'a str>) -> Self {
259        if let Some(b) = branch { return GitRef::Branch(b); }
260        if let Some(t) = tag    { return GitRef::Tag(t); }
261        if let Some(r) = rev    { return GitRef::Rev(r); }
262        GitRef::DefaultBranch
263    }
264
265    /// Slug appended to the cache directory name to prevent collisions between
266    /// different refs of the same repo.
267    fn cache_slug(&self) -> String {
268        match self {
269            GitRef::Branch(b)    => format!("@branch-{}", sanitize_ref(b)),
270            GitRef::Tag(t)       => format!("@tag-{}", sanitize_ref(t)),
271            GitRef::Rev(r)       => format!("@rev-{}", &r[..r.len().min(12)]),
272            GitRef::DefaultBranch => String::new(),
273        }
274    }
275}
276
277/// Replace characters that are not safe in directory names.
278fn sanitize_ref(r: &str) -> String {
279    r.chars().map(|c| if c.is_alphanumeric() || c == '-' || c == '.' { c } else { '_' }).collect()
280}
281
282/// Return the local cache directory for `pkg_name`, cloning from `url`
283/// at the given ref if it isn't there yet.
284///
285/// Cache root: `$LEX_PACKAGES_DIR` if set, otherwise `~/.lex/packages/`.
286/// Cache key:  `{pkg_name}{ref_slug}` so different tags/revs don't collide.
287fn git_ensure_cached(pkg_name: &str, url: &str, git_ref: &GitRef<'_>) -> Result<PathBuf, PackageError> {
288    let cache_root = packages_cache_dir()?;
289    let dir_name = format!("{}{}", pkg_name, git_ref.cache_slug());
290    let pkg_dir = cache_root.join(&dir_name);
291    if pkg_dir.exists() {
292        return Ok(pkg_dir);
293    }
294    std::fs::create_dir_all(&cache_root).map_err(|e| PackageError::Io {
295        path: cache_root.display().to_string(),
296        detail: e.to_string(),
297    })?;
298
299    let dest = pkg_dir.to_str().unwrap_or(&dir_name);
300
301    let status = match git_ref {
302        GitRef::Rev(rev) => {
303            // Shallow clone is not possible for arbitrary commits; do a full
304            // clone then check out the specific revision.
305            let s = run_git(&["clone", "--quiet", url, dest], url)?;
306            if s {
307                run_git(&["-C", dest, "checkout", "--quiet", rev], url)?;
308                true
309            } else {
310                false
311            }
312        }
313        GitRef::Tag(tag) => run_git(&["clone", "--quiet", "--depth=1", "--branch", tag, url, dest], url)?,
314        GitRef::Branch(branch) => run_git(&["clone", "--quiet", "--depth=1", "--branch", branch, url, dest], url)?,
315        GitRef::DefaultBranch  => run_git(&["clone", "--quiet", "--depth=1", url, dest], url)?,
316    };
317
318    if !status {
319        // Clean up partial clone so a retry doesn't hit the cache check above.
320        let _ = std::fs::remove_dir_all(&pkg_dir);
321        return Err(PackageError::GitFailed {
322            url: url.to_string(),
323            detail: "`git` exited with non-zero status".into(),
324        });
325    }
326
327    pkg_dir.canonicalize().map_err(|e| PackageError::Io {
328        path: pkg_dir.display().to_string(),
329        detail: e.to_string(),
330    })
331}
332
333/// Run a git command and return `Ok(true)` on success, `Ok(false)` on non-zero
334/// exit, or `Err` if git could not be spawned.
335fn run_git(args: &[&str], url: &str) -> Result<bool, PackageError> {
336    let status = std::process::Command::new("git")
337        .args(args)
338        .status()
339        .map_err(|e| PackageError::GitFailed {
340            url: url.to_string(),
341            detail: format!("could not run `git`: {e}"),
342        })?;
343    Ok(status.success())
344}
345
346/// Download a registry package archive and extract it to the local cache.
347///
348/// Cache path: `$LEX_PACKAGES_DIR/{name}-{version}/` (versioned to avoid
349/// collisions with git-cached packages at `{name}/`).
350///
351/// Download URL: `{registry}/v1/pkg/{name}/{version}/archive`
352fn registry_ensure_cached(
353    pkg_name: &str,
354    registry: &str,
355    version: &str,
356) -> Result<PathBuf, PackageError> {
357    let cache_root = packages_cache_dir()?;
358    // Registry packages are cached under `{name}-{version}` to keep multiple
359    // versions side-by-side and separate from git-cached directories.
360    let pkg_dir = cache_root.join(format!("{pkg_name}-{version}"));
361    if pkg_dir.exists() {
362        return Ok(pkg_dir);
363    }
364    std::fs::create_dir_all(&cache_root).map_err(|e| PackageError::Io {
365        path: cache_root.display().to_string(),
366        detail: e.to_string(),
367    })?;
368
369    let url = format!(
370        "{}/v1/pkg/{}/{}/archive",
371        registry.trim_end_matches('/'),
372        pkg_name,
373        version,
374    );
375    let response = ureq::get(&url).call().map_err(|e| PackageError::RegistryFailed {
376        name: pkg_name.to_string(),
377        registry: registry.to_string(),
378        version: version.to_string(),
379        detail: format!("GET {url}: {e}"),
380    })?;
381    if response.status() != 200 {
382        return Err(PackageError::RegistryFailed {
383            name: pkg_name.to_string(),
384            registry: registry.to_string(),
385            version: version.to_string(),
386            detail: format!("GET {url} returned HTTP {}", response.status()),
387        });
388    }
389
390    let archive_bytes = response
391        .into_body()
392        .read_to_vec()
393        .map_err(|e| PackageError::RegistryFailed {
394            name: pkg_name.to_string(),
395            registry: registry.to_string(),
396            version: version.to_string(),
397            detail: format!("reading response body: {e}"),
398        })?;
399
400    let gz = flate2::read::GzDecoder::new(std::io::Cursor::new(&archive_bytes));
401    let mut ar = tar::Archive::new(gz);
402    ar.unpack(&pkg_dir).map_err(|e| PackageError::RegistryFailed {
403        name: pkg_name.to_string(),
404        registry: registry.to_string(),
405        version: version.to_string(),
406        detail: format!("extracting archive: {e}"),
407    })?;
408
409    pkg_dir.canonicalize().map_err(|e| PackageError::Io {
410        path: pkg_dir.display().to_string(),
411        detail: e.to_string(),
412    })
413}
414
415/// The flat package cache root used by the resolver — `$LEX_PACKAGES_DIR`
416/// if set, otherwise `~/.lex/packages/`. Returns `None` only when neither is
417/// available. Exposed so tooling can recognise dependencies whose path
418/// resolves *into* this cache (and are therefore aliases of an
419/// already-installed package rather than independent sources).
420pub fn packages_cache_root() -> Option<PathBuf> {
421    packages_cache_dir().ok()
422}
423
424fn packages_cache_dir() -> Result<PathBuf, PackageError> {
425    if let Ok(dir) = std::env::var("LEX_PACKAGES_DIR") {
426        return Ok(PathBuf::from(dir));
427    }
428    let home = std::env::var("HOME")
429        .or_else(|_| std::env::var("USERPROFILE"))
430        .map_err(|_| PackageError::Io {
431            path: "~/.lex/packages".into(),
432            detail: "could not determine home directory (set LEX_PACKAGES_DIR)".into(),
433        })?;
434    Ok(PathBuf::from(home).join(".lex").join("packages"))
435}
436
437// ── Errors ────────────────────────────────────────────────────────────────────
438
439#[derive(Debug, thiserror::Error)]
440pub enum PackageError {
441    #[error("no lex.toml found searching up from {searched_from} (needed to resolve \"{reference}\")")]
442    NoManifest { reference: String, searched_from: String },
443
444    #[error("failed to parse {path}: {detail}")]
445    ManifestParse { path: String, detail: String },
446
447    #[error("package \"{name}\" not found in {manifest}")]
448    UnknownPackage { name: String, manifest: String },
449
450    #[error("module \"{module}\" not found in package \"{pkg}\" (looked in {pkg_root}/src/ and {pkg_root}/)")]
451    ModuleNotFound { pkg: String, module: String, pkg_root: String },
452
453    #[error("git clone of {url} failed: {detail}")]
454    GitFailed { url: String, detail: String },
455
456    #[error("registry fetch of {name}@{version} from {registry} failed: {detail}")]
457    RegistryFailed { name: String, registry: String, version: String, detail: String },
458
459    #[error("I/O error at {path}: {detail}")]
460    Io { path: String, detail: String },
461}
462
463#[cfg(test)]
464mod floor_tests {
465    use super::{parse_version, satisfies_floor};
466
467    #[test]
468    fn parses_plain_and_v_prefixed() {
469        assert_eq!(parse_version("0.10.18"), Some((0, 10, 18)));
470        assert_eq!(parse_version("v0.10.18"), Some((0, 10, 18)));
471        assert_eq!(parse_version("  1.2.3 "), Some((1, 2, 3)));
472    }
473
474    #[test]
475    fn refuses_what_it_cannot_read_rather_than_guessing() {
476        // A floor it cannot parse must be reported as unknown, never
477        // silently treated as met (which would defeat the check) or as
478        // violated (which would fail working installs).
479        for bad in ["nightly", "0.10", "0.10.18.1", "", "0.x.1", "1.2.3-rc1"] {
480            assert_eq!(parse_version(bad), None, "should not parse: {bad}");
481            assert_eq!(satisfies_floor("0.10.18", bad), None);
482            assert_eq!(satisfies_floor(bad, "0.10.18"), None);
483        }
484    }
485
486    #[test]
487    fn compares_by_component_not_lexically() {
488        // The case that motivated #803: 0.10.11 < 0.10.15, though a
489        // string comparison says otherwise ("0.10.11" > "0.10.15" is
490        // false, but "0.10.9" > "0.10.15" is true lexically).
491        assert_eq!(satisfies_floor("0.10.11", "0.10.15"), Some(false));
492        assert_eq!(satisfies_floor("0.10.18", "0.10.15"), Some(true));
493        assert_eq!(satisfies_floor("0.10.9", "0.10.15"), Some(false));
494        assert_eq!(satisfies_floor("0.9.20", "0.10.0"), Some(false));
495    }
496
497    #[test]
498    fn an_exactly_met_floor_is_met() {
499        assert_eq!(satisfies_floor("0.10.15", "0.10.15"), Some(true));
500    }
501
502    #[test]
503    fn the_field_is_read_from_a_manifest() {
504        // Before #803 this field existed in every lex.toml in the org and
505        // was dropped on the floor by serde, so assert it survives parsing.
506        let m: super::Manifest = toml::from_str(
507            "[package]\nname = \"p\"\nversion = \"0.1.0\"\nlex = \"0.10.15\"\n",
508        )
509        .expect("parse");
510        assert_eq!(m.package.unwrap().lex.as_deref(), Some("0.10.15"));
511    }
512
513    #[test]
514    fn a_manifest_without_the_field_still_parses() {
515        let m: super::Manifest =
516            toml::from_str("[package]\nname = \"p\"\nversion = \"0.1.0\"\n").expect("parse");
517        assert_eq!(m.package.unwrap().lex, None);
518    }
519}
lex_syntax/workspace.rs

lex_syntax/
workspace.rs
