Skip to main content

lockdocs_core/query/
packages.rs

1//! Dependency selection, exact-version sources, and index preparation.
2
3use super::{Answer, Engine};
4use crate::extract::Kind;
5use crate::index::{self, PackageIndex};
6use crate::locate::{self, Source};
7use crate::project::Spec;
8use crate::{embed, fetch, upstream, Dep, Eco};
9use rayon::prelude::*;
10use serde_json::{json, Value};
11use std::collections::HashSet;
12use std::path::PathBuf;
13use std::sync::Arc;
14
15/// Cross-dependency searches index at most this many direct dependencies.
16const MAX_PACKAGES: usize = 80;
17
18/// An indexed package, the dependency it answers for, and a drift note.
19pub(super) type ReadyPkg = (Arc<PackageIndex>, Dep, Option<String>);
20
21/// A resolved package ready to search.
22pub(super) enum Resolved {
23    Ready(Arc<PackageIndex>, Dep, Option<String>),
24    Missing(Dep, String),
25}
26
27/// Keep the original flat fields for CLI consumers; richer provenance is additive.
28pub(super) fn fetch_report(dep: &Dep, status: &upstream::Manifest) -> Value {
29    json!({"package": dep.id(), "repo": status.repo, "tag": status.tag,
30        "files": status.files, "note": status.note, "upstream": status})
31}
32
33impl Engine {
34    pub fn resolve(&self, filter: Option<&str>) -> Answer {
35        let p = &self.project;
36        let filter = filter.map(|f| f.trim().to_ascii_lowercase()).filter(|f| !f.is_empty());
37        let shown: Vec<&Dep> = match &filter {
38            Some(f) => p.deps.iter().filter(|d| d.name.to_ascii_lowercase().contains(f.as_str())).collect(),
39            None => p.direct().collect(),
40        };
41        let rows: Vec<(&Dep, Option<Source>)> = shown
42            .par_iter()
43            .map(|d| (*d, locate::locate(d, &p.root).or_else(|| fetch::cached_for(d, Some(&p.root)))))
44            .collect();
45        let mut text = format!(
46            "Project: {}\nLockfiles: {}\n",
47            p.root.display(),
48            if p.lockfiles.is_empty() {
49                "none found".to_string()
50            } else {
51                p.lockfiles.join(", ")
52            }
53        );
54        for w in &p.warnings {
55            text.push_str(&format!("Warning: {w}\n"));
56        }
57        let total_direct = p.direct().count();
58        text.push_str(&match &filter {
59            Some(f) => format!("\n{} of {} dependencies match `{f}`:\n", rows.len(), p.deps.len()),
60            None => format!(
61                "\n{} direct dependencies ({} total with transitive; pass a filter to search all):\n",
62                total_direct,
63                p.deps.len()
64            ),
65        });
66        let w = rows.iter().map(|(d, _)| d.name.len() + d.version.len() + 1).max().unwrap_or(10).min(48);
67        let mut items = Vec::new();
68        for (d, s) in &rows {
69            let id = format!("{}@{}", d.name, d.version);
70            let (status, src) = match s {
71                Some(s) if s.version == d.version => ("docs ready", s.label.clone()),
72                Some(s) => ("version drift", format!("installed {} at {}", s.version, s.label)),
73                None => (
74                    "not installed",
75                    if self.opts.fetch {
76                        "will fetch on first query".into()
77                    } else {
78                        "install it, or enable fetching".to_string()
79                    },
80                ),
81            };
82            text.push_str(&format!(
83                "  {:<6} {:<w$}  {}{}  {}\n",
84                d.eco.as_str(),
85                id,
86                if d.direct { "" } else { "(transitive) " },
87                status,
88                src
89            ));
90            items.push(json!({"ecosystem": d.eco.as_str(), "name": d.name, "version": d.version, "direct": d.direct, "lockfile": d.from, "status": status, "source": src}));
91        }
92        if rows.is_empty() {
93            text.push_str("  (none)\n");
94        }
95        Answer {
96            text,
97            json: json!({"root": p.root.display().to_string(), "lockfiles": p.lockfiles, "dependencies": items, "total": p.deps.len(), "warnings": p.warnings}),
98        }
99    }
100
101    /// Dependencies for a package spec. A version not in the project is
102    /// fetched when allowed.
103    fn deps_for(&self, spec_str: &str) -> Result<Vec<Dep>, String> {
104        let spec = Spec::parse(spec_str);
105        let found = self.project.find(&spec);
106        if let Some(d) = found.iter().find(|d| spec.version_matches(&d.version)) {
107            return Ok(vec![(*d).clone()]);
108        }
109        let eco = spec.eco.or_else(|| found.first().map(|d| d.eco));
110        if let (Some(v), Some(eco)) = (&spec.version, eco) {
111            if spec.is_exact() {
112                let version = if eco == Eco::Go && !v.starts_with('v') { format!("v{v}") } else { v.clone() };
113                let dep = Dep {
114                    eco,
115                    name: found.first().map_or(spec.name.clone(), |d| d.name.clone()),
116                    version,
117                    direct: false,
118                    from: "requested".into(),
119                };
120                if fetch::cached_for(&dep, Some(&self.project.root)).is_some() || self.opts.fetch {
121                    return Ok(vec![dep]);
122                }
123                return Err(format!(
124                    "{}@{} is not in this project{}; enable fetching (--fetch or LOCKDOCS_FETCH=1) to read that exact version.",
125                    dep.name,
126                    dep.version,
127                    found.first().map(|d| format!(" (it pins {})", d.version)).unwrap_or_default()
128                ));
129            }
130        }
131        if let Some(d) = found.first() {
132            return Err(format!("this project pins {} {}, which does not match `{}`.", d.name, d.version, spec_str));
133        }
134        if spec.eco.is_none() && spec.version.is_some() && self.opts.fetch {
135            return Err(format!(
136                "`{}` is not a dependency here; prefix the ecosystem to fetch it (npm:, pypi:, cargo:, go:).",
137                spec.name
138            ));
139        }
140        let near = self.project.similar(&spec.name, 6);
141        Err(if near.is_empty() {
142            format!(
143                "`{}` is not a dependency of {} (lockfiles: {}). Use `resolve` to list dependencies.",
144                spec.name,
145                self.project.root.display(),
146                if self.project.lockfiles.is_empty() {
147                    "none found".into()
148                } else {
149                    self.project.lockfiles.join(", ")
150                }
151            )
152        } else {
153            format!("`{}` is not a dependency here. Did you mean: {}?", spec.name, near.join(", "))
154        })
155    }
156
157    /// Direct dependencies mentioned by name in the query, if any.
158    fn mentioned(&self, query: &str) -> Vec<Dep> {
159        let q = query.to_ascii_lowercase();
160        let words: HashSet<&str> = q
161            .split(|c: char| c.is_whitespace() || matches!(c, ',' | '?' | '(' | ')' | '`' | '\'' | '"'))
162            .collect();
163        let mut out: Vec<Dep> = Vec::new();
164        for d in self.project.direct() {
165            let n = d.name.to_ascii_lowercase();
166            let short = n.rsplit('/').next().unwrap_or(&n);
167            if (words.contains(n.as_str())
168                || (d.eco == Eco::Go && words.contains(short))
169                || words.iter().any(|w| w.starts_with(&format!("{n}.")) || w.starts_with(&format!("{n}::"))))
170                && !out.iter().any(|o| o.name == d.name)
171            {
172                out.push(d.clone());
173            }
174        }
175        out
176    }
177
178    pub(super) fn select(&self, package: Option<&str>, query: &str) -> Result<(Vec<Dep>, bool), String> {
179        if let Some(p) = package.filter(|p| !p.trim().is_empty()) {
180            let mut out = Vec::new();
181            for spec in p.split(',').map(str::trim).filter(|s| !s.is_empty()) {
182                out.extend(self.deps_for(spec)?);
183            }
184            return Ok((out, false));
185        }
186        let m = self.mentioned(query);
187        if !m.is_empty() {
188            return Ok((m, false));
189        }
190        let mut seen = HashSet::new();
191        let direct: Vec<Dep> = self
192            .project
193            .direct()
194            .filter(|d| seen.insert((d.eco, d.name.clone())))
195            .take(MAX_PACKAGES)
196            .cloned()
197            .collect();
198        if direct.is_empty() {
199            return Err(format!(
200                "No dependencies found in {} (looked for {}). Pass `package`, or run from the project directory.",
201                self.project.root.display(),
202                "package-lock.json, pnpm-lock.yaml, yarn.lock, bun.lock, Cargo.lock, uv.lock, poetry.lock, requirements*.txt, go.mod"
203            ));
204        }
205        Ok((direct, true))
206    }
207
208    fn block(&self, req: &crate::private::PrivateRequired) {
209        let mut b = self.private_blocked.lock().unwrap();
210        if !b.iter().any(|r| r.package == req.package) {
211            b.push(req.clone());
212        }
213    }
214
215    /// Find sources for a dependency: installed files first, then a
216    /// previously fetched copy, then (only when enabled) the registry.
217    fn source(&self, dep: &Dep) -> Result<(Source, Option<String>), String> {
218        let local = locate::locate(dep, &self.project.root);
219        if let Some(s) = &local {
220            if s.version == dep.version {
221                return Ok((s.clone(), None));
222            }
223        }
224        if let Some(s) = fetch::cached_for(dep, Some(&self.project.root)) {
225            return Ok((s, None));
226        }
227        if !self.opts.private {
228            if let Some(req) = crate::private::needs_private(dep, &self.project.root) {
229                self.block(&req);
230                return Err(format!("{req}. Run `lockdocs licence status`."));
231            }
232        }
233        if self.opts.fetch {
234            match fetch::fetch_for(dep, Some(&self.project.root), self.opts.private) {
235                Ok(s) => return Ok((s, None)),
236                Err(e) => {
237                    if let Some(req) = e.downcast_ref::<crate::private::PrivateRequired>() {
238                        self.block(req);
239                        return Err(format!("{req}. Run `lockdocs licence status`."));
240                    }
241                    return Err(format!("{}: exact registry fetching failed: {e:#}; no other version substituted", dep.id()));
242                }
243            }
244        }
245        if let Some(s) = local {
246            return Err(format!(
247                "{} pins {}, but {} has {}; refusing to substitute the installed version. Reinstall to sync, or enable exact registry fetching (--fetch).",
248                dep.from,
249                dep.id(),
250                s.label,
251                s.version
252            ));
253        }
254        let how = match dep.eco {
255            Eco::Npm => "run your package manager's install",
256            Eco::PyPI => "create the virtualenv (.venv) and install",
257            Eco::Cargo => "run `cargo fetch`",
258            Eco::Go => "run `go mod download`",
259        };
260        Err(format!(
261            "{} is pinned in {} but its files are not on this machine: {how}, or enable fetching (--fetch or LOCKDOCS_FETCH=1) to download exactly this version.",
262            dep.id(),
263            dep.from
264        ))
265    }
266
267    pub(super) fn index(&self, dep: &Dep) -> Resolved {
268        match self.source(dep) {
269            Ok((src, mut note)) => {
270                let key = format!(
271                    "{}:{}@{}:{}:{}",
272                    dep.eco,
273                    dep.name,
274                    src.version,
275                    src.dir.display(),
276                    index::source_stamp(dep, &src, &self.project.root)
277                );
278                if let Some(i) = self.indexes.lock().unwrap().get(&key) {
279                    // Rebuild once the embedding model has arrived.
280                    if !i.embed.is_empty() || !embed::available() {
281                        if let Some(n) = self.upstream_notes.lock().unwrap().get(&dep.id()) {
282                            note = Some(match note {
283                                Some(old) => format!("{old} {n}"),
284                                None => n.clone(),
285                            });
286                        }
287                        return Resolved::Ready(i.clone(), dep.clone(), note);
288                    }
289                }
290                let (up, up_note) = self.upstream(dep, &src);
291                if let Some(n) = up_note {
292                    self.upstream_notes.lock().unwrap().insert(dep.id(), n.clone());
293                    note = Some(match note {
294                        Some(old) => format!("{old} {n}"),
295                        None => n,
296                    });
297                } else {
298                    self.upstream_notes.lock().unwrap().remove(&dep.id());
299                }
300                let idx = Arc::new(index::load_or_build(dep, &src, &self.project.root, up.as_ref()));
301                self.indexes.lock().unwrap().insert(key, idx.clone());
302                Resolved::Ready(idx, dep.clone(), note)
303            }
304            Err(e) => Resolved::Missing(dep.clone(), e),
305        }
306    }
307
308    /// Upstream docs for this exact version: a cached copy, or (when fetching
309    /// is enabled) a one-time download.
310    fn upstream(&self, dep: &Dep, src: &Source) -> (Option<(PathBuf, upstream::Manifest)>, Option<String>) {
311        if src.version != dep.version || std::env::var("LOCKDOCS_NO_UPSTREAM").is_ok() {
312            return (None, None);
313        }
314        let enabled = self.opts.fetch || self.opts.upstream;
315        if fetch::is_git(dep) {
316            return (
317                None,
318                enabled.then(|| "upstream docs skipped: git dependency's resolved commit is not a release tag; using its checkout files".into()),
319            );
320        }
321        let cached = upstream::cached(dep);
322        if let Some(c) = cached.as_ref().filter(|(_, m)| !enabled || !upstream::needs_refresh(m, self.opts.fetch)) {
323            let note = if upstream::needs_refresh(&c.1, false) {
324                Some(format!(
325                    "offline fallback: upstream cache format {} has not been exact-tag revalidated{}{}",
326                    c.1.format,
327                    if c.1.format < 4 {
328                        "; its old commit candidate may have been a branch"
329                    } else {
330                        ""
331                    },
332                    c.1.note.as_ref().map(|n| format!("; {n}")).unwrap_or_default()
333                ))
334            } else {
335                c.1.note.clone()
336            };
337            return (Some(c.clone()), note);
338        }
339        if enabled {
340            let result = if self.opts.fetch {
341                upstream::fetch_private(dep, src, self.opts.private)
342            } else {
343                upstream::fetch_automatic(dep, src)
344            };
345            return match result {
346                Ok(m) => (Some((upstream::dir(dep), m.clone())), m.note.clone()),
347                Err(e) => (None, Some(format!("upstream docs fetch failed: {e:#}; using package files"))),
348            };
349        }
350        (None, None)
351    }
352
353    /// `lockdocs fetch`: download what makes answers complete, once: missing
354    /// packages at their pinned versions, upstream docs at each version's git
355    /// tag, and the embedding model.
356    pub fn fetch_all(&self, package: Option<&str>) -> Result<Answer, String> {
357        let t = std::time::Instant::now();
358        let mut text = String::new();
359        if embed::enabled() {
360            match embed::ensure() {
361                Ok(()) => text.push_str(&format!("  model   {} ready\n", embed::MODEL_ID)),
362                Err(e) => text.push_str(&format!("  model   {} unavailable ({e:#}); keyword search only\n", embed::MODEL_ID)),
363            }
364        }
365        let deps: Vec<Dep> = match package.filter(|p| !p.trim().is_empty()) {
366            Some(p) => {
367                let mut out = Vec::new();
368                for spec in p.split(',').map(str::trim).filter(|s| !s.is_empty()) {
369                    out.extend(self.deps_for(spec)?);
370                }
371                out
372            }
373            None => {
374                let mut seen = HashSet::new();
375                self.project.direct().filter(|d| seen.insert((d.eco, d.name.clone()))).cloned().collect()
376            }
377        };
378        let rows: Vec<(Dep, String, Value)> = deps
379            .par_iter()
380            .map(|d| {
381                if fetch::is_git(d) {
382                    let src = locate::locate(d, &self.project.root);
383                    return (
384                        d.clone(),
385                        "upstream skipped: git dependency; use resolved checkout files".into(),
386                        json!({"package": d.id(), "status": "git-checkout", "source": src.map(|s| s.label)}),
387                    );
388                }
389                let src = match locate::locate(d, &self.project.root)
390                    .filter(|s| s.version == d.version)
391                    .or_else(|| fetch::cached_for(d, Some(&self.project.root)))
392                {
393                    Some(s) => Some(s),
394                    None if !fetch::is_git(d) => match fetch::fetch_for(d, Some(&self.project.root), self.opts.private) {
395                        Ok(s) => Some(s),
396                        Err(e) => {
397                            if let Some(req) = e.downcast_ref::<crate::private::PrivateRequired>() {
398                                self.block(req);
399                            }
400                            None
401                        }
402                    },
403                    None => None,
404                };
405                let Some(src) = src else {
406                    return (
407                        d.clone(),
408                        "package files unavailable".to_string(),
409                        json!({"package": d.id(), "status": "missing"}),
410                    );
411                };
412                let status = match upstream::fetch_private(d, &src, self.opts.private) {
413                    Ok(m) => m,
414                    Err(e) => {
415                        return (
416                            d.clone(),
417                            format!("upstream docs fetch failed: {e:#}"),
418                            json!({"package": d.id(), "status": "fetch-failed", "error": format!("{e:#}")}),
419                        )
420                    }
421                };
422                let line = match (&status.tag, status.files) {
423                    (_, n) if n > 0 => format!(
424                        "upstream docs {}{}: {n} files ({:.1} MB)",
425                        status.tag.as_ref().map(|t| format!("{}@{t}", status.repo)).unwrap_or_default(),
426                        status.site.as_ref().map(|s| format!(" + docs site {s}")).unwrap_or_default(),
427                        status.bytes as f64 / 1e6
428                    ),
429                    _ => format!("no upstream docs ({}: {})", status.repo, status.note.clone().unwrap_or_default()),
430                };
431                (
432                    d.clone(),
433                    if let Some(note) = &status.note { format!("{line}; {note}") } else { line },
434                    fetch_report(d, &status),
435                )
436            })
437            .collect();
438        for (d, line, _) in &rows {
439            text.push_str(&format!("  {:<40} {line}\n", d.id()));
440        }
441        // Rebuild indexes with what was fetched.
442        self.indexes.lock().unwrap().clear();
443        let ms = t.elapsed().as_millis();
444        Ok(Answer {
445            text: format!("Fetched for {} packages in {ms} ms (cached; later queries stay offline):\n{text}", rows.len()),
446            json: json!({"packages": rows.iter().map(|r| r.2.clone()).collect::<Vec<_>>(), "ms": ms, "model": embed::installed()}),
447        })
448    }
449
450    pub(super) fn resolve_all(&self, deps: &[Dep]) -> (Vec<ReadyPkg>, Vec<(Dep, String)>) {
451        let results: Vec<Resolved> = deps.par_iter().map(|d| self.index(d)).collect();
452        let mut ready = Vec::new();
453        let mut missing = Vec::new();
454        for r in results {
455            match r {
456                Resolved::Ready(i, d, n) => ready.push((i, d, n)),
457                Resolved::Missing(d, e) => missing.push((d, e)),
458            }
459        }
460        (ready, missing)
461    }
462
463    /// Build (or load) indexes ahead of time and report what they hold.
464    pub fn warm(&self, package: Option<&str>) -> Result<Answer, String> {
465        let t = std::time::Instant::now();
466        let deps: Vec<Dep> = match package.filter(|p| !p.trim().is_empty()) {
467            Some(p) => self.select(Some(p), "")?.0,
468            None => {
469                let mut seen = HashSet::new();
470                self.project.direct().filter(|d| seen.insert((d.eco, d.name.clone()))).cloned().collect()
471            }
472        };
473        let (ready, missing) = self.resolve_all(&deps);
474        let mut text = String::new();
475        let mut items = Vec::new();
476        for (idx, _, note) in &ready {
477            let symbols = idx.entries.iter().filter(|e| e.kind != Kind::Prose).count();
478            text.push_str(&format!(
479                "  {:<40} {:>6} symbols {:>6} doc sections  ({})\n",
480                idx.id(),
481                symbols,
482                idx.entries.len() - symbols,
483                idx.source
484            ));
485            if let Some(n) = note {
486                text.push_str(&format!("    note: {n}\n"));
487            }
488            items.push(json!({"package": idx.id(), "ecosystem": idx.eco.as_str(), "symbols": symbols, "sections": idx.entries.len() - symbols, "source": idx.source, "upstream": idx.upstream, "note": note, "build_ms": idx.build_ms}));
489        }
490        for (d, e) in &missing {
491            text.push_str(&format!("  {:<40} missing: {e}\n", d.id()));
492        }
493        let ms = t.elapsed().as_millis();
494        text = format!("Indexed {} of {} packages in {ms} ms.\n{text}", ready.len(), deps.len());
495        Ok(Answer {
496            text,
497            json: json!({"packages": items, "missing": missing.iter().map(|(d, e)| json!({"package": d.id(), "error": e})).collect::<Vec<_>>(), "ms": ms}),
498        })
499    }
500}