Skip to main content

lockdocs_core/
query.rs

1//! The three questions agents ask: which versions do I have (`resolve`),
2//! what do the docs say about X (`docs`), and what exactly is this symbol
3//! (`api`). Answers are token-budgeted and cite `package@version path:line`.
4
5use crate::bm25::{self, Bm25};
6use crate::extract::{Entry, Kind};
7use crate::index::{self, PackageIndex};
8use crate::locate::{self, Source};
9use crate::project::{Project, Spec};
10use crate::{embed, est_tokens, fetch, upstream, Dep, Eco};
11use rayon::prelude::*;
12use serde_json::{json, Value};
13use std::collections::{HashMap, HashSet};
14use std::path::{Path, PathBuf};
15use std::sync::{Arc, Mutex};
16
17pub const DEFAULT_TOKENS: usize = 1200;
18/// Results scoring below this fraction of the best are left out (after 3).
19const RELEVANCE_FLOOR: f32 = 0.3;
20/// Share of the fused rank score from the embedding ranking; the rest is the
21/// keyword ranking. Small on purpose: measured on the 105-question benchmark,
22/// anything above ~0.15 lets the embedding outvote exact API-name matches.
23const DENSE_WEIGHT: f32 = 0.1;
24/// Reciprocal-rank constant: `1 / (RRF_K + rank)`.
25const RRF_K: f32 = 10.0;
26/// Share of the embedding in the score mix used when upstream docs are indexed.
27const DOCS_DENSE_WEIGHT: f32 = 0.45;
28/// Embedding candidates considered per query.
29const DENSE_POOL: usize = 300;
30/// The keyword leader is final when it scores at least this many times the
31/// runner-up (after boosts); the embedding is not even looked at.
32const DECISIVE_RATIO: f32 = 2.0;
33/// Share of the keyword score from headings and first sentences.
34const HEAD_WEIGHT: f32 = 0.25;
35/// Cross-dependency searches index at most this many direct dependencies.
36const MAX_PACKAGES: usize = 80;
37
38#[derive(Debug, Clone, PartialEq, Eq)]
39pub struct Options {
40    /// Allow downloading exact versions that are not installed.
41    pub fetch: bool,
42    /// Anonymous release-tag docs on first use; independent of registry downloads.
43    pub upstream: bool,
44    /// lockdocs Pro: read the private registries and git hosts the user has
45    /// configured. Set by the binary after it has verified the licence; the
46    /// core never decides this itself.
47    pub private: bool,
48}
49
50impl Default for Options {
51    fn default() -> Self {
52        let mut fetch = std::env::var("LOCKDOCS_FETCH").is_ok_and(|v| matches!(v.as_str(), "1" | "true" | "yes"));
53        let mut upstream = automatic_upstream(std::env::var("LOCKDOCS_FETCH").ok().as_deref(), std::env::var("LOCKDOCS_NO_UPSTREAM").is_ok());
54        if std::env::var("LOCKDOCS_OFFLINE").is_ok_and(|v| v != "0") {
55            fetch = false;
56            upstream = false;
57        }
58        Options {
59            fetch,
60            upstream,
61            private: false,
62        }
63    }
64}
65
66fn automatic_upstream(fetch: Option<&str>, disabled: bool) -> bool {
67    !disabled && !fetch.is_some_and(|v| matches!(v, "0" | "false" | "no"))
68}
69
70pub struct Answer {
71    pub text: String,
72    pub json: Value,
73}
74
75pub struct Engine {
76    pub project: Project,
77    opts: Options,
78    indexes: Mutex<HashMap<String, Arc<PackageIndex>>>,
79    upstream_notes: Mutex<HashMap<String, String>>,
80    /// Packages that needed a private source this engine was not licensed to read.
81    private_blocked: Mutex<Vec<crate::private::PrivateRequired>>,
82}
83
84/// An indexed package, the dependency it answers for, and a drift note.
85type ReadyPkg = (Arc<PackageIndex>, Dep, Option<String>);
86
87/// A resolved package ready to search.
88enum Resolved {
89    Ready(Arc<PackageIndex>, Dep, Option<String>),
90    Missing(Dep, String),
91}
92
93/// Keep the original flat fields for CLI consumers; richer provenance is additive.
94fn fetch_report(dep: &Dep, status: &upstream::Manifest) -> Value {
95    json!({"package": dep.id(), "repo": status.repo, "tag": status.tag,
96        "files": status.files, "note": status.note, "upstream": status})
97}
98
99fn provenance(ready: &[ReadyPkg]) -> Vec<Value> {
100    ready
101        .iter()
102        .map(|(idx, dep, note)| {
103            json!({
104                "package": idx.id(), "requested_version": dep.version, "source": idx.source,
105                "registry_fetched": idx.fetched, "upstream": idx.upstream_provenance, "upstream_label": idx.upstream, "note": note,
106            })
107        })
108        .collect()
109}
110
111/// Keep network fallbacks visible on ordinary broad MCP answers without listing
112/// every dependency's long header or crowding out the actual answer.
113fn broad_header(ready: &[ReadyPkg], missing: &[(Dep, String)], tokens: usize) -> String {
114    let fetched = ready.iter().filter(|r| r.0.fetched).count();
115    let upstream = ready.iter().filter(|r| r.0.upstream.is_some()).count();
116    let notes = ready.iter().filter(|r| r.2.is_some()).count();
117    let mut header = format!("Searched {} direct dependencies: {} local package sources, {fetched} registry sources, {upstream} upstream sources; {notes} fallback notes, {} unavailable.\n", ready.len(), ready.len() - fetched, missing.len());
118    let max_chars = (tokens / 3).clamp(100, 400) * 3;
119    for (idx, _, note) in ready {
120        if let Some(note) = note {
121            let row = format!("Fallback {}: {}\n", idx.id(), truncate(note, 160));
122            if header.chars().count() + row.chars().count() > max_chars {
123                break;
124            }
125            header.push_str(&row);
126        }
127    }
128    for (dep, error) in missing {
129        let row = format!("Skipped {}: {}\n", dep.id(), truncate(error, 140));
130        if header.chars().count() + row.chars().count() > max_chars {
131            break;
132        }
133        header.push_str(&row);
134    }
135    for (idx, _, _) in ready {
136        if let Some(m) = &idx.upstream_provenance {
137            let row = format!(
138                "Upstream {}: {}@{} commit {}{}\n",
139                idx.id(),
140                m.repo,
141                m.tag.as_deref().unwrap_or("none"),
142                m.commit.as_deref().map(|s| &s[..s.len().min(7)]).unwrap_or("none"),
143                if m.site.is_some() { " + opted-in docs site" } else { "" }
144            );
145            if header.chars().count() + row.chars().count() > max_chars {
146                break;
147            }
148            header.push_str(&row);
149        }
150    }
151    header.push_str("Focus `package` or request JSON for full provenance.\n");
152    header
153}
154
155fn lang(eco: Eco) -> &'static str {
156    match eco {
157        Eco::Npm => "ts",
158        Eco::PyPI => "python",
159        Eco::Cargo => "rust",
160        Eco::Go => "go",
161    }
162}
163
164/// Changelog prose: a changelog file, or a section under a version heading
165/// (`v1.7 (2020-10-26)`, `## 2.0.0`), as in READMEs that embed history.
166fn is_changelog(e: &Entry) -> bool {
167    let f = e.file.rsplit('/').next().unwrap_or(&e.file).to_ascii_uppercase();
168    if ["CHANGELOG", "CHANGES", "HISTORY", "NEWS", "RELEASES"].iter().any(|p| f.starts_with(p)) {
169        return true;
170    }
171    e.name.split(" › ").skip(1).any(|h| {
172        let h = h.trim().trim_start_matches(['v', 'V', '[']);
173        let mut parts = h.split(|c: char| !c.is_ascii_digit());
174        matches!((parts.next(), parts.next()), (Some(a), Some(b)) if !a.is_empty() && !b.is_empty()) && h.chars().nth(a_len(h)) == Some('.')
175    })
176}
177
178fn a_len(h: &str) -> usize {
179    h.chars().take_while(|c| c.is_ascii_digit()).count()
180}
181
182fn change_intent(q: &str) -> bool {
183    let q = q.to_ascii_lowercase();
184    [
185        "change",
186        "migrat",
187        "upgrad",
188        "breaking",
189        "deprecat",
190        "removed",
191        "renamed",
192        "new in",
193        "since",
194        "release",
195        "what's new",
196        "differen",
197    ]
198    .iter()
199    .any(|w| q.contains(w))
200}
201
202fn private_path(e: &Entry) -> bool {
203    e.file.split('/').any(|s| s.starts_with('_') && !s.starts_with("__init__")) || e.path.contains("._") || e.file.contains("/internal/")
204}
205
206/// Tooling shipped inside a package (type-checker plugins, test helpers).
207fn tooling_path(e: &Entry) -> bool {
208    e.file.split('/').any(|s| {
209        let stem = s.split('.').next().unwrap_or(s);
210        matches!(
211            stem,
212            "mypy" | "pylint" | "plugin" | "plugins" | "testing" | "conftest" | "scripts" | "codemod" | "codemods" | "eslint" | "babel"
213        )
214    })
215}
216
217impl Engine {
218    pub fn new(root: &Path, opts: Options) -> Engine {
219        Engine {
220            project: Project::load(root),
221            opts,
222            indexes: Mutex::new(HashMap::new()),
223            upstream_notes: Mutex::new(HashMap::new()),
224            private_blocked: Mutex::new(Vec::new()),
225        }
226    }
227
228    fn block(&self, req: &crate::private::PrivateRequired) {
229        let mut b = self.private_blocked.lock().unwrap();
230        if !b.iter().any(|r| r.package == req.package) {
231            b.push(req.clone());
232        }
233    }
234
235    /// The packages that needed a private source without Pro since the last
236    /// call. The binary turns a non-empty list into the `pro_required` answer.
237    pub fn take_private_blocked(&self) -> Vec<crate::private::PrivateRequired> {
238        std::mem::take(&mut *self.private_blocked.lock().unwrap())
239    }
240
241    pub fn root(&self) -> &Path {
242        &self.project.root
243    }
244
245    /// Find sources for a dependency: installed files first, then a
246    /// previously fetched copy, then (only when enabled) the registry.
247    fn source(&self, dep: &Dep) -> Result<(Source, Option<String>), String> {
248        let local = locate::locate(dep, &self.project.root);
249        if let Some(s) = &local {
250            if s.version == dep.version {
251                return Ok((s.clone(), None));
252            }
253        }
254        if let Some(s) = fetch::cached_for(dep, Some(&self.project.root)) {
255            return Ok((s, None));
256        }
257        if !self.opts.private {
258            if let Some(req) = crate::private::needs_private(dep, &self.project.root) {
259                self.block(&req);
260                return Err(format!("{req}. Run `lockdocs licence status`."));
261            }
262        }
263        if self.opts.fetch {
264            match fetch::fetch_for(dep, Some(&self.project.root), self.opts.private) {
265                Ok(s) => return Ok((s, None)),
266                Err(e) => {
267                    if let Some(req) = e.downcast_ref::<crate::private::PrivateRequired>() {
268                        self.block(req);
269                        return Err(format!("{req}. Run `lockdocs licence status`."));
270                    }
271                    return Err(format!("{}: exact registry fetching failed: {e:#}; no other version substituted", dep.id()));
272                }
273            }
274        }
275        if let Some(s) = local {
276            return Err(format!(
277                "{} pins {}, but {} has {}; refusing to substitute the installed version. Reinstall to sync, or enable exact registry fetching (--fetch).",
278                dep.from,
279                dep.id(),
280                s.label,
281                s.version
282            ));
283        }
284        let how = match dep.eco {
285            Eco::Npm => "run your package manager's install",
286            Eco::PyPI => "create the virtualenv (.venv) and install",
287            Eco::Cargo => "run `cargo fetch`",
288            Eco::Go => "run `go mod download`",
289        };
290        Err(format!(
291            "{} is pinned in {} but its files are not on this machine: {how}, or enable fetching (--fetch or LOCKDOCS_FETCH=1) to download exactly this version.",
292            dep.id(),
293            dep.from
294        ))
295    }
296
297    fn index(&self, dep: &Dep) -> Resolved {
298        match self.source(dep) {
299            Ok((src, mut note)) => {
300                let key = format!(
301                    "{}:{}@{}:{}:{}",
302                    dep.eco,
303                    dep.name,
304                    src.version,
305                    src.dir.display(),
306                    index::source_stamp(dep, &src, &self.project.root)
307                );
308                if let Some(i) = self.indexes.lock().unwrap().get(&key) {
309                    // Rebuild once the embedding model has arrived.
310                    if !i.embed.is_empty() || !embed::available() {
311                        if let Some(n) = self.upstream_notes.lock().unwrap().get(&dep.id()) {
312                            note = Some(match note {
313                                Some(old) => format!("{old} {n}"),
314                                None => n.clone(),
315                            });
316                        }
317                        return Resolved::Ready(i.clone(), dep.clone(), note);
318                    }
319                }
320                let (up, up_note) = self.upstream(dep, &src);
321                if let Some(n) = up_note {
322                    self.upstream_notes.lock().unwrap().insert(dep.id(), n.clone());
323                    note = Some(match note {
324                        Some(old) => format!("{old} {n}"),
325                        None => n,
326                    });
327                } else {
328                    self.upstream_notes.lock().unwrap().remove(&dep.id());
329                }
330                let idx = Arc::new(index::load_or_build(dep, &src, &self.project.root, up.as_ref()));
331                self.indexes.lock().unwrap().insert(key, idx.clone());
332                Resolved::Ready(idx, dep.clone(), note)
333            }
334            Err(e) => Resolved::Missing(dep.clone(), e),
335        }
336    }
337
338    /// Upstream docs for this exact version: a cached copy, or (when fetching
339    /// is enabled) a one-time download.
340    fn upstream(&self, dep: &Dep, src: &Source) -> (Option<(PathBuf, upstream::Manifest)>, Option<String>) {
341        if src.version != dep.version || std::env::var("LOCKDOCS_NO_UPSTREAM").is_ok() {
342            return (None, None);
343        }
344        let enabled = self.opts.fetch || self.opts.upstream;
345        if fetch::is_git(dep) {
346            return (
347                None,
348                enabled.then(|| "upstream docs skipped: git dependency's resolved commit is not a release tag; using its checkout files".into()),
349            );
350        }
351        let cached = upstream::cached(dep);
352        if let Some(c) = cached.as_ref().filter(|(_, m)| !enabled || !upstream::needs_refresh(m, self.opts.fetch)) {
353            let note = if upstream::needs_refresh(&c.1, false) {
354                Some(format!(
355                    "offline fallback: upstream cache format {} has not been exact-tag revalidated{}{}",
356                    c.1.format,
357                    if c.1.format < 4 {
358                        "; its old commit candidate may have been a branch"
359                    } else {
360                        ""
361                    },
362                    c.1.note.as_ref().map(|n| format!("; {n}")).unwrap_or_default()
363                ))
364            } else {
365                c.1.note.clone()
366            };
367            return (Some(c.clone()), note);
368        }
369        if enabled {
370            let result = if self.opts.fetch {
371                upstream::fetch_private(dep, src, self.opts.private)
372            } else {
373                upstream::fetch_automatic(dep, src)
374            };
375            return match result {
376                Ok(m) => (Some((upstream::dir(dep), m.clone())), m.note.clone()),
377                Err(e) => (None, Some(format!("upstream docs fetch failed: {e:#}; using package files"))),
378            };
379        }
380        (None, None)
381    }
382
383    /// `lockdocs fetch`: download what makes answers complete, once: missing
384    /// packages at their pinned versions, upstream docs at each version's git
385    /// tag, and the embedding model.
386    pub fn fetch_all(&self, package: Option<&str>) -> Result<Answer, String> {
387        let t = std::time::Instant::now();
388        let mut text = String::new();
389        if embed::enabled() {
390            match embed::ensure() {
391                Ok(()) => text.push_str(&format!("  model   {} ready\n", embed::MODEL_ID)),
392                Err(e) => text.push_str(&format!("  model   {} unavailable ({e:#}); keyword search only\n", embed::MODEL_ID)),
393            }
394        }
395        let deps: Vec<Dep> = match package.filter(|p| !p.trim().is_empty()) {
396            Some(p) => {
397                let mut out = Vec::new();
398                for spec in p.split(',').map(str::trim).filter(|s| !s.is_empty()) {
399                    out.extend(self.deps_for(spec)?);
400                }
401                out
402            }
403            None => {
404                let mut seen = HashSet::new();
405                self.project.direct().filter(|d| seen.insert((d.eco, d.name.clone()))).cloned().collect()
406            }
407        };
408        let rows: Vec<(Dep, String, Value)> = deps
409            .par_iter()
410            .map(|d| {
411                if fetch::is_git(d) {
412                    let src = locate::locate(d, &self.project.root);
413                    return (
414                        d.clone(),
415                        "upstream skipped: git dependency; use resolved checkout files".into(),
416                        json!({"package": d.id(), "status": "git-checkout", "source": src.map(|s| s.label)}),
417                    );
418                }
419                let src = match locate::locate(d, &self.project.root)
420                    .filter(|s| s.version == d.version)
421                    .or_else(|| fetch::cached_for(d, Some(&self.project.root)))
422                {
423                    Some(s) => Some(s),
424                    None if !fetch::is_git(d) => match fetch::fetch_for(d, Some(&self.project.root), self.opts.private) {
425                        Ok(s) => Some(s),
426                        Err(e) => {
427                            if let Some(req) = e.downcast_ref::<crate::private::PrivateRequired>() {
428                                self.block(req);
429                            }
430                            None
431                        }
432                    },
433                    None => None,
434                };
435                let Some(src) = src else {
436                    return (
437                        d.clone(),
438                        "package files unavailable".to_string(),
439                        json!({"package": d.id(), "status": "missing"}),
440                    );
441                };
442                let status = match upstream::fetch_private(d, &src, self.opts.private) {
443                    Ok(m) => m,
444                    Err(e) => {
445                        return (
446                            d.clone(),
447                            format!("upstream docs fetch failed: {e:#}"),
448                            json!({"package": d.id(), "status": "fetch-failed", "error": format!("{e:#}")}),
449                        )
450                    }
451                };
452                let line = match (&status.tag, status.files) {
453                    (_, n) if n > 0 => format!(
454                        "upstream docs {}{}: {n} files ({:.1} MB)",
455                        status.tag.as_ref().map(|t| format!("{}@{t}", status.repo)).unwrap_or_default(),
456                        status.site.as_ref().map(|s| format!(" + docs site {s}")).unwrap_or_default(),
457                        status.bytes as f64 / 1e6
458                    ),
459                    _ => format!("no upstream docs ({}: {})", status.repo, status.note.clone().unwrap_or_default()),
460                };
461                (
462                    d.clone(),
463                    if let Some(note) = &status.note { format!("{line}; {note}") } else { line },
464                    fetch_report(d, &status),
465                )
466            })
467            .collect();
468        for (d, line, _) in &rows {
469            text.push_str(&format!("  {:<40} {line}\n", d.id()));
470        }
471        // Rebuild indexes with what was fetched.
472        self.indexes.lock().unwrap().clear();
473        let ms = t.elapsed().as_millis();
474        Ok(Answer {
475            text: format!("Fetched for {} packages in {ms} ms (cached; later queries stay offline):\n{text}", rows.len()),
476            json: json!({"packages": rows.iter().map(|r| r.2.clone()).collect::<Vec<_>>(), "ms": ms, "model": embed::installed()}),
477        })
478    }
479
480    /// Dependencies for a package spec. A version not in the project is
481    /// fetched when allowed.
482    fn deps_for(&self, spec_str: &str) -> Result<Vec<Dep>, String> {
483        let spec = Spec::parse(spec_str);
484        let found = self.project.find(&spec);
485        if let Some(d) = found.iter().find(|d| spec.version_matches(&d.version)) {
486            return Ok(vec![(*d).clone()]);
487        }
488        let eco = spec.eco.or_else(|| found.first().map(|d| d.eco));
489        if let (Some(v), Some(eco)) = (&spec.version, eco) {
490            if spec.is_exact() {
491                let version = if eco == Eco::Go && !v.starts_with('v') { format!("v{v}") } else { v.clone() };
492                let dep = Dep {
493                    eco,
494                    name: found.first().map_or(spec.name.clone(), |d| d.name.clone()),
495                    version,
496                    direct: false,
497                    from: "requested".into(),
498                };
499                if fetch::cached_for(&dep, Some(&self.project.root)).is_some() || self.opts.fetch {
500                    return Ok(vec![dep]);
501                }
502                return Err(format!(
503                    "{}@{} is not in this project{}; enable fetching (--fetch or LOCKDOCS_FETCH=1) to read that exact version.",
504                    dep.name,
505                    dep.version,
506                    found.first().map(|d| format!(" (it pins {})", d.version)).unwrap_or_default()
507                ));
508            }
509        }
510        if let Some(d) = found.first() {
511            return Err(format!("this project pins {} {}, which does not match `{}`.", d.name, d.version, spec_str));
512        }
513        if spec.eco.is_none() && spec.version.is_some() && self.opts.fetch {
514            return Err(format!(
515                "`{}` is not a dependency here; prefix the ecosystem to fetch it (npm:, pypi:, cargo:, go:).",
516                spec.name
517            ));
518        }
519        let near = self.project.similar(&spec.name, 6);
520        Err(if near.is_empty() {
521            format!(
522                "`{}` is not a dependency of {} (lockfiles: {}). Use `resolve` to list dependencies.",
523                spec.name,
524                self.project.root.display(),
525                if self.project.lockfiles.is_empty() {
526                    "none found".into()
527                } else {
528                    self.project.lockfiles.join(", ")
529                }
530            )
531        } else {
532            format!("`{}` is not a dependency here. Did you mean: {}?", spec.name, near.join(", "))
533        })
534    }
535
536    /// Direct dependencies mentioned by name in the query, if any.
537    fn mentioned(&self, query: &str) -> Vec<Dep> {
538        let q = query.to_ascii_lowercase();
539        let words: HashSet<&str> = q
540            .split(|c: char| c.is_whitespace() || matches!(c, ',' | '?' | '(' | ')' | '`' | '\'' | '"'))
541            .collect();
542        let mut out: Vec<Dep> = Vec::new();
543        for d in self.project.direct() {
544            let n = d.name.to_ascii_lowercase();
545            let short = n.rsplit('/').next().unwrap_or(&n);
546            if (words.contains(n.as_str())
547                || (d.eco == Eco::Go && words.contains(short))
548                || words.iter().any(|w| w.starts_with(&format!("{n}.")) || w.starts_with(&format!("{n}::"))))
549                && !out.iter().any(|o| o.name == d.name)
550            {
551                out.push(d.clone());
552            }
553        }
554        out
555    }
556
557    fn select(&self, package: Option<&str>, query: &str) -> Result<(Vec<Dep>, bool), String> {
558        if let Some(p) = package.filter(|p| !p.trim().is_empty()) {
559            let mut out = Vec::new();
560            for spec in p.split(',').map(str::trim).filter(|s| !s.is_empty()) {
561                out.extend(self.deps_for(spec)?);
562            }
563            return Ok((out, false));
564        }
565        let m = self.mentioned(query);
566        if !m.is_empty() {
567            return Ok((m, false));
568        }
569        let mut seen = HashSet::new();
570        let direct: Vec<Dep> = self
571            .project
572            .direct()
573            .filter(|d| seen.insert((d.eco, d.name.clone())))
574            .take(MAX_PACKAGES)
575            .cloned()
576            .collect();
577        if direct.is_empty() {
578            return Err(format!(
579                "No dependencies found in {} (looked for {}). Pass `package`, or run from the project directory.",
580                self.project.root.display(),
581                "package-lock.json, pnpm-lock.yaml, yarn.lock, bun.lock, Cargo.lock, uv.lock, poetry.lock, requirements*.txt, go.mod"
582            ));
583        }
584        Ok((direct, true))
585    }
586
587    fn resolve_all(&self, deps: &[Dep]) -> (Vec<ReadyPkg>, Vec<(Dep, String)>) {
588        let results: Vec<Resolved> = deps.par_iter().map(|d| self.index(d)).collect();
589        let mut ready = Vec::new();
590        let mut missing = Vec::new();
591        for r in results {
592            match r {
593                Resolved::Ready(i, d, n) => ready.push((i, d, n)),
594                Resolved::Missing(d, e) => missing.push((d, e)),
595            }
596        }
597        (ready, missing)
598    }
599
600    /// Build (or load) indexes ahead of time and report what they hold.
601    pub fn warm(&self, package: Option<&str>) -> Result<Answer, String> {
602        let t = std::time::Instant::now();
603        let deps: Vec<Dep> = match package.filter(|p| !p.trim().is_empty()) {
604            Some(p) => self.select(Some(p), "")?.0,
605            None => {
606                let mut seen = HashSet::new();
607                self.project.direct().filter(|d| seen.insert((d.eco, d.name.clone()))).cloned().collect()
608            }
609        };
610        let (ready, missing) = self.resolve_all(&deps);
611        let mut text = String::new();
612        let mut items = Vec::new();
613        for (idx, _, note) in &ready {
614            let symbols = idx.entries.iter().filter(|e| e.kind != Kind::Prose).count();
615            text.push_str(&format!(
616                "  {:<40} {:>6} symbols {:>6} doc sections  ({})\n",
617                idx.id(),
618                symbols,
619                idx.entries.len() - symbols,
620                idx.source
621            ));
622            if let Some(n) = note {
623                text.push_str(&format!("    note: {n}\n"));
624            }
625            items.push(json!({"package": idx.id(), "ecosystem": idx.eco.as_str(), "symbols": symbols, "sections": idx.entries.len() - symbols, "source": idx.source, "upstream": idx.upstream, "note": note, "build_ms": idx.build_ms}));
626        }
627        for (d, e) in &missing {
628            text.push_str(&format!("  {:<40} missing: {e}\n", d.id()));
629        }
630        let ms = t.elapsed().as_millis();
631        text = format!("Indexed {} of {} packages in {ms} ms.\n{text}", ready.len(), deps.len());
632        Ok(Answer {
633            text,
634            json: json!({"packages": items, "missing": missing.iter().map(|(d, e)| json!({"package": d.id(), "error": e})).collect::<Vec<_>>(), "ms": ms}),
635        })
636    }
637
638    // ------------------------------------------------------------ resolve
639
640    pub fn resolve(&self, filter: Option<&str>) -> Answer {
641        let p = &self.project;
642        let filter = filter.map(|f| f.trim().to_ascii_lowercase()).filter(|f| !f.is_empty());
643        let shown: Vec<&Dep> = match &filter {
644            Some(f) => p.deps.iter().filter(|d| d.name.to_ascii_lowercase().contains(f.as_str())).collect(),
645            None => p.direct().collect(),
646        };
647        let rows: Vec<(&Dep, Option<Source>)> = shown
648            .par_iter()
649            .map(|d| (*d, locate::locate(d, &p.root).or_else(|| fetch::cached_for(d, Some(&p.root)))))
650            .collect();
651        let mut text = format!(
652            "Project: {}\nLockfiles: {}\n",
653            p.root.display(),
654            if p.lockfiles.is_empty() {
655                "none found".to_string()
656            } else {
657                p.lockfiles.join(", ")
658            }
659        );
660        for w in &p.warnings {
661            text.push_str(&format!("Warning: {w}\n"));
662        }
663        let total_direct = p.direct().count();
664        text.push_str(&match &filter {
665            Some(f) => format!("\n{} of {} dependencies match `{f}`:\n", rows.len(), p.deps.len()),
666            None => format!(
667                "\n{} direct dependencies ({} total with transitive; pass a filter to search all):\n",
668                total_direct,
669                p.deps.len()
670            ),
671        });
672        let w = rows.iter().map(|(d, _)| d.name.len() + d.version.len() + 1).max().unwrap_or(10).min(48);
673        let mut items = Vec::new();
674        for (d, s) in &rows {
675            let id = format!("{}@{}", d.name, d.version);
676            let (status, src) = match s {
677                Some(s) if s.version == d.version => ("docs ready", s.label.clone()),
678                Some(s) => ("version drift", format!("installed {} at {}", s.version, s.label)),
679                None => (
680                    "not installed",
681                    if self.opts.fetch {
682                        "will fetch on first query".into()
683                    } else {
684                        "install it, or enable fetching".to_string()
685                    },
686                ),
687            };
688            text.push_str(&format!(
689                "  {:<6} {:<w$}  {}{}  {}\n",
690                d.eco.as_str(),
691                id,
692                if d.direct { "" } else { "(transitive) " },
693                status,
694                src
695            ));
696            items.push(json!({"ecosystem": d.eco.as_str(), "name": d.name, "version": d.version, "direct": d.direct, "lockfile": d.from, "status": status, "source": src}));
697        }
698        if rows.is_empty() {
699            text.push_str("  (none)\n");
700        }
701        Answer {
702            text,
703            json: json!({"root": p.root.display().to_string(), "lockfiles": p.lockfiles, "dependencies": items, "total": p.deps.len(), "warnings": p.warnings}),
704        }
705    }
706
707    // ------------------------------------------------------------ docs
708
709    /// Upgrade report (lockdocs Pro; the caller checks the licence): the API
710    /// difference between the pinned version of `package` and `target`,
711    /// limited to symbols this project uses. The target version is read from
712    /// the same sources as any other version (installed, cached, or fetched
713    /// when fetching is enabled).
714    pub fn upgrade_report(&self, package: &str, target: &str, tokens: usize) -> Result<Answer, String> {
715        let spec = Spec::parse(package);
716        let found = self.project.find(&spec);
717        let dep = found
718            .iter()
719            .find(|d| d.direct)
720            .or_else(|| found.first())
721            .map(|d| (*d).clone())
722            .ok_or_else(|| format!("`{package}` is not pinned in this project's lockfiles; run `resolve` to see packages."))?;
723        let target = target.trim().trim_start_matches('v');
724        if target.is_empty() {
725            return Err("`upgrade_to` needs a target version, e.g. 15.0.0".into());
726        }
727        if target == dep.version.trim_start_matches('v') {
728            return Err(format!("{} is already the pinned version; pick a different target.", dep.id()));
729        }
730        let old = match self.index(&dep) {
731            Resolved::Ready(i, _, _) => i,
732            Resolved::Missing(_, e) => return Err(e),
733        };
734        let target_dep = Dep {
735            eco: dep.eco,
736            name: dep.name.clone(),
737            version: target.to_string(),
738            direct: false,
739            from: "requested".into(),
740        };
741        let new = match self.index(&target_dep) {
742            Resolved::Ready(i, _, _) => i,
743            Resolved::Missing(_, e) => {
744                return Err(format!(
745                    "{e}\nThe target version's files are needed to compare; add --fetch (or LOCKDOCS_FETCH=1) to download exactly {}.",
746                    target_dep.id()
747                ))
748            }
749        };
750        let r = crate::upgrade::report(&self.project.root, &old, &new);
751        let max = tokens.clamp(200, 20000) * 3;
752        let text = if r.text.len() > max {
753            format!("{}\n... truncated to the token budget; raise `tokens`.\n", truncate(&r.text, max))
754        } else {
755            r.text
756        };
757        Ok(Answer { text, json: r.json })
758    }
759
760    pub fn docs(&self, package: Option<&str>, query: &str, tokens: usize) -> Result<Answer, String> {
761        let tokens = tokens.clamp(200, 20000);
762        let mut imports = HashMap::new();
763        let selector = package.map(|p| {
764            p.split(',')
765                .map(|p| {
766                    let spec = Spec::parse(p.trim());
767                    let base = npm_import_base(&spec.name);
768                    let npm = spec.eco == Some(Eco::Npm) || spec.eco.is_none() && self.project.deps.iter().any(|d| d.eco == Eco::Npm && d.name == base);
769                    if npm && base != spec.name {
770                        imports.entry(base.to_string()).or_insert_with(Vec::new).push(spec.name.clone());
771                        format!("npm:{base}{}", spec.version.as_ref().map(|v| format!("@{v}")).unwrap_or_default())
772                    } else {
773                        if npm {
774                            imports.entry(base.to_string()).or_insert_with(Vec::new).push(base.to_string());
775                        }
776                        p.trim().to_string()
777                    }
778                })
779                .collect::<Vec<_>>()
780                .join(",")
781        });
782        let (deps, broad) = self.select(selector.as_deref(), query)?;
783        let (ready, missing) = self.resolve_all(&deps);
784        if ready.is_empty() {
785            return Err(missing.into_iter().map(|(_, e)| e).collect::<Vec<_>>().join("\n"));
786        }
787        let mut scopes = Vec::new();
788        for (idx, _, _) in &ready {
789            let requested = imports.get(&idx.name).cloned().unwrap_or_else(|| vec![idx.name.clone()]);
790            if let Some(api) = &idx.npm_api {
791                for import in &requested {
792                    if import != &idx.name && !api.roots.iter().any(|r| &r.import == import) {
793                        return Err(format!("`{import}` has no public declaration entrypoint in {}.", idx.id()));
794                    }
795                }
796            }
797            let mut scope = std::collections::BTreeSet::new();
798            for import in requested {
799                scope.extend(idx.docs_indices(Some(&import)));
800            }
801            scopes.push(scope);
802        }
803        let q = query.trim();
804        if q.is_empty() {
805            return Ok(self.overview(&ready, &scopes, tokens));
806        }
807        let qterms = bm25::query_terms(q);
808        // One BM25 over the selected packages so scores compare.
809        let mut refs: Vec<(usize, usize)> = Vec::new();
810        for (pi, scope) in scopes.iter().enumerate() {
811            for &ei in scope {
812                refs.push((pi, ei));
813            }
814        }
815        let bm = Bm25::build(refs.iter().map(|&(pi, ei)| (ready[pi].0.terms[ei].as_slice(), ready[pi].0.lens[ei])));
816        let hb = Bm25::build(refs.iter().map(|&(pi, ei)| (ready[pi].0.head[ei].as_slice(), ready[pi].0.head_lens[ei])));
817        let mut raw_idents = identifiers(q);
818        raw_idents.extend(api_words(&ready, &refs, q));
819        let changes = change_intent(q);
820        let scored = hybrid_rank(&ready, &refs, (&bm, &hb), &qterms, q, &raw_idents, changes);
821        let mut out = Pack::new(tokens);
822        let mut header = String::new();
823        for (idx, dep, note) in &ready {
824            header.push_str(&format!(
825                "{} · {} · {}{}\n",
826                idx.id(),
827                idx.eco,
828                idx.source,
829                if dep.from == "requested" {
830                    String::new()
831                } else {
832                    format!(" · pinned in {}", dep.from)
833                }
834            ));
835            if let Some(u) = &idx.upstream {
836                header.push_str(&format!("Upstream docs: {u}\n"));
837            }
838            if let Some(n) = note {
839                header.push_str(&format!("Note: {n}\n"));
840            }
841        }
842        if broad {
843            header = broad_header(&ready, &missing, tokens);
844        }
845        for (d, e) in &missing {
846            if !broad {
847                header.push_str(&format!("Skipped {}: {e}\n", d.id()));
848            }
849        }
850        out.push_raw(&header);
851        let mut seen: HashSet<(usize, String, u32)> = HashSet::new();
852        let mut hits_json = Vec::new();
853        let mut used_pkgs: Vec<String> = Vec::new();
854        let top = scored.first().map_or(0.0, |s| s.0);
855        for (score, pi, ei) in scored {
856            let idx = &ready[pi].0;
857            let e = &idx.entries[ei];
858            // Stop at weak matches once a few good ones are in.
859            if out.blocks >= 3 && score < top * RELEVANCE_FLOOR {
860                break;
861            }
862            if !seen.insert((pi, e.file.clone(), e.line)) {
863                continue;
864            }
865            let e = idx.npm_api.as_ref().map_or_else(|| resolve_alias(idx, e), |api| api.search_alias(e));
866            // The top result also carries the lead of its page and parent
867            // section, within the same budget.
868            let ctx = if out.blocks == 0 { section_context(idx, e) } else { None };
869            let max_doc = if out.blocks == 0 {
870                2000 - ctx.as_ref().map_or(0, |c| c.len().min(800) / 2)
871            } else {
872                800
873            };
874            let mut block = render_entry(idx, e, max_doc);
875            if let (Some(ctx), Some(nl)) = (ctx, block.find('\n')) {
876                block.insert_str(nl + 1, &ctx);
877            }
878            if !out.push_block(&block) {
879                break;
880            }
881            if !used_pkgs.contains(&idx.id()) {
882                used_pkgs.push(idx.id());
883            }
884            hits_json.push(json!({"package": idx.id(), "kind": e.kind.as_str(), "path": e.path, "file": e.file, "line": e.line, "score": score}));
885        }
886        if out.blocks == 0 {
887            out.push_raw(&format!(
888                "No match for \"{q}\" in {}. Try other words, a symbol name (the `api` tool), or `package` to widen/narrow.\n",
889                ready.iter().map(|r| r.0.id()).collect::<Vec<_>>().join(", ")
890            ));
891        } else if broad {
892            out.prepend(&format!("Sources: {}\n", used_pkgs.join(", ")));
893        }
894        // Packages that ship few docs: point at the one-time upstream fetch.
895        for (idx, dep, _) in &ready {
896            let prose = idx.entries.iter().filter(|e| e.kind == Kind::Prose).count();
897            if !broad && idx.upstream.is_none() && prose < 40 && !self.opts.fetch && upstream::cached(dep).is_none() {
898                out.push_raw(&format!(
899                    "\nTip: {} ships few docs. `lockdocs fetch {}` adds its upstream docs at this version's git tag (one-time download, then offline).\n",
900                    idx.id(),
901                    idx.name
902                ));
903            }
904        }
905        Ok(Answer {
906            json: json!({"query": q, "packages": ready.iter().map(|r| r.0.id()).collect::<Vec<_>>(), "hits": hits_json, "provenance": provenance(&ready), "tokens": est_tokens(&out.text)}),
907            text: out.text,
908        })
909    }
910
911    fn overview(&self, ready: &[ReadyPkg], scopes: &[std::collections::BTreeSet<usize>], tokens: usize) -> Answer {
912        let mut out = Pack::new(tokens);
913        for ((idx, dep, note), scope) in ready.iter().zip(scopes) {
914            out.push_raw(&format!("{} · {} · {} · pinned in {}\n", idx.id(), idx.eco, idx.source, dep.from));
915            if let Some(u) = &idx.upstream {
916                out.push_raw(&format!("Upstream: {u}\n"));
917            }
918            if let Some(n) = note {
919                out.push_raw(&format!("Note: {n}\n"));
920            }
921            let entries: Vec<&Entry> = scope.iter().map(|&i| &idx.entries[i]).collect();
922            let symbols = entries
923                .iter()
924                .filter(|e| e.kind != Kind::Prose && e.kind != Kind::Alias && !private_path(e))
925                .count();
926            let prose = entries.len() - symbols;
927            out.push_raw(&format!("Indexed {symbols} API symbols and {prose} doc sections.\n"));
928            // Top-level API names, shortest paths first.
929            let mut top: Vec<&Entry> = entries
930                .iter()
931                .copied()
932                .filter(|e| e.kind != Kind::Prose && !private_path(e) && !e.legacy && !e.doc.is_empty())
933                .collect();
934            top.sort_by_key(|e| (e.path.matches(['.', ':']).count(), e.path.len()));
935            let names: Vec<String> = top.iter().take(60).map(|e| e.path.clone()).collect();
936            if !names.is_empty() {
937                out.push_raw(&format!("Key API: {}\n", names.join(", ")));
938            }
939            for e in entries
940                .iter()
941                .filter(|e| e.kind == Kind::Prose && e.file.to_ascii_uppercase().starts_with("README"))
942                .take(3)
943            {
944                if !out.push_block(&render_entry(idx, e, 1500)) {
945                    break;
946                }
947            }
948        }
949        Answer {
950            json: json!({"packages": ready.iter().map(|r| r.0.id()).collect::<Vec<_>>(), "provenance": provenance(ready), "tokens": est_tokens(&out.text)}),
951            text: out.text,
952        }
953    }
954
955    // ------------------------------------------------------------ api
956
957    pub fn api(&self, symbol: &str, package: Option<&str>, tokens: usize) -> Result<Answer, String> {
958        let tokens = tokens.clamp(200, 20000);
959        let symbol = symbol.trim().trim_end_matches("()").trim();
960        if symbol.is_empty() {
961            return Err("symbol is required, e.g. `z.object`, `tokio::spawn`, `BaseModel.model_dump`".into());
962        }
963        let mut requested_import = None;
964        let (deps, rest) = match package.filter(|p| !p.trim().is_empty()) {
965            Some(p) => {
966                let spec = Spec::parse(p);
967                let base = npm_import_base(&spec.name);
968                if spec.eco.is_none_or(|e| e == Eco::Npm) && base != spec.name && self.project.find(&spec).is_empty() {
969                    let selector = format!("npm:{base}{}", spec.version.as_ref().map(|v| format!("@{v}")).unwrap_or_default());
970                    requested_import = Some(spec.name);
971                    (self.select(Some(&selector), "")?.0, symbol.to_string())
972                } else {
973                    (self.select(Some(p), "")?.0, symbol.to_string())
974                }
975            }
976            None => match self
977                .npm_import_prefix(symbol)
978                .map(|(dep, import, rest)| {
979                    requested_import = Some(import);
980                    (dep, rest)
981                })
982                .or_else(|| self.package_prefix(symbol))
983            {
984                Some((dep, rest)) => (vec![dep], rest),
985                None => (self.select(None, symbol)?.0, symbol.to_string()),
986            },
987        };
988        let segs: Vec<String> = split_symbol(&rest);
989        let Some(last) = segs.last().map(|l| l.trim_end_matches('!').to_string()) else {
990            // Only a package name: give the overview.
991            return self.docs(
992                Some(&deps.iter().map(|d| format!("{}:{}@{}", d.eco, d.name, d.version)).collect::<Vec<_>>().join(",")),
993                "",
994                tokens,
995            );
996        };
997        let quals: Vec<String> = segs[..segs.len() - 1].iter().map(|s| s.to_ascii_lowercase()).collect();
998        let (ready, missing) = self.resolve_all(&deps);
999        if ready.is_empty() {
1000            return Err(missing.into_iter().map(|(_, e)| e).collect::<Vec<_>>().join("\n"));
1001        }
1002        let mut cands: Vec<(f32, usize, usize)> = Vec::new();
1003        let mut applicability = Vec::new();
1004        let mut alternatives = Vec::new();
1005        let mut unresolved = false;
1006        let mut npm_scoped = false;
1007        let last_l = last.to_ascii_lowercase();
1008        let public_path = segs.join(".");
1009        for (pi, (idx, _, _)) in ready.iter().enumerate() {
1010            if let Some(api) = &idx.npm_api {
1011                npm_scoped = true;
1012                let import = requested_import.as_deref().unwrap_or(&idx.name);
1013                let mut has_root = false;
1014                for root in &api.roots {
1015                    let found = api.resolve(root, &public_path, idx.api_entries());
1016                    if root.import == import {
1017                        has_root = true;
1018                        unresolved |= found.unresolved;
1019                        for ei in found.entries {
1020                            let e = &idx.api_entries()[ei];
1021                            cands.push((10.0, pi, ei));
1022                            applicability.push(
1023                                json!({"import": root.import, "conditions": root.conditions.trim_start_matches('/'), "entrypoint": root.file,
1024                                "file": e.file, "line": e.line, "path": e.path}),
1025                            );
1026                        }
1027                    } else if !found.unresolved {
1028                        for ei in found.entries {
1029                            let e = &idx.api_entries()[ei];
1030                            alternatives.push(
1031                                json!({"package": idx.id(), "import": root.import, "conditions": root.conditions.trim_start_matches('/'),
1032                                "entrypoint": root.file, "path": e.path, "file": e.file, "line": e.line, "signature": e.sig}),
1033                            );
1034                        }
1035                    }
1036                }
1037                unresolved |= !has_root;
1038                continue;
1039            }
1040            for (ei, e) in idx.api_entries().iter().enumerate() {
1041                if e.kind == Kind::Prose {
1042                    continue;
1043                }
1044                let name = e.name.trim_end_matches('!');
1045                let exact = name == last;
1046                if !exact && name.to_ascii_lowercase() != last_l {
1047                    continue;
1048                }
1049                cands.push((api_score(e, exact, &quals), pi, ei));
1050            }
1051        }
1052        cands.sort_by_key(|(_, pi, ei)| (*pi, *ei));
1053        cands.dedup_by_key(|(_, pi, ei)| (*pi, *ei));
1054        // Multiple overloads at one declaration are valid. Conflicting files,
1055        // packages or conditional editions are not a license to guess.
1056        let locations: HashSet<_> = cands
1057            .iter()
1058            .map(|(_, pi, ei)| (*pi, &ready[*pi].0.api_entries()[*ei].file, &ready[*pi].0.api_entries()[*ei].path))
1059            .collect();
1060        let ambiguous = npm_scoped && locations.len() > 1 && {
1061            let signatures: HashSet<_> = cands.iter().map(|(_, pi, ei)| &ready[*pi].0.api_entries()[*ei].sig).collect();
1062            signatures.len() > 1
1063        };
1064        if npm_scoped && (cands.is_empty() || unresolved || ambiguous) {
1065            let status = if ambiguous {
1066                "ambiguous"
1067            } else if unresolved {
1068                "unresolved"
1069            } else {
1070                "not_exported"
1071            };
1072            let imports: Vec<_> = ready
1073                .iter()
1074                .filter(|r| r.0.eco == Eco::Npm)
1075                .map(|r| requested_import.as_deref().unwrap_or(&r.0.name))
1076                .collect();
1077            let mut out = Pack::new(tokens);
1078            out.push_raw(&format!(
1079                "No verified exact public API match for `{public_path}` from `{}` ({}).\n",
1080                imports.join("`, `"),
1081                status
1082            ));
1083            out.push_block(if ambiguous {
1084                "Public declarations disagree across entrypoints or export conditions; no signature was selected.\n"
1085            } else if unresolved {
1086                "Import applicability could not be fully resolved from the installed declarations; no unrelated same-named symbol was substituted.\n"
1087            } else {
1088                "This symbol is not exported by the requested import entrypoint. Same-named internal declarations are not a match.\n"
1089            });
1090            for a in alternatives.iter().take(8) {
1091                out.push_block(&format!(
1092                    "Alternate import `{}` ({}): `{}` — {} {}:{}\n",
1093                    a["import"].as_str().unwrap_or(""),
1094                    a["conditions"].as_str().unwrap_or(""),
1095                    a["signature"].as_str().unwrap_or(""),
1096                    a["package"].as_str().unwrap_or(""),
1097                    a["file"].as_str().unwrap_or(""),
1098                    a["line"]
1099                ));
1100            }
1101            if ambiguous {
1102                for a in applicability.iter().take(8) {
1103                    out.push_block(&format!(
1104                        "Applicable import `{}` ({}) via {} → {}:{}\n",
1105                        a["import"].as_str().unwrap_or(""),
1106                        a["conditions"].as_str().unwrap_or(""),
1107                        a["entrypoint"].as_str().unwrap_or(""),
1108                        a["file"].as_str().unwrap_or(""),
1109                        a["line"]
1110                    ));
1111                }
1112                for (_, pi, ei) in cands.iter().take(8) {
1113                    let idx = &ready[*pi].0;
1114                    out.push_block(&render_entry(idx, &idx.api_entries()[*ei], 500));
1115                }
1116            }
1117            return Ok(Answer {
1118                json: json!({"status": status, "symbol": public_path, "imports": imports, "provenance": provenance(&ready),
1119                "applicability": applicability, "alternatives": alternatives, "tokens": est_tokens(&out.text)}),
1120                text: out.text,
1121            });
1122        }
1123        cands.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));
1124        if cands.is_empty() {
1125            // Nothing by that name: fall back to a docs search scoped to the same packages.
1126            let pk = ready
1127                .iter()
1128                .map(|r| format!("{}:{}@{}", r.0.eco, r.0.name, r.0.version))
1129                .collect::<Vec<_>>()
1130                .join(",");
1131            let mut a = self.docs(Some(&pk), &segs.join(" "), tokens)?;
1132            a.text = format!(
1133                "No symbol named `{last}` in {}; closest documentation:\n\n{}",
1134                ready.iter().map(|r| r.0.id()).collect::<Vec<_>>().join(", "),
1135                a.text
1136            );
1137            return Ok(a);
1138        }
1139        let (_, bpi, bei) = cands[0];
1140        let (idx, dep, note) = &ready[bpi];
1141        let best0 = &idx.api_entries()[bei];
1142        let best = if idx.npm_api.is_some() { best0 } else { resolve_alias(idx, best0) };
1143        let mut out = Pack::new(tokens);
1144        out.push_raw(&format!(
1145            "{} · {} · {}{}\n",
1146            idx.id(),
1147            idx.eco,
1148            idx.source,
1149            if dep.from == "requested" {
1150                String::new()
1151            } else {
1152                format!(" · pinned in {}", dep.from)
1153            }
1154        ));
1155        if let Some(u) = &idx.upstream {
1156            out.push_raw(&format!("Upstream: {u}\n"));
1157        }
1158        if let Some(n) = note {
1159            out.push_raw(&format!("Note: {n}\n"));
1160        }
1161        if !std::ptr::eq(best, best0) {
1162            out.push_raw(&format!(
1163                "`{}` is a re-export of `{}` ({}:{}).\n",
1164                best0.path, best.path, best0.file, best0.line
1165            ));
1166        }
1167        if idx.npm_api.is_some() {
1168            out.push_raw(&format!(
1169                "Public import: `{}` · symbol `{public_path}`\n",
1170                requested_import.as_deref().unwrap_or(&idx.name)
1171            ));
1172            for a in &applicability {
1173                out.push_block(&format!(
1174                    "Declaration entrypoint: {} ({})\n",
1175                    a["entrypoint"].as_str().unwrap_or(""),
1176                    a["conditions"].as_str().unwrap_or("")
1177                ));
1178            }
1179        }
1180        out.push_block(&render_entry(idx, best, 6000));
1181        // Overloads and declaration merging: same path, other entries.
1182        let overloads: Vec<&Entry> = idx
1183            .api_entries()
1184            .iter()
1185            .filter(|e| e.path == best.path && e.file == best.file && !std::ptr::eq(*e, best) && e.kind != Kind::Prose && e.kind != Kind::Alias)
1186            .take(8)
1187            .collect();
1188        if !overloads.is_empty() {
1189            let mut s = String::from("Other declarations:\n");
1190            for e in overloads {
1191                s.push_str(&format!("- `{}` — {} {}:{}\n", e.sig, idx.id(), e.file, e.line));
1192            }
1193            out.push_block(&s);
1194        }
1195        let mut members_json = Vec::new();
1196        if best.kind.is_container() {
1197            let sep = if idx.eco == Eco::Cargo { "::" } else { "." };
1198            let prefix = format!("{}{sep}", best.path);
1199            let members: Vec<&Entry> = idx
1200                .api_entries()
1201                .iter()
1202                .filter(|e| {
1203                    e.path.starts_with(&prefix)
1204                        && !e.path[prefix.len()..].contains(sep)
1205                        && e.kind != Kind::Prose
1206                        && (idx.npm_api.is_none() || e.file == best.file)
1207                })
1208                .collect();
1209            if !members.is_empty() {
1210                let mut s = format!("Members of {} ({}):\n", best.path, members.len());
1211                for m in members.iter().take(80) {
1212                    let doc1 = m.doc.lines().next().unwrap_or("");
1213                    s.push_str(&format!(
1214                        "- `{}`{}{} — {}:{}\n",
1215                        m.sig,
1216                        if doc1.is_empty() { "" } else { " — " },
1217                        truncate(doc1, 120),
1218                        m.file,
1219                        m.line
1220                    ));
1221                    members_json.push(json!({"path": m.path, "sig": m.sig, "file": m.file, "line": m.line}));
1222                }
1223                out.push_block(&s);
1224            }
1225        }
1226        let others: Vec<String> = cands
1227            .iter()
1228            .skip(1)
1229            .filter(|(_, pi, ei)| {
1230                let e = &ready[*pi].0.api_entries()[*ei];
1231                !(e.path == best.path && e.file == best.file) && !std::ptr::eq(e, best0)
1232            })
1233            .take(8)
1234            .map(|(_, pi, ei)| {
1235                let (ix, _, _) = &ready[*pi];
1236                let e = &ix.api_entries()[*ei];
1237                format!("- {} `{}` — {} {}:{}", e.kind.as_str(), e.path, ix.id(), e.file, e.line)
1238            })
1239            .collect();
1240        if !others.is_empty() {
1241            out.push_block(&format!("Other matches:\n{}\n", others.join("\n")));
1242        }
1243        Ok(Answer {
1244            json: json!({
1245                "provenance": provenance(&ready), "package": idx.id(), "kind": best.kind.as_str(), "path": best.path, "signature": best.sig, "doc": best.doc,
1246                "file": best.file, "line": best.line, "members": members_json, "tokens": est_tokens(&out.text),
1247                "status": "exact", "import": if idx.npm_api.is_some() { Some(requested_import.as_deref().unwrap_or(&idx.name)) } else { None },
1248                "applicability": applicability, "alternatives": alternatives,
1249            }),
1250            text: out.text,
1251        })
1252    }
1253
1254    fn npm_import_prefix(&self, symbol: &str) -> Option<(Dep, String, String)> {
1255        for d in &self.project.deps {
1256            if d.eco != Eco::Npm {
1257                continue;
1258            }
1259            let Some(rest) = symbol.strip_prefix(&format!("{}/", d.name)) else {
1260                continue;
1261            };
1262            if let Some((subpath, sym)) = rest.split_once('.') {
1263                return Some((d.clone(), format!("{}/{subpath}", d.name), sym.to_string()));
1264            }
1265        }
1266        None
1267    }
1268
1269    /// `zod.z.object` -> (zod, `z.object`); `tokio::spawn` -> (tokio, `spawn`);
1270    /// `@tanstack/react-query.useQuery`; `github.com/gin-gonic/gin.Context`.
1271    fn package_prefix(&self, symbol: &str) -> Option<(Dep, String)> {
1272        let mut best: Option<(usize, Dep)> = None;
1273        let bytes = symbol.as_bytes();
1274        let mut cuts: Vec<usize> = Vec::new();
1275        for i in 0..bytes.len() {
1276            if bytes[i] == b'.' || bytes[i] == b'#' || (bytes[i] == b':' && i + 1 < bytes.len() && bytes[i + 1] == b':') {
1277                cuts.push(i);
1278            }
1279        }
1280        cuts.push(symbol.len());
1281        for &c in &cuts {
1282            let cand = &symbol[..c];
1283            let spec = Spec::parse(cand);
1284            if spec.version.is_some() {
1285                continue;
1286            }
1287            let found = self.project.find(&spec);
1288            if let Some(d) = found.first() {
1289                if best.as_ref().is_none_or(|(bc, _)| c > *bc) {
1290                    best = Some((c, (*d).clone()));
1291                }
1292            }
1293        }
1294        let (c, dep) = best?;
1295        let rest = symbol[c..].trim_start_matches(['.', '#', ':']).to_string();
1296        Some((dep, rest))
1297    }
1298}
1299
1300fn npm_import_base(name: &str) -> &str {
1301    let skip = if name.starts_with('@') {
1302        name.find('/').map_or(name.len(), |i| i + 1)
1303    } else {
1304        0
1305    };
1306    name[skip..].find('/').map_or(name, |i| &name[..skip + i])
1307}
1308
1309fn split_symbol(s: &str) -> Vec<String> {
1310    s.replace("::", ".")
1311        .replace('#', ".")
1312        .split('.')
1313        .map(|x| x.trim().trim_end_matches("()").to_string())
1314        .filter(|x| !x.is_empty())
1315        .collect()
1316}
1317
1318/// Identifier-like words in a question (`z.object`, `useEffect`,
1319/// `model_dump`, `cookies()`); plain English words are not identifiers.
1320fn identifiers(q: &str) -> Vec<String> {
1321    let words: Vec<&str> = q
1322        .split(|c: char| c.is_whitespace() || matches!(c, ',' | '?' | '\'' | '"' | '[' | ']'))
1323        .filter(|w| !w.is_empty())
1324        .collect();
1325    let single = words.len() == 1;
1326    words
1327        .into_iter()
1328        .filter(|w| {
1329            let inner = w.trim_matches(|c: char| !c.is_alphanumeric());
1330            single || w.starts_with('`') || w.contains(['_', '.', '(', '$']) || w.contains("::") || inner.chars().skip(1).any(|c| c.is_ascii_uppercase())
1331        })
1332        .map(|w| w.trim_matches(|c: char| matches!(c, '`' | '(' | ')' | '.' | ':' | '!' | ';')))
1333        .filter(|w| w.len() >= 2 && w.chars().any(|c| c.is_ascii_alphabetic()))
1334        .flat_map(|w| {
1335            let mut v = vec![w.to_ascii_lowercase()];
1336            if let Some(last) = split_symbol(w).last() {
1337                v.push(last.to_ascii_lowercase());
1338            }
1339            v
1340        })
1341        .collect()
1342}
1343
1344/// Plain words in the question that name a documented top-level API of the
1345/// searched packages ("run code *after* the response" in Next.js, which
1346/// exports `after`). They count as identifiers.
1347fn api_words(ready: &[ReadyPkg], refs: &[(usize, usize)], q: &str) -> Vec<String> {
1348    let words: HashSet<String> = q
1349        .split(|c: char| !(c.is_ascii_alphanumeric() || c == '_'))
1350        .filter(|w| w.len() >= 4)
1351        .map(|w| w.to_ascii_lowercase())
1352        .filter(|w| !bm25::is_stop(w))
1353        .collect();
1354    let mut out = Vec::new();
1355    for &(pi, ei) in refs {
1356        let e = &ready[pi].0.entries[ei];
1357        let top = matches!(e.kind, Kind::Function | Kind::Macro | Kind::Class)
1358            && !e.doc.is_empty()
1359            && !private_path(e)
1360            && !e.legacy
1361            && e.path.matches(['.', ':']).count() <= 2;
1362        let n = e.name.trim_end_matches('!').to_ascii_lowercase();
1363        if top && words.contains(&n) && !out.contains(&n) {
1364            out.push(n);
1365        }
1366    }
1367    out
1368}
1369
1370/// Keyword ranking (BM25 over text and headings, then docs-specific boosts)
1371/// first; when no entry clearly leads, the embedding ranking is fused in by
1372/// weighted reciprocal rank. Then deprecation redirects ("use X instead")
1373/// lift the API they point to. Returns (score, package, entry), best first.
1374fn hybrid_rank(
1375    ready: &[ReadyPkg],
1376    refs: &[(usize, usize)],
1377    (bm, hb): (&Bm25, &Bm25),
1378    qterms: &[String],
1379    q: &str,
1380    idents: &[String],
1381    changes: bool,
1382) -> Vec<(f32, usize, usize)> {
1383    // (full-text BM25, heading BM25), each normalized to its best hit.
1384    let mut fused: HashMap<usize, (f32, f32)> = HashMap::new();
1385    let expanded = bm25::expand(qterms);
1386    let bm_hits = bm.search_weighted(&expanded);
1387    let bmax = bm_hits.first().map_or(1.0, |h| h.score).max(1e-6);
1388    for h in bm_hits.iter().take(1500) {
1389        fused.entry(h.doc as usize).or_default().0 = h.score / bmax;
1390    }
1391    let head_w = std::env::var("LOCKDOCS_HEAD_WEIGHT").ok().and_then(|v| v.parse().ok()).unwrap_or(HEAD_WEIGHT);
1392    let h_hits = hb.search_weighted(&expanded);
1393    let hmax = h_hits.first().map_or(1.0, |h| h.score).max(1e-6);
1394    for h in h_hits.iter().take(1500) {
1395        fused.entry(h.doc as usize).or_default().1 = h.score / hmax;
1396    }
1397    let dense_w: f32 = std::env::var("LOCKDOCS_DENSE_WEIGHT").ok().and_then(|v| v.parse().ok()).unwrap_or(DENSE_WEIGHT);
1398    // Query words that are rare in these packages; common ones ("futures" in
1399    // tokio) say little about which entry is meant.
1400    let n = refs.len().max(1) as f32;
1401    let rare: Vec<String> = qterms.iter().filter(|t| (bm.df(t) as f32) < n * 0.03).cloned().collect();
1402    let weight = |i: usize| {
1403        let (pi, ei) = refs[i];
1404        let e = &ready[pi].0.entries[ei];
1405        boost(e, idents, changes, major_of(&ready[pi].0.version)) * name_hit(e, &rare)
1406    };
1407    // Keyword ranking first, with the docs-specific boosts: this is what the
1408    // answer is built on.
1409    let mut lex: Vec<(usize, f32)> = fused.iter().map(|(&i, &(b, h))| (i, ((1.0 - head_w) * b + head_w * h) * weight(i))).collect();
1410    lex.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal).then(a.0.cmp(&b.0)));
1411    // A clear leader needs no second opinion: skip the embedding entirely.
1412    let decisive = match (lex.first(), lex.get(1)) {
1413        (Some(t), Some(s)) => t.1 >= s.1 * DECISIVE_RATIO,
1414        (Some(_), None) => true,
1415        _ => false,
1416    };
1417    let debug = std::env::var("LOCKDOCS_DEBUG").is_ok();
1418    let qv = if !decisive && dense_w > 0.0 && ready.iter().all(|r| !r.0.vecs.is_empty()) {
1419        embed::query_model().and_then(|m| m.embed(q))
1420    } else {
1421        None
1422    };
1423    // Embedding rank of each entry among the 300 nearest to the query.
1424    let mut dense_rank: HashMap<usize, usize> = HashMap::new();
1425    let mut dense_cos: HashMap<usize, f32> = HashMap::new();
1426    if let Some(qv) = &qv {
1427        let mut sims: Vec<(usize, f32)> = refs
1428            .par_iter()
1429            .enumerate()
1430            .map(|(i, &(pi, ei))| (i, embed::cosine(qv, &ready[pi].0.vecs[ei])))
1431            .collect();
1432        sims.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal).then(a.0.cmp(&b.0)));
1433        for (r, (i, _)) in sims.iter().take(DENSE_POOL).enumerate() {
1434            dense_rank.insert(*i, r);
1435        }
1436        let cmax = sims.first().map_or(1.0, |s| s.1);
1437        let floor = sims.get(300).map_or(0.0, |s| s.1);
1438        for (i, c) in sims.iter().take(300) {
1439            dense_cos.insert(*i, ((c - floor) / (cmax - floor).max(1e-6)).max(0.0));
1440        }
1441        if debug {
1442            for (i, c) in sims.iter().take(10) {
1443                let (pi, ei) = refs[*i];
1444                eprintln!("dense cos={c:.3} {}", ready[pi].0.entries[ei].path);
1445            }
1446        }
1447    }
1448    if debug {
1449        eprintln!("decisive={decisive} dense={}", qv.is_some());
1450    }
1451    // Package files only (API names and signatures): reciprocal-rank fusion
1452    // weighted towards the keyword ranking, so the embedding breaks near-ties
1453    // and rescues paraphrases but cannot outvote an exact name match. With
1454    // upstream docs in the index the answers are prose pages, where the
1455    // embedding is as reliable as keywords and score magnitudes matter (a
1456    // rank-only fusion buried select.md under Graceful Shutdown): there each
1457    // normalized score is mixed, as before. Scores are scaled so the best is 1.
1458    let with_docs = ready.iter().any(|r| r.0.upstream.is_some());
1459    let rrf = |r: usize| 1.0 / (RRF_K + r as f32);
1460    let mut scored: Vec<(f32, usize, usize)> = if qv.is_some() && with_docs {
1461        let mut ids: HashSet<usize> = lex.iter().map(|l| l.0).collect();
1462        ids.extend(dense_cos.keys().copied());
1463        let lexm: HashMap<usize, f32> = fused.iter().map(|(&i, &(b, h))| (i, (1.0 - head_w) * b + head_w * h)).collect();
1464        ids.into_iter()
1465            .map(|i| {
1466                let (pi, ei) = refs[i];
1467                let l = lexm.get(&i).copied().unwrap_or(0.0);
1468                let d = dense_cos.get(&i).copied().unwrap_or(0.0);
1469                (((1.0 - DOCS_DENSE_WEIGHT) * l + DOCS_DENSE_WEIGHT * d) * weight(i), pi, ei)
1470            })
1471            .collect()
1472    } else if qv.is_some() {
1473        let mut ids: HashSet<usize> = lex.iter().map(|l| l.0).collect();
1474        ids.extend(dense_rank.keys().copied());
1475        let lex_rank: HashMap<usize, usize> = lex.iter().enumerate().map(|(r, l)| (l.0, r)).collect();
1476        ids.into_iter()
1477            .map(|i| {
1478                let l = lex_rank.get(&i).map_or(0.0, |&r| rrf(r));
1479                let d = dense_rank.get(&i).map_or(0.0, |&r| rrf(r));
1480                let (pi, ei) = refs[i];
1481                ((1.0 - dense_w) * l + dense_w * d, pi, ei)
1482            })
1483            .collect()
1484    } else {
1485        lex.iter().map(|&(i, s)| (s, refs[i].0, refs[i].1)).collect()
1486    };
1487    if qv.is_some() {
1488        let top = scored.iter().map(|s| s.0).fold(0.0f32, f32::max).max(1e-9);
1489        scored.iter_mut().for_each(|s| s.0 /= top);
1490    }
1491    if debug {
1492        scored.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));
1493        for (s, pi, ei) in scored.iter().take(10) {
1494            eprintln!("{s:.3} {}", ready[*pi].0.entries[*ei].path);
1495        }
1496    }
1497    scored.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));
1498    // Deprecation redirects among the top results.
1499    let mut lifted: Vec<(f32, usize, String)> = Vec::new();
1500    for (s, pi, ei) in scored.iter().take(30) {
1501        let e = &ready[*pi].0.entries[*ei];
1502        for t in redirects(&format!("{}\n{}", e.sig, e.doc)) {
1503            if t != e.name {
1504                lifted.push((*s * 0.95, *pi, t));
1505            }
1506        }
1507    }
1508    if !lifted.is_empty() {
1509        let mut have: HashMap<(usize, usize), usize> = scored.iter().enumerate().map(|(i, (_, pi, ei))| ((*pi, *ei), i)).collect();
1510        for (s, pi, name) in lifted {
1511            for &(_, ei) in refs.iter().filter(|(p, _)| *p == pi) {
1512                let e = &ready[pi].0.entries[ei];
1513                if e.kind == Kind::Prose || e.kind == Kind::Alias || e.name != name || e.legacy {
1514                    continue;
1515                }
1516                match have.get(&(pi, ei)) {
1517                    Some(&i) => scored[i].0 = scored[i].0.max(s),
1518                    None => {
1519                        have.insert((pi, ei), scored.len());
1520                        scored.push((s, pi, ei));
1521                    }
1522                }
1523            }
1524        }
1525        scored.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));
1526    }
1527    scored
1528}
1529
1530/// API names a doc points readers to: "use `model_validate` instead",
1531/// "Consider `z.strictObject(A.shape)`", "renamed to X", "in favor of X".
1532pub fn redirects(text: &str) -> Vec<String> {
1533    let lower = text.to_ascii_lowercase();
1534    let mut out = Vec::new();
1535    for pat in [
1536        "use ",
1537        "consider ",
1538        "renamed to ",
1539        "replaced by ",
1540        "in favor of ",
1541        "in favour of ",
1542        "instead use ",
1543    ] {
1544        let mut from = 0;
1545        while let Some(i) = lower[from..].find(pat) {
1546            let at = from + i + pat.len();
1547            from = at;
1548            let rest = &text[at..];
1549            let rest = rest.trim_start_matches(['`', '\'', '"', '*', ' ']);
1550            let ident: String = rest.chars().take_while(|c| c.is_alphanumeric() || matches!(c, '_' | '.' | ':' | '$')).collect();
1551            let ident = ident.trim_end_matches(['.', ':']);
1552            // Only identifier-looking targets: code spans or names with _ . $ or camelCase.
1553            let quoted = text[at..].starts_with('`');
1554            let looks = ident.contains(['_', '.', '$']) || ident.chars().skip(1).any(|c| c.is_ascii_uppercase());
1555            if ident.len() >= 3 && (quoted || looks) {
1556                let last = ident.rsplit(['.', ':']).next().unwrap_or(ident).to_string();
1557                if last.len() >= 3 && !out.contains(&last) {
1558                    out.push(last);
1559                }
1560            }
1561        }
1562    }
1563    out
1564}
1565
1566/// Terms of a prose heading. Capitalized words are names (`TypeScript`,
1567/// `JavaScript`) and stay whole; identifiers (`useActionState`,
1568/// `model_dump`) are also split into their parts.
1569fn heading_terms(h: &str) -> Vec<String> {
1570    let mut out = Vec::new();
1571    for w in h.split(|c: char| !(c.is_ascii_alphanumeric() || c == '_')).filter(|w| !w.is_empty()) {
1572        if w.starts_with(|c: char| c.is_ascii_uppercase()) && !w.contains('_') {
1573            out.push(bm25::stem(&w.to_ascii_lowercase()));
1574        } else {
1575            out.extend(bm25::terms(w));
1576        }
1577    }
1578    out
1579}
1580
1581/// Entries whose name (or section heading) carries a query word are about it.
1582fn name_hit(e: &Entry, qterms: &[String]) -> f32 {
1583    let words = if e.kind == Kind::Prose {
1584        let last = e.name.rsplit(" › ").next().unwrap_or("");
1585        if crate::markdown::generic_heading(last) {
1586            Vec::new()
1587        } else {
1588            heading_terms(last)
1589        }
1590    } else {
1591        bm25::terms(&e.name)
1592    };
1593    let n = qterms.iter().filter(|q| words.contains(q)).count();
1594    match n {
1595        0 => 1.0,
1596        1 => 1.5,
1597        _ => 1.9,
1598    }
1599}
1600
1601fn major_of(version: &str) -> u64 {
1602    version.trim_start_matches('v').split('.').next().and_then(|m| m.parse().ok()).unwrap_or(0)
1603}
1604
1605/// A migration or upgrade guide to a major older than the pinned one
1606/// ("Migrating to v6.0.0" in ESLint 9, "Upgrade to Prisma ORM 4" in Prisma 6):
1607/// history, not how things work now.
1608fn old_upgrade_guide(e: &Entry, major: u64) -> bool {
1609    let title = e.name.split(" › ").next().unwrap_or("");
1610    let stem = e.file.rsplit('/').next().unwrap_or("");
1611    [title, stem].iter().any(|t| {
1612        let l = t.to_ascii_lowercase();
1613        (l.contains("migrat") || l.contains("upgrad"))
1614            && l.split(|c: char| !c.is_ascii_digit())
1615                .find(|d| !d.is_empty() && d.len() <= 3)
1616                .and_then(|d| d.parse::<u64>().ok())
1617                .is_some_and(|v| v < major)
1618    })
1619}
1620
1621/// A page its own title marks as deprecated ("Configure Language Options (Deprecated)").
1622fn deprecated_page(e: &Entry) -> bool {
1623    e.name.split(" › ").next().unwrap_or("").to_ascii_lowercase().contains("(deprecated)")
1624}
1625
1626fn boost(e: &Entry, idents: &[String], changes: bool, major: u64) -> f32 {
1627    let mut b = 1.0;
1628    // Curated guides from the project's own docs folder answer "how do I" better
1629    // than internal symbols do.
1630    if e.kind == Kind::Prose && e.file.starts_with("upstream:") {
1631        b *= std::env::var("LOCKDOCS_UPSTREAM_BOOST").ok().and_then(|v| v.parse().ok()).unwrap_or(1.35);
1632    }
1633    if e.kind == Kind::Prose {
1634        // A section headed by the API the question names (`after`, `select!`).
1635        let last = e
1636            .name
1637            .rsplit(" › ")
1638            .next()
1639            .unwrap_or("")
1640            .trim()
1641            .trim_end_matches("()")
1642            .trim_end_matches('!')
1643            .to_ascii_lowercase();
1644        if !last.is_empty() && idents.contains(&last) {
1645            b *= 1.4;
1646        }
1647        if !changes && old_upgrade_guide(e, major) {
1648            b *= 0.5;
1649        }
1650        if deprecated_page(e) {
1651            b *= 0.6;
1652        }
1653        if is_changelog(e) {
1654            b *= if changes { 1.5 } else { 0.5 };
1655        } else if e.file.to_ascii_uppercase().starts_with("README") {
1656            b *= 1.15;
1657        }
1658    } else {
1659        let n = e.name.trim_end_matches('!').to_ascii_lowercase();
1660        let p = e.path.to_ascii_lowercase();
1661        if idents.contains(&n) {
1662            b *= 1.8;
1663        }
1664        if idents
1665            .iter()
1666            .any(|i| i.len() > n.len() && (p.ends_with(i.as_str()) || i.ends_with(&format!(".{n}"))))
1667        {
1668            b *= 1.4;
1669        }
1670        if e.doc.is_empty() {
1671            b *= 0.6;
1672        }
1673        if e.kind == Kind::Alias {
1674            b *= 0.6;
1675        }
1676    }
1677    if private_path(e) {
1678        b *= 0.6;
1679    }
1680    if e.legacy {
1681        b *= 0.35;
1682    }
1683    if tooling_path(e) {
1684        b *= 0.5;
1685    }
1686    b
1687}
1688
1689fn api_score(e: &Entry, exact: bool, quals: &[String]) -> f32 {
1690    let mut s = if exact { 10.0 } else { 7.0 };
1691    let segs: Vec<String> = split_symbol(&e.path).into_iter().map(|x| x.to_ascii_lowercase()).collect();
1692    // A bare name means the top-level API, not a same-named member.
1693    if quals.is_empty() {
1694        s += if matches!(e.kind, Kind::Method | Kind::Field) { -1.5 } else { 1.5 };
1695    }
1696    let file = e.file.to_ascii_lowercase();
1697    for q in quals {
1698        if segs.iter().any(|x| x == q) {
1699            s += 3.0;
1700        } else if file.contains(q.as_str()) {
1701            s += 1.0;
1702        }
1703    }
1704    // The qualifier directly before the name matters most (`Router.route`).
1705    if let (Some(q), Some(parent)) = (quals.last(), segs.len().checked_sub(2).map(|i| &segs[i])) {
1706        if q == parent {
1707            s += 2.0;
1708        }
1709    }
1710    if !e.doc.is_empty() {
1711        s += 2.0;
1712    }
1713    if private_path(e) {
1714        s -= 3.0;
1715    }
1716    if e.legacy {
1717        s -= 5.0;
1718    }
1719    if tooling_path(e) {
1720        s -= 3.0;
1721    }
1722    if e.kind == Kind::Alias {
1723        s -= 0.5;
1724    }
1725    s -= 0.2 * segs.len() as f32;
1726    s
1727}
1728
1729fn resolve_alias<'a>(idx: &'a PackageIndex, e: &'a Entry) -> &'a Entry {
1730    let Some(target) = &e.alias_of else { return e };
1731    idx.entries
1732        .iter()
1733        .filter(|x| &x.name == target && x.kind != Kind::Alias && x.kind != Kind::Prose)
1734        .max_by_key(|x| (x.file == e.file, !x.doc.is_empty()))
1735        .unwrap_or(e)
1736}
1737
1738fn truncate(s: &str, n: usize) -> String {
1739    if s.chars().count() <= n {
1740        return s.to_string();
1741    }
1742    let t: String = s.chars().take(n).collect();
1743    format!("{t}…")
1744}
1745
1746/// One result block with its citation.
1747pub fn render_entry(idx: &PackageIndex, e: &Entry, max_doc: usize) -> String {
1748    let cite = format!("{} {}:{}", idx.id(), e.file, e.line);
1749    if e.kind == Kind::Prose {
1750        return format!("### {} — {}\n{}\n", e.name, cite, truncate(&e.doc, max_doc));
1751    }
1752    let mut s = format!("### {} ({}) — {}\n```{}\n{}\n```\n", e.path, e.kind.as_str(), cite, lang(idx.eco), e.sig);
1753    if !e.doc.is_empty() {
1754        s.push_str(&truncate(&e.doc, max_doc));
1755        s.push('\n');
1756    }
1757    s
1758}
1759
1760/// The first paragraph of text of a section, and the short list or code
1761/// block right after it (what the section offers). Tag-only lines
1762/// (`<Deprecated>`) are skipped.
1763fn lead(doc: &str) -> String {
1764    let tag_only = |l: &str| l.starts_with('<') && l.ends_with('>');
1765    let mut lines = doc.lines().map(str::trim_end).peekable();
1766    let mut para: Vec<&str> = Vec::new();
1767    for l in lines.by_ref() {
1768        let t = l.trim();
1769        if t.starts_with("```") || t.starts_with("~~~") {
1770            return String::new();
1771        }
1772        if t.is_empty() || tag_only(t) {
1773            if para.is_empty() {
1774                continue;
1775            }
1776            break;
1777        }
1778        para.push(t);
1779    }
1780    let mut out = truncate(&para.join(" "), 320);
1781    while lines.peek().is_some_and(|l| l.trim().is_empty()) {
1782        lines.next();
1783    }
1784    let bullet = |l: &str| {
1785        let t = l.trim_start();
1786        t.starts_with("* ") || t.starts_with("- ") || t.split_once(". ").is_some_and(|(n, _)| !n.is_empty() && n.chars().all(|c| c.is_ascii_digit()))
1787    };
1788    if lines.peek().is_some_and(|l| bullet(l)) {
1789        let mut list = String::new();
1790        for l in lines.by_ref() {
1791            if l.trim().is_empty() {
1792                break;
1793            }
1794            let t = l.trim();
1795            list.push_str(if bullet(l) { "\n" } else { " " });
1796            list.push_str(t);
1797        }
1798        out.push_str(&truncate(&list, 400));
1799    } else if lines.peek().is_some_and(|l| l.trim_start().starts_with("```")) {
1800        let mut code = vec![lines.next().unwrap_or_default()];
1801        for l in lines.by_ref() {
1802            code.push(l);
1803            if l.trim_start().starts_with("```") {
1804                break;
1805            }
1806        }
1807        let code = code.join("\n");
1808        if code.len() <= 300 && code.matches("```").count() == 2 {
1809            out.push('\n');
1810            out.push_str(&code);
1811        }
1812    }
1813    out
1814}
1815
1816/// Leads of a prose section's page and parent section, quoted, when they
1817/// say something the section does not (a deprecation notice on the page,
1818/// the setup a subsection builds on).
1819fn section_context(idx: &PackageIndex, e: &Entry) -> Option<String> {
1820    if e.kind != Kind::Prose {
1821        return None;
1822    }
1823    let parts: Vec<&str> = e.name.split(" › ").collect();
1824    if parts.len() < 2 {
1825        return None;
1826    }
1827    let mut wanted = vec![parts[0].to_string()];
1828    if parts.len() > 2 {
1829        wanted.push(parts[..parts.len() - 1].join(" › "));
1830    }
1831    let mut out = String::new();
1832    for name in wanted {
1833        let Some(p) = idx
1834            .entries
1835            .iter()
1836            .filter(|x| x.kind == Kind::Prose && x.file == e.file && x.name == name)
1837            .min_by_key(|x| x.line)
1838        else {
1839            continue;
1840        };
1841        let l = lead(&p.doc);
1842        if l.len() < 20 || e.doc.contains(&l) || out.contains(&l) {
1843            continue;
1844        }
1845        for line in format!("{}: {l}", p.name.rsplit(" › ").next().unwrap_or("")).lines() {
1846            out.push_str("> ");
1847            out.push_str(line);
1848            out.push('\n');
1849        }
1850    }
1851    (!out.is_empty()).then(|| out + "\n")
1852}
1853
1854/// Token-budgeted output.
1855struct Pack {
1856    text: String,
1857    budget: usize,
1858    blocks: usize,
1859    full: bool,
1860}
1861
1862impl Pack {
1863    fn new(budget: usize) -> Pack {
1864        Pack {
1865            text: String::new(),
1866            budget,
1867            blocks: 0,
1868            full: false,
1869        }
1870    }
1871    fn used(&self) -> usize {
1872        est_tokens(&self.text)
1873    }
1874    fn push_raw(&mut self, s: &str) {
1875        self.text.push_str(s);
1876    }
1877    fn prepend(&mut self, s: &str) {
1878        self.text.insert_str(0, s);
1879    }
1880    /// Add a block if it fits; a first oversize block is trimmed to fit.
1881    fn push_block(&mut self, b: &str) -> bool {
1882        if self.full {
1883            return false;
1884        }
1885        let need = est_tokens(b) + 1;
1886        let left = self.budget.saturating_sub(self.used());
1887        if need <= left {
1888            self.text.push('\n');
1889            self.text.push_str(b);
1890            self.blocks += 1;
1891            return true;
1892        }
1893        if left > 120 {
1894            // Trim to the remaining budget at a line boundary.
1895            let max_chars = left * 3;
1896            let mut cut = String::new();
1897            for line in b.lines() {
1898                if cut.len() + line.len() + 1 > max_chars {
1899                    break;
1900                }
1901                cut.push_str(line);
1902                cut.push('\n');
1903            }
1904            if cut.matches("```").count() % 2 == 1 {
1905                cut.push_str("```\n");
1906            }
1907            if cut.lines().count() > 1 {
1908                self.text.push('\n');
1909                self.text.push_str(&cut);
1910                self.text.push_str("[…truncated to fit the token budget]\n");
1911                self.blocks += 1;
1912            }
1913        }
1914        self.full = true;
1915        false
1916    }
1917}
1918
1919/// Where the engine keeps its per-root state for long-running servers.
1920#[derive(Default)]
1921pub struct Workspace {
1922    engines: Mutex<HashMap<PathBuf, (u64, Arc<Engine>)>>,
1923}
1924
1925fn lock_stamp(root: &Path) -> u64 {
1926    // Re-read the project when any lockfile in the root changes.
1927    let mut h: u64 = 0;
1928    if let Ok(rd) = std::fs::read_dir(root) {
1929        for e in rd.flatten() {
1930            let n = e.file_name().to_string_lossy().to_string();
1931            if crate::lockfile::LOCKFILES.iter().any(|(f, _)| *f == n) || crate::lockfile::is_requirements(&n) || n == "package.json" || n == "pyproject.toml" {
1932                if let Ok(m) = e.metadata() {
1933                    let t = m
1934                        .modified()
1935                        .ok()
1936                        .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
1937                        .map_or(0, |d| d.as_nanos() as u64);
1938                    h = h.wrapping_mul(31).wrapping_add(t ^ m.len());
1939                }
1940            }
1941        }
1942    }
1943    h
1944}
1945
1946impl Workspace {
1947    pub fn engine(&self, root: &Path, opts: &Options) -> Arc<Engine> {
1948        let root = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
1949        let stamp = lock_stamp(&root);
1950        let mut m = self.engines.lock().unwrap();
1951        if let Some((s, e)) = m.get(&root) {
1952            if *s == stamp && e.opts == *opts {
1953                return e.clone();
1954            }
1955        }
1956        let e = Arc::new(Engine::new(&root, opts.clone()));
1957        m.insert(root, (stamp, e.clone()));
1958        e
1959    }
1960}
1961
1962#[cfg(test)]
1963mod tests {
1964    use super::*;
1965
1966    #[test]
1967    fn fetch_report_preserves_flat_counts_and_adds_full_provenance() {
1968        let dep = Dep {
1969            eco: Eco::Cargo,
1970            name: "axum".into(),
1971            version: "0.7.9".into(),
1972            direct: true,
1973            from: "Cargo.lock".into(),
1974        };
1975        let status: upstream::Manifest = serde_json::from_value(json!({
1976            "format": upstream::FORMAT, "repo": "github.com/tokio-rs/axum", "tag": "axum-v0.7.9",
1977            "commit": "0123456789abcdef0123456789abcdef01234567", "files": 20, "bytes": 100, "note": null,
1978        }))
1979        .unwrap();
1980        let report = fetch_report(&dep, &status);
1981        assert_eq!(report["files"], 20);
1982        assert_eq!(report["files"], report["upstream"]["files"]);
1983        assert_eq!(report["tag"], report["upstream"]["tag"]);
1984        assert_eq!(report["repo"], report["upstream"]["repo"]);
1985        assert!(report.get("note").is_some());
1986    }
1987
1988    #[test]
1989    fn upstream_is_default_with_explicit_opt_out() {
1990        assert!(automatic_upstream(None, false));
1991        assert!(automatic_upstream(Some("1"), false));
1992        for value in ["0", "false", "no"] {
1993            assert!(!automatic_upstream(Some(value), false));
1994        }
1995        assert!(!automatic_upstream(None, true));
1996    }
1997
1998    #[test]
1999    fn workspace_does_not_reuse_network_policy() {
2000        let ws = Workspace::default();
2001        let root = std::env::temp_dir();
2002        let online = Options {
2003            fetch: false,
2004            upstream: true,
2005            private: false,
2006        };
2007        let offline = Options {
2008            fetch: false,
2009            upstream: false,
2010            private: false,
2011        };
2012        let a = ws.engine(&root, &online);
2013        let b = ws.engine(&root, &offline);
2014        assert!(!Arc::ptr_eq(&a, &b));
2015        assert!(!b.opts.upstream);
2016    }
2017
2018    #[test]
2019    fn old_upgrade_guides_and_deprecated_pages() {
2020        let e = |name: &str, file: &str| Entry {
2021            kind: Kind::Prose,
2022            name: name.into(),
2023            path: name.into(),
2024            file: file.into(),
2025            line: 1,
2026            sig: String::new(),
2027            doc: String::new(),
2028            alias_of: None,
2029            legacy: false,
2030        };
2031        assert!(old_upgrade_guide(
2032            &e("Migrating to v6.0.0 › x", "upstream:docs/src/use/migrating-to-6.0.0.md"),
2033            9
2034        ));
2035        assert!(old_upgrade_guide(
2036            &e("Upgrade to Prisma ORM 4 › x", "upstream:docs/700-upgrading-to-prisma-4.mdx"),
2037            6
2038        ));
2039        assert!(!old_upgrade_guide(
2040            &e("Upgrade to Prisma ORM 6 › x", "upstream:docs/500-upgrading-to-prisma-6.mdx"),
2041            6
2042        ));
2043        assert!(!old_upgrade_guide(&e("Migration Guide › x", "upstream:docs/migration.md"), 2));
2044        assert!(deprecated_page(&e("Configure Language Options (Deprecated) › Globals", "x.md")));
2045        assert!(!heading_terms("Seeding with TypeScript or JavaScript").contains(&"type".to_string()));
2046        assert!(heading_terms("useActionState reference").contains(&"action".to_string()));
2047        let l = lead("Intro line.\n\n```css\n@import \"tailwindcss\";\n\n@custom-variant dark (&:where(.dark, .dark *));\n```\n\nMore.");
2048        assert!(l.contains("@custom-variant") && l.ends_with("```"), "{l}");
2049        let l = lead("Three helpers:\n\n* **`parse_obj`**: from a dict\n  more.\n* `parse_raw`\n\nNext.");
2050        assert!(l.contains("parse_obj") && l.contains("parse_raw") && !l.contains("Next"), "{l}");
2051        assert!(lead("<Deprecated>\n\nIn React 19, it is no longer necessary.\n\n</Deprecated>").starts_with("In React 19"));
2052    }
2053
2054    #[test]
2055    fn deprecation_redirects() {
2056        assert_eq!(redirects("Consider `z.strictObject(A.shape)` instead"), vec!["strictObject"]);
2057        assert_eq!(
2058            redirects("The `parse_obj` method is deprecated; use `model_validate` instead."),
2059            vec!["model_validate"]
2060        );
2061        assert_eq!(redirects("@deprecated Use .extend instead"), vec!["extend"]);
2062        assert_eq!(redirects("You can use this to parse data"), Vec::<String>::new());
2063        assert_eq!(redirects("Renamed to OptionalFromRequestParts."), vec!["OptionalFromRequestParts"]);
2064    }
2065}