Skip to main content

lockdocs_core/
query.rs

1//! The three questions agents ask: which versions do I have (`resolve`),
2//! what do the docs say about X (`docs`), and what exactly is this symbol
3//! (`api`). Answers are token-budgeted and cite `package@version path:line`.
4
5use crate::bm25::{self, Bm25};
6use crate::extract::{Entry, Kind};
7use crate::index::{self, PackageIndex};
8use crate::locate::{self, Source};
9use crate::project::{Project, Spec};
10use crate::{embed, est_tokens, fetch, upstream, Dep, Eco};
11use rayon::prelude::*;
12use serde_json::{json, Value};
13use std::collections::{HashMap, HashSet};
14use std::path::{Path, PathBuf};
15use std::sync::{Arc, Mutex};
16
17pub const DEFAULT_TOKENS: usize = 1200;
18/// Results scoring below this fraction of the best are left out (after 3).
19const RELEVANCE_FLOOR: f32 = 0.3;
20/// Share of the fused rank score from the embedding ranking; the rest is the
21/// keyword ranking. Small on purpose: measured on the 105-question benchmark,
22/// anything above ~0.15 lets the embedding outvote exact API-name matches.
23const DENSE_WEIGHT: f32 = 0.1;
24/// Reciprocal-rank constant: `1 / (RRF_K + rank)`.
25const RRF_K: f32 = 10.0;
26/// Share of the embedding in the score mix used when upstream docs are indexed.
27const DOCS_DENSE_WEIGHT: f32 = 0.45;
28/// Embedding candidates considered per query.
29const DENSE_POOL: usize = 300;
30/// The keyword leader is final when it scores at least this many times the
31/// runner-up (after boosts); the embedding is not even looked at.
32const DECISIVE_RATIO: f32 = 2.0;
33/// Share of the keyword score from headings and first sentences.
34const HEAD_WEIGHT: f32 = 0.25;
35/// Cross-dependency searches index at most this many direct dependencies.
36const MAX_PACKAGES: usize = 80;
37
38#[derive(Debug, Clone, PartialEq, Eq)]
39pub struct Options {
40    /// Allow downloading exact versions that are not installed.
41    pub fetch: bool,
42    /// Anonymous release-tag docs on first use; independent of registry downloads.
43    pub upstream: bool,
44    /// lockdocs Pro: read the private registries and git hosts the user has
45    /// configured. Set by the binary after it has verified the licence; the
46    /// core never decides this itself.
47    pub private: bool,
48}
49
50impl Default for Options {
51    fn default() -> Self {
52        let mut fetch = std::env::var("LOCKDOCS_FETCH").is_ok_and(|v| matches!(v.as_str(), "1" | "true" | "yes"));
53        let mut upstream = automatic_upstream(std::env::var("LOCKDOCS_FETCH").ok().as_deref(), std::env::var("LOCKDOCS_NO_UPSTREAM").is_ok());
54        if std::env::var("LOCKDOCS_OFFLINE").is_ok_and(|v| v != "0") {
55            fetch = false;
56            upstream = false;
57        }
58        Options {
59            fetch,
60            upstream,
61            private: false,
62        }
63    }
64}
65
66fn automatic_upstream(fetch: Option<&str>, disabled: bool) -> bool {
67    !disabled && !fetch.is_some_and(|v| matches!(v, "0" | "false" | "no"))
68}
69
70pub struct Answer {
71    pub text: String,
72    pub json: Value,
73}
74
75pub struct Engine {
76    pub project: Project,
77    opts: Options,
78    indexes: Mutex<HashMap<String, Arc<PackageIndex>>>,
79    upstream_notes: Mutex<HashMap<String, String>>,
80    /// Packages that needed a private source this engine was not licensed to read.
81    private_blocked: Mutex<Vec<crate::private::PrivateRequired>>,
82}
83
84/// An indexed package, the dependency it answers for, and a drift note.
85type ReadyPkg = (Arc<PackageIndex>, Dep, Option<String>);
86
87/// A resolved package ready to search.
88enum Resolved {
89    Ready(Arc<PackageIndex>, Dep, Option<String>),
90    Missing(Dep, String),
91}
92
93/// Keep the original flat fields for CLI consumers; richer provenance is additive.
94fn fetch_report(dep: &Dep, status: &upstream::Manifest) -> Value {
95    json!({"package": dep.id(), "repo": status.repo, "tag": status.tag,
96        "files": status.files, "note": status.note, "upstream": status})
97}
98
99fn provenance(ready: &[ReadyPkg]) -> Vec<Value> {
100    ready
101        .iter()
102        .map(|(idx, dep, note)| {
103            json!({
104                "package": idx.id(), "requested_version": dep.version, "source": idx.source,
105                "registry_fetched": idx.fetched, "upstream": idx.upstream_provenance, "upstream_label": idx.upstream, "note": note,
106            })
107        })
108        .collect()
109}
110
111/// Keep network fallbacks visible on ordinary broad MCP answers without listing
112/// every dependency's long header or crowding out the actual answer.
113fn broad_header(ready: &[ReadyPkg], missing: &[(Dep, String)], tokens: usize) -> String {
114    let fetched = ready.iter().filter(|r| r.0.fetched).count();
115    let upstream = ready.iter().filter(|r| r.0.upstream.is_some()).count();
116    let notes = ready.iter().filter(|r| r.2.is_some()).count();
117    let mut header = format!("Searched {} direct dependencies: {} local package sources, {fetched} registry sources, {upstream} upstream sources; {notes} fallback notes, {} unavailable.\n", ready.len(), ready.len() - fetched, missing.len());
118    let max_chars = (tokens / 3).clamp(100, 400) * 3;
119    for (idx, _, note) in ready {
120        if let Some(note) = note {
121            let row = format!("Fallback {}: {}\n", idx.id(), truncate(note, 160));
122            if header.chars().count() + row.chars().count() > max_chars {
123                break;
124            }
125            header.push_str(&row);
126        }
127    }
128    for (dep, error) in missing {
129        let row = format!("Skipped {}: {}\n", dep.id(), truncate(error, 140));
130        if header.chars().count() + row.chars().count() > max_chars {
131            break;
132        }
133        header.push_str(&row);
134    }
135    for (idx, _, _) in ready {
136        if let Some(m) = &idx.upstream_provenance {
137            let row = format!(
138                "Upstream {}: {}@{} commit {}{}\n",
139                idx.id(),
140                m.repo,
141                m.tag.as_deref().unwrap_or("none"),
142                m.commit.as_deref().map(|s| &s[..s.len().min(7)]).unwrap_or("none"),
143                if m.site.is_some() { " + opted-in docs site" } else { "" }
144            );
145            if header.chars().count() + row.chars().count() > max_chars {
146                break;
147            }
148            header.push_str(&row);
149        }
150    }
151    header.push_str("Focus `package` or request JSON for full provenance.\n");
152    header
153}
154
155fn lang(eco: Eco) -> &'static str {
156    match eco {
157        Eco::Npm => "ts",
158        Eco::PyPI => "python",
159        Eco::Cargo => "rust",
160        Eco::Go => "go",
161    }
162}
163
164/// Changelog prose: a changelog file, or a section under a version heading
165/// (`v1.7 (2020-10-26)`, `## 2.0.0`), as in READMEs that embed history.
166fn is_changelog(e: &Entry) -> bool {
167    let f = e.file.rsplit('/').next().unwrap_or(&e.file).to_ascii_uppercase();
168    if ["CHANGELOG", "CHANGES", "HISTORY", "NEWS", "RELEASES"].iter().any(|p| f.starts_with(p)) {
169        return true;
170    }
171    e.name.split(" › ").skip(1).any(|h| {
172        let h = h.trim().trim_start_matches(['v', 'V', '[']);
173        let mut parts = h.split(|c: char| !c.is_ascii_digit());
174        matches!((parts.next(), parts.next()), (Some(a), Some(b)) if !a.is_empty() && !b.is_empty()) && h.chars().nth(a_len(h)) == Some('.')
175    })
176}
177
178fn a_len(h: &str) -> usize {
179    h.chars().take_while(|c| c.is_ascii_digit()).count()
180}
181
182fn change_intent(q: &str) -> bool {
183    let q = q.to_ascii_lowercase();
184    [
185        "change",
186        "migrat",
187        "upgrad",
188        "breaking",
189        "deprecat",
190        "removed",
191        "renamed",
192        "new in",
193        "since",
194        "release",
195        "what's new",
196        "differen",
197    ]
198    .iter()
199    .any(|w| q.contains(w))
200}
201
202fn private_path(e: &Entry) -> bool {
203    e.file.split('/').any(|s| s.starts_with('_') && !s.starts_with("__init__")) || e.path.contains("._") || e.file.contains("/internal/")
204}
205
206/// Tooling shipped inside a package (type-checker plugins, test helpers).
207fn tooling_path(e: &Entry) -> bool {
208    e.file.split('/').any(|s| {
209        let stem = s.split('.').next().unwrap_or(s);
210        matches!(
211            stem,
212            "mypy" | "pylint" | "plugin" | "plugins" | "testing" | "conftest" | "scripts" | "codemod" | "codemods" | "eslint" | "babel"
213        )
214    })
215}
216
217impl Engine {
218    pub fn new(root: &Path, opts: Options) -> Engine {
219        Engine {
220            project: Project::load(root),
221            opts,
222            indexes: Mutex::new(HashMap::new()),
223            upstream_notes: Mutex::new(HashMap::new()),
224            private_blocked: Mutex::new(Vec::new()),
225        }
226    }
227
228    fn block(&self, req: &crate::private::PrivateRequired) {
229        let mut b = self.private_blocked.lock().unwrap();
230        if !b.iter().any(|r| r.package == req.package) {
231            b.push(req.clone());
232        }
233    }
234
235    /// The packages that needed a private source without Pro since the last
236    /// call. The binary turns a non-empty list into the `pro_required` answer.
237    pub fn take_private_blocked(&self) -> Vec<crate::private::PrivateRequired> {
238        std::mem::take(&mut *self.private_blocked.lock().unwrap())
239    }
240
241    pub fn root(&self) -> &Path {
242        &self.project.root
243    }
244
245    /// Find sources for a dependency: installed files first, then a
246    /// previously fetched copy, then (only when enabled) the registry.
247    fn source(&self, dep: &Dep) -> Result<(Source, Option<String>), String> {
248        let local = locate::locate(dep, &self.project.root);
249        if let Some(s) = &local {
250            if s.version == dep.version {
251                return Ok((s.clone(), None));
252            }
253        }
254        if let Some(s) = fetch::cached_for(dep, Some(&self.project.root)) {
255            return Ok((s, None));
256        }
257        if !self.opts.private {
258            if let Some(req) = crate::private::needs_private(dep, &self.project.root) {
259                self.block(&req);
260                return Err(format!("{req}. Run `lockdocs licence status`."));
261            }
262        }
263        if self.opts.fetch {
264            match fetch::fetch_for(dep, Some(&self.project.root), self.opts.private) {
265                Ok(s) => return Ok((s, None)),
266                Err(e) => {
267                    if let Some(req) = e.downcast_ref::<crate::private::PrivateRequired>() {
268                        self.block(req);
269                        return Err(format!("{req}. Run `lockdocs licence status`."));
270                    }
271                    return Err(format!("{}: exact registry fetching failed: {e:#}; no other version substituted", dep.id()));
272                }
273            }
274        }
275        if let Some(s) = local {
276            return Err(format!(
277                "{} pins {}, but {} has {}; refusing to substitute the installed version. Reinstall to sync, or enable exact registry fetching (--fetch).",
278                dep.from,
279                dep.id(),
280                s.label,
281                s.version
282            ));
283        }
284        let how = match dep.eco {
285            Eco::Npm => "run your package manager's install",
286            Eco::PyPI => "create the virtualenv (.venv) and install",
287            Eco::Cargo => "run `cargo fetch`",
288            Eco::Go => "run `go mod download`",
289        };
290        Err(format!(
291            "{} is pinned in {} but its files are not on this machine: {how}, or enable fetching (--fetch or LOCKDOCS_FETCH=1) to download exactly this version.",
292            dep.id(),
293            dep.from
294        ))
295    }
296
297    fn index(&self, dep: &Dep) -> Resolved {
298        match self.source(dep) {
299            Ok((src, mut note)) => {
300                let key = format!(
301                    "{}:{}@{}:{}:{}",
302                    dep.eco,
303                    dep.name,
304                    src.version,
305                    src.dir.display(),
306                    index::source_stamp(dep, &src, &self.project.root)
307                );
308                if let Some(i) = self.indexes.lock().unwrap().get(&key) {
309                    // Rebuild once the embedding model has arrived.
310                    if !i.embed.is_empty() || !embed::available() {
311                        if let Some(n) = self.upstream_notes.lock().unwrap().get(&dep.id()) {
312                            note = Some(match note {
313                                Some(old) => format!("{old} {n}"),
314                                None => n.clone(),
315                            });
316                        }
317                        return Resolved::Ready(i.clone(), dep.clone(), note);
318                    }
319                }
320                let (up, up_note) = self.upstream(dep, &src);
321                if let Some(n) = up_note {
322                    self.upstream_notes.lock().unwrap().insert(dep.id(), n.clone());
323                    note = Some(match note {
324                        Some(old) => format!("{old} {n}"),
325                        None => n,
326                    });
327                } else {
328                    self.upstream_notes.lock().unwrap().remove(&dep.id());
329                }
330                let idx = Arc::new(index::load_or_build(dep, &src, &self.project.root, up.as_ref()));
331                self.indexes.lock().unwrap().insert(key, idx.clone());
332                Resolved::Ready(idx, dep.clone(), note)
333            }
334            Err(e) => Resolved::Missing(dep.clone(), e),
335        }
336    }
337
338    /// Upstream docs for this exact version: a cached copy, or (when fetching
339    /// is enabled) a one-time download.
340    fn upstream(&self, dep: &Dep, src: &Source) -> (Option<(PathBuf, upstream::Manifest)>, Option<String>) {
341        if src.version != dep.version || std::env::var("LOCKDOCS_NO_UPSTREAM").is_ok() {
342            return (None, None);
343        }
344        let enabled = self.opts.fetch || self.opts.upstream;
345        if fetch::is_git(dep) {
346            return (
347                None,
348                enabled.then(|| "upstream docs skipped: git dependency's resolved commit is not a release tag; using its checkout files".into()),
349            );
350        }
351        let cached = upstream::cached(dep);
352        if let Some(c) = cached.as_ref().filter(|(_, m)| !enabled || !upstream::needs_refresh(m, self.opts.fetch)) {
353            let note = if upstream::needs_refresh(&c.1, false) {
354                Some(format!(
355                    "offline fallback: upstream cache format {} has not been exact-tag revalidated{}{}",
356                    c.1.format,
357                    if c.1.format < 4 {
358                        "; its old commit candidate may have been a branch"
359                    } else {
360                        ""
361                    },
362                    c.1.note.as_ref().map(|n| format!("; {n}")).unwrap_or_default()
363                ))
364            } else {
365                c.1.note.clone()
366            };
367            return (Some(c.clone()), note);
368        }
369        if enabled {
370            let result = if self.opts.fetch {
371                upstream::fetch_private(dep, src, self.opts.private)
372            } else {
373                upstream::fetch_automatic(dep, src)
374            };
375            return match result {
376                Ok(m) => (Some((upstream::dir(dep), m.clone())), m.note.clone()),
377                Err(e) => (None, Some(format!("upstream docs fetch failed: {e:#}; using package files"))),
378            };
379        }
380        (None, None)
381    }
382
383    /// `lockdocs fetch`: download what makes answers complete, once: missing
384    /// packages at their pinned versions, upstream docs at each version's git
385    /// tag, and the embedding model.
386    pub fn fetch_all(&self, package: Option<&str>) -> Result<Answer, String> {
387        let t = std::time::Instant::now();
388        let mut text = String::new();
389        if embed::enabled() {
390            match embed::ensure() {
391                Ok(()) => text.push_str(&format!("  model   {} ready\n", embed::MODEL_ID)),
392                Err(e) => text.push_str(&format!("  model   {} unavailable ({e:#}); keyword search only\n", embed::MODEL_ID)),
393            }
394        }
395        let deps: Vec<Dep> = match package.filter(|p| !p.trim().is_empty()) {
396            Some(p) => {
397                let mut out = Vec::new();
398                for spec in p.split(',').map(str::trim).filter(|s| !s.is_empty()) {
399                    out.extend(self.deps_for(spec)?);
400                }
401                out
402            }
403            None => {
404                let mut seen = HashSet::new();
405                self.project.direct().filter(|d| seen.insert((d.eco, d.name.clone()))).cloned().collect()
406            }
407        };
408        let rows: Vec<(Dep, String, Value)> = deps
409            .par_iter()
410            .map(|d| {
411                if fetch::is_git(d) {
412                    let src = locate::locate(d, &self.project.root);
413                    return (
414                        d.clone(),
415                        "upstream skipped: git dependency; use resolved checkout files".into(),
416                        json!({"package": d.id(), "status": "git-checkout", "source": src.map(|s| s.label)}),
417                    );
418                }
419                let src = match locate::locate(d, &self.project.root)
420                    .filter(|s| s.version == d.version)
421                    .or_else(|| fetch::cached_for(d, Some(&self.project.root)))
422                {
423                    Some(s) => Some(s),
424                    None if !fetch::is_git(d) => match fetch::fetch_for(d, Some(&self.project.root), self.opts.private) {
425                        Ok(s) => Some(s),
426                        Err(e) => {
427                            if let Some(req) = e.downcast_ref::<crate::private::PrivateRequired>() {
428                                self.block(req);
429                            }
430                            None
431                        }
432                    },
433                    None => None,
434                };
435                let Some(src) = src else {
436                    return (
437                        d.clone(),
438                        "package files unavailable".to_string(),
439                        json!({"package": d.id(), "status": "missing"}),
440                    );
441                };
442                let status = match upstream::fetch_private(d, &src, self.opts.private) {
443                    Ok(m) => m,
444                    Err(e) => {
445                        return (
446                            d.clone(),
447                            format!("upstream docs fetch failed: {e:#}"),
448                            json!({"package": d.id(), "status": "fetch-failed", "error": format!("{e:#}")}),
449                        )
450                    }
451                };
452                let line = match (&status.tag, status.files) {
453                    (_, n) if n > 0 => format!(
454                        "upstream docs {}{}: {n} files ({:.1} MB)",
455                        status.tag.as_ref().map(|t| format!("{}@{t}", status.repo)).unwrap_or_default(),
456                        status.site.as_ref().map(|s| format!(" + docs site {s}")).unwrap_or_default(),
457                        status.bytes as f64 / 1e6
458                    ),
459                    _ => format!("no upstream docs ({}: {})", status.repo, status.note.clone().unwrap_or_default()),
460                };
461                (
462                    d.clone(),
463                    if let Some(note) = &status.note { format!("{line}; {note}") } else { line },
464                    fetch_report(d, &status),
465                )
466            })
467            .collect();
468        for (d, line, _) in &rows {
469            text.push_str(&format!("  {:<40} {line}\n", d.id()));
470        }
471        // Rebuild indexes with what was fetched.
472        self.indexes.lock().unwrap().clear();
473        let ms = t.elapsed().as_millis();
474        Ok(Answer {
475            text: format!("Fetched for {} packages in {ms} ms (cached; later queries stay offline):\n{text}", rows.len()),
476            json: json!({"packages": rows.iter().map(|r| r.2.clone()).collect::<Vec<_>>(), "ms": ms, "model": embed::installed()}),
477        })
478    }
479
480    /// Dependencies for a package spec. A version not in the project is
481    /// fetched when allowed.
482    fn deps_for(&self, spec_str: &str) -> Result<Vec<Dep>, String> {
483        let spec = Spec::parse(spec_str);
484        let found = self.project.find(&spec);
485        if let Some(d) = found.iter().find(|d| spec.version_matches(&d.version)) {
486            return Ok(vec![(*d).clone()]);
487        }
488        let eco = spec.eco.or_else(|| found.first().map(|d| d.eco));
489        if let (Some(v), Some(eco)) = (&spec.version, eco) {
490            if spec.is_exact() {
491                let version = if eco == Eco::Go && !v.starts_with('v') { format!("v{v}") } else { v.clone() };
492                let dep = Dep {
493                    eco,
494                    name: found.first().map_or(spec.name.clone(), |d| d.name.clone()),
495                    version,
496                    direct: false,
497                    from: "requested".into(),
498                };
499                if fetch::cached_for(&dep, Some(&self.project.root)).is_some() || self.opts.fetch {
500                    return Ok(vec![dep]);
501                }
502                return Err(format!(
503                    "{}@{} is not in this project{}; enable fetching (--fetch or LOCKDOCS_FETCH=1) to read that exact version.",
504                    dep.name,
505                    dep.version,
506                    found.first().map(|d| format!(" (it pins {})", d.version)).unwrap_or_default()
507                ));
508            }
509        }
510        if let Some(d) = found.first() {
511            return Err(format!("this project pins {} {}, which does not match `{}`.", d.name, d.version, spec_str));
512        }
513        if spec.eco.is_none() && spec.version.is_some() && self.opts.fetch {
514            return Err(format!(
515                "`{}` is not a dependency here; prefix the ecosystem to fetch it (npm:, pypi:, cargo:, go:).",
516                spec.name
517            ));
518        }
519        let near = self.project.similar(&spec.name, 6);
520        Err(if near.is_empty() {
521            format!(
522                "`{}` is not a dependency of {} (lockfiles: {}). Use `resolve` to list dependencies.",
523                spec.name,
524                self.project.root.display(),
525                if self.project.lockfiles.is_empty() {
526                    "none found".into()
527                } else {
528                    self.project.lockfiles.join(", ")
529                }
530            )
531        } else {
532            format!("`{}` is not a dependency here. Did you mean: {}?", spec.name, near.join(", "))
533        })
534    }
535
536    /// Direct dependencies mentioned by name in the query, if any.
537    fn mentioned(&self, query: &str) -> Vec<Dep> {
538        let q = query.to_ascii_lowercase();
539        let words: HashSet<&str> = q
540            .split(|c: char| c.is_whitespace() || matches!(c, ',' | '?' | '(' | ')' | '`' | '\'' | '"'))
541            .collect();
542        let mut out: Vec<Dep> = Vec::new();
543        for d in self.project.direct() {
544            let n = d.name.to_ascii_lowercase();
545            let short = n.rsplit('/').next().unwrap_or(&n);
546            if (words.contains(n.as_str())
547                || (d.eco == Eco::Go && words.contains(short))
548                || words.iter().any(|w| w.starts_with(&format!("{n}.")) || w.starts_with(&format!("{n}::"))))
549                && !out.iter().any(|o| o.name == d.name)
550            {
551                out.push(d.clone());
552            }
553        }
554        out
555    }
556
557    fn select(&self, package: Option<&str>, query: &str) -> Result<(Vec<Dep>, bool), String> {
558        if let Some(p) = package.filter(|p| !p.trim().is_empty()) {
559            let mut out = Vec::new();
560            for spec in p.split(',').map(str::trim).filter(|s| !s.is_empty()) {
561                out.extend(self.deps_for(spec)?);
562            }
563            return Ok((out, false));
564        }
565        let m = self.mentioned(query);
566        if !m.is_empty() {
567            return Ok((m, false));
568        }
569        let mut seen = HashSet::new();
570        let direct: Vec<Dep> = self
571            .project
572            .direct()
573            .filter(|d| seen.insert((d.eco, d.name.clone())))
574            .take(MAX_PACKAGES)
575            .cloned()
576            .collect();
577        if direct.is_empty() {
578            return Err(format!(
579                "No dependencies found in {} (looked for {}). Pass `package`, or run from the project directory.",
580                self.project.root.display(),
581                "package-lock.json, pnpm-lock.yaml, yarn.lock, bun.lock, Cargo.lock, uv.lock, poetry.lock, requirements*.txt, go.mod"
582            ));
583        }
584        Ok((direct, true))
585    }
586
587    fn resolve_all(&self, deps: &[Dep]) -> (Vec<ReadyPkg>, Vec<(Dep, String)>) {
588        let results: Vec<Resolved> = deps.par_iter().map(|d| self.index(d)).collect();
589        let mut ready = Vec::new();
590        let mut missing = Vec::new();
591        for r in results {
592            match r {
593                Resolved::Ready(i, d, n) => ready.push((i, d, n)),
594                Resolved::Missing(d, e) => missing.push((d, e)),
595            }
596        }
597        (ready, missing)
598    }
599
600    /// Build (or load) indexes ahead of time and report what they hold.
601    pub fn warm(&self, package: Option<&str>) -> Result<Answer, String> {
602        let t = std::time::Instant::now();
603        let deps: Vec<Dep> = match package.filter(|p| !p.trim().is_empty()) {
604            Some(p) => self.select(Some(p), "")?.0,
605            None => {
606                let mut seen = HashSet::new();
607                self.project.direct().filter(|d| seen.insert((d.eco, d.name.clone()))).cloned().collect()
608            }
609        };
610        let (ready, missing) = self.resolve_all(&deps);
611        let mut text = String::new();
612        let mut items = Vec::new();
613        for (idx, _, note) in &ready {
614            let symbols = idx.entries.iter().filter(|e| e.kind != Kind::Prose).count();
615            text.push_str(&format!(
616                "  {:<40} {:>6} symbols {:>6} doc sections  ({})\n",
617                idx.id(),
618                symbols,
619                idx.entries.len() - symbols,
620                idx.source
621            ));
622            if let Some(n) = note {
623                text.push_str(&format!("    note: {n}\n"));
624            }
625            items.push(json!({"package": idx.id(), "ecosystem": idx.eco.as_str(), "symbols": symbols, "sections": idx.entries.len() - symbols, "source": idx.source, "upstream": idx.upstream, "note": note, "build_ms": idx.build_ms}));
626        }
627        for (d, e) in &missing {
628            text.push_str(&format!("  {:<40} missing: {e}\n", d.id()));
629        }
630        let ms = t.elapsed().as_millis();
631        text = format!("Indexed {} of {} packages in {ms} ms.\n{text}", ready.len(), deps.len());
632        Ok(Answer {
633            text,
634            json: json!({"packages": items, "missing": missing.iter().map(|(d, e)| json!({"package": d.id(), "error": e})).collect::<Vec<_>>(), "ms": ms}),
635        })
636    }
637
638    // ------------------------------------------------------------ resolve
639
640    pub fn resolve(&self, filter: Option<&str>) -> Answer {
641        let p = &self.project;
642        let filter = filter.map(|f| f.trim().to_ascii_lowercase()).filter(|f| !f.is_empty());
643        let shown: Vec<&Dep> = match &filter {
644            Some(f) => p.deps.iter().filter(|d| d.name.to_ascii_lowercase().contains(f.as_str())).collect(),
645            None => p.direct().collect(),
646        };
647        let rows: Vec<(&Dep, Option<Source>)> = shown
648            .par_iter()
649            .map(|d| (*d, locate::locate(d, &p.root).or_else(|| fetch::cached_for(d, Some(&p.root)))))
650            .collect();
651        let mut text = format!(
652            "Project: {}\nLockfiles: {}\n",
653            p.root.display(),
654            if p.lockfiles.is_empty() {
655                "none found".to_string()
656            } else {
657                p.lockfiles.join(", ")
658            }
659        );
660        for w in &p.warnings {
661            text.push_str(&format!("Warning: {w}\n"));
662        }
663        let total_direct = p.direct().count();
664        text.push_str(&match &filter {
665            Some(f) => format!("\n{} of {} dependencies match `{f}`:\n", rows.len(), p.deps.len()),
666            None => format!(
667                "\n{} direct dependencies ({} total with transitive; pass a filter to search all):\n",
668                total_direct,
669                p.deps.len()
670            ),
671        });
672        let w = rows.iter().map(|(d, _)| d.name.len() + d.version.len() + 1).max().unwrap_or(10).min(48);
673        let mut items = Vec::new();
674        for (d, s) in &rows {
675            let id = format!("{}@{}", d.name, d.version);
676            let (status, src) = match s {
677                Some(s) if s.version == d.version => ("docs ready", s.label.clone()),
678                Some(s) => ("version drift", format!("installed {} at {}", s.version, s.label)),
679                None => (
680                    "not installed",
681                    if self.opts.fetch {
682                        "will fetch on first query".into()
683                    } else {
684                        "install it, or enable fetching".to_string()
685                    },
686                ),
687            };
688            text.push_str(&format!(
689                "  {:<6} {:<w$}  {}{}  {}\n",
690                d.eco.as_str(),
691                id,
692                if d.direct { "" } else { "(transitive) " },
693                status,
694                src
695            ));
696            items.push(json!({"ecosystem": d.eco.as_str(), "name": d.name, "version": d.version, "direct": d.direct, "lockfile": d.from, "status": status, "source": src}));
697        }
698        if rows.is_empty() {
699            text.push_str("  (none)\n");
700        }
701        Answer {
702            text,
703            json: json!({"root": p.root.display().to_string(), "lockfiles": p.lockfiles, "dependencies": items, "total": p.deps.len(), "warnings": p.warnings}),
704        }
705    }
706
707    // ------------------------------------------------------------ docs
708
709    /// Upgrade report (lockdocs Pro; the caller checks the licence): the API
710    /// difference between the pinned version of `package` and `target`,
711    /// limited to symbols this project uses. The target version is read from
712    /// the same sources as any other version (installed, cached, or fetched
713    /// when fetching is enabled).
714    pub fn upgrade_report(&self, package: &str, target: &str, tokens: usize) -> Result<Answer, String> {
715        let spec = Spec::parse(package);
716        let found = self.project.find(&spec);
717        let dep = found
718            .iter()
719            .find(|d| d.direct)
720            .or_else(|| found.first())
721            .map(|d| (*d).clone())
722            .ok_or_else(|| format!("`{package}` is not pinned in this project's lockfiles; run `resolve` to see packages."))?;
723        let target = target.trim().trim_start_matches('v');
724        if target.is_empty() {
725            return Err("`upgrade_to` needs a target version, e.g. 15.0.0".into());
726        }
727        if target == dep.version.trim_start_matches('v') {
728            return Err(format!("{} is already the pinned version; pick a different target.", dep.id()));
729        }
730        let old = match self.index(&dep) {
731            Resolved::Ready(i, _, _) => i,
732            Resolved::Missing(_, e) => return Err(e),
733        };
734        let target_dep = Dep {
735            eco: dep.eco,
736            name: dep.name.clone(),
737            version: target.to_string(),
738            direct: false,
739            from: "requested".into(),
740        };
741        let new = match self.index(&target_dep) {
742            Resolved::Ready(i, _, _) => i,
743            Resolved::Missing(_, e) => {
744                return Err(format!(
745                    "{e}\nThe target version's files are needed to compare; add --fetch (or LOCKDOCS_FETCH=1) to download exactly {}.",
746                    target_dep.id()
747                ))
748            }
749        };
750        let r = crate::upgrade::report(&self.project.root, &old, &new);
751        let max = tokens.clamp(200, 20000) * 3;
752        let text = if r.text.len() > max {
753            format!("{}\n... truncated to the token budget; raise `tokens`.\n", truncate(&r.text, max))
754        } else {
755            r.text
756        };
757        Ok(Answer { text, json: r.json })
758    }
759
760    pub fn docs(&self, package: Option<&str>, query: &str, tokens: usize) -> Result<Answer, String> {
761        let tokens = tokens.clamp(200, 20000);
762        let mut imports = HashMap::new();
763        let selector = package.map(|p| {
764            p.split(',')
765                .map(|p| {
766                    let spec = Spec::parse(p.trim());
767                    let base = npm_import_base(&spec.name);
768                    let npm = spec.eco == Some(Eco::Npm) || spec.eco.is_none() && self.project.deps.iter().any(|d| d.eco == Eco::Npm && d.name == base);
769                    if npm && base != spec.name {
770                        imports.entry(base.to_string()).or_insert_with(Vec::new).push(spec.name.clone());
771                        format!("npm:{base}{}", spec.version.as_ref().map(|v| format!("@{v}")).unwrap_or_default())
772                    } else {
773                        if npm {
774                            imports.entry(base.to_string()).or_insert_with(Vec::new).push(base.to_string());
775                        }
776                        p.trim().to_string()
777                    }
778                })
779                .collect::<Vec<_>>()
780                .join(",")
781        });
782        let (deps, broad) = self.select(selector.as_deref(), query)?;
783        let (ready, missing) = self.resolve_all(&deps);
784        if ready.is_empty() {
785            return Err(missing.into_iter().map(|(_, e)| e).collect::<Vec<_>>().join("\n"));
786        }
787        let mut scopes = Vec::new();
788        let mut public_calls = HashSet::new();
789        for (pi, (idx, _, _)) in ready.iter().enumerate() {
790            let mut requested = imports.get(&idx.name).cloned().unwrap_or_else(|| vec![idx.name.clone()]);
791            // A question naming known public imports can narrow a root search,
792            // but must never override an explicitly selected alternate edition.
793            if requested.iter().all(|import| import == &idx.name) {
794                if let Some(api) = &idx.npm_api {
795                    let named: std::collections::BTreeSet<String> = api
796                        .roots
797                        .iter()
798                        .filter(|r| r.import != idx.name && mentions_import(query, &r.import))
799                        .map(|r| r.import.clone())
800                        .collect();
801                    if !named.is_empty() {
802                        requested = named.into_iter().collect();
803                    }
804                }
805            }
806            if let Some(api) = &idx.npm_api {
807                for import in &requested {
808                    if import != &idx.name && !api.roots.iter().any(|r| &r.import == import) {
809                        return Err(format!("`{import}` has no public declaration entrypoint in {}.", idx.id()));
810                    }
811                }
812            }
813            let mut scope = std::collections::BTreeSet::new();
814            for import in requested {
815                scope.extend(idx.docs_indices(Some(&import)));
816                public_calls.extend(public_call_entries(idx, &import, query).into_iter().map(|ei| (pi, ei)));
817            }
818            scopes.push(scope);
819        }
820        let q = query.trim();
821        if q.is_empty() {
822            return Ok(self.overview(&ready, &scopes, tokens));
823        }
824        let qterms = bm25::query_terms(q);
825        // One BM25 over the selected packages so scores compare.
826        let mut refs: Vec<(usize, usize)> = Vec::new();
827        for (pi, scope) in scopes.iter().enumerate() {
828            for &ei in scope {
829                refs.push((pi, ei));
830            }
831        }
832        let bm = Bm25::build(refs.iter().map(|&(pi, ei)| (ready[pi].0.terms[ei].as_slice(), ready[pi].0.lens[ei])));
833        let hb = Bm25::build(refs.iter().map(|&(pi, ei)| (ready[pi].0.head[ei].as_slice(), ready[pi].0.head_lens[ei])));
834        let mut raw_idents = identifiers(q);
835        raw_idents.extend(api_words(&ready, &refs, q));
836        let changes = change_intent(q);
837        let mut scored = hybrid_rank(&ready, &refs, (&bm, &hb), &qterms, q, &raw_idents, changes);
838        // An explicitly called public API is the subject of the question.
839        // Referenced type members with the same name remain supporting hits.
840        scored.sort_by_key(|&(_, pi, ei)| !public_calls.contains(&(pi, ei)));
841        let mut out = Pack::new(tokens);
842        let mut header = String::new();
843        for (idx, dep, note) in &ready {
844            header.push_str(&format!(
845                "{} · {} · {}{}\n",
846                idx.id(),
847                idx.eco,
848                idx.source,
849                if dep.from == "requested" {
850                    String::new()
851                } else {
852                    format!(" · pinned in {}", dep.from)
853                }
854            ));
855            if let Some(u) = &idx.upstream {
856                header.push_str(&format!("Upstream docs: {u}\n"));
857            }
858            if let Some(n) = note {
859                header.push_str(&format!("Note: {n}\n"));
860            }
861        }
862        if broad {
863            header = broad_header(&ready, &missing, tokens);
864        }
865        for (d, e) in &missing {
866            if !broad {
867                header.push_str(&format!("Skipped {}: {e}\n", d.id()));
868            }
869        }
870        out.push_raw(&header);
871        let mut seen: HashSet<(usize, String, u32)> = HashSet::new();
872        let mut hits_json = Vec::new();
873        let mut used_pkgs: Vec<String> = Vec::new();
874        let top = scored.first().map_or(0.0, |s| s.0);
875        for (score, pi, ei) in scored {
876            let idx = &ready[pi].0;
877            let e = &idx.entries[ei];
878            // Stop at weak matches once a few good ones are in.
879            if out.blocks >= 3 && score < top * RELEVANCE_FLOOR {
880                break;
881            }
882            if !seen.insert((pi, e.file.clone(), e.line)) {
883                continue;
884            }
885            let e = idx.npm_api.as_ref().map_or_else(|| resolve_alias(idx, e), |api| api.search_alias(e));
886            // The top result also carries the lead of its page and parent
887            // section, within the same budget.
888            let ctx = if out.blocks == 0 { section_context(idx, e, &scopes[pi]) } else { None };
889            let max_doc = if out.blocks == 0 {
890                2000 - ctx.as_ref().map_or(0, |c| c.len().min(800) / 2)
891            } else {
892                800
893            };
894            let mut block = render_entry(idx, e, max_doc);
895            if let (Some(ctx), Some(nl)) = (ctx, block.find('\n')) {
896                block.insert_str(nl + 1, &ctx);
897            }
898            if !out.push_block(&block) {
899                break;
900            }
901            if !used_pkgs.contains(&idx.id()) {
902                used_pkgs.push(idx.id());
903            }
904            hits_json.push(json!({"package": idx.id(), "kind": e.kind.as_str(), "path": e.path, "file": e.file, "line": e.line, "score": score}));
905        }
906        if out.blocks == 0 {
907            out.push_raw(&format!(
908                "No match for \"{q}\" in {}. Try other words, a symbol name (the `api` tool), or `package` to widen/narrow.\n",
909                ready.iter().map(|r| r.0.id()).collect::<Vec<_>>().join(", ")
910            ));
911        } else if broad {
912            out.prepend(&format!("Sources: {}\n", used_pkgs.join(", ")));
913        }
914        // Packages that ship few docs: point at the one-time upstream fetch.
915        for (idx, dep, _) in &ready {
916            let prose = idx.entries.iter().filter(|e| e.kind == Kind::Prose).count();
917            if !broad && idx.upstream.is_none() && prose < 40 && !self.opts.fetch && upstream::cached(dep).is_none() {
918                out.push_raw(&format!(
919                    "\nTip: {} ships few docs. `lockdocs fetch {}` adds its upstream docs at this version's git tag (one-time download, then offline).\n",
920                    idx.id(),
921                    idx.name
922                ));
923            }
924        }
925        Ok(Answer {
926            json: json!({"query": q, "packages": ready.iter().map(|r| r.0.id()).collect::<Vec<_>>(), "hits": hits_json, "provenance": provenance(&ready), "tokens": est_tokens(&out.text)}),
927            text: out.text,
928        })
929    }
930
931    fn overview(&self, ready: &[ReadyPkg], scopes: &[std::collections::BTreeSet<usize>], tokens: usize) -> Answer {
932        let mut out = Pack::new(tokens);
933        for ((idx, dep, note), scope) in ready.iter().zip(scopes) {
934            out.push_raw(&format!("{} · {} · {} · pinned in {}\n", idx.id(), idx.eco, idx.source, dep.from));
935            if let Some(u) = &idx.upstream {
936                out.push_raw(&format!("Upstream: {u}\n"));
937            }
938            if let Some(n) = note {
939                out.push_raw(&format!("Note: {n}\n"));
940            }
941            let entries: Vec<&Entry> = scope.iter().map(|&i| &idx.entries[i]).collect();
942            let symbols = entries
943                .iter()
944                .filter(|e| e.kind != Kind::Prose && e.kind != Kind::Alias && !private_path(e))
945                .count();
946            let prose = entries.len() - symbols;
947            out.push_raw(&format!("Indexed {symbols} API symbols and {prose} doc sections.\n"));
948            // Top-level API names, shortest paths first.
949            let mut top: Vec<&Entry> = entries
950                .iter()
951                .copied()
952                .filter(|e| e.kind != Kind::Prose && !private_path(e) && !e.legacy && !e.doc.is_empty())
953                .collect();
954            top.sort_by_key(|e| (e.path.matches(['.', ':']).count(), e.path.len()));
955            let names: Vec<String> = top.iter().take(60).map(|e| e.path.clone()).collect();
956            if !names.is_empty() {
957                out.push_raw(&format!("Key API: {}\n", names.join(", ")));
958            }
959            for e in entries
960                .iter()
961                .filter(|e| e.kind == Kind::Prose && e.file.to_ascii_uppercase().starts_with("README"))
962                .take(3)
963            {
964                if !out.push_block(&render_entry(idx, e, 1500)) {
965                    break;
966                }
967            }
968        }
969        Answer {
970            json: json!({"packages": ready.iter().map(|r| r.0.id()).collect::<Vec<_>>(), "provenance": provenance(ready), "tokens": est_tokens(&out.text)}),
971            text: out.text,
972        }
973    }
974
975    // ------------------------------------------------------------ api
976
977    pub fn api(&self, symbol: &str, package: Option<&str>, tokens: usize) -> Result<Answer, String> {
978        let tokens = tokens.clamp(200, 20000);
979        let symbol = symbol.trim().trim_end_matches("()").trim();
980        if symbol.is_empty() {
981            return Err("symbol is required, e.g. `z.object`, `tokio::spawn`, `BaseModel.model_dump`".into());
982        }
983        let mut requested_import = None;
984        let (deps, rest) = match package.filter(|p| !p.trim().is_empty()) {
985            Some(p) => {
986                let spec = Spec::parse(p);
987                let base = npm_import_base(&spec.name);
988                if spec.eco.is_none_or(|e| e == Eco::Npm) && base != spec.name && self.project.find(&spec).is_empty() {
989                    let selector = format!("npm:{base}{}", spec.version.as_ref().map(|v| format!("@{v}")).unwrap_or_default());
990                    requested_import = Some(spec.name);
991                    (self.select(Some(&selector), "")?.0, symbol.to_string())
992                } else {
993                    (self.select(Some(p), "")?.0, symbol.to_string())
994                }
995            }
996            None => match self
997                .npm_import_prefix(symbol)
998                .map(|(dep, import, rest)| {
999                    requested_import = Some(import);
1000                    (dep, rest)
1001                })
1002                .or_else(|| self.package_prefix(symbol))
1003            {
1004                Some((dep, rest)) => (vec![dep], rest),
1005                None => (self.select(None, symbol)?.0, symbol.to_string()),
1006            },
1007        };
1008        let segs: Vec<String> = split_symbol(&rest);
1009        let Some(last) = segs.last().map(|l| l.trim_end_matches('!').to_string()) else {
1010            // Only a package name: give the overview.
1011            return self.docs(
1012                Some(&deps.iter().map(|d| format!("{}:{}@{}", d.eco, d.name, d.version)).collect::<Vec<_>>().join(",")),
1013                "",
1014                tokens,
1015            );
1016        };
1017        let quals: Vec<String> = segs[..segs.len() - 1].iter().map(|s| s.to_ascii_lowercase()).collect();
1018        let (ready, missing) = self.resolve_all(&deps);
1019        if ready.is_empty() {
1020            return Err(missing.into_iter().map(|(_, e)| e).collect::<Vec<_>>().join("\n"));
1021        }
1022        let mut cands: Vec<(f32, usize, usize)> = Vec::new();
1023        let mut applicability = Vec::new();
1024        let mut alternatives = Vec::new();
1025        let mut unresolved = false;
1026        let mut npm_scoped = false;
1027        let last_l = last.to_ascii_lowercase();
1028        let public_path = segs.join(".");
1029        for (pi, (idx, _, _)) in ready.iter().enumerate() {
1030            if let Some(api) = &idx.npm_api {
1031                npm_scoped = true;
1032                let import = requested_import.as_deref().unwrap_or(&idx.name);
1033                let mut has_root = false;
1034                for root in &api.roots {
1035                    let found = api.resolve(root, &public_path, idx.api_entries());
1036                    if root.import == import {
1037                        has_root = true;
1038                        unresolved |= found.unresolved;
1039                        for ei in found.entries {
1040                            let e = &idx.api_entries()[ei];
1041                            cands.push((10.0, pi, ei));
1042                            applicability.push(
1043                                json!({"import": root.import, "conditions": root.conditions.trim_start_matches('/'), "entrypoint": root.file,
1044                                "file": e.file, "line": e.line, "path": e.path}),
1045                            );
1046                        }
1047                    } else if !found.unresolved {
1048                        for ei in found.entries {
1049                            let e = &idx.api_entries()[ei];
1050                            alternatives.push(
1051                                json!({"package": idx.id(), "import": root.import, "conditions": root.conditions.trim_start_matches('/'),
1052                                "entrypoint": root.file, "path": e.path, "file": e.file, "line": e.line, "signature": e.sig}),
1053                            );
1054                        }
1055                    }
1056                }
1057                unresolved |= !has_root;
1058                continue;
1059            }
1060            for (ei, e) in idx.api_entries().iter().enumerate() {
1061                if e.kind == Kind::Prose {
1062                    continue;
1063                }
1064                let name = e.name.trim_end_matches('!');
1065                let exact = name == last;
1066                if !exact && name.to_ascii_lowercase() != last_l {
1067                    continue;
1068                }
1069                cands.push((api_score(e, exact, &quals), pi, ei));
1070            }
1071        }
1072        cands.sort_by_key(|(_, pi, ei)| (*pi, *ei));
1073        cands.dedup_by_key(|(_, pi, ei)| (*pi, *ei));
1074        // Multiple overloads at one declaration are valid. Conflicting files,
1075        // packages or conditional editions are not a license to guess.
1076        let locations: HashSet<_> = cands
1077            .iter()
1078            .map(|(_, pi, ei)| (*pi, &ready[*pi].0.api_entries()[*ei].file, &ready[*pi].0.api_entries()[*ei].path))
1079            .collect();
1080        let ambiguous = npm_scoped && locations.len() > 1 && {
1081            let signatures: HashSet<_> = cands.iter().map(|(_, pi, ei)| &ready[*pi].0.api_entries()[*ei].sig).collect();
1082            signatures.len() > 1
1083        };
1084        if npm_scoped && (cands.is_empty() || unresolved || ambiguous) {
1085            let status = if ambiguous {
1086                "ambiguous"
1087            } else if unresolved {
1088                "unresolved"
1089            } else {
1090                "not_exported"
1091            };
1092            let imports: Vec<_> = ready
1093                .iter()
1094                .filter(|r| r.0.eco == Eco::Npm)
1095                .map(|r| requested_import.as_deref().unwrap_or(&r.0.name))
1096                .collect();
1097            let mut out = Pack::new(tokens);
1098            out.push_raw(&format!(
1099                "No verified exact public API match for `{public_path}` from `{}` ({}).\n",
1100                imports.join("`, `"),
1101                status
1102            ));
1103            out.push_block(if ambiguous {
1104                "Public declarations disagree across entrypoints or export conditions; no signature was selected.\n"
1105            } else if unresolved {
1106                "Import applicability could not be fully resolved from the installed declarations; no unrelated same-named symbol was substituted.\n"
1107            } else {
1108                "This symbol is not exported by the requested import entrypoint. Same-named internal declarations are not a match.\n"
1109            });
1110            for a in alternatives.iter().take(8) {
1111                out.push_block(&format!(
1112                    "Alternate import `{}` ({}): `{}` — {} {}:{}\n",
1113                    a["import"].as_str().unwrap_or(""),
1114                    a["conditions"].as_str().unwrap_or(""),
1115                    a["signature"].as_str().unwrap_or(""),
1116                    a["package"].as_str().unwrap_or(""),
1117                    a["file"].as_str().unwrap_or(""),
1118                    a["line"]
1119                ));
1120            }
1121            if ambiguous {
1122                for a in applicability.iter().take(8) {
1123                    out.push_block(&format!(
1124                        "Applicable import `{}` ({}) via {} → {}:{}\n",
1125                        a["import"].as_str().unwrap_or(""),
1126                        a["conditions"].as_str().unwrap_or(""),
1127                        a["entrypoint"].as_str().unwrap_or(""),
1128                        a["file"].as_str().unwrap_or(""),
1129                        a["line"]
1130                    ));
1131                }
1132                for (_, pi, ei) in cands.iter().take(8) {
1133                    let idx = &ready[*pi].0;
1134                    out.push_block(&render_entry(idx, &idx.api_entries()[*ei], 500));
1135                }
1136            }
1137            return Ok(Answer {
1138                json: json!({"status": status, "symbol": public_path, "imports": imports, "provenance": provenance(&ready),
1139                "applicability": applicability, "alternatives": alternatives, "tokens": est_tokens(&out.text)}),
1140                text: out.text,
1141            });
1142        }
1143        cands.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));
1144        if cands.is_empty() {
1145            // Nothing by that name: fall back to a docs search scoped to the same packages.
1146            let pk = ready
1147                .iter()
1148                .map(|r| format!("{}:{}@{}", r.0.eco, r.0.name, r.0.version))
1149                .collect::<Vec<_>>()
1150                .join(",");
1151            let mut a = self.docs(Some(&pk), &segs.join(" "), tokens)?;
1152            a.text = format!(
1153                "No symbol named `{last}` in {}; closest documentation:\n\n{}",
1154                ready.iter().map(|r| r.0.id()).collect::<Vec<_>>().join(", "),
1155                a.text
1156            );
1157            return Ok(a);
1158        }
1159        let (_, bpi, bei) = cands[0];
1160        let (idx, dep, note) = &ready[bpi];
1161        let best0 = &idx.api_entries()[bei];
1162        let best = if idx.npm_api.is_some() { best0 } else { resolve_alias(idx, best0) };
1163        let mut out = Pack::new(tokens);
1164        out.push_raw(&format!(
1165            "{} · {} · {}{}\n",
1166            idx.id(),
1167            idx.eco,
1168            idx.source,
1169            if dep.from == "requested" {
1170                String::new()
1171            } else {
1172                format!(" · pinned in {}", dep.from)
1173            }
1174        ));
1175        if let Some(u) = &idx.upstream {
1176            out.push_raw(&format!("Upstream: {u}\n"));
1177        }
1178        if let Some(n) = note {
1179            out.push_raw(&format!("Note: {n}\n"));
1180        }
1181        if !std::ptr::eq(best, best0) {
1182            out.push_raw(&format!(
1183                "`{}` is a re-export of `{}` ({}:{}).\n",
1184                best0.path, best.path, best0.file, best0.line
1185            ));
1186        }
1187        if idx.npm_api.is_some() {
1188            out.push_raw(&format!(
1189                "Public import: `{}` · symbol `{public_path}`\n",
1190                requested_import.as_deref().unwrap_or(&idx.name)
1191            ));
1192            for a in &applicability {
1193                out.push_block(&format!(
1194                    "Declaration entrypoint: {} ({})\n",
1195                    a["entrypoint"].as_str().unwrap_or(""),
1196                    a["conditions"].as_str().unwrap_or("")
1197                ));
1198            }
1199        }
1200        out.push_block(&render_entry(idx, best, 6000));
1201        // Overloads and declaration merging: same path, other entries.
1202        let overloads: Vec<&Entry> = idx
1203            .api_entries()
1204            .iter()
1205            .filter(|e| e.path == best.path && e.file == best.file && !std::ptr::eq(*e, best) && e.kind != Kind::Prose && e.kind != Kind::Alias)
1206            .take(8)
1207            .collect();
1208        if !overloads.is_empty() {
1209            let mut s = String::from("Other declarations:\n");
1210            for e in overloads {
1211                s.push_str(&format!("- `{}` — {} {}:{}\n", e.sig, idx.id(), e.file, e.line));
1212            }
1213            out.push_block(&s);
1214        }
1215        let mut members_json = Vec::new();
1216        if best.kind.is_container() {
1217            let sep = if idx.eco == Eco::Cargo { "::" } else { "." };
1218            let prefix = format!("{}{sep}", best.path);
1219            let members: Vec<&Entry> = idx
1220                .api_entries()
1221                .iter()
1222                .filter(|e| {
1223                    e.path.starts_with(&prefix)
1224                        && !e.path[prefix.len()..].contains(sep)
1225                        && e.kind != Kind::Prose
1226                        && (idx.npm_api.is_none() || e.file == best.file)
1227                })
1228                .collect();
1229            if !members.is_empty() {
1230                let mut s = format!("Members of {} ({}):\n", best.path, members.len());
1231                for m in members.iter().take(80) {
1232                    let doc1 = m.doc.lines().next().unwrap_or("");
1233                    s.push_str(&format!(
1234                        "- `{}`{}{} — {}:{}\n",
1235                        m.sig,
1236                        if doc1.is_empty() { "" } else { " — " },
1237                        truncate(doc1, 120),
1238                        m.file,
1239                        m.line
1240                    ));
1241                    members_json.push(json!({"path": m.path, "sig": m.sig, "file": m.file, "line": m.line}));
1242                }
1243                out.push_block(&s);
1244            }
1245        }
1246        let others: Vec<String> = cands
1247            .iter()
1248            .skip(1)
1249            .filter(|(_, pi, ei)| {
1250                let e = &ready[*pi].0.api_entries()[*ei];
1251                !(e.path == best.path && e.file == best.file) && !std::ptr::eq(e, best0)
1252            })
1253            .take(8)
1254            .map(|(_, pi, ei)| {
1255                let (ix, _, _) = &ready[*pi];
1256                let e = &ix.api_entries()[*ei];
1257                format!("- {} `{}` — {} {}:{}", e.kind.as_str(), e.path, ix.id(), e.file, e.line)
1258            })
1259            .collect();
1260        if !others.is_empty() {
1261            out.push_block(&format!("Other matches:\n{}\n", others.join("\n")));
1262        }
1263        Ok(Answer {
1264            json: json!({
1265                "provenance": provenance(&ready), "package": idx.id(), "kind": best.kind.as_str(), "path": best.path, "signature": best.sig, "doc": best.doc,
1266                "file": best.file, "line": best.line, "members": members_json, "tokens": est_tokens(&out.text),
1267                "status": "exact", "import": if idx.npm_api.is_some() { Some(requested_import.as_deref().unwrap_or(&idx.name)) } else { None },
1268                "applicability": applicability, "alternatives": alternatives,
1269            }),
1270            text: out.text,
1271        })
1272    }
1273
1274    fn npm_import_prefix(&self, symbol: &str) -> Option<(Dep, String, String)> {
1275        for d in &self.project.deps {
1276            if d.eco != Eco::Npm {
1277                continue;
1278            }
1279            let Some(rest) = symbol.strip_prefix(&format!("{}/", d.name)) else {
1280                continue;
1281            };
1282            if let Some((subpath, sym)) = rest.split_once('.') {
1283                return Some((d.clone(), format!("{}/{subpath}", d.name), sym.to_string()));
1284            }
1285        }
1286        None
1287    }
1288
1289    /// `zod.z.object` -> (zod, `z.object`); `tokio::spawn` -> (tokio, `spawn`);
1290    /// `@tanstack/react-query.useQuery`; `github.com/gin-gonic/gin.Context`.
1291    fn package_prefix(&self, symbol: &str) -> Option<(Dep, String)> {
1292        let mut best: Option<(usize, Dep)> = None;
1293        let bytes = symbol.as_bytes();
1294        let mut cuts: Vec<usize> = Vec::new();
1295        for i in 0..bytes.len() {
1296            if bytes[i] == b'.' || bytes[i] == b'#' || (bytes[i] == b':' && i + 1 < bytes.len() && bytes[i + 1] == b':') {
1297                cuts.push(i);
1298            }
1299        }
1300        cuts.push(symbol.len());
1301        for &c in &cuts {
1302            let cand = &symbol[..c];
1303            let spec = Spec::parse(cand);
1304            if spec.version.is_some() {
1305                continue;
1306            }
1307            let found = self.project.find(&spec);
1308            if let Some(d) = found.first() {
1309                if best.as_ref().is_none_or(|(bc, _)| c > *bc) {
1310                    best = Some((c, (*d).clone()));
1311                }
1312            }
1313        }
1314        let (c, dep) = best?;
1315        let rest = symbol[c..].trim_start_matches(['.', '#', ':']).to_string();
1316        Some((dep, rest))
1317    }
1318}
1319
1320fn npm_import_base(name: &str) -> &str {
1321    let skip = if name.starts_with('@') {
1322        name.find('/').map_or(name.len(), |i| i + 1)
1323    } else {
1324        0
1325    };
1326    name[skip..].find('/').map_or(name, |i| &name[..skip + i])
1327}
1328
1329fn split_symbol(s: &str) -> Vec<String> {
1330    s.replace("::", ".")
1331        .replace('#', ".")
1332        .split('.')
1333        .map(|x| x.trim().trim_end_matches("()").to_string())
1334        .filter(|x| !x.is_empty())
1335        .collect()
1336}
1337
1338/// Identifier-like words in a question (`z.object`, `useEffect`,
1339/// `model_dump`, `cookies()`); plain English words are not identifiers.
1340fn identifiers(q: &str) -> Vec<String> {
1341    let words: Vec<&str> = q
1342        .split(|c: char| c.is_whitespace() || matches!(c, ',' | '?' | '\'' | '"' | '[' | ']'))
1343        .filter(|w| !w.is_empty())
1344        .collect();
1345    let single = words.len() == 1;
1346    words
1347        .into_iter()
1348        .filter(|w| {
1349            let inner = w.trim_matches(|c: char| !c.is_alphanumeric());
1350            single || w.starts_with('`') || w.contains(['_', '.', '(', '$']) || w.contains("::") || inner.chars().skip(1).any(|c| c.is_ascii_uppercase())
1351        })
1352        .map(|w| w.trim_matches(|c: char| matches!(c, '`' | '(' | ')' | '.' | ':' | '!' | ';')))
1353        .filter(|w| w.len() >= 2 && w.chars().any(|c| c.is_ascii_alphabetic()))
1354        .flat_map(|w| {
1355            let mut v = vec![w.to_ascii_lowercase()];
1356            if let Some(last) = split_symbol(w).last() {
1357                v.push(last.to_ascii_lowercase());
1358            }
1359            v
1360        })
1361        .collect()
1362}
1363
1364/// Plain words in the question that name a documented top-level API of the
1365/// searched packages ("run code *after* the response" in Next.js, which
1366/// exports `after`). They count as identifiers.
1367fn api_words(ready: &[ReadyPkg], refs: &[(usize, usize)], q: &str) -> Vec<String> {
1368    let words: HashSet<String> = q
1369        .split(|c: char| !(c.is_ascii_alphanumeric() || c == '_'))
1370        .filter(|w| w.len() >= 4)
1371        .map(|w| w.to_ascii_lowercase())
1372        .filter(|w| !bm25::is_stop(w))
1373        .collect();
1374    let mut out = Vec::new();
1375    for &(pi, ei) in refs {
1376        let e = &ready[pi].0.entries[ei];
1377        let top = matches!(e.kind, Kind::Function | Kind::Macro | Kind::Class)
1378            && !e.doc.is_empty()
1379            && !private_path(e)
1380            && !e.legacy
1381            && e.path.matches(['.', ':']).count() <= 2;
1382        let n = e.name.trim_end_matches('!').to_ascii_lowercase();
1383        if top && words.contains(&n) && !out.contains(&n) {
1384            out.push(n);
1385        }
1386    }
1387    out
1388}
1389
1390/// Keyword ranking (BM25 over text and headings, then docs-specific boosts)
1391/// first; when no entry clearly leads, the embedding ranking is fused in by
1392/// weighted reciprocal rank. Then deprecation redirects ("use X instead")
1393/// lift the API they point to. Returns (score, package, entry), best first.
1394fn hybrid_rank(
1395    ready: &[ReadyPkg],
1396    refs: &[(usize, usize)],
1397    (bm, hb): (&Bm25, &Bm25),
1398    qterms: &[String],
1399    q: &str,
1400    idents: &[String],
1401    changes: bool,
1402) -> Vec<(f32, usize, usize)> {
1403    // (full-text BM25, heading BM25), each normalized to its best hit.
1404    let mut fused: HashMap<usize, (f32, f32)> = HashMap::new();
1405    let expanded = bm25::expand(qterms);
1406    let bm_hits = bm.search_weighted(&expanded);
1407    let bmax = bm_hits.first().map_or(1.0, |h| h.score).max(1e-6);
1408    for h in bm_hits.iter().take(1500) {
1409        fused.entry(h.doc as usize).or_default().0 = h.score / bmax;
1410    }
1411    let head_w = std::env::var("LOCKDOCS_HEAD_WEIGHT").ok().and_then(|v| v.parse().ok()).unwrap_or(HEAD_WEIGHT);
1412    let h_hits = hb.search_weighted(&expanded);
1413    let hmax = h_hits.first().map_or(1.0, |h| h.score).max(1e-6);
1414    for h in h_hits.iter().take(1500) {
1415        fused.entry(h.doc as usize).or_default().1 = h.score / hmax;
1416    }
1417    let dense_w: f32 = std::env::var("LOCKDOCS_DENSE_WEIGHT").ok().and_then(|v| v.parse().ok()).unwrap_or(DENSE_WEIGHT);
1418    // Query words that are rare in these packages; common ones ("futures" in
1419    // tokio) say little about which entry is meant.
1420    let n = refs.len().max(1) as f32;
1421    let rare: Vec<String> = qterms.iter().filter(|t| (bm.df(t) as f32) < n * 0.03).cloned().collect();
1422    let weight = |i: usize| {
1423        let (pi, ei) = refs[i];
1424        let e = &ready[pi].0.entries[ei];
1425        boost(e, idents, changes, major_of(&ready[pi].0.version)) * name_hit(e, &rare)
1426    };
1427    // Keyword ranking first, with the docs-specific boosts: this is what the
1428    // answer is built on.
1429    let mut lex: Vec<(usize, f32)> = fused.iter().map(|(&i, &(b, h))| (i, ((1.0 - head_w) * b + head_w * h) * weight(i))).collect();
1430    lex.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal).then(a.0.cmp(&b.0)));
1431    // A clear leader needs no second opinion: skip the embedding entirely.
1432    let decisive = match (lex.first(), lex.get(1)) {
1433        (Some(t), Some(s)) => t.1 >= s.1 * DECISIVE_RATIO,
1434        (Some(_), None) => true,
1435        _ => false,
1436    };
1437    let debug = std::env::var("LOCKDOCS_DEBUG").is_ok();
1438    let qv = if !decisive && dense_w > 0.0 && ready.iter().all(|r| !r.0.vecs.is_empty()) {
1439        embed::query_model().and_then(|m| m.embed(q))
1440    } else {
1441        None
1442    };
1443    // Embedding rank of each entry among the 300 nearest to the query.
1444    let mut dense_rank: HashMap<usize, usize> = HashMap::new();
1445    let mut dense_cos: HashMap<usize, f32> = HashMap::new();
1446    if let Some(qv) = &qv {
1447        let mut sims: Vec<(usize, f32)> = refs
1448            .par_iter()
1449            .enumerate()
1450            .map(|(i, &(pi, ei))| (i, embed::cosine(qv, &ready[pi].0.vecs[ei])))
1451            .collect();
1452        sims.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal).then(a.0.cmp(&b.0)));
1453        for (r, (i, _)) in sims.iter().take(DENSE_POOL).enumerate() {
1454            dense_rank.insert(*i, r);
1455        }
1456        let cmax = sims.first().map_or(1.0, |s| s.1);
1457        let floor = sims.get(300).map_or(0.0, |s| s.1);
1458        for (i, c) in sims.iter().take(300) {
1459            dense_cos.insert(*i, ((c - floor) / (cmax - floor).max(1e-6)).max(0.0));
1460        }
1461        if debug {
1462            for (i, c) in sims.iter().take(10) {
1463                let (pi, ei) = refs[*i];
1464                eprintln!("dense cos={c:.3} {}", ready[pi].0.entries[ei].path);
1465            }
1466        }
1467    }
1468    if debug {
1469        eprintln!("decisive={decisive} dense={}", qv.is_some());
1470    }
1471    // Package files only (API names and signatures): reciprocal-rank fusion
1472    // weighted towards the keyword ranking, so the embedding breaks near-ties
1473    // and rescues paraphrases but cannot outvote an exact name match. With
1474    // upstream docs in the index the answers are prose pages, where the
1475    // embedding is as reliable as keywords and score magnitudes matter (a
1476    // rank-only fusion buried select.md under Graceful Shutdown): there each
1477    // normalized score is mixed, as before. Scores are scaled so the best is 1.
1478    let with_docs = ready.iter().any(|r| r.0.upstream.is_some());
1479    let rrf = |r: usize| 1.0 / (RRF_K + r as f32);
1480    let mut scored: Vec<(f32, usize, usize)> = if qv.is_some() && with_docs {
1481        let mut ids: HashSet<usize> = lex.iter().map(|l| l.0).collect();
1482        ids.extend(dense_cos.keys().copied());
1483        let lexm: HashMap<usize, f32> = fused.iter().map(|(&i, &(b, h))| (i, (1.0 - head_w) * b + head_w * h)).collect();
1484        ids.into_iter()
1485            .map(|i| {
1486                let (pi, ei) = refs[i];
1487                let l = lexm.get(&i).copied().unwrap_or(0.0);
1488                let d = dense_cos.get(&i).copied().unwrap_or(0.0);
1489                (((1.0 - DOCS_DENSE_WEIGHT) * l + DOCS_DENSE_WEIGHT * d) * weight(i), pi, ei)
1490            })
1491            .collect()
1492    } else if qv.is_some() {
1493        let mut ids: HashSet<usize> = lex.iter().map(|l| l.0).collect();
1494        ids.extend(dense_rank.keys().copied());
1495        let lex_rank: HashMap<usize, usize> = lex.iter().enumerate().map(|(r, l)| (l.0, r)).collect();
1496        ids.into_iter()
1497            .map(|i| {
1498                let l = lex_rank.get(&i).map_or(0.0, |&r| rrf(r));
1499                let d = dense_rank.get(&i).map_or(0.0, |&r| rrf(r));
1500                let (pi, ei) = refs[i];
1501                ((1.0 - dense_w) * l + dense_w * d, pi, ei)
1502            })
1503            .collect()
1504    } else {
1505        lex.iter().map(|&(i, s)| (s, refs[i].0, refs[i].1)).collect()
1506    };
1507    if qv.is_some() {
1508        let top = scored.iter().map(|s| s.0).fold(0.0f32, f32::max).max(1e-9);
1509        scored.iter_mut().for_each(|s| s.0 /= top);
1510    }
1511    if debug {
1512        scored.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));
1513        for (s, pi, ei) in scored.iter().take(10) {
1514            eprintln!("{s:.3} {}", ready[*pi].0.entries[*ei].path);
1515        }
1516    }
1517    scored.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));
1518    // Deprecation redirects among the top results.
1519    let mut lifted: Vec<(f32, usize, String)> = Vec::new();
1520    for (s, pi, ei) in scored.iter().take(30) {
1521        let e = &ready[*pi].0.entries[*ei];
1522        for t in redirects(&format!("{}\n{}", e.sig, e.doc)) {
1523            if t != e.name {
1524                lifted.push((*s * 0.95, *pi, t));
1525            }
1526        }
1527    }
1528    if !lifted.is_empty() {
1529        let mut have: HashMap<(usize, usize), usize> = scored.iter().enumerate().map(|(i, (_, pi, ei))| ((*pi, *ei), i)).collect();
1530        for (s, pi, name) in lifted {
1531            for &(_, ei) in refs.iter().filter(|(p, _)| *p == pi) {
1532                let e = &ready[pi].0.entries[ei];
1533                if e.kind == Kind::Prose || e.kind == Kind::Alias || e.name != name || e.legacy {
1534                    continue;
1535                }
1536                match have.get(&(pi, ei)) {
1537                    Some(&i) => scored[i].0 = scored[i].0.max(s),
1538                    None => {
1539                        have.insert((pi, ei), scored.len());
1540                        scored.push((s, pi, ei));
1541                    }
1542                }
1543            }
1544        }
1545        scored.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));
1546    }
1547    scored
1548}
1549
1550/// API names a doc points readers to: "use `model_validate` instead",
1551/// "Consider `z.strictObject(A.shape)`", "renamed to X", "in favor of X".
1552pub fn redirects(text: &str) -> Vec<String> {
1553    let lower = text.to_ascii_lowercase();
1554    let mut out = Vec::new();
1555    for pat in [
1556        "use ",
1557        "consider ",
1558        "renamed to ",
1559        "replaced by ",
1560        "in favor of ",
1561        "in favour of ",
1562        "instead use ",
1563    ] {
1564        let mut from = 0;
1565        while let Some(i) = lower[from..].find(pat) {
1566            let at = from + i + pat.len();
1567            from = at;
1568            let rest = &text[at..];
1569            let rest = rest.trim_start_matches(['`', '\'', '"', '*', ' ']);
1570            let ident: String = rest.chars().take_while(|c| c.is_alphanumeric() || matches!(c, '_' | '.' | ':' | '$')).collect();
1571            let ident = ident.trim_end_matches(['.', ':']);
1572            // Only identifier-looking targets: code spans or names with _ . $ or camelCase.
1573            let quoted = text[at..].starts_with('`');
1574            let looks = ident.contains(['_', '.', '$']) || ident.chars().skip(1).any(|c| c.is_ascii_uppercase());
1575            if ident.len() >= 3 && (quoted || looks) {
1576                let last = ident.rsplit(['.', ':']).next().unwrap_or(ident).to_string();
1577                if last.len() >= 3 && !out.contains(&last) {
1578                    out.push(last);
1579                }
1580            }
1581        }
1582    }
1583    out
1584}
1585
1586/// Terms of a prose heading. Capitalized words are names (`TypeScript`,
1587/// `JavaScript`) and stay whole; identifiers (`useActionState`,
1588/// `model_dump`) are also split into their parts.
1589fn heading_terms(h: &str) -> Vec<String> {
1590    let mut out = Vec::new();
1591    for w in h.split(|c: char| !(c.is_ascii_alphanumeric() || c == '_')).filter(|w| !w.is_empty()) {
1592        if w.starts_with(|c: char| c.is_ascii_uppercase()) && !w.contains('_') {
1593            out.push(bm25::stem(&w.to_ascii_lowercase()));
1594        } else {
1595            out.extend(bm25::terms(w));
1596        }
1597    }
1598    out
1599}
1600
1601/// Entries whose name (or section heading) carries a query word are about it.
1602fn name_hit(e: &Entry, qterms: &[String]) -> f32 {
1603    let words = if e.kind == Kind::Prose {
1604        let last = e.name.rsplit(" › ").next().unwrap_or("");
1605        if crate::markdown::generic_heading(last) {
1606            Vec::new()
1607        } else {
1608            heading_terms(last)
1609        }
1610    } else {
1611        bm25::terms(&e.name)
1612    };
1613    let n = qterms.iter().filter(|q| words.contains(q)).count();
1614    match n {
1615        0 => 1.0,
1616        1 => 1.5,
1617        _ => 1.9,
1618    }
1619}
1620
1621fn major_of(version: &str) -> u64 {
1622    version.trim_start_matches('v').split('.').next().and_then(|m| m.parse().ok()).unwrap_or(0)
1623}
1624
1625/// A migration or upgrade guide to a major older than the pinned one
1626/// ("Migrating to v6.0.0" in ESLint 9, "Upgrade to Prisma ORM 4" in Prisma 6):
1627/// history, not how things work now.
1628fn old_upgrade_guide(e: &Entry, major: u64) -> bool {
1629    let title = e.name.split(" › ").next().unwrap_or("");
1630    let stem = e.file.rsplit('/').next().unwrap_or("");
1631    [title, stem].iter().any(|t| {
1632        let l = t.to_ascii_lowercase();
1633        (l.contains("migrat") || l.contains("upgrad"))
1634            && l.split(|c: char| !c.is_ascii_digit())
1635                .find(|d| !d.is_empty() && d.len() <= 3)
1636                .and_then(|d| d.parse::<u64>().ok())
1637                .is_some_and(|v| v < major)
1638    })
1639}
1640
1641/// A page its own title marks as deprecated ("Configure Language Options (Deprecated)").
1642fn deprecated_page(e: &Entry) -> bool {
1643    e.name.split(" › ").next().unwrap_or("").to_ascii_lowercase().contains("(deprecated)")
1644}
1645
1646fn boost(e: &Entry, idents: &[String], changes: bool, major: u64) -> f32 {
1647    let mut b = 1.0;
1648    // Curated guides from the project's own docs folder answer "how do I" better
1649    // than internal symbols do.
1650    if e.kind == Kind::Prose && e.file.starts_with("upstream:") {
1651        b *= std::env::var("LOCKDOCS_UPSTREAM_BOOST").ok().and_then(|v| v.parse().ok()).unwrap_or(1.35);
1652    }
1653    if e.kind == Kind::Prose {
1654        // A section headed by the API the question names (`after`, `select!`).
1655        let last = e
1656            .name
1657            .rsplit(" › ")
1658            .next()
1659            .unwrap_or("")
1660            .trim()
1661            .trim_end_matches("()")
1662            .trim_end_matches('!')
1663            .to_ascii_lowercase();
1664        if !last.is_empty() && idents.contains(&last) {
1665            b *= 1.4;
1666        }
1667        if !changes && old_upgrade_guide(e, major) {
1668            b *= 0.5;
1669        }
1670        if deprecated_page(e) {
1671            b *= 0.6;
1672        }
1673        if is_changelog(e) {
1674            b *= if changes { 1.5 } else { 0.5 };
1675        } else if e.file.to_ascii_uppercase().starts_with("README") {
1676            b *= 1.15;
1677        }
1678    } else {
1679        let n = e.name.trim_end_matches('!').to_ascii_lowercase();
1680        let p = e.path.to_ascii_lowercase();
1681        if idents.contains(&n) {
1682            b *= 1.8;
1683        }
1684        if idents
1685            .iter()
1686            .any(|i| i.len() > n.len() && (p.ends_with(i.as_str()) || i.ends_with(&format!(".{n}"))))
1687        {
1688            b *= 1.4;
1689        }
1690        if e.doc.is_empty() {
1691            b *= 0.6;
1692        }
1693        if e.kind == Kind::Alias {
1694            b *= 0.6;
1695        }
1696    }
1697    if private_path(e) {
1698        b *= 0.6;
1699    }
1700    if e.legacy {
1701        b *= 0.35;
1702    }
1703    if tooling_path(e) {
1704        b *= 0.5;
1705    }
1706    b
1707}
1708
1709fn api_score(e: &Entry, exact: bool, quals: &[String]) -> f32 {
1710    let mut s = if exact { 10.0 } else { 7.0 };
1711    let segs: Vec<String> = split_symbol(&e.path).into_iter().map(|x| x.to_ascii_lowercase()).collect();
1712    // A bare name means the top-level API, not a same-named member.
1713    if quals.is_empty() {
1714        s += if matches!(e.kind, Kind::Method | Kind::Field) { -1.5 } else { 1.5 };
1715    }
1716    let file = e.file.to_ascii_lowercase();
1717    for q in quals {
1718        if segs.iter().any(|x| x == q) {
1719            s += 3.0;
1720        } else if file.contains(q.as_str()) {
1721            s += 1.0;
1722        }
1723    }
1724    // The qualifier directly before the name matters most (`Router.route`).
1725    if let (Some(q), Some(parent)) = (quals.last(), segs.len().checked_sub(2).map(|i| &segs[i])) {
1726        if q == parent {
1727            s += 2.0;
1728        }
1729    }
1730    if !e.doc.is_empty() {
1731        s += 2.0;
1732    }
1733    if private_path(e) {
1734        s -= 3.0;
1735    }
1736    if e.legacy {
1737        s -= 5.0;
1738    }
1739    if tooling_path(e) {
1740        s -= 3.0;
1741    }
1742    if e.kind == Kind::Alias {
1743        s -= 0.5;
1744    }
1745    s -= 0.2 * segs.len() as f32;
1746    s
1747}
1748
1749fn resolve_alias<'a>(idx: &'a PackageIndex, e: &'a Entry) -> &'a Entry {
1750    let Some(target) = &e.alias_of else { return e };
1751    idx.entries
1752        .iter()
1753        .filter(|x| &x.name == target && x.kind != Kind::Alias && x.kind != Kind::Prose)
1754        .max_by_key(|x| (x.file == e.file, !x.doc.is_empty()))
1755        .unwrap_or(e)
1756}
1757
1758fn truncate(s: &str, n: usize) -> String {
1759    if s.chars().count() <= n {
1760        return s.to_string();
1761    }
1762    let t: String = s.chars().take(n).collect();
1763    format!("{t}…")
1764}
1765
1766/// One result block with its citation.
1767pub fn render_entry(idx: &PackageIndex, e: &Entry, max_doc: usize) -> String {
1768    let cite = format!("{} {}:{}", idx.id(), e.file, e.line);
1769    if e.kind == Kind::Prose {
1770        return format!("### {} — {}\n{}\n", e.name, cite, truncate(&e.doc, max_doc));
1771    }
1772    let mut s = format!("### {} ({}) — {}\n```{}\n{}\n```\n", e.path, e.kind.as_str(), cite, lang(idx.eco), e.sig);
1773    if !e.doc.is_empty() {
1774        s.push_str(&truncate(&e.doc, max_doc));
1775        s.push('\n');
1776    }
1777    s
1778}
1779
1780/// The first paragraph of text of a section, and the short list or code
1781/// block right after it (what the section offers). Tag-only lines
1782/// (`<Deprecated>`) are skipped.
1783fn lead(doc: &str) -> String {
1784    let tag_only = |l: &str| l.starts_with('<') && l.ends_with('>');
1785    let mut lines = doc.lines().map(str::trim_end).peekable();
1786    let mut para: Vec<&str> = Vec::new();
1787    for l in lines.by_ref() {
1788        let t = l.trim();
1789        if t.starts_with("```") || t.starts_with("~~~") {
1790            return String::new();
1791        }
1792        if t.is_empty() || tag_only(t) {
1793            if para.is_empty() {
1794                continue;
1795            }
1796            break;
1797        }
1798        para.push(t);
1799    }
1800    let mut out = truncate(&para.join(" "), 320);
1801    while lines.peek().is_some_and(|l| l.trim().is_empty()) {
1802        lines.next();
1803    }
1804    let bullet = |l: &str| {
1805        let t = l.trim_start();
1806        t.starts_with("* ") || t.starts_with("- ") || t.split_once(". ").is_some_and(|(n, _)| !n.is_empty() && n.chars().all(|c| c.is_ascii_digit()))
1807    };
1808    if lines.peek().is_some_and(|l| bullet(l)) {
1809        let mut list = String::new();
1810        for l in lines.by_ref() {
1811            if l.trim().is_empty() {
1812                break;
1813            }
1814            let t = l.trim();
1815            list.push_str(if bullet(l) { "\n" } else { " " });
1816            list.push_str(t);
1817        }
1818        out.push_str(&truncate(&list, 400));
1819    } else if lines.peek().is_some_and(|l| l.trim_start().starts_with("```")) {
1820        let mut code = vec![lines.next().unwrap_or_default()];
1821        for l in lines.by_ref() {
1822            code.push(l);
1823            if l.trim_start().starts_with("```") {
1824                break;
1825            }
1826        }
1827        let code = code.join("\n");
1828        if code.len() <= 300 && code.matches("```").count() == 2 {
1829            out.push('\n');
1830            out.push_str(&code);
1831        }
1832    }
1833    out
1834}
1835
1836/// Leads of a prose section's page and parent section, quoted, when they
1837/// say something the section does not (a deprecation notice on the page,
1838/// the setup a subsection builds on).
1839fn section_context(idx: &PackageIndex, e: &Entry, scope: &std::collections::BTreeSet<usize>) -> Option<String> {
1840    if e.kind != Kind::Prose {
1841        return None;
1842    }
1843    // Keep the page and closest nonempty parent, using logical ancestry rather
1844    // than heading text (which can repeat), and the same applicability filter
1845    // as retrieval. A permitted child must not quote an incompatible setup.
1846    let mut wanted = Vec::new();
1847    if let Some(line) = e.prose_parents.first() {
1848        wanted.push(*line);
1849    }
1850    if let Some(line) = e.prose_parents.last() {
1851        if !wanted.contains(line) {
1852            wanted.push(*line);
1853        }
1854    }
1855    let mut out = String::new();
1856    for line in wanted {
1857        let Some(p) = scope
1858            .iter()
1859            .map(|&i| &idx.entries[i])
1860            .find(|p| p.kind == Kind::Prose && p.file == e.file && p.line == line)
1861        else {
1862            continue;
1863        };
1864        let l = lead(&p.doc);
1865        if l.len() < 20 || e.doc.contains(&l) || out.contains(&l) {
1866            continue;
1867        }
1868        // The enclosing hit already cites this same file. Keep the ancestor
1869        // line explicit without repeating its long path in the token budget.
1870        for line in format!("line {}: {l}", p.line).lines() {
1871            out.push_str("> ");
1872            out.push_str(line);
1873            out.push('\n');
1874        }
1875    }
1876    (!out.is_empty()).then(|| out + "\n")
1877}
1878
1879/// Exact case-sensitive module token, not a prefix or package-looking substring.
1880fn mentions_import(query: &str, import: &str) -> bool {
1881    let module_char = |c: char| c.is_alphanumeric() || matches!(c, '@' | '/' | '_' | '-' | '.');
1882    query.match_indices(import).any(|(i, _)| {
1883        let before = query[..i].chars().next_back();
1884        let after = &query[i + import.len()..];
1885        before.is_none_or(|c| !module_char(c))
1886            && (after.chars().next().is_none_or(|c| !module_char(c)) || after.starts_with('.') && after[1..].chars().next().is_none_or(|c| !module_char(c)))
1887    })
1888}
1889
1890/// Resolve explicit calls through the selected public entrypoint, rather than
1891/// treating every same-named member in its transitive type graph as the API.
1892fn public_call_entries(idx: &PackageIndex, import: &str, query: &str) -> Vec<usize> {
1893    let Some(api) = &idx.npm_api else {
1894        return Vec::new();
1895    };
1896    let names: HashSet<&str> = api
1897        .entries
1898        .iter()
1899        .flat_map(|e| [e.name.as_str(), e.path.as_str()])
1900        .filter(|name| {
1901            query.match_indices(*name).any(|(i, _)| {
1902                query[i + name.len()..].starts_with('(')
1903                    && query[..i]
1904                        .chars()
1905                        .next_back()
1906                        .is_none_or(|c| !(c.is_alphanumeric() || matches!(c, '_' | '$' | '.' | ':' | '/' | '-' | '@')))
1907            })
1908        })
1909        .collect();
1910    let roots: Vec<_> = api.roots.iter().filter(|r| r.import == import).collect();
1911    let mut targets = HashSet::new();
1912    for name in names {
1913        let resolved: Vec<_> = roots.iter().map(|root| api.resolve(root, name, &api.entries)).collect();
1914        let Some(first) = resolved.first() else {
1915            continue;
1916        };
1917        if first.entries.is_empty() || resolved.iter().any(|r| r.unresolved || r.entries != first.entries) {
1918            continue;
1919        }
1920        for &i in &first.entries {
1921            let e = &api.entries[i];
1922            targets.insert((e.file.as_str(), e.path.as_str(), e.line));
1923        }
1924    }
1925    idx.entries
1926        .iter()
1927        .enumerate()
1928        .filter_map(|(i, e)| targets.contains(&(e.file.as_str(), e.path.as_str(), e.line)).then_some(i))
1929        .collect()
1930}
1931
1932/// Token-budgeted output.
1933struct Pack {
1934    text: String,
1935    budget: usize,
1936    blocks: usize,
1937    full: bool,
1938}
1939
1940impl Pack {
1941    fn new(budget: usize) -> Pack {
1942        Pack {
1943            text: String::new(),
1944            budget,
1945            blocks: 0,
1946            full: false,
1947        }
1948    }
1949    fn used(&self) -> usize {
1950        est_tokens(&self.text)
1951    }
1952    fn push_raw(&mut self, s: &str) {
1953        self.text.push_str(s);
1954    }
1955    fn prepend(&mut self, s: &str) {
1956        self.text.insert_str(0, s);
1957    }
1958    /// Add a block if it fits; a first oversize block is trimmed to fit.
1959    fn push_block(&mut self, b: &str) -> bool {
1960        if self.full {
1961            return false;
1962        }
1963        let need = est_tokens(b) + 1;
1964        let left = self.budget.saturating_sub(self.used());
1965        if need <= left {
1966            self.text.push('\n');
1967            self.text.push_str(b);
1968            self.blocks += 1;
1969            return true;
1970        }
1971        if left > 120 {
1972            // Trim to the remaining budget at a line boundary.
1973            let max_chars = left * 3;
1974            let mut cut = String::new();
1975            for line in b.lines() {
1976                if cut.len() + line.len() + 1 > max_chars {
1977                    break;
1978                }
1979                cut.push_str(line);
1980                cut.push('\n');
1981            }
1982            if cut.matches("```").count() % 2 == 1 {
1983                cut.push_str("```\n");
1984            }
1985            if cut.lines().count() > 1 {
1986                self.text.push('\n');
1987                self.text.push_str(&cut);
1988                self.text.push_str("[…truncated to fit the token budget]\n");
1989                self.blocks += 1;
1990            }
1991        }
1992        self.full = true;
1993        false
1994    }
1995}
1996
1997/// Where the engine keeps its per-root state for long-running servers.
1998#[derive(Default)]
1999pub struct Workspace {
2000    engines: Mutex<HashMap<PathBuf, (u64, Arc<Engine>)>>,
2001}
2002
2003fn lock_stamp(root: &Path) -> u64 {
2004    // Re-read the project when any lockfile in the root changes.
2005    let mut h: u64 = 0;
2006    if let Ok(rd) = std::fs::read_dir(root) {
2007        for e in rd.flatten() {
2008            let n = e.file_name().to_string_lossy().to_string();
2009            if crate::lockfile::LOCKFILES.iter().any(|(f, _)| *f == n) || crate::lockfile::is_requirements(&n) || n == "package.json" || n == "pyproject.toml" {
2010                if let Ok(m) = e.metadata() {
2011                    let t = m
2012                        .modified()
2013                        .ok()
2014                        .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
2015                        .map_or(0, |d| d.as_nanos() as u64);
2016                    h = h.wrapping_mul(31).wrapping_add(t ^ m.len());
2017                }
2018            }
2019        }
2020    }
2021    h
2022}
2023
2024impl Workspace {
2025    pub fn engine(&self, root: &Path, opts: &Options) -> Arc<Engine> {
2026        let root = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
2027        let stamp = lock_stamp(&root);
2028        let mut m = self.engines.lock().unwrap();
2029        if let Some((s, e)) = m.get(&root) {
2030            if *s == stamp && e.opts == *opts {
2031                return e.clone();
2032            }
2033        }
2034        let e = Arc::new(Engine::new(&root, opts.clone()));
2035        m.insert(root, (stamp, e.clone()));
2036        e
2037    }
2038}
2039
2040#[cfg(test)]
2041mod tests {
2042    use super::*;
2043
2044    #[test]
2045    fn fetch_report_preserves_flat_counts_and_adds_full_provenance() {
2046        let dep = Dep {
2047            eco: Eco::Cargo,
2048            name: "axum".into(),
2049            version: "0.7.9".into(),
2050            direct: true,
2051            from: "Cargo.lock".into(),
2052        };
2053        let status: upstream::Manifest = serde_json::from_value(json!({
2054            "format": upstream::FORMAT, "repo": "github.com/tokio-rs/axum", "tag": "axum-v0.7.9",
2055            "commit": "0123456789abcdef0123456789abcdef01234567", "files": 20, "bytes": 100, "note": null,
2056        }))
2057        .unwrap();
2058        let report = fetch_report(&dep, &status);
2059        assert_eq!(report["files"], 20);
2060        assert_eq!(report["files"], report["upstream"]["files"]);
2061        assert_eq!(report["tag"], report["upstream"]["tag"]);
2062        assert_eq!(report["repo"], report["upstream"]["repo"]);
2063        assert!(report.get("note").is_some());
2064    }
2065
2066    #[test]
2067    fn upstream_is_default_with_explicit_opt_out() {
2068        assert!(automatic_upstream(None, false));
2069        assert!(automatic_upstream(Some("1"), false));
2070        for value in ["0", "false", "no"] {
2071            assert!(!automatic_upstream(Some(value), false));
2072        }
2073        assert!(!automatic_upstream(None, true));
2074    }
2075
2076    #[test]
2077    fn workspace_does_not_reuse_network_policy() {
2078        let ws = Workspace::default();
2079        let root = std::env::temp_dir();
2080        let online = Options {
2081            fetch: false,
2082            upstream: true,
2083            private: false,
2084        };
2085        let offline = Options {
2086            fetch: false,
2087            upstream: false,
2088            private: false,
2089        };
2090        let a = ws.engine(&root, &online);
2091        let b = ws.engine(&root, &offline);
2092        assert!(!Arc::ptr_eq(&a, &b));
2093        assert!(!b.opts.upstream);
2094    }
2095
2096    #[test]
2097    fn old_upgrade_guides_and_deprecated_pages() {
2098        let e = |name: &str, file: &str| Entry {
2099            kind: Kind::Prose,
2100            name: name.into(),
2101            path: name.into(),
2102            file: file.into(),
2103            line: 1,
2104            sig: String::new(),
2105            doc: String::new(),
2106            alias_of: None,
2107            legacy: false,
2108            prose_imports: Vec::new(),
2109            prose_parents: Vec::new(),
2110        };
2111        assert!(old_upgrade_guide(
2112            &e("Migrating to v6.0.0 › x", "upstream:docs/src/use/migrating-to-6.0.0.md"),
2113            9
2114        ));
2115        assert!(old_upgrade_guide(
2116            &e("Upgrade to Prisma ORM 4 › x", "upstream:docs/700-upgrading-to-prisma-4.mdx"),
2117            6
2118        ));
2119        assert!(!old_upgrade_guide(
2120            &e("Upgrade to Prisma ORM 6 › x", "upstream:docs/500-upgrading-to-prisma-6.mdx"),
2121            6
2122        ));
2123        assert!(!old_upgrade_guide(&e("Migration Guide › x", "upstream:docs/migration.md"), 2));
2124        assert!(deprecated_page(&e("Configure Language Options (Deprecated) › Globals", "x.md")));
2125        assert!(!heading_terms("Seeding with TypeScript or JavaScript").contains(&"type".to_string()));
2126        assert!(heading_terms("useActionState reference").contains(&"action".to_string()));
2127        let l = lead("Intro line.\n\n```css\n@import \"tailwindcss\";\n\n@custom-variant dark (&:where(.dark, .dark *));\n```\n\nMore.");
2128        assert!(l.contains("@custom-variant") && l.ends_with("```"), "{l}");
2129        let l = lead("Three helpers:\n\n* **`parse_obj`**: from a dict\n  more.\n* `parse_raw`\n\nNext.");
2130        assert!(l.contains("parse_obj") && l.contains("parse_raw") && !l.contains("Next"), "{l}");
2131        assert!(lead("<Deprecated>\n\nIn React 19, it is no longer necessary.\n\n</Deprecated>").starts_with("In React 19"));
2132    }
2133
2134    #[test]
2135    fn deprecation_redirects() {
2136        assert_eq!(redirects("Consider `z.strictObject(A.shape)` instead"), vec!["strictObject"]);
2137        assert_eq!(
2138            redirects("The `parse_obj` method is deprecated; use `model_validate` instead."),
2139            vec!["model_validate"]
2140        );
2141        assert_eq!(redirects("@deprecated Use .extend instead"), vec!["extend"]);
2142        assert_eq!(redirects("You can use this to parse data"), Vec::<String>::new());
2143        assert_eq!(redirects("Renamed to OptionalFromRequestParts."), vec!["OptionalFromRequestParts"]);
2144    }
2145}