Skip to main content

lockdocs_core/query/
docs.rs

1//! Documentation search, import applicability, and package overviews.
2
3use super::entries::{npm_import_base, private_path, resolve_alias};
4use super::packages::ReadyPkg;
5use super::ranking::{api_words, change_intent, hybrid_rank, identifiers};
6use super::render::{provenance, render_entry, section_context, truncate, Pack};
7use super::{Answer, Engine};
8use crate::bm25::{self, Bm25};
9use crate::extract::{Entry, Kind};
10use crate::index::PackageIndex;
11use crate::project::Spec;
12use crate::{est_tokens, upstream, Dep, Eco};
13use serde_json::json;
14use std::collections::{HashMap, HashSet};
15
16/// Results scoring below this fraction of the best are left out (after 3).
17const RELEVANCE_FLOOR: f32 = 0.3;
18
19impl Engine {
20    pub fn docs(&self, package: Option<&str>, query: &str, tokens: usize) -> Result<Answer, String> {
21        let tokens = tokens.clamp(200, 20000);
22        let mut imports = HashMap::new();
23        let selector = package.map(|p| {
24            p.split(',')
25                .map(|p| {
26                    let spec = Spec::parse(p.trim());
27                    let base = npm_import_base(&spec.name);
28                    let npm = spec.eco == Some(Eco::Npm) || spec.eco.is_none() && self.project.deps.iter().any(|d| d.eco == Eco::Npm && d.name == base);
29                    if npm && base != spec.name {
30                        imports.entry(base.to_string()).or_insert_with(Vec::new).push(spec.name.clone());
31                        format!("npm:{base}{}", spec.version.as_ref().map(|v| format!("@{v}")).unwrap_or_default())
32                    } else {
33                        if npm {
34                            imports.entry(base.to_string()).or_insert_with(Vec::new).push(base.to_string());
35                        }
36                        p.trim().to_string()
37                    }
38                })
39                .collect::<Vec<_>>()
40                .join(",")
41        });
42        let (deps, broad) = self.select(selector.as_deref(), query)?;
43        let (ready, missing) = self.resolve_all(&deps);
44        if ready.is_empty() {
45            return Err(missing.into_iter().map(|(_, e)| e).collect::<Vec<_>>().join("\n"));
46        }
47        let mut scopes = Vec::new();
48        let mut public_calls = HashSet::new();
49        for (pi, (idx, _, _)) in ready.iter().enumerate() {
50            let mut requested = imports.get(&idx.name).cloned().unwrap_or_else(|| vec![idx.name.clone()]);
51            // A question naming known public imports can narrow a root search,
52            // but must never override an explicitly selected alternate edition.
53            if requested.iter().all(|import| import == &idx.name) {
54                if let Some(api) = &idx.npm_api {
55                    let named: std::collections::BTreeSet<String> = api
56                        .roots
57                        .iter()
58                        .filter(|r| r.import != idx.name && mentions_import(query, &r.import))
59                        .map(|r| r.import.clone())
60                        .collect();
61                    if !named.is_empty() {
62                        requested = named.into_iter().collect();
63                    }
64                }
65            }
66            if let Some(api) = &idx.npm_api {
67                for import in &requested {
68                    if import != &idx.name && !api.roots.iter().any(|r| &r.import == import) {
69                        return Err(format!("`{import}` has no public declaration entrypoint in {}.", idx.id()));
70                    }
71                }
72            }
73            let mut scope = std::collections::BTreeSet::new();
74            for import in requested {
75                scope.extend(idx.docs_indices(Some(&import)));
76                public_calls.extend(public_call_entries(idx, &import, query).into_iter().map(|ei| (pi, ei)));
77            }
78            scopes.push(scope);
79        }
80        let q = query.trim();
81        if q.is_empty() {
82            return Ok(self.overview(&ready, &scopes, tokens));
83        }
84        let qterms = bm25::query_terms(q);
85        // One BM25 over the selected packages so scores compare.
86        let mut refs: Vec<(usize, usize)> = Vec::new();
87        for (pi, scope) in scopes.iter().enumerate() {
88            for &ei in scope {
89                refs.push((pi, ei));
90            }
91        }
92        let bm = Bm25::build(refs.iter().map(|&(pi, ei)| (ready[pi].0.terms[ei].as_slice(), ready[pi].0.lens[ei])));
93        let hb = Bm25::build(refs.iter().map(|&(pi, ei)| (ready[pi].0.head[ei].as_slice(), ready[pi].0.head_lens[ei])));
94        let mut raw_idents = identifiers(q);
95        raw_idents.extend(api_words(&ready, &refs, q));
96        let changes = change_intent(q);
97        let mut scored = hybrid_rank(&ready, &refs, (&bm, &hb), &qterms, q, &raw_idents, changes);
98        // An explicitly called public API is the subject of the question.
99        // Referenced type members with the same name remain supporting hits.
100        scored.sort_by_key(|&(_, pi, ei)| !public_calls.contains(&(pi, ei)));
101        let mut out = Pack::new(tokens);
102        let mut header = String::new();
103        for (idx, dep, note) in &ready {
104            header.push_str(&format!(
105                "{} · {} · {}{}\n",
106                idx.id(),
107                idx.eco,
108                idx.source,
109                if dep.from == "requested" {
110                    String::new()
111                } else {
112                    format!(" · pinned in {}", dep.from)
113                }
114            ));
115            if let Some(u) = &idx.upstream {
116                header.push_str(&format!("Upstream docs: {u}\n"));
117            }
118            if let Some(n) = note {
119                header.push_str(&format!("Note: {n}\n"));
120            }
121        }
122        if broad {
123            header = broad_header(&ready, &missing, tokens);
124        }
125        for (d, e) in &missing {
126            if !broad {
127                header.push_str(&format!("Skipped {}: {e}\n", d.id()));
128            }
129        }
130        out.push_raw(&header);
131        let mut seen: HashSet<(usize, String, u32)> = HashSet::new();
132        let mut hits_json = Vec::new();
133        let mut used_pkgs: Vec<String> = Vec::new();
134        let top = scored.first().map_or(0.0, |s| s.0);
135        for (score, pi, ei) in scored {
136            let idx = &ready[pi].0;
137            let e = &idx.entries[ei];
138            // Stop at weak matches once a few good ones are in.
139            if out.blocks >= 3 && score < top * RELEVANCE_FLOOR {
140                break;
141            }
142            if !seen.insert((pi, e.file.clone(), e.line)) {
143                continue;
144            }
145            let e = idx.npm_api.as_ref().map_or_else(|| resolve_alias(idx, e), |api| api.search_alias(e));
146            // The top result also carries the lead of its page and parent
147            // section, within the same budget.
148            let ctx = if out.blocks == 0 { section_context(idx, e, &scopes[pi]) } else { None };
149            let max_doc = if out.blocks == 0 {
150                2000 - ctx.as_ref().map_or(0, |c| c.len().min(800) / 2)
151            } else {
152                800
153            };
154            let mut block = render_entry(idx, e, max_doc);
155            if let (Some(ctx), Some(nl)) = (ctx, block.find('\n')) {
156                block.insert_str(nl + 1, &ctx);
157            }
158            if !out.push_block(&block) {
159                break;
160            }
161            if !used_pkgs.contains(&idx.id()) {
162                used_pkgs.push(idx.id());
163            }
164            hits_json.push(json!({"package": idx.id(), "kind": e.kind.as_str(), "path": e.path, "file": e.file, "line": e.line, "score": score}));
165        }
166        if out.blocks == 0 {
167            out.push_raw(&format!(
168                "No match for \"{q}\" in {}. Try other words, a symbol name (the `api` tool), or `package` to widen/narrow.\n",
169                ready.iter().map(|r| r.0.id()).collect::<Vec<_>>().join(", ")
170            ));
171        } else if broad {
172            out.prepend(&format!("Sources: {}\n", used_pkgs.join(", ")));
173        }
174        // Packages that ship few docs: point at the one-time upstream fetch.
175        for (idx, dep, _) in &ready {
176            let prose = idx.entries.iter().filter(|e| e.kind == Kind::Prose).count();
177            if !broad && idx.upstream.is_none() && prose < 40 && !self.opts.fetch && upstream::cached(dep).is_none() {
178                out.push_raw(&format!(
179                    "\nTip: {} ships few docs. `lockdocs fetch {}` adds its upstream docs at this version's git tag (one-time download, then offline).\n",
180                    idx.id(),
181                    idx.name
182                ));
183            }
184        }
185        Ok(Answer {
186            json: json!({"query": q, "packages": ready.iter().map(|r| r.0.id()).collect::<Vec<_>>(), "hits": hits_json, "provenance": provenance(&ready), "tokens": est_tokens(&out.text)}),
187            text: out.text,
188        })
189    }
190
191    fn overview(&self, ready: &[ReadyPkg], scopes: &[std::collections::BTreeSet<usize>], tokens: usize) -> Answer {
192        let mut out = Pack::new(tokens);
193        for ((idx, dep, note), scope) in ready.iter().zip(scopes) {
194            out.push_raw(&format!("{} · {} · {} · pinned in {}\n", idx.id(), idx.eco, idx.source, dep.from));
195            if let Some(u) = &idx.upstream {
196                out.push_raw(&format!("Upstream: {u}\n"));
197            }
198            if let Some(n) = note {
199                out.push_raw(&format!("Note: {n}\n"));
200            }
201            let entries: Vec<&Entry> = scope.iter().map(|&i| &idx.entries[i]).collect();
202            let symbols = entries
203                .iter()
204                .filter(|e| e.kind != Kind::Prose && e.kind != Kind::Alias && !private_path(e))
205                .count();
206            let prose = entries.len() - symbols;
207            out.push_raw(&format!("Indexed {symbols} API symbols and {prose} doc sections.\n"));
208            // Top-level API names, shortest paths first.
209            let mut top: Vec<&Entry> = entries
210                .iter()
211                .copied()
212                .filter(|e| e.kind != Kind::Prose && !private_path(e) && !e.legacy && !e.doc.is_empty())
213                .collect();
214            top.sort_by_key(|e| (e.path.matches(['.', ':']).count(), e.path.len()));
215            let names: Vec<String> = top.iter().take(60).map(|e| e.path.clone()).collect();
216            if !names.is_empty() {
217                out.push_raw(&format!("Key API: {}\n", names.join(", ")));
218            }
219            for e in entries
220                .iter()
221                .filter(|e| e.kind == Kind::Prose && e.file.to_ascii_uppercase().starts_with("README"))
222                .take(3)
223            {
224                if !out.push_block(&render_entry(idx, e, 1500)) {
225                    break;
226                }
227            }
228        }
229        Answer {
230            json: json!({"packages": ready.iter().map(|r| r.0.id()).collect::<Vec<_>>(), "provenance": provenance(ready), "tokens": est_tokens(&out.text)}),
231            text: out.text,
232        }
233    }
234}
235
236/// Keep network fallbacks visible on ordinary broad MCP answers without listing
237/// every dependency's long header or crowding out the actual answer.
238fn broad_header(ready: &[ReadyPkg], missing: &[(Dep, String)], tokens: usize) -> String {
239    let fetched = ready.iter().filter(|r| r.0.fetched).count();
240    let upstream = ready.iter().filter(|r| r.0.upstream.is_some()).count();
241    let notes = ready.iter().filter(|r| r.2.is_some()).count();
242    let mut header = format!("Searched {} direct dependencies: {} local package sources, {fetched} registry sources, {upstream} upstream sources; {notes} fallback notes, {} unavailable.\n", ready.len(), ready.len() - fetched, missing.len());
243    let max_chars = (tokens / 3).clamp(100, 400) * 3;
244    for (idx, _, note) in ready {
245        if let Some(note) = note {
246            let row = format!("Fallback {}: {}\n", idx.id(), truncate(note, 160));
247            if header.chars().count() + row.chars().count() > max_chars {
248                break;
249            }
250            header.push_str(&row);
251        }
252    }
253    for (dep, error) in missing {
254        let row = format!("Skipped {}: {}\n", dep.id(), truncate(error, 140));
255        if header.chars().count() + row.chars().count() > max_chars {
256            break;
257        }
258        header.push_str(&row);
259    }
260    for (idx, _, _) in ready {
261        if let Some(m) = &idx.upstream_provenance {
262            let row = format!(
263                "Upstream {}: {}@{} commit {}{}\n",
264                idx.id(),
265                m.repo,
266                m.tag.as_deref().unwrap_or("none"),
267                m.commit.as_deref().map(|s| &s[..s.len().min(7)]).unwrap_or("none"),
268                if m.site.is_some() { " + opted-in docs site" } else { "" }
269            );
270            if header.chars().count() + row.chars().count() > max_chars {
271                break;
272            }
273            header.push_str(&row);
274        }
275    }
276    header.push_str("Focus `package` or request JSON for full provenance.\n");
277    header
278}
279
280/// Exact case-sensitive module token, not a prefix or package-looking substring.
281fn mentions_import(query: &str, import: &str) -> bool {
282    let module_char = |c: char| c.is_alphanumeric() || matches!(c, '@' | '/' | '_' | '-' | '.');
283    query.match_indices(import).any(|(i, _)| {
284        let before = query[..i].chars().next_back();
285        let after = &query[i + import.len()..];
286        before.is_none_or(|c| !module_char(c))
287            && (after.chars().next().is_none_or(|c| !module_char(c)) || after.starts_with('.') && after[1..].chars().next().is_none_or(|c| !module_char(c)))
288    })
289}
290
291/// Resolve explicit calls through the selected public entrypoint, rather than
292/// treating every same-named member in its transitive type graph as the API.
293fn public_call_entries(idx: &PackageIndex, import: &str, query: &str) -> Vec<usize> {
294    let Some(api) = &idx.npm_api else {
295        return Vec::new();
296    };
297    let names: HashSet<&str> = api
298        .entries
299        .iter()
300        .flat_map(|e| [e.name.as_str(), e.path.as_str()])
301        .filter(|name| {
302            query.match_indices(*name).any(|(i, _)| {
303                query[i + name.len()..].starts_with('(')
304                    && query[..i]
305                        .chars()
306                        .next_back()
307                        .is_none_or(|c| !(c.is_alphanumeric() || matches!(c, '_' | '$' | '.' | ':' | '/' | '-' | '@')))
308            })
309        })
310        .collect();
311    let roots: Vec<_> = api.roots.iter().filter(|r| r.import == import).collect();
312    let mut targets = HashSet::new();
313    for name in names {
314        let resolved: Vec<_> = roots.iter().map(|root| api.resolve(root, name, &api.entries)).collect();
315        let Some(first) = resolved.first() else {
316            continue;
317        };
318        if first.entries.is_empty() || resolved.iter().any(|r| r.unresolved || r.entries != first.entries) {
319            continue;
320        }
321        for &i in &first.entries {
322            let e = &api.entries[i];
323            targets.insert((e.file.as_str(), e.path.as_str(), e.line));
324        }
325    }
326    idx.entries
327        .iter()
328        .enumerate()
329        .filter_map(|(i, e)| targets.contains(&(e.file.as_str(), e.path.as_str(), e.line)).then_some(i))
330        .collect()
331}