1use crate::bm25::{self, Bm25};
6use crate::extract::{Entry, Kind};
7use crate::index::{self, PackageIndex};
8use crate::locate::{self, Source};
9use crate::project::{Project, Spec};
10use crate::{embed, est_tokens, fetch, norm_name, upstream, Dep, Eco};
11use rayon::prelude::*;
12use serde_json::{json, Value};
13use std::collections::{HashMap, HashSet};
14use std::path::{Path, PathBuf};
15use std::sync::{Arc, Mutex};
16
17pub const DEFAULT_TOKENS: usize = 1200;
18const RELEVANCE_FLOOR: f32 = 0.3;
20const DENSE_WEIGHT: f32 = 0.1;
24const RRF_K: f32 = 10.0;
26const DOCS_DENSE_WEIGHT: f32 = 0.45;
28const DENSE_POOL: usize = 300;
30const DECISIVE_RATIO: f32 = 2.0;
33const HEAD_WEIGHT: f32 = 0.25;
35const MAX_PACKAGES: usize = 80;
37
38#[derive(Debug, Clone, PartialEq, Eq)]
39pub struct Options {
40 pub fetch: bool,
42 pub upstream: bool,
44}
45
46impl Default for Options {
47 fn default() -> Self {
48 let mut fetch = std::env::var("LOCKDOCS_FETCH").is_ok_and(|v| matches!(v.as_str(), "1" | "true" | "yes"));
49 let mut upstream = automatic_upstream(std::env::var("LOCKDOCS_FETCH").ok().as_deref(), std::env::var("LOCKDOCS_NO_UPSTREAM").is_ok());
50 if std::env::var("LOCKDOCS_OFFLINE").is_ok_and(|v| v != "0") {
51 fetch = false;
52 upstream = false;
53 }
54 Options { fetch, upstream }
55 }
56}
57
58fn automatic_upstream(fetch: Option<&str>, disabled: bool) -> bool {
59 !disabled && !fetch.is_some_and(|v| matches!(v, "0" | "false" | "no"))
60}
61
62pub struct Answer {
63 pub text: String,
64 pub json: Value,
65}
66
67pub struct Engine {
68 pub project: Project,
69 opts: Options,
70 indexes: Mutex<HashMap<String, Arc<PackageIndex>>>,
71 upstream_notes: Mutex<HashMap<String, String>>,
72}
73
74type ReadyPkg = (Arc<PackageIndex>, Dep, Option<String>);
76
77enum Resolved {
79 Ready(Arc<PackageIndex>, Dep, Option<String>),
80 Missing(Dep, String),
81}
82
83fn fetch_report(dep: &Dep, status: &upstream::Manifest) -> Value {
85 json!({"package": dep.id(), "repo": status.repo, "tag": status.tag,
86 "files": status.files, "note": status.note, "upstream": status})
87}
88
89fn provenance(ready: &[ReadyPkg]) -> Vec<Value> {
90 ready
91 .iter()
92 .map(|(idx, dep, note)| {
93 json!({
94 "package": idx.id(), "requested_version": dep.version, "source": idx.source,
95 "registry_fetched": idx.fetched, "upstream": idx.upstream_provenance, "upstream_label": idx.upstream, "note": note,
96 })
97 })
98 .collect()
99}
100
101fn broad_header(ready: &[ReadyPkg], missing: &[(Dep, String)], tokens: usize) -> String {
104 let fetched = ready.iter().filter(|r| r.0.fetched).count();
105 let upstream = ready.iter().filter(|r| r.0.upstream.is_some()).count();
106 let notes = ready.iter().filter(|r| r.2.is_some()).count();
107 let mut header = format!("Searched {} direct dependencies: {} local package sources, {fetched} registry sources, {upstream} upstream sources; {notes} fallback notes, {} unavailable.\n", ready.len(), ready.len() - fetched, missing.len());
108 let max_chars = (tokens / 3).clamp(100, 400) * 3;
109 for (idx, _, note) in ready {
110 if let Some(note) = note {
111 let row = format!("Fallback {}: {}\n", idx.id(), truncate(note, 160));
112 if header.chars().count() + row.chars().count() > max_chars {
113 break;
114 }
115 header.push_str(&row);
116 }
117 }
118 for (dep, error) in missing {
119 let row = format!("Skipped {}: {}\n", dep.id(), truncate(error, 140));
120 if header.chars().count() + row.chars().count() > max_chars {
121 break;
122 }
123 header.push_str(&row);
124 }
125 for (idx, _, _) in ready {
126 if let Some(m) = &idx.upstream_provenance {
127 let row = format!(
128 "Upstream {}: {}@{} commit {}{}\n",
129 idx.id(),
130 m.repo,
131 m.tag.as_deref().unwrap_or("none"),
132 m.commit.as_deref().map(|s| &s[..s.len().min(7)]).unwrap_or("none"),
133 if m.site.is_some() { " + opted-in docs site" } else { "" }
134 );
135 if header.chars().count() + row.chars().count() > max_chars {
136 break;
137 }
138 header.push_str(&row);
139 }
140 }
141 header.push_str("Focus `package` or request JSON for full provenance.\n");
142 header
143}
144
145fn lang(eco: Eco) -> &'static str {
146 match eco {
147 Eco::Npm => "ts",
148 Eco::PyPI => "python",
149 Eco::Cargo => "rust",
150 Eco::Go => "go",
151 }
152}
153
154fn is_changelog(e: &Entry) -> bool {
157 let f = e.file.rsplit('/').next().unwrap_or(&e.file).to_ascii_uppercase();
158 if ["CHANGELOG", "CHANGES", "HISTORY", "NEWS", "RELEASES"].iter().any(|p| f.starts_with(p)) {
159 return true;
160 }
161 e.name.split(" › ").skip(1).any(|h| {
162 let h = h.trim().trim_start_matches(['v', 'V', '[']);
163 let mut parts = h.split(|c: char| !c.is_ascii_digit());
164 matches!((parts.next(), parts.next()), (Some(a), Some(b)) if !a.is_empty() && !b.is_empty()) && h.chars().nth(a_len(h)) == Some('.')
165 })
166}
167
168fn a_len(h: &str) -> usize {
169 h.chars().take_while(|c| c.is_ascii_digit()).count()
170}
171
172fn change_intent(q: &str) -> bool {
173 let q = q.to_ascii_lowercase();
174 [
175 "change",
176 "migrat",
177 "upgrad",
178 "breaking",
179 "deprecat",
180 "removed",
181 "renamed",
182 "new in",
183 "since",
184 "release",
185 "what's new",
186 "differen",
187 ]
188 .iter()
189 .any(|w| q.contains(w))
190}
191
192fn private_path(e: &Entry) -> bool {
193 e.file.split('/').any(|s| s.starts_with('_') && !s.starts_with("__init__")) || e.path.contains("._") || e.file.contains("/internal/")
194}
195
196fn tooling_path(e: &Entry) -> bool {
198 e.file.split('/').any(|s| {
199 let stem = s.split('.').next().unwrap_or(s);
200 matches!(
201 stem,
202 "mypy" | "pylint" | "plugin" | "plugins" | "testing" | "conftest" | "scripts" | "codemod" | "codemods" | "eslint" | "babel"
203 )
204 })
205}
206
207impl Engine {
208 pub fn new(root: &Path, opts: Options) -> Engine {
209 Engine {
210 project: Project::load(root),
211 opts,
212 indexes: Mutex::new(HashMap::new()),
213 upstream_notes: Mutex::new(HashMap::new()),
214 }
215 }
216
217 pub fn root(&self) -> &Path {
218 &self.project.root
219 }
220
221 pub fn fetch_enabled(&self) -> bool {
222 self.opts.fetch
223 }
224
225 fn source(&self, dep: &Dep) -> Result<(Source, Option<String>), String> {
228 let local = locate::locate(dep, &self.project.root);
229 if let Some(s) = &local {
230 if s.version == dep.version {
231 return Ok((s.clone(), None));
232 }
233 }
234 if let Some(s) = fetch::cached(dep) {
235 return Ok((s, None));
236 }
237 if self.opts.fetch {
238 match fetch::fetch(dep) {
239 Ok(s) => return Ok((s, None)),
240 Err(e) => return Err(format!("{}: exact registry fetching failed: {e:#}; no other version substituted", dep.id())),
241 }
242 }
243 if let Some(s) = local {
244 return Err(format!(
245 "{} pins {}, but {} has {}; refusing to substitute the installed version. Reinstall to sync, or enable exact registry fetching (--fetch).",
246 dep.from,
247 dep.id(),
248 s.label,
249 s.version
250 ));
251 }
252 let how = match dep.eco {
253 Eco::Npm => "run your package manager's install",
254 Eco::PyPI => "create the virtualenv (.venv) and install",
255 Eco::Cargo => "run `cargo fetch`",
256 Eco::Go => "run `go mod download`",
257 };
258 Err(format!(
259 "{} is pinned in {} but its files are not on this machine: {how}, or enable fetching (--fetch or LOCKDOCS_FETCH=1) to download exactly this version.",
260 dep.id(),
261 dep.from
262 ))
263 }
264
265 fn index(&self, dep: &Dep) -> Resolved {
266 match self.source(dep) {
267 Ok((src, mut note)) => {
268 let key = format!("{}:{}@{}:{}", dep.eco, dep.name, src.version, src.dir.display());
269 if let Some(i) = self.indexes.lock().unwrap().get(&key) {
270 if !i.embed.is_empty() || !embed::available() {
272 if let Some(n) = self.upstream_notes.lock().unwrap().get(&dep.id()) {
273 note = Some(match note {
274 Some(old) => format!("{old} {n}"),
275 None => n.clone(),
276 });
277 }
278 return Resolved::Ready(i.clone(), dep.clone(), note);
279 }
280 }
281 let (up, up_note) = self.upstream(dep, &src);
282 if let Some(n) = up_note {
283 self.upstream_notes.lock().unwrap().insert(dep.id(), n.clone());
284 note = Some(match note {
285 Some(old) => format!("{old} {n}"),
286 None => n,
287 });
288 } else {
289 self.upstream_notes.lock().unwrap().remove(&dep.id());
290 }
291 let idx = Arc::new(index::load_or_build(dep, &src, &self.project.root, up.as_ref()));
292 self.indexes.lock().unwrap().insert(key, idx.clone());
293 Resolved::Ready(idx, dep.clone(), note)
294 }
295 Err(e) => Resolved::Missing(dep.clone(), e),
296 }
297 }
298
299 fn upstream(&self, dep: &Dep, src: &Source) -> (Option<(PathBuf, upstream::Manifest)>, Option<String>) {
302 if src.version != dep.version || std::env::var("LOCKDOCS_NO_UPSTREAM").is_ok() {
303 return (None, None);
304 }
305 let enabled = self.opts.fetch || self.opts.upstream;
306 if fetch::is_git(dep) {
307 return (
308 None,
309 enabled.then(|| "upstream docs skipped: git dependency's resolved commit is not a release tag; using its checkout files".into()),
310 );
311 }
312 let cached = upstream::cached(dep);
313 if let Some(c) = cached.as_ref().filter(|(_, m)| !enabled || !upstream::needs_refresh(m, self.opts.fetch)) {
314 let note = if upstream::needs_refresh(&c.1, false) {
315 Some(format!(
316 "offline fallback: upstream cache format {} has not been exact-tag revalidated{}{}",
317 c.1.format,
318 if c.1.format < 4 {
319 "; its old commit candidate may have been a branch"
320 } else {
321 ""
322 },
323 c.1.note.as_ref().map(|n| format!("; {n}")).unwrap_or_default()
324 ))
325 } else {
326 c.1.note.clone()
327 };
328 return (Some(c.clone()), note);
329 }
330 if enabled {
331 let result = if self.opts.fetch {
332 upstream::fetch(dep, src)
333 } else {
334 upstream::fetch_automatic(dep, src)
335 };
336 return match result {
337 Ok(m) => (Some((upstream::dir(dep), m.clone())), m.note.clone()),
338 Err(e) => (None, Some(format!("upstream docs fetch failed: {e:#}; using package files"))),
339 };
340 }
341 (None, None)
342 }
343
344 pub fn fetch_all(&self, package: Option<&str>) -> Result<Answer, String> {
348 let t = std::time::Instant::now();
349 let mut text = String::new();
350 if embed::enabled() {
351 match embed::ensure() {
352 Ok(()) => text.push_str(&format!(" model {} ready\n", embed::MODEL_ID)),
353 Err(e) => text.push_str(&format!(" model {} unavailable ({e:#}); keyword search only\n", embed::MODEL_ID)),
354 }
355 }
356 let deps: Vec<Dep> = match package.filter(|p| !p.trim().is_empty()) {
357 Some(p) => {
358 let mut out = Vec::new();
359 for spec in p.split(',').map(str::trim).filter(|s| !s.is_empty()) {
360 out.extend(self.deps_for(spec)?);
361 }
362 out
363 }
364 None => {
365 let mut seen = HashSet::new();
366 self.project.direct().filter(|d| seen.insert((d.eco, d.name.clone()))).cloned().collect()
367 }
368 };
369 let rows: Vec<(Dep, String, Value)> = deps
370 .par_iter()
371 .map(|d| {
372 if fetch::is_git(d) {
373 let src = locate::locate(d, &self.project.root);
374 return (
375 d.clone(),
376 "upstream skipped: git dependency; use resolved checkout files".into(),
377 json!({"package": d.id(), "status": "git-checkout", "source": src.map(|s| s.label)}),
378 );
379 }
380 let src = match locate::locate(d, &self.project.root)
381 .filter(|s| s.version == d.version)
382 .or_else(|| fetch::cached(d))
383 {
384 Some(s) => Some(s),
385 None if !fetch::is_git(d) => fetch::fetch(d).ok(),
386 None => None,
387 };
388 let Some(src) = src else {
389 return (
390 d.clone(),
391 "package files unavailable".to_string(),
392 json!({"package": d.id(), "status": "missing"}),
393 );
394 };
395 let status = match upstream::fetch(d, &src) {
396 Ok(m) => m,
397 Err(e) => {
398 return (
399 d.clone(),
400 format!("upstream docs fetch failed: {e:#}"),
401 json!({"package": d.id(), "status": "fetch-failed", "error": format!("{e:#}")}),
402 )
403 }
404 };
405 let line = match (&status.tag, status.files) {
406 (_, n) if n > 0 => format!(
407 "upstream docs {}{}: {n} files ({:.1} MB)",
408 status.tag.as_ref().map(|t| format!("{}@{t}", status.repo)).unwrap_or_default(),
409 status.site.as_ref().map(|s| format!(" + docs site {s}")).unwrap_or_default(),
410 status.bytes as f64 / 1e6
411 ),
412 _ => format!("no upstream docs ({}: {})", status.repo, status.note.clone().unwrap_or_default()),
413 };
414 (
415 d.clone(),
416 if let Some(note) = &status.note { format!("{line}; {note}") } else { line },
417 fetch_report(d, &status),
418 )
419 })
420 .collect();
421 for (d, line, _) in &rows {
422 text.push_str(&format!(" {:<40} {line}\n", d.id()));
423 }
424 self.indexes.lock().unwrap().clear();
426 let ms = t.elapsed().as_millis();
427 Ok(Answer {
428 text: format!("Fetched for {} packages in {ms} ms (cached; later queries stay offline):\n{text}", rows.len()),
429 json: json!({"packages": rows.iter().map(|r| r.2.clone()).collect::<Vec<_>>(), "ms": ms, "model": embed::installed()}),
430 })
431 }
432
433 fn deps_for(&self, spec_str: &str) -> Result<Vec<Dep>, String> {
436 let spec = Spec::parse(spec_str);
437 let found = self.project.find(&spec);
438 if let Some(d) = found.iter().find(|d| spec.version_matches(&d.version)) {
439 return Ok(vec![(*d).clone()]);
440 }
441 let eco = spec.eco.or_else(|| found.first().map(|d| d.eco));
442 if let (Some(v), Some(eco)) = (&spec.version, eco) {
443 if spec.is_exact() {
444 let version = if eco == Eco::Go && !v.starts_with('v') { format!("v{v}") } else { v.clone() };
445 let dep = Dep {
446 eco,
447 name: found.first().map_or(spec.name.clone(), |d| d.name.clone()),
448 version,
449 direct: false,
450 from: "requested".into(),
451 };
452 if fetch::cached(&dep).is_some() || self.opts.fetch {
453 return Ok(vec![dep]);
454 }
455 return Err(format!(
456 "{}@{} is not in this project{}; enable fetching (--fetch or LOCKDOCS_FETCH=1) to read that exact version.",
457 dep.name,
458 dep.version,
459 found.first().map(|d| format!(" (it pins {})", d.version)).unwrap_or_default()
460 ));
461 }
462 }
463 if let Some(d) = found.first() {
464 return Err(format!("this project pins {} {}, which does not match `{}`.", d.name, d.version, spec_str));
465 }
466 if spec.eco.is_none() && spec.version.is_some() && self.opts.fetch {
467 return Err(format!(
468 "`{}` is not a dependency here; prefix the ecosystem to fetch it (npm:, pypi:, cargo:, go:).",
469 spec.name
470 ));
471 }
472 let near = self.project.similar(&spec.name, 6);
473 Err(if near.is_empty() {
474 format!(
475 "`{}` is not a dependency of {} (lockfiles: {}). Use `resolve` to list dependencies.",
476 spec.name,
477 self.project.root.display(),
478 if self.project.lockfiles.is_empty() {
479 "none found".into()
480 } else {
481 self.project.lockfiles.join(", ")
482 }
483 )
484 } else {
485 format!("`{}` is not a dependency here. Did you mean: {}?", spec.name, near.join(", "))
486 })
487 }
488
489 fn mentioned(&self, query: &str) -> Vec<Dep> {
491 let q = query.to_ascii_lowercase();
492 let words: HashSet<&str> = q
493 .split(|c: char| c.is_whitespace() || matches!(c, ',' | '?' | '(' | ')' | '`' | '\'' | '"'))
494 .collect();
495 let mut out: Vec<Dep> = Vec::new();
496 for d in self.project.direct() {
497 let n = d.name.to_ascii_lowercase();
498 let short = n.rsplit('/').next().unwrap_or(&n);
499 if (words.contains(n.as_str())
500 || (d.eco == Eco::Go && words.contains(short))
501 || words.iter().any(|w| w.starts_with(&format!("{n}.")) || w.starts_with(&format!("{n}::"))))
502 && !out.iter().any(|o| o.name == d.name)
503 {
504 out.push(d.clone());
505 }
506 }
507 out
508 }
509
510 fn select(&self, package: Option<&str>, query: &str) -> Result<(Vec<Dep>, bool), String> {
511 if let Some(p) = package.filter(|p| !p.trim().is_empty()) {
512 let mut out = Vec::new();
513 for spec in p.split(',').map(str::trim).filter(|s| !s.is_empty()) {
514 out.extend(self.deps_for(spec)?);
515 }
516 return Ok((out, false));
517 }
518 let m = self.mentioned(query);
519 if !m.is_empty() {
520 return Ok((m, false));
521 }
522 let mut seen = HashSet::new();
523 let direct: Vec<Dep> = self
524 .project
525 .direct()
526 .filter(|d| seen.insert((d.eco, d.name.clone())))
527 .take(MAX_PACKAGES)
528 .cloned()
529 .collect();
530 if direct.is_empty() {
531 return Err(format!(
532 "No dependencies found in {} (looked for {}). Pass `package`, or run from the project directory.",
533 self.project.root.display(),
534 "package-lock.json, pnpm-lock.yaml, yarn.lock, bun.lock, Cargo.lock, uv.lock, poetry.lock, requirements*.txt, go.mod"
535 ));
536 }
537 Ok((direct, true))
538 }
539
540 fn resolve_all(&self, deps: &[Dep]) -> (Vec<ReadyPkg>, Vec<(Dep, String)>) {
541 let results: Vec<Resolved> = deps.par_iter().map(|d| self.index(d)).collect();
542 let mut ready = Vec::new();
543 let mut missing = Vec::new();
544 for r in results {
545 match r {
546 Resolved::Ready(i, d, n) => ready.push((i, d, n)),
547 Resolved::Missing(d, e) => missing.push((d, e)),
548 }
549 }
550 (ready, missing)
551 }
552
553 pub fn warm(&self, package: Option<&str>) -> Result<Answer, String> {
555 let t = std::time::Instant::now();
556 let deps: Vec<Dep> = match package.filter(|p| !p.trim().is_empty()) {
557 Some(p) => self.select(Some(p), "")?.0,
558 None => {
559 let mut seen = HashSet::new();
560 self.project.direct().filter(|d| seen.insert((d.eco, d.name.clone()))).cloned().collect()
561 }
562 };
563 let (ready, missing) = self.resolve_all(&deps);
564 let mut text = String::new();
565 let mut items = Vec::new();
566 for (idx, _, note) in &ready {
567 let symbols = idx.entries.iter().filter(|e| e.kind != Kind::Prose).count();
568 text.push_str(&format!(
569 " {:<40} {:>6} symbols {:>6} doc sections ({})\n",
570 idx.id(),
571 symbols,
572 idx.entries.len() - symbols,
573 idx.source
574 ));
575 if let Some(n) = note {
576 text.push_str(&format!(" note: {n}\n"));
577 }
578 items.push(json!({"package": idx.id(), "ecosystem": idx.eco.as_str(), "symbols": symbols, "sections": idx.entries.len() - symbols, "source": idx.source, "upstream": idx.upstream, "note": note, "build_ms": idx.build_ms}));
579 }
580 for (d, e) in &missing {
581 text.push_str(&format!(" {:<40} missing: {e}\n", d.id()));
582 }
583 let ms = t.elapsed().as_millis();
584 text = format!("Indexed {} of {} packages in {ms} ms.\n{text}", ready.len(), deps.len());
585 Ok(Answer {
586 text,
587 json: json!({"packages": items, "missing": missing.iter().map(|(d, e)| json!({"package": d.id(), "error": e})).collect::<Vec<_>>(), "ms": ms}),
588 })
589 }
590
591 pub fn resolve(&self, filter: Option<&str>) -> Answer {
594 let p = &self.project;
595 let filter = filter.map(|f| f.trim().to_ascii_lowercase()).filter(|f| !f.is_empty());
596 let shown: Vec<&Dep> = match &filter {
597 Some(f) => p.deps.iter().filter(|d| d.name.to_ascii_lowercase().contains(f.as_str())).collect(),
598 None => p.direct().collect(),
599 };
600 let rows: Vec<(&Dep, Option<Source>)> = shown
601 .par_iter()
602 .map(|d| (*d, locate::locate(d, &p.root).or_else(|| fetch::cached(d))))
603 .collect();
604 let mut text = format!(
605 "Project: {}\nLockfiles: {}\n",
606 p.root.display(),
607 if p.lockfiles.is_empty() {
608 "none found".to_string()
609 } else {
610 p.lockfiles.join(", ")
611 }
612 );
613 for w in &p.warnings {
614 text.push_str(&format!("Warning: {w}\n"));
615 }
616 let total_direct = p.direct().count();
617 text.push_str(&match &filter {
618 Some(f) => format!("\n{} of {} dependencies match `{f}`:\n", rows.len(), p.deps.len()),
619 None => format!(
620 "\n{} direct dependencies ({} total with transitive; pass a filter to search all):\n",
621 total_direct,
622 p.deps.len()
623 ),
624 });
625 let w = rows.iter().map(|(d, _)| d.name.len() + d.version.len() + 1).max().unwrap_or(10).min(48);
626 let mut items = Vec::new();
627 for (d, s) in &rows {
628 let id = format!("{}@{}", d.name, d.version);
629 let (status, src) = match s {
630 Some(s) if s.version == d.version => ("docs ready", s.label.clone()),
631 Some(s) => ("version drift", format!("installed {} at {}", s.version, s.label)),
632 None => (
633 "not installed",
634 if self.opts.fetch {
635 "will fetch on first query".into()
636 } else {
637 "install it, or enable fetching".to_string()
638 },
639 ),
640 };
641 text.push_str(&format!(
642 " {:<6} {:<w$} {}{} {}\n",
643 d.eco.as_str(),
644 id,
645 if d.direct { "" } else { "(transitive) " },
646 status,
647 src
648 ));
649 items.push(json!({"ecosystem": d.eco.as_str(), "name": d.name, "version": d.version, "direct": d.direct, "lockfile": d.from, "status": status, "source": src}));
650 }
651 if rows.is_empty() {
652 text.push_str(" (none)\n");
653 }
654 Answer {
655 text,
656 json: json!({"root": p.root.display().to_string(), "lockfiles": p.lockfiles, "dependencies": items, "total": p.deps.len(), "warnings": p.warnings}),
657 }
658 }
659
660 pub fn upgrade_report(&self, package: &str, target: &str, tokens: usize) -> Result<Answer, String> {
668 let spec = Spec::parse(package);
669 let found = self.project.find(&spec);
670 let dep = found
671 .iter()
672 .find(|d| d.direct)
673 .or_else(|| found.first())
674 .map(|d| (*d).clone())
675 .ok_or_else(|| format!("`{package}` is not pinned in this project's lockfiles; run `resolve` to see packages."))?;
676 let target = target.trim().trim_start_matches('v');
677 if target.is_empty() {
678 return Err("`upgrade_to` needs a target version, e.g. 15.0.0".into());
679 }
680 if target == dep.version.trim_start_matches('v') {
681 return Err(format!("{} is already the pinned version; pick a different target.", dep.id()));
682 }
683 let old = match self.index(&dep) {
684 Resolved::Ready(i, _, _) => i,
685 Resolved::Missing(_, e) => return Err(e),
686 };
687 let target_dep = Dep {
688 eco: dep.eco,
689 name: dep.name.clone(),
690 version: target.to_string(),
691 direct: false,
692 from: "requested".into(),
693 };
694 let new = match self.index(&target_dep) {
695 Resolved::Ready(i, _, _) => i,
696 Resolved::Missing(_, e) => {
697 return Err(format!(
698 "{e}\nThe target version's files are needed to compare; add --fetch (or LOCKDOCS_FETCH=1) to download exactly {}.",
699 target_dep.id()
700 ))
701 }
702 };
703 let r = crate::upgrade::report(&self.project.root, &old, &new);
704 let max = tokens.clamp(200, 20000) * 3;
705 let text = if r.text.len() > max {
706 format!("{}\n... truncated to the token budget; raise `tokens`.\n", truncate(&r.text, max))
707 } else {
708 r.text
709 };
710 Ok(Answer { text, json: r.json })
711 }
712
713 pub fn docs(&self, package: Option<&str>, query: &str, tokens: usize) -> Result<Answer, String> {
714 let tokens = tokens.clamp(200, 20000);
715 let (deps, broad) = self.select(package, query)?;
716 let (ready, missing) = self.resolve_all(&deps);
717 if ready.is_empty() {
718 return Err(missing.into_iter().map(|(_, e)| e).collect::<Vec<_>>().join("\n"));
719 }
720 let q = query.trim();
721 if q.is_empty() {
722 return Ok(self.overview(&ready, tokens));
723 }
724 let qterms = bm25::query_terms(q);
725 let mut refs: Vec<(usize, usize)> = Vec::new();
727 for (pi, (idx, _, _)) in ready.iter().enumerate() {
728 for ei in 0..idx.entries.len() {
729 refs.push((pi, ei));
730 }
731 }
732 let bm = Bm25::build(refs.iter().map(|&(pi, ei)| (ready[pi].0.terms[ei].as_slice(), ready[pi].0.lens[ei])));
733 let hb = Bm25::build(refs.iter().map(|&(pi, ei)| (ready[pi].0.head[ei].as_slice(), ready[pi].0.head_lens[ei])));
734 let mut raw_idents = identifiers(q);
735 raw_idents.extend(api_words(&ready, q));
736 let changes = change_intent(q);
737 let scored = hybrid_rank(&ready, &refs, (&bm, &hb), &qterms, q, &raw_idents, changes);
738 let mut out = Pack::new(tokens);
739 let mut header = String::new();
740 for (idx, dep, note) in &ready {
741 header.push_str(&format!(
742 "{} · {} · {}{}\n",
743 idx.id(),
744 idx.eco,
745 idx.source,
746 if dep.from == "requested" {
747 String::new()
748 } else {
749 format!(" · pinned in {}", dep.from)
750 }
751 ));
752 if let Some(u) = &idx.upstream {
753 header.push_str(&format!("Upstream docs: {u}\n"));
754 }
755 if let Some(n) = note {
756 header.push_str(&format!("Note: {n}\n"));
757 }
758 }
759 if broad {
760 header = broad_header(&ready, &missing, tokens);
761 }
762 for (d, e) in &missing {
763 if !broad {
764 header.push_str(&format!("Skipped {}: {e}\n", d.id()));
765 }
766 }
767 out.push_raw(&header);
768 let mut seen: HashSet<(usize, String, u32)> = HashSet::new();
769 let mut hits_json = Vec::new();
770 let mut used_pkgs: Vec<String> = Vec::new();
771 let top = scored.first().map_or(0.0, |s| s.0);
772 for (score, pi, ei) in scored {
773 let idx = &ready[pi].0;
774 let e = &idx.entries[ei];
775 if out.blocks >= 3 && score < top * RELEVANCE_FLOOR {
777 break;
778 }
779 if !seen.insert((pi, e.file.clone(), e.line)) {
780 continue;
781 }
782 let e = resolve_alias(idx, e);
783 let ctx = if out.blocks == 0 { section_context(idx, e) } else { None };
786 let max_doc = if out.blocks == 0 {
787 2000 - ctx.as_ref().map_or(0, |c| c.len().min(800) / 2)
788 } else {
789 800
790 };
791 let mut block = render_entry(idx, e, max_doc);
792 if let (Some(ctx), Some(nl)) = (ctx, block.find('\n')) {
793 block.insert_str(nl + 1, &ctx);
794 }
795 if !out.push_block(&block) {
796 break;
797 }
798 if !used_pkgs.contains(&idx.id()) {
799 used_pkgs.push(idx.id());
800 }
801 hits_json.push(json!({"package": idx.id(), "kind": e.kind.as_str(), "path": e.path, "file": e.file, "line": e.line, "score": score}));
802 }
803 if out.blocks == 0 {
804 out.push_raw(&format!(
805 "No match for \"{q}\" in {}. Try other words, a symbol name (the `api` tool), or `package` to widen/narrow.\n",
806 ready.iter().map(|r| r.0.id()).collect::<Vec<_>>().join(", ")
807 ));
808 } else if broad {
809 out.prepend(&format!("Sources: {}\n", used_pkgs.join(", ")));
810 }
811 for (idx, dep, _) in &ready {
813 let prose = idx.entries.iter().filter(|e| e.kind == Kind::Prose).count();
814 if !broad && idx.upstream.is_none() && prose < 40 && !self.opts.fetch && upstream::cached(dep).is_none() {
815 out.push_raw(&format!(
816 "\nTip: {} ships few docs. `lockdocs fetch {}` adds its upstream docs at this version's git tag (one-time download, then offline).\n",
817 idx.id(),
818 idx.name
819 ));
820 }
821 }
822 Ok(Answer {
823 json: json!({"query": q, "packages": ready.iter().map(|r| r.0.id()).collect::<Vec<_>>(), "hits": hits_json, "provenance": provenance(&ready), "tokens": est_tokens(&out.text)}),
824 text: out.text,
825 })
826 }
827
828 fn overview(&self, ready: &[ReadyPkg], tokens: usize) -> Answer {
829 let mut out = Pack::new(tokens);
830 for (idx, dep, note) in ready {
831 out.push_raw(&format!("{} · {} · {} · pinned in {}\n", idx.id(), idx.eco, idx.source, dep.from));
832 if let Some(u) = &idx.upstream {
833 out.push_raw(&format!("Upstream: {u}\n"));
834 }
835 if let Some(n) = note {
836 out.push_raw(&format!("Note: {n}\n"));
837 }
838 let symbols = idx
839 .entries
840 .iter()
841 .filter(|e| e.kind != Kind::Prose && e.kind != Kind::Alias && !private_path(e))
842 .count();
843 let prose = idx.entries.len() - symbols;
844 out.push_raw(&format!("Indexed {symbols} API symbols and {prose} doc sections.\n"));
845 let mut top: Vec<&Entry> = idx
847 .entries
848 .iter()
849 .filter(|e| e.kind != Kind::Prose && !private_path(e) && !e.legacy && !e.doc.is_empty())
850 .collect();
851 top.sort_by_key(|e| (e.path.matches(['.', ':']).count(), e.path.len()));
852 let names: Vec<String> = top.iter().take(60).map(|e| e.path.clone()).collect();
853 if !names.is_empty() {
854 out.push_raw(&format!("Key API: {}\n", names.join(", ")));
855 }
856 for e in idx
857 .entries
858 .iter()
859 .filter(|e| e.kind == Kind::Prose && e.file.to_ascii_uppercase().starts_with("README"))
860 .take(3)
861 {
862 if !out.push_block(&render_entry(idx, e, 1500)) {
863 break;
864 }
865 }
866 }
867 Answer {
868 json: json!({"packages": ready.iter().map(|r| r.0.id()).collect::<Vec<_>>(), "provenance": provenance(ready), "tokens": est_tokens(&out.text)}),
869 text: out.text,
870 }
871 }
872
873 pub fn api(&self, symbol: &str, package: Option<&str>, tokens: usize) -> Result<Answer, String> {
876 let tokens = tokens.clamp(200, 20000);
877 let symbol = symbol.trim().trim_end_matches("()").trim();
878 if symbol.is_empty() {
879 return Err("symbol is required, e.g. `z.object`, `tokio::spawn`, `BaseModel.model_dump`".into());
880 }
881 let (deps, rest) = match package.filter(|p| !p.trim().is_empty()) {
882 Some(p) => (self.select(Some(p), "")?.0, symbol.to_string()),
883 None => match self.package_prefix(symbol) {
884 Some((dep, rest)) => (vec![dep], rest),
885 None => (self.select(None, symbol)?.0, symbol.to_string()),
886 },
887 };
888 let segs: Vec<String> = split_symbol(&rest);
889 let Some(last) = segs.last().map(|l| l.trim_end_matches('!').to_string()) else {
890 return self.docs(
892 Some(&deps.iter().map(|d| format!("{}:{}@{}", d.eco, d.name, d.version)).collect::<Vec<_>>().join(",")),
893 "",
894 tokens,
895 );
896 };
897 let quals: Vec<String> = segs[..segs.len() - 1].iter().map(|s| s.to_ascii_lowercase()).collect();
898 let (ready, missing) = self.resolve_all(&deps);
899 if ready.is_empty() {
900 return Err(missing.into_iter().map(|(_, e)| e).collect::<Vec<_>>().join("\n"));
901 }
902 let mut cands: Vec<(f32, usize, usize)> = Vec::new();
903 let last_l = last.to_ascii_lowercase();
904 for (pi, (idx, _, _)) in ready.iter().enumerate() {
905 for (ei, e) in idx.entries.iter().enumerate() {
906 if e.kind == Kind::Prose {
907 continue;
908 }
909 let name = e.name.trim_end_matches('!');
910 let exact = name == last;
911 if !exact && name.to_ascii_lowercase() != last_l {
912 continue;
913 }
914 cands.push((api_score(e, exact, &quals), pi, ei));
915 }
916 }
917 cands.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));
918 if cands.is_empty() {
919 let pk = ready
921 .iter()
922 .map(|r| format!("{}:{}@{}", r.0.eco, r.0.name, r.0.version))
923 .collect::<Vec<_>>()
924 .join(",");
925 let mut a = self.docs(Some(&pk), &segs.join(" "), tokens)?;
926 a.text = format!(
927 "No symbol named `{last}` in {}; closest documentation:\n\n{}",
928 ready.iter().map(|r| r.0.id()).collect::<Vec<_>>().join(", "),
929 a.text
930 );
931 return Ok(a);
932 }
933 let (_, bpi, bei) = cands[0];
934 let (idx, dep, note) = &ready[bpi];
935 let best0 = &idx.entries[bei];
936 let best = resolve_alias(idx, best0);
937 let mut out = Pack::new(tokens);
938 out.push_raw(&format!(
939 "{} · {} · {}{}\n",
940 idx.id(),
941 idx.eco,
942 idx.source,
943 if dep.from == "requested" {
944 String::new()
945 } else {
946 format!(" · pinned in {}", dep.from)
947 }
948 ));
949 if let Some(u) = &idx.upstream {
950 out.push_raw(&format!("Upstream: {u}\n"));
951 }
952 if let Some(n) = note {
953 out.push_raw(&format!("Note: {n}\n"));
954 }
955 if !std::ptr::eq(best, best0) {
956 out.push_raw(&format!(
957 "`{}` is a re-export of `{}` ({}:{}).\n",
958 best0.path, best.path, best0.file, best0.line
959 ));
960 }
961 out.push_block(&render_entry(idx, best, 6000));
962 let overloads: Vec<&Entry> = idx
964 .entries
965 .iter()
966 .filter(|e| e.path == best.path && e.file == best.file && !std::ptr::eq(*e, best) && e.kind != Kind::Prose && e.kind != Kind::Alias)
967 .take(8)
968 .collect();
969 if !overloads.is_empty() {
970 let mut s = String::from("Other declarations:\n");
971 for e in overloads {
972 s.push_str(&format!("- `{}` — {} {}:{}\n", e.sig, idx.id(), e.file, e.line));
973 }
974 out.push_block(&s);
975 }
976 let mut members_json = Vec::new();
977 if best.kind.is_container() {
978 let sep = if idx.eco == Eco::Cargo { "::" } else { "." };
979 let prefix = format!("{}{sep}", best.path);
980 let members: Vec<&Entry> = idx
981 .entries
982 .iter()
983 .filter(|e| e.path.starts_with(&prefix) && !e.path[prefix.len()..].contains(sep) && e.kind != Kind::Prose)
984 .collect();
985 if !members.is_empty() {
986 let mut s = format!("Members of {} ({}):\n", best.path, members.len());
987 for m in members.iter().take(80) {
988 let doc1 = m.doc.lines().next().unwrap_or("");
989 s.push_str(&format!(
990 "- `{}`{}{} — {}:{}\n",
991 m.sig,
992 if doc1.is_empty() { "" } else { " — " },
993 truncate(doc1, 120),
994 m.file,
995 m.line
996 ));
997 members_json.push(json!({"path": m.path, "sig": m.sig, "file": m.file, "line": m.line}));
998 }
999 out.push_block(&s);
1000 }
1001 }
1002 let others: Vec<String> = cands
1003 .iter()
1004 .skip(1)
1005 .filter(|(_, pi, ei)| {
1006 let e = &ready[*pi].0.entries[*ei];
1007 !(e.path == best.path && e.file == best.file) && !std::ptr::eq(e, best0)
1008 })
1009 .take(8)
1010 .map(|(_, pi, ei)| {
1011 let (ix, _, _) = &ready[*pi];
1012 let e = &ix.entries[*ei];
1013 format!("- {} `{}` — {} {}:{}", e.kind.as_str(), e.path, ix.id(), e.file, e.line)
1014 })
1015 .collect();
1016 if !others.is_empty() {
1017 out.push_block(&format!("Other matches:\n{}\n", others.join("\n")));
1018 }
1019 Ok(Answer {
1020 json: json!({
1021 "provenance": provenance(&ready), "package": idx.id(), "kind": best.kind.as_str(), "path": best.path, "signature": best.sig, "doc": best.doc,
1022 "file": best.file, "line": best.line, "members": members_json, "tokens": est_tokens(&out.text),
1023 }),
1024 text: out.text,
1025 })
1026 }
1027
1028 fn package_prefix(&self, symbol: &str) -> Option<(Dep, String)> {
1031 let mut best: Option<(usize, Dep)> = None;
1032 let bytes = symbol.as_bytes();
1033 let mut cuts: Vec<usize> = Vec::new();
1034 for i in 0..bytes.len() {
1035 if bytes[i] == b'.' || bytes[i] == b'#' || (bytes[i] == b':' && i + 1 < bytes.len() && bytes[i + 1] == b':') {
1036 cuts.push(i);
1037 }
1038 }
1039 cuts.push(symbol.len());
1040 for &c in &cuts {
1041 let cand = &symbol[..c];
1042 let spec = Spec::parse(cand);
1043 if spec.version.is_some() {
1044 continue;
1045 }
1046 let found = self.project.find(&spec);
1047 if let Some(d) = found.first() {
1048 if best.as_ref().is_none_or(|(bc, _)| c > *bc) {
1049 best = Some((c, (*d).clone()));
1050 }
1051 }
1052 }
1053 let (c, dep) = best?;
1054 let rest = symbol[c..].trim_start_matches(['.', '#', ':']).to_string();
1055 Some((dep, rest))
1056 }
1057}
1058
1059fn split_symbol(s: &str) -> Vec<String> {
1060 s.replace("::", ".")
1061 .replace('#', ".")
1062 .split('.')
1063 .map(|x| x.trim().trim_end_matches("()").to_string())
1064 .filter(|x| !x.is_empty())
1065 .collect()
1066}
1067
1068fn identifiers(q: &str) -> Vec<String> {
1071 let words: Vec<&str> = q
1072 .split(|c: char| c.is_whitespace() || matches!(c, ',' | '?' | '\'' | '"' | '[' | ']'))
1073 .filter(|w| !w.is_empty())
1074 .collect();
1075 let single = words.len() == 1;
1076 words
1077 .into_iter()
1078 .filter(|w| {
1079 let inner = w.trim_matches(|c: char| !c.is_alphanumeric());
1080 single || w.starts_with('`') || w.contains(['_', '.', '(', '$']) || w.contains("::") || inner.chars().skip(1).any(|c| c.is_ascii_uppercase())
1081 })
1082 .map(|w| w.trim_matches(|c: char| matches!(c, '`' | '(' | ')' | '.' | ':' | '!' | ';')))
1083 .filter(|w| w.len() >= 2 && w.chars().any(|c| c.is_ascii_alphabetic()))
1084 .flat_map(|w| {
1085 let mut v = vec![w.to_ascii_lowercase()];
1086 if let Some(last) = split_symbol(w).last() {
1087 v.push(last.to_ascii_lowercase());
1088 }
1089 v
1090 })
1091 .collect()
1092}
1093
1094fn api_words(ready: &[ReadyPkg], q: &str) -> Vec<String> {
1098 let words: HashSet<String> = q
1099 .split(|c: char| !(c.is_ascii_alphanumeric() || c == '_'))
1100 .filter(|w| w.len() >= 4)
1101 .map(|w| w.to_ascii_lowercase())
1102 .filter(|w| !bm25::is_stop(w))
1103 .collect();
1104 let mut out = Vec::new();
1105 for (idx, _, _) in ready {
1106 for e in &idx.entries {
1107 let top = matches!(e.kind, Kind::Function | Kind::Macro | Kind::Class)
1108 && !e.doc.is_empty()
1109 && !private_path(e)
1110 && !e.legacy
1111 && e.path.matches(['.', ':']).count() <= 2;
1112 let n = e.name.trim_end_matches('!').to_ascii_lowercase();
1113 if top && words.contains(&n) && !out.contains(&n) {
1114 out.push(n);
1115 }
1116 }
1117 }
1118 out
1119}
1120
1121fn hybrid_rank(
1126 ready: &[ReadyPkg],
1127 refs: &[(usize, usize)],
1128 (bm, hb): (&Bm25, &Bm25),
1129 qterms: &[String],
1130 q: &str,
1131 idents: &[String],
1132 changes: bool,
1133) -> Vec<(f32, usize, usize)> {
1134 let mut fused: HashMap<usize, (f32, f32)> = HashMap::new();
1136 let expanded = bm25::expand(qterms);
1137 let bm_hits = bm.search_weighted(&expanded);
1138 let bmax = bm_hits.first().map_or(1.0, |h| h.score).max(1e-6);
1139 for h in bm_hits.iter().take(1500) {
1140 fused.entry(h.doc as usize).or_default().0 = h.score / bmax;
1141 }
1142 let head_w = std::env::var("LOCKDOCS_HEAD_WEIGHT").ok().and_then(|v| v.parse().ok()).unwrap_or(HEAD_WEIGHT);
1143 let h_hits = hb.search_weighted(&expanded);
1144 let hmax = h_hits.first().map_or(1.0, |h| h.score).max(1e-6);
1145 for h in h_hits.iter().take(1500) {
1146 fused.entry(h.doc as usize).or_default().1 = h.score / hmax;
1147 }
1148 let dense_w: f32 = std::env::var("LOCKDOCS_DENSE_WEIGHT").ok().and_then(|v| v.parse().ok()).unwrap_or(DENSE_WEIGHT);
1149 let n = refs.len().max(1) as f32;
1152 let rare: Vec<String> = qterms.iter().filter(|t| (bm.df(t) as f32) < n * 0.03).cloned().collect();
1153 let weight = |i: usize| {
1154 let (pi, ei) = refs[i];
1155 let e = &ready[pi].0.entries[ei];
1156 boost(e, idents, changes, major_of(&ready[pi].0.version)) * name_hit(e, &rare)
1157 };
1158 let mut lex: Vec<(usize, f32)> = fused.iter().map(|(&i, &(b, h))| (i, ((1.0 - head_w) * b + head_w * h) * weight(i))).collect();
1161 lex.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal).then(a.0.cmp(&b.0)));
1162 let decisive = match (lex.first(), lex.get(1)) {
1164 (Some(t), Some(s)) => t.1 >= s.1 * DECISIVE_RATIO,
1165 (Some(_), None) => true,
1166 _ => false,
1167 };
1168 let debug = std::env::var("LOCKDOCS_DEBUG").is_ok();
1169 let qv = if !decisive && dense_w > 0.0 && ready.iter().all(|r| !r.0.vecs.is_empty()) {
1170 embed::query_model().and_then(|m| m.embed(q))
1171 } else {
1172 None
1173 };
1174 let mut dense_rank: HashMap<usize, usize> = HashMap::new();
1176 let mut dense_cos: HashMap<usize, f32> = HashMap::new();
1177 if let Some(qv) = &qv {
1178 let mut sims: Vec<(usize, f32)> = refs
1179 .par_iter()
1180 .enumerate()
1181 .map(|(i, &(pi, ei))| (i, embed::cosine(qv, &ready[pi].0.vecs[ei])))
1182 .collect();
1183 sims.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal).then(a.0.cmp(&b.0)));
1184 for (r, (i, _)) in sims.iter().take(DENSE_POOL).enumerate() {
1185 dense_rank.insert(*i, r);
1186 }
1187 let cmax = sims.first().map_or(1.0, |s| s.1);
1188 let floor = sims.get(300).map_or(0.0, |s| s.1);
1189 for (i, c) in sims.iter().take(300) {
1190 dense_cos.insert(*i, ((c - floor) / (cmax - floor).max(1e-6)).max(0.0));
1191 }
1192 if debug {
1193 for (i, c) in sims.iter().take(10) {
1194 let (pi, ei) = refs[*i];
1195 eprintln!("dense cos={c:.3} {}", ready[pi].0.entries[ei].path);
1196 }
1197 }
1198 }
1199 if debug {
1200 eprintln!("decisive={decisive} dense={}", qv.is_some());
1201 }
1202 let with_docs = ready.iter().any(|r| r.0.upstream.is_some());
1210 let rrf = |r: usize| 1.0 / (RRF_K + r as f32);
1211 let mut scored: Vec<(f32, usize, usize)> = if qv.is_some() && with_docs {
1212 let mut ids: HashSet<usize> = lex.iter().map(|l| l.0).collect();
1213 ids.extend(dense_cos.keys().copied());
1214 let lexm: HashMap<usize, f32> = fused.iter().map(|(&i, &(b, h))| (i, (1.0 - head_w) * b + head_w * h)).collect();
1215 ids.into_iter()
1216 .map(|i| {
1217 let (pi, ei) = refs[i];
1218 let l = lexm.get(&i).copied().unwrap_or(0.0);
1219 let d = dense_cos.get(&i).copied().unwrap_or(0.0);
1220 (((1.0 - DOCS_DENSE_WEIGHT) * l + DOCS_DENSE_WEIGHT * d) * weight(i), pi, ei)
1221 })
1222 .collect()
1223 } else if qv.is_some() {
1224 let mut ids: HashSet<usize> = lex.iter().map(|l| l.0).collect();
1225 ids.extend(dense_rank.keys().copied());
1226 let lex_rank: HashMap<usize, usize> = lex.iter().enumerate().map(|(r, l)| (l.0, r)).collect();
1227 ids.into_iter()
1228 .map(|i| {
1229 let l = lex_rank.get(&i).map_or(0.0, |&r| rrf(r));
1230 let d = dense_rank.get(&i).map_or(0.0, |&r| rrf(r));
1231 let (pi, ei) = refs[i];
1232 ((1.0 - dense_w) * l + dense_w * d, pi, ei)
1233 })
1234 .collect()
1235 } else {
1236 lex.iter().map(|&(i, s)| (s, refs[i].0, refs[i].1)).collect()
1237 };
1238 if qv.is_some() {
1239 let top = scored.iter().map(|s| s.0).fold(0.0f32, f32::max).max(1e-9);
1240 scored.iter_mut().for_each(|s| s.0 /= top);
1241 }
1242 if debug {
1243 scored.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));
1244 for (s, pi, ei) in scored.iter().take(10) {
1245 eprintln!("{s:.3} {}", ready[*pi].0.entries[*ei].path);
1246 }
1247 }
1248 scored.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));
1249 let mut lifted: Vec<(f32, usize, String)> = Vec::new();
1251 for (s, pi, ei) in scored.iter().take(30) {
1252 let e = &ready[*pi].0.entries[*ei];
1253 for t in redirects(&format!("{}\n{}", e.sig, e.doc)) {
1254 if t != e.name {
1255 lifted.push((*s * 0.95, *pi, t));
1256 }
1257 }
1258 }
1259 if !lifted.is_empty() {
1260 let mut have: HashMap<(usize, usize), usize> = scored.iter().enumerate().map(|(i, (_, pi, ei))| ((*pi, *ei), i)).collect();
1261 for (s, pi, name) in lifted {
1262 for (ei, e) in ready[pi].0.entries.iter().enumerate() {
1263 if e.kind == Kind::Prose || e.kind == Kind::Alias || e.name != name || e.legacy {
1264 continue;
1265 }
1266 match have.get(&(pi, ei)) {
1267 Some(&i) => scored[i].0 = scored[i].0.max(s),
1268 None => {
1269 have.insert((pi, ei), scored.len());
1270 scored.push((s, pi, ei));
1271 }
1272 }
1273 }
1274 }
1275 scored.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));
1276 }
1277 scored
1278}
1279
1280pub fn redirects(text: &str) -> Vec<String> {
1283 let lower = text.to_ascii_lowercase();
1284 let mut out = Vec::new();
1285 for pat in [
1286 "use ",
1287 "consider ",
1288 "renamed to ",
1289 "replaced by ",
1290 "in favor of ",
1291 "in favour of ",
1292 "instead use ",
1293 ] {
1294 let mut from = 0;
1295 while let Some(i) = lower[from..].find(pat) {
1296 let at = from + i + pat.len();
1297 from = at;
1298 let rest = &text[at..];
1299 let rest = rest.trim_start_matches(['`', '\'', '"', '*', ' ']);
1300 let ident: String = rest.chars().take_while(|c| c.is_alphanumeric() || matches!(c, '_' | '.' | ':' | '$')).collect();
1301 let ident = ident.trim_end_matches(['.', ':']);
1302 let quoted = text[at..].starts_with('`');
1304 let looks = ident.contains(['_', '.', '$']) || ident.chars().skip(1).any(|c| c.is_ascii_uppercase());
1305 if ident.len() >= 3 && (quoted || looks) {
1306 let last = ident.rsplit(['.', ':']).next().unwrap_or(ident).to_string();
1307 if last.len() >= 3 && !out.contains(&last) {
1308 out.push(last);
1309 }
1310 }
1311 }
1312 }
1313 out
1314}
1315
1316fn heading_terms(h: &str) -> Vec<String> {
1320 let mut out = Vec::new();
1321 for w in h.split(|c: char| !(c.is_ascii_alphanumeric() || c == '_')).filter(|w| !w.is_empty()) {
1322 if w.starts_with(|c: char| c.is_ascii_uppercase()) && !w.contains('_') {
1323 out.push(bm25::stem(&w.to_ascii_lowercase()));
1324 } else {
1325 out.extend(bm25::terms(w));
1326 }
1327 }
1328 out
1329}
1330
1331fn name_hit(e: &Entry, qterms: &[String]) -> f32 {
1333 let words = if e.kind == Kind::Prose {
1334 let last = e.name.rsplit(" › ").next().unwrap_or("");
1335 if crate::markdown::generic_heading(last) {
1336 Vec::new()
1337 } else {
1338 heading_terms(last)
1339 }
1340 } else {
1341 bm25::terms(&e.name)
1342 };
1343 let n = qterms.iter().filter(|q| words.contains(q)).count();
1344 match n {
1345 0 => 1.0,
1346 1 => 1.5,
1347 _ => 1.9,
1348 }
1349}
1350
1351fn major_of(version: &str) -> u64 {
1352 version.trim_start_matches('v').split('.').next().and_then(|m| m.parse().ok()).unwrap_or(0)
1353}
1354
1355fn old_upgrade_guide(e: &Entry, major: u64) -> bool {
1359 let title = e.name.split(" › ").next().unwrap_or("");
1360 let stem = e.file.rsplit('/').next().unwrap_or("");
1361 [title, stem].iter().any(|t| {
1362 let l = t.to_ascii_lowercase();
1363 (l.contains("migrat") || l.contains("upgrad"))
1364 && l.split(|c: char| !c.is_ascii_digit())
1365 .find(|d| !d.is_empty() && d.len() <= 3)
1366 .and_then(|d| d.parse::<u64>().ok())
1367 .is_some_and(|v| v < major)
1368 })
1369}
1370
1371fn deprecated_page(e: &Entry) -> bool {
1373 e.name.split(" › ").next().unwrap_or("").to_ascii_lowercase().contains("(deprecated)")
1374}
1375
1376fn boost(e: &Entry, idents: &[String], changes: bool, major: u64) -> f32 {
1377 let mut b = 1.0;
1378 if e.kind == Kind::Prose && e.file.starts_with("upstream:") {
1381 b *= std::env::var("LOCKDOCS_UPSTREAM_BOOST").ok().and_then(|v| v.parse().ok()).unwrap_or(1.35);
1382 }
1383 if e.kind == Kind::Prose {
1384 let last = e
1386 .name
1387 .rsplit(" › ")
1388 .next()
1389 .unwrap_or("")
1390 .trim()
1391 .trim_end_matches("()")
1392 .trim_end_matches('!')
1393 .to_ascii_lowercase();
1394 if !last.is_empty() && idents.contains(&last) {
1395 b *= 1.4;
1396 }
1397 if !changes && old_upgrade_guide(e, major) {
1398 b *= 0.5;
1399 }
1400 if deprecated_page(e) {
1401 b *= 0.6;
1402 }
1403 if is_changelog(e) {
1404 b *= if changes { 1.5 } else { 0.5 };
1405 } else if e.file.to_ascii_uppercase().starts_with("README") {
1406 b *= 1.15;
1407 }
1408 } else {
1409 let n = e.name.trim_end_matches('!').to_ascii_lowercase();
1410 let p = e.path.to_ascii_lowercase();
1411 if idents.contains(&n) {
1412 b *= 1.8;
1413 }
1414 if idents
1415 .iter()
1416 .any(|i| i.len() > n.len() && (p.ends_with(i.as_str()) || i.ends_with(&format!(".{n}"))))
1417 {
1418 b *= 1.4;
1419 }
1420 if e.doc.is_empty() {
1421 b *= 0.6;
1422 }
1423 if e.kind == Kind::Alias {
1424 b *= 0.6;
1425 }
1426 }
1427 if private_path(e) {
1428 b *= 0.6;
1429 }
1430 if e.legacy {
1431 b *= 0.35;
1432 }
1433 if tooling_path(e) {
1434 b *= 0.5;
1435 }
1436 b
1437}
1438
1439fn api_score(e: &Entry, exact: bool, quals: &[String]) -> f32 {
1440 let mut s = if exact { 10.0 } else { 7.0 };
1441 let segs: Vec<String> = split_symbol(&e.path).into_iter().map(|x| x.to_ascii_lowercase()).collect();
1442 if quals.is_empty() {
1444 s += if matches!(e.kind, Kind::Method | Kind::Field) { -1.5 } else { 1.5 };
1445 }
1446 let file = e.file.to_ascii_lowercase();
1447 for q in quals {
1448 if segs.iter().any(|x| x == q) {
1449 s += 3.0;
1450 } else if file.contains(q.as_str()) {
1451 s += 1.0;
1452 }
1453 }
1454 if let (Some(q), Some(parent)) = (quals.last(), segs.len().checked_sub(2).map(|i| &segs[i])) {
1456 if q == parent {
1457 s += 2.0;
1458 }
1459 }
1460 if !e.doc.is_empty() {
1461 s += 2.0;
1462 }
1463 if private_path(e) {
1464 s -= 3.0;
1465 }
1466 if e.legacy {
1467 s -= 5.0;
1468 }
1469 if tooling_path(e) {
1470 s -= 3.0;
1471 }
1472 if e.kind == Kind::Alias {
1473 s -= 0.5;
1474 }
1475 s -= 0.2 * segs.len() as f32;
1476 s
1477}
1478
1479fn resolve_alias<'a>(idx: &'a PackageIndex, e: &'a Entry) -> &'a Entry {
1480 let Some(target) = &e.alias_of else { return e };
1481 idx.entries
1482 .iter()
1483 .filter(|x| &x.name == target && x.kind != Kind::Alias && x.kind != Kind::Prose)
1484 .max_by_key(|x| (x.file == e.file, !x.doc.is_empty()))
1485 .unwrap_or(e)
1486}
1487
1488fn truncate(s: &str, n: usize) -> String {
1489 if s.chars().count() <= n {
1490 return s.to_string();
1491 }
1492 let t: String = s.chars().take(n).collect();
1493 format!("{t}…")
1494}
1495
1496pub fn render_entry(idx: &PackageIndex, e: &Entry, max_doc: usize) -> String {
1498 let cite = format!("{} {}:{}", idx.id(), e.file, e.line);
1499 if e.kind == Kind::Prose {
1500 return format!("### {} — {}\n{}\n", e.name, cite, truncate(&e.doc, max_doc));
1501 }
1502 let mut s = format!("### {} ({}) — {}\n```{}\n{}\n```\n", e.path, e.kind.as_str(), cite, lang(idx.eco), e.sig);
1503 if !e.doc.is_empty() {
1504 s.push_str(&truncate(&e.doc, max_doc));
1505 s.push('\n');
1506 }
1507 s
1508}
1509
1510fn lead(doc: &str) -> String {
1514 let tag_only = |l: &str| l.starts_with('<') && l.ends_with('>');
1515 let mut lines = doc.lines().map(str::trim_end).peekable();
1516 let mut para: Vec<&str> = Vec::new();
1517 for l in lines.by_ref() {
1518 let t = l.trim();
1519 if t.starts_with("```") || t.starts_with("~~~") {
1520 return String::new();
1521 }
1522 if t.is_empty() || tag_only(t) {
1523 if para.is_empty() {
1524 continue;
1525 }
1526 break;
1527 }
1528 para.push(t);
1529 }
1530 let mut out = truncate(¶.join(" "), 320);
1531 while lines.peek().is_some_and(|l| l.trim().is_empty()) {
1532 lines.next();
1533 }
1534 let bullet = |l: &str| {
1535 let t = l.trim_start();
1536 t.starts_with("* ") || t.starts_with("- ") || t.split_once(". ").is_some_and(|(n, _)| !n.is_empty() && n.chars().all(|c| c.is_ascii_digit()))
1537 };
1538 if lines.peek().is_some_and(|l| bullet(l)) {
1539 let mut list = String::new();
1540 for l in lines.by_ref() {
1541 if l.trim().is_empty() {
1542 break;
1543 }
1544 let t = l.trim();
1545 list.push_str(if bullet(l) { "\n" } else { " " });
1546 list.push_str(t);
1547 }
1548 out.push_str(&truncate(&list, 400));
1549 } else if lines.peek().is_some_and(|l| l.trim_start().starts_with("```")) {
1550 let mut code = vec![lines.next().unwrap_or_default()];
1551 for l in lines.by_ref() {
1552 code.push(l);
1553 if l.trim_start().starts_with("```") {
1554 break;
1555 }
1556 }
1557 let code = code.join("\n");
1558 if code.len() <= 300 && code.matches("```").count() == 2 {
1559 out.push('\n');
1560 out.push_str(&code);
1561 }
1562 }
1563 out
1564}
1565
1566fn section_context(idx: &PackageIndex, e: &Entry) -> Option<String> {
1570 if e.kind != Kind::Prose {
1571 return None;
1572 }
1573 let parts: Vec<&str> = e.name.split(" › ").collect();
1574 if parts.len() < 2 {
1575 return None;
1576 }
1577 let mut wanted = vec![parts[0].to_string()];
1578 if parts.len() > 2 {
1579 wanted.push(parts[..parts.len() - 1].join(" › "));
1580 }
1581 let mut out = String::new();
1582 for name in wanted {
1583 let Some(p) = idx
1584 .entries
1585 .iter()
1586 .filter(|x| x.kind == Kind::Prose && x.file == e.file && x.name == name)
1587 .min_by_key(|x| x.line)
1588 else {
1589 continue;
1590 };
1591 let l = lead(&p.doc);
1592 if l.len() < 20 || e.doc.contains(&l) || out.contains(&l) {
1593 continue;
1594 }
1595 for line in format!("{}: {l}", p.name.rsplit(" › ").next().unwrap_or("")).lines() {
1596 out.push_str("> ");
1597 out.push_str(line);
1598 out.push('\n');
1599 }
1600 }
1601 (!out.is_empty()).then(|| out + "\n")
1602}
1603
1604struct Pack {
1606 text: String,
1607 budget: usize,
1608 blocks: usize,
1609 full: bool,
1610}
1611
1612impl Pack {
1613 fn new(budget: usize) -> Pack {
1614 Pack {
1615 text: String::new(),
1616 budget,
1617 blocks: 0,
1618 full: false,
1619 }
1620 }
1621 fn used(&self) -> usize {
1622 est_tokens(&self.text)
1623 }
1624 fn push_raw(&mut self, s: &str) {
1625 self.text.push_str(s);
1626 }
1627 fn prepend(&mut self, s: &str) {
1628 self.text.insert_str(0, s);
1629 }
1630 fn push_block(&mut self, b: &str) -> bool {
1632 if self.full {
1633 return false;
1634 }
1635 let need = est_tokens(b) + 1;
1636 let left = self.budget.saturating_sub(self.used());
1637 if need <= left {
1638 self.text.push('\n');
1639 self.text.push_str(b);
1640 self.blocks += 1;
1641 return true;
1642 }
1643 if left > 120 {
1644 let max_chars = left * 3;
1646 let mut cut = String::new();
1647 for line in b.lines() {
1648 if cut.len() + line.len() + 1 > max_chars {
1649 break;
1650 }
1651 cut.push_str(line);
1652 cut.push('\n');
1653 }
1654 if cut.matches("```").count() % 2 == 1 {
1655 cut.push_str("```\n");
1656 }
1657 if cut.lines().count() > 1 {
1658 self.text.push('\n');
1659 self.text.push_str(&cut);
1660 self.text.push_str("[…truncated to fit the token budget]\n");
1661 self.blocks += 1;
1662 }
1663 }
1664 self.full = true;
1665 false
1666 }
1667}
1668
1669#[derive(Default)]
1671pub struct Workspace {
1672 engines: Mutex<HashMap<PathBuf, (u64, Arc<Engine>)>>,
1673}
1674
1675fn lock_stamp(root: &Path) -> u64 {
1676 let mut h: u64 = 0;
1678 if let Ok(rd) = std::fs::read_dir(root) {
1679 for e in rd.flatten() {
1680 let n = e.file_name().to_string_lossy().to_string();
1681 if crate::lockfile::LOCKFILES.iter().any(|(f, _)| *f == n) || crate::lockfile::is_requirements(&n) || n == "package.json" || n == "pyproject.toml" {
1682 if let Ok(m) = e.metadata() {
1683 let t = m
1684 .modified()
1685 .ok()
1686 .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
1687 .map_or(0, |d| d.as_nanos() as u64);
1688 h = h.wrapping_mul(31).wrapping_add(t ^ m.len());
1689 }
1690 }
1691 }
1692 }
1693 h
1694}
1695
1696impl Workspace {
1697 pub fn engine(&self, root: &Path, opts: &Options) -> Arc<Engine> {
1698 let root = root.canonicalize().unwrap_or_else(|_| root.to_path_buf());
1699 let stamp = lock_stamp(&root);
1700 let mut m = self.engines.lock().unwrap();
1701 if let Some((s, e)) = m.get(&root) {
1702 if *s == stamp && e.opts == *opts {
1703 return e.clone();
1704 }
1705 }
1706 let e = Arc::new(Engine::new(&root, opts.clone()));
1707 m.insert(root, (stamp, e.clone()));
1708 e
1709 }
1710}
1711
1712pub fn dep_key(d: &Dep) -> String {
1714 format!("{}:{}", d.eco, norm_name(d.eco, &d.name))
1715}
1716
1717#[cfg(test)]
1718mod tests {
1719 use super::*;
1720
1721 #[test]
1722 fn fetch_report_preserves_flat_counts_and_adds_full_provenance() {
1723 let dep = Dep {
1724 eco: Eco::Cargo,
1725 name: "axum".into(),
1726 version: "0.7.9".into(),
1727 direct: true,
1728 from: "Cargo.lock".into(),
1729 };
1730 let status: upstream::Manifest = serde_json::from_value(json!({
1731 "format": upstream::FORMAT, "repo": "github.com/tokio-rs/axum", "tag": "axum-v0.7.9",
1732 "commit": "0123456789abcdef0123456789abcdef01234567", "files": 20, "bytes": 100, "note": null,
1733 }))
1734 .unwrap();
1735 let report = fetch_report(&dep, &status);
1736 assert_eq!(report["files"], 20);
1737 assert_eq!(report["files"], report["upstream"]["files"]);
1738 assert_eq!(report["tag"], report["upstream"]["tag"]);
1739 assert_eq!(report["repo"], report["upstream"]["repo"]);
1740 assert!(report.get("note").is_some());
1741 }
1742
1743 #[test]
1744 fn upstream_is_default_with_explicit_opt_out() {
1745 assert!(automatic_upstream(None, false));
1746 assert!(automatic_upstream(Some("1"), false));
1747 for value in ["0", "false", "no"] {
1748 assert!(!automatic_upstream(Some(value), false));
1749 }
1750 assert!(!automatic_upstream(None, true));
1751 }
1752
1753 #[test]
1754 fn workspace_does_not_reuse_network_policy() {
1755 let ws = Workspace::default();
1756 let root = std::env::temp_dir();
1757 let online = Options { fetch: false, upstream: true };
1758 let offline = Options { fetch: false, upstream: false };
1759 let a = ws.engine(&root, &online);
1760 let b = ws.engine(&root, &offline);
1761 assert!(!Arc::ptr_eq(&a, &b));
1762 assert!(!b.opts.upstream);
1763 }
1764
1765 #[test]
1766 fn old_upgrade_guides_and_deprecated_pages() {
1767 let e = |name: &str, file: &str| Entry {
1768 kind: Kind::Prose,
1769 name: name.into(),
1770 path: name.into(),
1771 file: file.into(),
1772 line: 1,
1773 sig: String::new(),
1774 doc: String::new(),
1775 alias_of: None,
1776 legacy: false,
1777 };
1778 assert!(old_upgrade_guide(
1779 &e("Migrating to v6.0.0 › x", "upstream:docs/src/use/migrating-to-6.0.0.md"),
1780 9
1781 ));
1782 assert!(old_upgrade_guide(
1783 &e("Upgrade to Prisma ORM 4 › x", "upstream:docs/700-upgrading-to-prisma-4.mdx"),
1784 6
1785 ));
1786 assert!(!old_upgrade_guide(
1787 &e("Upgrade to Prisma ORM 6 › x", "upstream:docs/500-upgrading-to-prisma-6.mdx"),
1788 6
1789 ));
1790 assert!(!old_upgrade_guide(&e("Migration Guide › x", "upstream:docs/migration.md"), 2));
1791 assert!(deprecated_page(&e("Configure Language Options (Deprecated) › Globals", "x.md")));
1792 assert!(!heading_terms("Seeding with TypeScript or JavaScript").contains(&"type".to_string()));
1793 assert!(heading_terms("useActionState reference").contains(&"action".to_string()));
1794 let l = lead("Intro line.\n\n```css\n@import \"tailwindcss\";\n\n@custom-variant dark (&:where(.dark, .dark *));\n```\n\nMore.");
1795 assert!(l.contains("@custom-variant") && l.ends_with("```"), "{l}");
1796 let l = lead("Three helpers:\n\n* **`parse_obj`**: from a dict\n more.\n* `parse_raw`\n\nNext.");
1797 assert!(l.contains("parse_obj") && l.contains("parse_raw") && !l.contains("Next"), "{l}");
1798 assert!(lead("<Deprecated>\n\nIn React 19, it is no longer necessary.\n\n</Deprecated>").starts_with("In React 19"));
1799 }
1800
1801 #[test]
1802 fn deprecation_redirects() {
1803 assert_eq!(redirects("Consider `z.strictObject(A.shape)` instead"), vec!["strictObject"]);
1804 assert_eq!(
1805 redirects("The `parse_obj` method is deprecated; use `model_validate` instead."),
1806 vec!["model_validate"]
1807 );
1808 assert_eq!(redirects("@deprecated Use .extend instead"), vec!["extend"]);
1809 assert_eq!(redirects("You can use this to parse data"), Vec::<String>::new());
1810 assert_eq!(redirects("Renamed to OptionalFromRequestParts."), vec!["OptionalFromRequestParts"]);
1811 }
1812}