Skip to main content

lockdocs_core/
index.rs

1//! Per-package index: entries plus their term frequencies, cached on disk
2//! by package, exact version and source location.
3
4use crate::bm25;
5use crate::embed::{self, Vec8};
6use crate::extract::{self, Entry, Kind};
7use crate::locate::{self, Source};
8use crate::upstream::Manifest;
9use crate::{cache, Dep, Eco};
10use serde::{Deserialize, Serialize};
11use std::path::{Path, PathBuf};
12use std::time::Instant;
13
14/// Bump when extraction or the on-disk format changes.
15pub const FORMAT: u32 = 14;
16
17#[derive(Serialize, Deserialize)]
18pub struct PackageIndex {
19    pub format: u32,
20    pub eco: Eco,
21    pub name: String,
22    /// The version whose files were indexed.
23    pub version: String,
24    pub source: String,
25    pub fetched: bool,
26    pub entries: Vec<Entry>,
27    pub npm_api: Option<crate::npm_api::PublicApi>,
28    pub terms: Vec<Vec<(String, u16)>>,
29    pub lens: Vec<u32>,
30    /// Terms of the heading or name plus the first sentence only, ranked on
31    /// their own so a long body does not bury what the entry says it is.
32    pub head: Vec<Vec<(String, u16)>>,
33    pub head_lens: Vec<u32>,
34    pub build_ms: u64,
35    /// `github.com/o/r@tag (N files)` when upstream docs are included.
36    pub upstream: Option<String>,
37    /// Fetch manifest used by this index, including immutable commit and failure notes.
38    pub upstream_provenance: Option<Manifest>,
39    /// Embedding model id, or empty for keyword-only.
40    pub embed: String,
41    /// One embedding per entry (empty without a model).
42    pub vecs: Vec<Vec8>,
43}
44
45impl PackageIndex {
46    pub fn api_entries(&self) -> &[Entry] {
47        self.npm_api.as_ref().map_or(self.entries.as_slice(), |api| api.entries.as_slice())
48    }
49
50    pub fn docs_indices(&self, import: Option<&str>) -> Vec<usize> {
51        let files = self.npm_api.as_ref().map(|api| api.search_files(import.unwrap_or(&self.name)));
52        let mut seen = std::collections::HashSet::new();
53        self.entries
54            .iter()
55            .enumerate()
56            .filter(|(_, e)| e.kind == Kind::Prose || files.as_ref().is_none_or(|files| files.contains(&e.file)))
57            // Deduplicate only after import selection: an identical declaration
58            // in another edition must not erase the requested file identity.
59            .filter(|(_, e)| seen.insert((e.path.as_str(), e.sig.as_str(), e.doc.as_str())))
60            .map(|(i, _)| i)
61            .collect()
62    }
63
64    pub fn id(&self) -> String {
65        format!("{}@{}", self.name, self.version)
66    }
67}
68
69/// Split text into (prose, code) where code is inside ``` fences.
70fn split_code(doc: &str) -> (String, String) {
71    let mut prose = String::new();
72    let mut code = String::new();
73    let mut in_fence = false;
74    for l in doc.lines() {
75        let t = l.trim_start();
76        if t.starts_with("```") || t.starts_with("~~~") {
77            in_fence = !in_fence;
78            continue;
79        }
80        let dst = if in_fence || l.starts_with("    ") && !l.trim().is_empty() && prose.ends_with("\n\n") {
81            &mut code
82        } else {
83            &mut prose
84        };
85        dst.push_str(l);
86        dst.push('\n');
87    }
88    (prose, code)
89}
90
91/// The first sentence of a doc comment.
92pub fn summary(prose: &str) -> &str {
93    let p = prose.trim_start();
94    let end = [p.find(". "), p.find(".\n"), p.find("\n\n")]
95        .into_iter()
96        .flatten()
97        .min()
98        .unwrap_or(p.len())
99        .min(300);
100    let mut end = end.min(p.len());
101    while !p.is_char_boundary(end) {
102        end -= 1;
103    }
104    &p[..end]
105}
106
107/// Weighted terms: names and headings count most, doc prose next, code
108/// examples and signatures least (they repeat incidental identifiers).
109pub fn entry_tf(e: &Entry) -> (Vec<(String, u16)>, u32) {
110    let (prose, code) = split_code(&e.doc);
111    if e.kind == Kind::Prose {
112        let head = e.name.split(" › ").skip(1).collect::<Vec<_>>().join(" ");
113        bm25::tf(&[(&head, 6), (&e.file, 1), (&prose, 2), (&code, 1)])
114    } else {
115        // The first sentence says what the symbol is for: weigh it like a heading.
116        let summary = summary(&prose);
117        bm25::tf(&[(&e.name, 8), (&e.path, 2), (&e.sig, 1), (summary, 4), (&prose, 2), (&code, 1)])
118    }
119}
120
121/// Heading (or name) and first sentence: what an entry says it is about.
122pub fn head_tf(e: &Entry) -> (Vec<(String, u16)>, u32) {
123    let (prose, _) = split_code(&e.doc);
124    let first = summary(&prose);
125    if e.kind == Kind::Prose {
126        bm25::tf(&[(crate::markdown::topic_heading(&e.name), 2), (first, 1)])
127    } else {
128        bm25::tf(&[(&e.name, 2), (first, 1)])
129    }
130}
131
132/// Files whose change means the package was modified in place.
133fn stamp(src: &Source) -> String {
134    let probe: Vec<PathBuf> = match &src.metadata {
135        Some(m) => vec![m.clone(), m.with_file_name("RECORD")],
136        None => ["package.json", "Cargo.toml", "go.mod", ".lockdocs-complete"]
137            .iter()
138            .map(|f| src.dir.join(f))
139            .collect(),
140    };
141    let mut s = String::new();
142    for p in probe {
143        if let Ok(m) = std::fs::metadata(&p) {
144            let t = m
145                .modified()
146                .ok()
147                .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
148                .map_or(0, |d| d.as_secs());
149            s.push_str(&format!("{}:{t};", m.len()));
150        }
151    }
152    s
153}
154
155/// What an entry means, for the embedding: its name and the start of its
156/// prose (code examples and long signatures dilute a mean-pooled vector).
157pub fn embed_text(e: &Entry) -> String {
158    let (prose, code) = split_code(&e.doc);
159    let mut body: String = prose.chars().take(600).collect();
160    if e.kind == Kind::Prose {
161        // A code-only section embedded by its title alone matches short
162        // questions far too well; its code says what it is about.
163        if body.trim().is_empty() {
164            body = code.chars().take(600).collect();
165        }
166        let head: Vec<&str> = e.name.split(" › ").collect();
167        let tail = head[head.len().saturating_sub(2)..].join(" ");
168        return format!("{tail}. {body}");
169    }
170    if body.trim().is_empty() {
171        body = e.sig.chars().take(200).collect();
172    }
173    let parent = e.path.rsplit(['.', ':']).nth(1).unwrap_or("");
174    format!("{} {parent}. {body}", e.name)
175}
176
177pub(crate) fn source_stamp(dep: &Dep, src: &Source, root: &Path) -> String {
178    npm_source_stamp(dep, src, types_pkg(dep, root).as_deref())
179}
180
181fn npm_source_stamp(dep: &Dep, src: &Source, types: Option<&Path>) -> String {
182    if dep.eco != Eco::Npm {
183        return stamp(src);
184    }
185    cache::hash(&[
186        &std::fs::read_to_string(src.dir.join("package.json")).unwrap_or_default(),
187        &extract::npm_stamp(src, types),
188        &types.and_then(|t| std::fs::read_to_string(t.join("package.json")).ok()).unwrap_or_default(),
189    ])
190}
191
192fn cache_path(dep: &Dep, src: &Source, types: Option<&Path>, up: Option<&(PathBuf, Manifest)>, embed: &str) -> PathBuf {
193    let up_key = up
194        .map(|(_, m)| {
195            format!(
196                "{}@{:?}:{}:{:?}:{:?}:{:?}:{}:{}",
197                m.repo, m.tag, m.files, m.site, m.commit, m.note, m.docs_sites_checked, m.format
198            )
199        })
200        .unwrap_or_default();
201    let key = cache::hash(&[
202        embed,
203        &up_key,
204        &FORMAT.to_string(),
205        dep.eco.as_str(),
206        &dep.name,
207        &src.version,
208        &src.dir.to_string_lossy(),
209        &npm_source_stamp(dep, src, types),
210        &types.map(|t| t.to_string_lossy().to_string()).unwrap_or_default(),
211    ]);
212    let safe = format!("{}@{}", dep.name, src.version).replace(['/', '\\', ':'], "+");
213    cache::dir().join("idx").join(dep.eco.as_str()).join(format!("{safe}-{key}.bin"))
214}
215
216/// `@types/<name>` next to an npm package that ships no declarations.
217fn types_pkg(dep: &Dep, root: &Path) -> Option<PathBuf> {
218    if dep.eco != Eco::Npm || dep.name.starts_with("@types/") {
219        return None;
220    }
221    let mangled = match dep.name.strip_prefix('@') {
222        Some(scoped) => scoped.replacen('/', "__", 1),
223        None => dep.name.clone(),
224    };
225    locate::npm_installed(&format!("@types/{mangled}"), root).map(|(d, _)| d)
226}
227
228pub fn build(dep: &Dep, src: &Source, root: &Path, up: Option<&(PathBuf, Manifest)>) -> PackageIndex {
229    let t = Instant::now();
230    let types = types_pkg(dep, root);
231    let up_dir = up.filter(|(_, m)| m.files > 0).map(|(d, _)| d.as_path());
232    let mut entries = extract::extract(dep.eco, &dep.name, src, types.as_deref(), up_dir);
233    let npm_api = (dep.eco == Eco::Npm).then(|| {
234        let sources = extract::npm_sources(src, types.as_deref());
235        // Separate @types packages keep their real citation paths, but their
236        // own package.json supplies the declaration entrypoint.
237        if let Some(types) = &types {
238            let prefix = types
239                .iter()
240                .rev()
241                .take(2)
242                .collect::<Vec<_>>()
243                .into_iter()
244                .rev()
245                .map(|s| s.to_string_lossy())
246                .collect::<Vec<_>>()
247                .join("/")
248                + "/";
249            if sources.keys().any(|f| f.starts_with(&prefix)) {
250                let relative = sources
251                    .iter()
252                    .filter_map(|(f, s)| f.strip_prefix(&prefix).map(|f| (f.to_string(), s.clone())))
253                    .collect();
254                return crate::npm_api::PublicApi::build(&dep.name, types, &relative).prefixed(&prefix);
255            }
256        }
257        crate::npm_api::PublicApi::build(&dep.name, &src.dir, &sources)
258    });
259    if let Some(api) = &npm_api {
260        entries.retain(|e| e.kind == Kind::Prose);
261        // Keep exact declaration identities, including .d.cts/.d.mts. The
262        // requested import selects and deduplicates its own search corpus.
263        entries.extend(api.entries.iter().cloned());
264    }
265    let (terms, lens): (Vec<_>, Vec<_>) = entries.iter().map(entry_tf).unzip();
266    let (head, head_lens): (Vec<_>, Vec<_>) = entries.iter().map(head_tf).unzip();
267    let model = embed::get();
268    let vecs: Vec<Vec8> = match &model {
269        Some(m) => {
270            use rayon::prelude::*;
271            entries.par_iter().map(|e| m.embed8(&embed_text(e)).unwrap_or_default()).collect()
272        }
273        None => Vec::new(),
274    };
275    PackageIndex {
276        format: FORMAT,
277        eco: dep.eco,
278        name: dep.name.clone(),
279        version: src.version.clone(),
280        source: src.label.clone(),
281        fetched: src.fetched,
282        entries,
283        npm_api,
284        terms,
285        lens,
286        head,
287        head_lens,
288        build_ms: t.elapsed().as_millis() as u64,
289        upstream: up.filter(|(_, m)| m.files > 0).map(|(_, m)| {
290            format!(
291                "{}@{}{}{} ({} files)",
292                m.repo,
293                m.tag.as_deref().unwrap_or("?"),
294                m.commit.as_ref().map(|c| format!(" commit {c}")).unwrap_or_default(),
295                m.site.as_ref().map(|s| format!(" + docs site {s}")).unwrap_or_default(),
296                m.files
297            )
298        }),
299        upstream_provenance: up.map(|(_, m)| m.clone()),
300        embed: if model.is_some() { embed::MODEL_ID.to_string() } else { String::new() },
301        vecs,
302    }
303}
304
305/// Load from the disk cache, or build and store.
306pub fn load_or_build(dep: &Dep, src: &Source, root: &Path, up: Option<&(PathBuf, Manifest)>) -> PackageIndex {
307    let types = types_pkg(dep, root);
308    let embed_id = if embed::available() { embed::MODEL_ID } else { "" };
309    let path = cache_path(dep, src, types.as_deref(), up, embed_id);
310    if let Ok(bytes) = std::fs::read(&path) {
311        if let Ok(idx) = postcard::from_bytes::<PackageIndex>(&bytes) {
312            if idx.format == FORMAT {
313                return idx;
314            }
315        }
316    }
317    let idx = build(dep, src, root, up);
318    if let Ok(bytes) = postcard::to_stdvec(&idx) {
319        if let Some(dir) = path.parent() {
320            let _ = std::fs::create_dir_all(dir);
321        }
322        let tmp = path.with_extension(format!("tmp{}", std::process::id()));
323        if std::fs::write(&tmp, bytes).is_ok() {
324            let _ = std::fs::rename(&tmp, &path);
325        }
326    }
327    idx
328}