Skip to main content

lockdocs_core/
index.rs

1//! Per-package index: entries plus their term frequencies, cached on disk
2//! by package, exact version and source location.
3
4use crate::bm25;
5use crate::embed::{self, Vec8};
6use crate::extract::{self, Entry, Kind};
7use crate::locate::{self, Source};
8use crate::upstream::Manifest;
9use crate::{cache, Dep, Eco};
10use serde::{Deserialize, Serialize};
11use std::path::{Path, PathBuf};
12use std::time::Instant;
13
14/// Bump when extraction or the on-disk format changes.
15pub const FORMAT: u32 = 11;
16
17#[derive(Serialize, Deserialize)]
18pub struct PackageIndex {
19    pub format: u32,
20    pub eco: Eco,
21    pub name: String,
22    /// The version whose files were indexed.
23    pub version: String,
24    pub source: String,
25    pub fetched: bool,
26    pub entries: Vec<Entry>,
27    pub terms: Vec<Vec<(String, u16)>>,
28    pub lens: Vec<u32>,
29    /// Terms of the heading or name plus the first sentence only, ranked on
30    /// their own so a long body does not bury what the entry says it is.
31    pub head: Vec<Vec<(String, u16)>>,
32    pub head_lens: Vec<u32>,
33    pub build_ms: u64,
34    /// `github.com/o/r@tag (N files)` when upstream docs are included.
35    pub upstream: Option<String>,
36    /// Fetch manifest used by this index, including immutable commit and failure notes.
37    pub upstream_provenance: Option<Manifest>,
38    /// Embedding model id, or empty for keyword-only.
39    pub embed: String,
40    /// One embedding per entry (empty without a model).
41    pub vecs: Vec<Vec8>,
42}
43
44impl PackageIndex {
45    pub fn id(&self) -> String {
46        format!("{}@{}", self.name, self.version)
47    }
48}
49
50/// Split text into (prose, code) where code is inside ``` fences.
51fn split_code(doc: &str) -> (String, String) {
52    let mut prose = String::new();
53    let mut code = String::new();
54    let mut in_fence = false;
55    for l in doc.lines() {
56        let t = l.trim_start();
57        if t.starts_with("```") || t.starts_with("~~~") {
58            in_fence = !in_fence;
59            continue;
60        }
61        let dst = if in_fence || l.starts_with("    ") && !l.trim().is_empty() && prose.ends_with("\n\n") {
62            &mut code
63        } else {
64            &mut prose
65        };
66        dst.push_str(l);
67        dst.push('\n');
68    }
69    (prose, code)
70}
71
72/// The first sentence of a doc comment.
73pub fn summary(prose: &str) -> &str {
74    let p = prose.trim_start();
75    let end = [p.find(". "), p.find(".\n"), p.find("\n\n")]
76        .into_iter()
77        .flatten()
78        .min()
79        .unwrap_or(p.len())
80        .min(300);
81    let mut end = end.min(p.len());
82    while !p.is_char_boundary(end) {
83        end -= 1;
84    }
85    &p[..end]
86}
87
88/// Weighted terms: names and headings count most, doc prose next, code
89/// examples and signatures least (they repeat incidental identifiers).
90pub fn entry_tf(e: &Entry) -> (Vec<(String, u16)>, u32) {
91    let (prose, code) = split_code(&e.doc);
92    if e.kind == Kind::Prose {
93        let head = e.name.split(" › ").skip(1).collect::<Vec<_>>().join(" ");
94        bm25::tf(&[(&head, 6), (&e.file, 1), (&prose, 2), (&code, 1)])
95    } else {
96        // The first sentence says what the symbol is for: weigh it like a heading.
97        let summary = summary(&prose);
98        bm25::tf(&[(&e.name, 8), (&e.path, 2), (&e.sig, 1), (summary, 4), (&prose, 2), (&code, 1)])
99    }
100}
101
102/// Heading (or name) and first sentence: what an entry says it is about.
103pub fn head_tf(e: &Entry) -> (Vec<(String, u16)>, u32) {
104    let (prose, _) = split_code(&e.doc);
105    let first = summary(&prose);
106    if e.kind == Kind::Prose {
107        bm25::tf(&[(crate::markdown::topic_heading(&e.name), 2), (first, 1)])
108    } else {
109        bm25::tf(&[(&e.name, 2), (first, 1)])
110    }
111}
112
113/// Files whose change means the package was modified in place.
114fn stamp(src: &Source) -> String {
115    let probe: Vec<PathBuf> = match &src.metadata {
116        Some(m) => vec![m.clone(), m.with_file_name("RECORD")],
117        None => ["package.json", "Cargo.toml", "go.mod", ".lockdocs-complete"]
118            .iter()
119            .map(|f| src.dir.join(f))
120            .collect(),
121    };
122    let mut s = String::new();
123    for p in probe {
124        if let Ok(m) = std::fs::metadata(&p) {
125            let t = m
126                .modified()
127                .ok()
128                .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
129                .map_or(0, |d| d.as_secs());
130            s.push_str(&format!("{}:{t};", m.len()));
131        }
132    }
133    s
134}
135
136/// What an entry means, for the embedding: its name and the start of its
137/// prose (code examples and long signatures dilute a mean-pooled vector).
138pub fn embed_text(e: &Entry) -> String {
139    let (prose, code) = split_code(&e.doc);
140    let mut body: String = prose.chars().take(600).collect();
141    if e.kind == Kind::Prose {
142        // A code-only section embedded by its title alone matches short
143        // questions far too well; its code says what it is about.
144        if body.trim().is_empty() {
145            body = code.chars().take(600).collect();
146        }
147        let head: Vec<&str> = e.name.split(" › ").collect();
148        let tail = head[head.len().saturating_sub(2)..].join(" ");
149        return format!("{tail}. {body}");
150    }
151    if body.trim().is_empty() {
152        body = e.sig.chars().take(200).collect();
153    }
154    let parent = e.path.rsplit(['.', ':']).nth(1).unwrap_or("");
155    format!("{} {parent}. {body}", e.name)
156}
157
158fn cache_path(dep: &Dep, src: &Source, types: Option<&Path>, up: Option<&(PathBuf, Manifest)>, embed: &str) -> PathBuf {
159    let up_key = up
160        .map(|(_, m)| {
161            format!(
162                "{}@{:?}:{}:{:?}:{:?}:{:?}:{}:{}",
163                m.repo, m.tag, m.files, m.site, m.commit, m.note, m.docs_sites_checked, m.format
164            )
165        })
166        .unwrap_or_default();
167    let key = cache::hash(&[
168        embed,
169        &up_key,
170        &FORMAT.to_string(),
171        dep.eco.as_str(),
172        &dep.name,
173        &src.version,
174        &src.dir.to_string_lossy(),
175        &stamp(src),
176        &types.map(|t| t.to_string_lossy().to_string()).unwrap_or_default(),
177    ]);
178    let safe = format!("{}@{}", dep.name, src.version).replace(['/', '\\', ':'], "+");
179    cache::dir().join("idx").join(dep.eco.as_str()).join(format!("{safe}-{key}.bin"))
180}
181
182/// `@types/<name>` next to an npm package that ships no declarations.
183fn types_pkg(dep: &Dep, root: &Path) -> Option<PathBuf> {
184    if dep.eco != Eco::Npm || dep.name.starts_with("@types/") {
185        return None;
186    }
187    let mangled = match dep.name.strip_prefix('@') {
188        Some(scoped) => scoped.replacen('/', "__", 1),
189        None => dep.name.clone(),
190    };
191    locate::npm_installed(&format!("@types/{mangled}"), root).map(|(d, _)| d)
192}
193
194pub fn build(dep: &Dep, src: &Source, root: &Path, up: Option<&(PathBuf, Manifest)>) -> PackageIndex {
195    let t = Instant::now();
196    let types = types_pkg(dep, root);
197    let up_dir = up.filter(|(_, m)| m.files > 0).map(|(d, _)| d.as_path());
198    let entries = extract::extract(dep.eco, &dep.name, src, types.as_deref(), up_dir);
199    let (terms, lens): (Vec<_>, Vec<_>) = entries.iter().map(entry_tf).unzip();
200    let (head, head_lens): (Vec<_>, Vec<_>) = entries.iter().map(head_tf).unzip();
201    let model = embed::get();
202    let vecs: Vec<Vec8> = match &model {
203        Some(m) => {
204            use rayon::prelude::*;
205            entries.par_iter().map(|e| m.embed8(&embed_text(e)).unwrap_or_default()).collect()
206        }
207        None => Vec::new(),
208    };
209    PackageIndex {
210        format: FORMAT,
211        eco: dep.eco,
212        name: dep.name.clone(),
213        version: src.version.clone(),
214        source: src.label.clone(),
215        fetched: src.fetched,
216        entries,
217        terms,
218        lens,
219        head,
220        head_lens,
221        build_ms: t.elapsed().as_millis() as u64,
222        upstream: up.filter(|(_, m)| m.files > 0).map(|(_, m)| {
223            format!(
224                "{}@{}{}{} ({} files)",
225                m.repo,
226                m.tag.as_deref().unwrap_or("?"),
227                m.commit.as_ref().map(|c| format!(" commit {c}")).unwrap_or_default(),
228                m.site.as_ref().map(|s| format!(" + docs site {s}")).unwrap_or_default(),
229                m.files
230            )
231        }),
232        upstream_provenance: up.map(|(_, m)| m.clone()),
233        embed: if model.is_some() { embed::MODEL_ID.to_string() } else { String::new() },
234        vecs,
235    }
236}
237
238/// Load from the disk cache, or build and store.
239pub fn load_or_build(dep: &Dep, src: &Source, root: &Path, up: Option<&(PathBuf, Manifest)>) -> PackageIndex {
240    let types = types_pkg(dep, root);
241    let embed_id = if embed::available() { embed::MODEL_ID } else { "" };
242    let path = cache_path(dep, src, types.as_deref(), up, embed_id);
243    if let Ok(bytes) = std::fs::read(&path) {
244        if let Ok(idx) = postcard::from_bytes::<PackageIndex>(&bytes) {
245            if idx.format == FORMAT {
246                return idx;
247            }
248        }
249    }
250    let idx = build(dep, src, root, up);
251    if let Ok(bytes) = postcard::to_stdvec(&idx) {
252        if let Some(dir) = path.parent() {
253            let _ = std::fs::create_dir_all(dir);
254        }
255        let tmp = path.with_extension(format!("tmp{}", std::process::id()));
256        if std::fs::write(&tmp, bytes).is_ok() {
257            let _ = std::fs::rename(&tmp, &path);
258        }
259    }
260    idx
261}