1use crate::bm25;
5use crate::embed::{self, Vec8};
6use crate::extract::{self, Entry, Kind};
7use crate::locate::{self, Source};
8use crate::upstream::Manifest;
9use crate::{cache, Dep, Eco};
10use serde::{Deserialize, Serialize};
11use std::path::{Path, PathBuf};
12use std::time::Instant;
13
14pub const FORMAT: u32 = 16;
16
17#[derive(Serialize, Deserialize)]
18pub struct PackageIndex {
19 pub format: u32,
20 pub eco: Eco,
21 pub name: String,
22 pub version: String,
24 pub source: String,
25 pub fetched: bool,
26 pub entries: Vec<Entry>,
27 pub npm_api: Option<crate::npm_api::PublicApi>,
28 pub terms: Vec<Vec<(String, u16)>>,
29 pub lens: Vec<u32>,
30 pub head: Vec<Vec<(String, u16)>>,
33 pub head_lens: Vec<u32>,
34 pub build_ms: u64,
35 pub upstream: Option<String>,
37 pub upstream_provenance: Option<Manifest>,
39 pub embed: String,
41 pub vecs: Vec<Vec8>,
43}
44
45impl PackageIndex {
46 pub fn api_entries(&self) -> &[Entry] {
47 self.npm_api.as_ref().map_or(self.entries.as_slice(), |api| api.entries.as_slice())
48 }
49
50 pub fn docs_indices(&self, import: Option<&str>) -> Vec<usize> {
51 let import = import.unwrap_or(&self.name);
52 let files = self.npm_api.as_ref().map(|api| api.search_files(import));
53 let compatible = self.npm_api.as_ref().map(|api| api.compatible_imports(import, &self.name));
54 let mut seen = std::collections::HashSet::new();
55 self.entries
56 .iter()
57 .enumerate()
58 .filter(|(_, e)| {
59 if e.kind == Kind::Prose {
60 compatible.as_ref().is_none_or(|imports| {
61 e.prose_imports
62 .iter()
63 .filter(|s| crate::markdown::import_base(s) == self.name)
64 .all(|s| imports.contains(s.as_str()))
65 })
66 } else {
67 files.as_ref().is_none_or(|files| files.contains(&e.file))
68 }
69 })
70 .filter(|(_, e)| seen.insert((e.path.as_str(), e.sig.as_str(), e.doc.as_str())))
73 .map(|(i, _)| i)
74 .collect()
75 }
76
77 pub fn id(&self) -> String {
78 format!("{}@{}", self.name, self.version)
79 }
80}
81
82fn split_code(doc: &str) -> (String, String) {
84 let mut prose = String::new();
85 let mut code = String::new();
86 let mut in_fence = false;
87 for l in doc.lines() {
88 let t = l.trim_start();
89 if t.starts_with("```") || t.starts_with("~~~") {
90 in_fence = !in_fence;
91 continue;
92 }
93 let dst = if in_fence || l.starts_with(" ") && !l.trim().is_empty() && prose.ends_with("\n\n") {
94 &mut code
95 } else {
96 &mut prose
97 };
98 dst.push_str(l);
99 dst.push('\n');
100 }
101 (prose, code)
102}
103
104pub fn summary(prose: &str) -> &str {
106 let p = prose.trim_start();
107 let end = [p.find(". "), p.find(".\n"), p.find("\n\n")]
108 .into_iter()
109 .flatten()
110 .min()
111 .unwrap_or(p.len())
112 .min(300);
113 let mut end = end.min(p.len());
114 while !p.is_char_boundary(end) {
115 end -= 1;
116 }
117 &p[..end]
118}
119
120pub fn entry_tf(e: &Entry) -> (Vec<(String, u16)>, u32) {
123 let (prose, code) = split_code(&e.doc);
124 if e.kind == Kind::Prose {
125 let head = e.name.split(" › ").skip(1).collect::<Vec<_>>().join(" ");
126 bm25::tf(&[(&head, 6), (&e.file, 1), (&prose, 2), (&code, 1)])
127 } else {
128 let summary = summary(&prose);
130 bm25::tf(&[(&e.name, 8), (&e.path, 2), (&e.sig, 1), (summary, 4), (&prose, 2), (&code, 1)])
131 }
132}
133
134pub fn head_tf(e: &Entry) -> (Vec<(String, u16)>, u32) {
136 let (prose, _) = split_code(&e.doc);
137 let first = summary(&prose);
138 if e.kind == Kind::Prose {
139 bm25::tf(&[(crate::markdown::topic_heading(&e.name), 2), (first, 1)])
140 } else {
141 bm25::tf(&[(&e.name, 2), (first, 1)])
142 }
143}
144
145fn stamp(src: &Source) -> String {
147 let probe: Vec<PathBuf> = match &src.metadata {
148 Some(m) => vec![m.clone(), m.with_file_name("RECORD")],
149 None => ["package.json", "Cargo.toml", "go.mod", ".lockdocs-complete"]
150 .iter()
151 .map(|f| src.dir.join(f))
152 .collect(),
153 };
154 let mut s = String::new();
155 for p in probe {
156 if let Ok(m) = std::fs::metadata(&p) {
157 let t = m
158 .modified()
159 .ok()
160 .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
161 .map_or(0, |d| d.as_secs());
162 s.push_str(&format!("{}:{t};", m.len()));
163 }
164 }
165 s
166}
167
168pub fn embed_text(e: &Entry) -> String {
171 let (prose, code) = split_code(&e.doc);
172 let mut body: String = prose.chars().take(600).collect();
173 if e.kind == Kind::Prose {
174 if body.trim().is_empty() {
177 body = code.chars().take(600).collect();
178 }
179 let head: Vec<&str> = e.name.split(" › ").collect();
180 let tail = head[head.len().saturating_sub(2)..].join(" ");
181 return format!("{tail}. {body}");
182 }
183 if body.trim().is_empty() {
184 body = e.sig.chars().take(200).collect();
185 }
186 let parent = e.path.rsplit(['.', ':']).nth(1).unwrap_or("");
187 format!("{} {parent}. {body}", e.name)
188}
189
190pub(crate) fn source_stamp(dep: &Dep, src: &Source, root: &Path) -> String {
191 npm_source_stamp(dep, src, types_pkg(dep, root).as_deref())
192}
193
194fn npm_source_stamp(dep: &Dep, src: &Source, types: Option<&Path>) -> String {
195 if dep.eco != Eco::Npm {
196 return stamp(src);
197 }
198 cache::hash(&[
199 &std::fs::read_to_string(src.dir.join("package.json")).unwrap_or_default(),
200 &extract::npm_stamp(src, types),
201 &types.and_then(|t| std::fs::read_to_string(t.join("package.json")).ok()).unwrap_or_default(),
202 ])
203}
204
205fn cache_path(dep: &Dep, src: &Source, types: Option<&Path>, up: Option<&(PathBuf, Manifest)>, embed: &str) -> PathBuf {
206 let up_key = up
207 .map(|(_, m)| {
208 format!(
209 "{}@{:?}:{}:{:?}:{:?}:{:?}:{}:{}",
210 m.repo, m.tag, m.files, m.site, m.commit, m.note, m.docs_sites_checked, m.format
211 )
212 })
213 .unwrap_or_default();
214 let key = cache::hash(&[
215 embed,
216 &up_key,
217 &FORMAT.to_string(),
218 dep.eco.as_str(),
219 &dep.name,
220 &src.version,
221 &src.dir.to_string_lossy(),
222 &npm_source_stamp(dep, src, types),
223 &types.map(|t| t.to_string_lossy().to_string()).unwrap_or_default(),
224 ]);
225 let safe = format!("{}@{}", dep.name, src.version).replace(['/', '\\', ':'], "+");
226 cache::dir().join("idx").join(dep.eco.as_str()).join(format!("{safe}-{key}.bin"))
227}
228
229fn types_pkg(dep: &Dep, root: &Path) -> Option<PathBuf> {
231 if dep.eco != Eco::Npm || dep.name.starts_with("@types/") {
232 return None;
233 }
234 let mangled = match dep.name.strip_prefix('@') {
235 Some(scoped) => scoped.replacen('/', "__", 1),
236 None => dep.name.clone(),
237 };
238 locate::npm_installed(&format!("@types/{mangled}"), root).map(|(d, _)| d)
239}
240
241pub fn build(dep: &Dep, src: &Source, root: &Path, up: Option<&(PathBuf, Manifest)>) -> PackageIndex {
242 let t = Instant::now();
243 let types = types_pkg(dep, root);
244 let up_dir = up.filter(|(_, m)| m.files > 0).map(|(d, _)| d.as_path());
245 let mut entries = extract::extract(dep.eco, &dep.name, src, types.as_deref(), up_dir);
246 let npm_api = (dep.eco == Eco::Npm).then(|| {
247 let sources = extract::npm_sources(src, types.as_deref());
248 if let Some(types) = &types {
251 let prefix = types
252 .iter()
253 .rev()
254 .take(2)
255 .collect::<Vec<_>>()
256 .into_iter()
257 .rev()
258 .map(|s| s.to_string_lossy())
259 .collect::<Vec<_>>()
260 .join("/")
261 + "/";
262 if sources.keys().any(|f| f.starts_with(&prefix)) {
263 let relative = sources
264 .iter()
265 .filter_map(|(f, s)| f.strip_prefix(&prefix).map(|f| (f.to_string(), s.clone())))
266 .collect();
267 return crate::npm_api::PublicApi::build(&dep.name, types, &relative).prefixed(&prefix);
268 }
269 }
270 crate::npm_api::PublicApi::build(&dep.name, &src.dir, &sources)
271 });
272 if let Some(api) = &npm_api {
273 entries.retain(|e| e.kind == Kind::Prose);
274 entries.extend(api.entries.iter().cloned());
277 }
278 let (terms, lens): (Vec<_>, Vec<_>) = entries.iter().map(entry_tf).unzip();
279 let (head, head_lens): (Vec<_>, Vec<_>) = entries.iter().map(head_tf).unzip();
280 let model = embed::get();
281 let vecs: Vec<Vec8> = match &model {
282 Some(m) => {
283 use rayon::prelude::*;
284 entries.par_iter().map(|e| m.embed8(&embed_text(e)).unwrap_or_default()).collect()
285 }
286 None => Vec::new(),
287 };
288 PackageIndex {
289 format: FORMAT,
290 eco: dep.eco,
291 name: dep.name.clone(),
292 version: src.version.clone(),
293 source: src.label.clone(),
294 fetched: src.fetched,
295 entries,
296 npm_api,
297 terms,
298 lens,
299 head,
300 head_lens,
301 build_ms: t.elapsed().as_millis() as u64,
302 upstream: up.filter(|(_, m)| m.files > 0).map(|(_, m)| {
303 format!(
304 "{}@{}{}{} ({} files)",
305 m.repo,
306 m.tag.as_deref().unwrap_or("?"),
307 m.commit.as_ref().map(|c| format!(" commit {c}")).unwrap_or_default(),
308 m.site.as_ref().map(|s| format!(" + docs site {s}")).unwrap_or_default(),
309 m.files
310 )
311 }),
312 upstream_provenance: up.map(|(_, m)| m.clone()),
313 embed: if model.is_some() { embed::MODEL_ID.to_string() } else { String::new() },
314 vecs,
315 }
316}
317
318pub fn load_or_build(dep: &Dep, src: &Source, root: &Path, up: Option<&(PathBuf, Manifest)>) -> PackageIndex {
320 let types = types_pkg(dep, root);
321 let embed_id = if embed::available() { embed::MODEL_ID } else { "" };
322 let path = cache_path(dep, src, types.as_deref(), up, embed_id);
323 if let Ok(bytes) = std::fs::read(&path) {
324 if let Ok(idx) = postcard::from_bytes::<PackageIndex>(&bytes) {
325 if idx.format == FORMAT {
326 return idx;
327 }
328 }
329 }
330 let idx = build(dep, src, root, up);
331 if let Ok(bytes) = postcard::to_stdvec(&idx) {
332 if let Some(dir) = path.parent() {
333 let _ = std::fs::create_dir_all(dir);
334 }
335 let tmp = path.with_extension(format!("tmp{}", std::process::id()));
336 if std::fs::write(&tmp, bytes).is_ok() {
337 let _ = std::fs::rename(&tmp, &path);
338 }
339 }
340 idx
341}