1use crate::bm25;
5use crate::embed::{self, Vec8};
6use crate::extract::{self, Entry, Kind};
7use crate::locate::{self, Source};
8use crate::upstream::Manifest;
9use crate::{cache, Dep, Eco};
10use serde::{Deserialize, Serialize};
11use std::path::{Path, PathBuf};
12use std::time::Instant;
13
14pub const FORMAT: u32 = 11;
16
17#[derive(Serialize, Deserialize)]
18pub struct PackageIndex {
19 pub format: u32,
20 pub eco: Eco,
21 pub name: String,
22 pub version: String,
24 pub source: String,
25 pub fetched: bool,
26 pub entries: Vec<Entry>,
27 pub terms: Vec<Vec<(String, u16)>>,
28 pub lens: Vec<u32>,
29 pub head: Vec<Vec<(String, u16)>>,
32 pub head_lens: Vec<u32>,
33 pub build_ms: u64,
34 pub upstream: Option<String>,
36 pub upstream_provenance: Option<Manifest>,
38 pub embed: String,
40 pub vecs: Vec<Vec8>,
42}
43
44impl PackageIndex {
45 pub fn id(&self) -> String {
46 format!("{}@{}", self.name, self.version)
47 }
48}
49
50fn split_code(doc: &str) -> (String, String) {
52 let mut prose = String::new();
53 let mut code = String::new();
54 let mut in_fence = false;
55 for l in doc.lines() {
56 let t = l.trim_start();
57 if t.starts_with("```") || t.starts_with("~~~") {
58 in_fence = !in_fence;
59 continue;
60 }
61 let dst = if in_fence || l.starts_with(" ") && !l.trim().is_empty() && prose.ends_with("\n\n") {
62 &mut code
63 } else {
64 &mut prose
65 };
66 dst.push_str(l);
67 dst.push('\n');
68 }
69 (prose, code)
70}
71
72pub fn summary(prose: &str) -> &str {
74 let p = prose.trim_start();
75 let end = [p.find(". "), p.find(".\n"), p.find("\n\n")]
76 .into_iter()
77 .flatten()
78 .min()
79 .unwrap_or(p.len())
80 .min(300);
81 let mut end = end.min(p.len());
82 while !p.is_char_boundary(end) {
83 end -= 1;
84 }
85 &p[..end]
86}
87
88pub fn entry_tf(e: &Entry) -> (Vec<(String, u16)>, u32) {
91 let (prose, code) = split_code(&e.doc);
92 if e.kind == Kind::Prose {
93 let head = e.name.split(" › ").skip(1).collect::<Vec<_>>().join(" ");
94 bm25::tf(&[(&head, 6), (&e.file, 1), (&prose, 2), (&code, 1)])
95 } else {
96 let summary = summary(&prose);
98 bm25::tf(&[(&e.name, 8), (&e.path, 2), (&e.sig, 1), (summary, 4), (&prose, 2), (&code, 1)])
99 }
100}
101
102pub fn head_tf(e: &Entry) -> (Vec<(String, u16)>, u32) {
104 let (prose, _) = split_code(&e.doc);
105 let first = summary(&prose);
106 if e.kind == Kind::Prose {
107 bm25::tf(&[(crate::markdown::topic_heading(&e.name), 2), (first, 1)])
108 } else {
109 bm25::tf(&[(&e.name, 2), (first, 1)])
110 }
111}
112
113fn stamp(src: &Source) -> String {
115 let probe: Vec<PathBuf> = match &src.metadata {
116 Some(m) => vec![m.clone(), m.with_file_name("RECORD")],
117 None => ["package.json", "Cargo.toml", "go.mod", ".lockdocs-complete"]
118 .iter()
119 .map(|f| src.dir.join(f))
120 .collect(),
121 };
122 let mut s = String::new();
123 for p in probe {
124 if let Ok(m) = std::fs::metadata(&p) {
125 let t = m
126 .modified()
127 .ok()
128 .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
129 .map_or(0, |d| d.as_secs());
130 s.push_str(&format!("{}:{t};", m.len()));
131 }
132 }
133 s
134}
135
136pub fn embed_text(e: &Entry) -> String {
139 let (prose, code) = split_code(&e.doc);
140 let mut body: String = prose.chars().take(600).collect();
141 if e.kind == Kind::Prose {
142 if body.trim().is_empty() {
145 body = code.chars().take(600).collect();
146 }
147 let head: Vec<&str> = e.name.split(" › ").collect();
148 let tail = head[head.len().saturating_sub(2)..].join(" ");
149 return format!("{tail}. {body}");
150 }
151 if body.trim().is_empty() {
152 body = e.sig.chars().take(200).collect();
153 }
154 let parent = e.path.rsplit(['.', ':']).nth(1).unwrap_or("");
155 format!("{} {parent}. {body}", e.name)
156}
157
158fn cache_path(dep: &Dep, src: &Source, types: Option<&Path>, up: Option<&(PathBuf, Manifest)>, embed: &str) -> PathBuf {
159 let up_key = up
160 .map(|(_, m)| {
161 format!(
162 "{}@{:?}:{}:{:?}:{:?}:{:?}:{}:{}",
163 m.repo, m.tag, m.files, m.site, m.commit, m.note, m.docs_sites_checked, m.format
164 )
165 })
166 .unwrap_or_default();
167 let key = cache::hash(&[
168 embed,
169 &up_key,
170 &FORMAT.to_string(),
171 dep.eco.as_str(),
172 &dep.name,
173 &src.version,
174 &src.dir.to_string_lossy(),
175 &stamp(src),
176 &types.map(|t| t.to_string_lossy().to_string()).unwrap_or_default(),
177 ]);
178 let safe = format!("{}@{}", dep.name, src.version).replace(['/', '\\', ':'], "+");
179 cache::dir().join("idx").join(dep.eco.as_str()).join(format!("{safe}-{key}.bin"))
180}
181
182fn types_pkg(dep: &Dep, root: &Path) -> Option<PathBuf> {
184 if dep.eco != Eco::Npm || dep.name.starts_with("@types/") {
185 return None;
186 }
187 let mangled = match dep.name.strip_prefix('@') {
188 Some(scoped) => scoped.replacen('/', "__", 1),
189 None => dep.name.clone(),
190 };
191 locate::npm_installed(&format!("@types/{mangled}"), root).map(|(d, _)| d)
192}
193
194pub fn build(dep: &Dep, src: &Source, root: &Path, up: Option<&(PathBuf, Manifest)>) -> PackageIndex {
195 let t = Instant::now();
196 let types = types_pkg(dep, root);
197 let up_dir = up.filter(|(_, m)| m.files > 0).map(|(d, _)| d.as_path());
198 let entries = extract::extract(dep.eco, &dep.name, src, types.as_deref(), up_dir);
199 let (terms, lens): (Vec<_>, Vec<_>) = entries.iter().map(entry_tf).unzip();
200 let (head, head_lens): (Vec<_>, Vec<_>) = entries.iter().map(head_tf).unzip();
201 let model = embed::get();
202 let vecs: Vec<Vec8> = match &model {
203 Some(m) => {
204 use rayon::prelude::*;
205 entries.par_iter().map(|e| m.embed8(&embed_text(e)).unwrap_or_default()).collect()
206 }
207 None => Vec::new(),
208 };
209 PackageIndex {
210 format: FORMAT,
211 eco: dep.eco,
212 name: dep.name.clone(),
213 version: src.version.clone(),
214 source: src.label.clone(),
215 fetched: src.fetched,
216 entries,
217 terms,
218 lens,
219 head,
220 head_lens,
221 build_ms: t.elapsed().as_millis() as u64,
222 upstream: up.filter(|(_, m)| m.files > 0).map(|(_, m)| {
223 format!(
224 "{}@{}{}{} ({} files)",
225 m.repo,
226 m.tag.as_deref().unwrap_or("?"),
227 m.commit.as_ref().map(|c| format!(" commit {c}")).unwrap_or_default(),
228 m.site.as_ref().map(|s| format!(" + docs site {s}")).unwrap_or_default(),
229 m.files
230 )
231 }),
232 upstream_provenance: up.map(|(_, m)| m.clone()),
233 embed: if model.is_some() { embed::MODEL_ID.to_string() } else { String::new() },
234 vecs,
235 }
236}
237
238pub fn load_or_build(dep: &Dep, src: &Source, root: &Path, up: Option<&(PathBuf, Manifest)>) -> PackageIndex {
240 let types = types_pkg(dep, root);
241 let embed_id = if embed::available() { embed::MODEL_ID } else { "" };
242 let path = cache_path(dep, src, types.as_deref(), up, embed_id);
243 if let Ok(bytes) = std::fs::read(&path) {
244 if let Ok(idx) = postcard::from_bytes::<PackageIndex>(&bytes) {
245 if idx.format == FORMAT {
246 return idx;
247 }
248 }
249 }
250 let idx = build(dep, src, root, up);
251 if let Ok(bytes) = postcard::to_stdvec(&idx) {
252 if let Some(dir) = path.parent() {
253 let _ = std::fs::create_dir_all(dir);
254 }
255 let tmp = path.with_extension(format!("tmp{}", std::process::id()));
256 if std::fs::write(&tmp, bytes).is_ok() {
257 let _ = std::fs::rename(&tmp, &path);
258 }
259 }
260 idx
261}