pray_core/
derived_metadata.rs

1use crate::hashing::normalize_line_endings;
2use crate::package_spec::parse_package_spec;
3use crate::{PrayError, PrayResult};
4use serde::{Deserialize, Serialize};
5use sha2::{Digest, Sha256};
6use std::collections::{BTreeMap, BTreeSet};
7use std::fs;
8use std::io::Cursor;
9use std::path::{Path, PathBuf};
10
11const DERIVED_EMBEDDING_MODEL: &str = "pray-hash-bucket-v1";
12const EMBEDDING_DIMENSIONS: usize = 16;
13const MAX_SUMMARY_PARTS: usize = 2;
14const MAX_TOKEN_COUNT: usize = 512;
15const MAX_SNIPPET_LENGTH: usize = 120;
16const STOPWORDS: &[&str] = &[
17    "a", "an", "and", "are", "as", "at", "be", "by", "for", "from", "has", "have", "in", "is",
18    "it", "its", "of", "on", "or", "package", "the", "this", "to", "was", "we", "with", "you",
19    "your",
20];
21
22#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
23pub struct RegistryDerivedMetadata {
24    #[serde(default, skip_serializing_if = "String::is_empty")]
25    pub summary: String,
26    #[serde(default, skip_serializing_if = "Vec::is_empty")]
27    pub topics: Vec<String>,
28    #[serde(default, skip_serializing_if = "Vec::is_empty")]
29    pub categories: Vec<String>,
30    #[serde(default, skip_serializing_if = "Vec::is_empty")]
31    pub possible_effects: Vec<String>,
32    #[serde(default, skip_serializing_if = "Vec::is_empty")]
33    pub possible_side_effects: Vec<String>,
34    #[serde(default, skip_serializing_if = "Vec::is_empty")]
35    pub embeddings: Vec<RegistryDerivedEmbedding>,
36    #[serde(default, skip_serializing_if = "Option::is_none")]
37    pub file_count: Option<usize>,
38    #[serde(default, skip_serializing_if = "Option::is_none")]
39    pub character_count: Option<usize>,
40    #[serde(default, skip_serializing_if = "Option::is_none")]
41    pub token_count: Option<usize>,
42}
43
44#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
45pub struct RegistryDerivedEmbedding {
46    pub model: String,
47    pub vector: Vec<i32>,
48}
49
50pub fn derive_registry_derived_metadata_from_root(
51    root: &Path,
52) -> PrayResult<RegistryDerivedMetadata> {
53    let spec_path = find_prayspec_file(root)?;
54    let spec_text = fs::read_to_string(&spec_path)?;
55    let spec = parse_package_spec(&normalize_line_endings(&spec_text))?.canonicalized();
56    derive_registry_derived_metadata_from_spec(root, &spec)
57}
58
59pub fn derive_registry_derived_metadata_from_archive_bytes(
60    archive_bytes: &[u8],
61) -> PrayResult<RegistryDerivedMetadata> {
62    let temp_dir = unique_temp_dir("pray-derived-metadata");
63    fs::create_dir_all(&temp_dir)?;
64    let unpack_result = unpack_archive_bytes(archive_bytes, &temp_dir)
65        .and_then(|_| derive_registry_derived_metadata_from_root(&temp_dir));
66    let _ = fs::remove_dir_all(&temp_dir);
67    unpack_result
68}
69
70fn derive_registry_derived_metadata_from_spec(
71    root: &Path,
72    spec: &crate::package_spec::PackageSpec,
73) -> PrayResult<RegistryDerivedMetadata> {
74    let mut summary_candidates = Vec::new();
75    if let Some(summary) = spec.summary.as_deref() {
76        push_candidate(&mut summary_candidates, summary);
77    }
78    if let Some(description) = spec.description.as_deref() {
79        push_candidate(&mut summary_candidates, description);
80    }
81    for export in spec.exports.values() {
82        if let Some(summary) = export.summary.as_deref() {
83            push_candidate(&mut summary_candidates, summary);
84        }
85    }
86
87    let mut analyzed_text = String::new();
88    let mut sample_lines = Vec::new();
89    let mut file_count = 0usize;
90    let mut character_count = 0usize;
91    for file in &spec.files {
92        let path = root.join(file);
93        let Ok(text) = fs::read_to_string(&path) else {
94            continue;
95        };
96        let normalized = normalize_line_endings(&text);
97        if let Some(line) = first_meaningful_line(&normalized) {
98            push_candidate(&mut sample_lines, &line);
99        }
100        file_count += 1;
101        character_count += normalized.chars().count();
102        analyzed_text.push_str(&normalized);
103        analyzed_text.push('\n');
104    }
105
106    for line in &sample_lines {
107        push_candidate(&mut summary_candidates, line);
108    }
109    if summary_candidates.is_empty() {
110        for token in top_topics(&analyzed_text) {
111            push_candidate(&mut summary_candidates, &token);
112        }
113    }
114
115    let summary = build_summary(&summary_candidates);
116    let tokens = tokenize(&analyzed_text);
117    let topics = top_topics(&analyzed_text);
118    let categories = infer_categories(&summary_candidates.join(" \n"), &topics);
119    let possible_effects = infer_possible_effects(&categories);
120    let possible_side_effects = infer_possible_side_effects(&analyzed_text, &topics);
121    let embeddings = vec![RegistryDerivedEmbedding {
122        model: DERIVED_EMBEDDING_MODEL.to_string(),
123        vector: hashed_embedding(&tokens),
124    }];
125
126    Ok(RegistryDerivedMetadata {
127        summary,
128        topics,
129        categories,
130        possible_effects,
131        possible_side_effects,
132        embeddings,
133        file_count: Some(file_count),
134        character_count: Some(character_count),
135        token_count: Some(tokens.len()),
136    })
137}
138
139fn unpack_archive_bytes(archive_bytes: &[u8], root: &Path) -> PrayResult<()> {
140    let decoder = zstd::stream::read::Decoder::new(Cursor::new(archive_bytes))?;
141    let mut archive = tar::Archive::new(decoder);
142    archive.unpack(root)?;
143    Ok(())
144}
145
146fn find_prayspec_file(root: &Path) -> PrayResult<PathBuf> {
147    let mut prayspec_files = Vec::new();
148    for entry in fs::read_dir(root)? {
149        let entry = entry?;
150        let path = entry.path();
151        if path.extension().and_then(|value| value.to_str()) == Some("prayspec") {
152            prayspec_files.push(path);
153        }
154    }
155    match prayspec_files.len() {
156        1 => Ok(prayspec_files.remove(0)),
157        0 => Err(PrayError::Resolution(format!(
158            "no prayspec file found in {:?}",
159            root
160        ))),
161        _ => Err(PrayError::Resolution(format!(
162            "multiple prayspec files found in {:?}",
163            root
164        ))),
165    }
166}
167
168fn build_summary(candidates: &[String]) -> String {
169    let mut summary = Vec::new();
170    let mut seen = BTreeSet::new();
171    for candidate in candidates {
172        let normalized = candidate.trim();
173        if normalized.is_empty() {
174            continue;
175        }
176        let key = normalized.to_lowercase();
177        if seen.insert(key) {
178            summary.push(truncate(normalized, MAX_SNIPPET_LENGTH));
179        }
180        if summary.len() == MAX_SUMMARY_PARTS {
181            break;
182        }
183    }
184    if summary.is_empty() {
185        return "Package metadata".to_string();
186    }
187    summary.join(" — ")
188}
189
190fn top_topics(text: &str) -> Vec<String> {
191    let mut counts: BTreeMap<String, usize> = BTreeMap::new();
192    for token in tokenize(text) {
193        *counts.entry(token).or_insert(0) += 1;
194    }
195    let mut ranked: Vec<(String, usize)> = counts.into_iter().collect();
196    ranked.sort_by(|left, right| right.1.cmp(&left.1).then(left.0.cmp(&right.0)));
197    ranked.into_iter().take(5).map(|(token, _)| token).collect()
198}
199
200fn infer_categories(summary_text: &str, topics: &[String]) -> Vec<String> {
201    let text = format!("{} {}", summary_text.to_lowercase(), topics.join(" "));
202    let mut categories = Vec::new();
203    if contains_any(
204        &text,
205        &["guide", "guidance", "readme", "documentation", "doc"],
206    ) {
207        categories.push("documentation".to_string());
208    }
209    if contains_any(&text, &["test", "testing", "fixture", "spec"]) {
210        categories.push("testing".to_string());
211    }
212    if contains_any(&text, &["auth", "security", "ssh", "key", "sign", "trust"]) {
213        categories.push("security".to_string());
214    }
215    if contains_any(
216        &text,
217        &["automation", "workflow", "render", "pipeline", "agent"],
218    ) {
219        categories.push("automation".to_string());
220    }
221    if categories.is_empty() {
222        categories.push("general".to_string());
223    }
224    categories.sort();
225    categories.dedup();
226    categories
227}
228
229fn infer_possible_effects(categories: &[String]) -> Vec<String> {
230    let mut effects = Vec::new();
231    if categories
232        .iter()
233        .any(|category| category == "documentation")
234    {
235        effects.push("clarifies usage".to_string());
236    }
237    if categories.iter().any(|category| category == "testing") {
238        effects.push("improves validation".to_string());
239    }
240    if categories.iter().any(|category| category == "security") {
241        effects.push("surfaces trust-sensitive behavior".to_string());
242    }
243    if categories.iter().any(|category| category == "automation") {
244        effects.push("reduces manual steps".to_string());
245    }
246    if effects.is_empty() {
247        effects.push("improves package discovery".to_string());
248    }
249    effects
250}
251
252fn infer_possible_side_effects(text: &str, topics: &[String]) -> Vec<String> {
253    let mut side_effects = Vec::new();
254    if contains_any(
255        text,
256        &[
257            "write",
258            "overwrite",
259            "replace",
260            "render",
261            "inject",
262            "delete",
263        ],
264    ) {
265        side_effects.push("may change generated output or managed files".to_string());
266    }
267    if contains_any(&topics.join(" "), &["publish", "release", "deploy"]) {
268        side_effects.push("may affect downstream package distribution".to_string());
269    }
270    side_effects.sort();
271    side_effects.dedup();
272    side_effects
273}
274
275fn hashed_embedding(tokens: &[String]) -> Vec<i32> {
276    let mut vector = vec![0i32; EMBEDDING_DIMENSIONS];
277    for token in tokens.iter().take(MAX_TOKEN_COUNT) {
278        let mut hasher = Sha256::new();
279        hasher.update(token.as_bytes());
280        let hash = hasher.finalize();
281        let bucket = hash[0] as usize % EMBEDDING_DIMENSIONS;
282        vector[bucket] += 1;
283    }
284    vector
285}
286
287fn tokenize(text: &str) -> Vec<String> {
288    let mut tokens = Vec::new();
289    for raw in text.split(|character: char| !character.is_alphanumeric()) {
290        let token = raw.trim().to_lowercase();
291        if token.len() < 4 || STOPWORDS.contains(&token.as_str()) {
292            continue;
293        }
294        tokens.push(token);
295        if tokens.len() == MAX_TOKEN_COUNT {
296            break;
297        }
298    }
299    tokens
300}
301
302fn contains_any(text: &str, needles: &[&str]) -> bool {
303    needles.iter().any(|needle| text.contains(needle))
304}
305
306fn push_candidate(candidates: &mut Vec<String>, candidate: &str) {
307    let trimmed = candidate.trim();
308    if trimmed.is_empty() {
309        return;
310    }
311    candidates.push(truncate(trimmed, MAX_SNIPPET_LENGTH));
312}
313
314fn first_meaningful_line(text: &str) -> Option<String> {
315    text.lines()
316        .map(str::trim)
317        .find(|line| !line.is_empty())
318        .map(|line| truncate(line, MAX_SNIPPET_LENGTH))
319}
320
321fn truncate(text: &str, limit: usize) -> String {
322    let mut shortened = String::new();
323    for character in text.chars().take(limit) {
324        shortened.push(character);
325    }
326    shortened
327}
328
329fn unique_temp_dir(prefix: &str) -> PathBuf {
330    let unique = std::time::SystemTime::now()
331        .duration_since(std::time::UNIX_EPOCH)
332        .expect("system clock before unix epoch")
333        .as_nanos();
334    std::env::temp_dir().join(format!("{prefix}-{unique}"))
335}
pray_core/derived_metadata.rs

pray_core/
derived_metadata.rs
