pray_core/
derived_metadata.rs

1use crate::hashing::normalize_line_endings;
2use crate::package_archive::unpack_praypkg;
3use crate::package_spec::parse_package_spec;
4use crate::paths::find_prayspec_file;
5use crate::PrayResult;
6use serde::{Deserialize, Serialize};
7use sha2::{Digest, Sha256};
8use std::collections::{BTreeMap, BTreeSet};
9use std::fs;
10use std::path::{Path, PathBuf};
11
12const DERIVED_EMBEDDING_MODEL: &str = "pray-hash-bucket-v1";
13const EMBEDDING_DIMENSIONS: usize = 16;
14const MAX_SUMMARY_PARTS: usize = 2;
15const MAX_TOKEN_COUNT: usize = 512;
16const MAX_SNIPPET_LENGTH: usize = 120;
17const STOPWORDS: &[&str] = &[
18    "a", "an", "and", "are", "as", "at", "be", "by", "for", "from", "has", "have", "in", "is",
19    "it", "its", "of", "on", "or", "package", "the", "this", "to", "was", "we", "with", "you",
20    "your",
21];
22
23#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
24pub struct RegistryDerivedMetadata {
25    #[serde(default, skip_serializing_if = "String::is_empty")]
26    pub summary: String,
27    #[serde(default, skip_serializing_if = "Vec::is_empty")]
28    pub topics: Vec<String>,
29    #[serde(default, skip_serializing_if = "Vec::is_empty")]
30    pub categories: Vec<String>,
31    #[serde(default, skip_serializing_if = "Vec::is_empty")]
32    pub possible_effects: Vec<String>,
33    #[serde(default, skip_serializing_if = "Vec::is_empty")]
34    pub possible_side_effects: Vec<String>,
35    #[serde(default, skip_serializing_if = "Vec::is_empty")]
36    pub embeddings: Vec<RegistryDerivedEmbedding>,
37    #[serde(default, skip_serializing_if = "Option::is_none")]
38    pub file_count: Option<usize>,
39    #[serde(default, skip_serializing_if = "Option::is_none")]
40    pub character_count: Option<usize>,
41    #[serde(default, skip_serializing_if = "Option::is_none")]
42    pub token_count: Option<usize>,
43}
44
45#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
46pub struct RegistryDerivedEmbedding {
47    pub model: String,
48    pub vector: Vec<i32>,
49}
50
51pub fn derive_registry_derived_metadata_from_root(
52    root: &Path,
53) -> PrayResult<RegistryDerivedMetadata> {
54    let spec_path = find_prayspec_file(root)?;
55    let spec_text = fs::read_to_string(&spec_path)?;
56    let spec = parse_package_spec(&normalize_line_endings(&spec_text))?.canonicalized();
57    derive_registry_derived_metadata_from_spec(root, &spec)
58}
59
60pub fn derive_registry_derived_metadata_from_archive_bytes(
61    archive_bytes: &[u8],
62) -> PrayResult<RegistryDerivedMetadata> {
63    let temp_dir = unique_temp_dir("pray-derived-metadata");
64    fs::create_dir_all(&temp_dir)?;
65    let unpack_result = unpack_praypkg(archive_bytes, &temp_dir)
66        .and_then(|_| derive_registry_derived_metadata_from_root(&temp_dir));
67    let _ = fs::remove_dir_all(&temp_dir);
68    unpack_result
69}
70
71fn derive_registry_derived_metadata_from_spec(
72    root: &Path,
73    spec: &crate::package_spec::PackageSpec,
74) -> PrayResult<RegistryDerivedMetadata> {
75    let mut summary_candidates = Vec::new();
76    if let Some(summary) = spec.summary.as_deref() {
77        push_candidate(&mut summary_candidates, summary);
78    }
79    if let Some(description) = spec.description.as_deref() {
80        push_candidate(&mut summary_candidates, description);
81    }
82    for export in spec.exports.values() {
83        if let Some(summary) = export.summary.as_deref() {
84            push_candidate(&mut summary_candidates, summary);
85        }
86    }
87
88    let mut analyzed_text = String::new();
89    let mut sample_lines = Vec::new();
90    let mut file_count = 0usize;
91    let mut character_count = 0usize;
92    for file in &spec.files {
93        let path = root.join(file);
94        let Ok(text) = fs::read_to_string(&path) else {
95            continue;
96        };
97        let normalized = normalize_line_endings(&text);
98        if let Some(line) = first_meaningful_line(&normalized) {
99            push_candidate(&mut sample_lines, &line);
100        }
101        file_count += 1;
102        character_count += normalized.chars().count();
103        analyzed_text.push_str(&normalized);
104        analyzed_text.push('\n');
105    }
106
107    for line in &sample_lines {
108        push_candidate(&mut summary_candidates, line);
109    }
110    if summary_candidates.is_empty() {
111        for token in top_topics(&analyzed_text) {
112            push_candidate(&mut summary_candidates, &token);
113        }
114    }
115
116    let summary = build_summary(&summary_candidates);
117    let tokens = tokenize(&analyzed_text);
118    let topics = top_topics(&analyzed_text);
119    let categories = infer_categories(&summary_candidates.join(" \n"), &topics);
120    let possible_effects = infer_possible_effects(&categories);
121    let possible_side_effects = infer_possible_side_effects(&analyzed_text, &topics);
122    let embeddings = vec![RegistryDerivedEmbedding {
123        model: DERIVED_EMBEDDING_MODEL.to_string(),
124        vector: hashed_embedding(&tokens),
125    }];
126
127    Ok(RegistryDerivedMetadata {
128        summary,
129        topics,
130        categories,
131        possible_effects,
132        possible_side_effects,
133        embeddings,
134        file_count: Some(file_count),
135        character_count: Some(character_count),
136        token_count: Some(tokens.len()),
137    })
138}
139
140fn build_summary(candidates: &[String]) -> String {
141    let mut summary = Vec::new();
142    let mut seen = BTreeSet::new();
143    for candidate in candidates {
144        let normalized = candidate.trim();
145        if normalized.is_empty() {
146            continue;
147        }
148        let key = normalized.to_lowercase();
149        if seen.insert(key) {
150            summary.push(truncate(normalized, MAX_SNIPPET_LENGTH));
151        }
152        if summary.len() == MAX_SUMMARY_PARTS {
153            break;
154        }
155    }
156    if summary.is_empty() {
157        return "Package metadata".to_string();
158    }
159    summary.join(" — ")
160}
161
162fn top_topics(text: &str) -> Vec<String> {
163    let mut counts: BTreeMap<String, usize> = BTreeMap::new();
164    for token in tokenize(text) {
165        *counts.entry(token).or_insert(0) += 1;
166    }
167    let mut ranked: Vec<(String, usize)> = counts.into_iter().collect();
168    ranked.sort_by(|left, right| right.1.cmp(&left.1).then(left.0.cmp(&right.0)));
169    ranked.into_iter().take(5).map(|(token, _)| token).collect()
170}
171
172fn infer_categories(summary_text: &str, topics: &[String]) -> Vec<String> {
173    let text = format!("{} {}", summary_text.to_lowercase(), topics.join(" "));
174    let mut categories = Vec::new();
175    if contains_any(
176        &text,
177        &["guide", "guidance", "readme", "documentation", "doc"],
178    ) {
179        categories.push("documentation".to_string());
180    }
181    if contains_any(&text, &["test", "testing", "fixture", "spec"]) {
182        categories.push("testing".to_string());
183    }
184    if contains_any(&text, &["auth", "security", "ssh", "key", "sign", "trust"]) {
185        categories.push("security".to_string());
186    }
187    if contains_any(
188        &text,
189        &["automation", "workflow", "render", "pipeline", "agent"],
190    ) {
191        categories.push("automation".to_string());
192    }
193    if categories.is_empty() {
194        categories.push("general".to_string());
195    }
196    categories.sort();
197    categories.dedup();
198    categories
199}
200
201fn infer_possible_effects(categories: &[String]) -> Vec<String> {
202    let mut effects = Vec::new();
203    if categories
204        .iter()
205        .any(|category| category == "documentation")
206    {
207        effects.push("clarifies usage".to_string());
208    }
209    if categories.iter().any(|category| category == "testing") {
210        effects.push("improves validation".to_string());
211    }
212    if categories.iter().any(|category| category == "security") {
213        effects.push("surfaces trust-sensitive behavior".to_string());
214    }
215    if categories.iter().any(|category| category == "automation") {
216        effects.push("reduces manual steps".to_string());
217    }
218    if effects.is_empty() {
219        effects.push("improves package discovery".to_string());
220    }
221    effects
222}
223
224fn infer_possible_side_effects(text: &str, topics: &[String]) -> Vec<String> {
225    let mut side_effects = Vec::new();
226    if contains_any(
227        text,
228        &[
229            "write",
230            "overwrite",
231            "replace",
232            "render",
233            "inject",
234            "delete",
235        ],
236    ) {
237        side_effects.push("may change generated output or managed files".to_string());
238    }
239    if contains_any(&topics.join(" "), &["publish", "release", "deploy"]) {
240        side_effects.push("may affect downstream package distribution".to_string());
241    }
242    side_effects.sort();
243    side_effects.dedup();
244    side_effects
245}
246
247fn hashed_embedding(tokens: &[String]) -> Vec<i32> {
248    let mut vector = vec![0i32; EMBEDDING_DIMENSIONS];
249    for token in tokens.iter().take(MAX_TOKEN_COUNT) {
250        let mut hasher = Sha256::new();
251        hasher.update(token.as_bytes());
252        let hash = hasher.finalize();
253        let bucket = hash[0] as usize % EMBEDDING_DIMENSIONS;
254        vector[bucket] += 1;
255    }
256    vector
257}
258
259fn tokenize(text: &str) -> Vec<String> {
260    let mut tokens = Vec::new();
261    for raw in text.split(|character: char| !character.is_alphanumeric()) {
262        let token = raw.trim().to_lowercase();
263        if token.len() < 4 || STOPWORDS.contains(&token.as_str()) {
264            continue;
265        }
266        tokens.push(token);
267        if tokens.len() == MAX_TOKEN_COUNT {
268            break;
269        }
270    }
271    tokens
272}
273
274fn contains_any(text: &str, needles: &[&str]) -> bool {
275    needles.iter().any(|needle| text.contains(needle))
276}
277
278fn push_candidate(candidates: &mut Vec<String>, candidate: &str) {
279    let trimmed = candidate.trim();
280    if trimmed.is_empty() {
281        return;
282    }
283    candidates.push(truncate(trimmed, MAX_SNIPPET_LENGTH));
284}
285
286fn first_meaningful_line(text: &str) -> Option<String> {
287    text.lines()
288        .map(str::trim)
289        .find(|line| !line.is_empty())
290        .map(|line| truncate(line, MAX_SNIPPET_LENGTH))
291}
292
293fn truncate(text: &str, limit: usize) -> String {
294    let mut shortened = String::new();
295    for character in text.chars().take(limit) {
296        shortened.push(character);
297    }
298    shortened
299}
300
301fn unique_temp_dir(prefix: &str) -> PathBuf {
302    use std::sync::atomic::{AtomicU64, Ordering};
303    static COUNTER: AtomicU64 = AtomicU64::new(0);
304    let unique = std::time::SystemTime::now()
305        .duration_since(std::time::UNIX_EPOCH)
306        .expect("system clock before unix epoch")
307        .as_nanos();
308    let sequence = COUNTER.fetch_add(1, Ordering::Relaxed);
309    std::env::temp_dir().join(format!("{prefix}-{unique}-{sequence}"))
310}
pray_core/derived_metadata.rs

pray_core/
derived_metadata.rs
