vissue_core/
digest.rs

1//! A content digest of the corpus, so a consumer can tell whether a projection
2//! it holds is still current.
3//!
4//! The digest is taken over the canonical JSONL export, which is already
5//! deterministic and byte-for-byte stable. Hashing that rather than the org
6//! files means formatting churn that does not change an issue does not change
7//! the digest either. Every function here reads; none writes.
8
9use anyhow::Result;
10use serde_json::{json, Value};
11use std::fmt::Write as _;
12use xxhash_rust::xxh3::xxh3_64;
13
14use crate::config::Layout;
15use crate::events;
16use crate::report;
17use crate::store::list_projects;
18
19/// One project's contribution to the digest.
20#[derive(Debug, Clone, PartialEq, Eq)]
21pub struct ProjectDigest {
22    pub project: String,
23    pub digest: String,
24    pub issues: usize,
25}
26
27/// The digest of a selected set of projects, plus what it was taken against.
28#[derive(Debug, Clone, PartialEq, Eq)]
29pub struct CorpusDigest {
30    /// Hash over the per-project digests, so it moves exactly when one of them
31    /// does.
32    pub combined: String,
33    pub projects: Vec<ProjectDigest>,
34    pub issues: usize,
35    pub generation: u64,
36}
37
38fn hex(value: u64) -> String {
39    format!("{value:016x}")
40}
41
42/// Digest one project's export bytes.
43pub fn project_digest(layout: &Layout, project: &str) -> Result<ProjectDigest> {
44    let export = report::export(layout, Some(project))?;
45    Ok(ProjectDigest {
46        project: project.to_string(),
47        digest: hex(xxh3_64(export.as_bytes())),
48        issues: export.lines().filter(|l| !l.trim().is_empty()).count(),
49    })
50}
51
52/// Digest the named projects, or every project when the list is empty.
53pub fn corpus_digest(layout: &Layout, projects: &[String]) -> Result<CorpusDigest> {
54    let mut selected: Vec<String> = if projects.is_empty() {
55        list_projects(layout)?
56    } else {
57        projects.to_vec()
58    };
59    selected.sort();
60    selected.dedup();
61
62    // One read of the corpus, grouped, rather than one read per project.
63    // `project_digest` filters a whole-corpus export down to a single
64    // project, so calling it in a loop is quadratic in the project count.
65    let grouped = report::export_by_project(layout)?;
66    let per_project: Vec<ProjectDigest> = selected
67        .iter()
68        .map(|project| {
69            let export = grouped.get(project).map(String::as_str).unwrap_or("");
70            ProjectDigest {
71                project: project.clone(),
72                digest: hex(xxh3_64(export.as_bytes())),
73                issues: export.lines().filter(|l| !l.trim().is_empty()).count(),
74            }
75        })
76        .collect();
77
78    // Combine over the sub-digests rather than over the raw bytes, so the
79    // combined value is a pure function of the parts a reader can inspect.
80    let mut material = String::new();
81    for p in &per_project {
82        let _ = writeln!(material, "{}={}", p.project, p.digest);
83    }
84
85    Ok(CorpusDigest {
86        combined: hex(xxh3_64(material.as_bytes())),
87        issues: per_project.iter().map(|p| p.issues).sum(),
88        projects: per_project,
89        generation: events::generation(layout),
90    })
91}
92
93impl CorpusDigest {
94    /// A summary line, then one line per project.
95    ///
96    /// The per-project lines are what turn "something moved" into "this moved",
97    /// which is why they are not folded into the summary.
98    pub fn render(&self) -> String {
99        let mut out = format!(
100            "combined={} issues={} generation={} projects={}\n",
101            self.combined,
102            self.issues,
103            self.generation,
104            self.projects.len()
105        );
106        for p in &self.projects {
107            let _ = writeln!(out, "{}  {:>6}  {}", p.digest, p.issues, p.project);
108        }
109        out
110    }
111
112    pub fn to_json(&self) -> Value {
113        json!({
114            "combined": self.combined,
115            "issues": self.issues,
116            "generation": self.generation,
117            "projects": self.projects.iter().map(|p| json!({
118                "project": p.project,
119                "digest": p.digest,
120                "issues": p.issues,
121            })).collect::<Vec<_>>(),
122        })
123    }
124
125    pub fn digest_of(&self, project: &str) -> Option<&str> {
126        self.projects
127            .iter()
128            .find(|p| p.project == project)
129            .map(|p| p.digest.as_str())
130    }
131
132    pub fn project_names(&self) -> Vec<String> {
133        self.projects.iter().map(|p| p.project.clone()).collect()
134    }
135}
136
137#[cfg(test)]
138mod tests {
139    use super::*;
140    use crate::config::DEFAULT_PREFIX;
141    use crate::ops::{create, CreateOpts};
142    use std::fs;
143
144    /// A tracker with `projects` projects, two issues each.
145    fn many_projects(projects: usize) -> (tempfile::TempDir, Layout) {
146        let dir = tempfile::tempdir().unwrap();
147        let layout = Layout::new(dir.path(), DEFAULT_PREFIX);
148        fs::create_dir_all(layout.projects_dir()).unwrap();
149        for p in 0..projects {
150            let project = format!("p{p:03}");
151            for i in 0..2 {
152                create(
153                    &layout,
154                    &project,
155                    &format!("issue {i}"),
156                    CreateOpts {
157                        quiet: true,
158                        body: Some("Body text, so a row is not trivially short."),
159                        ..CreateOpts::default()
160                    },
161                )
162                .unwrap();
163            }
164        }
165        (dir, layout)
166    }
167
168    /// Each project's digest is taken over exactly what `export` returns for
169    /// it. The grouped read exists for speed, and a digest that moved would
170    /// mark every mirror ever stamped as stale.
171    #[test]
172    fn a_project_digest_matches_its_own_export() {
173        let (_dir, layout) = many_projects(4);
174        let grouped = crate::report::export_by_project(&layout).unwrap();
175        for project in crate::store::list_projects(&layout).unwrap() {
176            let alone = crate::report::export(&layout, Some(&project)).unwrap();
177            assert_eq!(
178                grouped.get(&project).map(String::as_str).unwrap_or(""),
179                alone,
180                "{project}: grouped export differs from its own"
181            );
182            let single = project_digest(&layout, &project).unwrap();
183            let combined = corpus_digest(&layout, &[]).unwrap();
184            assert_eq!(
185                combined.digest_of(&project),
186                Some(single.digest.as_str()),
187                "{project}: corpus digest disagrees with the project's own"
188            );
189        }
190    }
191
192    /// Digesting the corpus reads it once, not once per project.
193    ///
194    /// Compared against `export`, which reads it once by definition, so the
195    /// bound calibrates itself to the machine rather than to a clock. Taking
196    /// one export per project costs one read each; the project count has to
197    /// be high enough that the difference clears the fixed slack, or a
198    /// quadratic version passes on a small corpus.
199    #[test]
200    fn the_corpus_digest_does_not_read_once_per_project() {
201        let (_dir, layout) = many_projects(120);
202        let started = std::time::Instant::now();
203        let _ = crate::report::export(&layout, None).unwrap();
204        let one_read = started.elapsed();
205
206        let started = std::time::Instant::now();
207        let digest = corpus_digest(&layout, &[]).unwrap();
208        let whole = started.elapsed();
209
210        assert_eq!(digest.projects.len(), 120);
211        assert!(
212            whole < one_read * 5 + std::time::Duration::from_millis(25),
213            "digest took {whole:?} against a single export of {one_read:?}, \
214             which is the shape of one read per project"
215        );
216    }
217
218    fn seeded() -> (tempfile::TempDir, Layout) {
219        let dir = tempfile::tempdir().unwrap();
220        let layout = Layout::new(dir.path(), DEFAULT_PREFIX);
221        fs::create_dir_all(layout.projects_dir()).unwrap();
222        create(&layout, "alpha", "first", CreateOpts::default()).unwrap();
223        create(&layout, "beta", "second", CreateOpts::default()).unwrap();
224        (dir, layout)
225    }
226
227    #[test]
228    fn a_digest_is_stable_across_runs() {
229        let (_dir, layout) = seeded();
230        let once = corpus_digest(&layout, &[]).unwrap();
231        let twice = corpus_digest(&layout, &[]).unwrap();
232        assert_eq!(once.combined, twice.combined);
233        assert_eq!(once.projects, twice.projects);
234        assert_eq!(once.combined.len(), 16, "{}", once.combined);
235    }
236
237    #[test]
238    fn the_combined_digest_follows_the_parts() {
239        let (_dir, layout) = seeded();
240        let before = corpus_digest(&layout, &[]).unwrap();
241        create(&layout, "alpha", "a third issue", CreateOpts::default()).unwrap();
242        let after = corpus_digest(&layout, &[]).unwrap();
243
244        assert_ne!(before.combined, after.combined, "the corpus moved");
245        assert_ne!(
246            before.digest_of("alpha"),
247            after.digest_of("alpha"),
248            "the changed project moved"
249        );
250        assert_eq!(
251            before.digest_of("beta"),
252            after.digest_of("beta"),
253            "an untouched project must not move"
254        );
255    }
256
257    #[test]
258    fn selecting_projects_narrows_the_digest() {
259        let (_dir, layout) = seeded();
260        let all = corpus_digest(&layout, &[]).unwrap();
261        let one = corpus_digest(&layout, &["alpha".to_string()]).unwrap();
262        assert_eq!(one.projects.len(), 1);
263        assert_ne!(all.combined, one.combined);
264        assert_eq!(all.digest_of("alpha"), one.digest_of("alpha"));
265    }
266
267    #[test]
268    fn selection_order_does_not_change_the_digest() {
269        let (_dir, layout) = seeded();
270        let forward = corpus_digest(&layout, &["alpha".into(), "beta".into()]).unwrap();
271        let backward = corpus_digest(&layout, &["beta".into(), "alpha".into()]).unwrap();
272        assert_eq!(forward.combined, backward.combined);
273    }
274
275    #[test]
276    fn the_rendered_form_names_every_project() {
277        let (_dir, layout) = seeded();
278        let digest = corpus_digest(&layout, &[]).unwrap();
279        let text = digest.render();
280        assert!(
281            text.starts_with(&format!("combined={}", digest.combined)),
282            "{text}"
283        );
284        assert!(text.contains("alpha"), "{text}");
285        assert!(text.contains("beta"), "{text}");
286        assert_eq!(text.lines().count(), 3, "{text}");
287
288        let value = digest.to_json();
289        assert_eq!(value["combined"], digest.combined);
290        assert_eq!(value["projects"].as_array().unwrap().len(), 2);
291    }
292}
vissue_core/digest.rs

vissue_core/
digest.rs
