use std::collections::{BTreeMap, BTreeSet};
use std::path::{Path, PathBuf};
use serde::{Deserialize, Serialize};
use crate::config::Layout;
use crate::error::{Error, Result};
use crate::views::IssueRec;
pub const VERSION: &str = "vissue-satchel/1";
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(default)]
pub struct Slice {
pub projects: Vec<String>,
pub issues: Vec<String>,
}
impl Slice {
#[must_use]
pub fn is_empty(&self) -> bool {
self.projects.is_empty() && self.issues.is_empty()
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Satchel {
pub version: String,
pub asked: Slice,
pub issues: Vec<String>,
pub carried: Vec<String>,
pub needs: Vec<String>,
pub packed_by: String,
pub packed_at: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Report {
pub issues: usize,
pub needs: usize,
pub files: usize,
pub notes: Vec<String>,
}
impl Report {
#[must_use]
pub fn render(&self) -> String {
let mut out = format!(
"issues={} deeds={} files={}\n",
self.issues, self.needs, self.files
);
for note in &self.notes {
out.push_str(note);
out.push('\n');
}
out
}
}
pub fn pack(layout: &Layout, asked: &Slice, dest: &Path) -> Result<Report> {
if asked.is_empty() {
return Err(Error::Other(anyhow::anyhow!(
"a satchel needs a project or an issue to pack"
)));
}
let recs = crate::catalog::load_recs(layout)?;
let by_id: BTreeMap<&str, &IssueRec> =
recs.iter().map(|r| (r.heading.id.as_str(), r)).collect();
let mut named: BTreeSet<String> = BTreeSet::new();
for project in &asked.projects {
for rec in &recs {
if rec.project == *project {
named.insert(rec.heading.id.clone());
}
}
}
for id in &asked.issues {
if !by_id.contains_key(id.as_str()) {
return Err(Error::IssueNotFound { id: id.clone() });
}
named.insert(id.clone());
}
if named.is_empty() {
return Err(Error::Other(anyhow::anyhow!(
"nothing to pack: {:?} matched no issue",
asked.projects
)));
}
let mut chosen = named.clone();
let mut frontier: Vec<String> = named.iter().cloned().collect();
while let Some(id) = frontier.pop() {
let Some(rec) = by_id.get(id.as_str()) else {
continue;
};
let mut reach: Vec<String> = rec.heading.blocked_by();
if let Some(parent) = rec.heading.parent().map(str::to_string) {
reach.push(parent);
}
for next in reach {
if by_id.contains_key(next.as_str()) && chosen.insert(next.clone()) {
frontier.push(next);
}
}
}
let carried: Vec<String> = chosen.difference(&named).cloned().collect();
let mut needs: BTreeSet<String> = BTreeSet::new();
for id in &chosen {
if let Some(rec) = by_id.get(id.as_str()) {
needs.extend(rec.heading.deeds());
}
}
let data = dest.join("data");
let issues_dir = data.join("issues");
std::fs::create_dir_all(&issues_dir).map_err(Error::from)?;
let mut payload: Vec<(PathBuf, String)> = Vec::new();
for id in &chosen {
let Some(rec) = by_id.get(id.as_str()) else {
continue;
};
let text = crate::catalog::org_text_from(rec)?;
let rel = PathBuf::from("data/issues").join(format!("{id}.org"));
write_payload(dest, &rel, text.as_bytes())?;
payload.push((rel, digest(text.as_bytes())));
}
let satchel = Satchel {
version: VERSION.to_string(),
asked: asked.clone(),
issues: chosen.iter().cloned().collect(),
carried,
needs: needs.iter().cloned().collect(),
packed_by: crate::config::identity(layout),
packed_at: crate::model::today_inactive_bracket(),
};
let described = serde_json::to_string_pretty(&satchel)
.map(|json| json + "\n")
.map_err(|e| Error::Other(anyhow::anyhow!("{e}")))?;
let rel = PathBuf::from("data/satchel.json");
write_payload(dest, &rel, described.as_bytes())?;
payload.push((rel, digest(described.as_bytes())));
payload.sort();
write_manifest(dest, &payload)?;
write_declaration(dest, &satchel)?;
let mut notes = Vec::new();
if !satchel.needs.is_empty() {
notes.push(format!(
"{} deed accessions are named and not enclosed; the deed store exports them",
satchel.needs.len()
));
}
if !satchel.carried.is_empty() {
notes.push(format!(
"{} issues came along as blockers or parents of what was asked for",
satchel.carried.len()
));
}
Ok(Report {
issues: satchel.issues.len(),
needs: satchel.needs.len(),
files: payload.len(),
notes,
})
}
pub fn seal(dir: &Path) -> Result<Report> {
let satchel = describe(dir)?;
let mut payload: Vec<(PathBuf, String)> = Vec::new();
for found in walk(&dir.join("data"))? {
let rel = found
.strip_prefix(dir)
.map_err(|e| Error::Other(anyhow::anyhow!("{e}")))?
.to_path_buf();
let bytes = std::fs::read(&found).map_err(Error::from)?;
payload.push((rel, digest(&bytes)));
}
payload.sort();
write_manifest(dir, &payload)?;
let mut notes = shortfall(dir, &satchel);
let atoms = atom_lines(dir);
if atoms > 0 {
notes.push(format!("{atoms} atoms arrived from a pack"));
}
Ok(Report {
issues: satchel.issues.len(),
needs: satchel.needs.len(),
files: payload.len(),
notes,
})
}
fn shortfall(dir: &Path, satchel: &Satchel) -> Vec<String> {
let enclosed = enclosed_deeds(dir);
let short = satchel
.needs
.iter()
.filter(|acc| !enclosed.contains(*acc))
.count();
if short == 0 {
return Vec::new();
}
vec![format!(
"{short} of {} deed accessions are named and not enclosed",
satchel.needs.len()
)]
}
fn signature_note(dir: &Path) -> String {
let manifest = dir.join("manifest-sha256.txt");
let signature = manifest.with_extension("txt.sig");
if signature.is_file() {
format!(
"the payload matches the manifest, and the manifest carries a signature this check \
did not verify: `deedar vouch check {}` says whether a key you accept made it",
manifest.display()
)
} else {
"the payload matches the manifest, and the manifest is unsigned, so this establishes \
that the bag arrived as written and nothing about who wrote it"
.to_string()
}
}
fn atom_lines(dir: &Path) -> usize {
let Ok(entries) = std::fs::read_dir(dir.join("data").join("atoms")) else {
return 0;
};
entries
.flatten()
.filter_map(|entry| std::fs::read_to_string(entry.path()).ok())
.map(|text| text.lines().filter(|l| !l.trim().is_empty()).count())
.sum()
}
fn enclosed_deeds(dir: &Path) -> BTreeSet<String> {
let mut out = BTreeSet::new();
let Ok(entries) = std::fs::read_dir(dir.join("data").join("deeds")) else {
return out;
};
for entry in entries.flatten() {
if entry.path().is_dir()
&& let Some(name) = entry.file_name().to_str()
{
out.insert(name.to_string());
}
}
out
}
pub fn verify(dir: &Path) -> Result<Report> {
let manifest = dir.join("manifest-sha256.txt");
let text = std::fs::read_to_string(&manifest)
.map_err(|_| Error::Other(anyhow::anyhow!("no manifest at {}", manifest.display())))?;
let mut listed: BTreeMap<PathBuf, String> = BTreeMap::new();
for line in text.lines().filter(|l| !l.trim().is_empty()) {
let Some((hash, rel)) = line.split_once(" ") else {
return Err(Error::Other(anyhow::anyhow!(
"manifest: not an entry: {line:?}"
)));
};
listed.insert(PathBuf::from(rel), hash.to_string());
}
let mut notes = Vec::new();
for (rel, want) in &listed {
let path = dir.join(rel);
let Ok(bytes) = std::fs::read(&path) else {
notes.push(format!("missing {}", rel.display()));
continue;
};
let got = digest(&bytes);
if got != *want {
notes.push(format!("changed {}", rel.display()));
}
}
for found in walk(&dir.join("data"))? {
let rel = found
.strip_prefix(dir)
.map_err(|e| Error::Other(anyhow::anyhow!("{e}")))?
.to_path_buf();
if !listed.contains_key(&rel) {
notes.push(format!("unlisted {}", rel.display()));
}
}
let described = dir.join("data").join("satchel.json");
let satchel: Satchel = std::fs::read_to_string(&described)
.map_err(|_| Error::Other(anyhow::anyhow!("no data/satchel.json")))
.and_then(|raw| serde_json::from_str(&raw).map_err(Error::from))?;
if satchel.version != VERSION {
notes.push(format!(
"packed as {} and read as {VERSION}",
satchel.version
));
}
if notes.is_empty() {
let mut notes = shortfall(dir, &satchel);
let atoms = atom_lines(dir);
if atoms > 0 {
notes.push(format!("{atoms} atoms arrived from a pack"));
}
notes.push(signature_note(dir));
Ok(Report {
issues: satchel.issues.len(),
needs: satchel.needs.len(),
files: listed.len(),
notes,
})
} else {
Err(Error::Other(anyhow::anyhow!(
"the satchel does not check out:\n{}",
notes.join("\n")
)))
}
}
pub fn describe(dir: &Path) -> Result<Satchel> {
let path = dir.join("data").join("satchel.json");
let raw = std::fs::read_to_string(&path)
.map_err(|_| Error::Other(anyhow::anyhow!("no satchel at {}", path.display())))?;
serde_json::from_str(&raw).map_err(Error::from)
}
fn write_payload(dest: &Path, rel: &Path, bytes: &[u8]) -> Result<()> {
let path = dest.join(rel);
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent).map_err(Error::from)?;
}
std::fs::write(&path, bytes).map_err(Error::from)
}
fn write_manifest(dest: &Path, payload: &[(PathBuf, String)]) -> Result<()> {
let mut out = String::new();
for (rel, hash) in payload {
out.push_str(&format!("{hash} {}\n", rel.display()));
}
std::fs::write(dest.join("manifest-sha256.txt"), out).map_err(Error::from)
}
fn write_declaration(dest: &Path, satchel: &Satchel) -> Result<()> {
std::fs::write(
dest.join("bagit.txt"),
"BagIt-Version: 1.0\nTag-File-Character-Encoding: UTF-8\n",
)
.map_err(Error::from)?;
let info = format!(
"Bag-Software-Agent: vissue {}\nBagging-Date: {}\nSource-Organization: {}\nExternal-Description: {} issues and {} deed accessions from a vissue tracker\nInternal-Sender-Identifier: {}\n",
env!("CARGO_PKG_VERSION"),
satchel.packed_at,
satchel.packed_by,
satchel.issues.len(),
satchel.needs.len(),
VERSION,
);
std::fs::write(dest.join("bag-info.txt"), info).map_err(Error::from)
}
fn walk(dir: &Path) -> Result<Vec<PathBuf>> {
let mut out = Vec::new();
if !dir.is_dir() {
return Ok(out);
}
let mut stack = vec![dir.to_path_buf()];
while let Some(at) = stack.pop() {
for entry in std::fs::read_dir(&at).map_err(Error::from)? {
let entry = entry.map_err(Error::from)?;
let path = entry.path();
if path.is_dir() {
stack.push(path);
} else {
out.push(path);
}
}
}
out.sort();
Ok(out)
}
fn digest(bytes: &[u8]) -> String {
use sha2::{Digest, Sha256};
let hash = Sha256::digest(bytes);
hash.iter().map(|b| format!("{b:02x}")).collect()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::config::DEFAULT_PREFIX;
use crate::ops::{self, CreateOpts};
fn made(layout: &Layout, title: &str, opts: CreateOpts<'_>) -> String {
let report = ops::create(layout, "sample", title, opts).expect("create");
report
.split_whitespace()
.next()
.expect("an id in the report")
.to_string()
}
fn tracker() -> (tempfile::TempDir, Layout) {
let dir = tempfile::tempdir().expect("tempdir");
let layout = Layout::new(dir.path(), DEFAULT_PREFIX);
std::fs::create_dir_all(layout.projects_dir()).expect("projects");
(dir, layout)
}
#[test]
fn a_named_issue_brings_its_blockers_and_its_plan() {
let (_dir, layout) = tracker();
let plan = made(&layout, "the plan", CreateOpts::default());
let blocker = made(&layout, "the blocker", CreateOpts::default());
let work = made(
&layout,
"the work",
CreateOpts {
parent: Some(plan.as_str()),
..CreateOpts::default()
},
);
ops::update(&layout, &work, None, None, Some(&blocker), None).expect("blocked by");
let bystander = made(&layout, "not asked for", CreateOpts::default());
let out = tempfile::tempdir().expect("out");
let report = pack(
&layout,
&Slice {
projects: Vec::new(),
issues: vec![work.clone()],
},
out.path(),
)
.expect("packs");
let described = describe(out.path()).expect("describes");
assert!(described.issues.contains(&work), "{described:?}");
assert!(described.issues.contains(&plan), "the plan is missing");
assert!(
described.issues.contains(&blocker),
"the blocker is missing"
);
assert!(
!described.issues.contains(&bystander),
"a sibling came along uninvited"
);
assert_eq!(described.carried.len(), 2, "{:?}", described.carried);
assert_eq!(report.issues, 3);
assert!(
out.path()
.join("data/issues")
.join(format!("{work}.org"))
.is_file()
);
}
#[test]
fn a_satchel_checks_out_until_something_moves() {
let (_dir, layout) = tracker();
let one = made(&layout, "first", CreateOpts::default());
let out = tempfile::tempdir().expect("out");
pack(
&layout,
&Slice {
projects: vec!["sample".into()],
issues: Vec::new(),
},
out.path(),
)
.expect("packs");
verify(out.path()).expect("a fresh satchel checks out");
let issue = out.path().join("data/issues").join(format!("{one}.org"));
std::fs::write(&issue, "* TODO something else\n").expect("write");
let err = verify(out.path()).expect_err("an edited payload passed");
assert!(format!("{err}").contains("changed"), "{err}");
}
#[test]
fn a_file_nobody_listed_is_a_finding() {
let (_dir, layout) = tracker();
made(&layout, "first", CreateOpts::default());
let out = tempfile::tempdir().expect("out");
pack(
&layout,
&Slice {
projects: vec!["sample".into()],
issues: Vec::new(),
},
out.path(),
)
.expect("packs");
verify(out.path()).expect("checks out");
std::fs::write(out.path().join("data/extra.sh"), "rm -rf /\n").expect("write");
let err = verify(out.path()).expect_err("an unlisted file passed");
assert!(format!("{err}").contains("unlisted"), "{err}");
}
#[test]
fn sealing_accounts_for_what_arrived_after_packing() {
let (_dir, layout) = tracker();
made(&layout, "first", CreateOpts::default());
let out = tempfile::tempdir().expect("out");
pack(
&layout,
&Slice {
projects: vec!["sample".into()],
issues: Vec::new(),
},
out.path(),
)
.expect("packs");
let deeds = out.path().join("data/deeds/deed-file-note");
std::fs::create_dir_all(&deeds).expect("mkdir");
std::fs::write(deeds.join("deed.bin"), b"deed bytes").expect("write");
let err = verify(out.path()).expect_err("an unsealed addition passed");
assert!(format!("{err}").contains("unlisted"), "{err}");
let report = seal(out.path()).expect("seals");
assert!(report.files >= 3, "{report:?}");
verify(out.path()).expect("a sealed satchel checks out");
}
#[test]
fn a_pack_can_put_what_the_seat_learned_in_too() {
let (_dir, layout) = tracker();
made(&layout, "first", CreateOpts::default());
let out = tempfile::tempdir().expect("out");
pack(
&layout,
&Slice {
projects: vec!["sample".into()],
issues: Vec::new(),
},
out.path(),
)
.expect("packs");
let atoms = out.path().join("data/atoms");
std::fs::create_dir_all(&atoms).expect("mkdir");
std::fs::write(
atoms.join("seat.jsonl"),
"{\"id\":\"a1\",\"text\":\"what was learned\"}\n {\"id\":\"a2\",\"text\":\"and this\"}\n",
)
.expect("write");
let sealed = seal(out.path()).expect("seals");
assert!(
sealed.notes.iter().any(|n| n.contains("2 atoms")),
"{:?}",
sealed.notes
);
let checked = verify(out.path()).expect("checks out");
assert!(
checked.notes.iter().any(|n| n.contains("2 atoms")),
"{:?}",
checked.notes
);
std::fs::write(atoms.join("late.jsonl"), "{\"id\":\"a3\"}\n").expect("write");
let err = verify(out.path()).expect_err("a late atom file passed");
assert!(format!("{err}").contains("unlisted"), "{err}");
}
#[test]
fn checking_a_satchel_says_what_it_did_not_check() {
let (_dir, layout) = tracker();
made(&layout, "first", CreateOpts::default());
let out = tempfile::tempdir().expect("out");
pack(
&layout,
&Slice {
projects: vec!["sample".into()],
issues: Vec::new(),
},
out.path(),
)
.expect("packs");
let unsigned = verify(out.path()).expect("checks out");
let said = unsigned.notes.join(" ");
assert!(
said.contains("nothing about who wrote it"),
"an unsigned satchel did not say so: {said}"
);
std::fs::write(
out.path().join("manifest-sha256.txt.sig"),
"ed25519 aa bb\n",
)
.expect("write");
let signed = verify(out.path()).expect("still checks out");
let said = signed.notes.join(" ");
assert!(said.contains("did not verify"), "{said}");
assert!(said.contains("vouch check"), "{said}");
}
#[test]
fn an_empty_or_unknown_slice_is_refused() {
let (_dir, layout) = tracker();
let out = tempfile::tempdir().expect("out");
assert!(pack(&layout, &Slice::default(), out.path()).is_err());
assert!(
pack(
&layout,
&Slice {
projects: Vec::new(),
issues: vec!["sample-nope".into()],
},
out.path(),
)
.is_err()
);
}
}