agora_agentkit/govlog/
data_set.rs

1//! The `data` of a `data_set` entry (`DATA-YYYY-NNNN`): the signed
2//! manifest of a data set whose bytes live in a git repository.
3//!
4//! The entry pins one [commit](GitCommitId) and lists every file with its
5//! size and [SHA-256](Sha256Hex), so the signature covers the bytes and not
6//! just a name for them. A server serves only the paths listed here and only
7//! after [checking](DataFile::check) each one; a correction is a revision
8//! naming a new commit, never a moving ref.
9//!
10//! Redactable like a [`StewardRecord`](super::StewardRecord): the prose
11//! fields are plain strings a redaction marker can replace.
12
13use serde::{Deserialize, Serialize};
14use sha2::{Digest, Sha256};
15
16use super::{GitCommitId, Sha256Hex};
17
18/// The only `agora_data_set` version there is
19pub const DATA_SET_VERSION: u32 = 1;
20
21/// The largest file a data set may list: what one `get_content` read shows
22/// at its largest `comment_budget`, so every file is readable whole. A
23/// server refuses to read a larger blob at all
24pub const MAX_DATA_FILE_BYTES: u64 =
25    crate::requests::GetContentInput::MAX_COMMENT_BUDGET as u64;
26
27/// The `data` of a `data_set` entry. See the [module docs](self).
28///
29/// Unknown keys are tolerated, as on a record: a stored manifest also
30/// carries the writer's `_blind`.
31#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
32#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
33#[cfg_attr(feature = "schemars", schemars(inline))]
34pub struct DataSet {
35    /// Always [`DATA_SET_VERSION`]
36    pub agora_data_set: u32,
37    pub title: String,
38    /// What the data is, where it came from and how to read it (markdown)
39    pub description: String,
40    /// Who compiled it, in their own words
41    pub compiled_by: String,
42    /// The data repository's name, as it is cloned (`agora-data`)
43    pub repo: String,
44    /// The commit every file is read at
45    pub commit: GitCommitId,
46    /// The file a reader should open first; one of `files`
47    pub index: DataFilePath,
48    /// Every file a server may serve, sorted by path
49    pub files: Vec<DataFile>,
50}
51
52/// One file of a [`DataSet`]
53#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
54#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
55#[cfg_attr(feature = "schemars", schemars(inline))]
56pub struct DataFile {
57    /// Relative to the repository root
58    pub path: DataFilePath,
59    /// Size in bytes
60    pub bytes: u64,
61    /// SHA-256 of the file's bytes (not git's object hash)
62    pub sha256: Sha256Hex,
63    /// One line on what the file holds
64    #[serde(default, skip_serializing_if = "Option::is_none")]
65    pub note: Option<String>,
66}
67
68impl DataSet {
69    /// What a writer refuses to sign: another version, a blank text field,
70    /// no files, paths out of order or repeated, an index that is not
71    /// listed, or a note that is blank or more than one line
72    pub fn validate(&self) -> Result<(), &'static str> {
73        if self.agora_data_set != DATA_SET_VERSION {
74            return Err("agora_data_set is not the version this writer writes");
75        }
76        let blank = |s: &str| s.trim().is_empty();
77        if blank(&self.title)
78            || blank(&self.description)
79            || blank(&self.compiled_by)
80            || blank(&self.repo)
81        {
82            return Err(
83                "title, description, compiled_by and repo must all say something",
84            );
85        }
86        if self.files.is_empty() {
87            return Err("a data set lists at least one file");
88        }
89        if self.files.windows(2).any(|w| w[0].path >= w[1].path) {
90            return Err("files must be sorted by path, each listed once");
91        }
92        if self.files.iter().any(|f| f.bytes > MAX_DATA_FILE_BYTES) {
93            return Err(
94                "a file is larger than MAX_DATA_FILE_BYTES (256 KiB); split it",
95            );
96        }
97        if self.file(&self.index).is_none() {
98            return Err("the index must be one of the listed files");
99        }
100        if self.files.iter().any(|f| {
101            f.note
102                .as_deref()
103                .is_some_and(|n| blank(n) || n.contains(['\n', '\r']))
104        }) {
105            return Err("a file's note is one line, or left out");
106        }
107        Ok(())
108    }
109
110    /// The listed file at `path`, if it is listed
111    pub fn file(&self, path: &DataFilePath) -> Option<&DataFile> {
112        self.files
113            .binary_search_by(|f| f.path.cmp(path))
114            .ok()
115            .map(|i| &self.files[i])
116            // Unsorted (an invalid manifest): still never a wrong answer.
117            .or_else(|| self.files.iter().find(|f| &f.path == path))
118    }
119
120    /// Every file's size, added up
121    pub fn total_bytes(&self) -> u64 {
122        self.files.iter().map(|f| f.bytes).sum()
123    }
124}
125
126impl DataFile {
127    /// The listing for `bytes` read at `path`, with no note
128    pub fn of(path: DataFilePath, bytes: &[u8]) -> Self {
129        Self {
130            path,
131            bytes: bytes.len() as u64,
132            sha256: Sha256Hex::from(<[u8; 32]>::from(Sha256::digest(bytes))),
133            note: None,
134        }
135    }
136
137    /// `Ok` when `bytes` are exactly the bytes this listing signs for
138    pub fn check(&self, bytes: &[u8]) -> Result<(), DataFileMismatch> {
139        let got = Self::of(self.path.clone(), bytes);
140        if got.bytes == self.bytes && got.sha256 == self.sha256 {
141            return Ok(());
142        }
143        Err(DataFileMismatch {
144            path: self.path.clone(),
145            expected_bytes: self.bytes,
146            expected_sha256: self.sha256,
147            got_bytes: got.bytes,
148            got_sha256: got.sha256,
149        })
150    }
151}
152
153/// The bytes read for a [`DataFile`] are not the ones its manifest signs
154/// for. Never serve them.
155#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
156#[error(
157    "{path}: the manifest signs for {expected_bytes} bytes with SHA-256 \
158     {expected_sha256}, but {got_bytes} bytes with SHA-256 {got_sha256} were read"
159)]
160pub struct DataFileMismatch {
161    pub path: DataFilePath,
162    pub expected_bytes: u64,
163    pub expected_sha256: Sha256Hex,
164    pub got_bytes: u64,
165    pub got_sha256: Sha256Hex,
166}
167
168/// A path that is not a [`DataFilePath`]
169#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
170#[error(
171    "not a data file path (relative, `/`-separated, no empty, `.` or `..` \
172     parts, no backslashes or control characters): {0:?}"
173)]
174pub struct DataFilePathError(pub String);
175
176/// A file's path in a data repository: relative to its root and
177/// `/`-separated, with nothing in it that could climb out of the tree
178#[derive(
179    Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize,
180)]
181#[serde(try_from = "String", into = "String")]
182pub struct DataFilePath(String);
183
184impl DataFilePath {
185    /// The longest path accepted, in bytes
186    pub const MAX_LEN: usize = 1024;
187
188    pub fn as_str(&self) -> &str {
189        &self.0
190    }
191}
192
193impl std::str::FromStr for DataFilePath {
194    type Err = DataFilePathError;
195
196    fn from_str(s: &str) -> Result<Self, Self::Err> {
197        let ok = !s.is_empty()
198            && s.len() <= Self::MAX_LEN
199            && !s.chars().any(|c| c == '\\' || c.is_control())
200            && s.split('/')
201                .all(|part| !part.is_empty() && part != "." && part != "..");
202        ok.then(|| Self(s.to_string()))
203            .ok_or_else(|| DataFilePathError(s.to_string()))
204    }
205}
206
207impl TryFrom<String> for DataFilePath {
208    type Error = DataFilePathError;
209
210    fn try_from(s: String) -> Result<Self, Self::Error> {
211        s.parse()
212    }
213}
214
215impl From<DataFilePath> for String {
216    fn from(path: DataFilePath) -> Self {
217        path.0
218    }
219}
220
221impl std::fmt::Display for DataFilePath {
222    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
223        f.write_str(&self.0)
224    }
225}
226
227impl AsRef<str> for DataFilePath {
228    fn as_ref(&self) -> &str {
229        &self.0
230    }
231}
232
233// Inline, like every id newtype: a derived schema would be a `$ref`.
234#[cfg(feature = "schemars")]
235impl schemars::JsonSchema for DataFilePath {
236    fn inline_schema() -> bool {
237        true
238    }
239
240    fn schema_name() -> std::borrow::Cow<'static, str> {
241        std::borrow::Cow::Borrowed("DataFilePath")
242    }
243
244    fn schema_id() -> std::borrow::Cow<'static, str> {
245        std::borrow::Cow::Borrowed(concat!(module_path!(), "::DataFilePath"))
246    }
247
248    fn json_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
249        schemars::json_schema!({
250            "type": "string",
251            "description": "A file's path in the data repository, relative \
252                            to its root, e.g. \"votes/index.md\"",
253        })
254    }
255}
256
257#[cfg(test)]
258pub(super) mod tests {
259    use super::super::{Blind, blind_data, non_integer_number};
260    use super::*;
261
262    fn path(s: &str) -> DataFilePath {
263        s.parse().unwrap()
264    }
265
266    pub(crate) fn manifest() -> DataSet {
267        let mut readme = DataFile::of(path("votes/README.md"), b"# Votes\n");
268        readme.note = Some("Read me first".into());
269        DataSet {
270            agora_data_set: DATA_SET_VERSION,
271            title: "Ratification votes".into(),
272            description: "Every ballot cast on GOV-2026-0001.".into(),
273            compiled_by: "claude-opus, from the governance log".into(),
274            repo: "agora-data".into(),
275            commit: "0123456789abcdef0123456789abcdef01234567".parse().unwrap(),
276            index: path("votes/README.md"),
277            // Sorted: "R" (0x52) before "b" (0x62).
278            files: vec![
279                readme,
280                DataFile::of(path("votes/ballots.csv"), b"seat,vote\n"),
281            ],
282        }
283    }
284
285    #[test]
286    fn a_manifest_validates_and_round_trips() {
287        let m = manifest();
288        assert_eq!(m.validate(), Ok(()));
289        let value = serde_json::to_value(&m).unwrap();
290        assert_eq!(value["agora_data_set"], 1);
291        assert_eq!(value["commit"], "0123456789abcdef0123456789abcdef01234567");
292        assert_eq!(value["files"][0]["path"], "votes/README.md");
293        assert_eq!(
294            value["files"][1]["sha256"],
295            // sha256("seat,vote\n")
296            hex::encode(Sha256::digest(b"seat,vote\n"))
297        );
298        assert!(value["files"][1].get("note").is_none());
299        assert_eq!(non_integer_number(&value), None);
300        let back: DataSet = serde_json::from_value(value).unwrap();
301        assert_eq!(back, m);
302        assert_eq!(m.total_bytes(), 8 + 10);
303    }
304
305    #[test]
306    fn a_stored_manifest_still_reads() {
307        let stored = blind_data(
308            &serde_json::to_value(manifest()).unwrap(),
309            Blind::from([7; 32]),
310        )
311        .unwrap();
312        let read: DataSet = serde_json::from_value(stored).unwrap();
313        assert_eq!(read, manifest());
314    }
315
316    #[test]
317    fn what_a_writer_refuses() {
318        let refused = |f: &dyn Fn(&mut DataSet)| {
319            let mut m = manifest();
320            f(&mut m);
321            m.validate().unwrap_err()
322        };
323        refused(&|m| m.agora_data_set = 2);
324        refused(&|m| m.compiled_by = " ".into());
325        refused(&|m| m.files.clear());
326        refused(&|m| m.files.reverse());
327        refused(&|m| {
328            let again = m.files[0].clone();
329            m.files.insert(0, again);
330        });
331        refused(&|m| m.index = path("votes/missing.md"));
332        refused(&|m| m.files[0].note = Some("two\nlines".into()));
333        refused(&|m| m.files[0].note = Some("".into()));
334        refused(&|m| m.files[1].bytes = MAX_DATA_FILE_BYTES + 1);
335    }
336
337    #[test]
338    fn a_tampered_file_is_refused() {
339        let m = manifest();
340        let ballots = m.file(&path("votes/ballots.csv")).unwrap();
341        assert_eq!(ballots.check(b"seat,vote\n"), Ok(()));
342        let err = ballots.check(b"seat,vote\nlawyer,yes\n").unwrap_err();
343        assert_eq!(err.expected_bytes, 10);
344        assert_eq!(err.got_bytes, 21);
345        // Same length, one byte changed: the hash is what catches it.
346        let err = ballots.check(b"seat,VOTE\n").unwrap_err();
347        assert_eq!(err.got_bytes, err.expected_bytes);
348        assert_ne!(err.got_sha256, err.expected_sha256);
349        assert!(err.to_string().contains("votes/ballots.csv"), "{err}");
350        assert!(m.file(&path("votes/other.csv")).is_none());
351    }
352
353    #[test]
354    fn paths_cannot_climb_out() {
355        for good in ["index.md", "a/b/c.csv", "with space/ü.txt", ".hidden"] {
356            assert_eq!(path(good).as_str(), good);
357        }
358        for bad in [
359            "",
360            "/etc/passwd",
361            "../secrets",
362            "a/../../b",
363            "a//b",
364            "a/./b",
365            "a/",
366            "a\\b",
367            "a\nb",
368            "nul\0",
369        ] {
370            assert!(bad.parse::<DataFilePath>().is_err(), "{bad:?}");
371            assert!(
372                serde_json::from_value::<DataFilePath>(serde_json::json!(bad))
373                    .is_err(),
374                "{bad:?} via serde"
375            );
376        }
377        assert!("a".repeat(1025).parse::<DataFilePath>().is_err());
378    }
379
380    #[test]
381    fn a_commit_is_twenty_bytes_of_hex() {
382        assert!("0123456789abcdef".parse::<GitCommitId>().is_err());
383        assert!(
384            "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
385                .parse::<GitCommitId>()
386                .is_err()
387        );
388        let id: GitCommitId =
389            "0123456789ABCDEF0123456789abcdef01234567".parse().unwrap();
390        assert_eq!(id.to_hex(), "0123456789abcdef0123456789abcdef01234567");
391    }
392}
agora_agentkit/govlog/data_set.rs

agora_agentkit/govlog/
data_set.rs
