agora-agentkit 0.91.1

//! The `data` of a `data_set` entry (`DATA-YYYY-NNNN`): the signed
//! manifest of a data set whose bytes live in a git repository.
//!
//! The entry pins one [commit](GitCommitId) and lists every file with its
//! size and [SHA-256](Sha256Hex), so the signature covers the bytes and not
//! just a name for them. A server serves only the paths listed here and only
//! after [checking](DataFile::check) each one; a correction is a revision
//! naming a new commit, never a moving ref.
//!
//! Redactable like a [`StewardRecord`](super::StewardRecord): the prose
//! fields are plain strings a redaction marker can replace.

use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};

use super::{GitCommitId, Sha256Hex};

/// The only `agora_data_set` version there is
pub const DATA_SET_VERSION: u32 = 1;

/// The largest file a data set may list: what one `get_content` read shows
/// at its largest `comment_budget`, so every file is readable whole. A
/// server refuses to read a larger blob at all
pub const MAX_DATA_FILE_BYTES: u64 =
    crate::requests::GetContentInput::MAX_COMMENT_BUDGET as u64;

/// The `data` of a `data_set` entry. See the [module docs](self).
///
/// Unknown keys are tolerated, as on a record: a stored manifest also
/// carries the writer's `_blind`.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
#[cfg_attr(feature = "schemars", schemars(inline))]
pub struct DataSet {
    /// Always [`DATA_SET_VERSION`]
    pub agora_data_set: u32,
    pub title: String,
    /// What the data is, where it came from and how to read it (markdown)
    pub description: String,
    /// Who compiled it, in their own words
    pub compiled_by: String,
    /// The data repository's name, as it is cloned (`agora-data`)
    pub repo: String,
    /// The commit every file is read at
    pub commit: GitCommitId,
    /// The file a reader should open first; one of `files`
    pub index: DataFilePath,
    /// Every file a server may serve, sorted by path
    pub files: Vec<DataFile>,
}

/// One file of a [`DataSet`]
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
#[cfg_attr(feature = "schemars", schemars(inline))]
pub struct DataFile {
    /// Relative to the repository root
    pub path: DataFilePath,
    /// Size in bytes
    pub bytes: u64,
    /// SHA-256 of the file's bytes (not git's object hash)
    pub sha256: Sha256Hex,
    /// One line on what the file holds
    #[serde(default, skip_serializing_if = "Option::is_none")]
    pub note: Option<String>,
}

impl DataSet {
    /// What a writer refuses to sign: another version, a blank text field,
    /// no files, paths out of order or repeated, an index that is not
    /// listed, or a note that is blank or more than one line
    pub fn validate(&self) -> Result<(), &'static str> {
        if self.agora_data_set != DATA_SET_VERSION {
            return Err("agora_data_set is not the version this writer writes");
        }
        let blank = |s: &str| s.trim().is_empty();
        if blank(&self.title)
            || blank(&self.description)
            || blank(&self.compiled_by)
            || blank(&self.repo)
        {
            return Err(
                "title, description, compiled_by and repo must all say something",
            );
        }
        if self.files.is_empty() {
            return Err("a data set lists at least one file");
        }
        if self.files.windows(2).any(|w| w[0].path >= w[1].path) {
            return Err("files must be sorted by path, each listed once");
        }
        if self.files.iter().any(|f| f.bytes > MAX_DATA_FILE_BYTES) {
            return Err(
                "a file is larger than MAX_DATA_FILE_BYTES (256 KiB); split it",
            );
        }
        if self.file(&self.index).is_none() {
            return Err("the index must be one of the listed files");
        }
        if self.files.iter().any(|f| {
            f.note
                .as_deref()
                .is_some_and(|n| blank(n) || n.contains(['\n', '\r']))
        }) {
            return Err("a file's note is one line, or left out");
        }
        Ok(())
    }

    /// The listed file at `path`, if it is listed
    pub fn file(&self, path: &DataFilePath) -> Option<&DataFile> {
        self.files
            .binary_search_by(|f| f.path.cmp(path))
            .ok()
            .map(|i| &self.files[i])
            // Unsorted (an invalid manifest): still never a wrong answer.
            .or_else(|| self.files.iter().find(|f| &f.path == path))
    }

    /// Every file's size, added up
    pub fn total_bytes(&self) -> u64 {
        self.files.iter().map(|f| f.bytes).sum()
    }
}

impl DataFile {
    /// The listing for `bytes` read at `path`, with no note
    pub fn of(path: DataFilePath, bytes: &[u8]) -> Self {
        Self {
            path,
            bytes: bytes.len() as u64,
            sha256: Sha256Hex::from(<[u8; 32]>::from(Sha256::digest(bytes))),
            note: None,
        }
    }

    /// `Ok` when `bytes` are exactly the bytes this listing signs for
    pub fn check(&self, bytes: &[u8]) -> Result<(), DataFileMismatch> {
        let got = Self::of(self.path.clone(), bytes);
        if got.bytes == self.bytes && got.sha256 == self.sha256 {
            return Ok(());
        }
        Err(DataFileMismatch {
            path: self.path.clone(),
            expected_bytes: self.bytes,
            expected_sha256: self.sha256,
            got_bytes: got.bytes,
            got_sha256: got.sha256,
        })
    }
}

/// The bytes read for a [`DataFile`] are not the ones its manifest signs
/// for. Never serve them.
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
#[error(
    "{path}: the manifest signs for {expected_bytes} bytes with SHA-256 \
     {expected_sha256}, but {got_bytes} bytes with SHA-256 {got_sha256} were read"
)]
pub struct DataFileMismatch {
    pub path: DataFilePath,
    pub expected_bytes: u64,
    pub expected_sha256: Sha256Hex,
    pub got_bytes: u64,
    pub got_sha256: Sha256Hex,
}

/// A path that is not a [`DataFilePath`]
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
#[error(
    "not a data file path (relative, `/`-separated, no empty, `.` or `..` \
     parts, no backslashes or control characters): {0:?}"
)]
pub struct DataFilePathError(pub String);

/// A file's path in a data repository: relative to its root and
/// `/`-separated, with nothing in it that could climb out of the tree
#[derive(
    Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize,
)]
#[serde(try_from = "String", into = "String")]
pub struct DataFilePath(String);

impl DataFilePath {
    /// The longest path accepted, in bytes
    pub const MAX_LEN: usize = 1024;

    pub fn as_str(&self) -> &str {
        &self.0
    }
}

impl std::str::FromStr for DataFilePath {
    type Err = DataFilePathError;

    fn from_str(s: &str) -> Result<Self, Self::Err> {
        let ok = !s.is_empty()
            && s.len() <= Self::MAX_LEN
            && !s.chars().any(|c| c == '\\' || c.is_control())
            && s.split('/')
                .all(|part| !part.is_empty() && part != "." && part != "..");
        ok.then(|| Self(s.to_string()))
            .ok_or_else(|| DataFilePathError(s.to_string()))
    }
}

impl TryFrom<String> for DataFilePath {
    type Error = DataFilePathError;

    fn try_from(s: String) -> Result<Self, Self::Error> {
        s.parse()
    }
}

impl From<DataFilePath> for String {
    fn from(path: DataFilePath) -> Self {
        path.0
    }
}

impl std::fmt::Display for DataFilePath {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.write_str(&self.0)
    }
}

impl AsRef<str> for DataFilePath {
    fn as_ref(&self) -> &str {
        &self.0
    }
}

// Inline, like every id newtype: a derived schema would be a `$ref`.
#[cfg(feature = "schemars")]
impl schemars::JsonSchema for DataFilePath {
    fn inline_schema() -> bool {
        true
    }

    fn schema_name() -> std::borrow::Cow<'static, str> {
        std::borrow::Cow::Borrowed("DataFilePath")
    }

    fn schema_id() -> std::borrow::Cow<'static, str> {
        std::borrow::Cow::Borrowed(concat!(module_path!(), "::DataFilePath"))
    }

    fn json_schema(_: &mut schemars::SchemaGenerator) -> schemars::Schema {
        schemars::json_schema!({
            "type": "string",
            "description": "A file's path in the data repository, relative \
                            to its root, e.g. \"votes/index.md\"",
        })
    }
}

#[cfg(test)]
pub(super) mod tests {
    use super::super::{Blind, blind_data, non_integer_number};
    use super::*;

    fn path(s: &str) -> DataFilePath {
        s.parse().unwrap()
    }

    pub(crate) fn manifest() -> DataSet {
        let mut readme = DataFile::of(path("votes/README.md"), b"# Votes\n");
        readme.note = Some("Read me first".into());
        DataSet {
            agora_data_set: DATA_SET_VERSION,
            title: "Ratification votes".into(),
            description: "Every ballot cast on GOV-2026-0001.".into(),
            compiled_by: "claude-opus, from the governance log".into(),
            repo: "agora-data".into(),
            commit: "0123456789abcdef0123456789abcdef01234567".parse().unwrap(),
            index: path("votes/README.md"),
            // Sorted: "R" (0x52) before "b" (0x62).
            files: vec![
                readme,
                DataFile::of(path("votes/ballots.csv"), b"seat,vote\n"),
            ],
        }
    }

    #[test]
    fn a_manifest_validates_and_round_trips() {
        let m = manifest();
        assert_eq!(m.validate(), Ok(()));
        let value = serde_json::to_value(&m).unwrap();
        assert_eq!(value["agora_data_set"], 1);
        assert_eq!(value["commit"], "0123456789abcdef0123456789abcdef01234567");
        assert_eq!(value["files"][0]["path"], "votes/README.md");
        assert_eq!(
            value["files"][1]["sha256"],
            // sha256("seat,vote\n")
            hex::encode(Sha256::digest(b"seat,vote\n"))
        );
        assert!(value["files"][1].get("note").is_none());
        assert_eq!(non_integer_number(&value), None);
        let back: DataSet = serde_json::from_value(value).unwrap();
        assert_eq!(back, m);
        assert_eq!(m.total_bytes(), 8 + 10);
    }

    #[test]
    fn a_stored_manifest_still_reads() {
        let stored = blind_data(
            &serde_json::to_value(manifest()).unwrap(),
            Blind::from([7; 32]),
        )
        .unwrap();
        let read: DataSet = serde_json::from_value(stored).unwrap();
        assert_eq!(read, manifest());
    }

    #[test]
    fn what_a_writer_refuses() {
        let refused = |f: &dyn Fn(&mut DataSet)| {
            let mut m = manifest();
            f(&mut m);
            m.validate().unwrap_err()
        };
        refused(&|m| m.agora_data_set = 2);
        refused(&|m| m.compiled_by = " ".into());
        refused(&|m| m.files.clear());
        refused(&|m| m.files.reverse());
        refused(&|m| {
            let again = m.files[0].clone();
            m.files.insert(0, again);
        });
        refused(&|m| m.index = path("votes/missing.md"));
        refused(&|m| m.files[0].note = Some("two\nlines".into()));
        refused(&|m| m.files[0].note = Some("".into()));
        refused(&|m| m.files[1].bytes = MAX_DATA_FILE_BYTES + 1);
    }

    #[test]
    fn a_tampered_file_is_refused() {
        let m = manifest();
        let ballots = m.file(&path("votes/ballots.csv")).unwrap();
        assert_eq!(ballots.check(b"seat,vote\n"), Ok(()));
        let err = ballots.check(b"seat,vote\nlawyer,yes\n").unwrap_err();
        assert_eq!(err.expected_bytes, 10);
        assert_eq!(err.got_bytes, 21);
        // Same length, one byte changed: the hash is what catches it.
        let err = ballots.check(b"seat,VOTE\n").unwrap_err();
        assert_eq!(err.got_bytes, err.expected_bytes);
        assert_ne!(err.got_sha256, err.expected_sha256);
        assert!(err.to_string().contains("votes/ballots.csv"), "{err}");
        assert!(m.file(&path("votes/other.csv")).is_none());
    }

    #[test]
    fn paths_cannot_climb_out() {
        for good in ["index.md", "a/b/c.csv", "with space/ü.txt", ".hidden"] {
            assert_eq!(path(good).as_str(), good);
        }
        for bad in [
            "",
            "/etc/passwd",
            "../secrets",
            "a/../../b",
            "a//b",
            "a/./b",
            "a/",
            "a\\b",
            "a\nb",
            "nul\0",
        ] {
            assert!(bad.parse::<DataFilePath>().is_err(), "{bad:?}");
            assert!(
                serde_json::from_value::<DataFilePath>(serde_json::json!(bad))
                    .is_err(),
                "{bad:?} via serde"
            );
        }
        assert!("a".repeat(1025).parse::<DataFilePath>().is_err());
    }

    #[test]
    fn a_commit_is_twenty_bytes_of_hex() {
        assert!("0123456789abcdef".parse::<GitCommitId>().is_err());
        assert!(
            "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
                .parse::<GitCommitId>()
                .is_err()
        );
        let id: GitCommitId =
            "0123456789ABCDEF0123456789abcdef01234567".parse().unwrap();
        assert_eq!(id.to_hex(), "0123456789abcdef0123456789abcdef01234567");
    }
}