rustev_eval/
holdout.rs

1//! The held-out ledger, `rustev.holdout-ledger/1` (spec 029, 3.7.3 to
2//! 3.7.5).
3//!
4//! A host-kept, hash-chained file: each entry binds one held-out repeat
5//! set's lineage, plan, repeat configuration and run reports, and the
6//! digest of the entry before it. Rustev reads a ledger it is given and
7//! returns a new one with one entry appended; it keeps no store and never
8//! modifies its input. The chain proves the ledger's internal consistency,
9//! not its completeness: a host can supply an older or truncated ledger,
10//! which the heads recorded in every repeat report let an auditor detect.
11
12use std::fmt;
13
14use rustev_contract::canonical::tagged_digest;
15use rustev_contract::ids::{ContentDigest, DatasetId, PlanId};
16use rustev_contract::limits::REPLAY_V1;
17use rustev_contract::scope::Handle;
18use serde::{Deserialize, Serialize};
19
20pub const HOLDOUT_LEDGER: &str = "rustev.holdout-ledger/1";
21/// The tag of an entry's chain digest.
22pub const ENTRY_TAG: &str = "rustev.holdout-ledger/1#entry";
23/// The most entries one ledger holds.
24pub const MAX_ENTRIES: usize = 10_000;
25/// The most run reports one entry binds (a repeat set's N).
26pub const MAX_ENTRY_RUNS: usize = 64;
27
28#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
29#[serde(deny_unknown_fields)]
30pub struct HoldoutLedger {
31    pub schema: String,
32    pub dataset: DatasetId,
33    pub entries: Vec<LedgerEntry>,
34}
35
36crate::doc::eval_document!(HoldoutLedger, HOLDOUT_LEDGER, REPLAY_V1, ContentDigest);
37
38#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
39#[serde(deny_unknown_fields)]
40pub struct LedgerEntry {
41    /// The host's lineage handle for what is being evaluated.
42    pub lineage: Handle,
43    pub plan: PlanId,
44    pub repeat_config: ContentDigest,
45    /// The record digests of the repeat set's run reports.
46    pub runs: Vec<ContentDigest>,
47    /// Supplied by the host, never read from a clock.
48    pub recorded_at_ms: i64,
49    /// Null for the first entry; otherwise the chain digest of the entry
50    /// before.
51    pub prev: Option<ContentDigest>,
52}
53
54/// What one held-out repeat report records (3.7.4): the entries with this
55/// lineage, with this plan and in total, each including the new one, and
56/// the heads before and after (none for an empty ledger).
57#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
58#[serde(deny_unknown_fields)]
59pub struct HeldOutRecord {
60    pub lineage: Handle,
61    pub count: u64,
62    pub plan_count: u64,
63    pub dataset_count: u64,
64    pub ledger_head_before: Option<ContentDigest>,
65    pub ledger_head_after: ContentDigest,
66}
67
68/// Why a ledger was refused (`invalid_input`).
69#[derive(Debug, Clone, PartialEq, Eq)]
70pub enum LedgerError {
71    Schema,
72    OtherDataset,
73    TooManyEntries(usize),
74    /// An entry with no run, or more than a repeat set's 64.
75    MalformedEntry(usize),
76    /// Entry `i`'s `prev` is not the chain digest of entry `i - 1`.
77    BrokenChain(usize),
78    NotCanonical,
79}
80
81impl fmt::Display for LedgerError {
82    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
83        match self {
84            LedgerError::Schema => write!(f, "not a rustev.holdout-ledger/1 document"),
85            LedgerError::OtherDataset => write!(f, "the ledger is for another dataset"),
86            LedgerError::TooManyEntries(n) => write!(f, "{n} entries exceed {MAX_ENTRIES}"),
87            LedgerError::MalformedEntry(i) => write!(f, "entry {i} binds no run or more than 64"),
88            LedgerError::BrokenChain(i) => write!(f, "entry {i} does not extend the chain"),
89            LedgerError::NotCanonical => write!(f, "an entry has no canonical form"),
90        }
91    }
92}
93
94impl std::error::Error for LedgerError {}
95
96/// `tagged_digest("rustev.holdout-ledger/1#entry", canonical bytes)`.
97pub fn entry_digest(entry: &LedgerEntry) -> Result<ContentDigest, LedgerError> {
98    let bytes = rustev_contract::canonical::canonical_bytes(entry)
99        .map_err(|_| LedgerError::NotCanonical)?;
100    ContentDigest::parse(&tagged_digest(ENTRY_TAG, &bytes)).map_err(|_| LedgerError::NotCanonical)
101}
102
103/// What one append binds.
104#[derive(Debug, Clone, PartialEq, Eq)]
105pub struct Append {
106    pub lineage: Handle,
107    pub plan: PlanId,
108    pub repeat_config: ContentDigest,
109    pub runs: Vec<ContentDigest>,
110    pub recorded_at_ms: i64,
111}
112
113impl HoldoutLedger {
114    /// The explicit new ledger of 3.7.3.
115    pub fn new(dataset: DatasetId) -> HoldoutLedger {
116        HoldoutLedger {
117            schema: HOLDOUT_LEDGER.into(),
118            dataset,
119            entries: vec![],
120        }
121    }
122
123    /// The schema, the dataset, the bound, every entry's shape and the
124    /// chain.
125    pub fn verify(&self, dataset: &DatasetId) -> Result<(), LedgerError> {
126        if self.schema != HOLDOUT_LEDGER {
127            return Err(LedgerError::Schema);
128        }
129        if &self.dataset != dataset {
130            return Err(LedgerError::OtherDataset);
131        }
132        if self.entries.len() > MAX_ENTRIES {
133            return Err(LedgerError::TooManyEntries(self.entries.len()));
134        }
135        let mut prev = None;
136        for (i, e) in self.entries.iter().enumerate() {
137            if e.runs.is_empty() || e.runs.len() > MAX_ENTRY_RUNS {
138                return Err(LedgerError::MalformedEntry(i));
139            }
140            if e.prev != prev {
141                return Err(LedgerError::BrokenChain(i));
142            }
143            prev = Some(entry_digest(e)?);
144        }
145        Ok(())
146    }
147
148    /// The chain digest of the last entry; none for an empty ledger.
149    pub fn head(&self) -> Result<Option<ContentDigest>, LedgerError> {
150        self.entries.last().map(entry_digest).transpose()
151    }
152
153    /// Verify this ledger and return a new one with one entry appended,
154    /// with the counts and heads a repeat report records. `self` is never
155    /// modified.
156    pub fn append(
157        &self,
158        dataset: &DatasetId,
159        a: Append,
160    ) -> Result<(HoldoutLedger, HeldOutRecord), LedgerError> {
161        self.verify(dataset)?;
162        if self.entries.len() >= MAX_ENTRIES {
163            return Err(LedgerError::TooManyEntries(self.entries.len() + 1));
164        }
165        if a.runs.is_empty() || a.runs.len() > MAX_ENTRY_RUNS {
166            return Err(LedgerError::MalformedEntry(self.entries.len()));
167        }
168        let before = self.head()?;
169        let entry = LedgerEntry {
170            lineage: a.lineage,
171            plan: a.plan,
172            repeat_config: a.repeat_config,
173            runs: a.runs,
174            recorded_at_ms: a.recorded_at_ms,
175            prev: before.clone(),
176        };
177        let after = entry_digest(&entry)?;
178        let mut next = self.clone();
179        next.entries.push(entry);
180        let new = next.entries.last().expect("just appended");
181        let count =
182            |f: &dyn Fn(&LedgerEntry) -> bool| next.entries.iter().filter(|e| f(e)).count() as u64;
183        let record = HeldOutRecord {
184            lineage: new.lineage.clone(),
185            count: count(&|e| e.lineage == new.lineage),
186            plan_count: count(&|e| e.plan == new.plan),
187            dataset_count: next.entries.len() as u64,
188            ledger_head_before: before,
189            ledger_head_after: after,
190        };
191        Ok((next, record))
192    }
193}
rustev_eval/holdout.rs

rustev_eval/
holdout.rs
