lean_ctx/core/
token_calibration.rs

1//! Provider-authoritative tokenizer calibration evidence (CO-05).
2//!
3//! Local tokenizer counts remain useful estimates, but they are not provider
4//! billing truth.  This module provides the bounded, deterministic report
5//! envelope required before a provider-count calibration can be consumed.
6//! Missing authority is deliberately rejected; callers must not silently
7//! promote local reference counts to provider evidence.
8
9use std::fmt::Write as _;
10
11use thiserror::Error;
12
13/// Stable schema identifier for tokenizer calibration reports.
14pub const TOKEN_CALIBRATION_SCHEMA_VERSION: &str = "leanctx.token-calibration/v1";
15/// Stable report version for the first calibration contract.
16pub const TOKEN_CALIBRATION_REPORT_VERSION: &str = "1.0.0";
17const MAX_ENTRIES: usize = 4_096;
18const MAX_TEXT_FIELD_CHARS: usize = 256;
19const MAX_AUTHORITY_REF_CHARS: usize = 512;
20
21/// A single provider-count observation, identified without retaining payload.
22#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
23pub struct TokenCalibrationEntry {
24    /// Immutable provider/corpus reference for the measured sample.
25    pub sample_ref: String,
26    /// Provider-reported input token count for the sample.
27    pub provider_tokens: u64,
28}
29
30/// Versioned provider-count calibration report.
31#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
32pub struct TokenCalibrationReportV1 {
33    pub schema_version: String,
34    pub report_version: String,
35    pub provider: String,
36    pub model: String,
37    pub tokenizer_family: String,
38    /// Immutable authority/evidence reference.  `None` is never consumable.
39    pub authority_ref: Option<String>,
40    /// Sorted, unique provider observations.
41    pub entries: Vec<TokenCalibrationEntry>,
42    /// `blake3:` digest over the canonical report payload excluding this field.
43    pub corpus_digest: String,
44}
45
46/// Fail-closed validation errors for calibration evidence.
47#[derive(Debug, Clone, PartialEq, Eq, Error)]
48pub enum TokenCalibrationError {
49    #[error("unsupported calibration schema version")]
50    UnsupportedSchema,
51    #[error("unsupported calibration report version")]
52    UnsupportedReportVersion,
53    #[error("{field} is empty or exceeds its bound")]
54    InvalidField { field: &'static str },
55    #[error("calibration authority is missing")]
56    MissingAuthority,
57    #[error("calibration authority reference is invalid")]
58    InvalidAuthority,
59    #[error("calibration entries are empty or exceed the bound")]
60    InvalidEntryCount,
61    #[error("calibration entries are not sorted and unique")]
62    NonCanonicalEntries,
63    #[error("calibration corpus digest is malformed")]
64    InvalidDigest,
65    #[error("calibration corpus digest does not match entries")]
66    DigestMismatch,
67    #[error("calibration report serialization failed")]
68    Serialization,
69}
70
71impl TokenCalibrationReportV1 {
72    /// Constructs an authoritative report and computes its deterministic digest.
73    pub fn new_provider_authoritative(
74        provider: impl Into<String>,
75        model: impl Into<String>,
76        tokenizer_family: impl Into<String>,
77        authority_ref: impl Into<String>,
78        mut entries: Vec<TokenCalibrationEntry>,
79    ) -> Result<Self, TokenCalibrationError> {
80        entries.sort_by(|left, right| left.sample_ref.cmp(&right.sample_ref));
81        let mut report = Self {
82            schema_version: TOKEN_CALIBRATION_SCHEMA_VERSION.to_string(),
83            report_version: TOKEN_CALIBRATION_REPORT_VERSION.to_string(),
84            provider: provider.into(),
85            model: model.into(),
86            tokenizer_family: tokenizer_family.into(),
87            authority_ref: Some(authority_ref.into()),
88            entries,
89            corpus_digest: String::new(),
90        };
91        report.corpus_digest = report.computed_digest();
92        report.validate()?;
93        Ok(report)
94    }
95
96    /// Validates all bounds, canonical ordering, authority, and digest fields.
97    pub fn validate(&self) -> Result<(), TokenCalibrationError> {
98        if self.schema_version != TOKEN_CALIBRATION_SCHEMA_VERSION {
99            return Err(TokenCalibrationError::UnsupportedSchema);
100        }
101        if self.report_version != TOKEN_CALIBRATION_REPORT_VERSION {
102            return Err(TokenCalibrationError::UnsupportedReportVersion);
103        }
104        for (field, value) in [
105            ("provider", self.provider.as_str()),
106            ("model", self.model.as_str()),
107            ("tokenizer_family", self.tokenizer_family.as_str()),
108        ] {
109            if !valid_text(value, MAX_TEXT_FIELD_CHARS) {
110                return Err(TokenCalibrationError::InvalidField { field });
111            }
112        }
113        let Some(authority_ref) = self.authority_ref.as_deref() else {
114            return Err(TokenCalibrationError::MissingAuthority);
115        };
116        if !valid_text(authority_ref, MAX_AUTHORITY_REF_CHARS) {
117            return Err(TokenCalibrationError::InvalidAuthority);
118        }
119        if self.entries.is_empty() || self.entries.len() > MAX_ENTRIES {
120            return Err(TokenCalibrationError::InvalidEntryCount);
121        }
122        let mut previous: Option<&str> = None;
123        for entry in &self.entries {
124            if !valid_text(&entry.sample_ref, MAX_AUTHORITY_REF_CHARS)
125                || previous.is_some_and(|value| value >= entry.sample_ref.as_str())
126            {
127                return Err(TokenCalibrationError::NonCanonicalEntries);
128            }
129            previous = Some(&entry.sample_ref);
130        }
131        if !is_digest(&self.corpus_digest) {
132            return Err(TokenCalibrationError::InvalidDigest);
133        }
134        if self.corpus_digest != self.computed_digest() {
135            return Err(TokenCalibrationError::DigestMismatch);
136        }
137        Ok(())
138    }
139
140    /// Returns canonical JSON only for a fully validated report.
141    pub fn canonical_json(&self) -> Result<Vec<u8>, TokenCalibrationError> {
142        self.validate()?;
143        serde_json::to_vec(self).map_err(|_| TokenCalibrationError::Serialization)
144    }
145
146    fn computed_digest(&self) -> String {
147        let mut payload = String::new();
148        append_field(&mut payload, TOKEN_CALIBRATION_SCHEMA_VERSION);
149        append_field(&mut payload, TOKEN_CALIBRATION_REPORT_VERSION);
150        append_field(&mut payload, &self.provider);
151        append_field(&mut payload, &self.model);
152        append_field(&mut payload, &self.tokenizer_family);
153        append_field(&mut payload, self.authority_ref.as_deref().unwrap_or(""));
154        for entry in &self.entries {
155            append_field(&mut payload, &entry.sample_ref);
156            let _ = write!(payload, "{}|", entry.provider_tokens);
157        }
158        format!("blake3:{}", blake3::hash(payload.as_bytes()).to_hex())
159    }
160}
161
162fn append_field(payload: &mut String, value: &str) {
163    let _ = write!(payload, "{}:", value.len());
164    payload.push_str(value);
165    payload.push('|');
166}
167
168fn valid_text(value: &str, max_chars: usize) -> bool {
169    !value.is_empty()
170        && value.chars().count() <= max_chars
171        && value.chars().all(|character| !character.is_control())
172}
173
174fn is_digest(value: &str) -> bool {
175    let Some(hex) = value.strip_prefix("blake3:") else {
176        return false;
177    };
178    hex.len() == 64
179        && hex
180            .bytes()
181            .all(|byte| byte.is_ascii_digit() || matches!(byte, b'a'..=b'f'))
182}
183
184#[cfg(test)]
185mod tests {
186    use super::*;
187
188    fn report() -> TokenCalibrationReportV1 {
189        TokenCalibrationReportV1::new_provider_authoritative(
190            "provider-a",
191            "model-a",
192            "provider-native-v1",
193            "https://evidence.example/report/1",
194            vec![
195                TokenCalibrationEntry {
196                    sample_ref: "sample:b".to_string(),
197                    provider_tokens: 11,
198                },
199                TokenCalibrationEntry {
200                    sample_ref: "sample:a".to_string(),
201                    provider_tokens: 7,
202                },
203            ],
204        )
205        .expect("valid report")
206    }
207
208    #[test]
209    fn constructor_sorts_entries_and_is_deterministic() {
210        let first = report();
211        let second = report();
212        assert_eq!(first, second);
213        assert_eq!(first.entries[0].sample_ref, "sample:a");
214        assert_eq!(first.canonical_json(), second.canonical_json());
215    }
216
217    #[test]
218    fn missing_authority_fails_closed() {
219        let mut value = report();
220        value.authority_ref = None;
221        assert_eq!(
222            value.validate(),
223            Err(TokenCalibrationError::MissingAuthority)
224        );
225        assert_eq!(
226            value.canonical_json(),
227            Err(TokenCalibrationError::MissingAuthority)
228        );
229    }
230
231    #[test]
232    fn digest_mutation_fails_closed() {
233        let mut value = report();
234        value.entries[0].provider_tokens += 1;
235        assert_eq!(value.validate(), Err(TokenCalibrationError::DigestMismatch));
236    }
237
238    #[test]
239    fn duplicate_or_unsorted_entries_fail_closed() {
240        let mut value = report();
241        value.entries.swap(0, 1);
242        value.corpus_digest = value.computed_digest();
243        assert_eq!(
244            value.validate(),
245            Err(TokenCalibrationError::NonCanonicalEntries)
246        );
247
248        value.entries[0].sample_ref = value.entries[1].sample_ref.clone();
249        value.corpus_digest = value.computed_digest();
250        assert_eq!(
251            value.validate(),
252            Err(TokenCalibrationError::NonCanonicalEntries)
253        );
254    }
255}
lean_ctx/core/token_calibration.rs

lean_ctx/core/
token_calibration.rs
