deepstrike_core/context/
measurement.rs

1//! spc_011-C-02: prompt token measurement — the three-way split the spec's §6.1 invariant
2//! ("measure provider-visible input, never underestimate") requires between what a request is
3//! *estimated* to cost before it is sent (`PromptMeasurement`, this module), what the provider
4//! actually reports back (`ProviderUsage`, Node/Python Host layer — not this crate). This module
5//! only defines a preflight fact. The `MeasurePrompt` wire tag is reserved, but A-00R removed its
6//! scheduler producer until request fingerprinting and durable replay semantics are defined.
7
8use serde::{Deserialize, Serialize};
9
10/// Where a token count came from — never just a bare `u32`, so a caller can tell "the provider's
11/// own count API said so" apart from "we guessed." The provenance remains part of the reserved
12/// contract, but no runtime currently persists or consumes this fact.
13#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
14#[serde(tag = "kind", rename_all = "snake_case")]
15pub enum MeasurementSource {
16    /// The provider's own token-counting endpoint answered (e.g. Anthropic
17    /// `messages/count_tokens`, OpenAI Responses token counting, Gemini `countTokens`).
18    Native { provider: String },
19    /// A real BPE tokenizer ran locally, but against a vocabulary that may not be the target
20    /// provider's own (e.g. cl100k standing in for a non-OpenAI vendor) — see
21    /// `FallbackEstimator` (spc_011-C-01) for the concrete counter this describes.
22    LocalExact { tokenizer: String },
23    /// No tokenizer ran at all; this is a coarse guess with a generous safety margin.
24    Heuristic,
25}
26
27/// How much to trust `PromptMeasurement.input_tokens` when deciding whether to compress. Kept
28/// separate from `MeasurementSource` — the *reason* a number exists and how much a caller should
29/// *lean on it* are different questions (a `LocalExact` cl100k count for an Anthropic request is
30/// still nontrivially uncertain, but it is not a bare guess either).
31#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
32#[serde(rename_all = "snake_case")]
33pub enum MeasurementConfidence {
34    Exact,
35    HighConfidence,
36    LowConfidence,
37}
38
39/// A single preflight token-count fact about a candidate render, for a specific provider/model.
40#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
41#[serde(deny_unknown_fields)]
42pub struct PromptMeasurement {
43    pub input_tokens: u32,
44    pub source: MeasurementSource,
45    pub confidence: MeasurementConfidence,
46}
47
48#[cfg(test)]
49mod tests {
50    use super::*;
51
52    #[test]
53    fn constructs_all_three_measurement_source_variants() {
54        let native = MeasurementSource::Native {
55            provider: "anthropic".to_string(),
56        };
57        let local_exact = MeasurementSource::LocalExact {
58            tokenizer: "cl100k_base".to_string(),
59        };
60        let heuristic = MeasurementSource::Heuristic;
61
62        assert_ne!(native, local_exact);
63        assert_ne!(local_exact, heuristic);
64    }
65
66    #[test]
67    fn prompt_measurement_round_trips_through_json() {
68        let m = PromptMeasurement {
69            input_tokens: 1234,
70            source: MeasurementSource::Native {
71                provider: "openai".to_string(),
72            },
73            confidence: MeasurementConfidence::Exact,
74        };
75        let json = serde_json::to_string(&m).unwrap();
76        let back: PromptMeasurement = serde_json::from_str(&json).unwrap();
77        assert_eq!(m, back);
78    }
79
80    #[test]
81    fn unknown_field_is_rejected() {
82        let raw = r#"{"input_tokens": 10, "source": {"kind": "heuristic"}, "confidence": "exact", "extra": true}"#;
83        let result: Result<PromptMeasurement, _> = serde_json::from_str(raw);
84        assert!(
85            result.is_err(),
86            "deny_unknown_fields must reject stray keys"
87        );
88    }
89}
deepstrike_core/context/measurement.rs

deepstrike_core/context/
measurement.rs
