deepstrike_core/context/
measurement.rs

1//! spc_011-C-02: prompt token measurement — the three-way split the spec's §6.1 invariant
2//! ("measure provider-visible input, never underestimate") requires between what a request is
3//! *estimated* to cost before it is sent (`PromptMeasurement`, this module), what the provider
4//! actually reports back (`ProviderUsage`, Node/Python Host layer — not this crate). This module
5//! only defines a preflight fact. The `MeasurePrompt` wire tag is reserved, but A-00R removed its
6//! scheduler producer until request fingerprinting and durable replay semantics are defined.
7
8use serde::{Deserialize, Serialize};
9
10/// Where a token count came from — never just a bare `u32`, so a caller can tell "the provider's
11/// own count API said so" apart from "we guessed." The provenance remains part of the reserved
12/// contract, but no runtime currently persists or consumes this fact.
13#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
14#[serde(tag = "kind", rename_all = "snake_case")]
15pub enum MeasurementSource {
16    /// The provider's own token-counting endpoint answered (e.g. Anthropic
17    /// `messages/count_tokens`, OpenAI Responses token counting, Gemini `countTokens`).
18    Native { provider: String },
19    /// A real BPE tokenizer ran locally, but against a vocabulary that may not be the target
20    /// provider's own (e.g. cl100k standing in for a non-OpenAI vendor) — see
21    /// `FallbackEstimator` (spc_011-C-01) for the concrete counter this describes.
22    LocalExact { tokenizer: String },
23    /// The provider's own postflight usage for this exact request, fed back after execution
24    /// (spc_024-06). The authority a replay reuses — never a preflight guess.
25    Postflight,
26    /// No tokenizer ran at all; this is a coarse guess with a generous safety margin.
27    Heuristic,
28}
29
30/// How much to trust `PromptMeasurement.input_tokens` when deciding whether to compress. Kept
31/// separate from `MeasurementSource` — the *reason* a number exists and how much a caller should
32/// *lean on it* are different questions (a `LocalExact` cl100k count for an Anthropic request is
33/// still nontrivially uncertain, but it is not a bare guess either).
34#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
35#[serde(rename_all = "snake_case")]
36pub enum MeasurementConfidence {
37    Exact,
38    HighConfidence,
39    LowConfidence,
40}
41
42/// A single preflight token-count fact about a candidate render, for a specific provider/model.
43#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
44#[serde(deny_unknown_fields)]
45pub struct PromptMeasurement {
46    pub input_tokens: u32,
47    pub source: MeasurementSource,
48    pub confidence: MeasurementConfidence,
49}
50
51#[cfg(test)]
52mod tests {
53    use super::*;
54
55    #[test]
56    fn constructs_all_three_measurement_source_variants() {
57        let native = MeasurementSource::Native {
58            provider: "anthropic".to_string(),
59        };
60        let local_exact = MeasurementSource::LocalExact {
61            tokenizer: "cl100k_base".to_string(),
62        };
63        let heuristic = MeasurementSource::Heuristic;
64
65        assert_ne!(native, local_exact);
66        assert_ne!(local_exact, heuristic);
67    }
68
69    #[test]
70    fn prompt_measurement_round_trips_through_json() {
71        let m = PromptMeasurement {
72            input_tokens: 1234,
73            source: MeasurementSource::Native {
74                provider: "openai".to_string(),
75            },
76            confidence: MeasurementConfidence::Exact,
77        };
78        let json = serde_json::to_string(&m).unwrap();
79        let back: PromptMeasurement = serde_json::from_str(&json).unwrap();
80        assert_eq!(m, back);
81    }
82
83    #[test]
84    fn postflight_source_round_trips_and_differs_from_preflight_kinds() {
85        // spc_024-06: observed usage fed back after execution — the authority a replay reuses.
86        let m = PromptMeasurement {
87            input_tokens: 1000,
88            source: MeasurementSource::Postflight,
89            confidence: MeasurementConfidence::Exact,
90        };
91        let json = serde_json::to_string(&m).unwrap();
92        let back: PromptMeasurement = serde_json::from_str(&json).unwrap();
93        assert_eq!(m, back);
94        assert_eq!(
95            serde_json::to_string(&m.source).unwrap(),
96            r#"{"kind":"postflight"}"#
97        );
98    }
99
100    #[test]
101    fn unknown_field_is_rejected() {
102        let raw = r#"{"input_tokens": 10, "source": {"kind": "heuristic"}, "confidence": "exact", "extra": true}"#;
103        let result: Result<PromptMeasurement, _> = serde_json::from_str(raw);
104        assert!(
105            result.is_err(),
106            "deny_unknown_fields must reject stray keys"
107        );
108    }
109}
deepstrike_core/context/measurement.rs

deepstrike_core/context/
measurement.rs
