ferrox_api/
usage.rs

1//! OpenAI-convention token accounting plus llama.cpp-style timings.
2//!
3//! Counted from the exact token ids the generation loop processed
4//! (prompt after BOS insertion, and every generated id), not
5//! re-tokenized after the fact -- re-tokenizing decoded text is not
6//! guaranteed to round-trip to the same count.
7//!
8//! Why the server reports timings at all, when a client can hold a
9//! stopwatch: the client's stopwatch measures the network, the proxy's
10//! buffer and its own event loop. More importantly it cannot separate
11//! **prefill from decode**, and a UI that divides total tokens by total
12//! wall time reports a 50 tok/s model as 5 tok/s whenever the prompt is
13//! long. Every downstream number built on that is then wrong in the
14//! same direction. So the phases are reported separately and the client
15//! is never asked to infer one from the other.
16//!
17//! Every timing is optional: a cached response, a batched decode, or an
18//! engine path that does not time itself must be able to answer
19//! honestly rather than emit a plausible zero.
20
21use serde::{Deserialize, Serialize};
22
23#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
24pub struct Usage {
25    pub prompt_tokens: usize,
26    pub completion_tokens: usize,
27    pub total_tokens: usize,
28    /// Prefill throughput (prompt tokens / prefill seconds), when timed.
29    #[serde(skip_serializing_if = "Option::is_none")]
30    pub prompt_per_second: Option<f64>,
31    /// Decode throughput (completion tokens / decode seconds), when timed.
32    #[serde(skip_serializing_if = "Option::is_none")]
33    pub predicted_per_second: Option<f64>,
34    /// Wall time spent processing the prompt, in milliseconds.
35    #[serde(skip_serializing_if = "Option::is_none")]
36    pub prompt_eval_duration_ms: Option<f64>,
37    /// Wall time spent in the decode loop, in milliseconds. Kept
38    /// separate from `prompt_eval_duration_ms` on purpose (see the
39    /// module docs).
40    #[serde(skip_serializing_if = "Option::is_none")]
41    pub generation_duration_ms: Option<f64>,
42    /// Time to first token: from the start of prefill to the moment the
43    /// first token was produced. `None` when no token was produced at
44    /// all (an immediate EOS), because a zero there would read as an
45    /// instantaneous response.
46    #[serde(skip_serializing_if = "Option::is_none")]
47    pub time_to_first_token_ms: Option<f64>,
48    /// Prompt tokens served from the KV prefix cache instead of being
49    /// recomputed. `Some(0)` means "the cache was consulted and missed";
50    /// `None` means "no prefix cache is configured" -- a distinction the
51    /// UI needs to decide whether to show the row at all.
52    #[serde(skip_serializing_if = "Option::is_none")]
53    pub cached_tokens: Option<usize>,
54}
55
56impl Usage {
57    pub fn new(prompt_tokens: usize, completion_tokens: usize) -> Self {
58        Usage {
59            prompt_tokens,
60            completion_tokens,
61            total_tokens: prompt_tokens + completion_tokens,
62            prompt_per_second: None,
63            predicted_per_second: None,
64            prompt_eval_duration_ms: None,
65            generation_duration_ms: None,
66            time_to_first_token_ms: None,
67            cached_tokens: None,
68        }
69    }
70
71    /// Records the two phase durations, in seconds, and the rates they
72    /// imply. A zero-length phase leaves the rate unset rather than
73    /// dividing by zero into infinity.
74    pub fn with_timings(mut self, prompt_secs: f64, predicted_secs: f64) -> Self {
75        self.prompt_eval_duration_ms = Some(prompt_secs * 1000.0);
76        self.generation_duration_ms = Some(predicted_secs * 1000.0);
77        if prompt_secs > 0.0 && self.prompt_tokens > 0 {
78            self.prompt_per_second = Some(self.prompt_tokens as f64 / prompt_secs);
79        }
80        if predicted_secs > 0.0 && self.completion_tokens > 0 {
81            self.predicted_per_second = Some(self.completion_tokens as f64 / predicted_secs);
82        }
83        self
84    }
85
86    /// Time-to-first-token, in seconds, measured from the start of
87    /// prefill. Ignored when no token was generated.
88    pub fn with_ttft(mut self, secs: f64) -> Self {
89        if self.completion_tokens > 0 {
90            self.time_to_first_token_ms = Some(secs * 1000.0);
91        }
92        self
93    }
94
95    /// Prompt tokens that came from the prefix cache. Call this only
96    /// when a prefix cache actually exists (see `cached_tokens`).
97    pub fn with_cached_tokens(mut self, cached: usize) -> Self {
98        self.cached_tokens = Some(cached);
99        self
100    }
101}
102
103#[cfg(test)]
104mod tests {
105    use super::*;
106
107    #[test]
108    fn totals_are_the_sum_of_the_two_phases() {
109        let usage = Usage::new(7, 3);
110        assert_eq!(usage.total_tokens, 10);
111    }
112
113    #[test]
114    fn phase_durations_stay_separate() {
115        // 100 prompt tokens in 1s, 10 generated in 1s. A client that
116        // conflated the phases would report 110/2 = 55 tok/s for both.
117        let usage = Usage::new(100, 10).with_timings(1.0, 1.0);
118        assert_eq!(usage.prompt_per_second, Some(100.0));
119        assert_eq!(usage.predicted_per_second, Some(10.0));
120        assert_eq!(usage.prompt_eval_duration_ms, Some(1000.0));
121        assert_eq!(usage.generation_duration_ms, Some(1000.0));
122    }
123
124    #[test]
125    fn zero_length_phases_do_not_become_infinite_rates() {
126        let usage = Usage::new(5, 5).with_timings(0.0, 0.0);
127        assert_eq!(usage.prompt_per_second, None);
128        assert_eq!(usage.predicted_per_second, None);
129        assert_eq!(usage.prompt_eval_duration_ms, Some(0.0));
130    }
131
132    #[test]
133    fn ttft_is_unset_when_nothing_was_generated() {
134        let usage = Usage::new(5, 0).with_ttft(0.25);
135        assert_eq!(usage.time_to_first_token_ms, None);
136        assert_eq!(
137            Usage::new(5, 1).with_ttft(0.25).time_to_first_token_ms,
138            Some(250.0)
139        );
140    }
141
142    #[test]
143    fn untimed_usage_serializes_to_the_plain_openai_shape() {
144        // Older clients must not start seeing null-valued extras.
145        let json = serde_json::to_string(&Usage::new(2, 3)).unwrap();
146        assert_eq!(
147            json,
148            "{\"prompt_tokens\":2,\"completion_tokens\":3,\"total_tokens\":5}"
149        );
150    }
151
152    #[test]
153    fn a_prefix_cache_miss_is_distinguishable_from_no_prefix_cache() {
154        assert_eq!(Usage::new(2, 3).cached_tokens, None);
155        assert_eq!(
156            Usage::new(2, 3).with_cached_tokens(0).cached_tokens,
157            Some(0)
158        );
159    }
160}
ferrox_api/usage.rs

ferrox_api/
usage.rs
