Skip to main content

jev_client/
response.rs

1//! The body returned by `POST /v1/systemone`.
2
3use indexmap::IndexMap;
4use schemars::JsonSchema;
5use serde::{Deserialize, Serialize};
6
7use crate::answer::Answer;
8
9/// An evaluation response: one answer per question, under the ids the request used.
10///
11/// Parsing is tolerant so that an upstream addition never breaks a caller: unknown fields are
12/// ignored, a missing `usage` reads as zero, and an answer of an unknown type is kept as raw JSON
13/// (see [`Answer::Unknown`]).
14#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)]
15#[schemars(
16    description = "An evaluation response: one answer per question, under the ids the request used."
17)]
18#[non_exhaustive]
19pub struct Response {
20    /// The versioned id of the model that answered, such as `jev-1.13.0`.
21    ///
22    /// This is the resolved id even when the request named an alias, so it is the value to log
23    /// and the value pricing is keyed on.
24    pub model: String,
25
26    /// One answer per question, keyed by the ids used in the request.
27    pub answers: IndexMap<String, Answer>,
28
29    /// Token usage for the request.
30    #[serde(default)]
31    pub usage: Usage,
32}
33
34impl Response {
35    /// A response, as a fake transport or a test would build one.
36    #[must_use]
37    pub fn new(model: impl Into<String>, answers: IndexMap<String, Answer>, usage: Usage) -> Self {
38        Self {
39            model: model.into(),
40            answers,
41            usage,
42        }
43    }
44}
45
46/// Token usage for one request. Only input tokens are billed.
47#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
48#[non_exhaustive]
49pub struct Usage {
50    /// Tokens read: the state, every question, and a fixed per-request overhead.
51    #[serde(default)]
52    pub input_tokens: u64,
53
54    /// Tokens produced. Reported by the API but not billed.
55    #[serde(default)]
56    pub output_tokens: u64,
57}
58
59impl Usage {
60    /// Usage with the given token counts.
61    #[must_use]
62    pub const fn new(input_tokens: u64, output_tokens: u64) -> Self {
63        Self {
64            input_tokens,
65            output_tokens,
66        }
67    }
68}
69
70#[cfg(test)]
71mod tests {
72    use indexmap::IndexMap;
73    use serde_json::json;
74
75    use super::{Response, Usage};
76    use crate::{Answer, NoulAnswer};
77
78    #[test]
79    fn unknown_fields_are_ignored_and_unknown_answers_are_kept() {
80        let rank = json!({ "type": "rank", "order": ["b", "a"] });
81        let json = json!({
82            "model": "jev-2.0.0",
83            "warnings": ["state was truncated"],
84            "answers": { "urgent": { "type": "noul", "noul": 0.9 }, "ranking": rank },
85            "usage": { "input_tokens": 312, "output_tokens": 48, "cached_tokens": 100 }
86        });
87
88        let response: Response = serde_json::from_value(json).unwrap();
89
90        assert_eq!(response.model, "jev-2.0.0");
91        assert_eq!(response.usage, Usage::new(312, 48));
92        assert_eq!(
93            response.answers.get("urgent"),
94            Some(&Answer::from(NoulAnswer::new(0.9)))
95        );
96        assert_eq!(
97            response.answers.get("ranking"),
98            Some(&Answer::Unknown(rank))
99        );
100    }
101
102    #[test]
103    fn missing_usage_reads_as_zero() {
104        let whole: Response =
105            serde_json::from_value(json!({ "model": "m", "answers": {} })).unwrap();
106        let partial: Response = serde_json::from_value(
107            json!({ "model": "m", "answers": {}, "usage": { "input_tokens": 7 } }),
108        )
109        .unwrap();
110
111        assert_eq!(whole.usage, Usage::default());
112        assert_eq!(partial.usage, Usage::new(7, 0));
113    }
114
115    #[test]
116    fn answers_keep_the_order_the_server_sent() {
117        let text = r#"{"model":"m","answers":{"z":{"type":"noul","noul":0.1},"a":{"type":"noul","noul":0.2}},"usage":{"input_tokens":1,"output_tokens":2}}"#;
118
119        let response: Response = serde_json::from_str(text).unwrap();
120
121        assert_eq!(response.answers.keys().collect::<Vec<_>>(), ["z", "a"]);
122        assert_eq!(serde_json::to_string(&response).unwrap(), text);
123    }
124
125    #[test]
126    fn a_response_can_be_built_for_fakes() {
127        let answers = IndexMap::from([("q".to_owned(), Answer::from(NoulAnswer::new(0.5)))]);
128
129        let response = Response::new("jev-1.13.0", answers, Usage::new(10, 0));
130
131        assert_eq!(response.answers.len(), 1);
132        assert_eq!(response.usage.input_tokens, 10);
133    }
134}