1use indexmap::IndexMap;
4use schemars::JsonSchema;
5use serde::{Deserialize, Serialize};
6
7use crate::answer::Answer;
8
9#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, JsonSchema)]
15#[schemars(
16 description = "An evaluation response: one answer per question, under the ids the request used."
17)]
18#[non_exhaustive]
19pub struct Response {
20 pub model: String,
25
26 pub answers: IndexMap<String, Answer>,
28
29 #[serde(default)]
31 pub usage: Usage,
32}
33
34impl Response {
35 #[must_use]
37 pub fn new(model: impl Into<String>, answers: IndexMap<String, Answer>, usage: Usage) -> Self {
38 Self {
39 model: model.into(),
40 answers,
41 usage,
42 }
43 }
44}
45
46#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
48#[non_exhaustive]
49pub struct Usage {
50 #[serde(default)]
52 pub input_tokens: u64,
53
54 #[serde(default)]
56 pub output_tokens: u64,
57}
58
59impl Usage {
60 #[must_use]
62 pub const fn new(input_tokens: u64, output_tokens: u64) -> Self {
63 Self {
64 input_tokens,
65 output_tokens,
66 }
67 }
68}
69
70#[cfg(test)]
71mod tests {
72 use indexmap::IndexMap;
73 use serde_json::json;
74
75 use super::{Response, Usage};
76 use crate::{Answer, NoulAnswer};
77
78 #[test]
79 fn unknown_fields_are_ignored_and_unknown_answers_are_kept() {
80 let rank = json!({ "type": "rank", "order": ["b", "a"] });
81 let json = json!({
82 "model": "jev-2.0.0",
83 "warnings": ["state was truncated"],
84 "answers": { "urgent": { "type": "noul", "noul": 0.9 }, "ranking": rank },
85 "usage": { "input_tokens": 312, "output_tokens": 48, "cached_tokens": 100 }
86 });
87
88 let response: Response = serde_json::from_value(json).unwrap();
89
90 assert_eq!(response.model, "jev-2.0.0");
91 assert_eq!(response.usage, Usage::new(312, 48));
92 assert_eq!(
93 response.answers.get("urgent"),
94 Some(&Answer::from(NoulAnswer::new(0.9)))
95 );
96 assert_eq!(
97 response.answers.get("ranking"),
98 Some(&Answer::Unknown(rank))
99 );
100 }
101
102 #[test]
103 fn missing_usage_reads_as_zero() {
104 let whole: Response =
105 serde_json::from_value(json!({ "model": "m", "answers": {} })).unwrap();
106 let partial: Response = serde_json::from_value(
107 json!({ "model": "m", "answers": {}, "usage": { "input_tokens": 7 } }),
108 )
109 .unwrap();
110
111 assert_eq!(whole.usage, Usage::default());
112 assert_eq!(partial.usage, Usage::new(7, 0));
113 }
114
115 #[test]
116 fn answers_keep_the_order_the_server_sent() {
117 let text = r#"{"model":"m","answers":{"z":{"type":"noul","noul":0.1},"a":{"type":"noul","noul":0.2}},"usage":{"input_tokens":1,"output_tokens":2}}"#;
118
119 let response: Response = serde_json::from_str(text).unwrap();
120
121 assert_eq!(response.answers.keys().collect::<Vec<_>>(), ["z", "a"]);
122 assert_eq!(serde_json::to_string(&response).unwrap(), text);
123 }
124
125 #[test]
126 fn a_response_can_be_built_for_fakes() {
127 let answers = IndexMap::from([("q".to_owned(), Answer::from(NoulAnswer::new(0.5)))]);
128
129 let response = Response::new("jev-1.13.0", answers, Usage::new(10, 0));
130
131 assert_eq!(response.answers.len(), 1);
132 assert_eq!(response.usage.input_tokens, 10);
133 }
134}