1use serde::Serialize;
10use serde_json::{Map, Value};
11use std::io;
12
13pub const MAX_OPTIONS: usize = 16;
17
18pub const MAX_SCORE_LEVELS: usize = 10;
20
21pub const CONFIDENCE_METHOD: &str = "one-minus-normalized-entropy";
26
27pub const PROBABILITY_STATUS: &str = "conditional option scores; uncalibrated as decision confidence";
29
30#[derive(Debug, Clone)]
32pub struct WireError(pub String);
33
34impl std::fmt::Display for WireError {
35 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
36 write!(f, "{}", self.0)
37 }
38}
39
40impl std::error::Error for WireError {}
41
42type Result<T> = std::result::Result<T, WireError>;
43
44fn fail<T>(path: &str, message: &str) -> Result<T> {
45 Err(WireError(format!("{path}: {message}")))
46}
47
48pub fn to_python_json(value: &Value) -> String {
54 let mut buffer = Vec::new();
55 let mut serializer = serde_json::Serializer::with_formatter(&mut buffer, PythonFormatter);
56 value
57 .serialize(&mut serializer)
58 .expect("serializing a serde_json::Value cannot fail");
59 String::from_utf8(buffer).expect("serde_json emits UTF-8")
60}
61
62struct PythonFormatter;
63
64impl serde_json::ser::Formatter for PythonFormatter {
65 fn begin_array_value<W>(&mut self, writer: &mut W, first: bool) -> io::Result<()>
66 where
67 W: ?Sized + io::Write,
68 {
69 if first {
70 Ok(())
71 } else {
72 writer.write_all(b", ")
73 }
74 }
75
76 fn begin_object_key<W>(&mut self, writer: &mut W, first: bool) -> io::Result<()>
77 where
78 W: ?Sized + io::Write,
79 {
80 if first {
81 Ok(())
82 } else {
83 writer.write_all(b", ")
84 }
85 }
86
87 fn begin_object_value<W>(&mut self, writer: &mut W) -> io::Result<()>
88 where
89 W: ?Sized + io::Write,
90 {
91 writer.write_all(b": ")
92 }
93}
94
95#[derive(Debug, Clone, Copy, PartialEq, Eq)]
97pub enum Kind {
98 Choice,
100 Noul,
102 Score,
104}
105
106impl Kind {
107 fn parse(value: &str) -> Option<Self> {
108 match value {
109 "choice" => Some(Self::Choice),
110 "noul" => Some(Self::Noul),
111 "score" => Some(Self::Score),
112 _ => None,
113 }
114 }
115}
116
117#[derive(Debug, Clone)]
119pub struct QuestionSpec {
120 pub id: String,
122 pub kind: Kind,
124 pub option_ids: Vec<String>,
126 pub legend: Vec<String>,
128}
129
130#[derive(Debug, Clone, serde::Deserialize)]
132pub struct RowOption {
133 pub id: String,
135 pub description: String,
137}
138
139#[derive(Debug, Clone, serde::Deserialize)]
141pub struct Row {
142 pub id: String,
144 pub state: Value,
146 pub question: String,
148 pub options: Vec<RowOption>,
150}
151
152pub fn request_rows(payload: &Value, served_model: &str) -> Result<(Vec<QuestionSpec>, Vec<Row>)> {
154 let body = match payload {
155 Value::Object(map) => map,
156 _ => return fail("body", "must be a JSON object"),
157 };
158 let state = body.get("state").cloned().unwrap_or(Value::Null);
159 validate_state(&state)?;
160
161 match body.get("model") {
162 Some(Value::String(model)) if model == served_model => {}
163 _ => {
164 return fail(
165 "model",
166 &format!("must be {served_model:?}; this server does not serve Jev aliases"),
167 )
168 }
169 }
170
171 let questions = match body.get("questions") {
172 Some(Value::Object(map)) if !map.is_empty() => map,
173 _ => return fail("questions", "must be a nonempty object"),
174 };
175
176 let mut specs = Vec::with_capacity(questions.len());
177 let mut rows = Vec::with_capacity(questions.len());
178 for (question_id, question) in questions {
179 if question_id.is_empty() {
180 return fail("questions", "question IDs must be nonempty strings");
181 }
182 let base = format!("questions.{question_id}");
183 let question = match question {
184 Value::Object(map) => map,
185 _ => return fail(&base, "must be an object"),
186 };
187 let kind_name = match question.get("type") {
188 Some(Value::String(name)) => name.as_str(),
189 _ => return fail(&format!("{base}.type"), "must be one of: choice, noul, score"),
190 };
191 let kind = match Kind::parse(kind_name) {
192 Some(kind) => kind,
193 None => return fail(&format!("{base}.type"), "must be one of: choice, noul, score"),
194 };
195 let instructions = instructions(question.get("instructions"), kind, &format!("{base}.instructions"))?;
196
197 let (option_ids, options, legend) = match kind {
198 Kind::Choice => choice_options(question, &base)?,
199 Kind::Noul => noul_options(question, &base)?,
200 Kind::Score => score_options(question, &base)?,
201 };
202
203 specs.push(QuestionSpec {
204 id: question_id.clone(),
205 kind,
206 option_ids,
207 legend,
208 });
209 rows.push(Row {
210 id: question_id.clone(),
211 state: state.clone(),
212 question: instructions,
213 options,
214 });
215 }
216 Ok((specs, rows))
217}
218
219fn validate_state(state: &Value) -> Result<()> {
220 match state {
221 Value::String(text) if text.is_empty() => fail("state", "must be a nonempty string, object, or array"),
222 Value::String(_) => Ok(()),
223 Value::Object(map) if map.is_empty() => {
224 fail("state", "must be a nonempty string, object, or array")
225 }
226 Value::Object(_) => Ok(()),
227 Value::Array(items) if items.is_empty() => {
228 fail("state", "must be a nonempty string, object, or array")
229 }
230 Value::Array(_) => Ok(()),
231 _ => fail("state", "must be a nonempty string, object, or array"),
232 }
233}
234
235fn json_text(value: &Value, path: &str) -> Result<String> {
236 match value {
237 Value::String(text) if text.is_empty() => fail(path, "must not be empty"),
238 Value::String(text) => Ok(text.clone()),
239 Value::Object(_) | Value::Array(_) => Ok(to_python_json(value)),
240 _ => fail(path, "must be a string, object, or array"),
241 }
242}
243
244fn description(option_id: &str, value: Option<&Value>, path: &str) -> Result<String> {
245 match value {
246 None | Some(Value::Null) => Ok(option_id.to_string()),
247 Some(value) => Ok(format!("{option_id}: {}", json_text(value, path)?)),
248 }
249}
250
251fn instructions(value: Option<&Value>, kind: Kind, path: &str) -> Result<String> {
252 match value {
253 Some(Value::Null) | None => Ok(match kind {
254 Kind::Choice => "Which option best matches the supplied state?".to_string(),
255 Kind::Noul => "Does the true outcome apply to the supplied state?".to_string(),
256 Kind::Score => "Which ordered level best matches the supplied state?".to_string(),
257 }),
258 Some(value) => json_text(value, path),
259 }
260}
261
262type Options = (Vec<String>, Vec<RowOption>, Vec<String>);
263
264fn choice_options(question: &Map<String, Value>, base: &str) -> Result<Options> {
265 let criteria = match question.get("criteria") {
266 Some(Value::Object(map)) if (2..=MAX_OPTIONS).contains(&map.len()) => map,
267 _ => {
268 return fail(
269 &format!("{base}.criteria"),
270 &format!("must contain 2-{MAX_OPTIONS} options"),
271 )
272 }
273 };
274 let mut option_ids = Vec::with_capacity(criteria.len());
275 let mut options = Vec::with_capacity(criteria.len());
276 for (option_id, value) in criteria {
277 if option_id.is_empty() {
278 return fail(&format!("{base}.criteria"), "option IDs must be nonempty strings");
279 }
280 if !matches!(value, Value::Null | Value::String(_) | Value::Object(_) | Value::Array(_)) {
281 return fail(
282 &format!("{base}.criteria.{option_id}"),
283 "must be a string, object, array, or null",
284 );
285 }
286 option_ids.push(option_id.clone());
287 options.push(RowOption {
288 id: option_id.clone(),
289 description: description(option_id, Some(value), &format!("{base}.criteria.{option_id}"))?,
290 });
291 }
292 Ok((option_ids, options, Vec::new()))
293}
294
295fn noul_options(question: &Map<String, Value>, base: &str) -> Result<Options> {
296 let criteria = match question.get("criteria") {
297 None | Some(Value::Null) => &Map::new(),
298 Some(Value::Object(map)) => map,
299 _ => return fail(&format!("{base}.criteria"), "must be an object when supplied"),
300 };
301 let unknown: Vec<&String> = criteria
302 .keys()
303 .filter(|key| key.as_str() != "true" && key.as_str() != "false")
304 .collect();
305 if !unknown.is_empty() {
306 let mut names: Vec<&str> = unknown.iter().map(|key| key.as_str()).collect();
307 names.sort_unstable();
308 return fail(
309 &format!("{base}.criteria"),
310 &format!("contains unsupported keys: {:?}", names),
311 );
312 }
313 let option_ids = vec!["true".to_string(), "false".to_string()];
314 let defaults = ["The answer is yes.", "The answer is no."];
315 let mut options = Vec::with_capacity(2);
316 for (index, option_id) in option_ids.iter().enumerate() {
317 let path = format!("{base}.criteria.{option_id}");
318 let fallback;
321 let value = match criteria.get(option_id) {
322 None => {
323 fallback = Value::String(defaults[index].to_string());
324 Some(&fallback)
325 }
326 Some(Value::Null) => None,
327 Some(value) => Some(value),
328 };
329 options.push(RowOption {
330 id: option_id.clone(),
331 description: description(option_id, value, &path)?,
332 });
333 }
334 Ok((option_ids, options, Vec::new()))
335}
336
337fn score_options(question: &Map<String, Value>, base: &str) -> Result<Options> {
338 let criteria = match question.get("criteria") {
339 Some(Value::Array(items)) if (2..=MAX_SCORE_LEVELS).contains(&items.len()) => items,
340 _ => {
341 return fail(
342 &format!("{base}.criteria"),
343 &format!("must contain 2-{MAX_SCORE_LEVELS} ordered levels"),
344 )
345 }
346 };
347 let mut option_ids = Vec::with_capacity(criteria.len());
348 let mut options = Vec::with_capacity(criteria.len());
349 let mut legend = Vec::with_capacity(criteria.len());
350 for (index, value) in criteria.iter().enumerate() {
351 let path = format!("{base}.criteria.{index}");
352 let text = json_text(value, &path)?;
353 let option_id = index.to_string();
354 legend.push(text.clone());
355 option_ids.push(option_id.clone());
356 options.push(RowOption {
357 id: option_id.clone(),
358 description: format!("Level {option_id}: {text}"),
359 });
360 }
361 Ok((option_ids, options, legend))
362}
363
364#[derive(Debug, Clone)]
366pub struct ScoredAnswer {
367 pub id: String,
369 pub option_ids: Vec<String>,
371 pub probabilities: Vec<f64>,
373 pub input_tokens: u64,
375 pub prompt_version: Option<String>,
377}
378
379pub fn distribution_confidence(probabilities: &[f64]) -> f64 {
381 let entropy: f64 = probabilities
382 .iter()
383 .filter(|value| **value > 0.0)
384 .map(|value| -value * value.ln())
385 .sum();
386 (1.0 - entropy / (probabilities.len() as f64).ln()).clamp(0.0, 1.0)
387}
388
389fn normalized_probabilities(spec: &QuestionSpec, result: &ScoredAnswer) -> Result<Vec<f64>> {
390 if result.id != spec.id || result.option_ids != spec.option_ids {
391 return Err(WireError(format!(
392 "Scorer result for {:?} does not match its declared options",
393 spec.id
394 )));
395 }
396 if result.probabilities.len() != spec.option_ids.len() {
397 return Err(WireError(format!(
398 "Scorer result for {:?} has an invalid probability vector",
399 spec.id
400 )));
401 }
402 if result
403 .probabilities
404 .iter()
405 .any(|value| !value.is_finite() || *value < 0.0)
406 {
407 return Err(WireError(format!(
408 "Scorer result for {:?} has non-finite or negative probabilities",
409 spec.id
410 )));
411 }
412 let total: f64 = result.probabilities.iter().sum();
413 if total <= 0.0 {
414 return Err(WireError(format!(
415 "Scorer result for {:?} has zero probability mass",
416 spec.id
417 )));
418 }
419 Ok(result.probabilities.iter().map(|value| value / total).collect())
420}
421
422#[derive(Debug)]
428pub struct OrderedMap<V>(Vec<(String, V)>);
429
430impl<V> OrderedMap<V> {
431 pub fn new() -> Self {
433 Self(Vec::new())
434 }
435
436 pub fn insert(&mut self, key: String, value: V) {
438 self.0.push((key, value));
439 }
440
441 pub fn is_empty(&self) -> bool {
443 self.0.is_empty()
444 }
445
446 pub fn len(&self) -> usize {
448 self.0.len()
449 }
450}
451
452impl<V> Default for OrderedMap<V> {
453 fn default() -> Self {
454 Self::new()
455 }
456}
457
458impl<V: Serialize> Serialize for OrderedMap<V> {
459 fn serialize<S: serde::Serializer>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error> {
460 use serde::ser::SerializeMap;
461 let mut map = serializer.serialize_map(Some(self.0.len()))?;
462 for (key, value) in &self.0 {
463 map.serialize_entry(key, value)?;
464 }
465 map.end()
466 }
467}
468
469#[derive(Debug, Serialize)]
476#[serde(tag = "type", rename_all = "lowercase")]
477pub enum Answer {
478 Noul {
480 noul: f64,
482 },
483 Choice {
485 choice: String,
487 probabilities: OrderedMap<f64>,
489 confidence: f64,
491 },
492 Score {
494 score: f64,
496 legend: OrderedMap<String>,
498 probabilities: OrderedMap<f64>,
500 confidence: f64,
502 },
503}
504
505#[derive(Debug, Serialize)]
506pub struct Usage {
508 pub input_tokens: u64,
510 pub output_tokens: u64,
512}
513
514#[derive(Debug, Serialize)]
516pub struct FastjevMeta {
517 pub probability_status: &'static str,
519 pub confidence_method: &'static str,
521 pub prompt_versions: Vec<String>,
523}
524
525#[derive(Debug, Serialize)]
527pub struct SystemOneResponse {
528 pub model: String,
530 pub answers: OrderedMap<Answer>,
532 pub usage: Usage,
534 pub fastjev: FastjevMeta,
536}
537
538pub fn response_from_results(
540 served_model: &str,
541 specs: &[QuestionSpec],
542 results: &[ScoredAnswer],
543) -> Result<SystemOneResponse> {
544 if results.len() != specs.len() {
545 return Err(WireError(
546 "Scorer returned a different number of results than requested".to_string(),
547 ));
548 }
549 let mut answers = OrderedMap::new();
550 let mut input_tokens: u64 = 0;
551 let mut prompt_versions: Vec<String> = Vec::new();
552 for (spec, result) in specs.iter().zip(results) {
553 let probabilities = normalized_probabilities(spec, result)?;
554 let winner_index = probabilities
555 .iter()
556 .enumerate()
557 .fold(0usize, |best, (index, value)| {
558 if *value > probabilities[best] {
559 index
560 } else {
561 best
562 }
563 });
564 let mut distribution = OrderedMap::new();
565 for (option_id, value) in spec.option_ids.iter().zip(&probabilities) {
566 distribution.insert(option_id.clone(), *value);
567 }
568 let answer = match spec.kind {
569 Kind::Noul => {
570 let true_index = spec
571 .option_ids
572 .iter()
573 .position(|option_id| option_id == "true")
574 .ok_or_else(|| {
575 WireError(format!("Scorer result for {:?} lost its true option", spec.id))
576 })?;
577 Answer::Noul {
578 noul: probabilities[true_index],
579 }
580 }
581 Kind::Choice => Answer::Choice {
582 choice: spec.option_ids[winner_index].clone(),
583 probabilities: distribution,
584 confidence: distribution_confidence(&probabilities),
585 },
586 Kind::Score => {
587 let score: f64 = probabilities
588 .iter()
589 .enumerate()
590 .map(|(index, value)| index as f64 * value)
591 .sum();
592 let mut legend: OrderedMap<String> = OrderedMap::new();
593 for (index, text) in spec.legend.iter().enumerate() {
594 legend.insert(index.to_string(), text.clone());
595 }
596 Answer::Score {
597 score,
598 legend,
599 probabilities: distribution,
600 confidence: distribution_confidence(&probabilities),
601 }
602 }
603 };
604 answers.insert(spec.id.clone(), answer);
605 input_tokens += result.input_tokens;
606 if let Some(version) = &result.prompt_version
607 && !prompt_versions.contains(version)
608 {
609 prompt_versions.push(version.clone());
610 }
611 }
612 prompt_versions.sort();
613 Ok(SystemOneResponse {
614 model: served_model.to_string(),
615 answers,
616 usage: Usage {
617 input_tokens,
618 output_tokens: 0,
619 },
620 fastjev: FastjevMeta {
621 probability_status: PROBABILITY_STATUS,
622 confidence_method: CONFIDENCE_METHOD,
623 prompt_versions,
624 },
625 })
626}
627
628#[cfg(test)]
629mod tests {
630 use super::*;
631 use serde_json::json;
632
633 fn sample_payload() -> Value {
634 json!({
635 "model": "bridge-model",
636 "state": "Help! My payouts have been failing for three days.",
637 "questions": {
638 "is_urgent": {"type": "noul", "instructions": "Does this convey urgency?"},
639 "department": {
640 "type": "choice",
641 "instructions": "Which team should handle this?",
642 "criteria": {"billing": "Payments, invoicing, and refunds", "technical": null},
643 },
644 "severity": {
645 "type": "score",
646 "instructions": "How severe is the problem?",
647 "criteria": ["Minor", "Degraded", "Blocking"],
648 },
649 },
650 })
651 }
652
653 #[test]
654 fn python_json_layout_matches_cpython_output() {
655 let value = json!({
659 "evidence": {"message": "已扣款两次 charged twice", "count": 2, "tags": ["a", "b"],
660 "ok": true, "missing": null, "ratio": 0.5},
661 "criterion": "Which queue?",
662 "options": [{"letter": "A", "description": "Billing, payments, and refunds."}],
663 });
664 assert_eq!(
665 to_python_json(&value),
666 r#"{"evidence": {"message": "已扣款两次 charged twice", "count": 2, "tags": ["a", "b"], "ok": true, "missing": null, "ratio": 0.5}, "criterion": "Which queue?", "options": [{"letter": "A", "description": "Billing, payments, and refunds."}]}"#
667 );
668 }
669
670 #[test]
671 fn python_json_layout_matches_python_separators() {
672 let value = json!({"a": [1, 2], "b": {"c": "x"}, "d": "中"});
673 assert_eq!(to_python_json(&value), r#"{"a": [1, 2], "b": {"c": "x"}, "d": "中"}"#);
674 }
675
676 #[test]
677 fn rows_render_descriptions_like_fastjev() {
678 let (specs, rows) = request_rows(&sample_payload(), "bridge-model").unwrap();
679 assert_eq!(specs.len(), 3);
680 assert_eq!(rows[1].options[0].description, "billing: Payments, invoicing, and refunds");
681 assert_eq!(rows[1].options[1].description, "technical");
682 assert_eq!(rows[2].options[2].description, "Level 2: Blocking");
683 assert_eq!(rows[0].options[0].description, "true: The answer is yes.");
684 assert_eq!(specs[2].legend, vec!["Minor", "Degraded", "Blocking"]);
685 assert_eq!(rows[1].question, "Which team should handle this?");
686 }
687
688 #[test]
689 fn noul_criteria_defaults_and_nulls_differ() {
690 let payload = json!({
691 "model": "m",
692 "state": "s",
693 "questions": {
694 "defaults": {"type": "noul"},
695 "explicit_null": {"type": "noul", "criteria": {"true": null, "false": "No way."}},
696 },
697 });
698 let (_specs, rows) = request_rows(&payload, "m").unwrap();
699 assert_eq!(rows[0].options[0].description, "true: The answer is yes.");
700 assert_eq!(rows[0].options[1].description, "false: The answer is no.");
701 assert_eq!(rows[1].options[0].description, "true");
702 assert_eq!(rows[1].options[1].description, "false: No way.");
703 }
704
705 #[test]
706 fn model_mismatch_is_rejected() {
707 let error = request_rows(&sample_payload(), "other-model").unwrap_err();
708 assert!(error.0.starts_with("model: "), "{}", error.0);
709 }
710
711 #[test]
712 fn empty_state_is_rejected() {
713 let payload = json!({"model": "m", "state": "", "questions": {"q": {"type": "noul"}}});
714 let error = request_rows(&payload, "m").unwrap_err();
715 assert!(error.0.starts_with("state: "), "{}", error.0);
716 }
717
718 #[test]
719 fn response_uses_probabilities_and_confidence() {
720 let (specs, _rows) = request_rows(&sample_payload(), "bridge-model").unwrap();
721 let results = vec![
722 ScoredAnswer {
723 id: "is_urgent".into(),
724 option_ids: vec!["true".into(), "false".into()],
725 probabilities: vec![0.9, 0.1],
726 input_tokens: 10,
727 prompt_version: Some("direct-options-v1".into()),
728 },
729 ScoredAnswer {
730 id: "department".into(),
731 option_ids: vec!["billing".into(), "technical".into()],
732 probabilities: vec![0.25, 0.75],
733 input_tokens: 20,
734 prompt_version: Some("direct-options-v1".into()),
735 },
736 ScoredAnswer {
737 id: "severity".into(),
738 option_ids: vec!["0".into(), "1".into(), "2".into()],
739 probabilities: vec![0.0, 1.0, 0.0],
740 input_tokens: 30,
741 prompt_version: Some("direct-options-v1".into()),
742 },
743 ];
744 let response = serde_json::to_value(
745 response_from_results("bridge-model", &specs, &results).unwrap(),
746 )
747 .unwrap();
748 assert_eq!(response["answers"]["is_urgent"]["noul"], 0.9);
749 assert_eq!(response["answers"]["department"]["choice"], "technical");
750 assert_eq!(response["answers"]["severity"]["score"], 1.0);
751 assert_eq!(response["usage"]["input_tokens"], 60);
752 assert_eq!(response["usage"]["output_tokens"], 0);
753 assert_eq!(response["answers"]["severity"]["legend"]["2"], "Blocking");
754 let confidence = response["answers"]["severity"]["confidence"].as_f64().unwrap();
755 assert!((confidence - 1.0).abs() < 1e-12);
756 }
757
758 #[test]
759 fn flat_distribution_has_zero_confidence() {
760 assert!(distribution_confidence(&[0.25, 0.25, 0.25, 0.25]).abs() < 1e-12);
761 }
762
763 #[test]
764 fn mismatched_result_is_rejected() {
765 let (specs, _rows) = request_rows(&sample_payload(), "bridge-model").unwrap();
766 let results = vec![ScoredAnswer {
767 id: "is_urgent".into(),
768 option_ids: vec!["false".into(), "true".into()],
769 probabilities: vec![0.5, 0.5],
770 input_tokens: 1,
771 prompt_version: None,
772 }];
773 assert!(response_from_results("bridge-model", &specs, &results).is_err());
774 }
775}