omni_dev/jev/
protocol.rs

1//! Wire types for TypeSafe's Jev "System One" REST API.
2//!
3//! The API answers a **map** of named questions about one opaque `state` in
4//! a single request — `choice` (pick one of a labelled set), `score` (rate
5//! against an ordered scale) and `noul` (a yes/no probability) all share the
6//! one `/v1/systemone` endpoint, distinguished by an internally-tagged
7//! `type` field. See `docs/jev.md` for the operator-facing guide.
8//!
9//! Every map here is a [`BTreeMap`], never a `HashMap`: neither `indexmap`
10//! nor serde_json's `preserve_order` feature is a dependency, so a
11//! `HashMap` would serialise in random order on every run.
12
13use std::collections::BTreeMap;
14
15use serde::{Deserialize, Serialize};
16
17use crate::jev::error::JevError;
18
19/// Default model identifier, used when no `--jev-model` / `TYPESAFE_MODEL`
20/// override is configured.
21pub const DEFAULT_MODEL: &str = "jev-latest";
22
23/// A `POST /v1/systemone` request body.
24#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
25pub struct SystemOneRequest {
26    /// The opaque program state the questions are asked about. Accepted as
27    /// a JSON string, object, or array.
28    pub state: serde_json::Value,
29
30    /// The Jev model to use, e.g. [`DEFAULT_MODEL`].
31    pub model: String,
32
33    /// The questions to ask about `state`, keyed by caller-chosen name.
34    /// Answered in one parallel pass; the response's `answers` map uses the
35    /// same keys.
36    pub questions: BTreeMap<String, Question>,
37}
38
39/// A single question asked of the `state`.
40///
41/// Internally tagged on `type` (`choice` | `score` | `noul`) — each variant's
42/// shape, in particular whether `criteria` is a map, an ordered list, or
43/// optional, differs by primitive and is not expressible as one flat struct.
44#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
45#[serde(tag = "type", rename_all = "lowercase")]
46pub enum Question {
47    /// Pick exactly one of a labelled set of options.
48    Choice {
49        /// Instructions describing what to choose and why.
50        instructions: String,
51        /// Option name to description, e.g. `{"billing": "Payment issues"}`.
52        /// A `BTreeMap` alphabetises option order in the outgoing request —
53        /// see `docs/jev.md` for the caveat this implies for `--option`.
54        criteria: BTreeMap<String, String>,
55    },
56
57    /// Rate the state against an ordered scale.
58    Score {
59        /// Instructions describing what to rate and how.
60        instructions: String,
61        /// The scale's levels, low to high. Order is significant and is
62        /// preserved (a `Vec`, not a map).
63        criteria: Vec<String>,
64    },
65
66    /// Estimate the probability of a yes/no condition.
67    Noul {
68        /// Instructions describing the yes/no condition to estimate.
69        instructions: String,
70        /// Optional descriptions of what "true" and "false" mean. Omitted
71        /// entirely from the request when `None`.
72        #[serde(skip_serializing_if = "Option::is_none")]
73        criteria: Option<NoulCriteria>,
74    },
75}
76
77/// Minimum number of options a [`Question::Choice`] must offer — picking
78/// "one of one" is not a judgment.
79pub const MIN_CHOICE_OPTIONS: usize = 2;
80
81/// Minimum number of levels a [`Question::Score`] scale must have.
82pub const MIN_SCORE_LEVELS: usize = 2;
83
84/// Most options a [`Question::Choice`] may offer on `jev-1.13`.
85pub const MAX_CHOICE_OPTIONS: usize = 255;
86
87/// Most levels a [`Question::Score`] scale may have on `jev-1.13`.
88pub const MAX_SCORE_LEVELS: usize = 10;
89
90/// The upper size limits a model accepts, per question type.
91///
92/// Unlike the minimums, which are omni-dev policy, these are **API limits that
93/// belong to a model version**: a later model may raise them. `None` means
94/// "not known, so not enforced", which is what every model without an entry in
95/// [`QuestionLimits::for_model`] gets, so a stale cap can never block a valid
96/// request.
97#[derive(Debug, Clone, Copy, PartialEq, Eq)]
98pub struct QuestionLimits {
99    /// Most options a `choice` may offer.
100    pub max_choice_options: Option<usize>,
101    /// Most levels a `score` scale may have.
102    pub max_score_levels: Option<usize>,
103}
104
105impl QuestionLimits {
106    /// No caps are enforced.
107    pub const UNLIMITED: Self = Self {
108        max_choice_options: None,
109        max_score_levels: None,
110    };
111
112    /// The documented `jev-1.13.0` caps.
113    const JEV_1_13: Self = Self {
114        max_choice_options: Some(MAX_CHOICE_OPTIONS),
115        max_score_levels: Some(MAX_SCORE_LEVELS),
116    };
117
118    /// The limits for `model`: the single table to extend when a model
119    /// documents different caps.
120    ///
121    /// `jev-1.13` and `jev-1.13.<digits>` match, but `jev-1.130` and
122    /// `jev-1.13.x` do not. `jev-latest` is enforced with the `jev-1.13` caps
123    /// because that is what the alias resolves to today. That is a bet: if
124    /// TypeSafe repoints the alias at a model with higher caps, the check goes
125    /// stale for default users until this table is updated, and pinning
126    /// `--jev-model` to a version without an entry here is the way round it.
127    /// Any other model gets [`Self::UNLIMITED`], so a cap that is not known is
128    /// never enforced.
129    #[must_use]
130    pub fn for_model(model: &str) -> Self {
131        let is_1_13 = model == "jev-1.13"
132            || model.strip_prefix("jev-1.13.").is_some_and(|patch| {
133                !patch.is_empty() && patch.bytes().all(|b| b.is_ascii_digit())
134            });
135        if model == DEFAULT_MODEL || is_1_13 {
136            Self::JEV_1_13
137        } else {
138            Self::UNLIMITED
139        }
140    }
141}
142
143impl Question {
144    /// Rejects degenerate questions: a `choice` needs at least
145    /// [`MIN_CHOICE_OPTIONS`] options and a `score` at least
146    /// [`MIN_SCORE_LEVELS`] levels. `noul` has none.
147    ///
148    /// This is omni-dev policy, not an API constraint: the live API accepts a
149    /// one-option `choice` and a one-level `score` with `200`, but can only
150    /// ever answer them with `confidence: 1.0`, so the paid call carries no
151    /// information and almost always means a mistyped spec.
152    ///
153    /// Duplicate `choice` option names cannot be detected here — a
154    /// [`BTreeMap`] has already collapsed them — so the `--option` parser
155    /// checks those itself.
156    pub fn validate(&self) -> Result<(), JevError> {
157        self.check_minimums()
158    }
159
160    /// [`Self::validate`], plus the upper size limits `model` is known to
161    /// enforce (see [`QuestionLimits::for_model`]). The minimums apply to every
162    /// model; the caps only to a model whose limits are known, so the error
163    /// names the model they came from.
164    pub fn validate_for_model(&self, model: &str) -> Result<(), JevError> {
165        self.check_minimums()?;
166        let limits = QuestionLimits::for_model(model);
167        let (kind, unit, count, max) = match self {
168            Self::Choice { criteria, .. } => (
169                "choice",
170                "options",
171                criteria.len(),
172                limits.max_choice_options,
173            ),
174            Self::Score { criteria, .. } => {
175                ("score", "levels", criteria.len(), limits.max_score_levels)
176            }
177            Self::Noul { .. } => return Ok(()),
178        };
179        match max {
180            Some(max) if count > max => Err(JevError::InvalidQuestionSpec(format!(
181                "a {kind} question accepts at most {max} {unit} on {model}, got {count}"
182            ))),
183            _ => Ok(()),
184        }
185    }
186
187    fn check_minimums(&self) -> Result<(), JevError> {
188        match self {
189            Self::Choice { criteria, .. } if criteria.len() < MIN_CHOICE_OPTIONS => {
190                Err(JevError::InvalidQuestionSpec(format!(
191                    "a choice question needs at least {MIN_CHOICE_OPTIONS} options, got {}",
192                    criteria.len()
193                )))
194            }
195            Self::Score { criteria, .. } if criteria.len() < MIN_SCORE_LEVELS => {
196                Err(JevError::InvalidQuestionSpec(format!(
197                    "a score question needs at least {MIN_SCORE_LEVELS} levels, got {}",
198                    criteria.len()
199                )))
200            }
201            _ => Ok(()),
202        }
203    }
204}
205
206/// Optional descriptions of the two poles of a [`Question::Noul`] question.
207///
208/// Field names are the JSON string keys `"true"`/`"false"` (both Rust
209/// keywords, hence the renames) rather than a nested `{value, description}`
210/// shape, matching the verified wire format `{"true": "…", "false": "…"}`.
211#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
212pub struct NoulCriteria {
213    /// What a `true`-leaning answer means. Skipped when unset.
214    #[serde(rename = "true", skip_serializing_if = "Option::is_none")]
215    pub true_means: Option<String>,
216
217    /// What a `false`-leaning answer means. Skipped when unset.
218    #[serde(rename = "false", skip_serializing_if = "Option::is_none")]
219    pub false_means: Option<String>,
220}
221
222/// A `POST /v1/systemone` response body.
223#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
224pub struct SystemOneResponse {
225    /// The concrete model version that produced the answers, e.g.
226    /// `"jev-1.13.0"` (distinct from the request's `model`, which may be an
227    /// alias like [`DEFAULT_MODEL`]).
228    pub model: String,
229
230    /// One answer per requested question, keyed identically to the
231    /// request's `questions` map.
232    pub answers: BTreeMap<String, Answer>,
233
234    /// Token accounting for the request. Defaults to zero counts if the API
235    /// ever omits the field.
236    #[serde(default)]
237    pub usage: Usage,
238}
239
240/// The answer to a single [`Question`].
241///
242/// Internally tagged on `type`, mirroring [`Question`]. Kept strict — no
243/// catch-all variant — because `#[serde(other)]` only applies to unit
244/// variants in an internally-tagged enum, so a new field on an existing
245/// answer (which serde already ignores by default) is the free extension
246/// path; a genuinely new `type` is a hard-error case that should name the
247/// unrecognised value and suggest upgrading omni-dev, not silently swallow
248/// it into a catch-all.
249#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
250#[serde(tag = "type", rename_all = "lowercase")]
251pub enum Answer {
252    /// The chosen option, with per-option probabilities.
253    Choice {
254        /// The selected option's name (one of the request's `criteria` keys).
255        choice: String,
256        /// Model confidence in `choice`, in `[0, 1]`.
257        confidence: f64,
258        /// Probability assigned to every option that was offered.
259        probabilities: BTreeMap<String, f64>,
260    },
261
262    /// The scored level, with a legend and per-level probabilities.
263    Score {
264        /// The estimated score, on the scale implied by the request's
265        /// `criteria` (index-like, not necessarily an integer).
266        score: f64,
267        /// Model confidence in `score`, in `[0, 1]`.
268        confidence: f64,
269        /// Scale index (as a string key, e.g. `"0"`) to its label.
270        legend: BTreeMap<String, String>,
271        /// Probability assigned to every scale level.
272        probabilities: BTreeMap<String, f64>,
273    },
274
275    /// The estimated yes/no probability.
276    ///
277    /// Deliberately has **no** `confidence` field — verified against a live
278    /// response; unlike `choice`/`score`, `noul`'s single probability value
279    /// already *is* its own confidence.
280    Noul {
281        /// Probability that the condition is true, in `[0, 1]`.
282        noul: f64,
283    },
284}
285
286/// Token accounting for one [`SystemOneRequest`]/[`SystemOneResponse`] pair.
287#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
288pub struct Usage {
289    /// Number of input tokens consumed (state + questions).
290    pub input_tokens: u64,
291    /// Number of output tokens produced (the answers).
292    pub output_tokens: u64,
293}
294
295#[cfg(test)]
296#[allow(clippy::unwrap_used, clippy::expect_used)]
297mod tests {
298    use super::*;
299
300    // ── Question → wire JSON ────────────────────────────────────────────
301
302    #[test]
303    fn choice_question_serialises_with_map_criteria() {
304        let question = Question::Choice {
305            instructions: "Route this".to_string(),
306            criteria: BTreeMap::from([
307                ("billing".to_string(), "Payments".to_string()),
308                ("technical".to_string(), "Bugs".to_string()),
309            ]),
310        };
311        let value = serde_json::to_value(&question).unwrap();
312        assert_eq!(
313            value,
314            serde_json::json!({
315                "type": "choice",
316                "instructions": "Route this",
317                "criteria": {"billing": "Payments", "technical": "Bugs"}
318            })
319        );
320    }
321
322    #[test]
323    fn score_question_serialises_with_ordered_array_criteria() {
324        let question = Question::Score {
325            instructions: "How urgent".to_string(),
326            criteria: vec!["Low".to_string(), "Medium".to_string(), "High".to_string()],
327        };
328        let value = serde_json::to_value(&question).unwrap();
329        assert_eq!(
330            value,
331            serde_json::json!({
332                "type": "score",
333                "instructions": "How urgent",
334                "criteria": ["Low", "Medium", "High"]
335            })
336        );
337    }
338
339    #[test]
340    fn noul_question_omits_criteria_when_none() {
341        let question = Question::Noul {
342            instructions: "Urgent?".to_string(),
343            criteria: None,
344        };
345        let value = serde_json::to_value(&question).unwrap();
346        assert_eq!(
347            value,
348            serde_json::json!({
349                "type": "noul",
350                "instructions": "Urgent?"
351            })
352        );
353    }
354
355    #[test]
356    fn noul_question_serialises_criteria_as_true_false_map() {
357        let question = Question::Noul {
358            instructions: "Urgent?".to_string(),
359            criteria: Some(NoulCriteria {
360                true_means: Some("Customer is frustrated".to_string()),
361                false_means: Some("Customer is calm".to_string()),
362            }),
363        };
364        let value = serde_json::to_value(&question).unwrap();
365        assert_eq!(
366            value,
367            serde_json::json!({
368                "type": "noul",
369                "instructions": "Urgent?",
370                "criteria": {
371                    "true": "Customer is frustrated",
372                    "false": "Customer is calm"
373                }
374            })
375        );
376    }
377
378    #[test]
379    fn noul_criteria_serialises_partial_sides() {
380        let criteria = NoulCriteria {
381            true_means: Some("yes".to_string()),
382            false_means: None,
383        };
384        let value = serde_json::to_value(&criteria).unwrap();
385        assert_eq!(value, serde_json::json!({"true": "yes"}));
386    }
387
388    // ── Real verified live response ─────────────────────────────────────
389
390    const LIVE_RESPONSE: &str = r#"{"model":"jev-1.13.0","answers":{"department":{"type":"choice","choice":"billing","confidence":0.97,"probabilities":{"sales":0.0,"technical":0.02,"billing":0.98}},"frustration":{"type":"score","score":1.01,"confidence":0.98,"legend":{"0":"Calm, just stating facts","1":"Frustrated but civil","2":"Very angry, strong language"},"probabilities":{"0":0.0,"1":0.99,"2":0.01}},"refund_requested":{"type":"noul","noul":0.98}},"usage":{"input_tokens":417,"output_tokens":71}}"#;
391
392    #[test]
393    fn live_response_deserialises() {
394        let response: SystemOneResponse = serde_json::from_str(LIVE_RESPONSE).unwrap();
395        assert_eq!(response.model, "jev-1.13.0");
396        assert_eq!(response.usage.input_tokens, 417);
397        assert_eq!(response.usage.output_tokens, 71);
398        assert_eq!(response.answers.len(), 3);
399
400        match &response.answers["department"] {
401            Answer::Choice {
402                choice,
403                confidence,
404                probabilities,
405            } => {
406                assert_eq!(choice, "billing");
407                assert!((confidence - 0.97).abs() < f64::EPSILON);
408                assert!((probabilities["billing"] - 0.98).abs() < f64::EPSILON);
409            }
410            other => panic!("expected Choice, got {other:?}"),
411        }
412
413        match &response.answers["frustration"] {
414            Answer::Score {
415                score,
416                confidence,
417                legend,
418                probabilities,
419            } => {
420                assert!((score - 1.01).abs() < f64::EPSILON);
421                assert!((confidence - 0.98).abs() < f64::EPSILON);
422                assert_eq!(legend["0"], "Calm, just stating facts");
423                assert!((probabilities["1"] - 0.99).abs() < f64::EPSILON);
424            }
425            other => panic!("expected Score, got {other:?}"),
426        }
427    }
428
429    #[test]
430    fn live_response_noul_answer_has_no_confidence_field() {
431        let response: SystemOneResponse = serde_json::from_str(LIVE_RESPONSE).unwrap();
432        match &response.answers["refund_requested"] {
433            Answer::Noul { noul } => assert!((noul - 0.98).abs() < f64::EPSILON),
434            other => panic!("expected Noul, got {other:?}"),
435        }
436    }
437
438    #[test]
439    fn unknown_extra_field_on_answer_is_ignored() {
440        let json = serde_json::json!({
441            "type": "noul",
442            "noul": 0.5,
443            "some_future_field": "unexpected"
444        });
445        let answer: Answer = serde_json::from_value(json).unwrap();
446        assert_eq!(answer, Answer::Noul { noul: 0.5 });
447    }
448
449    #[test]
450    fn usage_defaults_when_missing() {
451        let json = serde_json::json!({
452            "model": "jev-1.13.0",
453            "answers": {}
454        });
455        let response: SystemOneResponse = serde_json::from_value(json).unwrap();
456        assert_eq!(response.usage, Usage::default());
457    }
458
459    // ── Question::validate ──────────────────────────────────────────────
460
461    fn choice_with(n: usize) -> Question {
462        Question::Choice {
463            instructions: "Route this".to_string(),
464            criteria: (0..n)
465                .map(|i| (format!("o{i}"), "desc".to_string()))
466                .collect(),
467        }
468    }
469
470    fn score_with(n: usize) -> Question {
471        Question::Score {
472            instructions: "How urgent".to_string(),
473            criteria: (0..n).map(|i| format!("l{i}")).collect(),
474        }
475    }
476
477    #[test]
478    fn validate_rejects_choice_with_fewer_than_two_options() {
479        for n in [0, 1] {
480            let err = choice_with(n).validate().unwrap_err();
481            assert!(matches!(err, JevError::InvalidQuestionSpec(_)));
482            assert!(err.to_string().contains(&format!("got {n}")));
483        }
484        choice_with(2).validate().unwrap();
485    }
486
487    #[test]
488    fn validate_rejects_score_with_fewer_than_two_levels() {
489        for n in [0, 1] {
490            let err = score_with(n).validate().unwrap_err();
491            assert!(matches!(err, JevError::InvalidQuestionSpec(_)));
492            assert!(err.to_string().contains(&format!("got {n}")));
493        }
494        score_with(2).validate().unwrap();
495    }
496
497    #[test]
498    fn validate_for_model_enforces_the_caps_at_their_boundary() {
499        choice_with(255).validate_for_model("jev-1.13.0").unwrap();
500        let err = choice_with(256)
501            .validate_for_model("jev-1.13.0")
502            .unwrap_err();
503        assert!(matches!(err, JevError::InvalidQuestionSpec(_)));
504        assert_eq!(
505            err.to_string(),
506            "Invalid question specification: a choice question accepts at most 255 options on jev-1.13.0, got 256"
507        );
508        score_with(10).validate_for_model("jev-1.13.0").unwrap();
509        let err = score_with(11).validate_for_model("jev-1.13.0").unwrap_err();
510        assert!(err
511            .to_string()
512            .contains("a score question accepts at most 10 levels on jev-1.13.0, got 11"));
513    }
514
515    #[test]
516    fn validate_for_model_enforces_the_caps_for_the_default_alias() {
517        assert!(choice_with(256).validate_for_model(DEFAULT_MODEL).is_err());
518        assert!(score_with(11).validate_for_model(DEFAULT_MODEL).is_err());
519    }
520
521    #[test]
522    fn validate_for_model_leaves_unknown_models_uncapped() {
523        for model in [
524            "jev-2.0",
525            "jev-1.130",
526            "jev-1.13.",
527            "jev-1.13.x",
528            "jev-1.14.0",
529            "",
530        ] {
531            choice_with(300).validate_for_model(model).unwrap();
532            score_with(20).validate_for_model(model).unwrap();
533        }
534    }
535
536    #[test]
537    fn validate_for_model_keeps_the_minimums_for_every_model() {
538        for model in ["jev-1.13.0", "jev-2.0"] {
539            assert!(choice_with(1).validate_for_model(model).is_err());
540            assert!(score_with(1).validate_for_model(model).is_err());
541        }
542    }
543
544    #[test]
545    fn for_model_matches_the_1_13_family_only() {
546        let capped = QuestionLimits {
547            max_choice_options: Some(255),
548            max_score_levels: Some(10),
549        };
550        for model in ["jev-latest", "jev-1.13", "jev-1.13.0", "jev-1.13.7"] {
551            assert_eq!(QuestionLimits::for_model(model), capped, "{model}");
552        }
553        for model in ["jev-1.130", "jev-1.12.0", "jev-2.0.0", "jev-latest-x"] {
554            assert_eq!(
555                QuestionLimits::for_model(model),
556                QuestionLimits::UNLIMITED,
557                "{model}"
558            );
559        }
560    }
561
562    #[test]
563    fn validate_accepts_noul_without_criteria() {
564        Question::Noul {
565            instructions: "Urgent?".to_string(),
566            criteria: None,
567        }
568        .validate()
569        .unwrap();
570    }
571
572    #[test]
573    fn default_model_constant() {
574        assert_eq!(DEFAULT_MODEL, "jev-latest");
575    }
576}
omni_dev/jev/protocol.rs

omni_dev/jev/
protocol.rs
