kime_core/
render.rs

1//! The text the model reads for a question, before tokenizing.
2//!
3//! Only the compat family is here for now. It reproduces what Laya 0.3.7 feeds its tokenizer
4//! (`build_sequence`, `render_options`, `render_criterion` and `serialize_state` in
5//! `laya/common.py`, and `Agent._to_internal`), because a single differing space changes the ids.
6//! The native family renders differently (spec/06-input.md) and arrives with the native models.
7
8use serde_json::Value;
9
10use crate::pyjson::Dumps;
11use crate::request::{Criteria, Question};
12
13const RAW: Dumps = Dumps { ensure_ascii: false };
14const ASCII: Dumps = Dumps { ensure_ascii: true };
15
16/// The default noul descriptions Laya uses when a side is missing or empty.
17pub const NOUL_FALSE: &str = "no, the statement does not hold";
18/// See [`NOUL_FALSE`].
19pub const NOUL_TRUE: &str = "yes, the statement holds";
20
21/// The three pieces of text for one compat question. Every piece has had the tokenizer's mask
22/// literal replaced by a space, so user text cannot forge a marker.
23#[derive(Debug, Clone, PartialEq, Eq)]
24pub struct CompatText {
25    /// `"<type> question: <instructions>"`.
26    pub head: String,
27    /// One per option, in option order, each with a leading space.
28    pub options: Vec<String>,
29}
30
31/// Renders a question the way Laya does. `mask` is the tokenizer's mask literal, `[MASK]` for
32/// ModernBERT and `<mask>` for mmBERT.
33#[must_use]
34pub fn compat_question(q: &Question, mask: &str) -> CompatText {
35    // Laya turns non string instructions into json.dumps(ins) with the defaults, so ensure_ascii
36    // is on here and nowhere else, and null becomes the word null. Laya rejects a missing field and
37    // kime renders it as nothing.
38    let ins = match &q.instructions {
39        None => String::new(),
40        Some(Value::String(s)) => s.clone(),
41        Some(v) => ASCII.to_string(v),
42    };
43    let head = format!("{} question: {}", q.qtype.as_str(), ins.replace(mask, " "));
44    let options = compat_options(&q.criteria)
45        .into_iter()
46        .map(|o| format!(" {}", o.replace(mask, " ")))
47        .collect();
48    CompatText { head, options }
49}
50
51/// Laya's `render_options`, without the leading space and mask replacement.
52#[must_use]
53pub fn compat_options(c: &Criteria) -> Vec<String> {
54    match c {
55        Criteria::Choice(opts) => opts
56            .iter()
57            .map(|o| match &o.description {
58                // Only null and "" mean no description. 0 and false are real values.
59                None => o.label.clone(),
60                Some(Value::String(s)) if s.is_empty() => o.label.clone(),
61                Some(v) => format!("{}: {}", o.label, criterion(v)),
62            })
63            .collect(),
64        Criteria::Score(levels) => {
65            levels.iter().enumerate().map(|(i, v)| format!("level {i}: {}", criterion(v))).collect()
66        }
67        Criteria::Noul { when_false, when_true } => {
68            let side = |v: &Option<Value>, default: &str| match v {
69                None | Some(Value::Null) => default.to_string(),
70                Some(Value::String(s)) if s.is_empty() => default.to_string(),
71                Some(v) => criterion(v),
72            };
73            vec![
74                format!("false: {}", side(when_false, NOUL_FALSE)),
75                format!("true: {}", side(when_true, NOUL_TRUE)),
76            ]
77        }
78    }
79}
80
81/// Laya's `render_criterion`: strings as they are, anything else as JSON with raw Unicode.
82#[must_use]
83pub fn criterion(v: &Value) -> String {
84    match v {
85        Value::String(s) => s.clone(),
86        v => RAW.to_string(v),
87    }
88}
89
90/// Laya's `serialize_state` followed by the mask replacement.
91#[must_use]
92pub fn compat_state(state: &Value, mask: &str) -> String {
93    let s = match state {
94        Value::String(s) => s.clone(),
95        v => RAW.to_string(v),
96    };
97    s.replace(mask, " ")
98}
99
100#[cfg(test)]
101mod tests {
102    use super::*;
103    use crate::request::{Limits, parse};
104    use serde_json::json;
105
106    fn render(q: Value) -> CompatText {
107        let r = parse(&json!({"state": "", "questions": {"q": q}}), &Limits::LAYA).unwrap();
108        compat_question(&r.questions[0], "[MASK]")
109    }
110
111    #[test]
112    fn choice_score_noul() {
113        let t = render(
114            json!({"type": "choice", "instructions": "Pick [MASK] one", "criteria": {"a": "", "b": null, "c": 0, "d": {"x": [1, 2.5]}}}),
115        );
116        assert_eq!(t.head, "choice question: Pick   one");
117        assert_eq!(t.options, [" a", " b", " c: 0", " d: {\"x\": [1, 2.5]}"]);
118        let t = render(
119            json!({"type": "score", "instructions": {"ask": "é"}, "criteria": ["low", null, true]}),
120        );
121        assert_eq!(t.head, "score question: {\"ask\": \"\\u00e9\"}");
122        assert_eq!(t.options, [" level 0: low", " level 1: null", " level 2: true"]);
123        let t = render(
124            json!({"type": "noul", "instructions": null, "criteria": {"TRUE": "", "false": "nope"}}),
125        );
126        assert_eq!(t.head, "noul question: null");
127        assert_eq!(t.options, [" false: nope", " true: yes, the statement holds"]);
128    }
129
130    #[test]
131    fn state() {
132        assert_eq!(
133            compat_state(&json!({"k": "ü", "n": [1e-5]}), "[MASK]"),
134            "{\"k\": \"ü\", \"n\": [1e-05]}"
135        );
136        assert_eq!(compat_state(&json!("a<mask>b"), "<mask>"), "a b");
137    }
138}
kime_core/render.rs

kime_core/
render.rs
