turnframe_understand/tasks/
verify.rs

1//! `verify`: whether what was understood is what the user said. It can only take away.
2
3use std::collections::BTreeMap;
4use std::fmt::Write as _;
5
6use serde::{Deserialize, Serialize};
7use serde_json::Value;
8use turnframe_core::understanding::{ArgumentValue, MessageRef, RecordValue, UnderstoodArgument};
9use turnframe_provider::request::Message;
10use turnframe_tasks::{ModelTask, StructuralError, TaskKind};
11
12use crate::input::{Speaker, UnderstandingInput};
13use crate::render;
14use crate::schema::{object, one_of, text};
15use crate::tasks::not_one_of;
16use crate::words::Span;
17
18const BUILT_IN: &str = include_str!("../../prompts/understand/verify.md");
19
20/// The verification task, over one turn.
21#[derive(Debug, Clone, Copy)]
22pub struct Verify<'a> {
23    turn: &'a UnderstandingInput,
24}
25
26impl<'a> Verify<'a> {
27    /// The task for `turn`.
28    #[must_use]
29    pub const fn new(turn: &'a UnderstandingInput) -> Self {
30        Self { turn }
31    }
32}
33
34/// One act as understood, shown for a person to check.
35#[derive(Debug, Clone)]
36pub struct VerifyInput<'a> {
37    /// How the unit is shown.
38    pub label: &'static str,
39    /// Its words.
40    pub words: Span,
41    /// What the operation does, or what starting the workflow means.
42    pub meaning: String,
43    /// The record, as shown.
44    pub record: String,
45    /// The arguments understood, by name.
46    pub arguments: &'a BTreeMap<String, UnderstoodArgument>,
47    /// How each argument is labelled, by name.
48    pub labels: BTreeMap<String, String>,
49    /// How a record-valued argument's record is shown.
50    pub record_labels: BTreeMap<String, String>,
51    /// What a value of a closed set means, by argument, when the operation says.
52    pub meanings: BTreeMap<String, String>,
53    /// The operation, and which of the times the request asks for it this act is.
54    pub occurrence: Option<(String, usize, usize)>,
55    /// What the whole-turn check found, when this call reads the act again.
56    pub note: Option<String>,
57    /// Words of this message the unit continues, such as the request a correction changes.
58    pub continues: Option<Span>,
59}
60
61/// The verdict on each argument.
62#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
63#[serde(rename_all = "snake_case")]
64pub enum ArgumentVerdict {
65    /// The user gave that value.
66    Stated,
67    /// The user gave no value for it.
68    NotStated,
69    /// The user gave another value.
70    Different,
71    /// The user gave only part of it.
72    Incomplete,
73    /// It takes words that are not part of the value, such as the field's name.
74    TooMuch,
75}
76
77/// The verdict on the act as a whole.
78#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
79#[serde(rename_all = "snake_case")]
80pub enum Overall {
81    /// The user asked for this operation on this record.
82    Confirmed,
83    /// The user did not ask for it.
84    NotRequested,
85    /// The user meant another record.
86    WrongRecord,
87}
88
89/// What the verifier found.
90#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
91pub struct Verdict {
92    /// Why, in a sentence; it reaches the repair when the act is not confirmed.
93    pub reason: String,
94    /// Each argument's verdict, by name.
95    pub arguments: BTreeMap<String, ArgumentVerdict>,
96    /// The act's verdict.
97    pub overall: Overall,
98}
99
100impl Verdict {
101    /// Whether everything checked out.
102    #[must_use]
103    pub fn confirmed(&self) -> bool {
104        self.overall == Overall::Confirmed
105            && self
106                .arguments
107                .values()
108                .all(|verdict| *verdict == ArgumentVerdict::Stated)
109    }
110
111    /// The arguments found wanting, by name.
112    #[must_use]
113    pub fn at_fault(&self) -> Vec<String> {
114        self.arguments
115            .iter()
116            .filter(|(_, verdict)| **verdict != ArgumentVerdict::Stated)
117            .map(|(name, _)| name.clone())
118            .collect()
119    }
120
121    /// The feedback a repair of the extraction is given.
122    #[must_use]
123    pub fn feedback(&self) -> String {
124        self.feedback_given(|_| false)
125    }
126
127    /// The feedback, where `refers_back` names the arguments copied from this message in a
128    /// conversation with earlier words: words there may only point at a value said before.
129    #[must_use]
130    pub fn feedback_given(&self, refers_back: impl Fn(&str) -> bool) -> String {
131        let mut out = format!("A check of your answer found: {}", self.reason);
132        for (name, verdict) in &self.arguments {
133            let said = match verdict {
134                ArgumentVerdict::Stated => continue,
135                ArgumentVerdict::NotStated | ArgumentVerdict::Different if refers_back(name) => {
136                    "you copied it from this message, which gives no value of its own there; \
137                     words that only refer back to a value said before give that value: point \
138                     at the value alone in the earlier message that says it, or give not_given \
139                     when none does"
140                }
141                ArgumentVerdict::NotStated => "the user gave no value for it",
142                ArgumentVerdict::Different => "the user gave another value",
143                ArgumentVerdict::Incomplete => "the user gave more of it than you took",
144                ArgumentVerdict::TooMuch => {
145                    "it takes words that are not part of the value; point at the value alone"
146                }
147            };
148            let _ = write!(out, "\n- {name}: {said}");
149        }
150        out
151    }
152}
153
154/// Whether `argument` is words of this message copied as they stand, not a value computed
155/// from them: a date or a choice read from words is never the words themselves.
156fn copied_from_this_message(turn: &UnderstandingInput, argument: &UnderstoodArgument) -> bool {
157    let (Some(excerpt), ArgumentValue::Json(Value::String(text))) =
158        (argument.excerpt, &argument.value)
159    else {
160        return false;
161    };
162    let bare = |text: &str| -> String {
163        text.chars()
164            .filter(|c| c.is_alphanumeric() || c.is_whitespace())
165            .collect::<String>()
166            .split_whitespace()
167            .collect::<Vec<_>>()
168            .join(" ")
169            .to_lowercase()
170    };
171    excerpt.message == MessageRef::Current
172        && turn
173            .message
174            .slice(crate::words::Span::new(
175                excerpt.words.first,
176                excerpt.words.last,
177            ))
178            .is_ok_and(|said| bare(said) == bare(text))
179}
180
181impl<'a> ModelTask for Verify<'a> {
182    type Input = VerifyInput<'a>;
183    type Output = Verdict;
184
185    fn kind(&self) -> TaskKind {
186        TaskKind::Verify
187    }
188
189    fn prompt_name(&self) -> &str {
190        "understand.verify"
191    }
192
193    fn instructions(&self) -> &str {
194        BUILT_IN
195    }
196
197    fn schema(&self, input: &VerifyInput<'a>) -> Value {
198        let verdicts = one_of([
199            "stated",
200            "not_stated",
201            "different",
202            "incomplete",
203            "too_much",
204        ]);
205        let arguments = input
206            .arguments
207            .keys()
208            .map(|name| (name.as_str(), verdicts.clone()))
209            .collect();
210        object(vec![
211            ("reason", text("Why, in one sentence.")),
212            ("arguments", object(arguments)),
213            (
214                "overall",
215                one_of(["confirmed", "not_requested", "wrong_record"]),
216            ),
217        ])
218    }
219
220    fn render(&self, input: &VerifyInput<'a>) -> Vec<Message> {
221        let turn = self.turn;
222        let mut understood = String::from("Understood:");
223        if input.arguments.is_empty() {
224            understood.push_str(" no arguments.");
225        }
226        let mut cited = Vec::new();
227        for (name, argument) in input.arguments {
228            let label = input.labels.get(name).map_or(name.as_str(), String::as_str);
229            let shown = render::understood(argument, turn, |record: &RecordValue| {
230                input
231                    .record_labels
232                    .get(name)
233                    .cloned()
234                    .unwrap_or_else(|| format!("{record:?}"))
235            });
236            let _ = write!(understood, "\n- {name} ({label}): {shown}");
237            if let Some(meaning) = input.meanings.get(name) {
238                let _ = write!(understood, "\n  {meaning}");
239            }
240            // Only a copy from this message can be words pointing at an earlier value; a value
241            // taken from the earlier message is what they point at.
242            let copied_here = copied_from_this_message(turn, argument);
243            if copied_here && turn.transcript.iter().any(|m| m.speaker == Speaker::User) {
244                understood.push_str(
245                    "\n  Copied from this message: words that only refer back to a value said \
246                     before («what I told you before») are not that value.",
247                );
248            }
249            if let Some(excerpt) = argument.excerpt
250                && let MessageRef::Earlier { index } = excerpt.message
251                && !cited.contains(&index)
252            {
253                cited.push(index);
254            }
255        }
256        let earlier = cited
257            .iter()
258            .filter_map(|index| {
259                let message = turn.transcript.get(*index)?;
260                Some(format!(
261                    "Earlier message {}: {}",
262                    render::message_name(*index),
263                    render::quoted(message.words.text())
264                ))
265            })
266            .collect::<Vec<_>>();
267        let words = &turn.message;
268        // A correction answers for what the last turn did, never for the question asked next.
269        let correction = input.label == "Correction";
270        let said = |span: Span| render::quoted(words.slice(span).unwrap_or_default());
271        // A correction of this message follows the request it corrects, as the user said them.
272        let (unit, then) = match (correction, input.continues) {
273            (true, Some(request)) => (
274                format!("Request: {}", said(request)),
275                Some(format!(
276                    "Corrected by: {}, the user's last word on what it changes.",
277                    said(input.words)
278                )),
279            ),
280            // Named as the words the act reads, so they are never taken for its value.
281            (_, continues) => (
282                format!(
283                    "The part of the message this act reads, {}: {}",
284                    with_article(input.label),
285                    said(input.words)
286                ),
287                continues.map(|span| format!("It continues: {}", said(span))),
288            ),
289        };
290        vec![Message::user(render::sections([
291            Some(format!("Operation: {}", input.meaning)),
292            Some(format!("Record: {}", input.record)),
293            Some(understood),
294            Some(format!("Today: {}", turn.today.format("%A %-d %B %Y"))),
295            (!earlier.is_empty()).then(|| earlier.join("\n")),
296            correction.then(|| render::receipts(turn)).flatten(),
297            render::last_assistant(turn),
298            (!correction).then(|| render::expectation(turn)).flatten(),
299            Some(format!(
300                "The user's message: {}",
301                render::quoted(words.text())
302            )),
303            Some(unit),
304            then,
305            input.occurrence.as_ref().map(|(operation, number, of)| {
306                format!(
307                    "This request asks for {operation} {of} times: this act is the {}, and other \
308                     acts read the rest of the request. Judge only whether the user gave these \
309                     values for it.",
310                    ordinal(*number)
311                )
312            }),
313            input
314                .note
315                .as_ref()
316                .map(|note| format!("A check of the whole message found: {note}")),
317        ]))]
318    }
319
320    fn check(&self, input: &VerifyInput<'a>, output: &Verdict) -> Result<(), StructuralError> {
321        let expected: Vec<String> = input.arguments.keys().cloned().collect();
322        if let Some(unknown) = output.arguments.keys().find(|n| !expected.contains(n)) {
323            return Err(not_one_of("argument", unknown, &expected));
324        }
325        if let Some(missing) = expected.iter().find(|n| !output.arguments.contains_key(*n)) {
326            return Err(StructuralError::new(
327                "missing_argument",
328                format!("`arguments.{missing}` is missing"),
329            ));
330        }
331        Ok(())
332    }
333
334    fn agree(&self, left: &Verdict, right: &Verdict) -> bool {
335        left.overall == right.overall && left.arguments == right.arguments
336    }
337}
338
339/// «Answer» as «an answer», «Request» as «a request».
340fn with_article(label: &str) -> String {
341    let lower = label.to_lowercase();
342    let article = if lower.starts_with(['a', 'e', 'i', 'o', 'u']) {
343        "an"
344    } else {
345        "a"
346    };
347    format!("{article} {lower}")
348}
349
350/// `1` as «first», up to «tenth»; past it, «number 11».
351fn ordinal(number: usize) -> String {
352    const WORDS: [&str; 10] = [
353        "first", "second", "third", "fourth", "fifth", "sixth", "seventh", "eighth", "ninth",
354        "tenth",
355    ];
356    number
357        .checked_sub(1)
358        .and_then(|index| WORDS.get(index))
359        .map_or_else(|| format!("number {number}"), |word| (*word).to_owned())
360}
361
362#[cfg(test)]
363mod tests {
364    use super::*;
365
366    #[test]
367    fn a_note_from_the_whole_turn_check_is_shown() {
368        let turn = UnderstandingInput::new("name Lisbon", "en-GB", chrono::NaiveDate::MIN);
369        let arguments = BTreeMap::new();
370        let input = VerifyInput {
371            label: "Request",
372            words: Span::new(0, 1),
373            meaning: "Name the trip.".to_owned(),
374            record: "Trip 1".to_owned(),
375            arguments: &arguments,
376            labels: BTreeMap::new(),
377            record_labels: BTreeMap::new(),
378            meanings: BTreeMap::new(),
379            occurrence: None,
380            note: Some("the message may not ask for this act".to_owned()),
381            continues: None,
382        };
383        let rendered = format!("{:?}", Verify::new(&turn).render(&input));
384        assert!(
385            rendered.contains(
386                "A check of the whole message found: the message may not ask for this act"
387            ),
388            "{rendered}"
389        );
390    }
391
392    #[test]
393    fn an_answer_is_checked_against_the_question_it_answers() {
394        let turn = UnderstandingInput::new("X", "en-GB", chrono::NaiveDate::MIN)
395            .with_earlier(crate::Speaker::Assistant, "What is A?")
396            .with_expectation(crate::Expectation::Obligation {
397                record: turnframe_core::ids::TargetToken::new("t1"),
398                sentence: "What is A?".to_owned(),
399            });
400        let arguments = BTreeMap::new();
401        let input = VerifyInput {
402            label: "Answer",
403            words: Span::new(1, 1),
404            meaning: "Set A.".to_owned(),
405            record: "Record 1".to_owned(),
406            arguments: &arguments,
407            labels: BTreeMap::new(),
408            record_labels: BTreeMap::new(),
409            meanings: BTreeMap::new(),
410            occurrence: None,
411            note: None,
412            continues: None,
413        };
414        let rendered = format!("{:?}", Verify::new(&turn).render(&input));
415        assert!(
416            rendered.contains("Last assistant message: «What is A?»"),
417            "{rendered}"
418        );
419        assert!(
420            rendered.contains("The assistant asked about: What is A?"),
421            "{rendered}"
422        );
423    }
424
425    #[test]
426    fn a_correction_is_checked_with_the_words_it_continues() {
427        let turn =
428            UnderstandingInput::new("set A to X 2026, no, Y", "en-GB", chrono::NaiveDate::MIN);
429        let arguments = BTreeMap::new();
430        let input = VerifyInput {
431            label: "Correction",
432            words: Span::new(5, 6),
433            meaning: "Set A.".to_owned(),
434            record: "Record 1".to_owned(),
435            arguments: &arguments,
436            labels: BTreeMap::new(),
437            record_labels: BTreeMap::new(),
438            meanings: BTreeMap::new(),
439            occurrence: None,
440            note: None,
441            continues: Some(Span::new(0, 4)),
442        };
443        let rendered = format!("{:?}", Verify::new(&turn).render(&input));
444        let request = rendered
445            .find("Request: «set A to X 2026,»")
446            .expect(&rendered);
447        let corrected = rendered
448            .find("Corrected by: «no, Y», the user's last word on what it changes.")
449            .expect(&rendered);
450        assert!(request < corrected, "{rendered}");
451    }
452
453    #[test]
454    fn a_correction_is_checked_against_what_the_last_turn_did() {
455        let turn = UnderstandingInput::new("no, the other one", "en-GB", chrono::NaiveDate::MIN)
456            .with_receipt(crate::PreviousReceipt::new("r1", "Field A set to X."))
457            .with_expectation(crate::Expectation::Obligation {
458                record: turnframe_core::ids::TargetToken::new("t1"),
459                sentence: "What is B?".to_owned(),
460            });
461        let arguments = BTreeMap::new();
462        let input = VerifyInput {
463            label: "Correction",
464            words: Span::new(1, 4),
465            meaning: "Set A.".to_owned(),
466            record: "Record 1".to_owned(),
467            arguments: &arguments,
468            labels: BTreeMap::new(),
469            record_labels: BTreeMap::new(),
470            meanings: BTreeMap::new(),
471            occurrence: None,
472            note: None,
473            continues: None,
474        };
475        let rendered = format!("{:?}", Verify::new(&turn).render(&input));
476        assert!(rendered.contains("r1: Field A set to X."), "{rendered}");
477        assert!(!rendered.contains("What is B?"), "{rendered}");
478    }
479
480    #[test]
481    fn an_occurrence_is_judged_as_the_one_it_is() {
482        let turn = UnderstandingInput::new("add a bag and a meal", "en-GB", chrono::NaiveDate::MIN);
483        let arguments = BTreeMap::new();
484        let input = VerifyInput {
485            label: "Request",
486            words: Span::new(0, 4),
487            meaning: "trip.add_extra: Add an extra.".to_owned(),
488            record: "Trip 1".to_owned(),
489            arguments: &arguments,
490            labels: BTreeMap::new(),
491            record_labels: BTreeMap::new(),
492            meanings: BTreeMap::new(),
493            occurrence: Some(("trip.add_extra".to_owned(), 2, 2)),
494            note: None,
495            continues: None,
496        };
497        let rendered = format!("{:?}", Verify::new(&turn).render(&input));
498        assert!(
499            rendered.contains(
500                "This request asks for trip.add_extra 2 times: this act is the second, and \
501                 other acts read the rest of the request. Judge only whether the user gave these \
502                 values for it."
503            ),
504            "{rendered}"
505        );
506    }
507
508    #[test]
509    fn a_value_of_a_closed_set_is_shown_with_what_it_means() {
510        let turn =
511            UnderstandingInput::new("none, she never joined", "en-GB", chrono::NaiveDate::MIN);
512        let arguments = BTreeMap::from([(
513            "reason".to_owned(),
514            UnderstoodArgument {
515                value: turnframe_core::understanding::ArgumentValue::Json("not_applicable".into()),
516                excerpt: None,
517            },
518        )]);
519        let input = VerifyInput {
520            label: "Request",
521            words: Span::new(0, 3),
522            meaning: "Decline the number.".to_owned(),
523            record: "Traveler 1".to_owned(),
524            arguments: &arguments,
525            labels: BTreeMap::new(),
526            record_labels: BTreeMap::new(),
527            meanings: BTreeMap::from([(
528                "reason".to_owned(),
529                "not_applicable when there is none to give".to_owned(),
530            )]),
531            occurrence: None,
532            note: None,
533            continues: None,
534        };
535        let rendered = format!("{:?}", Verify::new(&turn).render(&input));
536        assert!(
537            rendered.contains("- reason (reason): «not_applicable», given earlier\\n  not_applicable when there is none to give"),
538            "{rendered}"
539        );
540    }
541}
turnframe_understand/tasks/verify.rs

turnframe_understand/tasks/
verify.rs
