harn_vm/orchestration/training_example/
validate.rs

1//! The `harn.agent_training_example.v1` call/result pairing invariant.
2//!
3//! One owner, two callers: the projector checks its own output before handing
4//! it out, and `harn models lora export` checks every projected example it
5//! consumes. Keeping the rule here means a trainer-facing consumer can never
6//! drift into a laxer reading of it than the producer used.
7//!
8//! The invariant: every tool call an assistant turn makes is answered by
9//! exactly one `role: "tool"` message carrying that call's id, the answers
10//! arrive in request order, they all arrive before the next non-tool turn, and
11//! the example ends on a no-tool assistant completion turn.
12
13use std::collections::BTreeSet;
14
15use super::TrainingMessage;
16
17#[derive(Clone, Debug, PartialEq, Eq)]
18pub struct TrainingPairingError {
19    pub kind: String,
20    pub message: String,
21    /// Index into the message list where the violation was detected.
22    pub message_index: usize,
23}
24
25impl TrainingPairingError {
26    fn new(kind: &str, message_index: usize, message: impl Into<String>) -> Self {
27        Self {
28            kind: kind.to_string(),
29            message: message.into(),
30            message_index,
31        }
32    }
33}
34
35impl std::fmt::Display for TrainingPairingError {
36    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
37        write!(
38            f,
39            "{} (message {}): {}",
40            self.kind, self.message_index, self.message
41        )
42    }
43}
44
45impl std::error::Error for TrainingPairingError {}
46
47pub fn validate_training_example_pairing(
48    messages: &[TrainingMessage],
49) -> Result<(), TrainingPairingError> {
50    let mut expected: Vec<(String, String)> = Vec::new();
51    let mut answered: BTreeSet<String> = BTreeSet::new();
52    let mut opened_by: usize = 0;
53
54    for (index, message) in messages.iter().enumerate() {
55        match message.role.as_str() {
56            "tool" => {
57                let Some(call_id) = message.tool_call_id.as_deref().filter(|id| !id.is_empty())
58                else {
59                    return Err(TrainingPairingError::new(
60                        "orphaned_tool_result",
61                        index,
62                        "tool message carries no tool_call_id, so it names no call",
63                    ));
64                };
65                if expected.is_empty() {
66                    return Err(TrainingPairingError::new(
67                        "orphaned_tool_result",
68                        index,
69                        format!("tool message answers {call_id} with no call outstanding"),
70                    ));
71                }
72                let (head_id, head_name) = &expected[0];
73                if head_id != call_id {
74                    let kind = if expected.iter().any(|(id, _)| id == call_id) {
75                        "out_of_order_tool_result"
76                    } else if answered.contains(call_id) {
77                        "duplicate_tool_result"
78                    } else {
79                        "orphaned_tool_result"
80                    };
81                    return Err(TrainingPairingError::new(
82                        kind,
83                        index,
84                        format!("expected the result for {head_id} ({head_name}), got {call_id}"),
85                    ));
86                }
87                answered.insert(call_id.to_string());
88                expected.remove(0);
89            }
90            "assistant" => {
91                if let Some((id, name)) = expected.first() {
92                    return Err(TrainingPairingError::new(
93                        "unpaired_tool_call",
94                        index,
95                        format!(
96                            "call {id} ({name}) from message {opened_by} is unanswered at the \
97                             next assistant turn"
98                        ),
99                    ));
100                }
101                let mut seen = BTreeSet::new();
102                for call in &message.tool_calls {
103                    if call.id.is_empty() {
104                        return Err(TrainingPairingError::new(
105                            "malformed_tool_call",
106                            index,
107                            format!("call to `{}` has no id", call.function.name),
108                        ));
109                    }
110                    if !seen.insert(call.id.clone()) {
111                        return Err(TrainingPairingError::new(
112                            "duplicate_tool_call_id",
113                            index,
114                            format!("assistant turn reuses call id {}", call.id),
115                        ));
116                    }
117                    if answered.contains(&call.id) {
118                        return Err(TrainingPairingError::new(
119                            "duplicate_tool_call_id",
120                            index,
121                            format!(
122                                "call id {} was already used earlier in this example",
123                                call.id
124                            ),
125                        ));
126                    }
127                    expected.push((call.id.clone(), call.function.name.clone()));
128                }
129                opened_by = index;
130            }
131            _ => {
132                if let Some((id, name)) = expected.first() {
133                    return Err(TrainingPairingError::new(
134                        "unpaired_tool_call",
135                        index,
136                        format!(
137                            "call {id} ({name}) from message {opened_by} is unanswered at the \
138                             next `{}` turn; a generic placeholder turn is not a tool result",
139                            message.role
140                        ),
141                    ));
142                }
143                if message.tool_call_id.is_some() {
144                    return Err(TrainingPairingError::new(
145                        "orphaned_tool_result",
146                        index,
147                        format!(
148                            "`{}` message carries a tool_call_id but is not a tool result",
149                            message.role
150                        ),
151                    ));
152                }
153            }
154        }
155    }
156    if let Some((id, name)) = expected.first() {
157        return Err(TrainingPairingError::new(
158            "unpaired_tool_call",
159            messages.len(),
160            format!("example ends with call {id} ({name}) unanswered"),
161        ));
162    }
163    match messages.last() {
164        Some(message) if message.role == "assistant" && message.tool_calls.is_empty() => Ok(()),
165        Some(message) => Err(TrainingPairingError::new(
166            "missing_terminal_assistant",
167            messages.len() - 1,
168            format!(
169                "example must end with a no-tool assistant completion turn, not `{}`",
170                message.role
171            ),
172        )),
173        None => Err(TrainingPairingError::new(
174            "missing_terminal_assistant",
175            0,
176            "example must end with a no-tool assistant completion turn",
177        )),
178    }
179}
harn_vm/orchestration/training_example/validate.rs

harn_vm/orchestration/training_example/
validate.rs
