harn_vm/orchestration/training_example/
mod.rs

1//! Project one authoritative Harn agent run into one lossless training example.
2//!
3//! # Why this exists
4//!
5//! Downstream corpus builders used to parse the LLM transcript sidecar
6//! themselves: they duplicated Harn's event vocabulary, guessed run boundaries
7//! from `iteration == 0`, and silently skipped rows they could not read. That
8//! is Harn-owned transcript semantics, and it drifts every time lifecycle or
9//! tool-call recording changes.
10//!
11//! This module is the single authoritative projection. It starts from the
12//! typed [`RunRecord`](crate::orchestration::RunRecord) and the verified
13//! transcript descriptor written by run persistence, replays the canonical
14//! event stream, and emits [`AgentTrainingExample`] — the
15//! `harn.agent_training_example.v1` contract.
16//!
17//! # What it is not
18//!
19//! It is not `std/agent/transcript`. That module is deliberately a
20//! compatibility/analysis normalizer: it infers iterations, defaults missing
21//! ids and names, regex-classifies tool-result errors, and stays lenient
22//! unless a caller opts into strict mode. Those forensic semantics are lossy,
23//! so they cannot be the authority for supervised fine-tuning.
24//!
25//! Every source fact here is either present and typed, or the projection
26//! fails with a structured [`TrainingExampleError`]. There is no lenient mode
27//! and no fallback to the analysis normalizer.
28//!
29//! # What it does not decide
30//!
31//! Eligibility. A caller chooses which explicit run should become a training
32//! example; Harn then projects that run faithfully or explains exactly why it
33//! cannot. Product verdicts stay out of the projector.
34
35mod project;
36mod source;
37mod validate;
38
39#[cfg(test)]
40mod tests;
41
42use std::path::{Path, PathBuf};
43
44use serde::{Deserialize, Serialize};
45use serde_json::Value as JsonValue;
46
47pub use crate::llm::tools::function_schema_from_catalog_row;
48pub use project::{TOOL_CALL_RECEIPT_VERSION, TOOL_RESULT_RECEIPT_VERSION};
49pub use validate::{validate_training_example_pairing, TrainingPairingError};
50
51/// Schema version of the projected example. Bump only for a breaking change to
52/// the shape below; consumers pin on this exact string.
53pub const TRAINING_EXAMPLE_SCHEMA_VERSION: &str = "harn.agent_training_example.v1";
54
55/// Structured reason a run could not be projected. `kind` is a small stable
56/// vocabulary so callers can branch without matching on prose.
57#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
58pub struct TrainingExampleError {
59    pub kind: String,
60    pub message: String,
61    /// 1-based JSONL row that produced the failure, when it came from a row.
62    #[serde(skip_serializing_if = "Option::is_none", default)]
63    pub event_index: Option<usize>,
64}
65
66impl TrainingExampleError {
67    pub(crate) fn new(kind: &str, message: impl Into<String>) -> Self {
68        Self {
69            kind: kind.to_string(),
70            message: message.into(),
71            event_index: None,
72        }
73    }
74
75    pub(crate) fn at(kind: &str, event_index: usize, message: impl Into<String>) -> Self {
76        Self {
77            kind: kind.to_string(),
78            message: message.into(),
79            event_index: Some(event_index),
80        }
81    }
82}
83
84impl std::fmt::Display for TrainingExampleError {
85    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
86        match self.event_index {
87            Some(index) => write!(f, "{} (event {index}): {}", self.kind, self.message),
88            None => write!(f, "{}: {}", self.kind, self.message),
89        }
90    }
91}
92
93impl std::error::Error for TrainingExampleError {}
94
95/// One provider-visible turn, in Harn's canonical provider-independent shape.
96///
97/// Tool results are always `role: "tool"` carrying `tool_call_id`, even when
98/// the run served them to the model as a text-channel `user` echo. `content`
99/// stays byte-exact with what the model saw, so a target renderer can
100/// reproduce either channel without re-parsing anything.
101#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
102pub struct TrainingMessage {
103    pub role: String,
104    pub content: String,
105    #[serde(default, skip_serializing_if = "Vec::is_empty")]
106    pub tool_calls: Vec<TrainingToolCall>,
107    #[serde(default, skip_serializing_if = "Option::is_none")]
108    pub tool_call_id: Option<String>,
109    #[serde(default, skip_serializing_if = "Option::is_none")]
110    pub name: Option<String>,
111}
112
113#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
114pub struct TrainingToolCall {
115    pub id: String,
116    #[serde(rename = "type")]
117    pub call_type: String,
118    pub function: TrainingToolFunction,
119}
120
121#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
122pub struct TrainingToolFunction {
123    pub name: String,
124    pub arguments: JsonValue,
125}
126
127#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
128pub struct TrainingUsage {
129    pub provider_calls: usize,
130    pub input_tokens: u64,
131    pub output_tokens: u64,
132}
133
134/// Identity and digest of the exact bytes this example was projected from.
135#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
136pub struct TrainingSource {
137    pub descriptor_schema_version: String,
138    pub transcript_path: String,
139    pub transcript_sha256: String,
140    pub transcript_byte_len: u64,
141    pub event_count: usize,
142    pub first_event_index: usize,
143    pub last_event_index: usize,
144    pub first_event_id: String,
145    pub last_event_id: String,
146}
147
148#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
149pub struct TrainingProvenance {
150    pub run_id: String,
151    pub session_id: String,
152    #[serde(default, skip_serializing_if = "Option::is_none")]
153    pub stage_id: Option<String>,
154    pub provider: String,
155    pub model: String,
156    #[serde(default, skip_serializing_if = "Option::is_none")]
157    pub route_policy: Option<String>,
158    /// The channel the session claimed and served tool results on, when the
159    /// run recorded one. An escalation can dispatch a call on a different
160    /// effective format without re-claiming the session's, so the two are
161    /// recorded separately rather than collapsed.
162    #[serde(default, skip_serializing_if = "Option::is_none")]
163    pub declared_tool_format: Option<String>,
164    /// Format the calls actually dispatched with.
165    pub effective_tool_format: String,
166    /// Hash of the exact schema array the model was served.
167    pub tool_catalog_hash: String,
168    #[serde(default, skip_serializing_if = "Option::is_none")]
169    pub tool_catalog_content_hash: Option<String>,
170    pub terminal_status: String,
171    pub usage: TrainingUsage,
172    pub source: TrainingSource,
173}
174
175/// The `harn.agent_training_example.v1` contract.
176#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
177pub struct AgentTrainingExample {
178    pub schema_version: String,
179    pub messages: Vec<TrainingMessage>,
180    /// The effective tool catalog exactly as served, never inferred from
181    /// prompt prose or observed argument values.
182    pub tools: Vec<JsonValue>,
183    pub provenance: TrainingProvenance,
184}
185
186/// Which run inside the artifact to project. Selection is always by explicit
187/// identity; nothing here may be chosen by output length, verbosity, or a
188/// `DONE` substring.
189#[derive(Clone, Debug, Default)]
190pub struct TrainingExampleRequest {
191    pub run_record_path: PathBuf,
192    pub run_id: Option<String>,
193    pub session_id: Option<String>,
194}
195
196impl TrainingExampleRequest {
197    pub fn new(run_record_path: impl Into<PathBuf>) -> Self {
198        Self {
199            run_record_path: run_record_path.into(),
200            run_id: None,
201            session_id: None,
202        }
203    }
204}
205
206/// Project one authoritative run into one training example.
207///
208/// Fails with a structured error rather than degrading whenever the run record
209/// cannot supply a required source fact, the transcript does not verify
210/// against its descriptor, or the recorded turns violate the call/result
211/// pairing invariant.
212pub fn project_agent_training_example(
213    request: &TrainingExampleRequest,
214) -> Result<AgentTrainingExample, TrainingExampleError> {
215    let run_record_path = request.run_record_path.as_path();
216    let run = load_run(run_record_path)?;
217    if let Some(expected) = request.run_id.as_deref() {
218        if run.id != expected {
219            return Err(TrainingExampleError::new(
220                "run_id_mismatch",
221                format!(
222                    "{} holds run {}, not the requested {expected}",
223                    run_record_path.display(),
224                    run.id
225                ),
226            ));
227        }
228    }
229    let transcript_path = super::verified_llm_transcript_pointer_path(&run, run_record_path)
230        .map_err(|error| TrainingExampleError::new(error.kind, error.message))?;
231    let descriptor = run
232        .observability
233        .as_ref()
234        .and_then(|observability| {
235            observability
236                .transcript_pointers
237                .iter()
238                .find(|pointer| pointer.kind == "llm_jsonl")
239        })
240        .and_then(|pointer| pointer.descriptor.clone())
241        .ok_or_else(|| {
242            TrainingExampleError::new("missing_descriptor", "run has no llm_jsonl descriptor")
243        })?;
244    if !descriptor.complete {
245        return Err(TrainingExampleError::new(
246            "incomplete_source",
247            format!(
248                "{} has no terminal record; the run did not finalize",
249                transcript_path.display()
250            ),
251        ));
252    }
253    let events = source::load_events(&transcript_path)?;
254    project::project(&run, &descriptor, &transcript_path, &events, request)
255}
256
257/// Read the run record exactly as it was persisted.
258///
259/// Deliberately not [`load_run_record`](crate::orchestration::load_run_record):
260/// that helper refreshes observability from the sidecar currently on disk, so
261/// the descriptor it hands back is a fresh re-derivation and its digest can
262/// never disagree with the bytes it was just computed from. Authority needs
263/// the descriptor as *written*, so a sidecar edited after the run finalized is
264/// caught by the digest comparison instead of being silently re-blessed.
265fn load_run(path: &Path) -> Result<super::RunRecord, TrainingExampleError> {
266    let content = std::fs::read_to_string(path).map_err(|error| {
267        TrainingExampleError::new(
268            "run_record_unreadable",
269            format!("failed to read {}: {error}", path.display()),
270        )
271    })?;
272    serde_json::from_str(&content).map_err(|error| {
273        TrainingExampleError::new(
274            "run_record_unreadable",
275            format!("failed to parse {}: {error}", path.display()),
276        )
277    })
278}
harn_vm/orchestration/training_example/mod.rs

harn_vm/orchestration/training_example/
mod.rs
