skilltest_core/
conversation.rs

1//! The conversation model: the transcript that flows between the runner and the
2//! provider, and is ultimately handed to evals.
3
4use schemars::JsonSchema;
5use serde::{Deserialize, Serialize};
6use serde_json::Value;
7
8/// Who produced a message.
9#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
10#[serde(rename_all = "lowercase")]
11pub enum Role {
12    /// The (real or simulated) user driving the skill.
13    User,
14    /// The skill / assistant under test.
15    Assistant,
16    /// System-level framing, if a provider chooses to surface it.
17    System,
18}
19
20/// One normalized tool-call / action event the skill took during a turn, lifted
21/// from oneharness's `events` array (its `--events` output). Harness-agnostic, so
22/// a consumer can inspect *what the skill did* — shell commands, file edits, tool
23/// uses — across any harness, not just the final text. Mirrors the oneharness
24/// action-event shape; `input` is the structured, tool-shaped args so a consumer
25/// can match on the command string or file path without re-parsing.
26///
27/// `input` is a free-form JSON value, so `Message`/`Transcript` are `PartialEq`
28/// but not `Eq`.
29#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
30pub struct ToolEvent {
31    /// `tool_call` (the skill invoked a tool) or `tool_result` (the observation).
32    pub kind: String,
33    /// Normalized tool name where knowable (e.g. `bash`, `edit_file`); `null` for
34    /// a `tool_result` or when the harness did not name it.
35    #[serde(default, skip_serializing_if = "Option::is_none")]
36    pub name: Option<String>,
37    /// Structured tool arguments (the command, the file path); `null` when none.
38    #[serde(default, skip_serializing_if = "Option::is_none")]
39    pub input: Option<Value>,
40    /// The result/observation text, when the transcript exposed it.
41    #[serde(default, skip_serializing_if = "Option::is_none")]
42    pub output: Option<String>,
43    /// Position within the turn, so ordering ("did X before Y") is expressible.
44    #[serde(default)]
45    pub index: usize,
46}
47
48/// A single turn in the conversation.
49#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
50pub struct Message {
51    pub role: Role,
52    pub content: String,
53    /// The normalized tool events the skill took producing this turn (assistant
54    /// turns only, and only when the harness exposed a tool transcript via
55    /// oneharness `--events`). Empty otherwise. Surfaced for post-hoc analysis
56    /// and streamed live for short-circuiting.
57    #[serde(default, skip_serializing_if = "Vec::is_empty")]
58    pub events: Vec<ToolEvent>,
59}
60
61impl Message {
62    /// Build a user message.
63    pub fn user(content: impl Into<String>) -> Self {
64        Self {
65            role: Role::User,
66            content: content.into(),
67            events: Vec::new(),
68        }
69    }
70
71    /// Build an assistant message.
72    pub fn assistant(content: impl Into<String>) -> Self {
73        Self {
74            role: Role::Assistant,
75            content: content.into(),
76            events: Vec::new(),
77        }
78    }
79
80    /// Attach the turn's normalized tool events (builder style).
81    #[must_use]
82    pub fn with_events(mut self, events: Vec<ToolEvent>) -> Self {
83        self.events = events;
84        self
85    }
86}
87
88/// An ordered list of messages. Thin wrapper so the type reads clearly at call
89/// sites and so we can grow conversation-level helpers without churn.
90#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)]
91pub struct Transcript {
92    pub messages: Vec<Message>,
93}
94
95impl Transcript {
96    /// Start a transcript from the initial user input given to the skill.
97    pub fn from_input(input: impl Into<String>) -> Self {
98        Self {
99            messages: vec![Message::user(input)],
100        }
101    }
102
103    /// Append a message.
104    pub fn push(&mut self, message: Message) {
105        self.messages.push(message);
106    }
107
108    /// Number of assistant turns produced so far.
109    #[must_use]
110    pub fn assistant_turns(&self) -> usize {
111        self.messages
112            .iter()
113            .filter(|m| m.role == Role::Assistant)
114            .count()
115    }
116}
skilltest_core/conversation.rs

skilltest_core/
conversation.rs
