#![allow(dead_code)]
use std::path::Path;
use async_trait::async_trait;
use serde::Deserialize;
use serde_json::Value;
use switchyard_protocol::{ContentBlock, Request, Role, WireFormat};
use crate::{LibsyError, Result};
use crate::core::processor::{Event, Processor};
use crate::core::state::State;
const SOFT: f32 = 0.3;
const HARD: f32 = 0.7;
const CRITICAL: f32 = 1.0;
static ERROR_PATTERNS: &[(&str, f32, &[&str])] = &[
(
"oom",
CRITICAL,
&["out of memory", "memoryerror", "cannot allocate memory"],
),
(
"connection_refused",
HARD,
&[
"connection refused",
"connectionrefusederror",
"econnrefused",
],
),
("traceback", HARD, &["traceback (most recent call last)"]),
(
"import_error",
HARD,
&["modulenotfounderror:", "importerror:", "no module named "],
),
(
"cmd_not_found",
HARD,
&["command not found", "not found\n", "/usr/bin/env: "],
),
("assertion", HARD, &["assertionerror"]),
("value_error", HARD, &["valueerror:"]),
("syntax_error", HARD, &["syntaxerror:"]),
(
"timeout",
HARD,
&[
"timed out",
"timeouterror",
"timeout expired",
"deadline exceeded",
],
),
(
"no_such_file",
HARD,
&[
"filenotfounderror:",
"no such file or directory",
"file does not exist",
],
),
("exit_nonzero", SOFT, &["returned non-zero"]),
];
static NONZERO_EXIT_PHRASES: &[&str] = &[
"exit code",
"exit status",
"exited with code",
"exited with status",
];
static EDIT_TOOL_NAMES: &[&str] = &[
"edit",
"multiedit",
"notebookedit",
"str_replace",
"str_replace_based_edit_tool",
"apply_patch", "text_editor",
"patch", ];
static WRITE_TOOL_NAMES: &[&str] = &["write", "create_file", "new_file", "write_file"];
static BASH_WRITE_PATTERNS: &[&str] = &[
"cat >",
"cat >>",
"echo >",
"echo >>",
"tee ",
"printf >",
"printf >>",
"> /",
">> /",
"<< 'eof'",
"<<eof",
"<<'eof'",
"<< eof",
];
static PYTHON_WRITE_PATTERNS: &[&str] = &["write_text(", "writelines(", ".write("];
static JAVASCRIPT_WRITE_PATTERNS: &[&str] = &[
"writefilesync(",
"writefile(",
"appendfilesync(",
"appendfile(",
];
static BASH_EDIT_PATTERNS: &[&str] = &[
"sed -i",
"sed --in-place",
"awk -i inplace",
"awk 'inplace=1'",
"patch ",
"patch -p",
"perl -i",
"perl -p -i",
"perl -pi",
];
static BASH_READ_PATTERNS: &[&str] = &[
"cat /", "cat ./", "cat ../", "grep ", "ls ", "ls -", "find ", "head ", "tail ", "wc ",
"diff ", "which ", "ps ", "df ", "du ", "stat ", "file ", "less ", "more ",
];
static BASH_READ_COMMANDS: &[&str] = &[
"cat", "rg", "nl", "jq", "pwd", "tree", "sed", "grep", "ls", "find", "head", "tail", "wc",
"diff", "which", "ps", "df", "du", "stat", "file", "less", "more", "readlink", "realpath",
"basename", "dirname", "printenv",
];
static GIT_READ_SUBCOMMANDS: &[&str] = &[
"status",
"diff",
"log",
"show",
"show-ref",
"rev-parse",
"ls-files",
"ls-remote",
"ls-tree",
"grep",
"blame",
"merge-base",
"check-ignore",
"tag",
];
static READ_TOOL_NAMES: &[&str] = &["read", "view", "read_file", "search_files"];
static PLAN_TOOL_NAMES: &[&str] = &["todowrite", "todo_write", "todo", "update_plan"];
static BASH_TOOL_NAMES: &[&str] = &[
"bash",
"shell_command",
"shell",
"local_shell_call",
"terminal",
"exec_command", ];
static TEST_PASS_PHRASES: &[&str] = &[
" passed",
"passed in",
"tests passed",
"all tests passed",
"test ok",
"test result: ok",
"passed.\n",
"tests pass",
"\nok ", "✓ ",
];
static TEST_FAILURE_LITERAL: &[&str] = &["✗ ", "fatal:", "assertionerror", "error:"];
static NUMERIC_FAILURE_KEYWORDS: &[&str] = &["failed", "failure", "failures", "errors", "error"];
pub const DEFAULT_RECENT_WINDOW: usize = 3;
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Eq)]
#[serde(default, deny_unknown_fields)]
pub struct ToolSemantics {
pub observe: Vec<String>,
pub mutate: Vec<String>,
pub plan: Vec<String>,
pub new: Vec<String>,
}
impl ToolSemantics {
pub fn validate(&self) -> Result<()> {
let mut seen: Vec<(String, &'static str)> = Vec::new();
for (category, names) in [
("observe", &self.observe),
("mutate", &self.mutate),
("plan", &self.plan),
("new", &self.new),
] {
for name in names {
if name.trim().is_empty() {
return Err(tool_semantics_error(format!(
"tool_semantics.{category} contains an empty tool name"
)));
}
let normalized = name.to_ascii_lowercase();
if is_builtin_tool_name(&name.to_lowercase()) {
return Err(tool_semantics_error(format!(
"tool {name:?} already has built-in semantics and cannot be reclassified"
)));
}
if let Some((_, previous)) = seen.iter().find(|(seen, _)| seen == &normalized) {
return Err(tool_semantics_error(format!(
"tool {name:?} appears in both tool_semantics.{previous} and tool_semantics.{category}"
)));
}
seen.push((normalized, category));
}
}
Ok(())
}
fn classify(&self, name: &str) -> Option<ToolSemantic> {
if contains_name(&self.observe, name) {
Some(ToolSemantic::Observe)
} else if contains_name(&self.mutate, name) {
Some(ToolSemantic::Mutate(MutationKind::Write))
} else if contains_name(&self.plan, name) {
Some(ToolSemantic::Plan)
} else if contains_name(&self.new, name) {
Some(ToolSemantic::New)
} else {
None
}
}
}
fn contains_name(names: &[String], candidate: &str) -> bool {
names
.iter()
.any(|name| name.eq_ignore_ascii_case(candidate))
}
fn tool_semantics_error(message: String) -> LibsyError {
LibsyError::AlgorithmError { message }
}
#[derive(Clone, Debug, Default)]
pub struct ToolSignals {
pub severity: f32,
pub repeated_failure: bool,
pub no_error_streak: u32,
pub edit_count: u32,
pub write_count: u32,
pub read_count: u32,
pub todowrite_count: u32,
pub recent_edit_count: u32,
pub recent_write_count: u32,
pub recent_read_count: u32,
pub recent_todowrite_count: u32,
pub new_count: u32,
pub recent_new_count: u32,
pub pure_bash_streak: u32,
pub tests_passed: bool,
pub tool_result_count: u32,
pub assistant_turn_count: u32,
pub turn_depth: u32,
pub compacted: bool,
}
impl ToolSignals {
pub fn from_request(request: &Request, window_size: Option<usize>) -> Self {
Self::from_request_with_semantics(request, window_size, &ToolSemantics::default())
}
pub fn from_request_with_semantics(
request: &Request,
window_size: Option<usize>,
semantics: &ToolSemantics,
) -> Self {
extract_tool_signals_with_window_and_semantics(
request,
window_size.unwrap_or(DEFAULT_RECENT_WINDOW),
semantics,
)
}
}
#[derive(Debug, Clone)]
struct ObservedToolCall {
name: String,
command: Option<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum MutationKind {
Write,
Edit,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ToolSemantic {
Mutate(MutationKind),
Observe,
Plan,
New,
Unknown,
}
#[derive(Debug, Clone)]
pub struct ToolSignalProcessor {
pub recent_window: usize,
pub tool_semantics: ToolSemantics,
}
impl Default for ToolSignalProcessor {
fn default() -> Self {
Self {
recent_window: DEFAULT_RECENT_WINDOW,
tool_semantics: ToolSemantics::default(),
}
}
}
#[async_trait]
impl Processor<State> for ToolSignalProcessor {
async fn process(&self, state: &mut State, event: Event<'_>) -> Result<()> {
if let Event::Request { request: req, .. } = event {
let tool_signal = ToolSignals::from_request_with_semantics(
req,
Some(self.recent_window),
&self.tool_semantics,
);
state.tool_signals = Some(tool_signal);
}
Ok(())
}
}
fn classify_tool_call(name: &str, command: Option<&str>) -> ToolSemantic {
classify_tool_call_with_semantics(name, command, &ToolSemantics::default())
}
fn classify_tool_call_with_semantics(
name: &str,
command: Option<&str>,
semantics: &ToolSemantics,
) -> ToolSemantic {
let lower = name.to_lowercase();
if WRITE_TOOL_NAMES.contains(&lower.as_str()) {
return ToolSemantic::Mutate(MutationKind::Write);
}
if EDIT_TOOL_NAMES.contains(&lower.as_str()) {
return ToolSemantic::Mutate(MutationKind::Edit);
}
if READ_TOOL_NAMES.contains(&lower.as_str()) {
return ToolSemantic::Observe;
}
if PLAN_TOOL_NAMES.contains(&lower.as_str()) {
return ToolSemantic::Plan;
}
if BASH_TOOL_NAMES.contains(&lower.as_str())
&& let Some(cmd) = command
{
if BASH_WRITE_PATTERNS.iter().any(|p| cmd.contains(p)) || shell_command_is_write(cmd) {
return ToolSemantic::Mutate(MutationKind::Write);
}
if cmd.contains("python") && PYTHON_WRITE_PATTERNS.iter().any(|p| cmd.contains(p)) {
return ToolSemantic::Mutate(MutationKind::Write);
}
if shell_invokes_program(cmd, "node")
&& JAVASCRIPT_WRITE_PATTERNS
.iter()
.any(|pattern| cmd.contains(pattern))
{
return ToolSemantic::Mutate(MutationKind::Write);
}
if BASH_EDIT_PATTERNS.iter().any(|p| cmd.contains(p)) || shell_command_is_edit(cmd) {
return ToolSemantic::Mutate(MutationKind::Edit);
}
if BASH_READ_PATTERNS.iter().any(|p| cmd.contains(p)) || shell_command_is_read(cmd) {
return ToolSemantic::Observe;
}
}
semantics.classify(name).unwrap_or(ToolSemantic::Unknown)
}
fn is_builtin_tool_name(lower: &str) -> bool {
WRITE_TOOL_NAMES.contains(&lower)
|| EDIT_TOOL_NAMES.contains(&lower)
|| READ_TOOL_NAMES.contains(&lower)
|| PLAN_TOOL_NAMES.contains(&lower)
|| BASH_TOOL_NAMES.contains(&lower)
}
fn shell_segments(command: &str) -> impl Iterator<Item = &str> {
let mut chars = command.char_indices();
let mut start = 0usize;
let mut quote = None;
let mut escaped = false;
let mut finished = false;
std::iter::from_fn(move || {
loop {
for (index, character) in chars.by_ref() {
if escaped {
escaped = false;
} else if character == '\\' && quote != Some('\'') {
escaped = true;
} else if quote == Some(character) {
quote = None;
} else if quote.is_none() && matches!(character, '\'' | '"') {
quote = Some(character);
} else if quote.is_none() && matches!(character, '\n' | ';' | '|' | '&') {
let segment = command[start..index].trim();
start = index + character.len_utf8();
if !segment.is_empty() {
return Some(segment);
}
}
}
if finished {
return None;
}
finished = true;
let segment = command[start..].trim();
if !segment.is_empty() {
return Some(segment);
}
}
})
}
fn shell_words(segment: &str) -> std::iter::Peekable<std::str::SplitAsciiWhitespace<'_>> {
let mut words = segment.split_ascii_whitespace().peekable();
if words.peek().copied() == Some("env") {
words.next();
while words.peek().is_some_and(|word| word.starts_with('-')) {
words.next();
}
}
while words
.peek()
.is_some_and(|word| word.contains('=') && !word.starts_with('='))
{
words.next();
}
words
}
fn program_name(word: &str) -> &str {
Path::new(word)
.file_name()
.and_then(|name| name.to_str())
.unwrap_or(word)
}
fn shell_invokes_program(command: &str, expected: &str) -> bool {
shell_segments(command).any(|segment| {
shell_words(segment)
.next()
.is_some_and(|word| program_name(word) == expected)
})
}
fn shell_command_is_write(command: &str) -> bool {
shell_segments(command).any(|segment| {
let mut words = shell_words(segment);
let Some(program) = words.next().map(program_name) else {
return false;
};
if matches!(program, "cp" | "mkdir" | "touch" | "install") {
return true;
}
let redirects_output = words.any(|word| matches!(word, ">" | ">>"));
redirects_output
&& (matches!(program, "echo" | "printf" | "git")
|| BASH_READ_COMMANDS.contains(&program))
})
}
fn shell_command_is_edit(command: &str) -> bool {
shell_segments(command).any(|segment| {
let mut words = shell_words(segment);
let Some(program) = words.next().map(program_name) else {
return false;
};
let has_arg = |arg: &str| words.clone().any(|word| word == arg);
match program {
"mv" | "rm" => true,
"perl" => words
.take_while(|word| word.starts_with('-'))
.any(|option| {
option
.trim_start_matches('-')
.chars()
.any(|flag| flag == 'i')
}),
"git" => words
.next()
.is_some_and(|subcommand| matches!(subcommand, "apply" | "am" | "restore")),
"gofmt" => has_arg("-w"),
"cargo" => words.clone().next() == Some("fmt") && !has_arg("--check"),
"ruff" => {
let subcommand = words.clone().next();
(subcommand == Some("format") && !has_arg("--check"))
|| (subcommand == Some("check") && has_arg("--fix"))
}
"prettier" => has_arg("--write"),
"black" => !has_arg("--check"),
_ => {
(words.clone().any(|word| program_name(word) == "prettier") && has_arg("--write"))
|| (words.clone().any(|word| program_name(word) == "ruff")
&& ((has_arg("format") && !has_arg("--check"))
|| (has_arg("check") && has_arg("--fix"))))
}
}
})
}
fn shell_command_is_read(command: &str) -> bool {
shell_segments(command).any(|segment| {
if segment == "env" {
return true;
}
let mut words = shell_words(segment);
let Some(program) = words.next().map(program_name) else {
return false;
};
if BASH_READ_COMMANDS.contains(&program) {
return true;
}
if program == "command" && words.next() == Some("-v") {
return true;
}
if program == "type" {
return true;
}
if program != "git" {
return false;
}
match words.next() {
Some("branch") => words.next().is_none_or(|arg| arg.starts_with('-')),
Some("remote") => words
.next()
.is_none_or(|arg| arg.starts_with('-') || arg == "get-url"),
Some("config") => words
.next()
.is_some_and(|arg| matches!(arg, "--get" | "--get-all" | "--list" | "-l")),
Some(subcommand) => GIT_READ_SUBCOMMANDS.contains(&subcommand),
None => false,
}
})
}
fn extract_tool_signals_with_window(request: &Request, recent_window: usize) -> ToolSignals {
extract_tool_signals_with_window_and_semantics(
request,
recent_window,
&ToolSemantics::default(),
)
}
fn extract_tool_signals_with_window_and_semantics(
request: &Request,
recent_window: usize,
semantics: &ToolSemantics,
) -> ToolSignals {
let messages = &request.llm_request.messages;
let mut tool_texts: Vec<(String, bool)> = Vec::new();
let mut tool_calls: Vec<ObservedToolCall> = Vec::new();
let mut compacted = false;
let mut tool_result_count = 0usize;
let mut assistant_turn_count = 0usize;
for message in messages {
if message.role == Role::Assistant {
assistant_turn_count += 1;
}
for block in &message.content {
match block {
ContentBlock::ToolCall(call) => {
tool_calls.push(ObservedToolCall {
name: call.name.clone(),
command: command_of(&call.arguments),
});
}
ContentBlock::ToolResult(result) => {
tool_result_count += 1;
let text = result
.content
.iter()
.filter_map(text_of)
.collect::<Vec<_>>()
.join("\n");
let is_error = result.is_error == Some(true);
if !text.is_empty() || is_error {
tool_texts.push((text, is_error));
}
}
ContentBlock::Unknown { provider, raw }
if provider.as_str() == WireFormat::OpenAiResponses.as_str()
&& raw.get("type").and_then(Value::as_str)
== Some("apply_patch_call_output") =>
{
tool_result_count += 1;
let text = raw
.get("output")
.and_then(Value::as_str)
.unwrap_or_default();
let is_error = raw.get("status").and_then(Value::as_str) == Some("failed");
tool_texts.push((text.to_owned(), is_error));
}
ContentBlock::Unknown { provider, raw }
if provider.as_str() == WireFormat::OpenAiResponses.as_str()
&& raw.get("type").and_then(Value::as_str) == Some("shell_call_output") =>
{
tool_result_count += 1;
let mut texts = Vec::new();
let mut is_error = false;
if let Some(outputs) = raw.get("output").and_then(Value::as_array) {
for output in outputs {
for field in ["stdout", "stderr"] {
if let Some(text) = output.get(field).and_then(Value::as_str)
&& !text.is_empty()
{
texts.push(text);
}
}
if let Some(outcome) = output.get("outcome") {
is_error |= match outcome.get("type").and_then(Value::as_str) {
Some("timeout") => true,
Some("exit")
if outcome
.get("exit_code")
.and_then(Value::as_i64)
.is_some_and(|code| code != 0) =>
{
texts.push("returned non-zero");
output
.get("stderr")
.and_then(Value::as_str)
.is_some_and(|text| !text.trim().is_empty())
}
_ => false,
};
}
}
}
let text = texts.join("\n");
tool_texts.push((text, is_error));
}
ContentBlock::Text { text } => {
compacted |= text.to_lowercase().contains(COMPACTION_MARKER);
}
_ => {}
}
}
}
let mut signal = build_signal(
tool_texts,
tool_calls,
messages.len() as u32,
recent_window,
semantics,
);
signal.compacted = compacted;
signal.tool_result_count = u32::try_from(tool_result_count).unwrap_or(u32::MAX);
signal.assistant_turn_count = u32::try_from(assistant_turn_count).unwrap_or(u32::MAX);
signal
}
const COMPACTION_MARKER: &str = "session is being continued";
fn command_of(arguments: &Value) -> Option<String> {
let decoded = arguments
.as_str()
.and_then(|raw| serde_json::from_str::<Value>(raw).ok());
let object = decoded.as_ref().unwrap_or(arguments);
["command", "cmd", "input"]
.iter()
.filter_map(|key| object.get(*key))
.find_map(command_text)
}
fn command_text(value: &Value) -> Option<String> {
match value {
Value::String(text) => Some(text.to_lowercase()),
Value::Array(parts) => {
let joined = parts
.iter()
.filter_map(Value::as_str)
.collect::<Vec<_>>()
.join(" ");
(!joined.is_empty()).then(|| joined.to_lowercase())
}
_ => None,
}
}
fn text_of(block: &ContentBlock) -> Option<&str> {
match block {
ContentBlock::Text { text } | ContentBlock::Refusal { text } => Some(text.as_str()),
_ => None,
}
}
fn build_signal(
tool_texts: Vec<(String, bool)>,
tool_calls: Vec<ObservedToolCall>,
turn_depth: u32,
recent_window: usize,
semantics: &ToolSemantics,
) -> ToolSignals {
let sev_start = tool_texts.len().saturating_sub(recent_window.max(1));
let mut severity = 0.0f32;
let mut failure_fingerprints = Vec::new();
let mut repeated_failure = false;
for (text, is_error) in &tool_texts[sev_start..] {
let (sev, _patterns) = classify_text(text);
let sev = if *is_error { sev.max(HARD) } else { sev };
if sev > severity {
severity = sev;
}
if let Some(fingerprint) = failure_fingerprint(text, *is_error) {
repeated_failure |= failure_fingerprints.contains(&fingerprint);
failure_fingerprints.push(fingerprint);
}
}
let no_error_streak = compute_no_error_streak(&tool_texts);
let recent_start = tool_calls.len().saturating_sub(recent_window);
let mut write_count = 0u32;
let mut edit_count = 0u32;
let mut read_count = 0u32;
let mut todowrite_count = 0u32;
let mut recent_write_count = 0u32;
let mut recent_edit_count = 0u32;
let mut recent_read_count = 0u32;
let mut recent_todowrite_count = 0u32;
let mut new_count = 0u32;
let mut recent_new_count = 0u32;
let mut pure_bash_streak = 0u32;
let mut streak_open = true;
for (i, tc) in tool_calls.iter().enumerate().rev() {
let cat = classify_tool_call_with_semantics(&tc.name, tc.command.as_deref(), semantics);
if streak_open {
if matches!(cat, ToolSemantic::Unknown) {
pure_bash_streak += 1;
} else {
streak_open = false;
}
}
match cat {
ToolSemantic::Mutate(MutationKind::Write) => {
write_count += 1;
if i >= recent_start {
recent_write_count += 1;
}
}
ToolSemantic::Mutate(MutationKind::Edit) => {
edit_count += 1;
if i >= recent_start {
recent_edit_count += 1;
}
}
ToolSemantic::Observe => {
read_count += 1;
if i >= recent_start {
recent_read_count += 1;
}
}
ToolSemantic::Plan => {
todowrite_count += 1;
if i >= recent_start {
recent_todowrite_count += 1;
}
}
ToolSemantic::New => {
new_count += 1;
if i >= recent_start {
recent_new_count += 1;
}
}
ToolSemantic::Unknown => {}
}
}
let tests_passed = detect_tests_passed(&tool_texts, recent_window);
ToolSignals {
severity,
repeated_failure,
no_error_streak,
edit_count,
write_count,
read_count,
todowrite_count,
recent_edit_count,
recent_write_count,
recent_read_count,
recent_todowrite_count,
new_count,
recent_new_count,
pure_bash_streak,
tests_passed,
turn_depth,
tool_result_count: 0,
assistant_turn_count: 0,
compacted: false,
}
}
fn content_to_text(content: Option<&Value>) -> Option<String> {
match content? {
Value::String(s) => Some(s.clone()),
Value::Array(blocks) => {
let parts: Vec<&str> = blocks
.iter()
.filter_map(|b| {
b.as_object()
.filter(|o| o.get("type").and_then(Value::as_str) == Some("text"))
.and_then(|o| o.get("text"))
.and_then(Value::as_str)
})
.collect();
if parts.is_empty() {
None
} else {
Some(parts.join("\n"))
}
}
_ => None,
}
}
pub(crate) fn classify_text(text: &str) -> (f32, Vec<String>) {
let lower = text.to_lowercase();
let mut patterns = Vec::new();
let mut severity: f32 = 0.0;
for (name, sev, substrings) in ERROR_PATTERNS {
if substrings.iter().any(|sub| lower.contains(sub)) {
patterns.push(name.to_string());
severity = severity.max(*sev);
}
}
if has_nonzero_exit_status(&lower) && !patterns.iter().any(|p| p == "exit_nonzero") {
patterns.push("exit_nonzero".to_string());
severity = severity.max(SOFT);
}
for (name, matched) in [
("compile_error", has_compiler_diagnostic(&lower)),
("runtime_exception", has_runtime_exception(&lower)),
("runtime_panic", has_runtime_panic(&lower)),
("patch_error", has_patch_failure(&lower)),
] {
if matched && !patterns.iter().any(|pattern| pattern == name) {
patterns.push(name.to_string());
severity = severity.max(HARD);
}
}
(severity, patterns)
}
fn failure_fingerprint(text: &str, is_error: bool) -> Option<String> {
let (severity, patterns) = classify_text(text);
if severity < HARD && !is_error {
return None;
}
let lower = text.to_lowercase();
let diagnostic = lower
.lines()
.find(|line| is_failure_diagnostic(line))
.or_else(|| lower.lines().find(|line| !line.trim().is_empty()))
.unwrap_or_default();
let normalized = normalize_failure_text(diagnostic);
Some(format!("{}|{normalized}", patterns.join(",")))
}
fn is_failure_diagnostic(line: &str) -> bool {
let line = line.trim();
[
"error",
"exception",
"panic",
"failed",
"timed out",
"timeout",
"connection refused",
"cannot allocate memory",
"out of memory",
"not found",
]
.iter()
.any(|marker| line.contains(marker))
}
fn normalize_failure_text(text: &str) -> String {
let mut normalized = String::new();
for word in text.split_whitespace() {
if !normalized.is_empty() {
normalized.push(' ');
}
let mut in_digits = false;
if word.starts_with('/') || word.contains("/src/") || word.contains("/tmp/") {
normalized.push_str("<path>");
continue;
}
for character in word.chars() {
if character.is_ascii_digit() {
if !in_digits {
normalized.push('#');
in_digits = true;
}
} else {
normalized.push(character);
in_digits = false;
}
}
}
normalized.chars().take(240).collect()
}
fn has_compiler_diagnostic(lower: &str) -> bool {
lower.lines().any(|line| {
let line = line.trim_start();
if matches!(
line,
"compilation failed" | "error: compilation failed" | "error: could not compile"
) || line.starts_with("error: could not compile ")
{
return true;
}
let Some(rest) = line.strip_prefix("error[e") else {
return false;
};
let Some((code, _)) = rest.split_once("]:") else {
return false;
};
!code.is_empty() && code.chars().all(|character| character.is_ascii_digit())
})
}
fn has_runtime_exception(lower: &str) -> bool {
let has_exception_line = lower.lines().any(|line| {
let line = line.trim_start();
[
"typeerror:",
"referenceerror:",
"rangeerror:",
"runtimeerror:",
"keyerror:",
"attributeerror:",
]
.iter()
.any(|prefix| line.starts_with(prefix))
});
has_exception_line && (lower.contains("\n at ") || lower.contains("\n at "))
}
fn has_runtime_panic(lower: &str) -> bool {
lower
.lines()
.any(|line| line.trim_start().starts_with("panic: runtime error:"))
&& (lower.contains("\ngoroutine ") || lower.contains("[signal sig"))
}
fn has_patch_failure(lower: &str) -> bool {
lower.lines().any(|line| {
let line = line.trim_start();
line.starts_with("error: patch failed:")
|| line.starts_with("patch failed:")
|| line.contains(": patch does not apply")
|| line.starts_with("invalid context")
})
}
fn has_nonzero_exit_status(lower: &str) -> bool {
NONZERO_EXIT_PHRASES
.iter()
.any(|phrase| phrase_followed_by_nonzero_integer(lower, phrase))
}
fn phrase_followed_by_nonzero_integer(lower: &str, phrase: &str) -> bool {
let mut cursor = 0usize;
while let Some(rel) = lower[cursor..].find(phrase) {
let value_start = cursor + rel + phrase.len();
let rest = lower[value_start..].trim_start_matches(|c: char| {
c.is_ascii_whitespace() || matches!(c, ':' | '=' | '\'' | '"' | '`')
});
let digits: String = rest.chars().take_while(|c| c.is_ascii_digit()).collect();
if !digits.is_empty() && digits.chars().any(|d| d != '0') {
return true;
}
cursor = value_start;
}
false
}
fn compute_no_error_streak(tool_texts: &[(String, bool)]) -> u32 {
let mut streak = 0u32;
for (text, is_error) in tool_texts.iter().rev() {
let (sev, _) = classify_text(text);
if *is_error || sev > 0.0 {
break;
}
streak += 1;
}
streak
}
fn detect_tests_passed(tool_texts: &[(String, bool)], recent_window: usize) -> bool {
let start = tool_texts.len().saturating_sub(recent_window.max(1));
let recent = &tool_texts[start..];
let after_latest_failure = recent
.iter()
.rposition(|(text, is_error)| *is_error || classify_text(text).0 > 0.0)
.map_or(recent, |index| &recent[index + 1..]);
after_latest_failure.iter().any(|(text, _)| {
let lower = text.to_lowercase();
TEST_PASS_PHRASES.iter().any(|p| lower.contains(p))
&& !TEST_FAILURE_LITERAL.iter().any(|p| lower.contains(p))
&& !has_nonzero_failure_count(&lower)
})
}
fn has_nonzero_failure_count(lower: &str) -> bool {
for kw in NUMERIC_FAILURE_KEYWORDS {
let mut cursor = 0usize;
while let Some(rel) = lower[cursor..].find(kw) {
let kw_start = cursor + rel;
let kw_end = kw_start + kw.len();
let boundary_after = lower[kw_end..]
.chars()
.next()
.is_none_or(|c| !c.is_ascii_alphanumeric());
if boundary_after {
let prefix = &lower[..kw_start];
let trimmed = prefix.trim_end_matches(|c: char| c.is_whitespace());
let digits_rev: String = trimmed
.chars()
.rev()
.take_while(|c| c.is_ascii_digit())
.collect();
if !digits_rev.is_empty() && digits_rev.chars().any(|d| d != '0') {
return true;
}
}
cursor = kw_start + kw.len();
}
}
false
}
#[cfg(test)]
mod tests {
use super::*;
use crate::algorithms::util::stage::score_signal;
use serde_json::json;
use switchyard_protocol::{
ContentBlock, LlmRequest, Message, Metadata, Role, ToolCall, ToolResult,
};
fn with_messages(messages: Vec<Message>) -> Request {
Request {
llm_request: LlmRequest {
messages,
..LlmRequest::default()
},
raw_request: None,
metadata: None,
}
}
fn tc(name: &str) -> Message {
Message {
role: Role::Assistant,
content: vec![ContentBlock::ToolCall(ToolCall {
id: String::new(),
name: name.to_string(),
arguments: json!({}),
})],
}
}
fn bash(command: &str) -> Message {
Message {
role: Role::Assistant,
content: vec![ContentBlock::ToolCall(ToolCall {
id: String::new(),
name: "Bash".to_string(),
arguments: json!({"command": command}),
})],
}
}
fn tr(text: &str) -> Message {
Message {
role: Role::User,
content: vec![ContentBlock::ToolResult(ToolResult {
tool_call_id: String::new(),
content: vec![ContentBlock::Text {
text: text.to_string(),
}],
is_error: None,
})],
}
}
#[test]
fn clean_text_has_zero_severity() {
let (sev, patterns) = classify_text("everything went fine");
assert_eq!(sev, 0.0);
assert!(patterns.is_empty());
}
#[test]
fn traceback_is_hard() {
let (sev, patterns) = classify_text("Traceback (most recent call last):\n ValueError");
assert_eq!(sev, HARD);
assert!(patterns.contains(&"traceback".to_string()));
}
#[test]
fn oom_is_critical() {
let (sev, _) = classify_text("Out of memory: kill process 1234");
assert_eq!(sev, CRITICAL);
}
#[test]
fn connection_refused_is_hard() {
let (severity, _) = classify_text("Connection refused on port 8000");
assert_eq!(severity, HARD);
}
#[test]
fn repeated_failure_ignores_volatile_paths_and_numbers() {
let request = with_messages(vec![
tr("error[E0308]: mismatched types at /tmp/a/src/lib.rs:12"),
tr("error[E0308]: mismatched types at /tmp/b/src/lib.rs:47"),
]);
assert!(ToolSignals::from_request(&request, None).repeated_failure);
}
#[test]
fn different_failures_are_not_repeated() {
let request = with_messages(vec![
tr("error[E0308]: mismatched types"),
tr("error[E0509]: cannot move out"),
]);
assert!(!ToolSignals::from_request(&request, None).repeated_failure);
}
#[test]
fn one_material_failure_is_not_repeated() {
let request = with_messages(vec![tr("Connection refused on port 8000")]);
assert!(!ToolSignals::from_request(&request, None).repeated_failure);
}
#[test]
fn structured_tool_failures_feed_error_and_recovery_signals() {
for text in ["Dependency unavailable", "", "5 passed in 0.12s"] {
let mut failed = tr(text);
let ContentBlock::ToolResult(result) = &mut failed.content[0] else {
panic!("expected tool result");
};
result.is_error = Some(true);
let mut request = with_messages(vec![tr("5 passed in 0.12s"), failed.clone()]);
let signals = ToolSignals::from_request(&request, Some(3));
assert_eq!(signals.severity, HARD);
assert!(!signals.repeated_failure);
assert_eq!(signals.no_error_streak, 0);
assert!(!signals.tests_passed);
request.llm_request.messages.push(failed);
assert!(ToolSignals::from_request(&request, Some(3)).repeated_failure);
request.llm_request.messages.push(tr("5 passed in 0.12s"));
let recovered = ToolSignals::from_request(&request, Some(1));
assert_eq!(recovered.severity, 0.0);
assert!(!recovered.repeated_failure);
assert_eq!(recovered.no_error_streak, 1);
assert!(recovered.tests_passed);
}
}
#[test]
fn severity_is_max_across_patterns() {
let (sev, _) = classify_text("exit code 1\nTraceback (most recent call last):");
assert_eq!(sev, HARD);
}
#[test]
fn codex_process_exit_zero_stays_clean() {
let (sev, patterns) =
classify_text("Chunk ID: abc\nProcess exited with code 0\nOutput:\nok");
assert_eq!(sev, 0.0);
assert!(!patterns.contains(&"exit_nonzero".to_string()));
}
#[test]
fn nonzero_exit_codes_are_soft_errors() {
let cases = [
"Process exited with code 1",
"Process exited with code 127",
"exit code: 2",
"exit status 3",
"exited with status 9",
];
for case in cases {
let (sev, patterns) = classify_text(case);
assert_eq!(sev, SOFT, "expected soft severity for {case}");
assert!(patterns.contains(&"exit_nonzero".to_string()));
}
}
#[test]
fn partial_process_failures_are_hard_errors() {
let cases = [
(
"Process running with session ID 12\nOutput:\nerror[E0509]: cannot move out",
"compile_error",
),
(
"Process exited with code 0\nOutput:\nTypeError: value is undefined\n at main.js:1:2",
"runtime_exception",
),
(
"Process running with session ID 13\nOutput:\npanic: runtime error: index out of range\n\ngoroutine 6 [running]:",
"runtime_panic",
),
(
"Process exited with code 0\nOutput:\nerror: patch failed: src/lib.rs:4\nerror: src/lib.rs: patch does not apply",
"patch_error",
),
];
for (text, expected_pattern) in cases {
let (severity, patterns) = classify_text(text);
assert_eq!(severity, HARD, "expected hard severity for {text}");
assert!(patterns.iter().any(|pattern| pattern == expected_pattern));
}
}
#[test]
fn source_text_that_names_exceptions_stays_clean() {
let text =
"pub enum TypeError: this is documentation\nlet sample = 'panic: runtime error:';";
assert_eq!(classify_text(text).0, 0.0);
}
#[test]
fn file_does_not_exist_is_hard() {
let (sev, patterns) =
classify_text("Error: File does not exist. Note: current working directory is /app.");
assert_eq!(sev, HARD);
assert!(patterns.contains(&"no_such_file".to_string()));
}
#[test]
fn bare_does_not_exist_stays_clean() {
let (sev, _) = classify_text("The directory does not exist yet, creating it now.");
assert_eq!(sev, 0.0);
}
#[test]
fn no_error_streak_all_clean() {
let texts = vec![("ok".to_string(), false), ("all good".to_string(), false)];
assert_eq!(compute_no_error_streak(&texts), 2);
}
#[test]
fn no_error_streak_stops_at_error() {
let texts = vec![
("Traceback (most recent call last):".to_string(), false),
("ok".to_string(), false),
("ok".to_string(), false),
];
assert_eq!(compute_no_error_streak(&texts), 2);
}
#[test]
fn tests_passed_detects_pytest_output() {
assert!(detect_tests_passed(
&[("====== 5 passed in 0.12s ======".to_string(), false)],
DEFAULT_RECENT_WINDOW
));
}
#[test]
fn tests_passed_ignores_partial_failures() {
assert!(!detect_tests_passed(
&[("2 failed, 5 passed in 0.56s".to_string(), false)],
DEFAULT_RECENT_WINDOW
));
}
#[test]
fn tests_passed_must_follow_the_latest_failure() {
assert!(!detect_tests_passed(
&[
("5 passed in 0.12s".to_string(), false),
(
"Traceback (most recent call last):\nValueError".to_string(),
false
),
("edit applied".to_string(), false),
],
DEFAULT_RECENT_WINDOW
));
assert!(detect_tests_passed(
&[
(
"Traceback (most recent call last):\nValueError".to_string(),
false
),
("5 passed in 0.12s".to_string(), false),
],
DEFAULT_RECENT_WINDOW
));
}
#[test]
fn severity_is_windowed_over_recent_results() {
let request = with_messages(vec![
tr("Traceback (most recent call last):\n ValueError"),
tr("ok"),
tr("ok"),
]);
assert_eq!(extract_tool_signals_with_window(&request, 3).severity, HARD);
assert_eq!(extract_tool_signals_with_window(&request, 1).severity, 0.0);
}
#[test]
fn extract_openai_chat_tool_results() {
let request = with_messages(vec![
Message::text(Role::User, "do something"),
tc("Edit"),
tr("Traceback (most recent call last):\n ValueError"),
]);
let sig = ToolSignals::from_request(&request, None);
assert_eq!(sig.severity, HARD);
assert_eq!(sig.edit_count, 1);
assert_eq!(sig.turn_depth, 3);
}
#[test]
fn extract_anthropic_tool_results() {
let request = with_messages(vec![tr("Traceback (most recent call last):\n ValueError")]);
let sig = ToolSignals::from_request(&request, None);
assert_eq!(sig.severity, HARD);
}
#[test]
fn extract_responses_api_tool_results() {
let request = with_messages(vec![tc("Write"), tr("file written successfully")]);
let sig = ToolSignals::from_request(&request, None);
assert_eq!(sig.severity, 0.0);
assert_eq!(sig.write_count, 1);
}
#[test]
fn responses_builtin_tool_failures_escalate() {
use crate::algorithms::util::stage::{PickOutcome, PickerMode, Tier, pick_tier};
let mut cases = Vec::new();
for (status, output) in [
(
"failed",
"Synthetic dependency unavailable; retry with the recovery path.",
),
(
"completed",
"Synthetic dependency unavailable; retry with the recovery path.",
),
("failed", ""),
] {
cases.push((
json!({
"type": "apply_patch_call_output",
"status": status,
"output": output,
}),
if status == "failed" { HARD } else { 0.0 },
));
}
for (outcome, stdout, stderr, severity) in [
(json!({"type": "exit", "exit_code": 1}), "", "", SOFT),
(json!({"type": "exit", "exit_code": 1}), "", " \n", SOFT),
(
json!({"type": "exit", "exit_code": 1}),
"",
"command failed",
HARD,
),
(json!({"type": "timeout"}), "", "", HARD),
(json!({"type": "exit", "exit_code": 0}), "done", "", 0.0),
(
json!({"type": "exit", "exit_code": 0}),
"Traceback (most recent call last):",
"",
HARD,
),
(
json!({"type": "exit", "exit_code": 0}),
"",
"Traceback (most recent call last):",
HARD,
),
] {
cases.push((
json!({
"type": "shell_call_output",
"output": [
{"stdout": stdout, "stderr": stderr, "outcome": outcome},
{"stdout": "", "stderr": "", "outcome": {"type": "exit", "exit_code": 0}}
],
}),
severity,
));
}
for (raw, severity) in cases {
let is_error = severity >= HARD;
let mut request = with_messages(
["call_1", "call_2"]
.into_iter()
.map(|call_id| {
let mut raw = raw.clone();
raw["call_id"] = json!(call_id);
Message {
role: Role::User,
content: vec![ContentBlock::Unknown {
provider: WireFormat::OpenAiResponses.into(),
raw,
}],
}
})
.collect(),
);
let signal = ToolSignals::from_request(&request, Some(3));
assert_eq!(signal.severity, severity, "{raw}");
assert_eq!(signal.repeated_failure, is_error, "{raw}");
assert_eq!(signal.tool_result_count, 2);
assert_eq!(
matches!(
pick_tier(&signal, PickerMode::EfficientFirst, 0.5),
PickOutcome::Resolved {
tier: Tier::Capable,
..
}
),
is_error,
"{raw}"
);
let mut success = match raw["type"].as_str() {
Some("apply_patch_call_output") => json!({
"type": "apply_patch_call_output", "status": "completed", "output": ""
}),
Some("shell_call_output") => json!({
"type": "shell_call_output",
"output": [{"stdout": "", "stderr": "", "outcome": {"type": "exit", "exit_code": 0}}]
}),
_ => unreachable!(),
};
for index in 0..3 {
success["call_id"] = json!(format!("success_{index}"));
request.llm_request.messages.push(Message {
role: Role::User,
content: vec![ContentBlock::Unknown {
provider: WireFormat::OpenAiResponses.into(),
raw: success.clone(),
}],
});
}
let recovered = ToolSignals::from_request(&request, Some(3));
assert_eq!(recovered.severity, 0.0, "{raw}");
assert!(!recovered.repeated_failure, "{raw}");
assert!(recovered.no_error_streak >= 3, "{raw}");
assert_eq!(recovered.tool_result_count, 5);
}
}
#[test]
fn conversation_counts_are_per_block_and_role_aware() {
let result = |content: Vec<ContentBlock>| {
ContentBlock::ToolResult(ToolResult {
tool_call_id: String::new(),
content,
is_error: None,
})
};
let request = with_messages(vec![
Message::text(Role::User, "do something"),
Message::text(Role::Assistant, "working"),
Message {
role: Role::User,
content: vec![
result(vec![ContentBlock::Text {
text: "ok".to_string(),
}]),
result(Vec::new()),
],
},
tc("Bash"),
]);
let sig = ToolSignals::from_request(&request, None);
assert_eq!(sig.tool_result_count, 2);
assert_eq!(sig.assistant_turn_count, 2);
assert_eq!(sig.turn_depth, 4);
}
#[test]
fn recent_window_counts_only_last_default_window_tool_calls() {
let request = with_messages(vec![
tc("Write"),
tr("ok"),
tc("Write"),
tr("ok"),
tc("Write"),
tr("ok"),
tc("Write"),
tr("ok"),
tc("Write"),
tr("ok"),
tc("Edit"),
tr("ok"),
]);
let sig = ToolSignals::from_request(&request, None);
assert_eq!(sig.write_count, 5);
assert_eq!(sig.edit_count, 1);
assert_eq!(sig.recent_write_count, 2);
assert_eq!(sig.recent_edit_count, 1);
}
#[test]
fn codex_apply_patch_counts_as_an_edit() {
let request = with_messages(vec![tc("apply_patch"), tr("Success. Updated the file")]);
let sig = ToolSignals::from_request(&request, None);
assert_eq!(sig.edit_count, 1);
assert_eq!(sig.recent_edit_count, 1);
}
fn exec_command(cmd: Value) -> Message {
Message {
role: Role::Assistant,
content: vec![ContentBlock::ToolCall(ToolCall {
id: String::new(),
name: "exec_command".to_string(),
arguments: cmd,
})],
}
}
#[test]
fn codex_exec_command_is_classified() {
let args = json!(r#"{"cmd":"sed -i s/a/b/ src/lib.rs","workdir":"/x"}"#);
let request = with_messages(vec![exec_command(args), tr("ok")]);
assert_eq!(
ToolSignals::from_request(&request, None).recent_edit_count,
1
);
}
#[test]
fn python_write_expressions_need_a_python_command() {
let write = with_messages(vec![
exec_command(json!({"cmd": "python3 - <<'PY'\np.write_text(s)\nPY"})),
tr("ok"),
]);
assert_eq!(
ToolSignals::from_request(&write, None).recent_write_count,
1
);
let search = with_messages(vec![
exec_command(json!({"cmd": "grep -R '.write(' src"})),
tr("ok"),
]);
assert_eq!(
ToolSignals::from_request(&search, None).recent_write_count,
0
);
}
#[test]
fn recent_window_size_is_caller_overridable() {
let request = with_messages(vec![
tc("Write"),
tr("ok"),
tc("Write"),
tr("ok"),
tc("Write"),
tr("ok"),
tc("Write"),
tr("ok"),
tc("Write"),
tr("ok"),
tc("Edit"),
tr("ok"),
]);
let narrow = extract_tool_signals_with_window(&request, 3);
assert_eq!(narrow.recent_write_count, 2);
assert_eq!(narrow.recent_edit_count, 1);
let wide = extract_tool_signals_with_window(&request, 6);
assert_eq!(wide.recent_write_count, 5);
assert_eq!(wide.recent_edit_count, 1);
}
#[test]
fn compaction_marker_sets_compacted() {
let request = with_messages(vec![
Message::text(
Role::User,
"This session is being continued from a previous conversation that ran out of context.",
),
bash("ls"),
]);
assert!(ToolSignals::from_request(&request, None).compacted);
}
#[test]
fn codex_compaction_metadata_stays_on_parent_route() {
let mut request = with_messages(vec![bash("ls")]);
request.metadata = Some(Metadata {
is_subagent: true,
agent_kind: Some("compact".to_string()),
..Default::default()
});
assert!(!ToolSignals::from_request(&request, None).compacted);
}
#[test]
fn no_compaction_marker_stays_uncompacted() {
let request = with_messages(vec![
Message::text(Role::User, "Write a script that parses the log file."),
bash("ls"),
]);
assert!(!ToolSignals::from_request(&request, None).compacted);
}
#[test]
fn bash_heredoc_counts_as_write() {
let request = with_messages(vec![bash("cat > /tmp/test.py <<'EOF'\nprint(1)\nEOF")]);
let sig = ToolSignals::from_request(&request, None);
assert_eq!(
sig.write_count, 1,
"Bash heredoc should bucket into write_count"
);
assert_eq!(sig.edit_count, 0);
}
#[test]
fn bash_sed_inplace_counts_as_edit() {
let request = with_messages(vec![bash("sed -i 's/foo/bar/g' /app/file.py")]);
let sig = ToolSignals::from_request(&request, None);
assert_eq!(
sig.edit_count, 1,
"Bash sed -i should bucket into edit_count"
);
assert_eq!(sig.write_count, 0);
}
#[test]
fn bash_non_mutating_does_not_count() {
let request = with_messages(vec![bash("ls -la /app"), bash("cat /app/main.py")]);
let sig = ToolSignals::from_request(&request, None);
assert_eq!(sig.write_count, 0);
assert_eq!(sig.edit_count, 0);
}
#[test]
fn tests_passed_detects_pytest_with_failure_block() {
assert!(!detect_tests_passed(
&[("2 failed, 5 passed in 0.56s".to_string(), false)],
DEFAULT_RECENT_WINDOW
));
}
#[test]
fn tests_passed_accepts_cargo_clean_summary() {
assert!(detect_tests_passed(
&[(
"running 3 tests\ntest result: ok. 3 passed; 0 failed; 0 ignored".to_string(),
false
)],
DEFAULT_RECENT_WINDOW
));
}
#[test]
fn tests_passed_rejects_cargo_real_failure() {
assert!(!detect_tests_passed(
&[(
"running 3 tests\ntest result: FAILED. 2 passed; 1 failed; 0 ignored".to_string(),
false
)],
DEFAULT_RECENT_WINDOW
));
}
#[test]
fn tests_passed_accepts_go_clean_summary() {
assert!(detect_tests_passed(
&[(
"ok github.com/foo/bar\t0.012s (5 passed, 0 errors)".to_string(),
false
)],
DEFAULT_RECENT_WINDOW
));
}
#[test]
fn tests_passed_accepts_pytest_zero_errors() {
assert!(detect_tests_passed(
&[("5 passed, 0 errors in 0.30s".to_string(), false)],
DEFAULT_RECENT_WINDOW
));
}
#[test]
fn tests_passed_detects_diy_checkmark() {
assert!(detect_tests_passed(
&[("✓ all checks passed".to_string(), false)],
DEFAULT_RECENT_WINDOW
));
}
#[test]
fn anthropic_bash_heredoc_extracts_command() {
let request = with_messages(vec![bash("cat > /tmp/foo.txt << 'EOF'\nhi\nEOF")]);
let sig = ToolSignals::from_request(&request, None);
assert_eq!(
sig.write_count, 1,
"Anthropic Bash heredoc must also be detected"
);
}
#[test]
fn recent_window_falls_back_to_full_history_when_short() {
let request = with_messages(vec![tc("Write")]);
let sig = ToolSignals::from_request(&request, None);
assert_eq!(sig.recent_write_count, 1);
assert_eq!(sig.recent_edit_count, 0);
}
#[test]
fn clean_tool_result_has_zero_severity_and_non_empty_streak() {
let request = with_messages(vec![tr("output ok"), tr("another ok")]);
let sig = ToolSignals::from_request(&request, None);
assert_eq!(sig.severity, 0.0);
assert_eq!(sig.no_error_streak, 2);
}
#[test]
fn todowrite_classifies_as_plan() {
assert_eq!(classify_tool_call("TodoWrite", None), ToolSemantic::Plan);
assert_eq!(classify_tool_call("todo_write", None), ToolSemantic::Plan);
}
#[test]
fn codex_update_plan_classifies_as_plan() {
assert_eq!(classify_tool_call("update_plan", None), ToolSemantic::Plan);
}
#[test]
fn codex_shell_command_runs_bash_pattern_match() {
assert_eq!(
classify_tool_call("shell_command", Some("cat > /app/foo.py <<'eof'\nx=1\neof")),
ToolSemantic::Mutate(MutationKind::Write),
);
assert_eq!(
classify_tool_call("shell_command", Some("ls /app")),
ToolSemantic::Observe,
);
assert_eq!(
classify_tool_call("shell_command", Some("./run_tests.sh")),
ToolSemantic::Unknown,
);
}
#[test]
fn read_tool_classifies_as_read() {
assert_eq!(classify_tool_call("Read", None), ToolSemantic::Observe);
assert_eq!(classify_tool_call("View", None), ToolSemantic::Observe);
}
#[test]
fn hermes_tool_names_classify() {
assert_eq!(
classify_tool_call("write_file", None),
ToolSemantic::Mutate(MutationKind::Write)
);
assert_eq!(
classify_tool_call("patch", None),
ToolSemantic::Mutate(MutationKind::Edit)
);
assert_eq!(classify_tool_call("read_file", None), ToolSemantic::Observe);
assert_eq!(
classify_tool_call("search_files", None),
ToolSemantic::Observe
);
assert_eq!(
classify_tool_call("terminal", Some("sed -i 's/a/b/' /app/x.py")),
ToolSemantic::Mutate(MutationKind::Edit),
);
assert_eq!(
classify_tool_call("terminal", Some("grep foo /app")),
ToolSemantic::Observe,
);
assert_eq!(
classify_tool_call("terminal", Some("./run_tests.sh")),
ToolSemantic::Unknown,
);
}
#[test]
fn bash_read_patterns_classify_as_read() {
let cases = [
"cat /etc/passwd",
"grep foo bar.txt",
"ls /app",
"find . -name '*.py'",
];
for cmd in cases {
assert_eq!(
classify_tool_call("Bash", Some(cmd)),
ToolSemantic::Observe,
"expected Read for {cmd}"
);
}
}
#[test]
fn codex_inspection_commands_classify_as_read() {
let cases = [
"sed -n '1,80p' src/lib.rs",
"rg -n 'needle' src",
"nl -ba src/lib.rs",
"cat package.json",
"jq '.scripts' package.json",
"git status --short",
"git log --oneline -5",
"git show HEAD:src/lib.rs",
"git branch --show-current",
"git remote -v",
"git config --get remote.origin.url",
];
for command in cases {
assert_eq!(
classify_tool_call("exec_command", Some(command)),
ToolSemantic::Observe,
"expected Read for {command}"
);
}
}
#[test]
fn quoted_shell_separators_do_not_create_commands() {
for command in ["rg 'foo|rm obsolete.rs'", "rg \"foo; rm obsolete.rs\""] {
assert_eq!(
classify_tool_call("exec_command", Some(command)),
ToolSemantic::Observe,
"quoted text must not be parsed as a command: {command}"
);
}
}
#[test]
fn codex_shell_mutations_classify_as_production() {
let writes = [
"cp source.rs destination.rs",
"mkdir -p src/generated",
"touch src/generated/mod.rs",
"git show HEAD:file.rs > file.rs",
"node <<'node'\nfs.writefilesync('file.js', text)\nnode",
];
for command in writes {
assert_eq!(
classify_tool_call("exec_command", Some(command)),
ToolSemantic::Mutate(MutationKind::Write),
"expected Write for {command}"
);
}
let edits = [
"mv old.rs new.rs",
"rm obsolete.rs",
"gofmt -w main.go",
"cargo fmt",
"ruff check --fix src",
"perl -0pi -e 's/old/new/' src/lib.rs",
"npx prettier --write src/lib.ts",
"uv run ruff format src",
"git apply fix.patch",
];
for command in edits {
assert_eq!(
classify_tool_call("exec_command", Some(command)),
ToolSemantic::Mutate(MutationKind::Edit),
"expected Edit for {command}"
);
}
}
#[test]
fn formatter_checks_are_not_edits() {
for command in [
"cargo fmt --check",
"ruff format --check src",
"black --check src",
] {
assert_ne!(
classify_tool_call("exec_command", Some(command)),
ToolSemantic::Mutate(MutationKind::Edit),
"read-only formatter check must not be Edit: {command}"
);
}
}
#[test]
fn embedded_comparison_is_not_a_shell_write() {
let command = "node <<'node'\nif (index > 0) console.log(index)\nnode";
assert_eq!(
classify_tool_call("exec_command", Some(command)),
ToolSemantic::Unknown
);
}
#[test]
fn bash_write_precedence_over_read() {
assert_eq!(
classify_tool_call("Bash", Some("cat /etc/hosts > /tmp/out")),
ToolSemantic::Mutate(MutationKind::Write),
);
}
#[test]
fn pure_bash_streak_counts_trailing_other() {
let request = with_messages(vec![
bash("make"),
tr("ok"),
bash("./configure"),
tr("ok"),
bash("make install"),
tr("ok"),
bash("./run.sh"),
tr("ok"),
bash("./test"),
tr("ok"),
]);
let sig = ToolSignals::from_request(&request, None);
assert_eq!(sig.pure_bash_streak, 5);
assert_eq!(sig.write_count, 0);
assert_eq!(sig.read_count, 0);
}
#[test]
fn pure_bash_streak_resets_on_write() {
let request = with_messages(vec![bash("make"), tr("ok"), tc("Write"), tr("ok")]);
let sig = ToolSignals::from_request(&request, None);
assert_eq!(sig.pure_bash_streak, 0);
assert_eq!(sig.write_count, 1);
}
#[test]
fn recent_window_tracks_todowrite_and_read() {
let request = with_messages(vec![
bash("make"),
tr("ok"),
tc("TodoWrite"),
tr("ok"),
tc("Read"),
tr("ok"),
tc("TodoWrite"),
tr("ok"),
]);
let sig = ToolSignals::from_request(&request, None);
assert_eq!(sig.todowrite_count, 2);
assert_eq!(sig.recent_todowrite_count, 2);
assert_eq!(sig.read_count, 1);
assert_eq!(sig.recent_read_count, 1);
}
#[test]
fn configured_tool_semantics_extend_the_builtin_vocabulary() {
let semantics = ToolSemantics {
observe: vec!["KB_search".to_string()],
mutate: vec!["send_payment_request".to_string()],
plan: vec!["create_research_plan".to_string()],
new: vec!["send_message_to_user".to_string()],
};
semantics.validate().expect("valid additive semantics");
let request = with_messages(vec![
tc("Read"),
tc("Write"),
tc("TodoWrite"),
tc("kb_SEARCH"),
tc("send_payment_request"),
tc("create_research_plan"),
tc("send_message_to_user"),
tc("unlisted_tool"),
]);
let signal = ToolSignals::from_request_with_semantics(&request, None, &semantics);
assert_eq!(signal.read_count, 2);
assert_eq!(signal.write_count, 2);
assert_eq!(signal.todowrite_count, 2);
assert_eq!(signal.new_count, 1);
assert_eq!(signal.recent_new_count, 1);
assert_eq!(signal.pure_bash_streak, 1);
}
#[test]
fn configured_tool_semantics_only_fold_ascii_case() {
let semantics = ToolSemantics {
observe: vec!["kb_search".to_string()],
..Default::default()
};
assert_eq!(
classify_tool_call_with_semantics("KB_SEARCH", None, &semantics),
ToolSemantic::Observe
);
assert_eq!(
classify_tool_call_with_semantics("KB_SEARCH", None, &semantics),
ToolSemantic::Unknown
);
}
#[test]
fn custom_semantics_preserve_builtin_unicode_lowercasing() {
let semantics = ToolSemantics {
observe: vec!["lookup_customer".to_string()],
..Default::default()
};
assert_eq!(
classify_tool_call_with_semantics("notebooKedit", None, &semantics),
ToolSemantic::Mutate(MutationKind::Edit)
);
}
#[test]
fn configured_semantics_never_replace_builtin_classifications() {
let semantics = ToolSemantics {
observe: vec!["lookup_customer".to_string()],
mutate: vec!["send_payment".to_string()],
plan: vec!["create_workflow".to_string()],
new: vec!["send_message".to_string()],
};
for name in WRITE_TOOL_NAMES {
assert_eq!(
classify_tool_call_with_semantics(name, None, &semantics),
ToolSemantic::Mutate(MutationKind::Write),
"write tool {name:?} changed classification"
);
}
for name in EDIT_TOOL_NAMES {
assert_eq!(
classify_tool_call_with_semantics(name, None, &semantics),
ToolSemantic::Mutate(MutationKind::Edit),
"edit tool {name:?} changed classification"
);
}
for name in READ_TOOL_NAMES {
assert_eq!(
classify_tool_call_with_semantics(name, None, &semantics),
ToolSemantic::Observe,
"read tool {name:?} changed classification"
);
}
for name in PLAN_TOOL_NAMES {
assert_eq!(
classify_tool_call_with_semantics(name, None, &semantics),
ToolSemantic::Plan,
"plan tool {name:?} changed classification"
);
}
for (command, expected) in [
("cat /tmp/input", ToolSemantic::Observe),
(
"cat /tmp/input > /tmp/output",
ToolSemantic::Mutate(MutationKind::Write),
),
(
"sed -i 's/a/b/' /tmp/file",
ToolSemantic::Mutate(MutationKind::Edit),
),
("./run_tests.sh", ToolSemantic::Unknown),
] {
assert_eq!(
classify_tool_call_with_semantics("BASH", Some(command), &semantics),
expected,
"bash command {command:?} changed classification"
);
}
}
#[test]
fn configured_semantics_score_like_their_builtin_equivalents() {
let semantics = ToolSemantics {
observe: vec!["lookup_customer".to_string()],
mutate: vec!["send_payment".to_string()],
plan: vec!["create_workflow".to_string()],
..Default::default()
};
for (builtin, configured) in [
("Read", "lookup_customer"),
("Write", "send_payment"),
("TodoWrite", "create_workflow"),
] {
let messages_before_tool = || {
vec![
Message::text(Role::User, "start"),
Message::text(Role::Assistant, "working"),
Message::text(Role::User, "continue"),
Message::text(Role::Assistant, "working"),
Message::text(Role::User, "continue"),
Message::text(Role::Assistant, "working"),
Message::text(Role::User, "continue"),
]
};
let mut builtin_messages = messages_before_tool();
builtin_messages.push(tc(builtin));
let mut configured_messages = messages_before_tool();
configured_messages.push(tc(configured));
let builtin_score = score_signal(&ToolSignals::from_request(
&with_messages(builtin_messages),
None,
));
let configured_score = score_signal(&ToolSignals::from_request_with_semantics(
&with_messages(configured_messages),
None,
&semantics,
));
assert_ne!(
builtin_score.score, 0.0,
"the {builtin:?} control must exercise a scoring dimension"
);
assert_eq!(
configured_score, builtin_score,
"configured tool {configured:?} must score exactly like {builtin:?}"
);
}
}
#[test]
fn tool_semantics_reject_duplicates_and_builtin_reclassification() {
let duplicate = ToolSemantics {
observe: vec!["lookup".to_string()],
mutate: vec!["LOOKUP".to_string()],
..Default::default()
};
assert!(
duplicate
.validate()
.expect_err("duplicate should fail")
.to_string()
.contains("appears in both")
);
let builtin = ToolSemantics {
new: vec!["write_file".to_string()],
..Default::default()
};
assert!(
builtin
.validate()
.expect_err("built-in should fail")
.to_string()
.contains("built-in semantics")
);
let empty = ToolSemantics {
observe: vec![" \t".to_string()],
..Default::default()
};
assert!(
empty
.validate()
.expect_err("empty name should fail")
.to_string()
.contains("empty tool name")
);
}
}