pub(crate) const DANGLING_TOOL_INTENT_REASON: &str = "dangling_tool_intent";
pub(crate) const DANGLING_TOOL_INTENT_PROMPT: &str = "You said you would invoke tools but emitted no tool call. Invoke the required tools now, or provide a final answer explaining why you cannot.";
use std::collections::HashSet;
const MAX_POLICY_TEXT_CHARS: usize = 4_096;
const MAX_POLICY_TOKENS: usize = 96;
const MAX_PROCEDURAL_SENTENCES: usize = 3;
const EXECUTION_VERBS: &str = "run use invoke call execute exercise test start";
const GENERIC_TARGET_WORDS: &str =
"a an the all any available each every have i my of one tool tools you your";
const GENERIC_QUANTIFIERS: &str = "a an all any available each every my one the your";
const BATCH_LABEL_DISALLOWED_WORDS: &str = "answer delete destroy erase execute explain if invoke later overwrite publish purge remove report result results run send use write";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ExpectationStatus {
NotRequested,
Requested,
Invalidated,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct ToolExecutionExpectation {
tools_available: bool,
status: ExpectationStatus,
}
impl ToolExecutionExpectation {
pub(crate) fn from_request(
request: &str,
tools_configured: bool,
advertised_tool_names: &HashSet<String>,
) -> Self {
let tools_available = tools_configured && !advertised_tool_names.is_empty();
let status = if tools_available
&& request_requests_immediate_tools(request, advertised_tool_names)
{
ExpectationStatus::Requested
} else {
ExpectationStatus::NotRequested
};
Self {
tools_available,
status,
}
}
pub(crate) fn invalidate(&mut self) {
self.status = ExpectationStatus::Invalidated;
}
fn allows_recovery(self) -> bool {
self.tools_available && self.status != ExpectationStatus::Invalidated
}
fn expects_tools(self) -> bool {
self.status == ExpectationStatus::Requested
}
}
pub(crate) fn completion_needs_tool_recovery_for_expectation(
expectation: &ToolExecutionExpectation,
assistant_response: &str,
advertised_tool_names: &HashSet<String>,
) -> bool {
if !expectation.allows_recovery() || advertised_tool_names.is_empty() {
return false;
}
is_dangling_tool_intent(assistant_response, advertised_tool_names)
|| (expectation.expects_tools()
&& is_procedural_tool_announcement(assistant_response, advertised_tool_names))
}
fn request_requests_immediate_tools(
request: &str,
advertised_tool_names: &HashSet<String>,
) -> bool {
let Some(request) = normalized_policy_text(request, false) else {
return false;
};
let tokens = policy_tokens(&request);
if tokens.is_empty() || tokens.len() > MAX_POLICY_TOKENS {
return false;
}
if matches_direct_smoke_request(&tokens, advertised_tool_names) {
return true;
}
let Some(action_index) = request_action_index(&tokens) else {
return false;
};
matches_execution_target(&tokens[action_index + 1..], advertised_tool_names)
}
fn is_procedural_tool_announcement(text: &str, advertised_tool_names: &HashSet<String>) -> bool {
let Some(normalized) = normalized_policy_text(text, true) else {
return false;
};
let sentences = normalized
.split(['.', '!', '?'])
.map(str::trim)
.filter(|sentence| !sentence.is_empty())
.collect::<Vec<_>>();
if sentences.is_empty() || sentences.len() > MAX_PROCEDURAL_SENTENCES {
return false;
}
let mut token_count = 0;
for (index, sentence) in sentences.iter().enumerate() {
let tokens = policy_tokens(sentence);
token_count += tokens.len();
let matches = if index == 0 {
matches_immediate_response(&tokens, advertised_tool_names)
} else {
matches_procedural_followup(&tokens, advertised_tool_names)
};
if tokens.is_empty() || !matches {
return false;
}
}
token_count <= MAX_POLICY_TOKENS
}
fn normalized_policy_text(text: &str, reject_answer_punctuation: bool) -> Option<String> {
let text = text.trim();
if text.is_empty()
|| text.chars().count() > MAX_POLICY_TEXT_CHARS
|| text
.chars()
.any(|character| matches!(character, '\n' | '\r' | '\t'))
{
return None;
}
let text = text.replace(['’', '‘'], "'").replace(['“', '”'], "\"");
if text.contains('"')
|| text.contains('`')
|| unsupported_apostrophe(&text)
|| (reject_answer_punctuation && text.chars().any(|c| matches!(c, ':' | ';')))
{
return None;
}
Some(text.to_ascii_lowercase())
}
fn unsupported_apostrophe(text: &str) -> bool {
let mut without_contractions = text.to_ascii_lowercase();
for contraction in ["i'll", "i'm", "i'd", "i've", "can't", "don't", "won't"] {
without_contractions = without_contractions.replace(contraction, "");
}
without_contractions.contains('\'')
}
fn policy_tokens(text: &str) -> Vec<String> {
text.split(|character: char| {
!(character.is_ascii_alphanumeric() || matches!(character, '_' | '-' | '\''))
})
.filter(|token| !token.is_empty())
.map(str::to_string)
.collect()
}
fn request_action_index(tokens: &[String]) -> Option<usize> {
let mut index = 0;
if tokens.first().map(String::as_str) == Some("please") {
index = 1;
} else if starts_with(tokens, &["can", "you"]) || starts_with(tokens, &["could", "you"]) {
index = 2;
} else if starts_with(tokens, &["go", "ahead", "and"]) {
index = 3;
} else if starts_with(tokens, &["i", "want", "you", "to"])
|| starts_with(tokens, &["i", "need", "you", "to"])
|| starts_with(tokens, &["i'd", "like", "you", "to"])
{
index = 4;
} else if tokens.is_empty() {
return None;
}
if tokens.get(index).map(String::as_str) == Some("please") {
index += 1;
}
(tokens
.get(index)
.is_some_and(|token| is_execution_verb(token)))
.then_some(index)
}
fn matches_direct_smoke_request(
tokens: &[String],
advertised_tool_names: &HashSet<String>,
) -> bool {
let start = usize::from(tokens.first().map(String::as_str) == Some("so"));
start < tokens.len() && matches_smoke_target(&tokens[start..], advertised_tool_names)
}
fn matches_immediate_response(tokens: &[String], advertised_tool_names: &HashSet<String>) -> bool {
let Some(action_index) = response_action_index(tokens) else {
return false;
};
matches_execution_target(&tokens[action_index + 1..], advertised_tool_names)
}
fn matches_procedural_followup(tokens: &[String], advertised_tool_names: &HashSet<String>) -> bool {
matches_immediate_response(tokens, advertised_tool_names) || matches_batch_continuation(tokens)
}
fn matches_batch_continuation(tokens: &[String]) -> bool {
let Some(action_index) = response_action_index(tokens) else {
return false;
};
let mut index = action_index + 1;
if tokens.get(index).map(String::as_str) != Some("them") {
return false;
}
index += 1;
if starts_with(&tokens[index..], &["in", "batches"]) {
index += 2;
}
if !starts_with(&tokens[index..], &["starting", "with"]) {
return false;
}
index += 2;
if tokens
.get(index)
.is_some_and(|token| is_leading_article(token))
{
index += 1;
}
let Some(last) = tokens.last().map(String::as_str) else {
return false;
};
if !matches!(last, "tool" | "tools") || index >= tokens.len().saturating_sub(1) {
return false;
}
let labels = &tokens[index..tokens.len() - 1];
(1..=6).contains(&labels.len()) && labels.iter().all(|label| is_batch_label(label))
}
fn is_batch_label(token: &str) -> bool {
let mut characters = token.chars();
characters
.next()
.is_some_and(|character| character.is_ascii_alphabetic())
&& characters
.all(|character| character.is_ascii_alphanumeric() || matches!(character, '_' | '-'))
&& !has_word(BATCH_LABEL_DISALLOWED_WORDS, token)
}
fn response_action_index(tokens: &[String]) -> Option<usize> {
let mut index = 0;
if tokens.get(index).map(String::as_str) == Some("now") {
index += 1;
}
if tokens.get(index).map(String::as_str) == Some("okay") {
index += 1;
}
let prefix_len = if tokens.get(index).map(String::as_str) == Some("i'll") {
1
} else if starts_with(&tokens[index..], &["i", "will"])
|| starts_with(&tokens[index..], &["let", "me"])
{
2
} else if starts_with(&tokens[index..], &["i'm", "going", "to"])
|| starts_with(&tokens[index..], &["i", "am", "going", "to"])
{
if tokens.get(index).map(String::as_str) == Some("i'm") {
3
} else {
4
}
} else {
return None;
};
index += prefix_len;
while tokens
.get(index)
.is_some_and(|token| matches!(token.as_str(), "now" | "actually" | "first"))
{
index += 1;
}
tokens
.get(index)
.is_some_and(|token| is_execution_verb(token))
.then_some(index)
}
fn matches_execution_target(target: &[String], advertised_tool_names: &HashSet<String>) -> bool {
let target = strip_terminal_words(target);
matches_tool_target_request(target, advertised_tool_names)
|| matches_smoke_target(target, advertised_tool_names)
}
fn matches_smoke_target(target: &[String], advertised_tool_names: &HashSet<String>) -> bool {
let target = strip_terminal_words(target);
let mut index = 0;
if target
.get(index)
.is_some_and(|token| matches!(token.as_str(), "a" | "an"))
{
index += 1;
} else {
return false;
}
if target.get(index).map(String::as_str) == Some("live") {
index += 1;
}
if target.get(index).map(String::as_str) != Some("smoke")
|| target.get(index + 1).map(String::as_str) != Some("test")
{
return false;
}
index += 2;
if target.get(index).map(String::as_str) == Some("of") {
index += 1;
}
index < target.len() && matches_tool_target_request(&target[index..], advertised_tool_names)
}
fn matches_tool_target_request(target: &[String], advertised_tool_names: &HashSet<String>) -> bool {
let target = strip_terminal_words(target);
let target = if target.first().map(String::as_str) == Some("of") {
&target[1..]
} else {
target
};
let mut index = 0;
while target
.get(index)
.is_some_and(|token| matches!(token.as_str(), "a" | "an" | "the" | "your" | "my"))
{
index += 1;
}
let target = &target[index..];
if target.is_empty() {
return false;
}
if target.len() == 1 && advertised_name(&target[0], advertised_tool_names) {
return true;
}
if target.len() == 2
&& advertised_name(&target[0], advertised_tool_names)
&& matches!(target[1].as_str(), "tool" | "tools")
{
return true;
}
matches_generic_target(target)
}
fn matches_generic_target(target: &[String]) -> bool {
target
.iter()
.any(|token| matches!(token.as_str(), "tool" | "tools"))
&& target
.iter()
.all(|token| has_word(GENERIC_TARGET_WORDS, token))
&& (target.len() == 1
|| target
.iter()
.any(|token| has_word(GENERIC_QUANTIFIERS, token)))
}
fn strip_terminal_words(target: &[String]) -> &[String] {
let mut end = target.len();
while end > 0 && matches!(target[end - 1].as_str(), "now" | "please") {
end -= 1;
}
&target[..end]
}
pub(crate) fn is_dangling_tool_intent(text: &str, advertised_tool_names: &HashSet<String>) -> bool {
let text = text.trim();
if text.is_empty()
|| text
.chars()
.any(|character| matches!(character, '"' | '`' | '‘' | '’' | '“' | '”'))
|| text
.chars()
.any(|character| character.is_whitespace() && character != ' ')
{
return false;
}
let Some(text) = text.strip_suffix('.') else {
return false;
};
let tokens = text.split_whitespace().collect::<Vec<_>>();
if tokens.is_empty() {
return false;
}
let lowered_tokens = tokens
.iter()
.map(|token| token.to_ascii_lowercase())
.collect::<Vec<_>>();
if lowered_tokens.first().map(String::as_str) == Some("now") {
let action_index =
1 + usize::from(lowered_tokens.get(1).map(String::as_str) == Some("actually"));
if lowered_tokens.get(action_index).map(String::as_str) == Some("running") {
let target_start = action_index + 1;
return target_start < tokens.len()
&& matches_tool_target(&tokens[target_start..], advertised_tool_names);
}
}
let Some(prefix_len) = promise_prefix_len(&lowered_tokens) else {
return false;
};
let action_index = prefix_len
+ usize::from(lowered_tokens.get(prefix_len).map(String::as_str) == Some("actually"));
if !matches!(
lowered_tokens.get(action_index).map(String::as_str),
Some("invoke" | "use" | "call")
) {
return false;
}
let target_start = action_index + 1;
let target_end = lowered_tokens.len().saturating_sub(1);
if target_start >= target_end || lowered_tokens.last().map(String::as_str) != Some("now") {
return false;
}
matches_tool_target(&tokens[target_start..target_end], advertised_tool_names)
}
fn has_word(words: &str, token: &str) -> bool {
words.split_whitespace().any(|word| word == token)
}
fn is_execution_verb(token: &str) -> bool {
has_word(EXECUTION_VERBS, token)
}
fn advertised_name(token: &str, advertised_tool_names: &HashSet<String>) -> bool {
advertised_tool_names
.iter()
.any(|name| name.eq_ignore_ascii_case(token))
}
fn matches_tool_target(target: &[&str], advertised_tool_names: &HashSet<String>) -> bool {
let mut index = 0;
if target
.get(index)
.is_some_and(|token| is_leading_article(token))
{
index += 1;
}
if index >= target.len() {
return false;
}
if target[index] == "tool" {
if index + 1 == target.len() {
return true;
}
return index + 2 == target.len() && target[index + 1].eq_ignore_ascii_case("x");
}
if target[index] == "tools" {
return index + 1 == target.len();
}
let Some(first_name) = target.get(index) else {
return false;
};
if !is_advertised_tool(first_name, advertised_tool_names) {
return false;
}
index += 1;
if index == target.len() {
return true;
}
if index + 1 == target.len() && is_trailing_tool_suffix(target[index]) {
return true;
}
if target.get(index) != Some(&"and") {
return false;
}
index += 1;
if target
.get(index)
.is_some_and(|token| is_leading_article(token))
{
index += 1;
}
let Some(second_name) = target.get(index) else {
return false;
};
if !is_advertised_tool(second_name, advertised_tool_names) {
return false;
}
index += 1;
if index == target.len() {
return first_name != second_name;
}
if index + 1 == target.len() && is_trailing_tool_suffix(target[index]) {
return first_name != second_name;
}
false
}
fn is_advertised_tool(token: &str, advertised_tool_names: &HashSet<String>) -> bool {
advertised_tool_names.contains(token)
}
fn is_trailing_tool_suffix(token: &str) -> bool {
matches!(token, "tool" | "tools")
}
fn is_leading_article(token: &str) -> bool {
matches!(token, "a" | "an" | "the")
}
fn starts_with(tokens: &[String], words: &[&str]) -> bool {
tokens.len() >= words.len() && tokens.iter().zip(words).all(|(token, word)| token == word)
}
fn promise_prefix_len(tokens: &[String]) -> Option<usize> {
if starts_with(tokens, &["okay,", "i'll"]) {
Some(2)
} else if tokens.first().map(String::as_str) == Some("i'll") {
Some(1)
} else if starts_with(tokens, &["let", "me"]) || starts_with(tokens, &["i", "will"]) {
Some(2)
} else if starts_with(tokens, &["i'm", "going", "to"]) {
Some(3)
} else if starts_with(tokens, &["i", "am", "going", "to"]) {
Some(4)
} else {
None
}
}
#[cfg(test)]
mod tests {
use super::is_dangling_tool_intent;
use std::collections::HashSet;
fn advertised(names: &[&str]) -> HashSet<String> {
names.iter().map(|name| (*name).to_string()).collect()
}
fn standard_tools() -> HashSet<String> {
advertised(&["browser", "hash_edit", "read", "subagents"])
}
#[test]
fn accepts_exact_generic_and_advertised_targets() {
let advertised_tools = standard_tools();
for text in [
"Let me actually invoke the browser and subagents tools now.",
"I'll invoke tool X now.",
"I'll invoke tool now.",
"I'll invoke tools now.",
"I'll invoke browser now.",
"I'll invoke the read now.",
"I'll invoke the read tool now.",
"I'll invoke read tools now.",
"I'll invoke browser and subagents now.",
"I'll invoke the browser and the subagents tools now.",
"Okay, I'll use hash_edit now.",
] {
assert!(
is_dangling_tool_intent(text, &advertised_tools),
"expected match: {text}"
);
}
}
#[test]
fn accepts_fronted_running_tool_announcements() {
let advertised_tools = standard_tools();
for text in [
"Now running the browser tool.",
"Now actually running the browser tool.",
] {
assert!(
is_dangling_tool_intent(text, &advertised_tools),
"expected match: {text}"
);
}
}
#[test]
fn fronted_running_tool_announcements_are_strict() {
let advertised_tools = standard_tools();
for text in [
"Now actually running unknown tool.",
"Now actually running the browser tool later.",
"Now actually running the browser tool because it is needed.",
"The process is now running the browser tool.",
"Now explaining the browser tool.",
"Now run the browser tool.",
"Now actually running the browser tool",
"Now actually running the browser tool!",
"\"Now actually running the browser tool.\"",
"`Now actually running the browser tool.`",
"Now actually running the browser\n tool.",
"Now actually running the browser\ttool.",
] {
assert!(
!is_dangling_tool_intent(text, &advertised_tools),
"unexpected match: {text}"
);
}
let browser_disabled = advertised(&["hash_edit", "read", "subagents"]);
assert!(!is_dangling_tool_intent(
"Now actually running the browser tool.",
&browser_disabled
));
}
#[test]
fn dynamic_provider_tool_names_are_exact_matches() {
let advertised_tools = advertised(&["mcp__server__search"]);
assert!(is_dangling_tool_intent(
"I'll invoke mcp__server__search now.",
&advertised_tools
));
for text in [
"I'll invoke mcp__other__search now.",
"I'll invoke arbitrary_tool now.",
"I'll invoke shell now.",
] {
assert!(
!is_dangling_tool_intent(text, &advertised_tools),
"unexpected match: {text}"
);
}
}
#[test]
fn generic_targets_are_bounded_and_prose_is_rejected() {
let advertised_tools = standard_tools();
for text in [
"I'll invoke tool documentation now.",
"I'll invoke documentation about the browser tool now.",
"I'll invoke the documentation about the browser tool now.",
"I'll invoke tool browser because now.",
"I'll invoke tool functions now.",
"I'll invoke tool X and read now.",
"I'll invoke tools X now.",
"I'll invoke tool tool now.",
"I'll invoke the tool read now.",
"I'll invoke the browser because now.",
"I'll invoke the browser to explain now.",
"I'll invoke browser and subagents and read now.",
"I'll invoke browser and browser now.",
"I'll invoke the browser tool tool now.",
"I'll invoke browser later.",
"I'll invoke browser now, because the next step is important.",
"I'll invoke browser now. This explains the next step.",
"I need to inspect the file. I'll invoke browser now.",
"\"I'll invoke browser now.\"",
"I'll invoke `browser` now.",
"```\nI'll invoke browser now.\n```",
"I'll invoke browser now!",
"I'll invoke browser, now.",
"I'll invoke browser now..",
] {
assert!(
!is_dangling_tool_intent(text, &advertised_tools),
"unexpected match: {text}"
);
}
}
}
#[cfg(test)]
mod completion_policy_tests {
use super::{
DANGLING_TOOL_INTENT_PROMPT, ToolExecutionExpectation,
completion_needs_tool_recovery_for_expectation,
};
use std::collections::HashSet;
fn advertised(names: &[&str]) -> HashSet<String> {
names.iter().map(|name| (*name).to_string()).collect()
}
fn needs(request: &str, response: &str, tools: &HashSet<String>) -> bool {
let expectation = ToolExecutionExpectation::from_request(request, true, tools);
completion_needs_tool_recovery_for_expectation(&expectation, response, tools)
}
#[test]
fn direct_requests_accept_immediate_assistant_tool_procedures() {
let tools = advertised(&["browser", "hash_edit", "read", "subagents"]);
assert!(needs(
"So a live smoke test of all of your tools please",
"I'll run a live smoke test of every tool I have. Let me exercise them in batches, starting with the file/tree/search tools.",
&tools,
));
assert!(needs(
"please run all tools",
"I'll run all tools. I will use the read tool.",
&tools,
));
assert!(needs(
"please run all tools",
"I'll run all tools. I will use all tools.",
&tools,
));
for (request, response) in [
("please run all tools", "I'll run all tools now."),
("please run all tools", "I will run all tools."),
("can you run the read tool", "I'll run the read tool now."),
("I want you to use browser", "Let me use browser now."),
] {
assert!(
needs(request, response, &tools),
"expected match: {request} / {response}"
);
}
}
#[test]
fn batch_followups_are_structurally_bounded() {
let tools = advertised(&["browser", "read"]);
for response in [
"I'll run all tools. Let me use them in batches, starting with alpha/beta/gamma tools.",
"I'll run all tools. Let me use them starting with the alpha tool.",
"I'll run all tools. Let me use them in batches, starting with alpha-beta/tool_2 tools.",
] {
assert!(
needs("please run all tools", response, &tools),
"expected response match: {response}"
);
}
for response in [
"I'll run all tools. Let me use them in batches, starting with tools.",
"I'll run all tools. Let me use them in batches, starting with one/two/three/four/five/six/seven tools.",
"I'll run all tools. Let me use them in batches, starting with delete tools.",
"I'll run all tools. Let me use them in batches, starting with alpha and delete tools.",
"I'll run all tools. Let me use them in batches, starting with alpha result tools.",
"I'll run all tools. Let me use them in batches, starting with alpha tools now.",
"I'll run all tools. Let me use them to delete every file.",
"I'll run all tools. Let me use them in batches, starting with alpha tools because they are ready.",
] {
assert!(
!needs("please run all tools", response, &tools),
"unexpected response match: {response}"
);
}
}
#[test]
fn indirect_requests_and_non_immediate_responses_do_not_match() {
let tools = advertised(&["browser", "read"]);
for request in [
"Tell me to run all tools",
"How would you run all tools?",
"Why run all tools?",
"What if you ran all tools?",
"Plan a live smoke test of all tools",
"Give me an example of running the read tool",
"If you approve, run all tools",
"Please do not run all tools",
"Please run all tools later",
"\"Please run all tools\"",
] {
assert!(
!needs(request, "I'll run all tools now.", &tools),
"unexpected request match: {request}"
);
}
for response in [
"Now run all tools.",
"Sure, I can run all tools.",
"I will run all tools later.",
"I will run all tools if approved.",
"I will run all tools. Here are the results.",
"I will run all tools. I'll use them to explain the results.",
"I will run all tools. I'll use the tool documentation.",
"I'll run all tools. Let me use them to delete every file.",
"I'll run all tools. Let me use them to overwrite every file.",
"I'll run all tools. Let me use them to send every file.",
"I'll run all tools. Let me use them to publish every file.",
"I ran all tools; here are the results.",
"The tools completed successfully; here is the result.",
"I'll explain the read tool: it accepts a path and returns text.",
"\"I'll run all tools now.\"",
"```text\nI'll run all tools now.\n```",
] {
assert!(
!needs("please run all tools", response, &tools),
"unexpected response match: {response}"
);
}
}
#[test]
fn advertised_names_and_runtime_state_bound_the_match() {
let tools = advertised(&["mcp__server__search"]);
assert!(needs(
"Please call mcp__server__search now",
"I'll call mcp__server__search now.",
&tools,
));
assert!(!needs(
"Please call mcp__other__search now",
"I'll call mcp__other__search now.",
&tools,
));
let expectation = ToolExecutionExpectation::from_request(
"please run all tools",
false,
&advertised(&["read"]),
);
assert!(!completion_needs_tool_recovery_for_expectation(
&expectation,
"I'll run all tools now.",
&advertised(&["read"]),
));
}
#[test]
fn canonical_recovery_prompt_remains_stable() {
assert!(DANGLING_TOOL_INTENT_PROMPT.contains("Invoke the required tools now"));
}
}