use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use tokio::sync::mpsc;
use crate::agent::plan_artifact;
use crate::config::AgentConfig;
use crate::llm::{
ChatCompletionsBackend, CompleteChatRetryingParams, LlmRetryingTransportOpts,
complete_chat_retrying, no_tools_chat_request,
};
use crate::types::{LlmSeedOverride, Message};
use crate::cm_agent::PlanSemanticLlmOutcome;
fn truncate_unicode(s: &str, max_chars: usize) -> String {
let n = s.chars().count();
if n <= max_chars {
return s.to_string();
}
let mut out = s.chars().take(max_chars).collect::<String>();
out.push_str("…(truncated)");
out
}
const SIDE_SYSTEM_JSON_ONLY: &str = "你是 CrabMate 的**规划一致性审计员**。只根据给定的「最近工具结果摘要」与「模型输出的 agent_reply_plan JSON」判断是否明显矛盾（例如计划声称成功但工具报错、计划步骤与工具输出冲突）。\n\
你必须**只输出一个 JSON 对象**（不要 Markdown 代码围栏、不要前后缀说明）。字段要求：\n\
- 若**无法判断**或**无明显矛盾**：{\"consistent\":true}\n\
- 若**明确矛盾**：{\"consistent\":false,\"violation_codes\":[\"…\"],\"rationale\":\"不超过 80 字的中文理由\"}\n\
其中 **violation_codes** 为字符串数组：1–8 个元素，每项仅含小写字母、数字、下划线，长度 1–64。建议码：`tool_outcome_contradiction`（与工具成功/失败状态冲突）、`plan_step_tool_mismatch`（步骤与工具输出明显不符）、`claim_not_supported_by_tools`（断言缺乏工具证据）、`semantic_mismatch_other`（其它明确矛盾）。";
const SIDE_SYSTEM_LEGACY_APPENDIX: &str = "\n**兼容**：若你只能输出纯文本，可在一行内写 INCONSISTENT 或 CONSISTENT（将按旧规则解析，但优先使用 JSON）。";
fn side_system_prompt(accept_legacy_text: bool) -> String {
if accept_legacy_text {
format!("{SIDE_SYSTEM_JSON_ONLY}{SIDE_SYSTEM_LEGACY_APPENDIX}")
} else {
SIDE_SYSTEM_JSON_ONLY.to_string()
}
}
const MAX_VIOLATION_CODES: usize = 8;
const MAX_CODE_LEN: usize = 64;
const MAX_RATIONALE_CHARS: usize = 120;
fn is_valid_violation_code_token(s: &str) -> bool {
let mut chars = s.chars();
let Some(first) = chars.next() else {
return false;
};
if !first.is_ascii_lowercase() && !first.is_ascii_digit() {
return false;
}
s.len() <= MAX_CODE_LEN
&& s.chars()
.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '_')
}
fn normalize_violation_codes(raw: &[serde_json::Value]) -> Vec<String> {
let mut out = Vec::new();
for v in raw {
let Some(s) = v.as_str() else {
continue;
};
let t = s.trim();
if is_valid_violation_code_token(t) {
out.push(t.to_string());
}
if out.len() >= MAX_VIOLATION_CODES {
break;
}
}
out.sort();
out.dedup();
out
}
fn truncate_rationale(s: &str) -> String {
let mut t = s.trim().to_string();
let n = t.chars().count();
if n > MAX_RATIONALE_CHARS {
t = t.chars().take(MAX_RATIONALE_CHARS).collect::<String>();
t.push('…');
}
t
}
fn outcome_from_json_value(v: &serde_json::Value) -> Option<PlanSemanticLlmOutcome> {
let consistent = v.get("consistent")?.as_bool()?;
let mut codes = v
.get("violation_codes")
.and_then(|x| x.as_array())
.map(|a| normalize_violation_codes(a.as_slice()))
.unwrap_or_default();
let rationale = v
.get("rationale")
.and_then(|x| x.as_str())
.map(truncate_rationale)
.filter(|s| !s.is_empty());
if !consistent && codes.is_empty() {
codes.push("semantic_mismatch_unspecified".to_string());
}
Some(PlanSemanticLlmOutcome {
consistent,
violation_codes: codes,
rationale,
user_cancelled: false,
})
}
fn extract_json_object_slice(s: &str) -> Option<&str> {
let start = s.find('{')?;
let end = s.rfind('}')?;
(end > start).then_some(&s[start..=end])
}
pub(crate) fn parse_plan_semantic_side_reply(
text: &str,
accept_legacy_text: bool,
) -> PlanSemanticLlmOutcome {
let trimmed = text.trim();
if trimmed.is_empty() {
return PlanSemanticLlmOutcome::consistent_ok();
}
if let Some(slice) = extract_json_object_slice(trimmed)
&& let Ok(v) = serde_json::from_str::<serde_json::Value>(slice)
&& let Some(o) = outcome_from_json_value(&v)
{
return o;
}
if let Ok(v) = serde_json::from_str::<serde_json::Value>(trimmed)
&& let Some(o) = outcome_from_json_value(&v)
{
return o;
}
if accept_legacy_text {
let upper = trimmed.to_uppercase();
if upper.contains("INCONSISTENT") {
log::debug!(
target: "crabmate::per",
"final_plan_semantic_check parse_path=legacy_text outcome=inconsistent"
);
return PlanSemanticLlmOutcome {
consistent: false,
violation_codes: vec!["semantic_mismatch_legacy".to_string()],
rationale: None,
user_cancelled: false,
};
}
if upper.contains("CONSISTENT") {
log::debug!(
target: "crabmate::per",
"final_plan_semantic_check parse_path=legacy_text outcome=consistent"
);
return PlanSemanticLlmOutcome::consistent_ok();
}
}
PlanSemanticLlmOutcome::consistent_ok()
}
pub(crate) struct PlanSemanticLlmCtx<'a> {
pub llm_backend: &'a (dyn ChatCompletionsBackend + Send + Sync),
pub client: &'a reqwest::Client,
pub api_key: &'a str,
pub cfg: &'a AgentConfig,
pub out: Option<&'a mpsc::Sender<String>>,
pub no_stream: bool,
pub cancel: Option<&'a AtomicBool>,
pub request_chrome_trace: Option<Arc<crate::request_chrome_trace::RequestTurnTrace>>,
pub temperature_override: Option<f32>,
pub model_override: Option<String>,
pub seed_override: LlmSeedOverride,
pub max_tokens: u32,
pub turn_budget: Option<&'a std::sync::Arc<crate::agent::turn_budget::TurnBudgetCounter>>,
}
pub(crate) async fn evaluate_plan_consistency_with_recent_tools_llm(
ctx: PlanSemanticLlmCtx<'_>,
plan_json: &str,
tool_digest: Option<&str>,
) -> PlanSemanticLlmOutcome {
let Some(digest) = tool_digest.map(str::trim).filter(|s| !s.is_empty()) else {
return PlanSemanticLlmOutcome::consistent_ok();
};
let plan_trim = plan_json.trim();
if plan_trim.is_empty() {
return PlanSemanticLlmOutcome::consistent_ok();
}
let accept_legacy = ctx
.cfg
.per_plan_policy
.final_plan_semantic_check_accept_legacy_text;
let user_body = format!(
"### 最近工具结果摘要（截断）\n{}\n\n### agent_reply_plan JSON\n{}",
truncate_unicode(digest, 6000),
truncate_unicode(plan_trim, 8000)
);
let side_messages = vec![
Message::system_only(side_system_prompt(accept_legacy)),
Message::user_only(user_body),
];
let model_override = ctx
.model_override
.as_deref()
.or(ctx.cfg.llm.planner_model.as_deref());
let llm_cfg = crate::cm_types::llm_config::LlmConfig {
llm: ctx.cfg.llm.clone(),
sampling: ctx.cfg.llm_sampling.clone(),
vendor_flags: ctx.cfg.llm_vendor_flags.clone(),
http_retry: ctx.cfg.llm_http_retry.clone(),
};
let mut req = no_tools_chat_request(
&llm_cfg,
&side_messages,
ctx.temperature_override,
model_override,
ctx.seed_override,
);
req.max_tokens = ctx.max_tokens.clamp(32, 1024);
let cc = CompleteChatRetryingParams::new(
ctx.llm_backend,
ctx.client,
ctx.api_key,
ctx.cfg,
LlmRetryingTransportOpts {
out: ctx.out,
no_stream: ctx.no_stream,
cancel: ctx.cancel,
},
ctx.request_chrome_trace,
model_override,
)
.with_turn_budget(ctx.turn_budget);
let (reply, finish) = match complete_chat_retrying(&cc, &req).await {
Ok(x) => x,
Err(e) => {
log::warn!(
target: "crabmate::per",
"final_plan_semantic_check llm_call_failed error={} (fail-open)",
e
);
return PlanSemanticLlmOutcome::consistent_ok();
}
};
if finish == crate::types::USER_CANCELLED_FINISH_REASON {
return PlanSemanticLlmOutcome::user_cancelled();
}
let text = crate::types::message_content_as_str(&reply.content)
.unwrap_or("")
.trim();
let outcome = parse_plan_semantic_side_reply(text, accept_legacy);
if outcome.consistent {
log::debug!(target: "crabmate::per", "final_plan_semantic_check outcome=consistent");
} else {
log::info!(
target: "crabmate::per",
"final_plan_semantic_check outcome=inconsistent codes={:?} preview={}",
outcome.violation_codes,
crate::redact::preview_chars(text, 120)
);
}
outcome
}
pub(crate) fn agent_reply_plan_json_compact(plan: &plan_artifact::AgentReplyPlanV1) -> String {
plan_artifact::agent_reply_plan_v1_to_json_string(plan).unwrap_or_else(|_| "{}".to_string())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn user_cancelled_outcome_flag() {
let o = PlanSemanticLlmOutcome::user_cancelled();
assert!(o.user_cancelled);
assert!(!o.consistent);
}
#[test]
fn parse_json_consistent_minimal() {
let o = parse_plan_semantic_side_reply(r#"{"consistent":true}"#, false);
assert!(o.consistent);
assert!(o.violation_codes.is_empty());
assert!(o.rationale.is_none());
}
#[test]
fn parse_json_inconsistent_with_codes() {
let o = parse_plan_semantic_side_reply(
r#"{"consistent":false,"violation_codes":["tool_outcome_contradiction"],"rationale":"工具报错但计划写成功"}"#,
false,
);
assert!(!o.consistent);
assert_eq!(
o.violation_codes,
vec!["tool_outcome_contradiction".to_string()]
);
assert!(o.rationale.is_some());
}
#[test]
fn parse_json_inconsistent_empty_codes_gets_fallback() {
let o = parse_plan_semantic_side_reply(r#"{"consistent":false}"#, false);
assert!(!o.consistent);
assert_eq!(
o.violation_codes,
vec!["semantic_mismatch_unspecified".to_string()]
);
}
#[test]
fn parse_embedded_json_object() {
let o = parse_plan_semantic_side_reply(
"here is {\"consistent\":false,\"violation_codes\":[\"plan_step_tool_mismatch\"]}",
false,
);
assert!(!o.consistent);
assert_eq!(
o.violation_codes,
vec!["plan_step_tool_mismatch".to_string()]
);
}
#[test]
fn parse_legacy_inconsistent_line_when_enabled() {
let o = parse_plan_semantic_side_reply("INCONSISTENT 与工具结果矛盾", true);
assert!(!o.consistent);
assert_eq!(
o.violation_codes,
vec!["semantic_mismatch_legacy".to_string()]
);
}
#[test]
fn parse_legacy_consistent_line_when_enabled() {
let o = parse_plan_semantic_side_reply("CONSISTENT", true);
assert!(o.consistent);
}
#[test]
fn parse_legacy_text_ignored_by_default() {
let o = parse_plan_semantic_side_reply("INCONSISTENT 与工具结果矛盾", false);
assert!(
o.consistent,
"default JSON-only: legacy INCONSISTENT must fail-open"
);
let o2 = parse_plan_semantic_side_reply("CONSISTENT", false);
assert!(o2.consistent);
}
#[test]
fn parse_invalid_code_tokens_dropped() {
let o = parse_plan_semantic_side_reply(
r#"{"consistent":false,"violation_codes":["ok_code","Bad-Code","also bad"]}"#,
false,
);
assert!(!o.consistent);
assert_eq!(o.violation_codes, vec!["ok_code".to_string()]);
}
}