use std::sync::Arc;
use std::time::Duration;
use evorule_reactor::{EventReceiver, Fact, FactId, FactSender, IoCallContext, IoType};
use evorule_tcb::JsonValue;
use crate::io_dispatcher::IoDispatcher;
use crate::metrics::{NoOpMetrics, SharedMetrics};
use crate::permission::{PermissionGate, Verdict};
const ID_OFFSET: u64 = 10000;
const MAX_RETRIES: u32 = 3;
const INITIAL_BACKOFF: Duration = Duration::from_millis(200);
fn is_retryable_error(err: &str) -> bool {
let lower = err.to_lowercase();
if lower.contains("timeout") || lower.contains("timed out") {
return true;
}
if lower.contains("connection") {
return true;
}
if lower.contains("temporarily") || lower.contains("temporary") {
return true;
}
for code in ["500", "502", "503", "504"] {
if lower.contains(code) {
return true;
}
}
false
}
#[derive(Debug, thiserror::Error)]
pub enum IoSubscriberError {
#[error("Event channel closed")]
ChannelClosed,
#[error("Command channel closed: {0}")]
CommandClosed(String),
}
pub type SkipPredicate = Arc<dyn Fn(&IoType, &JsonValue) -> bool + Send + Sync>;
pub struct IoSubscriber {
dispatcher: IoDispatcher,
next_id: u64,
metrics: SharedMetrics,
skip: Option<SkipPredicate>,
gate: Option<PermissionGate>,
}
impl IoSubscriber {
pub fn new(dispatcher: IoDispatcher) -> Self {
Self {
dispatcher,
next_id: ID_OFFSET,
metrics: Arc::new(NoOpMetrics),
skip: None,
gate: None,
}
}
pub fn with_permission_gate(mut self, gate: PermissionGate) -> Self {
self.gate = Some(gate);
self
}
pub fn with_skip(mut self, predicate: SkipPredicate) -> Self {
self.skip = Some(predicate);
self
}
pub fn with_metrics(mut self, metrics: SharedMetrics) -> Self {
self.metrics = metrics;
self
}
fn next_fact_id(&mut self) -> FactId {
let id = FactId(self.next_id);
self.next_id += 1;
id
}
pub async fn run(
mut self,
mut event_rx: EventReceiver,
command_tx: FactSender,
) -> Result<(), IoSubscriberError> {
tracing::info!(
id_offset = ID_OFFSET,
"IoSubscriber 启动,开始订阅 event broadcast 通道"
);
loop {
match event_rx.recv().await {
Ok(fact) => {
self.handle_fact(fact, &command_tx).await?;
}
Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
tracing::warn!(
skipped = n,
"IoSubscriber 落后于 event 通道,已跳过 {} 条 Fact",
n
);
continue;
}
Err(tokio::sync::broadcast::error::RecvError::Closed) => {
tracing::info!("Event 通道已关闭,IoSubscriber 正常退出");
return Ok(());
}
}
}
}
async fn handle_fact(
&mut self,
fact: Fact,
command_tx: &FactSender,
) -> Result<(), IoSubscriberError> {
match fact {
Fact::IoRequest {
id,
cause,
io_type,
params,
} => {
if let Some(skip) = &self.skip {
if skip(&io_type, ¶ms) {
tracing::trace!(
fact_id = %id,
io_type = %io_type,
"IoSubscriber 命中跳过谓词,IoRequest 留待外部执行者应答"
);
return Ok(());
}
}
self.dispatch_and_respond(id, cause, io_type, params, command_tx)
.await
}
other => {
tracing::trace!(
fact_id = %other.id(),
fact_type = other.type_name(),
"IoSubscriber 忽略非 IoRequest 事实"
);
Ok(())
}
}
}
#[allow(clippy::cognitive_complexity)]
async fn dispatch_and_respond(
&mut self,
request_id: FactId,
cause: FactId,
io_type: IoType,
params: JsonValue,
command_tx: &FactSender,
) -> Result<(), IoSubscriberError> {
tracing::info!(
request_id = %request_id,
io_type = %io_type,
"处理 IoRequest"
);
if let Some(gate) = &self.gate {
let mut ctx = IoCallContext::new(cause, 0, None);
let verdict = gate.check(&mut ctx, io_type.as_str(), Some(¶ms));
if verdict != Verdict::Allow {
let reason = match verdict {
Verdict::Deny => "denied by permission gate",
Verdict::Candidate => "pending permission approval (fail-closed)",
Verdict::Allow => unreachable!(),
};
tracing::warn!(
request_id = %request_id,
io_type = %io_type.as_str(),
caller_role = ?ctx.caller_role,
verdict = ?verdict,
"IoRequest 权限判定未通过,回写错误 IoResponse"
);
self.metrics.inc_io_errors(io_type.as_str());
let response = Fact::IoResponse {
id: self.next_fact_id(),
request_id,
result: JsonValue::Null,
error: Some(format!(
"permission denied: io_type={} {}, resource={}{}",
io_type.as_str(),
reason,
"io:",
io_type.as_str()
)),
};
return command_tx
.send(response)
.map_err(|_| IoSubscriberError::CommandClosed(format!("request_id={request_id}")));
}
}
let overall_start = std::time::Instant::now();
let io_type_str: &str = io_type.as_str();
let mut had_error = false;
let mut attempt: u32 = 0;
let response = loop {
attempt += 1;
match self.dispatcher.dispatch(&io_type, ¶ms).await {
Ok(result) => {
if attempt > 1 {
tracing::info!(
request_id = %request_id,
attempt,
"IoRequest 在重试后执行成功"
);
} else {
tracing::info!(
request_id = %request_id,
"IoRequest 执行成功,回写 IoResponse"
);
}
break Fact::IoResponse {
id: self.next_fact_id(),
request_id,
result,
error: None,
};
}
Err(err_msg) => {
if attempt <= MAX_RETRIES && is_retryable_error(&err_msg) {
let backoff =
INITIAL_BACKOFF.saturating_mul(2u32.saturating_pow(attempt - 1));
tracing::warn!(
request_id = %request_id,
attempt,
max_attempts = MAX_RETRIES + 1,
backoff_ms = backoff.as_millis() as u64,
error = %err_msg,
"IoRequest 瞬时错误,指数退避重试"
);
tokio::time::sleep(backoff).await;
continue;
}
tracing::warn!(
request_id = %request_id,
attempt,
error = %err_msg,
"IoRequest 执行失败(最终),回写错误 IoResponse"
);
had_error = true;
break Fact::IoResponse {
id: self.next_fact_id(),
request_id,
result: JsonValue::Null,
error: Some(err_msg),
};
}
}
};
self.metrics
.observe_io_duration(io_type_str, overall_start.elapsed());
if had_error {
self.metrics.inc_io_errors(io_type_str);
}
command_tx
.send(response)
.map_err(|_| IoSubscriberError::CommandClosed(format!("request_id={request_id}")))?;
Ok(())
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used)]
#![allow(clippy::panic, clippy::expect_used)]
use super::*;
use evorule_reactor::IoType;
#[test]
fn test_id_starts_at_offset() {
let next_id = ID_OFFSET;
assert_eq!(FactId(next_id), FactId(10000));
assert_eq!(FactId(next_id + 1), FactId(10001));
assert_eq!(FactId(next_id + 2), FactId(10002));
}
#[test]
fn test_io_subscriber_error_display() {
let e = IoSubscriberError::ChannelClosed;
assert_eq!(format!("{e}"), "Event channel closed");
let e = IoSubscriberError::CommandClosed("request_id=F42".to_string());
assert_eq!(format!("{e}"), "Command channel closed: request_id=F42");
}
#[test]
fn test_io_type_import_available() {
let t = IoType::call_external();
assert_eq!(t.as_str(), "call_external");
}
#[test]
fn test_is_retryable_error_timeout() {
assert!(is_retryable_error(
"http request failed: operation timed out"
));
assert!(is_retryable_error("db query timed out after 5s"));
assert!(is_retryable_error("request timeout"));
}
#[test]
fn test_is_retryable_error_connection() {
assert!(is_retryable_error("connection refused"));
assert!(is_retryable_error("connection reset by peer"));
assert!(is_retryable_error("Connection closed"));
}
#[test]
fn test_is_retryable_error_http_5xx() {
assert!(is_retryable_error(
"LLM API returned 503: service unavailable"
));
assert!(is_retryable_error("http request failed with status: 500"));
assert!(is_retryable_error("gateway 502 bad gateway"));
}
#[test]
fn test_is_retryable_error_temporary() {
assert!(is_retryable_error("service temporarily unavailable"));
assert!(is_retryable_error("Temporary failure in name resolution"));
}
#[test]
fn test_is_not_retryable_error_client_errors() {
assert!(!is_retryable_error("missing required param: prompt"));
assert!(!is_retryable_error("tool not found: foo"));
assert!(!is_retryable_error("LLM API returned 401: unauthorized"));
assert!(!is_retryable_error("http request failed with status: 404"));
assert!(!is_retryable_error("bad request 400"));
}
#[test]
fn test_retry_constants() {
assert_eq!(MAX_RETRIES, 3);
assert_eq!(INITIAL_BACKOFF, Duration::from_millis(200));
assert_eq!(
INITIAL_BACKOFF.saturating_mul(2u32.saturating_pow(0)),
Duration::from_millis(200)
);
assert_eq!(
INITIAL_BACKOFF.saturating_mul(2u32.saturating_pow(1)),
Duration::from_millis(400)
);
assert_eq!(
INITIAL_BACKOFF.saturating_mul(2u32.saturating_pow(2)),
Duration::from_millis(800)
);
}
fn llm_audit_params() -> JsonValue {
JsonValue::object_from_pairs(&[(
"messages",
JsonValue::array(vec![JsonValue::object_from_pairs(&[
("role", JsonValue::string("user")),
("content", JsonValue::string("hi")),
])]),
)])
}
#[tokio::test]
async fn test_skip_predicate_leaves_io_request_unanswered() {
let mut subscriber = IoSubscriber::new(IoDispatcher::builder().build()).with_skip(Arc::new(
|io_type: &IoType, params: &JsonValue| {
io_type.as_str() == "call_external"
&& params.get("messages").is_some()
&& params.get("service_name").is_none()
&& params.get("name").is_none()
},
));
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
let fact = Fact::IoRequest {
id: FactId(1),
cause: FactId(0),
io_type: IoType::call_external(),
params: llm_audit_params(),
};
let result = subscriber.handle_fact(fact, &tx).await;
assert!(result.is_ok());
assert!(
rx.try_recv().is_err(),
"skip 命中时不应回写任何 IoResponse"
);
}
#[tokio::test]
async fn test_without_skip_error_response_is_written() {
let mut subscriber = IoSubscriber::new(IoDispatcher::builder().build());
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
let fact = Fact::IoRequest {
id: FactId(2),
cause: FactId(0),
io_type: IoType::call_external(),
params: llm_audit_params(),
};
let result = subscriber.handle_fact(fact, &tx).await;
assert!(result.is_ok());
match rx.try_recv() {
Ok(Fact::IoResponse { request_id, error, .. }) => {
assert_eq!(request_id, FactId(2));
assert!(error.is_some(), "未注册类型应回写错误 IoResponse");
}
other => panic!("应回写错误 IoResponse,实际: {other:?}"),
}
}
#[tokio::test]
async fn test_skip_predicate_does_not_hit_service_calls() {
let mut subscriber = IoSubscriber::new(IoDispatcher::builder().build()).with_skip(Arc::new(
|io_type: &IoType, params: &JsonValue| {
io_type.as_str() == "call_external"
&& params.get("messages").is_some()
&& params.get("service_name").is_none()
&& params.get("name").is_none()
},
));
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
let params = JsonValue::object_from_pairs(&[(
"service_name",
JsonValue::string("inverse_kinematics_solver"),
)]);
let fact = Fact::IoRequest {
id: FactId(3),
cause: FactId(0),
io_type: IoType::call_external(),
params,
};
let result = subscriber.handle_fact(fact, &tx).await;
assert!(result.is_ok());
assert!(
rx.try_recv().is_ok(),
"带 service_name 的调用不应被跳过,应照常分发并回写"
);
}
fn service_call_request(id: u64) -> Fact {
let params = JsonValue::object_from_pairs(&[(
"service_name",
JsonValue::string("inverse_kinematics_solver"),
)]);
Fact::IoRequest {
id: FactId(id),
cause: FactId(1),
io_type: IoType::call_external(),
params,
}
}
#[tokio::test]
async fn test_permission_gate_deny_writes_error_response() {
use crate::permission::PermissionGate;
use std::sync::Arc as StdArc;
let gate = PermissionGate::new(StdArc::new(crate::shared_facts_log::SharedFactsLog::new()));
let mut subscriber = IoSubscriber::new(IoDispatcher::builder().build()).with_permission_gate(gate);
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
subscriber
.handle_fact(service_call_request(10), &tx)
.await
.unwrap();
match rx.try_recv() {
Ok(Fact::IoResponse { request_id, error, .. }) => {
assert_eq!(request_id, FactId(10));
let err = error.expect("Deny 必须回写错误 IoResponse");
assert!(
err.contains("permission denied"),
"错误消息应表明权限拒绝,实际: {err}"
);
}
other => panic!("Deny 应回写错误 IoResponse,实际: {other:?}"),
}
}
#[tokio::test]
async fn test_permission_gate_allow_proceeds_to_dispatch() {
use crate::permission::PermissionGate;
use evorule_reactor::CallerRole;
use std::sync::Arc as StdArc;
let gate = PermissionGate::new(StdArc::new(crate::shared_facts_log::SharedFactsLog::new()))
.with_caller_role_resolver(StdArc::new(|_| CallerRole::Human));
let mut subscriber = IoSubscriber::new(IoDispatcher::builder().build()).with_permission_gate(gate);
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
subscriber
.handle_fact(service_call_request(11), &tx)
.await
.unwrap();
match rx.try_recv() {
Ok(Fact::IoResponse { error, .. }) => {
let err = error.expect("空 dispatcher 必然分发失败");
assert!(
!err.contains("permission denied"),
"Allow 后错误应来自 dispatch 而非权限门,实际: {err}"
);
}
other => panic!("Allow 应照常 dispatch 并回写,实际: {other:?}"),
}
}
#[tokio::test]
async fn test_permission_gate_does_not_override_skip() {
use crate::permission::PermissionGate;
use std::sync::Arc as StdArc;
let gate = PermissionGate::new(StdArc::new(crate::shared_facts_log::SharedFactsLog::new()));
let mut subscriber = IoSubscriber::new(IoDispatcher::builder().build())
.with_permission_gate(gate)
.with_skip(StdArc::new(
|io_type: &IoType, params: &JsonValue| {
io_type.as_str() == "call_external"
&& params.get("messages").is_some()
&& params.get("service_name").is_none()
},
));
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
let fact = Fact::IoRequest {
id: FactId(12),
cause: FactId(1),
io_type: IoType::call_external(),
params: llm_audit_params(),
};
subscriber.handle_fact(fact, &tx).await.unwrap();
assert!(
rx.try_recv().is_err(),
"skip 命中时权限门不得抢答(审计回路依赖外部执行者应答)"
);
}
}