use std::collections::{HashMap, HashSet};
use std::sync::OnceLock;
use regex::{Regex, RegexBuilder};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use chio_core::capability::Constraint;
use chio_kernel::{Guard, GuardContext, KernelError, Verdict};
use crate::action::{extract_action, ToolAction};
#[derive(Debug, thiserror::Error)]
pub enum ContentReviewError {
#[error("invalid review pattern `{pattern}`: {source}")]
InvalidPattern {
pattern: String,
#[source]
source: regex::Error,
},
#[error("{0}")]
UnsafePattern(String),
}
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ContentReviewRules {
#[serde(default = "default_true")]
pub detect_pii: bool,
#[serde(default = "default_true")]
pub detect_profanity: bool,
#[serde(default)]
pub banned_words: Vec<String>,
#[serde(default)]
pub extra_patterns: Vec<String>,
#[serde(default = "default_max_scan_bytes")]
pub max_scan_bytes: usize,
}
fn default_true() -> bool {
true
}
fn default_max_scan_bytes() -> usize {
64 * 1024
}
#[derive(Clone, Debug, Deserialize, Serialize)]
#[serde(deny_unknown_fields)]
pub struct ContentReviewConfig {
#[serde(default = "default_true")]
pub enabled: bool,
#[serde(default = "default_rules")]
pub default_rules: ContentReviewRules,
#[serde(default)]
pub per_service: HashMap<String, ContentReviewRules>,
}
fn default_rules() -> ContentReviewRules {
ContentReviewRules {
detect_pii: true,
detect_profanity: true,
banned_words: Vec::new(),
extra_patterns: Vec::new(),
max_scan_bytes: default_max_scan_bytes(),
}
}
impl Default for ContentReviewConfig {
fn default() -> Self {
Self {
enabled: true,
default_rules: default_rules(),
per_service: HashMap::new(),
}
}
}
struct CompiledRules {
detect_pii: bool,
detect_profanity: bool,
banned_words: HashSet<String>,
extra_patterns: Vec<Regex>,
max_scan_bytes: usize,
}
const MAX_EXTRA_PATTERNS: usize = 64;
const MAX_EXTRA_PATTERN_LEN: usize = 512;
const MAX_EXTRA_PATTERN_COMPLEXITY: usize = 96;
const EXTRA_PATTERN_REGEX_SIZE_LIMIT: usize = 1 << 20;
const EXTRA_PATTERN_DFA_SIZE_LIMIT: usize = 1 << 20;
impl CompiledRules {
fn compile(rules: &ContentReviewRules) -> Result<Self, ContentReviewError> {
if rules.extra_patterns.len() > MAX_EXTRA_PATTERNS {
return Err(ContentReviewError::UnsafePattern(format!(
"content_review.extra_patterns allows at most {MAX_EXTRA_PATTERNS} patterns"
)));
}
let mut extra_patterns = Vec::with_capacity(rules.extra_patterns.len());
for pat in &rules.extra_patterns {
let trimmed = pat.trim();
if trimmed.is_empty() {
return Err(ContentReviewError::UnsafePattern(
"content_review.extra_patterns cannot contain empty patterns".to_string(),
));
}
if trimmed.len() > MAX_EXTRA_PATTERN_LEN {
return Err(ContentReviewError::UnsafePattern(format!(
"content_review.extra_patterns entries must be at most {MAX_EXTRA_PATTERN_LEN} characters"
)));
}
let complexity = review_pattern_complexity(trimmed);
if complexity > MAX_EXTRA_PATTERN_COMPLEXITY {
return Err(ContentReviewError::UnsafePattern(format!(
"content_review.extra_patterns entries must have complexity at most {MAX_EXTRA_PATTERN_COMPLEXITY}"
)));
}
let re = RegexBuilder::new(trimmed)
.size_limit(EXTRA_PATTERN_REGEX_SIZE_LIMIT)
.dfa_size_limit(EXTRA_PATTERN_DFA_SIZE_LIMIT)
.build()
.map_err(|e| ContentReviewError::InvalidPattern {
pattern: trimmed.to_string(),
source: e,
})?;
extra_patterns.push(re);
}
let banned_words = rules
.banned_words
.iter()
.map(|w| w.to_ascii_lowercase())
.collect();
Ok(Self {
detect_pii: rules.detect_pii,
detect_profanity: rules.detect_profanity,
banned_words,
extra_patterns,
max_scan_bytes: rules.max_scan_bytes.max(1),
})
}
}
fn review_pattern_complexity(pattern: &str) -> usize {
let mut score = 0usize;
let mut escaped = false;
for ch in pattern.chars() {
if escaped {
escaped = false;
continue;
}
match ch {
'\\' => escaped = true,
'|' | '*' | '+' | '?' => score = score.saturating_add(4),
'{' | '[' | '(' => score = score.saturating_add(2),
_ => {}
}
}
score
}
pub struct ContentReviewGuard {
enabled: bool,
default_rules: CompiledRules,
per_service: HashMap<String, CompiledRules>,
}
impl ContentReviewGuard {
pub fn new() -> Self {
match Self::with_config(ContentReviewConfig::default()) {
Ok(g) => g,
Err(_) => Self {
enabled: true,
default_rules: CompiledRules {
detect_pii: true,
detect_profanity: true,
banned_words: HashSet::new(),
extra_patterns: Vec::new(),
max_scan_bytes: default_max_scan_bytes(),
},
per_service: HashMap::new(),
},
}
}
pub fn with_config(config: ContentReviewConfig) -> Result<Self, ContentReviewError> {
let default_rules = CompiledRules::compile(&config.default_rules)?;
let mut per_service = HashMap::with_capacity(config.per_service.len());
for (service, rules) in &config.per_service {
per_service.insert(service.clone(), CompiledRules::compile(rules)?);
}
Ok(Self {
enabled: config.enabled,
default_rules,
per_service,
})
}
fn rules_for(&self, service: &str) -> &CompiledRules {
self.per_service.get(service).unwrap_or(&self.default_rules)
}
}
impl Default for ContentReviewGuard {
fn default() -> Self {
Self::new()
}
}
impl Guard for ContentReviewGuard {
fn name(&self) -> &str {
"content-review"
}
fn evaluate(&self, ctx: &GuardContext) -> Result<Verdict, KernelError> {
if !self.enabled {
return Ok(Verdict::Allow);
}
let action = extract_action(&ctx.request.tool_name, &ctx.request.arguments);
let (service, endpoint) = match action {
ToolAction::ExternalApiCall { service, endpoint } => (service, endpoint),
_ => return Ok(Verdict::Allow),
};
if let Some(verdict) = evaluate_amount_threshold(ctx, &service)? {
return Ok(verdict);
}
let text = extract_outbound_text(&ctx.request.arguments);
let text = match text {
Some(t) if !t.is_empty() => t,
_ => return Ok(Verdict::Allow),
};
let rules = self.rules_for(&service);
let truncated = truncate_utf8(&text, rules.max_scan_bytes);
let mut categories: Vec<&'static str> = Vec::new();
if rules.detect_pii {
for (category, re) in builtin_pii_patterns() {
if re.is_match(truncated) {
categories.push(*category);
}
}
}
if rules.detect_profanity && contains_banned_word(truncated, &rules.banned_words) {
categories.push("profanity");
}
for re in &rules.extra_patterns {
if re.is_match(truncated) {
categories.push("custom");
}
}
if !categories.is_empty() {
tracing::warn!(
guard = "content-review",
service = %service,
endpoint = %endpoint,
detected_categories = ?categories,
"content-review denied outbound message"
);
return Ok(Verdict::Deny);
}
Ok(Verdict::Allow)
}
}
fn evaluate_amount_threshold(
ctx: &GuardContext,
service: &str,
) -> Result<Option<Verdict>, KernelError> {
if !is_payment_service(service) {
return Ok(None);
}
let Some(grant) = ctx
.matched_grant_index
.and_then(|idx| ctx.scope.grants.get(idx))
else {
return Ok(None);
};
let threshold = grant.constraints.iter().find_map(|c| match c {
Constraint::RequireApprovalAbove { threshold_units } => Some(*threshold_units),
_ => None,
});
let Some(threshold) = threshold else {
return Ok(None);
};
let amount_units = extract_amount_units(ctx.request).or_else(|| {
ctx.request
.governed_intent
.as_ref()
.and_then(|intent| intent.max_amount.as_ref().map(|amt| amt.units))
});
let Some(units) = amount_units else {
return Ok(None);
};
if units >= threshold {
tracing::info!(
guard = "content-review",
service = %service,
units,
threshold,
"content-review requires human approval for monetary threshold"
);
return Ok(Some(Verdict::PendingApproval));
}
Ok(None)
}
fn is_payment_service(service: &str) -> bool {
matches!(
service,
"stripe" | "paypal" | "square" | "braintree" | "adyen" | "plaid"
)
}
fn extract_amount_units(request: &chio_kernel::ToolCallRequest) -> Option<u64> {
let args = &request.arguments;
for key in ["amount_units", "amountUnits", "amount"] {
if let Some(v) = args.get(key) {
if let Some(u) = v.as_u64() {
return Some(u);
}
if let Some(f) = v.as_f64() {
if f >= 0.0 && f.is_finite() {
return Some(f as u64);
}
}
}
}
None
}
fn extract_outbound_text(arguments: &Value) -> Option<String> {
let mut chunks: Vec<String> = Vec::new();
for key in [
"text",
"body",
"message",
"content",
"subject",
"html",
"description",
"summary",
"note",
] {
if let Some(v) = arguments.get(key).and_then(|v| v.as_str()) {
if !v.is_empty() {
chunks.push(v.to_string());
}
}
}
if let Some(arr) = arguments.get("blocks").and_then(|v| v.as_array()) {
for block in arr {
if let Some(text) = block
.get("text")
.and_then(|t| t.get("text"))
.and_then(|t| t.as_str())
{
chunks.push(text.to_string());
}
}
}
if chunks.is_empty() {
None
} else {
Some(chunks.join("\n"))
}
}
fn truncate_utf8(input: &str, max_bytes: usize) -> &str {
if input.len() <= max_bytes {
return input;
}
let mut end = max_bytes;
while end > 0 && !input.is_char_boundary(end) {
end -= 1;
}
&input[..end]
}
fn contains_banned_word(text: &str, banned: &HashSet<String>) -> bool {
if banned.is_empty() {
return false;
}
let lowered = text.to_ascii_lowercase();
for word in banned {
if word.is_empty() {
continue;
}
if lowered.contains(word) {
return true;
}
}
false
}
fn builtin_pii_patterns() -> &'static [(&'static str, Regex)] {
static PATS: OnceLock<Vec<(&'static str, Regex)>> = OnceLock::new();
PATS.get_or_init(|| {
let sources: &[(&'static str, &'static str)] = &[
("email", r"(?i)\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b"),
("ssn", r"\b\d{3}-\d{2}-\d{4}\b"),
("phone_us", r"\b(?:\+?1[-.\s]?)?\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}\b"),
("credit_card", r"\b(?:\d[ -]*?){13,19}\b"),
("ipv4", r"\b(?:\d{1,3}\.){3}\d{1,3}\b"),
];
sources
.iter()
.filter_map(|(cat, src)| match Regex::new(src) {
Ok(re) => Some((*cat, re)),
Err(err) => {
tracing::error!(error = %err, source = %src, category = %cat, "content-review: pii regex failed");
None
}
})
.collect()
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn extract_outbound_text_joins_chunks() {
let args = serde_json::json!({
"subject": "hi",
"body": "hello",
"blocks": [{"text": {"text": "b1"}}]
});
let text = extract_outbound_text(&args).unwrap();
assert!(text.contains("hi"));
assert!(text.contains("hello"));
assert!(text.contains("b1"));
}
#[test]
fn pii_patterns_detect_email() {
let pats = builtin_pii_patterns();
assert!(pats
.iter()
.any(|(cat, re)| *cat == "email" && re.is_match("user@example.com")));
}
#[test]
fn truncate_utf8_honors_boundaries() {
let s = "héllo";
let out = truncate_utf8(s, 2);
assert_eq!(out, "h");
}
}