use crate::scanner::detect::{resolve_overlaps, Span};
use crate::scanner::{
Direction, Disposition, HoldBack, ScanCtx, ScanResult, Scanner, ScannerId, Verdict,
};
use crate::scanners::redact::redact;
const BANNED_LABEL: &str = "BANNED_SUBSTRING";
const HIT_RISK: f32 = 1.0;
#[derive(Debug, Clone)]
pub struct BanSubstringsScanner {
substrings: Vec<String>,
case_sensitive: bool,
disposition: Disposition,
direction: Direction,
}
impl BanSubstringsScanner {
pub const ID: ScannerId = ScannerId("native:ban-substrings");
#[must_use]
pub fn new(substrings: impl IntoIterator<Item = String>) -> Self {
Self {
substrings: substrings.into_iter().collect(),
case_sensitive: true,
disposition: Disposition::Block,
direction: Direction::Input,
}
}
#[must_use]
pub fn output_phrase_gate(phrases: impl IntoIterator<Item = String>) -> Self {
Self::new(phrases)
.case_insensitive()
.with_direction(Direction::Output)
}
#[must_use]
pub fn case_insensitive(mut self) -> Self {
self.case_sensitive = false;
self
}
#[must_use]
pub fn redacting(mut self) -> Self {
self.disposition = Disposition::Transform;
self
}
#[must_use]
pub fn with_direction(mut self, direction: Direction) -> Self {
self.direction = direction;
self
}
fn hits(&self, text: &str) -> Vec<Span> {
let haystack = if self.case_sensitive {
text.to_owned()
} else {
text.to_ascii_lowercase()
};
let mut spans = Vec::new();
for needle in &self.substrings {
if needle.is_empty() {
continue;
}
let needle = if self.case_sensitive {
needle.clone()
} else {
needle.to_ascii_lowercase()
};
let mut from = 0;
while let Some(rel) = haystack[from..].find(&needle) {
let start = from + rel;
let end = start + needle.len();
spans.push(Span::new(start, end, BANNED_LABEL, HIT_RISK));
from = end;
}
}
spans
}
}
impl Scanner for BanSubstringsScanner {
fn id(&self) -> ScannerId {
Self::ID
}
fn direction(&self) -> Direction {
self.direction
}
fn disposition(&self) -> Disposition {
self.disposition
}
fn scan<'a>(&self, text: &'a str, ctx: &mut ScanCtx) -> ScanResult<'a> {
let spans = resolve_overlaps(self.hits(text));
match self.disposition {
Disposition::Block => Ok(Verdict::detected(
text,
if spans.is_empty() { 0.0 } else { HIT_RISK },
HIT_RISK,
)),
Disposition::Transform => {
Ok(Verdict::transformed(redact(
text,
&spans,
&mut ctx.vault,
false,
)))
}
}
}
fn hold_back(&self) -> HoldBack {
match self.disposition {
Disposition::Transform => HoldBack::TokenBoundary,
Disposition::Block => HoldBack::WholeStream,
}
}
fn stream_patterns(&self) -> Vec<String> {
if self.disposition != Disposition::Transform {
return Vec::new();
}
let mut patterns = Vec::new();
for needle in &self.substrings {
if needle.is_empty() {
continue;
}
let escaped = regex::escape(needle);
patterns.push(if self.case_sensitive {
escaped
} else {
format!("(?i){escaped}")
});
}
patterns
}
}
#[cfg(test)]
mod tests {
#![allow(
clippy::unwrap_used,
clippy::expect_used,
reason = "tests assert on known-good values"
)]
use super::*;
#[test]
fn blocks_on_configured_substring() {
let scanner = BanSubstringsScanner::new(["forbidden".to_owned()]);
let mut ctx = ScanCtx::new();
let verdict = scanner.scan("this is forbidden text", &mut ctx).unwrap();
assert!(!verdict.valid);
assert!((verdict.risk - HIT_RISK).abs() < f32::EPSILON);
}
#[test]
fn passes_when_no_substring_present() {
let scanner = BanSubstringsScanner::new(["forbidden".to_owned()]);
let mut ctx = ScanCtx::new();
let verdict = scanner.scan("perfectly fine text", &mut ctx).unwrap();
assert!(verdict.valid);
}
#[test]
fn case_sensitive_by_default() {
let scanner = BanSubstringsScanner::new(["Forbidden".to_owned()]);
let mut ctx = ScanCtx::new();
assert!(scanner.scan("this is forbidden", &mut ctx).unwrap().valid);
}
#[test]
fn case_insensitive_flag_matches_any_case() {
let scanner = BanSubstringsScanner::new(["forbidden".to_owned()]).case_insensitive();
let mut ctx = ScanCtx::new();
assert!(!scanner.scan("this is FORBIDDEN", &mut ctx).unwrap().valid);
}
#[test]
fn output_phrase_gate_blocks_whole_stream() {
let scanner =
BanSubstringsScanner::output_phrase_gate(["as an ai language model".to_owned()]);
assert_eq!(scanner.direction(), Direction::Output);
assert_eq!(scanner.disposition(), Disposition::Block);
assert_eq!(scanner.hold_back(), HoldBack::WholeStream);
let mut ctx = ScanCtx::new();
let verdict = scanner
.scan("Sure! As an AI language model, I cannot.", &mut ctx)
.unwrap();
assert!(!verdict.valid);
}
#[test]
fn output_phrase_gate_passes_clean_response() {
let scanner = BanSubstringsScanner::output_phrase_gate(["forbidden phrase".to_owned()]);
let mut ctx = ScanCtx::new();
assert!(
scanner
.scan("a perfectly ordinary answer", &mut ctx)
.unwrap()
.valid
);
}
#[test]
fn redacting_mode_tokenizes_the_hit() {
let scanner = BanSubstringsScanner::new(["forbidden".to_owned()]).redacting();
let mut ctx = ScanCtx::new();
let verdict = scanner.scan("this is forbidden text", &mut ctx).unwrap();
assert_eq!(scanner.disposition(), Disposition::Transform);
assert!(verdict.valid);
assert!(verdict.text.contains("[REDACTED_BANNED_SUBSTRING_1_"));
assert!(!verdict.text.contains("forbidden"));
assert_eq!(ctx.vault.len(), 1);
}
}