pub mod detect;
pub mod substitute;
pub mod types;
pub mod vault;
use std::collections::BTreeMap;
use std::time::Instant;
use thiserror::Error;
pub use substitute::RestoreEncoder;
pub use types::{
Direction, Disposition, HoldBack, RestorePolicy, ScanResult, ScannerId, Threshold, Verdict,
};
pub use vault::{NonceStrategy, Vault};
#[derive(Debug, Error)]
pub enum ScanError {
#[error("scanner {scanner} failed: {reason}")]
Detector { scanner: ScannerId, reason: String },
}
#[non_exhaustive]
pub struct ScanCtx {
pub vault: Vault,
pub prompt: Option<String>,
pub restore_encoder: RestoreEncoder,
}
impl ScanCtx {
#[must_use]
pub fn new() -> Self {
Self {
vault: Vault::new(),
prompt: None,
restore_encoder: RestoreEncoder::identity(),
}
}
#[must_use]
pub fn with_vault(mut self, vault: Vault) -> Self {
self.vault = vault;
self
}
#[must_use]
pub fn with_prompt(mut self, prompt: impl Into<String>) -> Self {
self.prompt = Some(prompt.into());
self
}
}
impl Default for ScanCtx {
fn default() -> Self {
Self::new()
}
}
pub trait Scanner: Send + Sync {
fn id(&self) -> ScannerId;
fn direction(&self) -> Direction;
fn disposition(&self) -> Disposition;
fn scan<'a>(&self, text: &'a str, ctx: &mut ScanCtx) -> ScanResult<'a>;
fn max_input_inflation_ratio(&self) -> f64 {
1.0
}
fn hold_back(&self) -> HoldBack {
HoldBack::TokenBoundary
}
fn stream_patterns(&self) -> Vec<String> {
Vec::new()
}
}
#[derive(Debug, Default, Clone, PartialEq)]
pub struct ScanMetrics {
pub detections: u32,
pub redacted: u32,
pub restored: u32,
pub blocked: u32,
pub by_entity_type: BTreeMap<String, u32>,
pub latency_us: u64,
}
#[derive(Debug, Clone, PartialEq)]
pub struct ScanEntry {
pub scanner: ScannerId,
pub valid: bool,
pub risk: f32,
pub metrics: ScanMetrics,
}
#[derive(Debug, Default, Clone)]
pub struct ScanReport {
entries: Vec<ScanEntry>,
}
impl ScanReport {
fn record(&mut self, scanner: ScannerId, verdict: &Verdict<'_>, metrics: ScanMetrics) {
self.entries.push(ScanEntry {
scanner,
valid: verdict.valid,
risk: verdict.risk,
metrics,
});
}
#[must_use]
pub fn entries(&self) -> &[ScanEntry] {
&self.entries
}
#[must_use]
pub fn aggregate_risk(&self) -> f32 {
self.entries
.iter()
.map(|e| e.risk)
.filter(|r| *r >= 0.0)
.fold(0.0_f32, f32::max)
}
}
#[derive(Debug, Clone, PartialEq, Error)]
#[error("blocked by scanner {scanner} (risk {risk})")]
pub struct Blocked {
pub scanner: ScannerId,
pub risk: f32,
}
pub struct ScannerStack {
input: Vec<Box<dyn Scanner>>,
output: Vec<Box<dyn Scanner>>,
fail_fast: bool,
ctx: ScanCtx,
report: ScanReport,
}
impl ScannerStack {
#[must_use]
pub fn new(scanners: Vec<Box<dyn Scanner>>, fail_fast: bool) -> Self {
Self::with_ctx(scanners, fail_fast, ScanCtx::new())
}
#[must_use]
pub fn with_ctx(scanners: Vec<Box<dyn Scanner>>, fail_fast: bool, ctx: ScanCtx) -> Self {
let mut input = Vec::new();
let mut output = Vec::new();
for scanner in scanners {
match scanner.direction() {
Direction::Input => input.push(scanner),
Direction::Output => output.push(scanner),
}
}
Self {
input,
output,
fail_fast,
ctx,
report: ScanReport::default(),
}
}
#[must_use]
pub fn report(&self) -> &ScanReport {
&self.report
}
#[must_use]
pub fn ctx(&self) -> &ScanCtx {
&self.ctx
}
pub fn set_restore_encoder(&mut self, encoder: RestoreEncoder) {
self.ctx.restore_encoder = encoder;
}
#[must_use]
pub fn output_stream_patterns(&self) -> Vec<String> {
self.output
.iter()
.flat_map(|s| s.stream_patterns())
.collect()
}
#[must_use]
pub fn output_needs_whole_stream(&self) -> bool {
self.output
.iter()
.any(|s| s.hold_back() == HoldBack::WholeStream)
}
pub fn run_input(&mut self, text: &str) -> Result<String, Blocked> {
self.ctx.prompt = Some(text.to_owned());
let mut current = text.to_owned();
for scanner in &self.input {
let before = vault_snapshot(&self.ctx.vault);
let started = Instant::now();
let Ok(verdict) = scanner.scan(¤t, &mut self.ctx) else {
continue;
};
let metrics = derive_metrics(
scanner.as_ref(),
&verdict,
&before,
&self.ctx.vault,
started.elapsed(),
);
self.report.record(scanner.id(), &verdict, metrics);
let blocked = !verdict.valid && scanner.disposition() == Disposition::Block;
current = verdict.text.into_owned();
if blocked && self.fail_fast {
return Err(Blocked {
scanner: scanner.id(),
risk: self.report.aggregate_risk(),
});
}
}
Ok(current)
}
pub fn run_output(&mut self, text: &str) -> Result<String, Blocked> {
let mut current = text.to_owned();
for scanner in self.output.iter().rev() {
let before = vault_snapshot(&self.ctx.vault);
let started = Instant::now();
let Ok(verdict) = scanner.scan(¤t, &mut self.ctx) else {
continue;
};
let metrics = derive_metrics(
scanner.as_ref(),
&verdict,
&before,
&self.ctx.vault,
started.elapsed(),
);
self.report.record(scanner.id(), &verdict, metrics);
let blocked = !verdict.valid && scanner.disposition() == Disposition::Block;
current = verdict.text.into_owned();
if blocked && self.fail_fast {
return Err(Blocked {
scanner: scanner.id(),
risk: self.report.aggregate_risk(),
});
}
}
Ok(current)
}
}
struct VaultSnapshot {
interned: BTreeMap<String, u32>,
restored: BTreeMap<String, u32>,
}
fn vault_snapshot(vault: &Vault) -> VaultSnapshot {
VaultSnapshot {
interned: vault
.interned_counts()
.map(|(ty, n)| (ty.to_owned(), n))
.collect(),
restored: vault
.restored_counts()
.map(|(ty, n)| (ty.to_owned(), n))
.collect(),
}
}
fn derive_metrics(
scanner: &dyn Scanner,
verdict: &Verdict<'_>,
before: &VaultSnapshot,
vault: &Vault,
latency: std::time::Duration,
) -> ScanMetrics {
let by_entity_type = positive_deltas(&before.interned, vault.interned_counts());
let redacted: u32 = by_entity_type.values().sum();
let restored: u32 = positive_deltas(&before.restored, vault.restored_counts())
.values()
.sum();
let (detections, blocked) = match scanner.disposition() {
Disposition::Block => {
let hit = u32::from(!verdict.valid);
(hit, hit)
}
Disposition::Transform => (redacted + restored, 0),
};
let latency_us = u64::try_from(latency.as_micros()).unwrap_or(u64::MAX);
ScanMetrics {
detections,
redacted,
restored,
blocked,
by_entity_type,
latency_us,
}
}
fn positive_deltas<'a>(
before: &BTreeMap<String, u32>,
after: impl Iterator<Item = (&'a str, u32)>,
) -> BTreeMap<String, u32> {
let mut deltas = BTreeMap::new();
for (ty, now) in after {
let was = before.get(ty).copied().unwrap_or(0);
if now > was {
deltas.insert(ty.to_owned(), now - was);
}
}
deltas
}
#[cfg(test)]
mod tests {
#![allow(
clippy::unwrap_used,
clippy::expect_used,
reason = "tests assert on known-good values"
)]
use std::borrow::Cow;
use super::*;
struct Upper;
impl Scanner for Upper {
fn id(&self) -> ScannerId {
ScannerId("test:upper")
}
fn direction(&self) -> Direction {
Direction::Input
}
fn disposition(&self) -> Disposition {
Disposition::Transform
}
fn scan<'a>(&self, text: &'a str, _ctx: &mut ScanCtx) -> ScanResult<'a> {
Ok(Verdict::transformed(Cow::Owned(text.to_uppercase())))
}
}
struct BlockBad;
impl Scanner for BlockBad {
fn id(&self) -> ScannerId {
ScannerId("test:blockbad")
}
fn direction(&self) -> Direction {
Direction::Input
}
fn disposition(&self) -> Disposition {
Disposition::Block
}
fn scan<'a>(&self, text: &'a str, _ctx: &mut ScanCtx) -> ScanResult<'a> {
let risk = if text.to_lowercase().contains("bad") {
1.0
} else {
0.0
};
Ok(Verdict::detected(text, risk, 0.5))
}
}
#[test]
fn input_threads_transformed_text() {
let mut stack = ScannerStack::new(vec![Box::new(Upper)], true);
assert_eq!(stack.run_input("hi").unwrap(), "HI");
}
#[test]
fn block_scanner_short_circuits_fail_fast() {
let mut stack = ScannerStack::new(vec![Box::new(BlockBad), Box::new(Upper)], true);
let err = stack.run_input("this is bad").unwrap_err();
assert_eq!(err.scanner, ScannerId("test:blockbad"));
assert_eq!(stack.report().entries().len(), 1);
}
#[test]
fn clean_input_passes_all_scanners() {
let mut stack = ScannerStack::new(vec![Box::new(BlockBad), Box::new(Upper)], true);
assert_eq!(stack.run_input("all good").unwrap(), "ALL GOOD");
assert!(stack.report().aggregate_risk().abs() < f32::EPSILON);
}
struct InternAll;
impl Scanner for InternAll {
fn id(&self) -> ScannerId {
ScannerId("test:intern")
}
fn direction(&self) -> Direction {
Direction::Input
}
fn disposition(&self) -> Disposition {
Disposition::Transform
}
fn scan<'a>(&self, text: &'a str, ctx: &mut ScanCtx) -> ScanResult<'a> {
let s = ctx.vault.intern("EMAIL", text, true);
Ok(Verdict::transformed(Cow::Owned(s)))
}
}
#[test]
fn with_ctx_threads_a_deterministic_vault() {
let key = b"conversation-key";
let run = |text: &str| {
let ctx = ScanCtx::new().with_vault(Vault::deterministic(key));
let mut stack = ScannerStack::with_ctx(vec![Box::new(InternAll)], true, ctx);
stack.run_input(text).unwrap()
};
assert_eq!(run("alice@x.com"), run("alice@x.com"));
assert_ne!(run("alice@x.com"), run("bob@y.com"));
}
#[test]
fn aggregate_risk_is_max_skipping_transformers() {
let mut report = ScanReport::default();
let m = ScanMetrics::default();
report.record(ScannerId("a"), &Verdict::detected("x", 0.3, 1.0), m.clone());
report.record(
ScannerId("b"),
&Verdict::transformed(Cow::Borrowed("x")),
m.clone(),
);
report.record(ScannerId("c"), &Verdict::detected("x", 0.7, 1.0), m);
assert!((report.aggregate_risk() - 0.7).abs() < f32::EPSILON);
}
}