pub mod holdback;
pub mod runner;
pub mod stream;
pub mod tier;
use thiserror::Error;
pub use runner::GuardrailRunner;
pub use tier::{ModelSpec, PrivacyTier, TierPolicy};
#[derive(Debug, Clone, PartialEq, Error)]
pub enum MiddlewareError {
#[error("blocked: {detail}")]
Blocked { detail: String },
}
#[derive(Debug, Clone, Default)]
pub struct Params {
pub prompt: String,
pub primary: Option<ModelSpec>,
pub fallback: Vec<ModelSpec>,
pub fusion: Vec<ModelSpec>,
pub consent_cross_tier: bool,
}
impl Params {
#[must_use]
pub fn new(prompt: impl Into<String>) -> Self {
Self {
prompt: prompt.into(),
..Self::default()
}
}
#[must_use]
pub fn with_primary(mut self, primary: ModelSpec) -> Self {
self.primary = Some(primary);
self
}
#[must_use]
pub fn with_fallback(mut self, fallback: Vec<ModelSpec>) -> Self {
self.fallback = fallback;
self
}
#[must_use]
pub fn with_fusion(mut self, fusion: Vec<ModelSpec>) -> Self {
self.fusion = fusion;
self
}
#[must_use]
pub fn with_cross_tier_consent(mut self, consent: bool) -> Self {
self.consent_cross_tier = consent;
self
}
}
pub trait Model {
fn generate(&self, params: &Params) -> Result<String, MiddlewareError>;
}
pub type Chunk = Result<String, MiddlewareError>;
pub trait StreamModel {
fn stream(&self, params: &Params) -> Box<dyn Iterator<Item = Chunk> + '_>;
}
pub type ChunkSource<'a> = dyn Iterator<Item = Chunk> + 'a;
pub type ChunkSink<'a> = dyn FnMut(String) -> Result<(), MiddlewareError> + 'a;
pub trait Middleware: Send + Sync {
fn id(&self) -> &str;
fn transform_params(&self, params: Params) -> Result<Params, MiddlewareError> {
Ok(params)
}
fn wrap_generate(
&self,
params: &Params,
next: &dyn Fn(&Params) -> Result<String, MiddlewareError>,
) -> Result<String, MiddlewareError> {
next(params)
}
fn wrap_stream(
&self,
next: &mut ChunkSource<'_>,
sink: &mut ChunkSink<'_>,
) -> Result<(), MiddlewareError> {
for chunk in next {
sink(chunk?)?;
}
Ok(())
}
}
pub struct MiddlewareChain<'m> {
layers: Vec<&'m dyn Middleware>,
}
impl<'m> MiddlewareChain<'m> {
#[must_use]
pub fn new(layers: Vec<&'m dyn Middleware>) -> Self {
Self { layers }
}
pub fn generate(&self, params: Params, model: &dyn Model) -> Result<String, MiddlewareError> {
let mut params = params;
for layer in &self.layers {
params = layer.transform_params(params)?;
}
self.wrap_at(0, ¶ms, model)
}
fn wrap_at(
&self,
i: usize,
params: &Params,
model: &dyn Model,
) -> Result<String, MiddlewareError> {
match self.layers.get(i) {
Some(layer) => {
let next = |p: &Params| self.wrap_at(i + 1, p, model);
layer.wrap_generate(params, &next)
}
None => model.generate(params),
}
}
pub fn stream(
&self,
params: Params,
model: &dyn StreamModel,
) -> Result<Vec<String>, MiddlewareError> {
let mut params = params;
for layer in &self.layers {
params = layer.transform_params(params)?;
}
let model_chunks: Vec<Chunk> = model.stream(¶ms).collect();
let mut chunks = model_chunks;
for layer in self.layers.iter().rev() {
let mut emitted: Vec<String> = Vec::new();
let mut source = chunks.into_iter();
let mut sink = |s: String| -> Result<(), MiddlewareError> {
emitted.push(s);
Ok(())
};
layer.wrap_stream(&mut source, &mut sink)?;
chunks = emitted.into_iter().map(Ok).collect();
}
chunks.into_iter().collect()
}
}
#[cfg(test)]
mod tests {
#![allow(
clippy::unwrap_used,
clippy::expect_used,
reason = "tests assert on known-good values"
)]
use super::*;
struct Echo;
impl Model for Echo {
fn generate(&self, params: &Params) -> Result<String, MiddlewareError> {
Ok(format!("echo: {}", params.prompt))
}
}
struct Tag(&'static str);
impl Middleware for Tag {
fn id(&self) -> &str {
self.0
}
fn wrap_generate(
&self,
params: &Params,
next: &dyn Fn(&Params) -> Result<String, MiddlewareError>,
) -> Result<String, MiddlewareError> {
let inner = next(params)?;
Ok(format!("[{}]{inner}[/{}]", self.0, self.0))
}
}
#[test]
fn chain_nests_first_outermost() {
let a = Tag("a");
let b = Tag("b");
let chain = MiddlewareChain::new(vec![&a, &b]);
let out = chain.generate(Params::new("hi"), &Echo).unwrap();
assert_eq!(out, "[a][b]echo: hi[/b][/a]");
}
#[test]
fn empty_chain_calls_model_directly() {
let chain = MiddlewareChain::new(vec![]);
let out = chain.generate(Params::new("hi"), &Echo).unwrap();
assert_eq!(out, "echo: hi");
}
}