use std::{fmt, sync::Arc};
use runifold_core::{CapabilitySet, RunContext};
use runifold_model::{ContentPart, FinishReason, Message, ModelResponse};
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use sha2::{Digest, Sha256};
use thiserror::Error;
use crate::AgentFuture;
const MAX_REVIEW_FEEDBACK_BYTES: usize = 65_536;
const MAX_REVIEW_REQUEST_BYTES: usize = 1_048_576;
const MAX_REJECTION_REASON_BYTES: usize = 4_096;
const MAX_REVIEWER_NAME_BYTES: usize = 128;
const MAX_REVIEW_ERROR_BYTES: usize = 4_096;
const MAX_DESCRIPTOR_CONFIGURATION_BYTES: usize = 65_536;
#[derive(Clone)]
pub(crate) struct TerminalReviewConfig {
pub(crate) reviewer: Arc<dyn TerminalReviewer>,
pub(crate) descriptor: TerminalReviewerDescriptor,
pub(crate) policy: TerminalReviewPolicy,
pub(crate) capabilities: CapabilitySet,
}
#[derive(Clone)]
pub(crate) struct TurnReviewConfig {
pub(crate) reviewer: Arc<dyn TurnReviewer>,
pub(crate) descriptor: TerminalReviewerDescriptor,
pub(crate) policy: TurnReviewPolicy,
pub(crate) capabilities: CapabilitySet,
}
impl fmt::Debug for TurnReviewConfig {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("TurnReviewConfig")
.field("descriptor", &self.descriptor)
.field("policy", &self.policy)
.field("capabilities", &self.capabilities)
.finish_non_exhaustive()
}
}
impl fmt::Debug for TerminalReviewConfig {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("TerminalReviewConfig")
.field("descriptor", &self.descriptor)
.field("policy", &self.policy)
.field("capabilities", &self.capabilities)
.finish_non_exhaustive()
}
}
#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
pub struct TerminalReviewerDescriptor {
name: String,
version: String,
configuration_sha256: String,
}
impl TerminalReviewerDescriptor {
pub fn new(
name: impl Into<String>,
version: impl Into<String>,
configuration: &Value,
) -> Result<Self, TerminalReviewError> {
let name = name.into();
let version = version.into();
validate_identifier("terminal reviewer name", &name)?;
validate_identifier("terminal reviewer version", &version)?;
let encoded = serde_json::to_vec(configuration)
.map_err(|error| TerminalReviewError::InvalidConfiguration(error.to_string()))?;
if encoded.len() > MAX_DESCRIPTOR_CONFIGURATION_BYTES {
return Err(TerminalReviewError::InvalidConfiguration(format!(
"terminal reviewer configuration exceeds {MAX_DESCRIPTOR_CONFIGURATION_BYTES} bytes"
)));
}
let digest = Sha256::digest(encoded);
let configuration_sha256 = lowercase_hex(&digest);
Ok(Self {
name,
version,
configuration_sha256,
})
}
pub fn name(&self) -> &str {
&self.name
}
pub fn version(&self) -> &str {
&self.version
}
pub fn configuration_sha256(&self) -> &str {
&self.configuration_sha256
}
}
#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
pub struct TerminalReviewPolicy {
max_repairs: u32,
}
impl TerminalReviewPolicy {
pub const fn new(max_repairs: u32) -> Self {
Self { max_repairs }
}
pub const fn max_repairs(self) -> u32 {
self.max_repairs
}
}
#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
pub enum TurnReviewScope {
#[default]
IntermediateOnly,
EveryModelResponse,
}
impl TurnReviewScope {
pub(crate) fn includes(self, response: &ModelResponse) -> bool {
match self {
Self::EveryModelResponse => true,
Self::IntermediateOnly => {
response
.content
.iter()
.any(|part| matches!(part, ContentPart::ToolCall(_)))
|| matches!(
&response.finish_reason,
FinishReason::Other(reason) if reason == "pause_turn"
)
}
}
}
}
#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
pub struct TurnReviewPolicy {
max_repairs: u32,
scope: TurnReviewScope,
}
impl TurnReviewPolicy {
pub const fn new(max_repairs: u32) -> Self {
Self {
max_repairs,
scope: TurnReviewScope::IntermediateOnly,
}
}
#[must_use]
pub const fn with_scope(mut self, scope: TurnReviewScope) -> Self {
self.scope = scope;
self
}
pub const fn max_repairs(self) -> u32 {
self.max_repairs
}
pub const fn scope(self) -> TurnReviewScope {
self.scope
}
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
pub struct TerminalReviewRequest {
pub agent: String,
pub turn: u32,
pub attempt: u32,
pub transcript: Vec<Message>,
pub candidate: ModelResponse,
}
impl TerminalReviewRequest {
pub fn validate(&self) -> Result<(), TerminalReviewError> {
let bytes = serde_json::to_vec(self)
.map_err(|error| TerminalReviewError::InvalidVerdict(error.to_string()))?
.len();
if bytes > MAX_REVIEW_REQUEST_BYTES {
return Err(TerminalReviewError::RequestTooLarge {
bytes,
maximum: MAX_REVIEW_REQUEST_BYTES,
});
}
Ok(())
}
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
pub struct TurnReviewRequest {
pub agent: String,
pub turn: u32,
pub transcript: Vec<Message>,
pub candidate: ModelResponse,
}
impl TurnReviewRequest {
pub fn validate(&self) -> Result<(), TerminalReviewError> {
let bytes = serde_json::to_vec(self)
.map_err(|error| TerminalReviewError::InvalidVerdict(error.to_string()))?
.len();
if bytes > MAX_REVIEW_REQUEST_BYTES {
return Err(TerminalReviewError::RequestTooLarge {
bytes,
maximum: MAX_REVIEW_REQUEST_BYTES,
});
}
Ok(())
}
}
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
pub enum TerminalReviewVerdictKind {
Approve,
Repair,
Reject,
}
impl TerminalReviewVerdictKind {
pub(crate) const fn as_str(self) -> &'static str {
match self {
Self::Approve => "approve",
Self::Repair => "repair",
Self::Reject => "reject",
}
}
}
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
#[serde(tag = "verdict", rename_all = "snake_case")]
#[non_exhaustive]
pub enum TerminalReviewVerdict {
Approve,
Repair {
feedback: Value,
},
Reject {
reason: String,
},
}
impl TerminalReviewVerdict {
pub const fn approve() -> Self {
Self::Approve
}
pub fn repair(feedback: Value) -> Result<Self, TerminalReviewError> {
let verdict = Self::Repair { feedback };
verdict.validate()?;
Ok(verdict)
}
pub fn reject(reason: impl Into<String>) -> Result<Self, TerminalReviewError> {
let verdict = Self::Reject {
reason: reason.into(),
};
verdict.validate()?;
Ok(verdict)
}
pub fn validate(&self) -> Result<(), TerminalReviewError> {
match self {
Self::Approve => Ok(()),
Self::Repair { feedback } => {
if feedback.is_null() {
return Err(TerminalReviewError::InvalidVerdict(
"repair feedback cannot be null".into(),
));
}
let length = serde_json::to_vec(feedback)
.map_err(|error| TerminalReviewError::InvalidVerdict(error.to_string()))?
.len();
if length > MAX_REVIEW_FEEDBACK_BYTES {
return Err(TerminalReviewError::InvalidVerdict(format!(
"repair feedback exceeds {MAX_REVIEW_FEEDBACK_BYTES} bytes"
)));
}
Ok(())
}
Self::Reject { reason } => validate_bounded_text(
"rejection reason",
reason,
MAX_REJECTION_REASON_BYTES,
TerminalReviewError::InvalidVerdict,
),
}
}
pub const fn kind(&self) -> TerminalReviewVerdictKind {
match self {
Self::Approve => TerminalReviewVerdictKind::Approve,
Self::Repair { .. } => TerminalReviewVerdictKind::Repair,
Self::Reject { .. } => TerminalReviewVerdictKind::Reject,
}
}
}
#[derive(Clone, Debug, Error, Eq, PartialEq)]
#[non_exhaustive]
pub enum TerminalReviewError {
#[error("invalid terminal reviewer configuration: {0}")]
InvalidConfiguration(String),
#[error("terminal review request is {bytes} bytes; maximum is {maximum}")]
RequestTooLarge {
bytes: usize,
maximum: usize,
},
#[error("terminal reviewer execution failed: {0}")]
Execution(String),
#[error("invalid terminal review verdict: {0}")]
InvalidVerdict(String),
}
impl TerminalReviewError {
pub(crate) fn bounded(self) -> Self {
match self {
Self::InvalidConfiguration(message) => {
Self::InvalidConfiguration(truncate_utf8(message, MAX_REVIEW_ERROR_BYTES))
}
Self::Execution(message) => {
Self::Execution(truncate_utf8(message, MAX_REVIEW_ERROR_BYTES))
}
Self::InvalidVerdict(message) => {
Self::InvalidVerdict(truncate_utf8(message, MAX_REVIEW_ERROR_BYTES))
}
Self::RequestTooLarge { bytes, maximum } => Self::RequestTooLarge { bytes, maximum },
}
}
}
pub type TerminalReviewFuture<'a> =
AgentFuture<'a, Result<TerminalReviewVerdict, TerminalReviewError>>;
pub trait TerminalReviewer: Send + Sync {
fn descriptor(&self) -> &TerminalReviewerDescriptor;
fn review_terminal<'a>(
&'a self,
request: TerminalReviewRequest,
run: &'a RunContext,
) -> TerminalReviewFuture<'a>;
}
pub type TurnReviewVerdict = TerminalReviewVerdict;
pub type TurnReviewError = TerminalReviewError;
pub type TurnReviewerDescriptor = TerminalReviewerDescriptor;
pub type TurnReviewFuture<'a> = AgentFuture<'a, Result<TurnReviewVerdict, TurnReviewError>>;
pub trait TurnReviewer: Send + Sync {
fn turn_descriptor(&self) -> &TurnReviewerDescriptor;
fn review_turn<'a>(
&'a self,
request: TurnReviewRequest,
run: &'a RunContext,
) -> TurnReviewFuture<'a>;
}
impl<T> TurnReviewer for T
where
T: TerminalReviewer + ?Sized,
{
fn turn_descriptor(&self) -> &TurnReviewerDescriptor {
TerminalReviewer::descriptor(self)
}
fn review_turn<'a>(
&'a self,
request: TurnReviewRequest,
run: &'a RunContext,
) -> TurnReviewFuture<'a> {
let terminal = TerminalReviewRequest {
agent: request.agent,
turn: request.turn,
attempt: 1,
transcript: request.transcript,
candidate: request.candidate,
};
self.review_terminal(terminal, run)
}
}
type TurnRuleFunction =
dyn Fn(&TurnReviewRequest) -> Result<TurnReviewVerdict, TurnReviewError> + Send + Sync;
#[derive(Clone)]
pub struct TurnRuleReviewer {
descriptor: TurnReviewerDescriptor,
rule: Arc<TurnRuleFunction>,
}
impl TurnRuleReviewer {
pub fn new<F>(
name: impl Into<String>,
version: impl Into<String>,
rule: F,
) -> Result<Self, TurnReviewError>
where
F: Fn(&TurnReviewRequest) -> Result<TurnReviewVerdict, TurnReviewError>
+ Send
+ Sync
+ 'static,
{
let name = name.into();
let version = version.into();
let descriptor = TurnReviewerDescriptor::new(name, version, &json!({"kind": "turn_rule"}))?;
Ok(Self {
descriptor,
rule: Arc::new(rule),
})
}
pub fn name(&self) -> &str {
self.descriptor.name()
}
}
impl TurnReviewer for TurnRuleReviewer {
fn turn_descriptor(&self) -> &TurnReviewerDescriptor {
&self.descriptor
}
fn review_turn<'a>(
&'a self,
request: TurnReviewRequest,
_run: &'a RunContext,
) -> TurnReviewFuture<'a> {
let verdict = request
.validate()
.and_then(|()| (self.rule)(&request))
.and_then(|verdict| {
verdict.validate()?;
Ok(verdict)
});
Box::pin(async move { verdict })
}
}
impl fmt::Debug for TurnRuleReviewer {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("TurnRuleReviewer")
.field("descriptor", &self.descriptor)
.finish_non_exhaustive()
}
}
type RuleFunction = dyn Fn(&TerminalReviewRequest) -> Result<TerminalReviewVerdict, TerminalReviewError>
+ Send
+ Sync;
#[derive(Clone)]
pub struct TerminalRuleReviewer {
descriptor: TerminalReviewerDescriptor,
rule: Arc<RuleFunction>,
}
impl TerminalRuleReviewer {
pub fn new<F>(
name: impl Into<String>,
version: impl Into<String>,
rule: F,
) -> Result<Self, TerminalReviewError>
where
F: Fn(&TerminalReviewRequest) -> Result<TerminalReviewVerdict, TerminalReviewError>
+ Send
+ Sync
+ 'static,
{
let name = name.into();
let version = version.into();
let descriptor =
TerminalReviewerDescriptor::new(name, version, &json!({"kind": "terminal_rule"}))?;
Ok(Self {
descriptor,
rule: Arc::new(rule),
})
}
pub fn name(&self) -> &str {
self.descriptor.name()
}
}
impl TerminalReviewer for TerminalRuleReviewer {
fn descriptor(&self) -> &TerminalReviewerDescriptor {
&self.descriptor
}
fn review_terminal<'a>(
&'a self,
request: TerminalReviewRequest,
_run: &'a RunContext,
) -> TerminalReviewFuture<'a> {
let verdict = request
.validate()
.and_then(|()| (self.rule)(&request))
.and_then(|verdict| {
verdict.validate()?;
Ok(verdict)
});
Box::pin(async move { verdict })
}
}
impl fmt::Debug for TerminalRuleReviewer {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("TerminalRuleReviewer")
.field("descriptor", &self.descriptor)
.finish_non_exhaustive()
}
}
#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
pub enum CompositeTerminalReviewMode {
#[default]
AllMustApprove,
FirstFailure,
}
#[derive(Clone)]
struct TerminalReviewerEntry {
name: String,
reviewer: Arc<dyn TerminalReviewer>,
}
#[derive(Clone)]
pub struct CompositeTerminalReviewer {
name: String,
version: String,
mode: CompositeTerminalReviewMode,
descriptor: TerminalReviewerDescriptor,
reviewers: Vec<TerminalReviewerEntry>,
}
impl CompositeTerminalReviewer {
pub fn new(
name: impl Into<String>,
version: impl Into<String>,
mode: CompositeTerminalReviewMode,
) -> Result<Self, TerminalReviewError> {
let name = name.into();
let version = version.into();
let descriptor = composite_descriptor(&name, &version, mode, &[])?;
Ok(Self {
name,
version,
mode,
descriptor,
reviewers: Vec::new(),
})
}
pub fn push<R>(
&mut self,
name: impl Into<String>,
reviewer: R,
) -> Result<(), TerminalReviewError>
where
R: TerminalReviewer + 'static,
{
self.push_shared(name, Arc::new(reviewer))
}
pub fn push_shared(
&mut self,
name: impl Into<String>,
reviewer: Arc<dyn TerminalReviewer>,
) -> Result<(), TerminalReviewError> {
let name = name.into();
validate_identifier("composite terminal reviewer entry", &name)?;
if self.reviewers.iter().any(|entry| entry.name == name) {
return Err(TerminalReviewError::InvalidConfiguration(format!(
"duplicate composite terminal reviewer entry `{name}`"
)));
}
let mut reviewers = self.reviewers.clone();
reviewers.push(TerminalReviewerEntry { name, reviewer });
let descriptor = composite_descriptor(&self.name, &self.version, self.mode, &reviewers)?;
self.reviewers = reviewers;
self.descriptor = descriptor;
Ok(())
}
pub const fn mode(&self) -> CompositeTerminalReviewMode {
self.mode
}
pub fn len(&self) -> usize {
self.reviewers.len()
}
pub fn is_empty(&self) -> bool {
self.reviewers.is_empty()
}
}
impl TerminalReviewer for CompositeTerminalReviewer {
fn descriptor(&self) -> &TerminalReviewerDescriptor {
&self.descriptor
}
fn review_terminal<'a>(
&'a self,
request: TerminalReviewRequest,
run: &'a RunContext,
) -> TerminalReviewFuture<'a> {
Box::pin(async move {
request.validate()?;
if self.reviewers.is_empty() {
return Err(TerminalReviewError::InvalidConfiguration(
"composite terminal reviewer requires at least one reviewer".into(),
));
}
let mut repairs = Vec::new();
for entry in &self.reviewers {
let verdict = entry
.reviewer
.review_terminal(request.clone(), run)
.await
.map_err(TerminalReviewError::bounded)?;
verdict.validate().map_err(TerminalReviewError::bounded)?;
match verdict {
TerminalReviewVerdict::Approve => {}
TerminalReviewVerdict::Reject { reason } => {
return TerminalReviewVerdict::reject(reason);
}
TerminalReviewVerdict::Repair { feedback } => {
if self.mode == CompositeTerminalReviewMode::FirstFailure {
return TerminalReviewVerdict::repair(feedback);
}
repairs.push(json!({
"reviewer": entry.name,
"feedback": feedback,
}));
}
}
}
if repairs.is_empty() {
Ok(TerminalReviewVerdict::approve())
} else {
TerminalReviewVerdict::repair(json!({
"kind": "composite",
"reviews": repairs,
}))
}
})
}
}
impl fmt::Debug for CompositeTerminalReviewer {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("CompositeTerminalReviewer")
.field("name", &self.name)
.field("version", &self.version)
.field("descriptor", &self.descriptor)
.field("mode", &self.mode)
.field(
"reviewers",
&self
.reviewers
.iter()
.map(|entry| entry.name.as_str())
.collect::<Vec<_>>(),
)
.finish()
}
}
fn composite_descriptor(
name: &str,
version: &str,
mode: CompositeTerminalReviewMode,
reviewers: &[TerminalReviewerEntry],
) -> Result<TerminalReviewerDescriptor, TerminalReviewError> {
TerminalReviewerDescriptor::new(
name,
version,
&json!({
"kind": "composite_terminal",
"mode": mode,
"reviewers": reviewers.iter().map(|entry| json!({
"entry": entry.name,
"descriptor": entry.reviewer.descriptor(),
})).collect::<Vec<_>>(),
}),
)
}
fn validate_identifier(field: &str, value: &str) -> Result<(), TerminalReviewError> {
if value.is_empty()
|| value.len() > MAX_REVIEWER_NAME_BYTES
|| !value
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'-' | b'.'))
{
return Err(TerminalReviewError::InvalidConfiguration(format!(
"{field} must contain 1..={MAX_REVIEWER_NAME_BYTES} ASCII letters, digits, `_`, `-`, or `.`"
)));
}
Ok(())
}
fn truncate_utf8(mut value: String, maximum: usize) -> String {
if value.len() <= maximum {
return value;
}
let mut end = maximum;
while !value.is_char_boundary(end) {
end = end.saturating_sub(1);
}
value.truncate(end);
value
}
fn lowercase_hex(bytes: &[u8]) -> String {
const DIGITS: &[u8; 16] = b"0123456789abcdef";
bytes.iter().fold(
String::with_capacity(bytes.len().saturating_mul(2)),
|mut encoded, byte| {
encoded.push(char::from(DIGITS[usize::from(byte >> 4)]));
encoded.push(char::from(DIGITS[usize::from(byte & 0x0f)]));
encoded
},
)
}
pub(crate) fn validate_bounded_text(
field: &str,
value: &str,
maximum: usize,
error: fn(String) -> TerminalReviewError,
) -> Result<(), TerminalReviewError> {
if value.trim().is_empty() {
return Err(error(format!("{field} cannot be blank")));
}
if value.len() > maximum {
return Err(error(format!("{field} exceeds {maximum} bytes")));
}
Ok(())
}
#[cfg(test)]
mod tests {
use serde_json::{Value, json};
use super::{
TerminalReviewError, TerminalReviewVerdict, TerminalReviewerDescriptor,
TerminalRuleReviewer,
};
#[test]
fn repair_feedback_and_rejection_reasons_are_bounded() {
assert!(matches!(
TerminalReviewVerdict::repair(Value::Null),
Err(TerminalReviewError::InvalidVerdict(_))
));
assert!(matches!(
TerminalReviewVerdict::repair(json!({"body": "x".repeat(65_537)})),
Err(TerminalReviewError::InvalidVerdict(_))
));
assert!(matches!(
TerminalReviewVerdict::reject(" "),
Err(TerminalReviewError::InvalidVerdict(_))
));
}
#[test]
fn rule_reviewer_requires_a_stable_name() {
let result = TerminalRuleReviewer::new("not a stable name", "v1", |_| {
Ok(TerminalReviewVerdict::approve())
});
assert!(matches!(
result,
Err(TerminalReviewError::InvalidConfiguration(_))
));
}
#[test]
fn descriptor_fingerprint_changes_with_configuration() {
let first =
TerminalReviewerDescriptor::new("review", "v1", &json!({"threshold": 1})).unwrap();
let same =
TerminalReviewerDescriptor::new("review", "v1", &json!({"threshold": 1})).unwrap();
let changed =
TerminalReviewerDescriptor::new("review", "v1", &json!({"threshold": 2})).unwrap();
assert_eq!(first, same);
assert_ne!(first, changed);
assert_eq!(first.configuration_sha256().len(), 64);
}
#[test]
fn reviewer_execution_errors_are_unicode_safely_bounded() {
let bounded = TerminalReviewError::Execution("界".repeat(2_000)).bounded();
let TerminalReviewError::Execution(message) = bounded else {
panic!("execution error kind must be retained");
};
assert!(message.len() <= 4_096);
assert!(message.is_char_boundary(message.len()));
}
}