use std::fmt;
use regex::Regex;
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Predicate {
Contains { needle: String },
Absent { needle: String },
Matches { pattern: String },
WordCount {
min: Option<usize>,
max: Option<usize>,
},
ParsesAsJson,
RequiredWith { trigger: String, required: String },
}
impl Predicate {
fn holds(&self, response: &str, lower: &str) -> Result<(), String> {
match self {
Predicate::Contains { needle } => {
if lower.contains(&needle.to_lowercase()) {
Ok(())
} else {
Err(format!("the response does not contain {needle:?}"))
}
}
Predicate::Absent { needle } => {
if lower.contains(&needle.to_lowercase()) {
Err(format!(
"the response contains {needle:?}, which is forbidden"
))
} else {
Ok(())
}
}
Predicate::Matches { pattern } => match Regex::new(pattern) {
Ok(re) => {
if re.is_match(response) {
Ok(())
} else {
Err(format!("the response does not match /{pattern}/"))
}
}
Err(e) => Err(format!("the constraint's pattern /{pattern}/ is not a valid regular expression: {e}")),
},
Predicate::WordCount { min, max } => {
let n = response.split_whitespace().count();
if let Some(lo) = min {
if n < *lo {
return Err(format!("the response has {n} words, fewer than the required minimum of {lo}"));
}
}
if let Some(hi) = max {
if n > *hi {
return Err(format!("the response has {n} words, more than the permitted maximum of {hi}"));
}
}
Ok(())
}
Predicate::ParsesAsJson => {
match serde_json::from_str::<serde_json::Value>(response.trim()) {
Ok(_) => Ok(()),
Err(e) => Err(format!("the response is not valid JSON: {e}")),
}
}
Predicate::RequiredWith { trigger, required } => {
if lower.contains(&trigger.to_lowercase())
&& !lower.contains(&required.to_lowercase())
{
Err(format!(
"the response contains {trigger:?} but not the {required:?} it requires"
))
} else {
Ok(())
}
}
}
}
fn obligation(&self) -> String {
match self {
Predicate::Contains { needle } => format!("must contain {needle:?}"),
Predicate::Absent { needle } => format!("must not contain {needle:?}"),
Predicate::Matches { pattern } => format!("must match /{pattern}/"),
Predicate::WordCount { min, max } => match (min, max) {
(Some(lo), Some(hi)) => format!("must be between {lo} and {hi} words"),
(Some(lo), None) => format!("must be at least {lo} words"),
(None, Some(hi)) => format!("must be at most {hi} words"),
(None, None) => "has no word-count bound".to_string(),
},
Predicate::ParsesAsJson => "must be valid JSON".to_string(),
Predicate::RequiredWith { trigger, required } => {
format!("must not contain {trigger:?} without also containing {required:?}")
}
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Constraint {
pub id: String,
pub source: String,
pub predicate: Predicate,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Clause {
Checkable(Constraint),
Uncheckable(String),
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ValidatorError {
NoCheckableConstraints { clauses: Vec<String> },
InvalidPattern { id: String, pattern: String, detail: String },
}
impl fmt::Display for ValidatorError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
ValidatorError::NoCheckableConstraints { clauses } => {
write!(
f,
"this mandate has no machine-checkable constraint, so its \
satisfaction rate is undefined and no response could ever \
violate it"
)?;
if !clauses.is_empty() {
write!(f, "; the clauses that could not be lowered were: ")?;
write!(f, "{}", clauses.join("; "))?;
}
Ok(())
}
ValidatorError::InvalidPattern { id, pattern, detail } => write!(
f,
"constraint {id} declares the pattern /{pattern}/, which is not a \
valid regular expression: {detail}"
),
}
}
}
impl std::error::Error for ValidatorError {}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ConstraintSet {
constraints: Vec<Constraint>,
unchecked: Vec<String>,
}
impl ConstraintSet {
pub fn new(clauses: Vec<Clause>) -> Result<Self, ValidatorError> {
let mut constraints = Vec::new();
let mut unchecked = Vec::new();
for clause in clauses {
match clause {
Clause::Checkable(c) => {
if let Predicate::Matches { pattern } = &c.predicate {
if let Err(e) = Regex::new(pattern) {
return Err(ValidatorError::InvalidPattern {
id: c.id.clone(),
pattern: pattern.clone(),
detail: e.to_string(),
});
}
}
constraints.push(c);
}
Clause::Uncheckable(text) => unchecked.push(text),
}
}
if constraints.is_empty() {
return Err(ValidatorError::NoCheckableConstraints { clauses: unchecked });
}
Ok(ConstraintSet {
constraints,
unchecked,
})
}
pub fn len(&self) -> usize {
self.constraints.len()
}
pub fn is_empty(&self) -> bool {
false
}
pub fn unchecked(&self) -> &[String] {
&self.unchecked
}
pub fn constraints(&self) -> &[Constraint] {
&self.constraints
}
pub fn evaluate(&self, response: &str) -> Verdict {
let lower = response.to_lowercase();
let mut satisfied = Vec::new();
let mut violated = Vec::new();
for c in &self.constraints {
match c.predicate.holds(response, &lower) {
Ok(()) => satisfied.push(c.id.clone()),
Err(reason) => violated.push(Violation {
id: c.id.clone(),
source: c.source.clone(),
obligation: c.predicate.obligation(),
reason,
}),
}
}
let csr = satisfied.len() as f64 / self.constraints.len() as f64;
Verdict {
csr,
error: 1.0 - csr,
satisfied,
violated,
}
}
pub fn lower(text: &str) -> Vec<Clause> {
let mut out = Vec::new();
for (i, raw) in split_clauses(text).into_iter().enumerate() {
let clause = raw.trim();
if clause.is_empty() {
continue;
}
let id = format!("c{}", i + 1);
match lower_clause(clause) {
Some(predicate) => out.push(Clause::Checkable(Constraint {
id,
source: clause.to_string(),
predicate,
})),
None => out.push(Clause::Uncheckable(clause.to_string())),
}
}
out
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Violation {
pub id: String,
pub source: String,
pub obligation: String,
pub reason: String,
}
#[derive(Debug, Clone, PartialEq)]
pub struct Verdict {
pub csr: f64,
pub error: f64,
pub satisfied: Vec<String>,
pub violated: Vec<Violation>,
}
impl Verdict {
pub fn is_satisfied(&self) -> bool {
self.violated.is_empty()
}
pub fn within(&self, epsilon: f64) -> bool {
self.error.abs() < epsilon
}
pub fn feedback(&self) -> String {
self.violated
.iter()
.map(|v| format!("- {} ({})", v.obligation, v.reason))
.collect::<Vec<_>>()
.join("\n")
}
}
fn split_clauses(text: &str) -> Vec<String> {
let mut parts = Vec::new();
for on_semi in text.split(';') {
let mut rest = on_semi;
while let Some(idx) = rest.find(", and ") {
parts.push(rest[..idx].to_string());
rest = &rest[idx + ", and ".len()..];
}
parts.push(rest.to_string());
}
parts
}
fn lower_clause(clause: &str) -> Option<Predicate> {
let c = clause.trim().to_lowercase();
let c = c
.trim_start_matches("output ")
.trim_start_matches("the output ")
.trim_start_matches("the response ")
.trim_start_matches("response ")
.trim();
for sep in [" without ", " unless "] {
if let Some(rest) = c.strip_prefix("no ") {
if let Some(idx) = rest.find(sep) {
let trigger = rest[..idx].trim();
let required = rest[idx + sep.len()..].trim();
if !trigger.is_empty() && !required.is_empty() {
return Some(Predicate::RequiredWith {
trigger: trigger.to_string(),
required: required.to_string(),
});
}
}
}
}
for form in [
"must be valid json",
"must be a valid json",
"must parse as json",
"must be valid json object",
] {
if c.starts_with(form) {
return Some(Predicate::ParsesAsJson);
}
}
if let Some(n) = trailing_count(&c, "must be at most ", " words") {
return Some(Predicate::WordCount {
min: None,
max: Some(n),
});
}
if let Some(n) = trailing_count(&c, "must be at least ", " words") {
return Some(Predicate::WordCount {
min: Some(n),
max: None,
});
}
if let Some(n) = trailing_count(&c, "must be no more than ", " words") {
return Some(Predicate::WordCount {
min: None,
max: Some(n),
});
}
if let Some(rest) = c.strip_prefix("must match /") {
if let Some(pattern) = rest.strip_suffix('/') {
if !pattern.is_empty() {
return Some(Predicate::Matches {
pattern: pattern.to_string(),
});
}
}
}
for (prefix, negated) in [
("must not contain ", true),
("must not include ", true),
("must never contain ", true),
("must contain ", false),
("must include ", false),
] {
if let Some(rest) = c.strip_prefix(prefix) {
let needle = unquote(rest.trim());
if !needle.is_empty() {
return Some(if negated {
Predicate::Absent { needle }
} else {
Predicate::Contains { needle }
});
}
}
}
None
}
fn trailing_count(c: &str, prefix: &str, suffix: &str) -> Option<usize> {
let rest = c.strip_prefix(prefix)?;
let digits = rest.strip_suffix(suffix).unwrap_or(rest);
digits.trim().parse::<usize>().ok()
}
fn unquote(s: &str) -> String {
let s = s.trim().trim_end_matches('.');
for q in ['"', '\'', '\u{201c}'] {
if let Some(rest) = s.strip_prefix(q) {
let close = if q == '\u{201c}' { '\u{201d}' } else { q };
if let Some(inner) = rest.strip_suffix(close) {
return inner.to_string();
}
}
}
s.to_string()
}
#[cfg(test)]
mod tests {
use super::*;
fn checkable(id: &str, predicate: Predicate) -> Clause {
Clause::Checkable(Constraint {
id: id.to_string(),
source: predicate.obligation(),
predicate,
})
}
fn set(predicates: Vec<Predicate>) -> ConstraintSet {
ConstraintSet::new(
predicates
.into_iter()
.enumerate()
.map(|(i, p)| checkable(&format!("c{}", i + 1), p))
.collect(),
)
.expect("fixture must have at least one checkable constraint")
}
#[test]
fn csr_is_the_fraction_satisfied_and_error_is_its_complement() {
let c = set(vec![
Predicate::Contains {
needle: "alpha".into(),
},
Predicate::Contains {
needle: "beta".into(),
},
Predicate::Contains {
needle: "gamma".into(),
},
Predicate::Contains {
needle: "delta".into(),
},
]);
let v = c.evaluate("alpha and beta only");
assert_eq!(v.csr, 0.5, "2 of 4 constraints hold");
assert_eq!(v.error, 0.5, "e = 1 − CSR");
assert_eq!(v.violated.len(), 2);
assert!(!v.is_satisfied());
}
#[test]
fn full_satisfaction_is_exactly_zero_error() {
let v = set(vec![
Predicate::Contains {
needle: "alpha".into(),
},
Predicate::Absent {
needle: "forbidden".into(),
},
])
.evaluate("alpha, and nothing else");
assert_eq!(v.error, 0.0);
assert!(v.is_satisfied());
assert!(v.within(1e-9), "zero error is inside every positive band");
}
#[test]
fn total_violation_is_error_one() {
let v = set(vec![Predicate::Contains {
needle: "alpha".into(),
}])
.evaluate("nothing relevant");
assert_eq!(v.error, 1.0);
}
#[test]
fn error_is_always_in_the_unit_interval() {
let c = set(vec![
Predicate::Contains {
needle: "a".into(),
},
Predicate::Absent {
needle: "a".into(),
},
]);
for response in ["a", "b", "", "aaaa"] {
let v = c.evaluate(response);
assert!(
(0.0..=1.0).contains(&v.error),
"e out of range for {response:?}: {}",
v.error
);
}
}
#[test]
fn an_empty_constraint_set_is_refused_not_scored_as_perfect() {
let err = ConstraintSet::new(vec![]).unwrap_err();
assert_eq!(
err,
ValidatorError::NoCheckableConstraints { clauses: vec![] }
);
assert!(
err.to_string().contains("no response could ever violate it"),
"the refusal must say WHY, got: {err}"
);
}
#[test]
fn a_set_of_only_unlowerable_clauses_is_refused() {
let err = ConstraintSet::new(vec![
Clause::Uncheckable("output must feel authoritative".into()),
Clause::Uncheckable("output must be insightful".into()),
])
.unwrap_err();
match err {
ValidatorError::NoCheckableConstraints { ref clauses } => {
assert_eq!(clauses.len(), 2, "the refusal names the clauses it could not lower");
}
other => panic!("expected NoCheckableConstraints, got {other:?}"),
}
}
#[test]
fn one_checkable_clause_among_unlowerable_ones_is_accepted_but_reports_the_gap() {
let c = ConstraintSet::new(vec![
Clause::Uncheckable("output must feel authoritative".into()),
checkable(
"c2",
Predicate::Absent {
needle: "guarantee".into(),
},
),
])
.expect("one checkable constraint is enough to define CSR");
assert_eq!(c.len(), 1, "|C| counts ONLY checkable constraints");
assert_eq!(
c.unchecked(),
["output must feel authoritative"],
"the unenforced clause stays visible instead of counting as satisfied"
);
assert_eq!(
c.evaluate("no forbidden word here").error,
0.0,
"an unenforced clause must not keep e above zero forever; if it entered |C| the mandate could never converge and would always exhaust its step budget"
);
assert_eq!(c.evaluate("we guarantee results").error, 1.0);
}
#[test]
fn an_invalid_regex_is_refused_at_construction() {
let err = ConstraintSet::new(vec![checkable(
"c1",
Predicate::Matches {
pattern: "([unclosed".into(),
},
)])
.unwrap_err();
match err {
ValidatorError::InvalidPattern { id, .. } => assert_eq!(id, "c1"),
other => panic!("expected InvalidPattern, got {other:?}"),
}
}
#[test]
fn an_invalid_regex_reached_at_runtime_counts_as_violated_never_as_satisfied() {
let p = Predicate::Matches {
pattern: "([unclosed".into(),
};
assert!(p.holds("anything at all", "anything at all").is_err());
}
#[test]
fn text_predicates_ignore_case_but_regex_does_not() {
let c = set(vec![Predicate::Contains {
needle: "Safe Harbor".into(),
}]);
assert!(c.evaluate("...SAFE HARBOR...").is_satisfied());
let re = set(vec![Predicate::Matches {
pattern: "Safe Harbor".into(),
}]);
assert!(!re.evaluate("...SAFE HARBOR...").is_satisfied());
assert!(re.evaluate("...Safe Harbor...").is_satisfied());
}
#[test]
fn word_count_bounds_are_inclusive() {
let c = set(vec![Predicate::WordCount {
min: Some(2),
max: Some(4),
}]);
assert!(!c.evaluate("one").is_satisfied());
assert!(c.evaluate("one two").is_satisfied());
assert!(c.evaluate("one two three four").is_satisfied());
assert!(!c.evaluate("one two three four five").is_satisfied());
}
#[test]
fn parses_as_json_tolerates_surrounding_whitespace_only() {
let c = set(vec![Predicate::ParsesAsJson]);
assert!(c.evaluate(" {\"a\": 1}\n").is_satisfied());
assert!(!c.evaluate("Here is your JSON: {\"a\": 1}").is_satisfied());
}
#[test]
fn required_with_is_an_implication_not_a_conjunction() {
let c = set(vec![Predicate::RequiredWith {
trigger: "we expect".into(),
required: "safe harbor".into(),
}]);
assert!(
c.evaluate("purely historical results").is_satisfied(),
"no trigger means no obligation"
);
assert!(!c.evaluate("we expect growth").is_satisfied());
assert!(c
.evaluate("we expect growth; see safe harbor statement")
.is_satisfied());
}
#[test]
fn the_readme_sec_constraint_lowers_partly_and_says_which_part() {
let clauses = ConstraintSet::lower(
"Output must be a valid SEC 10-K section with GAAP-compliant financial \
tables, footnote references, and no forward-looking statements without \
safe harbor language",
);
let checkable_count = clauses
.iter()
.filter(|c| matches!(c, Clause::Checkable(_)))
.count();
assert_eq!(
checkable_count, 1,
"exactly the 'no X without Y' clause is decidable; the rest is not, \
and must not be pretended into C"
);
let c = ConstraintSet::new(clauses).expect("one checkable clause suffices");
assert!(
matches!(
c.constraints()[0].predicate,
Predicate::RequiredWith { .. }
),
"got {:?}",
c.constraints()[0].predicate
);
assert_eq!(c.unchecked().len(), 1, "the prose half stays visible");
}
#[test]
fn lowering_recognises_each_form_in_the_closed_subset() {
let cases: Vec<(&str, Predicate)> = vec![
(
"output must contain \"safe harbor\"",
Predicate::Contains {
needle: "safe harbor".into(),
},
),
(
"must not contain 'guarantee'",
Predicate::Absent {
needle: "guarantee".into(),
},
),
(
"the response must include a citation",
Predicate::Contains {
needle: "a citation".into(),
},
),
("output must be valid JSON", Predicate::ParsesAsJson),
(
"must be at most 200 words",
Predicate::WordCount {
min: None,
max: Some(200),
},
),
(
"must be at least 50 words",
Predicate::WordCount {
min: Some(50),
max: None,
},
),
(
"must match /^SECTION [0-9]+/",
Predicate::Matches {
pattern: "^section [0-9]+".into(),
},
),
(
"no projections unless a disclaimer is present",
Predicate::RequiredWith {
trigger: "projections".into(),
required: "a disclaimer is present".into(),
},
),
];
for (text, expected) in cases {
let lowered = ConstraintSet::lower(text);
assert_eq!(lowered.len(), 1, "one clause expected from {text:?}");
match &lowered[0] {
Clause::Checkable(c) => assert_eq!(c.predicate, expected, "for {text:?}"),
Clause::Uncheckable(t) => panic!("{text:?} should lower, got Uncheckable({t:?})"),
}
}
}
#[test]
fn negation_is_never_eaten_by_the_positive_prefix() {
let lowered = ConstraintSet::lower("must not contain \"guarantee\"");
match &lowered[0] {
Clause::Checkable(c) => assert_eq!(
c.predicate,
Predicate::Absent {
needle: "guarantee".into()
},
"a negated clause lowered to its own opposite"
),
other => panic!("expected Checkable, got {other:?}"),
}
}
#[test]
fn prose_that_cannot_be_decided_is_reported_not_guessed() {
for text in [
"output must be insightful",
"the response should feel professional",
"must be accurate and well-reasoned",
"",
] {
for clause in ConstraintSet::lower(text) {
assert!(
matches!(clause, Clause::Uncheckable(_)),
"{text:?} was guessed into a predicate: {clause:?}"
);
}
}
}
#[test]
fn a_bare_comma_does_not_split_a_single_obligation() {
let clauses = ConstraintSet::lower(
"output must contain \"GAAP tables, footnote references\"",
);
assert_eq!(clauses.len(), 1, "a plain comma is not a clause separator");
}
#[test]
fn semicolons_and_and_separate_clauses() {
let clauses = ConstraintSet::lower(
"must contain \"alpha\"; must not contain \"beta\", and must be valid JSON",
);
assert_eq!(clauses.len(), 3);
let c = ConstraintSet::new(clauses).unwrap();
assert_eq!(c.len(), 3);
}
#[test]
fn feedback_names_the_obligation_and_the_reason_for_every_violation() {
let v = set(vec![
Predicate::Contains {
needle: "safe harbor".into(),
},
Predicate::Absent {
needle: "guaranteed".into(),
},
])
.evaluate("returns are guaranteed");
let fb = v.feedback();
assert!(fb.contains("must contain \"safe harbor\""), "{fb}");
assert!(fb.contains("must not contain \"guaranteed\""), "{fb}");
assert_eq!(fb.lines().count(), 2);
}
#[test]
fn a_satisfied_verdict_has_nothing_to_feed_back() {
let v = set(vec![Predicate::Contains {
needle: "alpha".into(),
}])
.evaluate("alpha");
assert_eq!(v.feedback(), "");
}
#[test]
fn within_is_a_strict_band_matching_the_published_convergence_test() {
let v = set(vec![
Predicate::Contains {
needle: "a".into(),
},
Predicate::Contains {
needle: "b".into(),
},
])
.evaluate("a only");
assert_eq!(v.error, 0.5);
assert!(!v.within(0.5), "|e| < ε is strict");
assert!(v.within(0.51));
}
}