use std::collections::HashMap;
use std::path::Path;
use serde::Deserialize;
#[derive(Debug, Clone, Deserialize, PartialEq)]
#[serde(untagged)]
pub enum StringMatch {
Substring(String),
Regex(RegexMatch),
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct RegexMatch {
pub regex: String,
#[serde(skip)]
compiled: Option<regex::Regex>,
}
impl PartialEq for RegexMatch {
fn eq(&self, other: &Self) -> bool {
self.regex == other.regex
}
}
impl RegexMatch {
fn compile(&mut self) -> Result<(), String> {
if self.compiled.is_some() {
return Ok(()); }
let re = regex::RegexBuilder::new(&self.regex)
.size_limit(1 << 20)
.dfa_size_limit(1 << 20) .build()
.map_err(|e| format!("Invalid regex '{}': {}", self.regex, e))?;
self.compiled = Some(re);
Ok(())
}
fn is_match(&self, haystack: &str) -> bool {
match &self.compiled {
Some(re) => re.is_match(haystack),
None => {
match regex::RegexBuilder::new(&self.regex)
.size_limit(1 << 20)
.dfa_size_limit(1 << 20)
.build()
{
Ok(re) => re.is_match(haystack),
Err(e) => {
eprintln!("[llmposter] Warning: invalid regex '{}': {}", self.regex, e);
false
}
}
}
}
}
}
impl StringMatch {
pub fn regex(pattern: &str) -> Self {
StringMatch::Regex(RegexMatch {
regex: pattern.to_string(),
compiled: None,
})
}
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct F64Range {
#[serde(default)]
pub min: Option<f64>,
#[serde(default)]
pub max: Option<f64>,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(untagged)]
pub enum F64Match {
Exact(f64),
Range(F64Range),
}
#[derive(Debug, Clone, Deserialize, Default)]
#[serde(deny_unknown_fields)]
pub struct FixtureMatch {
pub user_message: Option<StringMatch>,
pub model: Option<StringMatch>,
#[serde(default)]
pub headers: std::collections::HashMap<String, StringMatch>,
pub system_prompt: Option<StringMatch>,
pub temperature: Option<F64Match>,
#[serde(default)]
pub metadata: std::collections::HashMap<String, StringMatch>,
pub tool_schema: Option<StringMatch>,
pub body_jsonpath: Option<String>,
#[cfg(feature = "jsonpath")]
#[serde(skip)]
body_jsonpath_compiled: Option<jsonpath_rust::parser::model::JpQuery>,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ToolCall {
pub name: String,
pub arguments: serde_json::Value,
}
#[cfg(feature = "templating")]
#[derive(Default)]
pub struct TemplateCache {
cell: std::sync::OnceLock<Result<std::sync::Arc<minijinja::Environment<'static>>, String>>,
}
#[cfg(feature = "templating")]
impl std::fmt::Debug for TemplateCache {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("TemplateCache")
.field("initialized", &self.cell.get().is_some())
.finish()
}
}
#[cfg(feature = "templating")]
impl Clone for TemplateCache {
fn clone(&self) -> Self {
#[cfg(debug_assertions)]
if self.cell.get().is_some() {
eprintln!(
"[llmposter] Warning: TemplateCache cloned — compile cache defeated. \
This is expected during hot-reload but not on the request path."
);
}
Self::default()
}
}
#[cfg(feature = "templating")]
impl TemplateCache {
pub(crate) fn get_or_compile(
&self,
template_source: &str,
) -> Result<&std::sync::Arc<minijinja::Environment<'static>>, &str> {
let entry = self.cell.get_or_init(|| {
let mut env = minijinja::Environment::new();
env.add_template_owned("t", template_source.to_string())
.map_err(|e| format!("template compile error: {}", e))?;
Ok(std::sync::Arc::new(env))
});
match entry {
Ok(env) => Ok(env),
Err(msg) => Err(msg.as_str()),
}
}
}
#[derive(Debug, Clone, Deserialize, Default)]
#[serde(deny_unknown_fields)]
pub struct FixtureResponse {
pub content: Option<String>,
pub content_template: Option<String>,
pub tool_calls: Option<Vec<ToolCall>>,
pub stop_reason: Option<String>,
pub finish_reason: Option<String>,
#[cfg(feature = "templating")]
#[serde(skip)]
#[doc(hidden)]
pub template_cache: TemplateCache,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct FixtureError {
pub status: u16,
pub message: String,
#[serde(default)]
pub headers: HashMap<String, String>,
}
#[derive(Debug, Clone, Deserialize, Default)]
#[serde(deny_unknown_fields)]
pub struct FailureConfig {
pub latency_ms: Option<u64>,
pub corrupt_body: Option<bool>,
#[serde(alias = "truncate_after_chunks")]
pub truncate_after_frames: Option<u32>,
pub disconnect_after_ms: Option<u64>,
pub latency_jitter_ms: Option<u64>,
pub duplicate_frames: Option<bool>,
pub probability: Option<f32>,
pub chaos_seed: Option<u64>,
}
impl FailureConfig {
pub(crate) fn has_chaos(&self) -> bool {
self.latency_jitter_ms.is_some()
|| self.duplicate_frames.is_some()
|| self.probability.is_some()
|| self.chaos_seed.is_some()
}
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct StreamingConfig {
pub latency: Option<u64>,
pub chunk_size: Option<usize>,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ScenarioConfig {
pub name: String,
pub required_state: Option<String>,
pub set_state: Option<String>,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Refusal {
pub reason: String,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Fixture {
#[serde(rename = "match")]
pub match_rule: Option<FixtureMatch>,
pub provider: Option<crate::format::Provider>,
pub response: Option<FixtureResponse>,
pub error: Option<FixtureError>,
pub refusal: Option<Refusal>,
pub failure: Option<FailureConfig>,
pub streaming: Option<StreamingConfig>,
pub scenario: Option<ScenarioConfig>,
#[serde(default)]
pub priority: Option<i32>,
#[serde(default)]
pub catch_all: bool,
}
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct FixtureFile {
pub fixtures: Vec<Fixture>,
}
impl Fixture {
pub fn new() -> Self {
Self {
match_rule: None,
provider: None,
response: None,
error: None,
refusal: None,
failure: None,
streaming: None,
scenario: None,
priority: None,
catch_all: false,
}
}
pub fn with_priority(mut self, priority: i32) -> Self {
self.priority = Some(priority);
self
}
pub fn as_catch_all(mut self) -> Self {
self.catch_all = true;
self
}
pub fn respond_with_refusal(mut self, reason: &str) -> Self {
self.refusal = Some(Refusal {
reason: reason.to_string(),
});
self
}
pub fn match_user_message(mut self, pattern: &str) -> Self {
let m = self.match_rule.get_or_insert_with(FixtureMatch::default);
m.user_message = Some(StringMatch::Substring(pattern.to_string()));
self
}
pub fn match_model(mut self, pattern: &str) -> Self {
let m = self.match_rule.get_or_insert_with(FixtureMatch::default);
m.model = Some(StringMatch::Substring(pattern.to_string()));
self
}
pub fn match_header(mut self, name: &str, value: &str) -> Self {
let m = self.match_rule.get_or_insert_with(FixtureMatch::default);
m.headers.insert(
name.to_ascii_lowercase(),
StringMatch::Substring(value.to_string()),
);
self
}
pub fn match_system_prompt(mut self, pattern: &str) -> Self {
let m = self.match_rule.get_or_insert_with(FixtureMatch::default);
m.system_prompt = Some(StringMatch::Substring(pattern.to_string()));
self
}
pub fn match_temperature(mut self, value: f64) -> Self {
let m = self.match_rule.get_or_insert_with(FixtureMatch::default);
m.temperature = Some(F64Match::Exact(value));
self
}
pub fn match_temperature_range(mut self, min: Option<f64>, max: Option<f64>) -> Self {
let m = self.match_rule.get_or_insert_with(FixtureMatch::default);
m.temperature = Some(F64Match::Range(F64Range { min, max }));
self
}
pub fn match_metadata(mut self, key: &str, value: &str) -> Self {
let m = self.match_rule.get_or_insert_with(FixtureMatch::default);
m.metadata
.insert(key.to_string(), StringMatch::Substring(value.to_string()));
self
}
pub fn match_tool_schema(mut self, pattern: &str) -> Self {
let m = self.match_rule.get_or_insert_with(FixtureMatch::default);
m.tool_schema = Some(StringMatch::Substring(pattern.to_string()));
self
}
#[cfg(feature = "jsonpath")]
pub fn match_body_jsonpath(mut self, path: &str) -> Self {
let m = self.match_rule.get_or_insert_with(FixtureMatch::default);
m.body_jsonpath = Some(path.to_string());
self
}
pub fn respond_with_content(mut self, content: &str) -> Self {
let r = self.response.get_or_insert(FixtureResponse::default());
r.content = Some(content.to_string());
r.tool_calls = None;
self
}
pub fn with_error(mut self, status: u16, message: &str) -> Self {
self.error = Some(FixtureError {
status,
message: message.to_string(),
headers: HashMap::new(),
});
self
}
pub fn with_error_headers<I, K, V>(
mut self,
status: u16,
message: &str,
headers: I,
) -> Result<Self, String>
where
I: IntoIterator<Item = (K, V)>,
K: AsRef<str>,
V: AsRef<str>,
{
use axum::http::{HeaderName, HeaderValue};
use std::str::FromStr;
let mut map = HashMap::new();
for (k, v) in headers {
HeaderName::from_str(k.as_ref())
.map_err(|e| format!("invalid header name {:?}: {e}", k.as_ref()))?;
HeaderValue::from_str(v.as_ref())
.map_err(|e| format!("invalid header value {:?}: {e}", v.as_ref()))?;
let lower = k.as_ref().to_ascii_lowercase();
if map.contains_key(&lower) {
return Err(format!(
"duplicate header name (case-insensitive): {lower:?}"
));
}
map.insert(lower, v.as_ref().to_string());
}
self.error = Some(FixtureError {
status,
message: message.to_string(),
headers: map,
});
Ok(self)
}
pub fn with_failure(mut self, failure: FailureConfig) -> Self {
self.failure = Some(failure);
self
}
pub fn with_stop_reason(mut self, reason: &str) -> Self {
self.response
.get_or_insert(FixtureResponse::default())
.stop_reason = Some(reason.to_string());
self
}
pub fn with_finish_reason(mut self, reason: &str) -> Self {
self.response
.get_or_insert(FixtureResponse::default())
.finish_reason = Some(reason.to_string());
self
}
pub fn with_streaming(mut self, latency: Option<u64>, chunk_size: Option<usize>) -> Self {
self.streaming = Some(StreamingConfig {
latency,
chunk_size,
});
self
}
pub fn with_scenario(
mut self,
name: &str,
required_state: Option<&str>,
set_state: Option<&str>,
) -> Self {
self.scenario = Some(ScenarioConfig {
name: name.to_string(),
required_state: required_state.map(|s| s.to_string()),
set_state: set_state.map(|s| s.to_string()),
});
self
}
pub fn for_provider(mut self, provider: crate::format::Provider) -> Self {
self.provider = Some(provider);
self
}
pub fn respond_with_tool_calls(mut self, tool_calls: Vec<ToolCall>) -> Self {
let r = self.response.get_or_insert(FixtureResponse::default());
r.tool_calls = Some(tool_calls);
r.content = None;
self
}
}
impl Default for Fixture {
fn default() -> Self {
Self::new()
}
}
impl Fixture {
pub fn validate(&mut self) -> Result<(), String> {
if let Some(ref e) = self.error {
if !(400..=599).contains(&e.status) {
return Err("error.status must be an error HTTP status (400-599)".to_string());
}
use axum::http::{HeaderName, HeaderValue};
use std::str::FromStr;
for (name, value) in &e.headers {
HeaderName::from_str(name)
.map_err(|err| format!("invalid error header name {name:?}: {err}"))?;
HeaderValue::from_str(value)
.map_err(|err| format!("invalid error header value {value:?}: {err}"))?;
}
}
if let Some(ref mut e) = self.error {
let mut normalized: HashMap<String, String> = HashMap::new();
for (k, v) in e.headers.drain() {
let lower = k.to_ascii_lowercase();
if normalized.contains_key(&lower) {
return Err(format!(
"duplicate error header name (case-insensitive): {lower:?}"
));
}
normalized.insert(lower, v);
}
e.headers = normalized;
}
if self.response.is_some() && self.error.is_some() {
return Err("'error' and 'response' are mutually exclusive".to_string());
}
if self.error.is_some() && self.failure.is_some() {
return Err("'error' and 'failure' are mutually exclusive".to_string());
}
if self.refusal.is_some() && self.response.is_some() {
return Err("'refusal' and 'response' are mutually exclusive".to_string());
}
if self.refusal.is_some() && self.error.is_some() {
return Err("'refusal' and 'error' are mutually exclusive".to_string());
}
if self.refusal.is_some() && self.failure.is_some() {
return Err("'refusal' and 'failure' are mutually exclusive".to_string());
}
if self.refusal.is_some() && self.streaming.is_some() {
return Err("'refusal' and 'streaming' are mutually exclusive".to_string());
}
if let Some(ref r) = self.refusal {
if r.reason.trim().is_empty() {
return Err("refusal.reason must not be blank".to_string());
}
}
if self.failure.is_some() && self.response.is_none() {
return Err("'failure' requires response to also be present".to_string());
}
if let (Some(ref f), None) = (&self.failure, &self.streaming) {
let has_stream_failure =
f.truncate_after_frames.is_some() || f.disconnect_after_ms.is_some();
if has_stream_failure {
eprintln!(
"[llmposter] Warning: failure.truncate_after_frames/disconnect_after_ms \
have no effect without streaming configured"
);
}
if f.duplicate_frames == Some(true) {
eprintln!(
"[llmposter] Warning: failure.duplicate_frames has no effect \
without streaming configured"
);
}
}
if let Some(ref f) = self.failure {
if let Some(p) = f.probability {
if !p.is_finite() || !(0.0..=1.0).contains(&p) {
return Err(format!(
"failure.probability must be a finite number in [0.0, 1.0], got {}",
p
));
}
}
if let Some(jitter) = f.latency_jitter_ms {
if jitter > 0 {
let base_latency = self.streaming.as_ref().and_then(|s| s.latency).unwrap_or(0);
if base_latency == 0 {
return Err(
"failure.latency_jitter_ms requires a non-zero streaming.latency"
.to_string(),
);
}
const MAX_JITTER_MS: u64 = 60 * 60 * 1000;
if jitter > MAX_JITTER_MS {
return Err(format!(
"failure.latency_jitter_ms must be <= {} (got {})",
MAX_JITTER_MS, jitter
));
}
}
}
let has_effect_field = f.latency_jitter_ms.map(|j| j > 0).unwrap_or(false)
|| f.duplicate_frames == Some(true);
let has_gate_field = f.chaos_seed.is_some() || f.probability.is_some();
if has_gate_field && !has_effect_field {
eprintln!(
"[llmposter] Warning: failure.chaos_seed/probability set without \
latency_jitter_ms or duplicate_frames — chaos fields have no \
observable effect"
);
}
}
if let Some(ref r) = self.response {
#[cfg(not(feature = "templating"))]
if r.content_template.is_some() {
return Err(
"'content_template' requires the 'templating' feature — rebuild with \
`--features templating` to enable it"
.to_string(),
);
}
if r.content.is_some() && r.content_template.is_some() {
return Err(
"'content' and 'content_template' in response are mutually exclusive"
.to_string(),
);
}
if r.content_template.is_some() && r.tool_calls.is_some() {
return Err(
"'content_template' and 'tool_calls' in response are mutually exclusive"
.to_string(),
);
}
#[cfg(feature = "templating")]
if let Some(ref tmpl) = r.content_template {
let mut env = minijinja::Environment::new();
if let Err(e) = env.add_template_owned("t", tmpl.clone()) {
return Err(format!("content_template compile error: {}", e));
}
}
if r.content.is_some() && r.tool_calls.is_some() {
return Err(
"'content' and 'tool_calls' in response are mutually exclusive".to_string(),
);
}
if r.content.is_none() && r.tool_calls.is_none() && r.content_template.is_none() {
return Err(
"response must have either 'content', 'content_template', or 'tool_calls'"
.to_string(),
);
}
if let Some(ref tc) = r.tool_calls {
if tc.is_empty() {
return Err("tool_calls must not be empty".to_string());
}
for (i, call) in tc.iter().enumerate() {
if call.name.trim().is_empty() {
return Err(format!("tool_calls[{}].name must not be empty", i));
}
if !call.arguments.is_object() {
return Err(format!(
"tool_calls[{}].arguments must be a JSON object, got {}",
i,
match &call.arguments {
serde_json::Value::Array(_) => "array",
serde_json::Value::String(_) => "string",
serde_json::Value::Number(_) => "number",
serde_json::Value::Bool(_) => "boolean",
serde_json::Value::Null => "null",
_ => "non-object",
}
));
}
}
}
}
if self.response.is_none() && self.error.is_none() && self.refusal.is_none() {
return Err("Fixture must have either 'response', 'error', or 'refusal'".to_string());
}
if let Some(ref s) = self.streaming {
if s.chunk_size == Some(0) {
return Err("streaming.chunk_size must be > 0".to_string());
}
if self.error.is_some() {
return Err("'streaming' config has no effect on error-only fixtures".to_string());
}
}
if let Some(ref mut m) = self.match_rule {
validate_string_match_field(&mut m.user_message, "user_message")?;
validate_string_match_field(&mut m.model, "model")?;
validate_string_match_field(&mut m.system_prompt, "system_prompt")?;
validate_string_match_field(&mut m.tool_schema, "tool_schema")?;
for (name, pattern) in m.headers.iter_mut() {
if name.trim().is_empty() {
return Err("match.headers: header name must not be blank".to_string());
}
if !name.bytes().all(|b| {
matches!(b,
b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9' |
b'!' | b'#' | b'$' | b'%' | b'&' | b'\'' | b'*' |
b'+' | b'-' | b'.' | b'^' | b'_' | b'`' | b'|' | b'~')
}) {
return Err(format!(
"match.headers: '{}' is not a valid HTTP header name \
(must be RFC 7230 token characters)",
name
));
}
validate_string_match(pattern, &format!("headers[{}]", name))?;
}
for (key, pattern) in m.metadata.iter_mut() {
if key.trim().is_empty() {
return Err("match.metadata: key must not be blank".to_string());
}
validate_string_match(pattern, &format!("metadata[{}]", key))?;
}
#[cfg(not(feature = "jsonpath"))]
if m.body_jsonpath.is_some() {
return Err(
"'match.body_jsonpath' requires the 'jsonpath' feature — rebuild with \
`--features jsonpath` to enable it"
.to_string(),
);
}
#[cfg(feature = "jsonpath")]
{
m.body_jsonpath_compiled = None;
if let Some(ref path) = m.body_jsonpath {
if path.trim().is_empty() {
return Err("match.body_jsonpath must not be empty".to_string());
}
match jsonpath_rust::parser::parse_json_path(path) {
Ok(q) => m.body_jsonpath_compiled = Some(q),
Err(e) => {
return Err(format!("match.body_jsonpath is invalid: {}", e));
}
}
}
}
if !m.headers.is_empty() {
let raw = std::mem::take(&mut m.headers);
let mut normalized: std::collections::HashMap<String, StringMatch> =
std::collections::HashMap::with_capacity(raw.len());
let mut origins: std::collections::HashMap<String, String> =
std::collections::HashMap::with_capacity(raw.len());
for (name, pattern) in raw {
let key = name.to_ascii_lowercase();
if let Some(prior) = origins.get(&key) {
return Err(format!(
"match.headers: duplicate header name after case-folding: \
'{}' and '{}' both normalize to '{}'",
prior, name, key
));
}
origins.insert(key.clone(), name);
normalized.insert(key, pattern);
}
m.headers = normalized;
}
if let Some(ref tm) = m.temperature {
match tm {
F64Match::Exact(v) => {
if !v.is_finite() {
return Err(format!(
"match.temperature must be a finite number, got {}",
v
));
}
}
F64Match::Range(r) => {
if let Some(min) = r.min {
if !min.is_finite() {
return Err("match.temperature.min must be finite".to_string());
}
}
if let Some(max) = r.max {
if !max.is_finite() {
return Err("match.temperature.max must be finite".to_string());
}
}
if let (Some(min), Some(max)) = (r.min, r.max) {
if min > max {
return Err(format!(
"match.temperature range inverted: min={} > max={}",
min, max
));
}
}
if r.min.is_none() && r.max.is_none() {
return Err("match.temperature range must set at least one of min/max"
.to_string());
}
}
}
}
}
Ok(())
}
}
fn validate_string_match_field(field: &mut Option<StringMatch>, name: &str) -> Result<(), String> {
if let Some(pattern) = field {
validate_string_match(pattern, name)?;
}
Ok(())
}
fn validate_string_match(pattern: &mut StringMatch, name: &str) -> Result<(), String> {
match pattern {
StringMatch::Substring(s) => {
if s.is_empty() {
return Err(format!("match.{} substring must not be empty", name));
}
}
StringMatch::Regex(r) => {
if r.regex.is_empty() {
return Err(format!("match.{} regex must not be empty", name));
}
r.compile().map_err(|e| format!("{} {}", name, e))?;
}
}
Ok(())
}
#[doc(hidden)]
pub fn match_fixture<'a>(
fixtures: &'a [Fixture],
user_message: &str,
model: Option<&str>,
provider: Option<crate::format::Provider>,
scenario_states: Option<&std::collections::HashMap<String, String>>,
) -> Option<&'a Fixture> {
for f in fixtures {
let has_body_fields = f.match_rule.as_ref().is_some_and(|m| {
!m.headers.is_empty()
|| m.system_prompt.is_some()
|| m.temperature.is_some()
|| !m.metadata.is_empty()
|| m.tool_schema.is_some()
|| m.body_jsonpath.is_some()
});
let has_ordering = f.priority.is_some() || f.catch_all;
if has_body_fields || has_ordering {
eprintln!(
"[llmposter] Warning: match_fixture() cannot honor \
v0.4.6 features (priority / catch_all / headers / \
system_prompt / temperature / metadata / tool_schema / \
body_jsonpath) — this legacy helper uses first-match \
order and a fabricated empty request. Drive the server \
through ServerBuilder for full match semantics."
);
break;
}
}
let empty_headers = std::collections::HashMap::new();
let empty_body = serde_json::Value::Null;
let ctx = MatchContext::new(
user_message,
model,
provider,
scenario_states,
&empty_headers,
&empty_body,
);
fixtures.iter().find(|f| fixture_matches(f, &ctx))
}
pub(crate) struct MatchContext<'a> {
pub user_message: &'a str,
pub model: Option<&'a str>,
pub provider: Option<crate::format::Provider>,
pub scenario_states: Option<&'a std::collections::HashMap<String, String>>,
pub headers: &'a std::collections::HashMap<String, String>,
pub body: &'a serde_json::Value,
system_prompt_cache: std::cell::OnceCell<Option<String>>,
tool_names_cache: std::cell::OnceCell<Vec<&'a str>>,
}
impl<'a> MatchContext<'a> {
pub fn new(
user_message: &'a str,
model: Option<&'a str>,
provider: Option<crate::format::Provider>,
scenario_states: Option<&'a std::collections::HashMap<String, String>>,
headers: &'a std::collections::HashMap<String, String>,
body: &'a serde_json::Value,
) -> Self {
Self {
user_message,
model,
provider,
scenario_states,
headers,
body,
system_prompt_cache: std::cell::OnceCell::new(),
tool_names_cache: std::cell::OnceCell::new(),
}
}
pub(crate) fn system_prompt(&self) -> Option<&str> {
self.system_prompt_cache
.get_or_init(|| extract_system_prompt(self.body, self.provider))
.as_deref()
}
pub(crate) fn tool_names(&self) -> &[&'a str] {
self.tool_names_cache
.get_or_init(|| extract_tool_names(self.body, self.provider))
}
}
pub(crate) fn fixture_matches(fixture: &Fixture, ctx: &MatchContext<'_>) -> bool {
if let Some(fp) = fixture.provider {
match ctx.provider {
Some(p) if p == fp => {}
_ => return false,
}
}
if let Some(ref scenario) = fixture.scenario {
if let Some(ref required) = scenario.required_state {
let current = ctx
.scenario_states
.and_then(|states| states.get(&scenario.name))
.map(|s| s.as_str())
.unwrap_or("");
if current != required {
return false;
}
}
}
let Some(m) = fixture.match_rule.as_ref() else {
return true;
};
if let Some(ref um) = m.user_message {
if !string_matches(um, ctx.user_message) {
return false;
}
}
if let Some(ref mm) = m.model {
match ctx.model {
Some(model) => {
if !string_matches(mm, model) {
return false;
}
}
None => return false,
}
}
for (name, pattern) in &m.headers {
match ctx.headers.get(name) {
Some(value) => {
if !string_matches(pattern, value) {
return false;
}
}
None => return false,
}
}
if let Some(ref sp) = m.system_prompt {
match ctx.system_prompt() {
Some(text) => {
if !string_matches(sp, text) {
return false;
}
}
None => return false,
}
}
if let Some(ref tm) = m.temperature {
let temp = extract_temperature(ctx.body, ctx.provider);
match temp {
Some(t) => {
if !f64_matches(tm, t) {
return false;
}
}
None => return false,
}
}
if !m.metadata.is_empty() {
let Some(metadata) = ctx.body.get("metadata").and_then(|v| v.as_object()) else {
return false;
};
for (key, pattern) in &m.metadata {
let value_str: Option<std::borrow::Cow<str>> =
metadata.get(key).and_then(|v| match v {
serde_json::Value::String(s) => Some(std::borrow::Cow::Borrowed(s.as_str())),
serde_json::Value::Number(n) => Some(std::borrow::Cow::Owned(n.to_string())),
serde_json::Value::Bool(b) => Some(std::borrow::Cow::Owned(b.to_string())),
_ => None,
});
match value_str {
Some(value) => {
if !string_matches(pattern, &value) {
return false;
}
}
None => return false,
}
}
}
if let Some(ref ts) = m.tool_schema {
let names = ctx.tool_names();
if !names.iter().any(|name| string_matches(ts, name)) {
return false;
}
}
#[cfg(feature = "jsonpath")]
if let Some(ref compiled) = m.body_jsonpath_compiled {
match jsonpath_rust::query::js_path_process(compiled, ctx.body) {
Ok(matches) => {
if matches.is_empty() || matches.into_iter().all(|q| q.val().is_null()) {
return false;
}
}
Err(_) => {
return false;
}
}
} else if let Some(ref path_str) = m.body_jsonpath {
match jsonpath_rust::parser::parse_json_path(path_str) {
Ok(query) => match jsonpath_rust::query::js_path_process(&query, ctx.body) {
Ok(matches) => {
if matches.is_empty() || matches.into_iter().all(|q| q.val().is_null()) {
return false;
}
}
Err(_) => return false,
},
Err(e) => {
eprintln!(
"[llmposter] Warning: invalid body_jsonpath '{}': {}",
path_str, e
);
return false;
}
}
}
true
}
fn extract_system_prompt(
body: &serde_json::Value,
provider: Option<crate::format::Provider>,
) -> Option<String> {
use crate::format::Provider;
if provider == Some(Provider::Anthropic) {
if let Some(s) = body.get("system") {
if let Some(text) = s.as_str() {
return Some(text.to_string());
}
if let Some(arr) = s.as_array() {
let parts: Vec<&str> = arr
.iter()
.filter(|block| block.get("type").and_then(|v| v.as_str()) == Some("text"))
.filter_map(|block| block.get("text").and_then(|v| v.as_str()))
.collect();
if !parts.is_empty() {
return Some(parts.join("\n"));
}
}
}
return None;
}
if provider == Some(Provider::Gemini) {
if let Some(parts) = body
.get("systemInstruction")
.and_then(|v| v.get("parts"))
.and_then(|v| v.as_array())
{
let texts: Vec<&str> = parts
.iter()
.filter_map(|p| p.get("text").and_then(|v| v.as_str()))
.collect();
if !texts.is_empty() {
return Some(texts.join("\n"));
}
}
return None;
}
if provider == Some(Provider::Responses) {
if let Some(text) = body.get("instructions").and_then(|v| v.as_str()) {
return Some(text.to_string());
}
}
let array_key = match provider {
Some(Provider::Responses) => "input",
_ => "messages",
};
if let Some(arr) = body.get(array_key).and_then(|v| v.as_array()) {
let parts: Vec<String> = arr
.iter()
.filter(|m| m.get("role").and_then(|v| v.as_str()) == Some("system"))
.filter_map(|m| {
let content = m.get("content")?;
if let Some(s) = content.as_str() {
return Some(s.to_string());
}
if let Some(arr) = content.as_array() {
let texts: Vec<&str> = arr
.iter()
.filter_map(|part| part.get("text").and_then(|v| v.as_str()))
.collect();
if !texts.is_empty() {
return Some(texts.join("\n"));
}
}
None
})
.collect();
if !parts.is_empty() {
return Some(parts.join("\n"));
}
}
None
}
fn extract_tool_names(
body: &serde_json::Value,
provider: Option<crate::format::Provider>,
) -> Vec<&str> {
use crate::format::Provider;
let tools = match body.get("tools").and_then(|v| v.as_array()) {
Some(t) => t,
None => return Vec::new(),
};
let mut out: Vec<&str> = Vec::new();
for tool in tools {
match provider {
Some(Provider::Gemini) => {
if let Some(decls) = tool.get("functionDeclarations").and_then(|v| v.as_array()) {
for decl in decls {
if let Some(name) = decl.get("name").and_then(|v| v.as_str()) {
out.push(name);
}
}
}
}
Some(Provider::Anthropic) => {
if let Some(name) = tool.get("name").and_then(|v| v.as_str()) {
out.push(name);
}
}
_ => {
if let Some(name) = tool
.get("function")
.and_then(|v| v.get("name"))
.and_then(|v| v.as_str())
{
out.push(name);
} else if let Some(name) = tool.get("name").and_then(|v| v.as_str()) {
out.push(name);
}
}
}
}
out
}
#[cfg(feature = "ui")]
pub(crate) fn extract_temperature_for_debug(
body: &serde_json::Value,
provider: Option<crate::format::Provider>,
) -> Option<f64> {
extract_temperature(body, provider)
}
fn extract_temperature(
body: &serde_json::Value,
provider: Option<crate::format::Provider>,
) -> Option<f64> {
if provider == Some(crate::format::Provider::Gemini) {
return body
.get("generationConfig")
.and_then(|v| v.get("temperature"))
.and_then(|v| v.as_f64());
}
body.get("temperature").and_then(|v| v.as_f64())
}
fn f64_matches(pattern: &F64Match, value: f64) -> bool {
match pattern {
F64Match::Exact(target) => value == *target,
F64Match::Range(range) => {
if let Some(min) = range.min {
if value < min {
return false;
}
}
if let Some(max) = range.max {
if value > max {
return false;
}
}
true
}
}
}
pub(crate) fn string_matches(pattern: &StringMatch, haystack: &str) -> bool {
match pattern {
StringMatch::Substring(s) => haystack.contains(s.as_str()),
StringMatch::Regex(r) => r.is_match(haystack),
}
}
pub fn load_yaml_file(path: &Path) -> Result<Vec<Fixture>, Box<dyn std::error::Error>> {
let content = std::fs::read_to_string(path)
.map_err(|e| format!("Failed to read {}: {}", path.display(), e))?;
let file: FixtureFile = serde_yaml_ng::from_str(&content)
.map_err(|e| format!("Invalid YAML in {}: {}", path.display(), e))?;
let mut fixtures = file.fixtures;
for (i, fixture) in fixtures.iter_mut().enumerate() {
fixture
.validate()
.map_err(|e| format!("Fixture #{} in {}: {}", i + 1, path.display(), e))?;
}
Ok(fixtures)
}
pub(crate) fn reload_sources(sources: &[std::path::PathBuf]) -> Result<Vec<Fixture>, String> {
let mut fixtures = Vec::new();
for path in sources {
let loaded = if path.is_dir() {
load_yaml_dir(path).map_err(|e| format!("{}: {}", path.display(), e))?
} else {
load_yaml_file(path).map_err(|e| format!("{}: {}", path.display(), e))?
};
fixtures.extend(loaded);
}
Ok(fixtures)
}
pub fn load_yaml_dir(dir: &Path) -> Result<Vec<Fixture>, Box<dyn std::error::Error>> {
let mut entries: Vec<_> = std::fs::read_dir(dir)
.map_err(|e| format!("Failed to read directory {}: {}", dir.display(), e))?
.collect::<Result<Vec<_>, _>>()
.map_err(|e| format!("Error reading directory entry in {}: {}", dir.display(), e))?
.into_iter()
.filter(|e| {
let is_file = e.file_type().map(|ft| ft.is_file()).unwrap_or(false);
if !is_file {
return false;
}
let name = e.file_name();
let name = name.to_string_lossy();
name.ends_with(".yaml") || name.ends_with(".yml")
})
.collect();
entries.sort_by_key(|e| e.file_name());
let mut all_fixtures = Vec::new();
for entry in entries {
let fixtures = load_yaml_file(&entry.path())?;
all_fixtures.extend(fixtures);
}
Ok(all_fixtures)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn should_parse_simple_text_fixture() {
let yaml = r#"
fixtures:
- match:
user_message: "hello"
response:
content: "Hi there!"
"#;
let file: FixtureFile = serde_yaml_ng::from_str(yaml).unwrap();
assert_eq!(file.fixtures.len(), 1);
let f = &file.fixtures[0];
assert_eq!(
f.match_rule.as_ref().unwrap().user_message,
Some(StringMatch::Substring("hello".to_string()))
);
assert_eq!(
f.response.as_ref().unwrap().content.as_deref(),
Some("Hi there!")
);
}
#[test]
fn should_parse_regex_match() {
let yaml = r#"
fixtures:
- match:
user_message:
regex: "hello \\w+"
response:
content: "matched regex"
"#;
let file: FixtureFile = serde_yaml_ng::from_str(yaml).unwrap();
let f = &file.fixtures[0];
match &f.match_rule.as_ref().unwrap().user_message {
Some(StringMatch::Regex(r)) => assert_eq!(r.regex, "hello \\w+"),
other => panic!("Expected Regex, got {:?}", other),
}
}
#[test]
fn should_parse_error_fixture() {
let yaml = r#"
fixtures:
- match:
model: "fail-model"
error:
status: 429
message: "Rate limit exceeded"
"#;
let file: FixtureFile = serde_yaml_ng::from_str(yaml).unwrap();
let f = &file.fixtures[0];
assert!(f.response.is_none());
let err = f.error.as_ref().unwrap();
assert_eq!(err.status, 429);
assert_eq!(err.message, "Rate limit exceeded");
}
#[test]
fn should_parse_failure_config() {
let yaml = r#"
fixtures:
- match:
user_message: "slow"
response:
content: "delayed"
failure:
latency_ms: 5000
"#;
let file: FixtureFile = serde_yaml_ng::from_str(yaml).unwrap();
let f = &file.fixtures[0];
assert_eq!(f.failure.as_ref().unwrap().latency_ms, Some(5000));
}
#[test]
fn should_parse_streaming_config() {
let yaml = r#"
fixtures:
- match:
user_message: "stream"
response:
content: "streamed"
streaming:
latency: 50
chunk_size: 10
"#;
let file: FixtureFile = serde_yaml_ng::from_str(yaml).unwrap();
let f = &file.fixtures[0];
let s = f.streaming.as_ref().unwrap();
assert_eq!(s.latency, Some(50));
assert_eq!(s.chunk_size, Some(10));
}
#[test]
fn should_parse_tool_call_response() {
let yaml = r#"
fixtures:
- match:
user_message: "weather"
response:
tool_calls:
- name: get_weather
arguments:
location: "San Francisco"
"#;
let file: FixtureFile = serde_yaml_ng::from_str(yaml).unwrap();
let tc = &file.fixtures[0]
.response
.as_ref()
.unwrap()
.tool_calls
.as_ref()
.unwrap()[0];
assert_eq!(tc.name, "get_weather");
assert_eq!(tc.arguments["location"], "San Francisco");
}
#[test]
fn should_parse_provider_specific_fixture() {
let yaml = r#"
fixtures:
- match:
user_message: "test"
provider: anthropic
response:
content: "response"
stop_reason: end_turn
"#;
let file: FixtureFile = serde_yaml_ng::from_str(yaml).unwrap();
let f = &file.fixtures[0];
assert_eq!(f.provider, Some(crate::format::Provider::Anthropic));
}
#[test]
fn should_reject_invalid_yaml() {
let yaml = "not: [valid: yaml: {{{";
let result: Result<FixtureFile, _> = serde_yaml_ng::from_str(yaml);
assert!(result.is_err());
}
#[test]
fn should_parse_model_match() {
let yaml = r#"
fixtures:
- match:
model: "gpt-4"
user_message: "hello"
response:
content: "hi"
"#;
let file: FixtureFile = serde_yaml_ng::from_str(yaml).unwrap();
let m = file.fixtures[0].match_rule.as_ref().unwrap();
assert_eq!(m.model, Some(StringMatch::Substring("gpt-4".to_string())));
}
#[test]
fn should_parse_catch_all_fixture() {
let yaml = r#"
fixtures:
- response:
content: "default response"
"#;
let file: FixtureFile = serde_yaml_ng::from_str(yaml).unwrap();
let f = &file.fixtures[0];
assert!(f.match_rule.is_none());
}
#[test]
fn should_reject_fixture_with_both_error_and_response() {
let mut f = Fixture {
response: Some(FixtureResponse {
content: Some("hi".to_string()),
..Default::default()
}),
error: Some(FixtureError {
status: 500,
message: "fail".to_string(),
headers: HashMap::new(),
}),
..Fixture::new()
};
let result = f.validate();
assert!(result.is_err());
assert!(result.unwrap_err().contains("mutually exclusive"));
}
#[test]
fn should_reject_fixture_with_failure_but_no_response() {
let mut f = Fixture {
failure: Some(FailureConfig {
latency_ms: Some(1000),
..Default::default()
}),
..Fixture::new()
};
let result = f.validate();
assert!(result.is_err());
assert!(result.unwrap_err().contains("requires response"));
}
#[test]
fn should_reject_fixture_with_error_and_failure() {
let mut f = Fixture {
error: Some(FixtureError {
status: 429,
message: "rate limit".to_string(),
headers: HashMap::new(),
}),
failure: Some(FailureConfig {
latency_ms: Some(1000),
..Default::default()
}),
..Fixture::new()
};
let result = f.validate();
assert!(result.is_err());
}
#[test]
fn should_reject_fixture_with_no_response_and_no_error() {
let mut f = Fixture {
match_rule: Some(FixtureMatch::default()),
..Fixture::new()
};
let result = f.validate();
assert!(result.is_err());
assert!(result.unwrap_err().contains("must have either"));
}
#[test]
fn should_reject_failure_probability_above_one() {
let mut f = Fixture::new()
.respond_with_content("ok")
.with_failure(FailureConfig {
probability: Some(2.0),
..Default::default()
});
let err = f.validate().unwrap_err();
assert!(
err.contains("probability must be a finite number in"),
"got: {}",
err
);
}
#[test]
fn should_reject_failure_probability_below_zero() {
let mut f = Fixture::new()
.respond_with_content("ok")
.with_failure(FailureConfig {
probability: Some(-0.5),
..Default::default()
});
let err = f.validate().unwrap_err();
assert!(
err.contains("probability must be a finite number in"),
"got: {}",
err
);
}
#[test]
fn should_reject_failure_probability_nan() {
let mut f = Fixture::new()
.respond_with_content("ok")
.with_failure(FailureConfig {
probability: Some(f32::NAN),
..Default::default()
});
let err = f.validate().unwrap_err();
assert!(
err.contains("probability must be a finite number in"),
"got: {}",
err
);
}
#[test]
fn should_reject_latency_jitter_without_streaming_latency() {
let mut f = Fixture::new()
.respond_with_content("ok")
.with_failure(FailureConfig {
latency_jitter_ms: Some(5),
..Default::default()
});
let err = f.validate().unwrap_err();
assert!(err.contains("latency_jitter_ms requires"), "got: {}", err);
}
#[test]
fn should_reject_latency_jitter_with_zero_streaming_latency() {
let mut f = Fixture::new()
.respond_with_content("ok")
.with_streaming(Some(0), Some(10))
.with_failure(FailureConfig {
latency_jitter_ms: Some(5),
..Default::default()
});
let err = f.validate().unwrap_err();
assert!(err.contains("latency_jitter_ms requires"), "got: {}", err);
}
#[test]
fn should_accept_zero_latency_jitter_without_streaming() {
let mut f = Fixture::new()
.respond_with_content("ok")
.with_failure(FailureConfig {
latency_jitter_ms: Some(0),
..Default::default()
});
assert!(f.validate().is_ok());
}
#[test]
fn should_reject_latency_jitter_above_one_hour_cap() {
let mut f = Fixture::new()
.respond_with_content("ok")
.with_streaming(Some(10), Some(5))
.with_failure(FailureConfig {
latency_jitter_ms: Some(3_600_001),
..Default::default()
});
let err = f.validate().unwrap_err();
assert!(err.contains("latency_jitter_ms must be <="), "got: {}", err);
}
#[test]
fn should_accept_latency_jitter_at_one_hour_cap() {
let mut f = Fixture::new()
.respond_with_content("ok")
.with_streaming(Some(10), Some(5))
.with_failure(FailureConfig {
latency_jitter_ms: Some(3_600_000),
..Default::default()
});
assert!(f.validate().is_ok());
}
#[test]
fn should_accept_latency_jitter_with_positive_streaming_latency() {
let mut f = Fixture::new()
.respond_with_content("ok")
.with_streaming(Some(10), Some(5))
.with_failure(FailureConfig {
latency_jitter_ms: Some(5),
..Default::default()
});
assert!(f.validate().is_ok());
}
#[test]
fn should_accept_failure_probability_at_boundaries() {
let mut f = Fixture::new()
.respond_with_content("ok")
.with_failure(FailureConfig {
probability: Some(0.0),
..Default::default()
});
assert!(f.validate().is_ok());
let mut f = Fixture::new()
.respond_with_content("ok")
.with_failure(FailureConfig {
probability: Some(1.0),
..Default::default()
});
assert!(f.validate().is_ok());
}
#[test]
fn should_accept_valid_error_fixture() {
let mut f = Fixture::new().with_error(429, "rate limit");
assert!(f.validate().is_ok());
}
#[test]
fn should_accept_valid_response_fixture() {
let mut f = Fixture::new().respond_with_content("hi");
assert!(f.validate().is_ok());
}
#[test]
fn should_reject_invalid_regex() {
let mut f = Fixture {
match_rule: Some(FixtureMatch {
user_message: Some(StringMatch::regex("[invalid")),
model: None,
..Default::default()
}),
response: Some(FixtureResponse {
content: Some("hi".to_string()),
..Default::default()
}),
..Fixture::new()
};
let result = f.validate();
assert!(result.is_err());
assert!(result.unwrap_err().contains("regex"));
}
#[test]
fn should_match_substring_user_message() {
let fixtures = vec![Fixture::new()
.match_user_message("hello")
.respond_with_content("hi")];
let result = match_fixture(&fixtures, "say hello world", None, None, None);
assert!(result.is_some());
}
#[test]
fn should_not_match_wrong_substring() {
let fixtures = vec![Fixture::new()
.match_user_message("goodbye")
.respond_with_content("bye")];
let result = match_fixture(&fixtures, "say hello world", None, None, None);
assert!(result.is_none());
}
#[test]
fn should_match_regex_user_message() {
let fixtures = vec![Fixture {
match_rule: Some(FixtureMatch {
user_message: Some(StringMatch::regex("hello \\w+")),
model: None,
..Default::default()
}),
..Fixture::new().respond_with_content("matched")
}];
let result = match_fixture(&fixtures, "hello world", None, None, None);
assert!(result.is_some());
}
#[test]
fn should_match_model() {
let fixtures = vec![Fixture::new()
.match_model("gpt-4")
.respond_with_content("gpt4 response")];
let result = match_fixture(&fixtures, "anything", Some("gpt-4-turbo"), None, None);
assert!(result.is_some());
}
#[test]
fn should_match_first_fixture_wins() {
let fixtures = vec![
Fixture::new()
.match_user_message("hello")
.respond_with_content("first"),
Fixture::new()
.match_user_message("hello")
.respond_with_content("second"),
];
let result = match_fixture(&fixtures, "hello", None, None, None);
assert_eq!(
result
.unwrap()
.response
.as_ref()
.unwrap()
.content
.as_deref(),
Some("first")
);
}
#[test]
fn should_match_catch_all() {
let fixtures = vec![Fixture::new().respond_with_content("default")];
let result = match_fixture(&fixtures, "anything at all", None, None, None);
assert!(result.is_some());
}
#[test]
fn should_filter_by_provider() {
let fixtures = vec![Fixture {
provider: Some(crate::format::Provider::Anthropic),
..Fixture::new().respond_with_content("anthropic only")
}];
let result = match_fixture(
&fixtures,
"hello",
None,
Some(crate::format::Provider::Anthropic),
None,
);
assert!(result.is_some());
let result = match_fixture(
&fixtures,
"hello",
None,
Some(crate::format::Provider::OpenAI),
None,
);
assert!(result.is_none());
}
#[test]
fn should_build_fixture_programmatically() {
let mut f = Fixture::new()
.match_user_message("hello")
.respond_with_content("Hi there!");
assert!(f.validate().is_ok());
assert_eq!(
f.response.as_ref().unwrap().content.as_deref(),
Some("Hi there!")
);
}
#[test]
fn should_build_error_fixture_programmatically() {
let mut f = Fixture::new()
.match_model("fail-model")
.with_error(429, "Rate limited");
assert!(f.validate().is_ok());
assert_eq!(f.error.as_ref().unwrap().status, 429);
}
#[test]
fn should_use_default_trait_for_fixture() {
let f = Fixture::default();
assert!(f.response.is_none());
assert!(f.error.is_none());
assert!(f.match_rule.is_none());
}
#[test]
fn should_compare_regex_match_by_pattern_string() {
let a = RegexMatch {
regex: "hello".to_string(),
compiled: None,
};
let b = RegexMatch {
regex: "hello".to_string(),
compiled: None,
};
let c = RegexMatch {
regex: "world".to_string(),
compiled: None,
};
assert_eq!(a, b);
assert_ne!(a, c);
}
#[test]
fn should_reject_response_with_both_content_and_tool_calls() {
let mut f = Fixture {
response: Some(FixtureResponse {
content: Some("text".to_string()),
tool_calls: Some(vec![ToolCall {
name: "func".to_string(),
arguments: serde_json::json!({}),
}]),
..Default::default()
}),
..Fixture::new()
};
let result = f.validate();
assert!(result.is_err());
assert!(result.unwrap_err().contains("mutually exclusive"));
}
#[test]
fn should_reject_response_with_neither_content_nor_tool_calls() {
let mut f = Fixture {
response: Some(FixtureResponse::default()),
..Fixture::new()
};
let result = f.validate();
assert!(result.is_err());
assert!(result.unwrap_err().contains("must have either"));
}
#[test]
fn should_reject_zero_chunk_size() {
let mut f = Fixture {
response: Some(FixtureResponse {
content: Some("hi".to_string()),
..Default::default()
}),
streaming: Some(StreamingConfig {
latency: None,
chunk_size: Some(0),
}),
..Fixture::new()
};
let result = f.validate();
assert!(result.is_err());
assert!(result.unwrap_err().contains("chunk_size must be > 0"));
}
#[test]
fn should_compile_model_regex_on_validate() {
let mut f = Fixture {
match_rule: Some(FixtureMatch {
user_message: None,
model: Some(StringMatch::regex("gpt-4.*")),
..Default::default()
}),
response: Some(FixtureResponse {
content: Some("hi".to_string()),
..Default::default()
}),
..Fixture::new()
};
assert!(f.validate().is_ok());
let fixtures = vec![f];
let result = match_fixture(&fixtures, "hello", Some("gpt-4-turbo"), None, None);
assert!(result.is_some());
}
#[test]
fn should_match_compiled_user_message_regex() {
let mut f = Fixture {
match_rule: Some(FixtureMatch {
user_message: Some(StringMatch::regex("he.*ld")),
model: None,
..Default::default()
}),
response: Some(FixtureResponse {
content: Some("matched".to_string()),
..Default::default()
}),
..Fixture::new()
};
assert!(f.validate().is_ok());
let fixtures = vec![f];
let result = match_fixture(&fixtures, "hello world", None, None, None);
assert!(result.is_some());
}
#[test]
fn should_reject_invalid_model_regex() {
let mut f = Fixture {
match_rule: Some(FixtureMatch {
user_message: None,
model: Some(StringMatch::regex("[invalid")),
..Default::default()
}),
response: Some(FixtureResponse {
content: Some("hi".to_string()),
..Default::default()
}),
..Fixture::new()
};
let result = f.validate();
assert!(result.is_err());
assert!(result.unwrap_err().contains("model"));
}
#[test]
fn should_not_match_model_when_no_model_provided() {
let fixtures = vec![Fixture::new()
.match_model("gpt-4")
.respond_with_content("gpt4 only")];
let result = match_fixture(&fixtures, "hello", None, None, None);
assert!(result.is_none());
}
#[test]
fn should_use_regex_fallback_for_unvalidated_fixture() {
let fixtures = vec![Fixture {
match_rule: Some(FixtureMatch {
user_message: Some(StringMatch::regex("hel+o")),
model: None,
..Default::default()
}),
response: Some(FixtureResponse {
content: Some("matched".to_string()),
..Default::default()
}),
..Fixture::new()
}];
let result = match_fixture(&fixtures, "helllo world", None, None, None);
assert!(result.is_some());
}
#[test]
fn should_load_yaml_file() {
let dir = std::env::temp_dir().join("llmposter_test_load");
std::fs::create_dir_all(&dir).unwrap();
let file = dir.join("test.yaml");
std::fs::write(
&file,
r#"
fixtures:
- match:
user_message: "test"
response:
content: "loaded from file"
"#,
)
.unwrap();
let fixtures = load_yaml_file(&file).unwrap();
assert_eq!(fixtures.len(), 1);
assert_eq!(
fixtures[0].response.as_ref().unwrap().content.as_deref(),
Some("loaded from file")
);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn should_load_yaml_dir() {
let dir = std::env::temp_dir().join("llmposter_test_dir");
std::fs::create_dir_all(&dir).unwrap();
std::fs::write(
dir.join("a.yaml"),
"fixtures:\n - match:\n user_message: \"a\"\n response:\n content: \"a\"",
)
.unwrap();
std::fs::write(
dir.join("b.yml"),
"fixtures:\n - match:\n user_message: \"b\"\n response:\n content: \"b\"",
)
.unwrap();
std::fs::write(dir.join("not_yaml.txt"), "ignored").unwrap();
std::fs::create_dir_all(dir.join("subdir")).unwrap();
let fixtures = load_yaml_dir(&dir).unwrap();
assert_eq!(fixtures.len(), 2);
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn should_return_error_for_invalid_yaml_file() {
let dir = std::env::temp_dir().join("llmposter_test_invalid");
std::fs::create_dir_all(&dir).unwrap();
let file = dir.join("bad.yaml");
std::fs::write(&file, "not: [valid: {{{").unwrap();
let result = load_yaml_file(&file);
assert!(result.is_err());
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn should_return_error_for_missing_file() {
let result = load_yaml_file(Path::new("/nonexistent/file.yaml"));
assert!(result.is_err());
}
#[test]
fn should_validate_fixtures_on_load() {
let dir = std::env::temp_dir().join("llmposter_test_validate_load");
std::fs::create_dir_all(&dir).unwrap();
let file = dir.join("invalid_fixture.yaml");
std::fs::write(
&file,
r#"
fixtures:
- match:
user_message: "test"
response:
content: "hi"
error:
status: 500
message: "also error"
"#,
)
.unwrap();
let result = load_yaml_file(&file);
assert!(result.is_err());
assert!(result
.unwrap_err()
.to_string()
.contains("mutually exclusive"));
std::fs::remove_dir_all(&dir).ok();
}
#[test]
fn should_reject_oversized_regex_at_validation() {
let huge_pattern = format!("a{{{}}}", 999_999);
let mut f = Fixture {
match_rule: Some(FixtureMatch {
user_message: Some(StringMatch::regex(&huge_pattern)),
model: None,
..Default::default()
}),
response: Some(FixtureResponse {
content: Some("hi".to_string()),
..Default::default()
}),
..Fixture::new()
};
let result = f.validate();
assert!(result.is_err(), "oversized regex should be rejected");
}
#[test]
fn should_return_false_for_oversized_regex_in_fallback() {
let huge_pattern = format!("a{{{}}}", 999_999);
let rm = RegexMatch {
regex: huge_pattern,
compiled: None, };
assert!(!rm.is_match("aaaa"));
}
#[test]
fn should_reject_scalar_tool_call_arguments() {
let mut f = Fixture::new().respond_with_tool_calls(vec![ToolCall {
name: "test".to_string(),
arguments: serde_json::json!("not an object"),
}]);
let result = f.validate();
assert!(result.is_err());
assert!(result.unwrap_err().contains("must be a JSON object"));
}
#[test]
fn should_reject_array_tool_call_arguments() {
let mut f = Fixture::new().respond_with_tool_calls(vec![ToolCall {
name: "test".to_string(),
arguments: serde_json::json!([1, 2, 3]),
}]);
let result = f.validate();
assert!(result.is_err());
assert!(result.unwrap_err().contains("must be a JSON object"));
}
#[test]
fn should_accept_object_tool_call_arguments() {
let mut f = Fixture::new().respond_with_tool_calls(vec![ToolCall {
name: "test".to_string(),
arguments: serde_json::json!({"key": "value"}),
}]);
assert!(f.validate().is_ok());
}
#[test]
fn should_reject_blank_tool_call_name() {
let mut f = Fixture::new().respond_with_tool_calls(vec![ToolCall {
name: "".to_string(),
arguments: serde_json::json!({"key": "value"}),
}]);
let result = f.validate();
assert!(result.is_err());
assert!(result.unwrap_err().contains("name must not be empty"));
}
#[test]
fn should_reject_whitespace_only_tool_call_name() {
let mut f = Fixture::new().respond_with_tool_calls(vec![ToolCall {
name: " ".to_string(),
arguments: serde_json::json!({"key": "value"}),
}]);
let result = f.validate();
assert!(result.is_err());
assert!(result.unwrap_err().contains("name must not be empty"));
}
#[test]
fn should_reject_non_error_status_codes() {
for status in [200, 204, 301, 302] {
let mut f = Fixture::new().with_error(status, "test");
let result = f.validate();
assert!(result.is_err(), "status {} should be rejected", status);
assert!(result.unwrap_err().contains("400-599"));
}
}
#[test]
fn should_accept_error_status_codes() {
for status in [400, 401, 403, 404, 429, 500, 502, 503, 529] {
let mut f = Fixture::new().with_error(status, "test");
assert!(f.validate().is_ok(), "status {} should be accepted", status);
}
}
#[test]
fn should_reject_empty_user_message_substring() {
let mut f = Fixture::new()
.match_user_message("")
.respond_with_content("ok");
let result = f.validate();
assert!(result.is_err());
assert!(result.unwrap_err().contains("must not be empty"));
}
#[test]
fn should_reject_empty_model_substring() {
let mut f = Fixture::new().match_model("").respond_with_content("ok");
let result = f.validate();
assert!(result.is_err());
assert!(result.unwrap_err().contains("must not be empty"));
}
#[test]
fn should_reject_empty_user_message_regex() {
let mut f = Fixture::new().respond_with_content("ok");
let m = f.match_rule.get_or_insert_with(FixtureMatch::default);
m.user_message = Some(StringMatch::regex(""));
let result = f.validate();
assert!(result.is_err());
assert!(result.unwrap_err().contains("regex must not be empty"));
}
#[test]
fn should_reject_empty_model_regex() {
let mut f = Fixture::new().respond_with_content("ok");
let m = f.match_rule.get_or_insert_with(FixtureMatch::default);
m.model = Some(StringMatch::regex(""));
let result = f.validate();
assert!(result.is_err());
assert!(result.unwrap_err().contains("regex must not be empty"));
}
#[test]
fn should_reject_unknown_yaml_fields() {
let yaml =
"fixtures:\n - match:\n user_mesage: typo\n response:\n content: ok";
let result: Result<FixtureFile, _> = serde_yaml_ng::from_str(yaml);
assert!(result.is_err(), "typo field 'user_mesage' must be rejected");
}
#[test]
fn should_reject_unknown_fixture_fields() {
let yaml = "fixtures:\n - unknown_field: true\n response:\n content: ok";
let result: Result<FixtureFile, _> = serde_yaml_ng::from_str(yaml);
assert!(result.is_err(), "unknown fixture field must be rejected");
}
#[test]
fn should_set_stop_reason_via_builder() {
let f = Fixture::new()
.respond_with_content("test")
.with_stop_reason("max_tokens");
assert_eq!(
f.response.as_ref().unwrap().stop_reason.as_deref(),
Some("max_tokens")
);
}
#[test]
fn should_set_finish_reason_via_builder() {
let f = Fixture::new()
.respond_with_content("test")
.with_finish_reason("length");
assert_eq!(
f.response.as_ref().unwrap().finish_reason.as_deref(),
Some("length")
);
}
#[test]
fn should_set_stop_reason_on_empty_response() {
let f = Fixture::new().with_stop_reason("end_turn");
assert!(f.response.is_some());
assert_eq!(
f.response.as_ref().unwrap().stop_reason.as_deref(),
Some("end_turn")
);
}
#[test]
fn should_set_finish_reason_on_empty_response() {
let f = Fixture::new().with_finish_reason("stop");
assert!(f.response.is_some());
assert_eq!(
f.response.as_ref().unwrap().finish_reason.as_deref(),
Some("stop")
);
}
#[test]
fn should_warn_but_accept_truncate_without_streaming_config() {
let mut f = Fixture {
failure: Some(FailureConfig {
truncate_after_frames: Some(2),
..Default::default()
}),
..Fixture::new().respond_with_content("ok")
};
assert!(f.validate().is_ok());
}
#[test]
fn should_warn_but_accept_disconnect_without_streaming_config() {
let mut f = Fixture {
failure: Some(FailureConfig {
disconnect_after_ms: Some(100),
..Default::default()
}),
..Fixture::new().respond_with_content("ok")
};
assert!(f.validate().is_ok());
}
#[test]
fn should_warn_but_accept_duplicate_frames_without_streaming_config() {
let mut f = Fixture {
failure: Some(FailureConfig {
duplicate_frames: Some(true),
..Default::default()
}),
..Fixture::new().respond_with_content("ok")
};
assert!(f.validate().is_ok());
}
#[test]
fn should_accept_duplicate_frames_with_streaming_config() {
let mut f = Fixture::new()
.respond_with_content("ok")
.with_streaming(Some(5), Some(10))
.with_failure(FailureConfig {
duplicate_frames: Some(true),
..Default::default()
});
assert!(f.validate().is_ok());
}
#[test]
fn should_warn_but_accept_truncate_on_tool_calls_fixture() {
let mut f = Fixture {
failure: Some(FailureConfig {
truncate_after_frames: Some(2),
..Default::default()
}),
..Fixture::new().respond_with_tool_calls(vec![ToolCall {
name: "get_weather".to_string(),
arguments: serde_json::json!({"location": "SF"}),
}])
};
assert!(f.validate().is_ok());
}
#[test]
fn should_skip_compile_when_already_compiled() {
let mut f = Fixture {
match_rule: Some(FixtureMatch {
user_message: Some(StringMatch::Regex(RegexMatch {
regex: "hello \\w+".to_string(),
compiled: None,
})),
model: None,
..Default::default()
}),
..Fixture::new().respond_with_content("ok")
};
assert!(f.validate().is_ok());
assert!(f.validate().is_ok()); }
#[test]
fn should_reject_empty_tool_calls_vec() {
let mut f = Fixture {
response: Some(FixtureResponse {
tool_calls: Some(vec![]),
..Default::default()
}),
..Fixture::new()
};
let result = f.validate();
assert!(result.is_err());
assert!(result.unwrap_err().contains("must not be empty"));
}
#[test]
fn should_reject_number_tool_call_arguments() {
let mut f = Fixture::new().respond_with_tool_calls(vec![ToolCall {
name: "test".to_string(),
arguments: serde_json::json!(42),
}]);
let result = f.validate();
assert!(result.is_err());
assert!(result
.unwrap_err()
.contains("must be a JSON object, got number"));
}
#[test]
fn should_reject_bool_tool_call_arguments() {
let mut f = Fixture::new().respond_with_tool_calls(vec![ToolCall {
name: "test".to_string(),
arguments: serde_json::json!(true),
}]);
let result = f.validate();
assert!(result.is_err());
assert!(result
.unwrap_err()
.contains("must be a JSON object, got boolean"));
}
#[test]
fn should_reject_null_tool_call_arguments() {
let mut f = Fixture::new().respond_with_tool_calls(vec![ToolCall {
name: "test".to_string(),
arguments: serde_json::json!(null),
}]);
let result = f.validate();
assert!(result.is_err());
assert!(result
.unwrap_err()
.contains("must be a JSON object, got null"));
}
#[test]
fn should_reject_duplicate_header_name_in_validate() {
let mut f = Fixture {
error: Some(FixtureError {
status: 429,
message: "rate limit".to_string(),
headers: HashMap::from([
("x-custom".to_string(), "a".to_string()),
("X-Custom".to_string(), "b".to_string()),
]),
}),
..Fixture::new()
};
let result = f.validate();
assert!(result.is_err());
assert!(result.unwrap_err().contains("duplicate"));
}
#[test]
fn should_reject_invalid_header_name_in_validate() {
let mut f = Fixture {
error: Some(FixtureError {
status: 429,
message: "rate limit".to_string(),
headers: HashMap::from([("invalid name!".to_string(), "value".to_string())]),
}),
..Fixture::new()
};
let result = f.validate();
assert!(result.is_err());
assert!(result.unwrap_err().contains("invalid error header name"));
}
#[test]
fn should_reject_invalid_header_value_in_validate() {
let mut f = Fixture {
error: Some(FixtureError {
status: 429,
message: "rate limit".to_string(),
headers: HashMap::from([("x-custom".to_string(), "\x00bad".to_string())]),
}),
..Fixture::new()
};
let result = f.validate();
assert!(result.is_err());
assert!(result.unwrap_err().contains("invalid error header value"));
}
#[test]
fn should_reject_streaming_config_on_error_fixture() {
let mut f = Fixture {
error: Some(FixtureError {
status: 429,
message: "rate limit".to_string(),
headers: HashMap::new(),
}),
streaming: Some(StreamingConfig {
latency: None,
chunk_size: Some(10),
}),
..Fixture::new()
};
let result = f.validate();
assert!(result.is_err());
assert!(result.unwrap_err().contains("no effect on error-only"));
}
#[cfg(feature = "jsonpath")]
fn ctx<'a>(
body: &'a serde_json::Value,
provider: Option<crate::format::Provider>,
) -> MatchContext<'a> {
static EMPTY_HEADERS: std::sync::OnceLock<HashMap<String, String>> =
std::sync::OnceLock::new();
let headers = EMPTY_HEADERS.get_or_init(HashMap::new);
MatchContext::new("", None, provider, None, headers, body)
}
#[test]
fn extract_system_prompt_anthropic_array_without_text_blocks() {
let body = serde_json::json!({
"system": [{"type": "image", "data": "..."}]
});
assert_eq!(
extract_system_prompt(&body, Some(crate::format::Provider::Anthropic)),
None
);
}
#[test]
fn extract_system_prompt_anthropic_array_of_strings_ignored() {
let body = serde_json::json!({"system": 42});
assert_eq!(
extract_system_prompt(&body, Some(crate::format::Provider::Anthropic)),
None
);
}
#[test]
fn extract_system_prompt_gemini_with_empty_parts_returns_none() {
let body = serde_json::json!({
"systemInstruction": {"parts": []}
});
assert_eq!(
extract_system_prompt(&body, Some(crate::format::Provider::Gemini)),
None
);
let body = serde_json::json!({
"systemInstruction": {"parts": [{"data": "no text field"}]}
});
assert_eq!(
extract_system_prompt(&body, Some(crate::format::Provider::Gemini)),
None
);
}
#[test]
fn extract_system_prompt_gemini_without_system_instruction_returns_none() {
let body = serde_json::json!({
"contents": [{"role": "user", "parts": [{"text": "hi"}]}]
});
assert_eq!(
extract_system_prompt(&body, Some(crate::format::Provider::Gemini)),
None
);
}
#[test]
fn extract_system_prompt_openai_content_array_without_text_parts() {
let body = serde_json::json!({
"messages": [
{"role": "system", "content": [{"type": "image_url", "image_url": {}}]}
]
});
assert_eq!(extract_system_prompt(&body, None), None);
}
#[test]
fn extract_system_prompt_multiple_openai_system_messages_concatenated() {
let body = serde_json::json!({
"messages": [
{"role": "system", "content": "first"},
{"role": "user", "content": "hi"},
{"role": "system", "content": "second"}
]
});
assert_eq!(
extract_system_prompt(&body, None).as_deref(),
Some("first\nsecond")
);
}
#[test]
fn extract_tool_names_missing_tools_field_returns_empty() {
let body = serde_json::json!({"messages": []});
assert!(extract_tool_names(&body, None).is_empty());
}
#[test]
fn extract_tool_names_openai_fallback_to_tools_name_field() {
let body = serde_json::json!({
"tools": [{"name": "plain_tool"}]
});
let names = extract_tool_names(&body, None);
assert_eq!(names, vec!["plain_tool"]);
}
#[test]
fn extract_tool_names_gemini_without_function_declarations() {
let body = serde_json::json!({
"tools": [{"retrieval": {"source": "..."}}]
});
assert!(extract_tool_names(&body, Some(crate::format::Provider::Gemini)).is_empty());
}
#[test]
fn extract_temperature_gemini_nested_path() {
let body = serde_json::json!({
"generationConfig": {"temperature": 0.42}
});
assert_eq!(
extract_temperature(&body, Some(crate::format::Provider::Gemini)),
Some(0.42)
);
}
#[test]
fn extract_temperature_gemini_missing_generation_config() {
let body = serde_json::json!({"contents": []});
assert_eq!(
extract_temperature(&body, Some(crate::format::Provider::Gemini)),
None
);
}
#[test]
fn extract_temperature_non_gemini_top_level() {
let body = serde_json::json!({"temperature": 0.8});
assert_eq!(extract_temperature(&body, None), Some(0.8));
assert_eq!(
extract_temperature(&body, Some(crate::format::Provider::OpenAI)),
Some(0.8)
);
}
#[test]
fn f64_matches_exact_and_range_bounds() {
assert!(f64_matches(&F64Match::Exact(0.7), 0.7));
assert!(!f64_matches(&F64Match::Exact(0.7), 0.8));
let rng = F64Match::Range(F64Range {
min: Some(0.5),
max: Some(1.0),
});
assert!(f64_matches(&rng, 0.5));
assert!(f64_matches(&rng, 0.75));
assert!(f64_matches(&rng, 1.0));
assert!(!f64_matches(&rng, 0.4));
assert!(!f64_matches(&rng, 1.1));
let only_min = F64Match::Range(F64Range {
min: Some(0.5),
max: None,
});
assert!(f64_matches(&only_min, 5.0));
assert!(!f64_matches(&only_min, 0.4));
let only_max = F64Match::Range(F64Range {
min: None,
max: Some(0.5),
});
assert!(f64_matches(&only_max, 0.4));
assert!(!f64_matches(&only_max, 0.6));
}
#[cfg(feature = "jsonpath")]
#[test]
fn body_jsonpath_matches_via_onthefly_compile_when_not_validated() {
let f = Fixture::new()
.match_body_jsonpath("$.foo")
.respond_with_content("ok");
let body = serde_json::json!({"foo": "bar"});
let c = ctx(&body, None);
assert!(fixture_matches(&f, &c));
let body = serde_json::json!({"other": "value"});
let c = ctx(&body, None);
assert!(!fixture_matches(&f, &c));
}
#[cfg(feature = "jsonpath")]
#[test]
fn body_jsonpath_invalid_expression_fails_match_via_fallback() {
let f = Fixture::new()
.match_body_jsonpath("$[not-valid")
.respond_with_content("ok");
let body = serde_json::json!({"foo": 1});
let c = ctx(&body, None);
assert!(!fixture_matches(&f, &c));
}
}