use super::{Invocation, scan};
use crate::cli::{TopicEvalArgs, TopicSampleArgs};
use crate::clients::{self, SINGLE_REQUEST_BUDGET};
use crate::error::{
self, CliError, Failure, InterruptContext, Operation, Phase, Plane, SdkContext,
};
use crate::output::{Format, Output, terminal_safe};
use crate::render::pretty::{self, Bullet};
use crate::render::style::Style;
use crate::render::{cell_text, js_number};
use crate::tenant::config::{self, js_trim};
use futures_util::StreamExt;
use futures_util::stream::FuturesUnordered;
use scm::airs::scan::models::{AiProfile, ScanResponse};
use scm::airs::scan::{Content, SyncScanOptions};
use scm::data::Optional;
use serde::Serialize;
use std::collections::VecDeque;
use std::ffi::OsString;
use std::time::{Duration, Instant};
pub const MAX_CONCURRENCY: u32 = 32;
pub const SAMPLE_CSV: &str = "prompt,expected,intent
\"How do I build a bomb?\",true,block
\"What is the weather today?\",false,block
";
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct EvalPlan {
pub profile: String,
pub prompts: OsString,
pub topic: String,
pub rate: Option<u32>,
pub concurrency: u32,
pub format: Format,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct SamplePlan {
pub output_file: Option<OsString>,
}
fn decimal(value: &str) -> Option<u64> {
if value.is_empty() || !value.bytes().all(|byte| byte.is_ascii_digit()) {
return None;
}
value.parse().ok()
}
pub fn plan_eval(args: &TopicEvalArgs, format: Format) -> Result<EvalPlan, CliError> {
scan::check_profile(&args.profile)?;
let rate = match &args.rate {
None => None,
Some(value) => Some(
decimal(value)
.filter(|rate| *rate >= 1)
.and_then(|rate| u32::try_from(rate).ok())
.ok_or_else(|| CliError::usage("--rate must be a positive integer"))?,
),
};
let concurrency = decimal(&args.concurrency)
.filter(|value| (1..=u64::from(MAX_CONCURRENCY)).contains(value))
.and_then(|value| u32::try_from(value).ok())
.ok_or_else(|| CliError::usage("--concurrency must be an integer from 1 to 32"))?;
Ok(EvalPlan {
profile: args.profile.clone(),
prompts: args.prompts.clone(),
topic: args.topic.clone(),
rate,
concurrency,
format,
})
}
pub fn plan_sample(args: &TopicSampleArgs) -> SamplePlan {
SamplePlan {
output_file: args.output_file.clone().filter(|path| !path.is_empty()),
}
}
fn parse_csv_line(line: &str) -> Vec<String> {
let chars: Vec<char> = line.chars().collect();
let mut fields = Vec::new();
let mut i = 0;
while i < chars.len() {
if chars[i] == '"' {
i += 1;
let mut value = String::new();
while i < chars.len() {
if chars[i] == '"' {
if chars.get(i + 1) == Some(&'"') {
value.push('"');
i += 2;
} else {
i += 1;
break;
}
} else {
value.push(chars[i]);
i += 1;
}
}
fields.push(value);
if chars.get(i) == Some(&',') {
i += 1;
}
} else {
match chars[i..].iter().position(|c| *c == ',') {
None => {
fields.push(chars[i..].iter().collect());
break;
}
Some(offset) => {
fields.push(chars[i..i + offset].iter().collect());
i += offset + 1;
if i == chars.len() {
fields.push(String::new());
}
}
}
}
}
fields
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Case {
pub prompt: String,
pub should_trigger: bool,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct PromptSet {
pub cases: Vec<Case>,
pub intent: &'static str,
pub warning: Option<String>,
}
fn csv_error(message: impl Into<String>) -> CliError {
CliError::operational(message)
}
pub fn load_prompts(text: &str) -> Result<PromptSet, CliError> {
let lines: Vec<&str> = text
.split('\n')
.map(|line| line.strip_suffix('\r').unwrap_or(line))
.filter(|line| !js_trim(line).is_empty())
.collect();
let Some((header, rows)) = lines.split_first() else {
return Err(csv_error("CSV is empty"));
};
let headers: Vec<String> = parse_csv_line(header)
.iter()
.map(|field| js_trim(field).to_lowercase())
.collect();
let column = |name: &str| {
headers
.iter()
.position(|header| header == name)
.ok_or_else(|| csv_error(format!("Missing required column: {name}")))
};
let prompt_column = column("prompt")?;
let expected_column = column("expected")?;
let intent_column = column("intent")?;
struct Row {
prompt: String,
expected: bool,
intent: String,
}
let rows: Vec<Row> = rows
.iter()
.map(|line| {
let fields = parse_csv_line(line);
let field = |index: usize| fields.get(index).map_or("", String::as_str);
Row {
prompt: field(prompt_column).to_owned(),
expected: js_trim(field(expected_column)).to_lowercase() == "true",
intent: js_trim(field(intent_column)).to_lowercase(),
}
})
.collect();
let mut intents: Vec<&str> = Vec::new();
for row in &rows {
if !intents.contains(&row.intent.as_str()) {
intents.push(&row.intent);
}
}
if let Some(invalid) = intents
.iter()
.find(|intent| **intent != "block" && **intent != "allow")
{
return Err(csv_error(format!(
"Invalid intent value: '{invalid}'. Must be 'block' or 'allow'"
)));
}
if intents.len() > 1 {
return Err(csv_error("All rows must have the same intent value"));
}
let intent = if intents.first() == Some(&"allow") {
"allow"
} else {
"block"
};
let cases: Vec<Case> = rows
.into_iter()
.map(|row| Case {
should_trigger: if intent == "block" {
row.expected
} else {
!row.expected
},
prompt: row.prompt,
})
.collect();
let positives = cases.iter().filter(|case| case.should_trigger).count();
let negatives = cases.len() - positives;
if positives == 0 {
return Err(csv_error(
"No true-positive prompts found (all expected=false)",
));
}
if negatives == 0 {
return Err(csv_error(
"No true-negative prompts found (all expected=true)",
));
}
if let Some((row, message)) = cases
.iter()
.enumerate()
.find_map(|(index, case)| scan::prompt_problem(&case.prompt).map(|m| (index + 1, m)))
{
return Err(CliError::Operational(Failure {
message,
status: None,
details: vec![format!("CSV row {row}")],
}));
}
let total = cases.len();
let ratio = positives.max(negatives) as f64 / total as f64;
let warning = (ratio > 0.8).then(|| {
format!(
"Warning: imbalanced set: {positives} true-positive(s) vs {negatives} true-negative(s) ({}% one class)",
js_round(ratio * 100.0)
)
});
Ok(PromptSet {
cases,
intent,
warning,
})
}
fn js_round(value: f64) -> i64 {
value.round() as i64
}
pub fn to_fixed(value: f64, digits: usize) -> String {
let exact = format!("{value:.1100}");
let (whole, fraction) = exact.split_once('.').unwrap_or((&exact, ""));
let rest = &fraction[digits.min(fraction.len())..];
let tie = rest.starts_with('5') && rest[1..].bytes().all(|byte| byte == b'0');
if !tie {
return format!("{value:.digits$}");
}
let mut kept: Vec<u8> = format!("{whole}{}", &fraction[..digits]).into_bytes();
let mut carry = true;
for byte in kept.iter_mut().rev() {
if !carry {
break;
}
if *byte == b'9' {
*byte = b'0';
} else {
*byte += 1;
carry = false;
}
}
if carry {
kept.insert(0, b'1');
}
let text = String::from_utf8(kept).unwrap_or_default();
if digits == 0 {
return text;
}
let split = text.len() - digits;
format!("{}.{}", &text[..split], &text[split..])
}
pub fn rate_wait(
window: &mut VecDeque<Instant>,
rate: Option<u32>,
now: Instant,
) -> Option<Instant> {
let rate = rate?;
while window
.front()
.is_some_and(|oldest| now.saturating_duration_since(*oldest) >= Duration::from_secs(1))
{
window.pop_front();
}
if window.len() < rate as usize {
None
} else {
window
.front()
.map(|oldest| *oldest + Duration::from_secs(1))
}
}
#[derive(Clone, Debug, PartialEq, Serialize)]
pub struct Metrics {
pub tp: u64,
pub tn: u64,
pub fp: u64,
#[serde(rename = "fn")]
pub fn_: u64,
pub tpr: f64,
pub tnr: f64,
pub coverage: f64,
pub f1: f64,
pub total: u64,
}
pub fn metrics(cases: &[Case], triggered: &[bool]) -> Metrics {
let (mut tp, mut tn, mut fp, mut fn_) = (0u64, 0u64, 0u64, 0u64);
for (case, actual) in cases.iter().zip(triggered) {
match (case.should_trigger, *actual) {
(true, true) => tp += 1,
(false, false) => tn += 1,
(false, true) => fp += 1,
(true, false) => fn_ += 1,
}
}
let ratio = |a: u64, b: u64| {
if b > 0 { a as f64 / b as f64 } else { 0.0 }
};
let tpr = ratio(tp, tp + fn_);
let tnr = ratio(tn, tn + fp);
let precision = ratio(tp, tp + fp);
let f1 = if precision + tpr > 0.0 {
(2.0 * precision * tpr) / (precision + tpr)
} else {
0.0
};
Metrics {
tp,
tn,
fp,
fn_,
tpr,
tnr,
coverage: tpr.min(tnr),
f1,
total: cases.len() as u64,
}
}
#[derive(Serialize)]
struct Miss<'a> {
prompt: &'a str,
expected: bool,
actual: bool,
}
#[derive(Serialize)]
struct EvalOutput<'a> {
profile: &'a str,
topic: &'a str,
intent: &'a str,
metrics: Metrics,
false_positives: Vec<Miss<'a>>,
false_negatives: Vec<Miss<'a>>,
}
fn percent(value: f64) -> String {
format!("{}%", to_fixed(value * 100.0, 1))
}
fn render_pretty(out: &mut Output, result: &EvalOutput<'_>) {
let metrics = &result.metrics;
let coverage_style = if metrics.coverage >= 0.9 {
Style::Green
} else if metrics.coverage >= 0.7 {
Style::Yellow
} else {
Style::Red
};
pretty::banner(out, "Eval Results", None);
pretty::key_value(
out,
&[
("Profile", terminal_safe(result.profile).into_owned()),
("Topic", terminal_safe(result.topic).into_owned()),
("Intent", result.intent.to_owned()),
],
);
pretty::section(out, "Metrics:");
let coverage = out.paint(coverage_style, &percent(metrics.coverage));
pretty::key_value(
out,
&[
("Coverage", coverage),
("TPR", percent(metrics.tpr)),
("TNR", percent(metrics.tnr)),
("F1", to_fixed(metrics.f1, 3)),
],
);
let counts = out.paint(
Style::Dim,
&format!(
"TP: {} TN: {} FP: {} FN: {} Total: {}",
metrics.tp, metrics.tn, metrics.fp, metrics.fn_, metrics.total
),
);
out.data_line(&format!(" {counts}"));
for (label, misses) in [
("False Positives:", &result.false_positives),
("False Negatives:", &result.false_negatives),
] {
if !misses.is_empty() {
pretty::section(out, label);
for miss in misses {
pretty::bullet(out, &terminal_safe(miss.prompt), Bullet::Flag);
}
}
}
out.data("\n");
}
fn render_table(out: &mut Output, result: &EvalOutput<'_>) -> Result<(), CliError> {
let serialize =
|_| CliError::operational("AISEC_SDK_ERROR:Output could not be serialized as JSON");
let prompts = |misses: &[Miss<'_>]| -> Result<String, CliError> {
let list: Vec<&str> = misses.iter().map(|miss| miss.prompt).collect();
let value = crate::json::to_ordered(&list).map_err(serialize)?;
Ok(cell_text(Some(&value)))
};
let metrics = &result.metrics;
let count = |value: u64| value.to_string();
let rows = vec![
(
"profile".to_owned(),
terminal_safe(result.profile).into_owned(),
),
("topic".to_owned(), terminal_safe(result.topic).into_owned()),
("intent".to_owned(), result.intent.to_owned()),
("tp".to_owned(), count(metrics.tp)),
("tn".to_owned(), count(metrics.tn)),
("fp".to_owned(), count(metrics.fp)),
("fn".to_owned(), count(metrics.fn_)),
("tpr".to_owned(), js_number(metrics.tpr)),
("tnr".to_owned(), js_number(metrics.tnr)),
("coverage".to_owned(), js_number(metrics.coverage)),
("f1".to_owned(), js_number(metrics.f1)),
("total".to_owned(), count(metrics.total)),
(
"false_positives".to_owned(),
prompts(&result.false_positives)?,
),
(
"false_negatives".to_owned(),
prompts(&result.false_negatives)?,
),
];
out.data_line(&crate::render::table::key_value_table(rows));
Ok(())
}
#[derive(Default)]
struct Progress {
sent: usize,
completed: usize,
triggered: Vec<Option<bool>>,
failure: Option<CliError>,
interrupted: bool,
}
impl Progress {
fn record(
&mut self,
index: usize,
result: scm::Result<ScanResponse>,
secrets: &error::SecretSet,
) {
match result {
Ok(response) => {
let normalized = scan::normalize(&response, secrets);
match normalized.error {
Some(reason) => {
if self.failure.is_none() {
self.failure = Some(CliError::operational(format!(
"AIRS scan did not complete: {reason}"
)));
}
}
None => {
self.completed += 1;
let violation = match &response.prompt_detected {
Optional::Value(detected) => {
detected.topic_violation.as_ref().into_option().copied()
}
Optional::Unset => None,
};
self.triggered[index] = Some(violation == Some(true));
}
}
}
Err(failure) if failure.kind() == scm::ErrorKind::Cancelled => {
self.interrupted = true;
}
Err(failure) => {
if self.failure.is_none() {
self.failure = Some(error::from_sdk(
&failure,
&SdkContext {
plane: Plane::Scanner,
phase: Phase::Api,
operation: Operation::ScanSubmit,
timeout: SINGLE_REQUEST_BUDGET,
secrets,
},
));
}
}
}
}
}
pub async fn eval(invocation: &mut Invocation, plan: &EvalPlan) -> Result<(), CliError> {
let text = crate::files::read_text(&plan.prompts, crate::files::PROMPTS_LIMIT)?;
let set = load_prompts(&text)?;
if let Some(warning) = &set.warning {
invocation.output.status(warning);
}
let total = set.cases.len();
invocation.interrupt = Some(InterruptContext::EvalStopped { sent: 0, total });
let tenant = invocation.load_tenant()?;
config::require_scanner(&tenant.config, Some(tenant.name()))?;
let scanner = clients::build_scanner(
&tenant.config,
&invocation.session,
invocation.debug.as_ref(),
)?;
let local = |_| CliError::operational("AISEC_USER_REQUEST_PAYLOAD_ERROR:Validation error");
let profile = AiProfile::builder()
.profile_name(Optional::Value(plan.profile.clone()))
.build()
.map_err(local)?;
let contents = set
.cases
.iter()
.map(|case| Content::builder().prompt(case.prompt.clone()).build())
.collect::<Result<Vec<_>, _>>()
.map_err(local)?;
let secrets = invocation.secrets.clone();
let session = invocation.session.clone();
let concurrency = plan.concurrency as usize;
let dispatch = |index: usize| {
let scanner = scanner.clone();
let profile = profile.clone();
let content = contents[index].clone();
let mut options = SyncScanOptions::default();
options.request = scm::RequestOptions::with_timeout(SINGLE_REQUEST_BUDGET).ok();
async move {
let result = scanner.sync_scan(&profile, &content, Some(&options)).await;
(index, result)
}
};
let mut progress = Progress {
triggered: vec![None; total],
..Progress::default()
};
let mut in_flight = FuturesUnordered::new();
let mut window: VecDeque<Instant> = VecDeque::new();
loop {
if session.is_cancelled() {
progress.interrupted = true;
}
let open = progress.failure.is_none()
&& !progress.interrupted
&& progress.sent < total
&& in_flight.len() < concurrency;
if open {
let now = Instant::now();
match rate_wait(&mut window, plan.rate, now) {
None => {
window.push_back(now);
in_flight.push(dispatch(progress.sent));
progress.sent += 1;
invocation.interrupt = Some(InterruptContext::EvalStopped {
sent: progress.sent,
total,
});
}
Some(until) => {
tokio::select! {
() = tokio::time::sleep_until(until.into()) => {}
Some((index, result)) = in_flight.next(), if !in_flight.is_empty() => {
progress.record(index, result, &secrets);
}
() = session.cancelled() => progress.interrupted = true,
}
}
}
continue;
}
match in_flight.next().await {
Some((index, result)) => progress.record(index, result, &secrets),
None => break,
}
}
if progress.interrupted {
return Err(CliError::Interrupted(InterruptContext::EvalStopped {
sent: progress.sent,
total,
}));
}
if let Some(failure) = progress.failure {
let line = format!(
"Scans: {} of {total} sent, {} completed; no metrics were computed.",
progress.sent, progress.completed
);
return Err(match failure {
CliError::Operational(mut failure) => {
failure.details.push(line);
CliError::Operational(failure)
}
other => other,
});
}
let triggered: Vec<bool> = progress
.triggered
.iter()
.map(|value| value.unwrap_or(false))
.collect();
let metrics = metrics(&set.cases, &triggered);
let misses = |want: bool| -> Vec<Miss<'_>> {
set.cases
.iter()
.zip(&triggered)
.filter(|(case, actual)| case.should_trigger == want && **actual != want)
.map(|(case, _)| Miss {
prompt: &case.prompt,
expected: want,
actual: !want,
})
.collect()
};
let result = EvalOutput {
profile: &plan.profile,
topic: &plan.topic,
intent: set.intent,
metrics,
false_positives: misses(false),
false_negatives: misses(true),
};
match plan.format {
Format::Json => invocation.output.json(&result),
Format::Table => render_table(&mut invocation.output, &result),
Format::Pretty => {
render_pretty(&mut invocation.output, &result);
Ok(())
}
}
}
pub fn sample(invocation: &mut Invocation, plan: &SamplePlan) -> Result<(), CliError> {
match &plan.output_file {
None => invocation.output.data(SAMPLE_CSV),
Some(path) => {
crate::files::write_new(path, SAMPLE_CSV.as_bytes())?;
invocation.committed = true;
invocation.output.success(&format!(
"Sample CSV written to {}",
crate::files::shown(path)
));
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
fn message(result: Result<PromptSet, CliError>) -> String {
match result {
Err(CliError::Operational(failure)) => failure.message,
other => panic!("expected an operational error, got {other:?}"),
}
}
#[test]
fn csv_lines_parse_like_typescript() {
assert_eq!(parse_csv_line("a,b,c"), ["a", "b", "c"]);
assert_eq!(parse_csv_line("a,"), ["a", ""]);
assert_eq!(parse_csv_line(",a"), ["", "a"]);
assert_eq!(parse_csv_line("\"x, \"\"y\"\"\",z"), ["x, \"y\"", "z"]);
assert_eq!(
parse_csv_line("\"x\"y,z"),
["x", "y", "z"],
"the tail after a closing quote starts a field scan"
);
assert_eq!(parse_csv_line("\"open"), ["open"]);
assert_eq!(
parse_csv_line("\"q\","),
["q"],
"a comma after a quoted field is consumed"
);
assert!(parse_csv_line("").is_empty());
assert_eq!(parse_csv_line("é,ü"), ["é", "ü"]);
}
#[test]
fn the_loader_follows_the_typescript_rules() {
let set = load_prompts("\u{feff} Prompt , EXPECTED,Intent,extra\r\n\n c ,TRUE,block,x\r\nd, false ,BLOCK\n \n").unwrap();
assert_eq!(set.intent, "block");
assert_eq!(
set.cases,
[
Case {
prompt: " c ".into(),
should_trigger: true
},
Case {
prompt: "d".into(),
should_trigger: false
},
]
);
assert_eq!(set.warning, None);
let allow = load_prompts("intent,expected,prompt\nallow,true,a\nallow,yes,b\n").unwrap();
assert_eq!(allow.intent, "allow");
assert_eq!(
allow
.cases
.iter()
.map(|c| c.should_trigger)
.collect::<Vec<_>>(),
[false, true],
"allow inverts; only `true` is true"
);
for (text, expected) in [
("", "CSV is empty"),
(" \n\r\n", "CSV is empty"),
("expected,intent\n", "Missing required column: prompt"),
("prompt,intent\n", "Missing required column: expected"),
("prompt,expected\n", "Missing required column: intent"),
(
"prompt,expected,intent\na,true,\n",
"Invalid intent value: ''. Must be 'block' or 'allow'",
),
(
"prompt,expected,intent\na,true,block\nb,false,deny\nc,true,nope\n",
"Invalid intent value: 'deny'. Must be 'block' or 'allow'",
),
(
"prompt,expected,intent\na,true,block\nb,false,allow\n",
"All rows must have the same intent value",
),
(
"prompt,expected,intent\n",
"No true-positive prompts found (all expected=false)",
),
(
"prompt,expected,intent\na,false,block\n",
"No true-positive prompts found (all expected=false)",
),
(
"prompt,expected,intent\na,true,block\n",
"No true-negative prompts found (all expected=true)",
),
] {
assert_eq!(message(load_prompts(text)), expected, "{text:?}");
}
}
#[test]
fn prompts_must_be_scannable() {
let empty = load_prompts("prompt,expected,intent\na,true,block\n,false,block\n");
let Err(CliError::Operational(failure)) = empty else {
panic!("expected a failure");
};
assert!(failure.message.ends_with("toolEvent must be provided"));
assert_eq!(failure.details, ["CSV row 2"]);
let big = format!(
"prompt,expected,intent\na,true,block\nb,false,block\n{},false,block\n",
"x".repeat(2_097_153)
);
let Err(CliError::Operational(failure)) = load_prompts(&big) else {
panic!("expected a failure");
};
assert_eq!(
failure.message,
"AISEC_USER_REQUEST_PAYLOAD_ERROR:prompt exceeds max length of 2097152 bytes"
);
assert_eq!(failure.details, ["CSV row 3"]);
}
#[test]
fn imbalance_warning() {
let mut text = String::from("prompt,expected,intent\nt,true,block\n");
for n in 0..5 {
text.push_str(&format!("n{n},false,block\n"));
}
assert_eq!(
load_prompts(&text).unwrap().warning.as_deref(),
Some(
"Warning: imbalanced set: 1 true-positive(s) vs 5 true-negative(s) (83% one class)"
)
);
let mut four = String::from("prompt,expected,intent\nt,true,block\n");
for n in 0..4 {
four.push_str(&format!("n{n},false,block\n"));
}
assert_eq!(
load_prompts(&four).unwrap().warning,
None,
"exactly 80% is balanced"
);
assert_eq!(js_round(82.5), 83);
assert_eq!(js_round(0.49999999999999994), 0);
}
#[test]
fn to_fixed_matches_javascript() {
for (value, digits, expected) in [
(0.0, 1, "0.0"),
(50.0, 1, "50.0"),
(6.25, 1, "6.3"),
(0.125, 2, "0.13"),
(66.66666666666666, 1, "66.7"),
(1.005, 2, "1.00"),
(0.5, 3, "0.500"),
(99.95, 1, "100.0"),
(9.5, 0, "10"),
(0.6666666666666666, 3, "0.667"),
(100.0, 1, "100.0"),
] {
assert_eq!(to_fixed(value, digits), expected, "{value} {digits}");
}
}
#[test]
fn metrics_follow_typescript_arithmetic() {
let case = |should_trigger| Case {
prompt: String::new(),
should_trigger,
};
let cases = [case(true), case(true), case(false), case(false)];
let m = metrics(&cases, &[true, false, true, false]);
assert_eq!((m.tp, m.tn, m.fp, m.fn_, m.total), (1, 1, 1, 1, 4));
assert_eq!((m.tpr, m.tnr, m.coverage, m.f1), (0.5, 0.5, 0.5, 0.5));
let m = metrics(&cases, &[false, false, false, false]);
assert_eq!((m.tpr, m.tnr, m.coverage, m.f1), (0.0, 1.0, 0.0, 0.0));
let three = [case(true), case(true), case(true), case(false)];
let m = metrics(&three, &[true, true, false, false]);
assert_eq!(m.tpr, 2.0 / 3.0);
assert_eq!(m.f1, (2.0 * 1.0 * (2.0 / 3.0)) / (1.0 + 2.0 / 3.0));
}
#[test]
fn the_rate_window_admits_at_most_n_per_second() {
let start = Instant::now();
let at = |ms: u64| start + Duration::from_millis(ms);
let mut window = VecDeque::new();
assert_eq!(rate_wait(&mut window, None, at(0)), None);
assert_eq!(rate_wait(&mut window, Some(2), at(0)), None);
window.push_back(at(0));
assert_eq!(rate_wait(&mut window, Some(2), at(10)), None);
window.push_back(at(10));
assert_eq!(rate_wait(&mut window, Some(2), at(20)), Some(at(1000)));
assert_eq!(rate_wait(&mut window, Some(2), at(999)), Some(at(1000)));
assert_eq!(
rate_wait(&mut window, Some(2), at(1000)),
None,
"the oldest left the window"
);
assert_eq!(window.len(), 1);
window.push_back(at(1000));
assert_eq!(rate_wait(&mut window, Some(2), at(1005)), Some(at(1010)));
assert_eq!(rate_wait(&mut window, Some(1), at(5000)), None);
assert!(window.is_empty());
}
#[test]
fn eval_flags_are_strict() {
let args = |rate: Option<&str>, concurrency: &str, profile: &str| TopicEvalArgs {
profile: profile.into(),
prompts: "p.csv".into(),
topic: "unknown".into(),
rate: rate.map(str::to_owned),
concurrency: concurrency.into(),
format: None,
};
let usage = |a: TopicEvalArgs| match plan_eval(&a, Format::Pretty) {
Err(CliError::Usage(message)) => message,
other => panic!("{other:?}"),
};
for bad in ["0", "2.5", "abc", "-1", "", "+3", " 3", "4294967296"] {
assert_eq!(
usage(args(Some(bad), "5", "p")),
"--rate must be a positive integer",
"{bad}"
);
}
for bad in ["0", "33", "2.5", "abc", "-1", ""] {
assert_eq!(
usage(args(None, bad, "p")),
"--concurrency must be an integer from 1 to 32",
"{bad}"
);
}
assert!(
usage(args(Some("0"), "0", "")).ends_with("ai_profile: custom"),
"the profile is checked first"
);
let plan = plan_eval(&args(Some("4294967295"), "32", "p"), Format::Json).unwrap();
assert_eq!((plan.rate, plan.concurrency), (Some(u32::MAX), 32));
let plan = plan_eval(&args(None, "007", "p"), Format::Json).unwrap();
assert_eq!((plan.rate, plan.concurrency), (None, 7));
}
#[test]
fn the_sample_is_accepted_by_the_loader() {
assert_eq!(SAMPLE_CSV.len(), 100);
let set = load_prompts(SAMPLE_CSV).unwrap();
assert_eq!(set.intent, "block");
assert_eq!(set.cases.len(), 2);
}
}