use std::collections::{BTreeMap, BTreeSet};
use std::path::Path;
use anyhow::{bail, Context, Result};
use clap::ValueEnum;
use serde::{Deserialize, Serialize};
use tracing::warn;
use crate::jev::citations::Citation;
use crate::jev::client::JevClient;
use crate::jev::error::is_auth_failure;
use crate::jev::input::truncate_middle;
use crate::jev::protocol::{Answer, Question, SystemOneRequest, Usage};
use crate::provider::{IssueDoc, ItemState};
pub use crate::jev::input::TRUNCATION_MARKER;
pub mod effort;
const STAGE_QUESTIONS_YAML: &str = include_str!("../templates/jev-route-questions.yaml");
pub const NO_DESIGN: &str = "none";
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum Provider {
#[value(name = "anthropic")]
Anthropic,
#[value(name = "openai")]
OpenAi,
#[value(name = "gemini")]
Gemini,
}
impl Provider {
pub const ALL: [Self; 3] = [Self::Anthropic, Self::OpenAi, Self::Gemini];
#[must_use]
pub const fn name(self) -> &'static str {
match self {
Self::Anthropic => "anthropic",
Self::OpenAi => "openai",
Self::Gemini => "gemini",
}
}
const fn tiers_yaml(self) -> &'static str {
match self {
Self::Anthropic => include_str!("../templates/jev-route-tiers-anthropic.yaml"),
Self::OpenAi => include_str!("../templates/jev-route-tiers-openai.yaml"),
Self::Gemini => include_str!("../templates/jev-route-tiers-gemini.yaml"),
}
}
pub fn tiers(self) -> Result<Tiers> {
Tiers::parse(self.tiers_yaml())
.with_context(|| format!("Invalid embedded tiers for provider {}", self.name()))
}
}
pub const DEFAULT_CLOSE_CALL: f64 = 0.3;
pub const DEFAULT_MAX_INPUT_CHARS: usize = 60_000;
const MIN_TIERS: usize = 2;
pub type OpenDependencies = BTreeMap<(String, u64), Vec<Citation>>;
pub type ReferenceFetchFailures = BTreeMap<(String, u64), Vec<ReferenceFetchFailure>>;
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct ReferenceFetchFailure {
#[serde(rename = "ref")]
pub item_ref: String,
pub error: String,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum Stage {
Design,
Implement,
Review,
}
impl Stage {
const ALL: [Self; 3] = [Self::Design, Self::Implement, Self::Review];
const fn template_key(self) -> &'static str {
match self {
Self::Design => "stage_design",
Self::Implement => "stage_implement",
Self::Review => "stage_review",
}
}
fn question_key(self, provider: &str) -> String {
format!("{provider}.{}", self.template_key())
}
}
#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Tier {
pub name: String,
pub description: String,
pub models: Option<Vec<effort::Model>>,
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct TiersFile {
tiers: Vec<Tier>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Tiers(Vec<Tier>);
impl Tiers {
pub fn parse(yaml: &str) -> Result<Self> {
let file: TiersFile = serde_yaml::from_str(yaml).context("Failed to parse tiers file")?;
if file.tiers.len() < MIN_TIERS {
bail!(
"a tiers file needs at least {MIN_TIERS} tiers, got {}",
file.tiers.len()
);
}
let mut seen = BTreeSet::new();
for tier in &file.tiers {
if tier.name.trim().is_empty() || tier.description.trim().is_empty() {
bail!("every tier needs a non-empty name and description");
}
if tier.name == NO_DESIGN {
bail!("the tier name {NO_DESIGN:?} is reserved for \"no design work remains\"");
}
if !seen.insert(tier.name.as_str()) {
bail!("tier {:?} is defined more than once", tier.name);
}
}
let mut model_rungs = BTreeMap::new();
for tier in &file.tiers {
if let Some(models) = &tier.models {
effort::validate(models).with_context(|| format!("rung {:?}", tier.name))?;
for model in models {
for &stage in &model.stages {
if let Some(previous) = model_rungs.insert((&model.name, stage), &tier.name)
{
bail!(
"model {:?} is bound to both rungs {previous:?} and {:?} for {stage:?}",
model.name,
tier.name
);
}
}
}
}
}
Ok(Self(file.tiers))
}
pub fn load_file(path: &Path) -> Result<Self> {
let yaml = std::fs::read_to_string(path)
.with_context(|| format!("Failed to read tiers file {}", path.display()))?;
Self::parse(&yaml).with_context(|| format!("Invalid tiers file {}", path.display()))
}
#[must_use]
pub fn as_slice(&self) -> &[Tier] {
&self.0
}
fn rank(&self, choice: &str) -> Option<usize> {
if choice == NO_DESIGN {
return Some(0);
}
self.0.iter().position(|t| t.name == choice).map(|i| i + 1)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Ladder {
pub name: String,
pub tiers: Tiers,
}
impl Ladder {
pub fn builtin(provider: Provider) -> Result<Self> {
Ok(Self {
name: provider.name().to_string(),
tiers: provider.tiers()?,
})
}
#[must_use]
pub fn named(name: String, tiers: Tiers) -> Self {
Self { name, tiers }
}
}
pub fn build_route_questions(ladders: &[Ladder]) -> Result<BTreeMap<String, Question>> {
build_route_questions_for_mode(ladders, true)
}
fn build_route_questions_for_mode(
ladders: &[Ladder],
effort_advice: bool,
) -> Result<BTreeMap<String, Question>> {
let templates: BTreeMap<String, Question> = serde_yaml::from_str(STAGE_QUESTIONS_YAML)
.context("Failed to parse the embedded stage questions")?;
for stage in Stage::ALL {
if !templates.contains_key(stage.template_key()) {
bail!(
"the embedded stage questions have no {:?}",
stage.template_key()
);
}
}
let mut questions = BTreeMap::new();
for ladder in ladders {
for (key, template) in &templates {
let mut question = template.clone();
let Question::Choice { criteria, .. } = &mut question else {
bail!("embedded stage question {key:?} is not a choice question");
};
for tier in ladder.tiers.as_slice() {
if criteria
.insert(tier.name.clone(), tier.description.clone())
.is_some()
{
bail!(
"tier {:?} of ladder {:?} collides with a fixed option of {key:?}",
tier.name,
ladder.name
);
}
}
question
.validate()
.with_context(|| format!("stage question {key:?} for {:?}", ladder.name))?;
questions.insert(format!("{}.{key}", ladder.name), question);
}
}
if effort_advice {
for ladder in ladders {
effort::add_questions(ladder, &mut questions);
}
}
Ok(questions)
}
#[must_use]
pub fn build_route_state(doc: &IssueDoc, max_chars: usize) -> (String, bool) {
let mut text = format!("# #{} {}\n\n{}\n", doc.number, doc.title, doc.body.trim());
if !doc.comments.is_empty() {
text.push_str("\n\n---\n\n## Comments on this issue\n");
for comment in &doc.comments {
text.push_str(&format!(
"\n\n**Comment by {}:**\n\n{}\n",
comment.author,
comment.body.trim()
));
}
}
truncate_middle(&text, max_chars)
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct StageAnswer {
pub choice: String,
pub confidence: f64,
pub probabilities: BTreeMap<String, f64>,
#[serde(skip_serializing_if = "BTreeMap::is_empty")]
pub effort_by_model: BTreeMap<String, Vec<effort::ModelEffort>>,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct StageAnswers {
pub design: StageAnswer,
pub implement: StageAnswer,
pub review: StageAnswer,
}
impl StageAnswers {
fn get(&self, stage: Stage) -> &StageAnswer {
match stage {
Stage::Design => &self.design,
Stage::Implement => &self.implement,
Stage::Review => &self.review,
}
}
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct DependencyEntry {
#[serde(rename = "ref")]
pub item_ref: String,
#[serde(skip)]
pub url: Option<String>,
pub state: ItemState,
pub could_be_cheaper: BTreeMap<String, f64>,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct IssueRoute {
#[serde(rename = "ref")]
pub item_ref: String,
pub url: String,
pub title: String,
pub state: ItemState,
#[serde(flatten)]
pub outcome: RouteOutcome,
#[serde(skip_serializing_if = "std::ops::Not::not")]
pub truncated: bool,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct ProviderRoute {
pub stages: StageAnswers,
pub class: String,
pub close_calls: Vec<Stage>,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
#[serde(untagged)]
pub enum RouteOutcome {
Routed {
providers: BTreeMap<String, ProviderRoute>,
depends_on: Vec<DependencyEntry>,
#[serde(skip_serializing_if = "Vec::is_empty")]
reference_fetch_failures: Vec<ReferenceFetchFailure>,
},
Failed {
error: String,
#[serde(skip_serializing_if = "Vec::is_empty")]
reference_fetch_failures: Vec<ReferenceFetchFailure>,
},
}
impl IssueRoute {
#[must_use]
pub fn failed(&self) -> bool {
matches!(self.outcome, RouteOutcome::Failed { .. })
}
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct RouteReport {
pub model: String,
pub issues: Vec<IssueRoute>,
#[serde(skip_serializing_if = "Vec::is_empty")]
pub ignored_closed: Vec<IgnoredIssue>,
pub usage: Usage,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct IgnoredIssue {
#[serde(rename = "ref")]
pub item_ref: String,
pub url: String,
pub title: String,
}
#[derive(Debug, Clone)]
pub struct RouteOptions {
pub model: String,
pub close_call: f64,
pub effort_advice: bool,
pub max_input_chars: usize,
pub allow_closed: bool,
pub ignore_closed: bool,
}
pub async fn run_route(
jev: &JevClient,
docs: &[IssueDoc],
ladders: &[Ladder],
opts: &RouteOptions,
dependencies: &OpenDependencies,
) -> Result<RouteReport> {
run_route_with_reference_fetch_failures(
jev,
docs,
ladders,
opts,
dependencies,
&ReferenceFetchFailures::new(),
)
.await
}
pub async fn run_route_with_reference_fetch_failures(
jev: &JevClient,
docs: &[IssueDoc],
ladders: &[Ladder],
opts: &RouteOptions,
dependencies: &OpenDependencies,
reference_fetch_failures: &ReferenceFetchFailures,
) -> Result<RouteReport> {
let (docs, ignored_closed) = drop_closed(docs, opts)?;
validate_options(&docs, ladders, opts)?;
let questions = build_route_questions_for_mode(ladders, opts.effort_advice)?;
let mut models = BTreeSet::new();
let mut usage = Usage::default();
let mut issues = Vec::with_capacity(docs.len());
for doc in docs {
let item_ref = item_ref(doc);
let (state, truncated) = build_route_state(doc, opts.max_input_chars);
if truncated {
warn!(
"Issue {item_ref} is longer than {} characters; its input was truncated",
opts.max_input_chars
);
}
let citations = dependencies
.get(&(doc.project.clone(), doc.number))
.map_or(&[][..], Vec::as_slice);
let citation_failures = reference_fetch_failures
.get(&(doc.project.clone(), doc.number))
.cloned()
.unwrap_or_default();
let mut request_questions = questions.clone();
for (i, citation) in citations.iter().enumerate() {
request_questions.insert(
could_be_cheaper_key(i),
could_be_cheaper_question(&citation.raw),
);
}
let request = SystemOneRequest {
state: serde_json::Value::String(state),
model: opts.model.clone(),
questions: request_questions,
};
let outcome = match jev.system_one(&request).await {
Err(err) if is_auth_failure(&err) => {
return Err(err).with_context(|| format!("Failed to route issue {item_ref}"));
}
Err(err) => failed(
&item_ref,
&err.context("Jev request failed"),
&citation_failures,
),
Ok(response) => {
models.insert(response.model);
usage.input_tokens += response.usage.input_tokens;
usage.output_tokens += response.usage.output_tokens;
match provider_routes(
&response.answers,
ladders,
opts.close_call,
opts.effort_advice,
) {
Ok(providers) => RouteOutcome::Routed {
providers,
depends_on: dependency_entries(&item_ref, citations, &response.answers),
reference_fetch_failures: citation_failures.clone(),
},
Err(err) => failed(
&item_ref,
&err.context("Unexpected Jev answer"),
&citation_failures,
),
}
}
};
issues.push(IssueRoute {
item_ref,
url: doc.url.clone(),
title: doc.title.clone(),
state: doc.state,
outcome,
truncated,
});
}
Ok(RouteReport {
model: models.into_iter().collect::<Vec<_>>().join(", "),
issues,
ignored_closed,
usage,
})
}
#[must_use]
pub fn is_ignored_closed(doc: &IssueDoc, ignore_closed: bool) -> bool {
ignore_closed && doc.state == ItemState::Closed
}
fn item_ref(doc: &IssueDoc) -> String {
format!("{}#{}", doc.project, doc.number)
}
fn drop_closed<'a>(
docs: &'a [IssueDoc],
opts: &RouteOptions,
) -> Result<(Vec<&'a IssueDoc>, Vec<IgnoredIssue>)> {
if opts.allow_closed && opts.ignore_closed {
bail!("--allow-closed and --ignore-closed cannot be used together");
}
let (open, closed): (Vec<&IssueDoc>, Vec<&IssueDoc>) = docs
.iter()
.partition(|d| !is_ignored_closed(d, opts.ignore_closed));
if open.is_empty() && !closed.is_empty() {
bail!(
"every requested issue is closed ({}) and --ignore-closed skipped them; nothing to \
route",
closed
.iter()
.map(|d| item_ref(d))
.collect::<Vec<_>>()
.join(", ")
);
}
let ignored = closed
.into_iter()
.map(|d| IgnoredIssue {
item_ref: item_ref(d),
url: d.url.clone(),
title: d.title.clone(),
})
.collect();
Ok((open, ignored))
}
fn failed(
item_ref: &str,
err: &anyhow::Error,
reference_fetch_failures: &[ReferenceFetchFailure],
) -> RouteOutcome {
let error = format!("{err:#}");
warn!("Could not route issue {item_ref}: {error}");
RouteOutcome::Failed {
error,
reference_fetch_failures: reference_fetch_failures.to_vec(),
}
}
fn validate_options(docs: &[&IssueDoc], ladders: &[Ladder], opts: &RouteOptions) -> Result<()> {
if docs.is_empty() {
bail!("no issues to route");
}
if ladders.is_empty() {
bail!("no ladders to route against");
}
let mut seen = BTreeSet::new();
for ladder in ladders {
if !seen.insert(ladder.name.as_str()) {
bail!("ladder {:?} is listed more than once", ladder.name);
}
}
if !(0.0..=1.0).contains(&opts.close_call) {
bail!(
"the close-call threshold must be between 0 and 1, got {}",
opts.close_call
);
}
if opts.max_input_chars == 0 {
bail!("the input cap must be at least 1 character");
}
if !opts.allow_closed {
let closed: Vec<String> = docs
.iter()
.filter(|d| d.state == ItemState::Closed)
.map(|d| item_ref(d))
.collect();
if !closed.is_empty() {
bail!(
"refusing to route closed issues ({}): their comments often describe how the \
work was done, which leaks the answer. Pass --ignore-closed to skip them, or \
--allow-closed to route them anyway",
closed.join(", ")
);
}
}
Ok(())
}
fn provider_routes(
answers: &BTreeMap<String, Answer>,
ladders: &[Ladder],
close_call: f64,
effort_advice: bool,
) -> Result<BTreeMap<String, ProviderRoute>> {
ladders
.iter()
.map(|ladder| {
let mut stages = stage_answers(answers, ladder)?;
if effort_advice {
for (stage, answer) in [
(Stage::Design, &mut stages.design),
(Stage::Implement, &mut stages.implement),
(Stage::Review, &mut stages.review),
] {
answer.effort_by_model =
effort::decode(ladder, stage, answer, answers, close_call)?;
}
}
let route = ProviderRoute {
class: issue_class(&stages, &ladder.tiers),
close_calls: close_calls(&stages, close_call),
stages,
};
Ok((ladder.name.clone(), route))
})
.collect()
}
fn stage_answers(answers: &BTreeMap<String, Answer>, ladder: &Ladder) -> Result<StageAnswers> {
let tiers = &ladder.tiers;
let read = |stage: Stage| -> Result<StageAnswer> {
let key = stage.question_key(&ladder.name);
let Some(Answer::Choice {
choice,
confidence,
probabilities,
}) = answers.get(&key)
else {
bail!("no choice answer for {key:?}");
};
let offered = match tiers.rank(choice) {
Some(0) => stage == Stage::Design,
Some(_) => true,
None => false,
};
if !offered {
bail!("{key:?} chose {choice:?}, which was not offered");
}
Ok(StageAnswer {
choice: choice.clone(),
confidence: *confidence,
probabilities: probabilities.clone(),
effort_by_model: BTreeMap::new(),
})
};
Ok(StageAnswers {
design: read(Stage::Design)?,
implement: read(Stage::Implement)?,
review: read(Stage::Review)?,
})
}
fn issue_class(stages: &StageAnswers, tiers: &Tiers) -> String {
[&stages.design, &stages.implement]
.into_iter()
.max_by_key(|a| tiers.rank(&a.choice).unwrap_or_default())
.map(|a| a.choice.clone())
.unwrap_or_default()
}
fn close_calls(stages: &StageAnswers, threshold: f64) -> Vec<Stage> {
Stage::ALL
.into_iter()
.filter(|&s| stages.get(s).confidence < threshold)
.collect()
}
fn could_be_cheaper_key(i: usize) -> String {
format!("could_be_cheaper_{i}")
}
fn could_be_cheaper_question(citation: &str) -> Question {
Question::Noul {
instructions: format!(
"This issue cites {citation}, which is still open. If {citation} is resolved, how \
likely is it that LESS design work would remain for THIS issue than the current \
text implies — as opposed to this issue's own remaining work being unaffected, \
because it is already scoped separately, is a parallel/sibling effort, or \
{citation} is otherwise not a precondition for finishing this issue's own \
remaining work?"
),
criteria: None,
}
}
fn dependency_entries(
item_ref: &str,
citations: &[Citation],
answers: &BTreeMap<String, Answer>,
) -> Vec<DependencyEntry> {
citations
.iter()
.enumerate()
.filter_map(|(i, citation)| {
if let Some(Answer::Noul { noul }) = answers.get(&could_be_cheaper_key(i)) {
return Some(DependencyEntry {
item_ref: citation.raw.clone(),
url: citation.url.clone(),
state: ItemState::Open,
could_be_cheaper: BTreeMap::from([("design".to_string(), *noul)]),
});
}
warn!(
"Issue {item_ref}: no could_be_cheaper answer for citation {:?}",
citation.raw
);
None
})
.collect()
}
#[must_use]
pub fn render_route_text(report: &RouteReport, max_input_chars: usize) -> String {
render_route_text_styled(report, max_input_chars, &[], TerminalStyle::default())
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct TerminalStyle {
pub color: bool,
pub hyperlinks: bool,
}
impl TerminalStyle {
#[must_use]
pub fn new(tty: bool, term: &str, no_color: bool, supports_links: bool) -> Self {
let usable = tty && !term.is_empty() && term != "dumb";
Self {
color: usable && !no_color,
hyperlinks: usable && supports_links,
}
}
}
#[must_use]
pub fn render_route_text_styled(
report: &RouteReport,
max_input_chars: usize,
ladders: &[Ladder],
style: TerminalStyle,
) -> String {
let mut blocks: Vec<String> = report
.issues
.iter()
.map(|issue| render_issue_block(issue, max_input_chars, ladders, style))
.collect();
if !report.ignored_closed.is_empty() {
let refs: Vec<String> = report
.ignored_closed
.iter()
.map(|issue| hyperlink(&issue.item_ref, Some(&issue.url), style))
.collect();
blocks.push(format!("ignored closed: {}", refs.join(", ")));
}
blocks.push(format!(
"model: {}, usage: {} input tokens, {} output tokens",
report.model, report.usage.input_tokens, report.usage.output_tokens
));
let mut text = blocks.join("\n\n");
text.push('\n');
text
}
fn render_issue_block(
issue: &IssueRoute,
max_input_chars: usize,
ladders: &[Ladder],
style: TerminalStyle,
) -> String {
let item_ref = hyperlink(&issue.item_ref, Some(&issue.url), style);
let item_ref = if issue.failed() {
item_ref
} else {
colorize(&item_ref, Complexity::Low, style)
};
let header = if issue.state == ItemState::Closed {
format!("{item_ref} — {} (closed)", issue.title)
} else {
format!("{item_ref} — {}", issue.title)
};
let mut lines = vec![header];
match &issue.outcome {
RouteOutcome::Routed {
providers,
depends_on,
reference_fetch_failures,
} => {
for (provider, route) in providers {
let ladder = ladders.iter().find(|ladder| ladder.name == *provider);
lines.extend(render_provider_line(
provider,
route,
providers.len(),
ladder,
style,
));
}
lines.extend(render_depends_on_lines(depends_on, style));
lines.extend(render_reference_fetch_failure_lines(
reference_fetch_failures,
));
}
RouteOutcome::Failed {
error,
reference_fetch_failures,
} => {
lines.push(format!(
" {}",
colorize(&format!("failed: {error}"), Complexity::High, style)
));
lines.extend(render_reference_fetch_failure_lines(
reference_fetch_failures,
));
}
}
if issue.truncated {
lines.push(format!(
" input truncated at {} characters",
with_thousands(max_input_chars)
));
}
lines.join("\n")
}
fn route_has_multi_model_tier(route: &ProviderRoute) -> bool {
[
&route.stages.design,
&route.stages.implement,
&route.stages.review,
]
.into_iter()
.any(|answer| answer.choice.contains(','))
}
fn render_provider_line(
provider: &str,
route: &ProviderRoute,
provider_count: usize,
ladder: Option<&Ladder>,
style: TerminalStyle,
) -> Vec<String> {
let mut lines = if route_has_multi_model_tier(route) {
render_provider_block(provider, route, provider_count, ladder, style)
} else {
vec![render_provider_line_compact(
provider,
route,
provider_count,
ladder,
style,
)]
};
lines.extend(effort::render(&route.stages, ladder));
lines
}
fn render_provider_line_compact(
provider: &str,
route: &ProviderRoute,
provider_count: usize,
ladder: Option<&Ladder>,
style: TerminalStyle,
) -> String {
let class = color_choice(&route.class, ladder, style);
let lead = if provider_count > 1 {
format!("{provider}: {class}")
} else {
class
};
let clauses: Vec<String> = Stage::ALL
.into_iter()
.map(|stage| render_stage_clause(stage, &route.stages, &route.close_calls, ladder, style))
.collect();
format!(" {lead} — {}", clauses.join(", "))
}
fn render_stage_clause(
stage: Stage,
stages: &StageAnswers,
close_calls: &[Stage],
ladder: Option<&Ladder>,
style: TerminalStyle,
) -> String {
let answer = stages.get(stage);
let choice = color_choice(&answer.choice, ladder, style);
let label = match stage {
Stage::Design if answer.choice == NO_DESIGN => format!(
"design needs {}",
colorize("no further work", Complexity::Low, style)
),
Stage::Design => format!("design needs {choice}"),
Stage::Implement => format!("implementation {choice}"),
Stage::Review => format!("review {choice}"),
};
format!(
"{label} ({})",
render_stage_confidence(answer, close_calls.contains(&stage))
)
}
fn render_stage_confidence(answer: &StageAnswer, close_call: bool) -> String {
let mut detail = format!("{:.2}", answer.confidence);
if close_call {
detail.push_str(", close call");
if let Some((name, probability)) = answer
.probabilities
.iter()
.filter(|(name, probability)| name.as_str() != answer.choice && probability.is_finite())
.max_by(|a, b| a.1.total_cmp(b.1).then_with(|| b.0.cmp(a.0)))
{
detail.push_str(&format!(" — {name} {probability:.2}"));
}
}
detail
}
fn render_provider_block(
provider: &str,
route: &ProviderRoute,
provider_count: usize,
ladder: Option<&Ladder>,
style: TerminalStyle,
) -> Vec<String> {
let mut lines = Vec::with_capacity(5);
let indent = if provider_count > 1 {
lines.push(format!(" {provider}:"));
" "
} else {
" "
};
lines.push(format!(
"{indent}class: {}",
color_choice(&route.class, ladder, style)
));
for stage in Stage::ALL {
lines.push(format!(
"{indent}{}",
render_stage_line(stage, &route.stages, &route.close_calls, ladder, style)
));
}
lines
}
fn render_stage_line(
stage: Stage,
stages: &StageAnswers,
close_calls: &[Stage],
ladder: Option<&Ladder>,
style: TerminalStyle,
) -> String {
let answer = stages.get(stage);
let choice = color_choice(&answer.choice, ladder, style);
let (label, content) = match stage {
Stage::Design if answer.choice == NO_DESIGN => (
"design",
format!(
"needs {}",
colorize("no further work", Complexity::Low, style)
),
),
Stage::Design => ("design", format!("needs {choice}")),
Stage::Implement => ("implementation", choice),
Stage::Review => ("review", choice),
};
format!(
"{label}: {content} ({})",
render_stage_confidence(answer, close_calls.contains(&stage))
)
}
fn render_depends_on_lines(depends_on: &[DependencyEntry], style: TerminalStyle) -> Vec<String> {
depends_on
.iter()
.map(|dep| {
let item_ref = hyperlink(&dep.item_ref, dep.url.as_deref(), style);
match dep.could_be_cheaper.get("design") {
Some(prob) => format!(
" cites open {item_ref}, which could leave less design work if resolved ({prob:.2})"
),
None => format!(" cites open {item_ref}"),
}
})
.collect()
}
fn render_reference_fetch_failure_lines(failures: &[ReferenceFetchFailure]) -> Vec<String> {
failures
.iter()
.map(|failure| {
format!(
" reference fetch failed: {} ({})",
failure.item_ref, failure.error
)
})
.collect()
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Complexity {
Low,
Medium,
High,
}
fn choice_complexity(choice: &str, ladder: &Ladder) -> Option<Complexity> {
if choice == NO_DESIGN {
return Some(Complexity::Low);
}
let tiers = ladder.tiers.as_slice();
let index = tiers.iter().position(|tier| tier.name == choice)?;
Some(if index == 0 {
Complexity::Low
} else if index == tiers.len() - 1 {
Complexity::High
} else {
Complexity::Medium
})
}
fn color_choice(choice: &str, ladder: Option<&Ladder>, style: TerminalStyle) -> String {
ladder
.and_then(|ladder| choice_complexity(choice, ladder))
.map_or_else(
|| choice.to_string(),
|level| colorize(choice, level, style),
)
}
fn colorize(text: &str, level: Complexity, style: TerminalStyle) -> String {
if !style.color {
return text.to_string();
}
let code = match level {
Complexity::Low => 32,
Complexity::Medium => 33,
Complexity::High => 31,
};
format!("\x1b[{code}m{text}\x1b[0m")
}
fn hyperlink(text: &str, url: Option<&str>, style: TerminalStyle) -> String {
let Some(url) = url.filter(|url| style.hyperlinks && valid_github_item_url(url)) else {
return text.to_string();
};
format!("\x1b]8;;{url}\x1b\\{text}\x1b]8;;\x1b\\")
}
fn valid_github_item_url(raw: &str) -> bool {
if raw.chars().any(char::is_control) {
return false;
}
let Ok(url) = url::Url::parse(raw) else {
return false;
};
if url.scheme() != "https" || url.host_str() != Some("github.com") {
return false;
}
let Some(segments) = url.path_segments() else {
return false; };
let parts: Vec<_> = segments.collect();
parts.len() == 4
&& !parts[0].is_empty()
&& !parts[1].is_empty()
&& matches!(parts[2], "issues" | "pull")
&& parts[3].parse::<u64>().is_ok()
}
fn with_thousands(n: usize) -> String {
let digits = n.to_string();
let mut grouped = String::with_capacity(digits.len() + digits.len() / 3);
for (i, ch) in digits.chars().rev().enumerate() {
if i > 0 && i % 3 == 0 {
grouped.push(',');
}
grouped.push(ch);
}
grouped.chars().rev().collect()
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used)]
mod tests {
use super::*;
use crate::provider::{Comment, GitProvider, ItemKind, ItemRef};
fn doc(number: u64, state: ItemState) -> IssueDoc {
IssueDoc {
provider: GitProvider::GitHub,
project: "rust-works/omni-dev".to_string(),
number,
kind: ItemKind::Issue,
title: "Route issues".to_string(),
state,
body: " The body.\n".to_string(),
comments: vec![],
closed_by: vec![],
url: format!("https://github.com/rust-works/omni-dev/issues/{number}"),
}
}
fn opts() -> RouteOptions {
RouteOptions {
model: "jev-latest".to_string(),
close_call: DEFAULT_CLOSE_CALL,
effort_advice: false,
max_input_chars: DEFAULT_MAX_INPUT_CHARS,
allow_closed: false,
ignore_closed: false,
}
}
fn default_tiers() -> Tiers {
Provider::Anthropic.tiers().unwrap()
}
fn class_only(provider: Provider) -> Result<Ladder> {
let mut ladder = Ladder::builtin(provider)?;
for tier in &mut ladder.tiers.0 {
tier.models = None;
}
Ok(ladder)
}
fn anthropic() -> Vec<Ladder> {
vec![class_only(Provider::Anthropic).unwrap()]
}
fn tier_names(tiers: &Tiers) -> Vec<&str> {
tiers.as_slice().iter().map(|t| t.name.as_str()).collect()
}
#[test]
fn anthropic_tiers_are_sonnet_opus_in_order() {
assert_eq!(tier_names(&default_tiers()), ["sonnet", "opus"]);
}
#[test]
fn anthropic_tier_descriptions_are_the_tested_text() {
let tiers = default_tiers();
let opus = &tiers.as_slice()[1];
assert_eq!(
opus.description,
"Reliable at reasoning across many files, subtle library or platform semantics, \
concurrency and ordering, once the overall direction is set. Can under-explore a \
genuinely open design space."
);
}
#[test]
fn provider_names_are_the_documented_abbreviations() {
assert_eq!(
tier_names(&Provider::OpenAi.tiers().unwrap()),
["terra", "sol", "astra"]
);
assert_eq!(
tier_names(&Provider::Gemini.tiers().unwrap()),
["flash", "pro", "deep-think"]
);
assert_eq!(
Provider::ALL.map(Provider::name),
["anthropic", "openai", "gemini"]
);
for provider in Provider::ALL {
assert_eq!(
provider.to_possible_value().unwrap().get_name(),
provider.name()
);
}
}
#[test]
fn openai_and_gemini_ladders_reuse_the_anthropic_descriptions_rung_for_rung() {
let anthropic = default_tiers();
const RETIRED_ANTHROPIC_THIRD_RUNG: &str =
"Strongest at open-ended design and research: choosing between architectures, \
working in unfamiliar territory with no precedent in the codebase, anticipating \
failure modes, and security-critical judgement.";
for provider in [Provider::OpenAi, Provider::Gemini] {
let tiers = provider.tiers().unwrap();
assert_eq!(tiers.as_slice().len(), 3);
for (rung, reference) in tiers.as_slice().iter().zip(anthropic.as_slice()) {
assert_eq!(
rung.description,
reference.description,
"{}: {} differs from {}",
provider.name(),
rung.name,
reference.name
);
}
assert_eq!(
tiers.as_slice()[2].description,
RETIRED_ANTHROPIC_THIRD_RUNG,
"{}: {} no longer matches the retired anthropic fable rung",
provider.name(), tiers.as_slice()[2].name );
}
}
#[test]
fn ladders_are_named_by_provider_or_a_custom_name() {
let ladder = class_only(Provider::Gemini).unwrap();
assert_eq!(ladder.name, "gemini");
let custom = Ladder::named("mine".to_string(), default_tiers());
assert_eq!(custom.name, "mine");
assert_eq!(custom.tiers, default_tiers());
}
#[test]
fn tiers_need_at_least_two() {
let err = Tiers::parse("tiers:\n - {name: a, description: A}\n").unwrap_err();
assert!(err.to_string().contains("at least 2 tiers"), "{err}");
}
#[test]
fn tiers_reject_the_reserved_none() {
let err = Tiers::parse(
"tiers:\n - {name: a, description: A}\n - {name: none, description: B}\n",
)
.unwrap_err();
assert!(err.to_string().contains("reserved"), "{err}");
}
#[test]
fn tiers_reject_duplicate_names() {
let err =
Tiers::parse("tiers:\n - {name: a, description: A}\n - {name: a, description: B}\n")
.unwrap_err();
assert!(err.to_string().contains("more than once"), "{err}");
}
#[test]
fn tiers_reject_empty_descriptions() {
let err =
Tiers::parse("tiers:\n - {name: a, description: A}\n - {name: b, description: ''}\n")
.unwrap_err();
assert!(err.to_string().contains("non-empty"), "{err}");
}
#[test]
fn tiers_reject_unknown_fields() {
let err = Tiers::parse(
"tiers:\n - {name: a, description: A, rank: 1}\n - {name: b, description: B}\n",
)
.unwrap_err();
assert!(format!("{err:#}").contains("unknown field"), "{err:#}");
}
#[test]
fn tiers_load_reads_a_custom_file() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("tiers.yaml");
std::fs::write(
&path,
"tiers:\n - {name: small, description: S}\n - {name: big, description: B}\n",
)
.unwrap();
let tiers = Tiers::load_file(&path).unwrap();
assert_eq!(tiers.as_slice()[1].name, "big");
}
#[test]
fn tiers_load_names_a_missing_file() {
let err = Tiers::load_file(Path::new("/no/such/tiers.yaml")).unwrap_err();
assert!(err.to_string().contains("Failed to read tiers file"));
}
const BAR: &str = " Choose the least capable class likely to complete this stage correctly \
with no rework, about 9 times in 10. Judge the work that remains given \
the text, not the size of the text.";
fn instructions(questions: &BTreeMap<String, Question>, key: &str) -> String {
let Question::Choice { instructions, .. } = &questions[key] else {
panic!("{key} is not a choice");
};
instructions.clone()
}
fn options(questions: &BTreeMap<String, Question>, key: &str) -> Vec<String> {
let Question::Choice { criteria, .. } = &questions[key] else {
panic!("{key} is not a choice");
};
criteria.keys().cloned().collect()
}
#[test]
fn stage_questions_are_the_tested_wording() {
let questions = build_route_questions(&anthropic()).unwrap();
assert_eq!(
instructions(&questions, "anthropic.stage_design"),
format!(
"Which class should do the design work that remains before implementation can \
start: choosing the approach, settling open questions, and writing a plan?{BAR}"
)
);
assert_eq!(
instructions(&questions, "anthropic.stage_implement"),
format!(
"Assume any remaining design work has been completed well. Which class should \
write the code, tests and docs?{BAR}"
)
);
assert_eq!(
instructions(&questions, "anthropic.stage_review"),
format!(
"Which class should review the finished change before merge, so that mistakes \
the automated tests would miss are caught?{BAR}"
)
);
}
#[test]
fn only_the_design_stage_offers_none() {
let questions = build_route_questions(&anthropic()).unwrap();
assert_eq!(
options(&questions, "anthropic.stage_design"),
["none", "opus", "sonnet"]
);
assert_eq!(
options(&questions, "anthropic.stage_implement"),
["opus", "sonnet"]
);
assert_eq!(
options(&questions, "anthropic.stage_review"),
["opus", "sonnet"]
);
let Question::Choice { criteria, .. } = &questions["anthropic.stage_design"] else {
panic!();
};
assert_eq!(
criteria[NO_DESIGN],
"No design work remains: the text already settles the approach and the open \
questions."
);
}
#[test]
fn questions_are_keyed_per_provider_in_one_map() {
let ladders = [
class_only(Provider::Anthropic).unwrap(),
class_only(Provider::OpenAi).unwrap(),
];
let questions = build_route_questions(&ladders).unwrap();
let keys: Vec<&String> = questions.keys().collect();
assert_eq!(
keys,
[
"anthropic.stage_design",
"anthropic.stage_implement",
"anthropic.stage_review",
"openai.stage_design",
"openai.stage_implement",
"openai.stage_review",
]
);
assert_eq!(
options(&questions, "openai.stage_design"),
["astra", "none", "sol", "terra"]
);
assert_eq!(
instructions(&questions, "openai.stage_review"),
instructions(&questions, "anthropic.stage_review")
);
}
#[test]
fn a_named_ladder_is_keyed_by_its_own_name() {
let ladders = [Ladder::named(
"mine".to_string(),
Tiers::parse("tiers:\n - {name: a, description: A}\n - {name: b, description: B}\n")
.unwrap(),
)];
let questions = build_route_questions(&ladders).unwrap();
assert_eq!(options(&questions, "mine.stage_implement"), ["a", "b"]);
}
#[test]
fn route_state_without_comments_is_title_and_trimmed_body() {
let (state, truncated) = build_route_state(&doc(7, ItemState::Open), 1000);
assert_eq!(state, "# #7 Route issues\n\nThe body.\n");
assert!(!truncated);
}
#[test]
fn route_state_lists_comments_in_order_in_the_tested_format() {
let mut d = doc(7, ItemState::Open);
d.comments = vec![
Comment {
author: "alice".to_string(),
body: "first\n".to_string(),
id: None,
},
Comment {
author: "bob".to_string(),
body: "second".to_string(),
id: None,
},
];
let (state, _) = build_route_state(&d, 1000);
assert_eq!(
state,
"# #7 Route issues\n\nThe body.\n\n\n---\n\n## Comments on this issue\n\
\n\n**Comment by alice:**\n\nfirst\n\
\n\n**Comment by bob:**\n\nsecond\n"
);
}
#[test]
fn route_state_truncates_the_middle_on_a_char_boundary() {
let mut d = doc(7, ItemState::Open);
d.body = "é".repeat(100);
let (full, _) = build_route_state(&d, usize::MAX);
let (state, truncated) = build_route_state(&d, 20);
assert!(truncated);
let (head, tail) = state
.split_once(&format!("\n\n{TRUNCATION_MARKER}\n\n"))
.unwrap();
assert_eq!(head.chars().count(), 10);
assert_eq!(tail.chars().count(), 10);
assert!(full.starts_with(head));
assert!(full.ends_with(tail));
}
#[test]
fn route_state_truncation_keeps_the_latest_comment() {
let mut d = doc(7, ItemState::Open);
d.body = "x".repeat(10_000);
d.comments = vec![Comment {
author: "maintainer".to_string(),
body: "**Decision**: settled.".to_string(),
id: None,
}];
let (state, truncated) = build_route_state(&d, 1_000);
assert!(truncated);
assert!(state.starts_with("# #7 Route issues"));
assert!(state.ends_with("**Decision**: settled.\n"), "{state}");
}
#[test]
fn route_state_truncation_handles_a_one_char_cap() {
let (state, truncated) = build_route_state(&doc(7, ItemState::Open), 1);
assert!(truncated);
assert_eq!(state, format!("\n\n{TRUNCATION_MARKER}\n\n\n"));
}
#[test]
fn route_state_at_exactly_the_cap_is_not_truncated() {
let d = doc(7, ItemState::Open);
let (full, _) = build_route_state(&d, usize::MAX);
let (state, truncated) = build_route_state(&d, full.chars().count());
assert_eq!(state, full);
assert!(!truncated);
}
fn answer(choice: &str, confidence: f64) -> StageAnswer {
StageAnswer {
effort_by_model: BTreeMap::new(),
choice: choice.to_string(),
confidence,
probabilities: BTreeMap::new(),
}
}
fn stages(design: &str, implement: &str, review: &str) -> StageAnswers {
StageAnswers {
design: answer(design, 0.9),
implement: answer(implement, 0.9),
review: answer(review, 0.9),
}
}
#[test]
fn close_call_shows_the_best_alternative_in_compact_text() {
let mut stages = stages("sol", "terra", "sol");
stages.design.confidence = 0.21;
stages.design.probabilities = BTreeMap::from([
("none".to_string(), 0.44),
("sol".to_string(), 0.48),
("terra".to_string(), 0.08),
]);
assert_eq!(
render_stage_clause(
Stage::Design,
&stages,
&[Stage::Design],
None,
TerminalStyle::default(),
),
"design needs sol (0.21, close call — none 0.44)"
);
assert_eq!(
render_stage_clause(Stage::Design, &stages, &[], None, TerminalStyle::default(),),
"design needs sol (0.21)"
);
}
#[test]
fn close_call_uses_a_stable_tiebreaker_and_handles_missing_alternatives() {
let mut answer = answer("sol", 0.21);
answer.probabilities = BTreeMap::from([
("astra".to_string(), 0.4),
("sol".to_string(), 0.2),
("terra".to_string(), 0.4),
]);
assert_eq!(
render_stage_confidence(&answer, true),
"0.21, close call — astra 0.40"
);
answer.probabilities.clear();
assert_eq!(render_stage_confidence(&answer, true), "0.21, close call");
}
#[test]
fn class_is_the_higher_of_design_and_implement() {
let tiers = default_tiers();
assert_eq!(
issue_class(&stages("opus", "sonnet", "sonnet"), &tiers),
"opus"
);
assert_eq!(
issue_class(&stages("sonnet", "opus", "sonnet"), &tiers),
"opus"
);
}
#[test]
fn class_ignores_review_and_none() {
let tiers = default_tiers();
assert_eq!(
issue_class(&stages("none", "sonnet", "opus"), &tiers),
"sonnet"
);
}
#[test]
fn close_calls_are_below_the_threshold_in_stage_order() {
let mut s = stages("opus", "sonnet", "opus");
s.review.confidence = 0.1;
s.design.confidence = 0.29;
s.implement.confidence = 0.3;
assert_eq!(close_calls(&s, 0.3), [Stage::Design, Stage::Review]);
}
#[test]
fn could_be_cheaper_key_is_stable_and_indexed() {
assert_eq!(could_be_cheaper_key(0), "could_be_cheaper_0");
assert_eq!(could_be_cheaper_key(1), "could_be_cheaper_1");
}
#[test]
fn could_be_cheaper_question_is_the_tested_wording() {
let Question::Noul {
instructions,
criteria,
} = could_be_cheaper_question("#1129")
else {
panic!("expected a noul question");
};
assert_eq!(
instructions,
"This issue cites #1129, which is still open. If #1129 is resolved, how likely is \
it that LESS design work would remain for THIS issue than the current text \
implies — as opposed to this issue's own remaining work being unaffected, because \
it is already scoped separately, is a parallel/sibling effort, or #1129 is \
otherwise not a precondition for finishing this issue's own remaining work?"
);
assert!(criteria.is_none());
}
#[test]
fn dependency_entries_reads_the_noul_answer_per_citation() {
let citations = vec![
Citation {
item_ref: ItemRef {
provider: GitProvider::GitHub,
project: "rust-works/omni-dev".to_string(),
kind: ItemKind::Issue,
number: 1129,
},
raw: "#1129".to_string(),
url: None,
},
Citation {
item_ref: ItemRef {
provider: GitProvider::GitHub,
project: "rust-works/omni-dev".to_string(),
kind: ItemKind::Issue,
number: 1349,
},
raw: "#1349".to_string(),
url: None,
},
];
let answers = BTreeMap::from([(
"could_be_cheaper_0".to_string(),
Answer::Noul { noul: 0.75 },
)]);
let deps = dependency_entries("o/r#1", &citations, &answers);
assert_eq!(deps.len(), 1, "{deps:?}");
assert_eq!(deps[0].item_ref, "#1129");
assert_eq!(deps[0].state, ItemState::Open);
assert_eq!(deps[0].could_be_cheaper.get("design"), Some(&0.75));
}
#[test]
fn stage_answers_reject_none_outside_design() {
let choice = |c: &str| Answer::Choice {
choice: c.to_string(),
confidence: 0.5,
probabilities: BTreeMap::new(),
};
let answers = BTreeMap::from([
("anthropic.stage_design".to_string(), choice("none")),
("anthropic.stage_implement".to_string(), choice("none")),
("anthropic.stage_review".to_string(), choice("opus")),
]);
let err = stage_answers(&answers, &anthropic()[0]).unwrap_err();
assert!(err.to_string().contains("not offered"), "{err}");
}
#[test]
fn stage_answers_require_a_choice_per_stage() {
let answers = BTreeMap::from([(
"anthropic.stage_design".to_string(),
Answer::Noul { noul: 0.5 },
)]);
let err = stage_answers(&answers, &anthropic()[0]).unwrap_err();
assert!(err.to_string().contains("no choice answer"), "{err}");
assert!(err.to_string().contains("anthropic.stage_design"), "{err}");
}
#[test]
fn stage_answers_are_read_under_the_ladders_prefix() {
let choice = |c: &str| Answer::Choice {
choice: c.to_string(),
confidence: 0.5,
probabilities: BTreeMap::new(),
};
let answers = BTreeMap::from([
("anthropic.stage_design".to_string(), choice("none")),
("anthropic.stage_implement".to_string(), choice("sonnet")),
("anthropic.stage_review".to_string(), choice("opus")),
]);
let err = stage_answers(&answers, &class_only(Provider::OpenAi).unwrap()).unwrap_err();
assert!(err.to_string().contains("openai.stage_design"), "{err}");
}
#[test]
fn closed_issues_are_refused_by_default() {
let err =
validate_options(&[&doc(1, ItemState::Closed)], &anthropic(), &opts()).unwrap_err();
assert!(err.to_string().contains("rust-works/omni-dev#1"), "{err}");
assert!(err.to_string().contains("--allow-closed"), "{err}");
assert!(err.to_string().contains("--ignore-closed"), "{err}");
}
#[test]
fn allow_closed_and_ignore_closed_are_mutually_exclusive() {
let mut o = opts();
o.allow_closed = true;
o.ignore_closed = true;
for state in [ItemState::Open, ItemState::Closed] {
let err = drop_closed(&[doc(1, state)], &o).unwrap_err();
assert!(err.to_string().contains("cannot be used together"), "{err}");
}
}
#[test]
fn drop_closed_keeps_every_issue_without_the_flag() {
let docs = [doc(1, ItemState::Open), doc(2, ItemState::Closed)];
let (kept, ignored) = drop_closed(&docs, &opts()).unwrap();
assert_eq!(kept.len(), 2);
assert!(ignored.is_empty());
}
#[test]
fn drop_closed_skips_closed_issues_in_request_order() {
let mut o = opts();
o.ignore_closed = true;
let docs = [
doc(3, ItemState::Closed),
doc(1, ItemState::Open),
doc(2, ItemState::Closed),
];
let (kept, ignored) = drop_closed(&docs, &o).unwrap();
assert_eq!(kept.iter().map(|d| d.number).collect::<Vec<_>>(), [1]);
assert_eq!(
ignored,
[
IgnoredIssue {
item_ref: "rust-works/omni-dev#3".to_string(),
url: "https://github.com/rust-works/omni-dev/issues/3".to_string(),
title: "Route issues".to_string(),
},
IgnoredIssue {
item_ref: "rust-works/omni-dev#2".to_string(),
url: "https://github.com/rust-works/omni-dev/issues/2".to_string(),
title: "Route issues".to_string(),
},
]
);
}
#[test]
fn drop_closed_refuses_a_batch_with_no_open_issue_left() {
let mut o = opts();
o.ignore_closed = true;
let docs = [doc(1, ItemState::Closed), doc(2, ItemState::Closed)];
let err = drop_closed(&docs, &o).unwrap_err();
let msg = err.to_string();
assert!(
msg.contains("rust-works/omni-dev#1, rust-works/omni-dev#2"),
"{msg}"
);
assert!(msg.contains("nothing to route"), "{msg}");
}
#[test]
fn drop_closed_passes_an_empty_batch_through() {
let mut o = opts();
o.ignore_closed = true;
let (kept, ignored) = drop_closed(&[], &o).unwrap();
assert!(kept.is_empty());
assert!(ignored.is_empty());
}
#[test]
fn closed_issues_are_allowed_on_request() {
let mut o = opts();
o.allow_closed = true;
validate_options(&[&doc(1, ItemState::Closed)], &anthropic(), &o).unwrap();
}
#[test]
fn empty_runs_and_bad_knobs_are_rejected() {
assert!(validate_options(&[], &anthropic(), &opts()).is_err());
let mut o = opts();
o.close_call = 1.5;
assert!(validate_options(&[&doc(1, ItemState::Open)], &anthropic(), &o).is_err());
let mut o = opts();
o.max_input_chars = 0;
assert!(validate_options(&[&doc(1, ItemState::Open)], &anthropic(), &o).is_err());
}
#[test]
fn empty_and_duplicate_providers_are_rejected() {
let err = validate_options(&[&doc(1, ItemState::Open)], &[], &opts()).unwrap_err();
assert!(err.to_string().contains("no ladders"), "{err}");
let twice = [
class_only(Provider::Gemini).unwrap(),
class_only(Provider::Gemini).unwrap(),
];
let err = validate_options(&[&doc(1, ItemState::Open)], &twice, &opts()).unwrap_err();
assert!(
err.to_string()
.contains("\"gemini\" is listed more than once"),
"{err}"
);
}
fn choice_json(choice: &str, confidence: f64) -> serde_json::Value {
serde_json::json!({
"type": "choice", "choice": choice, "confidence": confidence,
"probabilities": {choice: confidence}
})
}
fn provider<'a>(outcome: &'a RouteOutcome, name: &str) -> &'a ProviderRoute {
let RouteOutcome::Routed { providers, .. } = outcome else {
panic!("expected a routed issue: {outcome:?}");
};
&providers[name]
}
#[tokio::test]
async fn run_route_asks_three_stages_per_issue_and_sums_usage() {
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("POST"))
.and(wiremock::matchers::path("/v1/systemone"))
.and(wiremock::matchers::body_partial_json(serde_json::json!({
"model": "jev-latest",
"state": "# #7 Route issues\n\nThe body.\n",
})))
.respond_with(
wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!({
"model": "jev-1.13.0",
"answers": {
"anthropic.stage_design": choice_json("opus", 0.52),
"anthropic.stage_implement": choice_json("sonnet", 0.83),
"anthropic.stage_review": choice_json("opus", 0.21),
},
"usage": {"input_tokens": 100, "output_tokens": 10}
})),
)
.expect(2)
.mount(&server)
.await;
let client = JevClient::new(&server.uri(), "key").unwrap();
let report = run_route(
&client,
&[doc(7, ItemState::Open), doc(7, ItemState::Open)],
&anthropic(),
&opts(),
&OpenDependencies::new(),
)
.await
.unwrap();
assert_eq!(report.model, "jev-1.13.0");
assert_eq!(report.usage.input_tokens, 200);
assert_eq!(report.usage.output_tokens, 20);
let issue = &report.issues[0];
assert_eq!(issue.item_ref, "rust-works/omni-dev#7");
let route = provider(&issue.outcome, "anthropic");
assert_eq!(route.class, "opus");
assert_eq!(route.close_calls, [Stage::Review]);
assert!(!issue.truncated);
let requests = server.received_requests().await.unwrap();
let body: serde_json::Value = requests[0].body_json().unwrap();
let keys: Vec<&String> = body["questions"].as_object().unwrap().keys().collect();
assert_eq!(
keys,
[
"anthropic.stage_design",
"anthropic.stage_implement",
"anthropic.stage_review"
]
);
}
#[tokio::test]
async fn run_route_ignore_closed_routes_only_open_issues() {
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("POST"))
.respond_with(
wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!({
"model": "jev-1.13.0",
"answers": {
"anthropic.stage_design": choice_json("opus", 0.52),
"anthropic.stage_implement": choice_json("sonnet", 0.83),
"anthropic.stage_review": choice_json("opus", 0.61),
},
"usage": {"input_tokens": 100, "output_tokens": 10}
})),
)
.expect(2)
.mount(&server)
.await;
let client = JevClient::new(&server.uri(), "key").unwrap();
let mut o = opts();
o.ignore_closed = true;
let report = run_route(
&client,
&[
doc(7, ItemState::Open),
doc(8, ItemState::Closed),
doc(9, ItemState::Open),
],
&anthropic(),
&o,
&OpenDependencies::new(),
)
.await
.unwrap();
let refs: Vec<&str> = report.issues.iter().map(|i| i.item_ref.as_str()).collect();
assert_eq!(refs, ["rust-works/omni-dev#7", "rust-works/omni-dev#9"]);
let json = serde_json::to_value(&report).unwrap();
assert_eq!(
json["ignored_closed"],
serde_json::json!([{
"ref": "rust-works/omni-dev#8",
"url": "https://github.com/rust-works/omni-dev/issues/8",
"title": "Route issues",
}])
);
let err = run_route(
&client,
&[doc(8, ItemState::Closed)],
&anthropic(),
&o,
&OpenDependencies::new(),
)
.await
.unwrap_err();
assert!(err.to_string().contains("nothing to route"), "{err}");
}
#[test]
fn route_report_omits_an_empty_ignored_closed() {
let report = RouteReport {
model: "m".to_string(),
issues: vec![],
ignored_closed: vec![],
usage: Usage::default(),
};
let json = serde_json::to_value(&report).unwrap();
assert!(json.get("ignored_closed").is_none(), "{json}");
}
#[tokio::test]
async fn run_route_routes_every_provider_in_one_call() {
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("POST"))
.respond_with(
wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!({
"model": "jev-1.13.0",
"answers": {
"anthropic.stage_design": choice_json("opus", 0.52),
"anthropic.stage_implement": choice_json("sonnet", 0.83),
"anthropic.stage_review": choice_json("opus", 0.9),
"openai.stage_design": choice_json("none", 0.9),
"openai.stage_implement": choice_json("sol", 0.2),
"openai.stage_review": choice_json("astra", 0.9),
"could_be_cheaper_0": {"type": "noul", "noul": 0.4},
},
"usage": {"input_tokens": 10, "output_tokens": 1}
})),
)
.expect(1)
.mount(&server)
.await;
let client = JevClient::new(&server.uri(), "key").unwrap();
let ladders = [
class_only(Provider::Anthropic).unwrap(),
class_only(Provider::OpenAi).unwrap(),
];
let dependencies = OpenDependencies::from([(
("rust-works/omni-dev".to_string(), 7),
vec![citation("#1129", 1129)],
)]);
let reference_fetch_failures = ReferenceFetchFailures::from([(
("rust-works/omni-dev".to_string(), 7),
vec![ReferenceFetchFailure {
item_ref: "#404".to_string(),
error: "not found".to_string(),
}],
)]);
let report = run_route_with_reference_fetch_failures(
&client,
&[doc(7, ItemState::Open)],
&ladders,
&opts(),
&dependencies,
&reference_fetch_failures,
)
.await
.unwrap();
let outcome = &report.issues[0].outcome;
let RouteOutcome::Routed {
providers,
depends_on,
reference_fetch_failures,
..
} = outcome
else {
panic!("expected a routed issue: {outcome:?}");
};
assert_eq!(
providers.keys().collect::<Vec<_>>(),
["anthropic", "openai"]
);
assert_eq!(provider(outcome, "anthropic").class, "opus");
assert!(provider(outcome, "anthropic").close_calls.is_empty());
assert_eq!(provider(outcome, "openai").class, "sol");
assert_eq!(provider(outcome, "openai").close_calls, [Stage::Implement]);
assert_eq!(depends_on.len(), 1);
assert_eq!(reference_fetch_failures[0].item_ref, "#404");
let requests = server.received_requests().await.unwrap();
let body: serde_json::Value = requests[0].body_json().unwrap();
let keys: Vec<&String> = body["questions"].as_object().unwrap().keys().collect();
assert_eq!(
keys,
[
"anthropic.stage_design",
"anthropic.stage_implement",
"anthropic.stage_review",
"could_be_cheaper_0",
"openai.stage_design",
"openai.stage_implement",
"openai.stage_review",
]
);
}
#[tokio::test]
async fn run_route_fails_the_issue_when_one_provider_is_unanswered() {
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("POST"))
.respond_with(wiremock::ResponseTemplate::new(200).set_body_json(routed_json()))
.mount(&server)
.await;
let client = JevClient::new(&server.uri(), "key").unwrap();
let ladders = [
class_only(Provider::Anthropic).unwrap(),
class_only(Provider::Gemini).unwrap(),
];
let report = run_route(
&client,
&[doc(7, ItemState::Open)],
&ladders,
&opts(),
&OpenDependencies::new(),
)
.await
.unwrap();
let RouteOutcome::Failed { error, .. } = &report.issues[0].outcome else {
panic!("expected a failed issue: {:?}", report.issues[0]);
};
assert!(error.contains("gemini.stage_design"), "{error}");
}
fn routed_json() -> serde_json::Value {
serde_json::json!({
"model": "jev-1.13.0",
"answers": {
"anthropic.stage_design": choice_json("none", 0.9),
"anthropic.stage_implement": choice_json("sonnet", 0.9),
"anthropic.stage_review": choice_json("opus", 0.9),
},
"usage": {"input_tokens": 5, "output_tokens": 1}
})
}
#[tokio::test]
async fn run_route_keeps_going_past_a_failed_issue() {
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("POST"))
.and(wiremock::matchers::body_partial_json(
serde_json::json!({"state": "# #1 Route issues\n\nThe body.\n"}),
))
.respond_with(wiremock::ResponseTemplate::new(500).set_body_string("boom"))
.mount(&server)
.await;
wiremock::Mock::given(wiremock::matchers::method("POST"))
.respond_with(wiremock::ResponseTemplate::new(200).set_body_json(routed_json()))
.mount(&server)
.await;
let client = JevClient::new(&server.uri(), "key").unwrap();
let reference_fetch_failures = ReferenceFetchFailures::from([(
("rust-works/omni-dev".to_string(), 1),
vec![ReferenceFetchFailure {
item_ref: "#404".to_string(),
error: "not found".to_string(),
}],
)]);
let report = run_route_with_reference_fetch_failures(
&client,
&[doc(1, ItemState::Open), doc(2, ItemState::Open)],
&anthropic(),
&opts(),
&OpenDependencies::new(),
&reference_fetch_failures,
)
.await
.unwrap();
assert!(report.issues[0].failed());
let RouteOutcome::Failed {
error,
reference_fetch_failures,
} = &report.issues[0].outcome
else {
panic!();
};
assert!(error.contains("500"), "{error}");
assert_eq!(reference_fetch_failures[0].item_ref, "#404");
let json = serde_json::to_value(&report).unwrap();
assert_eq!(
json["issues"][0]["reference_fetch_failures"][0]["ref"],
"#404"
);
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
assert!(
text.contains(" reference fetch failed: #404 (not found)"),
"{text}"
);
assert!(!report.issues[1].failed());
assert_eq!(report.usage.input_tokens, 5);
}
#[tokio::test]
async fn run_route_stops_on_an_auth_failure() {
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("POST"))
.respond_with(wiremock::ResponseTemplate::new(401).set_body_string("bad key"))
.expect(1)
.mount(&server)
.await;
let client = JevClient::new(&server.uri(), "key").unwrap();
let err = run_route(
&client,
&[doc(3, ItemState::Open), doc(4, ItemState::Open)],
&anthropic(),
&opts(),
&OpenDependencies::new(),
)
.await
.unwrap_err();
assert!(format!("{err:#}").contains("Failed to route issue rust-works/omni-dev#3"));
}
#[tokio::test]
async fn run_route_names_the_issue_on_an_unexpected_answer() {
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("POST"))
.respond_with(
wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!({
"model": "jev-1.13.0",
"answers": {
"anthropic.stage_design": choice_json("haiku", 0.9),
"anthropic.stage_implement": choice_json("sonnet", 0.9),
"anthropic.stage_review": choice_json("opus", 0.9),
},
})),
)
.mount(&server)
.await;
let client = JevClient::new(&server.uri(), "key").unwrap();
let report = run_route(
&client,
&[doc(9, ItemState::Open)],
&anthropic(),
&opts(),
&OpenDependencies::new(),
)
.await
.unwrap();
let RouteOutcome::Failed { error, .. } = &report.issues[0].outcome else {
panic!("expected a failed issue");
};
assert!(error.contains("Unexpected Jev answer"), "{error}");
assert!(error.contains("not offered"), "{error}");
}
fn citation(raw: &str, number: u64) -> Citation {
Citation {
item_ref: ItemRef {
provider: GitProvider::GitHub,
project: "rust-works/omni-dev".to_string(),
kind: ItemKind::Issue,
number,
},
raw: raw.to_string(),
url: None,
}
}
#[tokio::test]
async fn run_route_asks_could_be_cheaper_for_each_open_citation() {
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("POST"))
.respond_with(
wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!({
"model": "jev-1.13.0",
"answers": {
"anthropic.stage_design": choice_json("opus", 0.2),
"anthropic.stage_implement": choice_json("sonnet", 0.61),
"anthropic.stage_review": choice_json("opus", 0.9),
"could_be_cheaper_0": {"type": "noul", "noul": 0.75},
},
"usage": {"input_tokens": 10, "output_tokens": 1}
})),
)
.mount(&server)
.await;
let client = JevClient::new(&server.uri(), "key").unwrap();
let dependencies = OpenDependencies::from([(
("rust-works/omni-dev".to_string(), 7),
vec![citation("#1129", 1129)],
)]);
let report = run_route(
&client,
&[doc(7, ItemState::Open)],
&anthropic(),
&opts(),
&dependencies,
)
.await
.unwrap();
let RouteOutcome::Routed { depends_on, .. } = &report.issues[0].outcome else {
panic!("expected a routed issue: {:?}", report.issues[0]);
};
assert_eq!(depends_on.len(), 1);
assert_eq!(depends_on[0].item_ref, "#1129");
assert_eq!(depends_on[0].state, ItemState::Open);
assert_eq!(depends_on[0].could_be_cheaper.get("design"), Some(&0.75));
let requests = server.received_requests().await.unwrap();
let body: serde_json::Value = requests[0].body_json().unwrap();
let keys: Vec<&String> = body["questions"].as_object().unwrap().keys().collect();
assert_eq!(
keys,
[
"anthropic.stage_design",
"anthropic.stage_implement",
"anthropic.stage_review",
"could_be_cheaper_0",
]
);
}
#[tokio::test]
async fn run_route_drops_a_dependency_with_no_could_be_cheaper_answer() {
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("POST"))
.respond_with(wiremock::ResponseTemplate::new(200).set_body_json(routed_json()))
.mount(&server)
.await;
let client = JevClient::new(&server.uri(), "key").unwrap();
let dependencies = OpenDependencies::from([(
("rust-works/omni-dev".to_string(), 7),
vec![citation("#1129", 1129)],
)]);
let report = run_route(
&client,
&[doc(7, ItemState::Open)],
&anthropic(),
&opts(),
&dependencies,
)
.await
.unwrap();
let RouteOutcome::Routed { depends_on, .. } = &report.issues[0].outcome else {
panic!("expected a routed issue: {:?}", report.issues[0]);
};
assert!(depends_on.is_empty(), "{depends_on:?}");
}
#[test]
fn report_serialises_ref_and_omits_false_truncated() {
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![IssueRoute {
item_ref: "o/r#1".to_string(),
url: "u".to_string(),
title: "t".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Routed {
providers: BTreeMap::from([(
"anthropic".to_string(),
ProviderRoute {
stages: stages("none", "sonnet", "opus"),
class: "sonnet".to_string(),
close_calls: vec![],
},
)]),
depends_on: vec![],
reference_fetch_failures: vec![],
},
truncated: false,
}],
ignored_closed: vec![],
usage: Usage::default(),
};
let value = serde_json::to_value(&report).unwrap();
assert_eq!(value["issues"][0]["ref"], "o/r#1");
assert!(value["issues"][0].get("truncated").is_none());
let anthropic = &value["issues"][0]["providers"]["anthropic"];
assert_eq!(anthropic["stages"]["design"]["choice"], "none");
assert_eq!(anthropic["class"], "sonnet");
assert_eq!(anthropic["close_calls"], serde_json::json!([]));
assert!(value["issues"][0].get("stages").is_none());
assert!(value["issues"][0].get("class").is_none());
assert_eq!(value["issues"][0]["depends_on"], serde_json::json!([]));
assert!(value["issues"][0].get("reference_fetch_failures").is_none());
assert!(value["issues"][0].get("error").is_none());
}
#[test]
fn a_failed_issue_serialises_only_its_error() {
let route = IssueRoute {
item_ref: "o/r#1".to_string(),
url: "u".to_string(),
title: "t".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Failed {
error: "HTTP 529".to_string(),
reference_fetch_failures: vec![],
},
truncated: false,
};
let value = serde_json::to_value(&route).unwrap();
assert_eq!(value["error"], "HTTP 529");
assert!(value.get("providers").is_none());
assert!(value.get("depends_on").is_none());
}
#[test]
fn render_route_text_matches_the_worked_example() {
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![IssueRoute {
item_ref: "rust-works/omni-dev#1641".to_string(),
url: "u".to_string(),
title: "Some issue title".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Routed {
providers: BTreeMap::from([(
"anthropic".to_string(),
ProviderRoute {
stages: StageAnswers {
design: answer("opus", 0.52),
implement: answer("sonnet", 0.83),
review: answer("opus", 0.41),
},
class: "opus".to_string(),
close_calls: vec![Stage::Review],
},
)]),
depends_on: vec![DependencyEntry {
item_ref: "#1129".to_string(),
url: None,
state: ItemState::Open,
could_be_cheaper: BTreeMap::from([("design".to_string(), 0.75)]),
}],
reference_fetch_failures: vec![],
},
truncated: false,
}],
ignored_closed: vec![],
usage: Usage {
input_tokens: 1432,
output_tokens: 61,
},
};
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
assert_eq!(
text,
"rust-works/omni-dev#1641 — Some issue title\n\
\x20\x20opus — design needs opus (0.52), implementation sonnet (0.83), review \
opus (0.41, close call)\n\
\x20\x20cites open #1129, which could leave less design work if resolved (0.75)\n\n\
model: jev-1.13.0, usage: 1432 input tokens, 61 output tokens\n"
);
}
#[test]
fn render_route_text_puts_each_open_citation_on_its_own_line() {
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![IssueRoute {
item_ref: "rust-works/omni-dev#1845".to_string(),
url: "u".to_string(),
title: "t".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Routed {
providers: BTreeMap::from([(
"anthropic".to_string(),
ProviderRoute {
stages: stages("none", "sonnet", "sonnet"),
class: "sonnet".to_string(),
close_calls: vec![],
},
)]),
depends_on: vec![
DependencyEntry {
item_ref: "#1830".to_string(),
url: None,
state: ItemState::Open,
could_be_cheaper: BTreeMap::from([("design".to_string(), 0.46)]),
},
DependencyEntry {
item_ref: "#1831".to_string(),
url: None,
state: ItemState::Open,
could_be_cheaper: BTreeMap::new(),
},
],
reference_fetch_failures: vec![],
},
truncated: false,
}],
ignored_closed: vec![],
usage: Usage::default(),
};
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
assert!(
text.contains(
" cites open #1830, which could leave less design work if resolved (0.46)\n \
cites open #1831\n"
),
"{text}"
);
assert!(!text.contains("; "), "{text}");
}
#[test]
fn render_route_text_names_each_provider_when_there_are_several() {
let route = |class: &str| ProviderRoute {
stages: stages("none", "sonnet", "sonnet"),
class: class.to_string(),
close_calls: vec![],
};
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![IssueRoute {
item_ref: "o/r#1".to_string(),
url: "u".to_string(),
title: "t".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Routed {
providers: BTreeMap::from([
("anthropic".to_string(), route("sonnet")),
("openai".to_string(), route("terra")),
]),
depends_on: vec![],
reference_fetch_failures: vec![],
},
truncated: false,
}],
ignored_closed: vec![],
usage: Usage::default(),
};
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
assert!(text.contains(" anthropic: sonnet —"), "{text}");
assert!(text.contains(" openai: terra —"), "{text}");
}
#[test]
fn render_route_text_reports_a_failed_issue() {
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![IssueRoute {
item_ref: "o/r#1".to_string(),
url: "u".to_string(),
title: "t".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Failed {
error: "HTTP 529".to_string(),
reference_fetch_failures: vec![],
},
truncated: false,
}],
ignored_closed: vec![],
usage: Usage::default(),
};
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
assert!(text.contains("o/r#1 — t\n failed: HTTP 529"), "{text}");
}
#[test]
fn render_route_text_marks_a_closed_issue_in_the_header() {
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![IssueRoute {
item_ref: "o/r#1".to_string(),
url: "u".to_string(),
title: "t".to_string(),
state: ItemState::Closed,
outcome: RouteOutcome::Failed {
error: "HTTP 529".to_string(),
reference_fetch_failures: vec![],
},
truncated: false,
}],
ignored_closed: vec![],
usage: Usage::default(),
};
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
assert!(text.starts_with("o/r#1 — t (closed)\n"), "{text}");
assert!(!text.contains("ignored closed"), "{text}");
}
#[test]
fn render_route_text_lists_ignored_closed_issues_before_the_footer() {
let ignored = |n: u64| IgnoredIssue {
item_ref: format!("o/r#{n}"),
url: format!("https://github.com/o/r/issues/{n}"),
title: "t".to_string(),
};
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![],
ignored_closed: vec![ignored(2), ignored(3)],
usage: Usage::default(),
};
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
assert!(
text.contains("ignored closed: o/r#2, o/r#3\n\nmodel: jev-1.13.0"),
"{text}"
);
let styled = render_route_text_styled(
&report,
DEFAULT_MAX_INPUT_CHARS,
&[],
TerminalStyle {
color: false,
hyperlinks: true,
},
);
assert!(
styled.contains("https://github.com/o/r/issues/2"),
"{styled}"
);
}
#[test]
fn render_route_text_omits_the_depends_on_sentence_when_empty() {
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![IssueRoute {
item_ref: "o/r#1".to_string(),
url: "u".to_string(),
title: "t".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Routed {
providers: BTreeMap::from([(
"anthropic".to_string(),
ProviderRoute {
stages: stages("none", "sonnet", "sonnet"),
class: "sonnet".to_string(),
close_calls: vec![],
},
)]),
depends_on: vec![],
reference_fetch_failures: vec![],
},
truncated: false,
}],
ignored_closed: vec![],
usage: Usage::default(),
};
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
assert!(!text.contains("cites"), "{text}");
}
#[test]
fn render_route_text_cites_a_dependency_with_no_could_be_cheaper_probability() {
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![IssueRoute {
item_ref: "o/r#1".to_string(),
url: "u".to_string(),
title: "t".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Routed {
providers: BTreeMap::from([(
"anthropic".to_string(),
ProviderRoute {
stages: stages("none", "sonnet", "sonnet"),
class: "sonnet".to_string(),
close_calls: vec![],
},
)]),
depends_on: vec![DependencyEntry {
item_ref: "#1129".to_string(),
url: None,
state: ItemState::Open,
could_be_cheaper: BTreeMap::new(),
}],
reference_fetch_failures: vec![],
},
truncated: false,
}],
ignored_closed: vec![],
usage: Usage::default(),
};
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
assert!(text.contains("cites open #1129"), "{text}");
assert!(!text.contains("could leave less design work"), "{text}");
}
#[test]
fn render_route_text_notes_a_truncated_issue_and_omits_the_note_otherwise() {
let issue = |truncated: bool| IssueRoute {
item_ref: "o/r#1".to_string(),
url: "u".to_string(),
title: "t".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Routed {
providers: BTreeMap::from([(
"anthropic".to_string(),
ProviderRoute {
stages: stages("none", "sonnet", "sonnet"),
class: "sonnet".to_string(),
close_calls: vec![],
},
)]),
depends_on: vec![],
reference_fetch_failures: vec![],
},
truncated,
};
let report = |truncated| RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![issue(truncated)],
ignored_closed: vec![],
usage: Usage::default(),
};
let truncated_text = render_route_text(&report(true), 60_000);
assert!(
truncated_text.contains(" input truncated at 60,000 characters"),
"{truncated_text}"
);
let kept_text = render_route_text(&report(false), 60_000);
assert!(!kept_text.contains("truncated"), "{kept_text}");
}
#[test]
fn render_route_text_notes_no_further_design_work() {
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![IssueRoute {
item_ref: "o/r#1".to_string(),
url: "u".to_string(),
title: "t".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Routed {
providers: BTreeMap::from([(
"anthropic".to_string(),
ProviderRoute {
stages: stages("none", "sonnet", "sonnet"),
class: "sonnet".to_string(),
close_calls: vec![],
},
)]),
depends_on: vec![],
reference_fetch_failures: vec![],
},
truncated: false,
}],
ignored_closed: vec![],
usage: Usage::default(),
};
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
assert!(text.contains("design needs no further work"), "{text}");
}
#[test]
fn render_route_text_switches_to_one_line_per_stage_for_multi_model_tiers() {
let multi = "global.anthropic.claude-sonnet-4-6,global.anthropic.claude-sonnet-5";
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![IssueRoute {
item_ref: "rust-works/omni-dev#1832".to_string(),
url: "u".to_string(),
title: "feat(drive): banded ranges for drive sheets (#1830)".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Routed {
providers: BTreeMap::from([(
"anthropic".to_string(),
ProviderRoute {
stages: StageAnswers {
design: answer(NO_DESIGN, 0.94),
implement: answer(multi, 0.94),
review: answer(multi, 0.70),
},
class: multi.to_string(),
close_calls: vec![],
},
)]),
depends_on: vec![DependencyEntry {
item_ref: "#1830".to_string(),
url: None,
state: ItemState::Open,
could_be_cheaper: BTreeMap::from([("design".to_string(), 0.48)]),
}],
reference_fetch_failures: vec![],
},
truncated: false,
}],
ignored_closed: vec![],
usage: Usage::default(),
};
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
assert_eq!(
text,
format!(
"rust-works/omni-dev#1832 — feat(drive): banded ranges for drive sheets (#1830)\n\
\x20\x20class: {multi}\n\
\x20\x20design: needs no further work (0.94)\n\
\x20\x20implementation: {multi} (0.94)\n\
\x20\x20review: {multi} (0.70)\n\
\x20\x20cites open #1830, which could leave less design work if resolved (0.48)\n\n\
model: jev-1.13.0, usage: 0 input tokens, 0 output tokens\n"
)
);
}
#[test]
fn render_route_text_keeps_the_compact_line_for_single_word_tiers() {
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![IssueRoute {
item_ref: "o/r#1".to_string(),
url: "u".to_string(),
title: "t".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Routed {
providers: BTreeMap::from([(
"anthropic".to_string(),
ProviderRoute {
stages: stages("opus", "sonnet", "opus"),
class: "opus".to_string(),
close_calls: vec![],
},
)]),
depends_on: vec![],
reference_fetch_failures: vec![],
},
truncated: false,
}],
ignored_closed: vec![],
usage: Usage::default(),
};
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
assert!(
text.contains(
" opus — design needs opus (0.90), implementation sonnet (0.90), review \
opus (0.90)"
),
"{text}"
);
assert!(!text.contains("class:"), "{text}");
}
#[test]
fn render_route_text_headers_the_block_by_provider_when_multi_model_and_multiple_ladders() {
let route = |class: &str| ProviderRoute {
stages: StageAnswers {
design: answer(NO_DESIGN, 0.9),
implement: answer(class, 0.9),
review: answer(class, 0.9),
},
class: class.to_string(),
close_calls: vec![],
};
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![IssueRoute {
item_ref: "o/r#1".to_string(),
url: "u".to_string(),
title: "t".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Routed {
providers: BTreeMap::from([
("anthropic".to_string(), route("a,b")),
("openai".to_string(), route("c,d")),
]),
depends_on: vec![],
reference_fetch_failures: vec![],
},
truncated: false,
}],
ignored_closed: vec![],
usage: Usage::default(),
};
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
assert!(text.contains(" anthropic:\n class: a,b\n"), "{text}");
assert!(text.contains(" implementation: a,b (0.90)"), "{text}");
assert!(text.contains(" openai:\n class: c,d\n"), "{text}");
assert!(text.contains(" implementation: c,d (0.90)"), "{text}");
}
#[test]
fn render_route_text_multi_model_layout_is_decided_per_ladder() {
let multi_route = ProviderRoute {
stages: StageAnswers {
design: answer(NO_DESIGN, 0.9),
implement: answer("a,b", 0.9),
review: answer("a,b", 0.9),
},
class: "a,b".to_string(),
close_calls: vec![],
};
let compact_route = ProviderRoute {
stages: stages("none", "terra", "terra"),
class: "terra".to_string(),
close_calls: vec![],
};
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![IssueRoute {
item_ref: "o/r#1".to_string(),
url: "u".to_string(),
title: "t".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Routed {
providers: BTreeMap::from([
("anthropic".to_string(), multi_route),
("openai".to_string(), compact_route),
]),
depends_on: vec![],
reference_fetch_failures: vec![],
},
truncated: false,
}],
ignored_closed: vec![],
usage: Usage::default(),
};
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
assert!(text.contains(" anthropic:\n class: a,b\n"), "{text}");
assert!(
text.contains(" openai: terra — design needs no further work"),
"{text}"
);
}
#[test]
fn render_route_text_notes_a_close_call_in_the_multi_line_layout() {
let mut review = answer("a,b", 0.41);
review.probabilities =
BTreeMap::from([("a,b".to_string(), 0.56), ("c,d".to_string(), 0.44)]);
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![IssueRoute {
item_ref: "o/r#1".to_string(),
url: "u".to_string(),
title: "t".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Routed {
providers: BTreeMap::from([(
"anthropic".to_string(),
ProviderRoute {
stages: StageAnswers {
design: answer(NO_DESIGN, 0.9),
implement: answer("a,b", 0.94),
review,
},
class: "a,b".to_string(),
close_calls: vec![Stage::Review],
},
)]),
depends_on: vec![],
reference_fetch_failures: vec![],
},
truncated: false,
}],
ignored_closed: vec![],
usage: Usage::default(),
};
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
assert!(
text.contains("review: a,b (0.41, close call — c,d 0.44)"),
"{text}"
);
}
#[test]
fn render_route_text_separates_multiple_issues_with_a_blank_line_in_request_order() {
let issue = |item_ref: &str| IssueRoute {
item_ref: item_ref.to_string(),
url: "u".to_string(),
title: "t".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Failed {
error: "e".to_string(),
reference_fetch_failures: vec![],
},
truncated: false,
};
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![issue("o/r#1"), issue("o/r#2")],
ignored_closed: vec![],
usage: Usage::default(),
};
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
let first = text.find("o/r#1").unwrap();
let second = text.find("o/r#2").unwrap();
assert!(first < second, "{text}");
assert!(
text.contains("o/r#1 — t\n failed: e\n\no/r#2 — t\n failed: e"),
"{text}"
);
}
#[test]
fn render_route_text_ends_with_the_model_and_summed_usage() {
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![],
ignored_closed: vec![],
usage: Usage {
input_tokens: 10,
output_tokens: 20,
},
};
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
assert_eq!(
text,
"model: jev-1.13.0, usage: 10 input tokens, 20 output tokens\n"
);
}
#[test]
fn terminal_style_requires_a_usable_tty() {
assert_eq!(
TerminalStyle::new(false, "xterm", false, true),
TerminalStyle::default()
);
assert_eq!(
TerminalStyle::new(true, "dumb", false, true),
TerminalStyle::default()
);
assert_eq!(
TerminalStyle::new(true, "", false, true),
TerminalStyle::default()
);
assert_eq!(
TerminalStyle::new(true, "xterm", true, true),
TerminalStyle {
color: false,
hyperlinks: true
}
);
assert_eq!(
TerminalStyle::new(true, "xterm", false, false),
TerminalStyle {
color: true,
hyperlinks: false
}
);
}
#[test]
fn complexity_uses_tier_order_for_built_in_and_custom_ladders() {
let anthropic = class_only(Provider::Anthropic).unwrap();
assert_eq!(
choice_complexity(NO_DESIGN, &anthropic),
Some(Complexity::Low)
);
assert_eq!(
choice_complexity("sonnet", &anthropic),
Some(Complexity::Low)
);
assert_eq!(
choice_complexity("opus", &anthropic),
Some(Complexity::High)
);
let custom = |names: &[&str]| {
use std::fmt::Write;
let mut yaml = String::from("tiers:\n");
for name in names {
writeln!(yaml, " - {{name: {name}, description: work}}").unwrap();
}
Ladder::named("custom".to_string(), Tiers::parse(&yaml).unwrap())
};
let two = custom(&["a", "b"]);
assert_eq!(choice_complexity("a", &two), Some(Complexity::Low));
assert_eq!(choice_complexity("b", &two), Some(Complexity::High));
let four = custom(&["a", "b", "c", "d"]);
assert_eq!(choice_complexity("b", &four), Some(Complexity::Medium));
assert_eq!(choice_complexity("c", &four), Some(Complexity::Medium));
assert_eq!(choice_complexity("d", &four), Some(Complexity::High));
assert_eq!(choice_complexity("unknown", &four), None);
}
#[test]
fn hyperlink_uses_exact_osc_8_framing_and_rejects_unsafe_urls() {
let style = TerminalStyle {
color: false,
hyperlinks: true,
};
let url = "https://github.com/other/repo/pull/42";
assert_eq!(
hyperlink("PR #42", Some(url), style),
format!("\x1b]8;;{url}\x1b\\PR #42\x1b]8;;\x1b\\")
);
for bad in [
"https://example.com/o/r/issues/1",
"http://github.com/o/r/issues/1",
"https://github.com/o/r/issues/1\x1b]8;;evil",
"https://github.com/o/r/issues/nope",
"not a valid url",
] {
assert_eq!(hyperlink("#1", Some(bad), style), "#1");
}
assert_eq!(hyperlink("#1", None, style), "#1");
}
#[test]
fn colorize_applies_ansi_codes_per_complexity_when_color_enabled() {
let color = TerminalStyle {
color: true,
hyperlinks: false,
};
assert_eq!(colorize("x", Complexity::Low, color), "\x1b[32mx\x1b[0m");
assert_eq!(colorize("x", Complexity::Medium, color), "\x1b[33mx\x1b[0m");
assert_eq!(colorize("x", Complexity::High, color), "\x1b[31mx\x1b[0m");
assert_eq!(
colorize("x", Complexity::Medium, TerminalStyle::default()),
"x"
);
}
#[test]
fn styled_report_colors_classes_stages_and_failure_and_links_refs() {
let issue = IssueRoute {
item_ref: "o/r#1".to_string(),
url: "https://github.com/o/r/issues/1".to_string(),
title: "t".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Routed {
providers: BTreeMap::from([(
"anthropic".to_string(),
ProviderRoute {
stages: stages(NO_DESIGN, "opus", "sonnet"),
class: "opus".to_string(),
close_calls: vec![],
},
)]),
depends_on: vec![DependencyEntry {
item_ref: "#42".to_string(),
url: Some("https://github.com/other/repo/pull/42".to_string()),
state: ItemState::Open,
could_be_cheaper: BTreeMap::new(),
}],
reference_fetch_failures: vec![],
},
truncated: false,
};
let failed = IssueRoute {
item_ref: "o/r#2".to_string(),
url: "https://github.com/o/r/issues/2".to_string(),
title: "failed".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Failed {
error: "HTTP 529".to_string(),
reference_fetch_failures: vec![],
},
truncated: false,
};
let report = RouteReport {
model: "jev".to_string(),
issues: vec![issue, failed],
ignored_closed: vec![],
usage: Usage::default(),
};
let style = TerminalStyle {
color: true,
hyperlinks: true,
};
let text = render_route_text_styled(
&report,
DEFAULT_MAX_INPUT_CHARS,
&[class_only(Provider::Anthropic).unwrap()],
style,
);
assert!(
text.contains(
"\x1b[32m\x1b]8;;https://github.com/o/r/issues/1\x1b\\o/r#1\x1b]8;;\x1b\\\x1b[0m"
),
"{text}"
);
assert!(
text.contains("\x1b[31mopus\x1b[0m — design needs \x1b[32mno further work\x1b[0m"),
"{text}"
);
assert!(text.contains("review \x1b[32msonnet\x1b[0m"), "{text}");
assert!(
text.contains("\x1b]8;;https://github.com/other/repo/pull/42\x1b\\#42\x1b]8;;\x1b\\"),
"{text}"
);
assert!(text.contains("\x1b[31mfailed: HTTP 529\x1b[0m"), "{text}");
let plain = render_route_text_styled(
&report,
DEFAULT_MAX_INPUT_CHARS,
&[],
TerminalStyle::default(),
);
assert_eq!(plain, render_route_text(&report, DEFAULT_MAX_INPUT_CHARS));
assert!(!plain.contains('\x1b'));
let json = serde_json::to_string(&report).unwrap();
assert!(!json.contains("https://github.com/other/repo/pull/42"));
}
#[test]
fn styled_multi_model_layout_colors_the_class_and_stage_choices() {
let tier = "model-a,model-b";
let ladder = Ladder::named(
"custom".to_string(),
Tiers::parse(&format!(
"tiers:\n - {{name: basic, description: work}}\n - {{name: '{tier}', description: work}}\n"
))
.unwrap(),
);
let report = RouteReport {
model: "jev".to_string(),
issues: vec![IssueRoute {
item_ref: "o/r#1".to_string(),
url: "https://github.com/o/r/issues/1".to_string(),
title: "t".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Routed {
providers: BTreeMap::from([(
"custom".to_string(),
ProviderRoute {
stages: stages(NO_DESIGN, tier, "basic"),
class: tier.to_string(),
close_calls: vec![],
},
)]),
depends_on: vec![],
reference_fetch_failures: vec![],
},
truncated: false,
}],
ignored_closed: vec![],
usage: Usage::default(),
};
let text = render_route_text_styled(
&report,
DEFAULT_MAX_INPUT_CHARS,
&[ladder],
TerminalStyle {
color: true,
hyperlinks: false,
},
);
assert!(
text.contains(&format!(" class: \x1b[31m{tier}\x1b[0m")),
"{text}"
);
assert!(
text.contains(&format!(" implementation: \x1b[31m{tier}\x1b[0m")),
"{text}"
);
assert!(text.contains(" review: \x1b[32mbasic\x1b[0m"), "{text}");
}
#[test]
fn design_stage_with_a_named_tier_renders_needs_prefix() {
let line = render_stage_line(
Stage::Design,
&stages("opus", "sonnet", "sonnet"),
&[],
None,
TerminalStyle::default(),
);
assert_eq!(line, "design: needs opus (0.90)");
}
#[test]
fn with_thousands_groups_digits() {
assert_eq!(with_thousands(0), "0");
assert_eq!(with_thousands(999), "999");
assert_eq!(with_thousands(60_000), "60,000");
assert_eq!(with_thousands(1_000_000), "1,000,000");
}
}