use std::collections::{BTreeMap, BTreeSet};
use std::path::Path;
use anyhow::{bail, Context, Result};
use clap::ValueEnum;
use serde::{Deserialize, Serialize};
use tracing::warn;
use crate::jev::citations::{find_citation_mentions, Citation};
use crate::jev::client::JevClient;
use crate::jev::error::is_auth_failure;
use crate::jev::input::truncate_middle;
use crate::jev::markdown_code::mask_code;
use crate::jev::protocol::{Answer, Question, SystemOneRequest, Usage};
use crate::provider::{IssueDoc, ItemRef, ItemState};
pub use crate::jev::input::TRUNCATION_MARKER;
pub mod effort;
const STAGE_QUESTIONS_YAML: &str = include_str!("../templates/jev-route-questions.yaml");
pub const NO_DESIGN: &str = "none";
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
pub enum Provider {
#[value(name = "anthropic")]
Anthropic,
#[value(name = "openai")]
OpenAi,
#[value(name = "gemini")]
Gemini,
}
impl Provider {
pub const ALL: [Self; 3] = [Self::Anthropic, Self::OpenAi, Self::Gemini];
#[must_use]
pub const fn name(self) -> &'static str {
match self {
Self::Anthropic => "anthropic",
Self::OpenAi => "openai",
Self::Gemini => "gemini",
}
}
const fn tiers_yaml(self) -> &'static str {
match self {
Self::Anthropic => include_str!("../templates/jev-route-tiers-anthropic.yaml"),
Self::OpenAi => include_str!("../templates/jev-route-tiers-openai.yaml"),
Self::Gemini => include_str!("../templates/jev-route-tiers-gemini.yaml"),
}
}
pub fn tiers(self) -> Result<Tiers> {
Tiers::parse(self.tiers_yaml())
.with_context(|| format!("Invalid embedded tiers for provider {}", self.name()))
}
}
pub const DEFAULT_CLOSE_CALL: f64 = 0.3;
pub const DEFAULT_CLOSE_CALL_MARGIN: f64 = 0.2;
pub const DEFAULT_MAX_INPUT_CHARS: usize = 60_000;
const MIN_TIERS: usize = 2;
pub type OpenDependencies = BTreeMap<(String, u64), Vec<Citation>>;
pub type ReferenceFetchFailures = BTreeMap<(String, u64), Vec<ReferenceFetchFailure>>;
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct ReferenceFetchFailure {
#[serde(rename = "ref")]
pub item_ref: String,
pub error: String,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum Stage {
Design,
Implement,
Review,
}
impl Stage {
const ALL: [Self; 3] = [Self::Design, Self::Implement, Self::Review];
const fn template_key(self) -> &'static str {
match self {
Self::Design => "stage_design",
Self::Implement => "stage_implement",
Self::Review => "stage_review",
}
}
fn question_key(self, provider: &str) -> String {
format!("{provider}.{}", self.template_key())
}
}
#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Tier {
pub name: String,
pub description: String,
pub models: Option<Vec<effort::Model>>,
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct TiersFile {
tiers: Vec<Tier>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Tiers(Vec<Tier>);
impl Tiers {
pub fn parse(yaml: &str) -> Result<Self> {
let file: TiersFile = serde_yaml::from_str(yaml).context("Failed to parse tiers file")?;
if file.tiers.len() < MIN_TIERS {
bail!(
"a tiers file needs at least {MIN_TIERS} tiers, got {}",
file.tiers.len()
);
}
let mut seen = BTreeSet::new();
for tier in &file.tiers {
if tier.name.trim().is_empty() || tier.description.trim().is_empty() {
bail!("every tier needs a non-empty name and description");
}
if tier.name == NO_DESIGN {
bail!("the tier name {NO_DESIGN:?} is reserved for \"no design work remains\"");
}
if !seen.insert(tier.name.as_str()) {
bail!("tier {:?} is defined more than once", tier.name);
}
}
let mut model_rungs = BTreeMap::new();
for tier in &file.tiers {
if let Some(models) = &tier.models {
effort::validate(models).with_context(|| format!("rung {:?}", tier.name))?;
for model in models {
for &stage in &model.stages {
if let Some(previous) = model_rungs.insert((&model.name, stage), &tier.name)
{
bail!(
"model {:?} is bound to both rungs {previous:?} and {:?} for {stage:?}",
model.name,
tier.name
);
}
}
}
}
}
Ok(Self(file.tiers))
}
pub fn load_file(path: &Path) -> Result<Self> {
let yaml = std::fs::read_to_string(path)
.with_context(|| format!("Failed to read tiers file {}", path.display()))?;
Self::parse(&yaml).with_context(|| format!("Invalid tiers file {}", path.display()))
}
#[must_use]
pub fn as_slice(&self) -> &[Tier] {
&self.0
}
fn rank(&self, choice: &str) -> Option<usize> {
if choice == NO_DESIGN {
return Some(0);
}
self.0.iter().position(|t| t.name == choice).map(|i| i + 1)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Ladder {
pub name: String,
pub tiers: Tiers,
}
impl Ladder {
pub fn builtin(provider: Provider) -> Result<Self> {
Ok(Self {
name: provider.name().to_string(),
tiers: provider.tiers()?,
})
}
#[must_use]
pub fn named(name: String, tiers: Tiers) -> Self {
Self { name, tiers }
}
}
pub fn build_route_questions(ladders: &[Ladder]) -> Result<BTreeMap<String, Question>> {
build_route_questions_for_mode(ladders, true)
}
pub fn build_route_questions_for_mode(
ladders: &[Ladder],
effort_advice: bool,
) -> Result<BTreeMap<String, Question>> {
let templates: BTreeMap<String, Question> = serde_yaml::from_str(STAGE_QUESTIONS_YAML)
.context("Failed to parse the embedded stage questions")?;
for stage in Stage::ALL {
if !templates.contains_key(stage.template_key()) {
bail!(
"the embedded stage questions have no {:?}",
stage.template_key()
);
}
}
let mut questions = BTreeMap::from([("open_questions".to_string(), open_questions_question())]);
for ladder in ladders {
for (key, template) in &templates {
let mut question = template.clone();
let Question::Choice { criteria, .. } = &mut question else {
bail!("embedded stage question {key:?} is not a choice question");
};
for tier in ladder.tiers.as_slice() {
if criteria
.insert(tier.name.clone(), tier.description.clone())
.is_some()
{
bail!(
"tier {:?} of ladder {:?} collides with a fixed option of {key:?}",
tier.name,
ladder.name
);
}
}
question
.validate()
.with_context(|| format!("stage question {key:?} for {:?}", ladder.name))?;
questions.insert(format!("{}.{key}", ladder.name), question);
}
}
if effort_advice {
for ladder in ladders {
effort::add_questions(ladder, &mut questions);
}
}
Ok(questions)
}
#[must_use]
pub fn build_route_state(doc: &IssueDoc, max_chars: usize) -> (String, bool) {
build_route_state_with_drafts(doc, max_chars, &[])
}
#[must_use]
pub fn build_route_citation_text(doc: &IssueDoc, max_chars: usize) -> String {
build_route_citation_text_with_drafts(doc, max_chars, &[])
}
#[must_use]
pub fn build_route_state_with_drafts(
doc: &IssueDoc,
max_chars: usize,
drafts: &[String],
) -> (String, bool) {
assemble_route_state(doc, max_chars, drafts, str::to_string)
}
#[must_use]
pub fn build_route_citation_text_with_drafts(
doc: &IssueDoc,
max_chars: usize,
drafts: &[String],
) -> String {
assemble_route_state(doc, max_chars, drafts, mask_code).0
}
fn assemble_route_state(
doc: &IssueDoc,
max_chars: usize,
drafts: &[String],
prose: impl Fn(&str) -> String,
) -> (String, bool) {
let mut text = format!(
"# #{} {}\n\n{}\n",
doc.number,
prose(&doc.title),
prose(doc.body.trim())
);
if !doc.comments.is_empty() || !drafts.is_empty() {
text.push_str("\n\n---\n\n## Comments on this issue\n");
for comment in &doc.comments {
text.push_str(&format!(
"\n\n**Comment by {}:**\n\n{}\n",
comment.author,
prose(comment.body.trim())
));
}
}
for draft in drafts {
text.push_str(&format!(
"\n\n**Comment by draft:**\n\n{}\n",
prose(draft.trim())
));
}
truncate_middle(&text, max_chars)
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct StageAnswer {
pub choice: String,
pub confidence: f64,
pub probabilities: BTreeMap<String, f64>,
#[serde(skip_serializing_if = "BTreeMap::is_empty")]
pub effort_by_model: BTreeMap<String, Vec<effort::ModelEffort>>,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct StageAnswers {
pub design: StageAnswer,
pub implement: StageAnswer,
pub review: StageAnswer,
}
impl StageAnswers {
fn get(&self, stage: Stage) -> &StageAnswer {
match stage {
Stage::Design => &self.design,
Stage::Implement => &self.implement,
Stage::Review => &self.review,
}
}
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct DependencyEntry {
#[serde(rename = "ref")]
pub item_ref: String,
#[serde(skip)]
pub url: Option<String>,
pub state: ItemState,
pub relation: CitationRelation,
pub could_be_cheaper: BTreeMap<String, f64>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum CitationRelation {
Blocker,
Tracker,
Unspecified,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct IssueRoute {
#[serde(skip_serializing_if = "Vec::is_empty")]
pub draft_comments: Vec<std::path::PathBuf>,
#[serde(rename = "ref")]
pub item_ref: String,
pub url: String,
pub title: String,
pub state: ItemState,
#[serde(flatten)]
pub outcome: RouteOutcome,
#[serde(skip_serializing_if = "std::ops::Not::not")]
pub truncated: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum ClassSource {
Design,
Implement,
}
impl ClassSource {
const fn label(self) -> &'static str {
match self {
Self::Design => "design",
Self::Implement => "implementation",
}
}
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct ProviderRoute {
pub stages: StageAnswers,
pub class: String,
pub class_from: ClassSource,
pub close_calls: Vec<Stage>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum OpenQuestionKind {
None,
Factual,
Design,
Both,
}
impl OpenQuestionKind {
fn label(self) -> &'static str {
match self {
Self::None => "none",
Self::Factual => "factual",
Self::Design => "design",
Self::Both => "both",
}
}
fn parse(label: &str) -> Result<Self> {
match label {
"none" => Ok(Self::None),
"factual" => Ok(Self::Factual),
"design" => Ok(Self::Design),
"both" => Ok(Self::Both),
_ => bail!("open_questions label {label:?} was not offered"),
}
}
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct OpenQuestions {
pub choice: OpenQuestionKind,
pub confidence: f64,
pub probabilities: BTreeMap<String, f64>,
}
fn open_questions_question() -> Question {
Question::Choice {
instructions: "What kind of questions is still open in the issue after considering all \
comments? Unknown existing-code facts and genuine design tradeoffs are different \
kinds. A suggested fix direction does not settle unverified factual premises, \
such as whether a reusable helper already exists. A code-informed decision with \
a concrete plan does settle them: carrying out its prescribed caller audits, \
code edits, tests, or verification is execution work, not an open question. \
Routine coding choices do not count as design tradeoffs. Judge the remaining \
uncertainty, not the amount of work."
.to_string(),
criteria: BTreeMap::from([
(
"none".to_string(),
"The latest text establishes the relevant existing-code facts and prescribes the \
approach. Executing its specified plan, including audits and verification, does \
not itself create open questions."
.to_string(),
),
(
"factual".to_string(),
"Only existing-code facts remain unsettled: whether a reusable helper exists, \
what it does, or which existing sites need changes. A suggested fix with \
uncertain reuse or scope is factual even if its intended behavior is clear; \
inspecting code can settle it without a discretionary design tradeoff."
.to_string(),
),
(
"design".to_string(),
"Only genuine choices between competing approaches remain, requiring judgment \
about behavior, architecture, or tradeoffs even after existing-code facts are \
known. Routine choices while implementing a prescribed fix do not qualify."
.to_string(),
),
(
"both".to_string(),
"Both unresolved existing-code premises and genuine choices between competing \
designs remain. Code reading alone cannot settle the design tradeoffs."
.to_string(),
),
]),
}
}
fn open_questions_answer(answers: &BTreeMap<String, Answer>) -> Result<OpenQuestions> {
let Some(Answer::Choice {
choice,
confidence,
probabilities,
}) = answers.get("open_questions")
else {
bail!("no choice answer for open_questions");
};
let choice = OpenQuestionKind::parse(choice)?;
for label in probabilities.keys() {
OpenQuestionKind::parse(label)?;
}
Ok(OpenQuestions {
choice,
confidence: *confidence,
probabilities: probabilities.clone(),
})
}
#[derive(Debug, Clone, PartialEq, Serialize)]
#[serde(untagged)]
pub enum RouteOutcome {
Routed {
#[serde(skip_serializing_if = "Option::is_none")]
open_questions: Option<OpenQuestions>,
providers: BTreeMap<String, ProviderRoute>,
depends_on: Vec<DependencyEntry>,
#[serde(skip_serializing_if = "Vec::is_empty")]
reference_fetch_failures: Vec<ReferenceFetchFailure>,
},
Failed {
error: String,
#[serde(skip_serializing_if = "Vec::is_empty")]
reference_fetch_failures: Vec<ReferenceFetchFailure>,
},
}
impl IssueRoute {
#[must_use]
pub fn failed(&self) -> bool {
matches!(self.outcome, RouteOutcome::Failed { .. })
}
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct RouteReport {
pub model: String,
pub issues: Vec<IssueRoute>,
#[serde(skip_serializing_if = "Vec::is_empty")]
pub ignored_closed: Vec<IgnoredIssue>,
pub usage: Usage,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct IgnoredIssue {
#[serde(rename = "ref")]
pub item_ref: String,
pub url: String,
pub title: String,
}
#[derive(Debug, Clone)]
pub struct RouteOptions {
pub draft_comments: Vec<String>,
pub model: String,
pub close_call: f64,
pub close_call_margin: f64,
pub effort_advice: bool,
pub max_input_chars: usize,
pub allow_closed: bool,
pub ignore_closed: bool,
}
pub async fn run_route(
jev: &JevClient,
docs: &[IssueDoc],
ladders: &[Ladder],
opts: &RouteOptions,
dependencies: &OpenDependencies,
) -> Result<RouteReport> {
run_route_with_reference_fetch_failures(
jev,
docs,
ladders,
opts,
dependencies,
&ReferenceFetchFailures::new(),
)
.await
}
pub async fn run_route_with_reference_fetch_failures(
jev: &JevClient,
docs: &[IssueDoc],
ladders: &[Ladder],
opts: &RouteOptions,
dependencies: &OpenDependencies,
reference_fetch_failures: &ReferenceFetchFailures,
) -> Result<RouteReport> {
if !opts.draft_comments.is_empty() && docs.len() != 1 {
bail!("draft comments apply to exactly one issue; cannot preview a batch");
}
let (docs, ignored_closed) = drop_closed(docs, opts)?;
validate_options(&docs, ladders, opts)?;
let questions = build_route_questions_for_mode(ladders, opts.effort_advice)?;
for (key, question) in &questions {
question
.validate_for_model(&opts.model)
.with_context(|| format!("route question {key:?}"))?;
}
let mut models = BTreeSet::new();
let mut usage = Usage::default();
let mut issues = Vec::with_capacity(docs.len());
for doc in docs {
let item_ref = item_ref(doc);
let (state, truncated) =
build_route_state_with_drafts(doc, opts.max_input_chars, &opts.draft_comments);
if truncated {
warn!(
"Issue {item_ref} is longer than {} characters; its input was truncated",
opts.max_input_chars
);
}
let citations = dependencies
.get(&(doc.project.clone(), doc.number))
.map_or(&[][..], Vec::as_slice);
let relations = citation_relations(&state, doc);
let citation_failures = reference_fetch_failures
.get(&(doc.project.clone(), doc.number))
.cloned()
.unwrap_or_default();
let mut request_questions = questions.clone();
for (i, citation) in citations.iter().enumerate() {
request_questions.insert(
could_be_cheaper_key(i),
could_be_cheaper_question(&citation.raw),
);
}
let request = SystemOneRequest {
state: serde_json::Value::String(state),
model: opts.model.clone(),
questions: request_questions,
};
let outcome = match jev.system_one(&request).await {
Err(err) if is_auth_failure(&err) => {
return Err(err).with_context(|| format!("Failed to route issue {item_ref}"));
}
Err(err) => failed(
&item_ref,
&err.context("Jev request failed"),
&citation_failures,
),
Ok(response) => {
models.insert(response.model);
usage.input_tokens += response.usage.input_tokens;
usage.output_tokens += response.usage.output_tokens;
match provider_routes(
&response.answers,
ladders,
opts.close_call,
opts.close_call_margin,
opts.effort_advice,
) {
Ok(providers) => RouteOutcome::Routed {
open_questions: open_questions_answer(&response.answers)
.map_err(|err| {
warn!(
"Ignoring open_questions for {item_ref}: {err:#}; \
routing is unaffected"
);
})
.ok(),
providers,
depends_on: dependency_entries(
&item_ref,
citations,
&relations,
&response.answers,
),
reference_fetch_failures: citation_failures.clone(),
},
Err(err) => failed(
&item_ref,
&err.context("Unexpected Jev answer"),
&citation_failures,
),
}
}
};
issues.push(IssueRoute {
draft_comments: vec![],
item_ref,
url: doc.url.clone(),
title: doc.title.clone(),
state: doc.state,
outcome,
truncated,
});
}
Ok(RouteReport {
model: models.into_iter().collect::<Vec<_>>().join(", "),
issues,
ignored_closed,
usage,
})
}
#[must_use]
pub fn is_ignored_closed(doc: &IssueDoc, ignore_closed: bool) -> bool {
ignore_closed && doc.state == ItemState::Closed
}
fn item_ref(doc: &IssueDoc) -> String {
format!("{}#{}", doc.project, doc.number)
}
fn drop_closed<'a>(
docs: &'a [IssueDoc],
opts: &RouteOptions,
) -> Result<(Vec<&'a IssueDoc>, Vec<IgnoredIssue>)> {
if opts.allow_closed && opts.ignore_closed {
bail!("--allow-closed and --ignore-closed cannot be used together");
}
let (open, closed): (Vec<&IssueDoc>, Vec<&IssueDoc>) = docs
.iter()
.partition(|d| !is_ignored_closed(d, opts.ignore_closed));
if open.is_empty() && !closed.is_empty() {
bail!(
"every requested issue is closed ({}) and --ignore-closed skipped them; nothing to \
route",
closed
.iter()
.map(|d| item_ref(d))
.collect::<Vec<_>>()
.join(", ")
);
}
let ignored = closed
.into_iter()
.map(|d| IgnoredIssue {
item_ref: item_ref(d),
url: d.url.clone(),
title: d.title.clone(),
})
.collect();
Ok((open, ignored))
}
fn failed(
item_ref: &str,
err: &anyhow::Error,
reference_fetch_failures: &[ReferenceFetchFailure],
) -> RouteOutcome {
let error = format!("{err:#}");
warn!("Could not route issue {item_ref}: {error}");
RouteOutcome::Failed {
error,
reference_fetch_failures: reference_fetch_failures.to_vec(),
}
}
fn validate_options(docs: &[&IssueDoc], ladders: &[Ladder], opts: &RouteOptions) -> Result<()> {
if docs.is_empty() {
bail!("no issues to route");
}
if ladders.is_empty() {
bail!("no ladders to route against");
}
let mut seen = BTreeSet::new();
for ladder in ladders {
if !seen.insert(ladder.name.as_str()) {
bail!("ladder {:?} is listed more than once", ladder.name);
}
}
if !(0.0..=1.0).contains(&opts.close_call) {
bail!(
"the close-call threshold must be between 0 and 1, got {}",
opts.close_call
);
}
if !(0.0..=1.0).contains(&opts.close_call_margin) {
bail!(
"the close-call margin must be between 0 and 1, got {}",
opts.close_call_margin
);
}
if opts.max_input_chars == 0 {
bail!("the input cap must be at least 1 character");
}
if !opts.allow_closed {
let closed: Vec<String> = docs
.iter()
.filter(|d| d.state == ItemState::Closed)
.map(|d| item_ref(d))
.collect();
if !closed.is_empty() {
bail!(
"refusing to route closed issues ({}): their comments often describe how the \
work was done, which leaks the answer. Pass --ignore-closed to skip them, or \
--allow-closed to route them anyway",
closed.join(", ")
);
}
}
Ok(())
}
fn provider_routes(
answers: &BTreeMap<String, Answer>,
ladders: &[Ladder],
close_call: f64,
close_call_margin: f64,
effort_advice: bool,
) -> Result<BTreeMap<String, ProviderRoute>> {
ladders
.iter()
.map(|ladder| {
let mut stages = stage_answers(answers, ladder)?;
if effort_advice {
for (stage, answer) in [
(Stage::Design, &mut stages.design),
(Stage::Implement, &mut stages.implement),
(Stage::Review, &mut stages.review),
] {
answer.effort_by_model =
effort::decode(ladder, stage, answer, answers, close_call)?;
}
}
let (class, class_from) = issue_class(&stages, &ladder.tiers);
let route = ProviderRoute {
class,
class_from,
close_calls: close_calls(&stages, &ladder.tiers, close_call, close_call_margin),
stages,
};
Ok((ladder.name.clone(), route))
})
.collect()
}
fn stage_answers(answers: &BTreeMap<String, Answer>, ladder: &Ladder) -> Result<StageAnswers> {
let tiers = &ladder.tiers;
let read = |stage: Stage| -> Result<StageAnswer> {
let key = stage.question_key(&ladder.name);
let Some(Answer::Choice {
choice,
confidence,
probabilities,
}) = answers.get(&key)
else {
bail!("no choice answer for {key:?}");
};
let offered = match tiers.rank(choice) {
Some(0) => stage == Stage::Design,
Some(_) => true,
None => false,
};
if !offered {
bail!("{key:?} chose {choice:?}, which was not offered");
}
Ok(StageAnswer {
choice: choice.clone(),
confidence: *confidence,
probabilities: probabilities.clone(),
effort_by_model: BTreeMap::new(),
})
};
Ok(StageAnswers {
design: read(Stage::Design)?,
implement: read(Stage::Implement)?,
review: read(Stage::Review)?,
})
}
fn issue_class(stages: &StageAnswers, tiers: &Tiers) -> (String, ClassSource) {
let rank = |a: &StageAnswer| tiers.rank(&a.choice).unwrap_or_default();
if rank(&stages.design) > rank(&stages.implement) {
(stages.design.choice.clone(), ClassSource::Design)
} else {
(stages.implement.choice.clone(), ClassSource::Implement)
}
}
fn top_two_margin(answer: &StageAnswer, stage: Stage, tiers: &Tiers) -> Option<f64> {
let mut usable: Vec<_> = answer
.probabilities
.iter()
.filter(|(name, probability)| {
probability.is_finite()
&& (0.0..=1.0).contains(*probability)
&& match tiers.rank(name) {
Some(0) => stage == Stage::Design,
Some(_) => true,
None => false,
}
})
.collect();
usable.sort_by(|a, b| b.1.total_cmp(a.1).then_with(|| a.0.cmp(b.0)));
let [first, second, ..] = usable.as_slice() else {
return None;
};
Some(first.1 - second.1)
}
fn close_calls(stages: &StageAnswers, tiers: &Tiers, threshold: f64, margin: f64) -> Vec<Stage> {
Stage::ALL
.into_iter()
.filter(|&stage| {
let answer = stages.get(stage);
answer.confidence < threshold
|| top_two_margin(answer, stage, tiers).is_some_and(|gap| gap < margin)
})
.collect()
}
pub fn could_be_cheaper_key(i: usize) -> String {
format!("could_be_cheaper_{i}")
}
pub fn could_be_cheaper_question(citation: &str) -> Question {
Question::Noul {
instructions: format!(
"This issue cites {citation}, which is still open. If {citation} is resolved, how \
likely is it that LESS design work would remain for THIS issue than the current \
text implies — as opposed to this issue's own remaining work being unaffected, \
because it is already scoped separately, is a parallel/sibling effort, or \
{citation} is otherwise not a precondition for finishing this issue's own \
remaining work?"
),
criteria: None,
}
}
fn citation_relations(state: &str, doc: &IssueDoc) -> BTreeMap<(String, u64), CitationRelation> {
let judged = ItemRef {
provider: doc.provider,
project: doc.project.clone(),
kind: doc.kind,
number: doc.number,
};
let mut evidence: BTreeMap<(String, u64), (bool, bool, bool)> = BTreeMap::new();
for mention in find_citation_mentions(state, &doc.project, &judged) {
let key = (
mention.citation.item_ref.project,
mention.citation.item_ref.number,
);
let entry = evidence.entry(key).or_default();
match relation_at(state, &mention.range) {
MentionRelation::Blocker => entry.0 = true,
MentionRelation::Tracker => entry.1 = true,
MentionRelation::Negated => entry.2 = true,
MentionRelation::None => {}
}
}
evidence
.into_iter()
.map(|(key, (blocker, tracker, negated))| {
let relation = match (blocker, tracker, negated) {
(true, false, false) => CitationRelation::Blocker,
(false, true, false) => CitationRelation::Tracker,
_ => CitationRelation::Unspecified,
};
(key, relation)
})
.collect()
}
enum MentionRelation {
Blocker,
Tracker,
Negated,
None,
}
fn relation_at(state: &str, range: &std::ops::Range<usize>) -> MentionRelation {
if !state[..range.start]
.matches("```")
.count()
.is_multiple_of(2)
{
return MentionRelation::None;
}
let before = &state[..range.start];
let boundary = before.rfind(['.', '!', '?', '\n']).map_or(0, |i| i + 1);
let sentence = &state[boundary..];
if sentence.trim_start().starts_with('>') {
return MentionRelation::None;
}
let prefix = &state[boundary..range.start];
if !prefix.matches('`').count().is_multiple_of(2)
|| !prefix.matches('"').count().is_multiple_of(2)
{
return MentionRelation::None;
}
let nearby: String = prefix
.chars()
.rev()
.take(160)
.collect::<String>()
.chars()
.rev()
.collect();
let nearby = nearby.to_ascii_lowercase();
let after = &state[range.end..];
let end = after.find(['.', '!', '?', '\n']).unwrap_or(after.len());
let following = after[..end].trim_start().to_ascii_lowercase();
if nearby.contains("such as")
|| nearby.contains("for example")
|| nearby.contains("worked example")
|| nearby.contains("e.g.")
{
return MentionRelation::None;
}
if nearby.contains("not blocked on")
|| nearby.contains("no longer blocked on")
|| nearby.contains("isn't blocked on")
|| nearby.contains("not dependent on")
|| nearby.contains("unblocked")
|| nearby.contains("not a tracker")
|| nearby.contains("not split out of")
|| following.starts_with("does not block")
|| following.starts_with("no longer blocks")
{
return MentionRelation::Negated;
}
if following.starts_with("blocks ")
|| following.starts_with("is required for ")
|| following.starts_with("must land before ")
{
return MentionRelation::Blocker;
}
if nearby.contains("blocked on")
|| nearby.contains("depends on")
|| nearby.contains("requires")
|| nearby.contains("prerequisite")
{
return MentionRelation::Blocker;
}
if nearby.contains("split out of")
|| nearby.contains("split from")
|| nearby.contains("parent tracker")
|| nearby.contains("tracker checkbox")
{
return MentionRelation::Tracker;
}
MentionRelation::None
}
fn dependency_entries(
item_ref: &str,
citations: &[Citation],
relations: &BTreeMap<(String, u64), CitationRelation>,
answers: &BTreeMap<String, Answer>,
) -> Vec<DependencyEntry> {
citations
.iter()
.enumerate()
.map(|(i, citation)| {
let could_be_cheaper =
if let Some(Answer::Noul { noul }) = answers.get(&could_be_cheaper_key(i)) {
BTreeMap::from([("design".to_string(), *noul)])
} else {
warn!(
"Issue {item_ref}: no could_be_cheaper answer for citation {:?}",
citation.raw
);
BTreeMap::new()
};
DependencyEntry {
item_ref: citation.raw.clone(),
url: citation.url.clone(),
state: ItemState::Open,
relation: relations
.get(&(citation.item_ref.project.clone(), citation.item_ref.number))
.copied()
.unwrap_or(CitationRelation::Unspecified),
could_be_cheaper,
}
})
.collect()
}
#[must_use]
pub fn render_route_text(report: &RouteReport, max_input_chars: usize) -> String {
render_route_text_styled(report, max_input_chars, &[], TerminalStyle::default())
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct TerminalStyle {
pub color: bool,
pub hyperlinks: bool,
}
impl TerminalStyle {
#[must_use]
pub fn new(tty: bool, term: &str, no_color: bool, supports_links: bool) -> Self {
let usable = tty && !term.is_empty() && term != "dumb";
Self {
color: usable && !no_color,
hyperlinks: usable && supports_links,
}
}
}
#[must_use]
pub fn render_route_text_styled(
report: &RouteReport,
max_input_chars: usize,
ladders: &[Ladder],
style: TerminalStyle,
) -> String {
let mut blocks: Vec<String> = report
.issues
.iter()
.map(|issue| render_issue_block(issue, max_input_chars, ladders, style))
.collect();
if !report.ignored_closed.is_empty() {
let refs: Vec<String> = report
.ignored_closed
.iter()
.map(|issue| hyperlink(&issue.item_ref, Some(&issue.url), style))
.collect();
blocks.push(format!("ignored closed: {}", refs.join(", ")));
}
blocks.push(format!(
"model: {}, usage: {} input tokens, {} output tokens",
report.model, report.usage.input_tokens, report.usage.output_tokens
));
let mut text = blocks.join("\n\n");
text.push('\n');
text
}
fn render_issue_block(
issue: &IssueRoute,
max_input_chars: usize,
ladders: &[Ladder],
style: TerminalStyle,
) -> String {
let item_ref = hyperlink(&issue.item_ref, Some(&issue.url), style);
let item_ref = if issue.failed() {
item_ref
} else {
colorize(&item_ref, Complexity::Low, style)
};
let header = if issue.state == ItemState::Closed {
format!("{item_ref} — {} (closed)", issue.title)
} else {
format!("{item_ref} — {}", issue.title)
};
let mut lines = vec![header];
if !issue.draft_comments.is_empty() {
lines.push(format!(
" draft comments (preview): {:?}",
issue.draft_comments
));
}
match &issue.outcome {
RouteOutcome::Routed {
providers,
depends_on,
reference_fetch_failures,
open_questions,
} => {
if let Some(open_questions) = open_questions {
lines.push(format!(
" open_questions: {} ({:.2})",
open_questions.choice.label(),
open_questions.confidence
));
}
for (provider, route) in providers {
let ladder = ladders.iter().find(|ladder| ladder.name == *provider);
lines.extend(render_provider_line(
provider,
route,
providers.len(),
ladder,
style,
));
}
lines.extend(render_depends_on_lines(depends_on, style));
lines.extend(render_reference_fetch_failure_lines(
reference_fetch_failures,
));
}
RouteOutcome::Failed {
error,
reference_fetch_failures,
} => {
lines.push(format!(
" {}",
colorize(&format!("failed: {error}"), Complexity::High, style)
));
lines.extend(render_reference_fetch_failure_lines(
reference_fetch_failures,
));
}
}
if issue.truncated {
lines.push(format!(
" input truncated at {} characters",
with_thousands(max_input_chars)
));
}
lines.join("\n")
}
fn route_has_multi_model_tier(route: &ProviderRoute) -> bool {
[
&route.stages.design,
&route.stages.implement,
&route.stages.review,
]
.into_iter()
.any(|answer| answer.choice.contains(','))
}
fn render_provider_line(
provider: &str,
route: &ProviderRoute,
provider_count: usize,
ladder: Option<&Ladder>,
style: TerminalStyle,
) -> Vec<String> {
let mut lines = if route_has_multi_model_tier(route) {
render_provider_block(provider, route, provider_count, ladder, style)
} else {
vec![render_provider_line_compact(
provider,
route,
provider_count,
ladder,
style,
)]
};
lines.extend(effort::render(&route.stages, ladder));
lines
}
fn render_provider_line_compact(
provider: &str,
route: &ProviderRoute,
provider_count: usize,
ladder: Option<&Ladder>,
style: TerminalStyle,
) -> String {
let class = format!(
"{} (from {})",
color_choice(&route.class, ladder, style),
route.class_from.label()
);
let lead = if provider_count > 1 {
format!("{provider}: {class}")
} else {
class
};
let clauses: Vec<String> = Stage::ALL
.into_iter()
.map(|stage| render_stage_clause(stage, &route.stages, &route.close_calls, ladder, style))
.collect();
format!(" {lead} — {}", clauses.join(", "))
}
fn render_stage_clause(
stage: Stage,
stages: &StageAnswers,
close_calls: &[Stage],
ladder: Option<&Ladder>,
style: TerminalStyle,
) -> String {
let answer = stages.get(stage);
let choice = color_choice(&answer.choice, ladder, style);
let label = match stage {
Stage::Design if answer.choice == NO_DESIGN => format!(
"design needs {}",
colorize("no further work", Complexity::Low, style)
),
Stage::Design => format!("design needs {choice}"),
Stage::Implement => format!("implementation {choice}"),
Stage::Review => format!("review {choice}"),
};
format!(
"{label} ({})",
render_stage_confidence(answer, close_calls.contains(&stage))
)
}
fn render_stage_confidence(answer: &StageAnswer, close_call: bool) -> String {
let mut detail = format!("{:.2}", answer.confidence);
if close_call {
detail.push_str(", close call");
if let Some((name, probability)) = answer
.probabilities
.iter()
.filter(|(name, probability)| name.as_str() != answer.choice && probability.is_finite())
.max_by(|a, b| a.1.total_cmp(b.1).then_with(|| b.0.cmp(a.0)))
{
detail.push_str(&format!(" — {name} {probability:.2}"));
}
}
detail
}
fn render_provider_block(
provider: &str,
route: &ProviderRoute,
provider_count: usize,
ladder: Option<&Ladder>,
style: TerminalStyle,
) -> Vec<String> {
let mut lines = Vec::with_capacity(5);
let indent = if provider_count > 1 {
lines.push(format!(" {provider}:"));
" "
} else {
" "
};
lines.push(format!(
"{indent}class: {} (from {})",
color_choice(&route.class, ladder, style),
route.class_from.label()
));
for stage in Stage::ALL {
lines.push(format!(
"{indent}{}",
render_stage_line(stage, &route.stages, &route.close_calls, ladder, style)
));
}
lines
}
fn render_stage_line(
stage: Stage,
stages: &StageAnswers,
close_calls: &[Stage],
ladder: Option<&Ladder>,
style: TerminalStyle,
) -> String {
let answer = stages.get(stage);
let choice = color_choice(&answer.choice, ladder, style);
let (label, content) = match stage {
Stage::Design if answer.choice == NO_DESIGN => (
"design",
format!(
"needs {}",
colorize("no further work", Complexity::Low, style)
),
),
Stage::Design => ("design", format!("needs {choice}")),
Stage::Implement => ("implementation", choice),
Stage::Review => ("review", choice),
};
format!(
"{label}: {content} ({})",
render_stage_confidence(answer, close_calls.contains(&stage))
)
}
fn render_depends_on_lines(depends_on: &[DependencyEntry], style: TerminalStyle) -> Vec<String> {
depends_on
.iter()
.map(|dep| {
let item_ref = hyperlink(&dep.item_ref, dep.url.as_deref(), style);
let relation = match dep.relation {
CitationRelation::Blocker => "blocker",
CitationRelation::Tracker => "tracker",
CitationRelation::Unspecified => "relation unspecified",
};
match dep.could_be_cheaper.get("design") {
Some(prob) => format!(
" cites open {item_ref} ({relation}), which could leave less design work if resolved ({prob:.2})"
),
None => format!(" cites open {item_ref} ({relation})"),
}
})
.collect()
}
fn render_reference_fetch_failure_lines(failures: &[ReferenceFetchFailure]) -> Vec<String> {
failures
.iter()
.map(|failure| {
format!(
" reference fetch failed: {} ({})",
failure.item_ref, failure.error
)
})
.collect()
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Complexity {
Low,
Medium,
High,
}
fn choice_complexity(choice: &str, ladder: &Ladder) -> Option<Complexity> {
if choice == NO_DESIGN {
return Some(Complexity::Low);
}
let tiers = ladder.tiers.as_slice();
let index = tiers.iter().position(|tier| tier.name == choice)?;
Some(if index == 0 {
Complexity::Low
} else if index == tiers.len() - 1 {
Complexity::High
} else {
Complexity::Medium
})
}
fn color_choice(choice: &str, ladder: Option<&Ladder>, style: TerminalStyle) -> String {
ladder
.and_then(|ladder| choice_complexity(choice, ladder))
.map_or_else(
|| choice.to_string(),
|level| colorize(choice, level, style),
)
}
fn colorize(text: &str, level: Complexity, style: TerminalStyle) -> String {
if !style.color {
return text.to_string();
}
let code = match level {
Complexity::Low => 32,
Complexity::Medium => 33,
Complexity::High => 31,
};
format!("\x1b[{code}m{text}\x1b[0m")
}
fn hyperlink(text: &str, url: Option<&str>, style: TerminalStyle) -> String {
let Some(url) = url.filter(|url| style.hyperlinks && valid_github_item_url(url)) else {
return text.to_string();
};
format!("\x1b]8;;{url}\x1b\\{text}\x1b]8;;\x1b\\")
}
fn valid_github_item_url(raw: &str) -> bool {
if raw.chars().any(char::is_control) {
return false;
}
let Ok(url) = url::Url::parse(raw) else {
return false;
};
if url.scheme() != "https" || url.host_str() != Some("github.com") {
return false;
}
let Some(segments) = url.path_segments() else {
return false; };
let parts: Vec<_> = segments.collect();
parts.len() == 4
&& !parts[0].is_empty()
&& !parts[1].is_empty()
&& matches!(parts[2], "issues" | "pull")
&& parts[3].parse::<u64>().is_ok()
}
fn with_thousands(n: usize) -> String {
let digits = n.to_string();
let mut grouped = String::with_capacity(digits.len() + digits.len() / 3);
for (i, ch) in digits.chars().rev().enumerate() {
if i > 0 && i % 3 == 0 {
grouped.push(',');
}
grouped.push(ch);
}
grouped.chars().rev().collect()
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used)]
mod tests {
use super::*;
use crate::provider::{Comment, GitProvider, ItemKind, ItemRef};
fn doc(number: u64, state: ItemState) -> IssueDoc {
IssueDoc {
provider: GitProvider::GitHub,
project: "rust-works/omni-dev".to_string(),
number,
kind: ItemKind::Issue,
title: "Route issues".to_string(),
state,
body: " The body.\n".to_string(),
comments: vec![],
closed_by: vec![],
url: format!("https://github.com/rust-works/omni-dev/issues/{number}"),
}
}
fn opts() -> RouteOptions {
RouteOptions {
draft_comments: vec![],
model: "jev-latest".to_string(),
close_call: DEFAULT_CLOSE_CALL,
close_call_margin: DEFAULT_CLOSE_CALL_MARGIN,
effort_advice: false,
max_input_chars: DEFAULT_MAX_INPUT_CHARS,
allow_closed: false,
ignore_closed: false,
}
}
fn default_tiers() -> Tiers {
Provider::Anthropic.tiers().unwrap()
}
fn class_only(provider: Provider) -> Result<Ladder> {
let mut ladder = Ladder::builtin(provider)?;
for tier in &mut ladder.tiers.0 {
tier.models = None;
}
Ok(ladder)
}
fn anthropic() -> Vec<Ladder> {
vec![class_only(Provider::Anthropic).unwrap()]
}
fn tier_names(tiers: &Tiers) -> Vec<&str> {
tiers.as_slice().iter().map(|t| t.name.as_str()).collect()
}
#[test]
fn anthropic_tiers_are_sonnet_opus_in_order() {
let tiers = default_tiers();
assert_eq!(tier_names(&tiers), ["sonnet", "opus"]);
assert_eq!(
tiers.as_slice()[0].models.as_ref().unwrap()[0].name,
"claude-sonnet-5-5"
);
assert_eq!(
tiers.as_slice()[1].models.as_ref().unwrap()[0].name,
"claude-opus-5-5"
);
}
#[test]
fn anthropic_tier_descriptions_are_the_reevaluated_text() {
let tiers = default_tiers();
let sonnet = &tiers.as_slice()[0];
let opus = &tiers.as_slice()[1];
assert_eq!(
sonnet.description,
"Reliable at implementing and reviewing well-scoped work across several files, \
including debugging and non-obvious interactions when the approach is clear. May \
need help with complex, open-ended design requiring sustained judgement."
);
assert_eq!(
opus.description,
"Reliable at complex, open-ended design and research: weighing architectures and \
unfamiliar approaches, anticipating failure modes, and making security-critical \
judgements. Carries subtle multi-file implementation and review through with \
sustained judgement."
);
}
#[test]
fn provider_names_are_the_documented_abbreviations() {
assert_eq!(
tier_names(&Provider::OpenAi.tiers().unwrap()),
["terra", "sol", "astra"]
);
assert_eq!(
tier_names(&Provider::Gemini.tiers().unwrap()),
["flash", "pro", "deep-think"]
);
assert_eq!(
Provider::ALL.map(Provider::name),
["anthropic", "openai", "gemini"]
);
for provider in Provider::ALL {
assert_eq!(
provider.to_possible_value().unwrap().get_name(),
provider.name()
);
}
}
#[test]
fn openai_and_gemini_ladders_reuse_the_anthropic_descriptions_rung_for_rung() {
let anthropic = default_tiers();
const RETIRED_ANTHROPIC_THIRD_RUNG: &str =
"Strongest at open-ended design and research: choosing between architectures, \
working in unfamiliar territory with no precedent in the codebase, anticipating \
failure modes, and security-critical judgement.";
for provider in [Provider::OpenAi, Provider::Gemini] {
let tiers = provider.tiers().unwrap();
assert_eq!(tiers.as_slice().len(), 3);
for (rung, reference) in tiers.as_slice().iter().zip(anthropic.as_slice()) {
assert_eq!(
rung.description,
reference.description,
"{}: {} differs from {}",
provider.name(),
rung.name,
reference.name
);
}
assert_eq!(
tiers.as_slice()[2].description,
RETIRED_ANTHROPIC_THIRD_RUNG,
"{}: {} no longer matches the retired anthropic fable rung",
provider.name(), tiers.as_slice()[2].name );
}
}
#[test]
fn ladders_are_named_by_provider_or_a_custom_name() {
let ladder = class_only(Provider::Gemini).unwrap();
assert_eq!(ladder.name, "gemini");
let custom = Ladder::named("mine".to_string(), default_tiers());
assert_eq!(custom.name, "mine");
assert_eq!(custom.tiers, default_tiers());
}
#[test]
fn tiers_need_at_least_two() {
let err = Tiers::parse("tiers:\n - {name: a, description: A}\n").unwrap_err();
assert!(err.to_string().contains("at least 2 tiers"), "{err}");
}
#[test]
fn tiers_reject_the_reserved_none() {
let err = Tiers::parse(
"tiers:\n - {name: a, description: A}\n - {name: none, description: B}\n",
)
.unwrap_err();
assert!(err.to_string().contains("reserved"), "{err}");
}
#[test]
fn tiers_reject_duplicate_names() {
let err =
Tiers::parse("tiers:\n - {name: a, description: A}\n - {name: a, description: B}\n")
.unwrap_err();
assert!(err.to_string().contains("more than once"), "{err}");
}
#[test]
fn tiers_reject_empty_descriptions() {
let err =
Tiers::parse("tiers:\n - {name: a, description: A}\n - {name: b, description: ''}\n")
.unwrap_err();
assert!(err.to_string().contains("non-empty"), "{err}");
}
#[test]
fn tiers_reject_unknown_fields() {
let err = Tiers::parse(
"tiers:\n - {name: a, description: A, rank: 1}\n - {name: b, description: B}\n",
)
.unwrap_err();
assert!(format!("{err:#}").contains("unknown field"), "{err:#}");
}
#[test]
fn tiers_load_reads_a_custom_file() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("tiers.yaml");
std::fs::write(
&path,
"tiers:\n - {name: small, description: S}\n - {name: big, description: B}\n",
)
.unwrap();
let tiers = Tiers::load_file(&path).unwrap();
assert_eq!(tiers.as_slice()[1].name, "big");
}
#[test]
fn tiers_load_names_a_missing_file() {
let err = Tiers::load_file(Path::new("/no/such/tiers.yaml")).unwrap_err();
assert!(err.to_string().contains("Failed to read tiers file"));
}
const BAR: &str = " Choose the least capable class likely to complete this stage correctly \
with no rework, about 9 times in 10. Judge the work that remains given \
the text, not the size of the text.";
fn instructions(questions: &BTreeMap<String, Question>, key: &str) -> String {
let Question::Choice { instructions, .. } = &questions[key] else {
panic!("{key} is not a choice");
};
instructions.clone()
}
fn options(questions: &BTreeMap<String, Question>, key: &str) -> Vec<String> {
let Question::Choice { criteria, .. } = &questions[key] else {
panic!("{key} is not a choice");
};
criteria.keys().cloned().collect()
}
#[test]
fn stage_questions_are_the_tested_wording() {
let questions = build_route_questions(&anthropic()).unwrap();
let tiers = default_tiers();
for stage in ["stage_design", "stage_implement", "stage_review"] {
let Question::Choice { criteria, .. } = &questions[&format!("anthropic.{stage}")]
else {
panic!("{stage} is not a choice");
};
for tier in tiers.as_slice() {
assert_eq!(
criteria[&tier.name], tier.description,
"{stage}: {}",
tier.name
);
}
}
assert_eq!(
instructions(&questions, "anthropic.stage_design"),
format!(
"Which class should do the design work that remains before implementation can \
start: choosing the approach, settling open questions, and writing a plan?{BAR}"
)
);
assert_eq!(
instructions(&questions, "anthropic.stage_implement"),
format!(
"Assume any remaining design work has been completed well. Which class should \
write the code, tests and docs?{BAR}"
)
);
assert_eq!(
instructions(&questions, "anthropic.stage_review"),
format!(
"Which class should review the finished change before merge, so that mistakes \
the automated tests would miss are caught?{BAR}"
)
);
}
#[test]
fn only_the_design_stage_offers_none() {
let questions = build_route_questions(&anthropic()).unwrap();
assert_eq!(
options(&questions, "anthropic.stage_design"),
["none", "opus", "sonnet"]
);
assert_eq!(
options(&questions, "anthropic.stage_implement"),
["opus", "sonnet"]
);
assert_eq!(
options(&questions, "anthropic.stage_review"),
["opus", "sonnet"]
);
let Question::Choice { criteria, .. } = &questions["anthropic.stage_design"] else {
panic!();
};
assert_eq!(
criteria[NO_DESIGN],
"No design work remains: the text already settles the approach and the open \
questions."
);
}
#[test]
fn questions_are_keyed_per_provider_in_one_map() {
let ladders = [
class_only(Provider::Anthropic).unwrap(),
class_only(Provider::OpenAi).unwrap(),
];
let questions = build_route_questions(&ladders).unwrap();
let keys: Vec<&String> = questions.keys().collect();
assert_eq!(
keys,
[
"anthropic.stage_design",
"anthropic.stage_implement",
"anthropic.stage_review",
"open_questions",
"openai.stage_design",
"openai.stage_implement",
"openai.stage_review",
]
);
assert_eq!(
options(&questions, "openai.stage_design"),
["astra", "none", "sol", "terra"]
);
assert_eq!(
instructions(&questions, "openai.stage_review"),
instructions(&questions, "anthropic.stage_review")
);
}
#[test]
fn a_named_ladder_is_keyed_by_its_own_name() {
let ladders = [Ladder::named(
"mine".to_string(),
Tiers::parse("tiers:\n - {name: a, description: A}\n - {name: b, description: B}\n")
.unwrap(),
)];
let questions = build_route_questions(&ladders).unwrap();
assert_eq!(options(&questions, "mine.stage_implement"), ["a", "b"]);
}
#[test]
fn route_state_without_comments_is_title_and_trimmed_body() {
let (state, truncated) = build_route_state(&doc(7, ItemState::Open), 1000);
assert_eq!(state, "# #7 Route issues\n\nThe body.\n");
assert!(!truncated);
}
#[test]
fn route_state_lists_comments_in_order_in_the_tested_format() {
let mut d = doc(7, ItemState::Open);
d.comments = vec![
Comment {
author: "alice".to_string(),
body: "first\n".to_string(),
id: None,
},
Comment {
author: "bob".to_string(),
body: "second".to_string(),
id: None,
},
];
let (state, _) = build_route_state(&d, 1000);
assert_eq!(
state,
"# #7 Route issues\n\nThe body.\n\n\n---\n\n## Comments on this issue\n\
\n\n**Comment by alice:**\n\nfirst\n\
\n\n**Comment by bob:**\n\nsecond\n"
);
}
#[test]
fn route_state_truncates_the_middle_on_a_char_boundary() {
let mut d = doc(7, ItemState::Open);
d.body = "é".repeat(100);
let (full, _) = build_route_state(&d, usize::MAX);
let (state, truncated) = build_route_state(&d, 20);
assert!(truncated);
let (head, tail) = state
.split_once(&format!("\n\n{TRUNCATION_MARKER}\n\n"))
.unwrap();
assert_eq!(head.chars().count(), 10);
assert_eq!(tail.chars().count(), 10);
assert!(full.starts_with(head));
assert!(full.ends_with(tail));
}
fn comment(author: &str, body: &str) -> Comment {
Comment {
author: author.to_string(),
body: body.to_string(),
id: None,
}
}
#[test]
fn citation_text_is_the_state_with_code_masked() {
let mut d = doc(7, ItemState::Open);
d.title = "Fix `o/r#1`".to_string();
d.body = "See #5.\n\n```\nerror: o/r#2\n```".to_string();
d.comments = vec![comment("alice", "inline `o/r#3` and #6")];
let (state, _) = build_route_state(&d, 1000);
let text = build_route_citation_text(&d, 1000);
assert_eq!(text.chars().count(), state.chars().count());
assert!(text.contains("See #5."), "{text}");
assert!(text.contains("and #6"), "{text}");
assert!(text.contains("**Comment by alice:**"), "{text}");
for masked in ["o/r#1", "o/r#2", "o/r#3"] {
assert!(!text.contains(masked), "{masked} survived in {text}");
}
}
#[test]
fn citation_text_is_truncated_where_the_state_is() {
let mut d = doc(7, ItemState::Open);
d.body = format!("start #1 {} `o/r#9` end #2", "x".repeat(500));
let (state, truncated) = build_route_state(&d, 100);
let text = build_route_citation_text(&d, 100);
assert!(truncated);
assert_eq!(text.chars().count(), state.chars().count());
assert!(text.contains(TRUNCATION_MARKER), "{text}");
}
#[test]
fn citation_text_masks_each_comment_on_its_own() {
let mut d = doc(7, ItemState::Open);
d.comments = vec![
comment("alice", "```\nstray fence, never closed\nreal #11"),
comment("bob", "real #12"),
comment("carol", "```\nerror: o/r#9\n```"),
];
let text = build_route_citation_text(&d, 10_000);
assert!(text.contains("real #11"), "{text}");
assert!(text.contains("real #12"), "{text}");
assert!(!text.contains("o/r#9"), "{text}");
}
#[test]
fn route_state_truncation_keeps_the_latest_comment() {
let mut d = doc(7, ItemState::Open);
d.body = "x".repeat(10_000);
d.comments = vec![Comment {
author: "maintainer".to_string(),
body: "**Decision**: settled.".to_string(),
id: None,
}];
let (state, truncated) = build_route_state(&d, 1_000);
assert!(truncated);
assert!(state.starts_with("# #7 Route issues"));
assert!(state.ends_with("**Decision**: settled.\n"), "{state}");
}
#[test]
fn route_state_truncation_handles_a_one_char_cap() {
let (state, truncated) = build_route_state(&doc(7, ItemState::Open), 1);
assert!(truncated);
assert_eq!(state, format!("\n\n{TRUNCATION_MARKER}\n\n\n"));
}
#[test]
fn route_state_at_exactly_the_cap_is_not_truncated() {
let d = doc(7, ItemState::Open);
let (full, _) = build_route_state(&d, usize::MAX);
let (state, truncated) = build_route_state(&d, full.chars().count());
assert_eq!(state, full);
assert!(!truncated);
}
#[test]
fn drafts_follow_fetched_comments_and_default_input_is_identical() {
let mut d = doc(7, ItemState::Open);
for (author, body) in [("alice", "first"), ("bob", "second")] {
d.comments.push(Comment {
author: author.into(),
body: body.into(),
id: None,
});
}
let drafts = [" third\n\n indented\n".into(), "fourth".into()];
let (state, truncated) = build_route_state_with_drafts(&d, 1000, &drafts);
assert!(!truncated);
assert!(state.ends_with(
"**Comment by draft:**\n\nthird\n\n indented\n\n\n**Comment by draft:**\n\nfourth\n"
));
assert!(state.find("first").unwrap() < state.find("second").unwrap());
assert!(state.find("second").unwrap() < state.find("third").unwrap());
assert_eq!(
build_route_state_with_drafts(&d, 1000, &[]),
build_route_state(&d, 1000)
);
assert_eq!(
build_route_citation_text_with_drafts(&d, 1000, &[]),
build_route_citation_text(&d, 1000)
);
d.comments.clear();
let (state, _) = build_route_state_with_drafts(&d, 1000, &drafts);
assert!(state.contains("## Comments on this issue"));
}
#[test]
fn draft_truncation_and_code_masking_remain_aligned() {
let d = doc(7, ItemState::Open);
let drafts = [
format!("`#99` {}\n```\n#88\n```", "é".repeat(200)),
"See #2 at the tail".into(),
];
let (full, _) = build_route_state_with_drafts(&d, usize::MAX, &drafts);
let masked = build_route_citation_text_with_drafts(&d, usize::MAX, &drafts);
assert_eq!(full.chars().count(), masked.chars().count());
assert!(!masked.contains("#99"));
assert!(!masked.contains("#88"));
assert!(masked.contains("#2"));
let (cut, truncated) = build_route_state_with_drafts(&d, 120, &drafts);
let masked_cut = build_route_citation_text_with_drafts(&d, 120, &drafts);
assert!(truncated);
assert!(cut.ends_with("See #2 at the tail\n"));
assert_eq!(cut.chars().count(), masked_cut.chars().count());
let marker = crate::jev::input::TRUNCATION_MARKER;
let before = cut.split_once(marker).unwrap().0.chars().count();
let masked_before = masked_cut.split_once(marker).unwrap().0.chars().count();
assert_eq!(before, masked_before);
assert!(cut.find(marker).unwrap() < cut.find("See #2").unwrap());
let isolated =
build_route_citation_text_with_drafts(&d, 1000, &["`start".into(), "#2 end`".into()]);
assert!(isolated.contains("#2"));
}
#[tokio::test]
async fn drafts_reach_requests_and_batch_guard_precedes_closed_filtering() {
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("POST"))
.respond_with(wiremock::ResponseTemplate::new(200).set_body_json(routed_json()))
.expect(1)
.mount(&server)
.await;
let client = JevClient::new(&server.uri(), "key").unwrap();
let mut options = opts();
options.draft_comments = vec!["Blocked on #1129".into()];
let dependencies = OpenDependencies::from([(
("rust-works/omni-dev".into(), 7),
vec![citation("#1129", 1129)],
)]);
run_route(
&client,
&[doc(7, ItemState::Open)],
&anthropic(),
&options,
&dependencies,
)
.await
.unwrap();
options.ignore_closed = true;
let err = run_route(
&client,
&[doc(7, ItemState::Open), doc(8, ItemState::Closed)],
&anthropic(),
&options,
&dependencies,
)
.await
.unwrap_err();
assert!(err.to_string().contains("exactly one issue"));
let requests = server.received_requests().await.unwrap();
assert_eq!(requests.len(), 1);
let body: serde_json::Value = requests[0].body_json().unwrap();
assert!(body["state"]
.as_str()
.unwrap()
.ends_with("**Comment by draft:**\n\nBlocked on #1129\n"));
assert!(body["questions"].get("could_be_cheaper_0").is_some());
let expected = build_route_questions_for_mode(&anthropic(), false).unwrap();
for (key, question) in expected {
assert_eq!(
body["questions"][key],
serde_json::to_value(question).unwrap()
);
}
}
#[test]
fn preview_provenance_is_visible_in_all_formats_including_failures() {
let mut issue = IssueRoute {
draft_comments: vec!["triage draft.md".into(), "scope.md".into()],
item_ref: "o/r#1".into(),
url: "u".into(),
title: "t".into(),
state: ItemState::Open,
truncated: false,
outcome: RouteOutcome::Failed {
error: "failure".into(),
reference_fetch_failures: vec![],
},
};
for outcome in [
issue.outcome.clone(),
RouteOutcome::Routed {
open_questions: Some(test_open_questions()),
providers: BTreeMap::new(),
depends_on: vec![],
reference_fetch_failures: vec![],
},
] {
issue.outcome = outcome;
let json = serde_json::to_value(&issue).unwrap();
assert_eq!(
json["draft_comments"],
serde_json::json!(["triage draft.md", "scope.md"])
);
let yaml: serde_yaml::Value =
serde_yaml::from_str(&serde_yaml::to_string(&issue).unwrap()).unwrap();
assert_eq!(yaml["draft_comments"][0].as_str(), Some("triage draft.md"));
let text = render_issue_block(&issue, 1000, &[], TerminalStyle::default());
assert!(text.contains("draft comments (preview): [\"triage draft.md\", \"scope.md\"]"));
}
issue.draft_comments.clear();
assert!(serde_json::to_value(&issue)
.unwrap()
.get("draft_comments")
.is_none());
assert!(
!render_issue_block(&issue, 1000, &[], TerminalStyle::default()).contains("preview")
);
}
fn answer(choice: &str, confidence: f64) -> StageAnswer {
StageAnswer {
effort_by_model: BTreeMap::new(),
choice: choice.to_string(),
confidence,
probabilities: BTreeMap::new(),
}
}
fn stages(design: &str, implement: &str, review: &str) -> StageAnswers {
StageAnswers {
design: answer(design, 0.9),
implement: answer(implement, 0.9),
review: answer(review, 0.9),
}
}
#[test]
fn close_call_shows_the_best_alternative_in_compact_text() {
let mut stages = stages("sol", "terra", "sol");
stages.design.confidence = 0.21;
stages.design.probabilities = BTreeMap::from([
("none".to_string(), 0.44),
("sol".to_string(), 0.48),
("terra".to_string(), 0.08),
]);
assert_eq!(
render_stage_clause(
Stage::Design,
&stages,
&[Stage::Design],
None,
TerminalStyle::default(),
),
"design needs sol (0.21, close call — none 0.44)"
);
assert_eq!(
render_stage_clause(Stage::Design, &stages, &[], None, TerminalStyle::default(),),
"design needs sol (0.21)"
);
}
#[test]
fn close_call_uses_a_stable_tiebreaker_and_handles_missing_alternatives() {
let mut answer = answer("sol", 0.21);
answer.probabilities = BTreeMap::from([
("astra".to_string(), 0.4),
("sol".to_string(), 0.2),
("terra".to_string(), 0.4),
]);
assert_eq!(
render_stage_confidence(&answer, true),
"0.21, close call — astra 0.40"
);
answer.probabilities.clear();
assert_eq!(render_stage_confidence(&answer, true), "0.21, close call");
}
#[test]
fn design_supplies_the_class_when_it_outranks_implement() {
let tiers = default_tiers();
assert_eq!(
issue_class(&stages("opus", "sonnet", "sonnet"), &tiers),
("opus".to_string(), ClassSource::Design)
);
}
#[test]
fn implement_supplies_the_class_when_it_outranks_design() {
let tiers = default_tiers();
assert_eq!(
issue_class(&stages("sonnet", "opus", "sonnet"), &tiers),
("opus".to_string(), ClassSource::Implement)
);
}
#[test]
fn a_tie_goes_to_implement() {
let tiers = default_tiers();
for tier in ["sonnet", "opus"] {
assert_eq!(
issue_class(&stages(tier, tier, "sonnet"), &tiers),
(tier.to_string(), ClassSource::Implement),
"{tier}"
);
}
}
#[test]
fn class_ignores_review_and_none() {
let tiers = default_tiers();
assert_eq!(
issue_class(&stages("none", "sonnet", "opus"), &tiers),
("sonnet".to_string(), ClassSource::Implement)
);
}
#[test]
fn close_calls_are_below_the_threshold_in_stage_order() {
let mut s = stages("opus", "sonnet", "opus");
s.review.confidence = 0.1;
s.design.confidence = 0.29;
s.implement.confidence = 0.3;
assert_eq!(
close_calls(&s, &default_tiers(), 0.3, DEFAULT_CLOSE_CALL_MARGIN),
[Stage::Design, Stage::Review]
);
}
fn distribution(entries: &[(&str, f64)]) -> BTreeMap<String, f64> {
entries
.iter()
.map(|(name, p)| ((*name).into(), *p))
.collect()
}
#[test]
fn close_call_flags_3017_split_and_preserves_class() {
let mut s = stages("none", "sonnet", "opus");
s.implement.confidence = 0.32;
s.implement.probabilities = distribution(&[("sonnet", 0.55), ("opus", 0.45)]);
assert_eq!(
close_calls(&s, &default_tiers(), 0.3, 0.2),
[Stage::Implement]
);
assert!(close_calls(&s, &default_tiers(), 0.3, 0.0).is_empty());
assert_eq!(
issue_class(&s, &default_tiers()),
("sonnet".into(), ClassSource::Implement)
);
}
#[test]
fn close_call_predicates_are_strict_independent_and_ordered() {
let tiers = default_tiers();
let mut s = stages("sonnet", "sonnet", "sonnet");
for (confidence, gap, expected) in [
(0.25, 0.25, false), (0.125, 0.5, true), (0.5, 0.125, true), (0.125, 0.125, true), (0.5, 0.5, false), ] {
s.design.confidence = confidence;
s.design.probabilities = distribution(&[("sonnet", 0.5 + gap), ("opus", 0.5)]);
assert_eq!(
close_calls(&s, &tiers, 0.25, 0.25).contains(&Stage::Design),
expected
);
}
s.design.confidence = 0.1;
s.design.probabilities = distribution(&[("sonnet", 0.5), ("opus", 0.5)]);
s.implement.probabilities = s.design.probabilities.clone();
s.review.probabilities = s.design.probabilities.clone();
assert_eq!(close_calls(&s, &tiers, 0.3, 0.2), Stage::ALL);
assert_eq!(close_calls(&s, &tiers, 0.3, 0.0), [Stage::Design]);
s.design.confidence = 0.9;
s.design.probabilities = distribution(&[("sonnet", 1.0), ("opus", 0.0)]);
assert!(!close_calls(&s, &tiers, 0.3, 1.0).contains(&Stage::Design));
}
#[test]
fn margin_uses_only_finite_offered_probabilities_without_imputation() {
let tiers = default_tiers();
let mut a = answer("sonnet", 0.9);
assert_eq!(top_two_margin(&a, Stage::Design, &tiers), None);
a.probabilities = distribution(&[("sonnet", 0.5)]);
assert_eq!(top_two_margin(&a, Stage::Design, &tiers), None);
for invalid in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY, -0.1, 1.1] {
a.probabilities = distribution(&[("sonnet", 0.5), ("opus", invalid), ("unknown", 0.5)]);
assert_eq!(top_two_margin(&a, Stage::Implement, &tiers), None);
}
a.probabilities = distribution(&[("none", 0.5), ("sonnet", 0.5), ("unknown", 0.5)]);
assert_eq!(top_two_margin(&a, Stage::Design, &tiers), Some(0.0));
for stage in [Stage::Implement, Stage::Review] {
assert_eq!(top_two_margin(&a, stage, &tiers), None);
}
a.probabilities = distribution(&[("none", 0.375), ("opus", 0.25)]);
assert_eq!(top_two_margin(&a, Stage::Design, &tiers), Some(0.125));
let openai = Provider::OpenAi.tiers().unwrap();
a.choice = "terra".into();
a.probabilities = distribution(&[("terra", 0.0), ("sol", 0.5), ("astra", 0.5)]);
assert_eq!(top_two_margin(&a, Stage::Implement, &openai), Some(0.0));
}
#[test]
fn invalid_probabilities_retain_existing_confidence_behavior() {
let mut s = stages("none", "sonnet", "opus");
s.design.confidence = 0.1;
s.design.probabilities = distribution(&[("sonnet", f64::NAN), ("opus", -1.0)]);
s.implement.confidence = f64::NAN;
s.review.confidence = f64::NEG_INFINITY;
assert_eq!(
close_calls(&s, &default_tiers(), 0.3, 0.2),
[Stage::Design, Stage::Review]
);
s.implement.probabilities = distribution(&[("sonnet", 0.5), ("opus", 0.5)]);
assert_eq!(close_calls(&s, &default_tiers(), 0.3, 0.2), Stage::ALL);
}
#[test]
fn custom_and_multiple_ladders_share_flags_across_output_formats() {
let ladders = [
Ladder::named("custom".into(), Tiers::parse("tiers:\n - name: small,alternative\n description: Local work\n - name: large\n description: Complex work\n").unwrap()),
class_only(Provider::Anthropic).unwrap(),
];
let mut answers = BTreeMap::new();
for ladder in &ladders {
let first = &ladder.tiers.as_slice()[0].name;
let second = &ladder.tiers.as_slice()[1].name;
for stage in Stage::ALL {
let (p1, p2) = if ladder.name == "custom" && stage == Stage::Implement {
(0.55, 0.45)
} else {
(0.9, 0.1)
};
answers.insert(
stage.question_key(&ladder.name),
Answer::Choice {
choice: first.clone(),
confidence: 0.32,
probabilities: distribution(&[(first, p1), (second, p2)]),
},
);
}
}
let routes = provider_routes(&answers, &ladders, 0.3, 0.2, false).unwrap();
assert!(routes["anthropic"].close_calls.is_empty());
let route = &routes["custom"];
assert_eq!(route.close_calls, [Stage::Implement]);
let json = serde_json::to_value(route).unwrap();
let yaml: serde_json::Value =
serde_yaml::from_str(&serde_yaml::to_string(route).unwrap()).unwrap();
assert_eq!(json, yaml);
assert_eq!(json["close_calls"], serde_json::json!(["implement"]));
for stage in Stage::ALL {
let clause = render_stage_clause(
stage,
&route.stages,
&route.close_calls,
None,
TerminalStyle::default(),
);
let line = render_stage_line(
stage,
&route.stages,
&route.close_calls,
Some(&ladders[0]),
TerminalStyle::default(),
);
for text in [clause, line] {
assert_eq!(
text.contains("close call"),
stage == Stage::Implement,
"{text}"
);
if stage == Stage::Implement {
assert!(text.contains("0.32, close call — large 0.45"), "{text}");
}
}
}
let confidence_only = provider_routes(&answers, &ladders, 0.3, 0.0, false).unwrap();
assert!(confidence_only["custom"].close_calls.is_empty());
assert_eq!(route.class, confidence_only["custom"].class);
assert_eq!(route.class_from, confidence_only["custom"].class_from);
}
#[tokio::test]
async fn run_route_applies_the_configured_margin_without_changing_requests() {
let server = wiremock::MockServer::start().await;
let mut implementation = choice_json("sonnet", 0.32);
implementation["probabilities"] = serde_json::json!({"sonnet": 0.55, "opus": 0.45});
wiremock::Mock::given(wiremock::matchers::method("POST"))
.and(wiremock::matchers::path("/v1/systemone"))
.respond_with(
wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!({
"model": "jev-1.13.0", "answers": {
"open_questions": choice_json("factual", 0.8),
"anthropic.stage_design": choice_json("none", 0.9),
"anthropic.stage_implement": implementation,
"anthropic.stage_review": choice_json("opus", 0.9)
}, "usage": {"input_tokens": 100, "output_tokens": 10}
})),
)
.expect(2)
.mount(&server)
.await;
let client = JevClient::new(&server.uri(), "key").unwrap();
for margin in [0.0, DEFAULT_CLOSE_CALL_MARGIN] {
let mut options = opts();
options.close_call_margin = margin;
let report = run_route(
&client,
&[doc(1, ItemState::Open)],
&anthropic(),
&options,
&OpenDependencies::new(),
)
.await
.unwrap();
assert_eq!(
provider(&report.issues[0].outcome, "anthropic")
.close_calls
.contains(&Stage::Implement),
margin > 0.0
);
}
let requests = server.received_requests().await.unwrap();
assert_eq!(requests.len(), 2);
assert_eq!(requests[0].body, requests[1].body);
}
#[tokio::test]
async fn invalid_margins_fail_before_a_jev_request() {
let server = wiremock::MockServer::start().await;
let client = JevClient::new(&server.uri(), "key").unwrap();
for margin in [-0.1, 1.1, f64::NAN, f64::INFINITY, f64::NEG_INFINITY] {
let mut options = opts();
options.close_call_margin = margin;
let err = run_route(
&client,
&[doc(1, ItemState::Open)],
&anthropic(),
&options,
&OpenDependencies::new(),
)
.await
.unwrap_err();
assert!(err.to_string().contains("close-call margin"), "{err}");
}
for margin in [0.0, 1.0] {
let mut options = opts();
options.close_call_margin = margin;
validate_options(&[&doc(1, ItemState::Open)], &anthropic(), &options).unwrap();
}
assert!(server.received_requests().await.unwrap().is_empty());
}
#[test]
fn could_be_cheaper_key_is_stable_and_indexed() {
assert_eq!(could_be_cheaper_key(0), "could_be_cheaper_0");
assert_eq!(could_be_cheaper_key(1), "could_be_cheaper_1");
}
#[test]
fn could_be_cheaper_question_is_the_tested_wording() {
let Question::Noul {
instructions,
criteria,
} = could_be_cheaper_question("#1129")
else {
panic!("expected a noul question");
};
assert_eq!(
instructions,
"This issue cites #1129, which is still open. If #1129 is resolved, how likely is \
it that LESS design work would remain for THIS issue than the current text \
implies — as opposed to this issue's own remaining work being unaffected, because \
it is already scoped separately, is a parallel/sibling effort, or #1129 is \
otherwise not a precondition for finishing this issue's own remaining work?"
);
assert!(criteria.is_none());
}
#[test]
fn dependency_entries_reads_the_noul_answer_per_citation() {
let citations = vec![
Citation {
item_ref: ItemRef {
provider: GitProvider::GitHub,
project: "rust-works/omni-dev".to_string(),
kind: ItemKind::Issue,
number: 1129,
},
raw: "#1129".to_string(),
url: None,
},
Citation {
item_ref: ItemRef {
provider: GitProvider::GitHub,
project: "rust-works/omni-dev".to_string(),
kind: ItemKind::Issue,
number: 1349,
},
raw: "#1349".to_string(),
url: None,
},
];
let answers = BTreeMap::from([(
"could_be_cheaper_0".to_string(),
Answer::Noul { noul: 0.75 },
)]);
let deps = dependency_entries("o/r#1", &citations, &BTreeMap::new(), &answers);
assert_eq!(deps.len(), 2, "{deps:?}");
assert_eq!(deps[0].item_ref, "#1129");
assert_eq!(deps[0].state, ItemState::Open);
assert_eq!(deps[0].could_be_cheaper.get("design"), Some(&0.75));
assert_eq!(deps[0].relation, CitationRelation::Unspecified);
let json = serde_json::to_value(&deps[0]).unwrap();
assert_eq!(json["relation"], "unspecified");
assert_eq!(json["could_be_cheaper"]["design"], 0.75);
assert_eq!(deps[1].item_ref, "#1349");
assert!(deps[1].could_be_cheaper.is_empty());
assert_eq!(
serde_json::to_value(CitationRelation::Blocker).unwrap(),
"blocker"
);
assert_eq!(
serde_json::to_value(CitationRelation::Tracker).unwrap(),
"tracker"
);
}
fn relation_for(body: &str, target: u64) -> CitationRelation {
let mut issue = doc(9999, ItemState::Open);
issue.body = body.to_string();
citation_relations(body, &issue)
.get(&(issue.project, target))
.copied()
.unwrap_or(CitationRelation::Unspecified)
}
#[test]
fn citation_relation_uses_later_explicit_mentions() {
assert_eq!(
relation_for(
"See #1351 for background. Blocked on node identity in #1351.",
1351
),
CitationRelation::Blocker
);
assert_eq!(
relation_for("See #1830. Split out of #1830.", 1830),
CitationRelation::Tracker
);
}
#[test]
fn citation_relation_ignores_negation_quotes_and_conflicts() {
for body in [
"Not blocked on #123.",
"For example, \"blocked on #123\" is a blocker claim.",
"> Blocked on #123.",
"```\nBlocked on #123\n```",
"This is such as #123, a worked example.",
"See #123 for context.",
"Blocked on #123. Split out of #123.",
"Blocked on #123. No longer blocked on #123.",
"#123 blocks this issue. #123 no longer blocks it.",
] {
assert_eq!(
relation_for(body, 123),
CitationRelation::Unspecified,
"{body}"
);
}
}
#[test]
fn citation_relation_real_issue_phrasing() {
assert_eq!(
relation_for(
"Split out of #1830. Blocker #1831 closed as ADR-0083.",
1830
),
CitationRelation::Tracker
);
assert_eq!(
relation_for("## Triage — blocked on #1129's classifier change.", 1129),
CitationRelation::Blocker
);
assert_eq!(
relation_for(
"Blocked on real alias/node identity — the same work as #1351.",
1351
),
CitationRelation::Blocker
);
assert_eq!(
relation_for("Absorbed by #1753; neither suggested option remains.", 1753),
CitationRelation::Unspecified
);
assert_eq!(
relation_for("#1831 blocks this issue until its ADR lands.", 1831),
CitationRelation::Blocker
);
assert_eq!(
relation_for("See #1831. It blocks this issue.", 1831),
CitationRelation::Unspecified
);
}
#[test]
fn citation_relations_match_frozen_public_issues() {
for (inputs, labels) in [
(
include_str!("../../docs/evaluations/jev-route-1871/inputs.json"),
include_str!("../../docs/evaluations/jev-route-1871/labels.json"),
),
(
include_str!("../../docs/evaluations/jev-route-1871/holdout-inputs.json"),
include_str!("../../docs/evaluations/jev-route-1871/holdout-labels.json"),
),
] {
let cases: Vec<serde_json::Value> = serde_json::from_str(inputs).unwrap();
let labels: serde_json::Value = serde_json::from_str(labels).unwrap();
for case in cases {
let id = case["id"].as_str().unwrap();
let doc: IssueDoc = serde_json::from_value(case["doc"].clone()).unwrap();
let (state, _) = build_route_state(&doc, DEFAULT_MAX_INPUT_CHARS);
let relations = citation_relations(&state, &doc);
for (number, label) in labels[id]["relations"].as_object().into_iter().flatten() {
let actual = relations
.get(&(doc.project.clone(), number.parse().unwrap()))
.copied()
.unwrap_or(CitationRelation::Unspecified);
let want = [
("blocker", CitationRelation::Blocker),
("tracker", CitationRelation::Tracker),
("unspecified", CitationRelation::Unspecified),
]
.into_iter()
.find(|(name, _)| Some(*name) == label.as_str())
.expect("a known relation label")
.1;
assert_eq!(actual, want, "{id} -> #{number}");
}
}
}
}
#[test]
fn stage_answers_reject_none_outside_design() {
let choice = |c: &str| Answer::Choice {
choice: c.to_string(),
confidence: 0.5,
probabilities: BTreeMap::new(),
};
let answers = BTreeMap::from([
("anthropic.stage_design".to_string(), choice("none")),
("anthropic.stage_implement".to_string(), choice("none")),
("anthropic.stage_review".to_string(), choice("opus")),
]);
let err = stage_answers(&answers, &anthropic()[0]).unwrap_err();
assert!(err.to_string().contains("not offered"), "{err}");
}
#[test]
fn stage_answers_require_a_choice_per_stage() {
let answers = BTreeMap::from([(
"anthropic.stage_design".to_string(),
Answer::Noul { noul: 0.5 },
)]);
let err = stage_answers(&answers, &anthropic()[0]).unwrap_err();
assert!(err.to_string().contains("no choice answer"), "{err}");
assert!(err.to_string().contains("anthropic.stage_design"), "{err}");
}
#[test]
fn stage_answers_are_read_under_the_ladders_prefix() {
let choice = |c: &str| Answer::Choice {
choice: c.to_string(),
confidence: 0.5,
probabilities: BTreeMap::new(),
};
let answers = BTreeMap::from([
("anthropic.stage_design".to_string(), choice("none")),
("anthropic.stage_implement".to_string(), choice("sonnet")),
("anthropic.stage_review".to_string(), choice("opus")),
]);
let err = stage_answers(&answers, &class_only(Provider::OpenAi).unwrap()).unwrap_err();
assert!(err.to_string().contains("openai.stage_design"), "{err}");
}
#[test]
fn closed_issues_are_refused_by_default() {
let err =
validate_options(&[&doc(1, ItemState::Closed)], &anthropic(), &opts()).unwrap_err();
assert!(err.to_string().contains("rust-works/omni-dev#1"), "{err}");
assert!(err.to_string().contains("--allow-closed"), "{err}");
assert!(err.to_string().contains("--ignore-closed"), "{err}");
}
#[test]
fn allow_closed_and_ignore_closed_are_mutually_exclusive() {
let mut o = opts();
o.allow_closed = true;
o.ignore_closed = true;
for state in [ItemState::Open, ItemState::Closed] {
let err = drop_closed(&[doc(1, state)], &o).unwrap_err();
assert!(err.to_string().contains("cannot be used together"), "{err}");
}
}
#[test]
fn drop_closed_keeps_every_issue_without_the_flag() {
let docs = [doc(1, ItemState::Open), doc(2, ItemState::Closed)];
let (kept, ignored) = drop_closed(&docs, &opts()).unwrap();
assert_eq!(kept.len(), 2);
assert!(ignored.is_empty());
}
#[test]
fn drop_closed_skips_closed_issues_in_request_order() {
let mut o = opts();
o.ignore_closed = true;
let docs = [
doc(3, ItemState::Closed),
doc(1, ItemState::Open),
doc(2, ItemState::Closed),
];
let (kept, ignored) = drop_closed(&docs, &o).unwrap();
assert_eq!(kept.iter().map(|d| d.number).collect::<Vec<_>>(), [1]);
assert_eq!(
ignored,
[
IgnoredIssue {
item_ref: "rust-works/omni-dev#3".to_string(),
url: "https://github.com/rust-works/omni-dev/issues/3".to_string(),
title: "Route issues".to_string(),
},
IgnoredIssue {
item_ref: "rust-works/omni-dev#2".to_string(),
url: "https://github.com/rust-works/omni-dev/issues/2".to_string(),
title: "Route issues".to_string(),
},
]
);
}
#[test]
fn drop_closed_refuses_a_batch_with_no_open_issue_left() {
let mut o = opts();
o.ignore_closed = true;
let docs = [doc(1, ItemState::Closed), doc(2, ItemState::Closed)];
let err = drop_closed(&docs, &o).unwrap_err();
let msg = err.to_string();
assert!(
msg.contains("rust-works/omni-dev#1, rust-works/omni-dev#2"),
"{msg}"
);
assert!(msg.contains("nothing to route"), "{msg}");
}
#[test]
fn drop_closed_passes_an_empty_batch_through() {
let mut o = opts();
o.ignore_closed = true;
let (kept, ignored) = drop_closed(&[], &o).unwrap();
assert!(kept.is_empty());
assert!(ignored.is_empty());
}
#[test]
fn closed_issues_are_allowed_on_request() {
let mut o = opts();
o.allow_closed = true;
validate_options(&[&doc(1, ItemState::Closed)], &anthropic(), &o).unwrap();
}
#[test]
fn empty_runs_and_bad_knobs_are_rejected() {
assert!(validate_options(&[], &anthropic(), &opts()).is_err());
let mut o = opts();
o.close_call = 1.5;
assert!(validate_options(&[&doc(1, ItemState::Open)], &anthropic(), &o).is_err());
let mut o = opts();
o.max_input_chars = 0;
assert!(validate_options(&[&doc(1, ItemState::Open)], &anthropic(), &o).is_err());
}
#[test]
fn empty_and_duplicate_providers_are_rejected() {
let err = validate_options(&[&doc(1, ItemState::Open)], &[], &opts()).unwrap_err();
assert!(err.to_string().contains("no ladders"), "{err}");
let twice = [
class_only(Provider::Gemini).unwrap(),
class_only(Provider::Gemini).unwrap(),
];
let err = validate_options(&[&doc(1, ItemState::Open)], &twice, &opts()).unwrap_err();
assert!(
err.to_string()
.contains("\"gemini\" is listed more than once"),
"{err}"
);
}
#[test]
fn open_questions_uses_the_evaluated_wording() {
let evaluated: serde_json::Value = serde_json::from_str(include_str!(
"../../docs/evaluations/jev-route-2053/candidate-v3.json"
))
.unwrap();
assert_eq!(
serde_json::to_value(open_questions_question()).unwrap(),
evaluated
);
}
#[test]
fn open_questions_is_once_per_issue_in_both_modes() {
let ladders = [
Ladder::builtin(Provider::Anthropic).unwrap(),
Ladder::builtin(Provider::OpenAi).unwrap(),
];
for effort in [false, true] {
let questions = build_route_questions_for_mode(&ladders, effort).unwrap();
assert_eq!(
options(&questions, "open_questions"),
["both", "design", "factual", "none"]
);
assert_eq!(
questions
.keys()
.filter(|key| key.contains("open_questions"))
.count(),
1
);
assert_eq!(questions["open_questions"], open_questions_question());
}
}
#[test]
fn open_questions_parses_all_kinds_without_changing_class() {
let ladder = class_only(Provider::Anthropic).unwrap();
for (label, kind) in [
("none", OpenQuestionKind::None),
("factual", OpenQuestionKind::Factual),
("design", OpenQuestionKind::Design),
("both", OpenQuestionKind::Both),
] {
let choice = |label: &str| Answer::Choice {
choice: label.into(),
confidence: 0.8,
probabilities: BTreeMap::from([(label.into(), 0.8)]),
};
let answers = BTreeMap::from([
("open_questions".into(), choice(label)),
("anthropic.stage_design".into(), choice("none")),
("anthropic.stage_implement".into(), choice("sonnet")),
("anthropic.stage_review".into(), choice("opus")),
]);
let answer = open_questions_answer(&answers).unwrap();
assert_eq!(answer.choice, kind);
let routes =
provider_routes(&answers, std::slice::from_ref(&ladder), 0.3, 0.2, false).unwrap();
assert_eq!(routes["anthropic"].class, "sonnet");
assert_eq!(routes["anthropic"].class_from, ClassSource::Implement);
let issue = IssueRoute {
draft_comments: vec![],
item_ref: "o/r#1".into(),
url: String::new(),
title: "Example".into(),
state: ItemState::Open,
truncated: false,
outcome: RouteOutcome::Routed {
open_questions: Some(answer),
providers: routes,
depends_on: vec![],
reference_fetch_failures: vec![],
},
};
let json = serde_json::to_value(&issue).unwrap();
assert_eq!(json["open_questions"]["choice"], label);
assert_eq!(json["open_questions"]["confidence"], 0.8);
assert_eq!(json["open_questions"]["probabilities"][label], 0.8);
let yaml: serde_yaml::Value =
serde_yaml::from_str(&serde_yaml::to_string(&issue).unwrap()).unwrap();
assert_eq!(yaml["open_questions"]["choice"].as_str(), Some(label));
assert!(
render_issue_block(&issue, 1000, &[], TerminalStyle::default())
.contains(&format!(" open_questions: {label} (0.80)"))
);
}
}
#[test]
fn open_questions_rejects_absent_wrong_type_and_unknown_labels() {
assert!(open_questions_answer(&BTreeMap::new())
.unwrap_err()
.to_string()
.contains("no choice answer for open_questions"));
for answer in [
Answer::Noul { noul: 0.8 },
Answer::Choice {
choice: "unknown".into(),
confidence: 0.8,
probabilities: BTreeMap::new(),
},
Answer::Choice {
choice: "none".into(),
confidence: 0.8,
probabilities: BTreeMap::from([("unknown".into(), 0.2)]),
},
] {
assert!(
open_questions_answer(&BTreeMap::from([("open_questions".into(), answer)]))
.unwrap_err()
.to_string()
.contains("open_questions")
);
}
}
#[tokio::test]
async fn unusable_open_questions_never_fail_routing() {
let server = wiremock::MockServer::start().await;
for number in [7, 8, 9] {
let mut answers = serde_json::json!({
"anthropic.stage_design": choice_json("none", 0.9),
"anthropic.stage_implement": choice_json("sonnet", 0.9),
"anthropic.stage_review": choice_json("opus", 0.9),
});
match number {
8 => answers["open_questions"] = choice_json("both", 0.7),
9 => answers["open_questions"] = choice_json("maybe", 0.7),
_ => {}
}
wiremock::Mock::given(wiremock::matchers::body_partial_json(serde_json::json!({"state": format!("# #{number} Route issues\n\nThe body.\n")})))
.respond_with(wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!({"model": "jev-test", "answers": answers, "usage": {"input_tokens": 10, "output_tokens": 5}})))
.expect(1).mount(&server).await;
}
let client = JevClient::new(&server.uri(), "key").unwrap();
let report = run_route(
&client,
&[
doc(7, ItemState::Open),
doc(8, ItemState::Open),
doc(9, ItemState::Open),
],
&anthropic(),
&opts(),
&OpenDependencies::new(),
)
.await
.unwrap();
assert!(report.issues.iter().all(|issue| !issue.failed()));
assert_eq!(report.usage.input_tokens, 30);
let json = serde_json::to_value(&report).unwrap();
for index in [0, 2] {
let issue = &json["issues"][index];
assert!(issue.get("open_questions").is_none(), "{issue}");
assert_eq!(issue["providers"]["anthropic"]["class"], "sonnet");
assert!(issue.get("error").is_none(), "{issue}");
let text =
render_issue_block(&report.issues[index], 1000, &[], TerminalStyle::default());
assert!(!text.contains("open_questions"), "{text}");
}
assert_eq!(json["issues"][1]["open_questions"]["choice"], "both");
assert_eq!(
json["issues"][1]["providers"]["anthropic"]["class"],
"sonnet"
);
}
fn test_open_questions() -> OpenQuestions {
OpenQuestions {
choice: OpenQuestionKind::Factual,
confidence: 0.8,
probabilities: BTreeMap::from([("factual".into(), 0.8)]),
}
}
fn choice_json(choice: &str, confidence: f64) -> serde_json::Value {
serde_json::json!({
"type": "choice", "choice": choice, "confidence": confidence,
"probabilities": {choice: confidence}
})
}
fn provider<'a>(outcome: &'a RouteOutcome, name: &str) -> &'a ProviderRoute {
let RouteOutcome::Routed { providers, .. } = outcome else {
panic!("expected a routed issue: {outcome:?}");
};
&providers[name]
}
#[tokio::test]
async fn run_route_asks_stages_and_open_questions_per_issue_and_sums_usage() {
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("POST"))
.and(wiremock::matchers::path("/v1/systemone"))
.and(wiremock::matchers::body_partial_json(serde_json::json!({
"model": "jev-latest",
"state": "# #7 Route issues\n\nThe body.\n",
})))
.respond_with(
wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!({
"model": "jev-1.13.0",
"answers": {
"open_questions": choice_json("factual", 0.8),
"anthropic.stage_design": choice_json("opus", 0.52),
"anthropic.stage_implement": choice_json("sonnet", 0.83),
"anthropic.stage_review": choice_json("opus", 0.21),
},
"usage": {"input_tokens": 100, "output_tokens": 10}
})),
)
.expect(2)
.mount(&server)
.await;
let client = JevClient::new(&server.uri(), "key").unwrap();
let report = run_route(
&client,
&[doc(7, ItemState::Open), doc(7, ItemState::Open)],
&anthropic(),
&opts(),
&OpenDependencies::new(),
)
.await
.unwrap();
assert_eq!(report.model, "jev-1.13.0");
assert_eq!(report.usage.input_tokens, 200);
assert_eq!(report.usage.output_tokens, 20);
let issue = &report.issues[0];
assert_eq!(issue.item_ref, "rust-works/omni-dev#7");
let route = provider(&issue.outcome, "anthropic");
assert_eq!(route.class, "opus");
assert_eq!(route.class_from, ClassSource::Design);
assert_eq!(route.close_calls, [Stage::Review]);
assert!(!issue.truncated);
let requests = server.received_requests().await.unwrap();
let body: serde_json::Value = requests[0].body_json().unwrap();
let keys: Vec<&String> = body["questions"].as_object().unwrap().keys().collect();
assert_eq!(
keys,
[
"anthropic.stage_design",
"anthropic.stage_implement",
"anthropic.stage_review",
"open_questions",
]
);
}
#[tokio::test]
async fn run_route_ignore_closed_routes_only_open_issues() {
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("POST"))
.respond_with(
wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!({
"model": "jev-1.13.0",
"answers": {
"open_questions": choice_json("factual", 0.8),
"anthropic.stage_design": choice_json("opus", 0.52),
"anthropic.stage_implement": choice_json("sonnet", 0.83),
"anthropic.stage_review": choice_json("opus", 0.61),
},
"usage": {"input_tokens": 100, "output_tokens": 10}
})),
)
.expect(2)
.mount(&server)
.await;
let client = JevClient::new(&server.uri(), "key").unwrap();
let mut o = opts();
o.ignore_closed = true;
let report = run_route(
&client,
&[
doc(7, ItemState::Open),
doc(8, ItemState::Closed),
doc(9, ItemState::Open),
],
&anthropic(),
&o,
&OpenDependencies::new(),
)
.await
.unwrap();
let refs: Vec<&str> = report.issues.iter().map(|i| i.item_ref.as_str()).collect();
assert_eq!(refs, ["rust-works/omni-dev#7", "rust-works/omni-dev#9"]);
let json = serde_json::to_value(&report).unwrap();
assert_eq!(
json["ignored_closed"],
serde_json::json!([{
"ref": "rust-works/omni-dev#8",
"url": "https://github.com/rust-works/omni-dev/issues/8",
"title": "Route issues",
}])
);
let err = run_route(
&client,
&[doc(8, ItemState::Closed)],
&anthropic(),
&o,
&OpenDependencies::new(),
)
.await
.unwrap_err();
assert!(err.to_string().contains("nothing to route"), "{err}");
}
#[test]
fn route_report_omits_an_empty_ignored_closed() {
let report = RouteReport {
model: "m".to_string(),
issues: vec![],
ignored_closed: vec![],
usage: Usage::default(),
};
let json = serde_json::to_value(&report).unwrap();
assert!(json.get("ignored_closed").is_none(), "{json}");
}
#[tokio::test]
async fn run_route_routes_every_provider_in_one_call() {
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("POST"))
.respond_with(
wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!({
"model": "jev-1.13.0",
"answers": {
"open_questions": choice_json("factual", 0.8),
"anthropic.stage_design": choice_json("opus", 0.52),
"anthropic.stage_implement": choice_json("sonnet", 0.83),
"anthropic.stage_review": choice_json("opus", 0.9),
"openai.stage_design": choice_json("none", 0.9),
"openai.stage_implement": choice_json("sol", 0.2),
"openai.stage_review": choice_json("astra", 0.9),
"could_be_cheaper_0": {"type": "noul", "noul": 0.4},
},
"usage": {"input_tokens": 10, "output_tokens": 1}
})),
)
.expect(1)
.mount(&server)
.await;
let client = JevClient::new(&server.uri(), "key").unwrap();
let ladders = [
class_only(Provider::Anthropic).unwrap(),
class_only(Provider::OpenAi).unwrap(),
];
let dependencies = OpenDependencies::from([(
("rust-works/omni-dev".to_string(), 7),
vec![citation("#1129", 1129)],
)]);
let reference_fetch_failures = ReferenceFetchFailures::from([(
("rust-works/omni-dev".to_string(), 7),
vec![ReferenceFetchFailure {
item_ref: "#404".to_string(),
error: "not found".to_string(),
}],
)]);
let report = run_route_with_reference_fetch_failures(
&client,
&[doc(7, ItemState::Open)],
&ladders,
&opts(),
&dependencies,
&reference_fetch_failures,
)
.await
.unwrap();
let outcome = &report.issues[0].outcome;
let RouteOutcome::Routed {
providers,
depends_on,
reference_fetch_failures,
..
} = outcome
else {
panic!("expected a routed issue: {outcome:?}");
};
assert_eq!(
providers.keys().collect::<Vec<_>>(),
["anthropic", "openai"]
);
assert_eq!(provider(outcome, "anthropic").class, "opus");
assert_eq!(
provider(outcome, "anthropic").class_from,
ClassSource::Design
);
assert!(provider(outcome, "anthropic").close_calls.is_empty());
assert_eq!(provider(outcome, "openai").class, "sol");
assert_eq!(
provider(outcome, "openai").class_from,
ClassSource::Implement
);
assert_eq!(provider(outcome, "openai").close_calls, [Stage::Implement]);
assert_eq!(depends_on.len(), 1);
assert_eq!(reference_fetch_failures[0].item_ref, "#404");
let requests = server.received_requests().await.unwrap();
let body: serde_json::Value = requests[0].body_json().unwrap();
let keys: Vec<&String> = body["questions"].as_object().unwrap().keys().collect();
assert_eq!(
keys,
[
"anthropic.stage_design",
"anthropic.stage_implement",
"anthropic.stage_review",
"could_be_cheaper_0",
"open_questions",
"openai.stage_design",
"openai.stage_implement",
"openai.stage_review",
]
);
}
#[tokio::test]
async fn run_route_fails_the_issue_when_one_provider_is_unanswered() {
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("POST"))
.respond_with(wiremock::ResponseTemplate::new(200).set_body_json(routed_json()))
.mount(&server)
.await;
let client = JevClient::new(&server.uri(), "key").unwrap();
let ladders = [
class_only(Provider::Anthropic).unwrap(),
class_only(Provider::Gemini).unwrap(),
];
let report = run_route(
&client,
&[doc(7, ItemState::Open)],
&ladders,
&opts(),
&OpenDependencies::new(),
)
.await
.unwrap();
let RouteOutcome::Failed { error, .. } = &report.issues[0].outcome else {
panic!("expected a failed issue: {:?}", report.issues[0]);
};
assert!(error.contains("gemini.stage_design"), "{error}");
}
fn routed_json() -> serde_json::Value {
serde_json::json!({
"model": "jev-1.13.0",
"answers": {
"open_questions": choice_json("factual", 0.8),
"anthropic.stage_design": choice_json("none", 0.9),
"anthropic.stage_implement": choice_json("sonnet", 0.9),
"anthropic.stage_review": choice_json("opus", 0.9),
},
"usage": {"input_tokens": 5, "output_tokens": 1}
})
}
#[tokio::test]
async fn run_route_keeps_going_past_a_failed_issue() {
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("POST"))
.and(wiremock::matchers::body_partial_json(
serde_json::json!({"state": "# #1 Route issues\n\nThe body.\n"}),
))
.respond_with(wiremock::ResponseTemplate::new(500).set_body_string("boom"))
.mount(&server)
.await;
wiremock::Mock::given(wiremock::matchers::method("POST"))
.respond_with(wiremock::ResponseTemplate::new(200).set_body_json(routed_json()))
.mount(&server)
.await;
let client = JevClient::new(&server.uri(), "key").unwrap();
let reference_fetch_failures = ReferenceFetchFailures::from([(
("rust-works/omni-dev".to_string(), 1),
vec![ReferenceFetchFailure {
item_ref: "#404".to_string(),
error: "not found".to_string(),
}],
)]);
let report = run_route_with_reference_fetch_failures(
&client,
&[doc(1, ItemState::Open), doc(2, ItemState::Open)],
&anthropic(),
&opts(),
&OpenDependencies::new(),
&reference_fetch_failures,
)
.await
.unwrap();
assert!(report.issues[0].failed());
let RouteOutcome::Failed {
error,
reference_fetch_failures,
} = &report.issues[0].outcome
else {
panic!();
};
assert!(error.contains("500"), "{error}");
assert_eq!(reference_fetch_failures[0].item_ref, "#404");
let json = serde_json::to_value(&report).unwrap();
assert_eq!(
json["issues"][0]["reference_fetch_failures"][0]["ref"],
"#404"
);
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
assert!(
text.contains(" reference fetch failed: #404 (not found)"),
"{text}"
);
assert!(!report.issues[1].failed());
assert_eq!(report.usage.input_tokens, 5);
}
#[tokio::test]
async fn run_route_stops_on_an_auth_failure() {
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("POST"))
.respond_with(wiremock::ResponseTemplate::new(401).set_body_string("bad key"))
.expect(1)
.mount(&server)
.await;
let client = JevClient::new(&server.uri(), "key").unwrap();
let err = run_route(
&client,
&[doc(3, ItemState::Open), doc(4, ItemState::Open)],
&anthropic(),
&opts(),
&OpenDependencies::new(),
)
.await
.unwrap_err();
assert!(format!("{err:#}").contains("Failed to route issue rust-works/omni-dev#3"));
}
#[tokio::test]
async fn run_route_rejects_a_ladder_over_the_choice_cap_before_any_request() {
let yaml: String = std::iter::once("tiers:\n".to_string())
.chain((0..256).map(|i| format!(" - {{name: t{i}, description: d{i}}}\n")))
.collect();
let ladder = Ladder::named("big".to_string(), Tiers::parse(&yaml).unwrap());
let client = JevClient::new("http://127.0.0.1:1", "key").unwrap();
let err = run_route(
&client,
&[doc(3, ItemState::Open)],
&[ladder],
&opts(),
&OpenDependencies::new(),
)
.await
.unwrap_err();
let chain = format!("{err:#}");
assert!(chain.contains("route question \"big.stage_"), "{chain}");
assert!(
chain.contains("at most 255 options on jev-latest"),
"{chain}"
);
}
#[tokio::test]
async fn run_route_names_the_issue_on_an_unexpected_answer() {
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("POST"))
.respond_with(
wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!({
"model": "jev-1.13.0",
"answers": {
"open_questions": choice_json("factual", 0.8),
"anthropic.stage_design": choice_json("haiku", 0.9),
"anthropic.stage_implement": choice_json("sonnet", 0.9),
"anthropic.stage_review": choice_json("opus", 0.9),
},
})),
)
.mount(&server)
.await;
let client = JevClient::new(&server.uri(), "key").unwrap();
let report = run_route(
&client,
&[doc(9, ItemState::Open)],
&anthropic(),
&opts(),
&OpenDependencies::new(),
)
.await
.unwrap();
let RouteOutcome::Failed { error, .. } = &report.issues[0].outcome else {
panic!("expected a failed issue");
};
assert!(error.contains("Unexpected Jev answer"), "{error}");
assert!(error.contains("not offered"), "{error}");
}
fn citation(raw: &str, number: u64) -> Citation {
Citation {
item_ref: ItemRef {
provider: GitProvider::GitHub,
project: "rust-works/omni-dev".to_string(),
kind: ItemKind::Issue,
number,
},
raw: raw.to_string(),
url: None,
}
}
#[tokio::test]
async fn run_route_asks_could_be_cheaper_for_each_open_citation() {
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("POST"))
.respond_with(
wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!({
"model": "jev-1.13.0",
"answers": {
"open_questions": choice_json("factual", 0.8),
"anthropic.stage_design": choice_json("opus", 0.2),
"anthropic.stage_implement": choice_json("sonnet", 0.61),
"anthropic.stage_review": choice_json("opus", 0.9),
"could_be_cheaper_0": {"type": "noul", "noul": 0.75},
},
"usage": {"input_tokens": 10, "output_tokens": 1}
})),
)
.mount(&server)
.await;
let client = JevClient::new(&server.uri(), "key").unwrap();
let dependencies = OpenDependencies::from([(
("rust-works/omni-dev".to_string(), 7),
vec![citation("#1129", 1129)],
)]);
let report = run_route(
&client,
&[doc(7, ItemState::Open)],
&anthropic(),
&opts(),
&dependencies,
)
.await
.unwrap();
let RouteOutcome::Routed { depends_on, .. } = &report.issues[0].outcome else {
panic!("expected a routed issue: {:?}", report.issues[0]);
};
assert_eq!(depends_on.len(), 1);
assert_eq!(depends_on[0].item_ref, "#1129");
assert_eq!(depends_on[0].state, ItemState::Open);
assert_eq!(depends_on[0].could_be_cheaper.get("design"), Some(&0.75));
let requests = server.received_requests().await.unwrap();
let body: serde_json::Value = requests[0].body_json().unwrap();
let keys: Vec<&String> = body["questions"].as_object().unwrap().keys().collect();
assert_eq!(
keys,
[
"anthropic.stage_design",
"anthropic.stage_implement",
"anthropic.stage_review",
"could_be_cheaper_0",
"open_questions",
]
);
}
#[tokio::test]
async fn run_route_keeps_a_dependency_with_no_could_be_cheaper_answer() {
let server = wiremock::MockServer::start().await;
wiremock::Mock::given(wiremock::matchers::method("POST"))
.respond_with(wiremock::ResponseTemplate::new(200).set_body_json(routed_json()))
.mount(&server)
.await;
let client = JevClient::new(&server.uri(), "key").unwrap();
let dependencies = OpenDependencies::from([(
("rust-works/omni-dev".to_string(), 7),
vec![citation("#1129", 1129)],
)]);
let report = run_route(
&client,
&[doc(7, ItemState::Open)],
&anthropic(),
&opts(),
&dependencies,
)
.await
.unwrap();
let RouteOutcome::Routed { depends_on, .. } = &report.issues[0].outcome else {
panic!("expected a routed issue: {:?}", report.issues[0]);
};
assert_eq!(depends_on.len(), 1, "{depends_on:?}");
assert_eq!(depends_on[0].item_ref, "#1129");
assert!(depends_on[0].could_be_cheaper.is_empty());
}
#[test]
fn report_serialises_ref_and_omits_false_truncated() {
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![IssueRoute {
draft_comments: vec![],
item_ref: "o/r#1".to_string(),
url: "u".to_string(),
title: "t".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Routed {
open_questions: Some(test_open_questions()),
providers: BTreeMap::from([(
"anthropic".to_string(),
ProviderRoute {
stages: stages("none", "sonnet", "opus"),
class: "sonnet".to_string(),
class_from: ClassSource::Implement,
close_calls: vec![],
},
)]),
depends_on: vec![],
reference_fetch_failures: vec![],
},
truncated: false,
}],
ignored_closed: vec![],
usage: Usage::default(),
};
let value = serde_json::to_value(&report).unwrap();
assert_eq!(value["issues"][0]["ref"], "o/r#1");
assert!(value["issues"][0].get("truncated").is_none());
let anthropic = &value["issues"][0]["providers"]["anthropic"];
assert_eq!(anthropic["stages"]["design"]["choice"], "none");
assert_eq!(anthropic["class"], "sonnet");
assert_eq!(anthropic["close_calls"], serde_json::json!([]));
assert!(value["issues"][0].get("stages").is_none());
assert!(value["issues"][0].get("class").is_none());
assert_eq!(value["issues"][0]["depends_on"], serde_json::json!([]));
assert!(value["issues"][0].get("reference_fetch_failures").is_none());
assert!(value["issues"][0].get("error").is_none());
}
#[test]
fn a_failed_issue_serialises_only_its_error() {
let route = IssueRoute {
draft_comments: vec![],
item_ref: "o/r#1".to_string(),
url: "u".to_string(),
title: "t".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Failed {
error: "HTTP 529".to_string(),
reference_fetch_failures: vec![],
},
truncated: false,
};
let value = serde_json::to_value(&route).unwrap();
assert_eq!(value["error"], "HTTP 529");
assert!(value.get("providers").is_none());
assert!(value.get("depends_on").is_none());
}
#[test]
fn render_route_text_matches_the_worked_example() {
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![IssueRoute {
draft_comments: vec![],
item_ref: "rust-works/omni-dev#1641".to_string(),
url: "u".to_string(),
title: "Some issue title".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Routed {
open_questions: Some(test_open_questions()),
providers: BTreeMap::from([(
"anthropic".to_string(),
ProviderRoute {
stages: StageAnswers {
design: answer("opus", 0.52),
implement: answer("sonnet", 0.83),
review: answer("opus", 0.41),
},
class: "opus".to_string(),
class_from: ClassSource::Design,
close_calls: vec![Stage::Review],
},
)]),
depends_on: vec![DependencyEntry {
item_ref: "#1129".to_string(),
url: None,
state: ItemState::Open,
relation: CitationRelation::Unspecified,
could_be_cheaper: BTreeMap::from([("design".to_string(), 0.75)]),
}],
reference_fetch_failures: vec![],
},
truncated: false,
}],
ignored_closed: vec![],
usage: Usage {
input_tokens: 1432,
output_tokens: 61,
},
};
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
assert_eq!(
text,
"rust-works/omni-dev#1641 — Some issue title\n\
\x20\x20open_questions: factual (0.80)\n\
\x20\x20opus (from design) — design needs opus (0.52), implementation sonnet (0.83), review \
opus (0.41, close call)\n\
\x20\x20cites open #1129 (relation unspecified), which could leave less design work if resolved (0.75)\n\n\
model: jev-1.13.0, usage: 1432 input tokens, 61 output tokens\n"
);
}
#[test]
fn render_route_text_puts_each_open_citation_on_its_own_line() {
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![IssueRoute {
draft_comments: vec![],
item_ref: "rust-works/omni-dev#1845".to_string(),
url: "u".to_string(),
title: "t".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Routed {
open_questions: Some(test_open_questions()),
providers: BTreeMap::from([(
"anthropic".to_string(),
ProviderRoute {
stages: stages("none", "sonnet", "sonnet"),
class: "sonnet".to_string(),
class_from: ClassSource::Implement,
close_calls: vec![],
},
)]),
depends_on: vec![
DependencyEntry {
item_ref: "#1830".to_string(),
url: None,
state: ItemState::Open,
relation: CitationRelation::Unspecified,
could_be_cheaper: BTreeMap::from([("design".to_string(), 0.46)]),
},
DependencyEntry {
item_ref: "#1831".to_string(),
url: None,
state: ItemState::Open,
relation: CitationRelation::Unspecified,
could_be_cheaper: BTreeMap::new(),
},
],
reference_fetch_failures: vec![],
},
truncated: false,
}],
ignored_closed: vec![],
usage: Usage::default(),
};
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
assert!(
text.contains(
" cites open #1830 (relation unspecified), which could leave less design work if resolved (0.46)\n \
cites open #1831 (relation unspecified)\n"
),
"{text}"
);
assert!(!text.contains("; "), "{text}");
}
#[test]
fn render_depends_on_lines_labels_each_citation_relation() {
let entry = |item_ref: &str, relation| DependencyEntry {
item_ref: item_ref.to_string(),
url: None,
state: ItemState::Open,
relation,
could_be_cheaper: BTreeMap::new(),
};
let lines = render_depends_on_lines(
&[
entry("#1", CitationRelation::Blocker),
entry("#2", CitationRelation::Tracker),
entry("#3", CitationRelation::Unspecified),
],
TerminalStyle::default(),
);
assert_eq!(
lines,
[
" cites open #1 (blocker)",
" cites open #2 (tracker)",
" cites open #3 (relation unspecified)",
]
);
}
#[test]
fn render_route_text_names_each_provider_when_there_are_several() {
let route = |class: &str| ProviderRoute {
stages: stages("none", "sonnet", "sonnet"),
class: class.to_string(),
class_from: ClassSource::Implement,
close_calls: vec![],
};
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![IssueRoute {
draft_comments: vec![],
item_ref: "o/r#1".to_string(),
url: "u".to_string(),
title: "t".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Routed {
open_questions: Some(test_open_questions()),
providers: BTreeMap::from([
("anthropic".to_string(), route("sonnet")),
("openai".to_string(), route("terra")),
]),
depends_on: vec![],
reference_fetch_failures: vec![],
},
truncated: false,
}],
ignored_closed: vec![],
usage: Usage::default(),
};
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
assert!(
text.contains(" anthropic: sonnet (from implementation) —"),
"{text}"
);
assert!(
text.contains(" openai: terra (from implementation) —"),
"{text}"
);
}
#[test]
fn render_route_text_reports_a_failed_issue() {
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![IssueRoute {
draft_comments: vec![],
item_ref: "o/r#1".to_string(),
url: "u".to_string(),
title: "t".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Failed {
error: "HTTP 529".to_string(),
reference_fetch_failures: vec![],
},
truncated: false,
}],
ignored_closed: vec![],
usage: Usage::default(),
};
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
assert!(text.contains("o/r#1 — t\n failed: HTTP 529"), "{text}");
}
#[test]
fn render_route_text_marks_a_closed_issue_in_the_header() {
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![IssueRoute {
draft_comments: vec![],
item_ref: "o/r#1".to_string(),
url: "u".to_string(),
title: "t".to_string(),
state: ItemState::Closed,
outcome: RouteOutcome::Failed {
error: "HTTP 529".to_string(),
reference_fetch_failures: vec![],
},
truncated: false,
}],
ignored_closed: vec![],
usage: Usage::default(),
};
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
assert!(text.starts_with("o/r#1 — t (closed)\n"), "{text}");
assert!(!text.contains("ignored closed"), "{text}");
}
#[test]
fn render_route_text_lists_ignored_closed_issues_before_the_footer() {
let ignored = |n: u64| IgnoredIssue {
item_ref: format!("o/r#{n}"),
url: format!("https://github.com/o/r/issues/{n}"),
title: "t".to_string(),
};
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![],
ignored_closed: vec![ignored(2), ignored(3)],
usage: Usage::default(),
};
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
assert!(
text.contains("ignored closed: o/r#2, o/r#3\n\nmodel: jev-1.13.0"),
"{text}"
);
let styled = render_route_text_styled(
&report,
DEFAULT_MAX_INPUT_CHARS,
&[],
TerminalStyle {
color: false,
hyperlinks: true,
},
);
assert!(
styled.contains("https://github.com/o/r/issues/2"),
"{styled}"
);
}
#[test]
fn render_route_text_omits_the_depends_on_sentence_when_empty() {
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![IssueRoute {
draft_comments: vec![],
item_ref: "o/r#1".to_string(),
url: "u".to_string(),
title: "t".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Routed {
open_questions: Some(test_open_questions()),
providers: BTreeMap::from([(
"anthropic".to_string(),
ProviderRoute {
stages: stages("none", "sonnet", "sonnet"),
class: "sonnet".to_string(),
class_from: ClassSource::Implement,
close_calls: vec![],
},
)]),
depends_on: vec![],
reference_fetch_failures: vec![],
},
truncated: false,
}],
ignored_closed: vec![],
usage: Usage::default(),
};
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
assert!(!text.contains("cites"), "{text}");
}
#[test]
fn render_route_text_cites_a_dependency_with_no_could_be_cheaper_probability() {
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![IssueRoute {
draft_comments: vec![],
item_ref: "o/r#1".to_string(),
url: "u".to_string(),
title: "t".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Routed {
open_questions: Some(test_open_questions()),
providers: BTreeMap::from([(
"anthropic".to_string(),
ProviderRoute {
stages: stages("none", "sonnet", "sonnet"),
class: "sonnet".to_string(),
class_from: ClassSource::Implement,
close_calls: vec![],
},
)]),
depends_on: vec![DependencyEntry {
item_ref: "#1129".to_string(),
url: None,
state: ItemState::Open,
relation: CitationRelation::Unspecified,
could_be_cheaper: BTreeMap::new(),
}],
reference_fetch_failures: vec![],
},
truncated: false,
}],
ignored_closed: vec![],
usage: Usage::default(),
};
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
assert!(text.contains("cites open #1129"), "{text}");
assert!(!text.contains("could leave less design work"), "{text}");
}
#[test]
fn render_route_text_notes_a_truncated_issue_and_omits_the_note_otherwise() {
let issue = |truncated: bool| IssueRoute {
draft_comments: vec![],
item_ref: "o/r#1".to_string(),
url: "u".to_string(),
title: "t".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Routed {
open_questions: Some(test_open_questions()),
providers: BTreeMap::from([(
"anthropic".to_string(),
ProviderRoute {
stages: stages("none", "sonnet", "sonnet"),
class: "sonnet".to_string(),
class_from: ClassSource::Implement,
close_calls: vec![],
},
)]),
depends_on: vec![],
reference_fetch_failures: vec![],
},
truncated,
};
let report = |truncated| RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![issue(truncated)],
ignored_closed: vec![],
usage: Usage::default(),
};
let truncated_text = render_route_text(&report(true), 60_000);
assert!(
truncated_text.contains(" input truncated at 60,000 characters"),
"{truncated_text}"
);
let kept_text = render_route_text(&report(false), 60_000);
assert!(!kept_text.contains("truncated"), "{kept_text}");
}
#[test]
fn render_route_text_notes_no_further_design_work() {
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![IssueRoute {
draft_comments: vec![],
item_ref: "o/r#1".to_string(),
url: "u".to_string(),
title: "t".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Routed {
open_questions: Some(test_open_questions()),
providers: BTreeMap::from([(
"anthropic".to_string(),
ProviderRoute {
stages: stages("none", "sonnet", "sonnet"),
class: "sonnet".to_string(),
class_from: ClassSource::Implement,
close_calls: vec![],
},
)]),
depends_on: vec![],
reference_fetch_failures: vec![],
},
truncated: false,
}],
ignored_closed: vec![],
usage: Usage::default(),
};
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
assert!(text.contains("design needs no further work"), "{text}");
}
#[test]
fn render_route_text_switches_to_one_line_per_stage_for_multi_model_tiers() {
let multi = "global.anthropic.claude-sonnet-4-6,global.anthropic.claude-sonnet-5";
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![IssueRoute {
draft_comments: vec![],
item_ref: "rust-works/omni-dev#1832".to_string(),
url: "u".to_string(),
title: "feat(drive): banded ranges for drive sheets (#1830)".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Routed {
open_questions: Some(test_open_questions()),
providers: BTreeMap::from([(
"anthropic".to_string(),
ProviderRoute {
stages: StageAnswers {
design: answer(NO_DESIGN, 0.94),
implement: answer(multi, 0.94),
review: answer(multi, 0.70),
},
class: multi.to_string(),
class_from: ClassSource::Implement,
close_calls: vec![],
},
)]),
depends_on: vec![DependencyEntry {
item_ref: "#1830".to_string(),
url: None,
state: ItemState::Open,
relation: CitationRelation::Unspecified,
could_be_cheaper: BTreeMap::from([("design".to_string(), 0.48)]),
}],
reference_fetch_failures: vec![],
},
truncated: false,
}],
ignored_closed: vec![],
usage: Usage::default(),
};
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
assert_eq!(
text,
format!(
"rust-works/omni-dev#1832 — feat(drive): banded ranges for drive sheets (#1830)\n\
\x20\x20open_questions: factual (0.80)\n\
\x20\x20class: {multi} (from implementation)\n\
\x20\x20design: needs no further work (0.94)\n\
\x20\x20implementation: {multi} (0.94)\n\
\x20\x20review: {multi} (0.70)\n\
\x20\x20cites open #1830 (relation unspecified), which could leave less design work if resolved (0.48)\n\n\
model: jev-1.13.0, usage: 0 input tokens, 0 output tokens\n"
)
);
}
#[test]
fn render_route_text_keeps_the_compact_line_for_single_word_tiers() {
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![IssueRoute {
draft_comments: vec![],
item_ref: "o/r#1".to_string(),
url: "u".to_string(),
title: "t".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Routed {
open_questions: Some(test_open_questions()),
providers: BTreeMap::from([(
"anthropic".to_string(),
ProviderRoute {
stages: stages("opus", "sonnet", "opus"),
class: "opus".to_string(),
class_from: ClassSource::Design,
close_calls: vec![],
},
)]),
depends_on: vec![],
reference_fetch_failures: vec![],
},
truncated: false,
}],
ignored_closed: vec![],
usage: Usage::default(),
};
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
assert!(
text.contains(
" opus (from design) — design needs opus (0.90), implementation sonnet (0.90), \
review opus (0.90)"
),
"{text}"
);
assert!(!text.contains("class:"), "{text}");
}
#[test]
fn render_route_text_headers_the_block_by_provider_when_multi_model_and_multiple_ladders() {
let route = |class: &str| ProviderRoute {
stages: StageAnswers {
design: answer(NO_DESIGN, 0.9),
implement: answer(class, 0.9),
review: answer(class, 0.9),
},
class: class.to_string(),
class_from: ClassSource::Implement,
close_calls: vec![],
};
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![IssueRoute {
draft_comments: vec![],
item_ref: "o/r#1".to_string(),
url: "u".to_string(),
title: "t".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Routed {
open_questions: Some(test_open_questions()),
providers: BTreeMap::from([
("anthropic".to_string(), route("a,b")),
("openai".to_string(), route("c,d")),
]),
depends_on: vec![],
reference_fetch_failures: vec![],
},
truncated: false,
}],
ignored_closed: vec![],
usage: Usage::default(),
};
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
assert!(
text.contains(" anthropic:\n class: a,b (from implementation)\n"),
"{text}"
);
assert!(text.contains(" implementation: a,b (0.90)"), "{text}");
assert!(
text.contains(" openai:\n class: c,d (from implementation)\n"),
"{text}"
);
assert!(text.contains(" implementation: c,d (0.90)"), "{text}");
}
#[test]
fn render_route_text_multi_model_layout_is_decided_per_ladder() {
let multi_route = ProviderRoute {
stages: StageAnswers {
design: answer(NO_DESIGN, 0.9),
implement: answer("a,b", 0.9),
review: answer("a,b", 0.9),
},
class: "a,b".to_string(),
class_from: ClassSource::Implement,
close_calls: vec![],
};
let compact_route = ProviderRoute {
stages: stages("none", "terra", "terra"),
class: "terra".to_string(),
class_from: ClassSource::Implement,
close_calls: vec![],
};
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![IssueRoute {
draft_comments: vec![],
item_ref: "o/r#1".to_string(),
url: "u".to_string(),
title: "t".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Routed {
open_questions: Some(test_open_questions()),
providers: BTreeMap::from([
("anthropic".to_string(), multi_route),
("openai".to_string(), compact_route),
]),
depends_on: vec![],
reference_fetch_failures: vec![],
},
truncated: false,
}],
ignored_closed: vec![],
usage: Usage::default(),
};
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
assert!(
text.contains(" anthropic:\n class: a,b (from implementation)\n"),
"{text}"
);
assert!(
text.contains(" openai: terra (from implementation) — design needs no further work"),
"{text}"
);
}
#[test]
fn render_route_text_notes_a_close_call_in_the_multi_line_layout() {
let mut review = answer("a,b", 0.41);
review.probabilities =
BTreeMap::from([("a,b".to_string(), 0.56), ("c,d".to_string(), 0.44)]);
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![IssueRoute {
draft_comments: vec![],
item_ref: "o/r#1".to_string(),
url: "u".to_string(),
title: "t".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Routed {
open_questions: Some(test_open_questions()),
providers: BTreeMap::from([(
"anthropic".to_string(),
ProviderRoute {
stages: StageAnswers {
design: answer(NO_DESIGN, 0.9),
implement: answer("a,b", 0.94),
review,
},
class: "a,b".to_string(),
class_from: ClassSource::Implement,
close_calls: vec![Stage::Review],
},
)]),
depends_on: vec![],
reference_fetch_failures: vec![],
},
truncated: false,
}],
ignored_closed: vec![],
usage: Usage::default(),
};
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
assert!(
text.contains("review: a,b (0.41, close call — c,d 0.44)"),
"{text}"
);
}
#[test]
fn render_route_text_separates_multiple_issues_with_a_blank_line_in_request_order() {
let issue = |item_ref: &str| IssueRoute {
draft_comments: vec![],
item_ref: item_ref.to_string(),
url: "u".to_string(),
title: "t".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Failed {
error: "e".to_string(),
reference_fetch_failures: vec![],
},
truncated: false,
};
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![issue("o/r#1"), issue("o/r#2")],
ignored_closed: vec![],
usage: Usage::default(),
};
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
let first = text.find("o/r#1").unwrap();
let second = text.find("o/r#2").unwrap();
assert!(first < second, "{text}");
assert!(
text.contains("o/r#1 — t\n failed: e\n\no/r#2 — t\n failed: e"),
"{text}"
);
}
#[test]
fn render_route_text_ends_with_the_model_and_summed_usage() {
let report = RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![],
ignored_closed: vec![],
usage: Usage {
input_tokens: 10,
output_tokens: 20,
},
};
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
assert_eq!(
text,
"model: jev-1.13.0, usage: 10 input tokens, 20 output tokens\n"
);
}
#[test]
fn terminal_style_requires_a_usable_tty() {
assert_eq!(
TerminalStyle::new(false, "xterm", false, true),
TerminalStyle::default()
);
assert_eq!(
TerminalStyle::new(true, "dumb", false, true),
TerminalStyle::default()
);
assert_eq!(
TerminalStyle::new(true, "", false, true),
TerminalStyle::default()
);
assert_eq!(
TerminalStyle::new(true, "xterm", true, true),
TerminalStyle {
color: false,
hyperlinks: true
}
);
assert_eq!(
TerminalStyle::new(true, "xterm", false, false),
TerminalStyle {
color: true,
hyperlinks: false
}
);
}
#[test]
fn complexity_uses_tier_order_for_built_in_and_custom_ladders() {
let anthropic = class_only(Provider::Anthropic).unwrap();
assert_eq!(
choice_complexity(NO_DESIGN, &anthropic),
Some(Complexity::Low)
);
assert_eq!(
choice_complexity("sonnet", &anthropic),
Some(Complexity::Low)
);
assert_eq!(
choice_complexity("opus", &anthropic),
Some(Complexity::High)
);
let custom = |names: &[&str]| {
use std::fmt::Write;
let mut yaml = String::from("tiers:\n");
for name in names {
writeln!(yaml, " - {{name: {name}, description: work}}").unwrap();
}
Ladder::named("custom".to_string(), Tiers::parse(&yaml).unwrap())
};
let two = custom(&["a", "b"]);
assert_eq!(choice_complexity("a", &two), Some(Complexity::Low));
assert_eq!(choice_complexity("b", &two), Some(Complexity::High));
let four = custom(&["a", "b", "c", "d"]);
assert_eq!(choice_complexity("b", &four), Some(Complexity::Medium));
assert_eq!(choice_complexity("c", &four), Some(Complexity::Medium));
assert_eq!(choice_complexity("d", &four), Some(Complexity::High));
assert_eq!(choice_complexity("unknown", &four), None);
}
#[test]
fn hyperlink_uses_exact_osc_8_framing_and_rejects_unsafe_urls() {
let style = TerminalStyle {
color: false,
hyperlinks: true,
};
let url = "https://github.com/other/repo/pull/42";
assert_eq!(
hyperlink("PR #42", Some(url), style),
format!("\x1b]8;;{url}\x1b\\PR #42\x1b]8;;\x1b\\")
);
for bad in [
"https://example.com/o/r/issues/1",
"http://github.com/o/r/issues/1",
"https://github.com/o/r/issues/1\x1b]8;;evil",
"https://github.com/o/r/issues/nope",
"not a valid url",
] {
assert_eq!(hyperlink("#1", Some(bad), style), "#1");
}
assert_eq!(hyperlink("#1", None, style), "#1");
}
#[test]
fn colorize_applies_ansi_codes_per_complexity_when_color_enabled() {
let color = TerminalStyle {
color: true,
hyperlinks: false,
};
assert_eq!(colorize("x", Complexity::Low, color), "\x1b[32mx\x1b[0m");
assert_eq!(colorize("x", Complexity::Medium, color), "\x1b[33mx\x1b[0m");
assert_eq!(colorize("x", Complexity::High, color), "\x1b[31mx\x1b[0m");
assert_eq!(
colorize("x", Complexity::Medium, TerminalStyle::default()),
"x"
);
}
#[test]
fn styled_report_colors_classes_stages_and_failure_and_links_refs() {
let issue = IssueRoute {
draft_comments: vec![],
item_ref: "o/r#1".to_string(),
url: "https://github.com/o/r/issues/1".to_string(),
title: "t".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Routed {
open_questions: Some(test_open_questions()),
providers: BTreeMap::from([(
"anthropic".to_string(),
ProviderRoute {
stages: stages(NO_DESIGN, "opus", "sonnet"),
class: "opus".to_string(),
class_from: ClassSource::Implement,
close_calls: vec![],
},
)]),
depends_on: vec![DependencyEntry {
item_ref: "#42".to_string(),
url: Some("https://github.com/other/repo/pull/42".to_string()),
state: ItemState::Open,
relation: CitationRelation::Unspecified,
could_be_cheaper: BTreeMap::new(),
}],
reference_fetch_failures: vec![],
},
truncated: false,
};
let failed = IssueRoute {
draft_comments: vec![],
item_ref: "o/r#2".to_string(),
url: "https://github.com/o/r/issues/2".to_string(),
title: "failed".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Failed {
error: "HTTP 529".to_string(),
reference_fetch_failures: vec![],
},
truncated: false,
};
let report = RouteReport {
model: "jev".to_string(),
issues: vec![issue, failed],
ignored_closed: vec![],
usage: Usage::default(),
};
let style = TerminalStyle {
color: true,
hyperlinks: true,
};
let text = render_route_text_styled(
&report,
DEFAULT_MAX_INPUT_CHARS,
&[class_only(Provider::Anthropic).unwrap()],
style,
);
assert!(
text.contains(
"\x1b[32m\x1b]8;;https://github.com/o/r/issues/1\x1b\\o/r#1\x1b]8;;\x1b\\\x1b[0m"
),
"{text}"
);
assert!(
text.contains(
"\x1b[31mopus\x1b[0m (from implementation) — design needs \x1b[32mno further work\x1b[0m"
),
"{text}"
);
assert!(text.contains("review \x1b[32msonnet\x1b[0m"), "{text}");
assert!(
text.contains("\x1b]8;;https://github.com/other/repo/pull/42\x1b\\#42\x1b]8;;\x1b\\"),
"{text}"
);
assert!(text.contains("\x1b[31mfailed: HTTP 529\x1b[0m"), "{text}");
let plain = render_route_text_styled(
&report,
DEFAULT_MAX_INPUT_CHARS,
&[],
TerminalStyle::default(),
);
assert_eq!(plain, render_route_text(&report, DEFAULT_MAX_INPUT_CHARS));
assert!(!plain.contains('\x1b'));
let json = serde_json::to_string(&report).unwrap();
assert!(!json.contains("https://github.com/other/repo/pull/42"));
}
#[test]
fn styled_multi_model_layout_colors_the_class_and_stage_choices() {
let tier = "model-a,model-b";
let ladder = Ladder::named(
"custom".to_string(),
Tiers::parse(&format!(
"tiers:\n - {{name: basic, description: work}}\n - {{name: '{tier}', description: work}}\n"
))
.unwrap(),
);
let report = RouteReport {
model: "jev".to_string(),
issues: vec![IssueRoute {
draft_comments: vec![],
item_ref: "o/r#1".to_string(),
url: "https://github.com/o/r/issues/1".to_string(),
title: "t".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Routed {
open_questions: Some(test_open_questions()),
providers: BTreeMap::from([(
"custom".to_string(),
ProviderRoute {
stages: stages(NO_DESIGN, tier, "basic"),
class: tier.to_string(),
class_from: ClassSource::Implement,
close_calls: vec![],
},
)]),
depends_on: vec![],
reference_fetch_failures: vec![],
},
truncated: false,
}],
ignored_closed: vec![],
usage: Usage::default(),
};
let text = render_route_text_styled(
&report,
DEFAULT_MAX_INPUT_CHARS,
&[ladder],
TerminalStyle {
color: true,
hyperlinks: false,
},
);
assert!(
text.contains(&format!(" class: \x1b[31m{tier}\x1b[0m")),
"{text}"
);
assert!(
text.contains(&format!(" implementation: \x1b[31m{tier}\x1b[0m")),
"{text}"
);
assert!(text.contains(" review: \x1b[32mbasic\x1b[0m"), "{text}");
}
fn mixed_source_report() -> RouteReport {
RouteReport {
model: "jev-1.13.0".to_string(),
issues: vec![IssueRoute {
draft_comments: vec![],
item_ref: "o/r#1".to_string(),
url: "u".to_string(),
title: "t".to_string(),
state: ItemState::Open,
outcome: RouteOutcome::Routed {
open_questions: Some(test_open_questions()),
providers: BTreeMap::from([
(
"anthropic".to_string(),
ProviderRoute {
stages: stages("opus", "sonnet", "sonnet"),
class: "opus".to_string(),
class_from: ClassSource::Design,
close_calls: vec![],
},
),
(
"openai".to_string(),
ProviderRoute {
stages: stages(NO_DESIGN, "sol", "terra"),
class: "sol".to_string(),
class_from: ClassSource::Implement,
close_calls: vec![],
},
),
]),
depends_on: vec![],
reference_fetch_failures: vec![],
},
truncated: false,
}],
ignored_closed: vec![],
usage: Usage::default(),
}
}
#[test]
fn class_from_is_serialised_per_provider_in_json_and_yaml() {
let report = mixed_source_report();
let json = serde_json::to_value(&report).unwrap();
let providers = &json["issues"][0]["providers"];
assert_eq!(providers["anthropic"]["class_from"], "design");
assert_eq!(providers["openai"]["class_from"], "implement");
let yaml = serde_yaml::to_string(&report).unwrap();
assert!(
yaml.contains("class: opus\n class_from: design\n"),
"{yaml}"
);
assert!(
yaml.contains("class: sol\n class_from: implement\n"),
"{yaml}"
);
}
#[test]
fn text_output_names_the_class_source_for_each_ladder() {
let text = render_route_text(&mixed_source_report(), DEFAULT_MAX_INPUT_CHARS);
assert!(
text.contains(" anthropic: opus (from design) — "),
"{text}"
);
assert!(
text.contains(" openai: sol (from implementation) — "),
"{text}"
);
}
#[test]
fn text_output_names_the_class_source_in_the_multi_line_layout() {
let multi = "a,b";
let report = RouteReport {
issues: vec![IssueRoute {
outcome: RouteOutcome::Routed {
open_questions: Some(test_open_questions()),
providers: BTreeMap::from([(
"anthropic".to_string(),
ProviderRoute {
stages: StageAnswers {
design: answer(multi, 0.9),
implement: answer("basic", 0.9),
review: answer("basic", 0.9),
},
class: multi.to_string(),
class_from: ClassSource::Design,
close_calls: vec![],
},
)]),
depends_on: vec![],
reference_fetch_failures: vec![],
},
..mixed_source_report().issues.remove(0)
}],
..mixed_source_report()
};
let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
assert!(text.contains(" class: a,b (from design)\n"), "{text}");
}
#[test]
fn design_stage_with_a_named_tier_renders_needs_prefix() {
let line = render_stage_line(
Stage::Design,
&stages("opus", "sonnet", "sonnet"),
&[],
None,
TerminalStyle::default(),
);
assert_eq!(line, "design: needs opus (0.90)");
}
#[test]
fn with_thousands_groups_digits() {
assert_eq!(with_thousands(0), "0");
assert_eq!(with_thousands(999), "999");
assert_eq!(with_thousands(60_000), "60,000");
assert_eq!(with_thousands(1_000_000), "1,000,000");
}
}