1use std::collections::{BTreeMap, BTreeSet};
16use std::path::Path;
17
18use anyhow::{bail, Context, Result};
19use clap::ValueEnum;
20use serde::{Deserialize, Serialize};
21use tracing::warn;
22
23use crate::jev::citations::{find_citation_mentions, Citation};
24use crate::jev::client::JevClient;
25use crate::jev::error::is_auth_failure;
26use crate::jev::input::truncate_middle;
27use crate::jev::markdown_code::mask_code;
28use crate::jev::protocol::{Answer, Question, SystemOneRequest, Usage};
29use crate::provider::{IssueDoc, ItemRef, ItemState};
30
31pub use crate::jev::input::TRUNCATION_MARKER;
32
33pub mod effort;
34
35const STAGE_QUESTIONS_YAML: &str = include_str!("../templates/jev-route-questions.yaml");
37
38pub const NO_DESIGN: &str = "none";
41
42#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
52pub enum Provider {
53 #[value(name = "anthropic")]
55 Anthropic,
56 #[value(name = "openai")]
58 OpenAi,
59 #[value(name = "gemini")]
62 Gemini,
63}
64
65impl Provider {
66 pub const ALL: [Self; 3] = [Self::Anthropic, Self::OpenAi, Self::Gemini];
68
69 #[must_use]
72 pub const fn name(self) -> &'static str {
73 match self {
74 Self::Anthropic => "anthropic",
75 Self::OpenAi => "openai",
76 Self::Gemini => "gemini",
77 }
78 }
79
80 const fn tiers_yaml(self) -> &'static str {
82 match self {
83 Self::Anthropic => include_str!("../templates/jev-route-tiers-anthropic.yaml"),
84 Self::OpenAi => include_str!("../templates/jev-route-tiers-openai.yaml"),
85 Self::Gemini => include_str!("../templates/jev-route-tiers-gemini.yaml"),
86 }
87 }
88
89 pub fn tiers(self) -> Result<Tiers> {
91 Tiers::parse(self.tiers_yaml())
92 .with_context(|| format!("Invalid embedded tiers for provider {}", self.name()))
93 }
94}
95
96pub const DEFAULT_CLOSE_CALL: f64 = 0.3;
99
100pub const DEFAULT_CLOSE_CALL_MARGIN: f64 = 0.2;
103
104pub const DEFAULT_MAX_INPUT_CHARS: usize = 60_000;
107
108const MIN_TIERS: usize = 2;
111
112pub type OpenDependencies = BTreeMap<(String, u64), Vec<Citation>>;
118
119pub type ReferenceFetchFailures = BTreeMap<(String, u64), Vec<ReferenceFetchFailure>>;
125
126#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
128pub struct ReferenceFetchFailure {
129 #[serde(rename = "ref")]
131 pub item_ref: String,
132 pub error: String,
134}
135
136#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
138#[serde(rename_all = "lowercase")]
139pub enum Stage {
140 Design,
142 Implement,
144 Review,
146}
147
148impl Stage {
149 const ALL: [Self; 3] = [Self::Design, Self::Implement, Self::Review];
151
152 const fn template_key(self) -> &'static str {
154 match self {
155 Self::Design => "stage_design",
156 Self::Implement => "stage_implement",
157 Self::Review => "stage_review",
158 }
159 }
160
161 fn question_key(self, provider: &str) -> String {
164 format!("{provider}.{}", self.template_key())
165 }
166}
167
168#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
170#[serde(deny_unknown_fields)]
171pub struct Tier {
172 pub name: String,
174 pub description: String,
176 pub models: Option<Vec<effort::Model>>,
178}
179
180#[derive(Deserialize)]
182#[serde(deny_unknown_fields)]
183struct TiersFile {
184 tiers: Vec<Tier>,
185}
186
187#[derive(Debug, Clone, PartialEq, Eq)]
189pub struct Tiers(Vec<Tier>);
190
191impl Tiers {
192 pub fn parse(yaml: &str) -> Result<Self> {
195 let file: TiersFile = serde_yaml::from_str(yaml).context("Failed to parse tiers file")?;
196 if file.tiers.len() < MIN_TIERS {
197 bail!(
198 "a tiers file needs at least {MIN_TIERS} tiers, got {}",
199 file.tiers.len()
200 );
201 }
202 let mut seen = BTreeSet::new();
203 for tier in &file.tiers {
204 if tier.name.trim().is_empty() || tier.description.trim().is_empty() {
205 bail!("every tier needs a non-empty name and description");
206 }
207 if tier.name == NO_DESIGN {
208 bail!("the tier name {NO_DESIGN:?} is reserved for \"no design work remains\"");
209 }
210 if !seen.insert(tier.name.as_str()) {
211 bail!("tier {:?} is defined more than once", tier.name);
212 }
213 }
214 let mut model_rungs = BTreeMap::new();
215 for tier in &file.tiers {
216 if let Some(models) = &tier.models {
217 effort::validate(models).with_context(|| format!("rung {:?}", tier.name))?;
218 for model in models {
219 for &stage in &model.stages {
220 if let Some(previous) = model_rungs.insert((&model.name, stage), &tier.name)
221 {
222 bail!(
223 "model {:?} is bound to both rungs {previous:?} and {:?} for {stage:?}",
224 model.name,
225 tier.name
226 );
227 }
228 }
229 }
230 }
231 }
232 Ok(Self(file.tiers))
233 }
234
235 pub fn load_file(path: &Path) -> Result<Self> {
237 let yaml = std::fs::read_to_string(path)
238 .with_context(|| format!("Failed to read tiers file {}", path.display()))?;
239 Self::parse(&yaml).with_context(|| format!("Invalid tiers file {}", path.display()))
240 }
241
242 #[must_use]
244 pub fn as_slice(&self) -> &[Tier] {
245 &self.0
246 }
247
248 fn rank(&self, choice: &str) -> Option<usize> {
251 if choice == NO_DESIGN {
252 return Some(0);
253 }
254 self.0.iter().position(|t| t.name == choice).map(|i| i + 1)
255 }
256}
257
258#[derive(Debug, Clone, PartialEq, Eq)]
264pub struct Ladder {
265 pub name: String,
267 pub tiers: Tiers,
269}
270
271impl Ladder {
272 pub fn builtin(provider: Provider) -> Result<Self> {
274 Ok(Self {
275 name: provider.name().to_string(),
276 tiers: provider.tiers()?,
277 })
278 }
279
280 #[must_use]
282 pub fn named(name: String, tiers: Tiers) -> Self {
283 Self { name, tiers }
284 }
285}
286
287pub fn build_route_questions(ladders: &[Ladder]) -> Result<BTreeMap<String, Question>> {
293 build_route_questions_for_mode(ladders, true)
294}
295
296pub fn build_route_questions_for_mode(
301 ladders: &[Ladder],
302 effort_advice: bool,
303) -> Result<BTreeMap<String, Question>> {
304 let templates: BTreeMap<String, Question> = serde_yaml::from_str(STAGE_QUESTIONS_YAML)
305 .context("Failed to parse the embedded stage questions")?;
306 for stage in Stage::ALL {
307 if !templates.contains_key(stage.template_key()) {
308 bail!(
309 "the embedded stage questions have no {:?}",
310 stage.template_key()
312 );
313 }
314 }
315 let mut questions = BTreeMap::from([("open_questions".to_string(), open_questions_question())]);
316 for ladder in ladders {
317 for (key, template) in &templates {
318 let mut question = template.clone();
319 let Question::Choice { criteria, .. } = &mut question else {
320 bail!("embedded stage question {key:?} is not a choice question");
322 };
323 for tier in ladder.tiers.as_slice() {
324 if criteria
325 .insert(tier.name.clone(), tier.description.clone())
326 .is_some()
327 {
328 bail!(
330 "tier {:?} of ladder {:?} collides with a fixed option of {key:?}",
331 tier.name,
332 ladder.name
333 );
334 }
335 }
336 question
337 .validate()
338 .with_context(|| format!("stage question {key:?} for {:?}", ladder.name))?;
339 questions.insert(format!("{}.{key}", ladder.name), question);
340 }
341 }
342 if effort_advice {
343 for ladder in ladders {
344 effort::add_questions(ladder, &mut questions);
345 }
346 }
347 Ok(questions)
348}
349
350#[must_use]
366pub fn build_route_state(doc: &IssueDoc, max_chars: usize) -> (String, bool) {
367 build_route_state_with_drafts(doc, max_chars, &[])
368}
369
370#[must_use]
380pub fn build_route_citation_text(doc: &IssueDoc, max_chars: usize) -> String {
381 build_route_citation_text_with_drafts(doc, max_chars, &[])
382}
383
384#[must_use]
386pub fn build_route_state_with_drafts(
387 doc: &IssueDoc,
388 max_chars: usize,
389 drafts: &[String],
390) -> (String, bool) {
391 assemble_route_state(doc, max_chars, drafts, str::to_string)
392}
393
394#[must_use]
396pub fn build_route_citation_text_with_drafts(
397 doc: &IssueDoc,
398 max_chars: usize,
399 drafts: &[String],
400) -> String {
401 assemble_route_state(doc, max_chars, drafts, mask_code).0
402}
403
404fn assemble_route_state(
407 doc: &IssueDoc,
408 max_chars: usize,
409 drafts: &[String],
410 prose: impl Fn(&str) -> String,
411) -> (String, bool) {
412 let mut text = format!(
413 "# #{} {}\n\n{}\n",
414 doc.number,
415 prose(&doc.title),
416 prose(doc.body.trim())
417 );
418 if !doc.comments.is_empty() || !drafts.is_empty() {
419 text.push_str("\n\n---\n\n## Comments on this issue\n");
420 for comment in &doc.comments {
421 text.push_str(&format!(
422 "\n\n**Comment by {}:**\n\n{}\n",
423 comment.author,
424 prose(comment.body.trim())
425 ));
426 }
427 }
428 for draft in drafts {
429 text.push_str(&format!(
430 "\n\n**Comment by draft:**\n\n{}\n",
431 prose(draft.trim())
432 ));
433 }
434 truncate_middle(&text, max_chars)
435}
436
437#[derive(Debug, Clone, PartialEq, Serialize)]
439pub struct StageAnswer {
440 pub choice: String,
442 pub confidence: f64,
444 pub probabilities: BTreeMap<String, f64>,
446 #[serde(skip_serializing_if = "BTreeMap::is_empty")]
448 pub effort_by_model: BTreeMap<String, Vec<effort::ModelEffort>>,
449}
450
451#[derive(Debug, Clone, PartialEq, Serialize)]
453pub struct StageAnswers {
454 pub design: StageAnswer,
456 pub implement: StageAnswer,
458 pub review: StageAnswer,
460}
461
462impl StageAnswers {
463 fn get(&self, stage: Stage) -> &StageAnswer {
464 match stage {
465 Stage::Design => &self.design,
466 Stage::Implement => &self.implement,
467 Stage::Review => &self.review,
468 }
469 }
470}
471
472#[derive(Debug, Clone, PartialEq, Serialize)]
475pub struct DependencyEntry {
476 #[serde(rename = "ref")]
480 pub item_ref: String,
481 #[serde(skip)]
483 pub url: Option<String>,
484 pub state: ItemState,
487 pub relation: CitationRelation,
490 pub could_be_cheaper: BTreeMap<String, f64>,
496}
497
498#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
500#[serde(rename_all = "lowercase")]
501pub enum CitationRelation {
502 Blocker,
504 Tracker,
506 Unspecified,
508}
509
510#[derive(Debug, Clone, PartialEq, Serialize)]
512pub struct IssueRoute {
513 #[serde(skip_serializing_if = "Vec::is_empty")]
515 pub draft_comments: Vec<std::path::PathBuf>,
516 #[serde(rename = "ref")]
518 pub item_ref: String,
519 pub url: String,
521 pub title: String,
523 pub state: ItemState,
527 #[serde(flatten)]
529 pub outcome: RouteOutcome,
530 #[serde(skip_serializing_if = "std::ops::Not::not")]
532 pub truncated: bool,
533}
534
535#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
540#[serde(rename_all = "lowercase")]
541pub enum ClassSource {
542 Design,
544 Implement,
546}
547
548impl ClassSource {
549 const fn label(self) -> &'static str {
551 match self {
552 Self::Design => "design",
553 Self::Implement => "implementation",
554 }
555 }
556}
557
558#[derive(Debug, Clone, PartialEq, Serialize)]
560pub struct ProviderRoute {
561 pub stages: StageAnswers,
563 pub class: String,
565 pub class_from: ClassSource,
568 pub close_calls: Vec<Stage>,
570}
571
572#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
574#[serde(rename_all = "snake_case")]
575pub enum OpenQuestionKind {
576 None,
578 Factual,
580 Design,
582 Both,
584}
585
586impl OpenQuestionKind {
587 fn label(self) -> &'static str {
588 match self {
589 Self::None => "none",
590 Self::Factual => "factual",
591 Self::Design => "design",
592 Self::Both => "both",
593 }
594 }
595
596 fn parse(label: &str) -> Result<Self> {
597 match label {
598 "none" => Ok(Self::None),
599 "factual" => Ok(Self::Factual),
600 "design" => Ok(Self::Design),
601 "both" => Ok(Self::Both),
602 _ => bail!("open_questions label {label:?} was not offered"),
603 }
604 }
605}
606
607#[derive(Debug, Clone, PartialEq, Serialize)]
609pub struct OpenQuestions {
610 pub choice: OpenQuestionKind,
612 pub confidence: f64,
614 pub probabilities: BTreeMap<String, f64>,
616}
617
618fn open_questions_question() -> Question {
619 Question::Choice {
620 instructions: "What kind of questions is still open in the issue after considering all \
621 comments? Unknown existing-code facts and genuine design tradeoffs are different \
622 kinds. A suggested fix direction does not settle unverified factual premises, \
623 such as whether a reusable helper already exists. A code-informed decision with \
624 a concrete plan does settle them: carrying out its prescribed caller audits, \
625 code edits, tests, or verification is execution work, not an open question. \
626 Routine coding choices do not count as design tradeoffs. Judge the remaining \
627 uncertainty, not the amount of work."
628 .to_string(),
629 criteria: BTreeMap::from([
630 (
631 "none".to_string(),
632 "The latest text establishes the relevant existing-code facts and prescribes the \
633 approach. Executing its specified plan, including audits and verification, does \
634 not itself create open questions."
635 .to_string(),
636 ),
637 (
638 "factual".to_string(),
639 "Only existing-code facts remain unsettled: whether a reusable helper exists, \
640 what it does, or which existing sites need changes. A suggested fix with \
641 uncertain reuse or scope is factual even if its intended behavior is clear; \
642 inspecting code can settle it without a discretionary design tradeoff."
643 .to_string(),
644 ),
645 (
646 "design".to_string(),
647 "Only genuine choices between competing approaches remain, requiring judgment \
648 about behavior, architecture, or tradeoffs even after existing-code facts are \
649 known. Routine choices while implementing a prescribed fix do not qualify."
650 .to_string(),
651 ),
652 (
653 "both".to_string(),
654 "Both unresolved existing-code premises and genuine choices between competing \
655 designs remain. Code reading alone cannot settle the design tradeoffs."
656 .to_string(),
657 ),
658 ]),
659 }
660}
661
662fn open_questions_answer(answers: &BTreeMap<String, Answer>) -> Result<OpenQuestions> {
663 let Some(Answer::Choice {
664 choice,
665 confidence,
666 probabilities,
667 }) = answers.get("open_questions")
668 else {
669 bail!("no choice answer for open_questions");
670 };
671 let choice = OpenQuestionKind::parse(choice)?;
672 for label in probabilities.keys() {
673 OpenQuestionKind::parse(label)?;
674 }
675 Ok(OpenQuestions {
676 choice,
677 confidence: *confidence,
678 probabilities: probabilities.clone(),
679 })
680}
681
682#[derive(Debug, Clone, PartialEq, Serialize)]
688#[serde(untagged)]
689pub enum RouteOutcome {
690 Routed {
692 #[serde(skip_serializing_if = "Option::is_none")]
696 open_questions: Option<OpenQuestions>,
697 providers: BTreeMap<String, ProviderRoute>,
699 depends_on: Vec<DependencyEntry>,
702 #[serde(skip_serializing_if = "Vec::is_empty")]
705 reference_fetch_failures: Vec<ReferenceFetchFailure>,
706 },
707 Failed {
710 error: String,
712 #[serde(skip_serializing_if = "Vec::is_empty")]
714 reference_fetch_failures: Vec<ReferenceFetchFailure>,
715 },
716}
717
718impl IssueRoute {
719 #[must_use]
721 pub fn failed(&self) -> bool {
722 matches!(self.outcome, RouteOutcome::Failed { .. })
723 }
724}
725
726#[derive(Debug, Clone, PartialEq, Serialize)]
728pub struct RouteReport {
729 pub model: String,
732 pub issues: Vec<IssueRoute>,
734 #[serde(skip_serializing_if = "Vec::is_empty")]
738 pub ignored_closed: Vec<IgnoredIssue>,
739 pub usage: Usage,
741}
742
743#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
746pub struct IgnoredIssue {
747 #[serde(rename = "ref")]
749 pub item_ref: String,
750 pub url: String,
752 pub title: String,
754}
755
756#[derive(Debug, Clone)]
758pub struct RouteOptions {
759 pub draft_comments: Vec<String>,
761 pub model: String,
763 pub close_call: f64,
765 pub close_call_margin: f64,
768 pub effort_advice: bool,
770 pub max_input_chars: usize,
772 pub allow_closed: bool,
774 pub ignore_closed: bool,
778}
779
780pub async fn run_route(
805 jev: &JevClient,
806 docs: &[IssueDoc],
807 ladders: &[Ladder],
808 opts: &RouteOptions,
809 dependencies: &OpenDependencies,
810) -> Result<RouteReport> {
811 run_route_with_reference_fetch_failures(
812 jev,
813 docs,
814 ladders,
815 opts,
816 dependencies,
817 &ReferenceFetchFailures::new(),
818 )
819 .await
820}
821
822pub async fn run_route_with_reference_fetch_failures(
825 jev: &JevClient,
826 docs: &[IssueDoc],
827 ladders: &[Ladder],
828 opts: &RouteOptions,
829 dependencies: &OpenDependencies,
830 reference_fetch_failures: &ReferenceFetchFailures,
831) -> Result<RouteReport> {
832 if !opts.draft_comments.is_empty() && docs.len() != 1 {
833 bail!("draft comments apply to exactly one issue; cannot preview a batch");
834 }
835 let (docs, ignored_closed) = drop_closed(docs, opts)?;
836 validate_options(&docs, ladders, opts)?;
837 let questions = build_route_questions_for_mode(ladders, opts.effort_advice)?;
838 for (key, question) in &questions {
841 question
842 .validate_for_model(&opts.model)
843 .with_context(|| format!("route question {key:?}"))?;
844 }
845
846 let mut models = BTreeSet::new();
847 let mut usage = Usage::default();
848 let mut issues = Vec::with_capacity(docs.len());
849 for doc in docs {
850 let item_ref = item_ref(doc);
851 let (state, truncated) =
852 build_route_state_with_drafts(doc, opts.max_input_chars, &opts.draft_comments);
853 if truncated {
854 warn!(
855 "Issue {item_ref} is longer than {} characters; its input was truncated",
856 opts.max_input_chars
857 );
858 }
859 let citations = dependencies
860 .get(&(doc.project.clone(), doc.number))
861 .map_or(&[][..], Vec::as_slice);
862 let relations = citation_relations(&state, doc);
863 let citation_failures = reference_fetch_failures
864 .get(&(doc.project.clone(), doc.number))
865 .cloned()
866 .unwrap_or_default();
867 let mut request_questions = questions.clone();
868 for (i, citation) in citations.iter().enumerate() {
869 request_questions.insert(
870 could_be_cheaper_key(i),
871 could_be_cheaper_question(&citation.raw),
872 );
873 }
874 let request = SystemOneRequest {
875 state: serde_json::Value::String(state),
876 model: opts.model.clone(),
877 questions: request_questions,
878 };
879 let outcome = match jev.system_one(&request).await {
880 Err(err) if is_auth_failure(&err) => {
881 return Err(err).with_context(|| format!("Failed to route issue {item_ref}"));
882 }
883 Err(err) => failed(
884 &item_ref,
885 &err.context("Jev request failed"),
886 &citation_failures,
887 ),
888 Ok(response) => {
889 models.insert(response.model);
890 usage.input_tokens += response.usage.input_tokens;
891 usage.output_tokens += response.usage.output_tokens;
892 match provider_routes(
893 &response.answers,
894 ladders,
895 opts.close_call,
896 opts.close_call_margin,
897 opts.effort_advice,
898 ) {
899 Ok(providers) => RouteOutcome::Routed {
900 open_questions: open_questions_answer(&response.answers)
904 .map_err(|err| {
905 warn!(
906 "Ignoring open_questions for {item_ref}: {err:#}; \
907 routing is unaffected"
908 );
909 })
910 .ok(),
911 providers,
912 depends_on: dependency_entries(
913 &item_ref,
914 citations,
915 &relations,
916 &response.answers,
917 ),
918 reference_fetch_failures: citation_failures.clone(),
919 },
920 Err(err) => failed(
921 &item_ref,
922 &err.context("Unexpected Jev answer"),
923 &citation_failures,
924 ),
925 }
926 }
927 };
928 issues.push(IssueRoute {
929 draft_comments: vec![],
930 item_ref,
931 url: doc.url.clone(),
932 title: doc.title.clone(),
933 state: doc.state,
934 outcome,
935 truncated,
936 });
937 }
938
939 Ok(RouteReport {
940 model: models.into_iter().collect::<Vec<_>>().join(", "),
941 issues,
942 ignored_closed,
943 usage,
944 })
945}
946
947#[must_use]
950pub fn is_ignored_closed(doc: &IssueDoc, ignore_closed: bool) -> bool {
951 ignore_closed && doc.state == ItemState::Closed
952}
953
954fn item_ref(doc: &IssueDoc) -> String {
956 format!("{}#{}", doc.project, doc.number)
957}
958
959fn drop_closed<'a>(
966 docs: &'a [IssueDoc],
967 opts: &RouteOptions,
968) -> Result<(Vec<&'a IssueDoc>, Vec<IgnoredIssue>)> {
969 if opts.allow_closed && opts.ignore_closed {
970 bail!("--allow-closed and --ignore-closed cannot be used together");
971 }
972 let (open, closed): (Vec<&IssueDoc>, Vec<&IssueDoc>) = docs
973 .iter()
974 .partition(|d| !is_ignored_closed(d, opts.ignore_closed));
975 if open.is_empty() && !closed.is_empty() {
976 bail!(
977 "every requested issue is closed ({}) and --ignore-closed skipped them; nothing to \
978 route",
979 closed
980 .iter()
981 .map(|d| item_ref(d))
982 .collect::<Vec<_>>()
983 .join(", ")
984 );
985 }
986 let ignored = closed
987 .into_iter()
988 .map(|d| IgnoredIssue {
989 item_ref: item_ref(d),
990 url: d.url.clone(),
991 title: d.title.clone(),
992 })
993 .collect();
994 Ok((open, ignored))
995}
996
997fn failed(
1000 item_ref: &str,
1001 err: &anyhow::Error,
1002 reference_fetch_failures: &[ReferenceFetchFailure],
1003) -> RouteOutcome {
1004 let error = format!("{err:#}");
1005 warn!("Could not route issue {item_ref}: {error}");
1006 RouteOutcome::Failed {
1007 error,
1008 reference_fetch_failures: reference_fetch_failures.to_vec(),
1009 }
1010}
1011
1012fn validate_options(docs: &[&IssueDoc], ladders: &[Ladder], opts: &RouteOptions) -> Result<()> {
1015 if docs.is_empty() {
1016 bail!("no issues to route");
1017 }
1018 if ladders.is_empty() {
1019 bail!("no ladders to route against");
1020 }
1021 let mut seen = BTreeSet::new();
1022 for ladder in ladders {
1023 if !seen.insert(ladder.name.as_str()) {
1024 bail!("ladder {:?} is listed more than once", ladder.name);
1025 }
1026 }
1027 if !(0.0..=1.0).contains(&opts.close_call) {
1028 bail!(
1029 "the close-call threshold must be between 0 and 1, got {}",
1030 opts.close_call
1031 );
1032 }
1033 if !(0.0..=1.0).contains(&opts.close_call_margin) {
1034 bail!(
1035 "the close-call margin must be between 0 and 1, got {}",
1036 opts.close_call_margin
1037 );
1038 }
1039 if opts.max_input_chars == 0 {
1040 bail!("the input cap must be at least 1 character");
1041 }
1042 if !opts.allow_closed {
1043 let closed: Vec<String> = docs
1044 .iter()
1045 .filter(|d| d.state == ItemState::Closed)
1046 .map(|d| item_ref(d))
1047 .collect();
1048 if !closed.is_empty() {
1049 bail!(
1050 "refusing to route closed issues ({}): their comments often describe how the \
1051 work was done, which leaks the answer. Pass --ignore-closed to skip them, or \
1052 --allow-closed to route them anyway",
1053 closed.join(", ")
1054 );
1055 }
1056 }
1057 Ok(())
1058}
1059
1060fn provider_routes(
1064 answers: &BTreeMap<String, Answer>,
1065 ladders: &[Ladder],
1066 close_call: f64,
1067 close_call_margin: f64,
1068 effort_advice: bool,
1069) -> Result<BTreeMap<String, ProviderRoute>> {
1070 ladders
1071 .iter()
1072 .map(|ladder| {
1073 let mut stages = stage_answers(answers, ladder)?;
1074 if effort_advice {
1075 for (stage, answer) in [
1076 (Stage::Design, &mut stages.design),
1077 (Stage::Implement, &mut stages.implement),
1078 (Stage::Review, &mut stages.review),
1079 ] {
1080 answer.effort_by_model =
1081 effort::decode(ladder, stage, answer, answers, close_call)?;
1082 }
1083 }
1084 let (class, class_from) = issue_class(&stages, &ladder.tiers);
1085 let route = ProviderRoute {
1086 class,
1087 class_from,
1088 close_calls: close_calls(&stages, &ladder.tiers, close_call, close_call_margin),
1089 stages,
1090 };
1091 Ok((ladder.name.clone(), route))
1092 })
1093 .collect()
1094}
1095
1096fn stage_answers(answers: &BTreeMap<String, Answer>, ladder: &Ladder) -> Result<StageAnswers> {
1099 let tiers = &ladder.tiers;
1100 let read = |stage: Stage| -> Result<StageAnswer> {
1101 let key = stage.question_key(&ladder.name);
1102 let Some(Answer::Choice {
1103 choice,
1104 confidence,
1105 probabilities,
1106 }) = answers.get(&key)
1107 else {
1108 bail!("no choice answer for {key:?}");
1109 };
1110 let offered = match tiers.rank(choice) {
1111 Some(0) => stage == Stage::Design,
1112 Some(_) => true,
1113 None => false,
1114 };
1115 if !offered {
1116 bail!("{key:?} chose {choice:?}, which was not offered");
1117 }
1118 Ok(StageAnswer {
1119 choice: choice.clone(),
1120 confidence: *confidence,
1121 probabilities: probabilities.clone(),
1122 effort_by_model: BTreeMap::new(),
1123 })
1124 };
1125 Ok(StageAnswers {
1126 design: read(Stage::Design)?,
1127 implement: read(Stage::Implement)?,
1128 review: read(Stage::Review)?,
1129 })
1130}
1131
1132fn issue_class(stages: &StageAnswers, tiers: &Tiers) -> (String, ClassSource) {
1140 let rank = |a: &StageAnswer| tiers.rank(&a.choice).unwrap_or_default();
1141 if rank(&stages.design) > rank(&stages.implement) {
1142 (stages.design.choice.clone(), ClassSource::Design)
1143 } else {
1144 (stages.implement.choice.clone(), ClassSource::Implement)
1145 }
1146}
1147
1148fn top_two_margin(answer: &StageAnswer, stage: Stage, tiers: &Tiers) -> Option<f64> {
1151 let mut usable: Vec<_> = answer
1152 .probabilities
1153 .iter()
1154 .filter(|(name, probability)| {
1155 probability.is_finite()
1156 && (0.0..=1.0).contains(*probability)
1157 && match tiers.rank(name) {
1158 Some(0) => stage == Stage::Design,
1159 Some(_) => true,
1160 None => false,
1161 }
1162 })
1163 .collect();
1164 usable.sort_by(|a, b| b.1.total_cmp(a.1).then_with(|| a.0.cmp(b.0)));
1165 let [first, second, ..] = usable.as_slice() else {
1166 return None;
1167 };
1168 Some(first.1 - second.1)
1169}
1170
1171fn close_calls(stages: &StageAnswers, tiers: &Tiers, threshold: f64, margin: f64) -> Vec<Stage> {
1174 Stage::ALL
1175 .into_iter()
1176 .filter(|&stage| {
1177 let answer = stages.get(stage);
1178 answer.confidence < threshold
1179 || top_two_margin(answer, stage, tiers).is_some_and(|gap| gap < margin)
1180 })
1181 .collect()
1182}
1183
1184pub fn could_be_cheaper_key(i: usize) -> String {
1189 format!("could_be_cheaper_{i}")
1190}
1191
1192pub fn could_be_cheaper_question(citation: &str) -> Question {
1200 Question::Noul {
1201 instructions: format!(
1202 "This issue cites {citation}, which is still open. If {citation} is resolved, how \
1203 likely is it that LESS design work would remain for THIS issue than the current \
1204 text implies — as opposed to this issue's own remaining work being unaffected, \
1205 because it is already scoped separately, is a parallel/sibling effort, or \
1206 {citation} is otherwise not a precondition for finishing this issue's own \
1207 remaining work?"
1208 ),
1209 criteria: None,
1210 }
1211}
1212
1213fn citation_relations(state: &str, doc: &IssueDoc) -> BTreeMap<(String, u64), CitationRelation> {
1217 let judged = ItemRef {
1218 provider: doc.provider,
1219 project: doc.project.clone(),
1220 kind: doc.kind,
1221 number: doc.number,
1222 };
1223 let mut evidence: BTreeMap<(String, u64), (bool, bool, bool)> = BTreeMap::new();
1224 for mention in find_citation_mentions(state, &doc.project, &judged) {
1225 let key = (
1226 mention.citation.item_ref.project,
1227 mention.citation.item_ref.number,
1228 );
1229 let entry = evidence.entry(key).or_default();
1230 match relation_at(state, &mention.range) {
1231 MentionRelation::Blocker => entry.0 = true,
1232 MentionRelation::Tracker => entry.1 = true,
1233 MentionRelation::Negated => entry.2 = true,
1234 MentionRelation::None => {}
1235 }
1236 }
1237 evidence
1238 .into_iter()
1239 .map(|(key, (blocker, tracker, negated))| {
1240 let relation = match (blocker, tracker, negated) {
1241 (true, false, false) => CitationRelation::Blocker,
1242 (false, true, false) => CitationRelation::Tracker,
1243 _ => CitationRelation::Unspecified,
1244 };
1245 (key, relation)
1246 })
1247 .collect()
1248}
1249
1250enum MentionRelation {
1251 Blocker,
1252 Tracker,
1253 Negated,
1254 None,
1255}
1256
1257fn relation_at(state: &str, range: &std::ops::Range<usize>) -> MentionRelation {
1258 if !state[..range.start]
1259 .matches("```")
1260 .count()
1261 .is_multiple_of(2)
1262 {
1263 return MentionRelation::None;
1264 }
1265 let before = &state[..range.start];
1266 let boundary = before.rfind(['.', '!', '?', '\n']).map_or(0, |i| i + 1);
1267 let sentence = &state[boundary..];
1268 if sentence.trim_start().starts_with('>') {
1269 return MentionRelation::None;
1270 }
1271 let prefix = &state[boundary..range.start];
1272 if !prefix.matches('`').count().is_multiple_of(2)
1273 || !prefix.matches('"').count().is_multiple_of(2)
1274 {
1275 return MentionRelation::None;
1276 }
1277 let nearby: String = prefix
1278 .chars()
1279 .rev()
1280 .take(160)
1281 .collect::<String>()
1282 .chars()
1283 .rev()
1284 .collect();
1285 let nearby = nearby.to_ascii_lowercase();
1286 let after = &state[range.end..];
1287 let end = after.find(['.', '!', '?', '\n']).unwrap_or(after.len());
1288 let following = after[..end].trim_start().to_ascii_lowercase();
1289 if nearby.contains("such as")
1290 || nearby.contains("for example")
1291 || nearby.contains("worked example")
1292 || nearby.contains("e.g.")
1293 {
1294 return MentionRelation::None;
1295 }
1296 if nearby.contains("not blocked on")
1297 || nearby.contains("no longer blocked on")
1298 || nearby.contains("isn't blocked on")
1299 || nearby.contains("not dependent on")
1300 || nearby.contains("unblocked")
1301 || nearby.contains("not a tracker")
1302 || nearby.contains("not split out of")
1303 || following.starts_with("does not block")
1304 || following.starts_with("no longer blocks")
1305 {
1306 return MentionRelation::Negated;
1307 }
1308 if following.starts_with("blocks ")
1309 || following.starts_with("is required for ")
1310 || following.starts_with("must land before ")
1311 {
1312 return MentionRelation::Blocker;
1313 }
1314 if nearby.contains("blocked on")
1315 || nearby.contains("depends on")
1316 || nearby.contains("requires")
1317 || nearby.contains("prerequisite")
1318 {
1319 return MentionRelation::Blocker;
1320 }
1321 if nearby.contains("split out of")
1322 || nearby.contains("split from")
1323 || nearby.contains("parent tracker")
1324 || nearby.contains("tracker checkbox")
1325 {
1326 return MentionRelation::Tracker;
1327 }
1328 MentionRelation::None
1329}
1330
1331fn dependency_entries(
1337 item_ref: &str,
1338 citations: &[Citation],
1339 relations: &BTreeMap<(String, u64), CitationRelation>,
1340 answers: &BTreeMap<String, Answer>,
1341) -> Vec<DependencyEntry> {
1342 citations
1343 .iter()
1344 .enumerate()
1345 .map(|(i, citation)| {
1346 let could_be_cheaper =
1347 if let Some(Answer::Noul { noul }) = answers.get(&could_be_cheaper_key(i)) {
1348 BTreeMap::from([("design".to_string(), *noul)])
1349 } else {
1350 warn!(
1351 "Issue {item_ref}: no could_be_cheaper answer for citation {:?}",
1352 citation.raw
1353 );
1354 BTreeMap::new()
1355 };
1356 DependencyEntry {
1357 item_ref: citation.raw.clone(),
1358 url: citation.url.clone(),
1359 state: ItemState::Open,
1360 relation: relations
1361 .get(&(citation.item_ref.project.clone(), citation.item_ref.number))
1362 .copied()
1363 .unwrap_or(CitationRelation::Unspecified),
1364 could_be_cheaper,
1365 }
1366 })
1367 .collect()
1368}
1369
1370#[must_use]
1391pub fn render_route_text(report: &RouteReport, max_input_chars: usize) -> String {
1392 render_route_text_styled(report, max_input_chars, &[], TerminalStyle::default())
1393}
1394
1395#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
1398pub struct TerminalStyle {
1399 pub color: bool,
1401 pub hyperlinks: bool,
1403}
1404
1405impl TerminalStyle {
1406 #[must_use]
1408 pub fn new(tty: bool, term: &str, no_color: bool, supports_links: bool) -> Self {
1409 let usable = tty && !term.is_empty() && term != "dumb";
1410 Self {
1411 color: usable && !no_color,
1412 hyperlinks: usable && supports_links,
1413 }
1414 }
1415}
1416
1417#[must_use]
1419pub fn render_route_text_styled(
1420 report: &RouteReport,
1421 max_input_chars: usize,
1422 ladders: &[Ladder],
1423 style: TerminalStyle,
1424) -> String {
1425 let mut blocks: Vec<String> = report
1426 .issues
1427 .iter()
1428 .map(|issue| render_issue_block(issue, max_input_chars, ladders, style))
1429 .collect();
1430 if !report.ignored_closed.is_empty() {
1431 let refs: Vec<String> = report
1432 .ignored_closed
1433 .iter()
1434 .map(|issue| hyperlink(&issue.item_ref, Some(&issue.url), style))
1435 .collect();
1436 blocks.push(format!("ignored closed: {}", refs.join(", ")));
1437 }
1438 blocks.push(format!(
1439 "model: {}, usage: {} input tokens, {} output tokens",
1440 report.model, report.usage.input_tokens, report.usage.output_tokens
1441 ));
1442 let mut text = blocks.join("\n\n");
1443 text.push('\n');
1444 text
1445}
1446
1447fn render_issue_block(
1451 issue: &IssueRoute,
1452 max_input_chars: usize,
1453 ladders: &[Ladder],
1454 style: TerminalStyle,
1455) -> String {
1456 let item_ref = hyperlink(&issue.item_ref, Some(&issue.url), style);
1457 let item_ref = if issue.failed() {
1458 item_ref
1459 } else {
1460 colorize(&item_ref, Complexity::Low, style)
1461 };
1462 let header = if issue.state == ItemState::Closed {
1463 format!("{item_ref} — {} (closed)", issue.title)
1464 } else {
1465 format!("{item_ref} — {}", issue.title)
1466 };
1467 let mut lines = vec![header];
1468 if !issue.draft_comments.is_empty() {
1469 lines.push(format!(
1470 " draft comments (preview): {:?}",
1471 issue.draft_comments
1472 ));
1473 }
1474 match &issue.outcome {
1475 RouteOutcome::Routed {
1476 providers,
1477 depends_on,
1478 reference_fetch_failures,
1479 open_questions,
1480 } => {
1481 if let Some(open_questions) = open_questions {
1482 lines.push(format!(
1483 " open_questions: {} ({:.2})",
1484 open_questions.choice.label(),
1485 open_questions.confidence
1486 ));
1487 }
1488 for (provider, route) in providers {
1489 let ladder = ladders.iter().find(|ladder| ladder.name == *provider);
1490 lines.extend(render_provider_line(
1491 provider,
1492 route,
1493 providers.len(),
1494 ladder,
1495 style,
1496 ));
1497 }
1498 lines.extend(render_depends_on_lines(depends_on, style));
1499 lines.extend(render_reference_fetch_failure_lines(
1500 reference_fetch_failures,
1501 ));
1502 }
1503 RouteOutcome::Failed {
1504 error,
1505 reference_fetch_failures,
1506 } => {
1507 lines.push(format!(
1508 " {}",
1509 colorize(&format!("failed: {error}"), Complexity::High, style)
1510 ));
1511 lines.extend(render_reference_fetch_failure_lines(
1512 reference_fetch_failures,
1513 ));
1514 }
1515 }
1516 if issue.truncated {
1517 lines.push(format!(
1518 " input truncated at {} characters",
1519 with_thousands(max_input_chars)
1520 ));
1521 }
1522 lines.join("\n")
1523}
1524
1525fn route_has_multi_model_tier(route: &ProviderRoute) -> bool {
1534 [
1535 &route.stages.design,
1536 &route.stages.implement,
1537 &route.stages.review,
1538 ]
1539 .into_iter()
1540 .any(|answer| answer.choice.contains(','))
1541}
1542
1543fn render_provider_line(
1548 provider: &str,
1549 route: &ProviderRoute,
1550 provider_count: usize,
1551 ladder: Option<&Ladder>,
1552 style: TerminalStyle,
1553) -> Vec<String> {
1554 let mut lines = if route_has_multi_model_tier(route) {
1555 render_provider_block(provider, route, provider_count, ladder, style)
1556 } else {
1557 vec![render_provider_line_compact(
1558 provider,
1559 route,
1560 provider_count,
1561 ladder,
1562 style,
1563 )]
1564 };
1565 lines.extend(effort::render(&route.stages, ladder));
1566 lines
1567}
1568
1569fn render_provider_line_compact(
1574 provider: &str,
1575 route: &ProviderRoute,
1576 provider_count: usize,
1577 ladder: Option<&Ladder>,
1578 style: TerminalStyle,
1579) -> String {
1580 let class = format!(
1581 "{} (from {})",
1582 color_choice(&route.class, ladder, style),
1583 route.class_from.label()
1584 );
1585 let lead = if provider_count > 1 {
1586 format!("{provider}: {class}")
1587 } else {
1588 class
1589 };
1590 let clauses: Vec<String> = Stage::ALL
1591 .into_iter()
1592 .map(|stage| render_stage_clause(stage, &route.stages, &route.close_calls, ladder, style))
1593 .collect();
1594 format!(" {lead} — {}", clauses.join(", "))
1595}
1596
1597fn render_stage_clause(
1600 stage: Stage,
1601 stages: &StageAnswers,
1602 close_calls: &[Stage],
1603 ladder: Option<&Ladder>,
1604 style: TerminalStyle,
1605) -> String {
1606 let answer = stages.get(stage);
1607 let choice = color_choice(&answer.choice, ladder, style);
1608 let label = match stage {
1609 Stage::Design if answer.choice == NO_DESIGN => format!(
1610 "design needs {}",
1611 colorize("no further work", Complexity::Low, style)
1612 ),
1613 Stage::Design => format!("design needs {choice}"),
1614 Stage::Implement => format!("implementation {choice}"),
1615 Stage::Review => format!("review {choice}"),
1616 };
1617 format!(
1618 "{label} ({})",
1619 render_stage_confidence(answer, close_calls.contains(&stage))
1620 )
1621}
1622
1623fn render_stage_confidence(answer: &StageAnswer, close_call: bool) -> String {
1626 let mut detail = format!("{:.2}", answer.confidence);
1627 if close_call {
1628 detail.push_str(", close call");
1629 if let Some((name, probability)) = answer
1630 .probabilities
1631 .iter()
1632 .filter(|(name, probability)| name.as_str() != answer.choice && probability.is_finite())
1633 .max_by(|a, b| a.1.total_cmp(b.1).then_with(|| b.0.cmp(a.0)))
1634 {
1635 detail.push_str(&format!(" — {name} {probability:.2}"));
1636 }
1637 }
1638 detail
1639}
1640
1641fn render_provider_block(
1647 provider: &str,
1648 route: &ProviderRoute,
1649 provider_count: usize,
1650 ladder: Option<&Ladder>,
1651 style: TerminalStyle,
1652) -> Vec<String> {
1653 let mut lines = Vec::with_capacity(5);
1654 let indent = if provider_count > 1 {
1655 lines.push(format!(" {provider}:"));
1656 " "
1657 } else {
1658 " "
1659 };
1660 lines.push(format!(
1661 "{indent}class: {} (from {})",
1662 color_choice(&route.class, ladder, style),
1663 route.class_from.label()
1664 ));
1665 for stage in Stage::ALL {
1666 lines.push(format!(
1667 "{indent}{}",
1668 render_stage_line(stage, &route.stages, &route.close_calls, ladder, style)
1669 ));
1670 }
1671 lines
1672}
1673
1674fn render_stage_line(
1679 stage: Stage,
1680 stages: &StageAnswers,
1681 close_calls: &[Stage],
1682 ladder: Option<&Ladder>,
1683 style: TerminalStyle,
1684) -> String {
1685 let answer = stages.get(stage);
1686 let choice = color_choice(&answer.choice, ladder, style);
1687 let (label, content) = match stage {
1688 Stage::Design if answer.choice == NO_DESIGN => (
1689 "design",
1690 format!(
1691 "needs {}",
1692 colorize("no further work", Complexity::Low, style)
1693 ),
1694 ),
1695 Stage::Design => ("design", format!("needs {choice}")),
1696 Stage::Implement => ("implementation", choice),
1697 Stage::Review => ("review", choice),
1698 };
1699 format!(
1700 "{label}: {content} ({})",
1701 render_stage_confidence(answer, close_calls.contains(&stage))
1702 )
1703}
1704
1705fn render_depends_on_lines(depends_on: &[DependencyEntry], style: TerminalStyle) -> Vec<String> {
1708 depends_on
1709 .iter()
1710 .map(|dep| {
1711 let item_ref = hyperlink(&dep.item_ref, dep.url.as_deref(), style);
1712 let relation = match dep.relation {
1713 CitationRelation::Blocker => "blocker",
1714 CitationRelation::Tracker => "tracker",
1715 CitationRelation::Unspecified => "relation unspecified",
1716 };
1717 match dep.could_be_cheaper.get("design") {
1718 Some(prob) => format!(
1719 " cites open {item_ref} ({relation}), which could leave less design work if resolved ({prob:.2})"
1720 ),
1721 None => format!(" cites open {item_ref} ({relation})"),
1722 }
1723 })
1724 .collect()
1725}
1726
1727fn render_reference_fetch_failure_lines(failures: &[ReferenceFetchFailure]) -> Vec<String> {
1729 failures
1730 .iter()
1731 .map(|failure| {
1732 format!(
1733 " reference fetch failed: {} ({})",
1734 failure.item_ref, failure.error
1735 )
1736 })
1737 .collect()
1738}
1739
1740#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1741enum Complexity {
1742 Low,
1743 Medium,
1744 High,
1745}
1746
1747fn choice_complexity(choice: &str, ladder: &Ladder) -> Option<Complexity> {
1749 if choice == NO_DESIGN {
1750 return Some(Complexity::Low);
1751 }
1752 let tiers = ladder.tiers.as_slice();
1753 let index = tiers.iter().position(|tier| tier.name == choice)?;
1754 Some(if index == 0 {
1755 Complexity::Low
1756 } else if index == tiers.len() - 1 {
1757 Complexity::High
1758 } else {
1759 Complexity::Medium
1760 })
1761}
1762
1763fn color_choice(choice: &str, ladder: Option<&Ladder>, style: TerminalStyle) -> String {
1764 ladder
1765 .and_then(|ladder| choice_complexity(choice, ladder))
1766 .map_or_else(
1767 || choice.to_string(),
1768 |level| colorize(choice, level, style),
1769 )
1770}
1771
1772fn colorize(text: &str, level: Complexity, style: TerminalStyle) -> String {
1773 if !style.color {
1774 return text.to_string();
1775 }
1776 let code = match level {
1777 Complexity::Low => 32,
1778 Complexity::Medium => 33,
1779 Complexity::High => 31,
1780 };
1781 format!("\x1b[{code}m{text}\x1b[0m")
1782}
1783
1784fn hyperlink(text: &str, url: Option<&str>, style: TerminalStyle) -> String {
1787 let Some(url) = url.filter(|url| style.hyperlinks && valid_github_item_url(url)) else {
1788 return text.to_string();
1789 };
1790 format!("\x1b]8;;{url}\x1b\\{text}\x1b]8;;\x1b\\")
1791}
1792
1793fn valid_github_item_url(raw: &str) -> bool {
1794 if raw.chars().any(char::is_control) {
1795 return false;
1796 }
1797 let Ok(url) = url::Url::parse(raw) else {
1798 return false;
1799 };
1800 if url.scheme() != "https" || url.host_str() != Some("github.com") {
1801 return false;
1802 }
1803 let Some(segments) = url.path_segments() else {
1804 return false; };
1806 let parts: Vec<_> = segments.collect();
1807 parts.len() == 4
1808 && !parts[0].is_empty()
1809 && !parts[1].is_empty()
1810 && matches!(parts[2], "issues" | "pull")
1811 && parts[3].parse::<u64>().is_ok()
1812}
1813
1814fn with_thousands(n: usize) -> String {
1816 let digits = n.to_string();
1817 let mut grouped = String::with_capacity(digits.len() + digits.len() / 3);
1818 for (i, ch) in digits.chars().rev().enumerate() {
1819 if i > 0 && i % 3 == 0 {
1820 grouped.push(',');
1821 }
1822 grouped.push(ch);
1823 }
1824 grouped.chars().rev().collect()
1825}
1826
1827#[cfg(test)]
1828#[allow(clippy::unwrap_used, clippy::expect_used)]
1829mod tests {
1830 use super::*;
1831 use crate::provider::{Comment, GitProvider, ItemKind, ItemRef};
1832
1833 fn doc(number: u64, state: ItemState) -> IssueDoc {
1834 IssueDoc {
1835 provider: GitProvider::GitHub,
1836 project: "rust-works/omni-dev".to_string(),
1837 number,
1838 kind: ItemKind::Issue,
1839 title: "Route issues".to_string(),
1840 state,
1841 body: " The body.\n".to_string(),
1842 comments: vec![],
1843 closed_by: vec![],
1844 url: format!("https://github.com/rust-works/omni-dev/issues/{number}"),
1845 }
1846 }
1847
1848 fn opts() -> RouteOptions {
1849 RouteOptions {
1850 draft_comments: vec![],
1851 model: "jev-latest".to_string(),
1852 close_call: DEFAULT_CLOSE_CALL,
1853 close_call_margin: DEFAULT_CLOSE_CALL_MARGIN,
1854 effort_advice: false,
1855 max_input_chars: DEFAULT_MAX_INPUT_CHARS,
1856 allow_closed: false,
1857 ignore_closed: false,
1858 }
1859 }
1860
1861 fn default_tiers() -> Tiers {
1862 Provider::Anthropic.tiers().unwrap()
1863 }
1864
1865 fn class_only(provider: Provider) -> Result<Ladder> {
1867 let mut ladder = Ladder::builtin(provider)?;
1868 for tier in &mut ladder.tiers.0 {
1869 tier.models = None;
1870 }
1871 Ok(ladder)
1872 }
1873
1874 fn anthropic() -> Vec<Ladder> {
1875 vec![class_only(Provider::Anthropic).unwrap()]
1876 }
1877
1878 fn tier_names(tiers: &Tiers) -> Vec<&str> {
1879 tiers.as_slice().iter().map(|t| t.name.as_str()).collect()
1880 }
1881
1882 #[test]
1887 fn anthropic_tiers_are_sonnet_opus_in_order() {
1888 let tiers = default_tiers();
1889 assert_eq!(tier_names(&tiers), ["sonnet", "opus"]);
1890 assert_eq!(
1891 tiers.as_slice()[0].models.as_ref().unwrap()[0].name,
1892 "claude-sonnet-5-5"
1893 );
1894 assert_eq!(
1895 tiers.as_slice()[1].models.as_ref().unwrap()[0].name,
1896 "claude-opus-5-5"
1897 );
1898 }
1899
1900 #[test]
1901 fn anthropic_tier_descriptions_are_the_reevaluated_text() {
1902 let tiers = default_tiers();
1903 let sonnet = &tiers.as_slice()[0];
1904 let opus = &tiers.as_slice()[1];
1905 assert_eq!(
1906 sonnet.description,
1907 "Reliable at implementing and reviewing well-scoped work across several files, \
1908 including debugging and non-obvious interactions when the approach is clear. May \
1909 need help with complex, open-ended design requiring sustained judgement."
1910 );
1911 assert_eq!(
1912 opus.description,
1913 "Reliable at complex, open-ended design and research: weighing architectures and \
1914 unfamiliar approaches, anticipating failure modes, and making security-critical \
1915 judgements. Carries subtle multi-file implementation and review through with \
1916 sustained judgement."
1917 );
1918 }
1919
1920 #[test]
1921 fn provider_names_are_the_documented_abbreviations() {
1922 assert_eq!(
1923 tier_names(&Provider::OpenAi.tiers().unwrap()),
1924 ["terra", "sol", "astra"]
1925 );
1926 assert_eq!(
1927 tier_names(&Provider::Gemini.tiers().unwrap()),
1928 ["flash", "pro", "deep-think"]
1929 );
1930 assert_eq!(
1931 Provider::ALL.map(Provider::name),
1932 ["anthropic", "openai", "gemini"]
1933 );
1934 for provider in Provider::ALL {
1935 assert_eq!(
1936 provider.to_possible_value().unwrap().get_name(),
1937 provider.name()
1938 );
1939 }
1940 }
1941
1942 #[test]
1956 fn openai_and_gemini_ladders_reuse_the_anthropic_descriptions_rung_for_rung() {
1957 let anthropic = default_tiers();
1958 const RETIRED_ANTHROPIC_THIRD_RUNG: &str =
1959 "Strongest at open-ended design and research: choosing between architectures, \
1960 working in unfamiliar territory with no precedent in the codebase, anticipating \
1961 failure modes, and security-critical judgement.";
1962 for provider in [Provider::OpenAi, Provider::Gemini] {
1963 let tiers = provider.tiers().unwrap();
1964 assert_eq!(tiers.as_slice().len(), 3);
1965 for (rung, reference) in tiers.as_slice().iter().zip(anthropic.as_slice()) {
1966 assert_eq!(
1967 rung.description,
1968 reference.description,
1969 "{}: {} differs from {}",
1970 provider.name(),
1972 rung.name,
1973 reference.name
1974 );
1975 }
1976 assert_eq!(
1977 tiers.as_slice()[2].description,
1978 RETIRED_ANTHROPIC_THIRD_RUNG,
1979 "{}: {} no longer matches the retired anthropic fable rung",
1980 provider.name(), tiers.as_slice()[2].name );
1983 }
1984 }
1985
1986 #[test]
1987 fn ladders_are_named_by_provider_or_a_custom_name() {
1988 let ladder = class_only(Provider::Gemini).unwrap();
1989 assert_eq!(ladder.name, "gemini");
1990 let custom = Ladder::named("mine".to_string(), default_tiers());
1991 assert_eq!(custom.name, "mine");
1992 assert_eq!(custom.tiers, default_tiers());
1993 }
1994
1995 #[test]
1996 fn tiers_need_at_least_two() {
1997 let err = Tiers::parse("tiers:\n - {name: a, description: A}\n").unwrap_err();
1998 assert!(err.to_string().contains("at least 2 tiers"), "{err}");
1999 }
2000
2001 #[test]
2002 fn tiers_reject_the_reserved_none() {
2003 let err = Tiers::parse(
2004 "tiers:\n - {name: a, description: A}\n - {name: none, description: B}\n",
2005 )
2006 .unwrap_err();
2007 assert!(err.to_string().contains("reserved"), "{err}");
2008 }
2009
2010 #[test]
2011 fn tiers_reject_duplicate_names() {
2012 let err =
2013 Tiers::parse("tiers:\n - {name: a, description: A}\n - {name: a, description: B}\n")
2014 .unwrap_err();
2015 assert!(err.to_string().contains("more than once"), "{err}");
2016 }
2017
2018 #[test]
2019 fn tiers_reject_empty_descriptions() {
2020 let err =
2021 Tiers::parse("tiers:\n - {name: a, description: A}\n - {name: b, description: ''}\n")
2022 .unwrap_err();
2023 assert!(err.to_string().contains("non-empty"), "{err}");
2024 }
2025
2026 #[test]
2027 fn tiers_reject_unknown_fields() {
2028 let err = Tiers::parse(
2029 "tiers:\n - {name: a, description: A, rank: 1}\n - {name: b, description: B}\n",
2030 )
2031 .unwrap_err();
2032 assert!(format!("{err:#}").contains("unknown field"), "{err:#}");
2033 }
2034
2035 #[test]
2036 fn tiers_load_reads_a_custom_file() {
2037 let dir = tempfile::tempdir().unwrap();
2038 let path = dir.path().join("tiers.yaml");
2039 std::fs::write(
2040 &path,
2041 "tiers:\n - {name: small, description: S}\n - {name: big, description: B}\n",
2042 )
2043 .unwrap();
2044 let tiers = Tiers::load_file(&path).unwrap();
2045 assert_eq!(tiers.as_slice()[1].name, "big");
2046 }
2047
2048 #[test]
2049 fn tiers_load_names_a_missing_file() {
2050 let err = Tiers::load_file(Path::new("/no/such/tiers.yaml")).unwrap_err();
2051 assert!(err.to_string().contains("Failed to read tiers file"));
2052 }
2053
2054 const BAR: &str = " Choose the least capable class likely to complete this stage correctly \
2057 with no rework, about 9 times in 10. Judge the work that remains given \
2058 the text, not the size of the text.";
2059
2060 fn instructions(questions: &BTreeMap<String, Question>, key: &str) -> String {
2061 let Question::Choice { instructions, .. } = &questions[key] else {
2062 panic!("{key} is not a choice");
2063 };
2064 instructions.clone()
2065 }
2066
2067 fn options(questions: &BTreeMap<String, Question>, key: &str) -> Vec<String> {
2068 let Question::Choice { criteria, .. } = &questions[key] else {
2069 panic!("{key} is not a choice");
2070 };
2071 criteria.keys().cloned().collect()
2072 }
2073
2074 #[test]
2077 fn stage_questions_are_the_tested_wording() {
2078 let questions = build_route_questions(&anthropic()).unwrap();
2079 let tiers = default_tiers();
2080 for stage in ["stage_design", "stage_implement", "stage_review"] {
2081 let Question::Choice { criteria, .. } = &questions[&format!("anthropic.{stage}")]
2082 else {
2083 panic!("{stage} is not a choice");
2084 };
2085 for tier in tiers.as_slice() {
2086 assert_eq!(
2087 criteria[&tier.name], tier.description,
2088 "{stage}: {}",
2089 tier.name
2090 );
2091 }
2092 }
2093 assert_eq!(
2094 instructions(&questions, "anthropic.stage_design"),
2095 format!(
2096 "Which class should do the design work that remains before implementation can \
2097 start: choosing the approach, settling open questions, and writing a plan?{BAR}"
2098 )
2099 );
2100 assert_eq!(
2101 instructions(&questions, "anthropic.stage_implement"),
2102 format!(
2103 "Assume any remaining design work has been completed well. Which class should \
2104 write the code, tests and docs?{BAR}"
2105 )
2106 );
2107 assert_eq!(
2108 instructions(&questions, "anthropic.stage_review"),
2109 format!(
2110 "Which class should review the finished change before merge, so that mistakes \
2111 the automated tests would miss are caught?{BAR}"
2112 )
2113 );
2114 }
2115
2116 #[test]
2117 fn only_the_design_stage_offers_none() {
2118 let questions = build_route_questions(&anthropic()).unwrap();
2119 assert_eq!(
2120 options(&questions, "anthropic.stage_design"),
2121 ["none", "opus", "sonnet"]
2122 );
2123 assert_eq!(
2124 options(&questions, "anthropic.stage_implement"),
2125 ["opus", "sonnet"]
2126 );
2127 assert_eq!(
2128 options(&questions, "anthropic.stage_review"),
2129 ["opus", "sonnet"]
2130 );
2131 let Question::Choice { criteria, .. } = &questions["anthropic.stage_design"] else {
2132 panic!();
2133 };
2134 assert_eq!(
2135 criteria[NO_DESIGN],
2136 "No design work remains: the text already settles the approach and the open \
2137 questions."
2138 );
2139 }
2140
2141 #[test]
2144 fn questions_are_keyed_per_provider_in_one_map() {
2145 let ladders = [
2146 class_only(Provider::Anthropic).unwrap(),
2147 class_only(Provider::OpenAi).unwrap(),
2148 ];
2149 let questions = build_route_questions(&ladders).unwrap();
2150 let keys: Vec<&String> = questions.keys().collect();
2151 assert_eq!(
2152 keys,
2153 [
2154 "anthropic.stage_design",
2155 "anthropic.stage_implement",
2156 "anthropic.stage_review",
2157 "open_questions",
2158 "openai.stage_design",
2159 "openai.stage_implement",
2160 "openai.stage_review",
2161 ]
2162 );
2163 assert_eq!(
2164 options(&questions, "openai.stage_design"),
2165 ["astra", "none", "sol", "terra"]
2166 );
2167 assert_eq!(
2168 instructions(&questions, "openai.stage_review"),
2169 instructions(&questions, "anthropic.stage_review")
2170 );
2171 }
2172
2173 #[test]
2174 fn a_named_ladder_is_keyed_by_its_own_name() {
2175 let ladders = [Ladder::named(
2176 "mine".to_string(),
2177 Tiers::parse("tiers:\n - {name: a, description: A}\n - {name: b, description: B}\n")
2178 .unwrap(),
2179 )];
2180 let questions = build_route_questions(&ladders).unwrap();
2181 assert_eq!(options(&questions, "mine.stage_implement"), ["a", "b"]);
2182 }
2183
2184 #[test]
2187 fn route_state_without_comments_is_title_and_trimmed_body() {
2188 let (state, truncated) = build_route_state(&doc(7, ItemState::Open), 1000);
2189 assert_eq!(state, "# #7 Route issues\n\nThe body.\n");
2190 assert!(!truncated);
2191 }
2192
2193 #[test]
2194 fn route_state_lists_comments_in_order_in_the_tested_format() {
2195 let mut d = doc(7, ItemState::Open);
2196 d.comments = vec![
2197 Comment {
2198 author: "alice".to_string(),
2199 body: "first\n".to_string(),
2200 id: None,
2201 },
2202 Comment {
2203 author: "bob".to_string(),
2204 body: "second".to_string(),
2205 id: None,
2206 },
2207 ];
2208 let (state, _) = build_route_state(&d, 1000);
2209 assert_eq!(
2210 state,
2211 "# #7 Route issues\n\nThe body.\n\n\n---\n\n## Comments on this issue\n\
2212 \n\n**Comment by alice:**\n\nfirst\n\
2213 \n\n**Comment by bob:**\n\nsecond\n"
2214 );
2215 }
2216
2217 #[test]
2218 fn route_state_truncates_the_middle_on_a_char_boundary() {
2219 let mut d = doc(7, ItemState::Open);
2220 d.body = "é".repeat(100);
2221 let (full, _) = build_route_state(&d, usize::MAX);
2222 let (state, truncated) = build_route_state(&d, 20);
2223 assert!(truncated);
2224 let (head, tail) = state
2225 .split_once(&format!("\n\n{TRUNCATION_MARKER}\n\n"))
2226 .unwrap();
2227 assert_eq!(head.chars().count(), 10);
2228 assert_eq!(tail.chars().count(), 10);
2229 assert!(full.starts_with(head));
2230 assert!(full.ends_with(tail));
2231 }
2232
2233 fn comment(author: &str, body: &str) -> Comment {
2234 Comment {
2235 author: author.to_string(),
2236 body: body.to_string(),
2237 id: None,
2238 }
2239 }
2240
2241 #[test]
2244 fn citation_text_is_the_state_with_code_masked() {
2245 let mut d = doc(7, ItemState::Open);
2246 d.title = "Fix `o/r#1`".to_string();
2247 d.body = "See #5.\n\n```\nerror: o/r#2\n```".to_string();
2248 d.comments = vec![comment("alice", "inline `o/r#3` and #6")];
2249 let (state, _) = build_route_state(&d, 1000);
2250 let text = build_route_citation_text(&d, 1000);
2251 assert_eq!(text.chars().count(), state.chars().count());
2252 assert!(text.contains("See #5."), "{text}");
2253 assert!(text.contains("and #6"), "{text}");
2254 assert!(text.contains("**Comment by alice:**"), "{text}");
2255 for masked in ["o/r#1", "o/r#2", "o/r#3"] {
2256 assert!(!text.contains(masked), "{masked} survived in {text}");
2257 }
2258 }
2259
2260 #[test]
2261 fn citation_text_is_truncated_where_the_state_is() {
2262 let mut d = doc(7, ItemState::Open);
2263 d.body = format!("start #1 {} `o/r#9` end #2", "x".repeat(500));
2264 let (state, truncated) = build_route_state(&d, 100);
2265 let text = build_route_citation_text(&d, 100);
2266 assert!(truncated);
2267 assert_eq!(text.chars().count(), state.chars().count());
2268 assert!(text.contains(TRUNCATION_MARKER), "{text}");
2269 }
2270
2271 #[test]
2275 fn citation_text_masks_each_comment_on_its_own() {
2276 let mut d = doc(7, ItemState::Open);
2277 d.comments = vec![
2278 comment("alice", "```\nstray fence, never closed\nreal #11"),
2279 comment("bob", "real #12"),
2280 comment("carol", "```\nerror: o/r#9\n```"),
2281 ];
2282 let text = build_route_citation_text(&d, 10_000);
2283 assert!(text.contains("real #11"), "{text}");
2284 assert!(text.contains("real #12"), "{text}");
2285 assert!(!text.contains("o/r#9"), "{text}");
2286 }
2287
2288 #[test]
2290 fn route_state_truncation_keeps_the_latest_comment() {
2291 let mut d = doc(7, ItemState::Open);
2292 d.body = "x".repeat(10_000);
2293 d.comments = vec![Comment {
2294 author: "maintainer".to_string(),
2295 body: "**Decision**: settled.".to_string(),
2296 id: None,
2297 }];
2298 let (state, truncated) = build_route_state(&d, 1_000);
2299 assert!(truncated);
2300 assert!(state.starts_with("# #7 Route issues"));
2301 assert!(state.ends_with("**Decision**: settled.\n"), "{state}");
2302 }
2303
2304 #[test]
2305 fn route_state_truncation_handles_a_one_char_cap() {
2306 let (state, truncated) = build_route_state(&doc(7, ItemState::Open), 1);
2307 assert!(truncated);
2308 assert_eq!(state, format!("\n\n{TRUNCATION_MARKER}\n\n\n"));
2309 }
2310
2311 #[test]
2312 fn route_state_at_exactly_the_cap_is_not_truncated() {
2313 let d = doc(7, ItemState::Open);
2314 let (full, _) = build_route_state(&d, usize::MAX);
2315 let (state, truncated) = build_route_state(&d, full.chars().count());
2316 assert_eq!(state, full);
2317 assert!(!truncated);
2318 }
2319
2320 #[test]
2321 fn drafts_follow_fetched_comments_and_default_input_is_identical() {
2322 let mut d = doc(7, ItemState::Open);
2323 for (author, body) in [("alice", "first"), ("bob", "second")] {
2324 d.comments.push(Comment {
2325 author: author.into(),
2326 body: body.into(),
2327 id: None,
2328 });
2329 }
2330 let drafts = [" third\n\n indented\n".into(), "fourth".into()];
2331 let (state, truncated) = build_route_state_with_drafts(&d, 1000, &drafts);
2332 assert!(!truncated);
2333 assert!(state.ends_with(
2334 "**Comment by draft:**\n\nthird\n\n indented\n\n\n**Comment by draft:**\n\nfourth\n"
2335 ));
2336 assert!(state.find("first").unwrap() < state.find("second").unwrap());
2337 assert!(state.find("second").unwrap() < state.find("third").unwrap());
2338 assert_eq!(
2339 build_route_state_with_drafts(&d, 1000, &[]),
2340 build_route_state(&d, 1000)
2341 );
2342 assert_eq!(
2343 build_route_citation_text_with_drafts(&d, 1000, &[]),
2344 build_route_citation_text(&d, 1000)
2345 );
2346 d.comments.clear();
2347 let (state, _) = build_route_state_with_drafts(&d, 1000, &drafts);
2348 assert!(state.contains("## Comments on this issue"));
2349 }
2350
2351 #[test]
2352 fn draft_truncation_and_code_masking_remain_aligned() {
2353 let d = doc(7, ItemState::Open);
2354 let drafts = [
2355 format!("`#99` {}\n```\n#88\n```", "é".repeat(200)),
2356 "See #2 at the tail".into(),
2357 ];
2358 let (full, _) = build_route_state_with_drafts(&d, usize::MAX, &drafts);
2359 let masked = build_route_citation_text_with_drafts(&d, usize::MAX, &drafts);
2360 assert_eq!(full.chars().count(), masked.chars().count());
2361 assert!(!masked.contains("#99"));
2362 assert!(!masked.contains("#88"));
2363 assert!(masked.contains("#2"));
2364 let (cut, truncated) = build_route_state_with_drafts(&d, 120, &drafts);
2365 let masked_cut = build_route_citation_text_with_drafts(&d, 120, &drafts);
2366 assert!(truncated);
2367 assert!(cut.ends_with("See #2 at the tail\n"));
2368 assert_eq!(cut.chars().count(), masked_cut.chars().count());
2369 let marker = crate::jev::input::TRUNCATION_MARKER;
2370 let before = cut.split_once(marker).unwrap().0.chars().count();
2371 let masked_before = masked_cut.split_once(marker).unwrap().0.chars().count();
2372 assert_eq!(before, masked_before);
2373 assert!(cut.find(marker).unwrap() < cut.find("See #2").unwrap());
2374 let isolated =
2375 build_route_citation_text_with_drafts(&d, 1000, &["`start".into(), "#2 end`".into()]);
2376 assert!(isolated.contains("#2"));
2377 }
2378
2379 #[tokio::test]
2380 async fn drafts_reach_requests_and_batch_guard_precedes_closed_filtering() {
2381 let server = wiremock::MockServer::start().await;
2382 wiremock::Mock::given(wiremock::matchers::method("POST"))
2383 .respond_with(wiremock::ResponseTemplate::new(200).set_body_json(routed_json()))
2384 .expect(1)
2385 .mount(&server)
2386 .await;
2387 let client = JevClient::new(&server.uri(), "key").unwrap();
2388 let mut options = opts();
2389 options.draft_comments = vec!["Blocked on #1129".into()];
2390 let dependencies = OpenDependencies::from([(
2391 ("rust-works/omni-dev".into(), 7),
2392 vec![citation("#1129", 1129)],
2393 )]);
2394 run_route(
2395 &client,
2396 &[doc(7, ItemState::Open)],
2397 &anthropic(),
2398 &options,
2399 &dependencies,
2400 )
2401 .await
2402 .unwrap();
2403 options.ignore_closed = true;
2404 let err = run_route(
2405 &client,
2406 &[doc(7, ItemState::Open), doc(8, ItemState::Closed)],
2407 &anthropic(),
2408 &options,
2409 &dependencies,
2410 )
2411 .await
2412 .unwrap_err();
2413 assert!(err.to_string().contains("exactly one issue"));
2414 let requests = server.received_requests().await.unwrap();
2415 assert_eq!(requests.len(), 1);
2416 let body: serde_json::Value = requests[0].body_json().unwrap();
2417 assert!(body["state"]
2418 .as_str()
2419 .unwrap()
2420 .ends_with("**Comment by draft:**\n\nBlocked on #1129\n"));
2421 assert!(body["questions"].get("could_be_cheaper_0").is_some());
2422 let expected = build_route_questions_for_mode(&anthropic(), false).unwrap();
2423 for (key, question) in expected {
2424 assert_eq!(
2425 body["questions"][key],
2426 serde_json::to_value(question).unwrap()
2427 );
2428 }
2429 }
2430
2431 #[test]
2432 fn preview_provenance_is_visible_in_all_formats_including_failures() {
2433 let mut issue = IssueRoute {
2434 draft_comments: vec!["triage draft.md".into(), "scope.md".into()],
2435 item_ref: "o/r#1".into(),
2436 url: "u".into(),
2437 title: "t".into(),
2438 state: ItemState::Open,
2439 truncated: false,
2440 outcome: RouteOutcome::Failed {
2441 error: "failure".into(),
2442 reference_fetch_failures: vec![],
2443 },
2444 };
2445 for outcome in [
2446 issue.outcome.clone(),
2447 RouteOutcome::Routed {
2448 open_questions: Some(test_open_questions()),
2449 providers: BTreeMap::new(),
2450 depends_on: vec![],
2451 reference_fetch_failures: vec![],
2452 },
2453 ] {
2454 issue.outcome = outcome;
2455 let json = serde_json::to_value(&issue).unwrap();
2456 assert_eq!(
2457 json["draft_comments"],
2458 serde_json::json!(["triage draft.md", "scope.md"])
2459 );
2460 let yaml: serde_yaml::Value =
2461 serde_yaml::from_str(&serde_yaml::to_string(&issue).unwrap()).unwrap();
2462 assert_eq!(yaml["draft_comments"][0].as_str(), Some("triage draft.md"));
2463 let text = render_issue_block(&issue, 1000, &[], TerminalStyle::default());
2464 assert!(text.contains("draft comments (preview): [\"triage draft.md\", \"scope.md\"]"));
2465 }
2466 issue.draft_comments.clear();
2467 assert!(serde_json::to_value(&issue)
2468 .unwrap()
2469 .get("draft_comments")
2470 .is_none());
2471 assert!(
2472 !render_issue_block(&issue, 1000, &[], TerminalStyle::default()).contains("preview")
2473 );
2474 }
2475
2476 fn answer(choice: &str, confidence: f64) -> StageAnswer {
2479 StageAnswer {
2480 effort_by_model: BTreeMap::new(),
2481 choice: choice.to_string(),
2482 confidence,
2483 probabilities: BTreeMap::new(),
2484 }
2485 }
2486
2487 fn stages(design: &str, implement: &str, review: &str) -> StageAnswers {
2488 StageAnswers {
2489 design: answer(design, 0.9),
2490 implement: answer(implement, 0.9),
2491 review: answer(review, 0.9),
2492 }
2493 }
2494
2495 #[test]
2496 fn close_call_shows_the_best_alternative_in_compact_text() {
2497 let mut stages = stages("sol", "terra", "sol");
2498 stages.design.confidence = 0.21;
2499 stages.design.probabilities = BTreeMap::from([
2500 ("none".to_string(), 0.44),
2501 ("sol".to_string(), 0.48),
2502 ("terra".to_string(), 0.08),
2503 ]);
2504
2505 assert_eq!(
2506 render_stage_clause(
2507 Stage::Design,
2508 &stages,
2509 &[Stage::Design],
2510 None,
2511 TerminalStyle::default(),
2512 ),
2513 "design needs sol (0.21, close call — none 0.44)"
2514 );
2515 assert_eq!(
2516 render_stage_clause(Stage::Design, &stages, &[], None, TerminalStyle::default(),),
2517 "design needs sol (0.21)"
2518 );
2519 }
2520
2521 #[test]
2522 fn close_call_uses_a_stable_tiebreaker_and_handles_missing_alternatives() {
2523 let mut answer = answer("sol", 0.21);
2524 answer.probabilities = BTreeMap::from([
2525 ("astra".to_string(), 0.4),
2526 ("sol".to_string(), 0.2),
2527 ("terra".to_string(), 0.4),
2528 ]);
2529 assert_eq!(
2530 render_stage_confidence(&answer, true),
2531 "0.21, close call — astra 0.40"
2532 );
2533 answer.probabilities.clear();
2534 assert_eq!(render_stage_confidence(&answer, true), "0.21, close call");
2535 }
2536
2537 #[test]
2538 fn design_supplies_the_class_when_it_outranks_implement() {
2539 let tiers = default_tiers();
2540 assert_eq!(
2541 issue_class(&stages("opus", "sonnet", "sonnet"), &tiers),
2542 ("opus".to_string(), ClassSource::Design)
2543 );
2544 }
2545
2546 #[test]
2547 fn implement_supplies_the_class_when_it_outranks_design() {
2548 let tiers = default_tiers();
2549 assert_eq!(
2550 issue_class(&stages("sonnet", "opus", "sonnet"), &tiers),
2551 ("opus".to_string(), ClassSource::Implement)
2552 );
2553 }
2554
2555 #[test]
2558 fn a_tie_goes_to_implement() {
2559 let tiers = default_tiers();
2560 for tier in ["sonnet", "opus"] {
2561 assert_eq!(
2562 issue_class(&stages(tier, tier, "sonnet"), &tiers),
2563 (tier.to_string(), ClassSource::Implement),
2564 "{tier}"
2565 );
2566 }
2567 }
2568
2569 #[test]
2570 fn class_ignores_review_and_none() {
2571 let tiers = default_tiers();
2572 assert_eq!(
2573 issue_class(&stages("none", "sonnet", "opus"), &tiers),
2574 ("sonnet".to_string(), ClassSource::Implement)
2575 );
2576 }
2577
2578 #[test]
2579 fn close_calls_are_below_the_threshold_in_stage_order() {
2580 let mut s = stages("opus", "sonnet", "opus");
2581 s.review.confidence = 0.1;
2582 s.design.confidence = 0.29;
2583 s.implement.confidence = 0.3;
2584 assert_eq!(
2585 close_calls(&s, &default_tiers(), 0.3, DEFAULT_CLOSE_CALL_MARGIN),
2586 [Stage::Design, Stage::Review]
2587 );
2588 }
2589
2590 fn distribution(entries: &[(&str, f64)]) -> BTreeMap<String, f64> {
2591 entries
2592 .iter()
2593 .map(|(name, p)| ((*name).into(), *p))
2594 .collect()
2595 }
2596
2597 #[test]
2598 fn close_call_flags_3017_split_and_preserves_class() {
2599 let mut s = stages("none", "sonnet", "opus");
2600 s.implement.confidence = 0.32;
2601 s.implement.probabilities = distribution(&[("sonnet", 0.55), ("opus", 0.45)]);
2602 assert_eq!(
2603 close_calls(&s, &default_tiers(), 0.3, 0.2),
2604 [Stage::Implement]
2605 );
2606 assert!(close_calls(&s, &default_tiers(), 0.3, 0.0).is_empty());
2607 assert_eq!(
2608 issue_class(&s, &default_tiers()),
2609 ("sonnet".into(), ClassSource::Implement)
2610 );
2611 }
2612
2613 #[test]
2614 fn close_call_predicates_are_strict_independent_and_ordered() {
2615 let tiers = default_tiers();
2616 let mut s = stages("sonnet", "sonnet", "sonnet");
2617 for (confidence, gap, expected) in [
2618 (0.25, 0.25, false), (0.125, 0.5, true), (0.5, 0.125, true), (0.125, 0.125, true), (0.5, 0.5, false), ] {
2624 s.design.confidence = confidence;
2625 s.design.probabilities = distribution(&[("sonnet", 0.5 + gap), ("opus", 0.5)]);
2626 assert_eq!(
2627 close_calls(&s, &tiers, 0.25, 0.25).contains(&Stage::Design),
2628 expected
2629 );
2630 }
2631 s.design.confidence = 0.1;
2632 s.design.probabilities = distribution(&[("sonnet", 0.5), ("opus", 0.5)]);
2633 s.implement.probabilities = s.design.probabilities.clone();
2634 s.review.probabilities = s.design.probabilities.clone();
2635 assert_eq!(close_calls(&s, &tiers, 0.3, 0.2), Stage::ALL);
2636 assert_eq!(close_calls(&s, &tiers, 0.3, 0.0), [Stage::Design]);
2637 s.design.confidence = 0.9;
2638 s.design.probabilities = distribution(&[("sonnet", 1.0), ("opus", 0.0)]);
2639 assert!(!close_calls(&s, &tiers, 0.3, 1.0).contains(&Stage::Design));
2640 }
2641
2642 #[test]
2643 fn margin_uses_only_finite_offered_probabilities_without_imputation() {
2644 let tiers = default_tiers();
2645 let mut a = answer("sonnet", 0.9);
2646 assert_eq!(top_two_margin(&a, Stage::Design, &tiers), None);
2647 a.probabilities = distribution(&[("sonnet", 0.5)]);
2648 assert_eq!(top_two_margin(&a, Stage::Design, &tiers), None);
2649 for invalid in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY, -0.1, 1.1] {
2650 a.probabilities = distribution(&[("sonnet", 0.5), ("opus", invalid), ("unknown", 0.5)]);
2651 assert_eq!(top_two_margin(&a, Stage::Implement, &tiers), None);
2652 }
2653 a.probabilities = distribution(&[("none", 0.5), ("sonnet", 0.5), ("unknown", 0.5)]);
2654 assert_eq!(top_two_margin(&a, Stage::Design, &tiers), Some(0.0));
2655 for stage in [Stage::Implement, Stage::Review] {
2656 assert_eq!(top_two_margin(&a, stage, &tiers), None);
2657 }
2658 a.probabilities = distribution(&[("none", 0.375), ("opus", 0.25)]);
2660 assert_eq!(top_two_margin(&a, Stage::Design, &tiers), Some(0.125));
2661 let openai = Provider::OpenAi.tiers().unwrap();
2663 a.choice = "terra".into();
2664 a.probabilities = distribution(&[("terra", 0.0), ("sol", 0.5), ("astra", 0.5)]);
2665 assert_eq!(top_two_margin(&a, Stage::Implement, &openai), Some(0.0));
2666 }
2667
2668 #[test]
2669 fn invalid_probabilities_retain_existing_confidence_behavior() {
2670 let mut s = stages("none", "sonnet", "opus");
2671 s.design.confidence = 0.1;
2672 s.design.probabilities = distribution(&[("sonnet", f64::NAN), ("opus", -1.0)]);
2673 s.implement.confidence = f64::NAN;
2674 s.review.confidence = f64::NEG_INFINITY;
2675 assert_eq!(
2676 close_calls(&s, &default_tiers(), 0.3, 0.2),
2677 [Stage::Design, Stage::Review]
2678 );
2679 s.implement.probabilities = distribution(&[("sonnet", 0.5), ("opus", 0.5)]);
2680 assert_eq!(close_calls(&s, &default_tiers(), 0.3, 0.2), Stage::ALL);
2681 }
2682
2683 #[test]
2684 fn custom_and_multiple_ladders_share_flags_across_output_formats() {
2685 let ladders = [
2686 Ladder::named("custom".into(), Tiers::parse("tiers:\n - name: small,alternative\n description: Local work\n - name: large\n description: Complex work\n").unwrap()),
2687 class_only(Provider::Anthropic).unwrap(),
2688 ];
2689 let mut answers = BTreeMap::new();
2690 for ladder in &ladders {
2691 let first = &ladder.tiers.as_slice()[0].name;
2692 let second = &ladder.tiers.as_slice()[1].name;
2693 for stage in Stage::ALL {
2694 let (p1, p2) = if ladder.name == "custom" && stage == Stage::Implement {
2695 (0.55, 0.45)
2696 } else {
2697 (0.9, 0.1)
2698 };
2699 answers.insert(
2700 stage.question_key(&ladder.name),
2701 Answer::Choice {
2702 choice: first.clone(),
2703 confidence: 0.32,
2704 probabilities: distribution(&[(first, p1), (second, p2)]),
2705 },
2706 );
2707 }
2708 }
2709 let routes = provider_routes(&answers, &ladders, 0.3, 0.2, false).unwrap();
2710 assert!(routes["anthropic"].close_calls.is_empty());
2711 let route = &routes["custom"];
2712 assert_eq!(route.close_calls, [Stage::Implement]);
2713 let json = serde_json::to_value(route).unwrap();
2714 let yaml: serde_json::Value =
2715 serde_yaml::from_str(&serde_yaml::to_string(route).unwrap()).unwrap();
2716 assert_eq!(json, yaml);
2717 assert_eq!(json["close_calls"], serde_json::json!(["implement"]));
2718 for stage in Stage::ALL {
2719 let clause = render_stage_clause(
2720 stage,
2721 &route.stages,
2722 &route.close_calls,
2723 None,
2724 TerminalStyle::default(),
2725 );
2726 let line = render_stage_line(
2727 stage,
2728 &route.stages,
2729 &route.close_calls,
2730 Some(&ladders[0]),
2731 TerminalStyle::default(),
2732 );
2733 for text in [clause, line] {
2734 assert_eq!(
2735 text.contains("close call"),
2736 stage == Stage::Implement,
2737 "{text}"
2738 );
2739 if stage == Stage::Implement {
2740 assert!(text.contains("0.32, close call — large 0.45"), "{text}");
2741 }
2742 }
2743 }
2744 let confidence_only = provider_routes(&answers, &ladders, 0.3, 0.0, false).unwrap();
2745 assert!(confidence_only["custom"].close_calls.is_empty());
2746 assert_eq!(route.class, confidence_only["custom"].class);
2747 assert_eq!(route.class_from, confidence_only["custom"].class_from);
2748 }
2749
2750 #[tokio::test]
2751 async fn run_route_applies_the_configured_margin_without_changing_requests() {
2752 let server = wiremock::MockServer::start().await;
2753 let mut implementation = choice_json("sonnet", 0.32);
2754 implementation["probabilities"] = serde_json::json!({"sonnet": 0.55, "opus": 0.45});
2755 wiremock::Mock::given(wiremock::matchers::method("POST"))
2756 .and(wiremock::matchers::path("/v1/systemone"))
2757 .respond_with(
2758 wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!({
2759 "model": "jev-1.13.0", "answers": {
2760 "open_questions": choice_json("factual", 0.8),
2761 "anthropic.stage_design": choice_json("none", 0.9),
2762 "anthropic.stage_implement": implementation,
2763 "anthropic.stage_review": choice_json("opus", 0.9)
2764 }, "usage": {"input_tokens": 100, "output_tokens": 10}
2765 })),
2766 )
2767 .expect(2)
2768 .mount(&server)
2769 .await;
2770 let client = JevClient::new(&server.uri(), "key").unwrap();
2771 for margin in [0.0, DEFAULT_CLOSE_CALL_MARGIN] {
2772 let mut options = opts();
2773 options.close_call_margin = margin;
2774 let report = run_route(
2775 &client,
2776 &[doc(1, ItemState::Open)],
2777 &anthropic(),
2778 &options,
2779 &OpenDependencies::new(),
2780 )
2781 .await
2782 .unwrap();
2783 assert_eq!(
2784 provider(&report.issues[0].outcome, "anthropic")
2785 .close_calls
2786 .contains(&Stage::Implement),
2787 margin > 0.0
2788 );
2789 }
2790 let requests = server.received_requests().await.unwrap();
2791 assert_eq!(requests.len(), 2);
2792 assert_eq!(requests[0].body, requests[1].body);
2793 }
2794
2795 #[tokio::test]
2796 async fn invalid_margins_fail_before_a_jev_request() {
2797 let server = wiremock::MockServer::start().await;
2798 let client = JevClient::new(&server.uri(), "key").unwrap();
2799 for margin in [-0.1, 1.1, f64::NAN, f64::INFINITY, f64::NEG_INFINITY] {
2800 let mut options = opts();
2801 options.close_call_margin = margin;
2802 let err = run_route(
2803 &client,
2804 &[doc(1, ItemState::Open)],
2805 &anthropic(),
2806 &options,
2807 &OpenDependencies::new(),
2808 )
2809 .await
2810 .unwrap_err();
2811 assert!(err.to_string().contains("close-call margin"), "{err}");
2812 }
2813 for margin in [0.0, 1.0] {
2814 let mut options = opts();
2815 options.close_call_margin = margin;
2816 validate_options(&[&doc(1, ItemState::Open)], &anthropic(), &options).unwrap();
2817 }
2818 assert!(server.received_requests().await.unwrap().is_empty());
2819 }
2820
2821 #[test]
2822 fn could_be_cheaper_key_is_stable_and_indexed() {
2823 assert_eq!(could_be_cheaper_key(0), "could_be_cheaper_0");
2824 assert_eq!(could_be_cheaper_key(1), "could_be_cheaper_1");
2825 }
2826
2827 #[test]
2828 fn could_be_cheaper_question_is_the_tested_wording() {
2829 let Question::Noul {
2830 instructions,
2831 criteria,
2832 } = could_be_cheaper_question("#1129")
2833 else {
2834 panic!("expected a noul question");
2835 };
2836 assert_eq!(
2837 instructions,
2838 "This issue cites #1129, which is still open. If #1129 is resolved, how likely is \
2839 it that LESS design work would remain for THIS issue than the current text \
2840 implies — as opposed to this issue's own remaining work being unaffected, because \
2841 it is already scoped separately, is a parallel/sibling effort, or #1129 is \
2842 otherwise not a precondition for finishing this issue's own remaining work?"
2843 );
2844 assert!(criteria.is_none());
2845 }
2846
2847 #[test]
2848 fn dependency_entries_reads_the_noul_answer_per_citation() {
2849 let citations = vec![
2850 Citation {
2851 item_ref: ItemRef {
2852 provider: GitProvider::GitHub,
2853 project: "rust-works/omni-dev".to_string(),
2854 kind: ItemKind::Issue,
2855 number: 1129,
2856 },
2857 raw: "#1129".to_string(),
2858 url: None,
2859 },
2860 Citation {
2861 item_ref: ItemRef {
2862 provider: GitProvider::GitHub,
2863 project: "rust-works/omni-dev".to_string(),
2864 kind: ItemKind::Issue,
2865 number: 1349,
2866 },
2867 raw: "#1349".to_string(),
2868 url: None,
2869 },
2870 ];
2871 let answers = BTreeMap::from([(
2872 "could_be_cheaper_0".to_string(),
2873 Answer::Noul { noul: 0.75 },
2874 )]);
2875 let deps = dependency_entries("o/r#1", &citations, &BTreeMap::new(), &answers);
2876 assert_eq!(deps.len(), 2, "{deps:?}");
2877 assert_eq!(deps[0].item_ref, "#1129");
2878 assert_eq!(deps[0].state, ItemState::Open);
2879 assert_eq!(deps[0].could_be_cheaper.get("design"), Some(&0.75));
2880 assert_eq!(deps[0].relation, CitationRelation::Unspecified);
2881 let json = serde_json::to_value(&deps[0]).unwrap();
2882 assert_eq!(json["relation"], "unspecified");
2883 assert_eq!(json["could_be_cheaper"]["design"], 0.75);
2884 assert_eq!(deps[1].item_ref, "#1349");
2885 assert!(deps[1].could_be_cheaper.is_empty());
2886 assert_eq!(
2887 serde_json::to_value(CitationRelation::Blocker).unwrap(),
2888 "blocker"
2889 );
2890 assert_eq!(
2891 serde_json::to_value(CitationRelation::Tracker).unwrap(),
2892 "tracker"
2893 );
2894 }
2895
2896 fn relation_for(body: &str, target: u64) -> CitationRelation {
2897 let mut issue = doc(9999, ItemState::Open);
2898 issue.body = body.to_string();
2899 citation_relations(body, &issue)
2900 .get(&(issue.project, target))
2901 .copied()
2902 .unwrap_or(CitationRelation::Unspecified)
2903 }
2904
2905 #[test]
2906 fn citation_relation_uses_later_explicit_mentions() {
2907 assert_eq!(
2908 relation_for(
2909 "See #1351 for background. Blocked on node identity in #1351.",
2910 1351
2911 ),
2912 CitationRelation::Blocker
2913 );
2914 assert_eq!(
2915 relation_for("See #1830. Split out of #1830.", 1830),
2916 CitationRelation::Tracker
2917 );
2918 }
2919
2920 #[test]
2921 fn citation_relation_ignores_negation_quotes_and_conflicts() {
2922 for body in [
2923 "Not blocked on #123.",
2924 "For example, \"blocked on #123\" is a blocker claim.",
2925 "> Blocked on #123.",
2926 "```\nBlocked on #123\n```",
2927 "This is such as #123, a worked example.",
2928 "See #123 for context.",
2929 "Blocked on #123. Split out of #123.",
2930 "Blocked on #123. No longer blocked on #123.",
2931 "#123 blocks this issue. #123 no longer blocks it.",
2932 ] {
2933 assert_eq!(
2934 relation_for(body, 123),
2935 CitationRelation::Unspecified,
2936 "{body}"
2937 );
2938 }
2939 }
2940
2941 #[test]
2942 fn citation_relation_real_issue_phrasing() {
2943 assert_eq!(
2944 relation_for(
2945 "Split out of #1830. Blocker #1831 closed as ADR-0083.",
2946 1830
2947 ),
2948 CitationRelation::Tracker
2949 );
2950 assert_eq!(
2951 relation_for("## Triage — blocked on #1129's classifier change.", 1129),
2952 CitationRelation::Blocker
2953 );
2954 assert_eq!(
2955 relation_for(
2956 "Blocked on real alias/node identity — the same work as #1351.",
2957 1351
2958 ),
2959 CitationRelation::Blocker
2960 );
2961 assert_eq!(
2962 relation_for("Absorbed by #1753; neither suggested option remains.", 1753),
2963 CitationRelation::Unspecified
2964 );
2965 assert_eq!(
2966 relation_for("#1831 blocks this issue until its ADR lands.", 1831),
2967 CitationRelation::Blocker
2968 );
2969 assert_eq!(
2970 relation_for("See #1831. It blocks this issue.", 1831),
2971 CitationRelation::Unspecified
2972 );
2973 }
2974
2975 #[test]
2976 fn citation_relations_match_frozen_public_issues() {
2977 for (inputs, labels) in [
2978 (
2979 include_str!("../../docs/evaluations/jev-route-1871/inputs.json"),
2980 include_str!("../../docs/evaluations/jev-route-1871/labels.json"),
2981 ),
2982 (
2983 include_str!("../../docs/evaluations/jev-route-1871/holdout-inputs.json"),
2984 include_str!("../../docs/evaluations/jev-route-1871/holdout-labels.json"),
2985 ),
2986 ] {
2987 let cases: Vec<serde_json::Value> = serde_json::from_str(inputs).unwrap();
2988 let labels: serde_json::Value = serde_json::from_str(labels).unwrap();
2989 for case in cases {
2990 let id = case["id"].as_str().unwrap();
2991 let doc: IssueDoc = serde_json::from_value(case["doc"].clone()).unwrap();
2992 let (state, _) = build_route_state(&doc, DEFAULT_MAX_INPUT_CHARS);
2993 let relations = citation_relations(&state, &doc);
2994 for (number, label) in labels[id]["relations"].as_object().into_iter().flatten() {
2996 let actual = relations
2997 .get(&(doc.project.clone(), number.parse().unwrap()))
2998 .copied()
2999 .unwrap_or(CitationRelation::Unspecified);
3000 let want = [
3001 ("blocker", CitationRelation::Blocker),
3002 ("tracker", CitationRelation::Tracker),
3003 ("unspecified", CitationRelation::Unspecified),
3004 ]
3005 .into_iter()
3006 .find(|(name, _)| Some(*name) == label.as_str())
3007 .expect("a known relation label")
3008 .1;
3009 assert_eq!(actual, want, "{id} -> #{number}");
3010 }
3011 }
3012 }
3013 }
3014
3015 #[test]
3016 fn stage_answers_reject_none_outside_design() {
3017 let choice = |c: &str| Answer::Choice {
3018 choice: c.to_string(),
3019 confidence: 0.5,
3020 probabilities: BTreeMap::new(),
3021 };
3022 let answers = BTreeMap::from([
3023 ("anthropic.stage_design".to_string(), choice("none")),
3024 ("anthropic.stage_implement".to_string(), choice("none")),
3025 ("anthropic.stage_review".to_string(), choice("opus")),
3026 ]);
3027 let err = stage_answers(&answers, &anthropic()[0]).unwrap_err();
3028 assert!(err.to_string().contains("not offered"), "{err}");
3029 }
3030
3031 #[test]
3032 fn stage_answers_require_a_choice_per_stage() {
3033 let answers = BTreeMap::from([(
3034 "anthropic.stage_design".to_string(),
3035 Answer::Noul { noul: 0.5 },
3036 )]);
3037 let err = stage_answers(&answers, &anthropic()[0]).unwrap_err();
3038 assert!(err.to_string().contains("no choice answer"), "{err}");
3039 assert!(err.to_string().contains("anthropic.stage_design"), "{err}");
3040 }
3041
3042 #[test]
3045 fn stage_answers_are_read_under_the_ladders_prefix() {
3046 let choice = |c: &str| Answer::Choice {
3047 choice: c.to_string(),
3048 confidence: 0.5,
3049 probabilities: BTreeMap::new(),
3050 };
3051 let answers = BTreeMap::from([
3052 ("anthropic.stage_design".to_string(), choice("none")),
3053 ("anthropic.stage_implement".to_string(), choice("sonnet")),
3054 ("anthropic.stage_review".to_string(), choice("opus")),
3055 ]);
3056 let err = stage_answers(&answers, &class_only(Provider::OpenAi).unwrap()).unwrap_err();
3057 assert!(err.to_string().contains("openai.stage_design"), "{err}");
3058 }
3059
3060 #[test]
3063 fn closed_issues_are_refused_by_default() {
3064 let err =
3065 validate_options(&[&doc(1, ItemState::Closed)], &anthropic(), &opts()).unwrap_err();
3066 assert!(err.to_string().contains("rust-works/omni-dev#1"), "{err}");
3067 assert!(err.to_string().contains("--allow-closed"), "{err}");
3068 assert!(err.to_string().contains("--ignore-closed"), "{err}");
3069 }
3070
3071 #[test]
3074 fn allow_closed_and_ignore_closed_are_mutually_exclusive() {
3075 let mut o = opts();
3076 o.allow_closed = true;
3077 o.ignore_closed = true;
3078 for state in [ItemState::Open, ItemState::Closed] {
3079 let err = drop_closed(&[doc(1, state)], &o).unwrap_err();
3080 assert!(err.to_string().contains("cannot be used together"), "{err}");
3081 }
3082 }
3083
3084 #[test]
3087 fn drop_closed_keeps_every_issue_without_the_flag() {
3088 let docs = [doc(1, ItemState::Open), doc(2, ItemState::Closed)];
3089 let (kept, ignored) = drop_closed(&docs, &opts()).unwrap();
3090 assert_eq!(kept.len(), 2);
3091 assert!(ignored.is_empty());
3092 }
3093
3094 #[test]
3095 fn drop_closed_skips_closed_issues_in_request_order() {
3096 let mut o = opts();
3097 o.ignore_closed = true;
3098 let docs = [
3099 doc(3, ItemState::Closed),
3100 doc(1, ItemState::Open),
3101 doc(2, ItemState::Closed),
3102 ];
3103 let (kept, ignored) = drop_closed(&docs, &o).unwrap();
3104 assert_eq!(kept.iter().map(|d| d.number).collect::<Vec<_>>(), [1]);
3105 assert_eq!(
3106 ignored,
3107 [
3108 IgnoredIssue {
3109 item_ref: "rust-works/omni-dev#3".to_string(),
3110 url: "https://github.com/rust-works/omni-dev/issues/3".to_string(),
3111 title: "Route issues".to_string(),
3112 },
3113 IgnoredIssue {
3114 item_ref: "rust-works/omni-dev#2".to_string(),
3115 url: "https://github.com/rust-works/omni-dev/issues/2".to_string(),
3116 title: "Route issues".to_string(),
3117 },
3118 ]
3119 );
3120 }
3121
3122 #[test]
3123 fn drop_closed_refuses_a_batch_with_no_open_issue_left() {
3124 let mut o = opts();
3125 o.ignore_closed = true;
3126 let docs = [doc(1, ItemState::Closed), doc(2, ItemState::Closed)];
3127 let err = drop_closed(&docs, &o).unwrap_err();
3128 let msg = err.to_string();
3129 assert!(
3130 msg.contains("rust-works/omni-dev#1, rust-works/omni-dev#2"),
3131 "{msg}"
3132 );
3133 assert!(msg.contains("nothing to route"), "{msg}");
3134 }
3135
3136 #[test]
3139 fn drop_closed_passes_an_empty_batch_through() {
3140 let mut o = opts();
3141 o.ignore_closed = true;
3142 let (kept, ignored) = drop_closed(&[], &o).unwrap();
3143 assert!(kept.is_empty());
3144 assert!(ignored.is_empty());
3145 }
3146
3147 #[test]
3148 fn closed_issues_are_allowed_on_request() {
3149 let mut o = opts();
3150 o.allow_closed = true;
3151 validate_options(&[&doc(1, ItemState::Closed)], &anthropic(), &o).unwrap();
3152 }
3153
3154 #[test]
3155 fn empty_runs_and_bad_knobs_are_rejected() {
3156 assert!(validate_options(&[], &anthropic(), &opts()).is_err());
3157 let mut o = opts();
3158 o.close_call = 1.5;
3159 assert!(validate_options(&[&doc(1, ItemState::Open)], &anthropic(), &o).is_err());
3160 let mut o = opts();
3161 o.max_input_chars = 0;
3162 assert!(validate_options(&[&doc(1, ItemState::Open)], &anthropic(), &o).is_err());
3163 }
3164
3165 #[test]
3166 fn empty_and_duplicate_providers_are_rejected() {
3167 let err = validate_options(&[&doc(1, ItemState::Open)], &[], &opts()).unwrap_err();
3168 assert!(err.to_string().contains("no ladders"), "{err}");
3169 let twice = [
3170 class_only(Provider::Gemini).unwrap(),
3171 class_only(Provider::Gemini).unwrap(),
3172 ];
3173 let err = validate_options(&[&doc(1, ItemState::Open)], &twice, &opts()).unwrap_err();
3174 assert!(
3175 err.to_string()
3176 .contains("\"gemini\" is listed more than once"),
3177 "{err}"
3178 );
3179 }
3180
3181 #[test]
3184 fn open_questions_uses_the_evaluated_wording() {
3185 let evaluated: serde_json::Value = serde_json::from_str(include_str!(
3186 "../../docs/evaluations/jev-route-2053/candidate-v3.json"
3187 ))
3188 .unwrap();
3189 assert_eq!(
3190 serde_json::to_value(open_questions_question()).unwrap(),
3191 evaluated
3192 );
3193 }
3194
3195 #[test]
3196 fn open_questions_is_once_per_issue_in_both_modes() {
3197 let ladders = [
3198 Ladder::builtin(Provider::Anthropic).unwrap(),
3199 Ladder::builtin(Provider::OpenAi).unwrap(),
3200 ];
3201 for effort in [false, true] {
3202 let questions = build_route_questions_for_mode(&ladders, effort).unwrap();
3203 assert_eq!(
3204 options(&questions, "open_questions"),
3205 ["both", "design", "factual", "none"]
3206 );
3207 assert_eq!(
3208 questions
3209 .keys()
3210 .filter(|key| key.contains("open_questions"))
3211 .count(),
3212 1
3213 );
3214 assert_eq!(questions["open_questions"], open_questions_question());
3215 }
3216 }
3217
3218 #[test]
3219 fn open_questions_parses_all_kinds_without_changing_class() {
3220 let ladder = class_only(Provider::Anthropic).unwrap();
3221 for (label, kind) in [
3222 ("none", OpenQuestionKind::None),
3223 ("factual", OpenQuestionKind::Factual),
3224 ("design", OpenQuestionKind::Design),
3225 ("both", OpenQuestionKind::Both),
3226 ] {
3227 let choice = |label: &str| Answer::Choice {
3228 choice: label.into(),
3229 confidence: 0.8,
3230 probabilities: BTreeMap::from([(label.into(), 0.8)]),
3231 };
3232 let answers = BTreeMap::from([
3233 ("open_questions".into(), choice(label)),
3234 ("anthropic.stage_design".into(), choice("none")),
3235 ("anthropic.stage_implement".into(), choice("sonnet")),
3236 ("anthropic.stage_review".into(), choice("opus")),
3237 ]);
3238 let answer = open_questions_answer(&answers).unwrap();
3239 assert_eq!(answer.choice, kind);
3240 let routes =
3241 provider_routes(&answers, std::slice::from_ref(&ladder), 0.3, 0.2, false).unwrap();
3242 assert_eq!(routes["anthropic"].class, "sonnet");
3243 assert_eq!(routes["anthropic"].class_from, ClassSource::Implement);
3244 let issue = IssueRoute {
3245 draft_comments: vec![],
3246 item_ref: "o/r#1".into(),
3247 url: String::new(),
3248 title: "Example".into(),
3249 state: ItemState::Open,
3250 truncated: false,
3251 outcome: RouteOutcome::Routed {
3252 open_questions: Some(answer),
3253 providers: routes,
3254 depends_on: vec![],
3255 reference_fetch_failures: vec![],
3256 },
3257 };
3258 let json = serde_json::to_value(&issue).unwrap();
3259 assert_eq!(json["open_questions"]["choice"], label);
3260 assert_eq!(json["open_questions"]["confidence"], 0.8);
3261 assert_eq!(json["open_questions"]["probabilities"][label], 0.8);
3262 let yaml: serde_yaml::Value =
3263 serde_yaml::from_str(&serde_yaml::to_string(&issue).unwrap()).unwrap();
3264 assert_eq!(yaml["open_questions"]["choice"].as_str(), Some(label));
3265 assert!(
3266 render_issue_block(&issue, 1000, &[], TerminalStyle::default())
3267 .contains(&format!(" open_questions: {label} (0.80)"))
3268 );
3269 }
3270 }
3271
3272 #[test]
3273 fn open_questions_rejects_absent_wrong_type_and_unknown_labels() {
3274 assert!(open_questions_answer(&BTreeMap::new())
3275 .unwrap_err()
3276 .to_string()
3277 .contains("no choice answer for open_questions"));
3278 for answer in [
3279 Answer::Noul { noul: 0.8 },
3280 Answer::Choice {
3281 choice: "unknown".into(),
3282 confidence: 0.8,
3283 probabilities: BTreeMap::new(),
3284 },
3285 Answer::Choice {
3286 choice: "none".into(),
3287 confidence: 0.8,
3288 probabilities: BTreeMap::from([("unknown".into(), 0.2)]),
3289 },
3290 ] {
3291 assert!(
3292 open_questions_answer(&BTreeMap::from([("open_questions".into(), answer)]))
3293 .unwrap_err()
3294 .to_string()
3295 .contains("open_questions")
3296 );
3297 }
3298 }
3299
3300 #[tokio::test]
3301 async fn unusable_open_questions_never_fail_routing() {
3302 let server = wiremock::MockServer::start().await;
3303 for number in [7, 8, 9] {
3304 let mut answers = serde_json::json!({
3305 "anthropic.stage_design": choice_json("none", 0.9),
3306 "anthropic.stage_implement": choice_json("sonnet", 0.9),
3307 "anthropic.stage_review": choice_json("opus", 0.9),
3308 });
3309 match number {
3310 8 => answers["open_questions"] = choice_json("both", 0.7),
3312 9 => answers["open_questions"] = choice_json("maybe", 0.7),
3313 _ => {}
3314 }
3315 wiremock::Mock::given(wiremock::matchers::body_partial_json(serde_json::json!({"state": format!("# #{number} Route issues\n\nThe body.\n")})))
3316 .respond_with(wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!({"model": "jev-test", "answers": answers, "usage": {"input_tokens": 10, "output_tokens": 5}})))
3317 .expect(1).mount(&server).await;
3318 }
3319 let client = JevClient::new(&server.uri(), "key").unwrap();
3320 let report = run_route(
3321 &client,
3322 &[
3323 doc(7, ItemState::Open),
3324 doc(8, ItemState::Open),
3325 doc(9, ItemState::Open),
3326 ],
3327 &anthropic(),
3328 &opts(),
3329 &OpenDependencies::new(),
3330 )
3331 .await
3332 .unwrap();
3333 assert!(report.issues.iter().all(|issue| !issue.failed()));
3334 assert_eq!(report.usage.input_tokens, 30);
3335 let json = serde_json::to_value(&report).unwrap();
3336 for index in [0, 2] {
3337 let issue = &json["issues"][index];
3338 assert!(issue.get("open_questions").is_none(), "{issue}");
3339 assert_eq!(issue["providers"]["anthropic"]["class"], "sonnet");
3340 assert!(issue.get("error").is_none(), "{issue}");
3341 let text =
3342 render_issue_block(&report.issues[index], 1000, &[], TerminalStyle::default());
3343 assert!(!text.contains("open_questions"), "{text}");
3344 }
3345 assert_eq!(json["issues"][1]["open_questions"]["choice"], "both");
3346 assert_eq!(
3347 json["issues"][1]["providers"]["anthropic"]["class"],
3348 "sonnet"
3349 );
3350 }
3351
3352 fn test_open_questions() -> OpenQuestions {
3353 OpenQuestions {
3354 choice: OpenQuestionKind::Factual,
3355 confidence: 0.8,
3356 probabilities: BTreeMap::from([("factual".into(), 0.8)]),
3357 }
3358 }
3359
3360 fn choice_json(choice: &str, confidence: f64) -> serde_json::Value {
3361 serde_json::json!({
3362 "type": "choice", "choice": choice, "confidence": confidence,
3363 "probabilities": {choice: confidence}
3364 })
3365 }
3366
3367 fn provider<'a>(outcome: &'a RouteOutcome, name: &str) -> &'a ProviderRoute {
3368 let RouteOutcome::Routed { providers, .. } = outcome else {
3369 panic!("expected a routed issue: {outcome:?}");
3371 };
3372 &providers[name]
3373 }
3374
3375 #[tokio::test]
3376 async fn run_route_asks_stages_and_open_questions_per_issue_and_sums_usage() {
3377 let server = wiremock::MockServer::start().await;
3378 wiremock::Mock::given(wiremock::matchers::method("POST"))
3379 .and(wiremock::matchers::path("/v1/systemone"))
3380 .and(wiremock::matchers::body_partial_json(serde_json::json!({
3381 "model": "jev-latest",
3382 "state": "# #7 Route issues\n\nThe body.\n",
3383 })))
3384 .respond_with(
3385 wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!({
3386 "model": "jev-1.13.0",
3387 "answers": {
3388 "open_questions": choice_json("factual", 0.8),
3389 "anthropic.stage_design": choice_json("opus", 0.52),
3390 "anthropic.stage_implement": choice_json("sonnet", 0.83),
3391 "anthropic.stage_review": choice_json("opus", 0.21),
3392 },
3393 "usage": {"input_tokens": 100, "output_tokens": 10}
3394 })),
3395 )
3396 .expect(2)
3397 .mount(&server)
3398 .await;
3399 let client = JevClient::new(&server.uri(), "key").unwrap();
3400
3401 let report = run_route(
3402 &client,
3403 &[doc(7, ItemState::Open), doc(7, ItemState::Open)],
3404 &anthropic(),
3405 &opts(),
3406 &OpenDependencies::new(),
3407 )
3408 .await
3409 .unwrap();
3410
3411 assert_eq!(report.model, "jev-1.13.0");
3412 assert_eq!(report.usage.input_tokens, 200);
3413 assert_eq!(report.usage.output_tokens, 20);
3414 let issue = &report.issues[0];
3415 assert_eq!(issue.item_ref, "rust-works/omni-dev#7");
3416 let route = provider(&issue.outcome, "anthropic");
3417 assert_eq!(route.class, "opus");
3418 assert_eq!(route.class_from, ClassSource::Design);
3419 assert_eq!(route.close_calls, [Stage::Review]);
3420 assert!(!issue.truncated);
3421
3422 let requests = server.received_requests().await.unwrap();
3423 let body: serde_json::Value = requests[0].body_json().unwrap();
3424 let keys: Vec<&String> = body["questions"].as_object().unwrap().keys().collect();
3425 assert_eq!(
3426 keys,
3427 [
3428 "anthropic.stage_design",
3429 "anthropic.stage_implement",
3430 "anthropic.stage_review",
3431 "open_questions",
3432 ]
3433 );
3434 }
3435
3436 #[tokio::test]
3440 async fn run_route_ignore_closed_routes_only_open_issues() {
3441 let server = wiremock::MockServer::start().await;
3442 wiremock::Mock::given(wiremock::matchers::method("POST"))
3443 .respond_with(
3444 wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!({
3445 "model": "jev-1.13.0",
3446 "answers": {
3447 "open_questions": choice_json("factual", 0.8),
3448 "anthropic.stage_design": choice_json("opus", 0.52),
3449 "anthropic.stage_implement": choice_json("sonnet", 0.83),
3450 "anthropic.stage_review": choice_json("opus", 0.61),
3451 },
3452 "usage": {"input_tokens": 100, "output_tokens": 10}
3453 })),
3454 )
3455 .expect(2)
3456 .mount(&server)
3457 .await;
3458 let client = JevClient::new(&server.uri(), "key").unwrap();
3459 let mut o = opts();
3460 o.ignore_closed = true;
3461
3462 let report = run_route(
3463 &client,
3464 &[
3465 doc(7, ItemState::Open),
3466 doc(8, ItemState::Closed),
3467 doc(9, ItemState::Open),
3468 ],
3469 &anthropic(),
3470 &o,
3471 &OpenDependencies::new(),
3472 )
3473 .await
3474 .unwrap();
3475
3476 let refs: Vec<&str> = report.issues.iter().map(|i| i.item_ref.as_str()).collect();
3477 assert_eq!(refs, ["rust-works/omni-dev#7", "rust-works/omni-dev#9"]);
3478 let json = serde_json::to_value(&report).unwrap();
3479 assert_eq!(
3480 json["ignored_closed"],
3481 serde_json::json!([{
3482 "ref": "rust-works/omni-dev#8",
3483 "url": "https://github.com/rust-works/omni-dev/issues/8",
3484 "title": "Route issues",
3485 }])
3486 );
3487
3488 let err = run_route(
3489 &client,
3490 &[doc(8, ItemState::Closed)],
3491 &anthropic(),
3492 &o,
3493 &OpenDependencies::new(),
3494 )
3495 .await
3496 .unwrap_err();
3497 assert!(err.to_string().contains("nothing to route"), "{err}");
3498 }
3499
3500 #[test]
3503 fn route_report_omits_an_empty_ignored_closed() {
3504 let report = RouteReport {
3505 model: "m".to_string(),
3506 issues: vec![],
3507 ignored_closed: vec![],
3508 usage: Usage::default(),
3509 };
3510 let json = serde_json::to_value(&report).unwrap();
3511 assert!(json.get("ignored_closed").is_none(), "{json}");
3512 }
3513
3514 #[tokio::test]
3517 async fn run_route_routes_every_provider_in_one_call() {
3518 let server = wiremock::MockServer::start().await;
3519 wiremock::Mock::given(wiremock::matchers::method("POST"))
3520 .respond_with(
3521 wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!({
3522 "model": "jev-1.13.0",
3523 "answers": {
3524 "open_questions": choice_json("factual", 0.8),
3525 "anthropic.stage_design": choice_json("opus", 0.52),
3526 "anthropic.stage_implement": choice_json("sonnet", 0.83),
3527 "anthropic.stage_review": choice_json("opus", 0.9),
3528 "openai.stage_design": choice_json("none", 0.9),
3529 "openai.stage_implement": choice_json("sol", 0.2),
3530 "openai.stage_review": choice_json("astra", 0.9),
3531 "could_be_cheaper_0": {"type": "noul", "noul": 0.4},
3532 },
3533 "usage": {"input_tokens": 10, "output_tokens": 1}
3534 })),
3535 )
3536 .expect(1)
3537 .mount(&server)
3538 .await;
3539 let client = JevClient::new(&server.uri(), "key").unwrap();
3540 let ladders = [
3541 class_only(Provider::Anthropic).unwrap(),
3542 class_only(Provider::OpenAi).unwrap(),
3543 ];
3544 let dependencies = OpenDependencies::from([(
3545 ("rust-works/omni-dev".to_string(), 7),
3546 vec![citation("#1129", 1129)],
3547 )]);
3548 let reference_fetch_failures = ReferenceFetchFailures::from([(
3549 ("rust-works/omni-dev".to_string(), 7),
3550 vec![ReferenceFetchFailure {
3551 item_ref: "#404".to_string(),
3552 error: "not found".to_string(),
3553 }],
3554 )]);
3555
3556 let report = run_route_with_reference_fetch_failures(
3557 &client,
3558 &[doc(7, ItemState::Open)],
3559 &ladders,
3560 &opts(),
3561 &dependencies,
3562 &reference_fetch_failures,
3563 )
3564 .await
3565 .unwrap();
3566
3567 let outcome = &report.issues[0].outcome;
3568 let RouteOutcome::Routed {
3569 providers,
3570 depends_on,
3571 reference_fetch_failures,
3572 ..
3573 } = outcome
3574 else {
3575 panic!("expected a routed issue: {outcome:?}");
3577 };
3578 assert_eq!(
3579 providers.keys().collect::<Vec<_>>(),
3580 ["anthropic", "openai"]
3581 );
3582 assert_eq!(provider(outcome, "anthropic").class, "opus");
3583 assert_eq!(
3584 provider(outcome, "anthropic").class_from,
3585 ClassSource::Design
3586 );
3587 assert!(provider(outcome, "anthropic").close_calls.is_empty());
3588 assert_eq!(provider(outcome, "openai").class, "sol");
3589 assert_eq!(
3590 provider(outcome, "openai").class_from,
3591 ClassSource::Implement
3592 );
3593 assert_eq!(provider(outcome, "openai").close_calls, [Stage::Implement]);
3594 assert_eq!(depends_on.len(), 1);
3595 assert_eq!(reference_fetch_failures[0].item_ref, "#404");
3596
3597 let requests = server.received_requests().await.unwrap();
3598 let body: serde_json::Value = requests[0].body_json().unwrap();
3599 let keys: Vec<&String> = body["questions"].as_object().unwrap().keys().collect();
3600 assert_eq!(
3601 keys,
3602 [
3603 "anthropic.stage_design",
3604 "anthropic.stage_implement",
3605 "anthropic.stage_review",
3606 "could_be_cheaper_0",
3607 "open_questions",
3608 "openai.stage_design",
3609 "openai.stage_implement",
3610 "openai.stage_review",
3611 ]
3612 );
3613 }
3614
3615 #[tokio::test]
3618 async fn run_route_fails_the_issue_when_one_provider_is_unanswered() {
3619 let server = wiremock::MockServer::start().await;
3620 wiremock::Mock::given(wiremock::matchers::method("POST"))
3621 .respond_with(wiremock::ResponseTemplate::new(200).set_body_json(routed_json()))
3622 .mount(&server)
3623 .await;
3624 let client = JevClient::new(&server.uri(), "key").unwrap();
3625 let ladders = [
3626 class_only(Provider::Anthropic).unwrap(),
3627 class_only(Provider::Gemini).unwrap(),
3628 ];
3629 let report = run_route(
3630 &client,
3631 &[doc(7, ItemState::Open)],
3632 &ladders,
3633 &opts(),
3634 &OpenDependencies::new(),
3635 )
3636 .await
3637 .unwrap();
3638 let RouteOutcome::Failed { error, .. } = &report.issues[0].outcome else {
3639 panic!("expected a failed issue: {:?}", report.issues[0]);
3641 };
3642 assert!(error.contains("gemini.stage_design"), "{error}");
3643 }
3644
3645 fn routed_json() -> serde_json::Value {
3646 serde_json::json!({
3647 "model": "jev-1.13.0",
3648 "answers": {
3649 "open_questions": choice_json("factual", 0.8),
3650 "anthropic.stage_design": choice_json("none", 0.9),
3651 "anthropic.stage_implement": choice_json("sonnet", 0.9),
3652 "anthropic.stage_review": choice_json("opus", 0.9),
3653 },
3654 "usage": {"input_tokens": 5, "output_tokens": 1}
3655 })
3656 }
3657
3658 #[tokio::test]
3660 async fn run_route_keeps_going_past_a_failed_issue() {
3661 let server = wiremock::MockServer::start().await;
3662 wiremock::Mock::given(wiremock::matchers::method("POST"))
3663 .and(wiremock::matchers::body_partial_json(
3664 serde_json::json!({"state": "# #1 Route issues\n\nThe body.\n"}),
3665 ))
3666 .respond_with(wiremock::ResponseTemplate::new(500).set_body_string("boom"))
3667 .mount(&server)
3668 .await;
3669 wiremock::Mock::given(wiremock::matchers::method("POST"))
3670 .respond_with(wiremock::ResponseTemplate::new(200).set_body_json(routed_json()))
3671 .mount(&server)
3672 .await;
3673 let client = JevClient::new(&server.uri(), "key").unwrap();
3674 let reference_fetch_failures = ReferenceFetchFailures::from([(
3675 ("rust-works/omni-dev".to_string(), 1),
3676 vec![ReferenceFetchFailure {
3677 item_ref: "#404".to_string(),
3678 error: "not found".to_string(),
3679 }],
3680 )]);
3681 let report = run_route_with_reference_fetch_failures(
3682 &client,
3683 &[doc(1, ItemState::Open), doc(2, ItemState::Open)],
3684 &anthropic(),
3685 &opts(),
3686 &OpenDependencies::new(),
3687 &reference_fetch_failures,
3688 )
3689 .await
3690 .unwrap();
3691
3692 assert!(report.issues[0].failed());
3693 let RouteOutcome::Failed {
3694 error,
3695 reference_fetch_failures,
3696 } = &report.issues[0].outcome
3697 else {
3698 panic!();
3699 };
3700 assert!(error.contains("500"), "{error}");
3701 assert_eq!(reference_fetch_failures[0].item_ref, "#404");
3702 let json = serde_json::to_value(&report).unwrap();
3703 assert_eq!(
3704 json["issues"][0]["reference_fetch_failures"][0]["ref"],
3705 "#404"
3706 );
3707 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
3708 assert!(
3709 text.contains(" reference fetch failed: #404 (not found)"),
3710 "{text}"
3711 );
3712 assert!(!report.issues[1].failed());
3713 assert_eq!(report.usage.input_tokens, 5);
3714 }
3715
3716 #[tokio::test]
3717 async fn run_route_stops_on_an_auth_failure() {
3718 let server = wiremock::MockServer::start().await;
3719 wiremock::Mock::given(wiremock::matchers::method("POST"))
3720 .respond_with(wiremock::ResponseTemplate::new(401).set_body_string("bad key"))
3721 .expect(1)
3722 .mount(&server)
3723 .await;
3724 let client = JevClient::new(&server.uri(), "key").unwrap();
3725 let err = run_route(
3726 &client,
3727 &[doc(3, ItemState::Open), doc(4, ItemState::Open)],
3728 &anthropic(),
3729 &opts(),
3730 &OpenDependencies::new(),
3731 )
3732 .await
3733 .unwrap_err();
3734 assert!(format!("{err:#}").contains("Failed to route issue rust-works/omni-dev#3"));
3735 }
3736
3737 #[tokio::test]
3738 async fn run_route_rejects_a_ladder_over_the_choice_cap_before_any_request() {
3739 let yaml: String = std::iter::once("tiers:\n".to_string())
3740 .chain((0..256).map(|i| format!(" - {{name: t{i}, description: d{i}}}\n")))
3741 .collect();
3742 let ladder = Ladder::named("big".to_string(), Tiers::parse(&yaml).unwrap());
3743 let client = JevClient::new("http://127.0.0.1:1", "key").unwrap();
3744 let err = run_route(
3745 &client,
3746 &[doc(3, ItemState::Open)],
3747 &[ladder],
3748 &opts(),
3749 &OpenDependencies::new(),
3750 )
3751 .await
3752 .unwrap_err();
3753 let chain = format!("{err:#}");
3754 assert!(chain.contains("route question \"big.stage_"), "{chain}");
3755 assert!(
3756 chain.contains("at most 255 options on jev-latest"),
3757 "{chain}"
3758 );
3759 }
3760
3761 #[tokio::test]
3762 async fn run_route_names_the_issue_on_an_unexpected_answer() {
3763 let server = wiremock::MockServer::start().await;
3764 wiremock::Mock::given(wiremock::matchers::method("POST"))
3765 .respond_with(
3766 wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!({
3767 "model": "jev-1.13.0",
3768 "answers": {
3769 "open_questions": choice_json("factual", 0.8),
3770 "anthropic.stage_design": choice_json("haiku", 0.9),
3771 "anthropic.stage_implement": choice_json("sonnet", 0.9),
3772 "anthropic.stage_review": choice_json("opus", 0.9),
3773 },
3774 })),
3775 )
3776 .mount(&server)
3777 .await;
3778 let client = JevClient::new(&server.uri(), "key").unwrap();
3779 let report = run_route(
3780 &client,
3781 &[doc(9, ItemState::Open)],
3782 &anthropic(),
3783 &opts(),
3784 &OpenDependencies::new(),
3785 )
3786 .await
3787 .unwrap();
3788 let RouteOutcome::Failed { error, .. } = &report.issues[0].outcome else {
3789 panic!("expected a failed issue");
3790 };
3791 assert!(error.contains("Unexpected Jev answer"), "{error}");
3792 assert!(error.contains("not offered"), "{error}");
3793 }
3794
3795 fn citation(raw: &str, number: u64) -> Citation {
3796 Citation {
3797 item_ref: ItemRef {
3798 provider: GitProvider::GitHub,
3799 project: "rust-works/omni-dev".to_string(),
3800 kind: ItemKind::Issue,
3801 number,
3802 },
3803 raw: raw.to_string(),
3804 url: None,
3805 }
3806 }
3807
3808 #[tokio::test]
3809 async fn run_route_asks_could_be_cheaper_for_each_open_citation() {
3810 let server = wiremock::MockServer::start().await;
3811 wiremock::Mock::given(wiremock::matchers::method("POST"))
3812 .respond_with(
3813 wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!({
3814 "model": "jev-1.13.0",
3815 "answers": {
3816 "open_questions": choice_json("factual", 0.8),
3817 "anthropic.stage_design": choice_json("opus", 0.2),
3818 "anthropic.stage_implement": choice_json("sonnet", 0.61),
3819 "anthropic.stage_review": choice_json("opus", 0.9),
3820 "could_be_cheaper_0": {"type": "noul", "noul": 0.75},
3821 },
3822 "usage": {"input_tokens": 10, "output_tokens": 1}
3823 })),
3824 )
3825 .mount(&server)
3826 .await;
3827 let client = JevClient::new(&server.uri(), "key").unwrap();
3828 let dependencies = OpenDependencies::from([(
3829 ("rust-works/omni-dev".to_string(), 7),
3830 vec![citation("#1129", 1129)],
3831 )]);
3832
3833 let report = run_route(
3834 &client,
3835 &[doc(7, ItemState::Open)],
3836 &anthropic(),
3837 &opts(),
3838 &dependencies,
3839 )
3840 .await
3841 .unwrap();
3842
3843 let RouteOutcome::Routed { depends_on, .. } = &report.issues[0].outcome else {
3844 panic!("expected a routed issue: {:?}", report.issues[0]);
3845 };
3846 assert_eq!(depends_on.len(), 1);
3847 assert_eq!(depends_on[0].item_ref, "#1129");
3848 assert_eq!(depends_on[0].state, ItemState::Open);
3849 assert_eq!(depends_on[0].could_be_cheaper.get("design"), Some(&0.75));
3850
3851 let requests = server.received_requests().await.unwrap();
3852 let body: serde_json::Value = requests[0].body_json().unwrap();
3853 let keys: Vec<&String> = body["questions"].as_object().unwrap().keys().collect();
3854 assert_eq!(
3855 keys,
3856 [
3857 "anthropic.stage_design",
3858 "anthropic.stage_implement",
3859 "anthropic.stage_review",
3860 "could_be_cheaper_0",
3861 "open_questions",
3862 ]
3863 );
3864 }
3865
3866 #[tokio::test]
3867 async fn run_route_keeps_a_dependency_with_no_could_be_cheaper_answer() {
3868 let server = wiremock::MockServer::start().await;
3869 wiremock::Mock::given(wiremock::matchers::method("POST"))
3870 .respond_with(wiremock::ResponseTemplate::new(200).set_body_json(routed_json()))
3871 .mount(&server)
3872 .await;
3873 let client = JevClient::new(&server.uri(), "key").unwrap();
3874 let dependencies = OpenDependencies::from([(
3875 ("rust-works/omni-dev".to_string(), 7),
3876 vec![citation("#1129", 1129)],
3877 )]);
3878
3879 let report = run_route(
3880 &client,
3881 &[doc(7, ItemState::Open)],
3882 &anthropic(),
3883 &opts(),
3884 &dependencies,
3885 )
3886 .await
3887 .unwrap();
3888
3889 let RouteOutcome::Routed { depends_on, .. } = &report.issues[0].outcome else {
3892 panic!("expected a routed issue: {:?}", report.issues[0]);
3893 };
3894 assert_eq!(depends_on.len(), 1, "{depends_on:?}");
3895 assert_eq!(depends_on[0].item_ref, "#1129");
3896 assert!(depends_on[0].could_be_cheaper.is_empty());
3897 }
3898
3899 #[test]
3900 fn report_serialises_ref_and_omits_false_truncated() {
3901 let report = RouteReport {
3902 model: "jev-1.13.0".to_string(),
3903 issues: vec![IssueRoute {
3904 draft_comments: vec![],
3905 item_ref: "o/r#1".to_string(),
3906 url: "u".to_string(),
3907 title: "t".to_string(),
3908 state: ItemState::Open,
3909 outcome: RouteOutcome::Routed {
3910 open_questions: Some(test_open_questions()),
3911 providers: BTreeMap::from([(
3912 "anthropic".to_string(),
3913 ProviderRoute {
3914 stages: stages("none", "sonnet", "opus"),
3915 class: "sonnet".to_string(),
3916 class_from: ClassSource::Implement,
3917 close_calls: vec![],
3918 },
3919 )]),
3920 depends_on: vec![],
3921 reference_fetch_failures: vec![],
3922 },
3923 truncated: false,
3924 }],
3925 ignored_closed: vec![],
3926 usage: Usage::default(),
3927 };
3928 let value = serde_json::to_value(&report).unwrap();
3929 assert_eq!(value["issues"][0]["ref"], "o/r#1");
3930 assert!(value["issues"][0].get("truncated").is_none());
3931 let anthropic = &value["issues"][0]["providers"]["anthropic"];
3932 assert_eq!(anthropic["stages"]["design"]["choice"], "none");
3933 assert_eq!(anthropic["class"], "sonnet");
3934 assert_eq!(anthropic["close_calls"], serde_json::json!([]));
3935 assert!(value["issues"][0].get("stages").is_none());
3936 assert!(value["issues"][0].get("class").is_none());
3937 assert_eq!(value["issues"][0]["depends_on"], serde_json::json!([]));
3938 assert!(value["issues"][0].get("reference_fetch_failures").is_none());
3939 assert!(value["issues"][0].get("error").is_none());
3940 }
3941
3942 #[test]
3943 fn a_failed_issue_serialises_only_its_error() {
3944 let route = IssueRoute {
3945 draft_comments: vec![],
3946 item_ref: "o/r#1".to_string(),
3947 url: "u".to_string(),
3948 title: "t".to_string(),
3949 state: ItemState::Open,
3950 outcome: RouteOutcome::Failed {
3951 error: "HTTP 529".to_string(),
3952 reference_fetch_failures: vec![],
3953 },
3954 truncated: false,
3955 };
3956 let value = serde_json::to_value(&route).unwrap();
3957 assert_eq!(value["error"], "HTTP 529");
3958 assert!(value.get("providers").is_none());
3959 assert!(value.get("depends_on").is_none());
3960 }
3961
3962 #[test]
3965 fn render_route_text_matches_the_worked_example() {
3966 let report = RouteReport {
3967 model: "jev-1.13.0".to_string(),
3968 issues: vec![IssueRoute {
3969 draft_comments: vec![],
3970 item_ref: "rust-works/omni-dev#1641".to_string(),
3971 url: "u".to_string(),
3972 title: "Some issue title".to_string(),
3973 state: ItemState::Open,
3974 outcome: RouteOutcome::Routed {
3975 open_questions: Some(test_open_questions()),
3976 providers: BTreeMap::from([(
3977 "anthropic".to_string(),
3978 ProviderRoute {
3979 stages: StageAnswers {
3980 design: answer("opus", 0.52),
3981 implement: answer("sonnet", 0.83),
3982 review: answer("opus", 0.41),
3983 },
3984 class: "opus".to_string(),
3985 class_from: ClassSource::Design,
3986 close_calls: vec![Stage::Review],
3987 },
3988 )]),
3989 depends_on: vec![DependencyEntry {
3990 item_ref: "#1129".to_string(),
3991 url: None,
3992 state: ItemState::Open,
3993 relation: CitationRelation::Unspecified,
3994 could_be_cheaper: BTreeMap::from([("design".to_string(), 0.75)]),
3995 }],
3996 reference_fetch_failures: vec![],
3997 },
3998 truncated: false,
3999 }],
4000 ignored_closed: vec![],
4001 usage: Usage {
4002 input_tokens: 1432,
4003 output_tokens: 61,
4004 },
4005 };
4006 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
4007 assert_eq!(
4008 text,
4009 "rust-works/omni-dev#1641 — Some issue title\n\
4010 \x20\x20open_questions: factual (0.80)\n\
4011 \x20\x20opus (from design) — design needs opus (0.52), implementation sonnet (0.83), review \
4012 opus (0.41, close call)\n\
4013 \x20\x20cites open #1129 (relation unspecified), which could leave less design work if resolved (0.75)\n\n\
4014 model: jev-1.13.0, usage: 1432 input tokens, 61 output tokens\n"
4015 );
4016 }
4017
4018 #[test]
4019 fn render_route_text_puts_each_open_citation_on_its_own_line() {
4020 let report = RouteReport {
4021 model: "jev-1.13.0".to_string(),
4022 issues: vec![IssueRoute {
4023 draft_comments: vec![],
4024 item_ref: "rust-works/omni-dev#1845".to_string(),
4025 url: "u".to_string(),
4026 title: "t".to_string(),
4027 state: ItemState::Open,
4028 outcome: RouteOutcome::Routed {
4029 open_questions: Some(test_open_questions()),
4030 providers: BTreeMap::from([(
4031 "anthropic".to_string(),
4032 ProviderRoute {
4033 stages: stages("none", "sonnet", "sonnet"),
4034 class: "sonnet".to_string(),
4035 class_from: ClassSource::Implement,
4036 close_calls: vec![],
4037 },
4038 )]),
4039 depends_on: vec![
4040 DependencyEntry {
4041 item_ref: "#1830".to_string(),
4042 url: None,
4043 state: ItemState::Open,
4044 relation: CitationRelation::Unspecified,
4045 could_be_cheaper: BTreeMap::from([("design".to_string(), 0.46)]),
4046 },
4047 DependencyEntry {
4048 item_ref: "#1831".to_string(),
4049 url: None,
4050 state: ItemState::Open,
4051 relation: CitationRelation::Unspecified,
4052 could_be_cheaper: BTreeMap::new(),
4053 },
4054 ],
4055 reference_fetch_failures: vec![],
4056 },
4057 truncated: false,
4058 }],
4059 ignored_closed: vec![],
4060 usage: Usage::default(),
4061 };
4062 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
4063 assert!(
4064 text.contains(
4065 " cites open #1830 (relation unspecified), which could leave less design work if resolved (0.46)\n \
4066 cites open #1831 (relation unspecified)\n"
4067 ),
4068 "{text}"
4069 );
4070 assert!(!text.contains("; "), "{text}");
4071 }
4072
4073 #[test]
4074 fn render_depends_on_lines_labels_each_citation_relation() {
4075 let entry = |item_ref: &str, relation| DependencyEntry {
4076 item_ref: item_ref.to_string(),
4077 url: None,
4078 state: ItemState::Open,
4079 relation,
4080 could_be_cheaper: BTreeMap::new(),
4081 };
4082 let lines = render_depends_on_lines(
4083 &[
4084 entry("#1", CitationRelation::Blocker),
4085 entry("#2", CitationRelation::Tracker),
4086 entry("#3", CitationRelation::Unspecified),
4087 ],
4088 TerminalStyle::default(),
4089 );
4090 assert_eq!(
4091 lines,
4092 [
4093 " cites open #1 (blocker)",
4094 " cites open #2 (tracker)",
4095 " cites open #3 (relation unspecified)",
4096 ]
4097 );
4098 }
4099
4100 #[test]
4101 fn render_route_text_names_each_provider_when_there_are_several() {
4102 let route = |class: &str| ProviderRoute {
4103 stages: stages("none", "sonnet", "sonnet"),
4104 class: class.to_string(),
4105 class_from: ClassSource::Implement,
4106 close_calls: vec![],
4107 };
4108 let report = RouteReport {
4109 model: "jev-1.13.0".to_string(),
4110 issues: vec![IssueRoute {
4111 draft_comments: vec![],
4112 item_ref: "o/r#1".to_string(),
4113 url: "u".to_string(),
4114 title: "t".to_string(),
4115 state: ItemState::Open,
4116 outcome: RouteOutcome::Routed {
4117 open_questions: Some(test_open_questions()),
4118 providers: BTreeMap::from([
4119 ("anthropic".to_string(), route("sonnet")),
4120 ("openai".to_string(), route("terra")),
4121 ]),
4122 depends_on: vec![],
4123 reference_fetch_failures: vec![],
4124 },
4125 truncated: false,
4126 }],
4127 ignored_closed: vec![],
4128 usage: Usage::default(),
4129 };
4130 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
4131 assert!(
4132 text.contains(" anthropic: sonnet (from implementation) —"),
4133 "{text}"
4134 );
4135 assert!(
4136 text.contains(" openai: terra (from implementation) —"),
4137 "{text}"
4138 );
4139 }
4140
4141 #[test]
4142 fn render_route_text_reports_a_failed_issue() {
4143 let report = RouteReport {
4144 model: "jev-1.13.0".to_string(),
4145 issues: vec![IssueRoute {
4146 draft_comments: vec![],
4147 item_ref: "o/r#1".to_string(),
4148 url: "u".to_string(),
4149 title: "t".to_string(),
4150 state: ItemState::Open,
4151 outcome: RouteOutcome::Failed {
4152 error: "HTTP 529".to_string(),
4153 reference_fetch_failures: vec![],
4154 },
4155 truncated: false,
4156 }],
4157 ignored_closed: vec![],
4158 usage: Usage::default(),
4159 };
4160 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
4161 assert!(text.contains("o/r#1 — t\n failed: HTTP 529"), "{text}");
4162 }
4163
4164 #[test]
4165 fn render_route_text_marks_a_closed_issue_in_the_header() {
4166 let report = RouteReport {
4167 model: "jev-1.13.0".to_string(),
4168 issues: vec![IssueRoute {
4169 draft_comments: vec![],
4170 item_ref: "o/r#1".to_string(),
4171 url: "u".to_string(),
4172 title: "t".to_string(),
4173 state: ItemState::Closed,
4174 outcome: RouteOutcome::Failed {
4175 error: "HTTP 529".to_string(),
4176 reference_fetch_failures: vec![],
4177 },
4178 truncated: false,
4179 }],
4180 ignored_closed: vec![],
4181 usage: Usage::default(),
4182 };
4183 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
4184 assert!(text.starts_with("o/r#1 — t (closed)\n"), "{text}");
4185 assert!(!text.contains("ignored closed"), "{text}");
4186 }
4187
4188 #[test]
4189 fn render_route_text_lists_ignored_closed_issues_before_the_footer() {
4190 let ignored = |n: u64| IgnoredIssue {
4191 item_ref: format!("o/r#{n}"),
4192 url: format!("https://github.com/o/r/issues/{n}"),
4193 title: "t".to_string(),
4194 };
4195 let report = RouteReport {
4196 model: "jev-1.13.0".to_string(),
4197 issues: vec![],
4198 ignored_closed: vec![ignored(2), ignored(3)],
4199 usage: Usage::default(),
4200 };
4201 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
4202 assert!(
4203 text.contains("ignored closed: o/r#2, o/r#3\n\nmodel: jev-1.13.0"),
4204 "{text}"
4205 );
4206 let styled = render_route_text_styled(
4207 &report,
4208 DEFAULT_MAX_INPUT_CHARS,
4209 &[],
4210 TerminalStyle {
4211 color: false,
4212 hyperlinks: true,
4213 },
4214 );
4215 assert!(
4216 styled.contains("https://github.com/o/r/issues/2"),
4217 "{styled}"
4218 );
4219 }
4220
4221 #[test]
4222 fn render_route_text_omits_the_depends_on_sentence_when_empty() {
4223 let report = RouteReport {
4224 model: "jev-1.13.0".to_string(),
4225 issues: vec![IssueRoute {
4226 draft_comments: vec![],
4227 item_ref: "o/r#1".to_string(),
4228 url: "u".to_string(),
4229 title: "t".to_string(),
4230 state: ItemState::Open,
4231 outcome: RouteOutcome::Routed {
4232 open_questions: Some(test_open_questions()),
4233 providers: BTreeMap::from([(
4234 "anthropic".to_string(),
4235 ProviderRoute {
4236 stages: stages("none", "sonnet", "sonnet"),
4237 class: "sonnet".to_string(),
4238 class_from: ClassSource::Implement,
4239 close_calls: vec![],
4240 },
4241 )]),
4242 depends_on: vec![],
4243 reference_fetch_failures: vec![],
4244 },
4245 truncated: false,
4246 }],
4247 ignored_closed: vec![],
4248 usage: Usage::default(),
4249 };
4250 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
4251 assert!(!text.contains("cites"), "{text}");
4252 }
4253
4254 #[test]
4258 fn render_route_text_cites_a_dependency_with_no_could_be_cheaper_probability() {
4259 let report = RouteReport {
4260 model: "jev-1.13.0".to_string(),
4261 issues: vec![IssueRoute {
4262 draft_comments: vec![],
4263 item_ref: "o/r#1".to_string(),
4264 url: "u".to_string(),
4265 title: "t".to_string(),
4266 state: ItemState::Open,
4267 outcome: RouteOutcome::Routed {
4268 open_questions: Some(test_open_questions()),
4269 providers: BTreeMap::from([(
4270 "anthropic".to_string(),
4271 ProviderRoute {
4272 stages: stages("none", "sonnet", "sonnet"),
4273 class: "sonnet".to_string(),
4274 class_from: ClassSource::Implement,
4275 close_calls: vec![],
4276 },
4277 )]),
4278 depends_on: vec![DependencyEntry {
4279 item_ref: "#1129".to_string(),
4280 url: None,
4281 state: ItemState::Open,
4282 relation: CitationRelation::Unspecified,
4283 could_be_cheaper: BTreeMap::new(),
4284 }],
4285 reference_fetch_failures: vec![],
4286 },
4287 truncated: false,
4288 }],
4289 ignored_closed: vec![],
4290 usage: Usage::default(),
4291 };
4292 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
4293 assert!(text.contains("cites open #1129"), "{text}");
4294 assert!(!text.contains("could leave less design work"), "{text}");
4295 }
4296
4297 #[test]
4298 fn render_route_text_notes_a_truncated_issue_and_omits_the_note_otherwise() {
4299 let issue = |truncated: bool| IssueRoute {
4300 draft_comments: vec![],
4301 item_ref: "o/r#1".to_string(),
4302 url: "u".to_string(),
4303 title: "t".to_string(),
4304 state: ItemState::Open,
4305 outcome: RouteOutcome::Routed {
4306 open_questions: Some(test_open_questions()),
4307 providers: BTreeMap::from([(
4308 "anthropic".to_string(),
4309 ProviderRoute {
4310 stages: stages("none", "sonnet", "sonnet"),
4311 class: "sonnet".to_string(),
4312 class_from: ClassSource::Implement,
4313 close_calls: vec![],
4314 },
4315 )]),
4316 depends_on: vec![],
4317 reference_fetch_failures: vec![],
4318 },
4319 truncated,
4320 };
4321 let report = |truncated| RouteReport {
4322 model: "jev-1.13.0".to_string(),
4323 issues: vec![issue(truncated)],
4324 ignored_closed: vec![],
4325 usage: Usage::default(),
4326 };
4327 let truncated_text = render_route_text(&report(true), 60_000);
4328 assert!(
4329 truncated_text.contains(" input truncated at 60,000 characters"),
4330 "{truncated_text}"
4331 );
4332 let kept_text = render_route_text(&report(false), 60_000);
4333 assert!(!kept_text.contains("truncated"), "{kept_text}");
4334 }
4335
4336 #[test]
4337 fn render_route_text_notes_no_further_design_work() {
4338 let report = RouteReport {
4339 model: "jev-1.13.0".to_string(),
4340 issues: vec![IssueRoute {
4341 draft_comments: vec![],
4342 item_ref: "o/r#1".to_string(),
4343 url: "u".to_string(),
4344 title: "t".to_string(),
4345 state: ItemState::Open,
4346 outcome: RouteOutcome::Routed {
4347 open_questions: Some(test_open_questions()),
4348 providers: BTreeMap::from([(
4349 "anthropic".to_string(),
4350 ProviderRoute {
4351 stages: stages("none", "sonnet", "sonnet"),
4352 class: "sonnet".to_string(),
4353 class_from: ClassSource::Implement,
4354 close_calls: vec![],
4355 },
4356 )]),
4357 depends_on: vec![],
4358 reference_fetch_failures: vec![],
4359 },
4360 truncated: false,
4361 }],
4362 ignored_closed: vec![],
4363 usage: Usage::default(),
4364 };
4365 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
4366 assert!(text.contains("design needs no further work"), "{text}");
4367 }
4368
4369 #[test]
4373 fn render_route_text_switches_to_one_line_per_stage_for_multi_model_tiers() {
4374 let multi = "global.anthropic.claude-sonnet-4-6,global.anthropic.claude-sonnet-5";
4375 let report = RouteReport {
4376 model: "jev-1.13.0".to_string(),
4377 issues: vec![IssueRoute {
4378 draft_comments: vec![],
4379 item_ref: "rust-works/omni-dev#1832".to_string(),
4380 url: "u".to_string(),
4381 title: "feat(drive): banded ranges for drive sheets (#1830)".to_string(),
4382 state: ItemState::Open,
4383 outcome: RouteOutcome::Routed {
4384 open_questions: Some(test_open_questions()),
4385 providers: BTreeMap::from([(
4386 "anthropic".to_string(),
4387 ProviderRoute {
4388 stages: StageAnswers {
4389 design: answer(NO_DESIGN, 0.94),
4390 implement: answer(multi, 0.94),
4391 review: answer(multi, 0.70),
4392 },
4393 class: multi.to_string(),
4394 class_from: ClassSource::Implement,
4395 close_calls: vec![],
4396 },
4397 )]),
4398 depends_on: vec![DependencyEntry {
4399 item_ref: "#1830".to_string(),
4400 url: None,
4401 state: ItemState::Open,
4402 relation: CitationRelation::Unspecified,
4403 could_be_cheaper: BTreeMap::from([("design".to_string(), 0.48)]),
4404 }],
4405 reference_fetch_failures: vec![],
4406 },
4407 truncated: false,
4408 }],
4409 ignored_closed: vec![],
4410 usage: Usage::default(),
4411 };
4412 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
4413 assert_eq!(
4414 text,
4415 format!(
4416 "rust-works/omni-dev#1832 — feat(drive): banded ranges for drive sheets (#1830)\n\
4417 \x20\x20open_questions: factual (0.80)\n\
4418 \x20\x20class: {multi} (from implementation)\n\
4419 \x20\x20design: needs no further work (0.94)\n\
4420 \x20\x20implementation: {multi} (0.94)\n\
4421 \x20\x20review: {multi} (0.70)\n\
4422 \x20\x20cites open #1830 (relation unspecified), which could leave less design work if resolved (0.48)\n\n\
4423 model: jev-1.13.0, usage: 0 input tokens, 0 output tokens\n"
4424 )
4425 );
4426 }
4427
4428 #[test]
4431 fn render_route_text_keeps_the_compact_line_for_single_word_tiers() {
4432 let report = RouteReport {
4433 model: "jev-1.13.0".to_string(),
4434 issues: vec![IssueRoute {
4435 draft_comments: vec![],
4436 item_ref: "o/r#1".to_string(),
4437 url: "u".to_string(),
4438 title: "t".to_string(),
4439 state: ItemState::Open,
4440 outcome: RouteOutcome::Routed {
4441 open_questions: Some(test_open_questions()),
4442 providers: BTreeMap::from([(
4443 "anthropic".to_string(),
4444 ProviderRoute {
4445 stages: stages("opus", "sonnet", "opus"),
4446 class: "opus".to_string(),
4447 class_from: ClassSource::Design,
4448 close_calls: vec![],
4449 },
4450 )]),
4451 depends_on: vec![],
4452 reference_fetch_failures: vec![],
4453 },
4454 truncated: false,
4455 }],
4456 ignored_closed: vec![],
4457 usage: Usage::default(),
4458 };
4459 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
4460 assert!(
4461 text.contains(
4462 " opus (from design) — design needs opus (0.90), implementation sonnet (0.90), \
4463 review opus (0.90)"
4464 ),
4465 "{text}"
4466 );
4467 assert!(!text.contains("class:"), "{text}");
4468 }
4469
4470 #[test]
4473 fn render_route_text_headers_the_block_by_provider_when_multi_model_and_multiple_ladders() {
4474 let route = |class: &str| ProviderRoute {
4475 stages: StageAnswers {
4476 design: answer(NO_DESIGN, 0.9),
4477 implement: answer(class, 0.9),
4478 review: answer(class, 0.9),
4479 },
4480 class: class.to_string(),
4481 class_from: ClassSource::Implement,
4482 close_calls: vec![],
4483 };
4484 let report = RouteReport {
4485 model: "jev-1.13.0".to_string(),
4486 issues: vec![IssueRoute {
4487 draft_comments: vec![],
4488 item_ref: "o/r#1".to_string(),
4489 url: "u".to_string(),
4490 title: "t".to_string(),
4491 state: ItemState::Open,
4492 outcome: RouteOutcome::Routed {
4493 open_questions: Some(test_open_questions()),
4494 providers: BTreeMap::from([
4495 ("anthropic".to_string(), route("a,b")),
4496 ("openai".to_string(), route("c,d")),
4497 ]),
4498 depends_on: vec![],
4499 reference_fetch_failures: vec![],
4500 },
4501 truncated: false,
4502 }],
4503 ignored_closed: vec![],
4504 usage: Usage::default(),
4505 };
4506 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
4507 assert!(
4508 text.contains(" anthropic:\n class: a,b (from implementation)\n"),
4509 "{text}"
4510 );
4511 assert!(text.contains(" implementation: a,b (0.90)"), "{text}");
4512 assert!(
4513 text.contains(" openai:\n class: c,d (from implementation)\n"),
4514 "{text}"
4515 );
4516 assert!(text.contains(" implementation: c,d (0.90)"), "{text}");
4517 }
4518
4519 #[test]
4523 fn render_route_text_multi_model_layout_is_decided_per_ladder() {
4524 let multi_route = ProviderRoute {
4525 stages: StageAnswers {
4526 design: answer(NO_DESIGN, 0.9),
4527 implement: answer("a,b", 0.9),
4528 review: answer("a,b", 0.9),
4529 },
4530 class: "a,b".to_string(),
4531 class_from: ClassSource::Implement,
4532 close_calls: vec![],
4533 };
4534 let compact_route = ProviderRoute {
4535 stages: stages("none", "terra", "terra"),
4536 class: "terra".to_string(),
4537 class_from: ClassSource::Implement,
4538 close_calls: vec![],
4539 };
4540 let report = RouteReport {
4541 model: "jev-1.13.0".to_string(),
4542 issues: vec![IssueRoute {
4543 draft_comments: vec![],
4544 item_ref: "o/r#1".to_string(),
4545 url: "u".to_string(),
4546 title: "t".to_string(),
4547 state: ItemState::Open,
4548 outcome: RouteOutcome::Routed {
4549 open_questions: Some(test_open_questions()),
4550 providers: BTreeMap::from([
4551 ("anthropic".to_string(), multi_route),
4552 ("openai".to_string(), compact_route),
4553 ]),
4554 depends_on: vec![],
4555 reference_fetch_failures: vec![],
4556 },
4557 truncated: false,
4558 }],
4559 ignored_closed: vec![],
4560 usage: Usage::default(),
4561 };
4562 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
4563 assert!(
4564 text.contains(" anthropic:\n class: a,b (from implementation)\n"),
4565 "{text}"
4566 );
4567 assert!(
4568 text.contains(" openai: terra (from implementation) — design needs no further work"),
4569 "{text}"
4570 );
4571 }
4572
4573 #[test]
4575 fn render_route_text_notes_a_close_call_in_the_multi_line_layout() {
4576 let mut review = answer("a,b", 0.41);
4577 review.probabilities =
4578 BTreeMap::from([("a,b".to_string(), 0.56), ("c,d".to_string(), 0.44)]);
4579 let report = RouteReport {
4580 model: "jev-1.13.0".to_string(),
4581 issues: vec![IssueRoute {
4582 draft_comments: vec![],
4583 item_ref: "o/r#1".to_string(),
4584 url: "u".to_string(),
4585 title: "t".to_string(),
4586 state: ItemState::Open,
4587 outcome: RouteOutcome::Routed {
4588 open_questions: Some(test_open_questions()),
4589 providers: BTreeMap::from([(
4590 "anthropic".to_string(),
4591 ProviderRoute {
4592 stages: StageAnswers {
4593 design: answer(NO_DESIGN, 0.9),
4594 implement: answer("a,b", 0.94),
4595 review,
4596 },
4597 class: "a,b".to_string(),
4598 class_from: ClassSource::Implement,
4599 close_calls: vec![Stage::Review],
4600 },
4601 )]),
4602 depends_on: vec![],
4603 reference_fetch_failures: vec![],
4604 },
4605 truncated: false,
4606 }],
4607 ignored_closed: vec![],
4608 usage: Usage::default(),
4609 };
4610 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
4611 assert!(
4612 text.contains("review: a,b (0.41, close call — c,d 0.44)"),
4613 "{text}"
4614 );
4615 }
4616
4617 #[test]
4618 fn render_route_text_separates_multiple_issues_with_a_blank_line_in_request_order() {
4619 let issue = |item_ref: &str| IssueRoute {
4620 draft_comments: vec![],
4621 item_ref: item_ref.to_string(),
4622 url: "u".to_string(),
4623 title: "t".to_string(),
4624 state: ItemState::Open,
4625 outcome: RouteOutcome::Failed {
4626 error: "e".to_string(),
4627 reference_fetch_failures: vec![],
4628 },
4629 truncated: false,
4630 };
4631 let report = RouteReport {
4632 model: "jev-1.13.0".to_string(),
4633 issues: vec![issue("o/r#1"), issue("o/r#2")],
4634 ignored_closed: vec![],
4635 usage: Usage::default(),
4636 };
4637 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
4638 let first = text.find("o/r#1").unwrap();
4639 let second = text.find("o/r#2").unwrap();
4640 assert!(first < second, "{text}");
4641 assert!(
4642 text.contains("o/r#1 — t\n failed: e\n\no/r#2 — t\n failed: e"),
4643 "{text}"
4644 );
4645 }
4646
4647 #[test]
4648 fn render_route_text_ends_with_the_model_and_summed_usage() {
4649 let report = RouteReport {
4650 model: "jev-1.13.0".to_string(),
4651 issues: vec![],
4652 ignored_closed: vec![],
4653 usage: Usage {
4654 input_tokens: 10,
4655 output_tokens: 20,
4656 },
4657 };
4658 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
4659 assert_eq!(
4660 text,
4661 "model: jev-1.13.0, usage: 10 input tokens, 20 output tokens\n"
4662 );
4663 }
4664
4665 #[test]
4666 fn terminal_style_requires_a_usable_tty() {
4667 assert_eq!(
4668 TerminalStyle::new(false, "xterm", false, true),
4669 TerminalStyle::default()
4670 );
4671 assert_eq!(
4672 TerminalStyle::new(true, "dumb", false, true),
4673 TerminalStyle::default()
4674 );
4675 assert_eq!(
4676 TerminalStyle::new(true, "", false, true),
4677 TerminalStyle::default()
4678 );
4679 assert_eq!(
4680 TerminalStyle::new(true, "xterm", true, true),
4681 TerminalStyle {
4682 color: false,
4683 hyperlinks: true
4684 }
4685 );
4686 assert_eq!(
4687 TerminalStyle::new(true, "xterm", false, false),
4688 TerminalStyle {
4689 color: true,
4690 hyperlinks: false
4691 }
4692 );
4693 }
4694
4695 #[test]
4696 fn complexity_uses_tier_order_for_built_in_and_custom_ladders() {
4697 let anthropic = class_only(Provider::Anthropic).unwrap();
4698 assert_eq!(
4699 choice_complexity(NO_DESIGN, &anthropic),
4700 Some(Complexity::Low)
4701 );
4702 assert_eq!(
4703 choice_complexity("sonnet", &anthropic),
4704 Some(Complexity::Low)
4705 );
4706 assert_eq!(
4707 choice_complexity("opus", &anthropic),
4708 Some(Complexity::High)
4709 );
4710 let custom = |names: &[&str]| {
4711 use std::fmt::Write;
4712
4713 let mut yaml = String::from("tiers:\n");
4714 for name in names {
4715 writeln!(yaml, " - {{name: {name}, description: work}}").unwrap();
4716 }
4717 Ladder::named("custom".to_string(), Tiers::parse(&yaml).unwrap())
4718 };
4719 let two = custom(&["a", "b"]);
4720 assert_eq!(choice_complexity("a", &two), Some(Complexity::Low));
4721 assert_eq!(choice_complexity("b", &two), Some(Complexity::High));
4722 let four = custom(&["a", "b", "c", "d"]);
4723 assert_eq!(choice_complexity("b", &four), Some(Complexity::Medium));
4724 assert_eq!(choice_complexity("c", &four), Some(Complexity::Medium));
4725 assert_eq!(choice_complexity("d", &four), Some(Complexity::High));
4726 assert_eq!(choice_complexity("unknown", &four), None);
4727 }
4728
4729 #[test]
4730 fn hyperlink_uses_exact_osc_8_framing_and_rejects_unsafe_urls() {
4731 let style = TerminalStyle {
4732 color: false,
4733 hyperlinks: true,
4734 };
4735 let url = "https://github.com/other/repo/pull/42";
4736 assert_eq!(
4737 hyperlink("PR #42", Some(url), style),
4738 format!("\x1b]8;;{url}\x1b\\PR #42\x1b]8;;\x1b\\")
4739 );
4740 for bad in [
4741 "https://example.com/o/r/issues/1",
4742 "http://github.com/o/r/issues/1",
4743 "https://github.com/o/r/issues/1\x1b]8;;evil",
4744 "https://github.com/o/r/issues/nope",
4745 "not a valid url",
4746 ] {
4747 assert_eq!(hyperlink("#1", Some(bad), style), "#1");
4748 }
4749 assert_eq!(hyperlink("#1", None, style), "#1");
4750 }
4751
4752 #[test]
4753 fn colorize_applies_ansi_codes_per_complexity_when_color_enabled() {
4754 let color = TerminalStyle {
4755 color: true,
4756 hyperlinks: false,
4757 };
4758 assert_eq!(colorize("x", Complexity::Low, color), "\x1b[32mx\x1b[0m");
4759 assert_eq!(colorize("x", Complexity::Medium, color), "\x1b[33mx\x1b[0m");
4760 assert_eq!(colorize("x", Complexity::High, color), "\x1b[31mx\x1b[0m");
4761 assert_eq!(
4762 colorize("x", Complexity::Medium, TerminalStyle::default()),
4763 "x"
4764 );
4765 }
4766
4767 #[test]
4768 fn styled_report_colors_classes_stages_and_failure_and_links_refs() {
4769 let issue = IssueRoute {
4770 draft_comments: vec![],
4771 item_ref: "o/r#1".to_string(),
4772 url: "https://github.com/o/r/issues/1".to_string(),
4773 title: "t".to_string(),
4774 state: ItemState::Open,
4775 outcome: RouteOutcome::Routed {
4776 open_questions: Some(test_open_questions()),
4777 providers: BTreeMap::from([(
4778 "anthropic".to_string(),
4779 ProviderRoute {
4780 stages: stages(NO_DESIGN, "opus", "sonnet"),
4781 class: "opus".to_string(),
4782 class_from: ClassSource::Implement,
4783 close_calls: vec![],
4784 },
4785 )]),
4786 depends_on: vec![DependencyEntry {
4787 item_ref: "#42".to_string(),
4788 url: Some("https://github.com/other/repo/pull/42".to_string()),
4789 state: ItemState::Open,
4790 relation: CitationRelation::Unspecified,
4791 could_be_cheaper: BTreeMap::new(),
4792 }],
4793 reference_fetch_failures: vec![],
4794 },
4795 truncated: false,
4796 };
4797 let failed = IssueRoute {
4798 draft_comments: vec![],
4799 item_ref: "o/r#2".to_string(),
4800 url: "https://github.com/o/r/issues/2".to_string(),
4801 title: "failed".to_string(),
4802 state: ItemState::Open,
4803 outcome: RouteOutcome::Failed {
4804 error: "HTTP 529".to_string(),
4805 reference_fetch_failures: vec![],
4806 },
4807 truncated: false,
4808 };
4809 let report = RouteReport {
4810 model: "jev".to_string(),
4811 issues: vec![issue, failed],
4812 ignored_closed: vec![],
4813 usage: Usage::default(),
4814 };
4815 let style = TerminalStyle {
4816 color: true,
4817 hyperlinks: true,
4818 };
4819 let text = render_route_text_styled(
4820 &report,
4821 DEFAULT_MAX_INPUT_CHARS,
4822 &[class_only(Provider::Anthropic).unwrap()],
4823 style,
4824 );
4825 assert!(
4826 text.contains(
4827 "\x1b[32m\x1b]8;;https://github.com/o/r/issues/1\x1b\\o/r#1\x1b]8;;\x1b\\\x1b[0m"
4828 ),
4829 "{text}"
4830 );
4831 assert!(
4832 text.contains(
4833 "\x1b[31mopus\x1b[0m (from implementation) — design needs \x1b[32mno further work\x1b[0m"
4834 ),
4835 "{text}"
4836 );
4837 assert!(text.contains("review \x1b[32msonnet\x1b[0m"), "{text}");
4838 assert!(
4839 text.contains("\x1b]8;;https://github.com/other/repo/pull/42\x1b\\#42\x1b]8;;\x1b\\"),
4840 "{text}"
4841 );
4842 assert!(text.contains("\x1b[31mfailed: HTTP 529\x1b[0m"), "{text}");
4843 let plain = render_route_text_styled(
4844 &report,
4845 DEFAULT_MAX_INPUT_CHARS,
4846 &[],
4847 TerminalStyle::default(),
4848 );
4849 assert_eq!(plain, render_route_text(&report, DEFAULT_MAX_INPUT_CHARS));
4850 assert!(!plain.contains('\x1b'));
4851 let json = serde_json::to_string(&report).unwrap();
4852 assert!(!json.contains("https://github.com/other/repo/pull/42"));
4853 }
4854
4855 #[test]
4856 fn styled_multi_model_layout_colors_the_class_and_stage_choices() {
4857 let tier = "model-a,model-b";
4858 let ladder = Ladder::named(
4859 "custom".to_string(),
4860 Tiers::parse(&format!(
4861 "tiers:\n - {{name: basic, description: work}}\n - {{name: '{tier}', description: work}}\n"
4862 ))
4863 .unwrap(),
4864 );
4865 let report = RouteReport {
4866 model: "jev".to_string(),
4867 issues: vec![IssueRoute {
4868 draft_comments: vec![],
4869 item_ref: "o/r#1".to_string(),
4870 url: "https://github.com/o/r/issues/1".to_string(),
4871 title: "t".to_string(),
4872 state: ItemState::Open,
4873 outcome: RouteOutcome::Routed {
4874 open_questions: Some(test_open_questions()),
4875 providers: BTreeMap::from([(
4876 "custom".to_string(),
4877 ProviderRoute {
4878 stages: stages(NO_DESIGN, tier, "basic"),
4879 class: tier.to_string(),
4880 class_from: ClassSource::Implement,
4881 close_calls: vec![],
4882 },
4883 )]),
4884 depends_on: vec![],
4885 reference_fetch_failures: vec![],
4886 },
4887 truncated: false,
4888 }],
4889 ignored_closed: vec![],
4890 usage: Usage::default(),
4891 };
4892 let text = render_route_text_styled(
4893 &report,
4894 DEFAULT_MAX_INPUT_CHARS,
4895 &[ladder],
4896 TerminalStyle {
4897 color: true,
4898 hyperlinks: false,
4899 },
4900 );
4901 assert!(
4902 text.contains(&format!(" class: \x1b[31m{tier}\x1b[0m")),
4903 "{text}"
4904 );
4905 assert!(
4906 text.contains(&format!(" implementation: \x1b[31m{tier}\x1b[0m")),
4907 "{text}"
4908 );
4909 assert!(text.contains(" review: \x1b[32mbasic\x1b[0m"), "{text}");
4910 }
4911
4912 fn mixed_source_report() -> RouteReport {
4915 RouteReport {
4916 model: "jev-1.13.0".to_string(),
4917 issues: vec![IssueRoute {
4918 draft_comments: vec![],
4919 item_ref: "o/r#1".to_string(),
4920 url: "u".to_string(),
4921 title: "t".to_string(),
4922 state: ItemState::Open,
4923 outcome: RouteOutcome::Routed {
4924 open_questions: Some(test_open_questions()),
4925 providers: BTreeMap::from([
4926 (
4927 "anthropic".to_string(),
4928 ProviderRoute {
4929 stages: stages("opus", "sonnet", "sonnet"),
4930 class: "opus".to_string(),
4931 class_from: ClassSource::Design,
4932 close_calls: vec![],
4933 },
4934 ),
4935 (
4936 "openai".to_string(),
4937 ProviderRoute {
4938 stages: stages(NO_DESIGN, "sol", "terra"),
4939 class: "sol".to_string(),
4940 class_from: ClassSource::Implement,
4941 close_calls: vec![],
4942 },
4943 ),
4944 ]),
4945 depends_on: vec![],
4946 reference_fetch_failures: vec![],
4947 },
4948 truncated: false,
4949 }],
4950 ignored_closed: vec![],
4951 usage: Usage::default(),
4952 }
4953 }
4954
4955 #[test]
4956 fn class_from_is_serialised_per_provider_in_json_and_yaml() {
4957 let report = mixed_source_report();
4958 let json = serde_json::to_value(&report).unwrap();
4959 let providers = &json["issues"][0]["providers"];
4960 assert_eq!(providers["anthropic"]["class_from"], "design");
4961 assert_eq!(providers["openai"]["class_from"], "implement");
4962
4963 let yaml = serde_yaml::to_string(&report).unwrap();
4964 assert!(
4965 yaml.contains("class: opus\n class_from: design\n"),
4966 "{yaml}"
4967 );
4968 assert!(
4969 yaml.contains("class: sol\n class_from: implement\n"),
4970 "{yaml}"
4971 );
4972 }
4973
4974 #[test]
4975 fn text_output_names_the_class_source_for_each_ladder() {
4976 let text = render_route_text(&mixed_source_report(), DEFAULT_MAX_INPUT_CHARS);
4977 assert!(
4978 text.contains(" anthropic: opus (from design) — "),
4979 "{text}"
4980 );
4981 assert!(
4982 text.contains(" openai: sol (from implementation) — "),
4983 "{text}"
4984 );
4985 }
4986
4987 #[test]
4988 fn text_output_names_the_class_source_in_the_multi_line_layout() {
4989 let multi = "a,b";
4990 let report = RouteReport {
4991 issues: vec![IssueRoute {
4992 outcome: RouteOutcome::Routed {
4993 open_questions: Some(test_open_questions()),
4994 providers: BTreeMap::from([(
4995 "anthropic".to_string(),
4996 ProviderRoute {
4997 stages: StageAnswers {
4998 design: answer(multi, 0.9),
4999 implement: answer("basic", 0.9),
5000 review: answer("basic", 0.9),
5001 },
5002 class: multi.to_string(),
5003 class_from: ClassSource::Design,
5004 close_calls: vec![],
5005 },
5006 )]),
5007 depends_on: vec![],
5008 reference_fetch_failures: vec![],
5009 },
5010 ..mixed_source_report().issues.remove(0)
5011 }],
5012 ..mixed_source_report()
5013 };
5014 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
5015 assert!(text.contains(" class: a,b (from design)\n"), "{text}");
5016 }
5017
5018 #[test]
5019 fn design_stage_with_a_named_tier_renders_needs_prefix() {
5020 let line = render_stage_line(
5021 Stage::Design,
5022 &stages("opus", "sonnet", "sonnet"),
5023 &[],
5024 None,
5025 TerminalStyle::default(),
5026 );
5027 assert_eq!(line, "design: needs opus (0.90)");
5028 }
5029
5030 #[test]
5031 fn with_thousands_groups_digits() {
5032 assert_eq!(with_thousands(0), "0");
5033 assert_eq!(with_thousands(999), "999");
5034 assert_eq!(with_thousands(60_000), "60,000");
5035 assert_eq!(with_thousands(1_000_000), "1,000,000");
5036 }
5037}