1use std::collections::{BTreeMap, BTreeSet};
16use std::path::Path;
17
18use anyhow::{bail, Context, Result};
19use clap::ValueEnum;
20use serde::{Deserialize, Serialize};
21use tracing::warn;
22
23use crate::jev::citations::{find_citation_mentions, Citation};
24use crate::jev::client::JevClient;
25use crate::jev::error::is_auth_failure;
26use crate::jev::input::truncate_middle;
27use crate::jev::markdown_code::mask_code;
28use crate::jev::protocol::{Answer, Question, SystemOneRequest, Usage};
29use crate::provider::{IssueDoc, ItemRef, ItemState};
30
31pub use crate::jev::input::TRUNCATION_MARKER;
32
33pub mod effort;
34
35const STAGE_QUESTIONS_YAML: &str = include_str!("../templates/jev-route-questions.yaml");
37
38pub const NO_DESIGN: &str = "none";
41
42#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
52pub enum Provider {
53 #[value(name = "anthropic")]
55 Anthropic,
56 #[value(name = "openai")]
58 OpenAi,
59 #[value(name = "gemini")]
62 Gemini,
63}
64
65impl Provider {
66 pub const ALL: [Self; 3] = [Self::Anthropic, Self::OpenAi, Self::Gemini];
68
69 #[must_use]
72 pub const fn name(self) -> &'static str {
73 match self {
74 Self::Anthropic => "anthropic",
75 Self::OpenAi => "openai",
76 Self::Gemini => "gemini",
77 }
78 }
79
80 const fn tiers_yaml(self) -> &'static str {
82 match self {
83 Self::Anthropic => include_str!("../templates/jev-route-tiers-anthropic.yaml"),
84 Self::OpenAi => include_str!("../templates/jev-route-tiers-openai.yaml"),
85 Self::Gemini => include_str!("../templates/jev-route-tiers-gemini.yaml"),
86 }
87 }
88
89 pub fn tiers(self) -> Result<Tiers> {
91 Tiers::parse(self.tiers_yaml())
92 .with_context(|| format!("Invalid embedded tiers for provider {}", self.name()))
93 }
94}
95
96pub const DEFAULT_CLOSE_CALL: f64 = 0.3;
99
100pub const DEFAULT_CLOSE_CALL_MARGIN: f64 = 0.2;
103
104pub const DEFAULT_MAX_INPUT_CHARS: usize = 60_000;
107
108const MIN_TIERS: usize = 2;
111
112pub type OpenDependencies = BTreeMap<(String, u64), Vec<Citation>>;
118
119pub type ReferenceFetchFailures = BTreeMap<(String, u64), Vec<ReferenceFetchFailure>>;
125
126#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
128pub struct ReferenceFetchFailure {
129 #[serde(rename = "ref")]
131 pub item_ref: String,
132 pub error: String,
134}
135
136#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize)]
138#[serde(rename_all = "lowercase")]
139pub enum Stage {
140 Design,
142 Implement,
144 Review,
146}
147
148impl Stage {
149 const ALL: [Self; 3] = [Self::Design, Self::Implement, Self::Review];
151
152 const fn template_key(self) -> &'static str {
154 match self {
155 Self::Design => "stage_design",
156 Self::Implement => "stage_implement",
157 Self::Review => "stage_review",
158 }
159 }
160
161 fn question_key(self, provider: &str) -> String {
164 format!("{provider}.{}", self.template_key())
165 }
166}
167
168#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
170#[serde(deny_unknown_fields)]
171pub struct Tier {
172 pub name: String,
174 pub description: String,
176 pub models: Option<Vec<effort::Model>>,
178}
179
180#[derive(Deserialize)]
182#[serde(deny_unknown_fields)]
183struct TiersFile {
184 tiers: Vec<Tier>,
185}
186
187#[derive(Debug, Clone, PartialEq, Eq)]
189pub struct Tiers(Vec<Tier>);
190
191impl Tiers {
192 pub fn parse(yaml: &str) -> Result<Self> {
195 let file: TiersFile = serde_yaml::from_str(yaml).context("Failed to parse tiers file")?;
196 if file.tiers.len() < MIN_TIERS {
197 bail!(
198 "a tiers file needs at least {MIN_TIERS} tiers, got {}",
199 file.tiers.len()
200 );
201 }
202 let mut seen = BTreeSet::new();
203 for tier in &file.tiers {
204 if tier.name.trim().is_empty() || tier.description.trim().is_empty() {
205 bail!("every tier needs a non-empty name and description");
206 }
207 if tier.name == NO_DESIGN {
208 bail!("the tier name {NO_DESIGN:?} is reserved for \"no design work remains\"");
209 }
210 if !seen.insert(tier.name.as_str()) {
211 bail!("tier {:?} is defined more than once", tier.name);
212 }
213 }
214 let mut model_rungs = BTreeMap::new();
215 for tier in &file.tiers {
216 if let Some(models) = &tier.models {
217 effort::validate(models).with_context(|| format!("rung {:?}", tier.name))?;
218 for model in models {
219 for &stage in &model.stages {
220 if let Some(previous) = model_rungs.insert((&model.name, stage), &tier.name)
221 {
222 bail!(
223 "model {:?} is bound to both rungs {previous:?} and {:?} for {stage:?}",
224 model.name,
225 tier.name
226 );
227 }
228 }
229 }
230 }
231 }
232 Ok(Self(file.tiers))
233 }
234
235 pub fn load_file(path: &Path) -> Result<Self> {
237 let yaml = std::fs::read_to_string(path)
238 .with_context(|| format!("Failed to read tiers file {}", path.display()))?;
239 Self::parse(&yaml).with_context(|| format!("Invalid tiers file {}", path.display()))
240 }
241
242 #[must_use]
244 pub fn as_slice(&self) -> &[Tier] {
245 &self.0
246 }
247
248 fn rank(&self, choice: &str) -> Option<usize> {
251 if choice == NO_DESIGN {
252 return Some(0);
253 }
254 self.0.iter().position(|t| t.name == choice).map(|i| i + 1)
255 }
256}
257
258#[derive(Debug, Clone, PartialEq, Eq)]
264pub struct Ladder {
265 pub name: String,
267 pub tiers: Tiers,
269}
270
271impl Ladder {
272 pub fn builtin(provider: Provider) -> Result<Self> {
274 Ok(Self {
275 name: provider.name().to_string(),
276 tiers: provider.tiers()?,
277 })
278 }
279
280 #[must_use]
282 pub fn named(name: String, tiers: Tiers) -> Self {
283 Self { name, tiers }
284 }
285}
286
287pub fn build_route_questions(ladders: &[Ladder]) -> Result<BTreeMap<String, Question>> {
293 build_route_questions_for_mode(ladders, true)
294}
295
296fn build_route_questions_for_mode(
297 ladders: &[Ladder],
298 effort_advice: bool,
299) -> Result<BTreeMap<String, Question>> {
300 let templates: BTreeMap<String, Question> = serde_yaml::from_str(STAGE_QUESTIONS_YAML)
301 .context("Failed to parse the embedded stage questions")?;
302 for stage in Stage::ALL {
303 if !templates.contains_key(stage.template_key()) {
304 bail!(
305 "the embedded stage questions have no {:?}",
306 stage.template_key()
308 );
309 }
310 }
311 let mut questions = BTreeMap::from([("open_questions".to_string(), open_questions_question())]);
312 for ladder in ladders {
313 for (key, template) in &templates {
314 let mut question = template.clone();
315 let Question::Choice { criteria, .. } = &mut question else {
316 bail!("embedded stage question {key:?} is not a choice question");
318 };
319 for tier in ladder.tiers.as_slice() {
320 if criteria
321 .insert(tier.name.clone(), tier.description.clone())
322 .is_some()
323 {
324 bail!(
326 "tier {:?} of ladder {:?} collides with a fixed option of {key:?}",
327 tier.name,
328 ladder.name
329 );
330 }
331 }
332 question
333 .validate()
334 .with_context(|| format!("stage question {key:?} for {:?}", ladder.name))?;
335 questions.insert(format!("{}.{key}", ladder.name), question);
336 }
337 }
338 if effort_advice {
339 for ladder in ladders {
340 effort::add_questions(ladder, &mut questions);
341 }
342 }
343 Ok(questions)
344}
345
346#[must_use]
362pub fn build_route_state(doc: &IssueDoc, max_chars: usize) -> (String, bool) {
363 build_route_state_with_drafts(doc, max_chars, &[])
364}
365
366#[must_use]
376pub fn build_route_citation_text(doc: &IssueDoc, max_chars: usize) -> String {
377 build_route_citation_text_with_drafts(doc, max_chars, &[])
378}
379
380#[must_use]
382pub fn build_route_state_with_drafts(
383 doc: &IssueDoc,
384 max_chars: usize,
385 drafts: &[String],
386) -> (String, bool) {
387 assemble_route_state(doc, max_chars, drafts, str::to_string)
388}
389
390#[must_use]
392pub fn build_route_citation_text_with_drafts(
393 doc: &IssueDoc,
394 max_chars: usize,
395 drafts: &[String],
396) -> String {
397 assemble_route_state(doc, max_chars, drafts, mask_code).0
398}
399
400fn assemble_route_state(
403 doc: &IssueDoc,
404 max_chars: usize,
405 drafts: &[String],
406 prose: impl Fn(&str) -> String,
407) -> (String, bool) {
408 let mut text = format!(
409 "# #{} {}\n\n{}\n",
410 doc.number,
411 prose(&doc.title),
412 prose(doc.body.trim())
413 );
414 if !doc.comments.is_empty() || !drafts.is_empty() {
415 text.push_str("\n\n---\n\n## Comments on this issue\n");
416 for comment in &doc.comments {
417 text.push_str(&format!(
418 "\n\n**Comment by {}:**\n\n{}\n",
419 comment.author,
420 prose(comment.body.trim())
421 ));
422 }
423 }
424 for draft in drafts {
425 text.push_str(&format!(
426 "\n\n**Comment by draft:**\n\n{}\n",
427 prose(draft.trim())
428 ));
429 }
430 truncate_middle(&text, max_chars)
431}
432
433#[derive(Debug, Clone, PartialEq, Serialize)]
435pub struct StageAnswer {
436 pub choice: String,
438 pub confidence: f64,
440 pub probabilities: BTreeMap<String, f64>,
442 #[serde(skip_serializing_if = "BTreeMap::is_empty")]
444 pub effort_by_model: BTreeMap<String, Vec<effort::ModelEffort>>,
445}
446
447#[derive(Debug, Clone, PartialEq, Serialize)]
449pub struct StageAnswers {
450 pub design: StageAnswer,
452 pub implement: StageAnswer,
454 pub review: StageAnswer,
456}
457
458impl StageAnswers {
459 fn get(&self, stage: Stage) -> &StageAnswer {
460 match stage {
461 Stage::Design => &self.design,
462 Stage::Implement => &self.implement,
463 Stage::Review => &self.review,
464 }
465 }
466}
467
468#[derive(Debug, Clone, PartialEq, Serialize)]
471pub struct DependencyEntry {
472 #[serde(rename = "ref")]
476 pub item_ref: String,
477 #[serde(skip)]
479 pub url: Option<String>,
480 pub state: ItemState,
483 pub relation: CitationRelation,
486 pub could_be_cheaper: BTreeMap<String, f64>,
491}
492
493#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
495#[serde(rename_all = "lowercase")]
496pub enum CitationRelation {
497 Blocker,
499 Tracker,
501 Unspecified,
503}
504
505#[derive(Debug, Clone, PartialEq, Serialize)]
507pub struct IssueRoute {
508 #[serde(skip_serializing_if = "Vec::is_empty")]
510 pub draft_comments: Vec<std::path::PathBuf>,
511 #[serde(rename = "ref")]
513 pub item_ref: String,
514 pub url: String,
516 pub title: String,
518 pub state: ItemState,
522 #[serde(flatten)]
524 pub outcome: RouteOutcome,
525 #[serde(skip_serializing_if = "std::ops::Not::not")]
527 pub truncated: bool,
528}
529
530#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
535#[serde(rename_all = "lowercase")]
536pub enum ClassSource {
537 Design,
539 Implement,
541}
542
543impl ClassSource {
544 const fn label(self) -> &'static str {
546 match self {
547 Self::Design => "design",
548 Self::Implement => "implementation",
549 }
550 }
551}
552
553#[derive(Debug, Clone, PartialEq, Serialize)]
555pub struct ProviderRoute {
556 pub stages: StageAnswers,
558 pub class: String,
560 pub class_from: ClassSource,
563 pub close_calls: Vec<Stage>,
565}
566
567#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
569#[serde(rename_all = "snake_case")]
570pub enum OpenQuestionKind {
571 None,
573 Factual,
575 Design,
577 Both,
579}
580
581impl OpenQuestionKind {
582 fn label(self) -> &'static str {
583 match self {
584 Self::None => "none",
585 Self::Factual => "factual",
586 Self::Design => "design",
587 Self::Both => "both",
588 }
589 }
590
591 fn parse(label: &str) -> Result<Self> {
592 match label {
593 "none" => Ok(Self::None),
594 "factual" => Ok(Self::Factual),
595 "design" => Ok(Self::Design),
596 "both" => Ok(Self::Both),
597 _ => bail!("open_questions label {label:?} was not offered"),
598 }
599 }
600}
601
602#[derive(Debug, Clone, PartialEq, Serialize)]
604pub struct OpenQuestions {
605 pub choice: OpenQuestionKind,
607 pub confidence: f64,
609 pub probabilities: BTreeMap<String, f64>,
611}
612
613fn open_questions_question() -> Question {
614 Question::Choice {
615 instructions: "What kind of questions is still open in the issue after considering all \
616 comments? Unknown existing-code facts and genuine design tradeoffs are different \
617 kinds. A suggested fix direction does not settle unverified factual premises, \
618 such as whether a reusable helper already exists. A code-informed decision with \
619 a concrete plan does settle them: carrying out its prescribed caller audits, \
620 code edits, tests, or verification is execution work, not an open question. \
621 Routine coding choices do not count as design tradeoffs. Judge the remaining \
622 uncertainty, not the amount of work."
623 .to_string(),
624 criteria: BTreeMap::from([
625 (
626 "none".to_string(),
627 "The latest text establishes the relevant existing-code facts and prescribes the \
628 approach. Executing its specified plan, including audits and verification, does \
629 not itself create open questions."
630 .to_string(),
631 ),
632 (
633 "factual".to_string(),
634 "Only existing-code facts remain unsettled: whether a reusable helper exists, \
635 what it does, or which existing sites need changes. A suggested fix with \
636 uncertain reuse or scope is factual even if its intended behavior is clear; \
637 inspecting code can settle it without a discretionary design tradeoff."
638 .to_string(),
639 ),
640 (
641 "design".to_string(),
642 "Only genuine choices between competing approaches remain, requiring judgment \
643 about behavior, architecture, or tradeoffs even after existing-code facts are \
644 known. Routine choices while implementing a prescribed fix do not qualify."
645 .to_string(),
646 ),
647 (
648 "both".to_string(),
649 "Both unresolved existing-code premises and genuine choices between competing \
650 designs remain. Code reading alone cannot settle the design tradeoffs."
651 .to_string(),
652 ),
653 ]),
654 }
655}
656
657fn open_questions_answer(answers: &BTreeMap<String, Answer>) -> Result<OpenQuestions> {
658 let Some(Answer::Choice {
659 choice,
660 confidence,
661 probabilities,
662 }) = answers.get("open_questions")
663 else {
664 bail!("no choice answer for open_questions");
665 };
666 let choice = OpenQuestionKind::parse(choice)?;
667 for label in probabilities.keys() {
668 OpenQuestionKind::parse(label)?;
669 }
670 Ok(OpenQuestions {
671 choice,
672 confidence: *confidence,
673 probabilities: probabilities.clone(),
674 })
675}
676
677#[derive(Debug, Clone, PartialEq, Serialize)]
683#[serde(untagged)]
684pub enum RouteOutcome {
685 Routed {
687 open_questions: OpenQuestions,
689 providers: BTreeMap<String, ProviderRoute>,
691 depends_on: Vec<DependencyEntry>,
694 #[serde(skip_serializing_if = "Vec::is_empty")]
697 reference_fetch_failures: Vec<ReferenceFetchFailure>,
698 },
699 Failed {
702 error: String,
704 #[serde(skip_serializing_if = "Vec::is_empty")]
706 reference_fetch_failures: Vec<ReferenceFetchFailure>,
707 },
708}
709
710impl IssueRoute {
711 #[must_use]
713 pub fn failed(&self) -> bool {
714 matches!(self.outcome, RouteOutcome::Failed { .. })
715 }
716}
717
718#[derive(Debug, Clone, PartialEq, Serialize)]
720pub struct RouteReport {
721 pub model: String,
724 pub issues: Vec<IssueRoute>,
726 #[serde(skip_serializing_if = "Vec::is_empty")]
730 pub ignored_closed: Vec<IgnoredIssue>,
731 pub usage: Usage,
733}
734
735#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
738pub struct IgnoredIssue {
739 #[serde(rename = "ref")]
741 pub item_ref: String,
742 pub url: String,
744 pub title: String,
746}
747
748#[derive(Debug, Clone)]
750pub struct RouteOptions {
751 pub draft_comments: Vec<String>,
753 pub model: String,
755 pub close_call: f64,
757 pub close_call_margin: f64,
760 pub effort_advice: bool,
762 pub max_input_chars: usize,
764 pub allow_closed: bool,
766 pub ignore_closed: bool,
770}
771
772pub async fn run_route(
796 jev: &JevClient,
797 docs: &[IssueDoc],
798 ladders: &[Ladder],
799 opts: &RouteOptions,
800 dependencies: &OpenDependencies,
801) -> Result<RouteReport> {
802 run_route_with_reference_fetch_failures(
803 jev,
804 docs,
805 ladders,
806 opts,
807 dependencies,
808 &ReferenceFetchFailures::new(),
809 )
810 .await
811}
812
813pub async fn run_route_with_reference_fetch_failures(
816 jev: &JevClient,
817 docs: &[IssueDoc],
818 ladders: &[Ladder],
819 opts: &RouteOptions,
820 dependencies: &OpenDependencies,
821 reference_fetch_failures: &ReferenceFetchFailures,
822) -> Result<RouteReport> {
823 if !opts.draft_comments.is_empty() && docs.len() != 1 {
824 bail!("draft comments apply to exactly one issue; cannot preview a batch");
825 }
826 let (docs, ignored_closed) = drop_closed(docs, opts)?;
827 validate_options(&docs, ladders, opts)?;
828 let questions = build_route_questions_for_mode(ladders, opts.effort_advice)?;
829 for (key, question) in &questions {
832 question
833 .validate_for_model(&opts.model)
834 .with_context(|| format!("route question {key:?}"))?;
835 }
836
837 let mut models = BTreeSet::new();
838 let mut usage = Usage::default();
839 let mut issues = Vec::with_capacity(docs.len());
840 for doc in docs {
841 let item_ref = item_ref(doc);
842 let (state, truncated) =
843 build_route_state_with_drafts(doc, opts.max_input_chars, &opts.draft_comments);
844 if truncated {
845 warn!(
846 "Issue {item_ref} is longer than {} characters; its input was truncated",
847 opts.max_input_chars
848 );
849 }
850 let citations = dependencies
851 .get(&(doc.project.clone(), doc.number))
852 .map_or(&[][..], Vec::as_slice);
853 let relations = citation_relations(&state, doc);
854 let citation_failures = reference_fetch_failures
855 .get(&(doc.project.clone(), doc.number))
856 .cloned()
857 .unwrap_or_default();
858 let mut request_questions = questions.clone();
859 for (i, citation) in citations.iter().enumerate() {
860 request_questions.insert(
861 could_be_cheaper_key(i),
862 could_be_cheaper_question(&citation.raw),
863 );
864 }
865 let request = SystemOneRequest {
866 state: serde_json::Value::String(state),
867 model: opts.model.clone(),
868 questions: request_questions,
869 };
870 let outcome = match jev.system_one(&request).await {
871 Err(err) if is_auth_failure(&err) => {
872 return Err(err).with_context(|| format!("Failed to route issue {item_ref}"));
873 }
874 Err(err) => failed(
875 &item_ref,
876 &err.context("Jev request failed"),
877 &citation_failures,
878 ),
879 Ok(response) => {
880 models.insert(response.model);
881 usage.input_tokens += response.usage.input_tokens;
882 usage.output_tokens += response.usage.output_tokens;
883 match open_questions_answer(&response.answers).and_then(|open_questions| {
884 provider_routes(
885 &response.answers,
886 ladders,
887 opts.close_call,
888 opts.close_call_margin,
889 opts.effort_advice,
890 )
891 .map(|providers| (open_questions, providers))
892 }) {
893 Ok((open_questions, providers)) => RouteOutcome::Routed {
894 open_questions,
895 providers,
896 depends_on: dependency_entries(
897 &item_ref,
898 citations,
899 &relations,
900 &response.answers,
901 ),
902 reference_fetch_failures: citation_failures.clone(),
903 },
904 Err(err) => failed(
905 &item_ref,
906 &err.context("Unexpected Jev answer"),
907 &citation_failures,
908 ),
909 }
910 }
911 };
912 issues.push(IssueRoute {
913 draft_comments: vec![],
914 item_ref,
915 url: doc.url.clone(),
916 title: doc.title.clone(),
917 state: doc.state,
918 outcome,
919 truncated,
920 });
921 }
922
923 Ok(RouteReport {
924 model: models.into_iter().collect::<Vec<_>>().join(", "),
925 issues,
926 ignored_closed,
927 usage,
928 })
929}
930
931#[must_use]
934pub fn is_ignored_closed(doc: &IssueDoc, ignore_closed: bool) -> bool {
935 ignore_closed && doc.state == ItemState::Closed
936}
937
938fn item_ref(doc: &IssueDoc) -> String {
940 format!("{}#{}", doc.project, doc.number)
941}
942
943fn drop_closed<'a>(
950 docs: &'a [IssueDoc],
951 opts: &RouteOptions,
952) -> Result<(Vec<&'a IssueDoc>, Vec<IgnoredIssue>)> {
953 if opts.allow_closed && opts.ignore_closed {
954 bail!("--allow-closed and --ignore-closed cannot be used together");
955 }
956 let (open, closed): (Vec<&IssueDoc>, Vec<&IssueDoc>) = docs
957 .iter()
958 .partition(|d| !is_ignored_closed(d, opts.ignore_closed));
959 if open.is_empty() && !closed.is_empty() {
960 bail!(
961 "every requested issue is closed ({}) and --ignore-closed skipped them; nothing to \
962 route",
963 closed
964 .iter()
965 .map(|d| item_ref(d))
966 .collect::<Vec<_>>()
967 .join(", ")
968 );
969 }
970 let ignored = closed
971 .into_iter()
972 .map(|d| IgnoredIssue {
973 item_ref: item_ref(d),
974 url: d.url.clone(),
975 title: d.title.clone(),
976 })
977 .collect();
978 Ok((open, ignored))
979}
980
981fn failed(
984 item_ref: &str,
985 err: &anyhow::Error,
986 reference_fetch_failures: &[ReferenceFetchFailure],
987) -> RouteOutcome {
988 let error = format!("{err:#}");
989 warn!("Could not route issue {item_ref}: {error}");
990 RouteOutcome::Failed {
991 error,
992 reference_fetch_failures: reference_fetch_failures.to_vec(),
993 }
994}
995
996fn validate_options(docs: &[&IssueDoc], ladders: &[Ladder], opts: &RouteOptions) -> Result<()> {
999 if docs.is_empty() {
1000 bail!("no issues to route");
1001 }
1002 if ladders.is_empty() {
1003 bail!("no ladders to route against");
1004 }
1005 let mut seen = BTreeSet::new();
1006 for ladder in ladders {
1007 if !seen.insert(ladder.name.as_str()) {
1008 bail!("ladder {:?} is listed more than once", ladder.name);
1009 }
1010 }
1011 if !(0.0..=1.0).contains(&opts.close_call) {
1012 bail!(
1013 "the close-call threshold must be between 0 and 1, got {}",
1014 opts.close_call
1015 );
1016 }
1017 if !(0.0..=1.0).contains(&opts.close_call_margin) {
1018 bail!(
1019 "the close-call margin must be between 0 and 1, got {}",
1020 opts.close_call_margin
1021 );
1022 }
1023 if opts.max_input_chars == 0 {
1024 bail!("the input cap must be at least 1 character");
1025 }
1026 if !opts.allow_closed {
1027 let closed: Vec<String> = docs
1028 .iter()
1029 .filter(|d| d.state == ItemState::Closed)
1030 .map(|d| item_ref(d))
1031 .collect();
1032 if !closed.is_empty() {
1033 bail!(
1034 "refusing to route closed issues ({}): their comments often describe how the \
1035 work was done, which leaks the answer. Pass --ignore-closed to skip them, or \
1036 --allow-closed to route them anyway",
1037 closed.join(", ")
1038 );
1039 }
1040 }
1041 Ok(())
1042}
1043
1044fn provider_routes(
1048 answers: &BTreeMap<String, Answer>,
1049 ladders: &[Ladder],
1050 close_call: f64,
1051 close_call_margin: f64,
1052 effort_advice: bool,
1053) -> Result<BTreeMap<String, ProviderRoute>> {
1054 ladders
1055 .iter()
1056 .map(|ladder| {
1057 let mut stages = stage_answers(answers, ladder)?;
1058 if effort_advice {
1059 for (stage, answer) in [
1060 (Stage::Design, &mut stages.design),
1061 (Stage::Implement, &mut stages.implement),
1062 (Stage::Review, &mut stages.review),
1063 ] {
1064 answer.effort_by_model =
1065 effort::decode(ladder, stage, answer, answers, close_call)?;
1066 }
1067 }
1068 let (class, class_from) = issue_class(&stages, &ladder.tiers);
1069 let route = ProviderRoute {
1070 class,
1071 class_from,
1072 close_calls: close_calls(&stages, &ladder.tiers, close_call, close_call_margin),
1073 stages,
1074 };
1075 Ok((ladder.name.clone(), route))
1076 })
1077 .collect()
1078}
1079
1080fn stage_answers(answers: &BTreeMap<String, Answer>, ladder: &Ladder) -> Result<StageAnswers> {
1083 let tiers = &ladder.tiers;
1084 let read = |stage: Stage| -> Result<StageAnswer> {
1085 let key = stage.question_key(&ladder.name);
1086 let Some(Answer::Choice {
1087 choice,
1088 confidence,
1089 probabilities,
1090 }) = answers.get(&key)
1091 else {
1092 bail!("no choice answer for {key:?}");
1093 };
1094 let offered = match tiers.rank(choice) {
1095 Some(0) => stage == Stage::Design,
1096 Some(_) => true,
1097 None => false,
1098 };
1099 if !offered {
1100 bail!("{key:?} chose {choice:?}, which was not offered");
1101 }
1102 Ok(StageAnswer {
1103 choice: choice.clone(),
1104 confidence: *confidence,
1105 probabilities: probabilities.clone(),
1106 effort_by_model: BTreeMap::new(),
1107 })
1108 };
1109 Ok(StageAnswers {
1110 design: read(Stage::Design)?,
1111 implement: read(Stage::Implement)?,
1112 review: read(Stage::Review)?,
1113 })
1114}
1115
1116fn issue_class(stages: &StageAnswers, tiers: &Tiers) -> (String, ClassSource) {
1124 let rank = |a: &StageAnswer| tiers.rank(&a.choice).unwrap_or_default();
1125 if rank(&stages.design) > rank(&stages.implement) {
1126 (stages.design.choice.clone(), ClassSource::Design)
1127 } else {
1128 (stages.implement.choice.clone(), ClassSource::Implement)
1129 }
1130}
1131
1132fn top_two_margin(answer: &StageAnswer, stage: Stage, tiers: &Tiers) -> Option<f64> {
1135 let mut usable: Vec<_> = answer
1136 .probabilities
1137 .iter()
1138 .filter(|(name, probability)| {
1139 probability.is_finite()
1140 && (0.0..=1.0).contains(*probability)
1141 && match tiers.rank(name) {
1142 Some(0) => stage == Stage::Design,
1143 Some(_) => true,
1144 None => false,
1145 }
1146 })
1147 .collect();
1148 usable.sort_by(|a, b| b.1.total_cmp(a.1).then_with(|| a.0.cmp(b.0)));
1149 let [first, second, ..] = usable.as_slice() else {
1150 return None;
1151 };
1152 Some(first.1 - second.1)
1153}
1154
1155fn close_calls(stages: &StageAnswers, tiers: &Tiers, threshold: f64, margin: f64) -> Vec<Stage> {
1158 Stage::ALL
1159 .into_iter()
1160 .filter(|&stage| {
1161 let answer = stages.get(stage);
1162 answer.confidence < threshold
1163 || top_two_margin(answer, stage, tiers).is_some_and(|gap| gap < margin)
1164 })
1165 .collect()
1166}
1167
1168fn could_be_cheaper_key(i: usize) -> String {
1170 format!("could_be_cheaper_{i}")
1171}
1172
1173pub fn could_be_cheaper_question(citation: &str) -> Question {
1180 Question::Noul {
1181 instructions: format!(
1182 "This issue cites {citation}, which is still open. If {citation} is resolved, how \
1183 likely is it that LESS design work would remain for THIS issue than the current \
1184 text implies — as opposed to this issue's own remaining work being unaffected, \
1185 because it is already scoped separately, is a parallel/sibling effort, or \
1186 {citation} is otherwise not a precondition for finishing this issue's own \
1187 remaining work?"
1188 ),
1189 criteria: None,
1190 }
1191}
1192
1193fn citation_relations(state: &str, doc: &IssueDoc) -> BTreeMap<(String, u64), CitationRelation> {
1197 let judged = ItemRef {
1198 provider: doc.provider,
1199 project: doc.project.clone(),
1200 kind: doc.kind,
1201 number: doc.number,
1202 };
1203 let mut evidence: BTreeMap<(String, u64), (bool, bool, bool)> = BTreeMap::new();
1204 for mention in find_citation_mentions(state, &doc.project, &judged) {
1205 let key = (
1206 mention.citation.item_ref.project,
1207 mention.citation.item_ref.number,
1208 );
1209 let entry = evidence.entry(key).or_default();
1210 match relation_at(state, &mention.range) {
1211 MentionRelation::Blocker => entry.0 = true,
1212 MentionRelation::Tracker => entry.1 = true,
1213 MentionRelation::Negated => entry.2 = true,
1214 MentionRelation::None => {}
1215 }
1216 }
1217 evidence
1218 .into_iter()
1219 .map(|(key, (blocker, tracker, negated))| {
1220 let relation = match (blocker, tracker, negated) {
1221 (true, false, false) => CitationRelation::Blocker,
1222 (false, true, false) => CitationRelation::Tracker,
1223 _ => CitationRelation::Unspecified,
1224 };
1225 (key, relation)
1226 })
1227 .collect()
1228}
1229
1230enum MentionRelation {
1231 Blocker,
1232 Tracker,
1233 Negated,
1234 None,
1235}
1236
1237fn relation_at(state: &str, range: &std::ops::Range<usize>) -> MentionRelation {
1238 if !state[..range.start]
1239 .matches("```")
1240 .count()
1241 .is_multiple_of(2)
1242 {
1243 return MentionRelation::None;
1244 }
1245 let before = &state[..range.start];
1246 let boundary = before.rfind(['.', '!', '?', '\n']).map_or(0, |i| i + 1);
1247 let sentence = &state[boundary..];
1248 if sentence.trim_start().starts_with('>') {
1249 return MentionRelation::None;
1250 }
1251 let prefix = &state[boundary..range.start];
1252 if !prefix.matches('`').count().is_multiple_of(2)
1253 || !prefix.matches('"').count().is_multiple_of(2)
1254 {
1255 return MentionRelation::None;
1256 }
1257 let nearby: String = prefix
1258 .chars()
1259 .rev()
1260 .take(160)
1261 .collect::<String>()
1262 .chars()
1263 .rev()
1264 .collect();
1265 let nearby = nearby.to_ascii_lowercase();
1266 let after = &state[range.end..];
1267 let end = after.find(['.', '!', '?', '\n']).unwrap_or(after.len());
1268 let following = after[..end].trim_start().to_ascii_lowercase();
1269 if nearby.contains("such as")
1270 || nearby.contains("for example")
1271 || nearby.contains("worked example")
1272 || nearby.contains("e.g.")
1273 {
1274 return MentionRelation::None;
1275 }
1276 if nearby.contains("not blocked on")
1277 || nearby.contains("no longer blocked on")
1278 || nearby.contains("isn't blocked on")
1279 || nearby.contains("not dependent on")
1280 || nearby.contains("unblocked")
1281 || nearby.contains("not a tracker")
1282 || nearby.contains("not split out of")
1283 || following.starts_with("does not block")
1284 || following.starts_with("no longer blocks")
1285 {
1286 return MentionRelation::Negated;
1287 }
1288 if following.starts_with("blocks ")
1289 || following.starts_with("is required for ")
1290 || following.starts_with("must land before ")
1291 {
1292 return MentionRelation::Blocker;
1293 }
1294 if nearby.contains("blocked on")
1295 || nearby.contains("depends on")
1296 || nearby.contains("requires")
1297 || nearby.contains("prerequisite")
1298 {
1299 return MentionRelation::Blocker;
1300 }
1301 if nearby.contains("split out of")
1302 || nearby.contains("split from")
1303 || nearby.contains("parent tracker")
1304 || nearby.contains("tracker checkbox")
1305 {
1306 return MentionRelation::Tracker;
1307 }
1308 MentionRelation::None
1309}
1310
1311fn dependency_entries(
1317 item_ref: &str,
1318 citations: &[Citation],
1319 relations: &BTreeMap<(String, u64), CitationRelation>,
1320 answers: &BTreeMap<String, Answer>,
1321) -> Vec<DependencyEntry> {
1322 citations
1323 .iter()
1324 .enumerate()
1325 .map(|(i, citation)| {
1326 let could_be_cheaper =
1327 if let Some(Answer::Noul { noul }) = answers.get(&could_be_cheaper_key(i)) {
1328 BTreeMap::from([("design".to_string(), *noul)])
1329 } else {
1330 warn!(
1331 "Issue {item_ref}: no could_be_cheaper answer for citation {:?}",
1332 citation.raw
1333 );
1334 BTreeMap::new()
1335 };
1336 DependencyEntry {
1337 item_ref: citation.raw.clone(),
1338 url: citation.url.clone(),
1339 state: ItemState::Open,
1340 relation: relations
1341 .get(&(citation.item_ref.project.clone(), citation.item_ref.number))
1342 .copied()
1343 .unwrap_or(CitationRelation::Unspecified),
1344 could_be_cheaper,
1345 }
1346 })
1347 .collect()
1348}
1349
1350#[must_use]
1371pub fn render_route_text(report: &RouteReport, max_input_chars: usize) -> String {
1372 render_route_text_styled(report, max_input_chars, &[], TerminalStyle::default())
1373}
1374
1375#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
1378pub struct TerminalStyle {
1379 pub color: bool,
1381 pub hyperlinks: bool,
1383}
1384
1385impl TerminalStyle {
1386 #[must_use]
1388 pub fn new(tty: bool, term: &str, no_color: bool, supports_links: bool) -> Self {
1389 let usable = tty && !term.is_empty() && term != "dumb";
1390 Self {
1391 color: usable && !no_color,
1392 hyperlinks: usable && supports_links,
1393 }
1394 }
1395}
1396
1397#[must_use]
1399pub fn render_route_text_styled(
1400 report: &RouteReport,
1401 max_input_chars: usize,
1402 ladders: &[Ladder],
1403 style: TerminalStyle,
1404) -> String {
1405 let mut blocks: Vec<String> = report
1406 .issues
1407 .iter()
1408 .map(|issue| render_issue_block(issue, max_input_chars, ladders, style))
1409 .collect();
1410 if !report.ignored_closed.is_empty() {
1411 let refs: Vec<String> = report
1412 .ignored_closed
1413 .iter()
1414 .map(|issue| hyperlink(&issue.item_ref, Some(&issue.url), style))
1415 .collect();
1416 blocks.push(format!("ignored closed: {}", refs.join(", ")));
1417 }
1418 blocks.push(format!(
1419 "model: {}, usage: {} input tokens, {} output tokens",
1420 report.model, report.usage.input_tokens, report.usage.output_tokens
1421 ));
1422 let mut text = blocks.join("\n\n");
1423 text.push('\n');
1424 text
1425}
1426
1427fn render_issue_block(
1431 issue: &IssueRoute,
1432 max_input_chars: usize,
1433 ladders: &[Ladder],
1434 style: TerminalStyle,
1435) -> String {
1436 let item_ref = hyperlink(&issue.item_ref, Some(&issue.url), style);
1437 let item_ref = if issue.failed() {
1438 item_ref
1439 } else {
1440 colorize(&item_ref, Complexity::Low, style)
1441 };
1442 let header = if issue.state == ItemState::Closed {
1443 format!("{item_ref} — {} (closed)", issue.title)
1444 } else {
1445 format!("{item_ref} — {}", issue.title)
1446 };
1447 let mut lines = vec![header];
1448 if !issue.draft_comments.is_empty() {
1449 lines.push(format!(
1450 " draft comments (preview): {:?}",
1451 issue.draft_comments
1452 ));
1453 }
1454 match &issue.outcome {
1455 RouteOutcome::Routed {
1456 providers,
1457 depends_on,
1458 reference_fetch_failures,
1459 open_questions,
1460 } => {
1461 lines.push(format!(
1462 " open_questions: {} ({:.2})",
1463 open_questions.choice.label(),
1464 open_questions.confidence
1465 ));
1466 for (provider, route) in providers {
1467 let ladder = ladders.iter().find(|ladder| ladder.name == *provider);
1468 lines.extend(render_provider_line(
1469 provider,
1470 route,
1471 providers.len(),
1472 ladder,
1473 style,
1474 ));
1475 }
1476 lines.extend(render_depends_on_lines(depends_on, style));
1477 lines.extend(render_reference_fetch_failure_lines(
1478 reference_fetch_failures,
1479 ));
1480 }
1481 RouteOutcome::Failed {
1482 error,
1483 reference_fetch_failures,
1484 } => {
1485 lines.push(format!(
1486 " {}",
1487 colorize(&format!("failed: {error}"), Complexity::High, style)
1488 ));
1489 lines.extend(render_reference_fetch_failure_lines(
1490 reference_fetch_failures,
1491 ));
1492 }
1493 }
1494 if issue.truncated {
1495 lines.push(format!(
1496 " input truncated at {} characters",
1497 with_thousands(max_input_chars)
1498 ));
1499 }
1500 lines.join("\n")
1501}
1502
1503fn route_has_multi_model_tier(route: &ProviderRoute) -> bool {
1512 [
1513 &route.stages.design,
1514 &route.stages.implement,
1515 &route.stages.review,
1516 ]
1517 .into_iter()
1518 .any(|answer| answer.choice.contains(','))
1519}
1520
1521fn render_provider_line(
1526 provider: &str,
1527 route: &ProviderRoute,
1528 provider_count: usize,
1529 ladder: Option<&Ladder>,
1530 style: TerminalStyle,
1531) -> Vec<String> {
1532 let mut lines = if route_has_multi_model_tier(route) {
1533 render_provider_block(provider, route, provider_count, ladder, style)
1534 } else {
1535 vec![render_provider_line_compact(
1536 provider,
1537 route,
1538 provider_count,
1539 ladder,
1540 style,
1541 )]
1542 };
1543 lines.extend(effort::render(&route.stages, ladder));
1544 lines
1545}
1546
1547fn render_provider_line_compact(
1552 provider: &str,
1553 route: &ProviderRoute,
1554 provider_count: usize,
1555 ladder: Option<&Ladder>,
1556 style: TerminalStyle,
1557) -> String {
1558 let class = format!(
1559 "{} (from {})",
1560 color_choice(&route.class, ladder, style),
1561 route.class_from.label()
1562 );
1563 let lead = if provider_count > 1 {
1564 format!("{provider}: {class}")
1565 } else {
1566 class
1567 };
1568 let clauses: Vec<String> = Stage::ALL
1569 .into_iter()
1570 .map(|stage| render_stage_clause(stage, &route.stages, &route.close_calls, ladder, style))
1571 .collect();
1572 format!(" {lead} — {}", clauses.join(", "))
1573}
1574
1575fn render_stage_clause(
1578 stage: Stage,
1579 stages: &StageAnswers,
1580 close_calls: &[Stage],
1581 ladder: Option<&Ladder>,
1582 style: TerminalStyle,
1583) -> String {
1584 let answer = stages.get(stage);
1585 let choice = color_choice(&answer.choice, ladder, style);
1586 let label = match stage {
1587 Stage::Design if answer.choice == NO_DESIGN => format!(
1588 "design needs {}",
1589 colorize("no further work", Complexity::Low, style)
1590 ),
1591 Stage::Design => format!("design needs {choice}"),
1592 Stage::Implement => format!("implementation {choice}"),
1593 Stage::Review => format!("review {choice}"),
1594 };
1595 format!(
1596 "{label} ({})",
1597 render_stage_confidence(answer, close_calls.contains(&stage))
1598 )
1599}
1600
1601fn render_stage_confidence(answer: &StageAnswer, close_call: bool) -> String {
1604 let mut detail = format!("{:.2}", answer.confidence);
1605 if close_call {
1606 detail.push_str(", close call");
1607 if let Some((name, probability)) = answer
1608 .probabilities
1609 .iter()
1610 .filter(|(name, probability)| name.as_str() != answer.choice && probability.is_finite())
1611 .max_by(|a, b| a.1.total_cmp(b.1).then_with(|| b.0.cmp(a.0)))
1612 {
1613 detail.push_str(&format!(" — {name} {probability:.2}"));
1614 }
1615 }
1616 detail
1617}
1618
1619fn render_provider_block(
1625 provider: &str,
1626 route: &ProviderRoute,
1627 provider_count: usize,
1628 ladder: Option<&Ladder>,
1629 style: TerminalStyle,
1630) -> Vec<String> {
1631 let mut lines = Vec::with_capacity(5);
1632 let indent = if provider_count > 1 {
1633 lines.push(format!(" {provider}:"));
1634 " "
1635 } else {
1636 " "
1637 };
1638 lines.push(format!(
1639 "{indent}class: {} (from {})",
1640 color_choice(&route.class, ladder, style),
1641 route.class_from.label()
1642 ));
1643 for stage in Stage::ALL {
1644 lines.push(format!(
1645 "{indent}{}",
1646 render_stage_line(stage, &route.stages, &route.close_calls, ladder, style)
1647 ));
1648 }
1649 lines
1650}
1651
1652fn render_stage_line(
1657 stage: Stage,
1658 stages: &StageAnswers,
1659 close_calls: &[Stage],
1660 ladder: Option<&Ladder>,
1661 style: TerminalStyle,
1662) -> String {
1663 let answer = stages.get(stage);
1664 let choice = color_choice(&answer.choice, ladder, style);
1665 let (label, content) = match stage {
1666 Stage::Design if answer.choice == NO_DESIGN => (
1667 "design",
1668 format!(
1669 "needs {}",
1670 colorize("no further work", Complexity::Low, style)
1671 ),
1672 ),
1673 Stage::Design => ("design", format!("needs {choice}")),
1674 Stage::Implement => ("implementation", choice),
1675 Stage::Review => ("review", choice),
1676 };
1677 format!(
1678 "{label}: {content} ({})",
1679 render_stage_confidence(answer, close_calls.contains(&stage))
1680 )
1681}
1682
1683fn render_depends_on_lines(depends_on: &[DependencyEntry], style: TerminalStyle) -> Vec<String> {
1686 depends_on
1687 .iter()
1688 .map(|dep| {
1689 let item_ref = hyperlink(&dep.item_ref, dep.url.as_deref(), style);
1690 let relation = match dep.relation {
1691 CitationRelation::Blocker => "blocker",
1692 CitationRelation::Tracker => "tracker",
1693 CitationRelation::Unspecified => "relation unspecified",
1694 };
1695 match dep.could_be_cheaper.get("design") {
1696 Some(prob) => format!(
1697 " cites open {item_ref} ({relation}), which could leave less design work if resolved ({prob:.2})"
1698 ),
1699 None => format!(" cites open {item_ref} ({relation})"),
1700 }
1701 })
1702 .collect()
1703}
1704
1705fn render_reference_fetch_failure_lines(failures: &[ReferenceFetchFailure]) -> Vec<String> {
1707 failures
1708 .iter()
1709 .map(|failure| {
1710 format!(
1711 " reference fetch failed: {} ({})",
1712 failure.item_ref, failure.error
1713 )
1714 })
1715 .collect()
1716}
1717
1718#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1719enum Complexity {
1720 Low,
1721 Medium,
1722 High,
1723}
1724
1725fn choice_complexity(choice: &str, ladder: &Ladder) -> Option<Complexity> {
1727 if choice == NO_DESIGN {
1728 return Some(Complexity::Low);
1729 }
1730 let tiers = ladder.tiers.as_slice();
1731 let index = tiers.iter().position(|tier| tier.name == choice)?;
1732 Some(if index == 0 {
1733 Complexity::Low
1734 } else if index == tiers.len() - 1 {
1735 Complexity::High
1736 } else {
1737 Complexity::Medium
1738 })
1739}
1740
1741fn color_choice(choice: &str, ladder: Option<&Ladder>, style: TerminalStyle) -> String {
1742 ladder
1743 .and_then(|ladder| choice_complexity(choice, ladder))
1744 .map_or_else(
1745 || choice.to_string(),
1746 |level| colorize(choice, level, style),
1747 )
1748}
1749
1750fn colorize(text: &str, level: Complexity, style: TerminalStyle) -> String {
1751 if !style.color {
1752 return text.to_string();
1753 }
1754 let code = match level {
1755 Complexity::Low => 32,
1756 Complexity::Medium => 33,
1757 Complexity::High => 31,
1758 };
1759 format!("\x1b[{code}m{text}\x1b[0m")
1760}
1761
1762fn hyperlink(text: &str, url: Option<&str>, style: TerminalStyle) -> String {
1765 let Some(url) = url.filter(|url| style.hyperlinks && valid_github_item_url(url)) else {
1766 return text.to_string();
1767 };
1768 format!("\x1b]8;;{url}\x1b\\{text}\x1b]8;;\x1b\\")
1769}
1770
1771fn valid_github_item_url(raw: &str) -> bool {
1772 if raw.chars().any(char::is_control) {
1773 return false;
1774 }
1775 let Ok(url) = url::Url::parse(raw) else {
1776 return false;
1777 };
1778 if url.scheme() != "https" || url.host_str() != Some("github.com") {
1779 return false;
1780 }
1781 let Some(segments) = url.path_segments() else {
1782 return false; };
1784 let parts: Vec<_> = segments.collect();
1785 parts.len() == 4
1786 && !parts[0].is_empty()
1787 && !parts[1].is_empty()
1788 && matches!(parts[2], "issues" | "pull")
1789 && parts[3].parse::<u64>().is_ok()
1790}
1791
1792fn with_thousands(n: usize) -> String {
1794 let digits = n.to_string();
1795 let mut grouped = String::with_capacity(digits.len() + digits.len() / 3);
1796 for (i, ch) in digits.chars().rev().enumerate() {
1797 if i > 0 && i % 3 == 0 {
1798 grouped.push(',');
1799 }
1800 grouped.push(ch);
1801 }
1802 grouped.chars().rev().collect()
1803}
1804
1805#[cfg(test)]
1806#[allow(clippy::unwrap_used, clippy::expect_used)]
1807mod tests {
1808 use super::*;
1809 use crate::provider::{Comment, GitProvider, ItemKind, ItemRef};
1810
1811 fn doc(number: u64, state: ItemState) -> IssueDoc {
1812 IssueDoc {
1813 provider: GitProvider::GitHub,
1814 project: "rust-works/omni-dev".to_string(),
1815 number,
1816 kind: ItemKind::Issue,
1817 title: "Route issues".to_string(),
1818 state,
1819 body: " The body.\n".to_string(),
1820 comments: vec![],
1821 closed_by: vec![],
1822 url: format!("https://github.com/rust-works/omni-dev/issues/{number}"),
1823 }
1824 }
1825
1826 fn opts() -> RouteOptions {
1827 RouteOptions {
1828 draft_comments: vec![],
1829 model: "jev-latest".to_string(),
1830 close_call: DEFAULT_CLOSE_CALL,
1831 close_call_margin: DEFAULT_CLOSE_CALL_MARGIN,
1832 effort_advice: false,
1833 max_input_chars: DEFAULT_MAX_INPUT_CHARS,
1834 allow_closed: false,
1835 ignore_closed: false,
1836 }
1837 }
1838
1839 fn default_tiers() -> Tiers {
1840 Provider::Anthropic.tiers().unwrap()
1841 }
1842
1843 fn class_only(provider: Provider) -> Result<Ladder> {
1845 let mut ladder = Ladder::builtin(provider)?;
1846 for tier in &mut ladder.tiers.0 {
1847 tier.models = None;
1848 }
1849 Ok(ladder)
1850 }
1851
1852 fn anthropic() -> Vec<Ladder> {
1853 vec![class_only(Provider::Anthropic).unwrap()]
1854 }
1855
1856 fn tier_names(tiers: &Tiers) -> Vec<&str> {
1857 tiers.as_slice().iter().map(|t| t.name.as_str()).collect()
1858 }
1859
1860 #[test]
1865 fn anthropic_tiers_are_sonnet_opus_in_order() {
1866 let tiers = default_tiers();
1867 assert_eq!(tier_names(&tiers), ["sonnet", "opus"]);
1868 assert_eq!(
1869 tiers.as_slice()[0].models.as_ref().unwrap()[0].name,
1870 "claude-sonnet-5-5"
1871 );
1872 assert_eq!(
1873 tiers.as_slice()[1].models.as_ref().unwrap()[0].name,
1874 "claude-opus-5-5"
1875 );
1876 }
1877
1878 #[test]
1879 fn anthropic_tier_descriptions_are_the_reevaluated_text() {
1880 let tiers = default_tiers();
1881 let sonnet = &tiers.as_slice()[0];
1882 let opus = &tiers.as_slice()[1];
1883 assert_eq!(
1884 sonnet.description,
1885 "Reliable at implementing and reviewing well-scoped work across several files, \
1886 including debugging and non-obvious interactions when the approach is clear. May \
1887 need help with complex, open-ended design requiring sustained judgement."
1888 );
1889 assert_eq!(
1890 opus.description,
1891 "Reliable at complex, open-ended design and research: weighing architectures and \
1892 unfamiliar approaches, anticipating failure modes, and making security-critical \
1893 judgements. Carries subtle multi-file implementation and review through with \
1894 sustained judgement."
1895 );
1896 }
1897
1898 #[test]
1899 fn provider_names_are_the_documented_abbreviations() {
1900 assert_eq!(
1901 tier_names(&Provider::OpenAi.tiers().unwrap()),
1902 ["terra", "sol", "astra"]
1903 );
1904 assert_eq!(
1905 tier_names(&Provider::Gemini.tiers().unwrap()),
1906 ["flash", "pro", "deep-think"]
1907 );
1908 assert_eq!(
1909 Provider::ALL.map(Provider::name),
1910 ["anthropic", "openai", "gemini"]
1911 );
1912 for provider in Provider::ALL {
1913 assert_eq!(
1914 provider.to_possible_value().unwrap().get_name(),
1915 provider.name()
1916 );
1917 }
1918 }
1919
1920 #[test]
1934 fn openai_and_gemini_ladders_reuse_the_anthropic_descriptions_rung_for_rung() {
1935 let anthropic = default_tiers();
1936 const RETIRED_ANTHROPIC_THIRD_RUNG: &str =
1937 "Strongest at open-ended design and research: choosing between architectures, \
1938 working in unfamiliar territory with no precedent in the codebase, anticipating \
1939 failure modes, and security-critical judgement.";
1940 for provider in [Provider::OpenAi, Provider::Gemini] {
1941 let tiers = provider.tiers().unwrap();
1942 assert_eq!(tiers.as_slice().len(), 3);
1943 for (rung, reference) in tiers.as_slice().iter().zip(anthropic.as_slice()) {
1944 assert_eq!(
1945 rung.description,
1946 reference.description,
1947 "{}: {} differs from {}",
1948 provider.name(),
1950 rung.name,
1951 reference.name
1952 );
1953 }
1954 assert_eq!(
1955 tiers.as_slice()[2].description,
1956 RETIRED_ANTHROPIC_THIRD_RUNG,
1957 "{}: {} no longer matches the retired anthropic fable rung",
1958 provider.name(), tiers.as_slice()[2].name );
1961 }
1962 }
1963
1964 #[test]
1965 fn ladders_are_named_by_provider_or_a_custom_name() {
1966 let ladder = class_only(Provider::Gemini).unwrap();
1967 assert_eq!(ladder.name, "gemini");
1968 let custom = Ladder::named("mine".to_string(), default_tiers());
1969 assert_eq!(custom.name, "mine");
1970 assert_eq!(custom.tiers, default_tiers());
1971 }
1972
1973 #[test]
1974 fn tiers_need_at_least_two() {
1975 let err = Tiers::parse("tiers:\n - {name: a, description: A}\n").unwrap_err();
1976 assert!(err.to_string().contains("at least 2 tiers"), "{err}");
1977 }
1978
1979 #[test]
1980 fn tiers_reject_the_reserved_none() {
1981 let err = Tiers::parse(
1982 "tiers:\n - {name: a, description: A}\n - {name: none, description: B}\n",
1983 )
1984 .unwrap_err();
1985 assert!(err.to_string().contains("reserved"), "{err}");
1986 }
1987
1988 #[test]
1989 fn tiers_reject_duplicate_names() {
1990 let err =
1991 Tiers::parse("tiers:\n - {name: a, description: A}\n - {name: a, description: B}\n")
1992 .unwrap_err();
1993 assert!(err.to_string().contains("more than once"), "{err}");
1994 }
1995
1996 #[test]
1997 fn tiers_reject_empty_descriptions() {
1998 let err =
1999 Tiers::parse("tiers:\n - {name: a, description: A}\n - {name: b, description: ''}\n")
2000 .unwrap_err();
2001 assert!(err.to_string().contains("non-empty"), "{err}");
2002 }
2003
2004 #[test]
2005 fn tiers_reject_unknown_fields() {
2006 let err = Tiers::parse(
2007 "tiers:\n - {name: a, description: A, rank: 1}\n - {name: b, description: B}\n",
2008 )
2009 .unwrap_err();
2010 assert!(format!("{err:#}").contains("unknown field"), "{err:#}");
2011 }
2012
2013 #[test]
2014 fn tiers_load_reads_a_custom_file() {
2015 let dir = tempfile::tempdir().unwrap();
2016 let path = dir.path().join("tiers.yaml");
2017 std::fs::write(
2018 &path,
2019 "tiers:\n - {name: small, description: S}\n - {name: big, description: B}\n",
2020 )
2021 .unwrap();
2022 let tiers = Tiers::load_file(&path).unwrap();
2023 assert_eq!(tiers.as_slice()[1].name, "big");
2024 }
2025
2026 #[test]
2027 fn tiers_load_names_a_missing_file() {
2028 let err = Tiers::load_file(Path::new("/no/such/tiers.yaml")).unwrap_err();
2029 assert!(err.to_string().contains("Failed to read tiers file"));
2030 }
2031
2032 const BAR: &str = " Choose the least capable class likely to complete this stage correctly \
2035 with no rework, about 9 times in 10. Judge the work that remains given \
2036 the text, not the size of the text.";
2037
2038 fn instructions(questions: &BTreeMap<String, Question>, key: &str) -> String {
2039 let Question::Choice { instructions, .. } = &questions[key] else {
2040 panic!("{key} is not a choice");
2041 };
2042 instructions.clone()
2043 }
2044
2045 fn options(questions: &BTreeMap<String, Question>, key: &str) -> Vec<String> {
2046 let Question::Choice { criteria, .. } = &questions[key] else {
2047 panic!("{key} is not a choice");
2048 };
2049 criteria.keys().cloned().collect()
2050 }
2051
2052 #[test]
2055 fn stage_questions_are_the_tested_wording() {
2056 let questions = build_route_questions(&anthropic()).unwrap();
2057 let tiers = default_tiers();
2058 for stage in ["stage_design", "stage_implement", "stage_review"] {
2059 let Question::Choice { criteria, .. } = &questions[&format!("anthropic.{stage}")]
2060 else {
2061 panic!("{stage} is not a choice");
2062 };
2063 for tier in tiers.as_slice() {
2064 assert_eq!(
2065 criteria[&tier.name], tier.description,
2066 "{stage}: {}",
2067 tier.name
2068 );
2069 }
2070 }
2071 assert_eq!(
2072 instructions(&questions, "anthropic.stage_design"),
2073 format!(
2074 "Which class should do the design work that remains before implementation can \
2075 start: choosing the approach, settling open questions, and writing a plan?{BAR}"
2076 )
2077 );
2078 assert_eq!(
2079 instructions(&questions, "anthropic.stage_implement"),
2080 format!(
2081 "Assume any remaining design work has been completed well. Which class should \
2082 write the code, tests and docs?{BAR}"
2083 )
2084 );
2085 assert_eq!(
2086 instructions(&questions, "anthropic.stage_review"),
2087 format!(
2088 "Which class should review the finished change before merge, so that mistakes \
2089 the automated tests would miss are caught?{BAR}"
2090 )
2091 );
2092 }
2093
2094 #[test]
2095 fn only_the_design_stage_offers_none() {
2096 let questions = build_route_questions(&anthropic()).unwrap();
2097 assert_eq!(
2098 options(&questions, "anthropic.stage_design"),
2099 ["none", "opus", "sonnet"]
2100 );
2101 assert_eq!(
2102 options(&questions, "anthropic.stage_implement"),
2103 ["opus", "sonnet"]
2104 );
2105 assert_eq!(
2106 options(&questions, "anthropic.stage_review"),
2107 ["opus", "sonnet"]
2108 );
2109 let Question::Choice { criteria, .. } = &questions["anthropic.stage_design"] else {
2110 panic!();
2111 };
2112 assert_eq!(
2113 criteria[NO_DESIGN],
2114 "No design work remains: the text already settles the approach and the open \
2115 questions."
2116 );
2117 }
2118
2119 #[test]
2122 fn questions_are_keyed_per_provider_in_one_map() {
2123 let ladders = [
2124 class_only(Provider::Anthropic).unwrap(),
2125 class_only(Provider::OpenAi).unwrap(),
2126 ];
2127 let questions = build_route_questions(&ladders).unwrap();
2128 let keys: Vec<&String> = questions.keys().collect();
2129 assert_eq!(
2130 keys,
2131 [
2132 "anthropic.stage_design",
2133 "anthropic.stage_implement",
2134 "anthropic.stage_review",
2135 "open_questions",
2136 "openai.stage_design",
2137 "openai.stage_implement",
2138 "openai.stage_review",
2139 ]
2140 );
2141 assert_eq!(
2142 options(&questions, "openai.stage_design"),
2143 ["astra", "none", "sol", "terra"]
2144 );
2145 assert_eq!(
2146 instructions(&questions, "openai.stage_review"),
2147 instructions(&questions, "anthropic.stage_review")
2148 );
2149 }
2150
2151 #[test]
2152 fn a_named_ladder_is_keyed_by_its_own_name() {
2153 let ladders = [Ladder::named(
2154 "mine".to_string(),
2155 Tiers::parse("tiers:\n - {name: a, description: A}\n - {name: b, description: B}\n")
2156 .unwrap(),
2157 )];
2158 let questions = build_route_questions(&ladders).unwrap();
2159 assert_eq!(options(&questions, "mine.stage_implement"), ["a", "b"]);
2160 }
2161
2162 #[test]
2165 fn route_state_without_comments_is_title_and_trimmed_body() {
2166 let (state, truncated) = build_route_state(&doc(7, ItemState::Open), 1000);
2167 assert_eq!(state, "# #7 Route issues\n\nThe body.\n");
2168 assert!(!truncated);
2169 }
2170
2171 #[test]
2172 fn route_state_lists_comments_in_order_in_the_tested_format() {
2173 let mut d = doc(7, ItemState::Open);
2174 d.comments = vec![
2175 Comment {
2176 author: "alice".to_string(),
2177 body: "first\n".to_string(),
2178 id: None,
2179 },
2180 Comment {
2181 author: "bob".to_string(),
2182 body: "second".to_string(),
2183 id: None,
2184 },
2185 ];
2186 let (state, _) = build_route_state(&d, 1000);
2187 assert_eq!(
2188 state,
2189 "# #7 Route issues\n\nThe body.\n\n\n---\n\n## Comments on this issue\n\
2190 \n\n**Comment by alice:**\n\nfirst\n\
2191 \n\n**Comment by bob:**\n\nsecond\n"
2192 );
2193 }
2194
2195 #[test]
2196 fn route_state_truncates_the_middle_on_a_char_boundary() {
2197 let mut d = doc(7, ItemState::Open);
2198 d.body = "é".repeat(100);
2199 let (full, _) = build_route_state(&d, usize::MAX);
2200 let (state, truncated) = build_route_state(&d, 20);
2201 assert!(truncated);
2202 let (head, tail) = state
2203 .split_once(&format!("\n\n{TRUNCATION_MARKER}\n\n"))
2204 .unwrap();
2205 assert_eq!(head.chars().count(), 10);
2206 assert_eq!(tail.chars().count(), 10);
2207 assert!(full.starts_with(head));
2208 assert!(full.ends_with(tail));
2209 }
2210
2211 fn comment(author: &str, body: &str) -> Comment {
2212 Comment {
2213 author: author.to_string(),
2214 body: body.to_string(),
2215 id: None,
2216 }
2217 }
2218
2219 #[test]
2222 fn citation_text_is_the_state_with_code_masked() {
2223 let mut d = doc(7, ItemState::Open);
2224 d.title = "Fix `o/r#1`".to_string();
2225 d.body = "See #5.\n\n```\nerror: o/r#2\n```".to_string();
2226 d.comments = vec![comment("alice", "inline `o/r#3` and #6")];
2227 let (state, _) = build_route_state(&d, 1000);
2228 let text = build_route_citation_text(&d, 1000);
2229 assert_eq!(text.chars().count(), state.chars().count());
2230 assert!(text.contains("See #5."), "{text}");
2231 assert!(text.contains("and #6"), "{text}");
2232 assert!(text.contains("**Comment by alice:**"), "{text}");
2233 for masked in ["o/r#1", "o/r#2", "o/r#3"] {
2234 assert!(!text.contains(masked), "{masked} survived in {text}");
2235 }
2236 }
2237
2238 #[test]
2239 fn citation_text_is_truncated_where_the_state_is() {
2240 let mut d = doc(7, ItemState::Open);
2241 d.body = format!("start #1 {} `o/r#9` end #2", "x".repeat(500));
2242 let (state, truncated) = build_route_state(&d, 100);
2243 let text = build_route_citation_text(&d, 100);
2244 assert!(truncated);
2245 assert_eq!(text.chars().count(), state.chars().count());
2246 assert!(text.contains(TRUNCATION_MARKER), "{text}");
2247 }
2248
2249 #[test]
2253 fn citation_text_masks_each_comment_on_its_own() {
2254 let mut d = doc(7, ItemState::Open);
2255 d.comments = vec![
2256 comment("alice", "```\nstray fence, never closed\nreal #11"),
2257 comment("bob", "real #12"),
2258 comment("carol", "```\nerror: o/r#9\n```"),
2259 ];
2260 let text = build_route_citation_text(&d, 10_000);
2261 assert!(text.contains("real #11"), "{text}");
2262 assert!(text.contains("real #12"), "{text}");
2263 assert!(!text.contains("o/r#9"), "{text}");
2264 }
2265
2266 #[test]
2268 fn route_state_truncation_keeps_the_latest_comment() {
2269 let mut d = doc(7, ItemState::Open);
2270 d.body = "x".repeat(10_000);
2271 d.comments = vec![Comment {
2272 author: "maintainer".to_string(),
2273 body: "**Decision**: settled.".to_string(),
2274 id: None,
2275 }];
2276 let (state, truncated) = build_route_state(&d, 1_000);
2277 assert!(truncated);
2278 assert!(state.starts_with("# #7 Route issues"));
2279 assert!(state.ends_with("**Decision**: settled.\n"), "{state}");
2280 }
2281
2282 #[test]
2283 fn route_state_truncation_handles_a_one_char_cap() {
2284 let (state, truncated) = build_route_state(&doc(7, ItemState::Open), 1);
2285 assert!(truncated);
2286 assert_eq!(state, format!("\n\n{TRUNCATION_MARKER}\n\n\n"));
2287 }
2288
2289 #[test]
2290 fn route_state_at_exactly_the_cap_is_not_truncated() {
2291 let d = doc(7, ItemState::Open);
2292 let (full, _) = build_route_state(&d, usize::MAX);
2293 let (state, truncated) = build_route_state(&d, full.chars().count());
2294 assert_eq!(state, full);
2295 assert!(!truncated);
2296 }
2297
2298 #[test]
2299 fn drafts_follow_fetched_comments_and_default_input_is_identical() {
2300 let mut d = doc(7, ItemState::Open);
2301 for (author, body) in [("alice", "first"), ("bob", "second")] {
2302 d.comments.push(Comment {
2303 author: author.into(),
2304 body: body.into(),
2305 id: None,
2306 });
2307 }
2308 let drafts = [" third\n\n indented\n".into(), "fourth".into()];
2309 let (state, truncated) = build_route_state_with_drafts(&d, 1000, &drafts);
2310 assert!(!truncated);
2311 assert!(state.ends_with(
2312 "**Comment by draft:**\n\nthird\n\n indented\n\n\n**Comment by draft:**\n\nfourth\n"
2313 ));
2314 assert!(state.find("first").unwrap() < state.find("second").unwrap());
2315 assert!(state.find("second").unwrap() < state.find("third").unwrap());
2316 assert_eq!(
2317 build_route_state_with_drafts(&d, 1000, &[]),
2318 build_route_state(&d, 1000)
2319 );
2320 assert_eq!(
2321 build_route_citation_text_with_drafts(&d, 1000, &[]),
2322 build_route_citation_text(&d, 1000)
2323 );
2324 d.comments.clear();
2325 let (state, _) = build_route_state_with_drafts(&d, 1000, &drafts);
2326 assert!(state.contains("## Comments on this issue"));
2327 }
2328
2329 #[test]
2330 fn draft_truncation_and_code_masking_remain_aligned() {
2331 let d = doc(7, ItemState::Open);
2332 let drafts = [
2333 format!("`#99` {}\n```\n#88\n```", "é".repeat(200)),
2334 "See #2 at the tail".into(),
2335 ];
2336 let (full, _) = build_route_state_with_drafts(&d, usize::MAX, &drafts);
2337 let masked = build_route_citation_text_with_drafts(&d, usize::MAX, &drafts);
2338 assert_eq!(full.chars().count(), masked.chars().count());
2339 assert!(!masked.contains("#99"));
2340 assert!(!masked.contains("#88"));
2341 assert!(masked.contains("#2"));
2342 let (cut, truncated) = build_route_state_with_drafts(&d, 120, &drafts);
2343 let masked_cut = build_route_citation_text_with_drafts(&d, 120, &drafts);
2344 assert!(truncated);
2345 assert!(cut.ends_with("See #2 at the tail\n"));
2346 assert_eq!(cut.chars().count(), masked_cut.chars().count());
2347 let marker = crate::jev::input::TRUNCATION_MARKER;
2348 let before = cut.split_once(marker).unwrap().0.chars().count();
2349 let masked_before = masked_cut.split_once(marker).unwrap().0.chars().count();
2350 assert_eq!(before, masked_before);
2351 assert!(cut.find(marker).unwrap() < cut.find("See #2").unwrap());
2352 let isolated =
2353 build_route_citation_text_with_drafts(&d, 1000, &["`start".into(), "#2 end`".into()]);
2354 assert!(isolated.contains("#2"));
2355 }
2356
2357 #[tokio::test]
2358 async fn drafts_reach_requests_and_batch_guard_precedes_closed_filtering() {
2359 let server = wiremock::MockServer::start().await;
2360 wiremock::Mock::given(wiremock::matchers::method("POST"))
2361 .respond_with(wiremock::ResponseTemplate::new(200).set_body_json(routed_json()))
2362 .expect(1)
2363 .mount(&server)
2364 .await;
2365 let client = JevClient::new(&server.uri(), "key").unwrap();
2366 let mut options = opts();
2367 options.draft_comments = vec!["Blocked on #1129".into()];
2368 let dependencies = OpenDependencies::from([(
2369 ("rust-works/omni-dev".into(), 7),
2370 vec![citation("#1129", 1129)],
2371 )]);
2372 run_route(
2373 &client,
2374 &[doc(7, ItemState::Open)],
2375 &anthropic(),
2376 &options,
2377 &dependencies,
2378 )
2379 .await
2380 .unwrap();
2381 options.ignore_closed = true;
2382 let err = run_route(
2383 &client,
2384 &[doc(7, ItemState::Open), doc(8, ItemState::Closed)],
2385 &anthropic(),
2386 &options,
2387 &dependencies,
2388 )
2389 .await
2390 .unwrap_err();
2391 assert!(err.to_string().contains("exactly one issue"));
2392 let requests = server.received_requests().await.unwrap();
2393 assert_eq!(requests.len(), 1);
2394 let body: serde_json::Value = requests[0].body_json().unwrap();
2395 assert!(body["state"]
2396 .as_str()
2397 .unwrap()
2398 .ends_with("**Comment by draft:**\n\nBlocked on #1129\n"));
2399 assert!(body["questions"].get("could_be_cheaper_0").is_some());
2400 let expected = build_route_questions_for_mode(&anthropic(), false).unwrap();
2401 for (key, question) in expected {
2402 assert_eq!(
2403 body["questions"][key],
2404 serde_json::to_value(question).unwrap()
2405 );
2406 }
2407 }
2408
2409 #[test]
2410 fn preview_provenance_is_visible_in_all_formats_including_failures() {
2411 let mut issue = IssueRoute {
2412 draft_comments: vec!["triage draft.md".into(), "scope.md".into()],
2413 item_ref: "o/r#1".into(),
2414 url: "u".into(),
2415 title: "t".into(),
2416 state: ItemState::Open,
2417 truncated: false,
2418 outcome: RouteOutcome::Failed {
2419 error: "failure".into(),
2420 reference_fetch_failures: vec![],
2421 },
2422 };
2423 for outcome in [
2424 issue.outcome.clone(),
2425 RouteOutcome::Routed {
2426 open_questions: test_open_questions(),
2427 providers: BTreeMap::new(),
2428 depends_on: vec![],
2429 reference_fetch_failures: vec![],
2430 },
2431 ] {
2432 issue.outcome = outcome;
2433 let json = serde_json::to_value(&issue).unwrap();
2434 assert_eq!(
2435 json["draft_comments"],
2436 serde_json::json!(["triage draft.md", "scope.md"])
2437 );
2438 let yaml: serde_yaml::Value =
2439 serde_yaml::from_str(&serde_yaml::to_string(&issue).unwrap()).unwrap();
2440 assert_eq!(yaml["draft_comments"][0].as_str(), Some("triage draft.md"));
2441 let text = render_issue_block(&issue, 1000, &[], TerminalStyle::default());
2442 assert!(text.contains("draft comments (preview): [\"triage draft.md\", \"scope.md\"]"));
2443 }
2444 issue.draft_comments.clear();
2445 assert!(serde_json::to_value(&issue)
2446 .unwrap()
2447 .get("draft_comments")
2448 .is_none());
2449 assert!(
2450 !render_issue_block(&issue, 1000, &[], TerminalStyle::default()).contains("preview")
2451 );
2452 }
2453
2454 fn answer(choice: &str, confidence: f64) -> StageAnswer {
2457 StageAnswer {
2458 effort_by_model: BTreeMap::new(),
2459 choice: choice.to_string(),
2460 confidence,
2461 probabilities: BTreeMap::new(),
2462 }
2463 }
2464
2465 fn stages(design: &str, implement: &str, review: &str) -> StageAnswers {
2466 StageAnswers {
2467 design: answer(design, 0.9),
2468 implement: answer(implement, 0.9),
2469 review: answer(review, 0.9),
2470 }
2471 }
2472
2473 #[test]
2474 fn close_call_shows_the_best_alternative_in_compact_text() {
2475 let mut stages = stages("sol", "terra", "sol");
2476 stages.design.confidence = 0.21;
2477 stages.design.probabilities = BTreeMap::from([
2478 ("none".to_string(), 0.44),
2479 ("sol".to_string(), 0.48),
2480 ("terra".to_string(), 0.08),
2481 ]);
2482
2483 assert_eq!(
2484 render_stage_clause(
2485 Stage::Design,
2486 &stages,
2487 &[Stage::Design],
2488 None,
2489 TerminalStyle::default(),
2490 ),
2491 "design needs sol (0.21, close call — none 0.44)"
2492 );
2493 assert_eq!(
2494 render_stage_clause(Stage::Design, &stages, &[], None, TerminalStyle::default(),),
2495 "design needs sol (0.21)"
2496 );
2497 }
2498
2499 #[test]
2500 fn close_call_uses_a_stable_tiebreaker_and_handles_missing_alternatives() {
2501 let mut answer = answer("sol", 0.21);
2502 answer.probabilities = BTreeMap::from([
2503 ("astra".to_string(), 0.4),
2504 ("sol".to_string(), 0.2),
2505 ("terra".to_string(), 0.4),
2506 ]);
2507 assert_eq!(
2508 render_stage_confidence(&answer, true),
2509 "0.21, close call — astra 0.40"
2510 );
2511 answer.probabilities.clear();
2512 assert_eq!(render_stage_confidence(&answer, true), "0.21, close call");
2513 }
2514
2515 #[test]
2516 fn design_supplies_the_class_when_it_outranks_implement() {
2517 let tiers = default_tiers();
2518 assert_eq!(
2519 issue_class(&stages("opus", "sonnet", "sonnet"), &tiers),
2520 ("opus".to_string(), ClassSource::Design)
2521 );
2522 }
2523
2524 #[test]
2525 fn implement_supplies_the_class_when_it_outranks_design() {
2526 let tiers = default_tiers();
2527 assert_eq!(
2528 issue_class(&stages("sonnet", "opus", "sonnet"), &tiers),
2529 ("opus".to_string(), ClassSource::Implement)
2530 );
2531 }
2532
2533 #[test]
2536 fn a_tie_goes_to_implement() {
2537 let tiers = default_tiers();
2538 for tier in ["sonnet", "opus"] {
2539 assert_eq!(
2540 issue_class(&stages(tier, tier, "sonnet"), &tiers),
2541 (tier.to_string(), ClassSource::Implement),
2542 "{tier}"
2543 );
2544 }
2545 }
2546
2547 #[test]
2548 fn class_ignores_review_and_none() {
2549 let tiers = default_tiers();
2550 assert_eq!(
2551 issue_class(&stages("none", "sonnet", "opus"), &tiers),
2552 ("sonnet".to_string(), ClassSource::Implement)
2553 );
2554 }
2555
2556 #[test]
2557 fn close_calls_are_below_the_threshold_in_stage_order() {
2558 let mut s = stages("opus", "sonnet", "opus");
2559 s.review.confidence = 0.1;
2560 s.design.confidence = 0.29;
2561 s.implement.confidence = 0.3;
2562 assert_eq!(
2563 close_calls(&s, &default_tiers(), 0.3, DEFAULT_CLOSE_CALL_MARGIN),
2564 [Stage::Design, Stage::Review]
2565 );
2566 }
2567
2568 fn distribution(entries: &[(&str, f64)]) -> BTreeMap<String, f64> {
2569 entries
2570 .iter()
2571 .map(|(name, p)| ((*name).into(), *p))
2572 .collect()
2573 }
2574
2575 #[test]
2576 fn close_call_flags_3017_split_and_preserves_class() {
2577 let mut s = stages("none", "sonnet", "opus");
2578 s.implement.confidence = 0.32;
2579 s.implement.probabilities = distribution(&[("sonnet", 0.55), ("opus", 0.45)]);
2580 assert_eq!(
2581 close_calls(&s, &default_tiers(), 0.3, 0.2),
2582 [Stage::Implement]
2583 );
2584 assert!(close_calls(&s, &default_tiers(), 0.3, 0.0).is_empty());
2585 assert_eq!(
2586 issue_class(&s, &default_tiers()),
2587 ("sonnet".into(), ClassSource::Implement)
2588 );
2589 }
2590
2591 #[test]
2592 fn close_call_predicates_are_strict_independent_and_ordered() {
2593 let tiers = default_tiers();
2594 let mut s = stages("sonnet", "sonnet", "sonnet");
2595 for (confidence, gap, expected) in [
2596 (0.25, 0.25, false), (0.125, 0.5, true), (0.5, 0.125, true), (0.125, 0.125, true), (0.5, 0.5, false), ] {
2602 s.design.confidence = confidence;
2603 s.design.probabilities = distribution(&[("sonnet", 0.5 + gap), ("opus", 0.5)]);
2604 assert_eq!(
2605 close_calls(&s, &tiers, 0.25, 0.25).contains(&Stage::Design),
2606 expected
2607 );
2608 }
2609 s.design.confidence = 0.1;
2610 s.design.probabilities = distribution(&[("sonnet", 0.5), ("opus", 0.5)]);
2611 s.implement.probabilities = s.design.probabilities.clone();
2612 s.review.probabilities = s.design.probabilities.clone();
2613 assert_eq!(close_calls(&s, &tiers, 0.3, 0.2), Stage::ALL);
2614 assert_eq!(close_calls(&s, &tiers, 0.3, 0.0), [Stage::Design]);
2615 s.design.confidence = 0.9;
2616 s.design.probabilities = distribution(&[("sonnet", 1.0), ("opus", 0.0)]);
2617 assert!(!close_calls(&s, &tiers, 0.3, 1.0).contains(&Stage::Design));
2618 }
2619
2620 #[test]
2621 fn margin_uses_only_finite_offered_probabilities_without_imputation() {
2622 let tiers = default_tiers();
2623 let mut a = answer("sonnet", 0.9);
2624 assert_eq!(top_two_margin(&a, Stage::Design, &tiers), None);
2625 a.probabilities = distribution(&[("sonnet", 0.5)]);
2626 assert_eq!(top_two_margin(&a, Stage::Design, &tiers), None);
2627 for invalid in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY, -0.1, 1.1] {
2628 a.probabilities = distribution(&[("sonnet", 0.5), ("opus", invalid), ("unknown", 0.5)]);
2629 assert_eq!(top_two_margin(&a, Stage::Implement, &tiers), None);
2630 }
2631 a.probabilities = distribution(&[("none", 0.5), ("sonnet", 0.5), ("unknown", 0.5)]);
2632 assert_eq!(top_two_margin(&a, Stage::Design, &tiers), Some(0.0));
2633 for stage in [Stage::Implement, Stage::Review] {
2634 assert_eq!(top_two_margin(&a, stage, &tiers), None);
2635 }
2636 a.probabilities = distribution(&[("none", 0.375), ("opus", 0.25)]);
2638 assert_eq!(top_two_margin(&a, Stage::Design, &tiers), Some(0.125));
2639 let openai = Provider::OpenAi.tiers().unwrap();
2641 a.choice = "terra".into();
2642 a.probabilities = distribution(&[("terra", 0.0), ("sol", 0.5), ("astra", 0.5)]);
2643 assert_eq!(top_two_margin(&a, Stage::Implement, &openai), Some(0.0));
2644 }
2645
2646 #[test]
2647 fn invalid_probabilities_retain_existing_confidence_behavior() {
2648 let mut s = stages("none", "sonnet", "opus");
2649 s.design.confidence = 0.1;
2650 s.design.probabilities = distribution(&[("sonnet", f64::NAN), ("opus", -1.0)]);
2651 s.implement.confidence = f64::NAN;
2652 s.review.confidence = f64::NEG_INFINITY;
2653 assert_eq!(
2654 close_calls(&s, &default_tiers(), 0.3, 0.2),
2655 [Stage::Design, Stage::Review]
2656 );
2657 s.implement.probabilities = distribution(&[("sonnet", 0.5), ("opus", 0.5)]);
2658 assert_eq!(close_calls(&s, &default_tiers(), 0.3, 0.2), Stage::ALL);
2659 }
2660
2661 #[test]
2662 fn custom_and_multiple_ladders_share_flags_across_output_formats() {
2663 let ladders = [
2664 Ladder::named("custom".into(), Tiers::parse("tiers:\n - name: small,alternative\n description: Local work\n - name: large\n description: Complex work\n").unwrap()),
2665 class_only(Provider::Anthropic).unwrap(),
2666 ];
2667 let mut answers = BTreeMap::new();
2668 for ladder in &ladders {
2669 let first = &ladder.tiers.as_slice()[0].name;
2670 let second = &ladder.tiers.as_slice()[1].name;
2671 for stage in Stage::ALL {
2672 let (p1, p2) = if ladder.name == "custom" && stage == Stage::Implement {
2673 (0.55, 0.45)
2674 } else {
2675 (0.9, 0.1)
2676 };
2677 answers.insert(
2678 stage.question_key(&ladder.name),
2679 Answer::Choice {
2680 choice: first.clone(),
2681 confidence: 0.32,
2682 probabilities: distribution(&[(first, p1), (second, p2)]),
2683 },
2684 );
2685 }
2686 }
2687 let routes = provider_routes(&answers, &ladders, 0.3, 0.2, false).unwrap();
2688 assert!(routes["anthropic"].close_calls.is_empty());
2689 let route = &routes["custom"];
2690 assert_eq!(route.close_calls, [Stage::Implement]);
2691 let json = serde_json::to_value(route).unwrap();
2692 let yaml: serde_json::Value =
2693 serde_yaml::from_str(&serde_yaml::to_string(route).unwrap()).unwrap();
2694 assert_eq!(json, yaml);
2695 assert_eq!(json["close_calls"], serde_json::json!(["implement"]));
2696 for stage in Stage::ALL {
2697 let clause = render_stage_clause(
2698 stage,
2699 &route.stages,
2700 &route.close_calls,
2701 None,
2702 TerminalStyle::default(),
2703 );
2704 let line = render_stage_line(
2705 stage,
2706 &route.stages,
2707 &route.close_calls,
2708 Some(&ladders[0]),
2709 TerminalStyle::default(),
2710 );
2711 for text in [clause, line] {
2712 assert_eq!(
2713 text.contains("close call"),
2714 stage == Stage::Implement,
2715 "{text}"
2716 );
2717 if stage == Stage::Implement {
2718 assert!(text.contains("0.32, close call — large 0.45"), "{text}");
2719 }
2720 }
2721 }
2722 let confidence_only = provider_routes(&answers, &ladders, 0.3, 0.0, false).unwrap();
2723 assert!(confidence_only["custom"].close_calls.is_empty());
2724 assert_eq!(route.class, confidence_only["custom"].class);
2725 assert_eq!(route.class_from, confidence_only["custom"].class_from);
2726 }
2727
2728 #[tokio::test]
2729 async fn run_route_applies_the_configured_margin_without_changing_requests() {
2730 let server = wiremock::MockServer::start().await;
2731 let mut implementation = choice_json("sonnet", 0.32);
2732 implementation["probabilities"] = serde_json::json!({"sonnet": 0.55, "opus": 0.45});
2733 wiremock::Mock::given(wiremock::matchers::method("POST"))
2734 .and(wiremock::matchers::path("/v1/systemone"))
2735 .respond_with(
2736 wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!({
2737 "model": "jev-1.13.0", "answers": {
2738 "open_questions": choice_json("factual", 0.8),
2739 "anthropic.stage_design": choice_json("none", 0.9),
2740 "anthropic.stage_implement": implementation,
2741 "anthropic.stage_review": choice_json("opus", 0.9)
2742 }, "usage": {"input_tokens": 100, "output_tokens": 10}
2743 })),
2744 )
2745 .expect(2)
2746 .mount(&server)
2747 .await;
2748 let client = JevClient::new(&server.uri(), "key").unwrap();
2749 for margin in [0.0, DEFAULT_CLOSE_CALL_MARGIN] {
2750 let mut options = opts();
2751 options.close_call_margin = margin;
2752 let report = run_route(
2753 &client,
2754 &[doc(1, ItemState::Open)],
2755 &anthropic(),
2756 &options,
2757 &OpenDependencies::new(),
2758 )
2759 .await
2760 .unwrap();
2761 assert_eq!(
2762 provider(&report.issues[0].outcome, "anthropic")
2763 .close_calls
2764 .contains(&Stage::Implement),
2765 margin > 0.0
2766 );
2767 }
2768 let requests = server.received_requests().await.unwrap();
2769 assert_eq!(requests.len(), 2);
2770 assert_eq!(requests[0].body, requests[1].body);
2771 }
2772
2773 #[tokio::test]
2774 async fn invalid_margins_fail_before_a_jev_request() {
2775 let server = wiremock::MockServer::start().await;
2776 let client = JevClient::new(&server.uri(), "key").unwrap();
2777 for margin in [-0.1, 1.1, f64::NAN, f64::INFINITY, f64::NEG_INFINITY] {
2778 let mut options = opts();
2779 options.close_call_margin = margin;
2780 let err = run_route(
2781 &client,
2782 &[doc(1, ItemState::Open)],
2783 &anthropic(),
2784 &options,
2785 &OpenDependencies::new(),
2786 )
2787 .await
2788 .unwrap_err();
2789 assert!(err.to_string().contains("close-call margin"), "{err}");
2790 }
2791 for margin in [0.0, 1.0] {
2792 let mut options = opts();
2793 options.close_call_margin = margin;
2794 validate_options(&[&doc(1, ItemState::Open)], &anthropic(), &options).unwrap();
2795 }
2796 assert!(server.received_requests().await.unwrap().is_empty());
2797 }
2798
2799 #[test]
2800 fn could_be_cheaper_key_is_stable_and_indexed() {
2801 assert_eq!(could_be_cheaper_key(0), "could_be_cheaper_0");
2802 assert_eq!(could_be_cheaper_key(1), "could_be_cheaper_1");
2803 }
2804
2805 #[test]
2806 fn could_be_cheaper_question_is_the_tested_wording() {
2807 let Question::Noul {
2808 instructions,
2809 criteria,
2810 } = could_be_cheaper_question("#1129")
2811 else {
2812 panic!("expected a noul question");
2813 };
2814 assert_eq!(
2815 instructions,
2816 "This issue cites #1129, which is still open. If #1129 is resolved, how likely is \
2817 it that LESS design work would remain for THIS issue than the current text \
2818 implies — as opposed to this issue's own remaining work being unaffected, because \
2819 it is already scoped separately, is a parallel/sibling effort, or #1129 is \
2820 otherwise not a precondition for finishing this issue's own remaining work?"
2821 );
2822 assert!(criteria.is_none());
2823 }
2824
2825 #[test]
2826 fn dependency_entries_reads_the_noul_answer_per_citation() {
2827 let citations = vec![
2828 Citation {
2829 item_ref: ItemRef {
2830 provider: GitProvider::GitHub,
2831 project: "rust-works/omni-dev".to_string(),
2832 kind: ItemKind::Issue,
2833 number: 1129,
2834 },
2835 raw: "#1129".to_string(),
2836 url: None,
2837 },
2838 Citation {
2839 item_ref: ItemRef {
2840 provider: GitProvider::GitHub,
2841 project: "rust-works/omni-dev".to_string(),
2842 kind: ItemKind::Issue,
2843 number: 1349,
2844 },
2845 raw: "#1349".to_string(),
2846 url: None,
2847 },
2848 ];
2849 let answers = BTreeMap::from([(
2850 "could_be_cheaper_0".to_string(),
2851 Answer::Noul { noul: 0.75 },
2852 )]);
2853 let deps = dependency_entries("o/r#1", &citations, &BTreeMap::new(), &answers);
2854 assert_eq!(deps.len(), 2, "{deps:?}");
2855 assert_eq!(deps[0].item_ref, "#1129");
2856 assert_eq!(deps[0].state, ItemState::Open);
2857 assert_eq!(deps[0].could_be_cheaper.get("design"), Some(&0.75));
2858 assert_eq!(deps[0].relation, CitationRelation::Unspecified);
2859 let json = serde_json::to_value(&deps[0]).unwrap();
2860 assert_eq!(json["relation"], "unspecified");
2861 assert_eq!(json["could_be_cheaper"]["design"], 0.75);
2862 assert_eq!(deps[1].item_ref, "#1349");
2863 assert!(deps[1].could_be_cheaper.is_empty());
2864 assert_eq!(
2865 serde_json::to_value(CitationRelation::Blocker).unwrap(),
2866 "blocker"
2867 );
2868 assert_eq!(
2869 serde_json::to_value(CitationRelation::Tracker).unwrap(),
2870 "tracker"
2871 );
2872 }
2873
2874 fn relation_for(body: &str, target: u64) -> CitationRelation {
2875 let mut issue = doc(9999, ItemState::Open);
2876 issue.body = body.to_string();
2877 citation_relations(body, &issue)
2878 .get(&(issue.project, target))
2879 .copied()
2880 .unwrap_or(CitationRelation::Unspecified)
2881 }
2882
2883 #[test]
2884 fn citation_relation_uses_later_explicit_mentions() {
2885 assert_eq!(
2886 relation_for(
2887 "See #1351 for background. Blocked on node identity in #1351.",
2888 1351
2889 ),
2890 CitationRelation::Blocker
2891 );
2892 assert_eq!(
2893 relation_for("See #1830. Split out of #1830.", 1830),
2894 CitationRelation::Tracker
2895 );
2896 }
2897
2898 #[test]
2899 fn citation_relation_ignores_negation_quotes_and_conflicts() {
2900 for body in [
2901 "Not blocked on #123.",
2902 "For example, \"blocked on #123\" is a blocker claim.",
2903 "> Blocked on #123.",
2904 "```\nBlocked on #123\n```",
2905 "This is such as #123, a worked example.",
2906 "See #123 for context.",
2907 "Blocked on #123. Split out of #123.",
2908 "Blocked on #123. No longer blocked on #123.",
2909 "#123 blocks this issue. #123 no longer blocks it.",
2910 ] {
2911 assert_eq!(
2912 relation_for(body, 123),
2913 CitationRelation::Unspecified,
2914 "{body}"
2915 );
2916 }
2917 }
2918
2919 #[test]
2920 fn citation_relation_real_issue_phrasing() {
2921 assert_eq!(
2922 relation_for(
2923 "Split out of #1830. Blocker #1831 closed as ADR-0083.",
2924 1830
2925 ),
2926 CitationRelation::Tracker
2927 );
2928 assert_eq!(
2929 relation_for("## Triage — blocked on #1129's classifier change.", 1129),
2930 CitationRelation::Blocker
2931 );
2932 assert_eq!(
2933 relation_for(
2934 "Blocked on real alias/node identity — the same work as #1351.",
2935 1351
2936 ),
2937 CitationRelation::Blocker
2938 );
2939 assert_eq!(
2940 relation_for("Absorbed by #1753; neither suggested option remains.", 1753),
2941 CitationRelation::Unspecified
2942 );
2943 assert_eq!(
2944 relation_for("#1831 blocks this issue until its ADR lands.", 1831),
2945 CitationRelation::Blocker
2946 );
2947 assert_eq!(
2948 relation_for("See #1831. It blocks this issue.", 1831),
2949 CitationRelation::Unspecified
2950 );
2951 }
2952
2953 #[test]
2954 fn citation_relations_match_frozen_public_issues() {
2955 for (inputs, labels) in [
2956 (
2957 include_str!("../../docs/evaluations/jev-route-1871/inputs.json"),
2958 include_str!("../../docs/evaluations/jev-route-1871/labels.json"),
2959 ),
2960 (
2961 include_str!("../../docs/evaluations/jev-route-1871/holdout-inputs.json"),
2962 include_str!("../../docs/evaluations/jev-route-1871/holdout-labels.json"),
2963 ),
2964 ] {
2965 let cases: Vec<serde_json::Value> = serde_json::from_str(inputs).unwrap();
2966 let labels: serde_json::Value = serde_json::from_str(labels).unwrap();
2967 for case in cases {
2968 let id = case["id"].as_str().unwrap();
2969 let doc: IssueDoc = serde_json::from_value(case["doc"].clone()).unwrap();
2970 let (state, _) = build_route_state(&doc, DEFAULT_MAX_INPUT_CHARS);
2971 let relations = citation_relations(&state, &doc);
2972 for (number, label) in labels[id]["relations"].as_object().into_iter().flatten() {
2974 let actual = relations
2975 .get(&(doc.project.clone(), number.parse().unwrap()))
2976 .copied()
2977 .unwrap_or(CitationRelation::Unspecified);
2978 let want = [
2979 ("blocker", CitationRelation::Blocker),
2980 ("tracker", CitationRelation::Tracker),
2981 ("unspecified", CitationRelation::Unspecified),
2982 ]
2983 .into_iter()
2984 .find(|(name, _)| Some(*name) == label.as_str())
2985 .expect("a known relation label")
2986 .1;
2987 assert_eq!(actual, want, "{id} -> #{number}");
2988 }
2989 }
2990 }
2991 }
2992
2993 #[test]
2994 fn stage_answers_reject_none_outside_design() {
2995 let choice = |c: &str| Answer::Choice {
2996 choice: c.to_string(),
2997 confidence: 0.5,
2998 probabilities: BTreeMap::new(),
2999 };
3000 let answers = BTreeMap::from([
3001 ("anthropic.stage_design".to_string(), choice("none")),
3002 ("anthropic.stage_implement".to_string(), choice("none")),
3003 ("anthropic.stage_review".to_string(), choice("opus")),
3004 ]);
3005 let err = stage_answers(&answers, &anthropic()[0]).unwrap_err();
3006 assert!(err.to_string().contains("not offered"), "{err}");
3007 }
3008
3009 #[test]
3010 fn stage_answers_require_a_choice_per_stage() {
3011 let answers = BTreeMap::from([(
3012 "anthropic.stage_design".to_string(),
3013 Answer::Noul { noul: 0.5 },
3014 )]);
3015 let err = stage_answers(&answers, &anthropic()[0]).unwrap_err();
3016 assert!(err.to_string().contains("no choice answer"), "{err}");
3017 assert!(err.to_string().contains("anthropic.stage_design"), "{err}");
3018 }
3019
3020 #[test]
3023 fn stage_answers_are_read_under_the_ladders_prefix() {
3024 let choice = |c: &str| Answer::Choice {
3025 choice: c.to_string(),
3026 confidence: 0.5,
3027 probabilities: BTreeMap::new(),
3028 };
3029 let answers = BTreeMap::from([
3030 ("anthropic.stage_design".to_string(), choice("none")),
3031 ("anthropic.stage_implement".to_string(), choice("sonnet")),
3032 ("anthropic.stage_review".to_string(), choice("opus")),
3033 ]);
3034 let err = stage_answers(&answers, &class_only(Provider::OpenAi).unwrap()).unwrap_err();
3035 assert!(err.to_string().contains("openai.stage_design"), "{err}");
3036 }
3037
3038 #[test]
3041 fn closed_issues_are_refused_by_default() {
3042 let err =
3043 validate_options(&[&doc(1, ItemState::Closed)], &anthropic(), &opts()).unwrap_err();
3044 assert!(err.to_string().contains("rust-works/omni-dev#1"), "{err}");
3045 assert!(err.to_string().contains("--allow-closed"), "{err}");
3046 assert!(err.to_string().contains("--ignore-closed"), "{err}");
3047 }
3048
3049 #[test]
3052 fn allow_closed_and_ignore_closed_are_mutually_exclusive() {
3053 let mut o = opts();
3054 o.allow_closed = true;
3055 o.ignore_closed = true;
3056 for state in [ItemState::Open, ItemState::Closed] {
3057 let err = drop_closed(&[doc(1, state)], &o).unwrap_err();
3058 assert!(err.to_string().contains("cannot be used together"), "{err}");
3059 }
3060 }
3061
3062 #[test]
3065 fn drop_closed_keeps_every_issue_without_the_flag() {
3066 let docs = [doc(1, ItemState::Open), doc(2, ItemState::Closed)];
3067 let (kept, ignored) = drop_closed(&docs, &opts()).unwrap();
3068 assert_eq!(kept.len(), 2);
3069 assert!(ignored.is_empty());
3070 }
3071
3072 #[test]
3073 fn drop_closed_skips_closed_issues_in_request_order() {
3074 let mut o = opts();
3075 o.ignore_closed = true;
3076 let docs = [
3077 doc(3, ItemState::Closed),
3078 doc(1, ItemState::Open),
3079 doc(2, ItemState::Closed),
3080 ];
3081 let (kept, ignored) = drop_closed(&docs, &o).unwrap();
3082 assert_eq!(kept.iter().map(|d| d.number).collect::<Vec<_>>(), [1]);
3083 assert_eq!(
3084 ignored,
3085 [
3086 IgnoredIssue {
3087 item_ref: "rust-works/omni-dev#3".to_string(),
3088 url: "https://github.com/rust-works/omni-dev/issues/3".to_string(),
3089 title: "Route issues".to_string(),
3090 },
3091 IgnoredIssue {
3092 item_ref: "rust-works/omni-dev#2".to_string(),
3093 url: "https://github.com/rust-works/omni-dev/issues/2".to_string(),
3094 title: "Route issues".to_string(),
3095 },
3096 ]
3097 );
3098 }
3099
3100 #[test]
3101 fn drop_closed_refuses_a_batch_with_no_open_issue_left() {
3102 let mut o = opts();
3103 o.ignore_closed = true;
3104 let docs = [doc(1, ItemState::Closed), doc(2, ItemState::Closed)];
3105 let err = drop_closed(&docs, &o).unwrap_err();
3106 let msg = err.to_string();
3107 assert!(
3108 msg.contains("rust-works/omni-dev#1, rust-works/omni-dev#2"),
3109 "{msg}"
3110 );
3111 assert!(msg.contains("nothing to route"), "{msg}");
3112 }
3113
3114 #[test]
3117 fn drop_closed_passes_an_empty_batch_through() {
3118 let mut o = opts();
3119 o.ignore_closed = true;
3120 let (kept, ignored) = drop_closed(&[], &o).unwrap();
3121 assert!(kept.is_empty());
3122 assert!(ignored.is_empty());
3123 }
3124
3125 #[test]
3126 fn closed_issues_are_allowed_on_request() {
3127 let mut o = opts();
3128 o.allow_closed = true;
3129 validate_options(&[&doc(1, ItemState::Closed)], &anthropic(), &o).unwrap();
3130 }
3131
3132 #[test]
3133 fn empty_runs_and_bad_knobs_are_rejected() {
3134 assert!(validate_options(&[], &anthropic(), &opts()).is_err());
3135 let mut o = opts();
3136 o.close_call = 1.5;
3137 assert!(validate_options(&[&doc(1, ItemState::Open)], &anthropic(), &o).is_err());
3138 let mut o = opts();
3139 o.max_input_chars = 0;
3140 assert!(validate_options(&[&doc(1, ItemState::Open)], &anthropic(), &o).is_err());
3141 }
3142
3143 #[test]
3144 fn empty_and_duplicate_providers_are_rejected() {
3145 let err = validate_options(&[&doc(1, ItemState::Open)], &[], &opts()).unwrap_err();
3146 assert!(err.to_string().contains("no ladders"), "{err}");
3147 let twice = [
3148 class_only(Provider::Gemini).unwrap(),
3149 class_only(Provider::Gemini).unwrap(),
3150 ];
3151 let err = validate_options(&[&doc(1, ItemState::Open)], &twice, &opts()).unwrap_err();
3152 assert!(
3153 err.to_string()
3154 .contains("\"gemini\" is listed more than once"),
3155 "{err}"
3156 );
3157 }
3158
3159 #[test]
3162 fn open_questions_uses_the_evaluated_wording() {
3163 let evaluated: serde_json::Value = serde_json::from_str(include_str!(
3164 "../../docs/evaluations/jev-route-2053/candidate-v3.json"
3165 ))
3166 .unwrap();
3167 assert_eq!(
3168 serde_json::to_value(open_questions_question()).unwrap(),
3169 evaluated
3170 );
3171 }
3172
3173 #[test]
3174 fn open_questions_is_once_per_issue_in_both_modes() {
3175 let ladders = [
3176 Ladder::builtin(Provider::Anthropic).unwrap(),
3177 Ladder::builtin(Provider::OpenAi).unwrap(),
3178 ];
3179 for effort in [false, true] {
3180 let questions = build_route_questions_for_mode(&ladders, effort).unwrap();
3181 assert_eq!(
3182 options(&questions, "open_questions"),
3183 ["both", "design", "factual", "none"]
3184 );
3185 assert_eq!(
3186 questions
3187 .keys()
3188 .filter(|key| key.contains("open_questions"))
3189 .count(),
3190 1
3191 );
3192 assert_eq!(questions["open_questions"], open_questions_question());
3193 }
3194 }
3195
3196 #[test]
3197 fn open_questions_parses_all_kinds_without_changing_class() {
3198 let ladder = class_only(Provider::Anthropic).unwrap();
3199 for (label, kind) in [
3200 ("none", OpenQuestionKind::None),
3201 ("factual", OpenQuestionKind::Factual),
3202 ("design", OpenQuestionKind::Design),
3203 ("both", OpenQuestionKind::Both),
3204 ] {
3205 let choice = |label: &str| Answer::Choice {
3206 choice: label.into(),
3207 confidence: 0.8,
3208 probabilities: BTreeMap::from([(label.into(), 0.8)]),
3209 };
3210 let answers = BTreeMap::from([
3211 ("open_questions".into(), choice(label)),
3212 ("anthropic.stage_design".into(), choice("none")),
3213 ("anthropic.stage_implement".into(), choice("sonnet")),
3214 ("anthropic.stage_review".into(), choice("opus")),
3215 ]);
3216 let answer = open_questions_answer(&answers).unwrap();
3217 assert_eq!(answer.choice, kind);
3218 let routes =
3219 provider_routes(&answers, std::slice::from_ref(&ladder), 0.3, 0.2, false).unwrap();
3220 assert_eq!(routes["anthropic"].class, "sonnet");
3221 assert_eq!(routes["anthropic"].class_from, ClassSource::Implement);
3222 let issue = IssueRoute {
3223 draft_comments: vec![],
3224 item_ref: "o/r#1".into(),
3225 url: String::new(),
3226 title: "Example".into(),
3227 state: ItemState::Open,
3228 truncated: false,
3229 outcome: RouteOutcome::Routed {
3230 open_questions: answer,
3231 providers: routes,
3232 depends_on: vec![],
3233 reference_fetch_failures: vec![],
3234 },
3235 };
3236 let json = serde_json::to_value(&issue).unwrap();
3237 assert_eq!(json["open_questions"]["choice"], label);
3238 assert_eq!(json["open_questions"]["confidence"], 0.8);
3239 assert_eq!(json["open_questions"]["probabilities"][label], 0.8);
3240 let yaml: serde_yaml::Value =
3241 serde_yaml::from_str(&serde_yaml::to_string(&issue).unwrap()).unwrap();
3242 assert_eq!(yaml["open_questions"]["choice"].as_str(), Some(label));
3243 assert!(
3244 render_issue_block(&issue, 1000, &[], TerminalStyle::default())
3245 .contains(&format!(" open_questions: {label} (0.80)"))
3246 );
3247 }
3248 }
3249
3250 #[test]
3251 fn open_questions_rejects_absent_wrong_type_and_unknown_labels() {
3252 assert!(open_questions_answer(&BTreeMap::new())
3253 .unwrap_err()
3254 .to_string()
3255 .contains("no choice answer for open_questions"));
3256 for answer in [
3257 Answer::Noul { noul: 0.8 },
3258 Answer::Choice {
3259 choice: "unknown".into(),
3260 confidence: 0.8,
3261 probabilities: BTreeMap::new(),
3262 },
3263 Answer::Choice {
3264 choice: "none".into(),
3265 confidence: 0.8,
3266 probabilities: BTreeMap::from([("unknown".into(), 0.2)]),
3267 },
3268 ] {
3269 assert!(
3270 open_questions_answer(&BTreeMap::from([("open_questions".into(), answer)]))
3271 .unwrap_err()
3272 .to_string()
3273 .contains("open_questions")
3274 );
3275 }
3276 }
3277
3278 #[tokio::test]
3279 async fn missing_open_questions_fails_one_issue_and_batch_continues() {
3280 let server = wiremock::MockServer::start().await;
3281 for number in [7, 8] {
3282 let mut answers = serde_json::json!({
3283 "anthropic.stage_design": choice_json("none", 0.9),
3284 "anthropic.stage_implement": choice_json("sonnet", 0.9),
3285 "anthropic.stage_review": choice_json("opus", 0.9),
3286 });
3287 if number == 8 {
3288 answers["open_questions"] = choice_json("both", 0.7);
3289 }
3290 wiremock::Mock::given(wiremock::matchers::body_partial_json(serde_json::json!({"state": format!("# #{number} Route issues\n\nThe body.\n")})))
3291 .respond_with(wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!({"model": "jev-test", "answers": answers, "usage": {"input_tokens": 10, "output_tokens": 5}})))
3292 .expect(1).mount(&server).await;
3293 }
3294 let client = JevClient::new(&server.uri(), "key").unwrap();
3295 let report = run_route(
3296 &client,
3297 &[doc(7, ItemState::Open), doc(8, ItemState::Open)],
3298 &anthropic(),
3299 &opts(),
3300 &OpenDependencies::new(),
3301 )
3302 .await
3303 .unwrap();
3304 assert!(report.issues[0].failed());
3305 assert!(!report.issues[1].failed());
3306 assert_eq!(report.usage.input_tokens, 20);
3307 let json = serde_json::to_value(&report).unwrap();
3308 assert!(json["issues"][0].get("open_questions").is_none());
3309 assert_eq!(json["issues"][1]["open_questions"]["choice"], "both");
3310 }
3311
3312 fn test_open_questions() -> OpenQuestions {
3313 OpenQuestions {
3314 choice: OpenQuestionKind::Factual,
3315 confidence: 0.8,
3316 probabilities: BTreeMap::from([("factual".into(), 0.8)]),
3317 }
3318 }
3319
3320 fn choice_json(choice: &str, confidence: f64) -> serde_json::Value {
3321 serde_json::json!({
3322 "type": "choice", "choice": choice, "confidence": confidence,
3323 "probabilities": {choice: confidence}
3324 })
3325 }
3326
3327 fn provider<'a>(outcome: &'a RouteOutcome, name: &str) -> &'a ProviderRoute {
3328 let RouteOutcome::Routed { providers, .. } = outcome else {
3329 panic!("expected a routed issue: {outcome:?}");
3331 };
3332 &providers[name]
3333 }
3334
3335 #[tokio::test]
3336 async fn run_route_asks_stages_and_open_questions_per_issue_and_sums_usage() {
3337 let server = wiremock::MockServer::start().await;
3338 wiremock::Mock::given(wiremock::matchers::method("POST"))
3339 .and(wiremock::matchers::path("/v1/systemone"))
3340 .and(wiremock::matchers::body_partial_json(serde_json::json!({
3341 "model": "jev-latest",
3342 "state": "# #7 Route issues\n\nThe body.\n",
3343 })))
3344 .respond_with(
3345 wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!({
3346 "model": "jev-1.13.0",
3347 "answers": {
3348 "open_questions": choice_json("factual", 0.8),
3349 "anthropic.stage_design": choice_json("opus", 0.52),
3350 "anthropic.stage_implement": choice_json("sonnet", 0.83),
3351 "anthropic.stage_review": choice_json("opus", 0.21),
3352 },
3353 "usage": {"input_tokens": 100, "output_tokens": 10}
3354 })),
3355 )
3356 .expect(2)
3357 .mount(&server)
3358 .await;
3359 let client = JevClient::new(&server.uri(), "key").unwrap();
3360
3361 let report = run_route(
3362 &client,
3363 &[doc(7, ItemState::Open), doc(7, ItemState::Open)],
3364 &anthropic(),
3365 &opts(),
3366 &OpenDependencies::new(),
3367 )
3368 .await
3369 .unwrap();
3370
3371 assert_eq!(report.model, "jev-1.13.0");
3372 assert_eq!(report.usage.input_tokens, 200);
3373 assert_eq!(report.usage.output_tokens, 20);
3374 let issue = &report.issues[0];
3375 assert_eq!(issue.item_ref, "rust-works/omni-dev#7");
3376 let route = provider(&issue.outcome, "anthropic");
3377 assert_eq!(route.class, "opus");
3378 assert_eq!(route.class_from, ClassSource::Design);
3379 assert_eq!(route.close_calls, [Stage::Review]);
3380 assert!(!issue.truncated);
3381
3382 let requests = server.received_requests().await.unwrap();
3383 let body: serde_json::Value = requests[0].body_json().unwrap();
3384 let keys: Vec<&String> = body["questions"].as_object().unwrap().keys().collect();
3385 assert_eq!(
3386 keys,
3387 [
3388 "anthropic.stage_design",
3389 "anthropic.stage_implement",
3390 "anthropic.stage_review",
3391 "open_questions",
3392 ]
3393 );
3394 }
3395
3396 #[tokio::test]
3400 async fn run_route_ignore_closed_routes_only_open_issues() {
3401 let server = wiremock::MockServer::start().await;
3402 wiremock::Mock::given(wiremock::matchers::method("POST"))
3403 .respond_with(
3404 wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!({
3405 "model": "jev-1.13.0",
3406 "answers": {
3407 "open_questions": choice_json("factual", 0.8),
3408 "anthropic.stage_design": choice_json("opus", 0.52),
3409 "anthropic.stage_implement": choice_json("sonnet", 0.83),
3410 "anthropic.stage_review": choice_json("opus", 0.61),
3411 },
3412 "usage": {"input_tokens": 100, "output_tokens": 10}
3413 })),
3414 )
3415 .expect(2)
3416 .mount(&server)
3417 .await;
3418 let client = JevClient::new(&server.uri(), "key").unwrap();
3419 let mut o = opts();
3420 o.ignore_closed = true;
3421
3422 let report = run_route(
3423 &client,
3424 &[
3425 doc(7, ItemState::Open),
3426 doc(8, ItemState::Closed),
3427 doc(9, ItemState::Open),
3428 ],
3429 &anthropic(),
3430 &o,
3431 &OpenDependencies::new(),
3432 )
3433 .await
3434 .unwrap();
3435
3436 let refs: Vec<&str> = report.issues.iter().map(|i| i.item_ref.as_str()).collect();
3437 assert_eq!(refs, ["rust-works/omni-dev#7", "rust-works/omni-dev#9"]);
3438 let json = serde_json::to_value(&report).unwrap();
3439 assert_eq!(
3440 json["ignored_closed"],
3441 serde_json::json!([{
3442 "ref": "rust-works/omni-dev#8",
3443 "url": "https://github.com/rust-works/omni-dev/issues/8",
3444 "title": "Route issues",
3445 }])
3446 );
3447
3448 let err = run_route(
3449 &client,
3450 &[doc(8, ItemState::Closed)],
3451 &anthropic(),
3452 &o,
3453 &OpenDependencies::new(),
3454 )
3455 .await
3456 .unwrap_err();
3457 assert!(err.to_string().contains("nothing to route"), "{err}");
3458 }
3459
3460 #[test]
3463 fn route_report_omits_an_empty_ignored_closed() {
3464 let report = RouteReport {
3465 model: "m".to_string(),
3466 issues: vec![],
3467 ignored_closed: vec![],
3468 usage: Usage::default(),
3469 };
3470 let json = serde_json::to_value(&report).unwrap();
3471 assert!(json.get("ignored_closed").is_none(), "{json}");
3472 }
3473
3474 #[tokio::test]
3477 async fn run_route_routes_every_provider_in_one_call() {
3478 let server = wiremock::MockServer::start().await;
3479 wiremock::Mock::given(wiremock::matchers::method("POST"))
3480 .respond_with(
3481 wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!({
3482 "model": "jev-1.13.0",
3483 "answers": {
3484 "open_questions": choice_json("factual", 0.8),
3485 "anthropic.stage_design": choice_json("opus", 0.52),
3486 "anthropic.stage_implement": choice_json("sonnet", 0.83),
3487 "anthropic.stage_review": choice_json("opus", 0.9),
3488 "openai.stage_design": choice_json("none", 0.9),
3489 "openai.stage_implement": choice_json("sol", 0.2),
3490 "openai.stage_review": choice_json("astra", 0.9),
3491 "could_be_cheaper_0": {"type": "noul", "noul": 0.4},
3492 },
3493 "usage": {"input_tokens": 10, "output_tokens": 1}
3494 })),
3495 )
3496 .expect(1)
3497 .mount(&server)
3498 .await;
3499 let client = JevClient::new(&server.uri(), "key").unwrap();
3500 let ladders = [
3501 class_only(Provider::Anthropic).unwrap(),
3502 class_only(Provider::OpenAi).unwrap(),
3503 ];
3504 let dependencies = OpenDependencies::from([(
3505 ("rust-works/omni-dev".to_string(), 7),
3506 vec![citation("#1129", 1129)],
3507 )]);
3508 let reference_fetch_failures = ReferenceFetchFailures::from([(
3509 ("rust-works/omni-dev".to_string(), 7),
3510 vec![ReferenceFetchFailure {
3511 item_ref: "#404".to_string(),
3512 error: "not found".to_string(),
3513 }],
3514 )]);
3515
3516 let report = run_route_with_reference_fetch_failures(
3517 &client,
3518 &[doc(7, ItemState::Open)],
3519 &ladders,
3520 &opts(),
3521 &dependencies,
3522 &reference_fetch_failures,
3523 )
3524 .await
3525 .unwrap();
3526
3527 let outcome = &report.issues[0].outcome;
3528 let RouteOutcome::Routed {
3529 providers,
3530 depends_on,
3531 reference_fetch_failures,
3532 ..
3533 } = outcome
3534 else {
3535 panic!("expected a routed issue: {outcome:?}");
3537 };
3538 assert_eq!(
3539 providers.keys().collect::<Vec<_>>(),
3540 ["anthropic", "openai"]
3541 );
3542 assert_eq!(provider(outcome, "anthropic").class, "opus");
3543 assert_eq!(
3544 provider(outcome, "anthropic").class_from,
3545 ClassSource::Design
3546 );
3547 assert!(provider(outcome, "anthropic").close_calls.is_empty());
3548 assert_eq!(provider(outcome, "openai").class, "sol");
3549 assert_eq!(
3550 provider(outcome, "openai").class_from,
3551 ClassSource::Implement
3552 );
3553 assert_eq!(provider(outcome, "openai").close_calls, [Stage::Implement]);
3554 assert_eq!(depends_on.len(), 1);
3555 assert_eq!(reference_fetch_failures[0].item_ref, "#404");
3556
3557 let requests = server.received_requests().await.unwrap();
3558 let body: serde_json::Value = requests[0].body_json().unwrap();
3559 let keys: Vec<&String> = body["questions"].as_object().unwrap().keys().collect();
3560 assert_eq!(
3561 keys,
3562 [
3563 "anthropic.stage_design",
3564 "anthropic.stage_implement",
3565 "anthropic.stage_review",
3566 "could_be_cheaper_0",
3567 "open_questions",
3568 "openai.stage_design",
3569 "openai.stage_implement",
3570 "openai.stage_review",
3571 ]
3572 );
3573 }
3574
3575 #[tokio::test]
3578 async fn run_route_fails_the_issue_when_one_provider_is_unanswered() {
3579 let server = wiremock::MockServer::start().await;
3580 wiremock::Mock::given(wiremock::matchers::method("POST"))
3581 .respond_with(wiremock::ResponseTemplate::new(200).set_body_json(routed_json()))
3582 .mount(&server)
3583 .await;
3584 let client = JevClient::new(&server.uri(), "key").unwrap();
3585 let ladders = [
3586 class_only(Provider::Anthropic).unwrap(),
3587 class_only(Provider::Gemini).unwrap(),
3588 ];
3589 let report = run_route(
3590 &client,
3591 &[doc(7, ItemState::Open)],
3592 &ladders,
3593 &opts(),
3594 &OpenDependencies::new(),
3595 )
3596 .await
3597 .unwrap();
3598 let RouteOutcome::Failed { error, .. } = &report.issues[0].outcome else {
3599 panic!("expected a failed issue: {:?}", report.issues[0]);
3601 };
3602 assert!(error.contains("gemini.stage_design"), "{error}");
3603 }
3604
3605 fn routed_json() -> serde_json::Value {
3606 serde_json::json!({
3607 "model": "jev-1.13.0",
3608 "answers": {
3609 "open_questions": choice_json("factual", 0.8),
3610 "anthropic.stage_design": choice_json("none", 0.9),
3611 "anthropic.stage_implement": choice_json("sonnet", 0.9),
3612 "anthropic.stage_review": choice_json("opus", 0.9),
3613 },
3614 "usage": {"input_tokens": 5, "output_tokens": 1}
3615 })
3616 }
3617
3618 #[tokio::test]
3620 async fn run_route_keeps_going_past_a_failed_issue() {
3621 let server = wiremock::MockServer::start().await;
3622 wiremock::Mock::given(wiremock::matchers::method("POST"))
3623 .and(wiremock::matchers::body_partial_json(
3624 serde_json::json!({"state": "# #1 Route issues\n\nThe body.\n"}),
3625 ))
3626 .respond_with(wiremock::ResponseTemplate::new(500).set_body_string("boom"))
3627 .mount(&server)
3628 .await;
3629 wiremock::Mock::given(wiremock::matchers::method("POST"))
3630 .respond_with(wiremock::ResponseTemplate::new(200).set_body_json(routed_json()))
3631 .mount(&server)
3632 .await;
3633 let client = JevClient::new(&server.uri(), "key").unwrap();
3634 let reference_fetch_failures = ReferenceFetchFailures::from([(
3635 ("rust-works/omni-dev".to_string(), 1),
3636 vec![ReferenceFetchFailure {
3637 item_ref: "#404".to_string(),
3638 error: "not found".to_string(),
3639 }],
3640 )]);
3641 let report = run_route_with_reference_fetch_failures(
3642 &client,
3643 &[doc(1, ItemState::Open), doc(2, ItemState::Open)],
3644 &anthropic(),
3645 &opts(),
3646 &OpenDependencies::new(),
3647 &reference_fetch_failures,
3648 )
3649 .await
3650 .unwrap();
3651
3652 assert!(report.issues[0].failed());
3653 let RouteOutcome::Failed {
3654 error,
3655 reference_fetch_failures,
3656 } = &report.issues[0].outcome
3657 else {
3658 panic!();
3659 };
3660 assert!(error.contains("500"), "{error}");
3661 assert_eq!(reference_fetch_failures[0].item_ref, "#404");
3662 let json = serde_json::to_value(&report).unwrap();
3663 assert_eq!(
3664 json["issues"][0]["reference_fetch_failures"][0]["ref"],
3665 "#404"
3666 );
3667 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
3668 assert!(
3669 text.contains(" reference fetch failed: #404 (not found)"),
3670 "{text}"
3671 );
3672 assert!(!report.issues[1].failed());
3673 assert_eq!(report.usage.input_tokens, 5);
3674 }
3675
3676 #[tokio::test]
3677 async fn run_route_stops_on_an_auth_failure() {
3678 let server = wiremock::MockServer::start().await;
3679 wiremock::Mock::given(wiremock::matchers::method("POST"))
3680 .respond_with(wiremock::ResponseTemplate::new(401).set_body_string("bad key"))
3681 .expect(1)
3682 .mount(&server)
3683 .await;
3684 let client = JevClient::new(&server.uri(), "key").unwrap();
3685 let err = run_route(
3686 &client,
3687 &[doc(3, ItemState::Open), doc(4, ItemState::Open)],
3688 &anthropic(),
3689 &opts(),
3690 &OpenDependencies::new(),
3691 )
3692 .await
3693 .unwrap_err();
3694 assert!(format!("{err:#}").contains("Failed to route issue rust-works/omni-dev#3"));
3695 }
3696
3697 #[tokio::test]
3698 async fn run_route_rejects_a_ladder_over_the_choice_cap_before_any_request() {
3699 let yaml: String = std::iter::once("tiers:\n".to_string())
3700 .chain((0..256).map(|i| format!(" - {{name: t{i}, description: d{i}}}\n")))
3701 .collect();
3702 let ladder = Ladder::named("big".to_string(), Tiers::parse(&yaml).unwrap());
3703 let client = JevClient::new("http://127.0.0.1:1", "key").unwrap();
3704 let err = run_route(
3705 &client,
3706 &[doc(3, ItemState::Open)],
3707 &[ladder],
3708 &opts(),
3709 &OpenDependencies::new(),
3710 )
3711 .await
3712 .unwrap_err();
3713 let chain = format!("{err:#}");
3714 assert!(chain.contains("route question \"big.stage_"), "{chain}");
3715 assert!(
3716 chain.contains("at most 255 options on jev-latest"),
3717 "{chain}"
3718 );
3719 }
3720
3721 #[tokio::test]
3722 async fn run_route_names_the_issue_on_an_unexpected_answer() {
3723 let server = wiremock::MockServer::start().await;
3724 wiremock::Mock::given(wiremock::matchers::method("POST"))
3725 .respond_with(
3726 wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!({
3727 "model": "jev-1.13.0",
3728 "answers": {
3729 "open_questions": choice_json("factual", 0.8),
3730 "anthropic.stage_design": choice_json("haiku", 0.9),
3731 "anthropic.stage_implement": choice_json("sonnet", 0.9),
3732 "anthropic.stage_review": choice_json("opus", 0.9),
3733 },
3734 })),
3735 )
3736 .mount(&server)
3737 .await;
3738 let client = JevClient::new(&server.uri(), "key").unwrap();
3739 let report = run_route(
3740 &client,
3741 &[doc(9, ItemState::Open)],
3742 &anthropic(),
3743 &opts(),
3744 &OpenDependencies::new(),
3745 )
3746 .await
3747 .unwrap();
3748 let RouteOutcome::Failed { error, .. } = &report.issues[0].outcome else {
3749 panic!("expected a failed issue");
3750 };
3751 assert!(error.contains("Unexpected Jev answer"), "{error}");
3752 assert!(error.contains("not offered"), "{error}");
3753 }
3754
3755 fn citation(raw: &str, number: u64) -> Citation {
3756 Citation {
3757 item_ref: ItemRef {
3758 provider: GitProvider::GitHub,
3759 project: "rust-works/omni-dev".to_string(),
3760 kind: ItemKind::Issue,
3761 number,
3762 },
3763 raw: raw.to_string(),
3764 url: None,
3765 }
3766 }
3767
3768 #[tokio::test]
3769 async fn run_route_asks_could_be_cheaper_for_each_open_citation() {
3770 let server = wiremock::MockServer::start().await;
3771 wiremock::Mock::given(wiremock::matchers::method("POST"))
3772 .respond_with(
3773 wiremock::ResponseTemplate::new(200).set_body_json(serde_json::json!({
3774 "model": "jev-1.13.0",
3775 "answers": {
3776 "open_questions": choice_json("factual", 0.8),
3777 "anthropic.stage_design": choice_json("opus", 0.2),
3778 "anthropic.stage_implement": choice_json("sonnet", 0.61),
3779 "anthropic.stage_review": choice_json("opus", 0.9),
3780 "could_be_cheaper_0": {"type": "noul", "noul": 0.75},
3781 },
3782 "usage": {"input_tokens": 10, "output_tokens": 1}
3783 })),
3784 )
3785 .mount(&server)
3786 .await;
3787 let client = JevClient::new(&server.uri(), "key").unwrap();
3788 let dependencies = OpenDependencies::from([(
3789 ("rust-works/omni-dev".to_string(), 7),
3790 vec![citation("#1129", 1129)],
3791 )]);
3792
3793 let report = run_route(
3794 &client,
3795 &[doc(7, ItemState::Open)],
3796 &anthropic(),
3797 &opts(),
3798 &dependencies,
3799 )
3800 .await
3801 .unwrap();
3802
3803 let RouteOutcome::Routed { depends_on, .. } = &report.issues[0].outcome else {
3804 panic!("expected a routed issue: {:?}", report.issues[0]);
3805 };
3806 assert_eq!(depends_on.len(), 1);
3807 assert_eq!(depends_on[0].item_ref, "#1129");
3808 assert_eq!(depends_on[0].state, ItemState::Open);
3809 assert_eq!(depends_on[0].could_be_cheaper.get("design"), Some(&0.75));
3810
3811 let requests = server.received_requests().await.unwrap();
3812 let body: serde_json::Value = requests[0].body_json().unwrap();
3813 let keys: Vec<&String> = body["questions"].as_object().unwrap().keys().collect();
3814 assert_eq!(
3815 keys,
3816 [
3817 "anthropic.stage_design",
3818 "anthropic.stage_implement",
3819 "anthropic.stage_review",
3820 "could_be_cheaper_0",
3821 "open_questions",
3822 ]
3823 );
3824 }
3825
3826 #[tokio::test]
3827 async fn run_route_keeps_a_dependency_with_no_could_be_cheaper_answer() {
3828 let server = wiremock::MockServer::start().await;
3829 wiremock::Mock::given(wiremock::matchers::method("POST"))
3830 .respond_with(wiremock::ResponseTemplate::new(200).set_body_json(routed_json()))
3831 .mount(&server)
3832 .await;
3833 let client = JevClient::new(&server.uri(), "key").unwrap();
3834 let dependencies = OpenDependencies::from([(
3835 ("rust-works/omni-dev".to_string(), 7),
3836 vec![citation("#1129", 1129)],
3837 )]);
3838
3839 let report = run_route(
3840 &client,
3841 &[doc(7, ItemState::Open)],
3842 &anthropic(),
3843 &opts(),
3844 &dependencies,
3845 )
3846 .await
3847 .unwrap();
3848
3849 let RouteOutcome::Routed { depends_on, .. } = &report.issues[0].outcome else {
3852 panic!("expected a routed issue: {:?}", report.issues[0]);
3853 };
3854 assert_eq!(depends_on.len(), 1, "{depends_on:?}");
3855 assert_eq!(depends_on[0].item_ref, "#1129");
3856 assert!(depends_on[0].could_be_cheaper.is_empty());
3857 }
3858
3859 #[test]
3860 fn report_serialises_ref_and_omits_false_truncated() {
3861 let report = RouteReport {
3862 model: "jev-1.13.0".to_string(),
3863 issues: vec![IssueRoute {
3864 draft_comments: vec![],
3865 item_ref: "o/r#1".to_string(),
3866 url: "u".to_string(),
3867 title: "t".to_string(),
3868 state: ItemState::Open,
3869 outcome: RouteOutcome::Routed {
3870 open_questions: test_open_questions(),
3871 providers: BTreeMap::from([(
3872 "anthropic".to_string(),
3873 ProviderRoute {
3874 stages: stages("none", "sonnet", "opus"),
3875 class: "sonnet".to_string(),
3876 class_from: ClassSource::Implement,
3877 close_calls: vec![],
3878 },
3879 )]),
3880 depends_on: vec![],
3881 reference_fetch_failures: vec![],
3882 },
3883 truncated: false,
3884 }],
3885 ignored_closed: vec![],
3886 usage: Usage::default(),
3887 };
3888 let value = serde_json::to_value(&report).unwrap();
3889 assert_eq!(value["issues"][0]["ref"], "o/r#1");
3890 assert!(value["issues"][0].get("truncated").is_none());
3891 let anthropic = &value["issues"][0]["providers"]["anthropic"];
3892 assert_eq!(anthropic["stages"]["design"]["choice"], "none");
3893 assert_eq!(anthropic["class"], "sonnet");
3894 assert_eq!(anthropic["close_calls"], serde_json::json!([]));
3895 assert!(value["issues"][0].get("stages").is_none());
3896 assert!(value["issues"][0].get("class").is_none());
3897 assert_eq!(value["issues"][0]["depends_on"], serde_json::json!([]));
3898 assert!(value["issues"][0].get("reference_fetch_failures").is_none());
3899 assert!(value["issues"][0].get("error").is_none());
3900 }
3901
3902 #[test]
3903 fn a_failed_issue_serialises_only_its_error() {
3904 let route = IssueRoute {
3905 draft_comments: vec![],
3906 item_ref: "o/r#1".to_string(),
3907 url: "u".to_string(),
3908 title: "t".to_string(),
3909 state: ItemState::Open,
3910 outcome: RouteOutcome::Failed {
3911 error: "HTTP 529".to_string(),
3912 reference_fetch_failures: vec![],
3913 },
3914 truncated: false,
3915 };
3916 let value = serde_json::to_value(&route).unwrap();
3917 assert_eq!(value["error"], "HTTP 529");
3918 assert!(value.get("providers").is_none());
3919 assert!(value.get("depends_on").is_none());
3920 }
3921
3922 #[test]
3925 fn render_route_text_matches_the_worked_example() {
3926 let report = RouteReport {
3927 model: "jev-1.13.0".to_string(),
3928 issues: vec![IssueRoute {
3929 draft_comments: vec![],
3930 item_ref: "rust-works/omni-dev#1641".to_string(),
3931 url: "u".to_string(),
3932 title: "Some issue title".to_string(),
3933 state: ItemState::Open,
3934 outcome: RouteOutcome::Routed {
3935 open_questions: test_open_questions(),
3936 providers: BTreeMap::from([(
3937 "anthropic".to_string(),
3938 ProviderRoute {
3939 stages: StageAnswers {
3940 design: answer("opus", 0.52),
3941 implement: answer("sonnet", 0.83),
3942 review: answer("opus", 0.41),
3943 },
3944 class: "opus".to_string(),
3945 class_from: ClassSource::Design,
3946 close_calls: vec![Stage::Review],
3947 },
3948 )]),
3949 depends_on: vec![DependencyEntry {
3950 item_ref: "#1129".to_string(),
3951 url: None,
3952 state: ItemState::Open,
3953 relation: CitationRelation::Unspecified,
3954 could_be_cheaper: BTreeMap::from([("design".to_string(), 0.75)]),
3955 }],
3956 reference_fetch_failures: vec![],
3957 },
3958 truncated: false,
3959 }],
3960 ignored_closed: vec![],
3961 usage: Usage {
3962 input_tokens: 1432,
3963 output_tokens: 61,
3964 },
3965 };
3966 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
3967 assert_eq!(
3968 text,
3969 "rust-works/omni-dev#1641 — Some issue title\n\
3970 \x20\x20open_questions: factual (0.80)\n\
3971 \x20\x20opus (from design) — design needs opus (0.52), implementation sonnet (0.83), review \
3972 opus (0.41, close call)\n\
3973 \x20\x20cites open #1129 (relation unspecified), which could leave less design work if resolved (0.75)\n\n\
3974 model: jev-1.13.0, usage: 1432 input tokens, 61 output tokens\n"
3975 );
3976 }
3977
3978 #[test]
3979 fn render_route_text_puts_each_open_citation_on_its_own_line() {
3980 let report = RouteReport {
3981 model: "jev-1.13.0".to_string(),
3982 issues: vec![IssueRoute {
3983 draft_comments: vec![],
3984 item_ref: "rust-works/omni-dev#1845".to_string(),
3985 url: "u".to_string(),
3986 title: "t".to_string(),
3987 state: ItemState::Open,
3988 outcome: RouteOutcome::Routed {
3989 open_questions: test_open_questions(),
3990 providers: BTreeMap::from([(
3991 "anthropic".to_string(),
3992 ProviderRoute {
3993 stages: stages("none", "sonnet", "sonnet"),
3994 class: "sonnet".to_string(),
3995 class_from: ClassSource::Implement,
3996 close_calls: vec![],
3997 },
3998 )]),
3999 depends_on: vec![
4000 DependencyEntry {
4001 item_ref: "#1830".to_string(),
4002 url: None,
4003 state: ItemState::Open,
4004 relation: CitationRelation::Unspecified,
4005 could_be_cheaper: BTreeMap::from([("design".to_string(), 0.46)]),
4006 },
4007 DependencyEntry {
4008 item_ref: "#1831".to_string(),
4009 url: None,
4010 state: ItemState::Open,
4011 relation: CitationRelation::Unspecified,
4012 could_be_cheaper: BTreeMap::new(),
4013 },
4014 ],
4015 reference_fetch_failures: vec![],
4016 },
4017 truncated: false,
4018 }],
4019 ignored_closed: vec![],
4020 usage: Usage::default(),
4021 };
4022 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
4023 assert!(
4024 text.contains(
4025 " cites open #1830 (relation unspecified), which could leave less design work if resolved (0.46)\n \
4026 cites open #1831 (relation unspecified)\n"
4027 ),
4028 "{text}"
4029 );
4030 assert!(!text.contains("; "), "{text}");
4031 }
4032
4033 #[test]
4034 fn render_depends_on_lines_labels_each_citation_relation() {
4035 let entry = |item_ref: &str, relation| DependencyEntry {
4036 item_ref: item_ref.to_string(),
4037 url: None,
4038 state: ItemState::Open,
4039 relation,
4040 could_be_cheaper: BTreeMap::new(),
4041 };
4042 let lines = render_depends_on_lines(
4043 &[
4044 entry("#1", CitationRelation::Blocker),
4045 entry("#2", CitationRelation::Tracker),
4046 entry("#3", CitationRelation::Unspecified),
4047 ],
4048 TerminalStyle::default(),
4049 );
4050 assert_eq!(
4051 lines,
4052 [
4053 " cites open #1 (blocker)",
4054 " cites open #2 (tracker)",
4055 " cites open #3 (relation unspecified)",
4056 ]
4057 );
4058 }
4059
4060 #[test]
4061 fn render_route_text_names_each_provider_when_there_are_several() {
4062 let route = |class: &str| ProviderRoute {
4063 stages: stages("none", "sonnet", "sonnet"),
4064 class: class.to_string(),
4065 class_from: ClassSource::Implement,
4066 close_calls: vec![],
4067 };
4068 let report = RouteReport {
4069 model: "jev-1.13.0".to_string(),
4070 issues: vec![IssueRoute {
4071 draft_comments: vec![],
4072 item_ref: "o/r#1".to_string(),
4073 url: "u".to_string(),
4074 title: "t".to_string(),
4075 state: ItemState::Open,
4076 outcome: RouteOutcome::Routed {
4077 open_questions: test_open_questions(),
4078 providers: BTreeMap::from([
4079 ("anthropic".to_string(), route("sonnet")),
4080 ("openai".to_string(), route("terra")),
4081 ]),
4082 depends_on: vec![],
4083 reference_fetch_failures: vec![],
4084 },
4085 truncated: false,
4086 }],
4087 ignored_closed: vec![],
4088 usage: Usage::default(),
4089 };
4090 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
4091 assert!(
4092 text.contains(" anthropic: sonnet (from implementation) —"),
4093 "{text}"
4094 );
4095 assert!(
4096 text.contains(" openai: terra (from implementation) —"),
4097 "{text}"
4098 );
4099 }
4100
4101 #[test]
4102 fn render_route_text_reports_a_failed_issue() {
4103 let report = RouteReport {
4104 model: "jev-1.13.0".to_string(),
4105 issues: vec![IssueRoute {
4106 draft_comments: vec![],
4107 item_ref: "o/r#1".to_string(),
4108 url: "u".to_string(),
4109 title: "t".to_string(),
4110 state: ItemState::Open,
4111 outcome: RouteOutcome::Failed {
4112 error: "HTTP 529".to_string(),
4113 reference_fetch_failures: vec![],
4114 },
4115 truncated: false,
4116 }],
4117 ignored_closed: vec![],
4118 usage: Usage::default(),
4119 };
4120 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
4121 assert!(text.contains("o/r#1 — t\n failed: HTTP 529"), "{text}");
4122 }
4123
4124 #[test]
4125 fn render_route_text_marks_a_closed_issue_in_the_header() {
4126 let report = RouteReport {
4127 model: "jev-1.13.0".to_string(),
4128 issues: vec![IssueRoute {
4129 draft_comments: vec![],
4130 item_ref: "o/r#1".to_string(),
4131 url: "u".to_string(),
4132 title: "t".to_string(),
4133 state: ItemState::Closed,
4134 outcome: RouteOutcome::Failed {
4135 error: "HTTP 529".to_string(),
4136 reference_fetch_failures: vec![],
4137 },
4138 truncated: false,
4139 }],
4140 ignored_closed: vec![],
4141 usage: Usage::default(),
4142 };
4143 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
4144 assert!(text.starts_with("o/r#1 — t (closed)\n"), "{text}");
4145 assert!(!text.contains("ignored closed"), "{text}");
4146 }
4147
4148 #[test]
4149 fn render_route_text_lists_ignored_closed_issues_before_the_footer() {
4150 let ignored = |n: u64| IgnoredIssue {
4151 item_ref: format!("o/r#{n}"),
4152 url: format!("https://github.com/o/r/issues/{n}"),
4153 title: "t".to_string(),
4154 };
4155 let report = RouteReport {
4156 model: "jev-1.13.0".to_string(),
4157 issues: vec![],
4158 ignored_closed: vec![ignored(2), ignored(3)],
4159 usage: Usage::default(),
4160 };
4161 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
4162 assert!(
4163 text.contains("ignored closed: o/r#2, o/r#3\n\nmodel: jev-1.13.0"),
4164 "{text}"
4165 );
4166 let styled = render_route_text_styled(
4167 &report,
4168 DEFAULT_MAX_INPUT_CHARS,
4169 &[],
4170 TerminalStyle {
4171 color: false,
4172 hyperlinks: true,
4173 },
4174 );
4175 assert!(
4176 styled.contains("https://github.com/o/r/issues/2"),
4177 "{styled}"
4178 );
4179 }
4180
4181 #[test]
4182 fn render_route_text_omits_the_depends_on_sentence_when_empty() {
4183 let report = RouteReport {
4184 model: "jev-1.13.0".to_string(),
4185 issues: vec![IssueRoute {
4186 draft_comments: vec![],
4187 item_ref: "o/r#1".to_string(),
4188 url: "u".to_string(),
4189 title: "t".to_string(),
4190 state: ItemState::Open,
4191 outcome: RouteOutcome::Routed {
4192 open_questions: test_open_questions(),
4193 providers: BTreeMap::from([(
4194 "anthropic".to_string(),
4195 ProviderRoute {
4196 stages: stages("none", "sonnet", "sonnet"),
4197 class: "sonnet".to_string(),
4198 class_from: ClassSource::Implement,
4199 close_calls: vec![],
4200 },
4201 )]),
4202 depends_on: vec![],
4203 reference_fetch_failures: vec![],
4204 },
4205 truncated: false,
4206 }],
4207 ignored_closed: vec![],
4208 usage: Usage::default(),
4209 };
4210 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
4211 assert!(!text.contains("cites"), "{text}");
4212 }
4213
4214 #[test]
4218 fn render_route_text_cites_a_dependency_with_no_could_be_cheaper_probability() {
4219 let report = RouteReport {
4220 model: "jev-1.13.0".to_string(),
4221 issues: vec![IssueRoute {
4222 draft_comments: vec![],
4223 item_ref: "o/r#1".to_string(),
4224 url: "u".to_string(),
4225 title: "t".to_string(),
4226 state: ItemState::Open,
4227 outcome: RouteOutcome::Routed {
4228 open_questions: test_open_questions(),
4229 providers: BTreeMap::from([(
4230 "anthropic".to_string(),
4231 ProviderRoute {
4232 stages: stages("none", "sonnet", "sonnet"),
4233 class: "sonnet".to_string(),
4234 class_from: ClassSource::Implement,
4235 close_calls: vec![],
4236 },
4237 )]),
4238 depends_on: vec![DependencyEntry {
4239 item_ref: "#1129".to_string(),
4240 url: None,
4241 state: ItemState::Open,
4242 relation: CitationRelation::Unspecified,
4243 could_be_cheaper: BTreeMap::new(),
4244 }],
4245 reference_fetch_failures: vec![],
4246 },
4247 truncated: false,
4248 }],
4249 ignored_closed: vec![],
4250 usage: Usage::default(),
4251 };
4252 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
4253 assert!(text.contains("cites open #1129"), "{text}");
4254 assert!(!text.contains("could leave less design work"), "{text}");
4255 }
4256
4257 #[test]
4258 fn render_route_text_notes_a_truncated_issue_and_omits_the_note_otherwise() {
4259 let issue = |truncated: bool| IssueRoute {
4260 draft_comments: vec![],
4261 item_ref: "o/r#1".to_string(),
4262 url: "u".to_string(),
4263 title: "t".to_string(),
4264 state: ItemState::Open,
4265 outcome: RouteOutcome::Routed {
4266 open_questions: test_open_questions(),
4267 providers: BTreeMap::from([(
4268 "anthropic".to_string(),
4269 ProviderRoute {
4270 stages: stages("none", "sonnet", "sonnet"),
4271 class: "sonnet".to_string(),
4272 class_from: ClassSource::Implement,
4273 close_calls: vec![],
4274 },
4275 )]),
4276 depends_on: vec![],
4277 reference_fetch_failures: vec![],
4278 },
4279 truncated,
4280 };
4281 let report = |truncated| RouteReport {
4282 model: "jev-1.13.0".to_string(),
4283 issues: vec![issue(truncated)],
4284 ignored_closed: vec![],
4285 usage: Usage::default(),
4286 };
4287 let truncated_text = render_route_text(&report(true), 60_000);
4288 assert!(
4289 truncated_text.contains(" input truncated at 60,000 characters"),
4290 "{truncated_text}"
4291 );
4292 let kept_text = render_route_text(&report(false), 60_000);
4293 assert!(!kept_text.contains("truncated"), "{kept_text}");
4294 }
4295
4296 #[test]
4297 fn render_route_text_notes_no_further_design_work() {
4298 let report = RouteReport {
4299 model: "jev-1.13.0".to_string(),
4300 issues: vec![IssueRoute {
4301 draft_comments: vec![],
4302 item_ref: "o/r#1".to_string(),
4303 url: "u".to_string(),
4304 title: "t".to_string(),
4305 state: ItemState::Open,
4306 outcome: RouteOutcome::Routed {
4307 open_questions: test_open_questions(),
4308 providers: BTreeMap::from([(
4309 "anthropic".to_string(),
4310 ProviderRoute {
4311 stages: stages("none", "sonnet", "sonnet"),
4312 class: "sonnet".to_string(),
4313 class_from: ClassSource::Implement,
4314 close_calls: vec![],
4315 },
4316 )]),
4317 depends_on: vec![],
4318 reference_fetch_failures: vec![],
4319 },
4320 truncated: false,
4321 }],
4322 ignored_closed: vec![],
4323 usage: Usage::default(),
4324 };
4325 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
4326 assert!(text.contains("design needs no further work"), "{text}");
4327 }
4328
4329 #[test]
4333 fn render_route_text_switches_to_one_line_per_stage_for_multi_model_tiers() {
4334 let multi = "global.anthropic.claude-sonnet-4-6,global.anthropic.claude-sonnet-5";
4335 let report = RouteReport {
4336 model: "jev-1.13.0".to_string(),
4337 issues: vec![IssueRoute {
4338 draft_comments: vec![],
4339 item_ref: "rust-works/omni-dev#1832".to_string(),
4340 url: "u".to_string(),
4341 title: "feat(drive): banded ranges for drive sheets (#1830)".to_string(),
4342 state: ItemState::Open,
4343 outcome: RouteOutcome::Routed {
4344 open_questions: test_open_questions(),
4345 providers: BTreeMap::from([(
4346 "anthropic".to_string(),
4347 ProviderRoute {
4348 stages: StageAnswers {
4349 design: answer(NO_DESIGN, 0.94),
4350 implement: answer(multi, 0.94),
4351 review: answer(multi, 0.70),
4352 },
4353 class: multi.to_string(),
4354 class_from: ClassSource::Implement,
4355 close_calls: vec![],
4356 },
4357 )]),
4358 depends_on: vec![DependencyEntry {
4359 item_ref: "#1830".to_string(),
4360 url: None,
4361 state: ItemState::Open,
4362 relation: CitationRelation::Unspecified,
4363 could_be_cheaper: BTreeMap::from([("design".to_string(), 0.48)]),
4364 }],
4365 reference_fetch_failures: vec![],
4366 },
4367 truncated: false,
4368 }],
4369 ignored_closed: vec![],
4370 usage: Usage::default(),
4371 };
4372 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
4373 assert_eq!(
4374 text,
4375 format!(
4376 "rust-works/omni-dev#1832 — feat(drive): banded ranges for drive sheets (#1830)\n\
4377 \x20\x20open_questions: factual (0.80)\n\
4378 \x20\x20class: {multi} (from implementation)\n\
4379 \x20\x20design: needs no further work (0.94)\n\
4380 \x20\x20implementation: {multi} (0.94)\n\
4381 \x20\x20review: {multi} (0.70)\n\
4382 \x20\x20cites open #1830 (relation unspecified), which could leave less design work if resolved (0.48)\n\n\
4383 model: jev-1.13.0, usage: 0 input tokens, 0 output tokens\n"
4384 )
4385 );
4386 }
4387
4388 #[test]
4391 fn render_route_text_keeps_the_compact_line_for_single_word_tiers() {
4392 let report = RouteReport {
4393 model: "jev-1.13.0".to_string(),
4394 issues: vec![IssueRoute {
4395 draft_comments: vec![],
4396 item_ref: "o/r#1".to_string(),
4397 url: "u".to_string(),
4398 title: "t".to_string(),
4399 state: ItemState::Open,
4400 outcome: RouteOutcome::Routed {
4401 open_questions: test_open_questions(),
4402 providers: BTreeMap::from([(
4403 "anthropic".to_string(),
4404 ProviderRoute {
4405 stages: stages("opus", "sonnet", "opus"),
4406 class: "opus".to_string(),
4407 class_from: ClassSource::Design,
4408 close_calls: vec![],
4409 },
4410 )]),
4411 depends_on: vec![],
4412 reference_fetch_failures: vec![],
4413 },
4414 truncated: false,
4415 }],
4416 ignored_closed: vec![],
4417 usage: Usage::default(),
4418 };
4419 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
4420 assert!(
4421 text.contains(
4422 " opus (from design) — design needs opus (0.90), implementation sonnet (0.90), \
4423 review opus (0.90)"
4424 ),
4425 "{text}"
4426 );
4427 assert!(!text.contains("class:"), "{text}");
4428 }
4429
4430 #[test]
4433 fn render_route_text_headers_the_block_by_provider_when_multi_model_and_multiple_ladders() {
4434 let route = |class: &str| ProviderRoute {
4435 stages: StageAnswers {
4436 design: answer(NO_DESIGN, 0.9),
4437 implement: answer(class, 0.9),
4438 review: answer(class, 0.9),
4439 },
4440 class: class.to_string(),
4441 class_from: ClassSource::Implement,
4442 close_calls: vec![],
4443 };
4444 let report = RouteReport {
4445 model: "jev-1.13.0".to_string(),
4446 issues: vec![IssueRoute {
4447 draft_comments: vec![],
4448 item_ref: "o/r#1".to_string(),
4449 url: "u".to_string(),
4450 title: "t".to_string(),
4451 state: ItemState::Open,
4452 outcome: RouteOutcome::Routed {
4453 open_questions: test_open_questions(),
4454 providers: BTreeMap::from([
4455 ("anthropic".to_string(), route("a,b")),
4456 ("openai".to_string(), route("c,d")),
4457 ]),
4458 depends_on: vec![],
4459 reference_fetch_failures: vec![],
4460 },
4461 truncated: false,
4462 }],
4463 ignored_closed: vec![],
4464 usage: Usage::default(),
4465 };
4466 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
4467 assert!(
4468 text.contains(" anthropic:\n class: a,b (from implementation)\n"),
4469 "{text}"
4470 );
4471 assert!(text.contains(" implementation: a,b (0.90)"), "{text}");
4472 assert!(
4473 text.contains(" openai:\n class: c,d (from implementation)\n"),
4474 "{text}"
4475 );
4476 assert!(text.contains(" implementation: c,d (0.90)"), "{text}");
4477 }
4478
4479 #[test]
4483 fn render_route_text_multi_model_layout_is_decided_per_ladder() {
4484 let multi_route = ProviderRoute {
4485 stages: StageAnswers {
4486 design: answer(NO_DESIGN, 0.9),
4487 implement: answer("a,b", 0.9),
4488 review: answer("a,b", 0.9),
4489 },
4490 class: "a,b".to_string(),
4491 class_from: ClassSource::Implement,
4492 close_calls: vec![],
4493 };
4494 let compact_route = ProviderRoute {
4495 stages: stages("none", "terra", "terra"),
4496 class: "terra".to_string(),
4497 class_from: ClassSource::Implement,
4498 close_calls: vec![],
4499 };
4500 let report = RouteReport {
4501 model: "jev-1.13.0".to_string(),
4502 issues: vec![IssueRoute {
4503 draft_comments: vec![],
4504 item_ref: "o/r#1".to_string(),
4505 url: "u".to_string(),
4506 title: "t".to_string(),
4507 state: ItemState::Open,
4508 outcome: RouteOutcome::Routed {
4509 open_questions: test_open_questions(),
4510 providers: BTreeMap::from([
4511 ("anthropic".to_string(), multi_route),
4512 ("openai".to_string(), compact_route),
4513 ]),
4514 depends_on: vec![],
4515 reference_fetch_failures: vec![],
4516 },
4517 truncated: false,
4518 }],
4519 ignored_closed: vec![],
4520 usage: Usage::default(),
4521 };
4522 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
4523 assert!(
4524 text.contains(" anthropic:\n class: a,b (from implementation)\n"),
4525 "{text}"
4526 );
4527 assert!(
4528 text.contains(" openai: terra (from implementation) — design needs no further work"),
4529 "{text}"
4530 );
4531 }
4532
4533 #[test]
4535 fn render_route_text_notes_a_close_call_in_the_multi_line_layout() {
4536 let mut review = answer("a,b", 0.41);
4537 review.probabilities =
4538 BTreeMap::from([("a,b".to_string(), 0.56), ("c,d".to_string(), 0.44)]);
4539 let report = RouteReport {
4540 model: "jev-1.13.0".to_string(),
4541 issues: vec![IssueRoute {
4542 draft_comments: vec![],
4543 item_ref: "o/r#1".to_string(),
4544 url: "u".to_string(),
4545 title: "t".to_string(),
4546 state: ItemState::Open,
4547 outcome: RouteOutcome::Routed {
4548 open_questions: test_open_questions(),
4549 providers: BTreeMap::from([(
4550 "anthropic".to_string(),
4551 ProviderRoute {
4552 stages: StageAnswers {
4553 design: answer(NO_DESIGN, 0.9),
4554 implement: answer("a,b", 0.94),
4555 review,
4556 },
4557 class: "a,b".to_string(),
4558 class_from: ClassSource::Implement,
4559 close_calls: vec![Stage::Review],
4560 },
4561 )]),
4562 depends_on: vec![],
4563 reference_fetch_failures: vec![],
4564 },
4565 truncated: false,
4566 }],
4567 ignored_closed: vec![],
4568 usage: Usage::default(),
4569 };
4570 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
4571 assert!(
4572 text.contains("review: a,b (0.41, close call — c,d 0.44)"),
4573 "{text}"
4574 );
4575 }
4576
4577 #[test]
4578 fn render_route_text_separates_multiple_issues_with_a_blank_line_in_request_order() {
4579 let issue = |item_ref: &str| IssueRoute {
4580 draft_comments: vec![],
4581 item_ref: item_ref.to_string(),
4582 url: "u".to_string(),
4583 title: "t".to_string(),
4584 state: ItemState::Open,
4585 outcome: RouteOutcome::Failed {
4586 error: "e".to_string(),
4587 reference_fetch_failures: vec![],
4588 },
4589 truncated: false,
4590 };
4591 let report = RouteReport {
4592 model: "jev-1.13.0".to_string(),
4593 issues: vec![issue("o/r#1"), issue("o/r#2")],
4594 ignored_closed: vec![],
4595 usage: Usage::default(),
4596 };
4597 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
4598 let first = text.find("o/r#1").unwrap();
4599 let second = text.find("o/r#2").unwrap();
4600 assert!(first < second, "{text}");
4601 assert!(
4602 text.contains("o/r#1 — t\n failed: e\n\no/r#2 — t\n failed: e"),
4603 "{text}"
4604 );
4605 }
4606
4607 #[test]
4608 fn render_route_text_ends_with_the_model_and_summed_usage() {
4609 let report = RouteReport {
4610 model: "jev-1.13.0".to_string(),
4611 issues: vec![],
4612 ignored_closed: vec![],
4613 usage: Usage {
4614 input_tokens: 10,
4615 output_tokens: 20,
4616 },
4617 };
4618 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
4619 assert_eq!(
4620 text,
4621 "model: jev-1.13.0, usage: 10 input tokens, 20 output tokens\n"
4622 );
4623 }
4624
4625 #[test]
4626 fn terminal_style_requires_a_usable_tty() {
4627 assert_eq!(
4628 TerminalStyle::new(false, "xterm", false, true),
4629 TerminalStyle::default()
4630 );
4631 assert_eq!(
4632 TerminalStyle::new(true, "dumb", false, true),
4633 TerminalStyle::default()
4634 );
4635 assert_eq!(
4636 TerminalStyle::new(true, "", false, true),
4637 TerminalStyle::default()
4638 );
4639 assert_eq!(
4640 TerminalStyle::new(true, "xterm", true, true),
4641 TerminalStyle {
4642 color: false,
4643 hyperlinks: true
4644 }
4645 );
4646 assert_eq!(
4647 TerminalStyle::new(true, "xterm", false, false),
4648 TerminalStyle {
4649 color: true,
4650 hyperlinks: false
4651 }
4652 );
4653 }
4654
4655 #[test]
4656 fn complexity_uses_tier_order_for_built_in_and_custom_ladders() {
4657 let anthropic = class_only(Provider::Anthropic).unwrap();
4658 assert_eq!(
4659 choice_complexity(NO_DESIGN, &anthropic),
4660 Some(Complexity::Low)
4661 );
4662 assert_eq!(
4663 choice_complexity("sonnet", &anthropic),
4664 Some(Complexity::Low)
4665 );
4666 assert_eq!(
4667 choice_complexity("opus", &anthropic),
4668 Some(Complexity::High)
4669 );
4670 let custom = |names: &[&str]| {
4671 use std::fmt::Write;
4672
4673 let mut yaml = String::from("tiers:\n");
4674 for name in names {
4675 writeln!(yaml, " - {{name: {name}, description: work}}").unwrap();
4676 }
4677 Ladder::named("custom".to_string(), Tiers::parse(&yaml).unwrap())
4678 };
4679 let two = custom(&["a", "b"]);
4680 assert_eq!(choice_complexity("a", &two), Some(Complexity::Low));
4681 assert_eq!(choice_complexity("b", &two), Some(Complexity::High));
4682 let four = custom(&["a", "b", "c", "d"]);
4683 assert_eq!(choice_complexity("b", &four), Some(Complexity::Medium));
4684 assert_eq!(choice_complexity("c", &four), Some(Complexity::Medium));
4685 assert_eq!(choice_complexity("d", &four), Some(Complexity::High));
4686 assert_eq!(choice_complexity("unknown", &four), None);
4687 }
4688
4689 #[test]
4690 fn hyperlink_uses_exact_osc_8_framing_and_rejects_unsafe_urls() {
4691 let style = TerminalStyle {
4692 color: false,
4693 hyperlinks: true,
4694 };
4695 let url = "https://github.com/other/repo/pull/42";
4696 assert_eq!(
4697 hyperlink("PR #42", Some(url), style),
4698 format!("\x1b]8;;{url}\x1b\\PR #42\x1b]8;;\x1b\\")
4699 );
4700 for bad in [
4701 "https://example.com/o/r/issues/1",
4702 "http://github.com/o/r/issues/1",
4703 "https://github.com/o/r/issues/1\x1b]8;;evil",
4704 "https://github.com/o/r/issues/nope",
4705 "not a valid url",
4706 ] {
4707 assert_eq!(hyperlink("#1", Some(bad), style), "#1");
4708 }
4709 assert_eq!(hyperlink("#1", None, style), "#1");
4710 }
4711
4712 #[test]
4713 fn colorize_applies_ansi_codes_per_complexity_when_color_enabled() {
4714 let color = TerminalStyle {
4715 color: true,
4716 hyperlinks: false,
4717 };
4718 assert_eq!(colorize("x", Complexity::Low, color), "\x1b[32mx\x1b[0m");
4719 assert_eq!(colorize("x", Complexity::Medium, color), "\x1b[33mx\x1b[0m");
4720 assert_eq!(colorize("x", Complexity::High, color), "\x1b[31mx\x1b[0m");
4721 assert_eq!(
4722 colorize("x", Complexity::Medium, TerminalStyle::default()),
4723 "x"
4724 );
4725 }
4726
4727 #[test]
4728 fn styled_report_colors_classes_stages_and_failure_and_links_refs() {
4729 let issue = IssueRoute {
4730 draft_comments: vec![],
4731 item_ref: "o/r#1".to_string(),
4732 url: "https://github.com/o/r/issues/1".to_string(),
4733 title: "t".to_string(),
4734 state: ItemState::Open,
4735 outcome: RouteOutcome::Routed {
4736 open_questions: test_open_questions(),
4737 providers: BTreeMap::from([(
4738 "anthropic".to_string(),
4739 ProviderRoute {
4740 stages: stages(NO_DESIGN, "opus", "sonnet"),
4741 class: "opus".to_string(),
4742 class_from: ClassSource::Implement,
4743 close_calls: vec![],
4744 },
4745 )]),
4746 depends_on: vec![DependencyEntry {
4747 item_ref: "#42".to_string(),
4748 url: Some("https://github.com/other/repo/pull/42".to_string()),
4749 state: ItemState::Open,
4750 relation: CitationRelation::Unspecified,
4751 could_be_cheaper: BTreeMap::new(),
4752 }],
4753 reference_fetch_failures: vec![],
4754 },
4755 truncated: false,
4756 };
4757 let failed = IssueRoute {
4758 draft_comments: vec![],
4759 item_ref: "o/r#2".to_string(),
4760 url: "https://github.com/o/r/issues/2".to_string(),
4761 title: "failed".to_string(),
4762 state: ItemState::Open,
4763 outcome: RouteOutcome::Failed {
4764 error: "HTTP 529".to_string(),
4765 reference_fetch_failures: vec![],
4766 },
4767 truncated: false,
4768 };
4769 let report = RouteReport {
4770 model: "jev".to_string(),
4771 issues: vec![issue, failed],
4772 ignored_closed: vec![],
4773 usage: Usage::default(),
4774 };
4775 let style = TerminalStyle {
4776 color: true,
4777 hyperlinks: true,
4778 };
4779 let text = render_route_text_styled(
4780 &report,
4781 DEFAULT_MAX_INPUT_CHARS,
4782 &[class_only(Provider::Anthropic).unwrap()],
4783 style,
4784 );
4785 assert!(
4786 text.contains(
4787 "\x1b[32m\x1b]8;;https://github.com/o/r/issues/1\x1b\\o/r#1\x1b]8;;\x1b\\\x1b[0m"
4788 ),
4789 "{text}"
4790 );
4791 assert!(
4792 text.contains(
4793 "\x1b[31mopus\x1b[0m (from implementation) — design needs \x1b[32mno further work\x1b[0m"
4794 ),
4795 "{text}"
4796 );
4797 assert!(text.contains("review \x1b[32msonnet\x1b[0m"), "{text}");
4798 assert!(
4799 text.contains("\x1b]8;;https://github.com/other/repo/pull/42\x1b\\#42\x1b]8;;\x1b\\"),
4800 "{text}"
4801 );
4802 assert!(text.contains("\x1b[31mfailed: HTTP 529\x1b[0m"), "{text}");
4803 let plain = render_route_text_styled(
4804 &report,
4805 DEFAULT_MAX_INPUT_CHARS,
4806 &[],
4807 TerminalStyle::default(),
4808 );
4809 assert_eq!(plain, render_route_text(&report, DEFAULT_MAX_INPUT_CHARS));
4810 assert!(!plain.contains('\x1b'));
4811 let json = serde_json::to_string(&report).unwrap();
4812 assert!(!json.contains("https://github.com/other/repo/pull/42"));
4813 }
4814
4815 #[test]
4816 fn styled_multi_model_layout_colors_the_class_and_stage_choices() {
4817 let tier = "model-a,model-b";
4818 let ladder = Ladder::named(
4819 "custom".to_string(),
4820 Tiers::parse(&format!(
4821 "tiers:\n - {{name: basic, description: work}}\n - {{name: '{tier}', description: work}}\n"
4822 ))
4823 .unwrap(),
4824 );
4825 let report = RouteReport {
4826 model: "jev".to_string(),
4827 issues: vec![IssueRoute {
4828 draft_comments: vec![],
4829 item_ref: "o/r#1".to_string(),
4830 url: "https://github.com/o/r/issues/1".to_string(),
4831 title: "t".to_string(),
4832 state: ItemState::Open,
4833 outcome: RouteOutcome::Routed {
4834 open_questions: test_open_questions(),
4835 providers: BTreeMap::from([(
4836 "custom".to_string(),
4837 ProviderRoute {
4838 stages: stages(NO_DESIGN, tier, "basic"),
4839 class: tier.to_string(),
4840 class_from: ClassSource::Implement,
4841 close_calls: vec![],
4842 },
4843 )]),
4844 depends_on: vec![],
4845 reference_fetch_failures: vec![],
4846 },
4847 truncated: false,
4848 }],
4849 ignored_closed: vec![],
4850 usage: Usage::default(),
4851 };
4852 let text = render_route_text_styled(
4853 &report,
4854 DEFAULT_MAX_INPUT_CHARS,
4855 &[ladder],
4856 TerminalStyle {
4857 color: true,
4858 hyperlinks: false,
4859 },
4860 );
4861 assert!(
4862 text.contains(&format!(" class: \x1b[31m{tier}\x1b[0m")),
4863 "{text}"
4864 );
4865 assert!(
4866 text.contains(&format!(" implementation: \x1b[31m{tier}\x1b[0m")),
4867 "{text}"
4868 );
4869 assert!(text.contains(" review: \x1b[32mbasic\x1b[0m"), "{text}");
4870 }
4871
4872 fn mixed_source_report() -> RouteReport {
4875 RouteReport {
4876 model: "jev-1.13.0".to_string(),
4877 issues: vec![IssueRoute {
4878 draft_comments: vec![],
4879 item_ref: "o/r#1".to_string(),
4880 url: "u".to_string(),
4881 title: "t".to_string(),
4882 state: ItemState::Open,
4883 outcome: RouteOutcome::Routed {
4884 open_questions: test_open_questions(),
4885 providers: BTreeMap::from([
4886 (
4887 "anthropic".to_string(),
4888 ProviderRoute {
4889 stages: stages("opus", "sonnet", "sonnet"),
4890 class: "opus".to_string(),
4891 class_from: ClassSource::Design,
4892 close_calls: vec![],
4893 },
4894 ),
4895 (
4896 "openai".to_string(),
4897 ProviderRoute {
4898 stages: stages(NO_DESIGN, "sol", "terra"),
4899 class: "sol".to_string(),
4900 class_from: ClassSource::Implement,
4901 close_calls: vec![],
4902 },
4903 ),
4904 ]),
4905 depends_on: vec![],
4906 reference_fetch_failures: vec![],
4907 },
4908 truncated: false,
4909 }],
4910 ignored_closed: vec![],
4911 usage: Usage::default(),
4912 }
4913 }
4914
4915 #[test]
4916 fn class_from_is_serialised_per_provider_in_json_and_yaml() {
4917 let report = mixed_source_report();
4918 let json = serde_json::to_value(&report).unwrap();
4919 let providers = &json["issues"][0]["providers"];
4920 assert_eq!(providers["anthropic"]["class_from"], "design");
4921 assert_eq!(providers["openai"]["class_from"], "implement");
4922
4923 let yaml = serde_yaml::to_string(&report).unwrap();
4924 assert!(
4925 yaml.contains("class: opus\n class_from: design\n"),
4926 "{yaml}"
4927 );
4928 assert!(
4929 yaml.contains("class: sol\n class_from: implement\n"),
4930 "{yaml}"
4931 );
4932 }
4933
4934 #[test]
4935 fn text_output_names_the_class_source_for_each_ladder() {
4936 let text = render_route_text(&mixed_source_report(), DEFAULT_MAX_INPUT_CHARS);
4937 assert!(
4938 text.contains(" anthropic: opus (from design) — "),
4939 "{text}"
4940 );
4941 assert!(
4942 text.contains(" openai: sol (from implementation) — "),
4943 "{text}"
4944 );
4945 }
4946
4947 #[test]
4948 fn text_output_names_the_class_source_in_the_multi_line_layout() {
4949 let multi = "a,b";
4950 let report = RouteReport {
4951 issues: vec![IssueRoute {
4952 outcome: RouteOutcome::Routed {
4953 open_questions: test_open_questions(),
4954 providers: BTreeMap::from([(
4955 "anthropic".to_string(),
4956 ProviderRoute {
4957 stages: StageAnswers {
4958 design: answer(multi, 0.9),
4959 implement: answer("basic", 0.9),
4960 review: answer("basic", 0.9),
4961 },
4962 class: multi.to_string(),
4963 class_from: ClassSource::Design,
4964 close_calls: vec![],
4965 },
4966 )]),
4967 depends_on: vec![],
4968 reference_fetch_failures: vec![],
4969 },
4970 ..mixed_source_report().issues.remove(0)
4971 }],
4972 ..mixed_source_report()
4973 };
4974 let text = render_route_text(&report, DEFAULT_MAX_INPUT_CHARS);
4975 assert!(text.contains(" class: a,b (from design)\n"), "{text}");
4976 }
4977
4978 #[test]
4979 fn design_stage_with_a_named_tier_renders_needs_prefix() {
4980 let line = render_stage_line(
4981 Stage::Design,
4982 &stages("opus", "sonnet", "sonnet"),
4983 &[],
4984 None,
4985 TerminalStyle::default(),
4986 );
4987 assert_eq!(line, "design: needs opus (0.90)");
4988 }
4989
4990 #[test]
4991 fn with_thousands_groups_digits() {
4992 assert_eq!(with_thousands(0), "0");
4993 assert_eq!(with_thousands(999), "999");
4994 assert_eq!(with_thousands(60_000), "60,000");
4995 assert_eq!(with_thousands(1_000_000), "1,000,000");
4996 }
4997}