1use std::collections::BTreeMap;
32use std::path::{Path, PathBuf};
33
34use serde::Serialize;
35use serde_json::Value;
36
37use crate::collective::{self, Config, Route};
38use crate::harness::{self, Harness, Run, SubAgent};
39use crate::learn::{self, Episode, regret};
40
41pub const FLAGS: &[&str] = &["--home", "--since", "--json", "--apply", "--only"];
43
44const USAGE: &str = "usage: borg review-fix [--home DIR] [--since YYYY-MM-DD] [--json] [--apply [--only ID,…]]";
45
46#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
47#[serde(rename_all = "snake_case")]
48pub enum Verdict {
49 Confirmed,
50 Rejected,
51 Unchecked,
53}
54
55#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
56#[serde(rename_all = "snake_case")]
57pub enum Fixed {
58 Yes,
59 No,
61 NoFixStep,
63 Unlocated,
65}
66
67#[derive(Debug, Clone, Serialize)]
68pub struct Finding {
69 pub id: String,
71 pub session: String,
72 pub run: String,
73 pub workflow: String,
74 pub agent: String,
76 pub ms: i64,
77 pub cwd: String,
78 pub file: Option<String>,
79 pub title: String,
80 pub detail: String,
82 pub severity: String,
83 pub verdict: Verdict,
84 pub fixed: Fixed,
85 pub task: String,
88 pub built: String,
90}
91
92impl Finding {
93 pub fn strong(&self) -> bool {
95 self.fixed == Fixed::Yes && self.verdict != Verdict::Rejected
96 }
97
98 pub fn episode(&self, engineer: &str) -> Episode {
102 let red = |t: &str| crate::notes::redact(t);
103 let verdict = match self.verdict {
104 Verdict::Confirmed => "confirmed",
105 _ => "unchecked",
106 };
107 let detail = match &self.file {
108 Some(f) => format!("{f}: {}", self.detail),
109 None => self.detail.clone(),
110 };
111 let r = regret::Regret {
112 decision: self.run.clone(),
113 decision_session: self.session.clone(),
114 decision_task: red(&self.task),
115 decision_proposal: red(&self.built),
116 fix: self.id.clone(),
117 fix_subject: red(&self.title),
118 diff: red(&detail),
119 signals: vec![regret::REVIEW.into(), verdict.into()],
120 decision_ms: self.ms,
122 ..regret::Regret::default()
123 };
124 Episode {
125 id: format!("V{}", self.id),
126 ms: self.ms,
127 repo: borg_trace::repo_key(&self.cwd).rsplit('/').next().unwrap_or_default().to_string(),
128 session: self.session.clone(),
129 task: r.decision_task.clone(),
130 agent: r.decision_proposal.clone(),
131 rejected_tool: None,
132 correction: format!("Review finding ({}, {verdict}), fixed before any commit: {}\n{}", self.severity, r.fix_subject, r.diff),
133 engineer: engineer.to_string(),
134 kind: regret::KIND.into(),
135 regret: Some(r),
136 }
137 }
138}
139
140fn words(s: &str) -> Vec<String> {
143 let mut out: Vec<String> = Vec::new();
144 let mut prev_lower = false;
145 for c in s.chars() {
146 if !c.is_alphanumeric() {
147 prev_lower = false;
148 out.push(String::new());
149 continue;
150 }
151 if c.is_uppercase() && prev_lower || out.is_empty() {
152 out.push(String::new());
153 }
154 prev_lower = c.is_lowercase() || c.is_ascii_digit();
155 out.last_mut().expect("pushed").extend(c.to_lowercase());
156 }
157 out.retain(|w| !w.is_empty());
158 out
159}
160
161fn says_fix(s: &str) -> bool {
164 let w = words(s);
165 w.iter().any(|w| ["fix", "fixes", "fixer", "fixing", "repair", "repairs", "repairer"].contains(&w.as_str()))
166 && !w.iter().any(|w| ["review", "verif", "impl"].iter().any(|n| w.starts_with(n)))
167}
168
169fn is_fix(a: &SubAgent) -> bool {
173 let phase = words(&a.phase);
174 if phase.iter().any(|w| ["review", "verif", "impl"].iter().any(|n| w.starts_with(n))) {
175 return false;
176 }
177 let first = a.label.split([':', '#']).next().unwrap_or_default();
178 let listed = a.output.as_ref().is_some_and(|o| ["findings", "issues"].iter().any(|k| o.get(*k).is_some_and(Value::is_array)));
179 (says_fix(&a.phase) || says_fix(first) || says_fix(&a.kind)) && !listed && verdicts_of(a.output.as_ref()).is_empty()
180}
181
182fn output_of(a: &SubAgent) -> Option<Value> {
187 if a.output.is_some() {
188 return a.output.clone();
189 }
190 let w = a.final_text.as_str();
191 let listed = |v: &Value| ["findings", "issues"].iter().any(|k| v.get(*k).is_some_and(Value::is_array));
192 let starts = w.match_indices('{').map(|(i, _)| i).filter(|&i| w[w[..i].rfind('\n').map_or(0, |n| n + 1)..i].trim().is_empty());
193 starts.rev().find_map(|i| serde_json::Deserializer::from_str(&w[i..]).into_iter::<Value>().next()?.ok().filter(listed))
194}
195
196pub const DETAIL_CHARS: usize = 800;
199
200const TEXT_KEYS: &[&str] = &["title", "claim", "problem", "description", "issue", "body"];
201const DETAIL_KEYS: &[&str] = &["detail", "evidence", "reason", "body"];
202const FILE_KEYS: &[&str] = &["file", "path", "file_path", "fileLines", "where"];
203
204fn field<'a>(o: &'a Value, keys: &[&str]) -> Option<&'a str> {
205 keys.iter().find_map(|k| o.get(*k).and_then(Value::as_str)).map(str::trim).filter(|s| !s.is_empty())
206}
207
208fn verdict_of(o: &Value) -> Option<Verdict> {
210 let flag = |k: &str| o.get(k).and_then(Value::as_bool);
211 if let Some(r) = flag("real").or_else(|| flag("keep")).or_else(|| flag("refuted").map(|r| !r)) {
212 return Some(if r { Verdict::Confirmed } else { Verdict::Rejected });
213 }
214 let w = o.get("verdict").and_then(Value::as_str)?.to_lowercase();
215 if ["confirm", "real", "valid", "holds", "true", "plausible"].iter().any(|k| w.starts_with(k)) {
217 Some(Verdict::Confirmed)
218 } else if ["refut", "reject", "false", "invalid", "not"].iter().any(|k| w.starts_with(k)) {
219 Some(Verdict::Rejected)
220 } else {
221 None
222 }
223}
224
225#[derive(Debug, PartialEq)]
227struct Raw {
228 id: Option<String>,
229 title: String,
230 detail: String,
231 severity: String,
232 file: Option<String>,
233}
234
235fn findings_of(output: Option<&Value>) -> Vec<Raw> {
237 let Some(o) = output else { return Vec::new() };
238 ["findings", "issues"]
239 .iter()
240 .filter_map(|k| o.get(*k).and_then(Value::as_array))
241 .flatten()
242 .filter(|f| f.is_object() && verdict_of(f).is_none())
243 .filter_map(|f| {
244 let title = field(f, TEXT_KEYS)?;
245 let severity = severity_of(f)?;
246 let detail = field(f, DETAIL_KEYS).filter(|d| *d != title).unwrap_or_default().chars().take(DETAIL_CHARS).collect();
247 Some(Raw { id: field(f, &["id"]).map(String::from), title: title.to_string(), detail, severity, file: field(f, FILE_KEYS).and_then(path_in) })
248 })
249 .collect()
250}
251
252fn severity_of(f: &Value) -> Option<String> {
254 if let Some(s) = field(f, &["severity"]) {
255 return Some(s.to_lowercase());
256 }
257 match f.get("priority")? {
258 Value::Number(n) => n.as_u64().map(|n| format!("p{n}")),
259 Value::String(s) => {
260 let s = s.trim().to_lowercase();
261 let n = s.strip_prefix('p').unwrap_or(&s);
262 Some(if n.parse::<u8>().is_ok() { format!("p{n}") } else { s }).filter(|s| !s.is_empty())
263 }
264 _ => None,
265 }
266}
267
268#[derive(Debug)]
270struct Said {
271 id: Option<String>,
272 title: Option<String>,
273 verdict: Verdict,
274}
275
276fn verdicts_of(output: Option<&Value>) -> Vec<Said> {
278 let Some(o) = output else { return Vec::new() };
279 if o.get("findings").is_none()
280 && let Some(verdict) = verdict_of(o)
281 {
282 return vec![Said { id: None, title: None, verdict }];
283 }
284 ["verdicts", "results"]
285 .iter()
286 .filter_map(|k| o.get(*k).and_then(Value::as_array))
287 .flatten()
288 .filter_map(|x| Some(Said { id: field(x, &["id"]).map(String::from), title: field(x, TEXT_KEYS).map(String::from), verdict: verdict_of(x)? }))
289 .collect()
290}
291
292fn dim(label: &str) -> &str {
294 label.rsplit(':').next().unwrap_or_default()
295}
296
297fn path_in(s: &str) -> Option<String> {
299 s.split(|c: char| c.is_whitespace() || "`'\"(),;".contains(c))
300 .map(|t| t.split(':').next().unwrap_or(t).trim_end_matches(['.', '§']))
301 .find(|t| t.contains('.') && t.len() > 2 && !t.starts_with("http") && t.chars().any(char::is_alphabetic))
302 .map(String::from)
303}
304
305fn norm(s: &str) -> String {
306 s.chars().filter(|c| c.is_alphanumeric()).flat_map(char::to_lowercase).collect()
307}
308
309fn same_title(a: &str, b: &str) -> bool {
312 let (a, b) = (norm(a), norm(b));
313 let n = a.len().min(b.len());
314 n >= 12 && a[..n] == b[..n]
315}
316
317fn names(work: &str, file: &str) -> bool {
319 let tail: Vec<&str> = file.rsplit('/').take(2).collect();
320 let short = tail.iter().rev().copied().collect::<Vec<_>>().join("/");
321 work.contains(file) || (short.len() > 3 && work.contains(&short))
322}
323
324fn run_findings(run: &Run) -> Vec<Finding> {
326 let agents = &run.agents;
327 let verdicts: Vec<(&SubAgent, Said)> = agents.iter().flat_map(|a| verdicts_of(a.output.as_ref()).into_iter().map(move |v| (a, v))).collect();
328 let raws: Vec<(&SubAgent, Raw)> = agents.iter().flat_map(|a| findings_of(a.output.as_ref()).into_iter().map(move |r| (a, r))).collect();
329 let builder =
332 agents.iter().filter(|_| !run.loose).find(|b| !is_fix(b) && findings_of(b.output.as_ref()).is_empty() && verdicts_of(b.output.as_ref()).is_empty());
333 let n = DETAIL_CHARS * 2;
334 let builder_repo = builder.map(|b| borg_trace::repo_key(&b.cwd));
335 let (task, built) = builder.map_or_else(Default::default, |b| {
336 let w: Vec<char> = b.work.chars().collect();
337 (b.prompt.chars().take(n).collect::<String>(), w[w.len().saturating_sub(n)..].iter().collect::<String>())
338 });
339 let mut out = Vec::new();
340 for (k, (a, Raw { id, title, detail, severity, file })) in raws.iter().enumerate() {
341 let head: String = title.chars().take(60).collect();
342 let same_id = |v: &str| raws.iter().filter(|(_, r)| r.id.as_deref() == Some(v)).count();
346 let linked = verdicts.iter().filter(|(va, said)| match (&said.id, &said.title) {
347 (Some(v), _) if id.as_deref() == Some(v.as_str()) => same_id(v) == 1 || dim(&va.label) == dim(&a.label),
348 (_, Some(t)) => same_title(t, title),
349 (Some(_), None) => false,
350 (None, None) => same_title(dim(&va.label), title) || va.prompt.contains(&head),
351 });
352 let verdict =
353 linked.fold(
354 Verdict::Unchecked,
355 |acc, (_, said)| if said.verdict == Verdict::Rejected || acc == Verdict::Rejected { Verdict::Rejected } else { said.verdict },
356 );
357 let fixers: Vec<&SubAgent> = agents.iter().filter(|f| is_fix(f) && f.started >= a.started).collect();
358 let fixed = match file {
359 _ if fixers.is_empty() => Fixed::NoFixStep,
360 None => Fixed::Unlocated,
361 Some(f) if fixers.iter().any(|x| names(&x.work, f)) => Fixed::Yes,
362 Some(_) => Fixed::No,
363 };
364 let same_repo = builder_repo.as_deref() == Some(borg_trace::repo_key(&a.cwd).as_str());
367 out.push(Finding {
368 id: if run.loose {
371 format!("{}#{}.{}", run.id, a.id, raws[..k].iter().filter(|(b, _)| std::ptr::eq(*b, *a)).count())
372 } else {
373 format!("{}#{k}", run.id)
374 },
375 session: run.session.clone(),
376 run: run.id.clone(),
377 workflow: run.name.clone(),
378 agent: a.label.clone(),
379 ms: a.started,
380 cwd: a.cwd.clone(),
381 file: file.clone(),
382 title: title.clone(),
383 detail: detail.clone(),
384 severity: severity.clone(),
385 verdict,
386 fixed,
387 task: if same_repo { task.clone() } else { String::new() },
388 built: if same_repo { built.clone() } else { String::new() },
389 });
390 }
391 out
392}
393
394fn runs_of<'h>(harnesses: impl Iterator<Item = &'h dyn Harness>, home: &Path, since: Option<i64>, session: Option<&str>) -> Vec<Run> {
397 prepared(harnesses.flat_map(|h| h.sub_runs(home, session, since)).collect())
398}
399
400fn prepared(mut runs: Vec<Run>) -> Vec<Run> {
401 for a in runs.iter_mut().flat_map(|r| r.agents.iter_mut()) {
402 a.output = output_of(a);
403 }
404 runs.sort_by_key(|r| r.started);
405 runs
406}
407
408fn scan(runs: &[Run]) -> (Vec<Finding>, Vec<Fixers>) {
410 let mut findings = Vec::new();
411 let mut fixers = Vec::new();
412 for run in runs {
413 findings.extend(run_findings(run));
414 let fix: Vec<&SubAgent> = run.agents.iter().filter(|a| is_fix(a)).collect();
415 if let Some(first) = fix.first() {
416 let repo = borg_trace::repo_key(&first.cwd);
417 fixers.push(Fixers { session: run.session.clone(), run: run.id.clone(), repo, work: fix.iter().map(|a| a.work.clone()).collect() });
418 }
419 }
420 (findings, fixers)
421}
422
423#[derive(Debug)]
425struct Fixers {
426 session: String,
427 run: String,
428 repo: String,
429 work: Vec<String>,
430}
431
432#[derive(Debug, Default, Clone, Serialize)]
434pub struct Tally {
435 pub runs: usize,
436 pub findings: usize,
437 pub confirmed: usize,
438 pub rejected: usize,
439 pub unchecked: usize,
440 pub fixed: usize,
441 pub not_fixed: usize,
442 pub no_fix_step: usize,
443 pub unlocated: usize,
444 pub strong: usize,
445}
446
447impl Tally {
448 fn add(&mut self, f: &Finding) {
449 self.findings += 1;
450 match f.verdict {
451 Verdict::Confirmed => self.confirmed += 1,
452 Verdict::Rejected => self.rejected += 1,
453 Verdict::Unchecked => self.unchecked += 1,
454 }
455 match f.fixed {
456 Fixed::Yes => self.fixed += 1,
457 Fixed::No => self.not_fixed += 1,
458 Fixed::NoFixStep => self.no_fix_step += 1,
459 Fixed::Unlocated => self.unlocated += 1,
460 }
461 self.strong += usize::from(f.strong());
462 }
463}
464
465#[derive(Debug, Default, Clone, Serialize)]
469pub struct Control {
470 pub unpaired: usize,
471 pub located: usize,
472 pub own_named: usize,
473 pub other_named: usize,
474}
475
476#[derive(Debug, Default, Serialize)]
477pub struct Report {
478 pub collectives: BTreeMap<String, Tally>,
479 pub off: Tally,
480 pub control: Control,
481 pub findings: Vec<Finding>,
483}
484
485pub fn run(home: &Path, since: Option<i64>, cfg: Option<&Config>, base: &Path) -> Report {
487 report(&runs_of(harness::ALL.iter().copied(), home, since, None), cfg, base)
488}
489
490fn report(runs: &[Run], cfg: Option<&Config>, base: &Path) -> Report {
492 let (findings, fixers) = scan(runs);
493 let mut r = Report::default();
494 let mut runs: BTreeMap<String, std::collections::BTreeSet<String>> = BTreeMap::new();
495 for f in findings {
496 let name = match collective::route(cfg, &f.cwd, base) {
497 Route::Off => None,
498 Route::Collective { name, .. } => Some(name),
499 };
500 let Some(name) = name else {
501 r.off.add(&f);
502 continue;
503 };
504 if let Some(file) = f.file.as_deref().filter(|_| f.fixed != Fixed::NoFixStep) {
505 let repo = borg_trace::repo_key(&f.cwd);
508 let at = fixers.iter().position(|x| x.run == f.run).map_or(0, |i| i + 1);
509 match fixers.iter().cycle().skip(at).take(fixers.len()).find(|x| x.session != f.session && x.repo == repo) {
510 Some(other) => {
511 r.control.located += 1;
512 r.control.own_named += usize::from(f.fixed == Fixed::Yes);
513 r.control.other_named += usize::from(other.work.iter().any(|w| names(w, file)));
514 }
515 None => r.control.unpaired += 1,
516 }
517 }
518 runs.entry(name.clone()).or_default().insert(f.run.clone());
519 r.collectives.entry(name).or_default().add(&f);
520 r.findings.push(f);
521 }
522 for (name, set) in runs {
523 r.collectives.entry(name).or_default().runs = set.len();
524 }
525 r
526}
527
528fn pct(n: usize, d: usize) -> String {
529 if d == 0 { "-".into() } else { format!("{:.0}%", 100.0 * n as f64 / d as f64) }
530}
531
532pub fn render(r: &Report) -> String {
533 let mut s = String::new();
534 for (name, t) in &r.collectives {
535 s.push_str(&format!(
536 "{name}: {} findings in {} workflow runs; verdict: {} confirmed, {} rejected, {} unchecked; fix: {} fixed, {} not named by the fix step, {} without a fix step, {} without a file; {} strong\n",
537 t.findings, t.runs, t.confirmed, t.rejected, t.unchecked, t.fixed, t.not_fixed, t.no_fix_step, t.unlocated, t.strong
538 ));
539 }
540 if r.collectives.is_empty() {
541 s.push_str("no workflow findings in repos of a collective\n");
542 }
543 let c = &r.control;
544 s.push_str(&format!(
545 "control: of {} located findings, the run's own fix step names the file {} ({}), another session's fix step in the same repo {} ({}); {} without such a session\n",
546 c.located,
547 c.own_named,
548 pct(c.own_named, c.located),
549 c.other_named,
550 pct(c.other_named, c.located),
551 c.unpaired
552 ));
553 s.push_str(&format!("off: {} findings skipped (not shown)\n", r.off.findings));
554 s
555}
556
557pub fn apply(report: &Report, only: Option<&[String]>, cfg: Option<&Config>, base: &Path, ask: &dyn Fn(&str) -> Option<Value>) -> (usize, usize, usize) {
561 let (recorded, homes) = record(report.findings.iter().filter(|f| only.is_none_or(|o| o.contains(&f.id))), cfg, base);
562 let (mut new, mut merged) = (0, 0);
563 for home in &homes {
564 let (n, m, _) = learn::learn_in_reporting(home, None, ask, learn::learning(), learn::now_ms(), &mut learn::print_step);
565 new += n;
566 merged += m;
567 }
568 (recorded, new, merged)
569}
570
571fn record<'a>(findings: impl Iterator<Item = &'a Finding>, cfg: Option<&Config>, base: &Path) -> (usize, Vec<PathBuf>) {
574 let mut homes: Vec<PathBuf> = Vec::new();
575 let mut recorded = 0;
576 let mut engineers: BTreeMap<String, String> = BTreeMap::new();
577 for f in findings.filter(|f| f.strong()) {
578 let Route::Collective { home, .. } = collective::route(cfg, &f.cwd, base) else { continue };
579 let engineer = engineers.entry(f.cwd.clone()).or_insert_with(|| learn::engineer(&f.cwd)).clone();
580 let e = f.episode(&engineer);
581 if crate::notes::regret::known(&home, &e.id) {
582 continue;
583 }
584 if learn::record_episode_in(&home, &e).is_ok() {
585 recorded += 1;
586 crate::stats::count(&home, crate::stats::REVIEW_FINDING, &f.session, |ev| {
587 ev.engineer = engineer.clone();
588 ev.repo = e.repo.clone();
589 ev.why = f.severity.clone();
590 ev.text = e.regret.as_ref().map(|r| r.fix_subject.clone()).unwrap_or_default();
591 });
592 if !homes.contains(&home) {
593 homes.push(home);
594 }
595 }
596 }
597 (recorded, homes)
598}
599
600pub fn on() -> bool {
603 std::env::var("BORG_REVIEW_FIX").as_deref() != Ok("off")
604}
605
606pub fn record_session(home: &Path, session: &str, cfg: Option<&Config>, base: &Path) -> (usize, Vec<PathBuf>) {
610 if session.is_empty() || session.contains(['/', '\\']) || session.starts_with('.') {
612 return (0, Vec::new());
613 }
614 let runs = match harness::named_current() {
615 Some(h) => runs_of(std::iter::once(h), home, None, Some(session)),
616 None => runs_of(harness::ALL.iter().copied(), home, None, Some(session)),
617 };
618 record(scan(&runs).0.iter(), cfg, base)
619}
620
621pub fn home_dir() -> PathBuf {
623 PathBuf::from(std::env::var("HOME").unwrap_or_default())
624}
625
626pub fn main(args: &[&str], ask: &dyn Fn(&str) -> Option<Value>) -> i32 {
628 let mut home = None;
629 let mut since = None;
630 let mut json = false;
631 let mut apply_it = false;
632 let mut only: Option<Vec<String>> = None;
633 let mut it = args.iter();
634 while let Some(a) = it.next() {
635 match *a {
636 "--json" => json = true,
637 "--apply" => apply_it = true,
638 "--only" => only = it.next().map(|v| v.split(',').map(str::to_string).collect()),
639 "--home" => home = it.next().map(PathBuf::from),
640 "--since" => match it.next().map(|d| borg_trace::parse_ts(&format!("{d}T00:00:00Z"))) {
641 Some(ms) if ms > 0 => since = Some(ms),
642 _ => {
643 eprintln!("borg review-fix: --since wants YYYY-MM-DD\n{USAGE}");
644 return 2;
645 }
646 },
647 _ => {
648 eprintln!("{USAGE}");
649 return 2;
650 }
651 }
652 }
653 let home = home.unwrap_or_else(home_dir);
654 let cfg = match collective::load() {
656 Ok(c) => c,
657 Err(e) => {
658 eprintln!("borg review-fix: invalid collectives config ({e}): every repo is off");
659 return 1;
660 }
661 };
662 let report = run(&home, since, cfg.as_ref(), &learn::home());
663 if apply_it {
664 let (recorded, new, merged) = apply(&report, only.as_deref(), cfg.as_ref(), &learn::home(), ask);
665 println!("{recorded} review findings recorded as episodes; {new} new lessons, {merged} merged");
666 return 0;
667 }
668 if json {
669 println!("{}", serde_json::to_string(&report).unwrap_or_default());
670 } else {
671 print!("{}", render(&report));
672 }
673 0
674}
675
676#[cfg(test)]
677mod tests {
678 use super::*;
679 use serde_json::json;
680
681 fn claude_runs(projects: &Path, session: Option<&str>) -> Vec<Run> {
683 prepared(harness::claude::workflow_runs(projects, session, None))
684 }
685
686 fn found(projects: &Path) -> Vec<Finding> {
687 scan(&claude_runs(projects, None)).0
688 }
689
690 fn temp(name: &str) -> PathBuf {
691 let d = std::env::temp_dir().join(format!("borg-review-fix-{name}-{}", std::process::id()));
692 let _ = std::fs::remove_dir_all(&d);
693 d
694 }
695
696 fn transcript(prompt: &str, tools: &[(&str, Value)], output: Option<Value>) -> String {
698 let mut lines = vec![json!({"type": "user", "cwd": "/w/app", "message": {"role": "user", "content": prompt}})];
699 for (name, input) in tools {
700 lines.push(json!({"type": "assistant", "message": {"content": [{"type": "tool_use", "name": name, "input": input}]}}));
701 }
702 if let Some(o) = output {
703 lines.push(json!({"type": "assistant", "message": {"content": [{"type": "tool_use", "name": "StructuredOutput", "input": o}]}}));
704 }
705 lines.iter().map(Value::to_string).collect::<Vec<_>>().join("\n")
706 }
707
708 fn write_run(projects: &Path, session: &str, run: &str, agents: &[(&str, &str, &str, i64, String)]) {
710 let dir = projects.join("-w-app").join(session);
711 let ad = dir.join("subagents/workflows").join(run);
712 std::fs::create_dir_all(&ad).unwrap();
713 std::fs::create_dir_all(dir.join("workflows")).unwrap();
714 let progress: Vec<Value> = agents
715 .iter()
716 .map(|(id, label, phase, at, t)| {
717 std::fs::write(ad.join(format!("agent-{id}.jsonl")), t).unwrap();
718 json!({"type": "workflow_agent", "label": label, "phaseTitle": phase, "agentId": id, "startedAt": at.to_string()})
719 })
720 .collect();
721 let r = json!({"runId": run, "workflowName": "build", "startTime": agents[0].3, "workflowProgress": progress});
722 std::fs::write(dir.join("workflows").join(format!("{run}.json")), r.to_string()).unwrap();
723 }
724
725 const TITLE: &str = "Pre-aborted signal returns before the error listener is attached";
726
727 fn review() -> String {
728 transcript("review the diff", &[], Some(json!({"findings": [{"file": "adapters/omp/borg.ts", "line": 153, "severity": "Medium", "title": TITLE}]})))
729 }
730
731 #[test]
732 fn a_confirmed_finding_the_fix_step_touches_is_strong() {
733 let p = temp("strong");
734 let verify = transcript(&format!("Verify: {TITLE}"), &[], Some(json!({"real": true, "reason": "probe crashed"})));
735 let fix = transcript("apply the findings", &[("Bash", json!({"command": "sed -i 's/a/b/' adapters/omp/borg.ts"}))], None);
736 write_run(
737 &p,
738 "s1",
739 "wf_1",
740 &[
741 ("a1", "review:concurrency", "Review", 10, review()),
742 ("a2", "verify:concurrency:Pre-aborted signal ret", "Verify", 20, verify),
743 ("a3", "fix", "Fix", 30, fix),
744 ],
745 );
746 let f = found(&p);
747 assert_eq!(f.len(), 1);
748 assert_eq!((f[0].verdict, f[0].fixed, f[0].severity.as_str()), (Verdict::Confirmed, Fixed::Yes, "medium"));
749 assert!(f[0].strong());
750 assert_eq!(f[0].cwd, "/w/app");
751 let _ = std::fs::remove_dir_all(p);
752 }
753
754 #[test]
755 fn a_rejected_or_unfixed_finding_is_not_strong() {
756 let p = temp("weak");
757 let verify = transcript("check it", &[], Some(json!({"verdicts": [{"title": TITLE, "verdict": "refuted"}]})));
758 let fix = transcript("apply the findings", &[("Edit", json!({"file_path": "src/other.rs"}))], None);
759 write_run(&p, "s1", "wf_1", &[("a1", "review", "Review", 10, review()), ("a2", "verify", "Verify", 20, verify), ("a3", "fix", "Fix", 30, fix)]);
760 let f = found(&p);
761 assert_eq!((f[0].verdict, f[0].fixed), (Verdict::Rejected, Fixed::No));
762 assert!(!f[0].strong());
763 let _ = std::fs::remove_dir_all(p);
764 }
765
766 #[test]
767 fn a_review_without_a_later_fix_step_or_a_file_is_counted_apart() {
768 let p = temp("nofix");
769 write_run(&p, "s1", "wf_1", &[("a1", "review", "Review", 10, review())]);
770 let unlocated = transcript("review", &[], Some(json!({"issues": [{"severity": "high", "problem": "the budget is spent on noise"}]})));
771 write_run(&p, "s2", "wf_2", &[("b1", "review:queen", "Review", 10, unlocated), ("b2", "fix", "Fix", 20, transcript("fix", &[], None))]);
772 let f = found(&p);
773 assert_eq!(f.iter().map(|x| x.fixed).collect::<Vec<_>>(), [Fixed::NoFixStep, Fixed::Unlocated]);
774 assert_eq!(f[1].verdict, Verdict::Unchecked);
775 let _ = std::fs::remove_dir_all(p);
776 }
777
778 #[test]
779 fn string_findings_and_gate_results_are_not_findings() {
780 let out = json!({"findings": ["X1 the voice speaks the commentary 38/38 times"], "results": [{"command": "npm run check", "passed": true}]});
781 assert!(findings_of(Some(&out)).is_empty());
782 assert!(verdicts_of(Some(&out)).is_empty());
783 let v = json!({"findings": [{"title": TITLE, "severity": "low", "real": false}]});
785 assert!(findings_of(Some(&v)).is_empty());
786 }
787
788 #[test]
789 fn verdicts_link_by_id_within_the_verifiers_dimension() {
790 let p = temp("ids");
791 let f =
792 |id: &str, file: &str| json!({"id": id, "file": file, "severity": "minor", "claim": format!("{file} drops the queued lessons on a late answer")});
793 let rust = transcript("review", &[], Some(json!({"findings": [f("F1", "src/a.rs")]})));
794 let docs = transcript("review", &[], Some(json!({"findings": [f("F1", "docs/b.md")]})));
795 let verify = transcript("verify", &[], Some(json!({"verdicts": [{"id": "F1", "refuted": true}]})));
796 let keep = transcript("verify", &[], Some(json!({"results": [{"id": "F1", "keep": true}]})));
797 write_run(
798 &p,
799 "s1",
800 "wf_1",
801 &[
802 ("a1", "review:rust", "Review", 10, rust),
803 ("a2", "review:docs", "Review", 11, docs),
804 ("a3", "verify:rust", "Verify", 20, verify),
805 ("a4", "verify:docs", "Verify", 21, keep),
806 ],
807 );
808 let got = found(&p);
809 assert_eq!(
810 got.iter().map(|x| (x.agent.as_str(), x.verdict)).collect::<Vec<_>>(),
811 [("review:rust", Verdict::Rejected), ("review:docs", Verdict::Confirmed)]
812 );
813 assert_eq!(verdict_of(&json!({"verdict": "PLAUSIBLE"})), Some(Verdict::Confirmed));
814 assert_eq!(verdict_of(&json!({"verdict": "needs more info"})), None);
815 let _ = std::fs::remove_dir_all(p);
816 }
817
818 #[test]
819 fn a_strong_finding_becomes_a_review_regret_episode() {
820 let p = temp("episode");
821 let build = transcript("Port the voice lane to gpt-live.", &[], None);
822 let fix = transcript("apply", &[("Bash", json!({"command": "sed -i x adapters/omp/borg.ts"}))], None);
823 write_run(&p, "s1", "wf_1", &[("a0", "build", "Build", 5, build), ("a1", "review", "Review", 10, review()), ("a2", "fix", "Fix", 20, fix)]);
824 let f = found(&p);
825 assert_eq!((f[0].id.as_str(), f[0].task.as_str()), ("wf_1#0", "Port the voice lane to gpt-live."));
826 let e = f[0].episode("dev@x");
827 assert_eq!((e.id.as_str(), e.kind.as_str(), e.session.as_str(), e.engineer.as_str()), ("Vwf_1#0", regret::KIND, "s1", "dev@x"));
828 let r = e.regret.unwrap();
829 assert!(regret::is_review(&r) && r.signals == ["review", "unchecked"]);
830 assert_eq!(r.decision_ms, e.ms, "decided before the finding: asks up to it are blamed");
831 assert!(r.diff.starts_with("adapters/omp/borg.ts: ") && r.fix_subject == TITLE, "{r:?}");
832 let _ = std::fs::remove_dir_all(p);
833 }
834
835 #[test]
836 fn paths_come_out_of_location_text() {
837 assert_eq!(path_in("src/main/java/Tq.java:72-79 (resets at L74)").as_deref(), Some("src/main/java/Tq.java"));
838 assert_eq!(path_in("sliceA-spec.md §G.3/§G.8 (ii)").as_deref(), Some("sliceA-spec.md"));
839 assert_eq!(path_in("the whole design"), None);
840 assert!(names("cat > crates/borg/src/live.rs <<EOF", "/home/x/crates/borg/src/live.rs") || names("x src/live.rs", "crates/borg/src/live.rs"));
841 assert!(!names("edits main.rs", "src/live.rs"));
842 }
843
844 #[test]
845 fn the_control_pairs_a_finding_with_another_sessions_fix_step() {
846 let p = temp("control");
847 let fix = |file: &str| transcript("fix", &[("Bash", json!({"command": format!("vim {file}")}))], None);
848 write_run(&p, "s1", "wf_1", &[("a1", "review", "Review", 10, review()), ("a2", "fix", "Fix", 20, fix("adapters/omp/borg.ts"))]);
849 write_run(&p, "s2", "wf_2", &[("b1", "build", "Build", 30, transcript("build", &[], None)), ("b2", "fix", "Fix", 40, fix("src/other.rs"))]);
850 let r = report(&claude_runs(&p, None), None, &p.join("store"));
851 assert_eq!((r.control.located, r.control.own_named, r.control.other_named), (1, 1, 0));
852 assert_eq!(r.collectives.values().map(|t| (t.findings, t.strong, t.runs)).collect::<Vec<_>>(), [(1, 1, 1)]);
853 let _ = std::fs::remove_dir_all(p);
854 }
855
856 #[test]
857 fn session_end_records_only_that_sessions_strong_findings_once() {
858 let p = temp("session");
859 let fix = transcript("apply the findings", &[("Bash", json!({"command": "sed -i 's/a/b/' adapters/omp/borg.ts"}))], None);
860 for (s, run) in [("s1", "wf_1"), ("s2", "wf_2")] {
861 write_run(&p, s, run, &[("a1", "review:concurrency", "Review", 10, review()), ("a3", "fix", "Fix", 30, fix.clone())]);
862 }
863 let store = p.join("store");
864 let session = |s: &str| record(scan(&claude_runs(&p, Some(s))).0.iter(), None, &store);
865 let (n, homes) = session("s1");
866 assert_eq!((n, homes), (1, vec![store.clone()]));
867 let episodes = std::fs::read_to_string(store.join("episodes.jsonl")).unwrap();
868 assert!(episodes.contains("Vwf_1#") && !episodes.contains("wf_2"), "{episodes}");
869 let evs = crate::stats::events(&store);
871 assert_eq!(
872 evs.iter().filter(|e| e.kind == crate::stats::REVIEW_FINDING).map(|e| (e.session.as_str(), e.why.as_str())).collect::<Vec<_>>(),
873 [("s1", "medium")]
874 );
875 assert_eq!(session("s1").0, 0);
877 assert_eq!(record_session(&p, "../-w-app/s2", None, &store).0, 0);
878 assert_eq!(record_session(&p, "", None, &store).0, 0);
879 let _ = std::fs::remove_dir_all(p);
880 }
881
882 fn fixture(p: &str) -> PathBuf {
883 Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/review_fix").join(p)
884 }
885
886 fn rows(f: &[Finding]) -> Vec<(&str, &str, Fixed, bool)> {
888 f.iter().map(|x| (x.title.as_str(), x.severity.as_str(), x.fixed, x.strong())).collect()
889 }
890
891 #[test]
892 fn a_claude_workflow_run_gives_its_strong_finding() {
893 let f = found(&fixture("claude/projects"));
894 assert_eq!(rows(&f), [("The budget place is reset by an arc instead of drained", "high", Fixed::Yes, true)]);
895 assert_eq!((f[0].session.as_str(), f[0].file.as_deref(), f[0].workflow.as_str()), ("s1", Some("src/net.rs"), "build"));
896 }
897
898 #[test]
899 fn omp_findings_yielded_one_by_one_and_a_fix_task_give_one_strong_finding() {
900 let runs = prepared(harness::omp::session_runs(&fixture("omp/sessions"), Some("01a0a00d-0000-7000-8000-000000000001"), None));
901 let f = scan(&runs).0;
902 assert_eq!(
904 rows(&f),
905 [
906 ("Facet mapping drops the unit of a numeric range", "p1", Fixed::Yes, true),
907 ("Empty facet list renders a stray separator", "p3", Fixed::No, false)
908 ]
909 );
910 let x = &f[0];
911 assert_eq!(
912 (x.agent.as_str(), x.session.as_str(), x.cwd.as_str(), x.workflow.as_str()),
913 ("MappingReview", "01a0a00d-0000-7000-8000-000000000001", "/w/app", "Map the facets")
914 );
915 assert_eq!(x.file.as_deref(), Some("/w/app/src/facets/mapping.ts"));
916 assert!(x.detail.starts_with("`mapRange` in src/facets/mapping.ts") && x.task == "Implement the facet mapping in src/facets/mapping.ts.", "{x:?}");
917 }
918
919 #[test]
920 fn omp_findings_in_one_yielded_object_are_all_read() {
921 let runs = prepared(harness::omp::session_runs(&fixture("omp/sessions"), Some("01a0a00d-0000-7000-8000-000000000002"), None));
922 let f = scan(&runs).0;
923 assert_eq!(
924 rows(&f),
925 [
926 ("Retry loop never backs off", "p0", Fixed::NoFixStep, false),
927 ("Timeout is read in seconds and used as milliseconds", "p2", Fixed::NoFixStep, false)
928 ]
929 );
930 let all = harness::omp::session_runs(&fixture("omp/sessions"), None, None);
932 assert_eq!(all.iter().map(|r| r.agents.len()).collect::<Vec<_>>(), [3, 2]);
933 }
934
935 fn opencode_runs(session: Option<&str>) -> Vec<Run> {
937 let export = |id: &str, _: Option<&Path>| {
938 ["v1", "v2"]
939 .iter()
940 .find_map(|v| std::fs::read(fixture("opencode").join(v).join(format!("{id}.json"))).ok())
941 .and_then(|b| serde_json::from_slice(&b).ok())
942 };
943 let query = |_: &str| Some(vec![json!({"id": "ses_1root0000000000000000000"}), json!({"id": "ses_2root0000000000000000000"})]);
944 prepared(harness::opencode::session_runs(session, None, &export, &query))
945 }
946
947 #[test]
948 fn opencode_findings_come_from_a_reviewers_json_text_and_a_fix_child_fixes_them() {
949 let f = scan(&opencode_runs(Some("ses_1root0000000000000000000"))).0;
950 assert_eq!(rows(&f), [("Cache key ignores the tenant id", "high", Fixed::Yes, true)]);
951 assert_eq!((f[0].agent.as_str(), f[0].session.as_str(), f[0].cwd.as_str()), ("review cache", "ses_1root0000000000000000000", "/w/app"));
952 let all = scan(&opencode_runs(None)).0;
955 assert_eq!(rows(&all)[1], ("Queue bound is checked after the push", "medium", Fixed::Yes, true));
956 assert!(opencode_runs(Some("01a0a00d-0000-7000-8000-000000000001")).is_empty());
958 }
959
960 #[test]
961 fn harnesses_without_sub_agent_transcripts_give_no_runs() {
962 let home = fixture("omp");
963 let none: Vec<&str> = harness::ALL.iter().filter(|h| h.sub_runs(&home, None, None).is_empty()).map(|h| h.name()).collect();
964 assert!(none.contains(&"sim"), "{none:?}");
965 }
966
967 #[test]
968 fn severity_comes_from_a_priority_and_text_from_a_body() {
969 let f = |v: Value| findings_of(Some(&json!({"findings": [v]}))).into_iter().map(|r| (r.title, r.severity, r.detail, r.file)).next();
970 assert_eq!(f(json!({"body": "src/a.rs drops it", "priority": "P2"})), Some(("src/a.rs drops it".into(), "p2".into(), String::new(), None)));
971 assert_eq!(
972 f(json!({"title": "t", "body": "b", "priority": 0, "file_path": "src/a.rs"})).map(|x| (x.1, x.2, x.3)),
973 Some(("p0".into(), "b".into(), Some("src/a.rs".into())))
974 );
975 assert_eq!(f(json!({"title": "no severity"})), None);
976 let text = |w: &str| output_of(&SubAgent { final_text: w.into(), ..Default::default() });
977 assert_eq!(text("{\"command\": \"x\"}\nDone: {\"findings\": []} inline"), None, "only an object that starts a line");
978 assert!(text("Report\n```json\n{\"issues\": [{\"problem\": \"p\"}]}\n```").is_some());
979 }
980
981 #[test]
982 fn only_a_fix_step_is_a_fix_step_whatever_its_label_mentions() {
983 let a = |label: &str, phase: &str, kind: &str, output: Option<Value>| SubAgent {
984 label: label.into(),
985 phase: phase.into(),
986 kind: kind.into(),
987 output,
988 ..Default::default()
989 };
990 let fix = |label: &str, phase: &str| is_fix(&a(label, phase, "", None));
991 assert!(!fix("verify:fix-correctness", "Verify"));
993 assert!(!fix("review:harness-fix#0", "Review"));
994 assert!(!fix("impl:harness-fix", "Implement"));
995 assert!(!fix("verify:fix-correctness", ""), "the label's first segment decides");
996 assert!(fix("fix", "Fix") && fix("fix:rust", "") && fix("fix#1", "") && fix("anything", "Repair"));
997 assert!(fix("MappingValidityFix", "") && fix("DeriveFacetsRepairs", "") && !fix("Prefixed", "") && !fix("MappingReviewFix", ""));
998 assert!(fix("fix", "") && !fix("fix", "Review"), "a review phase is never a fix step");
999 assert!(is_fix(&a("ses", "", "fixer", None)), "an omp kind");
1000 assert!(!is_fix(&a("fix", "Fix", "", Some(json!({"findings": []})))));
1002 assert!(!is_fix(&a("fix", "Fix", "", Some(json!({"real": true})))));
1003 let p = temp("labels");
1005 let build = transcript("Build the harness fix.", &[], None);
1006 let fix_it = transcript("apply", &[("Bash", json!({"command": "sed -i x adapters/omp/borg.ts"}))], None);
1007 let verify = transcript("verify", &[("Bash", json!({"command": "cat adapters/omp/borg.ts"}))], Some(json!({"real": true})));
1008 write_run(
1009 &p,
1010 "s1",
1011 "wf_1",
1012 &[
1013 ("a0", "impl:harness-fix", "Implement", 5, build),
1014 ("a1", "review:harness-fix#0", "Review", 10, review()),
1015 ("a2", "verify:fix-correctness", "Verify", 20, verify),
1016 ],
1017 );
1018 let f = found(&p);
1019 assert_eq!((f[0].fixed, f[0].task.as_str()), (Fixed::NoFixStep, "Build the harness fix."));
1020 write_run(&p, "s2", "wf_2", &[("b1", "review:harness-fix#0", "Review", 10, review()), ("b2", "fix:rust", "", 30, fix_it)]);
1021 assert_eq!(found(&p)[1].fixed, Fixed::Yes);
1022 let _ = std::fs::remove_dir_all(p);
1023 }
1024
1025 #[test]
1026 fn findings_come_only_from_the_final_text_never_from_a_tool_input() {
1027 let echo = json!({"findings": [{"file": "src/a.rs", "severity": "high", "title": "the findings this fix step works through"}]});
1028 for last in ["", "Fixed src/a.rs."] {
1030 let a = SubAgent {
1031 label: "fix".into(),
1032 work: format!("{{\"command\":\"cargo test\"}}\n{echo}\n{last}\n"),
1033 final_text: last.into(),
1034 ..Default::default()
1035 };
1036 let a = SubAgent { output: output_of(&a), ..a };
1037 assert!(a.output.is_none() && is_fix(&a), "{last:?}");
1038 }
1039 let fenced = format!("Done.\n```json\n{}\n```\n", serde_json::to_string_pretty(&echo).unwrap());
1041 assert_eq!(output_of(&SubAgent { final_text: fenced, ..Default::default() }), Some(echo.clone()));
1042 assert_eq!(output_of(&SubAgent { final_text: echo.to_string(), ..Default::default() }), Some(echo));
1043 }
1044
1045 #[test]
1046 fn a_builder_in_another_repo_gives_no_task_text() {
1047 let p = temp("builder-repo");
1048 let fix = transcript("apply the findings", &[("Bash", json!({"command": "sed -i 's/a/b/' adapters/omp/borg.ts"}))], None);
1049 let build = transcript("build the secret feature", &[], None).replace("/w/app", "/w/private");
1050 write_run(&p, "s1", "wf_1", &[("a0", "build", "Build", 5, build), ("a1", "review:concurrency", "Review", 10, review()), ("a3", "fix", "Fix", 30, fix)]);
1051 let f = found(&p);
1052 assert_eq!(f.len(), 1);
1053 assert!(f[0].task.is_empty() && f[0].built.is_empty(), "{:?}", f[0].task);
1054 let _ = std::fs::remove_dir_all(p);
1055 }
1056}