1use std::collections::{BTreeMap, HashSet};
24use std::hash::BuildHasher;
25use std::path::{Path, PathBuf};
26use std::time::Duration;
27
28use serde::Serialize;
29
30use crate::notes::links::{self, Link, LinkRef, Side, Touched};
31use crate::notes::reasoning;
32use crate::notes::regress_cosmetic as cosmetic;
33use crate::notes::regress_diff::{self as diff, Hunk};
34pub use crate::notes::regress_git::Git;
35use crate::notes::regress_signals as signals;
36use crate::notes::{self, Excerpt, redact};
37use crate::{collective, learn};
38
39const TEXT_CAP: usize = 240;
42const SPAN_CAP: usize = 400;
45const FILES_CAP: usize = 10;
48const RANGES_CAP: usize = 20;
50const DAY: i64 = crate::clock::DAY_MS / 1000;
52
53#[derive(Debug, Clone, Copy, PartialEq, Eq)]
54pub enum Mode {
55 Off,
56 Links,
58 Learn,
59}
60
61#[derive(Debug, Clone)]
63pub struct Cfg {
64 pub mode: Mode,
65 pub min_age_s: i64,
66 pub max_links: usize,
67 pub max_files: usize,
68 pub max_lines: usize,
69 pub call_timeout: Duration,
70 pub budget: Duration,
71}
72
73impl Default for Cfg {
74 fn default() -> Cfg {
75 Cfg {
76 mode: Mode::Links,
77 min_age_s: 2 * DAY,
78 max_links: 5,
79 max_files: 40,
80 max_lines: 2000,
81 call_timeout: Duration::from_secs(10),
82 budget: Duration::from_secs(60),
83 }
84 }
85}
86
87impl Cfg {
88 pub fn from_env() -> Cfg {
89 let var = |k: &str| std::env::var(k).ok();
90 let mode = match var("BORG_REGRESS").as_deref() {
91 Some("off") => Mode::Off,
92 Some("learn") => Mode::Learn,
93 _ => Mode::Links,
94 };
95 let d = Cfg::default();
96 Cfg {
97 mode,
98 min_age_s: var("BORG_REGRESS_MIN_AGE").and_then(|s| signals::parse_age(&s)).unwrap_or(d.min_age_s),
99 call_timeout: var("BORG_REGRESS_TIMEOUT_S").and_then(|s| s.trim().parse().ok()).map_or(d.call_timeout, Duration::from_secs),
100 ..d
101 }
102 }
103
104 #[must_use]
106 pub fn git<'a>(&self, repo: &'a Path) -> Git<'a> {
107 Git::new(repo, self.call_timeout, self.budget)
108 }
109}
110
111#[derive(Debug, Clone, PartialEq, Eq)]
113pub struct Scope {
114 pub here: String,
115 pub store: PathBuf,
116 pub sync: bool,
117}
118
119impl Scope {
120 #[must_use]
122 pub fn of(route: &collective::Route, store: &Path) -> Option<Scope> {
123 let collective::Route::Collective { name, .. } = route else { return None };
124 Some(Scope { here: name.clone(), store: store.to_path_buf(), sync: collective::may_sync(route) })
125 }
126}
127
128#[derive(Debug, Clone, Default)]
130pub struct Fix {
131 pub sha: String,
132 pub session: String,
134 pub task: String,
135 pub subject: String,
136 pub body: String,
137 pub secs: i64,
139 pub date: String,
140 pub email: String,
141 pub session_text: String,
143 pub tests: Option<String>,
145 pub windows: Vec<(i64, i64)>,
146}
147
148#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
150pub struct Skipped {
151 pub fix: String,
152 pub origin: String,
154 pub reason: String,
155}
156
157#[derive(Debug, Clone, Default)]
159pub struct Detected {
160 pub links: Vec<Link>,
161 pub skipped: Vec<Skipped>,
162}
163
164impl Detected {
165 fn skip(&mut self, fix: &str, origin: &str, reason: impl Into<String>) {
166 self.skipped.push(Skipped { fix: fix.into(), origin: origin.into(), reason: reason.into() });
167 }
168}
169
170fn first_line(s: &str, cap: usize) -> String {
171 let line = s.lines().map(str::trim).find(|l| !l.is_empty()).unwrap_or_default();
172 redact(line).chars().take(cap).collect()
173}
174
175pub struct Run<'a> {
177 pub git: &'a Git<'a>,
178 pub scope: &'a Scope,
179 pub cfg: &'a Cfg,
180 pub any_origin: bool,
182 pub seen: &'a HashSet<(String, String)>,
184}
185
186#[derive(Debug, Default)]
188struct Origin {
189 sha: String,
190 lines: Vec<(String, u32)>,
191 secs: i64,
192 date: String,
193 renamed: bool,
195}
196
197fn shape(git: &Git, cfg: &Cfg, sha: &str) -> Result<String, String> {
199 let parents = git.run(&["rev-parse", &format!("{sha}^@")]).ok_or("timeout")?;
200 let parents: Vec<&str> = parents.lines().filter(|l| !l.is_empty()).collect();
201 let parent = match parents[..] {
202 [] => return Err("no-parent".into()),
203 [p] => p.to_string(),
204 _ => return Err("merge".into()),
205 };
206 let numstat = git.run(&["diff", "--numstat", "-z", "-M", "--no-ext-diff", &parent, sha]).ok_or("timeout")?;
207 let (files, lines) = diff::size(&numstat);
208 if files > cfg.max_files || lines > cfg.max_lines {
209 return Err(format!("too-large ({files} files, {lines} lines)"));
210 }
211 Ok(parent)
212}
213
214fn generated_by_attr(git: &Git, hunks: &[Hunk]) -> HashSet<String> {
216 let mut args = vec!["check-attr", "-z", "linguist-generated", "diff", "--"];
217 args.extend(hunks.iter().map(|h| h.path.as_str()).filter(|p| !p.is_empty()));
218 git.run(&args).map(|out| diff::generated_by_attr(&out)).unwrap_or_default()
219}
220
221fn unblamed(h: &Hunk) -> Option<(String, &'static str)> {
223 let at = |n: u32| format!("{}:{n}", if h.path.is_empty() { "(new file)" } else { &h.path });
224 match h.removed.first() {
225 None => Some((at(h.added_at), "additions-only")),
226 Some(&(n, _)) if h.path.is_empty() => Some((at(n), "unparsed-path")),
227 Some(&(n, _)) => cosmetic::formatting_only(h).then(|| (at(n), "format")),
228 }
229}
230
231fn skipped_files(git: &Git, all: &[Hunk]) -> BTreeMap<String, &'static str> {
234 let mut files: BTreeMap<String, &'static str> = cosmetic::reflowed_files(all).into_iter().map(|p| (p, "format")).collect();
235 let named = all.iter().map(|h| h.path.clone()).filter(|p| diff::generated(p));
236 files.extend(named.chain(generated_by_attr(git, all)).map(|p| (p, "generated")));
237 files
238}
239
240fn changed(git: &Git, parent: &str, sha: &str, d: &mut Detected) -> Vec<Hunk> {
243 let Some(all) = git.hunks(parent, sha) else {
244 d.skip(sha, "", "timeout");
245 return Vec::new();
246 };
247 let files = skipped_files(git, &all);
248 for (path, reason) in &files {
249 d.skip(sha, path, *reason);
250 }
251 let rest = all.into_iter().map(|mut h| {
253 let why = if files.contains_key(&h.path) { None } else { unblamed(&h) };
254 if let Some((at, reason)) = &why {
255 d.skip(sha, at, *reason);
256 }
257 if why.is_some() || files.contains_key(&h.path) {
258 h.removed.clear();
259 }
260 h
261 });
262 let (kept, moved) = cosmetic::drop_moves(rest.collect());
263 if moved > 0 {
264 d.skip(sha, &format!("{moved} line(s)"), "move");
265 }
266 kept
267}
268
269fn shallow_roots(git: &Git) -> HashSet<String> {
271 let Some(p) = git.run(&["rev-parse", "--git-path", "shallow"]) else { return HashSet::new() };
272 let p = git.repo().join(p.trim());
273 std::fs::read_to_string(p).unwrap_or_default().lines().map(|l| l.trim().to_string()).collect()
274}
275
276fn in_rename(hunks: &[Hunk], path: &str, n: u32) -> bool {
278 hunks.iter().any(|h| h.path == path && h.removed.iter().any(|(m, _)| *m == n) && cosmetic::renamed_only(h))
279}
280
281fn origins(git: &Git, parent: &str, hunks: &[Hunk], d: &mut Detected) -> BTreeMap<String, Origin> {
283 let shallow = shallow_roots(git);
284 let mut out: BTreeMap<String, Origin> = BTreeMap::new();
285 for (path, mut lines) in diff::removed_by_path(hunks) {
286 lines.truncate(SPAN_CAP);
287 let spans: Vec<String> = diff::ranges(&lines).iter().map(|(a, b)| format!("-L{a},{b}")).collect();
288 let mut args = vec!["blame", "--root", "-w", "-M", "-C", "--porcelain"];
289 args.extend(spans.iter().map(String::as_str));
290 args.extend([parent, "--", path.as_str()]);
291 let Some(porcelain) = git.run(&args) else {
292 d.skip(parent, &path, "blame-failed");
293 continue;
294 };
295 for (sha, line) in diff::blame_lines(&porcelain).into_iter().filter(|(s, _)| !shallow.contains(s)) {
296 let o = out.entry(sha.clone()).or_insert_with(|| Origin { sha, renamed: true, ..Origin::default() });
297 o.renamed &= in_rename(hunks, &path, line);
298 o.lines.push((path.clone(), line));
299 }
300 }
301 out
302}
303
304fn dated(git: &Git, origins: &mut BTreeMap<String, Origin>) {
306 let mut args = vec!["show", "-s", "--format=%H %ct %cs"];
307 args.extend(origins.keys().map(String::as_str));
308 for l in git.run(&args).unwrap_or_default().lines() {
309 let mut f = l.split(' ');
310 if let (Some(sha), Some(ct), Some(cs)) = (f.next(), f.next(), f.next())
311 && let Some(o) = origins.get_mut(sha)
312 {
313 (o.secs, o.date) = (ct.parse().unwrap_or(i64::MAX), cs.to_string());
314 }
315 }
316}
317
318fn days(secs: i64) -> String {
319 if secs % DAY == 0 { format!("{}d", secs / DAY) } else { format!("{}h", secs / 3_600) }
320}
321
322fn verdict(run: &Run, fix: &Fix, o: &Origin, why: Option<&Excerpt>) -> Result<(), String> {
324 if o.sha == fix.sha {
325 return Err("self".into());
326 }
327 if why.is_none() && !run.any_origin {
328 return Err("no-reasoning".into());
329 }
330 if why.is_some_and(|e| !fix.session.is_empty() && e.session == fix.session) {
331 return Err("same-session".into());
332 }
333 let age = fix.secs.saturating_sub(o.secs);
334 if age < run.cfg.min_age_s {
335 return Err(format!("too-young ({} < {})", days(age.max(0)), days(run.cfg.min_age_s)));
336 }
337 if run.seen.contains(&(o.sha.clone(), fix.sha.clone())) || why.is_some_and(|e| e.fixed_by.iter().any(|r| r.commit == fix.sha)) {
338 return Err("duplicate".into());
339 }
340 if run.git.run(&["merge-base", "--is-ancestor", &o.sha, &fix.sha]).is_none() {
341 return Err("not-ancestor".into());
342 }
343 Ok(())
344}
345
346#[must_use]
348pub fn signals_of(fix: &Fix) -> Vec<String> {
349 let checks = [
350 (signals::REVERT, signals::revert_signal(&fix.subject, &fix.body)),
351 (signals::SUBJECT, signals::subject_signal(&fix.subject)),
352 (signals::TESTS, fix.tests.as_deref().is_some_and(|t| signals::tests_signal(t, &fix.windows, fix.secs.saturating_mul(1000).saturating_add(999)))),
353 (signals::WORDS, signals::words_signal(&fix.session_text)),
354 ];
355 checks.iter().filter(|(_, on)| *on).map(|(s, _)| (*s).to_string()).collect()
356}
357
358fn touched(lines: &[(String, u32)]) -> Vec<Touched> {
360 let mut out: Vec<Touched> = Vec::new();
361 for (path, _) in lines {
362 if out.len() < FILES_CAP && !out.iter().any(|t| t.path == *path) {
363 let mine: Vec<u32> = lines.iter().filter(|(p, _)| p == path).map(|(_, n)| *n).collect();
364 out.push(Touched { path: path.clone(), lines: diff::ranges(&mine).into_iter().take(RANGES_CAP).collect() });
365 }
366 }
367 out
368}
369
370fn repo_name(repo: &Path) -> String {
371 repo.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default()
372}
373
374fn link(run: &Run, fix: &Fix, o: &Origin, why: Option<&Excerpt>, found: &[String]) -> Link {
375 let age = fix.secs.saturating_sub(o.secs).max(0) / DAY;
376 Link {
377 kind: links::KIND.into(),
378 commit: o.sha.clone(),
379 fix: fix.sha.clone(),
380 repo: repo_name(run.git.repo()),
381 collective: run.scope.here.clone(),
382 strength: if signals::strong(found) { links::FIX } else { links::REVISITED }.into(),
383 signals: found.to_vec(),
384 decision: Side {
385 session: why.map(|e| e.session.clone()).unwrap_or_default(),
386 task: why.map(|e| first_line(&e.task, TEXT_CAP)).unwrap_or_default(),
387 subject: String::new(),
388 date: o.date.clone(),
389 secs: if o.secs == i64::MAX { 0 } else { o.secs },
391 },
392 fixed: Side {
393 session: fix.session.clone(),
394 task: first_line(&fix.task, TEXT_CAP),
395 subject: first_line(&fix.subject, TEXT_CAP),
396 date: fix.date.clone(),
397 secs: fix.secs,
398 },
399 files: touched(&o.lines),
400 lines: o.lines.len(),
401 age_days: u32::try_from(age).unwrap_or(u32::MAX),
402 ms: learn::now_ms(),
403 }
404}
405
406#[must_use]
409pub fn detect(run: &Run, fix: &Fix) -> Detected {
410 let mut d = Detected::default();
411 let parent = match shape(run.git, run.cfg, &fix.sha) {
412 Ok(p) => p,
413 Err(reason) => {
414 d.skip(&fix.sha, "", reason);
415 return d;
416 }
417 };
418 let hunks = changed(run.git, &parent, &fix.sha, &mut d);
419 let mut found = origins(run.git, &parent, &hunks, &mut d);
420 if found.is_empty() {
421 return d;
422 }
423 dated(run.git, &mut found);
424 let shas: Vec<String> = found.keys().cloned().collect();
425 let why = reasoning::reasoning_for(run.git.repo(), &run.scope.store, &run.scope.here, &shas);
426 let strength = signals_of(fix);
427 let several = found.len() > 1;
429 for o in found.values() {
430 let signals = if several && o.renamed { &[][..] } else { &strength[..] };
431 match verdict(run, fix, o, why.get(&o.sha)) {
432 Ok(()) => d.links.push(link(run, fix, o, why.get(&o.sha), signals)),
433 Err(reason) => d.skip(&fix.sha, &o.sha, reason),
434 }
435 }
436 d
437}
438
439#[must_use]
442pub fn dedupe_cap(d: Detected, max: usize) -> Detected {
443 let Detected { links, mut skipped } = d;
444 let mut keys = HashSet::new();
445 let mut kept: Vec<Link> = Vec::new();
446 for l in links {
447 let by = if l.fixed.session.is_empty() { &l.fix } else { &l.fixed.session };
448 if keys.insert((l.commit.clone(), by.clone())) {
449 kept.push(l);
450 } else {
451 skipped.push(Skipped { fix: l.fix, origin: l.commit, reason: "duplicate".into() });
452 }
453 }
454 kept.sort_by_key(|l| (!l.strong(), std::cmp::Reverse(l.lines)));
455 for l in kept.split_off(max.min(kept.len())) {
456 skipped.push(Skipped { fix: l.fix, origin: l.commit, reason: "cap".into() });
457 }
458 Detected { links: kept, skipped }
459}
460
461fn stored_links(store: &Path, here: &str) -> Vec<Link> {
463 let text = std::fs::read_to_string(store.join("reasoning.jsonl")).unwrap_or_default();
464 links::records(&text).into_iter().filter(|l| reasoning::visible(&l.collective, here)).collect()
465}
466
467pub fn links_of<S: BuildHasher>(store: &Path, here: &str, shas: &HashSet<&str, S>) -> Vec<Link> {
469 stored_links(store, here).into_iter().filter(|l| shas.contains(l.commit.as_str()) || shas.contains(l.fix.as_str())).collect()
470}
471
472fn seen_in(store: &Path, here: &str) -> HashSet<(String, String)> {
473 stored_links(store, here).into_iter().map(|l| (l.commit, l.fix)).collect()
474}
475
476fn commit_info(git: &Git, sha: &str) -> Option<Fix> {
478 let text = git.run(&["show", "-s", "--format=%ct%x00%cs%x00%ae%x00%s%x00%b", sha])?;
479 let f: Vec<&str> = text.splitn(5, '\0').collect();
480 let [secs, date, email, subject, body] = f[..] else { return None };
481 Some(Fix {
482 sha: sha.into(),
483 secs: secs.trim().parse().ok()?,
484 date: date.into(),
485 email: email.into(),
486 subject: subject.into(),
487 body: body.into(),
488 ..Fix::default()
489 })
490}
491
492pub(crate) fn session_file(l: ¬es::Landed, ext: &str) -> Option<String> {
494 std::fs::read_to_string(l.dir.join(format!("{}.{ext}", notes::safe(l.session)))).ok()
495}
496
497fn session_fix(l: ¬es::Landed, git: &Git, sha: &str) -> Option<Fix> {
499 let read = |ext: &str| session_file(l, ext);
500 let task = read("task").unwrap_or_default();
501 Some(Fix {
502 session: l.session.into(),
503 session_text: format!("{}\n{task}", read("prompts").unwrap_or_default()),
504 task,
505 tests: read("tests"),
506 windows: l.start.map(|st| notes::windows(l.dir, l.session, st)).unwrap_or_default(),
507 ..commit_info(git, sha)?
508 })
509}
510
511#[must_use]
514pub fn session_links(l: ¬es::Landed, scope: &Scope, cfg: &Cfg) -> Detected {
515 links_with(&cfg.git(l.repo), l, scope, cfg)
516}
517
518#[must_use]
520pub fn links_with(git: &Git, l: ¬es::Landed, scope: &Scope, cfg: &Cfg) -> Detected {
521 let seen = seen_in(&scope.store, &scope.here);
522 let run = Run { git, scope, cfg, any_origin: false, seen: &seen };
523 let mut all = Detected::default();
524 for c in l.commits {
525 let Some(fix) = session_fix(l, git, c) else { continue };
526 let d = detect(&run, &fix);
527 all.links.extend(d.links);
528 all.skipped.extend(d.skipped);
529 }
530 dedupe_cap(all, cfg.max_links)
531}
532
533#[derive(Debug, Default, PartialEq, Eq)]
535pub struct Recorded {
536 pub links: usize,
537 pub strong: usize,
538 pub replicated: usize,
540 pub episodes: usize,
542 pub errors: Vec<String>,
543}
544
545fn add_ref(repo: &Path, scope: &Scope, commit: &str, r: LinkRef, decision: bool) -> Result<bool, String> {
549 let e = match notes::note_of(repo, commit) {
550 Some(e) => Some(e),
551 None if decision => reasoning::stored_of(&scope.store, &scope.here, &HashSet::from([commit])).remove(commit),
552 None => None,
553 };
554 let Some(mut e) = e.filter(|e| reasoning::visible(&e.collective, &scope.here)) else { return Ok(false) };
555 let list = if decision { &mut e.fixed_by } else { &mut e.fixes };
556 if links::push_ref(list, r) { notes::write_note(repo, &e).map(|()| true) } else { Ok(false) }
557}
558
559pub fn record(repo: &Path, scope: &Scope, links: &[Link]) -> Recorded {
562 let mut done = Recorded::default();
563 for l in links {
564 let (fixed_by, fixes) = links::refs(l);
565 let results = [add_ref(repo, scope, &l.commit, fixed_by, true), add_ref(repo, scope, &l.fix, fixes, false)];
566 done.errors.extend(results.into_iter().filter_map(Result::err));
567 if scope.sync {
568 match learn::append_jsonl(&scope.store.join("reasoning.jsonl"), l) {
569 Ok(()) => done.replicated += 1,
570 Err(e) => done.errors.push(format!("{}: {e}", scope.store.display())),
571 }
572 }
573 done.links += 1;
574 done.strong += usize::from(l.strong());
575 }
576 done
577}
578
579fn replay_fix(git: &Git, sha: &str) -> Option<Fix> {
581 let note = notes::note_of(git.repo(), sha);
582 let task = note.as_ref().map(|e| e.task.clone()).unwrap_or_default();
583 Some(Fix { session: note.map(|e| e.session).unwrap_or_default(), session_text: task.clone(), task, ..commit_info(git, sha)? })
584}
585
586#[derive(Debug, Default)]
588pub struct Replayed {
589 pub found: Detected,
590 pub scanned: usize,
591 pub capped: bool,
593 pub shallow: bool,
594}
595
596pub const REPLAY_CAP: usize = 500;
599
600pub fn replay(repo: &Path, range: &str, scope: &Scope, cfg: &Cfg, any_origin: bool) -> Result<Replayed, String> {
606 let list = cfg.git(repo).run(&["rev-list", "--no-merges", "--reverse", range, "--"]).ok_or(format!("{range} is not a commit range here"))?;
607 let commits: Vec<&str> = list.lines().filter(|l| !l.is_empty()).collect();
608 let seen = seen_in(&scope.store, &scope.here);
609 let mut out = Replayed { capped: commits.len() > REPLAY_CAP, ..Replayed::default() };
610 for sha in commits.into_iter().take(REPLAY_CAP) {
611 let git = cfg.git(repo);
612 out.shallow |= out.scanned == 0 && !shallow_roots(&git).is_empty();
613 let run = Run { git: &git, scope, cfg, any_origin, seen: &seen };
614 let Some(fix) = replay_fix(&git, sha) else { continue };
615 let d = dedupe_cap(detect(&run, &fix), cfg.max_links);
616 out.found.links.extend(d.links);
617 out.found.skipped.extend(d.skipped);
618 out.scanned += 1;
619 }
620 Ok(out)
621}
622
623#[cfg(test)]
624mod tests {
625 use super::*;
626
627 fn l(commit: &str, fix: &str, session: &str, strong: bool, lines: usize) -> Link {
628 Link {
629 commit: commit.into(),
630 fix: fix.into(),
631 strength: if strong { links::FIX } else { links::REVISITED }.into(),
632 fixed: Side { session: session.into(), ..Side::default() },
633 lines,
634 ..Link::default()
635 }
636 }
637
638 #[test]
639 fn dedupe_then_cap_strong_first() {
640 let mut links = vec![l("d1", "f2", "s-7", false, 1), l("d1", "f1", "s-7", true, 9)];
641 links.extend((2..8).map(|i| l(&format!("d{i}"), "f2", "s-7", i % 2 == 0, i)));
642 let d = dedupe_cap(Detected { links, skipped: vec![] }, 5);
643 let kept: Vec<(&str, bool)> = d.links.iter().map(|x| (x.commit.as_str(), x.strong())).collect();
644 assert_eq!(kept, [("d6", true), ("d4", true), ("d2", true), ("d7", false), ("d5", false)]);
645 let reasons: Vec<(&str, &str)> = d.skipped.iter().map(|s| (s.origin.as_str(), s.reason.as_str())).collect();
646 assert_eq!(reasons, [("d1", "duplicate"), ("d3", "cap"), ("d1", "cap")], "the newest fix of a session keeps the link");
647 }
648
649 #[test]
650 fn cfg_from_env_defaults() {
651 let d = Cfg::default();
652 assert_eq!((d.mode, d.min_age_s, d.max_links, d.max_files, d.max_lines), (Mode::Links, 172_800, 5, 40, 2000));
653 assert_eq!(days(172_800), "2d");
654 assert_eq!(days(43_200), "12h");
655 }
656}