1use std::collections::{BTreeMap, HashSet};
24use std::hash::BuildHasher;
25use std::path::{Path, PathBuf};
26use std::time::Duration;
27
28use serde::Serialize;
29
30use crate::notes::links::{self, Link, LinkRef, Side, Touched};
31use crate::notes::reasoning;
32use crate::notes::regress_cosmetic as cosmetic;
33use crate::notes::regress_diff::{self as diff, Hunk};
34pub use crate::notes::regress_git::Git;
35use crate::notes::regress_signals as signals;
36use crate::notes::{self, Excerpt, redact};
37use crate::{collective, learn};
38
39const TEXT_CAP: usize = 240;
41const SPAN_CAP: usize = 400;
43const FILES_CAP: usize = 10;
45const RANGES_CAP: usize = 20;
46const DAY: i64 = 86_400;
47
48#[derive(Debug, Clone, Copy, PartialEq, Eq)]
49pub enum Mode {
50 Off,
51 Links,
53 Learn,
54}
55
56#[derive(Debug, Clone)]
58pub struct Cfg {
59 pub mode: Mode,
60 pub min_age_s: i64,
61 pub max_links: usize,
62 pub max_files: usize,
63 pub max_lines: usize,
64 pub call_timeout: Duration,
65 pub budget: Duration,
66}
67
68impl Default for Cfg {
69 fn default() -> Cfg {
70 Cfg {
71 mode: Mode::Links,
72 min_age_s: 2 * DAY,
73 max_links: 5,
74 max_files: 40,
75 max_lines: 2000,
76 call_timeout: Duration::from_secs(10),
77 budget: Duration::from_secs(60),
78 }
79 }
80}
81
82impl Cfg {
83 pub fn from_env() -> Cfg {
84 let var = |k: &str| std::env::var(k).ok();
85 let mode = match var("BORG_REGRESS").as_deref() {
86 Some("off") => Mode::Off,
87 Some("learn") => Mode::Learn,
88 _ => Mode::Links,
89 };
90 let d = Cfg::default();
91 Cfg {
92 mode,
93 min_age_s: var("BORG_REGRESS_MIN_AGE").and_then(|s| signals::parse_age(&s)).unwrap_or(d.min_age_s),
94 call_timeout: var("BORG_REGRESS_TIMEOUT_S").and_then(|s| s.trim().parse().ok()).map_or(d.call_timeout, Duration::from_secs),
95 ..d
96 }
97 }
98
99 #[must_use]
101 pub fn git<'a>(&self, repo: &'a Path) -> Git<'a> {
102 Git::new(repo, self.call_timeout, self.budget)
103 }
104}
105
106#[derive(Debug, Clone, PartialEq, Eq)]
108pub struct Scope {
109 pub here: String,
110 pub store: PathBuf,
111 pub sync: bool,
112}
113
114impl Scope {
115 #[must_use]
117 pub fn of(route: &collective::Route, store: &Path) -> Option<Scope> {
118 let collective::Route::Collective { name, .. } = route else { return None };
119 Some(Scope { here: name.clone(), store: store.to_path_buf(), sync: collective::may_sync(route) })
120 }
121}
122
123#[derive(Debug, Clone, Default)]
125pub struct Fix {
126 pub sha: String,
127 pub session: String,
129 pub task: String,
130 pub subject: String,
131 pub body: String,
132 pub secs: i64,
134 pub date: String,
135 pub email: String,
136 pub session_text: String,
138 pub tests: Option<String>,
140 pub windows: Vec<(i64, i64)>,
141}
142
143#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
145pub struct Skipped {
146 pub fix: String,
147 pub origin: String,
149 pub reason: String,
150}
151
152#[derive(Debug, Clone, Default)]
154pub struct Detected {
155 pub links: Vec<Link>,
156 pub skipped: Vec<Skipped>,
157}
158
159impl Detected {
160 fn skip(&mut self, fix: &str, origin: &str, reason: impl Into<String>) {
161 self.skipped.push(Skipped { fix: fix.into(), origin: origin.into(), reason: reason.into() });
162 }
163}
164
165fn first_line(s: &str, cap: usize) -> String {
166 let line = s.lines().map(str::trim).find(|l| !l.is_empty()).unwrap_or_default();
167 redact(line).chars().take(cap).collect()
168}
169
170pub struct Run<'a> {
172 pub git: &'a Git<'a>,
173 pub scope: &'a Scope,
174 pub cfg: &'a Cfg,
175 pub any_origin: bool,
177 pub seen: &'a HashSet<(String, String)>,
179}
180
181#[derive(Debug, Default)]
183struct Origin {
184 sha: String,
185 lines: Vec<(String, u32)>,
186 secs: i64,
187 date: String,
188 renamed: bool,
190}
191
192fn shape(git: &Git, cfg: &Cfg, sha: &str) -> Result<String, String> {
194 let parents = git.run(&["rev-parse", &format!("{sha}^@")]).ok_or("timeout")?;
195 let parents: Vec<&str> = parents.lines().filter(|l| !l.is_empty()).collect();
196 let parent = match parents[..] {
197 [] => return Err("no-parent".into()),
198 [p] => p.to_string(),
199 _ => return Err("merge".into()),
200 };
201 let numstat = git.run(&["diff", "--numstat", "-z", "-M", "--no-ext-diff", &parent, sha]).ok_or("timeout")?;
202 let (files, lines) = diff::size(&numstat);
203 if files > cfg.max_files || lines > cfg.max_lines {
204 return Err(format!("too-large ({files} files, {lines} lines)"));
205 }
206 Ok(parent)
207}
208
209fn generated_by_attr(git: &Git, hunks: &[Hunk]) -> HashSet<String> {
211 let mut args = vec!["check-attr", "-z", "linguist-generated", "diff", "--"];
212 args.extend(hunks.iter().map(|h| h.path.as_str()).filter(|p| !p.is_empty()));
213 git.run(&args).map(|out| diff::generated_by_attr(&out)).unwrap_or_default()
214}
215
216fn unblamed(h: &Hunk) -> Option<(String, &'static str)> {
218 let at = |n: u32| format!("{}:{n}", if h.path.is_empty() { "(new file)" } else { &h.path });
219 match h.removed.first() {
220 None => Some((at(h.added_at), "additions-only")),
221 Some(&(n, _)) if h.path.is_empty() => Some((at(n), "unparsed-path")),
222 Some(&(n, _)) => cosmetic::formatting_only(h).then(|| (at(n), "format")),
223 }
224}
225
226fn skipped_files(git: &Git, all: &[Hunk]) -> BTreeMap<String, &'static str> {
229 let mut files: BTreeMap<String, &'static str> = cosmetic::reflowed_files(all).into_iter().map(|p| (p, "format")).collect();
230 let named = all.iter().map(|h| h.path.clone()).filter(|p| diff::generated(p));
231 files.extend(named.chain(generated_by_attr(git, all)).map(|p| (p, "generated")));
232 files
233}
234
235fn changed(git: &Git, parent: &str, sha: &str, d: &mut Detected) -> Vec<Hunk> {
238 let Some(all) = git.hunks(parent, sha) else {
239 d.skip(sha, "", "timeout");
240 return Vec::new();
241 };
242 let files = skipped_files(git, &all);
243 for (path, reason) in &files {
244 d.skip(sha, path, *reason);
245 }
246 let rest = all.into_iter().map(|mut h| {
248 let why = if files.contains_key(&h.path) { None } else { unblamed(&h) };
249 if let Some((at, reason)) = &why {
250 d.skip(sha, at, *reason);
251 }
252 if why.is_some() || files.contains_key(&h.path) {
253 h.removed.clear();
254 }
255 h
256 });
257 let (kept, moved) = cosmetic::drop_moves(rest.collect());
258 if moved > 0 {
259 d.skip(sha, &format!("{moved} line(s)"), "move");
260 }
261 kept
262}
263
264fn shallow_roots(git: &Git) -> HashSet<String> {
266 let Some(p) = git.run(&["rev-parse", "--git-path", "shallow"]) else { return HashSet::new() };
267 let p = git.repo().join(p.trim());
268 std::fs::read_to_string(p).unwrap_or_default().lines().map(|l| l.trim().to_string()).collect()
269}
270
271fn in_rename(hunks: &[Hunk], path: &str, n: u32) -> bool {
273 hunks.iter().any(|h| h.path == path && h.removed.iter().any(|(m, _)| *m == n) && cosmetic::renamed_only(h))
274}
275
276fn origins(git: &Git, parent: &str, hunks: &[Hunk], d: &mut Detected) -> BTreeMap<String, Origin> {
278 let shallow = shallow_roots(git);
279 let mut out: BTreeMap<String, Origin> = BTreeMap::new();
280 for (path, mut lines) in diff::removed_by_path(hunks) {
281 lines.truncate(SPAN_CAP);
282 let spans: Vec<String> = diff::ranges(&lines).iter().map(|(a, b)| format!("-L{a},{b}")).collect();
283 let mut args = vec!["blame", "--root", "-w", "-M", "-C", "--porcelain"];
284 args.extend(spans.iter().map(String::as_str));
285 args.extend([parent, "--", path.as_str()]);
286 let Some(porcelain) = git.run(&args) else {
287 d.skip(parent, &path, "blame-failed");
288 continue;
289 };
290 for (sha, line) in diff::blame_lines(&porcelain).into_iter().filter(|(s, _)| !shallow.contains(s)) {
291 let o = out.entry(sha.clone()).or_insert_with(|| Origin { sha, renamed: true, ..Origin::default() });
292 o.renamed &= in_rename(hunks, &path, line);
293 o.lines.push((path.clone(), line));
294 }
295 }
296 out
297}
298
299fn dated(git: &Git, origins: &mut BTreeMap<String, Origin>) {
301 let mut args = vec!["show", "-s", "--format=%H %ct %cs"];
302 args.extend(origins.keys().map(String::as_str));
303 for l in git.run(&args).unwrap_or_default().lines() {
304 let mut f = l.split(' ');
305 if let (Some(sha), Some(ct), Some(cs)) = (f.next(), f.next(), f.next())
306 && let Some(o) = origins.get_mut(sha)
307 {
308 (o.secs, o.date) = (ct.parse().unwrap_or(i64::MAX), cs.to_string());
309 }
310 }
311}
312
313fn days(secs: i64) -> String {
314 if secs % DAY == 0 { format!("{}d", secs / DAY) } else { format!("{}h", secs / 3_600) }
315}
316
317fn verdict(run: &Run, fix: &Fix, o: &Origin, why: Option<&Excerpt>) -> Result<(), String> {
319 if o.sha == fix.sha {
320 return Err("self".into());
321 }
322 if why.is_none() && !run.any_origin {
323 return Err("no-reasoning".into());
324 }
325 if why.is_some_and(|e| !fix.session.is_empty() && e.session == fix.session) {
326 return Err("same-session".into());
327 }
328 let age = fix.secs.saturating_sub(o.secs);
329 if age < run.cfg.min_age_s {
330 return Err(format!("too-young ({} < {})", days(age.max(0)), days(run.cfg.min_age_s)));
331 }
332 if run.seen.contains(&(o.sha.clone(), fix.sha.clone())) || why.is_some_and(|e| e.fixed_by.iter().any(|r| r.commit == fix.sha)) {
333 return Err("duplicate".into());
334 }
335 if run.git.run(&["merge-base", "--is-ancestor", &o.sha, &fix.sha]).is_none() {
336 return Err("not-ancestor".into());
337 }
338 Ok(())
339}
340
341#[must_use]
343pub fn signals_of(fix: &Fix) -> Vec<String> {
344 let checks = [
345 (signals::REVERT, signals::revert_signal(&fix.subject, &fix.body)),
346 (signals::SUBJECT, signals::subject_signal(&fix.subject)),
347 (signals::TESTS, fix.tests.as_deref().is_some_and(|t| signals::tests_signal(t, &fix.windows, fix.secs.saturating_mul(1000).saturating_add(999)))),
348 (signals::WORDS, signals::words_signal(&fix.session_text)),
349 ];
350 checks.iter().filter(|(_, on)| *on).map(|(s, _)| (*s).to_string()).collect()
351}
352
353fn touched(lines: &[(String, u32)]) -> Vec<Touched> {
355 let mut out: Vec<Touched> = Vec::new();
356 for (path, _) in lines {
357 if out.len() < FILES_CAP && !out.iter().any(|t| t.path == *path) {
358 let mine: Vec<u32> = lines.iter().filter(|(p, _)| p == path).map(|(_, n)| *n).collect();
359 out.push(Touched { path: path.clone(), lines: diff::ranges(&mine).into_iter().take(RANGES_CAP).collect() });
360 }
361 }
362 out
363}
364
365fn repo_name(repo: &Path) -> String {
366 repo.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default()
367}
368
369fn link(run: &Run, fix: &Fix, o: &Origin, why: Option<&Excerpt>, found: &[String]) -> Link {
370 let age = fix.secs.saturating_sub(o.secs).max(0) / DAY;
371 Link {
372 kind: links::KIND.into(),
373 commit: o.sha.clone(),
374 fix: fix.sha.clone(),
375 repo: repo_name(run.git.repo()),
376 collective: run.scope.here.clone(),
377 strength: if signals::strong(found) { links::FIX } else { links::REVISITED }.into(),
378 signals: found.to_vec(),
379 decision: Side {
380 session: why.map(|e| e.session.clone()).unwrap_or_default(),
381 task: why.map(|e| first_line(&e.task, TEXT_CAP)).unwrap_or_default(),
382 subject: String::new(),
383 date: o.date.clone(),
384 secs: if o.secs == i64::MAX { 0 } else { o.secs },
386 },
387 fixed: Side {
388 session: fix.session.clone(),
389 task: first_line(&fix.task, TEXT_CAP),
390 subject: first_line(&fix.subject, TEXT_CAP),
391 date: fix.date.clone(),
392 secs: fix.secs,
393 },
394 files: touched(&o.lines),
395 lines: o.lines.len(),
396 age_days: u32::try_from(age).unwrap_or(u32::MAX),
397 ms: learn::now_ms(),
398 }
399}
400
401#[must_use]
404pub fn detect(run: &Run, fix: &Fix) -> Detected {
405 let mut d = Detected::default();
406 let parent = match shape(run.git, run.cfg, &fix.sha) {
407 Ok(p) => p,
408 Err(reason) => {
409 d.skip(&fix.sha, "", reason);
410 return d;
411 }
412 };
413 let hunks = changed(run.git, &parent, &fix.sha, &mut d);
414 let mut found = origins(run.git, &parent, &hunks, &mut d);
415 if found.is_empty() {
416 return d;
417 }
418 dated(run.git, &mut found);
419 let shas: Vec<String> = found.keys().cloned().collect();
420 let why = reasoning::reasoning_for(run.git.repo(), &run.scope.store, &run.scope.here, &shas);
421 let strength = signals_of(fix);
422 let several = found.len() > 1;
424 for o in found.values() {
425 let signals = if several && o.renamed { &[][..] } else { &strength[..] };
426 match verdict(run, fix, o, why.get(&o.sha)) {
427 Ok(()) => d.links.push(link(run, fix, o, why.get(&o.sha), signals)),
428 Err(reason) => d.skip(&fix.sha, &o.sha, reason),
429 }
430 }
431 d
432}
433
434#[must_use]
437pub fn dedupe_cap(d: Detected, max: usize) -> Detected {
438 let Detected { links, mut skipped } = d;
439 let mut keys = HashSet::new();
440 let mut kept: Vec<Link> = Vec::new();
441 for l in links {
442 let by = if l.fixed.session.is_empty() { &l.fix } else { &l.fixed.session };
443 if keys.insert((l.commit.clone(), by.clone())) {
444 kept.push(l);
445 } else {
446 skipped.push(Skipped { fix: l.fix, origin: l.commit, reason: "duplicate".into() });
447 }
448 }
449 kept.sort_by_key(|l| (!l.strong(), std::cmp::Reverse(l.lines)));
450 for l in kept.split_off(max.min(kept.len())) {
451 skipped.push(Skipped { fix: l.fix, origin: l.commit, reason: "cap".into() });
452 }
453 Detected { links: kept, skipped }
454}
455
456fn stored_links(store: &Path, here: &str) -> Vec<Link> {
458 let text = std::fs::read_to_string(store.join("reasoning.jsonl")).unwrap_or_default();
459 links::records(&text).into_iter().filter(|l| reasoning::visible(&l.collective, here)).collect()
460}
461
462pub fn links_of<S: BuildHasher>(store: &Path, here: &str, shas: &HashSet<&str, S>) -> Vec<Link> {
464 stored_links(store, here).into_iter().filter(|l| shas.contains(l.commit.as_str()) || shas.contains(l.fix.as_str())).collect()
465}
466
467fn seen_in(store: &Path, here: &str) -> HashSet<(String, String)> {
468 stored_links(store, here).into_iter().map(|l| (l.commit, l.fix)).collect()
469}
470
471fn commit_info(git: &Git, sha: &str) -> Option<Fix> {
473 let text = git.run(&["show", "-s", "--format=%ct%x00%cs%x00%ae%x00%s%x00%b", sha])?;
474 let f: Vec<&str> = text.splitn(5, '\0').collect();
475 let [secs, date, email, subject, body] = f[..] else { return None };
476 Some(Fix {
477 sha: sha.into(),
478 secs: secs.trim().parse().ok()?,
479 date: date.into(),
480 email: email.into(),
481 subject: subject.into(),
482 body: body.into(),
483 ..Fix::default()
484 })
485}
486
487pub(crate) fn session_file(l: ¬es::Landed, ext: &str) -> Option<String> {
489 std::fs::read_to_string(l.dir.join(format!("{}.{ext}", notes::safe(l.session)))).ok()
490}
491
492fn session_fix(l: ¬es::Landed, git: &Git, sha: &str) -> Option<Fix> {
494 let read = |ext: &str| session_file(l, ext);
495 let task = read("task").unwrap_or_default();
496 Some(Fix {
497 session: l.session.into(),
498 session_text: format!("{}\n{task}", read("prompts").unwrap_or_default()),
499 task,
500 tests: read("tests"),
501 windows: l.start.map(|st| notes::windows(l.dir, l.session, st)).unwrap_or_default(),
502 ..commit_info(git, sha)?
503 })
504}
505
506#[must_use]
509pub fn session_links(l: ¬es::Landed, scope: &Scope, cfg: &Cfg) -> Detected {
510 links_with(&cfg.git(l.repo), l, scope, cfg)
511}
512
513#[must_use]
515pub fn links_with(git: &Git, l: ¬es::Landed, scope: &Scope, cfg: &Cfg) -> Detected {
516 let seen = seen_in(&scope.store, &scope.here);
517 let run = Run { git, scope, cfg, any_origin: false, seen: &seen };
518 let mut all = Detected::default();
519 for c in l.commits {
520 let Some(fix) = session_fix(l, git, c) else { continue };
521 let d = detect(&run, &fix);
522 all.links.extend(d.links);
523 all.skipped.extend(d.skipped);
524 }
525 dedupe_cap(all, cfg.max_links)
526}
527
528#[derive(Debug, Default, PartialEq, Eq)]
530pub struct Recorded {
531 pub links: usize,
532 pub strong: usize,
533 pub replicated: usize,
535 pub episodes: usize,
537 pub errors: Vec<String>,
538}
539
540fn add_ref(repo: &Path, scope: &Scope, commit: &str, r: LinkRef, decision: bool) -> Result<bool, String> {
544 let e = match notes::note_of(repo, commit) {
545 Some(e) => Some(e),
546 None if decision => reasoning::stored_of(&scope.store, &scope.here, &HashSet::from([commit])).remove(commit),
547 None => None,
548 };
549 let Some(mut e) = e.filter(|e| reasoning::visible(&e.collective, &scope.here)) else { return Ok(false) };
550 let list = if decision { &mut e.fixed_by } else { &mut e.fixes };
551 if links::push_ref(list, r) { notes::write_note(repo, &e).map(|()| true) } else { Ok(false) }
552}
553
554pub fn record(repo: &Path, scope: &Scope, links: &[Link]) -> Recorded {
557 let mut done = Recorded::default();
558 for l in links {
559 let (fixed_by, fixes) = links::refs(l);
560 let results = [add_ref(repo, scope, &l.commit, fixed_by, true), add_ref(repo, scope, &l.fix, fixes, false)];
561 done.errors.extend(results.into_iter().filter_map(Result::err));
562 if scope.sync {
563 match learn::append_jsonl(&scope.store.join("reasoning.jsonl"), l) {
564 Ok(()) => done.replicated += 1,
565 Err(e) => done.errors.push(format!("{}: {e}", scope.store.display())),
566 }
567 }
568 done.links += 1;
569 done.strong += usize::from(l.strong());
570 }
571 done
572}
573
574fn replay_fix(git: &Git, sha: &str) -> Option<Fix> {
576 let note = notes::note_of(git.repo(), sha);
577 let task = note.as_ref().map(|e| e.task.clone()).unwrap_or_default();
578 Some(Fix { session: note.map(|e| e.session).unwrap_or_default(), session_text: task.clone(), task, ..commit_info(git, sha)? })
579}
580
581#[derive(Debug, Default)]
583pub struct Replayed {
584 pub found: Detected,
585 pub scanned: usize,
586 pub capped: bool,
588 pub shallow: bool,
589}
590
591pub const REPLAY_CAP: usize = 500;
593
594pub fn replay(repo: &Path, range: &str, scope: &Scope, cfg: &Cfg, any_origin: bool) -> Result<Replayed, String> {
600 let list = cfg.git(repo).run(&["rev-list", "--no-merges", "--reverse", range, "--"]).ok_or(format!("{range} is not a commit range here"))?;
601 let commits: Vec<&str> = list.lines().filter(|l| !l.is_empty()).collect();
602 let seen = seen_in(&scope.store, &scope.here);
603 let mut out = Replayed { capped: commits.len() > REPLAY_CAP, ..Replayed::default() };
604 for sha in commits.into_iter().take(REPLAY_CAP) {
605 let git = cfg.git(repo);
606 out.shallow |= out.scanned == 0 && !shallow_roots(&git).is_empty();
607 let run = Run { git: &git, scope, cfg, any_origin, seen: &seen };
608 let Some(fix) = replay_fix(&git, sha) else { continue };
609 let d = dedupe_cap(detect(&run, &fix), cfg.max_links);
610 out.found.links.extend(d.links);
611 out.found.skipped.extend(d.skipped);
612 out.scanned += 1;
613 }
614 Ok(out)
615}
616
617#[cfg(test)]
618mod tests {
619 use super::*;
620
621 fn l(commit: &str, fix: &str, session: &str, strong: bool, lines: usize) -> Link {
622 Link {
623 commit: commit.into(),
624 fix: fix.into(),
625 strength: if strong { links::FIX } else { links::REVISITED }.into(),
626 fixed: Side { session: session.into(), ..Side::default() },
627 lines,
628 ..Link::default()
629 }
630 }
631
632 #[test]
633 fn dedupe_then_cap_strong_first() {
634 let mut links = vec![l("d1", "f2", "s-7", false, 1), l("d1", "f1", "s-7", true, 9)];
635 links.extend((2..8).map(|i| l(&format!("d{i}"), "f2", "s-7", i % 2 == 0, i)));
636 let d = dedupe_cap(Detected { links, skipped: vec![] }, 5);
637 let kept: Vec<(&str, bool)> = d.links.iter().map(|x| (x.commit.as_str(), x.strong())).collect();
638 assert_eq!(kept, [("d6", true), ("d4", true), ("d2", true), ("d7", false), ("d5", false)]);
639 let reasons: Vec<(&str, &str)> = d.skipped.iter().map(|s| (s.origin.as_str(), s.reason.as_str())).collect();
640 assert_eq!(reasons, [("d1", "duplicate"), ("d3", "cap"), ("d1", "cap")], "the newest fix of a session keeps the link");
641 }
642
643 #[test]
644 fn cfg_from_env_defaults() {
645 let d = Cfg::default();
646 assert_eq!((d.mode, d.min_age_s, d.max_links, d.max_files, d.max_lines), (Mode::Links, 172_800, 5, 40, 2000));
647 assert_eq!(days(172_800), "2d");
648 assert_eq!(days(43_200), "12h");
649 }
650}