use std::collections::BTreeSet;
use std::path::{Path, PathBuf};
use anyhow::Result;
use globset::{Glob, GlobSet, GlobSetBuilder};
use serde::{Deserialize, Serialize};
use crate::config::{Check, CheckKind, Config, Generated};
use crate::repo::{Repo, Worktree};
use crate::run;
pub struct Checker<'a> {
pub repo: &'a Repo,
pub config: &'a Config,
pub target: String,
pub scratch: PathBuf,
pub log_dir: PathBuf,
pub env: Vec<(String, String)>,
pub jobs: usize,
baseline: Option<Worktree>,
pub upstream_failures: BTreeSet<String>,
pub records: Vec<CheckRecord>,
comparisons: Vec<GoComparison>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct CheckRecord {
pub candidate: String,
pub target: String,
pub subject: String,
pub ordinal: usize,
pub name: String,
pub kind: String,
pub outcome: CheckStatus,
pub log: PathBuf,
pub go_comparisons: Vec<GoComparison>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum CheckStatus {
Passed,
Failed,
Skipped,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct GoComparison {
pub package: String,
pub test: String,
pub retries: usize,
pub upstream_runs: usize,
pub verdict: String,
}
pub enum Outcome {
Pass,
Fail {
check: String,
tier: String,
log: PathBuf,
},
}
impl<'a> Checker<'a> {
pub fn new(
repo: &'a Repo,
config: &'a Config,
target: String,
scratch: PathBuf,
log_dir: PathBuf,
) -> Checker<'a> {
let (jobs, env) = resource_env(config);
Checker {
repo,
config,
target,
scratch,
log_dir,
env,
jobs,
baseline: None,
upstream_failures: BTreeSet::new(),
records: Vec::new(),
comparisons: Vec::new(),
}
}
pub fn run(
&mut self,
dir: &Path,
checks: &[Check],
generated_if_changed: bool,
log_name: &str,
) -> Result<Outcome> {
let log = self
.log_dir
.join(format!("{}.log", log_name.replace('/', "_")));
let _ = std::fs::remove_file(&log);
let initial_head = run::output(dir, "git", &["rev-parse", "--verify", "HEAD"])?;
let packages = go_packages(dir, &self.target)?;
for (ordinal, check) in checks.iter().enumerate() {
self.comparisons.clear();
let kind = match check.kind {
CheckKind::Command => "command",
CheckKind::GoTest => "go_test",
};
let packages_arg = packages.join(" ");
if check.when.as_deref() == Some("go_packages") && packages.is_empty() {
self.record(
&initial_head,
log_name,
ordinal,
&check.name,
kind,
CheckStatus::Skipped,
&log,
);
continue;
}
let command = check
.run
.replace("{go_packages}", &packages_arg)
.replace("{jobs}", &self.jobs.to_string());
append(
&log,
&format!("jj-fork-check: {}\n$ {command}\n", check.name),
)?;
let ok = match check.kind {
CheckKind::Command => run::shell(dir, &command, &self.env, &log)?,
CheckKind::GoTest => self.go_test(dir, &command, &log)?,
};
self.record(
&initial_head,
log_name,
ordinal,
&check.name,
kind,
if ok {
CheckStatus::Passed
} else {
CheckStatus::Failed
},
&log,
);
if !ok {
let text = std::fs::read_to_string(&log).unwrap_or_default();
let tier = match check.low_if_errors_at_most {
Some(max) if compiler_errors(&text) <= max => "low".to_string(),
_ => check.tier.clone(),
};
return Ok(Outcome::Fail {
check: check.name.clone(),
tier,
log,
});
}
}
if let Some(generated) = &self.config.generated {
let changed = run::output(dir, "git", &["diff", "--name-only", &self.target, "HEAD"])?;
let inputs = globs(&generated.inputs)?;
let relevant = !generated_if_changed || changed.lines().any(|f| inputs.is_match(f));
if relevant {
append(&log, "jj-fork-check: generated\n")?;
let ok = generated_current(dir, generated, &self.env, &log)?;
self.comparisons.clear();
self.record(
&initial_head,
log_name,
checks.len(),
"generated",
"generated",
if ok {
CheckStatus::Passed
} else {
CheckStatus::Failed
},
&log,
);
if !ok {
return Ok(Outcome::Fail {
check: "generated".into(),
tier: "low".into(),
log,
});
}
} else {
self.comparisons.clear();
self.record(
&initial_head,
log_name,
checks.len(),
"generated",
"generated",
CheckStatus::Skipped,
&log,
);
}
}
let head = run::output(dir, "git", &["rev-parse", "--verify", "HEAD"]);
let status = run::output(
dir,
"git",
&["status", "--porcelain", "--untracked-files=no"],
)?;
self.comparisons.clear();
let intact = head.as_ref().ok() == Some(&initial_head) && status.is_empty();
self.record(
&initial_head,
log_name,
checks.len() + 1,
"candidate integrity",
"integrity",
if intact {
CheckStatus::Passed
} else {
CheckStatus::Failed
},
&log,
);
if !intact {
append(
&log,
&format!(
"jj-fork-check: candidate integrity\nchecks changed the candidate: expected HEAD {initial_head}, got {}\ntracked working-tree/index changes:\n{status}\n",
head.unwrap_or_else(|e| format!("unavailable ({e})")),
),
)?;
return Ok(Outcome::Fail {
check: "candidate integrity".into(),
tier: "medium".into(),
log,
});
}
Ok(Outcome::Pass)
}
#[allow(clippy::too_many_arguments)]
fn record(
&mut self,
candidate: &str,
subject: &str,
ordinal: usize,
name: &str,
kind: &str,
outcome: CheckStatus,
log: &Path,
) {
self.records.push(CheckRecord {
candidate: candidate.into(),
target: self.target.clone(),
subject: subject.into(),
ordinal,
name: name.into(),
kind: kind.into(),
outcome,
log: log.into(),
go_comparisons: self.comparisons.clone(),
});
}
fn go_test(&mut self, dir: &Path, command: &str, log: &Path) -> Result<bool> {
let attempt = self.scratch.join("go-test.log");
let _ = std::fs::remove_file(&attempt);
if run::shell(dir, command, &self.env, &attempt)? {
append(log, &std::fs::read_to_string(&attempt).unwrap_or_default())?;
return Ok(true);
}
let text = std::fs::read_to_string(&attempt).unwrap_or_default();
append(log, &text)?;
let mut real = false;
for failure in go_test_failures(&text) {
let Some(test) = failure.test else {
append(
log,
&format!(
"jj-fork: {} failed without a failing test (build failure)\n",
failure.package
),
)?;
real = true;
continue;
};
let env = self.env.clone();
let scratch_log = self.scratch.join("retry.log");
let rerun = |dir: &Path, count: usize| {
let command = format!("go test -count={count} -run '^{test}$' {}", failure.package);
run::shell(dir, &command, &env, &scratch_log)
};
let mut retries = 0;
let verdict = classify_failure(
|| {
retries += 1;
rerun(dir, 1)
},
|| {
let baseline = self.baseline()?;
rerun(&baseline, UPSTREAM_RUNS)
},
)?;
self.comparisons.push(GoComparison {
package: failure.package.clone(),
test: test.clone(),
retries,
upstream_runs: if verdict == Verdict::Flaky {
0
} else {
UPSTREAM_RUNS
},
verdict: match verdict {
Verdict::Flaky => "flaky",
Verdict::UpstreamFailure => "upstream_failure",
Verdict::Real => "real_failure",
}
.into(),
});
match verdict {
Verdict::Flaky => append(
log,
&format!(
"jj-fork: note: flaky: {} {test} failed, then passed on retry\n",
failure.package
),
)?,
Verdict::UpstreamFailure => {
append(
log,
&format!(
"jj-fork: note: {} {test} also fails on upstream; ignoring\n",
failure.package
),
)?;
self.upstream_failures
.insert(format!("{} {test}", failure.package));
}
Verdict::Real => {
append(
log,
&format!(
"jj-fork: {} {test} fails every retry here but passes {UPSTREAM_RUNS} runs on upstream\n",
failure.package
),
)?;
real = true;
}
}
}
Ok(!real)
}
fn baseline(&mut self) -> Result<PathBuf> {
if self.baseline.is_none() {
self.baseline = Some(
self.repo
.worktree(&self.scratch, "baseline", &self.target)?,
);
}
Ok(self.baseline.as_ref().unwrap().path.clone())
}
}
const RETRIES: usize = 3;
const UPSTREAM_RUNS: usize = 5;
#[derive(Debug, PartialEq, Eq)]
enum Verdict {
Flaky,
UpstreamFailure,
Real,
}
fn classify_failure(
mut retry: impl FnMut() -> Result<bool>,
upstream: impl FnOnce() -> Result<bool>,
) -> Result<Verdict> {
for _ in 0..RETRIES {
if retry()? {
return Ok(Verdict::Flaky);
}
}
Ok(if upstream()? {
Verdict::Real
} else {
Verdict::UpstreamFailure
})
}
fn resource_env(config: &Config) -> (usize, Vec<(String, String)>) {
let mib = std::fs::read_to_string("/proc/meminfo")
.ok()
.and_then(|s| {
s.lines()
.find(|l| l.starts_with("MemTotal:"))
.and_then(|l| l.split_whitespace().nth(1))
.and_then(|kb| kb.parse::<u64>().ok())
})
.map(|kb| kb / 1024)
.unwrap_or(0);
let cpus = std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(2);
if mib == 0 || mib >= config.low_memory.below_mib {
return (cpus, Vec::new());
}
let jobs = if mib < 4096 { 1 } else { 2 };
let limit = (mib * 60 / 100).to_string();
let env = config
.low_memory
.env
.iter()
.map(|(k, v)| {
(
k.clone(),
v.replace("{jobs}", &jobs.to_string())
.replace("{memory_limit_mib}", &limit),
)
})
.collect();
(jobs, env)
}
fn go_packages(dir: &Path, target: &str) -> Result<Vec<String>> {
let changed = run::output(
dir,
"git",
&["diff", "--name-only", target, "HEAD", "--", "*.go"],
)?;
let dirs: BTreeSet<String> = changed
.lines()
.filter_map(|f| {
Path::new(f)
.parent()
.map(|p| p.to_string_lossy().to_string())
})
.collect();
Ok(dirs
.into_iter()
.filter(|d| {
let path = if d.is_empty() {
dir.to_path_buf()
} else {
dir.join(d)
};
std::fs::read_dir(path)
.map(|entries| {
entries
.flatten()
.any(|e| e.path().extension().is_some_and(|x| x == "go"))
})
.unwrap_or(false)
})
.map(|d| {
if d.is_empty() {
".".to_string()
} else {
format!("./{d}")
}
})
.collect())
}
fn generated_current(
dir: &Path,
generated: &Generated,
env: &[(String, String)],
log: &Path,
) -> Result<bool> {
if !run::shell(dir, &generated.regenerate, env, log)? {
return Ok(false);
}
if let Some(header) = &generated.header {
let status = run::output(
dir,
"git",
&["status", "--porcelain", "--untracked-files=all"],
)?;
for line in status.lines() {
let file = dir.join(line.get(2..).unwrap_or_default().trim_start());
restore_header(&file, &header.text, &header.comments)?;
}
}
let status = run::output(
dir,
"git",
&["status", "--porcelain", "--untracked-files=all"],
)?;
if !status.is_empty() {
append(
log,
&format!("generated files are out of date:\n{status}\n"),
)?;
return Ok(false);
}
Ok(true)
}
fn restore_header(
file: &Path,
text: &str,
comments: &std::collections::BTreeMap<String, String>,
) -> Result<()> {
let Some(prefix) = file
.extension()
.and_then(|e| comments.get(&e.to_string_lossy().to_string()))
else {
return Ok(());
};
let Ok(content) = std::fs::read_to_string(file) else {
return Ok(());
};
let first = text.lines().next().unwrap_or_default();
if content.lines().next().is_some_and(|l| l.contains(first)) {
return Ok(());
}
let mut header: String = text.lines().map(|l| format!("{prefix} {l}\n")).collect();
header.push('\n');
std::fs::write(file, header + &content)?;
Ok(())
}
pub fn globs(patterns: &[String]) -> Result<GlobSet> {
let mut builder = GlobSetBuilder::new();
for p in patterns {
builder.add(Glob::new(p)?);
}
Ok(builder.build()?)
}
fn append(log: &Path, text: &str) -> Result<()> {
use std::io::Write;
let mut f = std::fs::File::options()
.create(true)
.append(true)
.open(log)?;
f.write_all(text.as_bytes())?;
Ok(())
}
#[derive(Debug, PartialEq)]
pub struct GoTestFailure {
pub package: String,
pub test: Option<String>,
}
pub fn go_test_failures(output: &str) -> Vec<GoTestFailure> {
let mut failures = Vec::new();
let mut pending: Vec<String> = Vec::new();
for line in output.lines() {
if let Some(rest) = line.strip_prefix("--- FAIL: ") {
if let Some(name) = rest.split_whitespace().next() {
pending.push(name.to_string());
}
} else if let Some(rest) = line.strip_prefix("FAIL\t") {
let package = rest.split('\t').next().unwrap_or_default().to_string();
if pending.is_empty() {
failures.push(GoTestFailure {
package,
test: None,
});
continue;
}
for test in pending.drain(..) {
failures.push(GoTestFailure {
package: package.clone(),
test: Some(test),
});
}
}
}
failures
}
pub fn compiler_errors(log: &str) -> usize {
log.lines()
.filter(|line| {
let mut parts = line.splitn(4, ':');
let (Some(path), Some(row), Some(col), Some(_)) =
(parts.next(), parts.next(), parts.next(), parts.next())
else {
return false;
};
!path.is_empty()
&& !path.contains(' ')
&& !row.is_empty()
&& row.chars().all(|c| c.is_ascii_digit())
&& !col.is_empty()
&& col.chars().all(|c| c.is_ascii_digit())
})
.count()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn successful_checks_must_leave_the_candidate_intact() {
for (command, passes, diagnostic) in [
("git checkout --detach HEAD^", false, "expected HEAD"),
("printf changed > tracked", false, "M tracked"),
(
"printf changed > tracked; git add tracked",
false,
"M tracked",
),
(
"printf artifact > artifact; printf ignored > ignored",
true,
"",
),
] {
let dir = tempfile::tempdir().unwrap();
let path = dir.path();
run::output(path, "git", &["init", "-q"]).unwrap();
run::output(path, "git", &["config", "user.name", "test"]).unwrap();
run::output(path, "git", &["config", "user.email", "test@example.com"]).unwrap();
std::fs::write(path.join("tracked"), "original").unwrap();
std::fs::write(path.join(".gitignore"), "ignored\n").unwrap();
run::output(path, "git", &["add", "."]).unwrap();
run::output(path, "git", &["commit", "-qm", "first"]).unwrap();
run::output(path, "git", &["commit", "--allow-empty", "-qm", "second"]).unwrap();
let repo = Repo { root: path.into() };
let config: Config = toml::from_str("[upstream]\nurl = 'unused'\n").unwrap();
let target = run::output(path, "git", &["rev-parse", "HEAD"]).unwrap();
let mut checker =
Checker::new(&repo, &config, target.clone(), path.into(), path.into());
let checks: Vec<Check> = toml::from_str::<crate::config::Checks>(&format!(
"patch = [{{name = 'mutator', run = {command:?}}}, {{name = 'skip', run = 'exit 1', when = 'go_packages'}}]"
))
.unwrap()
.patch;
match checker.run(path, &checks, true, "candidate").unwrap() {
Outcome::Pass => assert!(passes, "accepted {command}"),
Outcome::Fail { check, tier, log } => {
assert!(!passes, "rejected {command}");
assert_eq!(check, "candidate integrity");
assert_eq!(tier, "medium");
let text = std::fs::read_to_string(log).unwrap();
assert!(text.contains(diagnostic), "{text}");
}
}
assert_eq!(checker.records.len(), 3);
assert!(checker.records.iter().all(|r| r.candidate == target));
assert_eq!(checker.records[0].outcome, CheckStatus::Passed);
assert_eq!(checker.records[1].outcome, CheckStatus::Skipped);
assert_eq!(checker.records[2].kind, "integrity");
assert_eq!(
checker.records[2].outcome,
if passes {
CheckStatus::Passed
} else {
CheckStatus::Failed
}
);
if passes {
assert!(path.join("artifact").exists());
assert!(path.join("ignored").exists());
} else if command.contains("printf") {
assert_eq!(
std::fs::read_to_string(path.join("tracked")).unwrap(),
"changed"
);
}
}
}
#[test]
fn go_test_failures_attribute_tests_to_packages() {
let output = "\
--- FAIL: TestA (0.00s)
a_test.go:10: boom
--- FAIL: TestA/sub (0.00s)
--- FAIL: TestB (0.01s)
FAIL
FAIL\texample.com/m/a\t0.1s
ok \texample.com/m/b\t0.2s
# example.com/m/c
c.go:3:5: undefined: x
FAIL\texample.com/m/c [build failed]
";
assert_eq!(
go_test_failures(output),
vec![
GoTestFailure {
package: "example.com/m/a".into(),
test: Some("TestA".into())
},
GoTestFailure {
package: "example.com/m/a".into(),
test: Some("TestB".into())
},
GoTestFailure {
package: "example.com/m/c [build failed]".into(),
test: None
},
]
);
}
#[test]
fn a_retry_that_passes_is_flaky_and_skips_upstream() {
let mut runs = 0;
let verdict = classify_failure(
|| {
runs += 1;
Ok(runs == 3)
},
|| panic!("upstream must not run"),
)
.unwrap();
assert_eq!(verdict, Verdict::Flaky);
assert_eq!(runs, 3);
}
#[test]
fn failing_every_retry_then_upstream_decides() {
let mut runs = 0;
let verdict = classify_failure(
|| {
runs += 1;
Ok(false)
},
|| Ok(false),
)
.unwrap();
assert_eq!(verdict, Verdict::UpstreamFailure);
assert_eq!(runs, RETRIES);
assert_eq!(
classify_failure(|| Ok(false), || Ok(true)).unwrap(),
Verdict::Real
);
}
#[test]
fn compiler_errors_count_only_located_messages() {
let log = "# pkg\na/b.go:10:19: undefined: X\nc.go:1:2: bad\nnote: something: else\nhttp://host:80: no\n";
assert_eq!(compiler_errors(log), 2);
}
#[test]
fn header_is_prepended_once() {
let dir = tempfile::tempdir().unwrap();
let file = dir.path().join("x.yaml");
std::fs::write(&file, "---\nkind: X\n").unwrap();
let comments = [("yaml".to_string(), "#".to_string())]
.into_iter()
.collect();
restore_header(&file, "Copyright A\nLine two", &comments).unwrap();
restore_header(&file, "Copyright A\nLine two", &comments).unwrap();
assert_eq!(
std::fs::read_to_string(&file).unwrap(),
"# Copyright A\n# Line two\n\n---\nkind: X\n"
);
}
#[test]
fn header_is_restored_on_the_first_status_line() {
let dir = tempfile::tempdir().unwrap();
let git = |args: &[&str]| run::output(dir.path(), "git", args).unwrap();
git(&["init", "-q"]);
std::fs::write(dir.path().join("a.yaml"), "# Copyright A\n\nkind: X\n").unwrap();
git(&["add", "."]);
git(&[
"-c",
"user.name=t",
"-c",
"user.email=t@e",
"commit",
"-qm",
"a",
]);
let generated: Generated = toml::from_str(
"paths = ['a.yaml']\nregenerate = \"printf 'kind: X\\\\n' > a.yaml\"\n[header]\ntext = 'Copyright A'\ncomments = { yaml = '#' }\n",
)
.unwrap();
let log = dir.path().join(".git").join("log");
assert!(generated_current(dir.path(), &generated, &[], &log).unwrap());
}
}