use std::fs::{self, OpenOptions};
use std::io::Write;
use std::path::{Path, PathBuf};
use jiff::Timestamp;
use serde::{Deserialize, Serialize};
use crate::error::Error;
use crate::job::Weights;
use crate::stats::Stats;
const KEEP: usize = 20;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Entry {
pub ts: Timestamp,
pub cmd: Vec<String>,
pub cwd: String,
pub requested: Weights,
pub measured: Stats,
pub exit_code: i32,
#[serde(default)]
pub v: u32,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub repo: Option<String>,
}
#[must_use]
pub fn repo_key(cwd: &Path) -> Option<String> {
let root = cwd.ancestors().find(|d| d.join(".git").exists())?;
let git = root.join(".git");
let common = if git.is_dir() {
git
} else {
let text = fs::read_to_string(&git).ok()?;
let own = root.join(text.trim().strip_prefix("gitdir:")?.trim());
match fs::read_to_string(own.join("commondir")) {
Ok(c) => own.join(c.trim()),
Err(_) => own,
}
};
let common = common.canonicalize().unwrap_or(common);
let rel = cwd
.strip_prefix(root)
.map_or_else(|_| PathBuf::new(), Path::to_path_buf);
Some(format!("{}:{}", common.display(), rel.display()))
}
pub const VERSION: u32 = 1;
pub fn append(path: &Path, entry: &Entry) -> Result<(), Error> {
if let Some(dir) = path.parent() {
fs::create_dir_all(dir)?;
}
let mut line = serde_json::to_vec(entry)?;
line.push(b'\n');
OpenOptions::new()
.create(true)
.append(true)
.open(path)?
.write_all(&line)?;
Ok(())
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Defaults {
pub cpu: u32,
pub mem: u64,
pub vram: Option<u64>,
pub mem_peak_at_ms: Option<u64>,
pub cpu_peak_at_ms: Option<u64>,
pub cpu_peak_median_ms: Option<u64>,
}
fn median(mut v: Vec<u64>) -> Option<u64> {
if v.is_empty() {
return None;
}
v.sort_unstable();
v.get(v.len() / 2).copied()
}
#[must_use]
pub fn defaults_from(
entries: &[Entry],
cmd: &[String],
cwd: &str,
repo: Option<&str>,
) -> Option<Defaults> {
defaults_with(
entries,
cmd,
cwd,
repo,
if cfg!(target_os = "linux") {
VERSION
} else {
0
},
)
}
const PROGRAM_WORDS: usize = 2;
#[must_use]
pub fn defaults_with(
entries: &[Entry],
cmd: &[String],
cwd: &str,
repo: Option<&str>,
min_v: u32,
) -> Option<Defaults> {
let ok: Vec<&Entry> = entries
.iter()
.filter(|e| e.v >= min_v && e.exit_code == 0)
.collect();
let same_repo = |e: &Entry| repo.is_some() && e.repo.as_deref() == repo;
let pick =
|tier: &dyn Fn(&Entry) -> bool| ok.iter().copied().filter(|e| tier(e)).collect::<Vec<_>>();
if let Some(d) = aggregate(&pick(&|e| e.cwd == cwd && e.cmd == cmd), median) {
return Some(d);
}
if let Some(d) = aggregate(&pick(&|e| same_repo(e) && e.cmd == cmd), median) {
return Some(d);
}
if !program_pools(cmd) {
return None;
}
let program = &cmd[..PROGRAM_WORDS];
aggregate(
&pick(&|e| same_repo(e) && e.cmd.starts_with(program)),
|v| v.into_iter().max(),
)
}
const WRAPPERS: [&str; 11] = [
"env",
"bash",
"sh",
"zsh",
"sudo",
"nice",
"time",
"nohup",
"xargs",
"stokehold",
"heavy",
];
fn program_pools(cmd: &[String]) -> bool {
if cmd.len() <= PROGRAM_WORDS {
return false;
}
let base = Path::new(&cmd[0])
.file_name()
.and_then(|n| n.to_str())
.unwrap_or(&cmd[0]);
!WRAPPERS.contains(&base) && !cmd[1].starts_with('-')
}
fn aggregate(mine: &[&Entry], pick: impl Fn(Vec<u64>) -> Option<u64>) -> Option<Defaults> {
let mine = &mine[mine.len().saturating_sub(KEEP)..];
let cores = pick(
mine.iter()
.map(|e| {
if e.measured.peak_centi_cores > 0 {
e.measured.peak_centi_cores
} else {
e.measured.centi_cores
}
})
.collect(),
)?;
let mem = pick(
mine.iter()
.map(|e| e.measured.peak_tree_rss.max(e.measured.peak_rss))
.collect(),
)?;
let vram = pick(mine.iter().filter_map(|e| e.measured.peak_vram).collect());
let mem_peak_at_ms = pick(
mine.iter()
.map(|e| e.measured.peak_mem_at_ms)
.filter(|ms| *ms > 0)
.collect(),
);
let ramps: Vec<u64> = mine
.iter()
.map(|e| e.measured.peak_cpu_at_ms)
.filter(|ms| *ms > 0)
.collect();
let cpu_peak_at_ms = ramps.iter().copied().max();
let cpu_peak_median_ms = median(ramps);
Some(Defaults {
cpu: u32::try_from(cores.div_ceil(100).max(1)).unwrap_or(u32::MAX),
mem,
vram,
mem_peak_at_ms,
cpu_peak_at_ms,
cpu_peak_median_ms,
})
}
#[must_use]
pub fn defaults_for(
path: &Path,
cmd: &[String],
cwd: &str,
repo: Option<&str>,
) -> Option<Defaults> {
let text = fs::read_to_string(path).ok()?;
let entries: Vec<Entry> = text
.lines()
.filter_map(|l| serde_json::from_str(l).ok())
.collect();
defaults_from(&entries, cmd, cwd, repo)
}
#[cfg(test)]
mod tests {
use super::*;
fn entry(cmd: &str, cwd: &str, centi: u64, rss: u64, code: i32) -> Entry {
Entry {
ts: Timestamp::UNIX_EPOCH,
cmd: vec![cmd.into()],
cwd: cwd.into(),
requested: Weights::default(),
measured: Stats {
centi_cores: centi,
peak_rss: rss,
..Stats::default()
},
exit_code: code,
v: VERSION,
repo: None,
}
}
#[test]
fn median_defaults_for_same_cmd_and_cwd() {
let all = vec![
entry("t", "/a", 150, 100, 0),
entry("t", "/a", 350, 300, 0),
entry("t", "/a", 250, 200, 0),
entry("t", "/b", 900, 900, 0),
entry("t", "/a", 900, 900, 1),
];
let d = defaults_from(&all, &["t".to_string()], "/a", None).unwrap();
assert_eq!(
d,
Defaults {
cpu: 3,
mem: 200,
vram: None,
mem_peak_at_ms: None,
cpu_peak_at_ms: None,
cpu_peak_median_ms: None
}
);
assert!(defaults_from(&all, &["other".to_string()], "/a", None).is_none());
}
#[test]
fn cpu_time_to_peak_is_the_slowest_of_the_runs_that_recorded_it() {
let runs: Vec<Entry> = [0, 20_000, 30_000, 40_000]
.into_iter()
.map(|ms| {
let mut e = entry("t", "/a", 100, 50, 0);
e.measured.peak_cpu_at_ms = ms;
e
})
.collect();
let d = defaults_from(&runs, &["t".to_string()], "/a", None).unwrap();
assert_eq!(d.cpu_peak_at_ms, Some(40_000), "0 (unknown) is left out");
assert_eq!(
d.cpu_peak_median_ms,
Some(30_000),
"the median of 20, 30 and 40 s"
);
let old = defaults_from(&runs[..1], &["t".to_string()], "/a", None).unwrap();
assert_eq!(old.cpu_peak_at_ms, None);
}
#[test]
fn defaults_use_tree_peaks_not_averages() {
let mut e = entry("t", "/a", 100, 50, 0);
e.measured.peak_centi_cores = 650;
e.measured.peak_tree_rss = 900;
let d = defaults_from(&[e], &["t".to_string()], "/a", None).unwrap();
assert_eq!(
d,
Defaults {
cpu: 7,
mem: 900,
vram: None,
mem_peak_at_ms: None,
cpu_peak_at_ms: None,
cpu_peak_median_ms: None
}
);
}
#[test]
fn v0_entries_are_ignored_when_min_version_is_one() {
let mut old = entry("t", "/a", 100, 46_000, 0);
old.v = 0;
let new = entry("t", "/a", 100, 2_600, 0);
let c = ["t".to_string()];
assert_eq!(defaults_with(&[old.clone()], &c, "/a", None, 1), None);
assert_eq!(
defaults_with(&[old, new], &c, "/a", None, 1).unwrap().mem,
2_600
);
}
#[test]
fn v0_entries_are_used_when_min_version_is_zero() {
let mut old = entry("t", "/a", 100, 700, 0);
old.v = 0;
assert_eq!(
defaults_with(&[old], &["t".to_string()], "/a", None, 0)
.unwrap()
.mem,
700
);
}
#[test]
fn entries_without_a_version_field_parse_as_v0_and_new_ones_write_v1() {
let mut value = serde_json::to_value(entry("t", "/a", 1, 1, 0)).unwrap();
value.as_object_mut().unwrap().remove("v");
assert_eq!(serde_json::from_value::<Entry>(value).unwrap().v, 0);
let json = serde_json::to_string(&entry("t", "/a", 1, 1, 0)).unwrap();
assert!(json.contains("\"v\":1"), "{json}");
}
#[test]
fn append_writes_one_json_line() {
let t = tempfile::tempdir().unwrap();
let p = t.path().join("sub/history.jsonl");
append(&p, &entry("t", "/a", 100, 1, 0)).unwrap();
append(&p, &entry("t", "/a", 100, 1, 0)).unwrap();
assert_eq!(fs::read_to_string(&p).unwrap().lines().count(), 2);
assert_eq!(
defaults_for(&p, &["t".to_string()], "/a", None)
.unwrap()
.cpu,
1
);
}
fn in_repo(cmd: &[&str], cwd: &str, repo: &str, peak: u64) -> Entry {
let mut e = entry("x", cwd, peak, 1 << 30, 0);
e.cmd = cmd.iter().map(|s| (*s).to_string()).collect();
e.measured.peak_centi_cores = peak;
e.repo = Some(repo.into());
e
}
#[test]
fn a_fresh_worktree_learns_from_the_other_worktrees_of_its_repo() {
let s = |v: &[&str]| v.iter().map(|s| (*s).to_string()).collect::<Vec<_>>();
let bdg = "/code/bdg/.git:apple";
let all = vec![
in_repo(&["make", "ui-check"], "/s/wt216/apple", bdg, 200),
in_repo(&["make", "ui-check"], "/s/wt218/apple", bdg, 300),
in_repo(&["make", "ui-check"], "/s/wt219/apple", bdg, 250),
in_repo(&["make", "lint"], "/s/wt216/apple", bdg, 900),
in_repo(
&["make", "ui-check"],
"/s/wt9/web",
"/code/other/.git:web",
1600,
),
];
assert_eq!(
defaults_from(&all, &s(&["make", "ui-check"]), "/s/wt218/apple", Some(bdg))
.unwrap()
.cpu,
3
);
assert_eq!(
defaults_from(&all, &s(&["make", "ui-check"]), "/s/wt225/apple", Some(bdg))
.unwrap()
.cpu,
3
);
assert_eq!(
defaults_from(
&all,
&s(&["make", "ui-check", "SCREENS=a"]),
"/s/wt225/apple",
Some(bdg)
)
.unwrap()
.cpu,
3
);
assert!(
defaults_from(
&all,
&s(&["make", "ui-check"]),
"/s/wt225/apple",
Some("/code/bdg/.git:web")
)
.is_none()
);
assert!(defaults_from(&all, &s(&["make", "ui-check"]), "/s/wt225/apple", None).is_none());
assert!(defaults_from(&all, &s(&["make", "test"]), "/s/wt225/apple", Some(bdg)).is_none());
}
#[test]
fn pooled_tier_uses_the_max_not_the_median() {
let s = |v: &[&str]| v.iter().map(|s| (*s).to_string()).collect::<Vec<_>>();
let r = "/c/.git:a";
let all = vec![
in_repo(&["cargo", "test", "-p", "a"], "/w1", r, 200),
in_repo(&["cargo", "test", "-p", "b"], "/w1", r, 300),
in_repo(&["cargo", "test", "-p", "c"], "/w1", r, 1600),
];
assert_eq!(
defaults_from(&all, &s(&["cargo", "test", "-p", "d"]), "/w2", Some(r))
.unwrap()
.cpu,
16
);
}
#[test]
fn pooled_tier_skips_bare_programs_wrappers_and_flags() {
let s = |v: &[&str]| v.iter().map(|s| (*s).to_string()).collect::<Vec<_>>();
let r = "/c/.git:a";
let all = vec![
in_repo(&["make", "lint"], "/w1", r, 100),
in_repo(&["bash", "-c", "x"], "/w1", r, 100),
in_repo(&["env", "A=1", "make"], "/w1", r, 100),
in_repo(&["xcodebuild", "-scheme", "A"], "/w1", r, 100),
];
let d = |c: &[&str]| defaults_from(&all, &s(c), "/w2", Some(r));
assert!(d(&["make"]).is_none(), "a bare program is not pooled");
assert!(d(&["make", "other"]).is_none(), "two words is not pooled");
assert!(d(&["bash", "-c", "y"]).is_none());
assert!(d(&["env", "A=1", "make", "lint"]).is_none());
assert!(d(&["xcodebuild", "-scheme", "B"]).is_none());
assert!(d(&["/usr/bin/sudo", "make", "lint"]).is_none());
assert!(d(&["make", "lint", "X=1"]).is_some());
}
#[test]
fn repo_key_is_shared_by_worktrees_and_absent_outside_git() {
let t = tempfile::tempdir().unwrap();
let main = t.path().join("repo");
fs::create_dir_all(main.join(".git/worktrees/wt1")).unwrap();
fs::create_dir_all(main.join("apple")).unwrap();
fs::write(main.join(".git/worktrees/wt1/commondir"), "../..\n").unwrap();
let wt = t.path().join("scratch/wt1");
fs::create_dir_all(wt.join("apple")).unwrap();
fs::write(
wt.join(".git"),
format!("gitdir: {}\n", main.join(".git/worktrees/wt1").display()),
)
.unwrap();
let a = repo_key(&main.join("apple")).unwrap();
assert_eq!(
repo_key(&wt.join("apple")).unwrap(),
a,
"a worktree shares the main checkout's key"
);
assert!(a.ends_with(":apple"), "{a}");
assert_ne!(
repo_key(&wt).unwrap(),
a,
"the worktree root is another place in the repo"
);
assert_eq!(repo_key(t.path()), None);
let mut value = serde_json::to_value(entry("t", "/a", 1, 1, 0)).unwrap();
assert!(value.get("repo").is_none(), "absent repo is not written");
value
.as_object_mut()
.unwrap()
.insert("repo".into(), a.clone().into());
assert_eq!(
serde_json::from_value::<Entry>(value).unwrap().repo,
Some(a)
);
}
}