use crate::conflict_sides::SAVED_DIRECTORY;
use crate::diagnostic_log::debug;
use crate::discard_audit::{AUDIT_FILE, append_line, check_appendable};
use crate::discarded_store::{Interrupted, OpenFailure, Store, Tally};
use crate::iso_time::format_iso_milliseconds;
use crate::json_record::{quote, string_array};
use crate::service_protocol::RELEASE;
use std::ffi::OsStr;
use std::path::{Path, PathBuf};
pub use crate::discarded_store::Identity;
pub const DEFAULT_MAX_AGE_DAYS: u64 = 30;
pub const DEFAULT_MAX_BYTES: u64 = 1024 * 1024 * 1024;
pub const YOUNG_FLOOR_MILLISECONDS: i64 = 24 * 60 * 60 * 1000;
const DAY_MILLISECONDS: i64 = 24 * 60 * 60 * 1000;
pub const STAMP_RESOLUTION_MILLISECONDS: i64 = 1000;
const LISTED_COPIES: usize = 100;
const TAG: &str = "discardedPrune";
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct Limits {
pub max_age_days: u64,
pub max_bytes: u64,
}
impl Default for Limits {
fn default() -> Self {
return Self { max_age_days: DEFAULT_MAX_AGE_DAYS, max_bytes: DEFAULT_MAX_BYTES };
}
}
pub fn parse_size(text: &str) -> Option<u64> {
let trimmed: &str = text.trim();
let (digits, unit): (&str, u64) = match trimmed.chars().last()?.to_ascii_lowercase() {
'k' => (&trimmed[..trimmed.len() - 1], 1024),
'm' => (&trimmed[..trimmed.len() - 1], 1024 * 1024),
'g' => (&trimmed[..trimmed.len() - 1], 1024 * 1024 * 1024),
_ => (trimmed, 1),
};
if digits.is_empty() || !digits.bytes().all(|byte| { return byte.is_ascii_digit(); }) {
return None;
}
return digits.parse::<u64>().ok()?.checked_mul(unit).filter(|bytes| { return *bytes > 0; });
}
fn days_from_civil(year: i64, month: i64, day: i64) -> i64 {
let shifted_year: i64 = if month <= 2 { year - 1 } else { year };
let era: i64 = shifted_year.div_euclid(400);
let year_of_era: i64 = shifted_year.rem_euclid(400);
let shifted_month: i64 = if month > 2 { month - 3 } else { month + 9 };
let day_of_year: i64 = (153 * shifted_month + 2) / 5 + day - 1;
let day_of_era: i64 = year_of_era * 365 + year_of_era / 4 - year_of_era / 100 + day_of_year;
return era * 146_097 + day_of_era - 719_468;
}
fn days_in_month(year: i64, month: i64) -> i64 {
return match month {
1 | 3 | 5 | 7 | 8 | 10 | 12 => 31,
4 | 6 | 9 | 11 => 30,
_ if year % 4 == 0 && (year % 100 != 0 || year % 400 == 0) => 29,
_ => 28,
};
}
pub fn parse_stamp(name: &str) -> Option<i64> {
let bytes: &[u8] = name.as_bytes();
if bytes.len() < 19 || !name.is_ascii() {
return None;
}
let shape: &[u8; 19] = b"dddd-dd-ddTdd-dd-dd";
let fits = bytes[..19].iter().zip(shape).all(|(found, wanted)| { return if *wanted == b'd' { found.is_ascii_digit() } else { found == wanted }; });
if !fits {
return None;
}
let suffix: &str = &name[19..];
if !(suffix.is_empty() || (suffix.starts_with('-') && suffix.len() > 1 && suffix[1..].bytes().all(|byte| { return byte.is_ascii_digit(); }))) {
return None;
}
let number = |from: usize, to: usize| -> i64 { return name[from..to].parse::<i64>().unwrap_or(0); };
let (year, month, day): (i64, i64, i64) = (number(0, 4), number(5, 7), number(8, 10));
let (hour, minute, second): (i64, i64, i64) = (number(11, 13), number(14, 16), number(17, 19));
if !(1..=12).contains(&month) || day < 1 || day > days_in_month(year, month) || hour > 23 || minute > 59 || second > 59 {
return None;
}
return Some(((days_from_civil(year, month, day) * 24 + hour) * 60 + minute) * 60_000 + second * 1000);
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Stamped {
pub name: String,
pub millis: i64,
pub bytes: u128,
pub identity: Identity,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ProblemKind {
StoreUnusable,
EntryUnreadable,
EntryChanged,
PartiallyRemoved,
NotRemoved,
AuditUnwritable,
RemovedWithoutAudit,
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Problem {
pub path: PathBuf,
pub kind: ProblemKind,
pub detail: String,
}
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub struct Listing {
pub entries: Vec<Stamped>,
pub problems: Vec<Problem>,
pub files: usize,
pub listed_files: std::collections::HashMap<String, std::collections::HashSet<String>>,
}
impl Problem {
pub fn message(&self) -> String {
let path = self.path.display();
let detail: &str = self.detail.as_str();
return match self.kind {
ProblemKind::StoreUnusable => format!(
"cli-git: the saved copies of replaced files in {path} were not pruned, and the age and size limits did not apply: {detail}. \
Make it a real directory (move the copies there and remove the link) and the next save prunes it again.\n"
),
ProblemKind::EntryUnreadable => format!(
"cli-git: {path} was kept and not counted: it cannot be read ({detail}). Fix its permissions or remove it yourself; \
until then the size limit sees the folder as smaller than it is.\n"
),
ProblemKind::EntryChanged => format!(
"cli-git: {path} was kept: it was replaced or changed while the pass ran ({detail}), so it is not the directory the pass judged.\n"
),
ProblemKind::PartiallyRemoved => format!(
"cli-git: {path} was only partly removed ({detail}). The audit file lists the copies that went and the rest are still there; \
fix the cause and the next pass finishes the removal.\n"
),
ProblemKind::NotRemoved => format!("cli-git: {path} could not be removed and was left whole ({detail}). The next pass tries again.\n"),
ProblemKind::AuditUnwritable => format!(
"cli-git: no saved copy was removed because the audit file {path} cannot be appended to ({detail}); every removal needs its \
line there. Fix the file or its permissions and the next pass removes what the limits allow.\n"
),
ProblemKind::RemovedWithoutAudit => format!(
"cli-git: {path} was removed but its line in the audit file could not be written ({detail}). Its copies were older or \
larger than the limits allow.\n"
),
};
}
}
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub struct PruneOutcome {
pub removed: Vec<Removed>,
pub problems: Vec<Problem>,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum Reason {
Age,
Size,
}
impl Reason {
pub fn word(self) -> &'static str {
return match self {
Reason::Age => "age",
Reason::Size => "size",
};
}
}
pub fn plan(entries: &[Stamped], now_millis: i64, limits: Limits, keep: Option<&str>) -> Vec<(usize, Reason)> {
let age_limit: i128 = i128::from(limits.max_age_days) * i128::from(DAY_MILLISECONDS);
let age_of = |entry: &Stamped| -> i128 { return i128::from(now_millis) - (i128::from(entry.millis) + i128::from(STAMP_RESOLUTION_MILLISECONDS)); };
let reachable = |entry: &Stamped| -> bool { return age_of(entry) >= i128::from(YOUNG_FLOOR_MILLISECONDS) && keep != Some(entry.name.as_str()); };
let mut doomed: Vec<(usize, Reason)> = Vec::new();
for (index, entry) in entries.iter().enumerate() {
if reachable(entry) && age_of(entry) > age_limit {
doomed.push((index, Reason::Age));
}
}
let mut total: u128 = entries.iter().enumerate().filter(|(index, _)| { return !doomed.iter().any(|(gone, _)| { return gone == index; }); }).map(|(_, entry)| { return entry.bytes; }).sum();
let mut oldest_first: Vec<usize> = (0..entries.len()).filter(|index| { return reachable(&entries[*index]) && !doomed.iter().any(|(gone, _)| { return gone == index; }); }).collect();
oldest_first.sort_by(|left, right| { return entries[*left].millis.cmp(&entries[*right].millis).then_with(|| { return entries[*left].name.cmp(&entries[*right].name); }); });
for index in oldest_first {
if total <= u128::from(limits.max_bytes) {
break;
}
total = total.saturating_sub(entries[index].bytes);
doomed.push((index, Reason::Size));
}
return doomed;
}
pub fn list_stamped(common_dir: &Path) -> Listing {
return match Store::open(common_dir, SAVED_DIRECTORY) {
Ok(Some(store)) => list_in(&store, common_dir.join(SAVED_DIRECTORY).as_path()),
Ok(None) => Listing::default(),
Err(failure) => Listing { problems: vec![unusable(common_dir.join(SAVED_DIRECTORY).as_path(), &failure)], ..Listing::default() },
};
}
fn unusable(saved_root: &Path, failure: &OpenFailure) -> Problem {
let detail: String = match failure {
OpenFailure::Link => String::from("a symbolic link, or something that is not a directory, stands at it or at its parent `cli-git`, and a removal never follows one"),
OpenFailure::Refused(error) => format!("it cannot be opened: {error}"),
};
return Problem { path: saved_root.to_path_buf(), kind: ProblemKind::StoreUnusable, detail };
}
fn list_in(store: &Store, saved_root: &Path) -> Listing {
let mut listing: Listing = Listing::default();
let names = match store.names() {
Ok(found) => found,
Err(error) => {
listing.problems.push(Problem { path: saved_root.to_path_buf(), kind: ProblemKind::StoreUnusable, detail: format!("it cannot be read: {error}") });
return listing;
}
};
for name in names {
let text: String = name.to_string_lossy().into_owned();
let Some(millis) = parse_stamp(text.as_str()) else { continue };
let path: PathBuf = saved_root.join(text.as_str());
match survey_entry(store, name.as_os_str()) {
Ok(None) => {}
Ok(Some((identity, tally))) => {
listing.files += tally.names.len();
listing.entries.push(Stamped { name: text.clone(), millis, bytes: tally.bytes, identity });
listing.listed_files.insert(text, tally.names.into_iter().collect());
}
Err(detail) => listing.problems.push(Problem { path, kind: ProblemKind::EntryUnreadable, detail }),
}
}
listing.entries.sort_by(|left, right| { return left.millis.cmp(&right.millis).then_with(|| { return left.name.cmp(&right.name); }); });
return listing;
}
fn survey_entry(store: &Store, name: &OsStr) -> Result<Option<(Identity, Tally)>, String> {
let Some(seen) = store.directory_identity(name).map_err(|error| { return error.to_string(); })? else {
return Ok(None);
};
let Some(directory) = store.open_directory(name).map_err(|error| { return error.to_string(); })? else {
return Ok(None);
};
let opened: Identity = directory.identity().map_err(|error| { return error.to_string(); })?;
if opened != seen {
return Err(String::from("it was replaced while it was being read"));
}
return match directory.survey() {
Ok(tally) => Ok(Some((opened, tally))),
Err(Interrupted { error, .. }) if error.kind() == std::io::ErrorKind::NotFound => Ok(None),
Err(Interrupted { error, .. }) => Err(error.to_string()),
};
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Removed {
pub path: PathBuf,
pub bytes: u128,
pub reason: Reason,
}
pub fn prune_line(removed: &Removed, copies: &[String], limits: Limits, folder_bytes: u128, now_millis: i64) -> String {
return audit_line("pruned", removed, copies, limits, folder_bytes, now_millis);
}
fn audit_line(outcome: &str, removed: &Removed, copies: &[String], limits: Limits, folder_bytes: u128, now_millis: i64) -> String {
let limit: u64 = match removed.reason {
Reason::Age => limits.max_age_days,
Reason::Size => limits.max_bytes,
};
let listed: Vec<String> = copies.iter().take(LISTED_COPIES).cloned().collect();
return format!(
"{{\"schemaVersion\":1,\"time\":{},\"level\":\"info\",\"timeMillis\":{},\"release\":{},\"pid\":{},\"form\":\"PruneDiscarded\",\
\"outcome\":{},\"reason\":{},\"limit\":{limit},\"folderBytes\":{folder_bytes},\"path\":{},\"bytes\":{},\
\"copiesTotal\":{},\"copies\":{}}}\n",
quote(&format_iso_milliseconds(now_millis)),
quote(&now_millis.to_string()),
quote(RELEASE),
std::process::id(),
quote(outcome),
quote(removed.reason.word()),
quote(&removed.path.to_string_lossy()),
removed.bytes,
copies.len(),
string_array(&listed),
);
}
pub fn prune(common_dir: &Path, now_millis: i64, limits: Limits, keep: Option<&str>) -> PruneOutcome {
return prune_with(common_dir, now_millis, limits, keep, &mut |_candidate: &Path| {});
}
pub fn prune_with(common_dir: &Path, now_millis: i64, limits: Limits, keep: Option<&str>, before_removal: &mut dyn FnMut(&Path)) -> PruneOutcome {
let saved_root: PathBuf = common_dir.join(SAVED_DIRECTORY);
let mut outcome: PruneOutcome = PruneOutcome::default();
let store: Store = match Store::open(common_dir, SAVED_DIRECTORY) {
Ok(Some(opened)) => opened,
Ok(None) => return outcome,
Err(failure) => {
outcome.problems.push(unusable(saved_root.as_path(), &failure));
return outcome;
}
};
let listing: Listing = list_in(&store, saved_root.as_path());
outcome.problems.extend(listing.problems);
let doomed: Vec<(usize, Reason)> = plan(listing.entries.as_slice(), now_millis, limits, keep);
if doomed.is_empty() {
return outcome;
}
let audit_path: PathBuf = common_dir.join(AUDIT_FILE);
if let Err(error) = check_appendable(audit_path.as_path()) {
outcome.problems.push(Problem { path: audit_path, kind: ProblemKind::AuditUnwritable, detail: error.to_string() });
return outcome;
}
let folder_bytes: u128 = listing.entries.iter().map(|entry| { return entry.bytes; }).sum();
for (index, reason) in doomed {
let entry: &Stamped = &listing.entries[index];
let path: PathBuf = saved_root.join(entry.name.as_str());
before_removal(path.as_path());
let mut context: Removal = Removal { store: &store, audit_path: audit_path.as_path(), limits, folder_bytes, now_millis, listed_files: &listing.listed_files };
remove_entry(&mut context, entry, path, reason, &mut outcome);
}
return outcome;
}
struct Removal<'pass> {
store: &'pass Store,
audit_path: &'pass Path,
limits: Limits,
folder_bytes: u128,
now_millis: i64,
listed_files: &'pass std::collections::HashMap<String, std::collections::HashSet<String>>,
}
fn write_audit(context: &Removal, outcome_word: &str, removed: &Removed, tally: &Tally, outcome: &mut PruneOutcome) {
let line: String = audit_line(outcome_word, removed, tally.names.as_slice(), context.limits, context.folder_bytes, context.now_millis);
if let Err(error) = append_line(context.audit_path, line.as_bytes()) {
debug(TAG, &format!("the audit record of {} could not be written: {error}", removed.path.display()));
outcome.problems.push(Problem { path: removed.path.clone(), kind: ProblemKind::RemovedWithoutAudit, detail: error.to_string() });
}
}
fn remove_entry(context: &mut Removal, entry: &Stamped, path: PathBuf, reason: Reason, outcome: &mut PruneOutcome) {
let name: &OsStr = OsStr::new(entry.name.as_str());
let kept = |kind: ProblemKind, detail: String| -> Problem { return Problem { path: path.clone(), kind, detail }; };
let directory = match context.store.open_directory(name) {
Ok(Some(opened)) => opened,
Ok(None) => return,
Err(error) if error.kind() == std::io::ErrorKind::NotADirectory => {
outcome.problems.push(kept(ProblemKind::EntryChanged, String::from("the name is now a link or a file")));
return;
}
Err(error) => {
outcome.problems.push(kept(ProblemKind::NotRemoved, error.to_string()));
return;
}
};
match directory.identity() {
Ok(found) if found == entry.identity => {}
Ok(found) if found.device == entry.identity.device && found.inode == entry.identity.inode => match directory.survey() {
Ok(now) if now.names.iter().all(|file| { return context.listed_files.get(entry.name.as_str()).is_some_and(|listed| { return listed.contains(file); }); }) => {}
Ok(_) => {
outcome.problems.push(kept(ProblemKind::EntryChanged, String::from("a file was added to it after the pass listed it")));
return;
}
Err(Interrupted { error, .. }) if error.kind() == std::io::ErrorKind::NotFound => return,
Err(Interrupted { error, .. }) => {
outcome.problems.push(kept(ProblemKind::NotRemoved, error.to_string()));
return;
}
},
Ok(_) => {
outcome.problems.push(kept(ProblemKind::EntryChanged, String::from("the name now stands for another directory")));
return;
}
Err(error) => {
outcome.problems.push(kept(ProblemKind::NotRemoved, error.to_string()));
return;
}
}
let mut gone: Removed = Removed { path: path.clone(), bytes: 0, reason };
let tally: Tally = match directory.empty() {
Ok(emptied) => emptied,
Err(Interrupted { tally, error }) => {
if tally.names.is_empty() {
outcome.problems.push(kept(ProblemKind::NotRemoved, error.to_string()));
} else {
gone.bytes = tally.bytes;
write_audit(context, "partial", &gone, &tally, outcome);
outcome.problems.push(kept(ProblemKind::PartiallyRemoved, error.to_string()));
}
return;
}
};
gone.bytes = tally.bytes;
match context.store.remove_empty_directory(name, entry.identity) {
Ok(true) => {}
Ok(false) => match context.store.directory_identity(name) {
Ok(None) if tally.names.is_empty() => return,
Ok(None) => {}
_ => outcome.problems.push(kept(ProblemKind::EntryChanged, String::from("the name now stands for another directory, which was kept; the copies judged are gone"))),
},
Err(error) => {
write_audit(context, "partial", &gone, &tally, outcome);
outcome.problems.push(kept(ProblemKind::PartiallyRemoved, format!("its copies are gone but the empty directory could not be removed: {error}")));
return;
}
}
write_audit(context, "pruned", &gone, &tally, outcome);
outcome.removed.push(gone);
}
pub fn directory_name_of(saved_root: &Path, copy: &Path) -> Option<String> {
return copy.strip_prefix(saved_root).ok()?.components().next().map(|first| { return first.as_os_str().to_string_lossy().into_owned(); });
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct Report {
pub bytes: u128,
pub copies: usize,
pub saves: usize,
pub oldest: String,
pub problems: Vec<Problem>,
}
pub fn report(common_dir: &Path) -> Option<Report> {
let listing: Listing = list_stamped(common_dir);
if listing.entries.is_empty() && listing.problems.is_empty() {
return None;
}
return Some(Report {
bytes: listing.entries.iter().map(|entry| { return entry.bytes; }).sum(),
copies: listing.files,
saves: listing.entries.len(),
oldest: listing.entries.first().map_or_else(String::new, |entry| { return entry.name.clone(); }),
problems: listing.problems,
});
}
pub fn doctor_section(common_dir: &Path, limits: Limits) -> String {
let Some(found) = report(common_dir) else {
return String::new();
};
let mut text: String = String::new();
if found.saves > 0 {
text = format!(
"git cli-git doctor: discarded copies: {} holds {} bytes in {} copies from {} saves; the oldest save is {}. Copies older than {} days \
are removed, and the oldest are removed past {} bytes, never one younger than a day (`discardedCopies.maxAgeDays` and \
`discardedCopies.maxSize` in cli-git.config.jsonc change the limits).\n",
common_dir.join(SAVED_DIRECTORY).display(),
found.bytes,
found.copies,
found.saves,
found.oldest,
limits.max_age_days,
limits.max_bytes,
);
}
for problem in &found.problems {
text.push_str(problem.message().replacen("cli-git: ", "git cli-git doctor: discarded copies: ", 1).as_str());
}
return text;
}
#[cfg(test)]
#[path = "discarded_prune_tests.rs"]
mod tests;
#[cfg(test)]
#[path = "discarded_prune_guard_tests.rs"]
mod guard_tests;