use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::SystemTime;
use smallvec::SmallVec;
use time::OffsetDateTime;
use crate::config::ConfigSet;
use crate::environment::RepositoryOptions;
use crate::error::{BadNumericSource, ConfigError, Error, Result};
use crate::git_date::approx::approxidate_careful_at;
use crate::index_pack::{install_pack_path, IngestPackOptions};
use crate::objects::{
oid_hash_map, oid_hash_set, CommitRef, ObjectId, ObjectKind, OidHashMap, OidHashSet,
TreeRefIter,
};
use crate::odb::Odb;
use crate::offload::LOP_GIT_DIR_KEY;
use crate::pack::PackedType;
use crate::pack_index::PackIndex;
use crate::pack_objects::PackBuildOptions;
use crate::repo::Repository;
use crate::state::InProgressOperation;
use super::{GcLock, GcLockAttempt};
pub const DROP_LARGE_BLOBS_TASK: &str = "drop-large-blobs";
pub const DEFAULT_DROP_MIN_SIZE: u64 = 1 << 20;
pub const DEFAULT_DROP_EXPIRE: &str = "2.weeks.ago";
const ENABLED_KEY: &str = "maintenance.drop-large-blobs.enabled";
const MIN_SIZE_KEY: &str = "maintenance.drop-large-blobs.minsize";
const EXPIRE_KEY: &str = "maintenance.drop-large-blobs.expire";
const OPERATION: &str = "drop-large-blobs";
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct DropPromisedBlobsOptions {
pub now: OffsetDateTime,
pub min_size: u64,
pub keep_newer_than: Option<OffsetDateTime>,
pub dry_run: bool,
pub threads: Option<usize>,
}
impl DropPromisedBlobsOptions {
#[must_use]
pub fn new(
now: OffsetDateTime,
min_size: u64,
keep_newer_than: Option<OffsetDateTime>,
) -> Self {
Self {
now,
min_size,
keep_newer_than,
dry_run: false,
threads: None,
}
}
pub fn from_config(config: &ConfigSet, now: OffsetDateTime) -> Result<Option<Self>> {
Self::from_config_with_overrides(config, now, None, None)
}
pub fn from_config_with_overrides(
config: &ConfigSet,
now: OffsetDateTime,
min_size: Option<u64>,
expiry: Option<DropExpiry>,
) -> Result<Option<Self>> {
let expiry = match expiry {
Some(expiry) => expiry,
None => configured_expiry(config, now)?,
};
let keep_newer_than = match expiry {
DropExpiry::Never => return Ok(None),
DropExpiry::Immediately => None,
DropExpiry::KeepNewerThan(cutoff) => Some(cutoff),
};
let min_size = match min_size {
Some(min_size) => min_size,
None => configured_min_size(config)?,
};
Ok(Some(Self::new(now, min_size, keep_newer_than)))
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DropExpiry {
Never,
Immediately,
KeepNewerThan(OffsetDateTime),
}
impl DropExpiry {
#[must_use]
pub fn parse(spec: &str, now: OffsetDateTime) -> Option<Self> {
let spec = spec.trim();
if spec.eq_ignore_ascii_case("never") || spec.eq_ignore_ascii_case("false") {
return Some(Self::Never);
}
if spec.eq_ignore_ascii_case("now") || spec.eq_ignore_ascii_case("all") {
return Some(Self::Immediately);
}
if spec.is_empty() {
return None;
}
let mut err = 0;
let secs = approxidate_careful_at(spec, now.unix_timestamp(), Some(&mut err));
if err != 0 {
return None;
}
let cutoff = i64::try_from(secs)
.ok()
.and_then(|secs| OffsetDateTime::from_unix_timestamp(secs).ok())?;
Some(Self::KeepNewerThan(cutoff))
}
}
pub fn drop_large_blobs_enabled(config: &ConfigSet) -> Result<bool> {
match config.get_bool(ENABLED_KEY) {
None => Ok(false),
Some(Ok(value)) => Ok(value),
Some(Err(err)) => Err(Error::Config(err.into())),
}
}
fn configured_expiry(config: &ConfigSet, now: OffsetDateTime) -> Result<DropExpiry> {
let spec = config
.get(EXPIRE_KEY)
.map(|s| s.trim().to_owned())
.filter(|s| !s.is_empty())
.unwrap_or_else(|| DEFAULT_DROP_EXPIRE.to_owned());
DropExpiry::parse(&spec, now).ok_or_else(|| {
Error::Config(ConfigError::InvalidDate {
key: EXPIRE_KEY.to_owned(),
value: spec,
})
})
}
fn configured_min_size(config: &ConfigSet) -> Result<u64> {
if let Some(entry) = config.get_last_entry(MIN_SIZE_KEY) {
let raw = entry.value.as_deref().unwrap_or_default();
let bad = |reason| {
Error::Config(ConfigError::BadNumericValue {
key: MIN_SIZE_KEY.to_owned(),
value: raw.to_owned(),
reason,
})
};
let value = crate::config::parse_git_config_int_strict(raw).map_err(|e| {
bad(match e {
crate::config::GitConfigIntStrictError::InvalidUnit => {
BadNumericSource::InvalidUnit
}
crate::config::GitConfigIntStrictError::OutOfRange => BadNumericSource::OutOfRange,
})
})?;
return u64::try_from(value).map_err(|_| bad(BadNumericSource::OutOfRange));
}
Ok(
crate::promisor_remote::configured_promisor_remotes(config, &[])
.iter()
.filter_map(|remote| {
crate::partial_clone::stored_remote_filter(config, &remote.name)
.and_then(|filter| filter.blob_limit_bytes())
})
.min()
.unwrap_or(DEFAULT_DROP_MIN_SIZE),
)
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct DroppedBlob {
pub oid: ObjectId,
pub size: u64,
pub pack: PathBuf,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
#[non_exhaustive]
pub struct KeptBlobCounts {
pub kept_pack: usize,
pub outside_promisor_packs: usize,
pub not_in_promisor: usize,
pub unverified: usize,
pub checked_out: usize,
pub recently_checked_out: usize,
pub recently_fetched: usize,
pub not_promised_elsewhere: usize,
}
impl KeptBlobCounts {
#[must_use]
pub fn total(&self) -> usize {
self.kept_pack
+ self.outside_promisor_packs
+ self.checked_out
+ self.recently_checked_out
+ self.recently_fetched
+ self.not_promised_elsewhere
+ self.not_in_promisor
+ self.unverified
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum DropPromisedBlobsSkip {
Disabled,
NoPromisorRemote,
OperationInProgress(InProgressOperation),
PreciousObjects,
NoCurrentTime,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
#[non_exhaustive]
pub struct DropPromisedBlobsReport {
pub min_size: u64,
pub keep_newer_than: Option<OffsetDateTime>,
pub candidates: usize,
pub dropped: Vec<DroppedBlob>,
pub kept: KeptBlobCounts,
pub bytes_reclaimed: u64,
pub packs_rewritten: usize,
pub packs_removed: usize,
pub skipped: Option<DropPromisedBlobsSkip>,
}
impl DropPromisedBlobsReport {
fn skipped_for(reason: DropPromisedBlobsSkip) -> Self {
Self {
skipped: Some(reason),
..Self::default()
}
}
}
pub fn drop_promised_blobs(
repo: &Repository,
opts: &DropPromisedBlobsOptions,
) -> Result<DropPromisedBlobsReport> {
repo.odb().require_files_primary(OPERATION)?;
let config = repo.config()?;
if let Some(refusal) = gate(repo, &config)? {
return Err(refusal.into());
}
let _lock = if opts.dry_run {
None
} else {
match GcLock::try_acquire(repo, opts.now.unix_timestamp())? {
GcLockAttempt::Acquired(lock) => Some(lock),
GcLockAttempt::Held(holder) => return Err(Error::MaintenanceRunning { holder }),
}
};
run(repo, &config, opts)
}
impl Repository {
pub fn drop_promised_blobs(
&self,
opts: &DropPromisedBlobsOptions,
) -> Result<DropPromisedBlobsReport> {
drop_promised_blobs(self, opts)
}
}
pub(crate) enum GcDropPlan {
Skip(DropPromisedBlobsSkip),
Run(DropPromisedBlobsOptions),
}
pub(crate) fn plan_drop_for_gc(config: &ConfigSet, now_unix: i64) -> Result<GcDropPlan> {
if !drop_large_blobs_enabled(config)? {
return Ok(GcDropPlan::Skip(DropPromisedBlobsSkip::Disabled));
}
let now = (now_unix != 0)
.then(|| OffsetDateTime::from_unix_timestamp(now_unix).ok())
.flatten();
let Some(now) = now else {
configured_min_size(config)?;
return Ok(GcDropPlan::Skip(DropPromisedBlobsSkip::NoCurrentTime));
};
Ok(match DropPromisedBlobsOptions::from_config(config, now)? {
Some(opts) => GcDropPlan::Run(opts),
None => GcDropPlan::Skip(DropPromisedBlobsSkip::Disabled),
})
}
pub(crate) fn run_drop_for_gc(
repo: &Repository,
config: &ConfigSet,
plan: GcDropPlan,
dry_run: bool,
threads: Option<usize>,
) -> Result<DropPromisedBlobsReport> {
let mut opts = match plan {
GcDropPlan::Skip(reason) => return Ok(DropPromisedBlobsReport::skipped_for(reason)),
GcDropPlan::Run(opts) => opts,
};
if let Some(refusal) = gate(repo, config)? {
return Ok(DropPromisedBlobsReport::skipped_for(refusal.into()));
}
opts.dry_run = dry_run;
opts.threads = threads;
run(repo, config, &opts)
}
enum Refusal {
PreciousObjects,
NoPromisorRemote,
InProgress(InProgressOperation),
}
impl From<Refusal> for DropPromisedBlobsSkip {
fn from(refusal: Refusal) -> Self {
match refusal {
Refusal::PreciousObjects => Self::PreciousObjects,
Refusal::NoPromisorRemote => Self::NoPromisorRemote,
Refusal::InProgress(op) => Self::OperationInProgress(op),
}
}
}
impl From<Refusal> for Error {
fn from(refusal: Refusal) -> Self {
match refusal {
Refusal::PreciousObjects => Error::PreciousObjectsRepository {
operation: OPERATION,
},
Refusal::NoPromisorRemote => Error::NoPromisorRemote {
operation: OPERATION,
},
Refusal::InProgress(op) => Error::OperationInProgress(op),
}
}
}
fn gate(repo: &Repository, config: &ConfigSet) -> Result<Option<Refusal>> {
if config
.get_bool("extensions.preciousobjects")
.and_then(|v| v.ok())
.unwrap_or(false)
{
return Ok(Some(Refusal::PreciousObjects));
}
if !crate::promisor::repo_treats_promisor_packs(repo.git_dir(), config) {
return Ok(Some(Refusal::NoPromisorRemote));
}
for wt in crate::worktree::list_worktrees(repo)? {
if wt.is_bare {
continue;
}
if let Some(op) = crate::state::detect_in_progress(&wt.admin_dir).first() {
return Ok(Some(Refusal::InProgress(*op)));
}
}
Ok(None)
}
struct PromisorPack {
idx: Arc<PackIndex>,
mtime: Option<SystemTime>,
kept: bool,
referrers: Vec<u32>,
commits: Vec<u32>,
}
struct Candidate {
size: u64,
packs: SmallVec<[u32; 1]>,
entry_bytes: u64,
}
fn run(
repo: &Repository,
config: &ConfigSet,
opts: &DropPromisedBlobsOptions,
) -> Result<DropPromisedBlobsReport> {
let odb = repo.odb();
let pack_dir = odb.objects_dir().join("pack");
odb.reprepare_local_packs();
let (promisor, others): (Vec<_>, Vec<_>) = odb
.local_pack_indexes()?
.into_iter()
.partition(|idx| idx.is_promisor);
let mut packs: Vec<PromisorPack> = promisor
.into_iter()
.map(|idx| PromisorPack {
mtime: std::fs::metadata(&idx.pack_path)
.and_then(|m| m.modified())
.ok(),
kept: super::pack_has_keep_marker(&pack_dir, &idx.pack_path),
idx,
referrers: Vec::new(),
commits: Vec::new(),
})
.collect();
packs.sort_by(|a, b| a.idx.pack_path.cmp(&b.idx.pack_path));
let mut report = DropPromisedBlobsReport {
min_size: opts.min_size,
keep_newer_than: opts.keep_newer_than,
..DropPromisedBlobsReport::default()
};
let mut candidates: OidHashMap<Candidate> = oid_hash_map();
for (pack_no, pack) in packs.iter_mut().enumerate() {
scan_pack(odb, pack, pack_no as u32, opts.min_size, &mut candidates)?;
}
report.candidates = candidates.len();
if candidates.is_empty() {
return Ok(report);
}
let cutoff = opts.keep_newer_than.map(SystemTime::from);
let mut stored_only_here: OidHashSet = oid_hash_set();
for (oid, cand) in &candidates {
let holding = || cand.packs.iter().map(|&p| &packs[p as usize]);
if holding().any(|p| p.kept) {
report.kept.kept_pack += 1;
} else if odb.local_loose_file_size(oid).is_some()
|| others.iter().any(|idx| idx.find_position(oid).is_some())
{
report.kept.outside_promisor_packs += 1;
} else {
stored_only_here.insert(*oid);
}
}
let mut undecided =
PromisorSupply::load(repo, config).suppliable(stored_only_here, &mut report.kept)?;
let mut walk = ProtectionWalk::new(odb, &undecided);
walk.mark_checkouts(repo)?;
let checked_out = std::mem::take(&mut walk.hits);
if let Some(cutoff) = opts.keep_newer_than {
walk.mark_recent_heads(repo, cutoff)?;
}
let recently_checked_out = std::mem::take(&mut walk.hits);
drop(walk);
report.kept.checked_out = checked_out.len();
report.kept.recently_checked_out = recently_checked_out.len();
undecided.retain(|oid| !checked_out.contains(oid) && !recently_checked_out.contains(oid));
undecided.retain(|oid| {
let fresh = cutoff.is_some_and(|cutoff| {
candidates[oid]
.packs
.iter()
.filter_map(|&p| packs[p as usize].mtime)
.any(|mtime| mtime >= cutoff)
});
if fresh {
report.kept.recently_fetched += 1;
}
!fresh
});
let promised = promised_by_promisor_packs(odb, &packs, &undecided)?;
report.kept.not_promised_elsewhere = undecided.len() - promised.len();
let drop_set = promised;
let mut dropped: Vec<DroppedBlob> = drop_set
.iter()
.map(|oid| {
let cand = &candidates[oid];
DroppedBlob {
oid: *oid,
size: cand.size,
pack: cand
.packs
.first()
.map(|&p| packs[p as usize].idx.pack_path.clone())
.unwrap_or_default(),
}
})
.collect();
dropped.sort_by_key(|d| d.oid);
report.dropped = dropped;
if drop_set.is_empty() {
return Ok(report);
}
if opts.dry_run {
report.bytes_reclaimed = drop_set.iter().map(|oid| candidates[oid].entry_bytes).sum();
return Ok(report);
}
let mut affected: Vec<u32> = drop_set
.iter()
.flat_map(|oid| candidates[oid].packs.iter().copied())
.collect();
affected.sort_unstable();
affected.dedup();
rewrite_packs(
repo,
config,
opts,
&packs,
&affected,
&drop_set,
&mut report,
)?;
Ok(report)
}
struct PromisorSupply {
local: Vec<LocalPromisor>,
unreadable: bool,
any_local: bool,
network: bool,
offloaded: bool,
}
struct LocalPromisor {
repo: Repository,
lop: bool,
}
impl PromisorSupply {
fn load(repo: &Repository, config: &ConfigSet) -> Self {
let recorded: Vec<PathBuf> = config
.get_all(LOP_GIT_DIR_KEY)
.into_iter()
.filter(|v| !v.trim().is_empty())
.map(PathBuf::from)
.collect();
let mut dirs: Vec<(PathBuf, bool)> = recorded.iter().map(|d| (d.clone(), true)).collect();
let mut network = false;
for remote in crate::promisor_remote::configured_promisor_remotes(config, &[]) {
let Some(url) = remote.url else { continue };
if crate::transport_path::is_local_path_remote_url(&url) {
let dir = crate::transport_path::resolve_local_remote_git_dir(
&url,
repo.git_dir(),
repo.work_tree(),
);
dirs.push((dir, false));
} else {
network = true;
}
}
let mut supply = Self {
local: Vec::new(),
unreadable: false,
any_local: !dirs.is_empty(),
network,
offloaded: !recorded.is_empty(),
};
for (dir, lop) in dirs {
let opened = Repository::open(
&dir,
&RepositoryOptions::with_environment(repo.environment().clone()),
);
let other = match opened {
Ok(other) if other.odb().hash_algo() == repo.odb().hash_algo() => other,
_ => {
supply.unreadable = true;
continue;
}
};
if crate::offload::same_object_store(&other, repo) {
continue;
}
if supply
.local
.iter()
.any(|p| crate::offload::same_object_store(&other, &p.repo))
{
continue;
}
supply.local.push(LocalPromisor { repo: other, lop });
}
supply
}
fn suppliable(&self, promised: OidHashSet, kept: &mut KeptBlobCounts) -> Result<OidHashSet> {
if !self.any_local && !self.offloaded {
return Ok(promised);
}
if promised.is_empty() {
return Ok(promised);
}
let mut stores = Vec::with_capacity(self.local.len());
for promisor in &self.local {
let odb = promisor.repo.odb();
let indexes = if promisor.lop {
crate::offload::lop_keep_packs(odb)?
} else {
odb.local_pack_indexes()?
};
stores.push((odb, indexes, !promisor.lop));
}
let held = |oid: &ObjectId| {
stores.iter().any(|(odb, indexes, loose_counts)| {
indexes.iter().any(|idx| idx.find_position(oid).is_some())
|| (*loose_counts && odb.local_loose_file_size(oid).is_some())
})
};
let trust_network = self.network && !self.offloaded && !self.unreadable;
let mut out = oid_hash_set();
for oid in promised {
if held(&oid) || trust_network {
out.insert(oid);
} else if self.network || self.unreadable {
kept.unverified += 1;
} else {
kept.not_in_promisor += 1;
}
}
Ok(out)
}
}
fn scan_pack(
odb: &Odb,
pack: &mut PromisorPack,
pack_no: u32,
min_size: u64,
candidates: &mut OidHashMap<Candidate>,
) -> Result<()> {
let idx = Arc::clone(&pack.idx);
let data = crate::pack::read_pack_bytes_cached(&idx.pack_path)?;
let bytes: &[u8] = &data;
let pack_end = (bytes.len() as u64).saturating_sub(idx.hash_bytes() as u64);
let mut kinds = EntryKinds::new(odb, &idx, bytes);
for pos in 0..idx.len() {
match kinds.kind(pos)? {
Some(ObjectKind::Blob) => {}
Some(ObjectKind::Tree | ObjectKind::Tag) => {
pack.referrers.push(pos as u32);
continue;
}
Some(ObjectKind::Commit) => {
pack.commits.push(pos as u32);
continue;
}
None => continue,
}
let offset = idx.offset_at(pos)?;
let size = entry_result_size(bytes, offset, idx.hash_bytes())?;
if size < min_size {
continue;
}
let oid = ObjectId::from_bytes(idx.oid_at(pos))?;
let entry_bytes = idx
.next_pack_offset_after(offset, pack_end)?
.saturating_sub(offset);
let cand = candidates.entry(oid).or_insert_with(|| Candidate {
size,
packs: SmallVec::new(),
entry_bytes: 0,
});
cand.packs.push(pack_no);
cand.entry_bytes += entry_bytes;
}
Ok(())
}
fn entry_result_size(bytes: &[u8], offset: u64, hash_bytes: usize) -> Result<u64> {
let mut pos = usize::try_from(offset)
.map_err(|_| Error::CorruptObject("pack offset out of range".to_owned()))?;
let (packed, size) = crate::pack::parse_pack_object_header(bytes, &mut pos)?;
match packed {
PackedType::OfsDelta => {
crate::pack::parse_ofs_delta_base(bytes, &mut pos, offset)?;
}
PackedType::RefDelta => pos += hash_bytes,
_ => return Ok(size),
}
crate::pack::read_delta_result_size_from_pack_zlib(bytes, &mut pos)
}
struct EntryKinds<'a> {
odb: &'a Odb,
idx: &'a PackIndex,
bytes: &'a [u8],
kinds: Vec<u8>,
}
const MAX_CHAIN: usize = 10_000;
impl<'a> EntryKinds<'a> {
fn new(odb: &'a Odb, idx: &'a PackIndex, bytes: &'a [u8]) -> Self {
Self {
odb,
idx,
bytes,
kinds: vec![0; idx.len()],
}
}
fn encode(kind: Option<ObjectKind>) -> u8 {
match kind {
Some(ObjectKind::Commit) => 1,
Some(ObjectKind::Tree) => 2,
Some(ObjectKind::Blob) => 3,
Some(ObjectKind::Tag) => 4,
None => 5,
}
}
fn decode(code: u8) -> Option<ObjectKind> {
match code {
1 => Some(ObjectKind::Commit),
2 => Some(ObjectKind::Tree),
3 => Some(ObjectKind::Blob),
4 => Some(ObjectKind::Tag),
_ => None,
}
}
fn kind(&mut self, pos: usize) -> Result<Option<ObjectKind>> {
let corrupt = |what: &str| {
Error::CorruptObject(format!("{what} in {}", self.idx.pack_path.display()))
};
let mut chain: Vec<usize> = Vec::new();
let mut cur = pos;
let kind = loop {
if self.kinds[cur] != 0 {
break Self::decode(self.kinds[cur]);
}
if chain.len() > MAX_CHAIN {
return Err(corrupt("delta chain too deep"));
}
chain.push(cur);
let offset = self.idx.offset_at(cur)?;
let mut at =
usize::try_from(offset).map_err(|_| corrupt("pack offset out of range"))?;
let (packed, _) = crate::pack::parse_pack_object_header(self.bytes, &mut at)?;
match packed {
PackedType::Commit => break Some(ObjectKind::Commit),
PackedType::Tree => break Some(ObjectKind::Tree),
PackedType::Blob => break Some(ObjectKind::Blob),
PackedType::Tag => break Some(ObjectKind::Tag),
PackedType::OfsDelta => {
let base = crate::pack::parse_ofs_delta_base(self.bytes, &mut at, offset)?;
cur = self
.idx
.find_position_by_offset_sorted(base)?
.ok_or_else(|| corrupt("delta base offset not in index"))?;
}
PackedType::RefDelta => {
let hash_bytes = self.idx.hash_bytes();
let raw = self
.bytes
.get(at..at + hash_bytes)
.ok_or_else(|| corrupt("truncated delta base id"))?;
let base = ObjectId::from_bytes(raw)?;
match self.idx.find_position(&base) {
Some(base_pos) => cur = base_pos,
None => match self.odb.read_info_without_fetch(&base) {
Ok(info) => break Some(info.kind),
Err(Error::ObjectNotFound(_)) => break None,
Err(err) => return Err(err),
},
}
}
}
};
let code = Self::encode(kind);
for row in chain {
self.kinds[row] = code;
}
Ok(kind)
}
}
fn promised_by_promisor_packs(
odb: &Odb,
packs: &[PromisorPack],
wanted: &OidHashSet,
) -> Result<OidHashSet> {
let mut found = oid_hash_set();
if wanted.is_empty() {
return Ok(found);
}
let oid_len = odb.hash_algo().len();
let mut read: Vec<Vec<bool>> = packs.iter().map(|_| Vec::new()).collect();
let mut referrer = |pack_no: usize,
row: usize,
queue: Option<&mut std::collections::VecDeque<ObjectId>>,
found: &mut OidHashSet|
-> Result<()> {
let seen = &mut read[pack_no];
if seen.is_empty() {
*seen = vec![false; packs[pack_no].idx.len()];
}
if std::mem::replace(&mut seen[row], true) {
return Ok(());
}
let idx = &packs[pack_no].idx;
let obj = crate::pack::read_object_at(idx, idx.offset_at(row)?)?;
match obj.kind {
ObjectKind::Tree => {
let mut queue = queue;
for entry in TreeRefIter::with_oid_len(&obj.data, oid_len) {
let entry = entry?;
match entry.mode & 0o170000 {
0o040000 => {
if let Some(q) = queue.as_deref_mut() {
q.push_back(entry.oid);
}
}
0o160000 => {}
_ if wanted.contains(&entry.oid) => {
found.insert(entry.oid);
}
_ => {}
}
}
}
ObjectKind::Tag => {
if let Some(target) = crate::objects::tag_object_line_oid(&obj.data) {
if wanted.contains(&target) {
found.insert(target);
}
}
}
ObjectKind::Blob | ObjectKind::Commit => {}
}
Ok(())
};
let locate = |oid: &ObjectId| {
packs
.iter()
.enumerate()
.find_map(|(n, p)| p.idx.find_position(oid).map(|row| (n, row)))
};
let mut queue: std::collections::VecDeque<ObjectId> = std::collections::VecDeque::new();
for pack in packs {
for &row in &pack.commits {
let obj = crate::pack::read_object_at(&pack.idx, pack.idx.offset_at(row as usize)?)?;
if obj.kind == ObjectKind::Commit {
queue.push_back(CommitRef::parse(&obj.data)?.tree());
}
}
}
let mut trees_seen = oid_hash_set();
while let Some(tree) = queue.pop_front() {
if found.len() == wanted.len() {
return Ok(found);
}
if !trees_seen.insert(tree) {
continue;
}
if let Some((pack_no, row)) = locate(&tree) {
referrer(pack_no, row, Some(&mut queue), &mut found)?;
}
}
drop(trees_seen);
for (pack_no, pack) in packs.iter().enumerate() {
for &row in &pack.referrers {
if found.len() == wanted.len() {
return Ok(found);
}
referrer(pack_no, row as usize, None, &mut found)?;
}
}
Ok(found)
}
struct ProtectionWalk<'a> {
odb: &'a Odb,
wanted: &'a OidHashSet,
trees_seen: OidHashSet,
hits: OidHashSet,
}
impl<'a> ProtectionWalk<'a> {
fn new(odb: &'a Odb, wanted: &'a OidHashSet) -> Self {
Self {
odb,
wanted,
trees_seen: oid_hash_set(),
hits: oid_hash_set(),
}
}
fn mark(&mut self, oid: ObjectId) {
if self.wanted.contains(&oid) {
self.hits.insert(oid);
}
}
fn walk_tree(&mut self, root: ObjectId) -> Result<()> {
let oid_len = self.odb.hash_algo().len();
let mut stack = vec![root];
while let Some(tree) = stack.pop() {
if !self.trees_seen.insert(tree) {
continue;
}
let Some(obj) = self.odb.try_read_once(&tree)? else {
continue;
};
if obj.kind != ObjectKind::Tree {
continue;
}
for entry in TreeRefIter::with_oid_len(&obj.data, oid_len) {
let entry = entry?;
match entry.mode & 0o170000 {
0o040000 => stack.push(entry.oid),
0o160000 => {}
_ => self.mark(entry.oid),
}
}
}
Ok(())
}
fn walk_commit(&mut self, commit: ObjectId) -> Result<()> {
if commit.is_zero() {
return Ok(());
}
let Some(obj) = self.odb.try_read_once(&commit)? else {
return Ok(());
};
if obj.kind != ObjectKind::Commit {
return Ok(());
}
let tree = CommitRef::parse(&obj.data)?.tree();
self.walk_tree(tree)
}
fn mark_checkouts(&mut self, repo: &Repository) -> Result<()> {
use crate::state::HeadState;
for wt in crate::worktree::list_worktrees(repo)? {
if wt.is_bare {
continue;
}
self.mark_index(repo, &wt.admin_dir.join("index"))?;
if let HeadState::Detached { oid } | HeadState::Branch { oid: Some(oid), .. } = wt.head
{
self.walk_commit(oid)?;
}
}
if repo.environment().git_index_file.is_some() {
self.mark_index(repo, &repo.index_path_for_env()?)?;
}
Ok(())
}
fn mark_index(&mut self, repo: &Repository, path: &Path) -> Result<()> {
let index = repo.load_index_at_preserving_sparse(
path,
crate::index::IndexParseContext::from_policy(repo.thread_policy()),
)?;
for entry in index.entries() {
match entry.mode & 0o170000 {
0o040000 => self.walk_tree(entry.oid)?,
0o160000 => {}
_ => self.mark(entry.oid),
}
}
Ok(())
}
fn mark_recent_heads(&mut self, repo: &Repository, cutoff: OffsetDateTime) -> Result<()> {
for wt in crate::worktree::list_worktrees(repo)? {
if wt.is_bare {
continue;
}
for entry in crate::reflog::read_reflog(&wt.admin_dir, "HEAD")? {
if entry.signature().is_some_and(|sig| sig.time >= cutoff) {
self.walk_commit(entry.old_oid)?;
self.walk_commit(entry.new_oid)?;
}
}
}
Ok(())
}
}
#[allow(clippy::too_many_arguments)]
fn rewrite_packs(
repo: &Repository,
config: &ConfigSet,
opts: &DropPromisedBlobsOptions,
packs: &[PromisorPack],
affected: &[u32],
drop_set: &OidHashSet,
report: &mut DropPromisedBlobsReport,
) -> Result<()> {
let odb = repo.odb();
let pack_dir = odb.objects_dir().join("pack");
let had_midx =
pack_dir.join("multi-pack-index").is_file() || pack_dir.join("multi-pack-index.d").is_dir();
let env = repo.environment().clone();
let pack_opts = PackBuildOptions::for_local_copy(Some(config));
let mut bytes_before = 0u64;
let mut bytes_after = 0u64;
let mut stems: std::collections::HashSet<std::ffi::OsString> = std::collections::HashSet::new();
for &p in affected {
let pack = &packs[p as usize];
let old = &pack.idx.pack_path;
bytes_before += pack_files_size(old);
let mut keep: Vec<(u64, ObjectId)> = Vec::new();
for entry in pack.idx.iter() {
let oid = ObjectId::from_bytes(entry.oid())?;
if !drop_set.contains(&oid) {
keep.push((entry.offset()?, oid));
}
}
let old_stem = old.file_stem().map(|stem| stem.to_os_string());
if keep.is_empty() {
stems.extend(old_stem);
report.packs_removed += 1;
continue;
}
keep.sort_unstable();
let objects: Vec<ObjectId> = keep.into_iter().map(|(_, oid)| oid).collect();
let built = super::write_staged_repack_pack(
odb,
&pack_dir,
&objects,
&HashMap::new(),
&pack_opts,
opts.threads,
Some(config),
)?;
let Some(staging) = built.staging_path.as_ref() else {
continue;
};
let trusted_rows = (!built.index_rows.is_empty()).then_some(built.index_rows);
let ingested = install_pack_path(
staging,
odb,
&IngestPackOptions {
threads: opts.threads,
merged_config: Some(config.clone()),
environment: env.clone(),
skip_post_index_verify: trusted_rows.is_some(),
trusted_index_rows: trusted_rows,
mark_promisor_pack: true,
..IngestPackOptions::default()
},
)?;
let Some(new_pack) = ingested.pack_path else {
continue;
};
if let Ok(lines) = std::fs::read(old.with_extension("promisor")) {
std::fs::write(new_pack.with_extension("promisor"), lines)?;
}
if config.pack_write_reverse_index_default(&env) {
crate::pack_rev::write_pack_rev_sidecar_for_index_path(
&new_pack.with_extension("idx"),
)?;
}
if let Some(mtime) = pack.mtime {
filetime::set_file_mtime(&new_pack, filetime::FileTime::from_system_time(mtime))?;
}
crate::index_pack::sync_installed_pack(&new_pack)?;
bytes_after += pack_files_size(&new_pack);
if new_pack.file_stem() != old.file_stem() {
stems.extend(old_stem);
}
report.packs_rewritten += 1;
}
if had_midx {
crate::midx::clear_pack_midx_state(&pack_dir)?;
}
odb.invalidate_packs();
super::remove_pack_files_with_stems(&pack_dir, &stems)?;
crate::pack::clear_pack_cache();
odb.invalidate_packs();
odb.reprepare_local_packs();
let packs_left = odb.local_pack_indexes()?.len();
if had_midx && packs_left > 0 {
super::write_midx(repo, &super::MidxOptions::default())?;
}
report.bytes_reclaimed = bytes_before.saturating_sub(bytes_after);
Ok(())
}
fn pack_files_size(pack: &Path) -> u64 {
["pack", "idx", "rev"]
.iter()
.filter_map(|ext| std::fs::metadata(pack.with_extension(ext)).ok())
.map(|m| m.len())
.sum()
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used)]
mod tests {
use super::*;
use crate::config::{ConfigFile, ConfigScope};
fn config(text: &str) -> ConfigSet {
let file = ConfigFile::parse(Path::new("test.config"), text, ConfigScope::Local).unwrap();
let mut set = ConfigSet::new();
set.merge(&file);
set
}
fn now() -> OffsetDateTime {
OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap()
}
#[test]
fn defaults_are_one_mebibyte_and_two_weeks() {
let opts = DropPromisedBlobsOptions::from_config(&config(""), now())
.unwrap()
.unwrap();
assert_eq!(opts.min_size, DEFAULT_DROP_MIN_SIZE);
assert_eq!(opts.keep_newer_than, Some(now() - time::Duration::weeks(2)));
assert_eq!(opts.now, now());
assert!(!opts.dry_run);
}
#[test]
fn min_size_takes_units_and_rejects_junk() {
let min = |v: &str| {
DropPromisedBlobsOptions::from_config(
&config(&format!(
"[maintenance \"drop-large-blobs\"]\n\tminSize = {v}\n"
)),
now(),
)
.map(|o| o.unwrap().min_size)
};
assert_eq!(min("100k").unwrap(), 100 * 1024);
assert_eq!(min("1m").unwrap(), 1 << 20);
assert_eq!(min("0").unwrap(), 0);
for bad in ["lots", "-1"] {
assert!(
matches!(
min(bad),
Err(Error::Config(ConfigError::BadNumericValue { .. }))
),
"{bad}"
);
}
}
#[test]
fn min_size_follows_the_smallest_promisor_blob_limit() {
let cfg = config(
"[remote \"origin\"]\n\turl = /o\n\tpromisor = true\n\tpartialclonefilter = blob:limit=200k\n\
[remote \"lop\"]\n\turl = /l\n\tpromisor = true\n\tpartialclonefilter = blob:limit=50k\n\
[remote \"none\"]\n\turl = /n\n\tpromisor = true\n\tpartialclonefilter = blob:none\n",
);
let opts = DropPromisedBlobsOptions::from_config(&cfg, now())
.unwrap()
.unwrap();
assert_eq!(opts.min_size, 50 * 1024);
let blobless = config(
"[remote \"origin\"]\n\turl = /o\n\tpromisor = true\n\tpartialclonefilter = blob:none\n",
);
let opts = DropPromisedBlobsOptions::from_config(&blobless, now())
.unwrap()
.unwrap();
assert_eq!(opts.min_size, DEFAULT_DROP_MIN_SIZE);
}
#[test]
fn expire_values() {
let expire = |v: &str| {
DropPromisedBlobsOptions::from_config(
&config(&format!(
"[maintenance \"drop-large-blobs\"]\n\texpire = {v}\n"
)),
now(),
)
};
assert_eq!(expire("now").unwrap().unwrap().keep_newer_than, None);
assert_eq!(expire("all").unwrap().unwrap().keep_newer_than, None);
assert!(expire("never").unwrap().is_none());
assert!(expire("false").unwrap().is_none());
assert_eq!(
expire("3.days.ago").unwrap().unwrap().keep_newer_than,
Some(now() - time::Duration::days(3))
);
assert!(matches!(
expire("not a date at all"),
Err(Error::Config(ConfigError::InvalidDate { key, value }))
if key == EXPIRE_KEY && value == "not a date at all"
));
}
#[test]
fn expiry_keywords_dates_and_junk() {
assert_eq!(DropExpiry::parse(" Never ", now()), Some(DropExpiry::Never));
assert_eq!(DropExpiry::parse("false", now()), Some(DropExpiry::Never));
assert_eq!(
DropExpiry::parse("NOW", now()),
Some(DropExpiry::Immediately)
);
assert_eq!(
DropExpiry::parse("all", now()),
Some(DropExpiry::Immediately)
);
assert_eq!(
DropExpiry::parse("1.week.ago", now()),
Some(DropExpiry::KeepNewerThan(now() - time::Duration::weeks(1)))
);
for junk in ["", " ", "not a date at all"] {
assert_eq!(DropExpiry::parse(junk, now()), None, "{junk:?}");
}
}
#[test]
fn overrides_replace_only_their_own_key() {
let cfg = config("[maintenance \"drop-large-blobs\"]\n\texpire = never\n\tminSize = 10m\n");
assert!(DropPromisedBlobsOptions::from_config(&cfg, now())
.unwrap()
.is_none());
let opts = DropPromisedBlobsOptions::from_config_with_overrides(
&cfg,
now(),
None,
Some(DropExpiry::Immediately),
)
.unwrap()
.unwrap();
assert_eq!(opts.min_size, 10 << 20);
assert_eq!(opts.keep_newer_than, None);
let opts = DropPromisedBlobsOptions::from_config_with_overrides(
&cfg,
now(),
Some(5),
Some(DropExpiry::KeepNewerThan(now())),
)
.unwrap()
.unwrap();
assert_eq!((opts.min_size, opts.keep_newer_than), (5, Some(now())));
assert!(DropPromisedBlobsOptions::from_config_with_overrides(
&cfg,
now(),
Some(5),
Some(DropExpiry::Never)
)
.unwrap()
.is_none());
let bad =
config("[maintenance \"drop-large-blobs\"]\n\texpire = bogus\n\tminSize = lots\n");
let opts = DropPromisedBlobsOptions::from_config_with_overrides(
&bad,
now(),
Some(7),
Some(DropExpiry::Immediately),
)
.unwrap()
.unwrap();
assert_eq!(opts.min_size, 7);
assert!(matches!(
DropPromisedBlobsOptions::from_config_with_overrides(
&bad,
now(),
None,
Some(DropExpiry::Immediately)
),
Err(Error::Config(ConfigError::BadNumericValue { .. }))
));
assert!(matches!(
DropPromisedBlobsOptions::from_config_with_overrides(&bad, now(), Some(7), None),
Err(Error::Config(ConfigError::InvalidDate { .. }))
));
}
#[test]
fn gc_plan_checks_config_and_needs_a_time() {
let plan = |text: &str, now_unix: i64| plan_drop_for_gc(&config(text), now_unix);
let on = "[maintenance \"drop-large-blobs\"]\n\tenabled = true\n";
assert!(matches!(
plan("", 0).unwrap(),
GcDropPlan::Skip(DropPromisedBlobsSkip::Disabled)
));
assert!(matches!(
plan(&format!("{on}\texpire = never\n"), 1_700_000_000).unwrap(),
GcDropPlan::Skip(DropPromisedBlobsSkip::Disabled)
));
let GcDropPlan::Run(opts) = plan(on, 1_700_000_000).unwrap() else {
panic!("runs with a time");
};
assert_eq!(opts.now, now());
for no_time in [0, i64::MAX] {
assert!(matches!(
plan(on, no_time).unwrap(),
GcDropPlan::Skip(DropPromisedBlobsSkip::NoCurrentTime)
));
}
for (bad, now_unix) in [
("\texpire = bogus\n", 1_700_000_000),
("\tminSize = lots\n", 1_700_000_000),
("\tminSize = lots\n", 0),
] {
assert!(
matches!(plan(&format!("{on}{bad}"), now_unix), Err(Error::Config(_))),
"{bad}"
);
}
assert!(matches!(
plan("[maintenance \"drop-large-blobs\"]\n\tenabled = maybe\n", 1),
Err(Error::Config(_))
));
}
#[test]
fn enabled_is_off_by_default_and_must_be_a_boolean() {
assert!(!drop_large_blobs_enabled(&config("")).unwrap());
let on = config("[maintenance \"drop-large-blobs\"]\n\tenabled = true\n");
assert!(drop_large_blobs_enabled(&on).unwrap());
let bad = config("[maintenance \"drop-large-blobs\"]\n\tenabled = sometimes\n");
assert!(matches!(
drop_large_blobs_enabled(&bad),
Err(Error::Config(_))
));
}
#[test]
fn kept_counts_total() {
let kept = KeptBlobCounts {
kept_pack: 1,
outside_promisor_packs: 2,
checked_out: 3,
recently_checked_out: 4,
recently_fetched: 5,
not_promised_elsewhere: 6,
not_in_promisor: 7,
unverified: 8,
};
assert_eq!(kept.total(), 36);
}
}