1use std::collections::HashMap;
49use std::path::{Path, PathBuf};
50use std::sync::Arc;
51use std::time::SystemTime;
52
53use smallvec::SmallVec;
54use time::OffsetDateTime;
55
56use crate::config::ConfigSet;
57use crate::environment::RepositoryOptions;
58use crate::error::{BadNumericSource, ConfigError, Error, Result};
59use crate::git_date::approx::approxidate_careful_at;
60use crate::index_pack::{install_pack_path, IngestPackOptions};
61use crate::objects::{
62 oid_hash_map, oid_hash_set, CommitRef, ObjectId, ObjectKind, OidHashMap, OidHashSet,
63 TreeRefIter,
64};
65use crate::odb::Odb;
66use crate::offload::LOP_GIT_DIR_KEY;
67use crate::pack::PackedType;
68use crate::pack_index::PackIndex;
69use crate::pack_objects::PackBuildOptions;
70use crate::repo::Repository;
71use crate::state::InProgressOperation;
72
73use super::{GcLock, GcLockAttempt};
74
75pub const DROP_LARGE_BLOBS_TASK: &str = "drop-large-blobs";
78
79pub const DEFAULT_DROP_MIN_SIZE: u64 = 1 << 20;
82
83pub const DEFAULT_DROP_EXPIRE: &str = "2.weeks.ago";
86
87const ENABLED_KEY: &str = "maintenance.drop-large-blobs.enabled";
88const MIN_SIZE_KEY: &str = "maintenance.drop-large-blobs.minsize";
89const EXPIRE_KEY: &str = "maintenance.drop-large-blobs.expire";
90
91const OPERATION: &str = "drop-large-blobs";
93
94#[derive(Debug, Clone, PartialEq, Eq)]
96#[non_exhaustive]
97pub struct DropPromisedBlobsOptions {
98 pub now: OffsetDateTime,
101 pub min_size: u64,
104 pub keep_newer_than: Option<OffsetDateTime>,
107 pub dry_run: bool,
110 pub threads: Option<usize>,
112}
113
114impl DropPromisedBlobsOptions {
115 #[must_use]
118 pub fn new(
119 now: OffsetDateTime,
120 min_size: u64,
121 keep_newer_than: Option<OffsetDateTime>,
122 ) -> Self {
123 Self {
124 now,
125 min_size,
126 keep_newer_than,
127 dry_run: false,
128 threads: None,
129 }
130 }
131
132 pub fn from_config(config: &ConfigSet, now: OffsetDateTime) -> Result<Option<Self>> {
149 Self::from_config_with_overrides(config, now, None, None)
150 }
151
152 pub fn from_config_with_overrides(
171 config: &ConfigSet,
172 now: OffsetDateTime,
173 min_size: Option<u64>,
174 expiry: Option<DropExpiry>,
175 ) -> Result<Option<Self>> {
176 let expiry = match expiry {
177 Some(expiry) => expiry,
178 None => configured_expiry(config, now)?,
179 };
180 let keep_newer_than = match expiry {
181 DropExpiry::Never => return Ok(None),
182 DropExpiry::Immediately => None,
183 DropExpiry::KeepNewerThan(cutoff) => Some(cutoff),
184 };
185 let min_size = match min_size {
186 Some(min_size) => min_size,
187 None => configured_min_size(config)?,
188 };
189 Ok(Some(Self::new(now, min_size, keep_newer_than)))
190 }
191}
192
193#[derive(Debug, Clone, Copy, PartialEq, Eq)]
196pub enum DropExpiry {
197 Never,
199 Immediately,
201 KeepNewerThan(OffsetDateTime),
203}
204
205impl DropExpiry {
206 #[must_use]
213 pub fn parse(spec: &str, now: OffsetDateTime) -> Option<Self> {
214 let spec = spec.trim();
215 if spec.eq_ignore_ascii_case("never") || spec.eq_ignore_ascii_case("false") {
216 return Some(Self::Never);
217 }
218 if spec.eq_ignore_ascii_case("now") || spec.eq_ignore_ascii_case("all") {
219 return Some(Self::Immediately);
220 }
221 if spec.is_empty() {
222 return None;
223 }
224 let mut err = 0;
225 let secs = approxidate_careful_at(spec, now.unix_timestamp(), Some(&mut err));
226 if err != 0 {
227 return None;
228 }
229 let cutoff = i64::try_from(secs)
230 .ok()
231 .and_then(|secs| OffsetDateTime::from_unix_timestamp(secs).ok())?;
232 Some(Self::KeepNewerThan(cutoff))
233 }
234}
235
236pub fn drop_large_blobs_enabled(config: &ConfigSet) -> Result<bool> {
243 match config.get_bool(ENABLED_KEY) {
244 None => Ok(false),
245 Some(Ok(value)) => Ok(value),
246 Some(Err(err)) => Err(Error::Config(err.into())),
247 }
248}
249
250fn configured_expiry(config: &ConfigSet, now: OffsetDateTime) -> Result<DropExpiry> {
252 let spec = config
253 .get(EXPIRE_KEY)
254 .map(|s| s.trim().to_owned())
255 .filter(|s| !s.is_empty())
256 .unwrap_or_else(|| DEFAULT_DROP_EXPIRE.to_owned());
257 DropExpiry::parse(&spec, now).ok_or_else(|| {
258 Error::Config(ConfigError::InvalidDate {
259 key: EXPIRE_KEY.to_owned(),
260 value: spec,
261 })
262 })
263}
264
265fn configured_min_size(config: &ConfigSet) -> Result<u64> {
267 if let Some(entry) = config.get_last_entry(MIN_SIZE_KEY) {
268 let raw = entry.value.as_deref().unwrap_or_default();
269 let bad = |reason| {
270 Error::Config(ConfigError::BadNumericValue {
271 key: MIN_SIZE_KEY.to_owned(),
272 value: raw.to_owned(),
273 reason,
274 })
275 };
276 let value = crate::config::parse_git_config_int_strict(raw).map_err(|e| {
277 bad(match e {
278 crate::config::GitConfigIntStrictError::InvalidUnit => {
279 BadNumericSource::InvalidUnit
280 }
281 crate::config::GitConfigIntStrictError::OutOfRange => BadNumericSource::OutOfRange,
282 })
283 })?;
284 return u64::try_from(value).map_err(|_| bad(BadNumericSource::OutOfRange));
285 }
286 Ok(
287 crate::promisor_remote::configured_promisor_remotes(config, &[])
288 .iter()
289 .filter_map(|remote| {
290 crate::partial_clone::stored_remote_filter(config, &remote.name)
291 .and_then(|filter| filter.blob_limit_bytes())
292 })
293 .min()
294 .unwrap_or(DEFAULT_DROP_MIN_SIZE),
295 )
296}
297
298#[derive(Debug, Clone, PartialEq, Eq)]
300#[non_exhaustive]
301pub struct DroppedBlob {
302 pub oid: ObjectId,
304 pub size: u64,
306 pub pack: PathBuf,
308}
309
310#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
313#[non_exhaustive]
314pub struct KeptBlobCounts {
315 pub kept_pack: usize,
317 pub outside_promisor_packs: usize,
321 pub not_in_promisor: usize,
324 pub unverified: usize,
330 pub checked_out: usize,
332 pub recently_checked_out: usize,
335 pub recently_fetched: usize,
337 pub not_promised_elsewhere: usize,
340}
341
342impl KeptBlobCounts {
343 #[must_use]
345 pub fn total(&self) -> usize {
346 self.kept_pack
347 + self.outside_promisor_packs
348 + self.checked_out
349 + self.recently_checked_out
350 + self.recently_fetched
351 + self.not_promised_elsewhere
352 + self.not_in_promisor
353 + self.unverified
354 }
355}
356
357#[derive(Debug, Clone, Copy, PartialEq, Eq)]
359#[non_exhaustive]
360pub enum DropPromisedBlobsSkip {
361 Disabled,
363 NoPromisorRemote,
365 OperationInProgress(InProgressOperation),
367 PreciousObjects,
369 NoCurrentTime,
372}
373
374#[derive(Debug, Clone, Default, PartialEq, Eq)]
377#[non_exhaustive]
378pub struct DropPromisedBlobsReport {
379 pub min_size: u64,
381 pub keep_newer_than: Option<OffsetDateTime>,
383 pub candidates: usize,
385 pub dropped: Vec<DroppedBlob>,
387 pub kept: KeptBlobCounts,
389 pub bytes_reclaimed: u64,
392 pub packs_rewritten: usize,
394 pub packs_removed: usize,
396 pub skipped: Option<DropPromisedBlobsSkip>,
398}
399
400impl DropPromisedBlobsReport {
401 fn skipped_for(reason: DropPromisedBlobsSkip) -> Self {
402 Self {
403 skipped: Some(reason),
404 ..Self::default()
405 }
406 }
407}
408
409pub fn drop_promised_blobs(
442 repo: &Repository,
443 opts: &DropPromisedBlobsOptions,
444) -> Result<DropPromisedBlobsReport> {
445 repo.odb().require_files_primary(OPERATION)?;
446 let config = repo.config()?;
447 if let Some(refusal) = gate(repo, &config)? {
448 return Err(refusal.into());
449 }
450 let _lock = if opts.dry_run {
451 None
452 } else {
453 match GcLock::try_acquire(repo, opts.now.unix_timestamp())? {
454 GcLockAttempt::Acquired(lock) => Some(lock),
455 GcLockAttempt::Held(holder) => return Err(Error::MaintenanceRunning { holder }),
456 }
457 };
458 run(repo, &config, opts)
459}
460
461impl Repository {
462 pub fn drop_promised_blobs(
468 &self,
469 opts: &DropPromisedBlobsOptions,
470 ) -> Result<DropPromisedBlobsReport> {
471 drop_promised_blobs(self, opts)
472 }
473}
474
475pub(crate) enum GcDropPlan {
478 Skip(DropPromisedBlobsSkip),
480 Run(DropPromisedBlobsOptions),
482}
483
484pub(crate) fn plan_drop_for_gc(config: &ConfigSet, now_unix: i64) -> Result<GcDropPlan> {
492 if !drop_large_blobs_enabled(config)? {
493 return Ok(GcDropPlan::Skip(DropPromisedBlobsSkip::Disabled));
494 }
495 let now = (now_unix != 0)
496 .then(|| OffsetDateTime::from_unix_timestamp(now_unix).ok())
497 .flatten();
498 let Some(now) = now else {
499 configured_min_size(config)?;
501 return Ok(GcDropPlan::Skip(DropPromisedBlobsSkip::NoCurrentTime));
502 };
503 Ok(match DropPromisedBlobsOptions::from_config(config, now)? {
504 Some(opts) => GcDropPlan::Run(opts),
505 None => GcDropPlan::Skip(DropPromisedBlobsSkip::Disabled),
506 })
507}
508
509pub(crate) fn run_drop_for_gc(
512 repo: &Repository,
513 config: &ConfigSet,
514 plan: GcDropPlan,
515 dry_run: bool,
516 threads: Option<usize>,
517) -> Result<DropPromisedBlobsReport> {
518 let mut opts = match plan {
519 GcDropPlan::Skip(reason) => return Ok(DropPromisedBlobsReport::skipped_for(reason)),
520 GcDropPlan::Run(opts) => opts,
521 };
522 if let Some(refusal) = gate(repo, config)? {
523 return Ok(DropPromisedBlobsReport::skipped_for(refusal.into()));
524 }
525 opts.dry_run = dry_run;
526 opts.threads = threads;
527 run(repo, config, &opts)
528}
529
530enum Refusal {
532 PreciousObjects,
533 NoPromisorRemote,
534 InProgress(InProgressOperation),
535}
536
537impl From<Refusal> for DropPromisedBlobsSkip {
538 fn from(refusal: Refusal) -> Self {
539 match refusal {
540 Refusal::PreciousObjects => Self::PreciousObjects,
541 Refusal::NoPromisorRemote => Self::NoPromisorRemote,
542 Refusal::InProgress(op) => Self::OperationInProgress(op),
543 }
544 }
545}
546
547impl From<Refusal> for Error {
548 fn from(refusal: Refusal) -> Self {
549 match refusal {
550 Refusal::PreciousObjects => Error::PreciousObjectsRepository {
551 operation: OPERATION,
552 },
553 Refusal::NoPromisorRemote => Error::NoPromisorRemote {
554 operation: OPERATION,
555 },
556 Refusal::InProgress(op) => Error::OperationInProgress(op),
557 }
558 }
559}
560
561fn gate(repo: &Repository, config: &ConfigSet) -> Result<Option<Refusal>> {
563 if config
564 .get_bool("extensions.preciousobjects")
565 .and_then(|v| v.ok())
566 .unwrap_or(false)
567 {
568 return Ok(Some(Refusal::PreciousObjects));
569 }
570 if !crate::promisor::repo_treats_promisor_packs(repo.git_dir(), config) {
571 return Ok(Some(Refusal::NoPromisorRemote));
572 }
573 for wt in crate::worktree::list_worktrees(repo)? {
574 if wt.is_bare {
575 continue;
576 }
577 if let Some(op) = crate::state::detect_in_progress(&wt.admin_dir).first() {
578 return Ok(Some(Refusal::InProgress(*op)));
579 }
580 }
581 Ok(None)
582}
583
584struct PromisorPack {
586 idx: Arc<PackIndex>,
587 mtime: Option<SystemTime>,
588 kept: bool,
589 referrers: Vec<u32>,
591 commits: Vec<u32>,
593}
594
595struct Candidate {
597 size: u64,
598 packs: SmallVec<[u32; 1]>,
600 entry_bytes: u64,
602}
603
604fn run(
605 repo: &Repository,
606 config: &ConfigSet,
607 opts: &DropPromisedBlobsOptions,
608) -> Result<DropPromisedBlobsReport> {
609 let odb = repo.odb();
610 let pack_dir = odb.objects_dir().join("pack");
611 odb.reprepare_local_packs();
612 let (promisor, others): (Vec<_>, Vec<_>) = odb
613 .local_pack_indexes()?
614 .into_iter()
615 .partition(|idx| idx.is_promisor);
616 let mut packs: Vec<PromisorPack> = promisor
617 .into_iter()
618 .map(|idx| PromisorPack {
619 mtime: std::fs::metadata(&idx.pack_path)
620 .and_then(|m| m.modified())
621 .ok(),
622 kept: super::pack_has_keep_marker(&pack_dir, &idx.pack_path),
623 idx,
624 referrers: Vec::new(),
625 commits: Vec::new(),
626 })
627 .collect();
628 packs.sort_by(|a, b| a.idx.pack_path.cmp(&b.idx.pack_path));
630
631 let mut report = DropPromisedBlobsReport {
632 min_size: opts.min_size,
633 keep_newer_than: opts.keep_newer_than,
634 ..DropPromisedBlobsReport::default()
635 };
636 let mut candidates: OidHashMap<Candidate> = oid_hash_map();
637 for (pack_no, pack) in packs.iter_mut().enumerate() {
638 scan_pack(odb, pack, pack_no as u32, opts.min_size, &mut candidates)?;
639 }
640 report.candidates = candidates.len();
641 if candidates.is_empty() {
642 return Ok(report);
643 }
644
645 let cutoff = opts.keep_newer_than.map(SystemTime::from);
647 let mut stored_only_here: OidHashSet = oid_hash_set();
648 for (oid, cand) in &candidates {
649 let holding = || cand.packs.iter().map(|&p| &packs[p as usize]);
650 if holding().any(|p| p.kept) {
651 report.kept.kept_pack += 1;
652 } else if odb.local_loose_file_size(oid).is_some()
653 || others.iter().any(|idx| idx.find_position(oid).is_some())
654 {
655 report.kept.outside_promisor_packs += 1;
656 } else {
657 stored_only_here.insert(*oid);
658 }
659 }
660 let mut undecided =
662 PromisorSupply::load(repo, config).suppliable(stored_only_here, &mut report.kept)?;
663 let mut walk = ProtectionWalk::new(odb, &undecided);
664 walk.mark_checkouts(repo)?;
665 let checked_out = std::mem::take(&mut walk.hits);
666 if let Some(cutoff) = opts.keep_newer_than {
667 walk.mark_recent_heads(repo, cutoff)?;
668 }
669 let recently_checked_out = std::mem::take(&mut walk.hits);
670 drop(walk);
671 report.kept.checked_out = checked_out.len();
672 report.kept.recently_checked_out = recently_checked_out.len();
673 undecided.retain(|oid| !checked_out.contains(oid) && !recently_checked_out.contains(oid));
674 undecided.retain(|oid| {
675 let fresh = cutoff.is_some_and(|cutoff| {
676 candidates[oid]
677 .packs
678 .iter()
679 .filter_map(|&p| packs[p as usize].mtime)
680 .any(|mtime| mtime >= cutoff)
681 });
682 if fresh {
683 report.kept.recently_fetched += 1;
684 }
685 !fresh
686 });
687 let promised = promised_by_promisor_packs(odb, &packs, &undecided)?;
688 report.kept.not_promised_elsewhere = undecided.len() - promised.len();
689 let drop_set = promised;
690
691 let mut dropped: Vec<DroppedBlob> = drop_set
692 .iter()
693 .map(|oid| {
694 let cand = &candidates[oid];
695 DroppedBlob {
696 oid: *oid,
697 size: cand.size,
698 pack: cand
699 .packs
700 .first()
701 .map(|&p| packs[p as usize].idx.pack_path.clone())
702 .unwrap_or_default(),
703 }
704 })
705 .collect();
706 dropped.sort_by_key(|d| d.oid);
707 report.dropped = dropped;
708 if drop_set.is_empty() {
709 return Ok(report);
710 }
711 if opts.dry_run {
712 report.bytes_reclaimed = drop_set.iter().map(|oid| candidates[oid].entry_bytes).sum();
713 return Ok(report);
714 }
715
716 let mut affected: Vec<u32> = drop_set
717 .iter()
718 .flat_map(|oid| candidates[oid].packs.iter().copied())
719 .collect();
720 affected.sort_unstable();
721 affected.dedup();
722 rewrite_packs(
723 repo,
724 config,
725 opts,
726 &packs,
727 &affected,
728 &drop_set,
729 &mut report,
730 )?;
731 Ok(report)
732}
733
734struct PromisorSupply {
737 local: Vec<LocalPromisor>,
740 unreadable: bool,
744 any_local: bool,
746 network: bool,
748 offloaded: bool,
751}
752
753struct LocalPromisor {
755 repo: Repository,
756 lop: bool,
760}
761
762impl PromisorSupply {
763 fn load(repo: &Repository, config: &ConfigSet) -> Self {
770 let recorded: Vec<PathBuf> = config
771 .get_all(LOP_GIT_DIR_KEY)
772 .into_iter()
773 .filter(|v| !v.trim().is_empty())
774 .map(PathBuf::from)
775 .collect();
776 let mut dirs: Vec<(PathBuf, bool)> = recorded.iter().map(|d| (d.clone(), true)).collect();
777 let mut network = false;
778 for remote in crate::promisor_remote::configured_promisor_remotes(config, &[]) {
779 let Some(url) = remote.url else { continue };
780 if crate::transport_path::is_local_path_remote_url(&url) {
781 let dir = crate::transport_path::resolve_local_remote_git_dir(
782 &url,
783 repo.git_dir(),
784 repo.work_tree(),
785 );
786 dirs.push((dir, false));
787 } else {
788 network = true;
789 }
790 }
791 let mut supply = Self {
792 local: Vec::new(),
793 unreadable: false,
794 any_local: !dirs.is_empty(),
795 network,
796 offloaded: !recorded.is_empty(),
797 };
798 for (dir, lop) in dirs {
799 let opened = Repository::open(
800 &dir,
801 &RepositoryOptions::with_environment(repo.environment().clone()),
802 );
803 let other = match opened {
804 Ok(other) if other.odb().hash_algo() == repo.odb().hash_algo() => other,
805 _ => {
806 supply.unreadable = true;
807 continue;
808 }
809 };
810 if crate::offload::same_object_store(&other, repo) {
811 continue;
812 }
813 if supply
815 .local
816 .iter()
817 .any(|p| crate::offload::same_object_store(&other, &p.repo))
818 {
819 continue;
820 }
821 supply.local.push(LocalPromisor { repo: other, lop });
822 }
823 supply
824 }
825
826 fn suppliable(&self, promised: OidHashSet, kept: &mut KeptBlobCounts) -> Result<OidHashSet> {
834 if !self.any_local && !self.offloaded {
835 return Ok(promised);
836 }
837 if promised.is_empty() {
838 return Ok(promised);
839 }
840 let mut stores = Vec::with_capacity(self.local.len());
841 for promisor in &self.local {
842 let odb = promisor.repo.odb();
843 let indexes = if promisor.lop {
844 crate::offload::lop_keep_packs(odb)?
845 } else {
846 odb.local_pack_indexes()?
847 };
848 stores.push((odb, indexes, !promisor.lop));
849 }
850 let held = |oid: &ObjectId| {
851 stores.iter().any(|(odb, indexes, loose_counts)| {
852 indexes.iter().any(|idx| idx.find_position(oid).is_some())
853 || (*loose_counts && odb.local_loose_file_size(oid).is_some())
854 })
855 };
856 let trust_network = self.network && !self.offloaded && !self.unreadable;
857 let mut out = oid_hash_set();
858 for oid in promised {
859 if held(&oid) || trust_network {
860 out.insert(oid);
861 } else if self.network || self.unreadable {
862 kept.unverified += 1;
863 } else {
864 kept.not_in_promisor += 1;
865 }
866 }
867 Ok(out)
868 }
869}
870
871fn scan_pack(
875 odb: &Odb,
876 pack: &mut PromisorPack,
877 pack_no: u32,
878 min_size: u64,
879 candidates: &mut OidHashMap<Candidate>,
880) -> Result<()> {
881 let idx = Arc::clone(&pack.idx);
882 let data = crate::pack::read_pack_bytes_cached(&idx.pack_path)?;
883 let bytes: &[u8] = &data;
884 let pack_end = (bytes.len() as u64).saturating_sub(idx.hash_bytes() as u64);
885 let mut kinds = EntryKinds::new(odb, &idx, bytes);
886 for pos in 0..idx.len() {
887 match kinds.kind(pos)? {
888 Some(ObjectKind::Blob) => {}
889 Some(ObjectKind::Tree | ObjectKind::Tag) => {
890 pack.referrers.push(pos as u32);
891 continue;
892 }
893 Some(ObjectKind::Commit) => {
894 pack.commits.push(pos as u32);
895 continue;
896 }
897 None => continue,
898 }
899 let offset = idx.offset_at(pos)?;
900 let size = entry_result_size(bytes, offset, idx.hash_bytes())?;
901 if size < min_size {
902 continue;
903 }
904 let oid = ObjectId::from_bytes(idx.oid_at(pos))?;
905 let entry_bytes = idx
906 .next_pack_offset_after(offset, pack_end)?
907 .saturating_sub(offset);
908 let cand = candidates.entry(oid).or_insert_with(|| Candidate {
909 size,
910 packs: SmallVec::new(),
911 entry_bytes: 0,
912 });
913 cand.packs.push(pack_no);
914 cand.entry_bytes += entry_bytes;
915 }
916 Ok(())
917}
918
919fn entry_result_size(bytes: &[u8], offset: u64, hash_bytes: usize) -> Result<u64> {
922 let mut pos = usize::try_from(offset)
923 .map_err(|_| Error::CorruptObject("pack offset out of range".to_owned()))?;
924 let (packed, size) = crate::pack::parse_pack_object_header(bytes, &mut pos)?;
925 match packed {
926 PackedType::OfsDelta => {
927 crate::pack::parse_ofs_delta_base(bytes, &mut pos, offset)?;
928 }
929 PackedType::RefDelta => pos += hash_bytes,
930 _ => return Ok(size),
931 }
932 crate::pack::read_delta_result_size_from_pack_zlib(bytes, &mut pos)
933}
934
935struct EntryKinds<'a> {
938 odb: &'a Odb,
939 idx: &'a PackIndex,
940 bytes: &'a [u8],
941 kinds: Vec<u8>,
943}
944
945const MAX_CHAIN: usize = 10_000;
947
948impl<'a> EntryKinds<'a> {
949 fn new(odb: &'a Odb, idx: &'a PackIndex, bytes: &'a [u8]) -> Self {
950 Self {
951 odb,
952 idx,
953 bytes,
954 kinds: vec![0; idx.len()],
955 }
956 }
957
958 fn encode(kind: Option<ObjectKind>) -> u8 {
959 match kind {
960 Some(ObjectKind::Commit) => 1,
961 Some(ObjectKind::Tree) => 2,
962 Some(ObjectKind::Blob) => 3,
963 Some(ObjectKind::Tag) => 4,
964 None => 5,
965 }
966 }
967
968 fn decode(code: u8) -> Option<ObjectKind> {
969 match code {
970 1 => Some(ObjectKind::Commit),
971 2 => Some(ObjectKind::Tree),
972 3 => Some(ObjectKind::Blob),
973 4 => Some(ObjectKind::Tag),
974 _ => None,
975 }
976 }
977
978 fn kind(&mut self, pos: usize) -> Result<Option<ObjectKind>> {
981 let corrupt = |what: &str| {
982 Error::CorruptObject(format!("{what} in {}", self.idx.pack_path.display()))
983 };
984 let mut chain: Vec<usize> = Vec::new();
985 let mut cur = pos;
986 let kind = loop {
987 if self.kinds[cur] != 0 {
988 break Self::decode(self.kinds[cur]);
989 }
990 if chain.len() > MAX_CHAIN {
991 return Err(corrupt("delta chain too deep"));
992 }
993 chain.push(cur);
994 let offset = self.idx.offset_at(cur)?;
995 let mut at =
996 usize::try_from(offset).map_err(|_| corrupt("pack offset out of range"))?;
997 let (packed, _) = crate::pack::parse_pack_object_header(self.bytes, &mut at)?;
998 match packed {
999 PackedType::Commit => break Some(ObjectKind::Commit),
1000 PackedType::Tree => break Some(ObjectKind::Tree),
1001 PackedType::Blob => break Some(ObjectKind::Blob),
1002 PackedType::Tag => break Some(ObjectKind::Tag),
1003 PackedType::OfsDelta => {
1004 let base = crate::pack::parse_ofs_delta_base(self.bytes, &mut at, offset)?;
1005 cur = self
1006 .idx
1007 .find_position_by_offset_sorted(base)?
1008 .ok_or_else(|| corrupt("delta base offset not in index"))?;
1009 }
1010 PackedType::RefDelta => {
1011 let hash_bytes = self.idx.hash_bytes();
1012 let raw = self
1013 .bytes
1014 .get(at..at + hash_bytes)
1015 .ok_or_else(|| corrupt("truncated delta base id"))?;
1016 let base = ObjectId::from_bytes(raw)?;
1017 match self.idx.find_position(&base) {
1018 Some(base_pos) => cur = base_pos,
1019 None => match self.odb.read_info_without_fetch(&base) {
1020 Ok(info) => break Some(info.kind),
1021 Err(Error::ObjectNotFound(_)) => break None,
1022 Err(err) => return Err(err),
1023 },
1024 }
1025 }
1026 }
1027 };
1028 let code = Self::encode(kind);
1029 for row in chain {
1030 self.kinds[row] = code;
1031 }
1032 Ok(kind)
1033 }
1034}
1035
1036fn promised_by_promisor_packs(
1045 odb: &Odb,
1046 packs: &[PromisorPack],
1047 wanted: &OidHashSet,
1048) -> Result<OidHashSet> {
1049 let mut found = oid_hash_set();
1050 if wanted.is_empty() {
1051 return Ok(found);
1052 }
1053 let oid_len = odb.hash_algo().len();
1054 let mut read: Vec<Vec<bool>> = packs.iter().map(|_| Vec::new()).collect();
1055 let mut referrer = |pack_no: usize,
1056 row: usize,
1057 queue: Option<&mut std::collections::VecDeque<ObjectId>>,
1058 found: &mut OidHashSet|
1059 -> Result<()> {
1060 let seen = &mut read[pack_no];
1061 if seen.is_empty() {
1062 *seen = vec![false; packs[pack_no].idx.len()];
1063 }
1064 if std::mem::replace(&mut seen[row], true) {
1065 return Ok(());
1066 }
1067 let idx = &packs[pack_no].idx;
1068 let obj = crate::pack::read_object_at(idx, idx.offset_at(row)?)?;
1069 match obj.kind {
1070 ObjectKind::Tree => {
1071 let mut queue = queue;
1072 for entry in TreeRefIter::with_oid_len(&obj.data, oid_len) {
1073 let entry = entry?;
1074 match entry.mode & 0o170000 {
1075 0o040000 => {
1076 if let Some(q) = queue.as_deref_mut() {
1077 q.push_back(entry.oid);
1078 }
1079 }
1080 0o160000 => {}
1081 _ if wanted.contains(&entry.oid) => {
1082 found.insert(entry.oid);
1083 }
1084 _ => {}
1085 }
1086 }
1087 }
1088 ObjectKind::Tag => {
1089 if let Some(target) = crate::objects::tag_object_line_oid(&obj.data) {
1090 if wanted.contains(&target) {
1091 found.insert(target);
1092 }
1093 }
1094 }
1095 ObjectKind::Blob | ObjectKind::Commit => {}
1096 }
1097 Ok(())
1098 };
1099 let locate = |oid: &ObjectId| {
1100 packs
1101 .iter()
1102 .enumerate()
1103 .find_map(|(n, p)| p.idx.find_position(oid).map(|row| (n, row)))
1104 };
1105
1106 let mut queue: std::collections::VecDeque<ObjectId> = std::collections::VecDeque::new();
1107 for pack in packs {
1108 for &row in &pack.commits {
1109 let obj = crate::pack::read_object_at(&pack.idx, pack.idx.offset_at(row as usize)?)?;
1110 if obj.kind == ObjectKind::Commit {
1111 queue.push_back(CommitRef::parse(&obj.data)?.tree());
1112 }
1113 }
1114 }
1115 let mut trees_seen = oid_hash_set();
1116 while let Some(tree) = queue.pop_front() {
1117 if found.len() == wanted.len() {
1118 return Ok(found);
1119 }
1120 if !trees_seen.insert(tree) {
1121 continue;
1122 }
1123 if let Some((pack_no, row)) = locate(&tree) {
1126 referrer(pack_no, row, Some(&mut queue), &mut found)?;
1127 }
1128 }
1129 drop(trees_seen);
1130 for (pack_no, pack) in packs.iter().enumerate() {
1131 for &row in &pack.referrers {
1132 if found.len() == wanted.len() {
1133 return Ok(found);
1134 }
1135 referrer(pack_no, row as usize, None, &mut found)?;
1136 }
1137 }
1138 Ok(found)
1139}
1140
1141struct ProtectionWalk<'a> {
1144 odb: &'a Odb,
1145 wanted: &'a OidHashSet,
1146 trees_seen: OidHashSet,
1147 hits: OidHashSet,
1148}
1149
1150impl<'a> ProtectionWalk<'a> {
1151 fn new(odb: &'a Odb, wanted: &'a OidHashSet) -> Self {
1152 Self {
1153 odb,
1154 wanted,
1155 trees_seen: oid_hash_set(),
1156 hits: oid_hash_set(),
1157 }
1158 }
1159
1160 fn mark(&mut self, oid: ObjectId) {
1161 if self.wanted.contains(&oid) {
1162 self.hits.insert(oid);
1163 }
1164 }
1165
1166 fn walk_tree(&mut self, root: ObjectId) -> Result<()> {
1167 let oid_len = self.odb.hash_algo().len();
1168 let mut stack = vec![root];
1169 while let Some(tree) = stack.pop() {
1170 if !self.trees_seen.insert(tree) {
1171 continue;
1172 }
1173 let Some(obj) = self.odb.try_read_once(&tree)? else {
1174 continue;
1175 };
1176 if obj.kind != ObjectKind::Tree {
1177 continue;
1178 }
1179 for entry in TreeRefIter::with_oid_len(&obj.data, oid_len) {
1180 let entry = entry?;
1181 match entry.mode & 0o170000 {
1182 0o040000 => stack.push(entry.oid),
1183 0o160000 => {}
1184 _ => self.mark(entry.oid),
1185 }
1186 }
1187 }
1188 Ok(())
1189 }
1190
1191 fn walk_commit(&mut self, commit: ObjectId) -> Result<()> {
1192 if commit.is_zero() {
1193 return Ok(());
1194 }
1195 let Some(obj) = self.odb.try_read_once(&commit)? else {
1196 return Ok(());
1197 };
1198 if obj.kind != ObjectKind::Commit {
1199 return Ok(());
1200 }
1201 let tree = CommitRef::parse(&obj.data)?.tree();
1202 self.walk_tree(tree)
1203 }
1204
1205 fn mark_checkouts(&mut self, repo: &Repository) -> Result<()> {
1208 use crate::state::HeadState;
1209 for wt in crate::worktree::list_worktrees(repo)? {
1210 if wt.is_bare {
1211 continue;
1212 }
1213 self.mark_index(repo, &wt.admin_dir.join("index"))?;
1214 if let HeadState::Detached { oid } | HeadState::Branch { oid: Some(oid), .. } = wt.head
1215 {
1216 self.walk_commit(oid)?;
1217 }
1218 }
1219 if repo.environment().git_index_file.is_some() {
1221 self.mark_index(repo, &repo.index_path_for_env()?)?;
1222 }
1223 Ok(())
1224 }
1225
1226 fn mark_index(&mut self, repo: &Repository, path: &Path) -> Result<()> {
1229 let index = repo.load_index_at_preserving_sparse(
1230 path,
1231 crate::index::IndexParseContext::from_policy(repo.thread_policy()),
1232 )?;
1233 for entry in index.entries() {
1234 match entry.mode & 0o170000 {
1235 0o040000 => self.walk_tree(entry.oid)?,
1236 0o160000 => {}
1237 _ => self.mark(entry.oid),
1238 }
1239 }
1240 Ok(())
1241 }
1242
1243 fn mark_recent_heads(&mut self, repo: &Repository, cutoff: OffsetDateTime) -> Result<()> {
1245 for wt in crate::worktree::list_worktrees(repo)? {
1246 if wt.is_bare {
1247 continue;
1248 }
1249 for entry in crate::reflog::read_reflog(&wt.admin_dir, "HEAD")? {
1250 if entry.signature().is_some_and(|sig| sig.time >= cutoff) {
1251 self.walk_commit(entry.old_oid)?;
1252 self.walk_commit(entry.new_oid)?;
1253 }
1254 }
1255 }
1256 Ok(())
1257 }
1258}
1259
1260#[allow(clippy::too_many_arguments)]
1263fn rewrite_packs(
1264 repo: &Repository,
1265 config: &ConfigSet,
1266 opts: &DropPromisedBlobsOptions,
1267 packs: &[PromisorPack],
1268 affected: &[u32],
1269 drop_set: &OidHashSet,
1270 report: &mut DropPromisedBlobsReport,
1271) -> Result<()> {
1272 let odb = repo.odb();
1273 let pack_dir = odb.objects_dir().join("pack");
1274 let had_midx =
1275 pack_dir.join("multi-pack-index").is_file() || pack_dir.join("multi-pack-index.d").is_dir();
1276 let env = repo.environment().clone();
1277 let pack_opts = PackBuildOptions::for_local_copy(Some(config));
1278 let mut bytes_before = 0u64;
1279 let mut bytes_after = 0u64;
1280 let mut stems: std::collections::HashSet<std::ffi::OsString> = std::collections::HashSet::new();
1281 for &p in affected {
1282 let pack = &packs[p as usize];
1283 let old = &pack.idx.pack_path;
1284 bytes_before += pack_files_size(old);
1285 let mut keep: Vec<(u64, ObjectId)> = Vec::new();
1286 for entry in pack.idx.iter() {
1287 let oid = ObjectId::from_bytes(entry.oid())?;
1288 if !drop_set.contains(&oid) {
1289 keep.push((entry.offset()?, oid));
1290 }
1291 }
1292 let old_stem = old.file_stem().map(|stem| stem.to_os_string());
1295 if keep.is_empty() {
1296 stems.extend(old_stem);
1297 report.packs_removed += 1;
1298 continue;
1299 }
1300 keep.sort_unstable();
1301 let objects: Vec<ObjectId> = keep.into_iter().map(|(_, oid)| oid).collect();
1302 let built = super::write_staged_repack_pack(
1303 odb,
1304 &pack_dir,
1305 &objects,
1306 &HashMap::new(),
1307 &pack_opts,
1308 opts.threads,
1309 Some(config),
1310 )?;
1311 let Some(staging) = built.staging_path.as_ref() else {
1312 continue;
1313 };
1314 let trusted_rows = (!built.index_rows.is_empty()).then_some(built.index_rows);
1315 let ingested = install_pack_path(
1316 staging,
1317 odb,
1318 &IngestPackOptions {
1319 threads: opts.threads,
1320 merged_config: Some(config.clone()),
1321 environment: env.clone(),
1322 skip_post_index_verify: trusted_rows.is_some(),
1323 trusted_index_rows: trusted_rows,
1324 mark_promisor_pack: true,
1325 ..IngestPackOptions::default()
1326 },
1327 )?;
1328 let Some(new_pack) = ingested.pack_path else {
1329 continue;
1330 };
1331 if let Ok(lines) = std::fs::read(old.with_extension("promisor")) {
1333 std::fs::write(new_pack.with_extension("promisor"), lines)?;
1334 }
1335 if config.pack_write_reverse_index_default(&env) {
1336 crate::pack_rev::write_pack_rev_sidecar_for_index_path(
1337 &new_pack.with_extension("idx"),
1338 )?;
1339 }
1340 if let Some(mtime) = pack.mtime {
1342 filetime::set_file_mtime(&new_pack, filetime::FileTime::from_system_time(mtime))?;
1343 }
1344 crate::index_pack::sync_installed_pack(&new_pack)?;
1345 bytes_after += pack_files_size(&new_pack);
1346 if new_pack.file_stem() != old.file_stem() {
1347 stems.extend(old_stem);
1348 }
1349 report.packs_rewritten += 1;
1350 }
1351 if had_midx {
1352 crate::midx::clear_pack_midx_state(&pack_dir)?;
1353 }
1354 odb.invalidate_packs();
1356 super::remove_pack_files_with_stems(&pack_dir, &stems)?;
1357 crate::pack::clear_pack_cache();
1358 odb.invalidate_packs();
1359 odb.reprepare_local_packs();
1360 let packs_left = odb.local_pack_indexes()?.len();
1361 if had_midx && packs_left > 0 {
1362 super::write_midx(repo, &super::MidxOptions::default())?;
1363 }
1364 report.bytes_reclaimed = bytes_before.saturating_sub(bytes_after);
1365 Ok(())
1366}
1367
1368fn pack_files_size(pack: &Path) -> u64 {
1370 ["pack", "idx", "rev"]
1371 .iter()
1372 .filter_map(|ext| std::fs::metadata(pack.with_extension(ext)).ok())
1373 .map(|m| m.len())
1374 .sum()
1375}
1376
1377#[cfg(test)]
1378#[allow(clippy::unwrap_used, clippy::expect_used)]
1379mod tests {
1380 use super::*;
1381 use crate::config::{ConfigFile, ConfigScope};
1382
1383 fn config(text: &str) -> ConfigSet {
1384 let file = ConfigFile::parse(Path::new("test.config"), text, ConfigScope::Local).unwrap();
1385 let mut set = ConfigSet::new();
1386 set.merge(&file);
1387 set
1388 }
1389
1390 fn now() -> OffsetDateTime {
1391 OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap()
1392 }
1393
1394 #[test]
1395 fn defaults_are_one_mebibyte_and_two_weeks() {
1396 let opts = DropPromisedBlobsOptions::from_config(&config(""), now())
1397 .unwrap()
1398 .unwrap();
1399 assert_eq!(opts.min_size, DEFAULT_DROP_MIN_SIZE);
1400 assert_eq!(opts.keep_newer_than, Some(now() - time::Duration::weeks(2)));
1401 assert_eq!(opts.now, now());
1402 assert!(!opts.dry_run);
1403 }
1404
1405 #[test]
1406 fn min_size_takes_units_and_rejects_junk() {
1407 let min = |v: &str| {
1408 DropPromisedBlobsOptions::from_config(
1409 &config(&format!(
1410 "[maintenance \"drop-large-blobs\"]\n\tminSize = {v}\n"
1411 )),
1412 now(),
1413 )
1414 .map(|o| o.unwrap().min_size)
1415 };
1416 assert_eq!(min("100k").unwrap(), 100 * 1024);
1417 assert_eq!(min("1m").unwrap(), 1 << 20);
1418 assert_eq!(min("0").unwrap(), 0);
1419 for bad in ["lots", "-1"] {
1420 assert!(
1421 matches!(
1422 min(bad),
1423 Err(Error::Config(ConfigError::BadNumericValue { .. }))
1424 ),
1425 "{bad}"
1426 );
1427 }
1428 }
1429
1430 #[test]
1431 fn min_size_follows_the_smallest_promisor_blob_limit() {
1432 let cfg = config(
1433 "[remote \"origin\"]\n\turl = /o\n\tpromisor = true\n\tpartialclonefilter = blob:limit=200k\n\
1434 [remote \"lop\"]\n\turl = /l\n\tpromisor = true\n\tpartialclonefilter = blob:limit=50k\n\
1435 [remote \"none\"]\n\turl = /n\n\tpromisor = true\n\tpartialclonefilter = blob:none\n",
1436 );
1437 let opts = DropPromisedBlobsOptions::from_config(&cfg, now())
1438 .unwrap()
1439 .unwrap();
1440 assert_eq!(opts.min_size, 50 * 1024);
1441 let blobless = config(
1442 "[remote \"origin\"]\n\turl = /o\n\tpromisor = true\n\tpartialclonefilter = blob:none\n",
1443 );
1444 let opts = DropPromisedBlobsOptions::from_config(&blobless, now())
1445 .unwrap()
1446 .unwrap();
1447 assert_eq!(opts.min_size, DEFAULT_DROP_MIN_SIZE);
1448 }
1449
1450 #[test]
1451 fn expire_values() {
1452 let expire = |v: &str| {
1453 DropPromisedBlobsOptions::from_config(
1454 &config(&format!(
1455 "[maintenance \"drop-large-blobs\"]\n\texpire = {v}\n"
1456 )),
1457 now(),
1458 )
1459 };
1460 assert_eq!(expire("now").unwrap().unwrap().keep_newer_than, None);
1461 assert_eq!(expire("all").unwrap().unwrap().keep_newer_than, None);
1462 assert!(expire("never").unwrap().is_none());
1463 assert!(expire("false").unwrap().is_none());
1464 assert_eq!(
1465 expire("3.days.ago").unwrap().unwrap().keep_newer_than,
1466 Some(now() - time::Duration::days(3))
1467 );
1468 assert!(matches!(
1469 expire("not a date at all"),
1470 Err(Error::Config(ConfigError::InvalidDate { key, value }))
1471 if key == EXPIRE_KEY && value == "not a date at all"
1472 ));
1473 }
1474
1475 #[test]
1476 fn expiry_keywords_dates_and_junk() {
1477 assert_eq!(DropExpiry::parse(" Never ", now()), Some(DropExpiry::Never));
1478 assert_eq!(DropExpiry::parse("false", now()), Some(DropExpiry::Never));
1479 assert_eq!(
1480 DropExpiry::parse("NOW", now()),
1481 Some(DropExpiry::Immediately)
1482 );
1483 assert_eq!(
1484 DropExpiry::parse("all", now()),
1485 Some(DropExpiry::Immediately)
1486 );
1487 assert_eq!(
1488 DropExpiry::parse("1.week.ago", now()),
1489 Some(DropExpiry::KeepNewerThan(now() - time::Duration::weeks(1)))
1490 );
1491 for junk in ["", " ", "not a date at all"] {
1492 assert_eq!(DropExpiry::parse(junk, now()), None, "{junk:?}");
1493 }
1494 }
1495
1496 #[test]
1497 fn overrides_replace_only_their_own_key() {
1498 let cfg = config("[maintenance \"drop-large-blobs\"]\n\texpire = never\n\tminSize = 10m\n");
1499 assert!(DropPromisedBlobsOptions::from_config(&cfg, now())
1500 .unwrap()
1501 .is_none());
1502 let opts = DropPromisedBlobsOptions::from_config_with_overrides(
1504 &cfg,
1505 now(),
1506 None,
1507 Some(DropExpiry::Immediately),
1508 )
1509 .unwrap()
1510 .unwrap();
1511 assert_eq!(opts.min_size, 10 << 20);
1512 assert_eq!(opts.keep_newer_than, None);
1513 let opts = DropPromisedBlobsOptions::from_config_with_overrides(
1514 &cfg,
1515 now(),
1516 Some(5),
1517 Some(DropExpiry::KeepNewerThan(now())),
1518 )
1519 .unwrap()
1520 .unwrap();
1521 assert_eq!((opts.min_size, opts.keep_newer_than), (5, Some(now())));
1522 assert!(DropPromisedBlobsOptions::from_config_with_overrides(
1523 &cfg,
1524 now(),
1525 Some(5),
1526 Some(DropExpiry::Never)
1527 )
1528 .unwrap()
1529 .is_none());
1530
1531 let bad =
1533 config("[maintenance \"drop-large-blobs\"]\n\texpire = bogus\n\tminSize = lots\n");
1534 let opts = DropPromisedBlobsOptions::from_config_with_overrides(
1535 &bad,
1536 now(),
1537 Some(7),
1538 Some(DropExpiry::Immediately),
1539 )
1540 .unwrap()
1541 .unwrap();
1542 assert_eq!(opts.min_size, 7);
1543 assert!(matches!(
1544 DropPromisedBlobsOptions::from_config_with_overrides(
1545 &bad,
1546 now(),
1547 None,
1548 Some(DropExpiry::Immediately)
1549 ),
1550 Err(Error::Config(ConfigError::BadNumericValue { .. }))
1551 ));
1552 assert!(matches!(
1553 DropPromisedBlobsOptions::from_config_with_overrides(&bad, now(), Some(7), None),
1554 Err(Error::Config(ConfigError::InvalidDate { .. }))
1555 ));
1556 }
1557
1558 #[test]
1559 fn gc_plan_checks_config_and_needs_a_time() {
1560 let plan = |text: &str, now_unix: i64| plan_drop_for_gc(&config(text), now_unix);
1561 let on = "[maintenance \"drop-large-blobs\"]\n\tenabled = true\n";
1562 assert!(matches!(
1563 plan("", 0).unwrap(),
1564 GcDropPlan::Skip(DropPromisedBlobsSkip::Disabled)
1565 ));
1566 assert!(matches!(
1567 plan(&format!("{on}\texpire = never\n"), 1_700_000_000).unwrap(),
1568 GcDropPlan::Skip(DropPromisedBlobsSkip::Disabled)
1569 ));
1570 let GcDropPlan::Run(opts) = plan(on, 1_700_000_000).unwrap() else {
1571 panic!("runs with a time");
1572 };
1573 assert_eq!(opts.now, now());
1574 for no_time in [0, i64::MAX] {
1576 assert!(matches!(
1577 plan(on, no_time).unwrap(),
1578 GcDropPlan::Skip(DropPromisedBlobsSkip::NoCurrentTime)
1579 ));
1580 }
1581 for (bad, now_unix) in [
1583 ("\texpire = bogus\n", 1_700_000_000),
1584 ("\tminSize = lots\n", 1_700_000_000),
1585 ("\tminSize = lots\n", 0),
1586 ] {
1587 assert!(
1588 matches!(plan(&format!("{on}{bad}"), now_unix), Err(Error::Config(_))),
1589 "{bad}"
1590 );
1591 }
1592 assert!(matches!(
1593 plan("[maintenance \"drop-large-blobs\"]\n\tenabled = maybe\n", 1),
1594 Err(Error::Config(_))
1595 ));
1596 }
1597
1598 #[test]
1599 fn enabled_is_off_by_default_and_must_be_a_boolean() {
1600 assert!(!drop_large_blobs_enabled(&config("")).unwrap());
1601 let on = config("[maintenance \"drop-large-blobs\"]\n\tenabled = true\n");
1602 assert!(drop_large_blobs_enabled(&on).unwrap());
1603 let bad = config("[maintenance \"drop-large-blobs\"]\n\tenabled = sometimes\n");
1604 assert!(matches!(
1605 drop_large_blobs_enabled(&bad),
1606 Err(Error::Config(_))
1607 ));
1608 }
1609
1610 #[test]
1611 fn kept_counts_total() {
1612 let kept = KeptBlobCounts {
1613 kept_pack: 1,
1614 outside_promisor_packs: 2,
1615 checked_out: 3,
1616 recently_checked_out: 4,
1617 recently_fetched: 5,
1618 not_promised_elsewhere: 6,
1619 not_in_promisor: 7,
1620 unverified: 8,
1621 };
1622 assert_eq!(kept.total(), 36);
1623 }
1624}