1use std::collections::HashMap;
49use std::path::{Path, PathBuf};
50use std::sync::Arc;
51use std::time::SystemTime;
52
53use smallvec::SmallVec;
54use time::OffsetDateTime;
55
56use crate::config::ConfigSet;
57use crate::environment::RepositoryOptions;
58use crate::error::{BadNumericSource, ConfigError, Error, Result};
59use crate::git_date::approx::approxidate_careful_at;
60use crate::index_pack::{install_pack_path, IngestPackOptions};
61use crate::objects::{
62 oid_hash_map, oid_hash_set, CommitRef, ObjectId, ObjectKind, OidHashMap, OidHashSet,
63 TreeRefIter,
64};
65use crate::odb::Odb;
66use crate::offload::LOP_GIT_DIR_KEY;
67use crate::pack::PackedType;
68use crate::pack_index::PackIndex;
69use crate::pack_objects::PackBuildOptions;
70use crate::repo::Repository;
71use crate::state::InProgressOperation;
72
73use super::{GcLock, GcLockAttempt};
74
75pub const DROP_LARGE_BLOBS_TASK: &str = "drop-large-blobs";
78
79pub const DEFAULT_DROP_MIN_SIZE: u64 = 1 << 20;
82
83pub const DEFAULT_DROP_EXPIRE: &str = "2.weeks.ago";
86
87const ENABLED_KEY: &str = "maintenance.drop-large-blobs.enabled";
88const MIN_SIZE_KEY: &str = "maintenance.drop-large-blobs.minsize";
89const EXPIRE_KEY: &str = "maintenance.drop-large-blobs.expire";
90
91const OPERATION: &str = "drop-large-blobs";
93
94#[derive(Debug, Clone, PartialEq, Eq)]
96#[non_exhaustive]
97pub struct DropPromisedBlobsOptions {
98 pub now: OffsetDateTime,
101 pub min_size: u64,
104 pub keep_newer_than: Option<OffsetDateTime>,
107 pub dry_run: bool,
110 pub threads: Option<usize>,
112}
113
114impl DropPromisedBlobsOptions {
115 #[must_use]
118 pub fn new(
119 now: OffsetDateTime,
120 min_size: u64,
121 keep_newer_than: Option<OffsetDateTime>,
122 ) -> Self {
123 Self {
124 now,
125 min_size,
126 keep_newer_than,
127 dry_run: false,
128 threads: None,
129 }
130 }
131
132 pub fn from_config(config: &ConfigSet, now: OffsetDateTime) -> Result<Option<Self>> {
149 Self::from_config_with_overrides(config, now, None, None)
150 }
151
152 pub fn from_config_with_overrides(
171 config: &ConfigSet,
172 now: OffsetDateTime,
173 min_size: Option<u64>,
174 expiry: Option<DropExpiry>,
175 ) -> Result<Option<Self>> {
176 let expiry = match expiry {
177 Some(expiry) => expiry,
178 None => configured_expiry(config, now)?,
179 };
180 let keep_newer_than = match expiry {
181 DropExpiry::Never => return Ok(None),
182 DropExpiry::Immediately => None,
183 DropExpiry::KeepNewerThan(cutoff) => Some(cutoff),
184 };
185 let min_size = match min_size {
186 Some(min_size) => min_size,
187 None => configured_min_size(config)?,
188 };
189 Ok(Some(Self::new(now, min_size, keep_newer_than)))
190 }
191}
192
193#[derive(Debug, Clone, Copy, PartialEq, Eq)]
196pub enum DropExpiry {
197 Never,
199 Immediately,
201 KeepNewerThan(OffsetDateTime),
203}
204
205impl DropExpiry {
206 #[must_use]
213 pub fn parse(spec: &str, now: OffsetDateTime) -> Option<Self> {
214 let spec = spec.trim();
215 if spec.eq_ignore_ascii_case("never") || spec.eq_ignore_ascii_case("false") {
216 return Some(Self::Never);
217 }
218 if spec.eq_ignore_ascii_case("now") || spec.eq_ignore_ascii_case("all") {
219 return Some(Self::Immediately);
220 }
221 if spec.is_empty() {
222 return None;
223 }
224 let mut err = 0;
225 let secs = approxidate_careful_at(spec, now.unix_timestamp(), Some(&mut err));
226 if err != 0 {
227 return None;
228 }
229 let cutoff = i64::try_from(secs)
230 .ok()
231 .and_then(|secs| OffsetDateTime::from_unix_timestamp(secs).ok())?;
232 Some(Self::KeepNewerThan(cutoff))
233 }
234}
235
236pub fn drop_large_blobs_enabled(config: &ConfigSet) -> Result<bool> {
243 Ok(config.get_bool(ENABLED_KEY).transpose()?.unwrap_or(false))
244}
245
246fn configured_expiry(config: &ConfigSet, now: OffsetDateTime) -> Result<DropExpiry> {
248 let spec = config
249 .get(EXPIRE_KEY)
250 .map(|s| s.trim().to_owned())
251 .filter(|s| !s.is_empty())
252 .unwrap_or_else(|| DEFAULT_DROP_EXPIRE.to_owned());
253 DropExpiry::parse(&spec, now).ok_or_else(|| {
254 Error::Config(ConfigError::InvalidDate {
255 key: EXPIRE_KEY.to_owned(),
256 value: spec,
257 })
258 })
259}
260
261fn configured_min_size(config: &ConfigSet) -> Result<u64> {
263 if let Some(entry) = config.get_last_entry(MIN_SIZE_KEY) {
264 let raw = entry.value.as_deref().unwrap_or_default();
265 let bad = |reason| {
266 Error::Config(ConfigError::BadNumericValue {
267 key: MIN_SIZE_KEY.to_owned(),
268 value: raw.to_owned(),
269 reason,
270 })
271 };
272 let value = crate::config::parse_git_config_int_strict(raw).map_err(|e| {
273 bad(match e {
274 crate::config::GitConfigIntStrictError::InvalidUnit => {
275 BadNumericSource::InvalidUnit
276 }
277 crate::config::GitConfigIntStrictError::OutOfRange => BadNumericSource::OutOfRange,
278 })
279 })?;
280 return u64::try_from(value).map_err(|_| bad(BadNumericSource::OutOfRange));
281 }
282 Ok(
283 crate::promisor_remote::configured_promisor_remotes(config, &[])
284 .iter()
285 .filter_map(|remote| {
286 crate::partial_clone::stored_remote_filter(config, &remote.name)
287 .and_then(|filter| filter.blob_limit_bytes())
288 })
289 .min()
290 .unwrap_or(DEFAULT_DROP_MIN_SIZE),
291 )
292}
293
294#[derive(Debug, Clone, PartialEq, Eq)]
296#[non_exhaustive]
297pub struct DroppedBlob {
298 pub oid: ObjectId,
300 pub size: u64,
302 pub pack: PathBuf,
304}
305
306#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
309#[non_exhaustive]
310pub struct KeptBlobCounts {
311 pub kept_pack: usize,
313 pub outside_promisor_packs: usize,
317 pub not_in_promisor: usize,
320 pub unverified: usize,
326 pub checked_out: usize,
328 pub recently_checked_out: usize,
331 pub recently_fetched: usize,
333 pub not_promised_elsewhere: usize,
336}
337
338impl KeptBlobCounts {
339 #[must_use]
341 pub fn total(&self) -> usize {
342 self.kept_pack
343 + self.outside_promisor_packs
344 + self.checked_out
345 + self.recently_checked_out
346 + self.recently_fetched
347 + self.not_promised_elsewhere
348 + self.not_in_promisor
349 + self.unverified
350 }
351}
352
353#[derive(Debug, Clone, Copy, PartialEq, Eq)]
355#[non_exhaustive]
356pub enum DropPromisedBlobsSkip {
357 Disabled,
359 NoPromisorRemote,
361 OperationInProgress(InProgressOperation),
363 PreciousObjects,
365 NoCurrentTime,
368}
369
370#[derive(Debug, Clone, Default, PartialEq, Eq)]
373#[non_exhaustive]
374pub struct DropPromisedBlobsReport {
375 pub min_size: u64,
377 pub keep_newer_than: Option<OffsetDateTime>,
379 pub candidates: usize,
381 pub dropped: Vec<DroppedBlob>,
383 pub kept: KeptBlobCounts,
385 pub bytes_reclaimed: u64,
388 pub packs_rewritten: usize,
390 pub packs_removed: usize,
392 pub skipped: Option<DropPromisedBlobsSkip>,
394}
395
396impl DropPromisedBlobsReport {
397 fn skipped_for(reason: DropPromisedBlobsSkip) -> Self {
398 Self {
399 skipped: Some(reason),
400 ..Self::default()
401 }
402 }
403}
404
405pub fn drop_promised_blobs(
438 repo: &Repository,
439 opts: &DropPromisedBlobsOptions,
440) -> Result<DropPromisedBlobsReport> {
441 repo.odb().require_files_primary(OPERATION)?;
442 let config = repo.config()?;
443 if let Some(refusal) = gate(repo, &config)? {
444 return Err(refusal.into());
445 }
446 let _lock = if opts.dry_run {
447 None
448 } else {
449 match GcLock::try_acquire(repo, opts.now.unix_timestamp())? {
450 GcLockAttempt::Acquired(lock) => Some(lock),
451 GcLockAttempt::Held(holder) => return Err(Error::MaintenanceRunning { holder }),
452 }
453 };
454 run(repo, &config, opts)
455}
456
457impl Repository {
458 pub fn drop_promised_blobs(
464 &self,
465 opts: &DropPromisedBlobsOptions,
466 ) -> Result<DropPromisedBlobsReport> {
467 drop_promised_blobs(self, opts)
468 }
469}
470
471pub(crate) enum GcDropPlan {
474 Skip(DropPromisedBlobsSkip),
476 Run(DropPromisedBlobsOptions),
478}
479
480pub(crate) fn plan_drop_for_gc(config: &ConfigSet, now_unix: i64) -> Result<GcDropPlan> {
488 if !drop_large_blobs_enabled(config)? {
489 return Ok(GcDropPlan::Skip(DropPromisedBlobsSkip::Disabled));
490 }
491 let now = (now_unix != 0)
492 .then(|| OffsetDateTime::from_unix_timestamp(now_unix).ok())
493 .flatten();
494 let Some(now) = now else {
495 configured_min_size(config)?;
497 return Ok(GcDropPlan::Skip(DropPromisedBlobsSkip::NoCurrentTime));
498 };
499 Ok(match DropPromisedBlobsOptions::from_config(config, now)? {
500 Some(opts) => GcDropPlan::Run(opts),
501 None => GcDropPlan::Skip(DropPromisedBlobsSkip::Disabled),
502 })
503}
504
505pub(crate) fn run_drop_for_gc(
508 repo: &Repository,
509 config: &ConfigSet,
510 plan: GcDropPlan,
511 dry_run: bool,
512 threads: Option<usize>,
513) -> Result<DropPromisedBlobsReport> {
514 let mut opts = match plan {
515 GcDropPlan::Skip(reason) => return Ok(DropPromisedBlobsReport::skipped_for(reason)),
516 GcDropPlan::Run(opts) => opts,
517 };
518 if let Some(refusal) = gate(repo, config)? {
519 return Ok(DropPromisedBlobsReport::skipped_for(refusal.into()));
520 }
521 opts.dry_run = dry_run;
522 opts.threads = threads;
523 run(repo, config, &opts)
524}
525
526enum Refusal {
528 PreciousObjects,
529 NoPromisorRemote,
530 InProgress(InProgressOperation),
531}
532
533impl From<Refusal> for DropPromisedBlobsSkip {
534 fn from(refusal: Refusal) -> Self {
535 match refusal {
536 Refusal::PreciousObjects => Self::PreciousObjects,
537 Refusal::NoPromisorRemote => Self::NoPromisorRemote,
538 Refusal::InProgress(op) => Self::OperationInProgress(op),
539 }
540 }
541}
542
543impl From<Refusal> for Error {
544 fn from(refusal: Refusal) -> Self {
545 match refusal {
546 Refusal::PreciousObjects => Error::PreciousObjectsRepository {
547 operation: OPERATION,
548 },
549 Refusal::NoPromisorRemote => Error::NoPromisorRemote {
550 operation: OPERATION,
551 },
552 Refusal::InProgress(op) => Error::OperationInProgress(op),
553 }
554 }
555}
556
557fn gate(repo: &Repository, config: &ConfigSet) -> Result<Option<Refusal>> {
559 if config
560 .get_bool("extensions.preciousobjects")
561 .and_then(|v| v.ok())
562 .unwrap_or(false)
563 {
564 return Ok(Some(Refusal::PreciousObjects));
565 }
566 if !crate::promisor::repo_treats_promisor_packs(repo.git_dir(), config) {
567 return Ok(Some(Refusal::NoPromisorRemote));
568 }
569 for wt in crate::worktree::list_worktrees(repo)? {
570 if wt.is_bare {
571 continue;
572 }
573 if let Some(op) = crate::state::detect_in_progress(&wt.admin_dir).first() {
574 return Ok(Some(Refusal::InProgress(*op)));
575 }
576 }
577 Ok(None)
578}
579
580struct PromisorPack {
582 idx: Arc<PackIndex>,
583 mtime: Option<SystemTime>,
584 kept: bool,
585 referrers: Vec<u32>,
587 commits: Vec<u32>,
589}
590
591struct Candidate {
593 size: u64,
594 packs: SmallVec<[u32; 1]>,
596 entry_bytes: u64,
598}
599
600fn run(
601 repo: &Repository,
602 config: &ConfigSet,
603 opts: &DropPromisedBlobsOptions,
604) -> Result<DropPromisedBlobsReport> {
605 let odb = repo.odb();
606 let pack_dir = odb.objects_dir().join("pack");
607 odb.reprepare_local_packs();
608 let (promisor, others): (Vec<_>, Vec<_>) = odb
609 .local_pack_indexes()?
610 .into_iter()
611 .partition(|idx| idx.is_promisor);
612 let mut packs: Vec<PromisorPack> = promisor
613 .into_iter()
614 .map(|idx| PromisorPack {
615 mtime: std::fs::metadata(&idx.pack_path)
616 .and_then(|m| m.modified())
617 .ok(),
618 kept: super::pack_has_keep_marker(&pack_dir, &idx.pack_path),
619 idx,
620 referrers: Vec::new(),
621 commits: Vec::new(),
622 })
623 .collect();
624 packs.sort_by(|a, b| a.idx.pack_path.cmp(&b.idx.pack_path));
626
627 let mut report = DropPromisedBlobsReport {
628 min_size: opts.min_size,
629 keep_newer_than: opts.keep_newer_than,
630 ..DropPromisedBlobsReport::default()
631 };
632 let mut candidates: OidHashMap<Candidate> = oid_hash_map();
633 for (pack_no, pack) in packs.iter_mut().enumerate() {
634 scan_pack(odb, pack, pack_no as u32, opts.min_size, &mut candidates)?;
635 }
636 report.candidates = candidates.len();
637 if candidates.is_empty() {
638 return Ok(report);
639 }
640
641 let cutoff = opts.keep_newer_than.map(SystemTime::from);
643 let mut stored_only_here: OidHashSet = oid_hash_set();
644 for (oid, cand) in &candidates {
645 let holding = || cand.packs.iter().map(|&p| &packs[p as usize]);
646 if holding().any(|p| p.kept) {
647 report.kept.kept_pack += 1;
648 } else if odb.local_loose_file_size(oid).is_some()
649 || others.iter().any(|idx| idx.find_position(oid).is_some())
650 {
651 report.kept.outside_promisor_packs += 1;
652 } else {
653 stored_only_here.insert(*oid);
654 }
655 }
656 let mut undecided =
658 PromisorSupply::load(repo, config).suppliable(stored_only_here, &mut report.kept)?;
659 let mut walk = ProtectionWalk::new(odb, &undecided);
660 walk.mark_checkouts(repo)?;
661 let checked_out = std::mem::take(&mut walk.hits);
662 if let Some(cutoff) = opts.keep_newer_than {
663 walk.mark_recent_heads(repo, cutoff)?;
664 }
665 let recently_checked_out = std::mem::take(&mut walk.hits);
666 drop(walk);
667 report.kept.checked_out = checked_out.len();
668 report.kept.recently_checked_out = recently_checked_out.len();
669 undecided.retain(|oid| !checked_out.contains(oid) && !recently_checked_out.contains(oid));
670 undecided.retain(|oid| {
671 let fresh = cutoff.is_some_and(|cutoff| {
672 candidates[oid]
673 .packs
674 .iter()
675 .filter_map(|&p| packs[p as usize].mtime)
676 .any(|mtime| mtime >= cutoff)
677 });
678 if fresh {
679 report.kept.recently_fetched += 1;
680 }
681 !fresh
682 });
683 let promised = promised_by_promisor_packs(odb, &packs, &undecided)?;
684 report.kept.not_promised_elsewhere = undecided.len() - promised.len();
685 let drop_set = promised;
686
687 let mut dropped: Vec<DroppedBlob> = drop_set
688 .iter()
689 .map(|oid| {
690 let cand = &candidates[oid];
691 DroppedBlob {
692 oid: *oid,
693 size: cand.size,
694 pack: cand
695 .packs
696 .first()
697 .map(|&p| packs[p as usize].idx.pack_path.clone())
698 .unwrap_or_default(),
699 }
700 })
701 .collect();
702 dropped.sort_by_key(|d| d.oid);
703 report.dropped = dropped;
704 if drop_set.is_empty() {
705 return Ok(report);
706 }
707 if opts.dry_run {
708 report.bytes_reclaimed = drop_set.iter().map(|oid| candidates[oid].entry_bytes).sum();
709 return Ok(report);
710 }
711
712 let mut affected: Vec<u32> = drop_set
713 .iter()
714 .flat_map(|oid| candidates[oid].packs.iter().copied())
715 .collect();
716 affected.sort_unstable();
717 affected.dedup();
718 rewrite_packs(
719 repo,
720 config,
721 opts,
722 &packs,
723 &affected,
724 &drop_set,
725 &mut report,
726 )?;
727 Ok(report)
728}
729
730struct PromisorSupply {
733 local: Vec<LocalPromisor>,
736 unreadable: bool,
740 any_local: bool,
742 network: bool,
744 offloaded: bool,
747}
748
749struct LocalPromisor {
751 repo: Repository,
752 lop: bool,
756}
757
758impl PromisorSupply {
759 fn load(repo: &Repository, config: &ConfigSet) -> Self {
766 let recorded: Vec<PathBuf> = config
767 .get_all(LOP_GIT_DIR_KEY)
768 .into_iter()
769 .filter(|v| !v.trim().is_empty())
770 .map(PathBuf::from)
771 .collect();
772 let mut dirs: Vec<(PathBuf, bool)> = recorded.iter().map(|d| (d.clone(), true)).collect();
773 let mut network = false;
774 for remote in crate::promisor_remote::configured_promisor_remotes(config, &[]) {
775 let Some(url) = remote.url else { continue };
776 if crate::transport_path::is_local_path_remote_url(&url) {
777 let dir = crate::transport_path::resolve_local_remote_git_dir(
778 &url,
779 repo.git_dir(),
780 repo.work_tree(),
781 );
782 dirs.push((dir, false));
783 } else {
784 network = true;
785 }
786 }
787 let mut supply = Self {
788 local: Vec::new(),
789 unreadable: false,
790 any_local: !dirs.is_empty(),
791 network,
792 offloaded: !recorded.is_empty(),
793 };
794 for (dir, lop) in dirs {
795 let opened = Repository::open(
796 &dir,
797 &RepositoryOptions::with_environment(repo.environment().clone()),
798 );
799 let other = match opened {
800 Ok(other) if other.odb().hash_algo() == repo.odb().hash_algo() => other,
801 _ => {
802 supply.unreadable = true;
803 continue;
804 }
805 };
806 if crate::offload::same_object_store(&other, repo) {
807 continue;
808 }
809 if supply
811 .local
812 .iter()
813 .any(|p| crate::offload::same_object_store(&other, &p.repo))
814 {
815 continue;
816 }
817 supply.local.push(LocalPromisor { repo: other, lop });
818 }
819 supply
820 }
821
822 fn suppliable(&self, promised: OidHashSet, kept: &mut KeptBlobCounts) -> Result<OidHashSet> {
830 if !self.any_local && !self.offloaded {
831 return Ok(promised);
832 }
833 if promised.is_empty() {
834 return Ok(promised);
835 }
836 let mut stores = Vec::with_capacity(self.local.len());
837 for promisor in &self.local {
838 let odb = promisor.repo.odb();
839 let indexes = if promisor.lop {
840 crate::offload::lop_keep_packs(odb)?
841 } else {
842 odb.local_pack_indexes()?
843 };
844 stores.push((odb, indexes, !promisor.lop));
845 }
846 let held = |oid: &ObjectId| {
847 stores.iter().any(|(odb, indexes, loose_counts)| {
848 indexes.iter().any(|idx| idx.find_position(oid).is_some())
849 || (*loose_counts && odb.local_loose_file_size(oid).is_some())
850 })
851 };
852 let trust_network = self.network && !self.offloaded && !self.unreadable;
853 let mut out = oid_hash_set();
854 for oid in promised {
855 if held(&oid) || trust_network {
856 out.insert(oid);
857 } else if self.network || self.unreadable {
858 kept.unverified += 1;
859 } else {
860 kept.not_in_promisor += 1;
861 }
862 }
863 Ok(out)
864 }
865}
866
867fn scan_pack(
871 odb: &Odb,
872 pack: &mut PromisorPack,
873 pack_no: u32,
874 min_size: u64,
875 candidates: &mut OidHashMap<Candidate>,
876) -> Result<()> {
877 let idx = Arc::clone(&pack.idx);
878 let data = crate::pack::read_pack_bytes_cached(&idx.pack_path)?;
879 let bytes: &[u8] = &data;
880 let pack_end = (bytes.len() as u64).saturating_sub(idx.hash_bytes() as u64);
881 let mut kinds = EntryKinds::new(odb, &idx, bytes);
882 for pos in 0..idx.len() {
883 match kinds.kind(pos)? {
884 Some(ObjectKind::Blob) => {}
885 Some(ObjectKind::Tree | ObjectKind::Tag) => {
886 pack.referrers.push(pos as u32);
887 continue;
888 }
889 Some(ObjectKind::Commit) => {
890 pack.commits.push(pos as u32);
891 continue;
892 }
893 None => continue,
894 }
895 let offset = idx.offset_at(pos)?;
896 let size = entry_result_size(bytes, offset, idx.hash_bytes())?;
897 if size < min_size {
898 continue;
899 }
900 let oid = ObjectId::from_bytes(idx.oid_at(pos))?;
901 let entry_bytes = idx
902 .next_pack_offset_after(offset, pack_end)?
903 .saturating_sub(offset);
904 let cand = candidates.entry(oid).or_insert_with(|| Candidate {
905 size,
906 packs: SmallVec::new(),
907 entry_bytes: 0,
908 });
909 cand.packs.push(pack_no);
910 cand.entry_bytes += entry_bytes;
911 }
912 Ok(())
913}
914
915fn entry_result_size(bytes: &[u8], offset: u64, hash_bytes: usize) -> Result<u64> {
918 let mut pos = usize::try_from(offset)
919 .map_err(|_| Error::CorruptObject("pack offset out of range".to_owned()))?;
920 let (packed, size) = crate::pack::parse_pack_object_header(bytes, &mut pos)?;
921 match packed {
922 PackedType::OfsDelta => {
923 crate::pack::parse_ofs_delta_base(bytes, &mut pos, offset)?;
924 }
925 PackedType::RefDelta => pos += hash_bytes,
926 _ => return Ok(size),
927 }
928 crate::pack::read_delta_result_size_from_pack_zlib(bytes, &mut pos)
929}
930
931struct EntryKinds<'a> {
934 odb: &'a Odb,
935 idx: &'a PackIndex,
936 bytes: &'a [u8],
937 kinds: Vec<u8>,
939}
940
941const MAX_CHAIN: usize = 10_000;
943
944impl<'a> EntryKinds<'a> {
945 fn new(odb: &'a Odb, idx: &'a PackIndex, bytes: &'a [u8]) -> Self {
946 Self {
947 odb,
948 idx,
949 bytes,
950 kinds: vec![0; idx.len()],
951 }
952 }
953
954 fn encode(kind: Option<ObjectKind>) -> u8 {
955 match kind {
956 Some(ObjectKind::Commit) => 1,
957 Some(ObjectKind::Tree) => 2,
958 Some(ObjectKind::Blob) => 3,
959 Some(ObjectKind::Tag) => 4,
960 None => 5,
961 }
962 }
963
964 fn decode(code: u8) -> Option<ObjectKind> {
965 match code {
966 1 => Some(ObjectKind::Commit),
967 2 => Some(ObjectKind::Tree),
968 3 => Some(ObjectKind::Blob),
969 4 => Some(ObjectKind::Tag),
970 _ => None,
971 }
972 }
973
974 fn kind(&mut self, pos: usize) -> Result<Option<ObjectKind>> {
977 let corrupt = |what: &str| {
978 Error::CorruptObject(format!("{what} in {}", self.idx.pack_path.display()))
979 };
980 let mut chain: Vec<usize> = Vec::new();
981 let mut cur = pos;
982 let kind = loop {
983 if self.kinds[cur] != 0 {
984 break Self::decode(self.kinds[cur]);
985 }
986 if chain.len() > MAX_CHAIN {
987 return Err(corrupt("delta chain too deep"));
988 }
989 chain.push(cur);
990 let offset = self.idx.offset_at(cur)?;
991 let mut at =
992 usize::try_from(offset).map_err(|_| corrupt("pack offset out of range"))?;
993 let (packed, _) = crate::pack::parse_pack_object_header(self.bytes, &mut at)?;
994 match packed {
995 PackedType::Commit => break Some(ObjectKind::Commit),
996 PackedType::Tree => break Some(ObjectKind::Tree),
997 PackedType::Blob => break Some(ObjectKind::Blob),
998 PackedType::Tag => break Some(ObjectKind::Tag),
999 PackedType::OfsDelta => {
1000 let base = crate::pack::parse_ofs_delta_base(self.bytes, &mut at, offset)?;
1001 cur = self
1002 .idx
1003 .find_position_by_offset_sorted(base)?
1004 .ok_or_else(|| corrupt("delta base offset not in index"))?;
1005 }
1006 PackedType::RefDelta => {
1007 let hash_bytes = self.idx.hash_bytes();
1008 let raw = self
1009 .bytes
1010 .get(at..at + hash_bytes)
1011 .ok_or_else(|| corrupt("truncated delta base id"))?;
1012 let base = ObjectId::from_bytes(raw)?;
1013 match self.idx.find_position(&base) {
1014 Some(base_pos) => cur = base_pos,
1015 None => match self.odb.read_info_without_fetch(&base) {
1016 Ok(info) => break Some(info.kind),
1017 Err(Error::ObjectNotFound(_)) => break None,
1018 Err(err) => return Err(err),
1019 },
1020 }
1021 }
1022 }
1023 };
1024 let code = Self::encode(kind);
1025 for row in chain {
1026 self.kinds[row] = code;
1027 }
1028 Ok(kind)
1029 }
1030}
1031
1032fn promised_by_promisor_packs(
1041 odb: &Odb,
1042 packs: &[PromisorPack],
1043 wanted: &OidHashSet,
1044) -> Result<OidHashSet> {
1045 let mut found = oid_hash_set();
1046 if wanted.is_empty() {
1047 return Ok(found);
1048 }
1049 let oid_len = odb.hash_algo().len();
1050 let mut read: Vec<Vec<bool>> = packs.iter().map(|_| Vec::new()).collect();
1051 let mut referrer = |pack_no: usize,
1052 row: usize,
1053 queue: Option<&mut std::collections::VecDeque<ObjectId>>,
1054 found: &mut OidHashSet|
1055 -> Result<()> {
1056 let seen = &mut read[pack_no];
1057 if seen.is_empty() {
1058 *seen = vec![false; packs[pack_no].idx.len()];
1059 }
1060 if std::mem::replace(&mut seen[row], true) {
1061 return Ok(());
1062 }
1063 let idx = &packs[pack_no].idx;
1064 let obj = crate::pack::read_object_at(idx, idx.offset_at(row)?)?;
1065 match obj.kind {
1066 ObjectKind::Tree => {
1067 let mut queue = queue;
1068 for entry in TreeRefIter::with_oid_len(&obj.data, oid_len) {
1069 let entry = entry?;
1070 match entry.mode & 0o170000 {
1071 0o040000 => {
1072 if let Some(q) = queue.as_deref_mut() {
1073 q.push_back(entry.oid);
1074 }
1075 }
1076 0o160000 => {}
1077 _ if wanted.contains(&entry.oid) => {
1078 found.insert(entry.oid);
1079 }
1080 _ => {}
1081 }
1082 }
1083 }
1084 ObjectKind::Tag => {
1085 if let Some(target) = crate::objects::tag_object_line_oid(&obj.data) {
1086 if wanted.contains(&target) {
1087 found.insert(target);
1088 }
1089 }
1090 }
1091 ObjectKind::Blob | ObjectKind::Commit => {}
1092 }
1093 Ok(())
1094 };
1095 let locate = |oid: &ObjectId| {
1096 packs
1097 .iter()
1098 .enumerate()
1099 .find_map(|(n, p)| p.idx.find_position(oid).map(|row| (n, row)))
1100 };
1101
1102 let mut queue: std::collections::VecDeque<ObjectId> = std::collections::VecDeque::new();
1103 for pack in packs {
1104 for &row in &pack.commits {
1105 let obj = crate::pack::read_object_at(&pack.idx, pack.idx.offset_at(row as usize)?)?;
1106 if obj.kind == ObjectKind::Commit {
1107 queue.push_back(CommitRef::parse(&obj.data)?.tree());
1108 }
1109 }
1110 }
1111 let mut trees_seen = oid_hash_set();
1112 while let Some(tree) = queue.pop_front() {
1113 if found.len() == wanted.len() {
1114 return Ok(found);
1115 }
1116 if !trees_seen.insert(tree) {
1117 continue;
1118 }
1119 if let Some((pack_no, row)) = locate(&tree) {
1122 referrer(pack_no, row, Some(&mut queue), &mut found)?;
1123 }
1124 }
1125 drop(trees_seen);
1126 for (pack_no, pack) in packs.iter().enumerate() {
1127 for &row in &pack.referrers {
1128 if found.len() == wanted.len() {
1129 return Ok(found);
1130 }
1131 referrer(pack_no, row as usize, None, &mut found)?;
1132 }
1133 }
1134 Ok(found)
1135}
1136
1137struct ProtectionWalk<'a> {
1140 odb: &'a Odb,
1141 wanted: &'a OidHashSet,
1142 trees_seen: OidHashSet,
1143 hits: OidHashSet,
1144}
1145
1146impl<'a> ProtectionWalk<'a> {
1147 fn new(odb: &'a Odb, wanted: &'a OidHashSet) -> Self {
1148 Self {
1149 odb,
1150 wanted,
1151 trees_seen: oid_hash_set(),
1152 hits: oid_hash_set(),
1153 }
1154 }
1155
1156 fn mark(&mut self, oid: ObjectId) {
1157 if self.wanted.contains(&oid) {
1158 self.hits.insert(oid);
1159 }
1160 }
1161
1162 fn walk_tree(&mut self, root: ObjectId) -> Result<()> {
1163 let oid_len = self.odb.hash_algo().len();
1164 let mut stack = vec![root];
1165 while let Some(tree) = stack.pop() {
1166 if !self.trees_seen.insert(tree) {
1167 continue;
1168 }
1169 let Some(obj) = self.odb.try_read_once(&tree)? else {
1170 continue;
1171 };
1172 if obj.kind != ObjectKind::Tree {
1173 continue;
1174 }
1175 for entry in TreeRefIter::with_oid_len(&obj.data, oid_len) {
1176 let entry = entry?;
1177 match entry.mode & 0o170000 {
1178 0o040000 => stack.push(entry.oid),
1179 0o160000 => {}
1180 _ => self.mark(entry.oid),
1181 }
1182 }
1183 }
1184 Ok(())
1185 }
1186
1187 fn walk_commit(&mut self, commit: ObjectId) -> Result<()> {
1188 if commit.is_zero() {
1189 return Ok(());
1190 }
1191 let Some(obj) = self.odb.try_read_once(&commit)? else {
1192 return Ok(());
1193 };
1194 if obj.kind != ObjectKind::Commit {
1195 return Ok(());
1196 }
1197 let tree = CommitRef::parse(&obj.data)?.tree();
1198 self.walk_tree(tree)
1199 }
1200
1201 fn mark_checkouts(&mut self, repo: &Repository) -> Result<()> {
1204 use crate::state::HeadState;
1205 for wt in crate::worktree::list_worktrees(repo)? {
1206 if wt.is_bare {
1207 continue;
1208 }
1209 self.mark_index(repo, &wt.admin_dir.join("index"))?;
1210 if let HeadState::Detached { oid } | HeadState::Branch { oid: Some(oid), .. } = wt.head
1211 {
1212 self.walk_commit(oid)?;
1213 }
1214 }
1215 if repo.environment().git_index_file.is_some() {
1217 self.mark_index(repo, &repo.index_path_for_env()?)?;
1218 }
1219 Ok(())
1220 }
1221
1222 fn mark_index(&mut self, repo: &Repository, path: &Path) -> Result<()> {
1225 let index = repo.load_index_at_preserving_sparse(
1226 path,
1227 crate::index::IndexParseContext::from_policy(repo.thread_policy()),
1228 )?;
1229 for entry in index.entries() {
1230 match entry.mode & 0o170000 {
1231 0o040000 => self.walk_tree(entry.oid)?,
1232 0o160000 => {}
1233 _ => self.mark(entry.oid),
1234 }
1235 }
1236 Ok(())
1237 }
1238
1239 fn mark_recent_heads(&mut self, repo: &Repository, cutoff: OffsetDateTime) -> Result<()> {
1241 for wt in crate::worktree::list_worktrees(repo)? {
1242 if wt.is_bare {
1243 continue;
1244 }
1245 for entry in crate::reflog::read_reflog(&wt.admin_dir, "HEAD")? {
1246 if entry.signature().is_some_and(|sig| sig.time >= cutoff) {
1247 self.walk_commit(entry.old_oid)?;
1248 self.walk_commit(entry.new_oid)?;
1249 }
1250 }
1251 }
1252 Ok(())
1253 }
1254}
1255
1256#[allow(clippy::too_many_arguments)]
1259fn rewrite_packs(
1260 repo: &Repository,
1261 config: &ConfigSet,
1262 opts: &DropPromisedBlobsOptions,
1263 packs: &[PromisorPack],
1264 affected: &[u32],
1265 drop_set: &OidHashSet,
1266 report: &mut DropPromisedBlobsReport,
1267) -> Result<()> {
1268 let odb = repo.odb();
1269 let pack_dir = odb.objects_dir().join("pack");
1270 let had_midx =
1271 pack_dir.join("multi-pack-index").is_file() || pack_dir.join("multi-pack-index.d").is_dir();
1272 let env = repo.environment().clone();
1273 let pack_opts = PackBuildOptions::for_local_copy(Some(config));
1274 let mut bytes_before = 0u64;
1275 let mut bytes_after = 0u64;
1276 let mut stems: std::collections::HashSet<std::ffi::OsString> = std::collections::HashSet::new();
1277 for &p in affected {
1278 let pack = &packs[p as usize];
1279 let old = &pack.idx.pack_path;
1280 bytes_before += pack_files_size(old);
1281 let mut keep: Vec<(u64, ObjectId)> = Vec::new();
1282 for entry in pack.idx.iter() {
1283 let oid = ObjectId::from_bytes(entry.oid())?;
1284 if !drop_set.contains(&oid) {
1285 keep.push((entry.offset()?, oid));
1286 }
1287 }
1288 let old_stem = old.file_stem().map(|stem| stem.to_os_string());
1291 if keep.is_empty() {
1292 stems.extend(old_stem);
1293 report.packs_removed += 1;
1294 continue;
1295 }
1296 keep.sort_unstable();
1297 let objects: Vec<ObjectId> = keep.into_iter().map(|(_, oid)| oid).collect();
1298 let built = super::write_staged_repack_pack(
1299 odb,
1300 &pack_dir,
1301 &objects,
1302 &HashMap::new(),
1303 &pack_opts,
1304 opts.threads,
1305 Some(config),
1306 )?;
1307 let Some(staging) = built.staging_path.as_ref() else {
1308 continue;
1309 };
1310 let trusted_rows = (!built.index_rows.is_empty()).then_some(built.index_rows);
1311 let ingested = install_pack_path(
1312 staging,
1313 odb,
1314 &IngestPackOptions {
1315 threads: opts.threads,
1316 merged_config: Some(config.clone()),
1317 environment: env.clone(),
1318 skip_post_index_verify: trusted_rows.is_some(),
1319 trusted_index_rows: trusted_rows,
1320 mark_promisor_pack: true,
1321 ..IngestPackOptions::default()
1322 },
1323 )?;
1324 let Some(new_pack) = ingested.pack_path else {
1325 continue;
1326 };
1327 if let Ok(lines) = std::fs::read(old.with_extension("promisor")) {
1329 std::fs::write(new_pack.with_extension("promisor"), lines)?;
1330 }
1331 if config.pack_write_reverse_index_default(&env) {
1332 crate::pack_rev::write_pack_rev_sidecar_for_index_path(
1333 &new_pack.with_extension("idx"),
1334 )?;
1335 }
1336 if let Some(mtime) = pack.mtime {
1338 filetime::set_file_mtime(&new_pack, filetime::FileTime::from_system_time(mtime))?;
1339 }
1340 crate::index_pack::sync_installed_pack(&new_pack)?;
1341 bytes_after += pack_files_size(&new_pack);
1342 if new_pack.file_stem() != old.file_stem() {
1343 stems.extend(old_stem);
1344 }
1345 report.packs_rewritten += 1;
1346 }
1347 if had_midx {
1348 crate::midx::clear_pack_midx_state(&pack_dir)?;
1349 }
1350 odb.invalidate_packs();
1352 super::remove_pack_files_with_stems(&pack_dir, &stems)?;
1353 crate::pack::clear_pack_cache();
1354 odb.invalidate_packs();
1355 odb.reprepare_local_packs();
1356 let packs_left = odb.local_pack_indexes()?.len();
1357 if had_midx && packs_left > 0 {
1358 super::write_midx(repo, &super::MidxOptions::default())?;
1359 }
1360 report.bytes_reclaimed = bytes_before.saturating_sub(bytes_after);
1361 Ok(())
1362}
1363
1364fn pack_files_size(pack: &Path) -> u64 {
1366 ["pack", "idx", "rev"]
1367 .iter()
1368 .filter_map(|ext| std::fs::metadata(pack.with_extension(ext)).ok())
1369 .map(|m| m.len())
1370 .sum()
1371}
1372
1373#[cfg(test)]
1374#[allow(clippy::unwrap_used, clippy::expect_used)]
1375mod tests {
1376 use super::*;
1377 use crate::config::{ConfigFile, ConfigScope};
1378
1379 fn config(text: &str) -> ConfigSet {
1380 let file = ConfigFile::parse(Path::new("test.config"), text, ConfigScope::Local).unwrap();
1381 let mut set = ConfigSet::new();
1382 set.merge(&file);
1383 set
1384 }
1385
1386 fn now() -> OffsetDateTime {
1387 OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap()
1388 }
1389
1390 #[test]
1391 fn defaults_are_one_mebibyte_and_two_weeks() {
1392 let opts = DropPromisedBlobsOptions::from_config(&config(""), now())
1393 .unwrap()
1394 .unwrap();
1395 assert_eq!(opts.min_size, DEFAULT_DROP_MIN_SIZE);
1396 assert_eq!(opts.keep_newer_than, Some(now() - time::Duration::weeks(2)));
1397 assert_eq!(opts.now, now());
1398 assert!(!opts.dry_run);
1399 }
1400
1401 #[test]
1402 fn min_size_takes_units_and_rejects_junk() {
1403 let min = |v: &str| {
1404 DropPromisedBlobsOptions::from_config(
1405 &config(&format!(
1406 "[maintenance \"drop-large-blobs\"]\n\tminSize = {v}\n"
1407 )),
1408 now(),
1409 )
1410 .map(|o| o.unwrap().min_size)
1411 };
1412 assert_eq!(min("100k").unwrap(), 100 * 1024);
1413 assert_eq!(min("1m").unwrap(), 1 << 20);
1414 assert_eq!(min("0").unwrap(), 0);
1415 for bad in ["lots", "-1"] {
1416 assert!(
1417 matches!(
1418 min(bad),
1419 Err(Error::Config(ConfigError::BadNumericValue { .. }))
1420 ),
1421 "{bad}"
1422 );
1423 }
1424 }
1425
1426 #[test]
1427 fn min_size_follows_the_smallest_promisor_blob_limit() {
1428 let cfg = config(
1429 "[remote \"origin\"]\n\turl = /o\n\tpromisor = true\n\tpartialclonefilter = blob:limit=200k\n\
1430 [remote \"lop\"]\n\turl = /l\n\tpromisor = true\n\tpartialclonefilter = blob:limit=50k\n\
1431 [remote \"none\"]\n\turl = /n\n\tpromisor = true\n\tpartialclonefilter = blob:none\n",
1432 );
1433 let opts = DropPromisedBlobsOptions::from_config(&cfg, now())
1434 .unwrap()
1435 .unwrap();
1436 assert_eq!(opts.min_size, 50 * 1024);
1437 let blobless = config(
1438 "[remote \"origin\"]\n\turl = /o\n\tpromisor = true\n\tpartialclonefilter = blob:none\n",
1439 );
1440 let opts = DropPromisedBlobsOptions::from_config(&blobless, now())
1441 .unwrap()
1442 .unwrap();
1443 assert_eq!(opts.min_size, DEFAULT_DROP_MIN_SIZE);
1444 }
1445
1446 #[test]
1447 fn expire_values() {
1448 let expire = |v: &str| {
1449 DropPromisedBlobsOptions::from_config(
1450 &config(&format!(
1451 "[maintenance \"drop-large-blobs\"]\n\texpire = {v}\n"
1452 )),
1453 now(),
1454 )
1455 };
1456 assert_eq!(expire("now").unwrap().unwrap().keep_newer_than, None);
1457 assert_eq!(expire("all").unwrap().unwrap().keep_newer_than, None);
1458 assert!(expire("never").unwrap().is_none());
1459 assert!(expire("false").unwrap().is_none());
1460 assert_eq!(
1461 expire("3.days.ago").unwrap().unwrap().keep_newer_than,
1462 Some(now() - time::Duration::days(3))
1463 );
1464 assert!(matches!(
1465 expire("not a date at all"),
1466 Err(Error::Config(ConfigError::InvalidDate { key, value }))
1467 if key == EXPIRE_KEY && value == "not a date at all"
1468 ));
1469 }
1470
1471 #[test]
1472 fn expiry_keywords_dates_and_junk() {
1473 assert_eq!(DropExpiry::parse(" Never ", now()), Some(DropExpiry::Never));
1474 assert_eq!(DropExpiry::parse("false", now()), Some(DropExpiry::Never));
1475 assert_eq!(
1476 DropExpiry::parse("NOW", now()),
1477 Some(DropExpiry::Immediately)
1478 );
1479 assert_eq!(
1480 DropExpiry::parse("all", now()),
1481 Some(DropExpiry::Immediately)
1482 );
1483 assert_eq!(
1484 DropExpiry::parse("1.week.ago", now()),
1485 Some(DropExpiry::KeepNewerThan(now() - time::Duration::weeks(1)))
1486 );
1487 for junk in ["", " ", "not a date at all"] {
1488 assert_eq!(DropExpiry::parse(junk, now()), None, "{junk:?}");
1489 }
1490 }
1491
1492 #[test]
1493 fn overrides_replace_only_their_own_key() {
1494 let cfg = config("[maintenance \"drop-large-blobs\"]\n\texpire = never\n\tminSize = 10m\n");
1495 assert!(DropPromisedBlobsOptions::from_config(&cfg, now())
1496 .unwrap()
1497 .is_none());
1498 let opts = DropPromisedBlobsOptions::from_config_with_overrides(
1500 &cfg,
1501 now(),
1502 None,
1503 Some(DropExpiry::Immediately),
1504 )
1505 .unwrap()
1506 .unwrap();
1507 assert_eq!(opts.min_size, 10 << 20);
1508 assert_eq!(opts.keep_newer_than, None);
1509 let opts = DropPromisedBlobsOptions::from_config_with_overrides(
1510 &cfg,
1511 now(),
1512 Some(5),
1513 Some(DropExpiry::KeepNewerThan(now())),
1514 )
1515 .unwrap()
1516 .unwrap();
1517 assert_eq!((opts.min_size, opts.keep_newer_than), (5, Some(now())));
1518 assert!(DropPromisedBlobsOptions::from_config_with_overrides(
1519 &cfg,
1520 now(),
1521 Some(5),
1522 Some(DropExpiry::Never)
1523 )
1524 .unwrap()
1525 .is_none());
1526
1527 let bad =
1529 config("[maintenance \"drop-large-blobs\"]\n\texpire = bogus\n\tminSize = lots\n");
1530 let opts = DropPromisedBlobsOptions::from_config_with_overrides(
1531 &bad,
1532 now(),
1533 Some(7),
1534 Some(DropExpiry::Immediately),
1535 )
1536 .unwrap()
1537 .unwrap();
1538 assert_eq!(opts.min_size, 7);
1539 assert!(matches!(
1540 DropPromisedBlobsOptions::from_config_with_overrides(
1541 &bad,
1542 now(),
1543 None,
1544 Some(DropExpiry::Immediately)
1545 ),
1546 Err(Error::Config(ConfigError::BadNumericValue { .. }))
1547 ));
1548 assert!(matches!(
1549 DropPromisedBlobsOptions::from_config_with_overrides(&bad, now(), Some(7), None),
1550 Err(Error::Config(ConfigError::InvalidDate { .. }))
1551 ));
1552 }
1553
1554 #[test]
1555 fn gc_plan_checks_config_and_needs_a_time() {
1556 let plan = |text: &str, now_unix: i64| plan_drop_for_gc(&config(text), now_unix);
1557 let on = "[maintenance \"drop-large-blobs\"]\n\tenabled = true\n";
1558 assert!(matches!(
1559 plan("", 0).unwrap(),
1560 GcDropPlan::Skip(DropPromisedBlobsSkip::Disabled)
1561 ));
1562 assert!(matches!(
1563 plan(&format!("{on}\texpire = never\n"), 1_700_000_000).unwrap(),
1564 GcDropPlan::Skip(DropPromisedBlobsSkip::Disabled)
1565 ));
1566 let GcDropPlan::Run(opts) = plan(on, 1_700_000_000).unwrap() else {
1567 panic!("runs with a time");
1568 };
1569 assert_eq!(opts.now, now());
1570 for no_time in [0, i64::MAX] {
1572 assert!(matches!(
1573 plan(on, no_time).unwrap(),
1574 GcDropPlan::Skip(DropPromisedBlobsSkip::NoCurrentTime)
1575 ));
1576 }
1577 for (bad, now_unix) in [
1579 ("\texpire = bogus\n", 1_700_000_000),
1580 ("\tminSize = lots\n", 1_700_000_000),
1581 ("\tminSize = lots\n", 0),
1582 ] {
1583 assert!(
1584 matches!(plan(&format!("{on}{bad}"), now_unix), Err(Error::Config(_))),
1585 "{bad}"
1586 );
1587 }
1588 assert!(matches!(
1589 plan("[maintenance \"drop-large-blobs\"]\n\tenabled = maybe\n", 1),
1590 Err(Error::Config(_))
1591 ));
1592 }
1593
1594 #[test]
1595 fn enabled_is_off_by_default_and_must_be_a_boolean() {
1596 assert!(!drop_large_blobs_enabled(&config("")).unwrap());
1597 let on = config("[maintenance \"drop-large-blobs\"]\n\tenabled = true\n");
1598 assert!(drop_large_blobs_enabled(&on).unwrap());
1599 let bad = config("[maintenance \"drop-large-blobs\"]\n\tenabled = sometimes\n");
1600 assert!(matches!(
1601 drop_large_blobs_enabled(&bad),
1602 Err(Error::Config(_))
1603 ));
1604 }
1605
1606 #[test]
1607 fn kept_counts_total() {
1608 let kept = KeptBlobCounts {
1609 kept_pack: 1,
1610 outside_promisor_packs: 2,
1611 checked_out: 3,
1612 recently_checked_out: 4,
1613 recently_fetched: 5,
1614 not_promised_elsewhere: 6,
1615 not_in_promisor: 7,
1616 unverified: 8,
1617 };
1618 assert_eq!(kept.total(), 36);
1619 }
1620}