1#![deny(unsafe_code)]
2#![cfg_attr(
3 test,
4 allow(
5 clippy::unwrap_used,
6 clippy::expect_used,
7 clippy::indexing_slicing,
8 clippy::arithmetic_side_effects,
9 clippy::shadow_unrelated,
10 clippy::let_underscore_must_use,
11 clippy::format_push_string
12 )
13)]
14
15use std::{
22 collections::{HashMap, HashSet},
23 io::Error as IoError,
24 num::TryFromIntError,
25 path::{Path, PathBuf},
26};
27
28use serde::{Deserialize, Serialize};
29use shardline_index::{
30 AsyncIndexStore, FileRecordInvariantError, LocalIndexStore, LocalIndexStoreError,
31 MemoryIndexStoreError, MemoryRecordStoreError, PostgresMetadataStoreError, QuarantineCandidate,
32 QuarantineCandidateError, RecordStore, RetentionHoldError, WebhookDeliveryError,
33};
34use shardline_protocol::unix_now_seconds_lossy;
35use shardline_server_core::{
36 InvalidLifecycleMetadataError, ServerObjectStore, ServerObjectStoreError,
37 server_frontend::ServerFrontend,
38};
39use shardline_storage::{
40 LocalObjectStoreError, ObjectPrefixError, ObjectStore, S3ObjectStoreError,
41};
42use shardline_xet_adapter::XetAdapterError;
43use thiserror::Error;
44
45mod dispatch;
46mod quarantine;
47mod reachability;
48
49use quarantine::{
50 read_active_retention_hold_object_keys, read_quarantine_entries, reconcile_quarantine_entries,
51 sweep_quarantine_entries,
52};
53use reachability::{
54 OrphanObject, ReachabilityAccumulator, collect_referenced_object_keys,
55 managed_object_hash_or_object_key, scan_orphan_objects,
56};
57
58pub use shardline_server_core::server_frontend::{
59 ServerFrontend as ServerFrontendKind, ServerFrontendParseError,
60};
61
62#[derive(Debug, Error)]
64pub enum GcError {
65 #[error("local storage operation failed")]
67 Io(#[from] IoError),
68 #[error("json operation failed")]
70 Json(#[from] serde_json::Error),
71 #[error("numeric conversion exceeded supported bounds")]
73 NumericConversion(#[from] TryFromIntError),
74 #[error("object storage adapter operation failed")]
76 ObjectStore(#[from] ServerObjectStoreError),
77 #[error("local object storage operation failed")]
79 LocalObjectStore(#[from] LocalObjectStoreError),
80 #[error("s3 object storage operation failed")]
82 S3ObjectStore(#[from] S3ObjectStoreError),
83 #[error("object storage prefix validation failed")]
85 ObjectPrefix(#[from] ObjectPrefixError),
86 #[error("index adapter operation failed")]
88 IndexStore(#[from] LocalIndexStoreError),
89 #[error("memory index adapter operation failed")]
91 MemoryIndexStore(#[from] MemoryIndexStoreError),
92 #[error("memory record adapter operation failed")]
94 MemoryRecordStore(#[from] MemoryRecordStoreError),
95 #[error("postgres metadata adapter operation failed")]
97 PostgresMetadata(#[from] PostgresMetadataStoreError),
98 #[error("retention hold input was invalid")]
100 RetentionHold(#[from] RetentionHoldError),
101 #[error("quarantine candidate input was invalid")]
103 QuarantineCandidate(#[from] QuarantineCandidateError),
104 #[error("webhook delivery metadata was invalid")]
106 WebhookDelivery(#[from] WebhookDeliveryError),
107 #[error("stored file metadata was invalid")]
109 FileRecordInvariant(#[from] FileRecordInvariantError),
110 #[error("lifecycle metadata was internally inconsistent")]
112 InvalidLifecycleMetadata(#[from] InvalidLifecycleMetadataError),
113 #[error("content hash must be 64 hexadecimal characters")]
115 InvalidContentHash,
116 #[error("arithmetic overflow")]
118 Overflow,
119 #[error("xet adapter operation failed")]
121 XetAdapter(#[from] XetAdapterError),
122}
123
124impl From<shardline_server_core::ParseStoredFileRecordError> for GcError {
125 fn from(err: shardline_server_core::ParseStoredFileRecordError) -> Self {
126 match err {
127 shardline_server_core::ParseStoredFileRecordError::StoredFileMetadataTooLarge {
128 observed_bytes,
129 maximum_bytes,
130 } => Self::Io(IoError::new(
131 std::io::ErrorKind::InvalidData,
132 format!(
133 "stored file metadata exceeded the bounded parser ceiling: {observed_bytes} > {maximum_bytes}"
134 ),
135 )),
136 shardline_server_core::ParseStoredFileRecordError::Json(e) => Self::Json(e),
137 }
138 }
139}
140
141impl From<shardline_server_core::RebuildOverflowError> for GcError {
142 fn from(_: shardline_server_core::RebuildOverflowError) -> Self {
143 Self::Overflow
144 }
145}
146
147impl From<GcError> for shardline_server_core::ServerObjectStoreError {
148 fn from(err: GcError) -> Self {
149 match err {
150 GcError::ObjectStore(e) => e,
151 GcError::LocalObjectStore(e) => Self::Local(e),
152 GcError::S3ObjectStore(e) => Self::S3(e),
153 GcError::Io(e) => Self::Io(e),
154 GcError::NumericConversion(e) => Self::NumericConversion(e),
155 GcError::Json(_)
156 | GcError::ObjectPrefix(_)
157 | GcError::IndexStore(_)
158 | GcError::MemoryIndexStore(_)
159 | GcError::MemoryRecordStore(_)
160 | GcError::PostgresMetadata(_)
161 | GcError::RetentionHold(_)
162 | GcError::QuarantineCandidate(_)
163 | GcError::WebhookDelivery(_)
164 | GcError::FileRecordInvariant(_)
165 | GcError::InvalidLifecycleMetadata(_)
166 | GcError::InvalidContentHash
167 | GcError::Overflow
168 | GcError::XetAdapter(_) => Self::Overflow,
169 }
170 }
171}
172
173pub use shardline_server_core::DEFAULT_LOCAL_GC_RETENTION_SECONDS;
175
176#[derive(Debug, Clone, Copy, PartialEq, Eq)]
178pub struct LocalGcOptions {
179 pub mark: bool,
181 pub sweep: bool,
183 pub retention_seconds: u64,
185}
186
187impl Default for LocalGcOptions {
188 fn default() -> Self {
189 Self {
190 mark: false,
191 sweep: false,
192 retention_seconds: DEFAULT_LOCAL_GC_RETENTION_SECONDS,
193 }
194 }
195}
196
197impl LocalGcOptions {
198 #[must_use]
200 pub const fn dry_run() -> Self {
201 Self {
202 mark: false,
203 sweep: false,
204 retention_seconds: DEFAULT_LOCAL_GC_RETENTION_SECONDS,
205 }
206 }
207
208 #[must_use]
210 pub const fn mark_only(retention_seconds: u64) -> Self {
211 Self {
212 mark: true,
213 sweep: false,
214 retention_seconds,
215 }
216 }
217
218 #[must_use]
220 pub const fn sweep_only() -> Self {
221 Self {
222 mark: false,
223 sweep: true,
224 retention_seconds: DEFAULT_LOCAL_GC_RETENTION_SECONDS,
225 }
226 }
227
228 #[must_use]
230 pub const fn mark_and_sweep(retention_seconds: u64) -> Self {
231 Self {
232 mark: true,
233 sweep: true,
234 retention_seconds,
235 }
236 }
237
238 #[must_use]
240 pub const fn mode_name(&self) -> &'static str {
241 match (self.mark, self.sweep) {
242 (false, false) => "dry-run",
243 (true, false) => "mark",
244 (false, true) => "sweep",
245 (true, true) => "mark-and-sweep",
246 }
247 }
248}
249
250#[derive(Debug, Clone, PartialEq, Eq)]
252pub struct LocalGcReport {
253 pub scanned_records: u64,
255 pub referenced_chunks: u64,
257 pub orphan_chunks: u64,
259 pub orphan_chunk_bytes: u64,
261 pub active_quarantine_candidates: u64,
263 pub new_quarantine_candidates: u64,
265 pub retained_quarantine_candidates: u64,
267 pub released_quarantine_candidates: u64,
270 pub deleted_chunks: u64,
272 pub deleted_bytes: u64,
274}
275
276#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
278pub struct GcRetentionReportEntry {
279 pub hash: String,
281 pub object_key: String,
283 pub observed_length: u64,
285 pub first_seen_unreachable_at_unix_seconds: u64,
287 pub delete_after_unix_seconds: u64,
289 pub expired: bool,
291 pub seconds_until_delete: u64,
293}
294
295#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
297pub struct GcOrphanInventoryEntry {
298 pub hash: String,
300 pub object_key: String,
302 pub bytes: u64,
304 pub quarantine_state: GcOrphanQuarantineState,
306 pub first_seen_unreachable_at_unix_seconds: Option<u64>,
308 pub delete_after_unix_seconds: Option<u64>,
310}
311
312#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
314#[serde(rename_all = "snake_case")]
315pub enum GcOrphanQuarantineState {
316 Untracked,
318 Quarantined,
320}
321
322#[derive(Debug, Clone, PartialEq, Eq)]
324pub struct LocalGcDiagnostics {
325 pub report: LocalGcReport,
327 pub retention_report: Vec<GcRetentionReportEntry>,
329 pub orphan_inventory: Vec<GcOrphanInventoryEntry>,
331}
332
333pub async fn run_local_gc(
340 root: PathBuf,
341 options: LocalGcOptions,
342) -> Result<LocalGcReport, GcError> {
343 Ok(run_local_gc_diagnostics(root, options).await?.report)
344}
345
346pub async fn run_local_gc_diagnostics(
353 root: PathBuf,
354 options: LocalGcOptions,
355) -> Result<LocalGcDiagnostics, GcError> {
356 let start = std::time::Instant::now();
357 let object_store = ServerObjectStore::local(root.join("chunks"))?;
358 let index_store = LocalIndexStore::open(root.clone());
359 let record_store = LocalRecordStore::open(root);
360 let result = run_gc_with_stores(
361 &record_store,
362 &index_store,
363 &object_store,
364 &[ServerFrontend::Xet],
365 options,
366 )
367 .await;
368 if let Ok(ref diagnostics) = result {
369 let elapsed = start.elapsed();
370 shardline_metrics::record_gc_run(
371 elapsed,
372 diagnostics.report.deleted_chunks,
373 diagnostics.report.deleted_bytes,
374 );
375 }
376 result
377}
378
379pub async fn run_gc_with_stores<RecordAdapter, IndexAdapter>(
386 record_store: &RecordAdapter,
387 index_store: &IndexAdapter,
388 object_store: &ServerObjectStore,
389 frontends: &[ServerFrontend],
390 options: LocalGcOptions,
391) -> Result<LocalGcDiagnostics, GcError>
392where
393 RecordAdapter: RecordStore + Sync,
394 RecordAdapter::Error: Into<GcError>,
395 IndexAdapter: AsyncIndexStore + Sync,
396 IndexAdapter::Error: Into<GcError>,
397{
398 let mut reachability = ReachabilityAccumulator::default();
399 let now_unix_seconds = unix_now_seconds_lossy();
400
401 collect_referenced_object_keys(
402 record_store,
403 index_store,
404 object_store,
405 frontends,
406 &mut reachability,
407 )
408 .await?;
409 validate_gc_index_integrity(index_store, object_store, now_unix_seconds).await?;
410
411 let prune_expired_retention_holds = options.mark || options.sweep;
412 let active_retention_hold_object_keys = read_active_retention_hold_object_keys(
413 index_store,
414 now_unix_seconds,
415 prune_expired_retention_holds,
416 )
417 .await?;
418 let mut orphan_objects = scan_orphan_objects(
419 object_store,
420 frontends,
421 &reachability.referenced_object_keys,
422 )?;
423 orphan_objects
424 .retain(|object_key, _orphan| !active_retention_hold_object_keys.contains(object_key));
425 let orphan_chunk_bytes = orphan_objects.values().try_fold(0_u64, |total, orphan| {
426 shardline_server_core::checked_add(total, orphan.bytes)
427 })?;
428
429 let mut quarantine_entries = read_quarantine_entries(index_store).await?;
430
431 let mut report = LocalGcReport {
432 scanned_records: reachability.scanned_records,
433 referenced_chunks: u64::try_from(reachability.referenced_object_keys.len())?,
434 orphan_chunks: u64::try_from(orphan_objects.len())?,
435 orphan_chunk_bytes,
436 active_quarantine_candidates: 0,
437 new_quarantine_candidates: 0,
438 retained_quarantine_candidates: 0,
439 released_quarantine_candidates: 0,
440 deleted_chunks: 0,
441 deleted_bytes: 0,
442 };
443
444 if options.mark {
445 reconcile_quarantine_entries(
446 index_store,
447 &orphan_objects,
448 now_unix_seconds,
449 options.retention_seconds,
450 &mut quarantine_entries,
451 &mut report,
452 )
453 .await?;
454 }
455
456 if options.sweep {
457 sweep_quarantine_entries(
458 object_store,
459 index_store,
460 &orphan_objects,
461 now_unix_seconds,
462 &mut quarantine_entries,
463 &mut report,
464 )
465 .await?;
466 }
467
468 report.active_quarantine_candidates = u64::try_from(quarantine_entries.len())?;
469 Ok(build_gc_diagnostics(
470 report,
471 frontends,
472 &orphan_objects,
473 &quarantine_entries,
474 now_unix_seconds,
475 ))
476}
477
478async fn validate_gc_index_integrity<IndexAdapter>(
479 index_store: &IndexAdapter,
480 object_store: &ServerObjectStore,
481 now_unix_seconds: u64,
482) -> Result<(), GcError>
483where
484 IndexAdapter: AsyncIndexStore + Sync,
485 IndexAdapter::Error: Into<GcError>,
486{
487 let mut quarantined_object_keys = HashSet::new();
488
489 index_store
490 .visit_quarantine_candidates(|candidate| {
491 if candidate.delete_after_unix_seconds()
492 < candidate.first_seen_unreachable_at_unix_seconds()
493 {
494 return Err(
495 InvalidLifecycleMetadataError::QuarantineCandidateDeleteBeforeFirstSeen {
496 object_key: candidate.object_key().as_str().to_owned(),
497 delete_after_unix_seconds: candidate.delete_after_unix_seconds(),
498 first_seen_unreachable_at_unix_seconds: candidate
499 .first_seen_unreachable_at_unix_seconds(),
500 }
501 .into(),
502 );
503 }
504
505 let Some(metadata) = object_store.metadata(candidate.object_key())? else {
506 return Err(
507 InvalidLifecycleMetadataError::QuarantineCandidateMissingObject {
508 object_key: candidate.object_key().as_str().to_owned(),
509 }
510 .into(),
511 );
512 };
513 if metadata.length() != candidate.observed_length() {
514 return Err(
515 InvalidLifecycleMetadataError::QuarantineCandidateLengthMismatch {
516 object_key: candidate.object_key().as_str().to_owned(),
517 expected_length: candidate.observed_length(),
518 observed_length: metadata.length(),
519 }
520 .into(),
521 );
522 }
523
524 quarantined_object_keys.insert(candidate.object_key().as_str().to_owned());
525 Ok::<(), GcError>(())
526 })
527 .await?;
528
529 index_store
530 .visit_retention_holds(|hold| {
531 if let Some(release_after_unix_seconds) = hold.release_after_unix_seconds()
532 && release_after_unix_seconds < hold.held_at_unix_seconds()
533 {
534 return Err(
535 InvalidLifecycleMetadataError::RetentionHoldReleaseBeforeHeld {
536 object_key: hold.object_key().as_str().to_owned(),
537 release_after_unix_seconds,
538 held_at_unix_seconds: hold.held_at_unix_seconds(),
539 }
540 .into(),
541 );
542 }
543
544 if hold.is_active_at(now_unix_seconds) {
545 if object_store.metadata(hold.object_key())?.is_none() {
546 return Err(
547 InvalidLifecycleMetadataError::ActiveRetentionHoldMissingObject {
548 object_key: hold.object_key().as_str().to_owned(),
549 }
550 .into(),
551 );
552 }
553 if quarantined_object_keys.contains(hold.object_key().as_str()) {
554 return Err(
555 InvalidLifecycleMetadataError::ActiveRetentionHoldQuarantined {
556 object_key: hold.object_key().as_str().to_owned(),
557 }
558 .into(),
559 );
560 }
561 }
562
563 Ok::<(), GcError>(())
564 })
565 .await?;
566
567 index_store
568 .visit_webhook_deliveries(|_delivery| Ok::<(), GcError>(()))
569 .await?;
570 index_store
571 .visit_provider_repository_states(|_state| Ok::<(), GcError>(()))
572 .await?;
573
574 Ok(())
575}
576
577#[must_use]
579pub fn quarantine_root(root: &Path) -> PathBuf {
580 root.join("gc").join("quarantine")
581}
582
583#[must_use]
585pub fn quarantine_record_path(root: &Path, hash: &str) -> PathBuf {
586 let prefix = hash.chars().take(2).collect::<String>();
587 root.join(prefix).join(format!("{hash}.json"))
588}
589
590fn build_gc_diagnostics(
591 report: LocalGcReport,
592 frontends: &[ServerFrontend],
593 orphan_objects: &HashMap<String, OrphanObject>,
594 quarantine_entries: &HashMap<String, QuarantineCandidate>,
595 now_unix_seconds: u64,
596) -> LocalGcDiagnostics {
597 let mut retention_report = quarantine_entries
598 .values()
599 .map(|candidate| retention_report_entry(candidate, frontends, now_unix_seconds))
600 .collect::<Vec<_>>();
601 retention_report.sort_by(|left, right| {
602 left.delete_after_unix_seconds
603 .cmp(&right.delete_after_unix_seconds)
604 .then_with(|| left.object_key.cmp(&right.object_key))
605 });
606
607 let mut orphan_inventory = orphan_objects
608 .iter()
609 .map(|(object_key, orphan)| {
610 orphan_inventory_entry(orphan, quarantine_entries.get(object_key))
611 })
612 .collect::<Vec<_>>();
613 orphan_inventory.sort_by(|left, right| left.object_key.cmp(&right.object_key));
614
615 LocalGcDiagnostics {
616 report,
617 retention_report,
618 orphan_inventory,
619 }
620}
621
622fn retention_report_entry(
623 candidate: &QuarantineCandidate,
624 frontends: &[ServerFrontend],
625 now_unix_seconds: u64,
626) -> GcRetentionReportEntry {
627 let seconds_until_delete = candidate
628 .delete_after_unix_seconds()
629 .saturating_sub(now_unix_seconds);
630 GcRetentionReportEntry {
631 hash: managed_object_hash_or_object_key(candidate.object_key(), frontends),
632 object_key: candidate.object_key().as_str().to_owned(),
633 observed_length: candidate.observed_length(),
634 first_seen_unreachable_at_unix_seconds: candidate.first_seen_unreachable_at_unix_seconds(),
635 delete_after_unix_seconds: candidate.delete_after_unix_seconds(),
636 expired: candidate.delete_after_unix_seconds() <= now_unix_seconds,
637 seconds_until_delete,
638 }
639}
640
641fn orphan_inventory_entry(
642 orphan: &OrphanObject,
643 candidate: Option<&QuarantineCandidate>,
644) -> GcOrphanInventoryEntry {
645 let object_key = orphan.object_key.as_str().to_owned();
646 match candidate {
647 Some(candidate) => GcOrphanInventoryEntry {
648 hash: orphan.hash.clone(),
649 object_key,
650 bytes: orphan.bytes,
651 quarantine_state: GcOrphanQuarantineState::Quarantined,
652 first_seen_unreachable_at_unix_seconds: Some(
653 candidate.first_seen_unreachable_at_unix_seconds(),
654 ),
655 delete_after_unix_seconds: Some(candidate.delete_after_unix_seconds()),
656 },
657 None => GcOrphanInventoryEntry {
658 hash: orphan.hash.clone(),
659 object_key,
660 bytes: orphan.bytes,
661 quarantine_state: GcOrphanQuarantineState::Untracked,
662 first_seen_unreachable_at_unix_seconds: None,
663 delete_after_unix_seconds: None,
664 },
665 }
666}
667
668pub use shardline_index::LocalRecordStore;
670
671#[cfg(test)]
672mod tests {
673 use super::*;
674
675 #[test]
676 fn default_retention_seconds_value() {
677 assert_eq!(DEFAULT_LOCAL_GC_RETENTION_SECONDS, 86_400);
678 }
679
680 #[test]
681 fn default_gc_options_are_dry_run() {
682 let opts = LocalGcOptions::default();
683 assert!(!opts.mark);
684 assert!(!opts.sweep);
685 assert_eq!(opts.retention_seconds, DEFAULT_LOCAL_GC_RETENTION_SECONDS);
686 }
687
688 #[test]
689 fn dry_run_options() {
690 let opts = LocalGcOptions::dry_run();
691 assert!(!opts.mark);
692 assert!(!opts.sweep);
693 assert_eq!(opts.retention_seconds, DEFAULT_LOCAL_GC_RETENTION_SECONDS);
694 }
695
696 #[test]
697 fn mark_only_options() {
698 let opts = LocalGcOptions::mark_only(3600);
699 assert!(opts.mark);
700 assert!(!opts.sweep);
701 assert_eq!(opts.retention_seconds, 3600);
702 }
703
704 #[test]
705 fn sweep_only_options() {
706 let opts = LocalGcOptions::sweep_only();
707 assert!(!opts.mark);
708 assert!(opts.sweep);
709 assert_eq!(opts.retention_seconds, DEFAULT_LOCAL_GC_RETENTION_SECONDS);
710 }
711
712 #[test]
713 fn mark_and_sweep_options() {
714 let opts = LocalGcOptions::mark_and_sweep(7200);
715 assert!(opts.mark);
716 assert!(opts.sweep);
717 assert_eq!(opts.retention_seconds, 7200);
718 }
719
720 #[test]
721 fn mode_name_dry_run() {
722 assert_eq!(LocalGcOptions::dry_run().mode_name(), "dry-run");
723 }
724
725 #[test]
726 fn mode_name_mark() {
727 assert_eq!(LocalGcOptions::mark_only(100).mode_name(), "mark");
728 }
729
730 #[test]
731 fn mode_name_sweep() {
732 assert_eq!(LocalGcOptions::sweep_only().mode_name(), "sweep");
733 }
734
735 #[test]
736 fn mode_name_mark_and_sweep() {
737 assert_eq!(
738 LocalGcOptions::mark_and_sweep(100).mode_name(),
739 "mark-and-sweep"
740 );
741 }
742}