1use std::any::Any;
16
17use crate::storage_api::DeletedObject;
18use crate::{
19 DeletedObjectReplicationInfo, MrfReplicateEntry, REPLICATE_EXISTING, REPLICATE_HEAL, REPLICATE_HEAL_DELETE,
20 ReplicateObjectInfo, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision,
21 VersionPurgeStatusType,
22};
23
24#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
25pub enum ReplicationQueueAdmission {
26 #[default]
27 Skipped,
28 Queued,
29 Missed,
30}
31
32impl ReplicationQueueAdmission {
33 pub fn merge(&mut self, other: Self) {
34 *self = match (*self, other) {
35 (Self::Missed, _) | (_, Self::Missed) => Self::Missed,
36 (Self::Queued, _) | (_, Self::Queued) => Self::Queued,
37 (Self::Skipped, Self::Skipped) => Self::Skipped,
38 };
39 }
40}
41
42#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
43pub struct ReplicationBatchAdmission {
44 pub total: usize,
45 pub queued: usize,
46 pub missed: usize,
47}
48
49impl ReplicationBatchAdmission {
50 pub fn record(&mut self, admission: ReplicationQueueAdmission) {
51 self.total += 1;
52 match admission {
53 ReplicationQueueAdmission::Queued => self.queued += 1,
54 ReplicationQueueAdmission::Missed | ReplicationQueueAdmission::Skipped => self.missed += 1,
55 }
56 }
57
58 pub fn outcome(self) -> &'static str {
59 match (self.queued, self.missed) {
60 (0, 0) => "empty",
61 (_, 0) => "all_queued",
62 (0, _) => "all_missed",
63 _ => "partial",
64 }
65 }
66}
67
68pub fn mrf_save_admission(saved: bool) -> ReplicationQueueAdmission {
69 if saved {
70 ReplicationQueueAdmission::Queued
71 } else {
72 ReplicationQueueAdmission::Missed
73 }
74}
75
76#[derive(Debug, Clone, Copy, PartialEq, Eq)]
77pub enum ReplicationWorkerQueue {
78 Regular,
79 Mrf,
80}
81
82pub fn worker_queue_for_replication_type(op_type: &ReplicationType) -> ReplicationWorkerQueue {
83 match op_type {
84 ReplicationType::Heal | ReplicationType::ExistingObject => ReplicationWorkerQueue::Mrf,
85 _ => ReplicationWorkerQueue::Regular,
86 }
87}
88
89#[derive(Debug, Clone, Default)]
90pub struct ReplicationHealQueueResult {
91 pub object_info: ReplicateObjectInfo,
92 pub admission: ReplicationQueueAdmission,
93}
94
95#[derive(Debug, Clone, Default)]
96pub enum ReplicationHealQueueAction {
97 #[default]
98 Skip,
99 QueueObject,
100 QueueDelete(DeletedObjectReplicationInfo),
101 QueueResyncDeletes(ReplicationHealResyncDeletes),
102}
103
104#[derive(Debug, Clone, Default)]
105pub struct ReplicationHealResyncDeletes {
106 pub delete_info: DeletedObjectReplicationInfo,
107 pub existing_obj_resync: ResyncDecision,
108}
109
110impl ReplicationHealResyncDeletes {
111 pub fn target_delete_infos(&self) -> impl Iterator<Item = DeletedObjectReplicationInfo> + '_ {
112 self.existing_obj_resync
113 .targets
114 .iter()
115 .filter(|(_, target)| target.replicate)
116 .map(|(target_arn, target)| {
117 let mut delete_info = self.delete_info.clone();
118 delete_info.reset_id = target.reset_id.clone();
119 delete_info.target_arn = target_arn.clone();
120 delete_info
121 })
122 }
123}
124
125pub fn replication_heal_queue_action(roi: &mut ReplicateObjectInfo) -> ReplicationHealQueueAction {
126 if !roi.dsc.replicate_any() {
127 return ReplicationHealQueueAction::Skip;
128 }
129
130 if roi.replication_status == ReplicationStatusType::Completed
131 && roi.version_purge_status.is_empty()
132 && !roi.existing_obj_resync.must_resync()
133 {
134 return ReplicationHealQueueAction::Skip;
135 }
136
137 if roi.delete_marker || !roi.version_purge_status.is_empty() {
138 let delete_info = heal_deleted_object_replication_info(roi);
139
140 if is_pending_or_failed_object_heal(roi) || is_pending_or_failed_version_purge(roi) {
141 return ReplicationHealQueueAction::QueueDelete(delete_info);
142 }
143
144 if roi.existing_obj_resync.must_resync()
145 && (roi.replication_status == ReplicationStatusType::Completed || roi.replication_status.is_empty())
146 {
147 return ReplicationHealQueueAction::QueueResyncDeletes(ReplicationHealResyncDeletes {
148 delete_info,
149 existing_obj_resync: roi.existing_obj_resync.clone(),
150 });
151 }
152
153 return ReplicationHealQueueAction::Skip;
154 }
155
156 if roi.existing_obj_resync.must_resync() {
157 roi.op_type = ReplicationType::ExistingObject;
158 }
159
160 if is_pending_or_failed_object_heal(roi) {
161 roi.event_type = REPLICATE_HEAL.to_string();
162 return ReplicationHealQueueAction::QueueObject;
163 }
164
165 if roi.existing_obj_resync.must_resync() {
166 roi.event_type = REPLICATE_EXISTING.to_string();
167 return ReplicationHealQueueAction::QueueObject;
168 }
169
170 ReplicationHealQueueAction::Skip
171}
172
173fn heal_deleted_object_replication_info(roi: &ReplicateObjectInfo) -> DeletedObjectReplicationInfo {
174 let (version_id, delete_marker_version_id) = if roi.version_purge_status.is_empty() {
175 (None, roi.version_id)
176 } else {
177 (roi.version_id, None)
178 };
179
180 DeletedObjectReplicationInfo {
181 delete_object: DeletedObject {
182 object_name: roi.name.clone(),
183 delete_marker_version_id,
184 version_id,
185 replication_state: roi.replication_state.clone(),
186 delete_marker_mtime: roi.mod_time,
187 delete_marker: roi.delete_marker,
188 ..Default::default()
189 },
190 bucket: roi.bucket.clone(),
191 op_type: ReplicationType::Heal,
192 event_type: REPLICATE_HEAL_DELETE.to_string(),
193 ..Default::default()
194 }
195}
196
197fn is_pending_or_failed_object_heal(roi: &ReplicateObjectInfo) -> bool {
198 matches!(roi.replication_status, ReplicationStatusType::Pending | ReplicationStatusType::Failed)
199}
200
201fn is_pending_or_failed_version_purge(roi: &ReplicateObjectInfo) -> bool {
202 matches!(roi.version_purge_status, VersionPurgeStatusType::Pending | VersionPurgeStatusType::Failed)
203}
204
205#[derive(Debug, Clone, PartialEq)]
206pub enum ReplicationPriority {
207 Fast,
208 Slow,
209 Auto,
210}
211
212impl std::str::FromStr for ReplicationPriority {
213 type Err = ();
214
215 fn from_str(s: &str) -> Result<Self, Self::Err> {
216 match s {
217 "fast" => Ok(ReplicationPriority::Fast),
218 "slow" => Ok(ReplicationPriority::Slow),
219 "auto" => Ok(ReplicationPriority::Auto),
220 _ => Ok(ReplicationPriority::Auto),
221 }
222 }
223}
224
225impl ReplicationPriority {
226 pub fn as_str(&self) -> &'static str {
227 match self {
228 ReplicationPriority::Fast => "fast",
229 ReplicationPriority::Slow => "slow",
230 ReplicationPriority::Auto => "auto",
231 }
232 }
233}
234
235#[derive(Debug, Clone)]
236pub enum ReplicationOperation {
237 Object(Box<ReplicateObjectInfo>),
238 Delete(Box<DeletedObjectReplicationInfo>),
239}
240
241impl ReplicationWorkerOperation for ReplicationOperation {
242 fn as_any(&self) -> &dyn Any {
243 self
244 }
245
246 fn to_mrf_entry(&self) -> MrfReplicateEntry {
247 match self {
248 ReplicationOperation::Object(obj) => obj.to_mrf_entry(),
249 ReplicationOperation::Delete(del) => del.to_mrf_entry(),
250 }
251 }
252
253 fn get_bucket(&self) -> &str {
254 match self {
255 ReplicationOperation::Object(obj) => obj.get_bucket(),
256 ReplicationOperation::Delete(del) => del.get_bucket(),
257 }
258 }
259
260 fn get_object(&self) -> &str {
261 match self {
262 ReplicationOperation::Object(obj) => obj.get_object(),
263 ReplicationOperation::Delete(del) => del.get_object(),
264 }
265 }
266
267 fn get_size(&self) -> i64 {
268 match self {
269 ReplicationOperation::Object(obj) => obj.get_size(),
270 ReplicationOperation::Delete(del) => del.get_size(),
271 }
272 }
273
274 fn is_delete_marker(&self) -> bool {
275 match self {
276 ReplicationOperation::Object(obj) => obj.is_delete_marker(),
277 ReplicationOperation::Delete(del) => del.is_delete_marker(),
278 }
279 }
280
281 fn get_op_type(&self) -> ReplicationType {
282 match self {
283 ReplicationOperation::Object(obj) => obj.get_op_type(),
284 ReplicationOperation::Delete(del) => del.get_op_type(),
285 }
286 }
287}
288
289#[cfg(test)]
290mod tests {
291 use std::str::FromStr;
292
293 use uuid::Uuid;
294
295 use super::{
296 ReplicationBatchAdmission, ReplicationHealQueueAction, ReplicationOperation, ReplicationPriority,
297 ReplicationQueueAdmission, ReplicationWorkerQueue,
298 };
299 use crate::storage_api::DeletedObject;
300 use crate::{
301 DeletedObjectReplicationInfo, REPLICATE_EXISTING, REPLICATE_HEAL, REPLICATE_HEAL_DELETE, ReplicateDecision,
302 ReplicateObjectInfo, ReplicateTargetDecision, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation,
303 ResyncTargetDecision, VersionPurgeStatusType, mrf_save_admission, replication_heal_queue_action,
304 worker_queue_for_replication_type,
305 };
306
307 #[test]
308 fn replication_queue_admission_combines_target_results() {
309 let mut admission = ReplicationQueueAdmission::Skipped;
310
311 admission.merge(ReplicationQueueAdmission::Queued);
312 assert_eq!(admission, ReplicationQueueAdmission::Queued);
313
314 admission.merge(ReplicationQueueAdmission::Missed);
315 assert_eq!(admission, ReplicationQueueAdmission::Missed);
316
317 admission.merge(ReplicationQueueAdmission::Skipped);
318 assert_eq!(admission, ReplicationQueueAdmission::Missed);
319 }
320
321 #[test]
322 fn mrf_save_admission_maps_send_result() {
323 assert_eq!(mrf_save_admission(true), ReplicationQueueAdmission::Queued);
324 assert_eq!(mrf_save_admission(false), ReplicationQueueAdmission::Missed);
325 }
326
327 #[test]
328 fn replication_batch_admission_classifies_all_partial_and_empty_batches() {
329 let mut all = ReplicationBatchAdmission::default();
330 all.record(ReplicationQueueAdmission::Queued);
331 assert_eq!(all.outcome(), "all_queued");
332
333 let mut partial = ReplicationBatchAdmission::default();
334 partial.record(ReplicationQueueAdmission::Queued);
335 partial.record(ReplicationQueueAdmission::Missed);
336 assert_eq!(partial.outcome(), "partial");
337
338 let mut all_missed = ReplicationBatchAdmission::default();
339 all_missed.record(ReplicationQueueAdmission::Missed);
340 assert_eq!(all_missed.outcome(), "all_missed");
341
342 let empty = ReplicationBatchAdmission::default();
343 assert_eq!(empty.outcome(), "empty");
344 }
345
346 #[test]
347 fn worker_queue_routes_heal_and_existing_objects_to_mrf() {
348 assert_eq!(worker_queue_for_replication_type(&ReplicationType::Heal), ReplicationWorkerQueue::Mrf);
349 assert_eq!(
350 worker_queue_for_replication_type(&ReplicationType::ExistingObject),
351 ReplicationWorkerQueue::Mrf
352 );
353 assert_eq!(
354 worker_queue_for_replication_type(&ReplicationType::Object),
355 ReplicationWorkerQueue::Regular
356 );
357 assert_eq!(
358 worker_queue_for_replication_type(&ReplicationType::Delete),
359 ReplicationWorkerQueue::Regular
360 );
361 }
362
363 #[test]
370 fn heal_queue_action_skips_when_no_target_replicates() {
371 for status in [
372 ReplicationStatusType::Empty,
373 ReplicationStatusType::Failed,
374 ReplicationStatusType::Replica,
375 ] {
376 let mut roi = ReplicateObjectInfo {
377 bucket: "bucket".to_string(),
378 name: "object".to_string(),
379 replication_status: status,
380 dsc: ReplicateDecision::new(),
381 ..Default::default()
382 };
383
384 let action = replication_heal_queue_action(&mut roi);
385
386 assert!(
387 matches!(action, ReplicationHealQueueAction::Skip),
388 "an empty replicate decision must skip heal queueing (status {:?})",
389 roi.replication_status
390 );
391 }
392 }
393
394 #[test]
397 fn heal_queue_action_skips_completed_object_without_resync() {
398 let mut roi = replicate_object_info(ReplicationStatusType::Completed);
399
400 let action = replication_heal_queue_action(&mut roi);
401
402 assert!(matches!(action, ReplicationHealQueueAction::Skip));
403 }
404
405 #[test]
406 fn heal_queue_action_routes_failed_objects_to_heal_queue() {
407 let mut roi = replicate_object_info(ReplicationStatusType::Failed);
408
409 let action = replication_heal_queue_action(&mut roi);
410
411 assert!(matches!(action, ReplicationHealQueueAction::QueueObject));
412 assert_eq!(roi.event_type, REPLICATE_HEAL);
413 }
414
415 #[test]
416 fn heal_queue_action_routes_existing_object_resync() {
417 let mut roi = replicate_object_info(ReplicationStatusType::Completed);
418 roi.existing_obj_resync.targets.insert(
419 "arn:target".to_string(),
420 ResyncTargetDecision {
421 replicate: true,
422 reset_id: "reset-1".to_string(),
423 reset_before_date: None,
424 },
425 );
426
427 let action = replication_heal_queue_action(&mut roi);
428
429 assert!(matches!(action, ReplicationHealQueueAction::QueueObject));
430 assert_eq!(roi.op_type, ReplicationType::ExistingObject);
431 assert_eq!(roi.event_type, REPLICATE_EXISTING);
432 }
433
434 #[test]
435 fn heal_queue_action_routes_pending_delete_marker() {
436 let version_id = Uuid::new_v4();
437 let mut roi = replicate_object_info(ReplicationStatusType::Pending);
438 roi.delete_marker = true;
439 roi.version_id = Some(version_id);
440
441 let action = replication_heal_queue_action(&mut roi);
442
443 let ReplicationHealQueueAction::QueueDelete(delete_info) = action else {
444 panic!("expected delete queue action");
445 };
446 assert_eq!(delete_info.bucket, "bucket");
447 assert_eq!(delete_info.event_type, REPLICATE_HEAL_DELETE);
448 assert_eq!(delete_info.delete_object.object_name, "object");
449 assert_eq!(delete_info.delete_object.delete_marker_version_id, Some(version_id));
450 assert_eq!(delete_info.delete_object.version_id, None);
451 }
452
453 #[test]
454 fn heal_queue_action_expands_resync_delete_targets() {
455 let mut roi = replicate_object_info(ReplicationStatusType::Completed);
456 roi.delete_marker = true;
457 roi.existing_obj_resync.targets.insert(
458 "arn:replicate".to_string(),
459 ResyncTargetDecision {
460 replicate: true,
461 reset_id: "reset-1".to_string(),
462 reset_before_date: None,
463 },
464 );
465 roi.existing_obj_resync.targets.insert(
466 "arn:skip".to_string(),
467 ResyncTargetDecision {
468 replicate: false,
469 reset_id: "reset-2".to_string(),
470 reset_before_date: None,
471 },
472 );
473
474 let action = replication_heal_queue_action(&mut roi);
475
476 let ReplicationHealQueueAction::QueueResyncDeletes(batch) = action else {
477 panic!("expected resync delete queue action");
478 };
479 let target_deletes = batch.target_delete_infos().collect::<Vec<_>>();
480 assert_eq!(target_deletes.len(), 1);
481 assert_eq!(target_deletes[0].target_arn, "arn:replicate");
482 assert_eq!(target_deletes[0].reset_id, "reset-1");
483 }
484
485 #[test]
486 fn heal_queue_action_routes_pending_version_purge() {
487 let version_id = Uuid::new_v4();
488 let mut roi = replicate_object_info(ReplicationStatusType::Completed);
489 roi.version_id = Some(version_id);
490 roi.version_purge_status = VersionPurgeStatusType::Pending;
491
492 let action = replication_heal_queue_action(&mut roi);
493
494 let ReplicationHealQueueAction::QueueDelete(delete_info) = action else {
495 panic!("expected version purge delete queue action");
496 };
497 assert_eq!(delete_info.delete_object.version_id, Some(version_id));
498 assert_eq!(delete_info.delete_object.delete_marker_version_id, None);
499 }
500
501 #[test]
502 fn heal_queue_action_preserves_pending_null_version_purge() {
503 let mut roi = replicate_object_info(ReplicationStatusType::Completed);
504 roi.version_id = Some(Uuid::nil());
505 roi.version_purge_status = VersionPurgeStatusType::Pending;
506
507 let action = replication_heal_queue_action(&mut roi);
508
509 let ReplicationHealQueueAction::QueueDelete(delete_info) = action else {
510 panic!("expected null-version purge delete queue action");
511 };
512 assert_eq!(delete_info.delete_object.version_id, Some(Uuid::nil()));
513 assert_eq!(delete_info.delete_object.delete_marker_version_id, None);
514 }
515
516 #[test]
517 fn replication_priority_parses_known_values_and_defaults_unknown() {
518 assert_eq!(ReplicationPriority::from_str("fast"), Ok(ReplicationPriority::Fast));
519 assert_eq!(ReplicationPriority::from_str("slow"), Ok(ReplicationPriority::Slow));
520 assert_eq!(ReplicationPriority::from_str("auto"), Ok(ReplicationPriority::Auto));
521 assert_eq!(ReplicationPriority::from_str("unexpected"), Ok(ReplicationPriority::Auto));
522 assert_eq!(ReplicationPriority::Fast.as_str(), "fast");
523 assert_eq!(ReplicationPriority::Slow.as_str(), "slow");
524 assert_eq!(ReplicationPriority::Auto.as_str(), "auto");
525 }
526
527 #[test]
528 fn replication_operation_delegates_delete_contract() {
529 let info = DeletedObjectReplicationInfo {
530 bucket: "bucket".to_string(),
531 op_type: ReplicationType::Delete,
532 delete_object: DeletedObject {
533 object_name: "object".to_string(),
534 delete_marker: true,
535 ..Default::default()
536 },
537 ..Default::default()
538 };
539 let operation = ReplicationOperation::Delete(Box::new(info));
540
541 assert_eq!(operation.get_bucket(), "bucket");
542 assert_eq!(operation.get_object(), "object");
543 assert_eq!(operation.get_op_type(), ReplicationType::Delete);
544 assert!(operation.is_delete_marker());
545 }
546
547 #[test]
548 fn replication_operation_delegates_object_contract() {
549 let info = ReplicateObjectInfo {
550 bucket: "bucket".to_string(),
551 name: "object".to_string(),
552 size: 128,
553 op_type: ReplicationType::Object,
554 ..Default::default()
555 };
556 let operation = ReplicationOperation::Object(Box::new(info));
557
558 assert_eq!(operation.get_bucket(), "bucket");
559 assert_eq!(operation.get_object(), "object");
560 assert_eq!(operation.get_size(), 128);
561 assert_eq!(operation.get_op_type(), ReplicationType::Object);
562 }
563
564 fn replicate_object_info(replication_status: ReplicationStatusType) -> ReplicateObjectInfo {
565 let mut dsc = ReplicateDecision::new();
566 dsc.set(ReplicateTargetDecision::new("arn:target".to_string(), true, false));
567
568 ReplicateObjectInfo {
569 bucket: "bucket".to_string(),
570 name: "object".to_string(),
571 replication_status,
572 dsc,
573 ..Default::default()
574 }
575 }
576}