Skip to main content

rustfs_replication/
queue.rs

1// Copyright 2024 RustFS Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::any::Any;
16
17use crate::storage_api::DeletedObject;
18use crate::{
19    DeletedObjectReplicationInfo, MrfReplicateEntry, REPLICATE_EXISTING, REPLICATE_HEAL, REPLICATE_HEAL_DELETE,
20    ReplicateObjectInfo, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation, ResyncDecision,
21    VersionPurgeStatusType,
22};
23
24#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
25pub enum ReplicationQueueAdmission {
26    #[default]
27    Skipped,
28    Queued,
29    Missed,
30}
31
32impl ReplicationQueueAdmission {
33    pub fn merge(&mut self, other: Self) {
34        *self = match (*self, other) {
35            (Self::Missed, _) | (_, Self::Missed) => Self::Missed,
36            (Self::Queued, _) | (_, Self::Queued) => Self::Queued,
37            (Self::Skipped, Self::Skipped) => Self::Skipped,
38        };
39    }
40}
41
42#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
43pub struct ReplicationBatchAdmission {
44    pub total: usize,
45    pub queued: usize,
46    pub missed: usize,
47}
48
49impl ReplicationBatchAdmission {
50    pub fn record(&mut self, admission: ReplicationQueueAdmission) {
51        self.total += 1;
52        match admission {
53            ReplicationQueueAdmission::Queued => self.queued += 1,
54            ReplicationQueueAdmission::Missed | ReplicationQueueAdmission::Skipped => self.missed += 1,
55        }
56    }
57
58    pub fn outcome(self) -> &'static str {
59        match (self.queued, self.missed) {
60            (0, 0) => "empty",
61            (_, 0) => "all_queued",
62            (0, _) => "all_missed",
63            _ => "partial",
64        }
65    }
66}
67
68pub fn mrf_save_admission(saved: bool) -> ReplicationQueueAdmission {
69    if saved {
70        ReplicationQueueAdmission::Queued
71    } else {
72        ReplicationQueueAdmission::Missed
73    }
74}
75
76#[derive(Debug, Clone, Copy, PartialEq, Eq)]
77pub enum ReplicationWorkerQueue {
78    Regular,
79    Mrf,
80}
81
82pub fn worker_queue_for_replication_type(op_type: &ReplicationType) -> ReplicationWorkerQueue {
83    match op_type {
84        ReplicationType::Heal | ReplicationType::ExistingObject => ReplicationWorkerQueue::Mrf,
85        _ => ReplicationWorkerQueue::Regular,
86    }
87}
88
89#[derive(Debug, Clone, Default)]
90pub struct ReplicationHealQueueResult {
91    pub object_info: ReplicateObjectInfo,
92    pub admission: ReplicationQueueAdmission,
93}
94
95#[derive(Debug, Clone, Default)]
96pub enum ReplicationHealQueueAction {
97    #[default]
98    Skip,
99    QueueObject,
100    QueueDelete(DeletedObjectReplicationInfo),
101    QueueResyncDeletes(ReplicationHealResyncDeletes),
102}
103
104#[derive(Debug, Clone, Default)]
105pub struct ReplicationHealResyncDeletes {
106    pub delete_info: DeletedObjectReplicationInfo,
107    pub existing_obj_resync: ResyncDecision,
108}
109
110impl ReplicationHealResyncDeletes {
111    pub fn target_delete_infos(&self) -> impl Iterator<Item = DeletedObjectReplicationInfo> + '_ {
112        self.existing_obj_resync
113            .targets
114            .iter()
115            .filter(|(_, target)| target.replicate)
116            .map(|(target_arn, target)| {
117                let mut delete_info = self.delete_info.clone();
118                delete_info.reset_id = target.reset_id.clone();
119                delete_info.target_arn = target_arn.clone();
120                delete_info
121            })
122    }
123}
124
125pub fn replication_heal_queue_action(roi: &mut ReplicateObjectInfo) -> ReplicationHealQueueAction {
126    if !roi.dsc.replicate_any() {
127        return ReplicationHealQueueAction::Skip;
128    }
129
130    if roi.replication_status == ReplicationStatusType::Completed
131        && roi.version_purge_status.is_empty()
132        && !roi.existing_obj_resync.must_resync()
133    {
134        return ReplicationHealQueueAction::Skip;
135    }
136
137    if roi.delete_marker || !roi.version_purge_status.is_empty() {
138        let delete_info = heal_deleted_object_replication_info(roi);
139
140        if is_pending_or_failed_object_heal(roi) || is_pending_or_failed_version_purge(roi) {
141            return ReplicationHealQueueAction::QueueDelete(delete_info);
142        }
143
144        if roi.existing_obj_resync.must_resync()
145            && (roi.replication_status == ReplicationStatusType::Completed || roi.replication_status.is_empty())
146        {
147            return ReplicationHealQueueAction::QueueResyncDeletes(ReplicationHealResyncDeletes {
148                delete_info,
149                existing_obj_resync: roi.existing_obj_resync.clone(),
150            });
151        }
152
153        return ReplicationHealQueueAction::Skip;
154    }
155
156    if roi.existing_obj_resync.must_resync() {
157        roi.op_type = ReplicationType::ExistingObject;
158    }
159
160    if is_pending_or_failed_object_heal(roi) {
161        roi.event_type = REPLICATE_HEAL.to_string();
162        return ReplicationHealQueueAction::QueueObject;
163    }
164
165    if roi.existing_obj_resync.must_resync() {
166        roi.event_type = REPLICATE_EXISTING.to_string();
167        return ReplicationHealQueueAction::QueueObject;
168    }
169
170    ReplicationHealQueueAction::Skip
171}
172
173fn heal_deleted_object_replication_info(roi: &ReplicateObjectInfo) -> DeletedObjectReplicationInfo {
174    let (version_id, delete_marker_version_id) = if roi.version_purge_status.is_empty() {
175        (None, roi.version_id)
176    } else {
177        (roi.version_id, None)
178    };
179
180    DeletedObjectReplicationInfo {
181        delete_object: DeletedObject {
182            object_name: roi.name.clone(),
183            delete_marker_version_id,
184            version_id,
185            replication_state: roi.replication_state.clone(),
186            delete_marker_mtime: roi.mod_time,
187            delete_marker: roi.delete_marker,
188            ..Default::default()
189        },
190        bucket: roi.bucket.clone(),
191        op_type: ReplicationType::Heal,
192        event_type: REPLICATE_HEAL_DELETE.to_string(),
193        ..Default::default()
194    }
195}
196
197fn is_pending_or_failed_object_heal(roi: &ReplicateObjectInfo) -> bool {
198    matches!(roi.replication_status, ReplicationStatusType::Pending | ReplicationStatusType::Failed)
199}
200
201fn is_pending_or_failed_version_purge(roi: &ReplicateObjectInfo) -> bool {
202    matches!(roi.version_purge_status, VersionPurgeStatusType::Pending | VersionPurgeStatusType::Failed)
203}
204
205#[derive(Debug, Clone, PartialEq)]
206pub enum ReplicationPriority {
207    Fast,
208    Slow,
209    Auto,
210}
211
212impl std::str::FromStr for ReplicationPriority {
213    type Err = ();
214
215    fn from_str(s: &str) -> Result<Self, Self::Err> {
216        match s {
217            "fast" => Ok(ReplicationPriority::Fast),
218            "slow" => Ok(ReplicationPriority::Slow),
219            "auto" => Ok(ReplicationPriority::Auto),
220            _ => Ok(ReplicationPriority::Auto),
221        }
222    }
223}
224
225impl ReplicationPriority {
226    pub fn as_str(&self) -> &'static str {
227        match self {
228            ReplicationPriority::Fast => "fast",
229            ReplicationPriority::Slow => "slow",
230            ReplicationPriority::Auto => "auto",
231        }
232    }
233}
234
235#[derive(Debug, Clone)]
236pub enum ReplicationOperation {
237    Object(Box<ReplicateObjectInfo>),
238    Delete(Box<DeletedObjectReplicationInfo>),
239}
240
241impl ReplicationWorkerOperation for ReplicationOperation {
242    fn as_any(&self) -> &dyn Any {
243        self
244    }
245
246    fn to_mrf_entry(&self) -> MrfReplicateEntry {
247        match self {
248            ReplicationOperation::Object(obj) => obj.to_mrf_entry(),
249            ReplicationOperation::Delete(del) => del.to_mrf_entry(),
250        }
251    }
252
253    fn get_bucket(&self) -> &str {
254        match self {
255            ReplicationOperation::Object(obj) => obj.get_bucket(),
256            ReplicationOperation::Delete(del) => del.get_bucket(),
257        }
258    }
259
260    fn get_object(&self) -> &str {
261        match self {
262            ReplicationOperation::Object(obj) => obj.get_object(),
263            ReplicationOperation::Delete(del) => del.get_object(),
264        }
265    }
266
267    fn get_size(&self) -> i64 {
268        match self {
269            ReplicationOperation::Object(obj) => obj.get_size(),
270            ReplicationOperation::Delete(del) => del.get_size(),
271        }
272    }
273
274    fn is_delete_marker(&self) -> bool {
275        match self {
276            ReplicationOperation::Object(obj) => obj.is_delete_marker(),
277            ReplicationOperation::Delete(del) => del.is_delete_marker(),
278        }
279    }
280
281    fn get_op_type(&self) -> ReplicationType {
282        match self {
283            ReplicationOperation::Object(obj) => obj.get_op_type(),
284            ReplicationOperation::Delete(del) => del.get_op_type(),
285        }
286    }
287}
288
289#[cfg(test)]
290mod tests {
291    use std::str::FromStr;
292
293    use uuid::Uuid;
294
295    use super::{
296        ReplicationBatchAdmission, ReplicationHealQueueAction, ReplicationOperation, ReplicationPriority,
297        ReplicationQueueAdmission, ReplicationWorkerQueue,
298    };
299    use crate::storage_api::DeletedObject;
300    use crate::{
301        DeletedObjectReplicationInfo, REPLICATE_EXISTING, REPLICATE_HEAL, REPLICATE_HEAL_DELETE, ReplicateDecision,
302        ReplicateObjectInfo, ReplicateTargetDecision, ReplicationStatusType, ReplicationType, ReplicationWorkerOperation,
303        ResyncTargetDecision, VersionPurgeStatusType, mrf_save_admission, replication_heal_queue_action,
304        worker_queue_for_replication_type,
305    };
306
307    #[test]
308    fn replication_queue_admission_combines_target_results() {
309        let mut admission = ReplicationQueueAdmission::Skipped;
310
311        admission.merge(ReplicationQueueAdmission::Queued);
312        assert_eq!(admission, ReplicationQueueAdmission::Queued);
313
314        admission.merge(ReplicationQueueAdmission::Missed);
315        assert_eq!(admission, ReplicationQueueAdmission::Missed);
316
317        admission.merge(ReplicationQueueAdmission::Skipped);
318        assert_eq!(admission, ReplicationQueueAdmission::Missed);
319    }
320
321    #[test]
322    fn mrf_save_admission_maps_send_result() {
323        assert_eq!(mrf_save_admission(true), ReplicationQueueAdmission::Queued);
324        assert_eq!(mrf_save_admission(false), ReplicationQueueAdmission::Missed);
325    }
326
327    #[test]
328    fn replication_batch_admission_classifies_all_partial_and_empty_batches() {
329        let mut all = ReplicationBatchAdmission::default();
330        all.record(ReplicationQueueAdmission::Queued);
331        assert_eq!(all.outcome(), "all_queued");
332
333        let mut partial = ReplicationBatchAdmission::default();
334        partial.record(ReplicationQueueAdmission::Queued);
335        partial.record(ReplicationQueueAdmission::Missed);
336        assert_eq!(partial.outcome(), "partial");
337
338        let mut all_missed = ReplicationBatchAdmission::default();
339        all_missed.record(ReplicationQueueAdmission::Missed);
340        assert_eq!(all_missed.outcome(), "all_missed");
341
342        let empty = ReplicationBatchAdmission::default();
343        assert_eq!(empty.outcome(), "empty");
344    }
345
346    #[test]
347    fn worker_queue_routes_heal_and_existing_objects_to_mrf() {
348        assert_eq!(worker_queue_for_replication_type(&ReplicationType::Heal), ReplicationWorkerQueue::Mrf);
349        assert_eq!(
350            worker_queue_for_replication_type(&ReplicationType::ExistingObject),
351            ReplicationWorkerQueue::Mrf
352        );
353        assert_eq!(
354            worker_queue_for_replication_type(&ReplicationType::Object),
355            ReplicationWorkerQueue::Regular
356        );
357        assert_eq!(
358            worker_queue_for_replication_type(&ReplicationType::Delete),
359            ReplicationWorkerQueue::Regular
360        );
361    }
362
363    /// P1-20 truth-table pin (rustfs/backlog#1675): when no target replicates
364    /// — the decision is empty because ExistingObjectReplication is Disabled
365    /// for a never-replicated object, or because the object is an inbound
366    /// REPLICA (must_replicate returns an empty decision for those) — the
367    /// heal pass must skip entirely, whatever the recorded status says. The
368    /// scanner never compensates these objects.
369    #[test]
370    fn heal_queue_action_skips_when_no_target_replicates() {
371        for status in [
372            ReplicationStatusType::Empty,
373            ReplicationStatusType::Failed,
374            ReplicationStatusType::Replica,
375        ] {
376            let mut roi = ReplicateObjectInfo {
377                bucket: "bucket".to_string(),
378                name: "object".to_string(),
379                replication_status: status,
380                dsc: ReplicateDecision::new(),
381                ..Default::default()
382            };
383
384            let action = replication_heal_queue_action(&mut roi);
385
386            assert!(
387                matches!(action, ReplicationHealQueueAction::Skip),
388                "an empty replicate decision must skip heal queueing (status {:?})",
389                roi.replication_status
390            );
391        }
392    }
393
394    /// P1-20 truth-table pin: a Completed object with no resync decision has
395    /// nothing left to heal — the scanner must not requeue it.
396    #[test]
397    fn heal_queue_action_skips_completed_object_without_resync() {
398        let mut roi = replicate_object_info(ReplicationStatusType::Completed);
399
400        let action = replication_heal_queue_action(&mut roi);
401
402        assert!(matches!(action, ReplicationHealQueueAction::Skip));
403    }
404
405    #[test]
406    fn heal_queue_action_routes_failed_objects_to_heal_queue() {
407        let mut roi = replicate_object_info(ReplicationStatusType::Failed);
408
409        let action = replication_heal_queue_action(&mut roi);
410
411        assert!(matches!(action, ReplicationHealQueueAction::QueueObject));
412        assert_eq!(roi.event_type, REPLICATE_HEAL);
413    }
414
415    #[test]
416    fn heal_queue_action_routes_existing_object_resync() {
417        let mut roi = replicate_object_info(ReplicationStatusType::Completed);
418        roi.existing_obj_resync.targets.insert(
419            "arn:target".to_string(),
420            ResyncTargetDecision {
421                replicate: true,
422                reset_id: "reset-1".to_string(),
423                reset_before_date: None,
424            },
425        );
426
427        let action = replication_heal_queue_action(&mut roi);
428
429        assert!(matches!(action, ReplicationHealQueueAction::QueueObject));
430        assert_eq!(roi.op_type, ReplicationType::ExistingObject);
431        assert_eq!(roi.event_type, REPLICATE_EXISTING);
432    }
433
434    #[test]
435    fn heal_queue_action_routes_pending_delete_marker() {
436        let version_id = Uuid::new_v4();
437        let mut roi = replicate_object_info(ReplicationStatusType::Pending);
438        roi.delete_marker = true;
439        roi.version_id = Some(version_id);
440
441        let action = replication_heal_queue_action(&mut roi);
442
443        let ReplicationHealQueueAction::QueueDelete(delete_info) = action else {
444            panic!("expected delete queue action");
445        };
446        assert_eq!(delete_info.bucket, "bucket");
447        assert_eq!(delete_info.event_type, REPLICATE_HEAL_DELETE);
448        assert_eq!(delete_info.delete_object.object_name, "object");
449        assert_eq!(delete_info.delete_object.delete_marker_version_id, Some(version_id));
450        assert_eq!(delete_info.delete_object.version_id, None);
451    }
452
453    #[test]
454    fn heal_queue_action_expands_resync_delete_targets() {
455        let mut roi = replicate_object_info(ReplicationStatusType::Completed);
456        roi.delete_marker = true;
457        roi.existing_obj_resync.targets.insert(
458            "arn:replicate".to_string(),
459            ResyncTargetDecision {
460                replicate: true,
461                reset_id: "reset-1".to_string(),
462                reset_before_date: None,
463            },
464        );
465        roi.existing_obj_resync.targets.insert(
466            "arn:skip".to_string(),
467            ResyncTargetDecision {
468                replicate: false,
469                reset_id: "reset-2".to_string(),
470                reset_before_date: None,
471            },
472        );
473
474        let action = replication_heal_queue_action(&mut roi);
475
476        let ReplicationHealQueueAction::QueueResyncDeletes(batch) = action else {
477            panic!("expected resync delete queue action");
478        };
479        let target_deletes = batch.target_delete_infos().collect::<Vec<_>>();
480        assert_eq!(target_deletes.len(), 1);
481        assert_eq!(target_deletes[0].target_arn, "arn:replicate");
482        assert_eq!(target_deletes[0].reset_id, "reset-1");
483    }
484
485    #[test]
486    fn heal_queue_action_routes_pending_version_purge() {
487        let version_id = Uuid::new_v4();
488        let mut roi = replicate_object_info(ReplicationStatusType::Completed);
489        roi.version_id = Some(version_id);
490        roi.version_purge_status = VersionPurgeStatusType::Pending;
491
492        let action = replication_heal_queue_action(&mut roi);
493
494        let ReplicationHealQueueAction::QueueDelete(delete_info) = action else {
495            panic!("expected version purge delete queue action");
496        };
497        assert_eq!(delete_info.delete_object.version_id, Some(version_id));
498        assert_eq!(delete_info.delete_object.delete_marker_version_id, None);
499    }
500
501    #[test]
502    fn heal_queue_action_preserves_pending_null_version_purge() {
503        let mut roi = replicate_object_info(ReplicationStatusType::Completed);
504        roi.version_id = Some(Uuid::nil());
505        roi.version_purge_status = VersionPurgeStatusType::Pending;
506
507        let action = replication_heal_queue_action(&mut roi);
508
509        let ReplicationHealQueueAction::QueueDelete(delete_info) = action else {
510            panic!("expected null-version purge delete queue action");
511        };
512        assert_eq!(delete_info.delete_object.version_id, Some(Uuid::nil()));
513        assert_eq!(delete_info.delete_object.delete_marker_version_id, None);
514    }
515
516    #[test]
517    fn replication_priority_parses_known_values_and_defaults_unknown() {
518        assert_eq!(ReplicationPriority::from_str("fast"), Ok(ReplicationPriority::Fast));
519        assert_eq!(ReplicationPriority::from_str("slow"), Ok(ReplicationPriority::Slow));
520        assert_eq!(ReplicationPriority::from_str("auto"), Ok(ReplicationPriority::Auto));
521        assert_eq!(ReplicationPriority::from_str("unexpected"), Ok(ReplicationPriority::Auto));
522        assert_eq!(ReplicationPriority::Fast.as_str(), "fast");
523        assert_eq!(ReplicationPriority::Slow.as_str(), "slow");
524        assert_eq!(ReplicationPriority::Auto.as_str(), "auto");
525    }
526
527    #[test]
528    fn replication_operation_delegates_delete_contract() {
529        let info = DeletedObjectReplicationInfo {
530            bucket: "bucket".to_string(),
531            op_type: ReplicationType::Delete,
532            delete_object: DeletedObject {
533                object_name: "object".to_string(),
534                delete_marker: true,
535                ..Default::default()
536            },
537            ..Default::default()
538        };
539        let operation = ReplicationOperation::Delete(Box::new(info));
540
541        assert_eq!(operation.get_bucket(), "bucket");
542        assert_eq!(operation.get_object(), "object");
543        assert_eq!(operation.get_op_type(), ReplicationType::Delete);
544        assert!(operation.is_delete_marker());
545    }
546
547    #[test]
548    fn replication_operation_delegates_object_contract() {
549        let info = ReplicateObjectInfo {
550            bucket: "bucket".to_string(),
551            name: "object".to_string(),
552            size: 128,
553            op_type: ReplicationType::Object,
554            ..Default::default()
555        };
556        let operation = ReplicationOperation::Object(Box::new(info));
557
558        assert_eq!(operation.get_bucket(), "bucket");
559        assert_eq!(operation.get_object(), "object");
560        assert_eq!(operation.get_size(), 128);
561        assert_eq!(operation.get_op_type(), ReplicationType::Object);
562    }
563
564    fn replicate_object_info(replication_status: ReplicationStatusType) -> ReplicateObjectInfo {
565        let mut dsc = ReplicateDecision::new();
566        dsc.set(ReplicateTargetDecision::new("arn:target".to_string(), true, false));
567
568        ReplicateObjectInfo {
569            bucket: "bucket".to_string(),
570            name: "object".to_string(),
571            replication_status,
572            dsc,
573            ..Default::default()
574        }
575    }
576}