Skip to main content

rustfs_replication/
delete.rs

1// Copyright 2024 RustFS Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::any::Any;
16
17use uuid::Uuid;
18
19use crate::storage_api::DeletedObject;
20use crate::{
21    MrfOpKind, MrfReplicateEntry, NULL_VERSION_ID, REPLICATE_EXISTING_DELETE, ReplicateObjectInfo, ReplicationState,
22    ReplicationStatusType, ReplicationType, ReplicationWorkerOperation,
23};
24
25#[derive(Debug, Clone, Default)]
26pub struct DeletedObjectReplicationInfo {
27    pub delete_object: DeletedObject,
28    pub bucket: String,
29    pub event_type: String,
30    pub op_type: ReplicationType,
31    pub reset_id: String,
32    pub target_arn: String,
33}
34
35impl DeletedObjectReplicationInfo {
36    pub fn admitted_target_arns(&self) -> Vec<String> {
37        if !self.target_arn.is_empty() {
38            return vec![self.target_arn.clone()];
39        }
40
41        let mut target_arns = if !self.delete_object.force_delete_target_arns.is_empty() {
42            self.delete_object.force_delete_target_arns.clone()
43        } else {
44            self.delete_object
45                .replication_state
46                .as_ref()
47                .map(admitted_target_arns_from_replication_state)
48                .unwrap_or_default()
49        };
50        target_arns.sort();
51        target_arns.dedup();
52        target_arns
53    }
54}
55
56impl ReplicationWorkerOperation for DeletedObjectReplicationInfo {
57    fn as_any(&self) -> &dyn Any {
58        self
59    }
60
61    fn to_mrf_entry(&self) -> MrfReplicateEntry {
62        MrfReplicateEntry {
63            bucket: self.bucket.clone(),
64            object: self.delete_object.object_name.clone(),
65            version_id: self.delete_object.version_id,
66            retry_count: 0,
67            size: 0,
68            op: MrfOpKind::Delete,
69            force_delete: self.delete_object.force_delete,
70            delete_marker_version_id: self.delete_object.delete_marker_version_id,
71            delete_marker: self.delete_object.delete_marker,
72            // Persist the original delete-marker mtime as Unix nanoseconds so replay after a
73            // restart stamps the replica with the source timestamp rather than the replay time
74            // (backlog#867). None when unknown; replay then falls back to the current time.
75            delete_marker_mtime: self
76                .delete_object
77                .delete_marker_mtime
78                .and_then(|t| i64::try_from(t.unix_timestamp_nanos()).ok()),
79            // Carry the target-assigned marker version ids (and the fail-closed corrupt
80            // flag) into the journal so a purge intent replayed after a restart addresses
81            // the same version the live path did (backlog#2290). Only delete-marker state
82            // ever records these; other deletes serialize an empty map.
83            target_delete_marker_version_ids: self
84                .delete_object
85                .replication_state
86                .as_ref()
87                .map(|state| state.target_delete_marker_version_ids.clone())
88                .unwrap_or_default(),
89            target_delete_marker_version_ids_corrupt: self
90                .delete_object
91                .replication_state
92                .as_ref()
93                .is_some_and(|state| state.target_delete_marker_version_ids_corrupt),
94            target_arns: self.admitted_target_arns(),
95            force_delete_id: self.delete_object.force_delete_id,
96            force_delete_generation: self.delete_object.force_delete_generation,
97            force_delete_local_commit: self.delete_object.force_delete,
98        }
99    }
100
101    fn get_bucket(&self) -> &str {
102        &self.bucket
103    }
104
105    fn get_object(&self) -> &str {
106        &self.delete_object.object_name
107    }
108
109    fn get_size(&self) -> i64 {
110        0
111    }
112
113    fn is_delete_marker(&self) -> bool {
114        true
115    }
116
117    fn get_op_type(&self) -> ReplicationType {
118        self.op_type
119    }
120}
121
122pub fn is_version_delete_replication(dobj: &DeletedObject) -> bool {
123    dobj.version_id.is_some() || (dobj.delete_marker_version_id.is_some() && !dobj.delete_marker)
124}
125
126/// Whether the target DELETE for this delete replication may use
127/// marker-CREATION semantics (`replication_delete_marker`, which omits the
128/// `versionId` query so the target mints its own marker).
129///
130/// A version purge must never do so: the DELETE has to address the exact
131/// version, otherwise a generic S3 target — which ignores the internal
132/// source-version headers — mints a fresh delete marker on every retry
133/// instead of removing one (rustfs#6823). The `delete_marker` flag alone is
134/// not enough because heal/resync rebuilds carry `delete_marker: true`
135/// together with a purge-shaped entry.
136pub fn delete_replication_creates_marker(dobj: &DeletedObject) -> bool {
137    dobj.delete_marker && !is_version_delete_replication(dobj)
138}
139
140pub fn should_retry_delete_marker_purge(dobj: &DeletedObject) -> bool {
141    dobj.delete_marker_version_id.is_some()
142}
143
144/// True when the target denied a replicated delete because object-lock
145/// retention or a legal hold protects that version on the replica (its
146/// deletion gate answers `AccessDenied` with the lock reason, and a
147/// replication request carries no governance bypass, rustfs#6850). Retrying
148/// cannot succeed until the lock itself lapses, so callers treat this as a
149/// policy denial rather than a transient fault.
150///
151/// The reason text is the RustFS deletion-gate wording; a MinIO/AWS peer
152/// phrases its WORM denial differently and simply stays unclassified — the
153/// caller then falls back to plain retry behavior, never a wrong state.
154pub fn is_object_lock_denied_delete(code: Option<&str>, message: Option<&str>) -> bool {
155    if !matches!(code, Some("AccessDenied")) {
156        return false;
157    }
158    let Some(message) = message else {
159        return false;
160    };
161    let message = message.to_ascii_lowercase();
162    message.contains("retention") || message.contains("legal hold")
163}
164
165fn admitted_target_arns_from_replication_state(state: &ReplicationState) -> Vec<String> {
166    let mut target_arns = state.targets.keys().cloned().collect::<Vec<_>>();
167    target_arns.extend(state.purge_targets.keys().cloned());
168    target_arns
169}
170
171pub fn is_retryable_delete_replication_head_error(is_not_found: bool, code: Option<&str>) -> bool {
172    !(is_not_found || matches!(code, Some("MethodNotAllowed" | "405")))
173}
174
175/// Build the delete-replication work item for an existing delete marker or
176/// version purge discovered during a resync scan.
177pub fn resync_existing_delete_replication_info(roi: &ReplicateObjectInfo, target_arn: &str) -> DeletedObjectReplicationInfo {
178    let (version_id, dm_version_id) = if roi.version_purge_status.is_empty() {
179        (None, roi.version_id)
180    } else {
181        (roi.version_id, None)
182    };
183
184    DeletedObjectReplicationInfo {
185        delete_object: DeletedObject {
186            object_name: roi.name.clone(),
187            delete_marker_version_id: dm_version_id,
188            version_id,
189            replication_state: roi.replication_state.clone(),
190            delete_marker: roi.delete_marker,
191            delete_marker_mtime: roi.mod_time,
192            ..Default::default()
193        },
194        bucket: roi.bucket.clone(),
195        event_type: REPLICATE_EXISTING_DELETE.to_string(),
196        op_type: ReplicationType::ExistingObject,
197        target_arn: target_arn.to_string(),
198        ..Default::default()
199    }
200}
201
202/// Whether a delete replication fully succeeded — the MRF replay acknowledges
203/// (drops) an entry exactly when this returns true.
204///
205/// The delayed purge is deliberately NOT an input: holding the outcome hostage
206/// to it (`&& !requires_delayed_purge`) forced `false` for every delete-marker
207/// entry and retained them all in the durable MRF journal forever. Purge
208/// failures persist their own purge-intent entry instead
209/// (`watch_and_purge_source_delete_marker`), and replays of those entries
210/// report purge success through `purge_stale_delete_marker_targets`.
211pub fn replicate_delete_outcome(
212    expected_targets: usize,
213    replicated_targets: usize,
214    state_persisted: bool,
215    source_state_verified: bool,
216    replication_status: &ReplicationStatusType,
217) -> bool {
218    expected_targets > 0
219        && replicated_targets == expected_targets
220        && state_persisted
221        && source_state_verified
222        && *replication_status == ReplicationStatusType::Completed
223}
224
225pub fn target_delete_version_id(version_id: Uuid, version_purge: bool) -> Option<String> {
226    if version_id.is_nil() {
227        version_purge.then(|| NULL_VERSION_ID.to_string())
228    } else {
229        Some(version_id.to_string())
230    }
231}
232
233/// Which version a delete-marker purge should address on one target.
234///
235/// `None` means do not purge at all: the recorded mapping disagreed across the
236/// dual internal prefixes, and guessing an id could destroy a live version on
237/// the target. `Some(id)` is the exact version the target reported when it
238/// accepted the marker; falling back to a source-derived id is only correct
239/// when the target mirrors source version ids, which a generic S3 target does
240/// not.
241pub fn delete_marker_purge_version_id(
242    state: Option<&ReplicationState>,
243    arn: &str,
244    delete_marker_version_id: Uuid,
245) -> Option<Option<String>> {
246    if state.is_some_and(|state| state.target_delete_marker_version_ids_corrupt) {
247        return None;
248    }
249    let recorded = state.and_then(|state| state.target_delete_marker_version_ids.get(arn).cloned());
250    Some(match recorded {
251        Some(version_id) => Some(version_id),
252        None => target_delete_version_id(delete_marker_version_id, true),
253    })
254}
255
256/// The version a delete replication addresses on `arn`, or `None` to refuse.
257///
258/// A version purge whose purged version is a delete marker must address the
259/// marker version the TARGET assigned — the recorded mapping, exactly as the
260/// delayed-purge watcher does. The source-side `DELETE ?versionId=<marker>`
261/// replicates as such a purge, and a generic S3 target answers a DELETE of an
262/// unknown versionId with 204 while keeping its marker, so addressing it by
263/// the source id reported success and left the marker behind (backlog#2290,
264/// R6.1 on the VMs). Nothing recorded falls back to the source-derived id
265/// (id-mirroring peers); a corrupt record refuses, as the watcher does.
266pub fn delete_replication_target_version_id(dobj: &DeletedObject, arn: &str) -> Option<Option<String>> {
267    let is_version_purge = is_version_delete_replication(dobj);
268    if is_version_purge
269        && !dobj.delete_marker
270        && let Some(marker) = dobj.delete_marker_version_id
271    {
272        return delete_marker_purge_version_id(dobj.replication_state.as_ref(), arn, marker);
273    }
274    let source_version = dobj.delete_marker_version_id.or(dobj.version_id).unwrap_or_default();
275    Some(target_delete_version_id(source_version, is_version_purge))
276}
277
278/// Shape an exhausted purge intent as a marker-creation delete entry. Replay
279/// reconstructs it with `delete_marker: true`, finds the source marker gone,
280/// and funnels into the stale-marker branch of `replicate_delete_with_outcome`
281/// — which re-runs the purge without touching source state and reports purge
282/// success as the replay outcome.
283pub fn delete_marker_purge_mrf_entry(dobj: &DeletedObjectReplicationInfo, failed_arns: Vec<String>) -> MrfReplicateEntry {
284    let mut entry = dobj.to_mrf_entry();
285    entry.delete_marker = true;
286    entry.version_id = None;
287    entry.retry_count = 0;
288    entry.target_arns = failed_arns;
289    entry
290}
291
292#[cfg(test)]
293mod tests {
294    use std::collections::HashMap;
295
296    use super::{
297        DeletedObjectReplicationInfo, delete_marker_purge_mrf_entry, delete_marker_purge_version_id,
298        delete_replication_creates_marker, delete_replication_target_version_id, is_object_lock_denied_delete,
299        is_retryable_delete_replication_head_error, is_version_delete_replication, replicate_delete_outcome,
300        resync_existing_delete_replication_info, should_retry_delete_marker_purge, target_delete_version_id,
301    };
302    use crate::storage_api::DeletedObject;
303    use crate::{
304        MrfOpKind, NULL_VERSION_ID, ReplicateObjectInfo, ReplicationState, ReplicationStatusType, ReplicationType,
305        ReplicationWorkerOperation, VersionPurgeStatusType,
306    };
307    use uuid::Uuid;
308
309    #[test]
310    fn deleted_object_replication_info_encodes_delete_mrf_entry() {
311        let version_id = Uuid::new_v4();
312        let delete_marker_version_id = Uuid::new_v4();
313        let mtime = time::OffsetDateTime::from_unix_timestamp_nanos(1_705_312_200_123_456_789).expect("valid mtime");
314        let info = DeletedObjectReplicationInfo {
315            bucket: "bucket".to_string(),
316            op_type: ReplicationType::Delete,
317            delete_object: DeletedObject {
318                object_name: "object".to_string(),
319                version_id: Some(version_id),
320                delete_marker_version_id: Some(delete_marker_version_id),
321                delete_marker: true,
322                delete_marker_mtime: Some(mtime),
323                ..Default::default()
324            },
325            target_arn: "arn:target-a".to_string(),
326            ..Default::default()
327        };
328
329        let entry = info.to_mrf_entry();
330
331        assert_eq!(entry.bucket, "bucket");
332        assert_eq!(entry.object, "object");
333        assert_eq!(entry.version_id, Some(version_id));
334        assert!(!entry.force_delete);
335        assert_eq!(entry.delete_marker_version_id, Some(delete_marker_version_id));
336        assert_eq!(entry.op, MrfOpKind::Delete);
337        assert!(entry.delete_marker);
338        // The original mtime must be persisted (as Unix nanos) so replay keeps the source
339        // timestamp instead of stamping the replica with the replay time (backlog#867).
340        assert_eq!(
341            entry.delete_marker_mtime,
342            Some(mtime.unix_timestamp_nanos() as i64),
343            "delete-marker mtime must be persisted in the MRF entry"
344        );
345        assert_eq!(entry.target_arns, vec!["arn:target-a".to_string()]);
346        assert_eq!(info.get_object(), "object");
347    }
348
349    #[test]
350    fn deleted_object_replication_info_without_mtime_yields_none() {
351        // Absent source mtime must persist as None so replay falls back to the current time,
352        // preserving pre-#867 behaviour.
353        let info = DeletedObjectReplicationInfo {
354            bucket: "bucket".to_string(),
355            delete_object: DeletedObject {
356                object_name: "object".to_string(),
357                delete_marker: true,
358                delete_marker_mtime: None,
359                ..Default::default()
360            },
361            ..Default::default()
362        };
363
364        assert_eq!(info.to_mrf_entry().delete_marker_mtime, None);
365        assert!(info.to_mrf_entry().target_arns.is_empty());
366        assert!(!info.to_mrf_entry().force_delete);
367    }
368
369    #[test]
370    fn deleted_object_replication_info_uses_explicit_target_over_replay_state() {
371        let info = DeletedObjectReplicationInfo {
372            bucket: "bucket".to_string(),
373            delete_object: DeletedObject {
374                object_name: "object".to_string(),
375                replication_state: Some(ReplicationState {
376                    targets: HashMap::from([("arn:target-state".to_string(), ReplicationStatusType::Pending)]),
377                    purge_targets: HashMap::from([("arn:purge-state".to_string(), VersionPurgeStatusType::Pending)]),
378                    ..Default::default()
379                }),
380                ..Default::default()
381            },
382            target_arn: "arn:target-explicit".to_string(),
383            ..Default::default()
384        };
385
386        assert_eq!(info.to_mrf_entry().target_arns, vec!["arn:target-explicit".to_string()]);
387    }
388
389    #[test]
390    fn deleted_object_replication_info_serializes_replay_targets_when_target_is_implicit() {
391        let info = DeletedObjectReplicationInfo {
392            bucket: "bucket".to_string(),
393            delete_object: DeletedObject {
394                object_name: "object".to_string(),
395                replication_state: Some(ReplicationState {
396                    targets: HashMap::from([("arn:target-b".to_string(), ReplicationStatusType::Pending)]),
397                    purge_targets: HashMap::from([
398                        ("arn:target-a".to_string(), VersionPurgeStatusType::Pending),
399                        ("arn:target-b".to_string(), VersionPurgeStatusType::Complete),
400                    ]),
401                    ..Default::default()
402                }),
403                ..Default::default()
404            },
405            ..Default::default()
406        };
407
408        assert_eq!(
409            info.to_mrf_entry().target_arns,
410            vec!["arn:target-a".to_string(), "arn:target-b".to_string()],
411            "MRF deletes must preserve the admitted target identities in stable order"
412        );
413    }
414
415    #[test]
416    fn deleted_object_replication_info_preserves_force_delete_handoff() {
417        let operation_id = Uuid::new_v4();
418        let info = DeletedObjectReplicationInfo {
419            bucket: "bucket".to_string(),
420            delete_object: DeletedObject {
421                object_name: "prefix/".to_string(),
422                force_delete: true,
423                force_delete_id: Some(operation_id),
424                force_delete_target_arns: vec![
425                    "arn:target-b".to_string(),
426                    "arn:target-a".to_string(),
427                    "arn:target-b".to_string(),
428                ],
429                force_delete_generation: Some(17),
430                ..Default::default()
431            },
432            ..Default::default()
433        };
434
435        let entry = info.to_mrf_entry();
436
437        assert!(entry.force_delete);
438        assert_eq!(entry.force_delete_id, Some(operation_id));
439        assert_eq!(entry.force_delete_generation, Some(17));
440        assert!(entry.force_delete_local_commit);
441        assert_eq!(entry.target_arns, vec!["arn:target-a", "arn:target-b"]);
442    }
443
444    #[test]
445    fn version_delete_replication_tracks_delete_marker_version_purge() {
446        let dobj = DeletedObject {
447            delete_marker: false,
448            delete_marker_version_id: Some(Uuid::new_v4()),
449            ..Default::default()
450        };
451
452        assert!(is_version_delete_replication(&dobj));
453    }
454
455    #[test]
456    fn version_delete_replication_tracks_explicit_version_id() {
457        let dobj = DeletedObject {
458            version_id: Some(Uuid::new_v4()),
459            ..Default::default()
460        };
461
462        assert!(is_version_delete_replication(&dobj));
463    }
464
465    #[test]
466    fn version_delete_replication_keeps_delete_marker_creation_separate() {
467        let dobj = DeletedObject {
468            delete_marker: true,
469            delete_marker_version_id: Some(Uuid::new_v4()),
470            ..Default::default()
471        };
472
473        assert!(!is_version_delete_replication(&dobj));
474    }
475
476    /// rustfs#6823 regression guard: a purge-shaped entry must never take
477    /// marker-creation semantics, which would drop the `versionId` from the
478    /// target DELETE and let a generic S3 target mint a fresh delete marker
479    /// on every heal/resync/MRF retry.
480    #[test]
481    fn purge_shapes_never_take_marker_creation_semantics() {
482        // Heal/resync rebuild of a delete-marker version purge: the entry
483        // carries `delete_marker: true` together with the purged version id.
484        let heal_purge = DeletedObject {
485            delete_marker: true,
486            version_id: Some(Uuid::new_v4()),
487            ..Default::default()
488        };
489        assert!(
490            !delete_replication_creates_marker(&heal_purge),
491            "a version purge must address the version, not mint a marker"
492        );
493
494        // Live delete-marker version purge addressed via the marker id.
495        let marker_purge = DeletedObject {
496            delete_marker: false,
497            delete_marker_version_id: Some(Uuid::new_v4()),
498            ..Default::default()
499        };
500        assert!(!delete_replication_creates_marker(&marker_purge));
501
502        // Only a plain delete-marker creation may let the target mint one.
503        let marker_creation = DeletedObject {
504            delete_marker: true,
505            delete_marker_version_id: Some(Uuid::new_v4()),
506            ..Default::default()
507        };
508        assert!(delete_replication_creates_marker(&marker_creation));
509
510        let versionless = DeletedObject {
511            delete_marker: false,
512            ..Default::default()
513        };
514        assert!(!delete_replication_creates_marker(&versionless));
515    }
516
517    /// The resync scan rebuilds purge work items with `delete_marker: true`
518    /// (crates/replication/src/delete.rs `resync_existing_delete_replication_info`);
519    /// pin that this shape flows into purge — not marker-creation — semantics.
520    #[test]
521    fn resync_rebuilt_purge_entry_keeps_versioned_delete_semantics() {
522        let roi = ReplicateObjectInfo {
523            bucket: "bucket".to_string(),
524            name: "object".to_string(),
525            version_id: Some(Uuid::new_v4()),
526            delete_marker: true,
527            version_purge_status: VersionPurgeStatusType::Pending,
528            ..Default::default()
529        };
530
531        let info = resync_existing_delete_replication_info(&roi, "arn:target-a");
532
533        assert!(is_version_delete_replication(&info.delete_object));
534        assert!(
535            !delete_replication_creates_marker(&info.delete_object),
536            "a rebuilt purge must not re-mint delete markers on the target"
537        );
538
539        // Without a pending purge the rebuild is a marker creation again.
540        let roi = ReplicateObjectInfo {
541            delete_marker: true,
542            version_id: Some(Uuid::new_v4()),
543            version_purge_status: VersionPurgeStatusType::Empty,
544            ..roi
545        };
546        let info = resync_existing_delete_replication_info(&roi, "arn:target-a");
547        assert!(delete_replication_creates_marker(&info.delete_object));
548    }
549
550    #[test]
551    fn delete_marker_purge_retry_covers_version_purge_and_marker_creation() {
552        let version_purge = DeletedObject {
553            delete_marker: false,
554            delete_marker_version_id: Some(Uuid::new_v4()),
555            ..Default::default()
556        };
557        let marker_creation = DeletedObject {
558            delete_marker: true,
559            delete_marker_version_id: Some(Uuid::new_v4()),
560            ..Default::default()
561        };
562
563        assert!(should_retry_delete_marker_purge(&version_purge));
564        assert!(should_retry_delete_marker_purge(&marker_creation));
565
566        let marker_without_version = DeletedObject {
567            delete_marker: true,
568            ..Default::default()
569        };
570        assert!(!should_retry_delete_marker_purge(&marker_without_version));
571    }
572
573    #[test]
574    fn retryable_delete_replication_head_error_allows_expected_delete_marker_responses() {
575        assert!(!is_retryable_delete_replication_head_error(false, Some("405")));
576        assert!(!is_retryable_delete_replication_head_error(false, Some("MethodNotAllowed")));
577        assert!(!is_retryable_delete_replication_head_error(true, Some("NoSuchKey")));
578        assert!(is_retryable_delete_replication_head_error(false, Some("AccessDenied")));
579    }
580
581    /// P1-21 regression guard for the outcome formula. A fully successful
582    /// delete-marker replication must acknowledge its MRF entry: the formula
583    /// once carried `&& !requires_delayed_purge`, which pinned every
584    /// delete-marker entry to Missed and retained the whole backlog forever.
585    /// (Deterministically staging a marker-creation entry in the durable
586    /// journal from e2e would require saturating the worker queues, so the
587    /// formula is pinned here instead; the purge-intent replay half is pinned
588    /// by the delayed-purge e2e pair.)
589    #[test]
590    fn test_replicate_delete_outcome_is_not_held_hostage_by_the_delayed_purge() {
591        assert!(
592            replicate_delete_outcome(1, 1, true, true, &ReplicationStatusType::Completed),
593            "a completed delete-marker replication must be acknowledgeable even though a delayed purge watch is pending"
594        );
595        assert!(!replicate_delete_outcome(0, 0, true, true, &ReplicationStatusType::Completed));
596        assert!(!replicate_delete_outcome(2, 1, true, true, &ReplicationStatusType::Completed));
597        assert!(!replicate_delete_outcome(1, 1, false, true, &ReplicationStatusType::Completed));
598        assert!(!replicate_delete_outcome(1, 1, true, false, &ReplicationStatusType::Completed));
599        assert!(!replicate_delete_outcome(1, 1, true, true, &ReplicationStatusType::Failed));
600    }
601
602    #[test]
603    fn test_delete_marker_purge_mrf_entry_replays_through_the_stale_marker_branch() {
604        let delete_marker_version_id = Uuid::new_v4();
605        let dobj = DeletedObjectReplicationInfo {
606            delete_object: DeletedObject {
607                object_name: "doc.txt".to_string(),
608                // A version-purge flavored source event: the entry must still
609                // be reshaped as a marker-creation delete so replay funnels
610                // into the stale-marker branch instead of re-running the full
611                // delete replication (whose source-state stamping would fail
612                // against the already-purged version).
613                delete_marker: false,
614                version_id: Some(Uuid::new_v4()),
615                delete_marker_version_id: Some(delete_marker_version_id),
616                ..Default::default()
617            },
618            bucket: "bucket-a".to_string(),
619            ..Default::default()
620        };
621
622        let entry = delete_marker_purge_mrf_entry(&dobj, vec!["arn:a".to_string()]);
623
624        assert!(entry.delete_marker, "purge intents must replay as marker-creation deletes");
625        assert_eq!(entry.version_id, None, "the purged data version must not leak into the replay");
626        assert_eq!(entry.delete_marker_version_id, Some(delete_marker_version_id));
627        assert_eq!(
628            entry.target_arns,
629            vec!["arn:a".to_string()],
630            "only the targets whose purge failed may be retried"
631        );
632        assert_eq!(entry.retry_count, 0);
633        assert_eq!(entry.bucket, "bucket-a");
634        assert_eq!(entry.object, "doc.txt");
635        assert!(
636            entry.target_delete_marker_version_ids.is_empty(),
637            "no recorded target marker ids means the journal carries none"
638        );
639        assert!(!entry.target_delete_marker_version_ids_corrupt);
640    }
641
642    /// backlog#2290: a purge intent journaled to MRF must carry the marker
643    /// version ids the targets assigned, plus the fail-closed corrupt flag,
644    /// so a replay after restart addresses the same version the live path did.
645    #[test]
646    fn delete_marker_purge_mrf_entry_carries_target_assigned_marker_versions() {
647        let delete_marker_version_id = Uuid::new_v4();
648        let mut state = ReplicationState::default();
649        state
650            .target_delete_marker_version_ids
651            .insert("arn:a".to_string(), "remote-marker-a".to_string());
652        state
653            .target_delete_marker_version_ids
654            .insert("arn:b".to_string(), "remote-marker-b".to_string());
655        let mut dobj = DeletedObjectReplicationInfo {
656            delete_object: DeletedObject {
657                object_name: "doc.txt".to_string(),
658                delete_marker: false,
659                version_id: Some(Uuid::new_v4()),
660                delete_marker_version_id: Some(delete_marker_version_id),
661                replication_state: Some(state),
662                ..Default::default()
663            },
664            bucket: "bucket-a".to_string(),
665            ..Default::default()
666        };
667
668        let entry = delete_marker_purge_mrf_entry(&dobj, vec!["arn:a".to_string()]);
669        assert_eq!(
670            entry.target_delete_marker_version_ids,
671            HashMap::from([
672                ("arn:a".to_string(), "remote-marker-a".to_string()),
673                ("arn:b".to_string(), "remote-marker-b".to_string()),
674            ]),
675            "every recorded target marker id survives the journal, regardless of the retried ARN subset"
676        );
677        assert!(!entry.target_delete_marker_version_ids_corrupt);
678        assert_eq!(
679            delete_marker_purge_version_id(
680                Some(&ReplicationState {
681                    target_delete_marker_version_ids: entry.target_delete_marker_version_ids,
682                    ..Default::default()
683                }),
684                "arn:a",
685                delete_marker_version_id
686            ),
687            Some(Some("remote-marker-a".to_string()))
688        );
689
690        // The live path refuses to purge on inconsistent metadata and reports the target
691        // as failed; the journaled intent must keep refusing after a restart.
692        dobj.delete_object
693            .replication_state
694            .as_mut()
695            .expect("state was set above")
696            .target_delete_marker_version_ids_corrupt = true;
697        let entry = delete_marker_purge_mrf_entry(&dobj, vec!["arn:a".to_string()]);
698        assert!(entry.target_delete_marker_version_ids_corrupt);
699
700        // A delete without replication state journals an empty map.
701        dobj.delete_object.replication_state = None;
702        let entry = dobj.to_mrf_entry();
703        assert!(entry.target_delete_marker_version_ids.is_empty());
704        assert!(!entry.target_delete_marker_version_ids_corrupt);
705    }
706
707    #[test]
708    fn target_delete_version_id_preserves_explicit_null_purges() {
709        let version_id = Uuid::new_v4();
710
711        assert_eq!(target_delete_version_id(version_id, true), Some(version_id.to_string()));
712        assert_eq!(target_delete_version_id(Uuid::nil(), true).as_deref(), Some(NULL_VERSION_ID));
713        assert_eq!(target_delete_version_id(Uuid::nil(), false), None);
714    }
715
716    #[test]
717    fn delete_marker_purge_prefers_the_recorded_target_version() {
718        let source = Uuid::new_v4();
719        let arn = "arn:rustfs:replication::target:bucket";
720
721        // No recorded mapping: fall back to deriving from the source uuid.
722        assert_eq!(delete_marker_purge_version_id(None, arn, source), Some(Some(source.to_string())));
723
724        // Recorded mapping wins — a generic S3 target assigns its own id, so the
725        // derived one would purge the wrong version or nothing at all.
726        let mut state = ReplicationState::default();
727        state
728            .target_delete_marker_version_ids
729            .insert(arn.to_string(), "target-assigned-id".to_string());
730        assert_eq!(
731            delete_marker_purge_version_id(Some(&state), arn, source),
732            Some(Some("target-assigned-id".to_string()))
733        );
734
735        // A mapping recorded for a different ARN must not be reused.
736        assert_eq!(
737            delete_marker_purge_version_id(Some(&state), "arn:rustfs:replication::other:bucket", source),
738            Some(Some(source.to_string()))
739        );
740
741        // Inconsistent persisted metadata: refuse to purge rather than guess.
742        let mut corrupt = state.clone();
743        corrupt.target_delete_marker_version_ids_corrupt = true;
744        assert_eq!(delete_marker_purge_version_id(Some(&corrupt), arn, source), None);
745    }
746
747    #[test]
748    fn object_lock_denied_delete_is_recognized_by_code_and_reason() {
749        // The peer's deletion gate answers AccessDenied with the lock reason.
750        assert!(is_object_lock_denied_delete(
751            Some("AccessDenied"),
752            Some("Object is under GOVERNANCE retention and cannot be deleted until 2026-09-01T00:00:00Z")
753        ));
754        assert!(is_object_lock_denied_delete(
755            Some("AccessDenied"),
756            Some("Object has a legal hold and cannot be deleted. Remove the legal hold first.")
757        ));
758
759        // A plain policy denial (misconfigured replicator) is not a lock denial.
760        assert!(!is_object_lock_denied_delete(Some("AccessDenied"), Some("Access Denied.")));
761        assert!(!is_object_lock_denied_delete(Some("AccessDenied"), None));
762        // Other errors mentioning retention must not match.
763        assert!(!is_object_lock_denied_delete(Some("InternalError"), Some("retention lookup failed")));
764        assert!(!is_object_lock_denied_delete(None, Some("legal hold")));
765    }
766
767    fn purge_of_marker(marker: Uuid, state: Option<ReplicationState>) -> DeletedObject {
768        DeletedObject {
769            object_name: "obj".to_string(),
770            delete_marker: false,
771            delete_marker_version_id: Some(marker),
772            version_id: None,
773            replication_state: state,
774            ..Default::default()
775        }
776    }
777
778    #[test]
779    fn delete_replication_target_version_id_addresses_recorded_marker_for_purges() {
780        let arn = "arn:minio:replication::generic:photos";
781        let marker = Uuid::new_v4();
782        let mut state = ReplicationState::default();
783        state
784            .target_delete_marker_version_ids
785            .insert(arn.to_string(), "remote-marker".to_string());
786
787        // purge of a replicated marker: the target's own version
788        assert_eq!(
789            delete_replication_target_version_id(&purge_of_marker(marker, Some(state.clone())), arn),
790            Some(Some("remote-marker".to_string()))
791        );
792        // nothing recorded for this arn: the source-derived id (id-mirroring peers)
793        assert_eq!(
794            delete_replication_target_version_id(&purge_of_marker(marker, None), arn),
795            Some(Some(marker.to_string()))
796        );
797        // corrupt record: refuse instead of guessing
798        state.target_delete_marker_version_ids_corrupt = true;
799        assert_eq!(delete_replication_target_version_id(&purge_of_marker(marker, Some(state)), arn), None);
800
801        // marker creation keeps the source id (the target mints its own on a
802        // versionless DELETE; the id only travels in the source header)
803        let creation = DeletedObject {
804            object_name: "obj".to_string(),
805            delete_marker: true,
806            delete_marker_version_id: Some(marker),
807            ..Default::default()
808        };
809        assert_eq!(delete_replication_target_version_id(&creation, arn), Some(Some(marker.to_string())));
810        // plain version purge: the source version id
811        let version = Uuid::new_v4();
812        let purge = DeletedObject {
813            object_name: "obj".to_string(),
814            version_id: Some(version),
815            ..Default::default()
816        };
817        assert_eq!(delete_replication_target_version_id(&purge, arn), Some(Some(version.to_string())));
818    }
819}