Skip to main content

rustfs_replication/
resync.rs

1// Copyright 2024 RustFS Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::HashMap;
16use std::fmt;
17use std::io::{Cursor, Read, Write};
18
19use byteorder::{BigEndian, ByteOrder, LittleEndian};
20use rmp::Marker;
21use serde::{Deserialize, Serialize};
22use time::OffsetDateTime;
23
24pub const RESYNC_META_FORMAT: u16 = 1;
25pub const RESYNC_META_VERSION: u16 = 1;
26
27// A resync snapshot contains small fixed-shape records for configured targets. These
28// limits leave room for thousands of targets while bounding corrupt MessagePack work.
29const RESYNC_FILE_HEADER_LEN: usize = 4;
30pub const RESYNC_FILE_MAX_BYTES: usize = 16 * 1024 * 1024;
31const RESYNC_MSGP_MAX_BYTES: usize = RESYNC_FILE_MAX_BYTES - RESYNC_FILE_HEADER_LEN;
32const RESYNC_MSGP_MAX_ELEMENT_BYTES: usize = 1024 * 1024;
33const RESYNC_MSGP_MAX_COLLECTION_ITEMS: usize = 4096;
34const RESYNC_MSGP_MAX_VALUES: usize = 131_072;
35const RESYNC_MSGP_MAX_DEPTH: usize = 32;
36const MSGP_TIME_EXT_TYPE: i8 = 5;
37const MSGP_TIME_LEN: u8 = 12;
38pub const WIRE_ZERO_TIME_UNIX: i64 = -62_135_596_800;
39const RESYNC_ERROR_DETAIL_MAX_CHARS: usize = 512;
40const RESYNC_ERROR_REDACTED: &str = "[redacted sensitive resync error detail]";
41const RESYNC_ERROR_TRUNCATED_SUFFIX: &str = "... (truncated)";
42const RESYNC_ERROR_SENSITIVE_MARKERS: &[&str] = &[
43    "authorization",
44    "bearer",
45    "credential",
46    "secret",
47    "token",
48    "signature",
49    "password",
50];
51
52pub type Result<T> = std::result::Result<T, ResyncStateError>;
53
54/// Error type for the persisted resync/MRF state files (backlog#1845 step 7:
55/// renamed from the crate-generic `Error`, with `Io` kept typed instead of
56/// collapsing into a string so callers can still classify the io failure).
57#[derive(Debug)]
58pub enum ResyncStateError {
59    CorruptedFormat,
60    Io(std::io::Error),
61    Other(String),
62}
63
64impl ResyncStateError {
65    fn other(err: impl Into<String>) -> Self {
66        Self::Other(err.into())
67    }
68}
69
70impl fmt::Display for ResyncStateError {
71    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
72        match self {
73            Self::CorruptedFormat => write!(f, "corrupted format"),
74            Self::Io(err) => write!(f, "{err}"),
75            Self::Other(err) => write!(f, "{err}"),
76        }
77    }
78}
79
80impl std::error::Error for ResyncStateError {
81    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
82        match self {
83            Self::Io(err) => Some(err),
84            _ => None,
85        }
86    }
87}
88
89impl From<std::io::Error> for ResyncStateError {
90    fn from(err: std::io::Error) -> Self {
91        Self::Io(err)
92    }
93}
94
95impl From<std::string::FromUtf8Error> for ResyncStateError {
96    fn from(err: std::string::FromUtf8Error) -> Self {
97        Self::other(err.to_string())
98    }
99}
100
101impl From<rmp::encode::ValueWriteError> for ResyncStateError {
102    fn from(err: rmp::encode::ValueWriteError) -> Self {
103        Self::other(err.to_string())
104    }
105}
106
107impl From<rmp::decode::ValueReadError> for ResyncStateError {
108    fn from(err: rmp::decode::ValueReadError) -> Self {
109        Self::other(err.to_string())
110    }
111}
112
113impl From<rmp::decode::NumValueReadError> for ResyncStateError {
114    fn from(err: rmp::decode::NumValueReadError) -> Self {
115        Self::other(err.to_string())
116    }
117}
118
119impl From<rmp_serde::decode::Error> for ResyncStateError {
120    fn from(err: rmp_serde::decode::Error) -> Self {
121        Self::other(err.to_string())
122    }
123}
124
125#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
126pub enum ResyncStatusType {
127    #[default]
128    NoResync,
129    ResyncPending,
130    ResyncCanceled,
131    ResyncStarted,
132    ResyncCompleted,
133    ResyncFailed,
134}
135
136impl ResyncStatusType {
137    pub fn is_valid(&self) -> bool {
138        *self != ResyncStatusType::NoResync
139    }
140}
141
142pub fn should_auto_resume_resync(status: ResyncStatusType) -> bool {
143    matches!(status, ResyncStatusType::ResyncPending | ResyncStatusType::ResyncStarted)
144}
145
146impl fmt::Display for ResyncStatusType {
147    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
148        let s = match self {
149            ResyncStatusType::ResyncStarted => "Ongoing",
150            ResyncStatusType::ResyncCompleted => "Completed",
151            ResyncStatusType::ResyncFailed => "Failed",
152            ResyncStatusType::ResyncPending => "Pending",
153            ResyncStatusType::ResyncCanceled => "Canceled",
154            ResyncStatusType::NoResync => "",
155        };
156        write!(f, "{s}")
157    }
158}
159
160#[derive(Debug, Clone, Default)]
161pub struct ResyncOpts {
162    pub bucket: String,
163    pub arn: String,
164    pub resync_id: String,
165    pub resync_before: Option<OffsetDateTime>,
166}
167
168#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq)]
169pub struct TargetReplicationResyncStatus {
170    pub start_time: Option<OffsetDateTime>,
171    pub last_update: Option<OffsetDateTime>,
172    pub resync_id: String,
173    pub resync_before_date: Option<OffsetDateTime>,
174    pub resync_status: ResyncStatusType,
175    pub failed_size: i64,
176    pub failed_count: i64,
177    pub replicated_size: i64,
178    pub replicated_count: i64,
179    pub bucket: String,
180    pub object: String,
181    pub error: Option<String>,
182}
183
184impl TargetReplicationResyncStatus {
185    pub fn new() -> Self {
186        Self::default()
187    }
188
189    fn marshal_wire_msg(&self, wr: &mut Vec<u8>) -> Result<()> {
190        rmp::encode::write_map_len(wr, 12)?;
191        rmp::encode::write_str(wr, "st")?;
192        write_msgp_time(wr, wire_time_or_default(self.start_time))?;
193        rmp::encode::write_str(wr, "lst")?;
194        write_msgp_time(wr, wire_time_or_default(self.last_update))?;
195        rmp::encode::write_str(wr, "id")?;
196        rmp::encode::write_str(wr, &self.resync_id)?;
197        rmp::encode::write_str(wr, "rdt")?;
198        write_msgp_time(wr, wire_time_or_default(self.resync_before_date))?;
199        rmp::encode::write_str(wr, "rst")?;
200        rmp::encode::write_i32(wr, resync_status_to_i32(self.resync_status))?;
201        rmp::encode::write_str(wr, "fs")?;
202        rmp::encode::write_i64(wr, self.failed_size)?;
203        rmp::encode::write_str(wr, "frc")?;
204        rmp::encode::write_i64(wr, self.failed_count)?;
205        rmp::encode::write_str(wr, "rs")?;
206        rmp::encode::write_i64(wr, self.replicated_size)?;
207        rmp::encode::write_str(wr, "rrc")?;
208        rmp::encode::write_i64(wr, self.replicated_count)?;
209        rmp::encode::write_str(wr, "bkt")?;
210        rmp::encode::write_str(wr, &self.bucket)?;
211        rmp::encode::write_str(wr, "obj")?;
212        rmp::encode::write_str(wr, &self.object)?;
213        rmp::encode::write_str(wr, "err")?;
214        let error = self.error.as_deref().and_then(sanitize_resync_error_detail);
215        rmp::encode::write_str(wr, error.as_deref().unwrap_or_default())?;
216        Ok(())
217    }
218
219    fn unmarshal_wire_msg<R: Read>(rd: &mut R) -> Result<Self> {
220        let mut out = Self::new();
221        let mut fields = rmp::decode::read_map_len(rd)?;
222
223        while fields > 0 {
224            fields -= 1;
225            let key = read_msgp_str(rd)?;
226            match key.as_str() {
227                "st" => out.start_time = normalize_wire_time(read_msgp_time_or_nil(rd)?),
228                "lst" => out.last_update = normalize_wire_time(read_msgp_time_or_nil(rd)?),
229                "id" => out.resync_id = read_msgp_str(rd)?,
230                "rdt" => out.resync_before_date = normalize_wire_time(read_msgp_time_or_nil(rd)?),
231                "rst" => {
232                    let v: i32 = rmp::decode::read_int(rd)?;
233                    out.resync_status = resync_status_from_i32(v)?;
234                }
235                "fs" => out.failed_size = rmp::decode::read_int(rd)?,
236                "frc" => out.failed_count = rmp::decode::read_int(rd)?,
237                "rs" => out.replicated_size = rmp::decode::read_int(rd)?,
238                "rrc" => out.replicated_count = rmp::decode::read_int(rd)?,
239                "bkt" => out.bucket = read_msgp_str(rd)?,
240                "obj" => out.object = read_msgp_str(rd)?,
241                "err" => {
242                    let error = read_msgp_str(rd)?;
243                    out.error = sanitize_resync_error_detail(&error);
244                }
245                _ => skip_msgp_value(rd)?,
246            }
247        }
248        Ok(out)
249    }
250}
251
252pub fn sanitize_resync_error_detail(detail: &str) -> Option<String> {
253    let mut normalized = String::new();
254    let mut char_count = 0usize;
255    let mut truncated = false;
256    'words: for word in detail.split_whitespace() {
257        if !normalized.is_empty() {
258            if char_count == RESYNC_ERROR_DETAIL_MAX_CHARS {
259                truncated = true;
260                break;
261            }
262            normalized.push(' ');
263            char_count += 1;
264        }
265        for ch in word.chars() {
266            if char_count == RESYNC_ERROR_DETAIL_MAX_CHARS {
267                truncated = true;
268                break 'words;
269            }
270            normalized.push(ch);
271            char_count += 1;
272        }
273    }
274    if normalized.is_empty() {
275        return None;
276    }
277
278    let lower = normalized.to_ascii_lowercase();
279    let is_service_code = normalized.len() <= 128
280        && normalized
281            .bytes()
282            .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_'));
283    if !is_service_code
284        && (RESYNC_ERROR_SENSITIVE_MARKERS.iter().any(|marker| lower.contains(marker))
285            || (lower.contains("://") && lower.contains('@')))
286    {
287        return Some(RESYNC_ERROR_REDACTED.to_string());
288    }
289
290    if !truncated {
291        return Some(normalized);
292    }
293
294    let keep = RESYNC_ERROR_DETAIL_MAX_CHARS.saturating_sub(RESYNC_ERROR_TRUNCATED_SUFFIX.chars().count());
295    let mut summary: String = normalized.chars().take(keep).collect();
296    summary.push_str(RESYNC_ERROR_TRUNCATED_SUFFIX);
297    Some(summary)
298}
299
300pub fn resync_state_accepts_update(state: &TargetReplicationResyncStatus, opts: &ResyncOpts) -> bool {
301    state.resync_id.is_empty() || state.resync_id == opts.resync_id
302}
303
304pub fn should_count_head_proxy_failure(is_not_found: bool, code: Option<&str>, raw_status: Option<u16>) -> bool {
305    if is_not_found || matches!(code, Some("MethodNotAllowed" | "405")) {
306        return false;
307    }
308    if matches!(raw_status, Some(404 | 405)) {
309        return false;
310    }
311    !is_version_id_mismatch(code, raw_status)
312}
313
314// AWS returns 400 for root callers and 403 for IAM users when a UUID version ID
315// is rejected. The 403 case is safe: a real auth failure also returns 403 on the
316// versionId-less fallback, propagating as a hard error instead of silently skipping.
317pub fn is_version_id_mismatch(code: Option<&str>, raw_status: Option<u16>) -> bool {
318    match code {
319        Some(c) if !c.is_empty() => c == "InvalidArgument",
320        _ => matches!(raw_status, Some(400 | 403)),
321    }
322}
323
324pub fn resync_status_duration(
325    status: ResyncStatusType,
326    start_time: Option<OffsetDateTime>,
327    now: OffsetDateTime,
328) -> Option<std::time::Duration> {
329    if !matches!(
330        status,
331        ResyncStatusType::ResyncCompleted | ResyncStatusType::ResyncFailed | ResyncStatusType::ResyncCanceled
332    ) {
333        return None;
334    }
335
336    let millis = (now - start_time?).whole_milliseconds();
337    if millis < 0 {
338        return None;
339    }
340
341    let millis = if millis > i128::from(u64::MAX) {
342        u64::MAX
343    } else {
344        u64::try_from(millis).ok()?
345    };
346    Some(std::time::Duration::from_millis(millis))
347}
348
349#[derive(Debug, Clone, Serialize, Deserialize, Default)]
350pub struct BucketReplicationResyncStatus {
351    pub version: u16,
352    pub targets_map: HashMap<String, TargetReplicationResyncStatus>,
353    pub id: i32,
354    pub last_update: Option<OffsetDateTime>,
355}
356
357impl BucketReplicationResyncStatus {
358    pub fn new() -> Self {
359        Self {
360            version: RESYNC_META_VERSION,
361            ..Default::default()
362        }
363    }
364
365    pub fn clone_tgt_stats(&self) -> HashMap<String, TargetReplicationResyncStatus> {
366        self.targets_map.clone()
367    }
368
369    pub fn marshal_msg(&self) -> Result<Vec<u8>> {
370        let mut wr = Vec::new();
371        rmp::encode::write_map_len(&mut wr, 4)?;
372        rmp::encode::write_str(&mut wr, "v")?;
373        rmp::encode::write_i32(&mut wr, i32::from(self.version))?;
374        rmp::encode::write_str(&mut wr, "brs")?;
375        let target_count = u32::try_from(self.targets_map.len()).map_err(|_| ResyncStateError::CorruptedFormat)?;
376        rmp::encode::write_map_len(&mut wr, target_count)?;
377        for (arn, status) in &self.targets_map {
378            rmp::encode::write_str(&mut wr, arn)?;
379            status.marshal_wire_msg(&mut wr)?;
380        }
381        rmp::encode::write_str(&mut wr, "id")?;
382        rmp::encode::write_i32(&mut wr, self.id)?;
383        rmp::encode::write_str(&mut wr, "lu")?;
384        write_msgp_time(&mut wr, wire_time_or_default(self.last_update))?;
385        validate_msgp_payload(&wr)?;
386        Ok(wr)
387    }
388
389    pub fn unmarshal_msg(data: &[u8]) -> Result<Self> {
390        validate_msgp_payload(data)?;
391        let mut rd = Cursor::new(data);
392        let mut out = Self::new();
393        let mut fields = rmp::decode::read_map_len(&mut rd)?;
394
395        while fields > 0 {
396            fields -= 1;
397            let key = read_msgp_str(&mut rd)?;
398            match key.as_str() {
399                "v" => {
400                    let v: i32 = rmp::decode::read_int(&mut rd)?;
401                    out.version = u16::try_from(v).map_err(|_| ResyncStateError::other("invalid resync version"))?;
402                }
403                "brs" => {
404                    let map_len = rmp::decode::read_map_len(&mut rd)?;
405                    let target_count = usize::try_from(map_len).map_err(|_| ResyncStateError::CorruptedFormat)?;
406                    let mut targets = HashMap::with_capacity(target_count);
407                    for _ in 0..map_len {
408                        let arn = read_msgp_str(&mut rd)?;
409                        let status = TargetReplicationResyncStatus::unmarshal_wire_msg(&mut rd)?;
410                        targets.insert(arn, status);
411                    }
412                    out.targets_map = targets;
413                }
414                "id" => {
415                    out.id = rmp::decode::read_int::<i32, _>(&mut rd)?;
416                }
417                "lu" => {
418                    out.last_update = normalize_wire_time(read_msgp_time_or_nil(&mut rd)?);
419                }
420                _ => skip_msgp_value(&mut rd)?,
421            }
422        }
423        Ok(out)
424    }
425
426    pub fn unmarshal_legacy_msg(data: &[u8]) -> Result<Self> {
427        validate_msgp_payload(data)?;
428        let mut status: Self = rmp_serde::from_slice(data)?;
429        for target in status.targets_map.values_mut() {
430            target.error = target.error.as_deref().and_then(sanitize_resync_error_detail);
431        }
432        Ok(status)
433    }
434}
435
436pub fn encode_resync_file(status: &BucketReplicationResyncStatus) -> Result<Vec<u8>> {
437    let payload = status.marshal_msg()?;
438    let mut data = Vec::with_capacity(RESYNC_FILE_HEADER_LEN + payload.len());
439    let mut major = [0u8; 2];
440    LittleEndian::write_u16(&mut major, RESYNC_META_FORMAT);
441    data.extend_from_slice(&major);
442    let mut minor = [0u8; 2];
443    LittleEndian::write_u16(&mut minor, RESYNC_META_VERSION);
444    data.extend_from_slice(&minor);
445    data.extend_from_slice(&payload);
446    Ok(data)
447}
448
449pub fn decode_resync_file(data: &[u8]) -> Result<BucketReplicationResyncStatus> {
450    if data.len() <= RESYNC_FILE_HEADER_LEN || data.len() > RESYNC_FILE_MAX_BYTES {
451        return Err(ResyncStateError::CorruptedFormat);
452    }
453
454    let mut major = [0u8; 2];
455    major.copy_from_slice(&data[0..2]);
456    if LittleEndian::read_u16(&major) != RESYNC_META_FORMAT {
457        return Err(ResyncStateError::CorruptedFormat);
458    }
459
460    let mut minor = [0u8; 2];
461    minor.copy_from_slice(&data[2..4]);
462    if LittleEndian::read_u16(&minor) != RESYNC_META_VERSION {
463        return Err(ResyncStateError::CorruptedFormat);
464    }
465
466    let status = match BucketReplicationResyncStatus::unmarshal_msg(&data[4..]) {
467        Ok(v) => v,
468        Err(_) => BucketReplicationResyncStatus::unmarshal_legacy_msg(&data[4..])?,
469    };
470    if status.version != RESYNC_META_VERSION {
471        return Err(ResyncStateError::CorruptedFormat);
472    }
473    Ok(status)
474}
475
476fn wire_time_or_default(value: Option<OffsetDateTime>) -> OffsetDateTime {
477    value.unwrap_or_else(wire_zero_time)
478}
479
480fn normalize_wire_time(value: Option<OffsetDateTime>) -> Option<OffsetDateTime> {
481    match value {
482        Some(v) if v == wire_zero_time() || v == OffsetDateTime::UNIX_EPOCH => None,
483        other => other,
484    }
485}
486
487fn wire_zero_time() -> OffsetDateTime {
488    OffsetDateTime::from_unix_timestamp(WIRE_ZERO_TIME_UNIX).unwrap_or(OffsetDateTime::UNIX_EPOCH)
489}
490
491fn validate_msgp_payload(data: &[u8]) -> Result<()> {
492    if data.len() > RESYNC_MSGP_MAX_BYTES {
493        return Err(ResyncStateError::CorruptedFormat);
494    }
495
496    let mut rd = Cursor::new(data);
497    let mut values = 0usize;
498    validate_msgp_value(&mut rd, 0, &mut values)?;
499    if usize::try_from(rd.position()).ok() != Some(data.len()) {
500        return Err(ResyncStateError::CorruptedFormat);
501    }
502    Ok(())
503}
504
505fn validate_msgp_value<R: Read>(rd: &mut R, depth: usize, values: &mut usize) -> Result<()> {
506    if depth > RESYNC_MSGP_MAX_DEPTH {
507        return Err(ResyncStateError::CorruptedFormat);
508    }
509    *values = values.checked_add(1).ok_or(ResyncStateError::CorruptedFormat)?;
510    if *values > RESYNC_MSGP_MAX_VALUES {
511        return Err(ResyncStateError::CorruptedFormat);
512    }
513
514    let marker = rmp::decode::read_marker(rd).map_err(|e| ResyncStateError::other(format!("{e:?}")))?;
515    let skip_len = match marker {
516        Marker::Null | Marker::False | Marker::True | Marker::FixPos(_) | Marker::FixNeg(_) => 0,
517        Marker::U8 | Marker::I8 => 1,
518        Marker::U16 | Marker::I16 => 2,
519        Marker::U32 | Marker::I32 | Marker::F32 => 4,
520        Marker::U64 | Marker::I64 | Marker::F64 => 8,
521        Marker::FixStr(len) => usize::from(len),
522        Marker::Str8 | Marker::Bin8 => read_skip_len(rd, 1)?,
523        Marker::Str16 | Marker::Bin16 => read_skip_len(rd, 2)?,
524        Marker::Str32 | Marker::Bin32 => read_skip_len(rd, 4)?,
525        Marker::FixArray(len) => {
526            validate_msgp_collection(rd, usize::from(len), depth, values, false)?;
527            return Ok(());
528        }
529        Marker::Array16 => {
530            let len = read_skip_len(rd, 2)?;
531            validate_msgp_collection(rd, len, depth, values, false)?;
532            return Ok(());
533        }
534        Marker::Array32 => {
535            let len = read_skip_len(rd, 4)?;
536            validate_msgp_collection(rd, len, depth, values, false)?;
537            return Ok(());
538        }
539        Marker::FixMap(len) => {
540            validate_msgp_collection(rd, usize::from(len), depth, values, true)?;
541            return Ok(());
542        }
543        Marker::Map16 => {
544            let len = read_skip_len(rd, 2)?;
545            validate_msgp_collection(rd, len, depth, values, true)?;
546            return Ok(());
547        }
548        Marker::Map32 => {
549            let len = read_skip_len(rd, 4)?;
550            validate_msgp_collection(rd, len, depth, values, true)?;
551            return Ok(());
552        }
553        Marker::FixExt1 => 2,
554        Marker::FixExt2 => 3,
555        Marker::FixExt4 => 5,
556        Marker::FixExt8 => 9,
557        Marker::FixExt16 => 17,
558        Marker::Ext8 => {
559            let len = validate_msgp_element_len(read_skip_len(rd, 1)?)?;
560            skip_exact(rd, 1)?;
561            return skip_exact(rd, len);
562        }
563        Marker::Ext16 => {
564            let len = validate_msgp_element_len(read_skip_len(rd, 2)?)?;
565            skip_exact(rd, 1)?;
566            return skip_exact(rd, len);
567        }
568        Marker::Ext32 => {
569            let len = validate_msgp_element_len(read_skip_len(rd, 4)?)?;
570            skip_exact(rd, 1)?;
571            return skip_exact(rd, len);
572        }
573        Marker::Reserved => return Err(ResyncStateError::CorruptedFormat),
574    };
575    let skip_len = validate_msgp_element_len(skip_len)?;
576    skip_exact(rd, skip_len)
577}
578
579fn validate_msgp_collection<R: Read>(rd: &mut R, len: usize, depth: usize, values: &mut usize, is_map: bool) -> Result<()> {
580    if len > RESYNC_MSGP_MAX_COLLECTION_ITEMS {
581        return Err(ResyncStateError::CorruptedFormat);
582    }
583    let values_per_item = if is_map { 2 } else { 1 };
584    let child_count = len.checked_mul(values_per_item).ok_or(ResyncStateError::CorruptedFormat)?;
585    for _ in 0..child_count {
586        validate_msgp_value(rd, depth + 1, values)?;
587    }
588    Ok(())
589}
590
591fn validate_msgp_element_len(len: usize) -> Result<usize> {
592    if len > RESYNC_MSGP_MAX_ELEMENT_BYTES {
593        return Err(ResyncStateError::CorruptedFormat);
594    }
595    Ok(len)
596}
597
598fn read_msgp_str<R: Read>(rd: &mut R) -> Result<String> {
599    let len = rmp::decode::read_str_len(rd)? as usize;
600    let mut buf = vec![0u8; len];
601    rd.read_exact(&mut buf)?;
602    Ok(String::from_utf8(buf)?)
603}
604
605fn read_msgp_time_or_nil<R: Read>(rd: &mut R) -> Result<Option<OffsetDateTime>> {
606    let marker = rmp::decode::read_marker(rd).map_err(|e| ResyncStateError::other(format!("{e:?}")))?;
607    match marker {
608        Marker::Null => Ok(None),
609        Marker::Ext8 => Ok(Some(read_msgp_ext8_time(rd)?)),
610        other => Err(ResyncStateError::other(format!("expected time ext or nil, got marker: {other:?}"))),
611    }
612}
613
614fn read_msgp_ext8_time<R: Read>(rd: &mut R) -> Result<OffsetDateTime> {
615    let mut len_buf = [0u8; 1];
616    rd.read_exact(&mut len_buf)?;
617    let len = len_buf[0] as usize;
618    if len != MSGP_TIME_LEN as usize {
619        return Err(ResyncStateError::other(format!("invalid msgp time len: {len}")));
620    }
621    let mut type_buf = [0u8; 1];
622    rd.read_exact(&mut type_buf)?;
623    if type_buf[0] != MSGP_TIME_EXT_TYPE as u8 {
624        return Err(ResyncStateError::other(format!("invalid msgp time type: {}", type_buf[0])));
625    }
626    let mut buf = [0u8; 12];
627    rd.read_exact(&mut buf)?;
628    let sec = BigEndian::read_i64(&buf[0..8]);
629    let nsec = BigEndian::read_u32(&buf[8..12]);
630    OffsetDateTime::from_unix_timestamp(sec)
631        .map_err(|_| ResyncStateError::other("invalid timestamp"))?
632        .replace_nanosecond(nsec)
633        .map_err(|_| ResyncStateError::other("invalid nanosecond"))
634}
635
636fn write_msgp_time<W: Write>(wr: &mut W, time: OffsetDateTime) -> Result<()> {
637    wr.write_all(&[0xc7, MSGP_TIME_LEN, MSGP_TIME_EXT_TYPE as u8])?;
638    let mut buf = [0u8; 12];
639    BigEndian::write_i64(&mut buf[0..8], time.unix_timestamp());
640    BigEndian::write_u32(&mut buf[8..12], time.nanosecond());
641    wr.write_all(&buf)?;
642    Ok(())
643}
644
645fn skip_msgp_value<R: Read>(rd: &mut R) -> Result<()> {
646    let mut values = 0usize;
647    validate_msgp_value(rd, 0, &mut values)
648}
649
650fn skip_exact<R: Read>(rd: &mut R, mut len: usize) -> Result<()> {
651    let mut buf = [0u8; 1024];
652    while len > 0 {
653        let take = len.min(buf.len());
654        rd.read_exact(&mut buf[..take])?;
655        len -= take;
656    }
657    Ok(())
658}
659
660fn read_skip_len<R: Read>(rd: &mut R, bytes: usize) -> Result<usize> {
661    match bytes {
662        1 => {
663            let mut buf = [0u8; 1];
664            rd.read_exact(&mut buf)?;
665            Ok(usize::from(buf[0]))
666        }
667        2 => {
668            let mut buf = [0u8; 2];
669            rd.read_exact(&mut buf)?;
670            Ok(usize::from(u16::from_be_bytes(buf)))
671        }
672        4 => {
673            let mut buf = [0u8; 4];
674            rd.read_exact(&mut buf)?;
675            usize::try_from(u32::from_be_bytes(buf)).map_err(|_| ResyncStateError::CorruptedFormat)
676        }
677        _ => Err(ResyncStateError::other("invalid MessagePack length width")),
678    }
679}
680
681fn resync_status_to_i32(status: ResyncStatusType) -> i32 {
682    match status {
683        ResyncStatusType::NoResync => 0,
684        ResyncStatusType::ResyncPending => 1,
685        ResyncStatusType::ResyncCanceled => 2,
686        ResyncStatusType::ResyncStarted => 3,
687        ResyncStatusType::ResyncCompleted => 4,
688        ResyncStatusType::ResyncFailed => 5,
689    }
690}
691
692fn resync_status_from_i32(code: i32) -> Result<ResyncStatusType> {
693    match code {
694        0 => Ok(ResyncStatusType::NoResync),
695        1 => Ok(ResyncStatusType::ResyncPending),
696        2 => Ok(ResyncStatusType::ResyncCanceled),
697        3 => Ok(ResyncStatusType::ResyncStarted),
698        4 => Ok(ResyncStatusType::ResyncCompleted),
699        5 => Ok(ResyncStatusType::ResyncFailed),
700        _ => Err(ResyncStateError::other(format!("invalid resync status code: {code}"))),
701    }
702}
703
704#[cfg(test)]
705mod tests {
706    use super::*;
707
708    fn wrap_resync_payload(payload: &[u8]) -> Vec<u8> {
709        let mut data = Vec::with_capacity(RESYNC_FILE_HEADER_LEN + payload.len());
710        data.extend_from_slice(&RESYNC_META_FORMAT.to_le_bytes());
711        data.extend_from_slice(&RESYNC_META_VERSION.to_le_bytes());
712        data.extend_from_slice(payload);
713        data
714    }
715
716    fn resync_file_with_unknown_value(value: &[u8]) -> Vec<u8> {
717        let mut payload = Vec::with_capacity(32 + value.len());
718        rmp::encode::write_map_len(&mut payload, 2).expect("test payload map length should encode");
719        rmp::encode::write_str(&mut payload, "v").expect("test version key should encode");
720        rmp::encode::write_i32(&mut payload, i32::from(RESYNC_META_VERSION)).expect("test version value should encode");
721        rmp::encode::write_str(&mut payload, "future").expect("test unknown key should encode");
722        payload.extend_from_slice(value);
723        wrap_resync_payload(&payload)
724    }
725
726    fn msgp_str32(len: usize) -> Vec<u8> {
727        let wire_len = u32::try_from(len).expect("test string length should fit u32");
728        let mut value = Vec::with_capacity(5 + len);
729        value.push(0xdb);
730        value.extend_from_slice(&wire_len.to_be_bytes());
731        value.resize(5 + len, b'x');
732        value
733    }
734
735    #[test]
736    fn resync_status_display_matches_admin_contract() {
737        assert_eq!(ResyncStatusType::ResyncStarted.to_string(), "Ongoing");
738        assert_eq!(ResyncStatusType::ResyncCompleted.to_string(), "Completed");
739        assert_eq!(ResyncStatusType::NoResync.to_string(), "");
740    }
741
742    #[test]
743    fn auto_resume_resync_only_for_inflight_states() {
744        assert!(should_auto_resume_resync(ResyncStatusType::ResyncPending));
745        assert!(should_auto_resume_resync(ResyncStatusType::ResyncStarted));
746        assert!(!should_auto_resume_resync(ResyncStatusType::NoResync));
747        assert!(!should_auto_resume_resync(ResyncStatusType::ResyncCanceled));
748        assert!(!should_auto_resume_resync(ResyncStatusType::ResyncCompleted));
749        assert!(!should_auto_resume_resync(ResyncStatusType::ResyncFailed));
750    }
751
752    #[test]
753    fn test_resync_status_duration_only_tracks_terminal_status() {
754        let start = match OffsetDateTime::from_unix_timestamp(1_700_000_000) {
755            Ok(start) => start,
756            Err(err) => panic!("valid test timestamp: {err}"),
757        };
758        let end = start + time::Duration::seconds(2);
759
760        assert_eq!(
761            resync_status_duration(ResyncStatusType::ResyncCompleted, Some(start), end),
762            Some(std::time::Duration::from_millis(2000))
763        );
764        assert_eq!(resync_status_duration(ResyncStatusType::ResyncStarted, Some(start), end), None);
765        assert_eq!(resync_status_duration(ResyncStatusType::ResyncFailed, None, end), None);
766    }
767
768    #[test]
769    fn resync_state_accepts_update_only_for_matching_run() {
770        let current = TargetReplicationResyncStatus {
771            resync_id: "run-new".to_string(),
772            ..Default::default()
773        };
774        let matching = ResyncOpts {
775            bucket: "bucket".to_string(),
776            arn: "arn:replication::dest".to_string(),
777            resync_id: "run-new".to_string(),
778            resync_before: None,
779        };
780        let stale = ResyncOpts {
781            bucket: "bucket".to_string(),
782            arn: "arn:replication::dest".to_string(),
783            resync_id: "run-old".to_string(),
784            resync_before: None,
785        };
786
787        assert!(resync_state_accepts_update(&TargetReplicationResyncStatus::default(), &matching));
788        assert!(resync_state_accepts_update(&current, &matching));
789        assert!(!resync_state_accepts_update(&current, &stale));
790        assert!(!resync_state_accepts_update(
791            &current,
792            &ResyncOpts {
793                resync_id: String::new(),
794                ..matching
795            }
796        ));
797    }
798
799    #[test]
800    fn head_proxy_failure_ignores_expected_target_responses() {
801        assert!(!should_count_head_proxy_failure(true, Some("NoSuchKey"), Some(404)));
802        assert!(!should_count_head_proxy_failure(false, Some("MethodNotAllowed"), Some(405)));
803        assert!(!should_count_head_proxy_failure(false, Some("405"), Some(405)));
804        assert!(!should_count_head_proxy_failure(false, Some("InvalidArgument"), Some(400)));
805        assert!(!should_count_head_proxy_failure(false, None, Some(400)));
806        assert!(!should_count_head_proxy_failure(false, None, Some(403)));
807    }
808
809    #[test]
810    fn head_proxy_failure_counts_unexpected_errors() {
811        assert!(should_count_head_proxy_failure(false, Some("AccessDenied"), Some(403)));
812        assert!(should_count_head_proxy_failure(false, None, Some(500)));
813    }
814
815    #[test]
816    fn version_id_mismatch_detects_aws_rejections() {
817        assert!(is_version_id_mismatch(Some("InvalidArgument"), Some(400)));
818        assert!(is_version_id_mismatch(None, Some(400)));
819        assert!(is_version_id_mismatch(Some(""), Some(400)));
820        assert!(is_version_id_mismatch(None, Some(403)));
821        assert!(is_version_id_mismatch(Some(""), Some(403)));
822        assert!(!is_version_id_mismatch(Some("AccessDenied"), Some(403)));
823        assert!(!is_version_id_mismatch(Some("NoSuchKey"), Some(404)));
824        assert!(!is_version_id_mismatch(Some("MalformedXML"), Some(400)));
825        assert!(!is_version_id_mismatch(Some("EntityTooLarge"), Some(400)));
826        assert!(!is_version_id_mismatch(None, Some(500)));
827        assert!(!is_version_id_mismatch(None, Some(404)));
828    }
829
830    #[test]
831    fn resync_file_round_trips_status() {
832        let mut status = BucketReplicationResyncStatus::new();
833        status.targets_map.insert(
834            "arn:replication:a".to_string(),
835            TargetReplicationResyncStatus {
836                resync_id: "rid-1".to_string(),
837                resync_status: ResyncStatusType::ResyncStarted,
838                bucket: "bucket-a".to_string(),
839                object: "object-a".to_string(),
840                replicated_count: 7,
841                error: Some("durable failure".to_string()),
842                ..Default::default()
843            },
844        );
845
846        let data = encode_resync_file(&status).expect("resync status should encode");
847        let got = decode_resync_file(&data).expect("resync status should decode");
848
849        assert_eq!(got.version, RESYNC_META_VERSION);
850        assert_eq!(got.targets_map["arn:replication:a"].resync_id, "rid-1");
851        assert_eq!(got.targets_map["arn:replication:a"].resync_status, ResyncStatusType::ResyncStarted);
852        assert_eq!(got.targets_map["arn:replication:a"].replicated_count, 7);
853        assert_eq!(got.targets_map["arn:replication:a"].error.as_deref(), Some("durable failure"));
854    }
855
856    #[test]
857    fn resync_file_rejects_trailing_messagepack_value() {
858        let status = BucketReplicationResyncStatus::new();
859        let mut data = encode_resync_file(&status).expect("resync status should encode");
860        data.push(0xc0);
861
862        assert!(matches!(decode_resync_file(&data), Err(ResyncStateError::CorruptedFormat)));
863    }
864
865    #[test]
866    fn resync_file_rejects_reserved_messagepack_marker() {
867        let data = resync_file_with_unknown_value(&[0xc1]);
868
869        assert!(matches!(decode_resync_file(&data), Err(ResyncStateError::CorruptedFormat)));
870    }
871
872    #[test]
873    fn resync_file_rejects_oversized_messagepack_element() {
874        let accepted = decode_resync_file(&resync_file_with_unknown_value(&msgp_str32(RESYNC_MSGP_MAX_ELEMENT_BYTES)))
875            .expect("element ending at the byte limit should decode");
876        assert_eq!(accepted.version, RESYNC_META_VERSION);
877
878        let value = msgp_str32(RESYNC_MSGP_MAX_ELEMENT_BYTES + 1);
879        let data = resync_file_with_unknown_value(&value);
880
881        assert!(matches!(decode_resync_file(&data), Err(ResyncStateError::CorruptedFormat)));
882    }
883
884    #[test]
885    fn resync_file_rejects_oversized_messagepack_collection() {
886        let accepted_wire_len = u32::try_from(RESYNC_MSGP_MAX_COLLECTION_ITEMS).expect("test collection length should fit u32");
887        let mut accepted = Vec::with_capacity(5 + RESYNC_MSGP_MAX_COLLECTION_ITEMS);
888        accepted.push(0xdd);
889        accepted.extend_from_slice(&accepted_wire_len.to_be_bytes());
890        accepted.resize(5 + RESYNC_MSGP_MAX_COLLECTION_ITEMS, 0xc0);
891        let mut accepted_map = Vec::with_capacity(5 + RESYNC_MSGP_MAX_COLLECTION_ITEMS * 2);
892        accepted_map.push(0xdf);
893        accepted_map.extend_from_slice(&accepted_wire_len.to_be_bytes());
894        accepted_map.resize(5 + RESYNC_MSGP_MAX_COLLECTION_ITEMS * 2, 0xc0);
895        for value in [accepted, accepted_map] {
896            let accepted = decode_resync_file(&resync_file_with_unknown_value(&value))
897                .expect("collection ending at the item limit should decode");
898            assert_eq!(accepted.version, RESYNC_META_VERSION);
899        }
900
901        let collection_len = RESYNC_MSGP_MAX_COLLECTION_ITEMS + 1;
902        let wire_len = u32::try_from(collection_len).expect("test collection length should fit u32");
903        let mut array = Vec::with_capacity(5 + collection_len);
904        array.push(0xdd);
905        array.extend_from_slice(&wire_len.to_be_bytes());
906        array.resize(5 + collection_len, 0xc0);
907        let mut map = Vec::with_capacity(5 + collection_len * 2);
908        map.push(0xdf);
909        map.extend_from_slice(&wire_len.to_be_bytes());
910        map.resize(5 + collection_len * 2, 0xc0);
911
912        for value in [array, map] {
913            let data = resync_file_with_unknown_value(&value);
914            assert!(matches!(decode_resync_file(&data), Err(ResyncStateError::CorruptedFormat)));
915        }
916    }
917
918    #[test]
919    fn resync_file_rejects_oversized_messagepack_extension() {
920        let extension = |len: usize| {
921            let wire_len = u32::try_from(len).expect("test extension length should fit u32");
922            let mut extension = Vec::with_capacity(6 + len);
923            extension.push(0xc9);
924            extension.extend_from_slice(&wire_len.to_be_bytes());
925            extension.push(u8::try_from(MSGP_TIME_EXT_TYPE).expect("test extension type should fit u8"));
926            extension.resize(6 + len, 0xaa);
927            extension
928        };
929
930        let accepted = decode_resync_file(&resync_file_with_unknown_value(&extension(RESYNC_MSGP_MAX_ELEMENT_BYTES)))
931            .expect("extension ending at the byte limit should decode");
932        assert_eq!(accepted.version, RESYNC_META_VERSION);
933
934        let data = resync_file_with_unknown_value(&extension(RESYNC_MSGP_MAX_ELEMENT_BYTES + 1));
935
936        assert!(matches!(decode_resync_file(&data), Err(ResyncStateError::CorruptedFormat)));
937    }
938
939    #[test]
940    fn resync_file_enforces_messagepack_depth_limit() {
941        let mut accepted = vec![0x91; RESYNC_MSGP_MAX_DEPTH - 1];
942        accepted.push(0xc0);
943        let accepted = decode_resync_file(&resync_file_with_unknown_value(&accepted))
944            .expect("payload ending at the depth limit should decode");
945        assert_eq!(accepted.version, RESYNC_META_VERSION);
946
947        let mut rejected = vec![0x91; RESYNC_MSGP_MAX_DEPTH];
948        rejected.push(0xc0);
949        let data = resync_file_with_unknown_value(&rejected);
950        assert!(matches!(decode_resync_file(&data), Err(ResyncStateError::CorruptedFormat)));
951    }
952
953    #[test]
954    fn resync_file_rejects_excessive_messagepack_values() {
955        const ENVELOPE_VALUES: usize = 4;
956        const INNER_ARRAYS: usize = 32;
957
958        let nested_arrays = |leaf_count: usize| {
959            let base_len = leaf_count / INNER_ARRAYS;
960            let longer_arrays = leaf_count % INNER_ARRAYS;
961            assert!(base_len + usize::from(longer_arrays > 0) <= RESYNC_MSGP_MAX_COLLECTION_ITEMS);
962
963            let mut value = Vec::new();
964            rmp::encode::write_array_len(
965                &mut value,
966                u32::try_from(INNER_ARRAYS).expect("test outer array length should fit u32"),
967            )
968            .expect("test outer array should encode");
969            for index in 0..INNER_ARRAYS {
970                let inner_len = base_len + usize::from(index < longer_arrays);
971                rmp::encode::write_array_len(
972                    &mut value,
973                    u32::try_from(inner_len).expect("test inner array length should fit u32"),
974                )
975                .expect("test inner array should encode");
976                value.resize(value.len() + inner_len, 0xc0);
977            }
978            value
979        };
980
981        let accepted_leaf_count = RESYNC_MSGP_MAX_VALUES - ENVELOPE_VALUES - 1 - INNER_ARRAYS;
982        let accepted = decode_resync_file(&resync_file_with_unknown_value(&nested_arrays(accepted_leaf_count)))
983            .expect("payload ending at the value limit should decode");
984        assert_eq!(accepted.version, RESYNC_META_VERSION);
985
986        let data = resync_file_with_unknown_value(&nested_arrays(accepted_leaf_count + 1));
987
988        assert!(matches!(decode_resync_file(&data), Err(ResyncStateError::CorruptedFormat)));
989    }
990
991    #[test]
992    fn resync_file_rejects_payload_over_file_limit() {
993        const FULL_CHUNKS: usize = 15;
994
995        let field_count = u32::try_from(FULL_CHUNKS + 2).expect("test field count should fit u32");
996        let chunk = msgp_str32(RESYNC_MSGP_MAX_ELEMENT_BYTES);
997        let mut payload = Vec::with_capacity(RESYNC_MSGP_MAX_BYTES);
998        rmp::encode::write_map_len(&mut payload, field_count).expect("test payload map should encode");
999        rmp::encode::write_str(&mut payload, "v").expect("test version key should encode");
1000        rmp::encode::write_i32(&mut payload, i32::from(RESYNC_META_VERSION)).expect("test version value should encode");
1001        for _ in 0..FULL_CHUNKS {
1002            rmp::encode::write_str(&mut payload, "future").expect("test unknown key should encode");
1003            payload.extend_from_slice(&chunk);
1004        }
1005        rmp::encode::write_str(&mut payload, "future").expect("test final unknown key should encode");
1006        let remaining = RESYNC_MSGP_MAX_BYTES
1007            .checked_sub(payload.len() + 5)
1008            .expect("test payload should leave room for the final string");
1009        assert!(remaining <= RESYNC_MSGP_MAX_ELEMENT_BYTES);
1010        payload.extend_from_slice(&msgp_str32(remaining));
1011
1012        let mut data = wrap_resync_payload(&payload);
1013        assert_eq!(data.len(), RESYNC_FILE_MAX_BYTES);
1014        let accepted = decode_resync_file(&data).expect("payload ending at the file limit should decode");
1015        assert_eq!(accepted.version, RESYNC_META_VERSION);
1016
1017        data.push(0xc0);
1018        assert_eq!(data.len(), RESYNC_FILE_MAX_BYTES + 1);
1019        assert!(matches!(decode_resync_file(&data), Err(ResyncStateError::CorruptedFormat)));
1020    }
1021
1022    #[test]
1023    fn resync_file_encoder_rejects_payload_over_file_limit() {
1024        let mut status = BucketReplicationResyncStatus::new();
1025        status.targets_map.insert(
1026            "arn:replication:0".to_string(),
1027            TargetReplicationResyncStatus {
1028                object: "x".repeat(RESYNC_MSGP_MAX_ELEMENT_BYTES),
1029                ..Default::default()
1030            },
1031        );
1032        assert!(encode_resync_file(&status).is_ok(), "a single maximum-sized element should encode");
1033
1034        let target_count = RESYNC_FILE_MAX_BYTES / RESYNC_MSGP_MAX_ELEMENT_BYTES + 1;
1035        for index in 1..target_count {
1036            status.targets_map.insert(
1037                format!("arn:replication:{index}"),
1038                TargetReplicationResyncStatus {
1039                    object: "x".repeat(RESYNC_MSGP_MAX_ELEMENT_BYTES),
1040                    ..Default::default()
1041                },
1042            );
1043        }
1044
1045        assert!(matches!(encode_resync_file(&status), Err(ResyncStateError::CorruptedFormat)));
1046    }
1047
1048    #[test]
1049    fn resync_file_encoder_rejects_oversized_messagepack_element() {
1050        let mut status = BucketReplicationResyncStatus::new();
1051        status.targets_map.insert(
1052            "arn:replication:oversized-element".to_string(),
1053            TargetReplicationResyncStatus {
1054                object: "x".repeat(RESYNC_MSGP_MAX_ELEMENT_BYTES + 1),
1055                ..Default::default()
1056            },
1057        );
1058
1059        assert!(matches!(encode_resync_file(&status), Err(ResyncStateError::CorruptedFormat)));
1060    }
1061
1062    #[test]
1063    fn resync_error_detail_is_bounded_and_unicode_safe() {
1064        let detail = format!("{}尾", "x".repeat(RESYNC_ERROR_DETAIL_MAX_CHARS + 32));
1065
1066        let sanitized = sanitize_resync_error_detail(&detail).expect("non-empty detail should remain");
1067
1068        assert_eq!(sanitized.chars().count(), RESYNC_ERROR_DETAIL_MAX_CHARS);
1069        assert!(sanitized.ends_with(RESYNC_ERROR_TRUNCATED_SUFFIX));
1070        assert!(!sanitized.contains('尾'));
1071    }
1072
1073    #[test]
1074    fn resync_error_detail_redacts_credentials_and_headers() {
1075        for detail in [
1076            "Authorization: Bearer status-secret",
1077            "request failed with secret_key=status-secret",
1078            "x-amz-security-token=status-secret",
1079            "https://user:status-secret@example.test",
1080        ] {
1081            assert_eq!(
1082                sanitize_resync_error_detail(detail).as_deref(),
1083                Some(RESYNC_ERROR_REDACTED),
1084                "sensitive detail should be replaced: {detail}"
1085            );
1086        }
1087        assert_eq!(sanitize_resync_error_detail("ExpiredToken").as_deref(), Some("ExpiredToken"));
1088    }
1089
1090    #[test]
1091    fn resync_file_sanitizes_legacy_error_on_decode() {
1092        let mut status = BucketReplicationResyncStatus::new();
1093        status.targets_map.insert(
1094            "arn:replication:a".to_string(),
1095            TargetReplicationResyncStatus {
1096                resync_id: "rid-legacy".to_string(),
1097                resync_status: ResyncStatusType::ResyncFailed,
1098                error: Some("Authorization: Bearer persisted-secret".to_string()),
1099                ..Default::default()
1100            },
1101        );
1102        let legacy_payload = rmp_serde::to_vec(&status).expect("legacy status should encode");
1103        let direct =
1104            BucketReplicationResyncStatus::unmarshal_legacy_msg(&legacy_payload).expect("legacy status should decode directly");
1105        let mut data = Vec::new();
1106        data.extend_from_slice(&RESYNC_META_FORMAT.to_le_bytes());
1107        data.extend_from_slice(&RESYNC_META_VERSION.to_le_bytes());
1108        data.extend_from_slice(&legacy_payload);
1109
1110        let decoded = decode_resync_file(&data).expect("legacy status should decode");
1111
1112        assert_eq!(direct.targets_map["arn:replication:a"].error.as_deref(), Some(RESYNC_ERROR_REDACTED));
1113        assert_eq!(decoded.targets_map["arn:replication:a"].error.as_deref(), Some(RESYNC_ERROR_REDACTED));
1114    }
1115
1116    #[test]
1117    fn legacy_resync_payload_rejects_oversized_messagepack_element() {
1118        let mut status = BucketReplicationResyncStatus::new();
1119        status.targets_map.insert(
1120            "arn:replication:legacy-oversized".to_string(),
1121            TargetReplicationResyncStatus {
1122                object: "x".repeat(RESYNC_MSGP_MAX_ELEMENT_BYTES + 1),
1123                ..Default::default()
1124            },
1125        );
1126        let payload = rmp_serde::to_vec(&status).expect("legacy test status should encode");
1127
1128        assert!(matches!(
1129            BucketReplicationResyncStatus::unmarshal_legacy_msg(&payload),
1130            Err(ResyncStateError::CorruptedFormat)
1131        ));
1132    }
1133
1134    #[test]
1135    fn resync_file_retains_error_for_restartable_and_failed_states() {
1136        let mut status = BucketReplicationResyncStatus::new();
1137        for (arn, resync_status) in [
1138            ("arn:replication:pending", ResyncStatusType::ResyncPending),
1139            ("arn:replication:ongoing", ResyncStatusType::ResyncStarted),
1140            ("arn:replication:failed", ResyncStatusType::ResyncFailed),
1141        ] {
1142            status.targets_map.insert(
1143                arn.to_string(),
1144                TargetReplicationResyncStatus {
1145                    resync_id: format!("{arn}-run"),
1146                    resync_status,
1147                    failed_count: 1,
1148                    error: Some("AccessDenied".to_string()),
1149                    ..Default::default()
1150                },
1151            );
1152        }
1153
1154        let data = encode_resync_file(&status).expect("resync status should encode");
1155        let decoded = decode_resync_file(&data).expect("resync status should decode after restart");
1156
1157        for arn in ["arn:replication:pending", "arn:replication:ongoing", "arn:replication:failed"] {
1158            assert_eq!(decoded.targets_map[arn].error.as_deref(), Some("AccessDenied"));
1159        }
1160        assert!(should_auto_resume_resync(decoded.targets_map["arn:replication:pending"].resync_status));
1161        assert!(should_auto_resume_resync(decoded.targets_map["arn:replication:ongoing"].resync_status));
1162        assert!(!should_auto_resume_resync(decoded.targets_map["arn:replication:failed"].resync_status));
1163    }
1164
1165    #[test]
1166    fn skip_msgp_value_consumes_ext_type_and_payload() {
1167        let mut ext16 = Cursor::new(vec![0xc8, 0, 2, MSGP_TIME_EXT_TYPE as u8, 0xaa, 0xbb, 0x01]);
1168        skip_msgp_value(&mut ext16).expect("ext16 should skip");
1169        assert!(matches!(rmp::decode::read_marker(&mut ext16), Ok(Marker::FixPos(1))));
1170
1171        let mut ext32 = Cursor::new(vec![0xc9, 0, 0, 0, 2, MSGP_TIME_EXT_TYPE as u8, 0xaa, 0xbb, 0x01]);
1172        skip_msgp_value(&mut ext32).expect("ext32 should skip");
1173        assert!(matches!(rmp::decode::read_marker(&mut ext32), Ok(Marker::FixPos(1))));
1174    }
1175
1176    #[test]
1177    fn skip_msgp_value_consumes_fixext_type_and_payload() {
1178        for (marker, payload_len) in [(0xd4, 1), (0xd5, 2), (0xd6, 4), (0xd7, 8), (0xd8, 16)] {
1179            let mut data = vec![marker, MSGP_TIME_EXT_TYPE as u8];
1180            data.resize(data.len() + payload_len, 0xaa);
1181            data.push(0x01);
1182
1183            let mut cursor = Cursor::new(data);
1184            skip_msgp_value(&mut cursor).expect("fixext should skip");
1185            assert!(matches!(rmp::decode::read_marker(&mut cursor), Ok(Marker::FixPos(1))));
1186        }
1187    }
1188
1189    #[test]
1190    fn io_errors_stay_typed_in_resync_state_error() {
1191        let io_err = std::io::Error::new(std::io::ErrorKind::PermissionDenied, "state file locked");
1192        let err: ResyncStateError = io_err.into();
1193        match &err {
1194            ResyncStateError::Io(inner) => {
1195                assert_eq!(inner.kind(), std::io::ErrorKind::PermissionDenied);
1196                assert_eq!(inner.to_string(), "state file locked");
1197            }
1198            other => panic!("io::Error must stay typed, got {other:?}"),
1199        }
1200        assert_eq!(err.to_string(), "state file locked");
1201        assert!(std::error::Error::source(&err).is_some(), "Io must expose its source");
1202    }
1203}