Skip to main content

rustfs_replication/
multipart.rs

1// Copyright 2024 RustFS Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::collections::HashMap;
16use std::fmt;
17
18use crate::http::{SUFFIX_ACTUAL_SIZE, get_internal_metadata};
19
20#[derive(Debug, Clone, Copy, PartialEq, Eq)]
21pub struct ReplicationMultipartPartInput {
22    pub offset: i64,
23    pub part_number: usize,
24    pub part_size: i64,
25}
26
27#[derive(Debug, Clone, Copy, PartialEq, Eq)]
28pub struct ReplicationMultipartPartPlan {
29    pub part_number: i32,
30    pub part_size: i64,
31    pub range: ReplicationMultipartRange,
32    pub next_offset: i64,
33}
34
35#[derive(Debug, Clone, Copy, PartialEq, Eq)]
36pub struct ReplicationMultipartRange {
37    pub start: i64,
38    pub end: i64,
39}
40
41#[derive(Debug, Clone, Copy, PartialEq, Eq)]
42pub enum ReplicationMultipartPlanError {
43    InvalidOffset { offset: i64 },
44    InvalidPartSize { part_size: i64 },
45    PartRangeOverflow { offset: i64, part_size: i64 },
46    PartOffsetOverflow { offset: i64, part_size: i64 },
47    PartNumberOverflow { part_number: usize },
48}
49
50impl fmt::Display for ReplicationMultipartPlanError {
51    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
52        match self {
53            Self::InvalidOffset { offset } => write!(f, "invalid multipart replication part offset {offset}"),
54            Self::InvalidPartSize { part_size } => write!(f, "invalid multipart replication part size {part_size}"),
55            Self::PartRangeOverflow { offset, part_size } => {
56                write!(f, "multipart replication part range overflows for offset {offset} and size {part_size}")
57            }
58            Self::PartOffsetOverflow { offset, part_size } => {
59                write!(f, "multipart replication next offset overflows for offset {offset} and size {part_size}")
60            }
61            Self::PartNumberOverflow { part_number } => {
62                write!(f, "multipart replication part number {part_number} overflows i32")
63            }
64        }
65    }
66}
67
68impl std::error::Error for ReplicationMultipartPlanError {}
69
70pub fn replication_multipart_part_plan(
71    input: ReplicationMultipartPartInput,
72) -> Result<ReplicationMultipartPartPlan, ReplicationMultipartPlanError> {
73    if input.offset < 0 {
74        return Err(ReplicationMultipartPlanError::InvalidOffset { offset: input.offset });
75    }
76    if input.part_size <= 0 {
77        return Err(ReplicationMultipartPlanError::InvalidPartSize {
78            part_size: input.part_size,
79        });
80    }
81
82    let part_number = i32::try_from(input.part_number).map_err(|_| ReplicationMultipartPlanError::PartNumberOverflow {
83        part_number: input.part_number,
84    })?;
85    let end = input
86        .offset
87        .checked_add(input.part_size - 1)
88        .ok_or(ReplicationMultipartPlanError::PartRangeOverflow {
89            offset: input.offset,
90            part_size: input.part_size,
91        })?;
92    let next_offset = end.checked_add(1).ok_or(ReplicationMultipartPlanError::PartOffsetOverflow {
93        offset: input.offset,
94        part_size: input.part_size,
95    })?;
96
97    Ok(ReplicationMultipartPartPlan {
98        part_number,
99        part_size: input.part_size,
100        range: ReplicationMultipartRange {
101            start: input.offset,
102            end,
103        },
104        next_offset,
105    })
106}
107
108pub fn replication_multipart_complete_actual_size(user_defined: &HashMap<String, String>) -> String {
109    get_internal_metadata(user_defined, SUFFIX_ACTUAL_SIZE).unwrap_or_default()
110}
111
112/// Largest body S3 accepts on a single `PutObject`. Anything above this has to
113/// be uploaded as multipart; the limit is part of the S3 API, not a RustFS
114/// tunable, so every generic S3 target enforces it.
115pub const REPLICATION_MAX_SINGLE_PUT_SIZE: i64 = 5 * 1024 * 1024 * 1024;
116
117/// Reject a single-`PutObject` replication transfer the target can never accept.
118///
119/// Replication mirrors the object's *source-side storage shape*, which is
120/// read from the ETag: a multipart upload carries a `-<part count>` suffix,
121/// a single `PutObject` does not. A single-`PutObject` object replicates with
122/// one `PutObject` whatever its size, and a multipart object replays the
123/// source's own part layout. So a source object larger than
124/// [`REPLICATION_MAX_SINGLE_PUT_SIZE`] whose ETag has no part-count suffix can
125/// never reach a generic S3 target — the remote rejects it with
126/// `EntityTooLarge`, but only after the whole body has been streamed to it
127/// (rustfs#6825).
128///
129/// Returning the failure up front turns an unbounded wasted transfer plus an
130/// opaque remote error into a stated, diagnosable limit. The message quotes
131/// the ETag the decision was made from rather than asserting a conclusion
132/// about how the object was uploaded: an operator reading the line can check
133/// the evidence against the object's own listing, and a transport-selection
134/// bug upstream of this guard shows up as a visible contradiction instead of
135/// as advice to re-upload an object that was already multipart.
136///
137/// RustFS deliberately does not re-chunk such an object into multipart on the
138/// replication side: the target's part layout is the source's, and rewriting
139/// it would break the ETag/part identity that heal and delete convergence
140/// address.
141pub fn replication_single_put_size_error(is_multipart: bool, transfer_size: i64, etag: Option<&str>) -> Option<String> {
142    if is_multipart || transfer_size <= REPLICATION_MAX_SINGLE_PUT_SIZE {
143        return None;
144    }
145    let etag = etag.filter(|value| !value.is_empty()).unwrap_or("<none>");
146    Some(format!(
147        "object of {transfer_size} bytes exceeds the {REPLICATION_MAX_SINGLE_PUT_SIZE} byte single-PutObject limit \
148         of an S3 target; its ETag {etag} has no part-count suffix, so replication uses a single PutObject and \
149         does not re-chunk; if the object is in fact multipart this is a transport-selection defect, otherwise \
150         re-upload it as a multipart upload to make it replicable"
151    ))
152}
153
154#[cfg(test)]
155mod tests {
156    use super::{
157        REPLICATION_MAX_SINGLE_PUT_SIZE, ReplicationMultipartPartInput, ReplicationMultipartPartPlan,
158        ReplicationMultipartPlanError, ReplicationMultipartRange, replication_multipart_complete_actual_size,
159        replication_multipart_part_plan, replication_single_put_size_error,
160    };
161    use crate::http::{SUFFIX_ACTUAL_SIZE, insert_internal_metadata};
162    use std::collections::HashMap;
163
164    const MIB: i64 = 1024 * 1024;
165
166    #[test]
167    fn multipart_part_plan_builds_range_and_next_offset() {
168        assert_eq!(
169            replication_multipart_part_plan(ReplicationMultipartPartInput {
170                offset: 0,
171                part_number: 1,
172                part_size: 10
173            }),
174            Ok(ReplicationMultipartPartPlan {
175                part_number: 1,
176                part_size: 10,
177                range: ReplicationMultipartRange { start: 0, end: 9 },
178                next_offset: 10
179            })
180        );
181
182        assert_eq!(
183            replication_multipart_part_plan(ReplicationMultipartPartInput {
184                offset: 10,
185                part_number: 2,
186                part_size: 5
187            }),
188            Ok(ReplicationMultipartPartPlan {
189                part_number: 2,
190                part_size: 5,
191                range: ReplicationMultipartRange { start: 10, end: 14 },
192                next_offset: 15
193            })
194        );
195    }
196
197    #[test]
198    fn multipart_part_plan_rejects_invalid_offsets_and_sizes() {
199        assert_eq!(
200            replication_multipart_part_plan(ReplicationMultipartPartInput {
201                offset: -1,
202                part_number: 1,
203                part_size: 10
204            }),
205            Err(ReplicationMultipartPlanError::InvalidOffset { offset: -1 })
206        );
207        assert_eq!(
208            replication_multipart_part_plan(ReplicationMultipartPartInput {
209                offset: 0,
210                part_number: 1,
211                part_size: 0
212            }),
213            Err(ReplicationMultipartPlanError::InvalidPartSize { part_size: 0 })
214        );
215    }
216
217    #[test]
218    fn multipart_part_plan_rejects_overflow() {
219        assert_eq!(
220            replication_multipart_part_plan(ReplicationMultipartPartInput {
221                offset: i64::MAX - 5,
222                part_number: 1,
223                part_size: 10
224            }),
225            Err(ReplicationMultipartPlanError::PartRangeOverflow {
226                offset: i64::MAX - 5,
227                part_size: 10
228            })
229        );
230        assert_eq!(
231            replication_multipart_part_plan(ReplicationMultipartPartInput {
232                offset: i64::MAX,
233                part_number: 1,
234                part_size: 1
235            }),
236            Err(ReplicationMultipartPlanError::PartOffsetOverflow {
237                offset: i64::MAX,
238                part_size: 1
239            })
240        );
241    }
242
243    #[test]
244    fn multipart_part_plan_rejects_part_number_overflow() {
245        let overflowing_part_number = usize::MAX;
246
247        assert_eq!(
248            replication_multipart_part_plan(ReplicationMultipartPartInput {
249                offset: 0,
250                part_number: overflowing_part_number,
251                part_size: 10
252            }),
253            Err(ReplicationMultipartPlanError::PartNumberOverflow {
254                part_number: overflowing_part_number
255            })
256        );
257    }
258
259    #[test]
260    fn multipart_complete_actual_size_reads_compatible_metadata() {
261        let mut user_defined = HashMap::new();
262        insert_internal_metadata(&mut user_defined, SUFFIX_ACTUAL_SIZE, "123".to_string());
263
264        assert_eq!(replication_multipart_complete_actual_size(&user_defined), "123");
265        assert!(replication_multipart_complete_actual_size(&HashMap::new()).is_empty());
266    }
267
268    #[test]
269    fn single_put_size_guard_admits_transfers_a_target_can_accept() {
270        for size in [
271            0,
272            1,
273            MIB,
274            REPLICATION_MAX_SINGLE_PUT_SIZE - 1,
275            REPLICATION_MAX_SINGLE_PUT_SIZE,
276        ] {
277            assert_eq!(
278                replication_single_put_size_error(false, size, Some("0123456789abcdef0123456789abcdef")),
279                None,
280                "single PUT of {size} bytes is within the S3 limit and must not be rejected"
281            );
282        }
283    }
284
285    #[test]
286    fn single_put_size_guard_rejects_an_oversized_single_put() {
287        let size = REPLICATION_MAX_SINGLE_PUT_SIZE + 1;
288        let etag = "0123456789abcdef0123456789abcdef";
289        let err = replication_single_put_size_error(false, size, Some(etag)).expect("oversized single PUT must be rejected");
290
291        // The message is the operator's diagnosis: it has to name the actual
292        // size, the limit, and the remedy.
293        assert!(err.contains(&size.to_string()), "message must name the object size: {err}");
294        assert!(
295            err.contains(&REPLICATION_MAX_SINGLE_PUT_SIZE.to_string()),
296            "message must name the limit: {err}"
297        );
298        assert!(err.contains("multipart"), "message must name the remedy: {err}");
299    }
300
301    #[test]
302    fn single_put_size_guard_quotes_the_etag_it_decided_from() {
303        // rustfs#6825, second report: a 768-part object was misrouted to the
304        // single-PUT path and this guard told the operator it "was not written
305        // as multipart" and to re-upload it. The message must carry the
306        // evidence the decision came from, so a misroute reads as a visible
307        // contradiction against the object's own listing, not as advice.
308        let etag = "767e7a8379c0f62c39e0ceeea0e13de9";
309        let err = replication_single_put_size_error(false, REPLICATION_MAX_SINGLE_PUT_SIZE + 1, Some(etag))
310            .expect("oversized single PUT must be rejected");
311
312        assert!(err.contains(etag), "message must quote the ETag: {err}");
313        assert!(
314            err.contains("part-count suffix"),
315            "message must say what about the ETag was read, not assert an upload method: {err}"
316        );
317        assert!(
318            !err.contains("was not written as multipart"),
319            "message must not assert how the object was uploaded: {err}"
320        );
321        assert!(
322            err.contains("transport-selection defect"),
323            "message must tell the operator what a contradiction with the listing means: {err}"
324        );
325
326        let missing = replication_single_put_size_error(false, REPLICATION_MAX_SINGLE_PUT_SIZE + 1, None)
327            .expect("oversized single PUT must be rejected");
328        assert!(missing.contains("<none>"), "a missing ETag must be stated, not hidden: {missing}");
329        let empty = replication_single_put_size_error(false, REPLICATION_MAX_SINGLE_PUT_SIZE + 1, Some(""))
330            .expect("oversized single PUT must be rejected");
331        assert!(empty.contains("<none>"), "an empty ETag must be stated, not hidden: {empty}");
332    }
333
334    #[test]
335    fn single_put_size_guard_never_rejects_the_multipart_route() {
336        // Multipart replays the source part layout, so object size alone says
337        // nothing about whether the target will accept it; the per-part limits
338        // are the target's to enforce.
339        assert_eq!(
340            replication_single_put_size_error(
341                true,
342                REPLICATION_MAX_SINGLE_PUT_SIZE * 1024,
343                Some("0123456789abcdef0123456789abcdef-768")
344            ),
345            None
346        );
347    }
348}