Skip to main content

rustfs_replication/
runtime.rs

1// Copyright 2024 RustFS Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use crate::ReplicationPriority;
16
17pub const WORKER_MAX_LIMIT: usize = 500;
18pub const WORKER_MIN_LIMIT: usize = 50;
19pub const WORKER_AUTO_DEFAULT: usize = 100;
20pub const MRF_WORKER_MAX_LIMIT: usize = 8;
21pub const MRF_WORKER_MIN_LIMIT: usize = 2;
22pub const MRF_WORKER_AUTO_DEFAULT: usize = 4;
23pub const LARGE_WORKER_COUNT: usize = 10;
24pub const MIN_LARGE_OBJ_SIZE: i64 = 128 * 1024 * 1024;
25
26#[derive(Debug, Clone, Copy, PartialEq, Eq)]
27pub struct ReplicationWorkerResize {
28    pub new_count: usize,
29    pub existing_count: usize,
30}
31
32#[derive(Debug, Clone, Copy, PartialEq, Eq)]
33pub struct ReplicationBackpressureState {
34    pub current_workers: usize,
35    pub active_workers: i32,
36    pub current_mrf_workers: i32,
37    pub active_mrf_workers: i32,
38    pub max_workers: usize,
39    pub include_mrf_workers: bool,
40}
41
42#[derive(Debug, Clone, Copy, PartialEq, Eq)]
43pub struct ReplicationBackpressureResize {
44    pub regular_workers: Option<ReplicationWorkerResize>,
45    pub mrf_workers: Option<i32>,
46}
47
48#[derive(Debug, Clone, Copy, PartialEq, Eq)]
49pub enum ReplicationBackpressureRecommendation {
50    KeepFast,
51    SetPriorityAuto,
52    Resize(ReplicationBackpressureResize),
53    Noop,
54}
55
56#[derive(Debug, Clone)]
57pub struct ReplicationPoolOpts {
58    pub priority: ReplicationPriority,
59    pub max_workers: Option<usize>,
60    pub max_l_workers: Option<usize>,
61}
62
63impl Default for ReplicationPoolOpts {
64    fn default() -> Self {
65        Self {
66            priority: ReplicationPriority::Auto,
67            max_workers: None,
68            max_l_workers: None,
69        }
70    }
71}
72
73#[derive(Debug, Clone, Copy, PartialEq, Eq)]
74pub struct ReplicationWorkerCounts {
75    pub workers: usize,
76    pub mrf_workers: usize,
77}
78
79impl ReplicationWorkerCounts {
80    pub const fn new(workers: usize, mrf_workers: usize) -> Self {
81        Self { workers, mrf_workers }
82    }
83
84    pub fn capped_by_worker_limit(self, max_workers: usize) -> Self {
85        Self {
86            workers: self.workers.min(max_workers),
87            mrf_workers: self.mrf_workers.min(max_workers),
88        }
89    }
90
91    pub fn mrf_workers_i32(self) -> i32 {
92        usize_to_i32_saturating(self.mrf_workers)
93    }
94}
95
96pub fn initial_worker_counts(opts: &ReplicationPoolOpts) -> ReplicationWorkerCounts {
97    worker_counts_for_priority(&opts.priority, WORKER_AUTO_DEFAULT, MRF_WORKER_AUTO_DEFAULT)
98        .capped_by_worker_limit(opts.max_workers.unwrap_or(WORKER_MAX_LIMIT))
99}
100
101pub fn resized_worker_counts(
102    priority: &ReplicationPriority,
103    max_workers: Option<usize>,
104    current_workers: usize,
105    current_mrf_workers: usize,
106) -> ReplicationWorkerCounts {
107    let counts = worker_counts_for_priority(priority, current_workers, current_mrf_workers);
108    if let Some(max_workers) = max_workers {
109        counts.capped_by_worker_limit(max_workers)
110    } else {
111        counts
112    }
113}
114
115pub fn mrf_worker_size_to_count(size: i32) -> usize {
116    let non_negative = size.max(0);
117    usize::try_from(non_negative).unwrap_or_default()
118}
119
120pub fn worker_counts_for_priority(
121    priority: &ReplicationPriority,
122    current_workers: usize,
123    current_mrf_workers: usize,
124) -> ReplicationWorkerCounts {
125    match priority {
126        ReplicationPriority::Fast => ReplicationWorkerCounts::new(WORKER_MAX_LIMIT, MRF_WORKER_MAX_LIMIT),
127        ReplicationPriority::Slow => ReplicationWorkerCounts::new(WORKER_MIN_LIMIT, MRF_WORKER_MIN_LIMIT),
128        ReplicationPriority::Auto => ReplicationWorkerCounts::new(
129            auto_worker_count(current_workers, WORKER_AUTO_DEFAULT),
130            auto_worker_count(current_mrf_workers, MRF_WORKER_AUTO_DEFAULT),
131        ),
132    }
133}
134
135pub fn should_queue_large_object(size: i64) -> bool {
136    size >= MIN_LARGE_OBJ_SIZE
137}
138
139pub fn should_grow_large_workers(active_lrg_workers: i32, max_l_workers: usize) -> bool {
140    active_lrg_workers < usize_to_i32_saturating(max_l_workers.min(LARGE_WORKER_COUNT))
141}
142
143pub fn next_large_worker_count(existing_workers: usize, max_l_workers: usize) -> usize {
144    existing_workers.saturating_add(1).min(max_l_workers)
145}
146
147pub fn next_regular_worker_count(current_workers: usize, active_workers: i32, max_workers: usize) -> Option<usize> {
148    let max_workers = max_workers.min(WORKER_MAX_LIMIT);
149    if active_workers < usize_to_i32_saturating(max_workers) {
150        Some(current_workers.saturating_add(1).min(max_workers))
151    } else {
152        None
153    }
154}
155
156pub fn next_mrf_worker_count(current_mrf_workers: i32, active_mrf_workers: i32, max_workers: usize) -> Option<i32> {
157    let max_mrf_workers = max_workers.min(MRF_WORKER_MAX_LIMIT);
158    let max_mrf_workers = usize_to_i32_saturating(max_mrf_workers);
159    if active_mrf_workers < max_mrf_workers {
160        Some(current_mrf_workers.saturating_add(1).min(max_mrf_workers))
161    } else {
162        None
163    }
164}
165
166pub fn large_worker_backpressure_resize(
167    existing_workers: usize,
168    active_lrg_workers: i32,
169    max_l_workers: usize,
170) -> Option<ReplicationWorkerResize> {
171    should_grow_large_workers(active_lrg_workers, max_l_workers).then_some(ReplicationWorkerResize {
172        new_count: next_large_worker_count(existing_workers, max_l_workers),
173        existing_count: existing_workers,
174    })
175}
176
177pub fn replication_backpressure_recommendation(
178    priority: &ReplicationPriority,
179    state: ReplicationBackpressureState,
180) -> ReplicationBackpressureRecommendation {
181    match priority {
182        ReplicationPriority::Fast => ReplicationBackpressureRecommendation::KeepFast,
183        ReplicationPriority::Slow => ReplicationBackpressureRecommendation::SetPriorityAuto,
184        ReplicationPriority::Auto => {
185            let regular_workers =
186                next_regular_worker_count(state.current_workers, state.active_workers, state.max_workers).map(|new_count| {
187                    ReplicationWorkerResize {
188                        new_count,
189                        existing_count: state.current_workers,
190                    }
191                });
192            let mrf_workers = state
193                .include_mrf_workers
194                .then(|| next_mrf_worker_count(state.current_mrf_workers, state.active_mrf_workers, state.max_workers))
195                .flatten();
196
197            if regular_workers.is_none() && mrf_workers.is_none() {
198                ReplicationBackpressureRecommendation::Noop
199            } else {
200                ReplicationBackpressureRecommendation::Resize(ReplicationBackpressureResize {
201                    regular_workers,
202                    mrf_workers,
203                })
204            }
205        }
206    }
207}
208
209fn auto_worker_count(current_workers: usize, auto_default: usize) -> usize {
210    if current_workers < auto_default {
211        current_workers.saturating_add(1).min(auto_default)
212    } else {
213        auto_default
214    }
215}
216
217fn usize_to_i32_saturating(value: usize) -> i32 {
218    i32::try_from(value).unwrap_or(i32::MAX)
219}
220
221#[cfg(test)]
222mod tests {
223    use super::*;
224
225    #[test]
226    fn initial_worker_counts_preserve_priority_defaults() {
227        let fast = ReplicationPoolOpts {
228            priority: ReplicationPriority::Fast,
229            ..Default::default()
230        };
231        assert_eq!(
232            initial_worker_counts(&fast),
233            ReplicationWorkerCounts::new(WORKER_MAX_LIMIT, MRF_WORKER_MAX_LIMIT)
234        );
235
236        let slow = ReplicationPoolOpts {
237            priority: ReplicationPriority::Slow,
238            ..Default::default()
239        };
240        assert_eq!(
241            initial_worker_counts(&slow),
242            ReplicationWorkerCounts::new(WORKER_MIN_LIMIT, MRF_WORKER_MIN_LIMIT)
243        );
244
245        let auto = ReplicationPoolOpts::default();
246        assert_eq!(
247            initial_worker_counts(&auto),
248            ReplicationWorkerCounts::new(WORKER_AUTO_DEFAULT, MRF_WORKER_AUTO_DEFAULT)
249        );
250    }
251
252    #[test]
253    fn worker_counts_respect_configured_max_workers() {
254        let opts = ReplicationPoolOpts {
255            priority: ReplicationPriority::Fast,
256            max_workers: Some(3),
257            max_l_workers: None,
258        };
259
260        assert_eq!(initial_worker_counts(&opts), ReplicationWorkerCounts::new(3, 3));
261        assert_eq!(
262            resized_worker_counts(&ReplicationPriority::Fast, Some(5), WORKER_AUTO_DEFAULT, MRF_WORKER_AUTO_DEFAULT),
263            ReplicationWorkerCounts::new(5, 5)
264        );
265    }
266
267    #[test]
268    fn mrf_worker_size_to_count_clamps_negative_values() {
269        assert_eq!(mrf_worker_size_to_count(-1), 0);
270        assert_eq!(mrf_worker_size_to_count(0), 0);
271        assert_eq!(mrf_worker_size_to_count(3), 3);
272    }
273
274    #[test]
275    fn auto_priority_grows_toward_defaults() {
276        assert_eq!(
277            worker_counts_for_priority(&ReplicationPriority::Auto, 0, 0),
278            ReplicationWorkerCounts::new(1, 1)
279        );
280        assert_eq!(
281            worker_counts_for_priority(&ReplicationPriority::Auto, WORKER_AUTO_DEFAULT, MRF_WORKER_AUTO_DEFAULT),
282            ReplicationWorkerCounts::new(WORKER_AUTO_DEFAULT, MRF_WORKER_AUTO_DEFAULT)
283        );
284    }
285
286    #[test]
287    fn backpressure_growth_caps_at_limits() {
288        assert_eq!(next_regular_worker_count(4, 4, WORKER_MAX_LIMIT), Some(5));
289        assert_eq!(
290            next_regular_worker_count(WORKER_MAX_LIMIT, usize_to_i32_saturating(WORKER_MAX_LIMIT), WORKER_MAX_LIMIT),
291            None
292        );
293
294        assert_eq!(next_mrf_worker_count(2, 2, WORKER_MAX_LIMIT), Some(3));
295        assert_eq!(
296            next_mrf_worker_count(8, usize_to_i32_saturating(MRF_WORKER_MAX_LIMIT), WORKER_MAX_LIMIT),
297            None
298        );
299
300        assert!(should_queue_large_object(MIN_LARGE_OBJ_SIZE));
301        assert!(!should_queue_large_object(MIN_LARGE_OBJ_SIZE - 1));
302        assert!(should_grow_large_workers(1, LARGE_WORKER_COUNT));
303        assert_eq!(next_large_worker_count(LARGE_WORKER_COUNT, LARGE_WORKER_COUNT), LARGE_WORKER_COUNT);
304    }
305
306    #[test]
307    fn large_worker_resize_recommendation_respects_growth_limit() {
308        assert_eq!(
309            large_worker_backpressure_resize(3, 2, LARGE_WORKER_COUNT),
310            Some(ReplicationWorkerResize {
311                new_count: 4,
312                existing_count: 3
313            })
314        );
315        assert_eq!(
316            large_worker_backpressure_resize(LARGE_WORKER_COUNT, usize_to_i32_saturating(LARGE_WORKER_COUNT), LARGE_WORKER_COUNT),
317            None
318        );
319    }
320
321    #[test]
322    fn queue_backpressure_recommendation_preserves_priority_semantics() {
323        let state = ReplicationBackpressureState {
324            current_workers: 4,
325            active_workers: 4,
326            current_mrf_workers: 2,
327            active_mrf_workers: 2,
328            max_workers: WORKER_MAX_LIMIT,
329            include_mrf_workers: true,
330        };
331
332        assert_eq!(
333            replication_backpressure_recommendation(&ReplicationPriority::Fast, state),
334            ReplicationBackpressureRecommendation::KeepFast
335        );
336        assert_eq!(
337            replication_backpressure_recommendation(&ReplicationPriority::Slow, state),
338            ReplicationBackpressureRecommendation::SetPriorityAuto
339        );
340
341        assert_eq!(
342            replication_backpressure_recommendation(&ReplicationPriority::Auto, state),
343            ReplicationBackpressureRecommendation::Resize(ReplicationBackpressureResize {
344                regular_workers: Some(ReplicationWorkerResize {
345                    new_count: 5,
346                    existing_count: 4
347                }),
348                mrf_workers: Some(3),
349            })
350        );
351    }
352
353    #[test]
354    fn delete_queue_backpressure_skips_mrf_growth() {
355        let state = ReplicationBackpressureState {
356            current_workers: 4,
357            active_workers: 4,
358            current_mrf_workers: 2,
359            active_mrf_workers: 2,
360            max_workers: WORKER_MAX_LIMIT,
361            include_mrf_workers: false,
362        };
363
364        assert_eq!(
365            replication_backpressure_recommendation(&ReplicationPriority::Auto, state),
366            ReplicationBackpressureRecommendation::Resize(ReplicationBackpressureResize {
367                regular_workers: Some(ReplicationWorkerResize {
368                    new_count: 5,
369                    existing_count: 4
370                }),
371                mrf_workers: None,
372            })
373        );
374    }
375}