1use crate::ReplicationPriority;
16
17pub const WORKER_MAX_LIMIT: usize = 500;
18pub const WORKER_MIN_LIMIT: usize = 50;
19pub const WORKER_AUTO_DEFAULT: usize = 100;
20pub const MRF_WORKER_MAX_LIMIT: usize = 8;
21pub const MRF_WORKER_MIN_LIMIT: usize = 2;
22pub const MRF_WORKER_AUTO_DEFAULT: usize = 4;
23pub const LARGE_WORKER_COUNT: usize = 10;
24pub const MIN_LARGE_OBJ_SIZE: i64 = 128 * 1024 * 1024;
25
26#[derive(Debug, Clone, Copy, PartialEq, Eq)]
27pub struct ReplicationWorkerResize {
28 pub new_count: usize,
29 pub existing_count: usize,
30}
31
32#[derive(Debug, Clone, Copy, PartialEq, Eq)]
33pub struct ReplicationBackpressureState {
34 pub current_workers: usize,
35 pub active_workers: i32,
36 pub current_mrf_workers: i32,
37 pub active_mrf_workers: i32,
38 pub max_workers: usize,
39 pub include_mrf_workers: bool,
40}
41
42#[derive(Debug, Clone, Copy, PartialEq, Eq)]
43pub struct ReplicationBackpressureResize {
44 pub regular_workers: Option<ReplicationWorkerResize>,
45 pub mrf_workers: Option<i32>,
46}
47
48#[derive(Debug, Clone, Copy, PartialEq, Eq)]
49pub enum ReplicationBackpressureRecommendation {
50 KeepFast,
51 SetPriorityAuto,
52 Resize(ReplicationBackpressureResize),
53 Noop,
54}
55
56#[derive(Debug, Clone)]
57pub struct ReplicationPoolOpts {
58 pub priority: ReplicationPriority,
59 pub max_workers: Option<usize>,
60 pub max_l_workers: Option<usize>,
61}
62
63impl Default for ReplicationPoolOpts {
64 fn default() -> Self {
65 Self {
66 priority: ReplicationPriority::Auto,
67 max_workers: None,
68 max_l_workers: None,
69 }
70 }
71}
72
73#[derive(Debug, Clone, Copy, PartialEq, Eq)]
74pub struct ReplicationWorkerCounts {
75 pub workers: usize,
76 pub mrf_workers: usize,
77}
78
79impl ReplicationWorkerCounts {
80 pub const fn new(workers: usize, mrf_workers: usize) -> Self {
81 Self { workers, mrf_workers }
82 }
83
84 pub fn capped_by_worker_limit(self, max_workers: usize) -> Self {
85 Self {
86 workers: self.workers.min(max_workers),
87 mrf_workers: self.mrf_workers.min(max_workers),
88 }
89 }
90
91 pub fn mrf_workers_i32(self) -> i32 {
92 usize_to_i32_saturating(self.mrf_workers)
93 }
94}
95
96pub fn initial_worker_counts(opts: &ReplicationPoolOpts) -> ReplicationWorkerCounts {
97 worker_counts_for_priority(&opts.priority, WORKER_AUTO_DEFAULT, MRF_WORKER_AUTO_DEFAULT)
98 .capped_by_worker_limit(opts.max_workers.unwrap_or(WORKER_MAX_LIMIT))
99}
100
101pub fn resized_worker_counts(
102 priority: &ReplicationPriority,
103 max_workers: Option<usize>,
104 current_workers: usize,
105 current_mrf_workers: usize,
106) -> ReplicationWorkerCounts {
107 let counts = worker_counts_for_priority(priority, current_workers, current_mrf_workers);
108 if let Some(max_workers) = max_workers {
109 counts.capped_by_worker_limit(max_workers)
110 } else {
111 counts
112 }
113}
114
115pub fn mrf_worker_size_to_count(size: i32) -> usize {
116 let non_negative = size.max(0);
117 usize::try_from(non_negative).unwrap_or_default()
118}
119
120pub fn worker_counts_for_priority(
121 priority: &ReplicationPriority,
122 current_workers: usize,
123 current_mrf_workers: usize,
124) -> ReplicationWorkerCounts {
125 match priority {
126 ReplicationPriority::Fast => ReplicationWorkerCounts::new(WORKER_MAX_LIMIT, MRF_WORKER_MAX_LIMIT),
127 ReplicationPriority::Slow => ReplicationWorkerCounts::new(WORKER_MIN_LIMIT, MRF_WORKER_MIN_LIMIT),
128 ReplicationPriority::Auto => ReplicationWorkerCounts::new(
129 auto_worker_count(current_workers, WORKER_AUTO_DEFAULT),
130 auto_worker_count(current_mrf_workers, MRF_WORKER_AUTO_DEFAULT),
131 ),
132 }
133}
134
135pub fn should_queue_large_object(size: i64) -> bool {
136 size >= MIN_LARGE_OBJ_SIZE
137}
138
139pub fn should_grow_large_workers(active_lrg_workers: i32, max_l_workers: usize) -> bool {
140 active_lrg_workers < usize_to_i32_saturating(max_l_workers.min(LARGE_WORKER_COUNT))
141}
142
143pub fn next_large_worker_count(existing_workers: usize, max_l_workers: usize) -> usize {
144 existing_workers.saturating_add(1).min(max_l_workers)
145}
146
147pub fn next_regular_worker_count(current_workers: usize, active_workers: i32, max_workers: usize) -> Option<usize> {
148 let max_workers = max_workers.min(WORKER_MAX_LIMIT);
149 if active_workers < usize_to_i32_saturating(max_workers) {
150 Some(current_workers.saturating_add(1).min(max_workers))
151 } else {
152 None
153 }
154}
155
156pub fn next_mrf_worker_count(current_mrf_workers: i32, active_mrf_workers: i32, max_workers: usize) -> Option<i32> {
157 let max_mrf_workers = max_workers.min(MRF_WORKER_MAX_LIMIT);
158 let max_mrf_workers = usize_to_i32_saturating(max_mrf_workers);
159 if active_mrf_workers < max_mrf_workers {
160 Some(current_mrf_workers.saturating_add(1).min(max_mrf_workers))
161 } else {
162 None
163 }
164}
165
166pub fn large_worker_backpressure_resize(
167 existing_workers: usize,
168 active_lrg_workers: i32,
169 max_l_workers: usize,
170) -> Option<ReplicationWorkerResize> {
171 should_grow_large_workers(active_lrg_workers, max_l_workers).then_some(ReplicationWorkerResize {
172 new_count: next_large_worker_count(existing_workers, max_l_workers),
173 existing_count: existing_workers,
174 })
175}
176
177pub fn replication_backpressure_recommendation(
178 priority: &ReplicationPriority,
179 state: ReplicationBackpressureState,
180) -> ReplicationBackpressureRecommendation {
181 match priority {
182 ReplicationPriority::Fast => ReplicationBackpressureRecommendation::KeepFast,
183 ReplicationPriority::Slow => ReplicationBackpressureRecommendation::SetPriorityAuto,
184 ReplicationPriority::Auto => {
185 let regular_workers =
186 next_regular_worker_count(state.current_workers, state.active_workers, state.max_workers).map(|new_count| {
187 ReplicationWorkerResize {
188 new_count,
189 existing_count: state.current_workers,
190 }
191 });
192 let mrf_workers = state
193 .include_mrf_workers
194 .then(|| next_mrf_worker_count(state.current_mrf_workers, state.active_mrf_workers, state.max_workers))
195 .flatten();
196
197 if regular_workers.is_none() && mrf_workers.is_none() {
198 ReplicationBackpressureRecommendation::Noop
199 } else {
200 ReplicationBackpressureRecommendation::Resize(ReplicationBackpressureResize {
201 regular_workers,
202 mrf_workers,
203 })
204 }
205 }
206 }
207}
208
209fn auto_worker_count(current_workers: usize, auto_default: usize) -> usize {
210 if current_workers < auto_default {
211 current_workers.saturating_add(1).min(auto_default)
212 } else {
213 auto_default
214 }
215}
216
217fn usize_to_i32_saturating(value: usize) -> i32 {
218 i32::try_from(value).unwrap_or(i32::MAX)
219}
220
221#[cfg(test)]
222mod tests {
223 use super::*;
224
225 #[test]
226 fn initial_worker_counts_preserve_priority_defaults() {
227 let fast = ReplicationPoolOpts {
228 priority: ReplicationPriority::Fast,
229 ..Default::default()
230 };
231 assert_eq!(
232 initial_worker_counts(&fast),
233 ReplicationWorkerCounts::new(WORKER_MAX_LIMIT, MRF_WORKER_MAX_LIMIT)
234 );
235
236 let slow = ReplicationPoolOpts {
237 priority: ReplicationPriority::Slow,
238 ..Default::default()
239 };
240 assert_eq!(
241 initial_worker_counts(&slow),
242 ReplicationWorkerCounts::new(WORKER_MIN_LIMIT, MRF_WORKER_MIN_LIMIT)
243 );
244
245 let auto = ReplicationPoolOpts::default();
246 assert_eq!(
247 initial_worker_counts(&auto),
248 ReplicationWorkerCounts::new(WORKER_AUTO_DEFAULT, MRF_WORKER_AUTO_DEFAULT)
249 );
250 }
251
252 #[test]
253 fn worker_counts_respect_configured_max_workers() {
254 let opts = ReplicationPoolOpts {
255 priority: ReplicationPriority::Fast,
256 max_workers: Some(3),
257 max_l_workers: None,
258 };
259
260 assert_eq!(initial_worker_counts(&opts), ReplicationWorkerCounts::new(3, 3));
261 assert_eq!(
262 resized_worker_counts(&ReplicationPriority::Fast, Some(5), WORKER_AUTO_DEFAULT, MRF_WORKER_AUTO_DEFAULT),
263 ReplicationWorkerCounts::new(5, 5)
264 );
265 }
266
267 #[test]
268 fn mrf_worker_size_to_count_clamps_negative_values() {
269 assert_eq!(mrf_worker_size_to_count(-1), 0);
270 assert_eq!(mrf_worker_size_to_count(0), 0);
271 assert_eq!(mrf_worker_size_to_count(3), 3);
272 }
273
274 #[test]
275 fn auto_priority_grows_toward_defaults() {
276 assert_eq!(
277 worker_counts_for_priority(&ReplicationPriority::Auto, 0, 0),
278 ReplicationWorkerCounts::new(1, 1)
279 );
280 assert_eq!(
281 worker_counts_for_priority(&ReplicationPriority::Auto, WORKER_AUTO_DEFAULT, MRF_WORKER_AUTO_DEFAULT),
282 ReplicationWorkerCounts::new(WORKER_AUTO_DEFAULT, MRF_WORKER_AUTO_DEFAULT)
283 );
284 }
285
286 #[test]
287 fn backpressure_growth_caps_at_limits() {
288 assert_eq!(next_regular_worker_count(4, 4, WORKER_MAX_LIMIT), Some(5));
289 assert_eq!(
290 next_regular_worker_count(WORKER_MAX_LIMIT, usize_to_i32_saturating(WORKER_MAX_LIMIT), WORKER_MAX_LIMIT),
291 None
292 );
293
294 assert_eq!(next_mrf_worker_count(2, 2, WORKER_MAX_LIMIT), Some(3));
295 assert_eq!(
296 next_mrf_worker_count(8, usize_to_i32_saturating(MRF_WORKER_MAX_LIMIT), WORKER_MAX_LIMIT),
297 None
298 );
299
300 assert!(should_queue_large_object(MIN_LARGE_OBJ_SIZE));
301 assert!(!should_queue_large_object(MIN_LARGE_OBJ_SIZE - 1));
302 assert!(should_grow_large_workers(1, LARGE_WORKER_COUNT));
303 assert_eq!(next_large_worker_count(LARGE_WORKER_COUNT, LARGE_WORKER_COUNT), LARGE_WORKER_COUNT);
304 }
305
306 #[test]
307 fn large_worker_resize_recommendation_respects_growth_limit() {
308 assert_eq!(
309 large_worker_backpressure_resize(3, 2, LARGE_WORKER_COUNT),
310 Some(ReplicationWorkerResize {
311 new_count: 4,
312 existing_count: 3
313 })
314 );
315 assert_eq!(
316 large_worker_backpressure_resize(LARGE_WORKER_COUNT, usize_to_i32_saturating(LARGE_WORKER_COUNT), LARGE_WORKER_COUNT),
317 None
318 );
319 }
320
321 #[test]
322 fn queue_backpressure_recommendation_preserves_priority_semantics() {
323 let state = ReplicationBackpressureState {
324 current_workers: 4,
325 active_workers: 4,
326 current_mrf_workers: 2,
327 active_mrf_workers: 2,
328 max_workers: WORKER_MAX_LIMIT,
329 include_mrf_workers: true,
330 };
331
332 assert_eq!(
333 replication_backpressure_recommendation(&ReplicationPriority::Fast, state),
334 ReplicationBackpressureRecommendation::KeepFast
335 );
336 assert_eq!(
337 replication_backpressure_recommendation(&ReplicationPriority::Slow, state),
338 ReplicationBackpressureRecommendation::SetPriorityAuto
339 );
340
341 assert_eq!(
342 replication_backpressure_recommendation(&ReplicationPriority::Auto, state),
343 ReplicationBackpressureRecommendation::Resize(ReplicationBackpressureResize {
344 regular_workers: Some(ReplicationWorkerResize {
345 new_count: 5,
346 existing_count: 4
347 }),
348 mrf_workers: Some(3),
349 })
350 );
351 }
352
353 #[test]
354 fn delete_queue_backpressure_skips_mrf_growth() {
355 let state = ReplicationBackpressureState {
356 current_workers: 4,
357 active_workers: 4,
358 current_mrf_workers: 2,
359 active_mrf_workers: 2,
360 max_workers: WORKER_MAX_LIMIT,
361 include_mrf_workers: false,
362 };
363
364 assert_eq!(
365 replication_backpressure_recommendation(&ReplicationPriority::Auto, state),
366 ReplicationBackpressureRecommendation::Resize(ReplicationBackpressureResize {
367 regular_workers: Some(ReplicationWorkerResize {
368 new_count: 5,
369 existing_count: 4
370 }),
371 mrf_workers: None,
372 })
373 );
374 }
375}