1#[derive(Debug, Clone, PartialEq)]
27pub struct PartitionBoundary {
28 pub shard_id: String,
30 pub start_idx: u64,
32 pub end_idx: u64,
34 pub vector_count: u64,
36 pub query_load: f64,
38}
39
40impl PartitionBoundary {
41 #[inline]
46 pub fn load_ratio(&self, total_load: f64) -> f64 {
47 self.query_load / total_load.max(1e-9)
48 }
49}
50
51#[derive(Debug, Clone, PartialEq)]
61pub enum RebalanceAction {
62 Split {
64 shard_id: String,
66 split_at: u64,
68 },
69 Merge {
71 shard_a: String,
73 shard_b: String,
75 },
76 Migrate {
78 from_shard: String,
80 to_shard: String,
82 count: u64,
84 },
85 NoChange,
87}
88
89#[derive(Debug, Clone)]
95pub struct PartitionerConfig {
96 pub max_vectors_per_shard: u64,
98 pub min_vectors_per_shard: u64,
100 pub load_imbalance_threshold: f64,
103 pub target_shard_count: usize,
105}
106
107impl Default for PartitionerConfig {
108 fn default() -> Self {
109 Self {
110 max_vectors_per_shard: 50_000,
111 min_vectors_per_shard: 1_000,
112 load_imbalance_threshold: 2.0,
113 target_shard_count: 8,
114 }
115 }
116}
117
118#[derive(Debug, Clone)]
124pub struct PartitionStats {
125 pub shard_count: usize,
127 pub total_vectors: u64,
129 pub avg_vectors_per_shard: f64,
131 pub max_load_shard: String,
133 pub min_load_shard: String,
135 pub imbalance_ratio: f64,
140}
141
142#[derive(Debug, Clone)]
168pub struct AdaptiveIndexPartitioner {
169 pub partitions: Vec<PartitionBoundary>,
171 pub config: PartitionerConfig,
173}
174
175impl AdaptiveIndexPartitioner {
176 pub fn new(config: PartitionerConfig) -> Self {
178 Self {
179 partitions: Vec::new(),
180 config,
181 }
182 }
183
184 pub fn add_partition(&mut self, p: PartitionBoundary) {
190 self.partitions.push(p);
191 self.partitions.sort_by_key(|p| p.start_idx);
192 }
193
194 pub fn total_load(&self) -> f64 {
196 self.partitions.iter().map(|p| p.query_load).sum()
197 }
198
199 pub fn total_vectors(&self) -> u64 {
201 self.partitions.iter().map(|p| p.vector_count).sum()
202 }
203
204 pub fn find_shard(&self, vector_idx: u64) -> Option<&PartitionBoundary> {
208 let pos = self
210 .partitions
211 .partition_point(|p| p.start_idx <= vector_idx);
212 if pos == 0 {
213 return None;
214 }
215 let candidate = &self.partitions[pos - 1];
216 if vector_idx < candidate.end_idx {
217 Some(candidate)
218 } else {
219 None
220 }
221 }
222
223 pub fn stats(&self) -> PartitionStats {
225 let shard_count = self.partitions.len();
226 let total_vectors = self.total_vectors();
227
228 let avg_vectors_per_shard = if shard_count == 0 {
229 0.0
230 } else {
231 total_vectors as f64 / shard_count as f64
232 };
233
234 if shard_count == 0 {
235 return PartitionStats {
236 shard_count: 0,
237 total_vectors: 0,
238 avg_vectors_per_shard: 0.0,
239 max_load_shard: String::new(),
240 min_load_shard: String::new(),
241 imbalance_ratio: 0.0,
242 };
243 }
244
245 let (max_idx, min_idx) =
248 self.partitions
249 .iter()
250 .enumerate()
251 .fold((0usize, 0usize), |(max_i, min_i), (i, p)| {
252 let new_max = if p.query_load > self.partitions[max_i].query_load {
253 i
254 } else {
255 max_i
256 };
257 let new_min = if p.query_load < self.partitions[min_i].query_load {
258 i
259 } else {
260 min_i
261 };
262 (new_max, new_min)
263 });
264
265 let max_load = self.partitions[max_idx].query_load;
266 let avg_load = self.total_load() / shard_count as f64;
267 let imbalance_ratio = max_load / avg_load.max(1e-9);
268
269 PartitionStats {
270 shard_count,
271 total_vectors,
272 avg_vectors_per_shard,
273 max_load_shard: self.partitions[max_idx].shard_id.clone(),
274 min_load_shard: self.partitions[min_idx].shard_id.clone(),
275 imbalance_ratio,
276 }
277 }
278
279 pub fn rebalance_needed(&self) -> bool {
282 self.suggest_rebalance()
283 .iter()
284 .any(|a| !matches!(a, RebalanceAction::NoChange))
285 }
286
287 pub fn suggest_rebalance(&self) -> Vec<RebalanceAction> {
305 if self.partitions.is_empty() {
306 return vec![RebalanceAction::NoChange];
307 }
308
309 let mut actions: Vec<RebalanceAction> = Vec::new();
310 let mut split_shards: std::collections::HashSet<String> = std::collections::HashSet::new();
313
314 for p in &self.partitions {
318 if p.vector_count > self.config.max_vectors_per_shard {
319 let split_at = p.start_idx + (p.end_idx - p.start_idx) / 2;
320 split_shards.insert(p.shard_id.clone());
321 actions.push(RebalanceAction::Split {
322 shard_id: p.shard_id.clone(),
323 split_at,
324 });
325 }
326 }
327
328 let n = self.partitions.len();
333 let mut i = 0usize;
334 while i + 1 < n {
335 let a = &self.partitions[i];
336 let b = &self.partitions[i + 1];
337 let both_underfull = a.vector_count < self.config.min_vectors_per_shard
338 && b.vector_count < self.config.min_vectors_per_shard;
339 let neither_splitting =
340 !split_shards.contains(&a.shard_id) && !split_shards.contains(&b.shard_id);
341 if both_underfull && neither_splitting {
342 actions.push(RebalanceAction::Merge {
343 shard_a: a.shard_id.clone(),
344 shard_b: b.shard_id.clone(),
345 });
346 i += 2;
348 } else {
349 i += 1;
350 }
351 }
352
353 if self.partitions.len() > 1 {
357 let total_load = self.total_load();
358 let avg_load = total_load / self.partitions.len() as f64;
359 let threshold = avg_load * self.config.load_imbalance_threshold;
360
361 let lightest_idx = self
363 .partitions
364 .iter()
365 .enumerate()
366 .min_by(|(_, a), (_, b)| {
367 a.query_load
368 .partial_cmp(&b.query_load)
369 .unwrap_or(std::cmp::Ordering::Equal)
370 })
371 .map(|(i, _)| i)
372 .unwrap_or(0);
373
374 for (idx, p) in self.partitions.iter().enumerate() {
375 if idx == lightest_idx {
376 continue;
377 }
378 if p.query_load > threshold {
379 let migrate_count = (p.vector_count / 2).max(1);
382 actions.push(RebalanceAction::Migrate {
383 from_shard: p.shard_id.clone(),
384 to_shard: self.partitions[lightest_idx].shard_id.clone(),
385 count: migrate_count,
386 });
387 }
388 }
389 }
390
391 if actions.is_empty() {
392 vec![RebalanceAction::NoChange]
393 } else {
394 actions
395 }
396 }
397}
398
399#[cfg(test)]
404mod tests {
405 use super::*;
406
407 fn make_partition(
409 shard_id: &str,
410 start_idx: u64,
411 end_idx: u64,
412 vector_count: u64,
413 query_load: f64,
414 ) -> PartitionBoundary {
415 PartitionBoundary {
416 shard_id: shard_id.to_string(),
417 start_idx,
418 end_idx,
419 vector_count,
420 query_load,
421 }
422 }
423
424 #[test]
426 fn test_new_empty() {
427 let p = AdaptiveIndexPartitioner::new(PartitionerConfig::default());
428 assert!(p.partitions.is_empty());
429 assert_eq!(p.total_vectors(), 0);
430 assert_eq!(p.total_load(), 0.0);
431 }
432
433 #[test]
435 fn test_add_partition_total_vectors() {
436 let mut p = AdaptiveIndexPartitioner::new(PartitionerConfig::default());
437 p.add_partition(make_partition("s0", 0, 1000, 1000, 10.0));
438 p.add_partition(make_partition("s1", 1000, 2000, 1000, 20.0));
439 assert_eq!(p.total_vectors(), 2000);
440 assert_eq!(p.partitions.len(), 2);
441 }
442
443 #[test]
445 fn test_load_ratio() {
446 let pb = make_partition("s0", 0, 1000, 1000, 50.0);
447 assert!((pb.load_ratio(100.0) - 0.5).abs() < 1e-9);
449 assert!((pb.load_ratio(0.0) - 50.0 / 1e-9).abs() < 1e-3);
451 }
452
453 #[test]
455 fn test_find_shard_found() {
456 let mut p = AdaptiveIndexPartitioner::new(PartitionerConfig::default());
457 p.add_partition(make_partition("s0", 0, 500, 500, 5.0));
458 p.add_partition(make_partition("s1", 500, 1000, 500, 5.0));
459 let found = p.find_shard(750).expect("should find shard for 750");
460 assert_eq!(found.shard_id, "s1");
461 }
462
463 #[test]
465 fn test_find_shard_not_found() {
466 let mut p = AdaptiveIndexPartitioner::new(PartitionerConfig::default());
467 p.add_partition(make_partition("s0", 100, 200, 100, 1.0));
468 assert!(p.find_shard(50).is_none()); assert!(p.find_shard(200).is_none()); assert!(p.find_shard(300).is_none()); }
472
473 #[test]
475 fn test_find_shard_first() {
476 let mut p = AdaptiveIndexPartitioner::new(PartitionerConfig::default());
477 p.add_partition(make_partition("s0", 0, 100, 100, 1.0));
478 p.add_partition(make_partition("s1", 100, 200, 100, 1.0));
479 let found = p.find_shard(0).expect("idx 0 must be in s0");
480 assert_eq!(found.shard_id, "s0");
481 }
482
483 #[test]
485 fn test_find_shard_last() {
486 let mut p = AdaptiveIndexPartitioner::new(PartitionerConfig::default());
487 p.add_partition(make_partition("s0", 0, 100, 100, 1.0));
488 p.add_partition(make_partition("s1", 100, 200, 100, 1.0));
489 let found = p.find_shard(199).expect("idx 199 must be in s1");
490 assert_eq!(found.shard_id, "s1");
491 }
492
493 #[test]
495 fn test_stats_empty() {
496 let p = AdaptiveIndexPartitioner::new(PartitionerConfig::default());
497 let s = p.stats();
498 assert_eq!(s.shard_count, 0);
499 assert_eq!(s.total_vectors, 0);
500 assert_eq!(s.avg_vectors_per_shard, 0.0);
501 assert!(s.max_load_shard.is_empty());
502 assert!(s.min_load_shard.is_empty());
503 assert_eq!(s.imbalance_ratio, 0.0);
504 }
505
506 #[test]
508 fn test_stats_single_shard() {
509 let mut p = AdaptiveIndexPartitioner::new(PartitionerConfig::default());
510 p.add_partition(make_partition("only", 0, 1000, 1000, 42.0));
511 let s = p.stats();
512 assert_eq!(s.shard_count, 1);
513 assert_eq!(s.total_vectors, 1000);
514 assert!((s.avg_vectors_per_shard - 1000.0).abs() < 1e-9);
515 assert_eq!(s.max_load_shard, "only");
516 assert_eq!(s.min_load_shard, "only");
517 assert!((s.imbalance_ratio - 1.0).abs() < 1e-9);
519 }
520
521 #[test]
523 fn test_stats_multiple_shards() {
524 let mut p = AdaptiveIndexPartitioner::new(PartitionerConfig::default());
525 p.add_partition(make_partition("s0", 0, 1000, 1000, 10.0));
526 p.add_partition(make_partition("s1", 1000, 2000, 500, 30.0));
527 let s = p.stats();
528 assert_eq!(s.shard_count, 2);
529 assert_eq!(s.total_vectors, 1500);
530 assert!((s.avg_vectors_per_shard - 750.0).abs() < 1e-9);
531 assert_eq!(s.max_load_shard, "s1");
532 assert_eq!(s.min_load_shard, "s0");
533 }
534
535 #[test]
537 fn test_imbalance_ratio_unbalanced() {
538 let mut p = AdaptiveIndexPartitioner::new(PartitionerConfig::default());
539 p.add_partition(make_partition("low", 0, 1000, 1000, 5.0));
540 p.add_partition(make_partition("high", 1000, 2000, 1000, 95.0));
541 let s = p.stats();
542 assert!(s.imbalance_ratio > 1.0);
544 }
545
546 #[test]
548 fn test_suggest_rebalance_overfull_split() {
549 let cfg = PartitionerConfig {
550 max_vectors_per_shard: 1_000,
551 ..Default::default()
552 };
553 let mut p = AdaptiveIndexPartitioner::new(cfg);
554 p.add_partition(make_partition("big", 0, 4000, 2000, 10.0));
556 let actions = p.suggest_rebalance();
557 let split = actions
558 .iter()
559 .find(|a| matches!(a, RebalanceAction::Split { .. }));
560 assert!(split.is_some(), "expected a Split action");
561 if let Some(RebalanceAction::Split { shard_id, .. }) = split {
562 assert_eq!(shard_id, "big");
563 }
564 }
565
566 #[test]
568 fn test_suggest_rebalance_underfull_merge() {
569 let cfg = PartitionerConfig {
570 min_vectors_per_shard: 1_000,
571 max_vectors_per_shard: 50_000,
572 ..Default::default()
573 };
574 let mut p = AdaptiveIndexPartitioner::new(cfg);
575 p.add_partition(make_partition("a", 0, 500, 500, 1.0));
577 p.add_partition(make_partition("b", 500, 1000, 500, 1.0));
578 let actions = p.suggest_rebalance();
579 let merge = actions
580 .iter()
581 .find(|a| matches!(a, RebalanceAction::Merge { .. }));
582 assert!(merge.is_some(), "expected a Merge action");
583 if let Some(RebalanceAction::Merge { shard_a, shard_b }) = merge {
584 assert_eq!(shard_a, "a");
585 assert_eq!(shard_b, "b");
586 }
587 }
588
589 #[test]
591 fn test_suggest_rebalance_overloaded_migrate() {
592 let cfg = PartitionerConfig {
593 load_imbalance_threshold: 2.0,
594 max_vectors_per_shard: 50_000,
595 min_vectors_per_shard: 0,
596 ..Default::default()
597 };
598 let mut p = AdaptiveIndexPartitioner::new(cfg);
599 p.add_partition(make_partition("s0", 0, 10_000, 10_000, 1000.0));
602 p.add_partition(make_partition("s1", 10_000, 20_000, 10_000, 1.0));
603 p.add_partition(make_partition("s2", 20_000, 30_000, 10_000, 1.0));
604 let actions = p.suggest_rebalance();
605 let migrate = actions
606 .iter()
607 .find(|a| matches!(a, RebalanceAction::Migrate { .. }));
608 assert!(migrate.is_some(), "expected a Migrate action");
609 if let Some(RebalanceAction::Migrate {
610 from_shard,
611 to_shard,
612 ..
613 }) = migrate
614 {
615 assert_eq!(from_shard, "s0");
616 assert_eq!(to_shard, "s1");
617 }
618 }
619
620 #[test]
622 fn test_suggest_rebalance_balanced_nochange() {
623 let cfg = PartitionerConfig {
624 max_vectors_per_shard: 50_000,
625 min_vectors_per_shard: 100,
626 load_imbalance_threshold: 2.0,
627 target_shard_count: 8,
628 };
629 let mut p = AdaptiveIndexPartitioner::new(cfg);
630 p.add_partition(make_partition("s0", 0, 5000, 5000, 50.0));
632 p.add_partition(make_partition("s1", 5000, 10_000, 5000, 60.0));
633 let actions = p.suggest_rebalance();
634 assert_eq!(actions.len(), 1);
635 assert!(matches!(actions[0], RebalanceAction::NoChange));
636 }
637
638 #[test]
640 fn test_rebalance_needed() {
641 let cfg = PartitionerConfig {
642 max_vectors_per_shard: 1_000,
643 ..Default::default()
644 };
645 let mut p = AdaptiveIndexPartitioner::new(cfg);
646 assert!(!p.rebalance_needed());
648 p.add_partition(make_partition("big", 0, 4000, 4000, 10.0));
650 assert!(p.rebalance_needed());
651 }
652
653 #[test]
655 fn test_total_load_sum() {
656 let mut p = AdaptiveIndexPartitioner::new(PartitionerConfig::default());
657 p.add_partition(make_partition("s0", 0, 1000, 1000, 33.3));
658 p.add_partition(make_partition("s1", 1000, 2000, 1000, 66.7));
659 assert!((p.total_load() - 100.0).abs() < 1e-6);
660 }
661
662 #[test]
664 fn test_split_at_midpoint() {
665 let cfg = PartitionerConfig {
666 max_vectors_per_shard: 1_000,
667 ..Default::default()
668 };
669 let mut p = AdaptiveIndexPartitioner::new(cfg);
670 p.add_partition(make_partition("m", 0, 4000, 2000, 5.0));
672 let actions = p.suggest_rebalance();
673 let split = actions
674 .iter()
675 .find_map(|a| {
676 if let RebalanceAction::Split { shard_id, split_at } = a {
677 if shard_id == "m" {
678 Some(*split_at)
679 } else {
680 None
681 }
682 } else {
683 None
684 }
685 })
686 .expect("Split action for 'm' must be present");
687 assert_eq!(split, 2000);
688 }
689
690 #[test]
692 fn test_migrate_targets_lightest_shard() {
693 let cfg = PartitionerConfig {
694 load_imbalance_threshold: 2.0,
695 max_vectors_per_shard: 50_000,
696 min_vectors_per_shard: 0,
697 target_shard_count: 8,
698 };
699 let mut p = AdaptiveIndexPartitioner::new(cfg);
700 p.add_partition(make_partition("heavy", 0, 10_000, 10_000, 900.0));
701 p.add_partition(make_partition("medium", 10_000, 20_000, 10_000, 100.0));
702 p.add_partition(make_partition("light", 20_000, 30_000, 10_000, 1.0));
703 let actions = p.suggest_rebalance();
704 let targets: Vec<&str> = actions
706 .iter()
707 .filter_map(|a| {
708 if let RebalanceAction::Migrate {
709 from_shard,
710 to_shard,
711 ..
712 } = a
713 {
714 if from_shard == "heavy" {
715 Some(to_shard.as_str())
716 } else {
717 None
718 }
719 } else {
720 None
721 }
722 })
723 .collect();
724 assert!(!targets.is_empty(), "expected a Migrate from 'heavy'");
725 assert!(
726 targets.iter().all(|&t| t == "light"),
727 "migrate target must be 'light', got {targets:?}"
728 );
729 }
730}