1use std::io;
16use std::time::Duration;
17use std::time::Instant;
18
19use crate::StatsOptions;
20use crate::cache::runtime::metrics::ActivityMetrics;
21use crate::config::CacheConfig;
22use crate::config::StorageLayout;
23use crate::config::invalid_config;
24use crate::error::Error;
25use crate::error::ErrorOperation;
26use crate::error::from_io;
27use crate::io::engine::CACHE_IO_COMPLETION_TIMEOUT;
28use crate::io::engine::IO_QUEUE_ENTRY_RESERVATION_BYTES;
29use crate::io::engine::MAX_IO_REQUESTS_PER_ENGINE;
30use crate::io::engine::io_uring_extra_memory_bytes;
31use crate::io::fill_control::FillController;
32use crate::managed_memory::BUFFER_ALIGNMENT;
33use crate::managed_memory::CACHE_THREAD_STACK_BYTES;
34use crate::memory::MemoryStore;
35use crate::region::recovery::DataGeometry;
36use crate::region::runtime_fixed_memory_bytes;
37use crate::region::staging::AppendStaging;
38use crate::stats::recording::Recorder;
39
40const DEFAULT_L1_SHARDS: usize = 32;
41const MAX_CONFIG_COUNT: usize = 65_536;
42const MAX_APPEND_SHARDS: u32 = 256;
43pub const MAX_WRITE_FLUSH_THRESHOLD_BYTES: usize = 4 * 1024 * 1024;
44const MAX_READ_IO_WAIT_TIMEOUT: Duration = Duration::from_secs(5);
45const DEFAULT_L1_CAPACITY_BYTES: usize = 256 * 1024 * 1024;
46const DEFAULT_APPEND_SHARDS: u32 = 4;
47const DEFAULT_POSIX_IO_WORKERS: usize = 4;
48const DEFAULT_IO_URING_MAX_IN_FLIGHT: usize = 64;
49const DEFAULT_RECLAIM_IO_CONCURRENCY: usize = 1;
50
51#[non_exhaustive]
56#[derive(Clone, Copy, Debug, Eq, PartialEq)]
57pub struct PosixIoOptions {
58 pub read_workers: usize,
60 pub write_workers: usize,
62 pub reclaim_workers: usize,
64}
65
66impl Default for PosixIoOptions {
67 fn default() -> Self {
68 Self {
69 read_workers: DEFAULT_POSIX_IO_WORKERS,
70 write_workers: DEFAULT_POSIX_IO_WORKERS,
71 reclaim_workers: DEFAULT_RECLAIM_IO_CONCURRENCY,
72 }
73 }
74}
75
76#[non_exhaustive]
85#[derive(Clone, Copy, Debug, Eq, PartialEq)]
86pub struct IoUringPoolOptions {
87 pub rings: usize,
93 pub max_in_flight: usize,
98 pub sq_poll: Option<IoUringSqPollOptions>,
103 pub io_poll: bool,
110}
111
112impl Default for IoUringPoolOptions {
113 fn default() -> Self {
114 Self {
115 rings: 1,
116 max_in_flight: DEFAULT_IO_URING_MAX_IN_FLIGHT,
117 sq_poll: None,
118 io_poll: false,
119 }
120 }
121}
122
123#[non_exhaustive]
129#[derive(Clone, Copy, Debug, Eq, PartialEq)]
130pub struct IoUringSqPollOptions {
131 pub idle_millis: u32,
133 pub cpu: Option<u32>,
138}
139
140impl IoUringSqPollOptions {
141 pub const fn new(idle_millis: u32) -> Self {
143 Self {
144 idle_millis,
145 cpu: None,
146 }
147 }
148}
149
150#[non_exhaustive]
155#[derive(Clone, Copy, Debug, Eq, PartialEq)]
156pub struct IoUringOptions {
157 pub read: IoUringPoolOptions,
159 pub write: IoUringPoolOptions,
161 pub reclaim: IoUringPoolOptions,
163}
164
165impl Default for IoUringOptions {
166 fn default() -> Self {
167 Self {
168 read: IoUringPoolOptions::default(),
169 write: IoUringPoolOptions::default(),
170 reclaim: IoUringPoolOptions {
171 max_in_flight: DEFAULT_RECLAIM_IO_CONCURRENCY,
172 ..IoUringPoolOptions::default()
173 },
174 }
175 }
176}
177
178#[non_exhaustive]
181#[derive(Clone, Copy, Debug, Eq, PartialEq)]
182pub enum IoEngineOptions {
183 Posix(PosixIoOptions),
185 IoUring(IoUringOptions),
191}
192
193impl Default for IoEngineOptions {
194 fn default() -> Self {
195 Self::Posix(PosixIoOptions::default())
196 }
197}
198
199const IO_URING_UNAVAILABLE_FEATURE: &str = "io_uring requires the io-uring crate feature";
200const IO_URING_UNAVAILABLE_PLATFORM: &str = "io_uring is unavailable on this platform";
201const IO_URING_UNAVAILABLE_ARCHITECTURE: &str = "io_uring is unavailable on this architecture";
202
203pub const fn io_uring_unavailability() -> Option<&'static str> {
205 if cfg!(not(target_os = "linux")) {
206 Some(IO_URING_UNAVAILABLE_PLATFORM)
207 } else if cfg!(not(any(
208 target_arch = "x86_64",
209 target_arch = "aarch64",
210 target_arch = "riscv64",
211 target_arch = "loongarch64",
212 target_arch = "powerpc64"
213 ))) {
214 Some(IO_URING_UNAVAILABLE_ARCHITECTURE)
215 } else if cfg!(not(feature = "io-uring")) {
216 Some(IO_URING_UNAVAILABLE_FEATURE)
217 } else {
218 None
219 }
220}
221
222#[derive(Clone, Copy, Debug, Eq, PartialEq)]
225pub enum IoPoolTopology {
226 Posix { workers: usize },
227 IoUring(IoUringPoolOptions),
228}
229
230impl IoPoolTopology {
231 pub const fn read(options: IoEngineOptions) -> Self {
232 match options {
233 IoEngineOptions::Posix(options) => Self::Posix {
234 workers: options.read_workers,
235 },
236 IoEngineOptions::IoUring(options) => Self::IoUring(options.read),
237 }
238 }
239
240 pub const fn write(options: IoEngineOptions) -> Self {
241 match options {
242 IoEngineOptions::Posix(options) => Self::Posix {
243 workers: options.write_workers,
244 },
245 IoEngineOptions::IoUring(options) => Self::IoUring(options.write),
246 }
247 }
248
249 pub const fn reclaim(options: IoEngineOptions) -> Self {
250 match options {
251 IoEngineOptions::Posix(options) => Self::Posix {
252 workers: options.reclaim_workers,
253 },
254 IoEngineOptions::IoUring(options) => Self::IoUring(options.reclaim),
255 }
256 }
257
258 pub const fn engine_count(self) -> usize {
259 match self {
260 Self::Posix { .. } => 1,
261 Self::IoUring(options) => options.rings,
262 }
263 }
264
265 pub const fn max_in_flight(self) -> usize {
266 match self {
267 Self::Posix { workers } => workers,
268 Self::IoUring(options) => options.max_in_flight,
269 }
270 }
271
272 pub const fn worker_threads(self) -> usize {
273 match self {
274 Self::Posix { workers } => workers,
275 Self::IoUring(options) => options.rings,
276 }
277 }
278
279 pub fn extra_memory_bytes(self) -> Option<usize> {
280 match self {
281 Self::Posix { .. } => Some(0),
282 Self::IoUring(options) => {
283 io_uring_extra_memory_bytes(options.max_in_flight, options.rings)
284 }
285 }
286 }
287
288 pub const fn engine_config(self, engine: usize) -> IoEngineConfig {
289 assert!(
290 engine < self.engine_count(),
291 "engine index exceeds pool topology"
292 );
293 match self {
294 Self::Posix { workers } => IoEngineConfig::Posix { workers },
295 Self::IoUring(options) => {
296 let base = options.max_in_flight / options.rings;
297 let remainder = options.max_in_flight % options.rings;
298 IoEngineConfig::IoUring(IoUringEngineConfig {
299 max_in_flight: base + if engine < remainder { 1 } else { 0 },
300 sq_poll: options.sq_poll,
301 io_poll: options.io_poll,
302 })
303 }
304 }
305 }
306}
307
308#[derive(Clone, Copy, Debug, Eq, PartialEq)]
309pub enum IoEngineConfig {
310 Posix { workers: usize },
311 IoUring(IoUringEngineConfig),
312}
313
314#[derive(Clone, Copy, Debug, Eq, PartialEq)]
315pub struct IoUringEngineConfig {
316 pub max_in_flight: usize,
317 pub sq_poll: Option<IoUringSqPollOptions>,
318 pub io_poll: bool,
319}
320
321#[non_exhaustive]
326#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)]
327pub enum IoMode {
328 #[default]
330 Buffered,
331 Direct,
336}
337
338#[non_exhaustive]
340#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)]
341pub enum L1EvictionPolicy {
342 #[default]
344 Clock,
345 S3Fifo,
347}
348
349#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
351pub enum ReadAdmission {
352 #[default]
354 Immediate,
355 Wait {
358 timeout: Duration,
360 max_waiters: Option<usize>,
363 },
364}
365
366#[non_exhaustive]
368#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
369pub enum FillControlOptions {
370 #[default]
372 Disabled,
373 Observe(FillLimits),
375 Adaptive(FillLimits),
377}
378
379#[non_exhaustive]
386#[derive(Clone, Copy, Debug, Eq, PartialEq)]
387pub struct FillLimits {
388 pub max_bytes_per_second: u64,
391 pub max_records_per_second: u32,
394}
395
396impl FillLimits {
397 pub const fn new(max_bytes_per_second: u64, max_records_per_second: u32) -> Self {
399 Self {
400 max_bytes_per_second,
401 max_records_per_second,
402 }
403 }
404}
405
406#[non_exhaustive]
412#[derive(Clone, Debug, Eq, PartialEq)]
413pub struct RuntimeOptions {
414 pub io_engine: IoEngineOptions,
417 pub io_mode: IoMode,
420 pub read_admission: ReadAdmission,
422 pub reclaim_io_timeout: Duration,
430 pub io_recovery_timeout: Option<Duration>,
441 pub fill_control: FillControlOptions,
447 pub append_shards: u32,
452 pub l1_capacity_bytes: usize,
457 pub l1_eviction_policy: L1EvictionPolicy,
460 pub managed_memory_limit_bytes: usize,
466 pub l1_shards: usize,
471 pub write_flush_threshold_bytes: usize,
477 pub stats: StatsOptions,
482}
483
484impl Default for RuntimeOptions {
485 fn default() -> Self {
486 Self {
487 io_engine: IoEngineOptions::default(),
488 io_mode: IoMode::Buffered,
489 read_admission: ReadAdmission::Immediate,
490 reclaim_io_timeout: Duration::from_secs(5),
491 io_recovery_timeout: None,
492 fill_control: FillControlOptions::Disabled,
493 append_shards: DEFAULT_APPEND_SHARDS,
494 l1_capacity_bytes: DEFAULT_L1_CAPACITY_BYTES,
495 l1_eviction_policy: L1EvictionPolicy::Clock,
496 managed_memory_limit_bytes: 1024 * 1024 * 1024,
497 l1_shards: DEFAULT_L1_SHARDS,
498 write_flush_threshold_bytes: MAX_WRITE_FLUSH_THRESHOLD_BYTES,
499 stats: StatsOptions::default(),
500 }
501 }
502}
503
504pub const fn read_io_wait_capacity(options: &RuntimeOptions) -> usize {
505 match options.read_admission {
506 ReadAdmission::Immediate => 0,
507 ReadAdmission::Wait { max_waiters, .. } => match max_waiters {
508 Some(capacity) => capacity,
509 None => IoPoolTopology::read(options.io_engine).max_in_flight(),
510 },
511 }
512}
513
514pub const fn read_io_wait_timeout(options: &RuntimeOptions) -> Duration {
515 match options.read_admission {
516 ReadAdmission::Immediate => Duration::ZERO,
517 ReadAdmission::Wait { timeout, .. } => timeout,
518 }
519}
520
521const RUNTIME_CONTROL_RESERVATION_BYTES: usize = 4096;
524const MIN_L1_SIZING_ENTRY_BYTES: usize = 4 * 1024;
528
529impl CacheConfig {
530 pub fn new(storage: StorageLayout, mut runtime: RuntimeOptions) -> Result<Self, Error> {
569 let build = || -> io::Result<Self> {
570 let geometry = storage.geometry();
571 let index_slots = storage.index_slots();
572 runtime.resolve()?;
573 let stats_bytes = Recorder::allocation_bytes(runtime.stats)?;
574 let fill_bytes = FillController::allocation_bytes(
575 runtime.fill_control,
576 runtime.append_shards as usize
577 + IoPoolTopology::reclaim(runtime.io_engine).max_in_flight(),
578 )?;
579 if geometry.region_count <= runtime.append_shards {
580 return Err(invalid_config(
581 "append shards require valid geometry with one Active Region each plus one spare Region",
582 ));
583 }
584 let l1_entry_capacity = runtime.l1_entry_capacity(geometry, index_slots)?;
585 let l1_metadata_bytes = MemoryStore::allocation_bytes(
586 runtime.l1_capacity_bytes,
587 l1_entry_capacity,
588 runtime.l1_shards,
589 runtime.l1_eviction_policy,
590 )?;
591 let fixed_bytes = runtime_fixed_memory_bytes(index_slots, geometry.region_count)?
592 .checked_add(l1_metadata_bytes)
593 .and_then(|bytes| bytes.checked_add(stats_bytes))
594 .and_then(|bytes| bytes.checked_add(fill_bytes))
595 .ok_or_else(|| invalid_config("fixed memory requirements overflow"))?;
596 let (reserved_memory_bytes, minimum_memory_bytes) =
597 runtime.memory_requirements(geometry, fixed_bytes)?;
598 if minimum_memory_bytes > runtime.managed_memory_limit_bytes {
599 return Err(io::Error::new(
600 io::ErrorKind::InvalidInput,
601 format!(
602 "managed memory limit cannot hold the cache memory requirements: requires {minimum_memory_bytes} bytes, configured {} bytes",
603 runtime.managed_memory_limit_bytes
604 ),
605 ));
606 }
607 Ok(Self {
608 storage,
609 runtime,
610 l1_entry_capacity,
611 reserved_memory_bytes,
612 minimum_memory_bytes,
613 })
614 };
615 build().map_err(|error| from_io(ErrorOperation::BuildConfig, error))
616 }
617}
618
619impl RuntimeOptions {
620 fn resolve(&mut self) -> io::Result<()> {
621 if matches!(self.io_engine, IoEngineOptions::IoUring(_))
622 && let Some(reason) = io_uring_unavailability()
623 {
624 return Err(io::Error::new(io::ErrorKind::Unsupported, reason));
625 }
626 if self.io_mode == IoMode::Direct && cfg!(not(target_os = "linux")) {
627 return Err(io::Error::new(
628 io::ErrorKind::Unsupported,
629 "direct I/O is unavailable on this platform",
630 ));
631 }
632 if self.append_shards == 0 || self.append_shards > MAX_APPEND_SHARDS {
633 return Err(invalid_config("append shards must be in 1..=256"));
634 }
635 if self.reclaim_io_timeout.is_zero()
636 || Instant::now()
637 .checked_add(self.reclaim_io_timeout)
638 .is_none()
639 {
640 return Err(invalid_config(
641 "reclaim I/O timeout must be positive and fit an absolute deadline",
642 ));
643 }
644 if let Some(recovery_timeout) = self.io_recovery_timeout
645 && self
646 .reclaim_io_timeout
647 .max(CACHE_IO_COMPLETION_TIMEOUT)
648 .checked_add(recovery_timeout)
649 .and_then(|timeout| Instant::now().checked_add(timeout))
650 .is_none()
651 {
652 return Err(invalid_config(
653 "I/O recovery timeout must fit the combined absolute deadline",
654 ));
655 }
656 let read_topology = IoPoolTopology::read(self.io_engine);
657 let write_topology = IoPoolTopology::write(self.io_engine);
658 let reclaim_topology = IoPoolTopology::reclaim(self.io_engine);
659 match self.io_engine {
660 IoEngineOptions::Posix(_) => {
661 validate_posix_pool("read", read_topology)?;
662 validate_posix_pool("write", write_topology)?;
663 validate_posix_pool("reclaim", reclaim_topology)?;
664 }
665 IoEngineOptions::IoUring(options) => {
666 validate_io_uring_pool("read", options.read)?;
667 validate_io_uring_pool("write", options.write)?;
668 validate_io_uring_pool("reclaim", options.reclaim)?;
669 if self.io_mode != IoMode::Direct
670 && (options.read.io_poll || options.write.io_poll || options.reclaim.io_poll)
671 {
672 return Err(invalid_config("io_uring IOPOLL requires direct I/O mode"));
673 }
674 }
675 }
676 if reclaim_topology.max_in_flight() > self.append_shards as usize {
677 return Err(invalid_config(
678 "reclaim I/O concurrency must be no greater than append shards",
679 ));
680 }
681 if let ReadAdmission::Wait {
682 timeout,
683 max_waiters,
684 } = &mut self.read_admission
685 {
686 if timeout.is_zero() || *timeout > MAX_READ_IO_WAIT_TIMEOUT {
687 return Err(invalid_config(
688 "read wait timeout must be greater than zero and at most five seconds",
689 ));
690 }
691 let capacity = max_waiters.unwrap_or(read_topology.max_in_flight());
692 if !(1..=MAX_CONFIG_COUNT).contains(&capacity) {
693 return Err(invalid_config("maximum read waiters must be in 1..=65536"));
694 }
695 *max_waiters = Some(capacity);
696 }
697 if self.managed_memory_limit_bytes == 0 {
698 return Err(invalid_config("managed memory limit must be non-zero"));
699 }
700 if self.l1_capacity_bytes > self.managed_memory_limit_bytes {
701 return Err(invalid_config(
702 "L1 capacity must not exceed the managed memory limit",
703 ));
704 }
705 if self.l1_shards == 0 || self.l1_shards > MAX_CONFIG_COUNT {
706 return Err(invalid_config("L1 shards must be in 1..=65536"));
707 }
708 if self.write_flush_threshold_bytes == 0
709 || self.write_flush_threshold_bytes > MAX_WRITE_FLUSH_THRESHOLD_BYTES
710 || !self
711 .write_flush_threshold_bytes
712 .is_multiple_of(BUFFER_ALIGNMENT)
713 {
714 return Err(invalid_config(
715 "write flush threshold must be 4 KiB aligned and within 4 KiB..=4 MiB",
716 ));
717 }
718 Ok(())
719 }
720
721 fn l1_entry_capacity(&self, geometry: DataGeometry, index_slots: usize) -> io::Result<usize> {
722 if self.l1_capacity_bytes == 0 {
723 return Ok(0);
724 }
725 let l2_capacity = u128::from(geometry.region_size)
726 .checked_mul(u128::from(geometry.region_count))
727 .filter(|capacity| *capacity != 0)
728 .ok_or_else(|| invalid_config("L2 capacity does not fit the L1 sizing"))?;
729 let expected_entries = index_slots.div_ceil(2).max(1);
730 let proportional = (expected_entries as u128)
731 .checked_mul(self.l1_capacity_bytes as u128)
732 .and_then(|entries| entries.checked_add(l2_capacity - 1))
733 .map(|entries| entries / l2_capacity)
734 .and_then(|entries| usize::try_from(entries).ok())
735 .ok_or_else(|| invalid_config("L1 entry capacity does not fit usize"))?;
736 let four_kib_density = self.l1_capacity_bytes.div_ceil(MIN_L1_SIZING_ENTRY_BYTES);
737 let maximum = MemoryStore::maximum_entry_capacity(self.l1_capacity_bytes, self.l1_shards);
738 let minimum = self.l1_shards.min(maximum);
739 Ok(proportional
740 .max(four_kib_density)
741 .min(expected_entries)
742 .max(minimum)
743 .min(maximum))
744 }
745
746 fn memory_requirements(
747 &self,
748 geometry: DataGeometry,
749 fixed_bytes: usize,
750 ) -> io::Result<(usize, usize)> {
751 let shard_count = self.append_shards as usize;
752 let topology_bytes = runtime_topology_memory_bytes(self)
753 .ok_or_else(|| invalid_config("runtime topology memory requirements overflow"))?;
754 let region_bytes = usize::try_from(geometry.region_size)
755 .map_err(|_| invalid_config("Region size does not fit the memory requirements"))?;
756 let write_buffer_reservation = AppendStaging::reservation_bytes(shard_count, region_bytes)
757 .ok_or_else(|| invalid_config("write buffer memory requirements overflow"))?;
758 let reserved_memory = fixed_bytes
759 .checked_add(self.l1_capacity_bytes)
760 .and_then(|bytes| bytes.checked_add(topology_bytes))
761 .ok_or_else(|| invalid_config("reserved memory requirements overflow"))?;
762 let reclaim_buffers = region_bytes
763 .checked_mul(IoPoolTopology::reclaim(self.io_engine).max_in_flight())
764 .ok_or_else(|| invalid_config("reclaim buffer memory requirements overflow"))?;
765 let minimum = reserved_memory
766 .checked_add(write_buffer_reservation)
767 .and_then(|bytes| bytes.checked_add(reclaim_buffers))
770 .and_then(|bytes| bytes.checked_add(region_bytes))
771 .ok_or_else(|| invalid_config("minimum memory requirements overflow"))?;
772 Ok((reserved_memory, minimum))
773 }
774}
775
776fn runtime_topology_memory_bytes(options: &RuntimeOptions) -> Option<usize> {
777 let shard_count = options.append_shards as usize;
778 let read = IoPoolTopology::read(options.io_engine);
781 let write = IoPoolTopology::write(options.io_engine);
782 let reclaim = IoPoolTopology::reclaim(options.io_engine);
783 let engine_count = read
784 .engine_count()
785 .checked_add(write.engine_count())?
786 .checked_add(reclaim.engine_count())?;
787 let stack_count = read
788 .worker_threads()
789 .checked_add(write.worker_threads())?
790 .checked_add(reclaim.worker_threads())?
791 .checked_add(engine_count)?
792 .checked_add(shard_count)?
793 .checked_add(reclaim.max_in_flight())?;
794 let stacks = stack_count.checked_mul(CACHE_THREAD_STACK_BYTES)?;
795 let read_wait_queue = read_io_wait_capacity(options);
796 let queue = write
797 .max_in_flight()
798 .checked_add(read.max_in_flight())?
799 .checked_add(read_wait_queue)?
800 .checked_add(reclaim.max_in_flight())?
801 .checked_mul(IO_QUEUE_ENTRY_RESERVATION_BYTES)?;
802 let uring = [read, write, reclaim]
803 .into_iter()
804 .try_fold(0_usize, |bytes, pool| {
805 bytes.checked_add(pool.extra_memory_bytes()?)
806 })?;
807 let controls = engine_count
808 .checked_add(shard_count)?
809 .checked_add(options.l1_shards)?
810 .checked_add(reclaim.max_in_flight())?
811 .checked_mul(RUNTIME_CONTROL_RESERVATION_BYTES)?;
812 let metrics = shard_count.checked_mul(size_of::<ActivityMetrics>())?;
813 stacks
814 .checked_add(queue)?
815 .checked_add(uring)?
816 .checked_add(controls)?
817 .checked_add(metrics)
818}
819
820fn validate_posix_pool(name: &str, topology: IoPoolTopology) -> io::Result<()> {
821 if !(1..=MAX_IO_REQUESTS_PER_ENGINE).contains(&topology.max_in_flight()) {
822 return Err(io::Error::new(
823 io::ErrorKind::InvalidInput,
824 format!("POSIX {name} worker count must be in 1..={MAX_IO_REQUESTS_PER_ENGINE}"),
825 ));
826 }
827 Ok(())
828}
829
830fn validate_io_uring_pool(name: &str, options: IoUringPoolOptions) -> io::Result<()> {
831 if !(1..=MAX_CONFIG_COUNT).contains(&options.rings) {
832 return Err(io::Error::new(
833 io::ErrorKind::InvalidInput,
834 format!("io_uring {name} ring count must be in 1..={MAX_CONFIG_COUNT}"),
835 ));
836 }
837 if !(1..=MAX_CONFIG_COUNT).contains(&options.max_in_flight) {
838 return Err(io::Error::new(
839 io::ErrorKind::InvalidInput,
840 format!("io_uring {name} maximum in-flight requests must be in 1..={MAX_CONFIG_COUNT}"),
841 ));
842 }
843 if options.rings > options.max_in_flight {
844 return Err(io::Error::new(
845 io::ErrorKind::InvalidInput,
846 format!("io_uring {name} ring count must not exceed its in-flight limit"),
847 ));
848 }
849 if options.max_in_flight.div_ceil(options.rings) > MAX_IO_REQUESTS_PER_ENGINE {
850 return Err(io::Error::new(
851 io::ErrorKind::InvalidInput,
852 format!("io_uring {name} per-ring depth must not exceed {MAX_IO_REQUESTS_PER_ENGINE}"),
853 ));
854 }
855 Ok(())
856}
857
858#[cfg(test)]
859mod tests {
860 use super::*;
861 use crate::ErrorKind;
862 use crate::StorageOptions;
863
864 #[test]
865 fn fill_control_validates_ceilings_and_accounts_observation_memory() {
866 let storage = StorageOptions::new(1024 * 1024 * 1024).build().unwrap();
867 let base = CacheConfig::new(storage.clone(), RuntimeOptions::default()).unwrap();
868 assert_eq!(base.runtime().fill_control, FillControlOptions::Disabled);
869 for (bytes, operations) in [(639, 100), ((1 << 40) + 1, 100), (640, 9)] {
870 let options = RuntimeOptions {
871 fill_control: FillControlOptions::Adaptive(FillLimits::new(bytes, operations)),
872 ..RuntimeOptions::default()
873 };
874 assert_eq!(
875 CacheConfig::new(storage.clone(), options)
876 .unwrap_err()
877 .kind(),
878 ErrorKind::InvalidInput
879 );
880 }
881 CacheConfig::new(
882 storage.clone(),
883 RuntimeOptions {
884 fill_control: FillControlOptions::Adaptive(FillLimits::new(640, u32::MAX)),
885 ..RuntimeOptions::default()
886 },
887 )
888 .unwrap();
889 let mut minimum = None;
890 for mode in [FillControlOptions::Observe, FillControlOptions::Adaptive] {
891 let config = CacheConfig::new(
892 storage.clone(),
893 RuntimeOptions {
894 fill_control: mode(FillLimits::new(64_000, 100)),
895 ..RuntimeOptions::default()
896 },
897 )
898 .unwrap();
899 let extra = config.minimum_memory_bytes() - base.minimum_memory_bytes();
900 assert!(extra > 0);
901 assert!(extra < CACHE_THREAD_STACK_BYTES);
902 if let Some(previous) = minimum {
903 assert_eq!(extra, previous);
904 }
905 minimum = Some(extra);
906 }
907 }
908
909 #[test]
910 fn recovery_timeout_allows_zero_and_rejects_overflow() {
911 let storage = StorageOptions::new(1024 * 1024 * 1024).build().unwrap();
912 assert_eq!(RuntimeOptions::default().io_recovery_timeout, None);
913 for timeout in [Duration::ZERO, Duration::from_secs(30)] {
914 let options = RuntimeOptions {
915 io_recovery_timeout: Some(timeout),
916 ..RuntimeOptions::default()
917 };
918 CacheConfig::new(storage.clone(), options).unwrap();
919 }
920 let options = RuntimeOptions {
921 io_recovery_timeout: Some(Duration::MAX),
922 ..RuntimeOptions::default()
923 };
924 assert_eq!(
925 CacheConfig::new(storage, options).unwrap_err().kind(),
926 ErrorKind::InvalidInput
927 );
928 }
929
930 #[test]
931 fn reclaim_timeout_is_validated_at_config_construction() {
932 let storage = StorageOptions::new(1024 * 1024 * 1024).build().unwrap();
933 assert_eq!(
934 RuntimeOptions::default().reclaim_io_timeout,
935 Duration::from_secs(5)
936 );
937 for timeout in [Duration::ZERO, Duration::MAX] {
938 let error = CacheConfig::new(
939 storage.clone(),
940 RuntimeOptions {
941 reclaim_io_timeout: timeout,
942 ..RuntimeOptions::default()
943 },
944 )
945 .unwrap_err();
946 assert_eq!(error.kind(), ErrorKind::InvalidInput);
947 }
948 CacheConfig::new(
949 storage,
950 RuntimeOptions {
951 reclaim_io_timeout: Duration::from_secs(30),
952 ..RuntimeOptions::default()
953 },
954 )
955 .unwrap();
956 }
957
958 #[test]
959 fn optional_read_wait_queue_is_memory_accounted() {
960 let base = RuntimeOptions {
961 io_engine: IoEngineOptions::Posix(PosixIoOptions {
962 read_workers: 7,
963 write_workers: 4,
964 reclaim_workers: 1,
965 }),
966 ..RuntimeOptions::default()
967 };
968 let no_wait = runtime_topology_memory_bytes(&base).unwrap();
969 let with_wait = runtime_topology_memory_bytes(&RuntimeOptions {
970 read_admission: ReadAdmission::Wait {
971 timeout: Duration::from_millis(1),
972 max_waiters: Some(11),
973 },
974 ..base
975 })
976 .unwrap();
977
978 assert_eq!(with_wait - no_wait, 11 * IO_QUEUE_ENTRY_RESERVATION_BYTES);
979 }
980
981 #[test]
982 fn each_additional_reclaimer_is_fully_memory_accounted() {
983 let geometry = DataGeometry {
984 data_file_len: DataGeometry::expected_file_len(512 * 1024, 10).unwrap(),
985 region_size: 512 * 1024,
986 region_count: 10,
987 };
988 let base = RuntimeOptions {
989 append_shards: 4,
990 l1_capacity_bytes: 0,
991 ..RuntimeOptions::default()
992 };
993 let (_, base_minimum) = base.memory_requirements(geometry, 0).unwrap();
994 let (_, parallel_minimum) = RuntimeOptions {
995 io_engine: IoEngineOptions::Posix(PosixIoOptions {
996 read_workers: 4,
997 write_workers: 4,
998 reclaim_workers: 2,
999 }),
1000 ..base
1001 }
1002 .memory_requirements(geometry, 0)
1003 .unwrap();
1004
1005 assert_eq!(
1006 parallel_minimum - base_minimum,
1007 geometry.region_size as usize
1008 + 2 * CACHE_THREAD_STACK_BYTES
1009 + IO_QUEUE_ENTRY_RESERVATION_BYTES
1010 + RUNTIME_CONTROL_RESERVATION_BYTES
1011 );
1012 }
1013
1014 #[test]
1015 fn io_engine_topology_matches_backend_shape() {
1016 let posix = IoEngineOptions::Posix(PosixIoOptions {
1017 read_workers: 7,
1018 write_workers: 5,
1019 reclaim_workers: 2,
1020 });
1021 assert_eq!(
1022 IoPoolTopology::read(posix),
1023 IoPoolTopology::Posix { workers: 7 }
1024 );
1025
1026 let io_uring = IoEngineOptions::IoUring(IoUringOptions {
1027 read: IoUringPoolOptions {
1028 rings: 3,
1029 max_in_flight: 8,
1030 ..IoUringPoolOptions::default()
1031 },
1032 write: IoUringPoolOptions {
1033 rings: 2,
1034 max_in_flight: 5,
1035 ..IoUringPoolOptions::default()
1036 },
1037 reclaim: IoUringPoolOptions {
1038 rings: 1,
1039 max_in_flight: 2,
1040 ..IoUringPoolOptions::default()
1041 },
1042 });
1043 let read = IoPoolTopology::read(io_uring);
1044 assert_eq!(read.engine_count(), 3);
1045 assert_eq!(read.max_in_flight(), 8);
1046 assert_eq!(read.worker_threads(), 3);
1047 let depths: Vec<_> = (0..read.engine_count())
1048 .map(|engine| match read.engine_config(engine) {
1049 IoEngineConfig::IoUring(config) => config.max_in_flight,
1050 IoEngineConfig::Posix { .. } => panic!("expected an io_uring engine"),
1051 })
1052 .collect();
1053 assert_eq!(depths, [3, 3, 2]);
1054 assert_eq!(depths.iter().sum::<usize>(), read.max_in_flight());
1055 }
1056
1057 #[test]
1058 fn io_uring_depth_reserves_more_than_common_request_bookkeeping() {
1059 let pool = IoUringPoolOptions {
1060 rings: 1,
1061 max_in_flight: 1,
1062 ..IoUringPoolOptions::default()
1063 };
1064 let shallow = RuntimeOptions {
1065 io_engine: IoEngineOptions::IoUring(IoUringOptions {
1066 read: pool,
1067 write: pool,
1068 reclaim: pool,
1069 }),
1070 ..RuntimeOptions::default()
1071 };
1072 let deep = RuntimeOptions {
1073 io_engine: IoEngineOptions::IoUring(IoUringOptions {
1074 read: IoUringPoolOptions {
1075 rings: 1,
1076 max_in_flight: MAX_IO_REQUESTS_PER_ENGINE,
1077 ..IoUringPoolOptions::default()
1078 },
1079 write: pool,
1080 reclaim: pool,
1081 }),
1082 ..shallow.clone()
1083 };
1084 let growth = runtime_topology_memory_bytes(&deep).unwrap()
1085 - runtime_topology_memory_bytes(&shallow).unwrap();
1086 assert!(growth > (MAX_IO_REQUESTS_PER_ENGINE - 1) * IO_QUEUE_ENTRY_RESERVATION_BYTES);
1087 }
1088
1089 #[test]
1090 fn io_uring_pool_validation_bounds_rings_and_depth() {
1091 validate_io_uring_pool(
1092 "read",
1093 IoUringPoolOptions {
1094 rings: 3,
1095 max_in_flight: 8,
1096 ..IoUringPoolOptions::default()
1097 },
1098 )
1099 .unwrap();
1100
1101 for config in [
1102 IoUringPoolOptions {
1103 rings: 0,
1104 max_in_flight: 8,
1105 ..IoUringPoolOptions::default()
1106 },
1107 IoUringPoolOptions {
1108 rings: 1,
1109 max_in_flight: 0,
1110 ..IoUringPoolOptions::default()
1111 },
1112 IoUringPoolOptions {
1113 rings: 3,
1114 max_in_flight: 2,
1115 ..IoUringPoolOptions::default()
1116 },
1117 IoUringPoolOptions {
1118 rings: 1,
1119 max_in_flight: MAX_IO_REQUESTS_PER_ENGINE + 1,
1120 ..IoUringPoolOptions::default()
1121 },
1122 ] {
1123 assert_eq!(
1124 validate_io_uring_pool("read", config).unwrap_err().kind(),
1125 io::ErrorKind::InvalidInput
1126 );
1127 }
1128 }
1129
1130 #[cfg(all(
1131 feature = "io-uring",
1132 target_os = "linux",
1133 any(
1134 target_arch = "x86_64",
1135 target_arch = "aarch64",
1136 target_arch = "riscv64",
1137 target_arch = "loongarch64",
1138 target_arch = "powerpc64"
1139 )
1140 ))]
1141 #[test]
1142 fn io_poll_requires_direct_mode() {
1143 let pool = IoUringPoolOptions {
1144 io_poll: true,
1145 ..IoUringPoolOptions::default()
1146 };
1147 let mut config = RuntimeOptions {
1148 io_engine: IoEngineOptions::IoUring(IoUringOptions {
1149 read: pool,
1150 write: IoUringPoolOptions::default(),
1151 reclaim: IoUringPoolOptions {
1152 rings: 1,
1153 max_in_flight: 1,
1154 ..IoUringPoolOptions::default()
1155 },
1156 }),
1157 ..RuntimeOptions::default()
1158 };
1159
1160 assert_eq!(
1161 config.resolve().unwrap_err().kind(),
1162 io::ErrorKind::InvalidInput
1163 );
1164 }
1165}