use std::time::Duration;
pub(crate) const DEFAULT_L1_SHARDS: usize = 32;
pub(crate) const MAX_APPEND_SHARDS: u32 = 256;
pub(crate) const MAX_WRITE_FLUSH_THRESHOLD_BYTES: usize = 4 * 1024 * 1024;
pub(crate) const MAX_READ_IO_WAIT_TIMEOUT: Duration = Duration::from_secs(5);
const DEFAULT_L1_CAPACITY_BYTES: usize = 256 * 1024 * 1024;
const DEFAULT_APPEND_SHARDS: u32 = 4;
const DEFAULT_POSIX_IO_WORKERS: usize = 4;
const DEFAULT_IO_URING_MAX_IN_FLIGHT: usize = 64;
const DEFAULT_RECLAIM_IO_CONCURRENCY: usize = 1;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct PosixIoConfig {
read_workers: usize,
write_workers: usize,
reclaim_workers: usize,
}
impl PosixIoConfig {
pub const fn new(read_workers: usize, write_workers: usize, reclaim_workers: usize) -> Self {
Self {
read_workers,
write_workers,
reclaim_workers,
}
}
pub const fn read_workers(self) -> usize {
self.read_workers
}
pub const fn write_workers(self) -> usize {
self.write_workers
}
pub const fn reclaim_workers(self) -> usize {
self.reclaim_workers
}
}
impl Default for PosixIoConfig {
fn default() -> Self {
Self::new(
DEFAULT_POSIX_IO_WORKERS,
DEFAULT_POSIX_IO_WORKERS,
DEFAULT_RECLAIM_IO_CONCURRENCY,
)
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct IoUringPoolConfig {
rings: usize,
max_in_flight: usize,
sq_poll: Option<IoUringSqPollConfig>,
io_poll: bool,
}
impl IoUringPoolConfig {
pub const fn new(rings: usize, max_in_flight: usize) -> Self {
Self {
rings,
max_in_flight,
sq_poll: None,
io_poll: false,
}
}
pub const fn with_sq_poll(mut self, sq_poll: IoUringSqPollConfig) -> Self {
self.sq_poll = Some(sq_poll);
self
}
pub const fn with_io_poll(mut self, enabled: bool) -> Self {
self.io_poll = enabled;
self
}
pub const fn rings(self) -> usize {
self.rings
}
pub const fn max_in_flight(self) -> usize {
self.max_in_flight
}
pub const fn sq_poll(self) -> Option<IoUringSqPollConfig> {
self.sq_poll
}
pub const fn io_poll(self) -> bool {
self.io_poll
}
}
impl Default for IoUringPoolConfig {
fn default() -> Self {
Self::new(1, DEFAULT_IO_URING_MAX_IN_FLIGHT)
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct IoUringSqPollConfig {
idle_millis: u32,
cpu: Option<u32>,
}
impl IoUringSqPollConfig {
pub const fn new(idle_millis: u32) -> Self {
Self {
idle_millis,
cpu: None,
}
}
pub const fn with_cpu(mut self, cpu: u32) -> Self {
self.cpu = Some(cpu);
self
}
pub const fn idle_millis(self) -> u32 {
self.idle_millis
}
pub const fn cpu(self) -> Option<u32> {
self.cpu
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct IoUringConfig {
read: IoUringPoolConfig,
write: IoUringPoolConfig,
reclaim: IoUringPoolConfig,
}
impl IoUringConfig {
pub const fn new(
read: IoUringPoolConfig,
write: IoUringPoolConfig,
reclaim: IoUringPoolConfig,
) -> Self {
Self {
read,
write,
reclaim,
}
}
pub const fn read(self) -> IoUringPoolConfig {
self.read
}
pub const fn write(self) -> IoUringPoolConfig {
self.write
}
pub const fn reclaim(self) -> IoUringPoolConfig {
self.reclaim
}
}
impl Default for IoUringConfig {
fn default() -> Self {
Self::new(
IoUringPoolConfig::new(1, DEFAULT_IO_URING_MAX_IN_FLIGHT),
IoUringPoolConfig::new(1, DEFAULT_IO_URING_MAX_IN_FLIGHT),
IoUringPoolConfig::new(1, DEFAULT_RECLAIM_IO_CONCURRENCY),
)
}
}
#[non_exhaustive]
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum IoEngine {
Posix(PosixIoConfig),
IoUring(IoUringConfig),
}
impl Default for IoEngine {
fn default() -> Self {
Self::Posix(PosixIoConfig::default())
}
}
impl IoEngine {
pub(crate) const fn is_available(self) -> bool {
match self {
Self::Posix(_) => true,
Self::IoUring(_) => cfg!(all(
feature = "io-uring",
target_os = "linux",
any(
target_arch = "x86_64",
target_arch = "aarch64",
target_arch = "riscv64",
target_arch = "loongarch64",
target_arch = "powerpc64"
)
)),
}
}
pub(crate) const fn read_topology(self) -> IoPoolTopology {
match self {
Self::Posix(config) => IoPoolTopology::posix(config.read_workers),
Self::IoUring(config) => IoPoolTopology::io_uring(config.read),
}
}
pub(crate) const fn write_topology(self) -> IoPoolTopology {
match self {
Self::Posix(config) => IoPoolTopology::posix(config.write_workers),
Self::IoUring(config) => IoPoolTopology::io_uring(config.write),
}
}
pub(crate) const fn reclaim_topology(self) -> IoPoolTopology {
match self {
Self::Posix(config) => IoPoolTopology::posix(config.reclaim_workers),
Self::IoUring(config) => IoPoolTopology::io_uring(config.reclaim),
}
}
pub(crate) const fn is_posix(self) -> bool {
matches!(self, Self::Posix(_))
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) struct IoPoolTopology {
pub(crate) engine_count: usize,
pub(crate) max_in_flight: usize,
pub(crate) worker_threads: usize,
pub(crate) io_uring: Option<IoUringPoolConfig>,
}
impl IoPoolTopology {
const fn posix(workers: usize) -> Self {
Self {
engine_count: 1,
max_in_flight: workers,
worker_threads: workers,
io_uring: None,
}
}
const fn io_uring(config: IoUringPoolConfig) -> Self {
Self {
engine_count: config.rings,
max_in_flight: config.max_in_flight,
worker_threads: config.rings,
io_uring: Some(config),
}
}
pub(crate) const fn depth_for_engine(self, engine: usize) -> usize {
let base = self.max_in_flight / self.engine_count;
let remainder = self.max_in_flight % self.engine_count;
base + if engine < remainder { 1 } else { 0 }
}
}
#[non_exhaustive]
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub enum IoMode {
#[default]
Buffered,
Direct,
}
impl IoMode {
pub(crate) const fn is_available(self) -> bool {
match self {
Self::Buffered => true,
Self::Direct => cfg!(target_os = "linux"),
}
}
}
#[non_exhaustive]
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub enum L1EvictionPolicy {
#[default]
Clock,
S3Fifo,
}
#[derive(Clone, Debug)]
pub struct RuntimeConfig {
pub(crate) io_engine: IoEngine,
pub(crate) io_mode: IoMode,
pub(crate) read_io_wait_capacity: Option<usize>,
pub(crate) read_io_wait_timeout: Duration,
pub(crate) append_shards: u32,
pub(crate) l1_capacity_bytes: usize,
pub(crate) l1_eviction_policy: L1EvictionPolicy,
pub(crate) managed_memory_limit_bytes: usize,
pub(crate) l1_shards: usize,
pub(crate) write_flush_threshold_bytes: usize,
pub(crate) statistics: bool,
}
impl Default for RuntimeConfig {
fn default() -> Self {
Self {
io_engine: IoEngine::default(),
io_mode: IoMode::Buffered,
read_io_wait_capacity: None,
read_io_wait_timeout: Duration::ZERO,
append_shards: DEFAULT_APPEND_SHARDS,
l1_capacity_bytes: DEFAULT_L1_CAPACITY_BYTES,
l1_eviction_policy: L1EvictionPolicy::Clock,
managed_memory_limit_bytes: 1024 * 1024 * 1024,
l1_shards: DEFAULT_L1_SHARDS,
write_flush_threshold_bytes: MAX_WRITE_FLUSH_THRESHOLD_BYTES,
statistics: false,
}
}
}
impl RuntimeConfig {
pub fn with_io_engine(mut self, engine: IoEngine) -> Self {
self.io_engine = engine;
self
}
pub fn with_io_mode(mut self, mode: IoMode) -> Self {
self.io_mode = mode;
self
}
pub fn with_read_io_wait_capacity(mut self, capacity: usize) -> Self {
self.read_io_wait_capacity = Some(capacity);
self
}
pub fn with_read_io_wait_timeout(mut self, timeout: Duration) -> Self {
self.read_io_wait_timeout = timeout;
self
}
pub fn with_append_shards(mut self, shards: u32) -> Self {
self.append_shards = shards;
self
}
pub fn with_l1_capacity_bytes(mut self, bytes: usize) -> Self {
self.l1_capacity_bytes = bytes;
self
}
pub fn with_l1_eviction_policy(mut self, policy: L1EvictionPolicy) -> Self {
self.l1_eviction_policy = policy;
self
}
pub fn with_managed_memory_limit_bytes(mut self, bytes: usize) -> Self {
self.managed_memory_limit_bytes = bytes;
self
}
pub fn with_l1_shards(mut self, shards: usize) -> Self {
self.l1_shards = shards;
self
}
pub fn with_write_flush_threshold_bytes(mut self, bytes: usize) -> Self {
self.write_flush_threshold_bytes = bytes;
self
}
pub fn with_statistics(mut self, enabled: bool) -> Self {
self.statistics = enabled;
self
}
pub const fn io_engine(&self) -> IoEngine {
self.io_engine
}
pub const fn io_mode(&self) -> IoMode {
self.io_mode
}
pub const fn read_io_max_in_flight(&self) -> usize {
self.io_engine.read_topology().max_in_flight
}
pub const fn read_io_wait_capacity(&self) -> usize {
match self.read_io_wait_capacity {
Some(capacity) => capacity,
None => self.read_io_max_in_flight(),
}
}
pub const fn read_io_wait_timeout(&self) -> Duration {
self.read_io_wait_timeout
}
pub const fn write_io_max_in_flight(&self) -> usize {
self.io_engine.write_topology().max_in_flight
}
pub const fn reclaim_io_max_in_flight(&self) -> usize {
self.io_engine.reclaim_topology().max_in_flight
}
pub const fn append_shards(&self) -> u32 {
self.append_shards
}
pub const fn l1_capacity_bytes(&self) -> usize {
self.l1_capacity_bytes
}
pub const fn l1_eviction_policy(&self) -> L1EvictionPolicy {
self.l1_eviction_policy
}
pub const fn managed_memory_limit_bytes(&self) -> usize {
self.managed_memory_limit_bytes
}
pub const fn l1_shards(&self) -> usize {
self.l1_shards
}
pub const fn write_flush_threshold_bytes(&self) -> usize {
self.write_flush_threshold_bytes
}
pub const fn statistics_enabled(&self) -> bool {
self.statistics
}
pub(crate) const fn read_io_topology(&self) -> IoPoolTopology {
self.io_engine.read_topology()
}
pub(crate) const fn write_io_topology(&self) -> IoPoolTopology {
self.io_engine.write_topology()
}
pub(crate) const fn reclaim_io_topology(&self) -> IoPoolTopology {
self.io_engine.reclaim_topology()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn runtime_defaults_are_stable() {
let config = RuntimeConfig::default();
assert_eq!(config.io_engine(), IoEngine::default());
assert_eq!(config.io_mode(), IoMode::Buffered);
assert_eq!(config.read_io_max_in_flight(), 4);
assert_eq!(config.read_io_wait_capacity(), 4);
assert_eq!(config.read_io_wait_timeout(), Duration::ZERO);
assert_eq!(config.write_io_max_in_flight(), 4);
assert_eq!(config.reclaim_io_max_in_flight(), 1);
assert_eq!(config.append_shards(), DEFAULT_APPEND_SHARDS);
assert_eq!(config.l1_capacity_bytes(), DEFAULT_L1_CAPACITY_BYTES);
assert_eq!(config.l1_eviction_policy(), L1EvictionPolicy::Clock);
assert_eq!(config.managed_memory_limit_bytes(), 1024 * 1024 * 1024);
assert_eq!(config.l1_shards(), DEFAULT_L1_SHARDS);
assert_eq!(
config.write_flush_threshold_bytes(),
MAX_WRITE_FLUSH_THRESHOLD_BYTES
);
assert!(!config.statistics_enabled());
}
#[test]
fn explicit_read_wait_capacity_is_independent_from_execution() {
let config = RuntimeConfig::default()
.with_read_io_wait_capacity(11)
.with_io_engine(IoEngine::Posix(PosixIoConfig::new(7, 4, 1)));
assert_eq!(config.read_io_max_in_flight(), 7);
assert_eq!(config.read_io_wait_capacity(), 11);
}
#[test]
fn io_engine_topology_matches_backend_shape() {
let posix = IoEngine::Posix(PosixIoConfig::new(7, 5, 2));
assert_eq!(
posix.read_topology(),
IoPoolTopology {
engine_count: 1,
max_in_flight: 7,
worker_threads: 7,
io_uring: None,
}
);
let io_uring = IoEngine::IoUring(IoUringConfig::new(
IoUringPoolConfig::new(3, 8),
IoUringPoolConfig::new(2, 5),
IoUringPoolConfig::new(1, 2),
));
let read = io_uring.read_topology();
assert_eq!(read.engine_count, 3);
assert_eq!(read.max_in_flight, 8);
assert_eq!(read.worker_threads, 3);
assert_eq!(read.depth_for_engine(0), 3);
assert_eq!(read.depth_for_engine(1), 3);
assert_eq!(read.depth_for_engine(2), 2);
}
#[test]
fn io_uring_pool_exposes_polling_options() {
let sq_poll = IoUringSqPollConfig::new(2_000).with_cpu(3);
let pool = IoUringPoolConfig::new(2, 96)
.with_sq_poll(sq_poll)
.with_io_poll(true);
assert_eq!(pool.rings(), 2);
assert_eq!(pool.max_in_flight(), 96);
assert_eq!(pool.sq_poll(), Some(sq_poll));
assert!(pool.io_poll());
}
}