Skip to main content

cache2/config/
runtime.rs

1// Copyright 2026 ScopeDB, Inc.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::io;
16use std::time::Duration;
17use std::time::Instant;
18
19use crate::StatsOptions;
20use crate::cache::runtime::metrics::ActivityMetrics;
21use crate::config::CacheConfig;
22use crate::config::StorageLayout;
23use crate::config::invalid_config;
24use crate::error::Error;
25use crate::error::ErrorOperation;
26use crate::error::from_io;
27use crate::io::engine::CACHE_IO_COMPLETION_TIMEOUT;
28use crate::io::engine::IO_QUEUE_ENTRY_RESERVATION_BYTES;
29use crate::io::engine::MAX_IO_REQUESTS_PER_ENGINE;
30use crate::io::engine::io_uring_extra_memory_bytes;
31use crate::io::fill_control::FillController;
32use crate::managed_memory::BUFFER_ALIGNMENT;
33use crate::managed_memory::CACHE_THREAD_STACK_BYTES;
34use crate::memory::MemoryStore;
35use crate::region::recovery::DataGeometry;
36use crate::region::runtime_fixed_memory_bytes;
37use crate::region::staging::AppendStaging;
38use crate::stats::recording::Recorder;
39
40const DEFAULT_L1_SHARDS: usize = 32;
41const MAX_CONFIG_COUNT: usize = 65_536;
42const MAX_APPEND_SHARDS: u32 = 256;
43pub const MAX_WRITE_FLUSH_THRESHOLD_BYTES: usize = 4 * 1024 * 1024;
44const MAX_READ_IO_WAIT_TIMEOUT: Duration = Duration::from_secs(5);
45const DEFAULT_L1_CAPACITY_BYTES: usize = 256 * 1024 * 1024;
46const DEFAULT_APPEND_SHARDS: u32 = 4;
47const DEFAULT_POSIX_IO_WORKERS: usize = 4;
48const DEFAULT_IO_URING_MAX_IN_FLIGHT: usize = 64;
49const DEFAULT_RECLAIM_IO_CONCURRENCY: usize = 1;
50
51/// Worker topology for POSIX positioned I/O.
52///
53/// Every admitted request occupies one worker until its blocking system call
54/// completes, so worker counts are also the per-pool in-flight limits.
55#[non_exhaustive]
56#[derive(Clone, Copy, Debug, Eq, PartialEq)]
57pub struct PosixIoOptions {
58    /// Number of read workers. Defaults to 4.
59    pub read_workers: usize,
60    /// Number of write workers. Defaults to 4.
61    pub write_workers: usize,
62    /// Number of reclaim workers. Defaults to 1.
63    pub reclaim_workers: usize,
64}
65
66impl Default for PosixIoOptions {
67    fn default() -> Self {
68        Self {
69            read_workers: DEFAULT_POSIX_IO_WORKERS,
70            write_workers: DEFAULT_POSIX_IO_WORKERS,
71            reclaim_workers: DEFAULT_RECLAIM_IO_CONCURRENCY,
72        }
73    }
74}
75
76/// Ring count and aggregate execution bound for one pool of the experimental
77/// `io_uring` engine.
78///
79/// Keep [`Self::rings`] at 1 and size [`Self::max_in_flight`] to concurrent work
80/// for this pool. Extra rings split the same depth across driver threads and do
81/// not add execution slots. Leave [`Self::sq_poll`] and [`Self::io_poll`]
82/// unset unless a host profile needs one of them; they are independent advanced
83/// opt-ins with distinct kernel requirements.
84#[non_exhaustive]
85#[derive(Clone, Copy, Debug, Eq, PartialEq)]
86pub struct IoUringPoolOptions {
87    /// Independent rings and driver threads.
88    ///
89    /// Defaults to 1. Raise this only after `max_in_flight` matches caller
90    /// concurrency and a single driver thread is CPU-bound; must not exceed
91    /// `max_in_flight`.
92    pub rings: usize,
93    /// Aggregate maximum in-flight requests across all rings in the pool.
94    ///
95    /// Defaults to 64 (or 1 for reclaim). For the read pool, size this to
96    /// concurrent L2 `get` calls.
97    pub max_in_flight: usize,
98    /// Kernel submission-queue polling configuration for every ring in this pool.
99    ///
100    /// Defaults to `None` (disabled). SQPOLL does not require Direct I/O.
101    /// Optional CPU affinity applies to every ring in the pool.
102    pub sq_poll: Option<IoUringSqPollOptions>,
103    /// Completion polling toggle for every ring in this pool.
104    ///
105    /// Defaults to `false`. Requires Direct I/O on a polling-capable
106    /// filesystem and block device. Consumes CPU while requests are
107    /// outstanding, and cancellation stays advisory until the polled operation
108    /// completes. Independent of [`Self::sq_poll`].
109    pub io_poll: bool,
110}
111
112impl Default for IoUringPoolOptions {
113    fn default() -> Self {
114        Self {
115            rings: 1,
116            max_in_flight: DEFAULT_IO_URING_MAX_IN_FLIGHT,
117            sq_poll: None,
118            io_poll: false,
119        }
120    }
121}
122
123/// Kernel submission-queue polling parameters for the experimental `io_uring`
124/// engine.
125///
126/// For a multi-ring pool, every ring's polling thread shares the same optional
127/// CPU affinity.
128#[non_exhaustive]
129#[derive(Clone, Copy, Debug, Eq, PartialEq)]
130pub struct IoUringSqPollOptions {
131    /// Idle time in milliseconds before the kernel polling thread can sleep.
132    pub idle_millis: u32,
133    /// Optional CPU affinity.
134    ///
135    /// Defaults to `None`, leaving the polling thread unpinned. For a multi-ring
136    /// pool, every ring's polling thread uses this CPU.
137    pub cpu: Option<u32>,
138}
139
140impl IoUringSqPollOptions {
141    /// Selects a submission polling idle time with no CPU affinity.
142    pub const fn new(idle_millis: u32) -> Self {
143        Self {
144            idle_millis,
145            cpu: None,
146        }
147    }
148}
149
150/// Pool topology for the experimental `io_uring` engine's independent read,
151/// write, and reclaim pools.
152///
153/// See [`IoUringPoolOptions`] for ring sizing and polling guidance.
154#[non_exhaustive]
155#[derive(Clone, Copy, Debug, Eq, PartialEq)]
156pub struct IoUringOptions {
157    /// Read pool (defaults to 1 ring and 64 in-flight requests).
158    pub read: IoUringPoolOptions,
159    /// Write pool (defaults to 1 ring and 64 in-flight requests).
160    pub write: IoUringPoolOptions,
161    /// Reclaim pool (defaults to 1 ring and 1 in-flight request).
162    pub reclaim: IoUringPoolOptions,
163}
164
165impl Default for IoUringOptions {
166    fn default() -> Self {
167        Self {
168            read: IoUringPoolOptions::default(),
169            write: IoUringPoolOptions::default(),
170            reclaim: IoUringPoolOptions {
171                max_in_flight: DEFAULT_RECLAIM_IO_CONCURRENCY,
172                ..IoUringPoolOptions::default()
173            },
174        }
175    }
176}
177
178/// Engine selection and topology for the independent read, write, and reclaim
179/// I/O pools, validated by [`CacheConfig::new`].
180#[non_exhaustive]
181#[derive(Clone, Copy, Debug, Eq, PartialEq)]
182pub enum IoEngineOptions {
183    /// Worker-backed POSIX positioned I/O with explicit thread counts.
184    Posix(PosixIoOptions),
185    /// Experimental Linux `io_uring` engine with independent pool options.
186    ///
187    /// Its API, configuration, and runtime behavior may change between
188    /// releases. Available only with the `io-uring` crate feature on a
189    /// supported Linux target. See [`IoUringPoolOptions`] for tuning guidance.
190    IoUring(IoUringOptions),
191}
192
193impl Default for IoEngineOptions {
194    fn default() -> Self {
195        Self::Posix(PosixIoOptions::default())
196    }
197}
198
199const IO_URING_UNAVAILABLE_FEATURE: &str = "io_uring requires the io-uring crate feature";
200const IO_URING_UNAVAILABLE_PLATFORM: &str = "io_uring is unavailable on this platform";
201const IO_URING_UNAVAILABLE_ARCHITECTURE: &str = "io_uring is unavailable on this architecture";
202
203/// Names the missing io_uring requirement, if any, for this build and target.
204pub const fn io_uring_unavailability() -> Option<&'static str> {
205    if cfg!(not(target_os = "linux")) {
206        Some(IO_URING_UNAVAILABLE_PLATFORM)
207    } else if cfg!(not(any(
208        target_arch = "x86_64",
209        target_arch = "aarch64",
210        target_arch = "riscv64",
211        target_arch = "loongarch64",
212        target_arch = "powerpc64"
213    ))) {
214        Some(IO_URING_UNAVAILABLE_ARCHITECTURE)
215    } else if cfg!(not(feature = "io-uring")) {
216        Some(IO_URING_UNAVAILABLE_FEATURE)
217    } else {
218        None
219    }
220}
221
222// Pool topology owns aggregate bounds; each engine config contains one instance's
223// execution parameters. Both accounting and construction derive from this shape.
224#[derive(Clone, Copy, Debug, Eq, PartialEq)]
225pub enum IoPoolTopology {
226    Posix { workers: usize },
227    IoUring(IoUringPoolOptions),
228}
229
230impl IoPoolTopology {
231    pub const fn read(options: IoEngineOptions) -> Self {
232        match options {
233            IoEngineOptions::Posix(options) => Self::Posix {
234                workers: options.read_workers,
235            },
236            IoEngineOptions::IoUring(options) => Self::IoUring(options.read),
237        }
238    }
239
240    pub const fn write(options: IoEngineOptions) -> Self {
241        match options {
242            IoEngineOptions::Posix(options) => Self::Posix {
243                workers: options.write_workers,
244            },
245            IoEngineOptions::IoUring(options) => Self::IoUring(options.write),
246        }
247    }
248
249    pub const fn reclaim(options: IoEngineOptions) -> Self {
250        match options {
251            IoEngineOptions::Posix(options) => Self::Posix {
252                workers: options.reclaim_workers,
253            },
254            IoEngineOptions::IoUring(options) => Self::IoUring(options.reclaim),
255        }
256    }
257
258    pub const fn engine_count(self) -> usize {
259        match self {
260            Self::Posix { .. } => 1,
261            Self::IoUring(options) => options.rings,
262        }
263    }
264
265    pub const fn max_in_flight(self) -> usize {
266        match self {
267            Self::Posix { workers } => workers,
268            Self::IoUring(options) => options.max_in_flight,
269        }
270    }
271
272    pub const fn worker_threads(self) -> usize {
273        match self {
274            Self::Posix { workers } => workers,
275            Self::IoUring(options) => options.rings,
276        }
277    }
278
279    pub fn extra_memory_bytes(self) -> Option<usize> {
280        match self {
281            Self::Posix { .. } => Some(0),
282            Self::IoUring(options) => {
283                io_uring_extra_memory_bytes(options.max_in_flight, options.rings)
284            }
285        }
286    }
287
288    pub const fn engine_config(self, engine: usize) -> IoEngineConfig {
289        assert!(
290            engine < self.engine_count(),
291            "engine index exceeds pool topology"
292        );
293        match self {
294            Self::Posix { workers } => IoEngineConfig::Posix { workers },
295            Self::IoUring(options) => {
296                let base = options.max_in_flight / options.rings;
297                let remainder = options.max_in_flight % options.rings;
298                IoEngineConfig::IoUring(IoUringEngineConfig {
299                    max_in_flight: base + if engine < remainder { 1 } else { 0 },
300                    sq_poll: options.sq_poll,
301                    io_poll: options.io_poll,
302                })
303            }
304        }
305    }
306}
307
308#[derive(Clone, Copy, Debug, Eq, PartialEq)]
309pub enum IoEngineConfig {
310    Posix { workers: usize },
311    IoUring(IoUringEngineConfig),
312}
313
314#[derive(Clone, Copy, Debug, Eq, PartialEq)]
315pub struct IoUringEngineConfig {
316    pub max_in_flight: usize,
317    pub sq_poll: Option<IoUringSqPollOptions>,
318    pub io_poll: bool,
319}
320
321/// Buffered/direct policy for runtime cache-record I/O.
322///
323/// Control files, recovery images, and any necessarily unaligned remainder use
324/// buffered I/O in every mode.
325#[non_exhaustive]
326#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)]
327pub enum IoMode {
328    /// Always use buffered positioned I/O.
329    #[default]
330    Buffered,
331    /// Require Linux `O_DIRECT` support for aligned record I/O.
332    ///
333    /// Aligned direct-I/O errors are returned instead of falling back. A
334    /// necessarily unaligned remainder still uses the buffered descriptor.
335    Direct,
336}
337
338/// Eviction policy for the process-local L1 tier.
339#[non_exhaustive]
340#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)]
341pub enum L1EvictionPolicy {
342    /// One-bit shard-local CLOCK with bounded victim scans.
343    #[default]
344    Clock,
345    /// Three static FIFO queues: small, main, and a metadata-only ghost queue.
346    S3Fifo,
347}
348
349/// Admission after an L2 index candidate has been selected.
350#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
351pub enum ReadAdmission {
352    /// Return a miss when execution or read-memory capacity is unavailable.
353    #[default]
354    Immediate,
355    /// Wait for execution capacity within a bounded queue and deadline.
356    /// Queue saturation, read-memory pressure, and timeout return overload.
357    Wait {
358        /// Maximum wait, greater than zero and no longer than five seconds.
359        timeout: Duration,
360        /// Maximum queued readers, from one through 65536. `None` follows the
361        /// aggregate read in-flight limit when [`CacheConfig`] is built.
362        max_waiters: Option<usize>,
363    },
364}
365
366/// Optional pre-timeout pressure observation and fill admission control.
367#[non_exhaustive]
368#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
369pub enum FillControlOptions {
370    /// No controller, timer, or additional request accounting.
371    #[default]
372    Disabled,
373    /// Report pressure and hypothetical rejections without changing admission.
374    Observe(FillLimits),
375    /// Reject new fills under pressure. Accepted writes and essential reclaim continue.
376    Adaptive(FillLimits),
377}
378
379/// Logical fill-rate ceilings shared by [`FillControlOptions::Observe`] and
380/// [`FillControlOptions::Adaptive`].
381///
382/// Limits are instance-wide, not per worker, and are not device bandwidth or
383/// IOPS guarantees. Foreground `put` does not consume them; `Adaptive` uses
384/// them to pace non-essential background flush.
385#[non_exhaustive]
386#[derive(Clone, Copy, Debug, Eq, PartialEq)]
387pub struct FillLimits {
388    /// Maximum encoded fill bytes per second, from 640 through 1 TiB/s,
389    /// instance-wide across all shard workers.
390    pub max_bytes_per_second: u64,
391    /// Maximum fill records per second, from 10 through 4,294,967,295,
392    /// instance-wide across all shard workers.
393    pub max_records_per_second: u32,
394}
395
396impl FillLimits {
397    /// Creates fill-rate ceilings validated when [`CacheConfig::new`] runs.
398    pub const fn new(max_bytes_per_second: u64, max_records_per_second: u32) -> Self {
399        Self {
400            max_bytes_per_second,
401            max_records_per_second,
402        }
403    }
404}
405
406/// Process-local runtime configuration checked together by [`CacheConfig::new`].
407///
408/// These options may change across opens. Warm recovery rebinds append shards
409/// from recovered Active and Free Regions when the requested topology fits.
410/// Start with [`Self::default`] and assign the fields to customize.
411#[non_exhaustive]
412#[derive(Clone, Debug, Eq, PartialEq)]
413pub struct RuntimeOptions {
414    /// Independent read, write, and reclaim pools (defaults to POSIX with 4,
415    /// 4, and 1 workers).
416    pub io_engine: IoEngineOptions,
417    /// Record I/O mode (defaults to buffered; direct I/O requires supported
418    /// Linux storage).
419    pub io_mode: IoMode,
420    /// Admission policy after an L2 candidate has been selected.
421    pub read_admission: ReadAdmission,
422    /// Deadline for each background reclaim read, including I/O admission.
423    ///
424    /// Defaults to five seconds. Must be positive and representable as an
425    /// absolute deadline. Does not change foreground reads or writes. Longer
426    /// deadlines can delay shutdown and exhaust free Regions while waiting;
427    /// expiration enters `io_recovery_timeout` recovery before cancellation
428    /// can fail the cache instance.
429    pub reclaim_io_timeout: Duration,
430    /// Background timeout recovery budget after the normal
431    /// admission/completion deadline.
432    ///
433    /// Defaults to `None` (recover until completion or close). Use
434    /// `Some(Duration::ZERO)` for immediate cancellation, or a finite duration
435    /// to limit recovery. While recovering, new fills return overload while
436    /// existing reads and deletes remain available; original requests retain
437    /// their slots, buffers, and Regions and are never resubmitted. Close
438    /// interrupts recovery, whereas drain can wait indefinitely with `None`.
439    /// Actual I/O errors and invalid completions still fail the instance.
440    pub io_recovery_timeout: Option<Duration>,
441    /// Optional pre-timeout fill pressure control.
442    ///
443    /// Disabled by default. Enabled modes reserve bounded worker observations;
444    /// `Adaptive` paces background flush and pauses new fills when outstanding
445    /// work is old.
446    pub fill_control: FillControlOptions,
447    /// Hash-routed append paths, from 1 through 256 (default 4).
448    ///
449    /// Each shard needs one Active Region, two Region-sized buffers, and a
450    /// worker; the layout also needs one spare Region.
451    pub append_shards: u32,
452    /// Retained L1 byte budget, including keys, values, and ownership charges.
453    ///
454    /// Defaults to 256 MiB; zero disables L1. Fixed L1 metadata is charged
455    /// separately.
456    pub l1_capacity_bytes: usize,
457    /// Bounded L1 eviction policy (defaults to CLOCK; S3-FIFO adds ghost
458    /// metadata).
459    pub l1_eviction_policy: L1EvictionPolicy,
460    /// Aggregate cache-managed memory limit, defaulting to 1 GiB.
461    ///
462    /// Covers index mappings, L1, buffers, metadata, queues, and cache threads.
463    /// Allocator overhead, Tokio, application memory, and the kernel page
464    /// cache are outside it.
465    pub managed_memory_limit_bytes: usize,
466    /// Independently locked L1 shards, from 1 through 65536 (default 32).
467    ///
468    /// Powers of two give the cheapest routing; more shards require more
469    /// metadata.
470    pub l1_shards: usize,
471    /// Per-append-shard threshold for requesting a flush, in 4 KiB multiples
472    /// through 4 MiB (the default).
473    ///
474    /// Partial buffers also flush on a bounded delay, pressure, and completion
475    /// barriers.
476    pub write_flush_threshold_bytes: usize,
477    /// Activity counters, request outcomes, and latency distributions.
478    ///
479    /// Defaults disable collection; health and resource gauges remain
480    /// available.
481    pub stats: StatsOptions,
482}
483
484impl Default for RuntimeOptions {
485    fn default() -> Self {
486        Self {
487            io_engine: IoEngineOptions::default(),
488            io_mode: IoMode::Buffered,
489            read_admission: ReadAdmission::Immediate,
490            reclaim_io_timeout: Duration::from_secs(5),
491            io_recovery_timeout: None,
492            fill_control: FillControlOptions::Disabled,
493            append_shards: DEFAULT_APPEND_SHARDS,
494            l1_capacity_bytes: DEFAULT_L1_CAPACITY_BYTES,
495            l1_eviction_policy: L1EvictionPolicy::Clock,
496            managed_memory_limit_bytes: 1024 * 1024 * 1024,
497            l1_shards: DEFAULT_L1_SHARDS,
498            write_flush_threshold_bytes: MAX_WRITE_FLUSH_THRESHOLD_BYTES,
499            stats: StatsOptions::default(),
500        }
501    }
502}
503
504pub const fn read_io_wait_capacity(options: &RuntimeOptions) -> usize {
505    match options.read_admission {
506        ReadAdmission::Immediate => 0,
507        ReadAdmission::Wait { max_waiters, .. } => match max_waiters {
508            Some(capacity) => capacity,
509            None => IoPoolTopology::read(options.io_engine).max_in_flight(),
510        },
511    }
512}
513
514pub const fn read_io_wait_timeout(options: &RuntimeOptions) -> Duration {
515    match options.read_admission {
516        ReadAdmission::Immediate => Duration::ZERO,
517        ReadAdmission::Wait { timeout, .. } => timeout,
518    }
519}
520
521// Covers worker/shard controls and handles whose size does not scale with the
522// payload or engine depth.
523const RUNTIME_CONTROL_RESERVATION_BYTES: usize = 4096;
524// Keep the fixed L1 directory useful when the configured L2 has deliberate
525// headroom. Smaller entries may still bypass before the byte budget fills;
526// this avoids sizing metadata for the theoretical 64-byte minimum.
527const MIN_L1_SIZING_ENTRY_BYTES: usize = 4 * 1024;
528
529impl CacheConfig {
530    /// Checks the complete combination and resolves dependent runtime defaults.
531    ///
532    /// # Errors
533    ///
534    /// Returns [`ErrorOperation::BuildConfig`] for incompatible Region/shard
535    /// counts, invalid runtime settings, unavailable build/platform features,
536    /// or insufficient managed memory. Device capabilities are checked at open.
537    ///
538    /// # Examples
539    ///
540    /// ```no_run
541    /// # async fn example() -> Result<(), cache2::Error> {
542    /// use cache2::Cache;
543    /// use cache2::CacheConfig;
544    /// use cache2::IoEngineOptions;
545    /// use cache2::PosixIoOptions;
546    /// use cache2::RuntimeOptions;
547    /// use cache2::StorageOptions;
548    ///
549    /// let mut storage = StorageOptions::new(1024 * 1024 * 1024);
550    /// storage.expected_entries = Some(100_000);
551    /// let storage = storage.build()?;
552    ///
553    /// let mut io = PosixIoOptions::default();
554    /// io.read_workers = 8;
555    ///
556    /// let mut runtime = RuntimeOptions::default();
557    /// runtime.io_engine = IoEngineOptions::Posix(io);
558    /// runtime.stats.activity_counters = true;
559    ///
560    /// let config = CacheConfig::new(storage, runtime)?;
561    /// let disk_peak = config.storage().peak_disk_bytes();
562    /// let memory_floor = config.minimum_memory_bytes();
563    /// let cache = Cache::open("cache.data", config).await?;
564    /// # cache.close_fast().await?;
565    /// # Ok(())
566    /// # }
567    /// ```
568    pub fn new(storage: StorageLayout, mut runtime: RuntimeOptions) -> Result<Self, Error> {
569        let build = || -> io::Result<Self> {
570            let geometry = storage.geometry();
571            let index_slots = storage.index_slots();
572            runtime.resolve()?;
573            let stats_bytes = Recorder::allocation_bytes(runtime.stats)?;
574            let fill_bytes = FillController::allocation_bytes(
575                runtime.fill_control,
576                runtime.append_shards as usize
577                    + IoPoolTopology::reclaim(runtime.io_engine).max_in_flight(),
578            )?;
579            if geometry.region_count <= runtime.append_shards {
580                return Err(invalid_config(
581                    "append shards require valid geometry with one Active Region each plus one spare Region",
582                ));
583            }
584            let l1_entry_capacity = runtime.l1_entry_capacity(geometry, index_slots)?;
585            let l1_metadata_bytes = MemoryStore::allocation_bytes(
586                runtime.l1_capacity_bytes,
587                l1_entry_capacity,
588                runtime.l1_shards,
589                runtime.l1_eviction_policy,
590            )?;
591            let fixed_bytes = runtime_fixed_memory_bytes(index_slots, geometry.region_count)?
592                .checked_add(l1_metadata_bytes)
593                .and_then(|bytes| bytes.checked_add(stats_bytes))
594                .and_then(|bytes| bytes.checked_add(fill_bytes))
595                .ok_or_else(|| invalid_config("fixed memory requirements overflow"))?;
596            let (reserved_memory_bytes, minimum_memory_bytes) =
597                runtime.memory_requirements(geometry, fixed_bytes)?;
598            if minimum_memory_bytes > runtime.managed_memory_limit_bytes {
599                return Err(io::Error::new(
600                    io::ErrorKind::InvalidInput,
601                    format!(
602                        "managed memory limit cannot hold the cache memory requirements: requires {minimum_memory_bytes} bytes, configured {} bytes",
603                        runtime.managed_memory_limit_bytes
604                    ),
605                ));
606            }
607            Ok(Self {
608                storage,
609                runtime,
610                l1_entry_capacity,
611                reserved_memory_bytes,
612                minimum_memory_bytes,
613            })
614        };
615        build().map_err(|error| from_io(ErrorOperation::BuildConfig, error))
616    }
617}
618
619impl RuntimeOptions {
620    fn resolve(&mut self) -> io::Result<()> {
621        if matches!(self.io_engine, IoEngineOptions::IoUring(_))
622            && let Some(reason) = io_uring_unavailability()
623        {
624            return Err(io::Error::new(io::ErrorKind::Unsupported, reason));
625        }
626        if self.io_mode == IoMode::Direct && cfg!(not(target_os = "linux")) {
627            return Err(io::Error::new(
628                io::ErrorKind::Unsupported,
629                "direct I/O is unavailable on this platform",
630            ));
631        }
632        if self.append_shards == 0 || self.append_shards > MAX_APPEND_SHARDS {
633            return Err(invalid_config("append shards must be in 1..=256"));
634        }
635        if self.reclaim_io_timeout.is_zero()
636            || Instant::now()
637                .checked_add(self.reclaim_io_timeout)
638                .is_none()
639        {
640            return Err(invalid_config(
641                "reclaim I/O timeout must be positive and fit an absolute deadline",
642            ));
643        }
644        if let Some(recovery_timeout) = self.io_recovery_timeout
645            && self
646                .reclaim_io_timeout
647                .max(CACHE_IO_COMPLETION_TIMEOUT)
648                .checked_add(recovery_timeout)
649                .and_then(|timeout| Instant::now().checked_add(timeout))
650                .is_none()
651        {
652            return Err(invalid_config(
653                "I/O recovery timeout must fit the combined absolute deadline",
654            ));
655        }
656        let read_topology = IoPoolTopology::read(self.io_engine);
657        let write_topology = IoPoolTopology::write(self.io_engine);
658        let reclaim_topology = IoPoolTopology::reclaim(self.io_engine);
659        match self.io_engine {
660            IoEngineOptions::Posix(_) => {
661                validate_posix_pool("read", read_topology)?;
662                validate_posix_pool("write", write_topology)?;
663                validate_posix_pool("reclaim", reclaim_topology)?;
664            }
665            IoEngineOptions::IoUring(options) => {
666                validate_io_uring_pool("read", options.read)?;
667                validate_io_uring_pool("write", options.write)?;
668                validate_io_uring_pool("reclaim", options.reclaim)?;
669                if self.io_mode != IoMode::Direct
670                    && (options.read.io_poll || options.write.io_poll || options.reclaim.io_poll)
671                {
672                    return Err(invalid_config("io_uring IOPOLL requires direct I/O mode"));
673                }
674            }
675        }
676        if reclaim_topology.max_in_flight() > self.append_shards as usize {
677            return Err(invalid_config(
678                "reclaim I/O concurrency must be no greater than append shards",
679            ));
680        }
681        if let ReadAdmission::Wait {
682            timeout,
683            max_waiters,
684        } = &mut self.read_admission
685        {
686            if timeout.is_zero() || *timeout > MAX_READ_IO_WAIT_TIMEOUT {
687                return Err(invalid_config(
688                    "read wait timeout must be greater than zero and at most five seconds",
689                ));
690            }
691            let capacity = max_waiters.unwrap_or(read_topology.max_in_flight());
692            if !(1..=MAX_CONFIG_COUNT).contains(&capacity) {
693                return Err(invalid_config("maximum read waiters must be in 1..=65536"));
694            }
695            *max_waiters = Some(capacity);
696        }
697        if self.managed_memory_limit_bytes == 0 {
698            return Err(invalid_config("managed memory limit must be non-zero"));
699        }
700        if self.l1_capacity_bytes > self.managed_memory_limit_bytes {
701            return Err(invalid_config(
702                "L1 capacity must not exceed the managed memory limit",
703            ));
704        }
705        if self.l1_shards == 0 || self.l1_shards > MAX_CONFIG_COUNT {
706            return Err(invalid_config("L1 shards must be in 1..=65536"));
707        }
708        if self.write_flush_threshold_bytes == 0
709            || self.write_flush_threshold_bytes > MAX_WRITE_FLUSH_THRESHOLD_BYTES
710            || !self
711                .write_flush_threshold_bytes
712                .is_multiple_of(BUFFER_ALIGNMENT)
713        {
714            return Err(invalid_config(
715                "write flush threshold must be 4 KiB aligned and within 4 KiB..=4 MiB",
716            ));
717        }
718        Ok(())
719    }
720
721    fn l1_entry_capacity(&self, geometry: DataGeometry, index_slots: usize) -> io::Result<usize> {
722        if self.l1_capacity_bytes == 0 {
723            return Ok(0);
724        }
725        let l2_capacity = u128::from(geometry.region_size)
726            .checked_mul(u128::from(geometry.region_count))
727            .filter(|capacity| *capacity != 0)
728            .ok_or_else(|| invalid_config("L2 capacity does not fit the L1 sizing"))?;
729        let expected_entries = index_slots.div_ceil(2).max(1);
730        let proportional = (expected_entries as u128)
731            .checked_mul(self.l1_capacity_bytes as u128)
732            .and_then(|entries| entries.checked_add(l2_capacity - 1))
733            .map(|entries| entries / l2_capacity)
734            .and_then(|entries| usize::try_from(entries).ok())
735            .ok_or_else(|| invalid_config("L1 entry capacity does not fit usize"))?;
736        let four_kib_density = self.l1_capacity_bytes.div_ceil(MIN_L1_SIZING_ENTRY_BYTES);
737        let maximum = MemoryStore::maximum_entry_capacity(self.l1_capacity_bytes, self.l1_shards);
738        let minimum = self.l1_shards.min(maximum);
739        Ok(proportional
740            .max(four_kib_density)
741            .min(expected_entries)
742            .max(minimum)
743            .min(maximum))
744    }
745
746    fn memory_requirements(
747        &self,
748        geometry: DataGeometry,
749        fixed_bytes: usize,
750    ) -> io::Result<(usize, usize)> {
751        let shard_count = self.append_shards as usize;
752        let topology_bytes = runtime_topology_memory_bytes(self)
753            .ok_or_else(|| invalid_config("runtime topology memory requirements overflow"))?;
754        let region_bytes = usize::try_from(geometry.region_size)
755            .map_err(|_| invalid_config("Region size does not fit the memory requirements"))?;
756        let write_buffer_reservation = AppendStaging::reservation_bytes(shard_count, region_bytes)
757            .ok_or_else(|| invalid_config("write buffer memory requirements overflow"))?;
758        let reserved_memory = fixed_bytes
759            .checked_add(self.l1_capacity_bytes)
760            .and_then(|bytes| bytes.checked_add(topology_bytes))
761            .ok_or_else(|| invalid_config("reserved memory requirements overflow"))?;
762        let reclaim_buffers = region_bytes
763            .checked_mul(IoPoolTopology::reclaim(self.io_engine).max_in_flight())
764            .ok_or_else(|| invalid_config("reclaim buffer memory requirements overflow"))?;
765        let minimum = reserved_memory
766            .checked_add(write_buffer_reservation)
767            // Every reclaimer permanently owns one Region-sized buffer. Keep
768            // one additional maximum-size bounded read for the foreground.
769            .and_then(|bytes| bytes.checked_add(reclaim_buffers))
770            .and_then(|bytes| bytes.checked_add(region_bytes))
771            .ok_or_else(|| invalid_config("minimum memory requirements overflow"))?;
772        Ok((reserved_memory, minimum))
773    }
774}
775
776fn runtime_topology_memory_bytes(options: &RuntimeOptions) -> Option<usize> {
777    let shard_count = options.append_shards as usize;
778    // Reserve one stack per physical I/O thread, one possible shutdown reaper
779    // per engine, and every append/reclaim worker.
780    let read = IoPoolTopology::read(options.io_engine);
781    let write = IoPoolTopology::write(options.io_engine);
782    let reclaim = IoPoolTopology::reclaim(options.io_engine);
783    let engine_count = read
784        .engine_count()
785        .checked_add(write.engine_count())?
786        .checked_add(reclaim.engine_count())?;
787    let stack_count = read
788        .worker_threads()
789        .checked_add(write.worker_threads())?
790        .checked_add(reclaim.worker_threads())?
791        .checked_add(engine_count)?
792        .checked_add(shard_count)?
793        .checked_add(reclaim.max_in_flight())?;
794    let stacks = stack_count.checked_mul(CACHE_THREAD_STACK_BYTES)?;
795    let read_wait_queue = read_io_wait_capacity(options);
796    let queue = write
797        .max_in_flight()
798        .checked_add(read.max_in_flight())?
799        .checked_add(read_wait_queue)?
800        .checked_add(reclaim.max_in_flight())?
801        .checked_mul(IO_QUEUE_ENTRY_RESERVATION_BYTES)?;
802    let uring = [read, write, reclaim]
803        .into_iter()
804        .try_fold(0_usize, |bytes, pool| {
805            bytes.checked_add(pool.extra_memory_bytes()?)
806        })?;
807    let controls = engine_count
808        .checked_add(shard_count)?
809        .checked_add(options.l1_shards)?
810        .checked_add(reclaim.max_in_flight())?
811        .checked_mul(RUNTIME_CONTROL_RESERVATION_BYTES)?;
812    let metrics = shard_count.checked_mul(size_of::<ActivityMetrics>())?;
813    stacks
814        .checked_add(queue)?
815        .checked_add(uring)?
816        .checked_add(controls)?
817        .checked_add(metrics)
818}
819
820fn validate_posix_pool(name: &str, topology: IoPoolTopology) -> io::Result<()> {
821    if !(1..=MAX_IO_REQUESTS_PER_ENGINE).contains(&topology.max_in_flight()) {
822        return Err(io::Error::new(
823            io::ErrorKind::InvalidInput,
824            format!("POSIX {name} worker count must be in 1..={MAX_IO_REQUESTS_PER_ENGINE}"),
825        ));
826    }
827    Ok(())
828}
829
830fn validate_io_uring_pool(name: &str, options: IoUringPoolOptions) -> io::Result<()> {
831    if !(1..=MAX_CONFIG_COUNT).contains(&options.rings) {
832        return Err(io::Error::new(
833            io::ErrorKind::InvalidInput,
834            format!("io_uring {name} ring count must be in 1..={MAX_CONFIG_COUNT}"),
835        ));
836    }
837    if !(1..=MAX_CONFIG_COUNT).contains(&options.max_in_flight) {
838        return Err(io::Error::new(
839            io::ErrorKind::InvalidInput,
840            format!("io_uring {name} maximum in-flight requests must be in 1..={MAX_CONFIG_COUNT}"),
841        ));
842    }
843    if options.rings > options.max_in_flight {
844        return Err(io::Error::new(
845            io::ErrorKind::InvalidInput,
846            format!("io_uring {name} ring count must not exceed its in-flight limit"),
847        ));
848    }
849    if options.max_in_flight.div_ceil(options.rings) > MAX_IO_REQUESTS_PER_ENGINE {
850        return Err(io::Error::new(
851            io::ErrorKind::InvalidInput,
852            format!("io_uring {name} per-ring depth must not exceed {MAX_IO_REQUESTS_PER_ENGINE}"),
853        ));
854    }
855    Ok(())
856}
857
858#[cfg(test)]
859mod tests {
860    use super::*;
861    use crate::ErrorKind;
862    use crate::StorageOptions;
863
864    #[test]
865    fn fill_control_validates_ceilings_and_accounts_observation_memory() {
866        let storage = StorageOptions::new(1024 * 1024 * 1024).build().unwrap();
867        let base = CacheConfig::new(storage.clone(), RuntimeOptions::default()).unwrap();
868        assert_eq!(base.runtime().fill_control, FillControlOptions::Disabled);
869        for (bytes, operations) in [(639, 100), ((1 << 40) + 1, 100), (640, 9)] {
870            let options = RuntimeOptions {
871                fill_control: FillControlOptions::Adaptive(FillLimits::new(bytes, operations)),
872                ..RuntimeOptions::default()
873            };
874            assert_eq!(
875                CacheConfig::new(storage.clone(), options)
876                    .unwrap_err()
877                    .kind(),
878                ErrorKind::InvalidInput
879            );
880        }
881        CacheConfig::new(
882            storage.clone(),
883            RuntimeOptions {
884                fill_control: FillControlOptions::Adaptive(FillLimits::new(640, u32::MAX)),
885                ..RuntimeOptions::default()
886            },
887        )
888        .unwrap();
889        let mut minimum = None;
890        for mode in [FillControlOptions::Observe, FillControlOptions::Adaptive] {
891            let config = CacheConfig::new(
892                storage.clone(),
893                RuntimeOptions {
894                    fill_control: mode(FillLimits::new(64_000, 100)),
895                    ..RuntimeOptions::default()
896                },
897            )
898            .unwrap();
899            let extra = config.minimum_memory_bytes() - base.minimum_memory_bytes();
900            assert!(extra > 0);
901            assert!(extra < CACHE_THREAD_STACK_BYTES);
902            if let Some(previous) = minimum {
903                assert_eq!(extra, previous);
904            }
905            minimum = Some(extra);
906        }
907    }
908
909    #[test]
910    fn recovery_timeout_allows_zero_and_rejects_overflow() {
911        let storage = StorageOptions::new(1024 * 1024 * 1024).build().unwrap();
912        assert_eq!(RuntimeOptions::default().io_recovery_timeout, None);
913        for timeout in [Duration::ZERO, Duration::from_secs(30)] {
914            let options = RuntimeOptions {
915                io_recovery_timeout: Some(timeout),
916                ..RuntimeOptions::default()
917            };
918            CacheConfig::new(storage.clone(), options).unwrap();
919        }
920        let options = RuntimeOptions {
921            io_recovery_timeout: Some(Duration::MAX),
922            ..RuntimeOptions::default()
923        };
924        assert_eq!(
925            CacheConfig::new(storage, options).unwrap_err().kind(),
926            ErrorKind::InvalidInput
927        );
928    }
929
930    #[test]
931    fn reclaim_timeout_is_validated_at_config_construction() {
932        let storage = StorageOptions::new(1024 * 1024 * 1024).build().unwrap();
933        assert_eq!(
934            RuntimeOptions::default().reclaim_io_timeout,
935            Duration::from_secs(5)
936        );
937        for timeout in [Duration::ZERO, Duration::MAX] {
938            let error = CacheConfig::new(
939                storage.clone(),
940                RuntimeOptions {
941                    reclaim_io_timeout: timeout,
942                    ..RuntimeOptions::default()
943                },
944            )
945            .unwrap_err();
946            assert_eq!(error.kind(), ErrorKind::InvalidInput);
947        }
948        CacheConfig::new(
949            storage,
950            RuntimeOptions {
951                reclaim_io_timeout: Duration::from_secs(30),
952                ..RuntimeOptions::default()
953            },
954        )
955        .unwrap();
956    }
957
958    #[test]
959    fn optional_read_wait_queue_is_memory_accounted() {
960        let base = RuntimeOptions {
961            io_engine: IoEngineOptions::Posix(PosixIoOptions {
962                read_workers: 7,
963                write_workers: 4,
964                reclaim_workers: 1,
965            }),
966            ..RuntimeOptions::default()
967        };
968        let no_wait = runtime_topology_memory_bytes(&base).unwrap();
969        let with_wait = runtime_topology_memory_bytes(&RuntimeOptions {
970            read_admission: ReadAdmission::Wait {
971                timeout: Duration::from_millis(1),
972                max_waiters: Some(11),
973            },
974            ..base
975        })
976        .unwrap();
977
978        assert_eq!(with_wait - no_wait, 11 * IO_QUEUE_ENTRY_RESERVATION_BYTES);
979    }
980
981    #[test]
982    fn each_additional_reclaimer_is_fully_memory_accounted() {
983        let geometry = DataGeometry {
984            data_file_len: DataGeometry::expected_file_len(512 * 1024, 10).unwrap(),
985            region_size: 512 * 1024,
986            region_count: 10,
987        };
988        let base = RuntimeOptions {
989            append_shards: 4,
990            l1_capacity_bytes: 0,
991            ..RuntimeOptions::default()
992        };
993        let (_, base_minimum) = base.memory_requirements(geometry, 0).unwrap();
994        let (_, parallel_minimum) = RuntimeOptions {
995            io_engine: IoEngineOptions::Posix(PosixIoOptions {
996                read_workers: 4,
997                write_workers: 4,
998                reclaim_workers: 2,
999            }),
1000            ..base
1001        }
1002        .memory_requirements(geometry, 0)
1003        .unwrap();
1004
1005        assert_eq!(
1006            parallel_minimum - base_minimum,
1007            geometry.region_size as usize
1008                + 2 * CACHE_THREAD_STACK_BYTES
1009                + IO_QUEUE_ENTRY_RESERVATION_BYTES
1010                + RUNTIME_CONTROL_RESERVATION_BYTES
1011        );
1012    }
1013
1014    #[test]
1015    fn io_engine_topology_matches_backend_shape() {
1016        let posix = IoEngineOptions::Posix(PosixIoOptions {
1017            read_workers: 7,
1018            write_workers: 5,
1019            reclaim_workers: 2,
1020        });
1021        assert_eq!(
1022            IoPoolTopology::read(posix),
1023            IoPoolTopology::Posix { workers: 7 }
1024        );
1025
1026        let io_uring = IoEngineOptions::IoUring(IoUringOptions {
1027            read: IoUringPoolOptions {
1028                rings: 3,
1029                max_in_flight: 8,
1030                ..IoUringPoolOptions::default()
1031            },
1032            write: IoUringPoolOptions {
1033                rings: 2,
1034                max_in_flight: 5,
1035                ..IoUringPoolOptions::default()
1036            },
1037            reclaim: IoUringPoolOptions {
1038                rings: 1,
1039                max_in_flight: 2,
1040                ..IoUringPoolOptions::default()
1041            },
1042        });
1043        let read = IoPoolTopology::read(io_uring);
1044        assert_eq!(read.engine_count(), 3);
1045        assert_eq!(read.max_in_flight(), 8);
1046        assert_eq!(read.worker_threads(), 3);
1047        let depths: Vec<_> = (0..read.engine_count())
1048            .map(|engine| match read.engine_config(engine) {
1049                IoEngineConfig::IoUring(config) => config.max_in_flight,
1050                IoEngineConfig::Posix { .. } => panic!("expected an io_uring engine"),
1051            })
1052            .collect();
1053        assert_eq!(depths, [3, 3, 2]);
1054        assert_eq!(depths.iter().sum::<usize>(), read.max_in_flight());
1055    }
1056
1057    #[test]
1058    fn io_uring_depth_reserves_more_than_common_request_bookkeeping() {
1059        let pool = IoUringPoolOptions {
1060            rings: 1,
1061            max_in_flight: 1,
1062            ..IoUringPoolOptions::default()
1063        };
1064        let shallow = RuntimeOptions {
1065            io_engine: IoEngineOptions::IoUring(IoUringOptions {
1066                read: pool,
1067                write: pool,
1068                reclaim: pool,
1069            }),
1070            ..RuntimeOptions::default()
1071        };
1072        let deep = RuntimeOptions {
1073            io_engine: IoEngineOptions::IoUring(IoUringOptions {
1074                read: IoUringPoolOptions {
1075                    rings: 1,
1076                    max_in_flight: MAX_IO_REQUESTS_PER_ENGINE,
1077                    ..IoUringPoolOptions::default()
1078                },
1079                write: pool,
1080                reclaim: pool,
1081            }),
1082            ..shallow.clone()
1083        };
1084        let growth = runtime_topology_memory_bytes(&deep).unwrap()
1085            - runtime_topology_memory_bytes(&shallow).unwrap();
1086        assert!(growth > (MAX_IO_REQUESTS_PER_ENGINE - 1) * IO_QUEUE_ENTRY_RESERVATION_BYTES);
1087    }
1088
1089    #[test]
1090    fn io_uring_pool_validation_bounds_rings_and_depth() {
1091        validate_io_uring_pool(
1092            "read",
1093            IoUringPoolOptions {
1094                rings: 3,
1095                max_in_flight: 8,
1096                ..IoUringPoolOptions::default()
1097            },
1098        )
1099        .unwrap();
1100
1101        for config in [
1102            IoUringPoolOptions {
1103                rings: 0,
1104                max_in_flight: 8,
1105                ..IoUringPoolOptions::default()
1106            },
1107            IoUringPoolOptions {
1108                rings: 1,
1109                max_in_flight: 0,
1110                ..IoUringPoolOptions::default()
1111            },
1112            IoUringPoolOptions {
1113                rings: 3,
1114                max_in_flight: 2,
1115                ..IoUringPoolOptions::default()
1116            },
1117            IoUringPoolOptions {
1118                rings: 1,
1119                max_in_flight: MAX_IO_REQUESTS_PER_ENGINE + 1,
1120                ..IoUringPoolOptions::default()
1121            },
1122        ] {
1123            assert_eq!(
1124                validate_io_uring_pool("read", config).unwrap_err().kind(),
1125                io::ErrorKind::InvalidInput
1126            );
1127        }
1128    }
1129
1130    #[cfg(all(
1131        feature = "io-uring",
1132        target_os = "linux",
1133        any(
1134            target_arch = "x86_64",
1135            target_arch = "aarch64",
1136            target_arch = "riscv64",
1137            target_arch = "loongarch64",
1138            target_arch = "powerpc64"
1139        )
1140    ))]
1141    #[test]
1142    fn io_poll_requires_direct_mode() {
1143        let pool = IoUringPoolOptions {
1144            io_poll: true,
1145            ..IoUringPoolOptions::default()
1146        };
1147        let mut config = RuntimeOptions {
1148            io_engine: IoEngineOptions::IoUring(IoUringOptions {
1149                read: pool,
1150                write: IoUringPoolOptions::default(),
1151                reclaim: IoUringPoolOptions {
1152                    rings: 1,
1153                    max_in_flight: 1,
1154                    ..IoUringPoolOptions::default()
1155                },
1156            }),
1157            ..RuntimeOptions::default()
1158        };
1159
1160        assert_eq!(
1161            config.resolve().unwrap_err().kind(),
1162            io::ErrorKind::InvalidInput
1163        );
1164    }
1165}