greplm_core/
io_backend.rs

1//! Ingest read path.
2//!
3//! Reading thousands of small source files is dominated by per-file syscall
4//! overhead, not bandwidth, so the goal is high effective queue depth (many
5//! reads in flight) rather than exotic zero-copy. The portable backend achieves
6//! that with a rayon worker pool over buffered reads; the page cache keeps
7//! re-reads (and the watch loop) cheap.
8//!
9//! This is a trait rather than a bare function because a batched-submission
10//! backend (io_uring on Linux) is a plausible future addition. It is not worth
11//! building yet: with a warm page cache, reads are ~5.6% of per-file indexing
12//! CPU against ~92% for tree-sitter parsing, and reads already overlap parsing
13//! across the worker pool. See docs/ROADMAP.md.
14
15use std::path::Path;
16
17use crate::config::Backend;
18use crate::error::{Error, Result};
19
20/// Abstraction over how file bytes are pulled in during indexing.
21pub trait IoBackend: Send + Sync {
22    /// Read the full contents of a file.
23    fn read(&self, path: &Path) -> Result<Vec<u8>>;
24
25    /// Human-readable backend name (for `status`).
26    fn name(&self) -> &'static str;
27}
28
29/// Portable backend: a plain buffered read. Concurrency is supplied by the
30/// caller running `read` from many rayon worker threads at once.
31#[derive(Debug, Default, Clone, Copy)]
32pub struct RayonBackend;
33
34impl IoBackend for RayonBackend {
35    fn read(&self, path: &Path) -> Result<Vec<u8>> {
36        std::fs::read(path).map_err(|e| Error::io(path, e))
37    }
38
39    fn name(&self) -> &'static str {
40        "rayon"
41    }
42}
43
44/// Resolve the configured [`Backend`] to an implementation.
45///
46/// `config.backend` used to be parsed and then ignored — selection came from
47/// build features alone — so setting it changed nothing. It is honored here, and
48/// [`IoBackend::name`] (what `greplm status` prints) always names the backend
49/// actually in use.
50pub fn select(backend: Backend) -> Box<dyn IoBackend> {
51    if backend == Backend::IoUringRemoved {
52        tracing::warn!(
53            "config.toml sets backend = \"io-uring\", which was removed in 0.5.0: it selected a \
54             stub that did buffered reads anyway. Using the portable backend; set \
55             backend = \"auto\" to silence this."
56        );
57    }
58    match backend {
59        Backend::Auto | Backend::Rayon | Backend::IoUringRemoved => Box::new(RayonBackend),
60    }
61}
62
63/// Select a backend using the default configuration.
64pub fn default_backend() -> Box<dyn IoBackend> {
65    select(Backend::default())
66}
67
68#[cfg(test)]
69mod tests {
70    use super::*;
71
72    /// `backend` in config.toml was parsed and then ignored: selection came from
73    /// build features alone, so setting it changed nothing. This pins that it is
74    /// honored, and that the reported name is the backend actually in use — which
75    /// is what `greplm status` prints.
76    #[test]
77    fn config_backend_is_honored_and_reported() {
78        assert_eq!(select(Backend::Rayon).name(), "rayon");
79        assert_eq!(select(Backend::Auto).name(), "rayon");
80    }
81
82    /// A `config.toml` written before 0.5.0 may still say `backend = "io-uring"`.
83    /// That must keep working — the value never selected anything real — and must
84    /// report the backend actually in use, not the one requested.
85    #[test]
86    fn pre_0_5_io_uring_config_still_loads() {
87        let cfg: crate::config::Config =
88            toml::from_str("backend = \"io-uring\"").expect("legacy value must still parse");
89        assert_eq!(cfg.backend, Backend::IoUringRemoved);
90        assert_eq!(select(cfg.backend).name(), "rayon");
91    }
92
93    #[test]
94    fn backend_reads_file_bytes_verbatim() {
95        let dir = std::env::temp_dir().join(format!("greplm-io-{}", std::process::id()));
96        std::fs::create_dir_all(&dir).unwrap();
97        let f = dir.join("sample.txt");
98        // Includes invalid UTF-8: the ingest path deals in bytes, not text.
99        let body: &[u8] = b"line one\nline two\n\xff\xfe binary-ish\n";
100        std::fs::write(&f, body).unwrap();
101        for b in [Backend::Auto, Backend::Rayon] {
102            assert_eq!(select(b).read(&f).unwrap(), body, "backend {b:?}");
103        }
104        assert!(select(Backend::Rayon).read(&dir.join("missing")).is_err());
105        let _ = std::fs::remove_dir_all(&dir);
106    }
107}
greplm_core/io_backend.rs

greplm_core/
io_backend.rs
