Skip to main content

rypipe_core/
auto.rs

1//! Auto engine selection: heuristic for picking the best execution mode.
2
3/// The execution mode selected by the auto heuristic.
4#[derive(Debug, Clone, Copy, PartialEq, Eq)]
5pub enum EngineMode {
6    /// Single-threaded, full table in RAM.
7    Columnar,
8    /// Multi-core parallel, full table in RAM.
9    Parallel,
10    /// Single-threaded streaming, bounded memory.
11    Stream,
12    /// Multi-core parallel streaming, bounded memory.
13    ParallelStreaming,
14}
15
16impl EngineMode {
17    /// Return the string representation used by Python adapters.
18    pub fn as_str(&self) -> &'static str {
19        match self {
20            EngineMode::Columnar => "columnar",
21            EngineMode::Parallel => "parallel",
22            EngineMode::Stream => "stream",
23            EngineMode::ParallelStreaming => "parallel_streaming",
24        }
25    }
26}
27
28/// Configuration for the auto engine selection heuristic.
29pub struct AutoConfig {
30    /// Size of the input file in bytes.
31    pub file_size: u64,
32    /// Memory budget (e.g. 64 MiB). `None` = no budget (full table).
33    pub memory: Option<u64>,
34    /// Number of threads for parallel processing. `None` = 1 (single-threaded).
35    pub threads: Option<usize>,
36    /// Explicit schema (column names). `Some` = schema provided, `None` = discover.
37    pub schema: Option<Vec<String>>,
38    /// Whether the adapter has parallel support.
39    pub has_parallel: bool,
40    /// Whether the adapter has columnar support.
41    pub has_columnar: bool,
42}
43
44/// Resolve the best engine mode based on configuration.
45///
46/// The heuristic considers:
47/// - `memory`: if provided, user wants streaming
48/// - `threads`: if > 1, prefer parallel modes
49/// - `schema`: if provided, streaming is 11% faster (no discovery overhead)
50/// - `file_size`: small files use columnar, large files use parallel
51pub fn resolve_engine(config: &AutoConfig) -> EngineMode {
52    let threads = config.threads.unwrap_or(1);
53
54    // 1. User explicitly requested streaming via memory=
55    if config.memory.is_some() {
56        if threads > 1 {
57            return EngineMode::ParallelStreaming;
58        }
59        return EngineMode::Stream;
60    }
61
62    // 2. User explicitly requested parallel via threads=
63    if threads > 1 {
64        // For parallel streaming, we need a memory budget.
65        // Without memory=, prefer parallel (full table) if it fits.
66        // If we can't determine available memory, use parallel streaming
67        // as the safer choice for large files.
68        if config.file_size >= 100 * 1024 * 1024 {
69            // Large file: prefer parallel streaming for bounded memory
70            return EngineMode::ParallelStreaming;
71        }
72        return EngineMode::Parallel;
73    }
74
75    // 3. Schema provided: prefer streaming for 11% boost on large files
76    if config.schema.is_some() && config.file_size >= 100 * 1024 * 1024 {
77        return EngineMode::Stream;
78    }
79
80    // 4. Default: file size based
81    if config.file_size < 8 * 1024 * 1024 {
82        // Small file: prefer columnar (no chunk overhead)
83        if config.has_columnar {
84            return EngineMode::Columnar;
85        }
86        return EngineMode::Stream;
87    }
88
89    // Large file: prefer parallel if available
90    if config.has_parallel {
91        return EngineMode::Parallel;
92    }
93
94    // Fallback
95    EngineMode::Stream
96}
97
98#[cfg(test)]
99mod tests {
100    use super::*;
101
102    #[test]
103    fn test_memory_only() {
104        let config = AutoConfig {
105            file_size: 1_000_000_000,
106            memory: Some(64 * 1024 * 1024),
107            threads: None,
108            schema: None,
109            has_parallel: true,
110            has_columnar: true,
111        };
112        assert_eq!(resolve_engine(&config), EngineMode::Stream);
113    }
114
115    #[test]
116    fn test_memory_and_threads() {
117        let config = AutoConfig {
118            file_size: 1_000_000_000,
119            memory: Some(64 * 1024 * 1024),
120            threads: Some(16),
121            schema: None,
122            has_parallel: true,
123            has_columnar: true,
124        };
125        assert_eq!(resolve_engine(&config), EngineMode::ParallelStreaming);
126    }
127
128    #[test]
129    fn test_threads_only_large_file() {
130        let config = AutoConfig {
131            file_size: 1_000_000_000,
132            memory: None,
133            threads: Some(16),
134            schema: None,
135            has_parallel: true,
136            has_columnar: true,
137        };
138        assert_eq!(resolve_engine(&config), EngineMode::ParallelStreaming);
139    }
140
141    #[test]
142    fn test_threads_only_small_file() {
143        let config = AutoConfig {
144            file_size: 50 * 1024 * 1024,
145            memory: None,
146            threads: Some(16),
147            schema: None,
148            has_parallel: true,
149            has_columnar: true,
150        };
151        assert_eq!(resolve_engine(&config), EngineMode::Parallel);
152    }
153
154    #[test]
155    fn test_schema_large_file() {
156        let config = AutoConfig {
157            file_size: 500 * 1024 * 1024,
158            memory: None,
159            threads: None,
160            schema: Some(vec!["a".into(), "b".into()]),
161            has_parallel: true,
162            has_columnar: true,
163        };
164        assert_eq!(resolve_engine(&config), EngineMode::Stream);
165    }
166
167    #[test]
168    fn test_small_file_columnar() {
169        let config = AutoConfig {
170            file_size: 5 * 1024 * 1024,
171            memory: None,
172            threads: None,
173            schema: None,
174            has_parallel: true,
175            has_columnar: true,
176        };
177        assert_eq!(resolve_engine(&config), EngineMode::Columnar);
178    }
179
180    #[test]
181    fn test_small_file_no_columnar() {
182        let config = AutoConfig {
183            file_size: 5 * 1024 * 1024,
184            memory: None,
185            threads: None,
186            schema: None,
187            has_parallel: true,
188            has_columnar: false,
189        };
190        assert_eq!(resolve_engine(&config), EngineMode::Stream);
191    }
192
193    #[test]
194    fn test_medium_file_parallel() {
195        let config = AutoConfig {
196            file_size: 100 * 1024 * 1024,
197            memory: None,
198            threads: None,
199            schema: None,
200            has_parallel: true,
201            has_columnar: true,
202        };
203        assert_eq!(resolve_engine(&config), EngineMode::Parallel);
204    }
205
206    #[test]
207    fn test_large_file_no_parallel() {
208        let config = AutoConfig {
209            file_size: 100 * 1024 * 1024,
210            memory: None,
211            threads: None,
212            schema: None,
213            has_parallel: false,
214            has_columnar: true,
215        };
216        assert_eq!(resolve_engine(&config), EngineMode::Stream);
217    }
218}