1#[cfg(test)]
9use std::borrow::Cow;
10use std::hash::{Hash, Hasher};
11use std::sync::atomic::{AtomicU64, Ordering};
12use std::sync::{Arc, LazyLock, RwLock};
13
14use rustc_hash::{FxHashMap, FxHasher};
15
16use crate::plan::{ExecutionPlan, FieldType};
17
18#[derive(Clone, Debug)]
21pub struct FrozenSchema {
22 names: Vec<Arc<str>>,
24 index: FxHashMap<Box<str>, Option<u32>>,
28 types: Vec<FieldType>,
30 exact: bool,
33}
34
35pub struct DiscoveryOpts {
37 pub full_scan_threshold: u64,
39 pub windows: usize,
41 pub window_bytes: usize,
43 pub always_scan_tail: bool,
45}
46
47impl Default for DiscoveryOpts {
48 fn default() -> Self {
49 Self {
50 full_scan_threshold: 128 * 1024 * 1024, windows: 16,
52 window_bytes: 2 * 1024 * 1024, always_scan_tail: true,
54 }
55 }
56}
57
58pub fn dynamic_window_count(file_size: u64) -> usize {
63 if file_size < 128 * 1024 * 1024 {
64 0 } else if file_size < 1024 * 1024 * 1024 {
66 16 } else if file_size < 10 * 1024 * 1024 * 1024 {
68 32 } else {
70 64 }
72}
73
74pub fn dynamic_window_size(file_size: u64) -> usize {
76 if file_size < 1024 * 1024 * 1024 {
77 2 * 1024 * 1024 } else {
79 4 * 1024 * 1024 }
81}
82
83pub const SCHEMA_CACHE_CAP: usize = 128;
87
88type SchemaCacheKey = (u64, u64);
89type SchemaCacheValue = Arc<Vec<String>>;
90
91pub static SCHEMA_CACHE: LazyLock<RwLock<FxHashMap<SchemaCacheKey, SchemaCacheValue>>> =
95 LazyLock::new(|| RwLock::new(FxHashMap::default()));
96
97pub static SCHEMA_CACHE_HITS: AtomicU64 = AtomicU64::new(0);
99pub static SCHEMA_CACHE_MISSES: AtomicU64 = AtomicU64::new(0);
101
102pub fn layout_signature(bytes: &[u8], opts: &DiscoveryOpts) -> (u64, u64) {
106 let mut hasher = FxHasher::default();
107 if (bytes.len() as u64) < opts.full_scan_threshold {
108 bytes.hash(&mut hasher);
109 } else {
110 let n = 4usize;
114 let w = opts.window_bytes.min(bytes.len());
115 for i in 0..n {
116 let start = (bytes.len() - w) * i / n.max(1);
117 bytes[start..start + w].hash(&mut hasher);
118 }
119 }
120 (bytes.len() as u64, hasher.finish())
121}
122
123pub fn insert_schema_cache(sig: (u64, u64), order: Arc<Vec<String>>) {
128 let mut cache = SCHEMA_CACHE.write().unwrap();
129 if cache.len() >= SCHEMA_CACHE_CAP {
130 if let Some(oldest) = cache.keys().next().copied() {
131 cache.remove(&oldest);
132 }
133 }
134 cache.insert(sig, order);
135}
136
137pub fn clear_schema_cache() {
139 SCHEMA_CACHE.write().unwrap().clear();
140 SCHEMA_CACHE_HITS.store(0, Ordering::Relaxed);
141 SCHEMA_CACHE_MISSES.store(0, Ordering::Relaxed);
142}
143
144pub fn schema_cache_stats() -> (u64, u64) {
146 (
147 SCHEMA_CACHE_HITS.load(Ordering::Relaxed),
148 SCHEMA_CACHE_MISSES.load(Ordering::Relaxed),
149 )
150}
151
152#[derive(Clone, Copy, Debug, PartialEq, Eq)]
154pub enum UnknownFieldPolicy {
155 Error,
157 Skip,
159}
160
161impl FrozenSchema {
162 pub fn from_plan(names: &[&str], plan: &ExecutionPlan) -> Self {
168 let mut index = FxHashMap::default();
169 let mut types = Vec::with_capacity(names.len());
170
171 for (slot, &name) in names.iter().enumerate() {
172 let ty = plan.column_type(name);
177 types.push(ty);
178 index.insert(Box::from(name), Some(slot as u32));
179 }
180
181 FrozenSchema {
182 names: names.iter().map(|n| Arc::from(*n)).collect(),
183 index,
184 types,
185 exact: true,
186 }
187 }
188
189 pub fn from_partial_plan(names: &[&str], plan: &ExecutionPlan) -> Self {
195 let mut index = FxHashMap::default();
196 let mut types = Vec::with_capacity(names.len());
197
198 for (slot, &name) in names.iter().enumerate() {
199 let ty = plan.column_type(name);
200 types.push(ty);
201 index.insert(Box::from(name), Some(slot as u32));
202 }
203
204 FrozenSchema {
205 names: names.iter().map(|n| Arc::from(*n)).collect(),
206 index,
207 types,
208 exact: false,
209 }
210 }
211
212 pub fn from_discovered(names_in_order: &[String], plan: &ExecutionPlan) -> Self {
215 let mut index = FxHashMap::default();
216 let mut out_names = Vec::new();
217 let mut types = Vec::new();
218
219 for name in names_in_order {
220 let resolved = plan.resolve_field(name);
222 match resolved {
223 Some(resolved_name) => {
224 if plan.drop_fields.contains(resolved_name) {
226 index.insert(Box::from(name.as_str()), None);
227 continue;
228 }
229 let slot = out_names.len() as u32;
230 let ty = plan.column_type(resolved_name);
231 index.insert(Box::from(name.as_str()), Some(slot));
232 out_names.push(Arc::from(resolved_name));
233 types.push(ty);
234 }
235 None => {
236 index.insert(Box::from(name.as_str()), None);
238 }
239 }
240 }
241
242 FrozenSchema {
243 names: out_names,
244 index,
245 types,
246 exact: false,
247 }
248 }
249
250 pub fn num_columns(&self) -> usize {
252 self.names.len()
253 }
254
255 pub fn column_names(&self) -> &[Arc<str>] {
257 &self.names
258 }
259
260 pub fn column_types(&self) -> &[FieldType] {
262 &self.types
263 }
264
265 #[inline]
267 pub fn resolve(&self, raw_name: &str) -> Option<u32> {
268 self.index.get(raw_name).copied().flatten()
269 }
270
271 pub fn is_exact(&self) -> bool {
273 self.exact
274 }
275}
276
277#[cfg(test)]
278mod tests {
279 use super::*;
280
281 #[test]
282 fn test_from_plan_explicit() {
283 let plan = ExecutionPlan::new();
284 let schema = FrozenSchema::from_plan(&["A", "B", "C"], &plan);
285 assert_eq!(schema.num_columns(), 3);
286 assert_eq!(schema.resolve("A"), Some(0));
287 assert_eq!(schema.resolve("B"), Some(1));
288 assert_eq!(schema.resolve("C"), Some(2));
289 assert_eq!(schema.resolve("D"), None);
290 assert!(schema.is_exact());
291 }
292
293 #[test]
294 fn test_from_plan_with_drop() {
295 let mut plan = ExecutionPlan::new();
296 plan.drop_fields.insert("B".to_string());
297 let schema = FrozenSchema::from_plan(&["A", "B", "C"], &plan);
298 assert_eq!(schema.num_columns(), 3);
301 }
302
303 #[test]
304 fn test_from_discovered_with_rename() {
305 let mut plan = ExecutionPlan::new();
306 plan.field_map.insert("old".to_string(), "new".to_string());
307 let names = vec!["old".to_string(), "keep".to_string()];
308 let schema = FrozenSchema::from_discovered(&names, &plan);
309 assert_eq!(schema.num_columns(), 2);
310 assert_eq!(schema.resolve("old"), Some(0)); assert_eq!(schema.resolve("keep"), Some(1));
312 assert_eq!(schema.column_names()[0].as_ref(), "new");
313 }
314
315 #[test]
316 fn test_from_discovered_with_drop() {
317 let mut plan = ExecutionPlan::new();
318 plan.drop_fields.insert("drop_me".to_string());
319 let names = vec!["keep".to_string(), "drop_me".to_string()];
320 let schema = FrozenSchema::from_discovered(&names, &plan);
321 assert_eq!(schema.num_columns(), 1);
322 assert_eq!(schema.resolve("keep"), Some(0));
323 assert_eq!(schema.resolve("drop_me"), None);
324 }
325
326 #[test]
327 fn test_deterministic_order() {
328 let plan = ExecutionPlan::new();
329 let names = vec!["C".to_string(), "A".to_string(), "B".to_string()];
330 let s1 = FrozenSchema::from_discovered(&names, &plan);
331 let s2 = FrozenSchema::from_discovered(&names, &plan);
332 assert_eq!(s1.column_names(), s2.column_names());
333 }
334
335 #[test]
336 fn test_ensure_schema_preserves_existing_columns() {
337 use crate::decoder::ColumnarSink;
338 use crate::engine::TableBuilder;
339 let plan = ExecutionPlan::new();
340 let schema = FrozenSchema::from_plan(&["X", "Y", "Z"], &plan);
341 let mut builder = TableBuilder::with_plan(100, Arc::new(plan));
342 builder.begin_row();
344 builder.put_field("X", crate::value::Value::Str(Cow::Borrowed("hello")));
345 builder.end_row();
346 let rows_before = builder.num_rows();
347 let cols_before = builder.num_columns();
348 builder.ensure_schema(&schema).unwrap();
350 assert_eq!(builder.num_rows(), rows_before);
351 assert_eq!(builder.num_columns(), cols_before + 2);
352 assert!(builder.column_names().contains(&"X".to_string()));
353 assert!(builder.column_names().contains(&"Y".to_string()));
354 assert!(builder.column_names().contains(&"Z".to_string()));
355 }
356
357 #[test]
358 fn test_dynamic_window_count() {
359 assert_eq!(dynamic_window_count(50 * 1024 * 1024), 0);
361 assert_eq!(dynamic_window_count(500 * 1024 * 1024), 16);
363 assert_eq!(dynamic_window_count(2 * 1024 * 1024 * 1024), 32);
365 assert_eq!(dynamic_window_count(20 * 1024 * 1024 * 1024), 64);
367 }
368
369 #[test]
370 fn test_dynamic_window_size() {
371 assert_eq!(dynamic_window_size(500 * 1024 * 1024), 2 * 1024 * 1024);
373 assert_eq!(dynamic_window_size(2 * 1024 * 1024 * 1024), 4 * 1024 * 1024);
375 }
376}