1#[cfg(test)]
9use std::borrow::Cow;
10use std::hash::{Hash, Hasher};
11use std::sync::atomic::{AtomicU64, Ordering};
12use std::sync::{Arc, LazyLock, RwLock};
13
14use rustc_hash::{FxHashMap, FxHasher};
15
16use crate::plan::{ExecutionPlan, FieldType};
17
18#[derive(Clone, Debug)]
21pub struct FrozenSchema {
22 names: Vec<Arc<str>>,
24 index: FxHashMap<Box<str>, Option<u32>>,
28 types: Vec<FieldType>,
30 exact: bool,
33}
34
35pub struct DiscoveryOpts {
37 pub full_scan_threshold: u64,
39 pub windows: usize,
41 pub window_bytes: usize,
43 pub always_scan_tail: bool,
45}
46
47impl Default for DiscoveryOpts {
48 fn default() -> Self {
49 Self {
50 full_scan_threshold: 128 * 1024 * 1024, windows: 16,
52 window_bytes: 2 * 1024 * 1024, always_scan_tail: true,
54 }
55 }
56}
57
58pub fn dynamic_window_count(file_size: u64) -> usize {
63 if file_size < 128 * 1024 * 1024 {
64 0 } else if file_size < 1024 * 1024 * 1024 {
66 16 } else if file_size < 10 * 1024 * 1024 * 1024 {
68 32 } else {
70 64 }
72}
73
74pub fn dynamic_window_size(file_size: u64) -> usize {
76 if file_size < 1024 * 1024 * 1024 {
77 2 * 1024 * 1024 } else {
79 4 * 1024 * 1024 }
81}
82
83pub const SCHEMA_CACHE_CAP: usize = 128;
87
88type SchemaCacheKey = (u64, u64);
89type SchemaCacheValue = Arc<Vec<String>>;
90
91pub static SCHEMA_CACHE: LazyLock<RwLock<FxHashMap<SchemaCacheKey, SchemaCacheValue>>> =
95 LazyLock::new(|| RwLock::new(FxHashMap::default()));
96
97pub static SCHEMA_CACHE_HITS: AtomicU64 = AtomicU64::new(0);
99pub static SCHEMA_CACHE_MISSES: AtomicU64 = AtomicU64::new(0);
101
102pub fn layout_signature(bytes: &[u8], opts: &DiscoveryOpts) -> (u64, u64) {
106 let mut hasher = FxHasher::default();
107 for range in signature_chunks(bytes.len(), opts) {
108 bytes[range].hash(&mut hasher);
109 }
110 (bytes.len() as u64, hasher.finish())
111}
112
113pub(crate) fn signature_chunks(
114 len: usize,
115 opts: &DiscoveryOpts,
116) -> impl Iterator<Item = std::ops::Range<usize>> {
117 let full = (len as u64) < opts.full_scan_threshold;
118 let count = if full { 1 } else { 4 };
119 let width = if full {
120 len
121 } else {
122 opts.window_bytes.min(len)
123 };
124 (0..count).flat_map(move |i| {
125 let start = (len - width) * i / count;
126 let end = start + width;
127 (start..end)
128 .step_by(65536)
129 .map(move |offset| offset..end.min(offset.saturating_add(65536)))
130 })
131}
132
133pub fn insert_schema_cache(sig: (u64, u64), order: Arc<Vec<String>>) {
138 let mut cache = SCHEMA_CACHE.write().unwrap_or_else(|e| e.into_inner());
139 if cache.len() >= SCHEMA_CACHE_CAP {
140 if let Some(oldest) = cache.keys().next().copied() {
141 cache.remove(&oldest);
142 }
143 }
144 cache.insert(sig, order);
145}
146
147pub fn clear_schema_cache() {
149 SCHEMA_CACHE
150 .write()
151 .unwrap_or_else(|e| e.into_inner())
152 .clear();
153 SCHEMA_CACHE_HITS.store(0, Ordering::Relaxed);
154 SCHEMA_CACHE_MISSES.store(0, Ordering::Relaxed);
155}
156
157pub fn schema_cache_stats() -> (u64, u64) {
159 (
160 SCHEMA_CACHE_HITS.load(Ordering::Relaxed),
161 SCHEMA_CACHE_MISSES.load(Ordering::Relaxed),
162 )
163}
164
165#[derive(Clone, Copy, Debug, PartialEq, Eq)]
167pub enum UnknownFieldPolicy {
168 Error,
170 Skip,
172}
173
174impl FrozenSchema {
175 pub fn from_plan(names: &[&str], plan: &ExecutionPlan) -> Self {
181 let mut index = FxHashMap::default();
182 let mut types = Vec::with_capacity(names.len());
183
184 for (slot, &name) in names.iter().enumerate() {
185 let ty = plan.column_type(name);
190 types.push(ty);
191 index.insert(Box::from(name), Some(slot as u32));
192 }
193
194 FrozenSchema {
195 names: names.iter().map(|n| Arc::from(*n)).collect(),
196 index,
197 types,
198 exact: true,
199 }
200 }
201
202 pub fn from_partial_plan(names: &[&str], plan: &ExecutionPlan) -> Self {
208 let mut index = FxHashMap::default();
209 let mut types = Vec::with_capacity(names.len());
210
211 for (slot, &name) in names.iter().enumerate() {
212 let ty = plan.column_type(name);
213 types.push(ty);
214 index.insert(Box::from(name), Some(slot as u32));
215 }
216
217 FrozenSchema {
218 names: names.iter().map(|n| Arc::from(*n)).collect(),
219 index,
220 types,
221 exact: false,
222 }
223 }
224
225 pub fn from_discovered(names_in_order: &[String], plan: &ExecutionPlan) -> Self {
228 let mut index = FxHashMap::default();
229 let mut out_names = Vec::new();
230 let mut types = Vec::new();
231
232 for name in names_in_order {
233 let resolved = plan.resolve_field(name);
235 match resolved {
236 Some(resolved_name) => {
237 if plan.drop_fields.contains(resolved_name) {
239 index.insert(Box::from(name.as_str()), None);
240 continue;
241 }
242 let slot = out_names.len() as u32;
243 let ty = plan.column_type(resolved_name);
244 index.insert(Box::from(name.as_str()), Some(slot));
245 out_names.push(Arc::from(resolved_name));
246 types.push(ty);
247 }
248 None => {
249 index.insert(Box::from(name.as_str()), None);
251 }
252 }
253 }
254
255 FrozenSchema {
256 names: out_names,
257 index,
258 types,
259 exact: false,
260 }
261 }
262
263 pub fn num_columns(&self) -> usize {
265 self.names.len()
266 }
267
268 pub fn column_names(&self) -> &[Arc<str>] {
270 &self.names
271 }
272
273 pub fn column_types(&self) -> &[FieldType] {
275 &self.types
276 }
277
278 #[inline]
280 pub fn resolve(&self, raw_name: &str) -> Option<u32> {
281 self.index.get(raw_name).copied().flatten()
282 }
283
284 pub fn is_exact(&self) -> bool {
286 self.exact
287 }
288}
289
290#[cfg(test)]
291mod tests {
292 use super::*;
293
294 #[test]
295 fn test_from_plan_explicit() {
296 let plan = ExecutionPlan::new();
297 let schema = FrozenSchema::from_plan(&["A", "B", "C"], &plan);
298 assert_eq!(schema.num_columns(), 3);
299 assert_eq!(schema.resolve("A"), Some(0));
300 assert_eq!(schema.resolve("B"), Some(1));
301 assert_eq!(schema.resolve("C"), Some(2));
302 assert_eq!(schema.resolve("D"), None);
303 assert!(schema.is_exact());
304 }
305
306 #[test]
307 fn test_from_plan_with_drop() {
308 let mut plan = ExecutionPlan::new();
309 plan.drop_fields.insert("B".to_string());
310 let schema = FrozenSchema::from_plan(&["A", "B", "C"], &plan);
311 assert_eq!(schema.num_columns(), 3);
314 }
315
316 #[test]
317 fn test_from_discovered_with_rename() {
318 let mut plan = ExecutionPlan::new();
319 plan.field_map.insert("old".to_string(), "new".to_string());
320 let names = vec!["old".to_string(), "keep".to_string()];
321 let schema = FrozenSchema::from_discovered(&names, &plan);
322 assert_eq!(schema.num_columns(), 2);
323 assert_eq!(schema.resolve("old"), Some(0)); assert_eq!(schema.resolve("keep"), Some(1));
325 assert_eq!(schema.column_names()[0].as_ref(), "new");
326 }
327
328 #[test]
329 fn test_from_discovered_with_drop() {
330 let mut plan = ExecutionPlan::new();
331 plan.drop_fields.insert("drop_me".to_string());
332 let names = vec!["keep".to_string(), "drop_me".to_string()];
333 let schema = FrozenSchema::from_discovered(&names, &plan);
334 assert_eq!(schema.num_columns(), 1);
335 assert_eq!(schema.resolve("keep"), Some(0));
336 assert_eq!(schema.resolve("drop_me"), None);
337 }
338
339 #[test]
340 fn test_deterministic_order() {
341 let plan = ExecutionPlan::new();
342 let names = vec!["C".to_string(), "A".to_string(), "B".to_string()];
343 let s1 = FrozenSchema::from_discovered(&names, &plan);
344 let s2 = FrozenSchema::from_discovered(&names, &plan);
345 assert_eq!(s1.column_names(), s2.column_names());
346 }
347
348 #[test]
349 fn test_ensure_schema_preserves_existing_columns() {
350 use crate::decoder::ColumnarSink;
351 use crate::engine::TableBuilder;
352 let plan = ExecutionPlan::new();
353 let schema = FrozenSchema::from_plan(&["X", "Y", "Z"], &plan);
354 let mut builder = TableBuilder::with_plan(100, Arc::new(plan));
355 builder.begin_row();
357 builder.put_field("X", crate::value::Value::Str(Cow::Borrowed("hello")));
358 builder.end_row();
359 let rows_before = builder.num_rows();
360 let cols_before = builder.num_columns();
361 builder.ensure_schema(&schema).unwrap();
363 assert_eq!(builder.num_rows(), rows_before);
364 assert_eq!(builder.num_columns(), cols_before + 2);
365 assert!(builder.column_names().contains(&"X".to_string()));
366 assert!(builder.column_names().contains(&"Y".to_string()));
367 assert!(builder.column_names().contains(&"Z".to_string()));
368 }
369
370 #[test]
371 fn test_dynamic_window_count() {
372 assert_eq!(dynamic_window_count(50 * 1024 * 1024), 0);
374 assert_eq!(dynamic_window_count(500 * 1024 * 1024), 16);
376 assert_eq!(dynamic_window_count(2 * 1024 * 1024 * 1024), 32);
378 assert_eq!(dynamic_window_count(20 * 1024 * 1024 * 1024), 64);
380 }
381
382 #[test]
383 fn test_dynamic_window_size() {
384 assert_eq!(dynamic_window_size(500 * 1024 * 1024), 2 * 1024 * 1024);
386 assert_eq!(dynamic_window_size(2 * 1024 * 1024 * 1024), 4 * 1024 * 1024);
388 }
389}