1#[cfg(test)]
9use std::borrow::Cow;
10use std::hash::{Hash, Hasher};
11use std::sync::atomic::{AtomicU64, Ordering};
12use std::sync::{Arc, LazyLock, RwLock};
13
14use rustc_hash::{FxHashMap, FxHasher};
15
16use crate::plan::{ExecutionPlan, FieldType};
17
18#[derive(Clone, Debug)]
21pub struct FrozenSchema {
22 names: Vec<Arc<str>>,
24 index: FxHashMap<Box<str>, Option<u32>>,
28 types: Vec<FieldType>,
30 exact: bool,
33}
34
35pub struct DiscoveryOpts {
37 pub full_scan_threshold: u64,
39 pub windows: usize,
41 pub window_bytes: usize,
43}
44
45impl Default for DiscoveryOpts {
46 fn default() -> Self {
47 Self {
48 full_scan_threshold: 128 * 1024 * 1024, windows: 16,
50 window_bytes: 2 * 1024 * 1024, }
52 }
53}
54
55pub const SCHEMA_CACHE_CAP: usize = 128;
59
60type SchemaCacheKey = (u64, u64);
61type SchemaCacheValue = Arc<Vec<String>>;
62
63pub static SCHEMA_CACHE: LazyLock<RwLock<FxHashMap<SchemaCacheKey, SchemaCacheValue>>> =
67 LazyLock::new(|| RwLock::new(FxHashMap::default()));
68
69pub static SCHEMA_CACHE_HITS: AtomicU64 = AtomicU64::new(0);
71pub static SCHEMA_CACHE_MISSES: AtomicU64 = AtomicU64::new(0);
73
74pub fn layout_signature(bytes: &[u8], opts: &DiscoveryOpts) -> (u64, u64) {
78 let mut hasher = FxHasher::default();
79 if (bytes.len() as u64) < opts.full_scan_threshold {
80 bytes.hash(&mut hasher);
81 } else {
82 let n = 4usize;
86 let w = opts.window_bytes.min(bytes.len());
87 for i in 0..n {
88 let start = (bytes.len() - w) * i / n.max(1);
89 bytes[start..start + w].hash(&mut hasher);
90 }
91 }
92 (bytes.len() as u64, hasher.finish())
93}
94
95pub fn insert_schema_cache(sig: (u64, u64), order: Arc<Vec<String>>) {
100 let mut cache = SCHEMA_CACHE.write().unwrap();
101 if cache.len() >= SCHEMA_CACHE_CAP {
102 if let Some(oldest) = cache.keys().next().copied() {
103 cache.remove(&oldest);
104 }
105 }
106 cache.insert(sig, order);
107}
108
109pub fn clear_schema_cache() {
111 SCHEMA_CACHE.write().unwrap().clear();
112 SCHEMA_CACHE_HITS.store(0, Ordering::Relaxed);
113 SCHEMA_CACHE_MISSES.store(0, Ordering::Relaxed);
114}
115
116pub fn schema_cache_stats() -> (u64, u64) {
118 (
119 SCHEMA_CACHE_HITS.load(Ordering::Relaxed),
120 SCHEMA_CACHE_MISSES.load(Ordering::Relaxed),
121 )
122}
123
124#[derive(Clone, Copy, Debug, PartialEq, Eq)]
126pub enum UnknownFieldPolicy {
127 Error,
129 Skip,
131}
132
133impl FrozenSchema {
134 pub fn from_plan(names: &[&str], plan: &ExecutionPlan) -> Self {
140 let mut index = FxHashMap::default();
141 let mut types = Vec::with_capacity(names.len());
142
143 for (slot, &name) in names.iter().enumerate() {
144 let ty = plan.column_type(name);
149 types.push(ty);
150 index.insert(Box::from(name), Some(slot as u32));
151 }
152
153 FrozenSchema {
154 names: names.iter().map(|n| Arc::from(*n)).collect(),
155 index,
156 types,
157 exact: true,
158 }
159 }
160
161 pub fn from_discovered(names_in_order: &[String], plan: &ExecutionPlan) -> Self {
164 let mut index = FxHashMap::default();
165 let mut out_names = Vec::new();
166 let mut types = Vec::new();
167
168 for name in names_in_order {
169 let resolved = plan.resolve_field(name);
171 match resolved {
172 Some(resolved_name) => {
173 if plan.drop_fields.contains(resolved_name) {
175 index.insert(Box::from(name.as_str()), None);
176 continue;
177 }
178 let slot = out_names.len() as u32;
179 let ty = plan.column_type(resolved_name);
180 index.insert(Box::from(name.as_str()), Some(slot));
181 out_names.push(Arc::from(resolved_name));
182 types.push(ty);
183 }
184 None => {
185 index.insert(Box::from(name.as_str()), None);
187 }
188 }
189 }
190
191 FrozenSchema {
192 names: out_names,
193 index,
194 types,
195 exact: false,
196 }
197 }
198
199 pub fn num_columns(&self) -> usize {
201 self.names.len()
202 }
203
204 pub fn column_names(&self) -> &[Arc<str>] {
206 &self.names
207 }
208
209 pub fn column_types(&self) -> &[FieldType] {
211 &self.types
212 }
213
214 #[inline]
216 pub fn resolve(&self, raw_name: &str) -> Option<u32> {
217 self.index.get(raw_name).copied().flatten()
218 }
219
220 pub fn is_exact(&self) -> bool {
222 self.exact
223 }
224}
225
226#[cfg(test)]
227mod tests {
228 use super::*;
229
230 #[test]
231 fn test_from_plan_explicit() {
232 let plan = ExecutionPlan::new();
233 let schema = FrozenSchema::from_plan(&["A", "B", "C"], &plan);
234 assert_eq!(schema.num_columns(), 3);
235 assert_eq!(schema.resolve("A"), Some(0));
236 assert_eq!(schema.resolve("B"), Some(1));
237 assert_eq!(schema.resolve("C"), Some(2));
238 assert_eq!(schema.resolve("D"), None);
239 assert!(schema.is_exact());
240 }
241
242 #[test]
243 fn test_from_plan_with_drop() {
244 let mut plan = ExecutionPlan::new();
245 plan.drop_fields.insert("B".to_string());
246 let schema = FrozenSchema::from_plan(&["A", "B", "C"], &plan);
247 assert_eq!(schema.num_columns(), 3);
250 }
251
252 #[test]
253 fn test_from_discovered_with_rename() {
254 let mut plan = ExecutionPlan::new();
255 plan.field_map.insert("old".to_string(), "new".to_string());
256 let names = vec!["old".to_string(), "keep".to_string()];
257 let schema = FrozenSchema::from_discovered(&names, &plan);
258 assert_eq!(schema.num_columns(), 2);
259 assert_eq!(schema.resolve("old"), Some(0)); assert_eq!(schema.resolve("keep"), Some(1));
261 assert_eq!(schema.column_names()[0].as_ref(), "new");
262 }
263
264 #[test]
265 fn test_from_discovered_with_drop() {
266 let mut plan = ExecutionPlan::new();
267 plan.drop_fields.insert("drop_me".to_string());
268 let names = vec!["keep".to_string(), "drop_me".to_string()];
269 let schema = FrozenSchema::from_discovered(&names, &plan);
270 assert_eq!(schema.num_columns(), 1);
271 assert_eq!(schema.resolve("keep"), Some(0));
272 assert_eq!(schema.resolve("drop_me"), None);
273 }
274
275 #[test]
276 fn test_deterministic_order() {
277 let plan = ExecutionPlan::new();
278 let names = vec!["C".to_string(), "A".to_string(), "B".to_string()];
279 let s1 = FrozenSchema::from_discovered(&names, &plan);
280 let s2 = FrozenSchema::from_discovered(&names, &plan);
281 assert_eq!(s1.column_names(), s2.column_names());
282 }
283
284 #[test]
285 fn test_ensure_schema_preserves_existing_columns() {
286 use crate::decoder::ColumnarSink;
287 use crate::engine::TableBuilder;
288 let plan = ExecutionPlan::new();
289 let schema = FrozenSchema::from_plan(&["X", "Y", "Z"], &plan);
290 let mut builder = TableBuilder::with_plan(100, Arc::new(plan));
291 builder.begin_row();
293 builder.put_field("X", crate::value::Value::Str(Cow::Borrowed("hello")));
294 builder.end_row();
295 let rows_before = builder.num_rows();
296 let cols_before = builder.num_columns();
297 builder.ensure_schema(&schema).unwrap();
299 assert_eq!(builder.num_rows(), rows_before);
300 assert_eq!(builder.num_columns(), cols_before + 2);
301 assert!(builder.column_names().contains(&"X".to_string()));
302 assert!(builder.column_names().contains(&"Y".to_string()));
303 assert!(builder.column_names().contains(&"Z".to_string()));
304 }
305}