Skip to main content

rypipe_core/
schema.rs

1//! Frozen schema for parallel streaming.
2//!
3//! A `FrozenSchema` captures the output column layout after discovery or
4//! explicit declaration.  Once yielded to the consumer it cannot change,
5//! which is the correctness guarantee that makes parallel streaming safe:
6//! chunk 1 cannot discover a column that chunk 7 later introduces.
7
8#[cfg(test)]
9use std::borrow::Cow;
10use std::hash::{Hash, Hasher};
11use std::sync::atomic::{AtomicU64, Ordering};
12use std::sync::{Arc, LazyLock, RwLock};
13
14use rustc_hash::{FxHashMap, FxHasher};
15
16use crate::plan::{ExecutionPlan, FieldType};
17
18/// Immutable, shared across all workers.  No lock, read-only after
19/// construction.
20#[derive(Clone, Debug)]
21pub struct FrozenSchema {
22    /// Output column names in final order.
23    names: Vec<Arc<str>>,
24    /// Raw input field name → output slot index.  `None` = dropped by plan.
25    /// Collapses `field_map` (rename) + `drop_fields` + `field_index` into
26    /// ONE lookup.
27    index: FxHashMap<Box<str>, Option<u32>>,
28    /// Output column types, parallel to `names`.
29    types: Vec<FieldType>,
30    /// `true` = full scan, `false` = sampled.  Affects the unknown-field
31    /// message.
32    exact: bool,
33}
34
35/// Options controlling schema discovery.
36pub struct DiscoveryOpts {
37    /// Files smaller than this are fully scanned (cheap, exact).
38    pub full_scan_threshold: u64,
39    /// Number of strided windows for large files.
40    pub windows: usize,
41    /// Bytes per window.
42    pub window_bytes: usize,
43    /// Whether to always scan the tail of the file to catch late-appearing columns.
44    pub always_scan_tail: bool,
45}
46
47impl Default for DiscoveryOpts {
48    fn default() -> Self {
49        Self {
50            full_scan_threshold: 128 * 1024 * 1024, // 128 MiB
51            windows: 16,
52            window_bytes: 2 * 1024 * 1024, // 2 MiB
53            always_scan_tail: true,
54        }
55    }
56}
57
58/// Compute dynamic window count based on file size.
59///
60/// Smaller files use fewer windows (or full scan), larger files use more
61/// to improve coverage of rare columns.
62pub fn dynamic_window_count(file_size: u64) -> usize {
63    if file_size < 128 * 1024 * 1024 {
64        0 // full scan
65    } else if file_size < 1024 * 1024 * 1024 {
66        16 // 32 MiB total at 2 MiB/window
67    } else if file_size < 10 * 1024 * 1024 * 1024 {
68        32 // 128 MiB total at 4 MiB/window
69    } else {
70        64 // 256 MiB total at 4 MiB/window
71    }
72}
73
74/// Compute dynamic window size based on file size.
75pub fn dynamic_window_size(file_size: u64) -> usize {
76    if file_size < 1024 * 1024 * 1024 {
77        2 * 1024 * 1024 // 2 MiB for files < 1 GiB
78    } else {
79        4 * 1024 * 1024 // 4 MiB for files >= 1 GiB
80    }
81}
82
83/// Maximum number of entries kept in the schema-discovery cache.  The cache
84/// is per-process and unbounded by default; cap it to avoid slow growth over
85/// long-running sessions with many distinct layouts.
86pub const SCHEMA_CACHE_CAP: usize = 128;
87
88type SchemaCacheKey = (u64, u64);
89type SchemaCacheValue = Arc<Vec<String>>;
90
91/// Global layout-keyed cache for discovered raw field-name order.
92/// Key is `(file_len, sample_hash)`; value is the raw order from a previous
93/// full discovery. The current plan is applied to the cached order on hit.
94pub static SCHEMA_CACHE: LazyLock<RwLock<FxHashMap<SchemaCacheKey, SchemaCacheValue>>> =
95    LazyLock::new(|| RwLock::new(FxHashMap::default()));
96
97/// Number of schema-discovery cache hits.
98pub static SCHEMA_CACHE_HITS: AtomicU64 = AtomicU64::new(0);
99/// Number of schema-discovery cache misses.
100pub static SCHEMA_CACHE_MISSES: AtomicU64 = AtomicU64::new(0);
101
102/// Compute a cheap layout signature for schema-discovery caching.
103/// Small files are hashed in full; large files hash several strided windows
104/// so that identical length + leading window is no longer enough to collide.
105pub fn layout_signature(bytes: &[u8], opts: &DiscoveryOpts) -> (u64, u64) {
106    let mut hasher = FxHasher::default();
107    if (bytes.len() as u64) < opts.full_scan_threshold {
108        bytes.hash(&mut hasher);
109    } else {
110        // Hash a few windows spread through the file.  This makes the key
111        // much less likely to collide for same-sized exports from the same
112        // template with different trailing columns.
113        let n = 4usize;
114        let w = opts.window_bytes.min(bytes.len());
115        for i in 0..n {
116            let start = (bytes.len() - w) * i / n.max(1);
117            bytes[start..start + w].hash(&mut hasher);
118        }
119    }
120    (bytes.len() as u64, hasher.finish())
121}
122
123/// Insert a discovered layout into the cache, evicting the oldest entry if
124/// the cache is at capacity.  `FxHashMap` does not track insertion order, so
125/// we evict an arbitrary key; for a bounded cache this is sufficient and
126/// avoids adding an LRU dependency.
127pub fn insert_schema_cache(sig: (u64, u64), order: Arc<Vec<String>>) {
128    let mut cache = SCHEMA_CACHE.write().unwrap();
129    if cache.len() >= SCHEMA_CACHE_CAP {
130        if let Some(oldest) = cache.keys().next().copied() {
131            cache.remove(&oldest);
132        }
133    }
134    cache.insert(sig, order);
135}
136
137/// Clear the discovery cache and reset its hit/miss counters.
138pub fn clear_schema_cache() {
139    SCHEMA_CACHE.write().unwrap().clear();
140    SCHEMA_CACHE_HITS.store(0, Ordering::Relaxed);
141    SCHEMA_CACHE_MISSES.store(0, Ordering::Relaxed);
142}
143
144/// Return the current cache hit and miss counts.
145pub fn schema_cache_stats() -> (u64, u64) {
146    (
147        SCHEMA_CACHE_HITS.load(Ordering::Relaxed),
148        SCHEMA_CACHE_MISSES.load(Ordering::Relaxed),
149    )
150}
151
152/// What to do when a field appears in the file but not in the schema.
153#[derive(Clone, Copy, Debug, PartialEq, Eq)]
154pub enum UnknownFieldPolicy {
155    /// Return an error (default).
156    Error,
157    /// Ignore the field, count occurrences, report at end.
158    Skip,
159}
160
161impl FrozenSchema {
162    /// Build from explicit column names and an execution plan.
163    ///
164    /// Declares the exact output columns.  Fields in the file not listed
165    /// here cause a hard error.  Columns declared here but absent from the
166    /// file become all-null.
167    pub fn from_plan(names: &[&str], plan: &ExecutionPlan) -> Self {
168        let mut index = FxHashMap::default();
169        let mut types = Vec::with_capacity(names.len());
170
171        for (slot, &name) in names.iter().enumerate() {
172            // Collapse rename: if the plan renames `name` to something
173            // else, we still map the *raw* name to the slot.  The caller
174            // should pass the *output* names; renames are applied by the
175            // plan already.
176            let ty = plan.column_type(name);
177            types.push(ty);
178            index.insert(Box::from(name), Some(slot as u32));
179        }
180
181        FrozenSchema {
182            names: names.iter().map(|n| Arc::from(*n)).collect(),
183            index,
184            types,
185            exact: true,
186        }
187    }
188
189    /// Build from explicit column names with non-exact matching.
190    ///
191    /// Like `from_plan`, but fields in the file not listed in `names` are
192    /// appended in discovery order rather than causing an error.  The declared
193    /// columns keep their specified order; unknown columns appear after them.
194    pub fn from_partial_plan(names: &[&str], plan: &ExecutionPlan) -> Self {
195        let mut index = FxHashMap::default();
196        let mut types = Vec::with_capacity(names.len());
197
198        for (slot, &name) in names.iter().enumerate() {
199            let ty = plan.column_type(name);
200            types.push(ty);
201            index.insert(Box::from(name), Some(slot as u32));
202        }
203
204        FrozenSchema {
205            names: names.iter().map(|n| Arc::from(*n)).collect(),
206            index,
207            types,
208            exact: false,
209        }
210    }
211
212    /// Build from discovered names (file order, then document order within
213    /// a window).  Applies the execution plan's renames and drops.
214    pub fn from_discovered(names_in_order: &[String], plan: &ExecutionPlan) -> Self {
215        let mut index = FxHashMap::default();
216        let mut out_names = Vec::new();
217        let mut types = Vec::new();
218
219        for name in names_in_order {
220            // Apply rename
221            let resolved = plan.resolve_field(name);
222            match resolved {
223                Some(resolved_name) => {
224                    // Check if renamed name is dropped
225                    if plan.drop_fields.contains(resolved_name) {
226                        index.insert(Box::from(name.as_str()), None);
227                        continue;
228                    }
229                    let slot = out_names.len() as u32;
230                    let ty = plan.column_type(resolved_name);
231                    index.insert(Box::from(name.as_str()), Some(slot));
232                    out_names.push(Arc::from(resolved_name));
233                    types.push(ty);
234                }
235                None => {
236                    // Field is dropped by plan
237                    index.insert(Box::from(name.as_str()), None);
238                }
239            }
240        }
241
242        FrozenSchema {
243            names: out_names,
244            index,
245            types,
246            exact: false,
247        }
248    }
249
250    /// Number of output columns.
251    pub fn num_columns(&self) -> usize {
252        self.names.len()
253    }
254
255    /// Output column names in order.
256    pub fn column_names(&self) -> &[Arc<str>] {
257        &self.names
258    }
259
260    /// Output column types, parallel to `column_names`.
261    pub fn column_types(&self) -> &[FieldType] {
262        &self.types
263    }
264
265    /// Resolve a raw field name to its output slot, or `None` if dropped.
266    #[inline]
267    pub fn resolve(&self, raw_name: &str) -> Option<u32> {
268        self.index.get(raw_name).copied().flatten()
269    }
270
271    /// Whether the schema was derived from a full scan (exact) or sampling.
272    pub fn is_exact(&self) -> bool {
273        self.exact
274    }
275}
276
277#[cfg(test)]
278mod tests {
279    use super::*;
280
281    #[test]
282    fn test_from_plan_explicit() {
283        let plan = ExecutionPlan::new();
284        let schema = FrozenSchema::from_plan(&["A", "B", "C"], &plan);
285        assert_eq!(schema.num_columns(), 3);
286        assert_eq!(schema.resolve("A"), Some(0));
287        assert_eq!(schema.resolve("B"), Some(1));
288        assert_eq!(schema.resolve("C"), Some(2));
289        assert_eq!(schema.resolve("D"), None);
290        assert!(schema.is_exact());
291    }
292
293    #[test]
294    fn test_from_plan_with_drop() {
295        let mut plan = ExecutionPlan::new();
296        plan.drop_fields.insert("B".to_string());
297        let schema = FrozenSchema::from_plan(&["A", "B", "C"], &plan);
298        // from_plan does NOT drop; it declares the schema as-is.
299        // Drops are handled at parse time via resolve() → None.
300        assert_eq!(schema.num_columns(), 3);
301    }
302
303    #[test]
304    fn test_from_discovered_with_rename() {
305        let mut plan = ExecutionPlan::new();
306        plan.field_map.insert("old".to_string(), "new".to_string());
307        let names = vec!["old".to_string(), "keep".to_string()];
308        let schema = FrozenSchema::from_discovered(&names, &plan);
309        assert_eq!(schema.num_columns(), 2);
310        assert_eq!(schema.resolve("old"), Some(0)); // slot 0 = "new"
311        assert_eq!(schema.resolve("keep"), Some(1));
312        assert_eq!(schema.column_names()[0].as_ref(), "new");
313    }
314
315    #[test]
316    fn test_from_discovered_with_drop() {
317        let mut plan = ExecutionPlan::new();
318        plan.drop_fields.insert("drop_me".to_string());
319        let names = vec!["keep".to_string(), "drop_me".to_string()];
320        let schema = FrozenSchema::from_discovered(&names, &plan);
321        assert_eq!(schema.num_columns(), 1);
322        assert_eq!(schema.resolve("keep"), Some(0));
323        assert_eq!(schema.resolve("drop_me"), None);
324    }
325
326    #[test]
327    fn test_deterministic_order() {
328        let plan = ExecutionPlan::new();
329        let names = vec!["C".to_string(), "A".to_string(), "B".to_string()];
330        let s1 = FrozenSchema::from_discovered(&names, &plan);
331        let s2 = FrozenSchema::from_discovered(&names, &plan);
332        assert_eq!(s1.column_names(), s2.column_names());
333    }
334
335    #[test]
336    fn test_ensure_schema_preserves_existing_columns() {
337        use crate::decoder::ColumnarSink;
338        use crate::engine::TableBuilder;
339        let plan = ExecutionPlan::new();
340        let schema = FrozenSchema::from_plan(&["X", "Y", "Z"], &plan);
341        let mut builder = TableBuilder::with_plan(100, Arc::new(plan));
342        // Pre-add column "X" with a push.
343        builder.begin_row();
344        builder.put_field("X", crate::value::Value::Str(Cow::Borrowed("hello")));
345        builder.end_row();
346        let rows_before = builder.num_rows();
347        let cols_before = builder.num_columns();
348        // ensure_schema should add Y and Z, but not duplicate X.
349        builder.ensure_schema(&schema).unwrap();
350        assert_eq!(builder.num_rows(), rows_before);
351        assert_eq!(builder.num_columns(), cols_before + 2);
352        assert!(builder.column_names().contains(&"X".to_string()));
353        assert!(builder.column_names().contains(&"Y".to_string()));
354        assert!(builder.column_names().contains(&"Z".to_string()));
355    }
356
357    #[test]
358    fn test_dynamic_window_count() {
359        // Small file: full scan
360        assert_eq!(dynamic_window_count(50 * 1024 * 1024), 0);
361        // Medium file: 16 windows
362        assert_eq!(dynamic_window_count(500 * 1024 * 1024), 16);
363        // Large file: 32 windows
364        assert_eq!(dynamic_window_count(2 * 1024 * 1024 * 1024), 32);
365        // Very large file: 64 windows
366        assert_eq!(dynamic_window_count(20 * 1024 * 1024 * 1024), 64);
367    }
368
369    #[test]
370    fn test_dynamic_window_size() {
371        // Small file: 2 MiB windows
372        assert_eq!(dynamic_window_size(500 * 1024 * 1024), 2 * 1024 * 1024);
373        // Large file: 4 MiB windows
374        assert_eq!(dynamic_window_size(2 * 1024 * 1024 * 1024), 4 * 1024 * 1024);
375    }
376}