Skip to main content

rypipe_core/
schema.rs

1//! Frozen schema for parallel streaming.
2//!
3//! A `FrozenSchema` captures the output column layout after discovery or
4//! explicit declaration.  Once yielded to the consumer it cannot change,
5//! which is the correctness guarantee that makes parallel streaming safe:
6//! chunk 1 cannot discover a column that chunk 7 later introduces.
7
8#[cfg(test)]
9use std::borrow::Cow;
10use std::hash::{Hash, Hasher};
11use std::sync::atomic::{AtomicU64, Ordering};
12use std::sync::{Arc, LazyLock, RwLock};
13
14use rustc_hash::{FxHashMap, FxHasher};
15
16use crate::plan::{ExecutionPlan, FieldType};
17
18/// Immutable, shared across all workers.  No lock, read-only after
19/// construction.
20#[derive(Clone, Debug)]
21pub struct FrozenSchema {
22    /// Output column names in final order.
23    names: Vec<Arc<str>>,
24    /// Raw input field name → output slot index.  `None` = dropped by plan.
25    /// Collapses `field_map` (rename) + `drop_fields` + `field_index` into
26    /// ONE lookup.
27    index: FxHashMap<Box<str>, Option<u32>>,
28    /// Output column types, parallel to `names`.
29    types: Vec<FieldType>,
30    /// `true` = full scan, `false` = sampled.  Affects the unknown-field
31    /// message.
32    exact: bool,
33}
34
35/// Options controlling schema discovery.
36pub struct DiscoveryOpts {
37    /// Files smaller than this are fully scanned (cheap, exact).
38    pub full_scan_threshold: u64,
39    /// Number of strided windows for large files.
40    pub windows: usize,
41    /// Bytes per window.
42    pub window_bytes: usize,
43    /// Whether to always scan the tail of the file to catch late-appearing columns.
44    pub always_scan_tail: bool,
45}
46
47impl Default for DiscoveryOpts {
48    fn default() -> Self {
49        Self {
50            full_scan_threshold: 128 * 1024 * 1024, // 128 MiB
51            windows: 16,
52            window_bytes: 2 * 1024 * 1024, // 2 MiB
53            always_scan_tail: true,
54        }
55    }
56}
57
58/// Compute dynamic window count based on file size.
59///
60/// Smaller files use fewer windows (or full scan), larger files use more
61/// to improve coverage of rare columns.
62pub fn dynamic_window_count(file_size: u64) -> usize {
63    if file_size < 128 * 1024 * 1024 {
64        0 // full scan
65    } else if file_size < 1024 * 1024 * 1024 {
66        16 // 32 MiB total at 2 MiB/window
67    } else if file_size < 10 * 1024 * 1024 * 1024 {
68        32 // 128 MiB total at 4 MiB/window
69    } else {
70        64 // 256 MiB total at 4 MiB/window
71    }
72}
73
74/// Compute dynamic window size based on file size.
75pub fn dynamic_window_size(file_size: u64) -> usize {
76    if file_size < 1024 * 1024 * 1024 {
77        2 * 1024 * 1024 // 2 MiB for files < 1 GiB
78    } else {
79        4 * 1024 * 1024 // 4 MiB for files >= 1 GiB
80    }
81}
82
83/// Maximum number of entries kept in the schema-discovery cache.  The cache
84/// is per-process and unbounded by default; cap it to avoid slow growth over
85/// long-running sessions with many distinct layouts.
86pub const SCHEMA_CACHE_CAP: usize = 128;
87
88type SchemaCacheKey = (u64, u64);
89type SchemaCacheValue = Arc<Vec<String>>;
90
91/// Global layout-keyed cache for discovered raw field-name order.
92/// Key is `(file_len, sample_hash)`; value is the raw order from a previous
93/// full discovery. The current plan is applied to the cached order on hit.
94pub static SCHEMA_CACHE: LazyLock<RwLock<FxHashMap<SchemaCacheKey, SchemaCacheValue>>> =
95    LazyLock::new(|| RwLock::new(FxHashMap::default()));
96
97/// Number of schema-discovery cache hits.
98pub static SCHEMA_CACHE_HITS: AtomicU64 = AtomicU64::new(0);
99/// Number of schema-discovery cache misses.
100pub static SCHEMA_CACHE_MISSES: AtomicU64 = AtomicU64::new(0);
101
102/// Compute a cheap layout signature for schema-discovery caching.
103/// Small files are hashed in full; large files hash several strided windows
104/// so that identical length + leading window is no longer enough to collide.
105pub fn layout_signature(bytes: &[u8], opts: &DiscoveryOpts) -> (u64, u64) {
106    let mut hasher = FxHasher::default();
107    for range in signature_chunks(bytes.len(), opts) {
108        bytes[range].hash(&mut hasher);
109    }
110    (bytes.len() as u64, hasher.finish())
111}
112
113pub(crate) fn signature_chunks(
114    len: usize,
115    opts: &DiscoveryOpts,
116) -> impl Iterator<Item = std::ops::Range<usize>> {
117    let full = (len as u64) < opts.full_scan_threshold;
118    let count = if full { 1 } else { 4 };
119    let width = if full {
120        len
121    } else {
122        opts.window_bytes.min(len)
123    };
124    (0..count).flat_map(move |i| {
125        let start = (len - width) * i / count;
126        let end = start + width;
127        (start..end)
128            .step_by(65536)
129            .map(move |offset| offset..end.min(offset.saturating_add(65536)))
130    })
131}
132
133/// Insert a discovered layout into the cache, evicting the oldest entry if
134/// the cache is at capacity.  `FxHashMap` does not track insertion order, so
135/// we evict an arbitrary key; for a bounded cache this is sufficient and
136/// avoids adding an LRU dependency.
137pub fn insert_schema_cache(sig: (u64, u64), order: Arc<Vec<String>>) {
138    let mut cache = SCHEMA_CACHE.write().unwrap_or_else(|e| e.into_inner());
139    if cache.len() >= SCHEMA_CACHE_CAP {
140        if let Some(oldest) = cache.keys().next().copied() {
141            cache.remove(&oldest);
142        }
143    }
144    cache.insert(sig, order);
145}
146
147/// Clear the discovery cache and reset its hit/miss counters.
148pub fn clear_schema_cache() {
149    SCHEMA_CACHE
150        .write()
151        .unwrap_or_else(|e| e.into_inner())
152        .clear();
153    SCHEMA_CACHE_HITS.store(0, Ordering::Relaxed);
154    SCHEMA_CACHE_MISSES.store(0, Ordering::Relaxed);
155}
156
157/// Return the current cache hit and miss counts.
158pub fn schema_cache_stats() -> (u64, u64) {
159    (
160        SCHEMA_CACHE_HITS.load(Ordering::Relaxed),
161        SCHEMA_CACHE_MISSES.load(Ordering::Relaxed),
162    )
163}
164
165/// What to do when a field appears in the file but not in the schema.
166#[derive(Clone, Copy, Debug, PartialEq, Eq)]
167pub enum UnknownFieldPolicy {
168    /// Return an error (default).
169    Error,
170    /// Ignore the field, count occurrences, report at end.
171    Skip,
172}
173
174impl FrozenSchema {
175    /// Build from explicit column names and an execution plan.
176    ///
177    /// Declares the exact output columns.  Fields in the file not listed
178    /// here cause a hard error.  Columns declared here but absent from the
179    /// file become all-null.
180    pub fn from_plan(names: &[&str], plan: &ExecutionPlan) -> Self {
181        let mut index = FxHashMap::default();
182        let mut types = Vec::with_capacity(names.len());
183
184        for (slot, &name) in names.iter().enumerate() {
185            // Collapse rename: if the plan renames `name` to something
186            // else, we still map the *raw* name to the slot.  The caller
187            // should pass the *output* names; renames are applied by the
188            // plan already.
189            let ty = plan.column_type(name);
190            types.push(ty);
191            index.insert(Box::from(name), Some(slot as u32));
192        }
193
194        FrozenSchema {
195            names: names.iter().map(|n| Arc::from(*n)).collect(),
196            index,
197            types,
198            exact: true,
199        }
200    }
201
202    /// Build from explicit column names with non-exact matching.
203    ///
204    /// Like `from_plan`, but fields in the file not listed in `names` are
205    /// appended in discovery order rather than causing an error.  The declared
206    /// columns keep their specified order; unknown columns appear after them.
207    pub fn from_partial_plan(names: &[&str], plan: &ExecutionPlan) -> Self {
208        let mut index = FxHashMap::default();
209        let mut types = Vec::with_capacity(names.len());
210
211        for (slot, &name) in names.iter().enumerate() {
212            let ty = plan.column_type(name);
213            types.push(ty);
214            index.insert(Box::from(name), Some(slot as u32));
215        }
216
217        FrozenSchema {
218            names: names.iter().map(|n| Arc::from(*n)).collect(),
219            index,
220            types,
221            exact: false,
222        }
223    }
224
225    /// Build from discovered names (file order, then document order within
226    /// a window).  Applies the execution plan's renames and drops.
227    pub fn from_discovered(names_in_order: &[String], plan: &ExecutionPlan) -> Self {
228        let mut index = FxHashMap::default();
229        let mut out_names = Vec::new();
230        let mut types = Vec::new();
231
232        for name in names_in_order {
233            // Apply rename
234            let resolved = plan.resolve_field(name);
235            match resolved {
236                Some(resolved_name) => {
237                    // Check if renamed name is dropped
238                    if plan.drop_fields.contains(resolved_name) {
239                        index.insert(Box::from(name.as_str()), None);
240                        continue;
241                    }
242                    let slot = out_names.len() as u32;
243                    let ty = plan.column_type(resolved_name);
244                    index.insert(Box::from(name.as_str()), Some(slot));
245                    out_names.push(Arc::from(resolved_name));
246                    types.push(ty);
247                }
248                None => {
249                    // Field is dropped by plan
250                    index.insert(Box::from(name.as_str()), None);
251                }
252            }
253        }
254
255        FrozenSchema {
256            names: out_names,
257            index,
258            types,
259            exact: false,
260        }
261    }
262
263    /// Number of output columns.
264    pub fn num_columns(&self) -> usize {
265        self.names.len()
266    }
267
268    /// Output column names in order.
269    pub fn column_names(&self) -> &[Arc<str>] {
270        &self.names
271    }
272
273    /// Output column types, parallel to `column_names`.
274    pub fn column_types(&self) -> &[FieldType] {
275        &self.types
276    }
277
278    /// Resolve a raw field name to its output slot, or `None` if dropped.
279    #[inline]
280    pub fn resolve(&self, raw_name: &str) -> Option<u32> {
281        self.index.get(raw_name).copied().flatten()
282    }
283
284    /// Whether the schema was derived from a full scan (exact) or sampling.
285    pub fn is_exact(&self) -> bool {
286        self.exact
287    }
288}
289
290#[cfg(test)]
291mod tests {
292    use super::*;
293
294    #[test]
295    fn test_from_plan_explicit() {
296        let plan = ExecutionPlan::new();
297        let schema = FrozenSchema::from_plan(&["A", "B", "C"], &plan);
298        assert_eq!(schema.num_columns(), 3);
299        assert_eq!(schema.resolve("A"), Some(0));
300        assert_eq!(schema.resolve("B"), Some(1));
301        assert_eq!(schema.resolve("C"), Some(2));
302        assert_eq!(schema.resolve("D"), None);
303        assert!(schema.is_exact());
304    }
305
306    #[test]
307    fn test_from_plan_with_drop() {
308        let mut plan = ExecutionPlan::new();
309        plan.drop_fields.insert("B".to_string());
310        let schema = FrozenSchema::from_plan(&["A", "B", "C"], &plan);
311        // from_plan does NOT drop; it declares the schema as-is.
312        // Drops are handled at parse time via resolve() → None.
313        assert_eq!(schema.num_columns(), 3);
314    }
315
316    #[test]
317    fn test_from_discovered_with_rename() {
318        let mut plan = ExecutionPlan::new();
319        plan.field_map.insert("old".to_string(), "new".to_string());
320        let names = vec!["old".to_string(), "keep".to_string()];
321        let schema = FrozenSchema::from_discovered(&names, &plan);
322        assert_eq!(schema.num_columns(), 2);
323        assert_eq!(schema.resolve("old"), Some(0)); // slot 0 = "new"
324        assert_eq!(schema.resolve("keep"), Some(1));
325        assert_eq!(schema.column_names()[0].as_ref(), "new");
326    }
327
328    #[test]
329    fn test_from_discovered_with_drop() {
330        let mut plan = ExecutionPlan::new();
331        plan.drop_fields.insert("drop_me".to_string());
332        let names = vec!["keep".to_string(), "drop_me".to_string()];
333        let schema = FrozenSchema::from_discovered(&names, &plan);
334        assert_eq!(schema.num_columns(), 1);
335        assert_eq!(schema.resolve("keep"), Some(0));
336        assert_eq!(schema.resolve("drop_me"), None);
337    }
338
339    #[test]
340    fn test_deterministic_order() {
341        let plan = ExecutionPlan::new();
342        let names = vec!["C".to_string(), "A".to_string(), "B".to_string()];
343        let s1 = FrozenSchema::from_discovered(&names, &plan);
344        let s2 = FrozenSchema::from_discovered(&names, &plan);
345        assert_eq!(s1.column_names(), s2.column_names());
346    }
347
348    #[test]
349    fn test_ensure_schema_preserves_existing_columns() {
350        use crate::decoder::ColumnarSink;
351        use crate::engine::TableBuilder;
352        let plan = ExecutionPlan::new();
353        let schema = FrozenSchema::from_plan(&["X", "Y", "Z"], &plan);
354        let mut builder = TableBuilder::with_plan(100, Arc::new(plan));
355        // Pre-add column "X" with a push.
356        builder.begin_row();
357        builder.put_field("X", crate::value::Value::Str(Cow::Borrowed("hello")));
358        builder.end_row();
359        let rows_before = builder.num_rows();
360        let cols_before = builder.num_columns();
361        // ensure_schema should add Y and Z, but not duplicate X.
362        builder.ensure_schema(&schema).unwrap();
363        assert_eq!(builder.num_rows(), rows_before);
364        assert_eq!(builder.num_columns(), cols_before + 2);
365        assert!(builder.column_names().contains(&"X".to_string()));
366        assert!(builder.column_names().contains(&"Y".to_string()));
367        assert!(builder.column_names().contains(&"Z".to_string()));
368    }
369
370    #[test]
371    fn test_dynamic_window_count() {
372        // Small file: full scan
373        assert_eq!(dynamic_window_count(50 * 1024 * 1024), 0);
374        // Medium file: 16 windows
375        assert_eq!(dynamic_window_count(500 * 1024 * 1024), 16);
376        // Large file: 32 windows
377        assert_eq!(dynamic_window_count(2 * 1024 * 1024 * 1024), 32);
378        // Very large file: 64 windows
379        assert_eq!(dynamic_window_count(20 * 1024 * 1024 * 1024), 64);
380    }
381
382    #[test]
383    fn test_dynamic_window_size() {
384        // Small file: 2 MiB windows
385        assert_eq!(dynamic_window_size(500 * 1024 * 1024), 2 * 1024 * 1024);
386        // Large file: 4 MiB windows
387        assert_eq!(dynamic_window_size(2 * 1024 * 1024 * 1024), 4 * 1024 * 1024);
388    }
389}