Skip to main content

rypipe_core/
schema.rs

1//! Frozen schema for parallel streaming.
2//!
3//! A `FrozenSchema` captures the output column layout after discovery or
4//! explicit declaration.  Once yielded to the consumer it cannot change,
5//! which is the correctness guarantee that makes parallel streaming safe:
6//! chunk 1 cannot discover a column that chunk 7 later introduces.
7
8#[cfg(test)]
9use std::borrow::Cow;
10use std::hash::{Hash, Hasher};
11use std::sync::atomic::{AtomicU64, Ordering};
12use std::sync::{Arc, LazyLock, RwLock};
13
14use rustc_hash::{FxHashMap, FxHasher};
15
16use crate::plan::{ExecutionPlan, FieldType};
17
18/// Immutable, shared across all workers.  No lock, read-only after
19/// construction.
20#[derive(Clone, Debug)]
21pub struct FrozenSchema {
22    /// Output column names in final order.
23    names: Vec<Arc<str>>,
24    /// Raw input field name → output slot index.  `None` = dropped by plan.
25    /// Collapses `field_map` (rename) + `drop_fields` + `field_index` into
26    /// ONE lookup.
27    index: FxHashMap<Box<str>, Option<u32>>,
28    /// Output column types, parallel to `names`.
29    types: Vec<FieldType>,
30    /// `true` = full scan, `false` = sampled.  Affects the unknown-field
31    /// message.
32    exact: bool,
33}
34
35/// Options controlling schema discovery.
36pub struct DiscoveryOpts {
37    /// Files smaller than this are fully scanned (cheap, exact).
38    pub full_scan_threshold: u64,
39    /// Number of strided windows for large files.
40    pub windows: usize,
41    /// Bytes per window.
42    pub window_bytes: usize,
43}
44
45impl Default for DiscoveryOpts {
46    fn default() -> Self {
47        Self {
48            full_scan_threshold: 128 * 1024 * 1024, // 128 MiB
49            windows: 16,
50            window_bytes: 2 * 1024 * 1024, // 2 MiB
51        }
52    }
53}
54
55/// Maximum number of entries kept in the schema-discovery cache.  The cache
56/// is per-process and unbounded by default; cap it to avoid slow growth over
57/// long-running sessions with many distinct layouts.
58pub const SCHEMA_CACHE_CAP: usize = 128;
59
60type SchemaCacheKey = (u64, u64);
61type SchemaCacheValue = Arc<Vec<String>>;
62
63/// Global layout-keyed cache for discovered raw field-name order.
64/// Key is `(file_len, sample_hash)`; value is the raw order from a previous
65/// full discovery. The current plan is applied to the cached order on hit.
66pub static SCHEMA_CACHE: LazyLock<RwLock<FxHashMap<SchemaCacheKey, SchemaCacheValue>>> =
67    LazyLock::new(|| RwLock::new(FxHashMap::default()));
68
69/// Number of schema-discovery cache hits.
70pub static SCHEMA_CACHE_HITS: AtomicU64 = AtomicU64::new(0);
71/// Number of schema-discovery cache misses.
72pub static SCHEMA_CACHE_MISSES: AtomicU64 = AtomicU64::new(0);
73
74/// Compute a cheap layout signature for schema-discovery caching.
75/// Small files are hashed in full; large files hash several strided windows
76/// so that identical length + leading window is no longer enough to collide.
77pub fn layout_signature(bytes: &[u8], opts: &DiscoveryOpts) -> (u64, u64) {
78    let mut hasher = FxHasher::default();
79    if (bytes.len() as u64) < opts.full_scan_threshold {
80        bytes.hash(&mut hasher);
81    } else {
82        // Hash a few windows spread through the file.  This makes the key
83        // much less likely to collide for same-sized exports from the same
84        // template with different trailing columns.
85        let n = 4usize;
86        let w = opts.window_bytes.min(bytes.len());
87        for i in 0..n {
88            let start = (bytes.len() - w) * i / n.max(1);
89            bytes[start..start + w].hash(&mut hasher);
90        }
91    }
92    (bytes.len() as u64, hasher.finish())
93}
94
95/// Insert a discovered layout into the cache, evicting the oldest entry if
96/// the cache is at capacity.  `FxHashMap` does not track insertion order, so
97/// we evict an arbitrary key; for a bounded cache this is sufficient and
98/// avoids adding an LRU dependency.
99pub fn insert_schema_cache(sig: (u64, u64), order: Arc<Vec<String>>) {
100    let mut cache = SCHEMA_CACHE.write().unwrap();
101    if cache.len() >= SCHEMA_CACHE_CAP {
102        if let Some(oldest) = cache.keys().next().copied() {
103            cache.remove(&oldest);
104        }
105    }
106    cache.insert(sig, order);
107}
108
109/// Clear the discovery cache and reset its hit/miss counters.
110pub fn clear_schema_cache() {
111    SCHEMA_CACHE.write().unwrap().clear();
112    SCHEMA_CACHE_HITS.store(0, Ordering::Relaxed);
113    SCHEMA_CACHE_MISSES.store(0, Ordering::Relaxed);
114}
115
116/// Return the current cache hit and miss counts.
117pub fn schema_cache_stats() -> (u64, u64) {
118    (
119        SCHEMA_CACHE_HITS.load(Ordering::Relaxed),
120        SCHEMA_CACHE_MISSES.load(Ordering::Relaxed),
121    )
122}
123
124/// What to do when a field appears in the file but not in the schema.
125#[derive(Clone, Copy, Debug, PartialEq, Eq)]
126pub enum UnknownFieldPolicy {
127    /// Return an error (default).
128    Error,
129    /// Ignore the field, count occurrences, report at end.
130    Skip,
131}
132
133impl FrozenSchema {
134    /// Build from explicit column names and an execution plan.
135    ///
136    /// Declares the exact output columns.  Fields in the file not listed
137    /// here are governed by `UnknownFieldPolicy`.  Columns declared here
138    /// but absent from the file become all-null.
139    pub fn from_plan(names: &[&str], plan: &ExecutionPlan) -> Self {
140        let mut index = FxHashMap::default();
141        let mut types = Vec::with_capacity(names.len());
142
143        for (slot, &name) in names.iter().enumerate() {
144            // Collapse rename: if the plan renames `name` to something
145            // else, we still map the *raw* name to the slot.  The caller
146            // should pass the *output* names; renames are applied by the
147            // plan already.
148            let ty = plan.column_type(name);
149            types.push(ty);
150            index.insert(Box::from(name), Some(slot as u32));
151        }
152
153        FrozenSchema {
154            names: names.iter().map(|n| Arc::from(*n)).collect(),
155            index,
156            types,
157            exact: true,
158        }
159    }
160
161    /// Build from discovered names (file order, then document order within
162    /// a window).  Applies the execution plan's renames and drops.
163    pub fn from_discovered(names_in_order: &[String], plan: &ExecutionPlan) -> Self {
164        let mut index = FxHashMap::default();
165        let mut out_names = Vec::new();
166        let mut types = Vec::new();
167
168        for name in names_in_order {
169            // Apply rename
170            let resolved = plan.resolve_field(name);
171            match resolved {
172                Some(resolved_name) => {
173                    // Check if renamed name is dropped
174                    if plan.drop_fields.contains(resolved_name) {
175                        index.insert(Box::from(name.as_str()), None);
176                        continue;
177                    }
178                    let slot = out_names.len() as u32;
179                    let ty = plan.column_type(resolved_name);
180                    index.insert(Box::from(name.as_str()), Some(slot));
181                    out_names.push(Arc::from(resolved_name));
182                    types.push(ty);
183                }
184                None => {
185                    // Field is dropped by plan
186                    index.insert(Box::from(name.as_str()), None);
187                }
188            }
189        }
190
191        FrozenSchema {
192            names: out_names,
193            index,
194            types,
195            exact: false,
196        }
197    }
198
199    /// Number of output columns.
200    pub fn num_columns(&self) -> usize {
201        self.names.len()
202    }
203
204    /// Output column names in order.
205    pub fn column_names(&self) -> &[Arc<str>] {
206        &self.names
207    }
208
209    /// Output column types, parallel to `column_names`.
210    pub fn column_types(&self) -> &[FieldType] {
211        &self.types
212    }
213
214    /// Resolve a raw field name to its output slot, or `None` if dropped.
215    #[inline]
216    pub fn resolve(&self, raw_name: &str) -> Option<u32> {
217        self.index.get(raw_name).copied().flatten()
218    }
219
220    /// Whether the schema was derived from a full scan (exact) or sampling.
221    pub fn is_exact(&self) -> bool {
222        self.exact
223    }
224}
225
226#[cfg(test)]
227mod tests {
228    use super::*;
229
230    #[test]
231    fn test_from_plan_explicit() {
232        let plan = ExecutionPlan::new();
233        let schema = FrozenSchema::from_plan(&["A", "B", "C"], &plan);
234        assert_eq!(schema.num_columns(), 3);
235        assert_eq!(schema.resolve("A"), Some(0));
236        assert_eq!(schema.resolve("B"), Some(1));
237        assert_eq!(schema.resolve("C"), Some(2));
238        assert_eq!(schema.resolve("D"), None);
239        assert!(schema.is_exact());
240    }
241
242    #[test]
243    fn test_from_plan_with_drop() {
244        let mut plan = ExecutionPlan::new();
245        plan.drop_fields.insert("B".to_string());
246        let schema = FrozenSchema::from_plan(&["A", "B", "C"], &plan);
247        // from_plan does NOT drop; it declares the schema as-is.
248        // Drops are handled at parse time via resolve() → None.
249        assert_eq!(schema.num_columns(), 3);
250    }
251
252    #[test]
253    fn test_from_discovered_with_rename() {
254        let mut plan = ExecutionPlan::new();
255        plan.field_map.insert("old".to_string(), "new".to_string());
256        let names = vec!["old".to_string(), "keep".to_string()];
257        let schema = FrozenSchema::from_discovered(&names, &plan);
258        assert_eq!(schema.num_columns(), 2);
259        assert_eq!(schema.resolve("old"), Some(0)); // slot 0 = "new"
260        assert_eq!(schema.resolve("keep"), Some(1));
261        assert_eq!(schema.column_names()[0].as_ref(), "new");
262    }
263
264    #[test]
265    fn test_from_discovered_with_drop() {
266        let mut plan = ExecutionPlan::new();
267        plan.drop_fields.insert("drop_me".to_string());
268        let names = vec!["keep".to_string(), "drop_me".to_string()];
269        let schema = FrozenSchema::from_discovered(&names, &plan);
270        assert_eq!(schema.num_columns(), 1);
271        assert_eq!(schema.resolve("keep"), Some(0));
272        assert_eq!(schema.resolve("drop_me"), None);
273    }
274
275    #[test]
276    fn test_deterministic_order() {
277        let plan = ExecutionPlan::new();
278        let names = vec!["C".to_string(), "A".to_string(), "B".to_string()];
279        let s1 = FrozenSchema::from_discovered(&names, &plan);
280        let s2 = FrozenSchema::from_discovered(&names, &plan);
281        assert_eq!(s1.column_names(), s2.column_names());
282    }
283
284    #[test]
285    fn test_ensure_schema_preserves_existing_columns() {
286        use crate::decoder::ColumnarSink;
287        use crate::engine::TableBuilder;
288        let plan = ExecutionPlan::new();
289        let schema = FrozenSchema::from_plan(&["X", "Y", "Z"], &plan);
290        let mut builder = TableBuilder::with_plan(100, Arc::new(plan));
291        // Pre-add column "X" with a push.
292        builder.begin_row();
293        builder.put_field("X", crate::value::Value::Str(Cow::Borrowed("hello")));
294        builder.end_row();
295        let rows_before = builder.num_rows();
296        let cols_before = builder.num_columns();
297        // ensure_schema should add Y and Z, but not duplicate X.
298        builder.ensure_schema(&schema).unwrap();
299        assert_eq!(builder.num_rows(), rows_before);
300        assert_eq!(builder.num_columns(), cols_before + 2);
301        assert!(builder.column_names().contains(&"X".to_string()));
302        assert!(builder.column_names().contains(&"Y".to_string()));
303        assert!(builder.column_names().contains(&"Z".to_string()));
304    }
305}