Skip to main content

rypipe_core/parallel_stream/
discovery.rs

1use super::*;
2
3#[cfg(feature = "mmap")]
4pub(super) fn discover_schema_for_file<P: RecordParser>(
5    input: &crate::input::MmapHandle,
6    splitter: &dyn Splitter,
7    parser: &P,
8    plan: &ExecutionPlan,
9    budget: MemoryBudget,
10) -> Result<FrozenSchema> {
11    let bytes = &input.mmap[..];
12    let opts = DiscoveryOpts::default();
13    use std::hash::{Hash, Hasher};
14    let mut file = input.file.try_clone()?;
15    let mut buffer = Vec::new();
16    let allowance = budget.share(4);
17    let mut hasher = rustc_hash::FxHasher::default();
18    for range in crate::schema::signature_chunks(bytes.len(), &opts) {
19        allowance.check(range.len())?;
20        buffer.resize(range.len(), 0);
21        allowance.check(buffer.capacity())?;
22        file.seek(SeekFrom::Start(range.start as u64))?;
23        file.read_exact(&mut buffer)?;
24        buffer.hash(&mut hasher);
25    }
26    let signature = (bytes.len() as u64, hasher.finish());
27    let cached = crate::schema::SCHEMA_CACHE
28        .read()
29        .unwrap_or_else(|e| e.into_inner())
30        .get(&signature)
31        .cloned();
32    if let Some(order) = cached {
33        crate::schema::SCHEMA_CACHE_HITS.fetch_add(1, Ordering::Relaxed);
34        return Ok(FrozenSchema::from_discovered(&order, plan));
35    }
36    crate::schema::SCHEMA_CACHE_MISSES.fetch_add(1, Ordering::Relaxed);
37    let start = std::time::Instant::now();
38    let chunk_size = (budget.bytes() / 8).max(1);
39    let count = bytes
40        .len()
41        .div_ceil(chunk_size)
42        .clamp(1, crate::MAX_SPLIT_CHUNKS);
43    let points = splitter.find_split_points_with(
44        bytes,
45        count,
46        plan.min_chunk_bytes
47            .unwrap_or(crate::decoder::MIN_CHUNK_BYTES),
48    );
49    let ranges = crate::decoder::split_points_to_ranges(&points, bytes.len());
50    let planning_bytes = points
51        .capacity()
52        .saturating_mul(std::mem::size_of::<usize>())
53        .saturating_add(ranges.capacity() * std::mem::size_of::<std::ops::Range<usize>>());
54    allowance.check(planning_bytes)?;
55    let windows = crate::schema::dynamic_window_count(bytes.len() as u64);
56    let window_bytes = crate::schema::dynamic_window_size(bytes.len() as u64);
57    let mut sink = DiscoverySink::new();
58    for range in ranges {
59        if windows > 0
60            && !(0..windows).any(|i| {
61                let offset = bytes.len() / windows * i;
62                range.start < offset.saturating_add(window_bytes) && range.end > offset
63            })
64            && range.end <= bytes.len().saturating_sub(window_bytes)
65        {
66            continue;
67        }
68        allowance.check(range.len().saturating_add(planning_bytes))?;
69        buffer.resize(range.len(), 0);
70        file.seek(SeekFrom::Start(range.start as u64))?;
71        file.read_exact(&mut buffer)?;
72        let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
73            parser.parse_chunk_generic(&buffer, &mut sink)
74        }))
75        .map_err(|_| crate::Error::Parser("parser panicked during schema discovery".into()))?;
76        let order = sink.order.borrow();
77        let seen = sink.seen.borrow();
78        let storage = order.capacity() * std::mem::size_of::<String>()
79            + order.iter().map(String::capacity).sum::<usize>()
80            + seen.capacity() * (std::mem::size_of::<Box<str>>() + 1)
81            + seen.iter().map(|name| name.len()).sum::<usize>();
82        allowance.check(
83            storage
84                .saturating_add(buffer.capacity())
85                .saturating_add(planning_bytes),
86        )?;
87    }
88    let order = sink.into_order();
89    let schema = FrozenSchema::from_discovered(&order, plan);
90    if !order.is_empty() {
91        crate::schema::insert_schema_cache(signature, Arc::new(order));
92    }
93    DISCOVERY_NS.store(start.elapsed().as_nanos() as u64, Ordering::Relaxed);
94    Ok(schema)
95}
96
97static DISCOVERY_NS: AtomicU64 = AtomicU64::new(0);
98
99/// Return the elapsed time (in nanoseconds) spent in schema discovery.
100pub fn discovery_profile() -> u64 {
101    DISCOVERY_NS.load(Ordering::Relaxed)
102}
103/// Reset the discovery timer to zero.
104pub fn reset_discovery_profile() {
105    DISCOVERY_NS.store(0, Ordering::Relaxed);
106}
107
108/// Options for parallel streaming.
109pub struct ParallelStreamOpts {
110    /// Number of worker threads.
111    pub threads: usize,
112    /// Whether to preserve row order (default: true).
113    pub ordered: bool,
114    /// Reorder memory limit in multiples of the parsing budget (0 = threads).
115    /// Ordered reads return an error if this limit is exceeded.
116    /// Workers may run at most this many chunks ahead of the next output chunk.
117    pub max_reorder: usize,
118    /// Explicit schema.  If `Some`, no discovery pass is needed.
119    /// Workers pre-size all columns from construction.
120    pub schema: Option<FrozenSchema>,
121}
122
123impl Default for ParallelStreamOpts {
124    fn default() -> Self {
125        Self {
126            threads: std::thread::available_parallelism()
127                .map(|n| n.get())
128                .unwrap_or(1),
129            ordered: true,
130            max_reorder: 0, // 0 = use threads value
131            schema: None,
132        }
133    }
134}
135
136/// Discovery sink that collects raw field names in encounter order.
137/// `needs_value=false` keeps the scanner in locate-only mode (no value
138/// extraction), but we still need `resolve()` calls, so `needs_resolve`
139/// stays true. Capture happens in `resolve()` via interior mutability.
140struct DiscoverySink {
141    seen: std::cell::RefCell<rustc_hash::FxHashSet<Box<str>>>,
142    order: std::cell::RefCell<Vec<String>>,
143}
144
145impl DiscoverySink {
146    fn new() -> Self {
147        Self {
148            seen: std::cell::RefCell::new(rustc_hash::FxHashSet::default()),
149            order: std::cell::RefCell::new(Vec::new()),
150        }
151    }
152    fn into_order(self) -> Vec<String> {
153        self.order.into_inner()
154    }
155}
156
157impl crate::decoder::ColumnarSink for DiscoverySink {
158    #[inline]
159    fn begin_row(&mut self) {}
160    #[inline]
161    fn put_field(&mut self, name: &str, _value: crate::value::Value<'_>) {
162        // Full-value fallback (if needs_value were true); keep for completeness.
163        let mut seen = self.seen.borrow_mut();
164        if seen.insert(Box::from(name)) {
165            self.order.borrow_mut().push(name.to_string());
166        }
167    }
168    #[inline]
169    fn end_row(&mut self) {}
170    #[inline]
171    fn wants(&self, _name: &str) -> bool {
172        true
173    }
174    #[inline]
175    fn resolve<'a>(&'a self, name: &'a str) -> Option<&'a str> {
176        // Locate-only path calls `resolve` without `put_field`; capture here.
177        let mut seen = self.seen.borrow_mut();
178        if seen.insert(Box::from(name)) {
179            self.order.borrow_mut().push(name.to_string());
180        }
181        Some(name)
182    }
183    #[inline]
184    fn needs_value(&self) -> bool {
185        false
186    }
187    // needs_resolve defaults to true → scanner calls resolve() for each field.
188    fn finish(&mut self) -> crate::Result<arrow::record_batch::RecordBatch> {
189        Ok(arrow::record_batch::RecordBatch::new_empty(
190            std::sync::Arc::new(arrow::datatypes::Schema::empty()),
191        ))
192    }
193}
194
195pub(crate) fn discover_schema<P: crate::decoder::RecordParser>(
196    bytes: &[u8],
197    parser: &P,
198    plan: &crate::plan::ExecutionPlan,
199    splitter: &dyn crate::decoder::Splitter,
200) -> (FrozenSchema, Vec<String>) {
201    let t0 = std::time::Instant::now();
202    // Explicit schema already handled by caller; this is auto-discovery.
203    let opts = DiscoveryOpts::default();
204
205    let file_size = bytes.len() as u64;
206
207    // Dynamic sizing: scale window count and size by file size
208    let n = crate::schema::dynamic_window_count(file_size);
209    let wbytes = crate::schema::dynamic_window_size(file_size);
210
211    let order: Vec<String> = if file_size < opts.full_scan_threshold || n == 0 {
212        // Small file or dynamic sizing says full scan
213        let mut sink = DiscoverySink::new();
214        let _ = parser.parse_chunk_generic(bytes, &mut sink);
215        sink.into_order()
216    } else {
217        use rayon::prelude::*;
218        // Parallelise windows: n×wbytes independent parses
219        let per_window: Vec<Vec<String>> = (0..n)
220            .into_par_iter()
221            .map(|i| {
222                let start = (file_size * i as u64 / n as u64) as usize;
223                let end = (start + wbytes).min(bytes.len());
224                if start >= end {
225                    return Vec::new();
226                }
227                let mut sink = DiscoverySink::new();
228                let slice = &bytes[start..end];
229                let _ = parser.parse_chunk_generic(slice, &mut sink);
230                sink.into_order()
231            })
232            .collect();
233        // Merge in file order, deduplicating, so global order approximates file order.
234        let mut seen = rustc_hash::FxHashSet::<String>::default();
235        let mut merged = Vec::new();
236        for mut v in per_window {
237            for name in v.drain(..) {
238                if seen.insert(name.clone()) {
239                    merged.push(name);
240                }
241            }
242        }
243        // Always scan the tail to catch late-appearing columns
244        if opts.always_scan_tail {
245            let tail_start = bytes.len().saturating_sub(wbytes);
246            let mut tail_sink = DiscoverySink::new();
247            let _ = parser.parse_chunk_generic(&bytes[tail_start..], &mut tail_sink);
248            for name in tail_sink.into_order() {
249                if seen.insert(name.clone()) {
250                    merged.push(name);
251                }
252            }
253        }
254        if merged.is_empty() && !bytes.is_empty() {
255            let mut sink = DiscoverySink::new();
256            let _ = parser.parse_chunk_generic(bytes, &mut sink);
257            sink.into_order()
258        } else {
259            merged
260        }
261    };
262    let elapsed = t0.elapsed().as_nanos() as u64;
263    DISCOVERY_NS.store(elapsed, Ordering::Relaxed);
264    let _ = splitter; // keep splitter in signature for future alignment
265    let schema = if order.is_empty() {
266        FrozenSchema::from_plan(&[], plan)
267    } else {
268        FrozenSchema::from_discovered(&order, plan)
269    };
270    (schema, order)
271}
272
273/// Public helper for batch workloads: discover the schema once and reuse.
274/// Example:
275/// ```rust,no_run
276/// # use rypipe_core::{discover_schema_for_bytes, ExecutionPlan, RecordParser, Splitter};
277/// fn example<S: Splitter, P: RecordParser>(bytes: &[u8], splitter: &S, parser: &P) {
278///     let plan = ExecutionPlan::new();
279///     let schema = discover_schema_for_bytes(bytes, splitter, parser, &plan);
280///     let _reused = schema.clone();
281/// }
282/// ```
283pub fn discover_schema_for_bytes<P: crate::decoder::RecordParser>(
284    bytes: &[u8],
285    splitter: &dyn crate::decoder::Splitter,
286    parser: &P,
287    plan: &crate::plan::ExecutionPlan,
288) -> FrozenSchema {
289    if !plan.schema_order.is_empty() {
290        let names: Vec<&str> = plan.schema_order.iter().map(|s| s.as_str()).collect();
291        return FrozenSchema::from_partial_plan(&names, plan);
292    }
293    let opts = DiscoveryOpts::default();
294    let sig = crate::schema::layout_signature(bytes, &opts);
295    {
296        let cache = crate::schema::SCHEMA_CACHE
297            .read()
298            .unwrap_or_else(|e| e.into_inner());
299        if let Some(order) = cache.get(&sig) {
300            crate::schema::SCHEMA_CACHE_HITS.fetch_add(1, Ordering::Relaxed);
301            return FrozenSchema::from_discovered(order, plan);
302        }
303    }
304    crate::schema::SCHEMA_CACHE_MISSES.fetch_add(1, Ordering::Relaxed);
305    let (schema, order) = discover_schema(bytes, parser, plan, splitter);
306    if !order.is_empty() {
307        crate::schema::insert_schema_cache(sig, Arc::new(order));
308    }
309    schema
310}