Skip to main content

rypipe_core/parallel_stream/
discovery.rs

1use super::*;
2
3#[cfg(feature = "mmap")]
4pub(super) fn discover_schema_for_file<P: RecordParser>(
5    input: &crate::input::MmapHandle,
6    splitter: &dyn Splitter,
7    parser: &P,
8    plan: &ExecutionPlan,
9    budget: MemoryBudget,
10) -> Result<FrozenSchema> {
11    let bytes = &input.mmap[..];
12    let opts = DiscoveryOpts::default();
13    use std::hash::{Hash, Hasher};
14    let mut file = input.file.try_clone()?;
15    let mut buffer = Vec::new();
16    let allowance = budget.share(4);
17    let mut hasher = rustc_hash::FxHasher::default();
18    for range in crate::schema::signature_chunks(bytes.len(), &opts) {
19        allowance.check(range.len())?;
20        buffer.resize(range.len(), 0);
21        allowance.check(buffer.capacity())?;
22        file.seek(SeekFrom::Start(range.start as u64))?;
23        file.read_exact(&mut buffer)?;
24        buffer.hash(&mut hasher);
25    }
26    let signature = (bytes.len() as u64, hasher.finish());
27    let cached = crate::schema::SCHEMA_CACHE
28        .read()
29        .unwrap_or_else(|e| e.into_inner())
30        .get(&signature)
31        .cloned();
32    if let Some(order) = cached {
33        crate::schema::SCHEMA_CACHE_HITS.fetch_add(1, Ordering::Relaxed);
34        return Ok(FrozenSchema::from_discovered(&order, plan));
35    }
36    crate::schema::SCHEMA_CACHE_MISSES.fetch_add(1, Ordering::Relaxed);
37    let start = std::time::Instant::now();
38    let chunk_size = (budget.bytes() / 8).max(1);
39    let count = bytes
40        .len()
41        .div_ceil(chunk_size)
42        .clamp(1, crate::MAX_SPLIT_CHUNKS);
43    let points = splitter.find_split_points(bytes, count);
44    let ranges = crate::decoder::split_points_to_ranges(&points, bytes.len());
45    let planning_bytes = points
46        .capacity()
47        .saturating_mul(std::mem::size_of::<usize>())
48        .saturating_add(ranges.capacity() * std::mem::size_of::<std::ops::Range<usize>>());
49    allowance.check(planning_bytes)?;
50    let windows = crate::schema::dynamic_window_count(bytes.len() as u64);
51    let window_bytes = crate::schema::dynamic_window_size(bytes.len() as u64);
52    let mut sink = DiscoverySink::new();
53    for range in ranges {
54        if windows > 0
55            && !(0..windows).any(|i| {
56                let offset = bytes.len() / windows * i;
57                range.start < offset.saturating_add(window_bytes) && range.end > offset
58            })
59            && range.end <= bytes.len().saturating_sub(window_bytes)
60        {
61            continue;
62        }
63        allowance.check(range.len().saturating_add(planning_bytes))?;
64        buffer.resize(range.len(), 0);
65        file.seek(SeekFrom::Start(range.start as u64))?;
66        file.read_exact(&mut buffer)?;
67        let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
68            parser.parse_chunk_generic(&buffer, &mut sink)
69        }))
70        .map_err(|_| crate::Error::Parser("parser panicked during schema discovery".into()))?;
71        let order = sink.order.borrow();
72        let seen = sink.seen.borrow();
73        let storage = order.capacity() * std::mem::size_of::<String>()
74            + order.iter().map(String::capacity).sum::<usize>()
75            + seen.capacity() * (std::mem::size_of::<Box<str>>() + 1)
76            + seen.iter().map(|name| name.len()).sum::<usize>();
77        allowance.check(
78            storage
79                .saturating_add(buffer.capacity())
80                .saturating_add(planning_bytes),
81        )?;
82    }
83    let order = sink.into_order();
84    let schema = FrozenSchema::from_discovered(&order, plan);
85    if !order.is_empty() {
86        crate::schema::insert_schema_cache(signature, Arc::new(order));
87    }
88    DISCOVERY_NS.store(start.elapsed().as_nanos() as u64, Ordering::Relaxed);
89    Ok(schema)
90}
91
92static DISCOVERY_NS: AtomicU64 = AtomicU64::new(0);
93
94/// Return the elapsed time (in nanoseconds) spent in schema discovery.
95pub fn discovery_profile() -> u64 {
96    DISCOVERY_NS.load(Ordering::Relaxed)
97}
98/// Reset the discovery timer to zero.
99pub fn reset_discovery_profile() {
100    DISCOVERY_NS.store(0, Ordering::Relaxed);
101}
102
103/// Options for parallel streaming.
104pub struct ParallelStreamOpts {
105    /// Number of worker threads.
106    pub threads: usize,
107    /// Whether to preserve row order (default: true).
108    pub ordered: bool,
109    /// Reorder memory limit in multiples of the parsing budget (0 = threads).
110    /// Ordered reads return an error if this limit is exceeded.
111    /// Workers may run at most this many chunks ahead of the next output chunk.
112    pub max_reorder: usize,
113    /// Explicit schema.  If `Some`, no discovery pass is needed.
114    /// Workers pre-size all columns from construction.
115    pub schema: Option<FrozenSchema>,
116}
117
118impl Default for ParallelStreamOpts {
119    fn default() -> Self {
120        Self {
121            threads: std::thread::available_parallelism()
122                .map(|n| n.get())
123                .unwrap_or(1),
124            ordered: true,
125            max_reorder: 0, // 0 = use threads value
126            schema: None,
127        }
128    }
129}
130
131/// Discovery sink that collects raw field names in encounter order.
132/// `needs_value=false` keeps the scanner in locate-only mode (no value
133/// extraction), but we still need `resolve()` calls, so `needs_resolve`
134/// stays true. Capture happens in `resolve()` via interior mutability.
135struct DiscoverySink {
136    seen: std::cell::RefCell<rustc_hash::FxHashSet<Box<str>>>,
137    order: std::cell::RefCell<Vec<String>>,
138}
139
140impl DiscoverySink {
141    fn new() -> Self {
142        Self {
143            seen: std::cell::RefCell::new(rustc_hash::FxHashSet::default()),
144            order: std::cell::RefCell::new(Vec::new()),
145        }
146    }
147    fn into_order(self) -> Vec<String> {
148        self.order.into_inner()
149    }
150}
151
152impl crate::decoder::ColumnarSink for DiscoverySink {
153    #[inline]
154    fn begin_row(&mut self) {}
155    #[inline]
156    fn put_field(&mut self, name: &str, _value: crate::value::Value<'_>) {
157        // Full-value fallback (if needs_value were true); keep for completeness.
158        let mut seen = self.seen.borrow_mut();
159        if seen.insert(Box::from(name)) {
160            self.order.borrow_mut().push(name.to_string());
161        }
162    }
163    #[inline]
164    fn end_row(&mut self) {}
165    #[inline]
166    fn wants(&self, _name: &str) -> bool {
167        true
168    }
169    #[inline]
170    fn resolve<'a>(&'a self, name: &'a str) -> Option<&'a str> {
171        // Locate-only path calls `resolve` without `put_field`; capture here.
172        let mut seen = self.seen.borrow_mut();
173        if seen.insert(Box::from(name)) {
174            self.order.borrow_mut().push(name.to_string());
175        }
176        Some(name)
177    }
178    #[inline]
179    fn needs_value(&self) -> bool {
180        false
181    }
182    // needs_resolve defaults to true → scanner calls resolve() for each field.
183    fn finish(&mut self) -> crate::Result<arrow::record_batch::RecordBatch> {
184        Ok(arrow::record_batch::RecordBatch::new_empty(
185            std::sync::Arc::new(arrow::datatypes::Schema::empty()),
186        ))
187    }
188}
189
190pub(crate) fn discover_schema<P: crate::decoder::RecordParser>(
191    bytes: &[u8],
192    parser: &P,
193    plan: &crate::plan::ExecutionPlan,
194    splitter: &dyn crate::decoder::Splitter,
195) -> (FrozenSchema, Vec<String>) {
196    let t0 = std::time::Instant::now();
197    // Explicit schema already handled by caller; this is auto-discovery.
198    let opts = DiscoveryOpts::default();
199
200    let file_size = bytes.len() as u64;
201
202    // Dynamic sizing: scale window count and size by file size
203    let n = crate::schema::dynamic_window_count(file_size);
204    let wbytes = crate::schema::dynamic_window_size(file_size);
205
206    let order: Vec<String> = if file_size < opts.full_scan_threshold || n == 0 {
207        // Small file or dynamic sizing says full scan
208        let mut sink = DiscoverySink::new();
209        let _ = parser.parse_chunk_generic(bytes, &mut sink);
210        sink.into_order()
211    } else {
212        use rayon::prelude::*;
213        // Parallelise windows: n×wbytes independent parses
214        let per_window: Vec<Vec<String>> = (0..n)
215            .into_par_iter()
216            .map(|i| {
217                let start = (file_size * i as u64 / n as u64) as usize;
218                let end = (start + wbytes).min(bytes.len());
219                if start >= end {
220                    return Vec::new();
221                }
222                let mut sink = DiscoverySink::new();
223                let slice = &bytes[start..end];
224                let _ = parser.parse_chunk_generic(slice, &mut sink);
225                sink.into_order()
226            })
227            .collect();
228        // Merge in file order, deduplicating, so global order approximates file order.
229        let mut seen = rustc_hash::FxHashSet::<String>::default();
230        let mut merged = Vec::new();
231        for mut v in per_window {
232            for name in v.drain(..) {
233                if seen.insert(name.clone()) {
234                    merged.push(name);
235                }
236            }
237        }
238        // Always scan the tail to catch late-appearing columns
239        if opts.always_scan_tail {
240            let tail_start = bytes.len().saturating_sub(wbytes);
241            let mut tail_sink = DiscoverySink::new();
242            let _ = parser.parse_chunk_generic(&bytes[tail_start..], &mut tail_sink);
243            for name in tail_sink.into_order() {
244                if seen.insert(name.clone()) {
245                    merged.push(name);
246                }
247            }
248        }
249        if merged.is_empty() && !bytes.is_empty() {
250            let mut sink = DiscoverySink::new();
251            let _ = parser.parse_chunk_generic(bytes, &mut sink);
252            sink.into_order()
253        } else {
254            merged
255        }
256    };
257    let elapsed = t0.elapsed().as_nanos() as u64;
258    DISCOVERY_NS.store(elapsed, Ordering::Relaxed);
259    let _ = splitter; // keep splitter in signature for future alignment
260    let schema = if order.is_empty() {
261        FrozenSchema::from_plan(&[], plan)
262    } else {
263        FrozenSchema::from_discovered(&order, plan)
264    };
265    (schema, order)
266}
267
268/// Public helper for batch workloads: discover the schema once and reuse.
269/// Example:
270/// ```rust,no_run
271/// # use rypipe_core::{discover_schema_for_bytes, ExecutionPlan, RecordParser, Splitter};
272/// fn example<S: Splitter, P: RecordParser>(bytes: &[u8], splitter: &S, parser: &P) {
273///     let plan = ExecutionPlan::new();
274///     let schema = discover_schema_for_bytes(bytes, splitter, parser, &plan);
275///     let _reused = schema.clone();
276/// }
277/// ```
278pub fn discover_schema_for_bytes<P: crate::decoder::RecordParser>(
279    bytes: &[u8],
280    splitter: &dyn crate::decoder::Splitter,
281    parser: &P,
282    plan: &crate::plan::ExecutionPlan,
283) -> FrozenSchema {
284    if !plan.schema_order.is_empty() {
285        let names: Vec<&str> = plan.schema_order.iter().map(|s| s.as_str()).collect();
286        return FrozenSchema::from_partial_plan(&names, plan);
287    }
288    let opts = DiscoveryOpts::default();
289    let sig = crate::schema::layout_signature(bytes, &opts);
290    {
291        let cache = crate::schema::SCHEMA_CACHE
292            .read()
293            .unwrap_or_else(|e| e.into_inner());
294        if let Some(order) = cache.get(&sig) {
295            crate::schema::SCHEMA_CACHE_HITS.fetch_add(1, Ordering::Relaxed);
296            return FrozenSchema::from_discovered(order, plan);
297        }
298    }
299    crate::schema::SCHEMA_CACHE_MISSES.fetch_add(1, Ordering::Relaxed);
300    let (schema, order) = discover_schema(bytes, parser, plan, splitter);
301    if !order.is_empty() {
302        crate::schema::insert_schema_cache(sig, Arc::new(order));
303    }
304    schema
305}