rypipe_core/parallel_stream/
discovery.rs1use super::*;
2
3#[cfg(feature = "mmap")]
4pub(super) fn discover_schema_for_file<P: RecordParser>(
5 input: &crate::input::MmapHandle,
6 splitter: &dyn Splitter,
7 parser: &P,
8 plan: &ExecutionPlan,
9 budget: MemoryBudget,
10) -> Result<FrozenSchema> {
11 let bytes = &input.mmap[..];
12 let opts = DiscoveryOpts::default();
13 use std::hash::{Hash, Hasher};
14 let mut file = input.file.try_clone()?;
15 let mut buffer = Vec::new();
16 let allowance = budget.share(4);
17 let mut hasher = rustc_hash::FxHasher::default();
18 for range in crate::schema::signature_chunks(bytes.len(), &opts) {
19 allowance.check(range.len())?;
20 buffer.resize(range.len(), 0);
21 allowance.check(buffer.capacity())?;
22 file.seek(SeekFrom::Start(range.start as u64))?;
23 file.read_exact(&mut buffer)?;
24 buffer.hash(&mut hasher);
25 }
26 let signature = (bytes.len() as u64, hasher.finish());
27 let cached = crate::schema::SCHEMA_CACHE
28 .read()
29 .unwrap_or_else(|e| e.into_inner())
30 .get(&signature)
31 .cloned();
32 if let Some(order) = cached {
33 crate::schema::SCHEMA_CACHE_HITS.fetch_add(1, Ordering::Relaxed);
34 return Ok(FrozenSchema::from_discovered(&order, plan));
35 }
36 crate::schema::SCHEMA_CACHE_MISSES.fetch_add(1, Ordering::Relaxed);
37 let start = std::time::Instant::now();
38 let chunk_size = (budget.bytes() / 8).max(1);
39 let count = bytes
40 .len()
41 .div_ceil(chunk_size)
42 .clamp(1, crate::MAX_SPLIT_CHUNKS);
43 let points = splitter.find_split_points_with(
44 bytes,
45 count,
46 plan.min_chunk_bytes
47 .unwrap_or(crate::decoder::MIN_CHUNK_BYTES),
48 );
49 let ranges = crate::decoder::split_points_to_ranges(&points, bytes.len());
50 let planning_bytes = points
51 .capacity()
52 .saturating_mul(std::mem::size_of::<usize>())
53 .saturating_add(ranges.capacity() * std::mem::size_of::<std::ops::Range<usize>>());
54 allowance.check(planning_bytes)?;
55 let windows = crate::schema::dynamic_window_count(bytes.len() as u64);
56 let window_bytes = crate::schema::dynamic_window_size(bytes.len() as u64);
57 let mut sink = DiscoverySink::new();
58 for range in ranges {
59 if windows > 0
60 && !(0..windows).any(|i| {
61 let offset = bytes.len() / windows * i;
62 range.start < offset.saturating_add(window_bytes) && range.end > offset
63 })
64 && range.end <= bytes.len().saturating_sub(window_bytes)
65 {
66 continue;
67 }
68 allowance.check(range.len().saturating_add(planning_bytes))?;
69 buffer.resize(range.len(), 0);
70 file.seek(SeekFrom::Start(range.start as u64))?;
71 file.read_exact(&mut buffer)?;
72 let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
73 parser.parse_chunk_generic(&buffer, &mut sink)
74 }))
75 .map_err(|_| crate::Error::Parser("parser panicked during schema discovery".into()))?;
76 let order = sink.order.borrow();
77 let seen = sink.seen.borrow();
78 let storage = order.capacity() * std::mem::size_of::<String>()
79 + order.iter().map(String::capacity).sum::<usize>()
80 + seen.capacity() * (std::mem::size_of::<Box<str>>() + 1)
81 + seen.iter().map(|name| name.len()).sum::<usize>();
82 allowance.check(
83 storage
84 .saturating_add(buffer.capacity())
85 .saturating_add(planning_bytes),
86 )?;
87 }
88 let order = sink.into_order();
89 let schema = FrozenSchema::from_discovered(&order, plan);
90 if !order.is_empty() {
91 crate::schema::insert_schema_cache(signature, Arc::new(order));
92 }
93 DISCOVERY_NS.store(start.elapsed().as_nanos() as u64, Ordering::Relaxed);
94 Ok(schema)
95}
96
97static DISCOVERY_NS: AtomicU64 = AtomicU64::new(0);
98
99pub fn discovery_profile() -> u64 {
101 DISCOVERY_NS.load(Ordering::Relaxed)
102}
103pub fn reset_discovery_profile() {
105 DISCOVERY_NS.store(0, Ordering::Relaxed);
106}
107
108pub struct ParallelStreamOpts {
110 pub threads: usize,
112 pub ordered: bool,
114 pub max_reorder: usize,
118 pub schema: Option<FrozenSchema>,
121}
122
123impl Default for ParallelStreamOpts {
124 fn default() -> Self {
125 Self {
126 threads: std::thread::available_parallelism()
127 .map(|n| n.get())
128 .unwrap_or(1),
129 ordered: true,
130 max_reorder: 0, schema: None,
132 }
133 }
134}
135
136struct DiscoverySink {
141 seen: std::cell::RefCell<rustc_hash::FxHashSet<Box<str>>>,
142 order: std::cell::RefCell<Vec<String>>,
143}
144
145impl DiscoverySink {
146 fn new() -> Self {
147 Self {
148 seen: std::cell::RefCell::new(rustc_hash::FxHashSet::default()),
149 order: std::cell::RefCell::new(Vec::new()),
150 }
151 }
152 fn into_order(self) -> Vec<String> {
153 self.order.into_inner()
154 }
155}
156
157impl crate::decoder::ColumnarSink for DiscoverySink {
158 #[inline]
159 fn begin_row(&mut self) {}
160 #[inline]
161 fn put_field(&mut self, name: &str, _value: crate::value::Value<'_>) {
162 let mut seen = self.seen.borrow_mut();
164 if seen.insert(Box::from(name)) {
165 self.order.borrow_mut().push(name.to_string());
166 }
167 }
168 #[inline]
169 fn end_row(&mut self) {}
170 #[inline]
171 fn wants(&self, _name: &str) -> bool {
172 true
173 }
174 #[inline]
175 fn resolve<'a>(&'a self, name: &'a str) -> Option<&'a str> {
176 let mut seen = self.seen.borrow_mut();
178 if seen.insert(Box::from(name)) {
179 self.order.borrow_mut().push(name.to_string());
180 }
181 Some(name)
182 }
183 #[inline]
184 fn needs_value(&self) -> bool {
185 false
186 }
187 fn finish(&mut self) -> crate::Result<arrow::record_batch::RecordBatch> {
189 Ok(arrow::record_batch::RecordBatch::new_empty(
190 std::sync::Arc::new(arrow::datatypes::Schema::empty()),
191 ))
192 }
193}
194
195pub(crate) fn discover_schema<P: crate::decoder::RecordParser>(
196 bytes: &[u8],
197 parser: &P,
198 plan: &crate::plan::ExecutionPlan,
199 splitter: &dyn crate::decoder::Splitter,
200) -> (FrozenSchema, Vec<String>) {
201 let t0 = std::time::Instant::now();
202 let opts = DiscoveryOpts::default();
204
205 let file_size = bytes.len() as u64;
206
207 let n = crate::schema::dynamic_window_count(file_size);
209 let wbytes = crate::schema::dynamic_window_size(file_size);
210
211 let order: Vec<String> = if file_size < opts.full_scan_threshold || n == 0 {
212 let mut sink = DiscoverySink::new();
214 let _ = parser.parse_chunk_generic(bytes, &mut sink);
215 sink.into_order()
216 } else {
217 use rayon::prelude::*;
218 let per_window: Vec<Vec<String>> = (0..n)
220 .into_par_iter()
221 .map(|i| {
222 let start = (file_size * i as u64 / n as u64) as usize;
223 let end = (start + wbytes).min(bytes.len());
224 if start >= end {
225 return Vec::new();
226 }
227 let mut sink = DiscoverySink::new();
228 let slice = &bytes[start..end];
229 let _ = parser.parse_chunk_generic(slice, &mut sink);
230 sink.into_order()
231 })
232 .collect();
233 let mut seen = rustc_hash::FxHashSet::<String>::default();
235 let mut merged = Vec::new();
236 for mut v in per_window {
237 for name in v.drain(..) {
238 if seen.insert(name.clone()) {
239 merged.push(name);
240 }
241 }
242 }
243 if opts.always_scan_tail {
245 let tail_start = bytes.len().saturating_sub(wbytes);
246 let mut tail_sink = DiscoverySink::new();
247 let _ = parser.parse_chunk_generic(&bytes[tail_start..], &mut tail_sink);
248 for name in tail_sink.into_order() {
249 if seen.insert(name.clone()) {
250 merged.push(name);
251 }
252 }
253 }
254 if merged.is_empty() && !bytes.is_empty() {
255 let mut sink = DiscoverySink::new();
256 let _ = parser.parse_chunk_generic(bytes, &mut sink);
257 sink.into_order()
258 } else {
259 merged
260 }
261 };
262 let elapsed = t0.elapsed().as_nanos() as u64;
263 DISCOVERY_NS.store(elapsed, Ordering::Relaxed);
264 let _ = splitter; let schema = if order.is_empty() {
266 FrozenSchema::from_plan(&[], plan)
267 } else {
268 FrozenSchema::from_discovered(&order, plan)
269 };
270 (schema, order)
271}
272
273pub fn discover_schema_for_bytes<P: crate::decoder::RecordParser>(
284 bytes: &[u8],
285 splitter: &dyn crate::decoder::Splitter,
286 parser: &P,
287 plan: &crate::plan::ExecutionPlan,
288) -> FrozenSchema {
289 if !plan.schema_order.is_empty() {
290 let names: Vec<&str> = plan.schema_order.iter().map(|s| s.as_str()).collect();
291 return FrozenSchema::from_partial_plan(&names, plan);
292 }
293 let opts = DiscoveryOpts::default();
294 let sig = crate::schema::layout_signature(bytes, &opts);
295 {
296 let cache = crate::schema::SCHEMA_CACHE
297 .read()
298 .unwrap_or_else(|e| e.into_inner());
299 if let Some(order) = cache.get(&sig) {
300 crate::schema::SCHEMA_CACHE_HITS.fetch_add(1, Ordering::Relaxed);
301 return FrozenSchema::from_discovered(order, plan);
302 }
303 }
304 crate::schema::SCHEMA_CACHE_MISSES.fetch_add(1, Ordering::Relaxed);
305 let (schema, order) = discover_schema(bytes, parser, plan, splitter);
306 if !order.is_empty() {
307 crate::schema::insert_schema_cache(sig, Arc::new(order));
308 }
309 schema
310}