rypipe_core/parallel_stream/
discovery.rs1use super::*;
2
3#[cfg(feature = "mmap")]
4pub(super) fn discover_schema_for_file<P: RecordParser>(
5 input: &crate::input::MmapHandle,
6 splitter: &dyn Splitter,
7 parser: &P,
8 plan: &ExecutionPlan,
9 budget: MemoryBudget,
10) -> Result<FrozenSchema> {
11 let bytes = &input.mmap[..];
12 let opts = DiscoveryOpts::default();
13 use std::hash::{Hash, Hasher};
14 let mut file = input.file.try_clone()?;
15 let mut buffer = Vec::new();
16 let allowance = budget.share(4);
17 let mut hasher = rustc_hash::FxHasher::default();
18 for range in crate::schema::signature_chunks(bytes.len(), &opts) {
19 allowance.check(range.len())?;
20 buffer.resize(range.len(), 0);
21 allowance.check(buffer.capacity())?;
22 file.seek(SeekFrom::Start(range.start as u64))?;
23 file.read_exact(&mut buffer)?;
24 buffer.hash(&mut hasher);
25 }
26 let signature = (bytes.len() as u64, hasher.finish());
27 let cached = crate::schema::SCHEMA_CACHE
28 .read()
29 .unwrap_or_else(|e| e.into_inner())
30 .get(&signature)
31 .cloned();
32 if let Some(order) = cached {
33 crate::schema::SCHEMA_CACHE_HITS.fetch_add(1, Ordering::Relaxed);
34 return Ok(FrozenSchema::from_discovered(&order, plan));
35 }
36 crate::schema::SCHEMA_CACHE_MISSES.fetch_add(1, Ordering::Relaxed);
37 let start = std::time::Instant::now();
38 let chunk_size = (budget.bytes() / 8).max(1);
39 let count = bytes
40 .len()
41 .div_ceil(chunk_size)
42 .clamp(1, crate::MAX_SPLIT_CHUNKS);
43 let points = splitter.find_split_points(bytes, count);
44 let ranges = crate::decoder::split_points_to_ranges(&points, bytes.len());
45 let planning_bytes = points
46 .capacity()
47 .saturating_mul(std::mem::size_of::<usize>())
48 .saturating_add(ranges.capacity() * std::mem::size_of::<std::ops::Range<usize>>());
49 allowance.check(planning_bytes)?;
50 let windows = crate::schema::dynamic_window_count(bytes.len() as u64);
51 let window_bytes = crate::schema::dynamic_window_size(bytes.len() as u64);
52 let mut sink = DiscoverySink::new();
53 for range in ranges {
54 if windows > 0
55 && !(0..windows).any(|i| {
56 let offset = bytes.len() / windows * i;
57 range.start < offset.saturating_add(window_bytes) && range.end > offset
58 })
59 && range.end <= bytes.len().saturating_sub(window_bytes)
60 {
61 continue;
62 }
63 allowance.check(range.len().saturating_add(planning_bytes))?;
64 buffer.resize(range.len(), 0);
65 file.seek(SeekFrom::Start(range.start as u64))?;
66 file.read_exact(&mut buffer)?;
67 let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
68 parser.parse_chunk_generic(&buffer, &mut sink)
69 }))
70 .map_err(|_| crate::Error::Parser("parser panicked during schema discovery".into()))?;
71 let order = sink.order.borrow();
72 let seen = sink.seen.borrow();
73 let storage = order.capacity() * std::mem::size_of::<String>()
74 + order.iter().map(String::capacity).sum::<usize>()
75 + seen.capacity() * (std::mem::size_of::<Box<str>>() + 1)
76 + seen.iter().map(|name| name.len()).sum::<usize>();
77 allowance.check(
78 storage
79 .saturating_add(buffer.capacity())
80 .saturating_add(planning_bytes),
81 )?;
82 }
83 let order = sink.into_order();
84 let schema = FrozenSchema::from_discovered(&order, plan);
85 if !order.is_empty() {
86 crate::schema::insert_schema_cache(signature, Arc::new(order));
87 }
88 DISCOVERY_NS.store(start.elapsed().as_nanos() as u64, Ordering::Relaxed);
89 Ok(schema)
90}
91
92static DISCOVERY_NS: AtomicU64 = AtomicU64::new(0);
93
94pub fn discovery_profile() -> u64 {
96 DISCOVERY_NS.load(Ordering::Relaxed)
97}
98pub fn reset_discovery_profile() {
100 DISCOVERY_NS.store(0, Ordering::Relaxed);
101}
102
103pub struct ParallelStreamOpts {
105 pub threads: usize,
107 pub ordered: bool,
109 pub max_reorder: usize,
113 pub schema: Option<FrozenSchema>,
116}
117
118impl Default for ParallelStreamOpts {
119 fn default() -> Self {
120 Self {
121 threads: std::thread::available_parallelism()
122 .map(|n| n.get())
123 .unwrap_or(1),
124 ordered: true,
125 max_reorder: 0, schema: None,
127 }
128 }
129}
130
131struct DiscoverySink {
136 seen: std::cell::RefCell<rustc_hash::FxHashSet<Box<str>>>,
137 order: std::cell::RefCell<Vec<String>>,
138}
139
140impl DiscoverySink {
141 fn new() -> Self {
142 Self {
143 seen: std::cell::RefCell::new(rustc_hash::FxHashSet::default()),
144 order: std::cell::RefCell::new(Vec::new()),
145 }
146 }
147 fn into_order(self) -> Vec<String> {
148 self.order.into_inner()
149 }
150}
151
152impl crate::decoder::ColumnarSink for DiscoverySink {
153 #[inline]
154 fn begin_row(&mut self) {}
155 #[inline]
156 fn put_field(&mut self, name: &str, _value: crate::value::Value<'_>) {
157 let mut seen = self.seen.borrow_mut();
159 if seen.insert(Box::from(name)) {
160 self.order.borrow_mut().push(name.to_string());
161 }
162 }
163 #[inline]
164 fn end_row(&mut self) {}
165 #[inline]
166 fn wants(&self, _name: &str) -> bool {
167 true
168 }
169 #[inline]
170 fn resolve<'a>(&'a self, name: &'a str) -> Option<&'a str> {
171 let mut seen = self.seen.borrow_mut();
173 if seen.insert(Box::from(name)) {
174 self.order.borrow_mut().push(name.to_string());
175 }
176 Some(name)
177 }
178 #[inline]
179 fn needs_value(&self) -> bool {
180 false
181 }
182 fn finish(&mut self) -> crate::Result<arrow::record_batch::RecordBatch> {
184 Ok(arrow::record_batch::RecordBatch::new_empty(
185 std::sync::Arc::new(arrow::datatypes::Schema::empty()),
186 ))
187 }
188}
189
190pub(crate) fn discover_schema<P: crate::decoder::RecordParser>(
191 bytes: &[u8],
192 parser: &P,
193 plan: &crate::plan::ExecutionPlan,
194 splitter: &dyn crate::decoder::Splitter,
195) -> (FrozenSchema, Vec<String>) {
196 let t0 = std::time::Instant::now();
197 let opts = DiscoveryOpts::default();
199
200 let file_size = bytes.len() as u64;
201
202 let n = crate::schema::dynamic_window_count(file_size);
204 let wbytes = crate::schema::dynamic_window_size(file_size);
205
206 let order: Vec<String> = if file_size < opts.full_scan_threshold || n == 0 {
207 let mut sink = DiscoverySink::new();
209 let _ = parser.parse_chunk_generic(bytes, &mut sink);
210 sink.into_order()
211 } else {
212 use rayon::prelude::*;
213 let per_window: Vec<Vec<String>> = (0..n)
215 .into_par_iter()
216 .map(|i| {
217 let start = (file_size * i as u64 / n as u64) as usize;
218 let end = (start + wbytes).min(bytes.len());
219 if start >= end {
220 return Vec::new();
221 }
222 let mut sink = DiscoverySink::new();
223 let slice = &bytes[start..end];
224 let _ = parser.parse_chunk_generic(slice, &mut sink);
225 sink.into_order()
226 })
227 .collect();
228 let mut seen = rustc_hash::FxHashSet::<String>::default();
230 let mut merged = Vec::new();
231 for mut v in per_window {
232 for name in v.drain(..) {
233 if seen.insert(name.clone()) {
234 merged.push(name);
235 }
236 }
237 }
238 if opts.always_scan_tail {
240 let tail_start = bytes.len().saturating_sub(wbytes);
241 let mut tail_sink = DiscoverySink::new();
242 let _ = parser.parse_chunk_generic(&bytes[tail_start..], &mut tail_sink);
243 for name in tail_sink.into_order() {
244 if seen.insert(name.clone()) {
245 merged.push(name);
246 }
247 }
248 }
249 if merged.is_empty() && !bytes.is_empty() {
250 let mut sink = DiscoverySink::new();
251 let _ = parser.parse_chunk_generic(bytes, &mut sink);
252 sink.into_order()
253 } else {
254 merged
255 }
256 };
257 let elapsed = t0.elapsed().as_nanos() as u64;
258 DISCOVERY_NS.store(elapsed, Ordering::Relaxed);
259 let _ = splitter; let schema = if order.is_empty() {
261 FrozenSchema::from_plan(&[], plan)
262 } else {
263 FrozenSchema::from_discovered(&order, plan)
264 };
265 (schema, order)
266}
267
268pub fn discover_schema_for_bytes<P: crate::decoder::RecordParser>(
279 bytes: &[u8],
280 splitter: &dyn crate::decoder::Splitter,
281 parser: &P,
282 plan: &crate::plan::ExecutionPlan,
283) -> FrozenSchema {
284 if !plan.schema_order.is_empty() {
285 let names: Vec<&str> = plan.schema_order.iter().map(|s| s.as_str()).collect();
286 return FrozenSchema::from_partial_plan(&names, plan);
287 }
288 let opts = DiscoveryOpts::default();
289 let sig = crate::schema::layout_signature(bytes, &opts);
290 {
291 let cache = crate::schema::SCHEMA_CACHE
292 .read()
293 .unwrap_or_else(|e| e.into_inner());
294 if let Some(order) = cache.get(&sig) {
295 crate::schema::SCHEMA_CACHE_HITS.fetch_add(1, Ordering::Relaxed);
296 return FrozenSchema::from_discovered(order, plan);
297 }
298 }
299 crate::schema::SCHEMA_CACHE_MISSES.fetch_add(1, Ordering::Relaxed);
300 let (schema, order) = discover_schema(bytes, parser, plan, splitter);
301 if !order.is_empty() {
302 crate::schema::insert_schema_cache(sig, Arc::new(order));
303 }
304 schema
305}