Skip to main content

rypipe_core/
bounded.rs

1#[cfg(feature = "mmap")]
2use std::io::{Read, Seek, SeekFrom};
3use std::ops::Range;
4use std::path::Path;
5use std::sync::Arc;
6
7use arrow::record_batch::RecordBatch;
8
9use crate::arrow_export::apply_compare_filter;
10use crate::budget::{BudgetLedger, StreamStats};
11use crate::consumer::CollectingConsumer;
12use crate::decoder::{split_points_to_ranges, RecordParser, Splitter};
13use crate::engine::TableBuilder;
14use crate::input::InputBuffer;
15use crate::plan::ExecutionPlan;
16use crate::Result;
17
18/// A memory budget expressed in bytes.
19#[derive(Clone, Copy, Debug)]
20pub struct MemoryBudget {
21    bytes: usize,
22    strict: bool,
23}
24
25impl MemoryBudget {
26    pub fn new(bytes: usize) -> Self {
27        Self {
28            bytes,
29            strict: false,
30        }
31    }
32
33    pub fn bytes(&self) -> usize {
34        self.bytes
35    }
36
37    /// Fail when an executor memory check exceeds its allocation allowance.
38    /// This does not impose an OS limit on process RSS or adapter-owned memory.
39    pub fn with_strict(mut self, strict: bool) -> Self {
40        self.strict = strict;
41        self
42    }
43
44    pub fn is_strict(&self) -> bool {
45        self.strict
46    }
47
48    pub(crate) fn share(self, count: usize) -> Self {
49        Self {
50            bytes: self.bytes / count.max(1),
51            ..self
52        }
53    }
54
55    pub(crate) fn check(self, used: usize) -> Result<()> {
56        if self.strict && used > self.bytes {
57            return Err(crate::Error::Memory {
58                used,
59                limit: self.bytes,
60            });
61        }
62        Ok(())
63    }
64}
65
66/// Internal safeguard: never request more than this many split points, so a
67/// pathological row-size estimate cannot explode per-chunk overhead. The
68/// batch count is otherwise derived from the budget, input size, and
69/// estimated bytes per row; batches may still exceed the budget when the
70/// required count exceeds this cap. Increased for 64KB streaming (50GB/64KB
71/// ≈ 800k batches); still bounded by file scan cost.
72pub const MAX_SPLIT_CHUNKS: usize = 100_000;
73
74/// Minimum rows per emitted batch when the memory budget cannot hold a
75/// single row. Batches are oversize in that regime regardless (ordinary
76/// mode permits this), and per-row batches would let batch count — and
77/// consumer cost — grow linearly with row count.
78pub const MIN_ROWS_OVERSIZE_BATCH: usize = 64;
79
80/// Parse an input in bounded batches to stay within a memory budget.
81pub struct BoundedExecutor {
82    budget: MemoryBudget,
83    split_cap: usize,
84}
85
86impl BoundedExecutor {
87    pub fn new(budget: MemoryBudget) -> Self {
88        Self {
89            budget,
90            split_cap: MAX_SPLIT_CHUNKS,
91        }
92    }
93
94    /// Override the split cap (default [`MAX_SPLIT_CHUNKS`]). Raising it may be
95    /// needed for very large files with small budgets, where capped batches
96    /// would otherwise overshoot the budget.
97    pub fn with_split_cap(mut self, cap: usize) -> Self {
98        self.split_cap = cap;
99        self
100    }
101
102    /// Payload bytes the executor targets per emitted batch. Exported
103    /// batches are exact-sized, so a single-record batch larger than this
104    /// is a flagged oversize batch (see [`StreamStats::oversize_batches`]).
105    fn batch_target_bytes(&self) -> usize {
106        (self.budget.bytes() / 64).max(1)
107    }
108
109    /// Derive chunk ranges and batch sizing from an in-memory sample of the
110    /// input. Returns `(chunk_ranges, rows_per_batch, bytes_per_row)`.
111    fn plan_chunks(
112        &self,
113        bytes: &[u8],
114        splitter: &dyn Splitter,
115        min_chunk_bytes: usize,
116    ) -> (Vec<Range<usize>>, usize, usize, bool) {
117        let bytes_per_row = splitter
118            .estimate_bytes_per_row(&bytes[..bytes.len().min(65536)])
119            .max(1);
120        // Input, growing columns, merge buffers and Arrow output overlap.
121        let batch_bytes = self.batch_target_bytes();
122        let total_rows_est = bytes.len() / bytes_per_row;
123        // When the budget cannot hold even one row, every batch is oversize
124        // anyway (permitted in ordinary mode), so per-row batches only
125        // explode the batch count — consumers concatenating thousands of
126        // single-row batches degrade quadratically. Floor at a minimum
127        // batch size in that regime, and report the regime so consume_chunks
128        // does not adapt it back down (the capacity-based adaptive estimate
129        // is inflated by builder preallocation and collapses to 1).
130        let oversize = batch_bytes < bytes_per_row;
131        let min_rows = if oversize { MIN_ROWS_OVERSIZE_BATCH } else { 1 };
132        let rows_per_batch = (batch_bytes / bytes_per_row)
133            .max(min_rows)
134            .min(total_rows_est.max(1));
135
136        let num_batches = bytes.len().div_ceil(batch_bytes).max(1);
137        let capped = num_batches.min(self.split_cap);
138        let t = std::time::Instant::now();
139        let split_points = splitter.find_split_points_with(bytes, capped, min_chunk_bytes);
140        crate::profiling::add(crate::profiling::SPLIT, t.elapsed().as_nanos() as u64);
141        let chunks = split_points_to_ranges(&split_points, bytes.len());
142        (chunks, rows_per_batch, bytes_per_row, oversize)
143    }
144
145    fn parse_chunk<P>(
146        parser: &P,
147        bytes: &[u8],
148        plan: Arc<ExecutionPlan>,
149        bytes_per_row: usize,
150        budget: MemoryBudget,
151    ) -> Result<TableBuilder>
152    where
153        P: RecordParser,
154    {
155        budget.check(bytes.len())?;
156        let mut engine =
157            TableBuilder::with_plan((bytes.len() / bytes_per_row.max(512)).max(64), plan);
158        engine.set_memory_budget(budget)?;
159        std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
160            let t = std::time::Instant::now();
161            parser.validate(bytes)?;
162            crate::profiling::add(crate::profiling::VALIDATE, t.elapsed().as_nanos() as u64);
163            let t = std::time::Instant::now();
164            let result = parser.parse_chunk_generic(bytes, &mut engine);
165            crate::profiling::add(crate::profiling::PARSE, t.elapsed().as_nanos() as u64);
166            result
167        }))
168        .unwrap_or_else(|payload| {
169            let msg = if let Some(s) = payload.downcast_ref::<&str>() {
170                (*s).to_string()
171            } else if let Some(s) = payload.downcast_ref::<String>() {
172                s.clone()
173            } else {
174                "unknown panic".to_string()
175            };
176            Err(crate::Error::Parser(format!(
177                "worker panicked during bounded parse: {msg}"
178            )))
179        })?;
180        engine.check_memory_budget()?;
181        Ok(engine)
182    }
183
184    fn consume_batch<C: crate::consumer::BatchConsumer>(
185        &self,
186        mut engine: TableBuilder,
187        plan: &ExecutionPlan,
188        consumer: &mut C,
189        ledger: &mut BudgetLedger,
190        stats: &mut StreamStats,
191    ) -> Result<()> {
192        let t = std::time::Instant::now();
193        let mut batch = engine.finish()?;
194        crate::profiling::add(crate::profiling::EXPORT, t.elapsed().as_nanos() as u64);
195        if let Some(ref filter) = plan.filter {
196            batch = apply_compare_filter(batch, filter)?;
197        }
198        let batch_bytes = batch.get_array_memory_size();
199        self.budget.share(4).check(batch_bytes)?;
200        if batch.num_rows() == 1 && batch_bytes > self.batch_target_bytes() {
201            stats.oversize_batches += 1;
202        }
203        let rows = batch.num_rows();
204        ledger.charge_in_flight(batch_bytes);
205        let res = consumer.consume(batch);
206        ledger.release_in_flight(batch_bytes);
207        res?;
208        stats.batches += 1;
209        stats.rows += rows;
210        Ok(())
211    }
212
213    #[allow(clippy::too_many_arguments)]
214    fn consume_chunks<I, C>(
215        &self,
216        chunks: I,
217        rows_per_batch: usize,
218        oversize: bool,
219        plan: Arc<ExecutionPlan>,
220        consumer: &mut C,
221        ledger: &mut BudgetLedger,
222        stats: &mut StreamStats,
223    ) -> Result<()>
224    where
225        I: IntoIterator<Item = Result<TableBuilder>>,
226        C: crate::consumer::BatchConsumer,
227    {
228        let mut batch_engine = TableBuilder::with_plan(rows_per_batch.min(64), Arc::clone(&plan));
229        batch_engine.set_memory_budget(self.budget.share(4))?;
230        let mut rows_in_batch = 0usize;
231        for chunk_engine in chunks {
232            let chunk_engine = chunk_engine?;
233            let chunk_rows = chunk_engine.num_rows();
234            ledger.set_builders(
235                batch_engine
236                    .capacity_bytes()
237                    .saturating_add(chunk_engine.capacity_bytes()),
238            );
239            let adaptive = (self.budget.bytes() / 8)
240                .saturating_mul(chunk_rows)
241                .checked_div(chunk_engine.capacity_bytes().max(1))
242                .unwrap_or(1)
243                .max(1);
244            // Oversize regime (budget cannot hold one input row): every
245            // batch is oversize regardless (permitted in ordinary mode), so
246            // keep the planned minimum batch size instead of adapting to
247            // per-row splits, and do not flush early — accumulation across
248            // chunks is what keeps the batch count (and consumer cost)
249            // sub-linear.
250            let rows_per_batch = if oversize {
251                rows_per_batch
252            } else {
253                rows_per_batch.min(adaptive)
254            };
255            if !oversize
256                && rows_in_batch > 0
257                && batch_engine
258                    .capacity_bytes()
259                    .saturating_add(chunk_engine.capacity_bytes())
260                    > self.budget.bytes() / 4
261            {
262                let batch = batch_engine.split_off(batch_engine.num_rows());
263                self.consume_batch(batch, &plan, consumer, ledger, stats)?;
264                ledger.set_builders(batch_engine.capacity_bytes());
265                rows_in_batch = 0;
266            }
267            batch_engine.extend(chunk_engine)?;
268            batch_engine.check_memory_budget()?;
269            ledger.set_builders(batch_engine.capacity_bytes());
270            rows_in_batch += chunk_rows;
271            while rows_in_batch >= rows_per_batch
272                || (!oversize && batch_engine.bytes_used() >= self.budget.bytes())
273            {
274                if batch_engine.num_rows() == 0 {
275                    break;
276                }
277                let n = rows_per_batch.min(batch_engine.num_rows());
278                let n = if !oversize
279                    && batch_engine.bytes_used() >= self.budget.bytes()
280                    && batch_engine.num_rows() > 1
281                {
282                    let est = (batch_engine.num_rows() as f64 * self.budget.bytes() as f64
283                        / batch_engine.bytes_used() as f64) as usize;
284                    est.clamp(1, n)
285                } else {
286                    n
287                };
288                self.consume_batch(batch_engine.split_off(n), &plan, consumer, ledger, stats)?;
289                ledger.set_builders(batch_engine.capacity_bytes());
290                rows_in_batch = rows_in_batch.saturating_sub(n);
291            }
292        }
293        if let Some(err) = batch_engine.unknown_error.take() {
294            return Err(crate::Error::Merge(err));
295        }
296        if let Some(err) = batch_engine.strict_error.take() {
297            return Err(err);
298        }
299        if batch_engine.num_rows() > 0 {
300            self.consume_batch(batch_engine, &plan, consumer, ledger, stats)?;
301        }
302        Ok(())
303    }
304
305    /// Parse an in-memory byte slice in bounded batches, calling `consumer` per batch.
306    ///
307    /// Chunks are sliced directly from `bytes`; no file I/O occurs. This is
308    /// the streaming entry point for adapters holding decompressed data.
309    /// The budget controls batch sizing; allocator, parser, input, and Arrow
310    /// overhead can raise process RSS above this target.
311    pub fn run_bytes_stream<P, C>(
312        &self,
313        bytes: &[u8],
314        splitter: &dyn Splitter,
315        parser: P,
316        plan: Arc<ExecutionPlan>,
317        consumer: &mut C,
318    ) -> Result<()>
319    where
320        P: RecordParser + Clone + Send + Sync,
321        C: crate::consumer::BatchConsumer,
322    {
323        self.run_bytes_stream_with_stats(bytes, splitter, parser, plan, consumer)
324            .map(|_| ())
325    }
326
327    /// Like [`BoundedExecutor::run_bytes_stream`], but reports a
328    /// [`StreamStats`] summary including the peak tracked engine memory.
329    ///
330    /// The input slice is caller-owned, so it is not charged to the ledger;
331    /// `peak_tracked_bytes` covers builders and in-flight batches only.
332    pub fn run_bytes_stream_with_stats<P, C>(
333        &self,
334        bytes: &[u8],
335        splitter: &dyn Splitter,
336        parser: P,
337        plan: Arc<ExecutionPlan>,
338        consumer: &mut C,
339    ) -> Result<StreamStats>
340    where
341        P: RecordParser + Clone + Send + Sync,
342        C: crate::consumer::BatchConsumer,
343    {
344        let mut ledger = BudgetLedger::new(self.budget);
345        let mut stats = StreamStats::default();
346        self.run_bytes_stream_inner(
347            bytes,
348            splitter,
349            parser,
350            plan,
351            consumer,
352            &mut ledger,
353            &mut stats,
354        )?;
355        stats.peak_tracked_bytes = ledger.peak();
356        Ok(stats)
357    }
358
359    #[allow(clippy::too_many_arguments)]
360    fn run_bytes_stream_inner<P, C>(
361        &self,
362        bytes: &[u8],
363        splitter: &dyn Splitter,
364        parser: P,
365        plan: Arc<ExecutionPlan>,
366        consumer: &mut C,
367        ledger: &mut BudgetLedger,
368        stats: &mut StreamStats,
369    ) -> Result<()>
370    where
371        P: RecordParser + Clone + Send + Sync,
372        C: crate::consumer::BatchConsumer,
373    {
374        if bytes.is_empty() {
375            return Ok(());
376        }
377
378        let (mut chunks, rows_per_batch, bytes_per_row, oversize) = self.plan_chunks(
379            bytes,
380            splitter,
381            plan.min_chunk_bytes
382                .unwrap_or(crate::decoder::MIN_CHUNK_BYTES),
383        );
384
385        // Guard against degenerate splitter output that produces no ranges.
386        if chunks.is_empty() {
387            chunks.push(0..bytes.len());
388        }
389
390        let parse_plan = Arc::clone(&plan);
391        let budget = self.budget.share(4);
392        budget.check(chunks.capacity() * std::mem::size_of::<Range<usize>>())?;
393        let chunks = chunks.into_iter().map(move |chunk| {
394            Self::parse_chunk(
395                &parser,
396                &bytes[chunk.start..chunk.end],
397                Arc::clone(&parse_plan),
398                bytes_per_row,
399                budget,
400            )
401        });
402        self.consume_chunks(
403            chunks,
404            rows_per_batch,
405            oversize,
406            plan,
407            consumer,
408            ledger,
409            stats,
410        )
411    }
412
413    /// Parse an in-memory byte slice in bounded batches, returning one
414    /// `RecordBatch` per batch.
415    ///
416    /// Chunks are sliced directly from `bytes`; no file I/O occurs. This is
417    /// the entry point for adapters holding decompressed or streamed-in data.
418    pub fn run_bytes<P>(
419        &self,
420        bytes: &[u8],
421        splitter: &dyn Splitter,
422        parser: P,
423        plan: Arc<ExecutionPlan>,
424    ) -> Result<Vec<RecordBatch>>
425    where
426        P: RecordParser + Clone + Send + Sync,
427    {
428        let batches = Vec::new();
429        let mut consumer = CollectingConsumer(batches);
430        self.run_bytes_stream(bytes, splitter, parser, plan, &mut consumer)?;
431        consumer.finish()
432    }
433
434    /// Parse `path` in batches, calling `consumer` per batch (streaming).
435    ///
436    /// Compressed inputs are decompressed up front and served via
437    /// `run_bytes_stream`. Uncompressed inputs with `mmap` use the seek-based
438    /// path with a reusable chunk buffer to keep RSS constant.
439    pub fn run_stream<P, C>(
440        &self,
441        path: &Path,
442        splitter: &dyn Splitter,
443        parser: P,
444        plan: Arc<ExecutionPlan>,
445        prefault: bool,
446        consumer: &mut C,
447    ) -> Result<()>
448    where
449        P: RecordParser + Clone + Send + Sync,
450        C: crate::consumer::BatchConsumer,
451    {
452        self.run_stream_with_stats(path, splitter, parser, plan, prefault, consumer)
453            .map(|_| ())
454    }
455
456    /// Like [`BoundedExecutor::run_stream`], but reports a [`StreamStats`]
457    /// summary including the peak tracked engine memory.
458    pub fn run_stream_with_stats<P, C>(
459        &self,
460        path: &Path,
461        splitter: &dyn Splitter,
462        parser: P,
463        plan: Arc<ExecutionPlan>,
464        prefault: bool,
465        consumer: &mut C,
466    ) -> Result<StreamStats>
467    where
468        P: RecordParser + Clone + Send + Sync,
469        C: crate::consumer::BatchConsumer,
470    {
471        let mut ledger = BudgetLedger::new(self.budget);
472        let mut stats = StreamStats::default();
473        let use_mmap = cfg!(feature = "mmap");
474        let input = InputBuffer::open(path, use_mmap, prefault)?;
475
476        #[cfg(feature = "mmap")]
477        if matches!(input, InputBuffer::Mmap(_)) {
478            self.run_mapped_stream(
479                input,
480                splitter,
481                parser,
482                plan,
483                consumer,
484                &mut ledger,
485                &mut stats,
486            )?;
487            stats.peak_tracked_bytes = ledger.peak();
488            return Ok(stats);
489        }
490
491        self.budget.share(4).check(input.len())?;
492        ledger.set_input(input.len());
493        self.run_bytes_stream_inner(
494            input.as_slice(),
495            splitter,
496            parser,
497            plan,
498            consumer,
499            &mut ledger,
500            &mut stats,
501        )?;
502        stats.peak_tracked_bytes = ledger.peak();
503        Ok(stats)
504    }
505
506    /// Parse `path` in batches, returning one `RecordBatch` per batch.
507    ///
508    /// The caller is responsible for concatenating the batches if a single
509    /// table is desired.
510    ///
511    /// Compressed inputs (see [`InputBuffer::open`]) are decompressed up
512    /// front and served from memory via [`BoundedExecutor::run_bytes`].
513    /// Uncompressed inputs on platforms with the `mmap` feature retain the
514    /// seek-based path: the mapping is dropped after planning and each chunk
515    /// is read from disk on demand, keeping resident memory low.
516    pub fn run<P>(
517        &self,
518        path: &Path,
519        splitter: &dyn Splitter,
520        parser: P,
521        plan: Arc<ExecutionPlan>,
522        prefault: bool,
523    ) -> Result<Vec<RecordBatch>>
524    where
525        P: RecordParser + Clone + Send + Sync,
526    {
527        let batches = Vec::new();
528        let mut consumer = CollectingConsumer(batches);
529        self.run_stream(path, splitter, parser, plan, prefault, &mut consumer)?;
530        consumer.finish()
531    }
532
533    /// Streaming path for mapped inputs: plan against the mapping, drop it,
534    /// then read each chunk with a reusable buffer.
535    #[cfg(feature = "mmap")]
536    #[allow(clippy::too_many_arguments)]
537    fn run_mapped_stream<P, C>(
538        &self,
539        input: InputBuffer,
540        splitter: &dyn Splitter,
541        parser: P,
542        plan: Arc<ExecutionPlan>,
543        consumer: &mut C,
544        ledger: &mut BudgetLedger,
545        stats: &mut StreamStats,
546    ) -> Result<()>
547    where
548        P: RecordParser + Clone + Send + Sync,
549        C: crate::consumer::BatchConsumer,
550    {
551        let bytes = input.as_slice();
552        if bytes.is_empty() {
553            return Ok(());
554        }
555
556        let (mut chunks, rows_per_batch, bytes_per_row, oversize) = self.plan_chunks(
557            bytes,
558            splitter,
559            plan.min_chunk_bytes
560                .unwrap_or(crate::decoder::MIN_CHUNK_BYTES),
561        );
562
563        // Guard against degenerate splitter output that produces no ranges.
564        if chunks.is_empty() {
565            chunks.push(0..bytes.len());
566        }
567
568        let InputBuffer::Mmap(handle) = input else {
569            unreachable!("mapped input required");
570        };
571        let mut file = handle.file;
572        drop(handle.mmap);
573
574        // Reusable buffer sized to the largest chunk to avoid per-chunk alloc.
575        let max_chunk = chunks.iter().map(|r| r.len()).max().unwrap_or(0);
576        let budget = self.budget.share(4);
577        budget.check(
578            max_chunk.saturating_add(chunks.capacity() * std::mem::size_of::<Range<usize>>()),
579        )?;
580        let mut chunk_buf = Vec::with_capacity(max_chunk);
581        ledger.set_input(chunk_buf.capacity());
582
583        let parse_plan = Arc::clone(&plan);
584        let parsed_chunks = chunks.into_iter().map(move |chunk| {
585            let chunk_len = chunk.len();
586            chunk_buf.resize(chunk_len, 0);
587            file.seek(SeekFrom::Start(chunk.start as u64))?;
588            file.read_exact(&mut chunk_buf)?;
589            Self::parse_chunk(
590                &parser,
591                &chunk_buf,
592                Arc::clone(&parse_plan),
593                bytes_per_row,
594                budget,
595            )
596        });
597        self.consume_chunks(
598            parsed_chunks,
599            rows_per_batch,
600            oversize,
601            plan,
602            consumer,
603            ledger,
604            stats,
605        )
606    }
607}
608
609#[cfg(test)]
610mod tests {
611    use super::*;
612    use crate::decoder::Splitter;
613    use crate::ExecutionPlan;
614
615    struct NewlineSplitter;
616
617    impl Splitter for NewlineSplitter {
618        fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize> {
619            if from == 0 && !bytes.is_empty() {
620                return Some(0);
621            }
622            bytes[from.saturating_sub(1)..]
623                .iter()
624                .position(|&b| b == b'\n')
625                .map(|i| from + i + 1)
626                .filter(|&p| p < bytes.len())
627        }
628
629        fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize {
630            let newlines = sample.iter().filter(|&&b| b == b'\n').count().max(1);
631            (sample.len() / newlines).max(1)
632        }
633    }
634
635    fn rows(n: usize) -> Vec<u8> {
636        "abcdefghij\n".repeat(n).into_bytes()
637    }
638
639    #[test]
640    fn test_split_cap_limits_chunk_count() {
641        let data = rows(10_000);
642        let budget = MemoryBudget::new(1024);
643
644        let default_chunks = BoundedExecutor::new(budget)
645            .plan_chunks(&data, &NewlineSplitter, crate::decoder::MIN_CHUNK_BYTES)
646            .0
647            .len();
648        assert!(
649            default_chunks > 2,
650            "uncapped run must split: {default_chunks}"
651        );
652
653        for cap in [1, 2, 3] {
654            let chunks = BoundedExecutor::new(budget)
655                .with_split_cap(cap)
656                .plan_chunks(&data, &NewlineSplitter, crate::decoder::MIN_CHUNK_BYTES)
657                .0;
658            assert!(
659                chunks.len() <= cap.max(1),
660                "cap {cap} must bound chunk count, got {}",
661                chunks.len()
662            );
663        }
664    }
665
666    /// Newline-delimited `key=value` rows, for bounded-path tests.
667    #[derive(Clone)]
668    struct KeyValueParser;
669
670    impl crate::decoder::RecordParser for KeyValueParser {
671        fn validate(&self, bytes: &[u8]) -> crate::Result<()> {
672            simdutf8::basic::from_utf8(bytes)?;
673            Ok(())
674        }
675
676        fn parse_chunk(
677            &self,
678            bytes: &[u8],
679            sink: &mut dyn crate::decoder::ColumnarSink,
680        ) -> crate::Result<()> {
681            let text = std::str::from_utf8(bytes).map_err(|e| crate::Error::Plan(e.to_string()))?;
682            for line in text.lines() {
683                if line.is_empty() {
684                    continue;
685                }
686                sink.begin_row();
687                for token in line.split_whitespace() {
688                    if let Some((k, v)) = token.split_once('=') {
689                        sink.put_field(k, crate::value::Value::Str(std::borrow::Cow::Borrowed(v)));
690                    }
691                }
692                sink.end_row();
693            }
694            Ok(())
695        }
696    }
697
698    /// Records start exactly at every multiple of 12, so chunk boundaries
699    /// never land mid-record (unlike NewlineSplitter, which is off-by-one
700    /// for this purpose).
701    struct AlignedSplitter;
702
703    impl Splitter for AlignedSplitter {
704        fn next_record_start(&self, bytes: &[u8], from: usize) -> Option<usize> {
705            let start = from.div_ceil(12) * 12;
706            (start < bytes.len()).then_some(start)
707        }
708
709        fn estimate_bytes_per_row(&self, _sample: &[u8]) -> usize {
710            12
711        }
712    }
713
714    #[test]
715    fn tiny_budget_below_row_size_keeps_batch_count_bounded() {
716        // Budget (8 bytes) smaller than a single row (~11 bytes): without the
717        // oversize floor this emits one batch per row (10_000 batches), which
718        // degrades quadratically for consumers concatenating batches.
719        // ~12 bytes/row; budget (8 bytes) cannot hold a single row. Without
720        // the oversize floor this emits one batch per row (100_000 batches),
721        // which degrades quadratically for consumers concatenating batches.
722        let data = "a=1 b=2 c=3\n".repeat(100_000).into_bytes();
723        let mut consumer = crate::consumer::CollectingConsumer(Vec::new());
724        let stats = BoundedExecutor::new(MemoryBudget::new(8))
725            .run_bytes_stream_with_stats(
726                &data,
727                &AlignedSplitter,
728                KeyValueParser,
729                std::sync::Arc::new(ExecutionPlan::new()),
730                &mut consumer,
731            )
732            .unwrap();
733        assert_eq!(stats.rows, 100_000, "all rows must be delivered");
734        assert!(
735            stats.batches <= 2048,
736            "batch count must stay far below row count for pathological              budgets, got {}",
737            stats.batches
738        );
739    }
740
741    #[test]
742    fn test_default_split_cap_is_max_split_chunks() {
743        assert_eq!(
744            BoundedExecutor::new(MemoryBudget::new(1)).split_cap,
745            MAX_SPLIT_CHUNKS
746        );
747        let plan = ExecutionPlan::new().with_max_split_chunks(7);
748        assert_eq!(plan.max_split_chunks, Some(7));
749        assert_eq!(ExecutionPlan::new().max_split_chunks, None);
750    }
751}