Skip to main content

uqa_execution/
spill.rs

1//
2// Unified Query Algebra
3//
4// Copyright (c) 2023-2026 Cognica, Inc.
5//
6
7//! Disk-backed spill buffer for blocking operators (`Sort`,
8//! `HashAggregate`, `Window`).
9//!
10//! The budget is measured in the exact number of bytes each batch occupies in
11//! the spill encoding. [`SpillBuffer::push`] automatically flushes before a
12//! successful push could leave more encoded bytes in memory than the budget.
13//! Draining restores spilled batches first and then any in-memory tail,
14//! preserving input order. The temporary file is removed when the buffer (or
15//! its active drain iterator) is dropped.
16
17use std::io::{BufReader, Take};
18use std::ops::Range;
19use std::path::{Path, PathBuf};
20use std::sync::Arc;
21use uqa_core::Value;
22use uqa_storage::temporary_file::TemporaryFile as File;
23
24use crate::batch::{Batch, OwnedPhysicalRow, PhysicalRow, RowSchema};
25use crate::physical::ExecResult;
26use uqa_storage::temporary_file::TemporaryFile as NamedTempFile;
27
28mod file;
29mod format;
30mod indexed;
31mod match_flags;
32mod segments;
33pub(crate) use file::SpillFileArena;
34pub(crate) use match_flags::MatchFlags;
35pub(crate) use segments::SpillSegmentDescriptor;
36
37use format::{
38    append_batches, decode_batch, encoded_batch_overhead_size, encoded_batch_size,
39    encoded_borrowed_row_record_size, encoded_physical_row_record_size, open_spill_reader,
40    read_bounded_spill_record, spill_error,
41};
42pub(crate) use format::{
43    decode_document, decode_physical_row_record, encode_document, encode_physical_row_record,
44    encoded_document_size,
45};
46mod batch_writer;
47pub(crate) use batch_writer::SpillBatchWriter;
48mod batch_layout;
49pub(crate) use batch_layout::EncodedBatchLayout;
50pub(crate) use indexed::BufferedIndexedSpill;
51pub use indexed::IndexedSpill;
52
53const SPILL_MAGIC: &[u8] = b"UQA-SPILL\x02\n";
54
55/// Incremental exact size of one not-yet-encoded batch. When the first row with lock origins arrives, the binary format adds an empty origin-count field to every preceding origin-free row, so accounting must update those already-buffered rows as well as the new record.
56#[derive(Clone, Copy)]
57pub(crate) struct EncodedBatchSizer {
58    physical_width: usize,
59    bytes: usize,
60    origin_free_rows: usize,
61    lock_origin_rows: usize,
62}
63
64impl EncodedBatchSizer {
65    pub(crate) fn new(schema: &RowSchema) -> ExecResult<Self> {
66        Ok(Self {
67            physical_width: schema.physical_width(),
68            bytes: encoded_batch_overhead_size(schema)?,
69            origin_free_rows: 0,
70            lock_origin_rows: 0,
71        })
72    }
73
74    pub(crate) fn append(&mut self, row: &PhysicalRow) -> ExecResult<()> {
75        self.append_size(
76            encoded_physical_row_record_size(row, self.physical_width)?,
77            !row.lock_origins().is_empty(),
78        )
79    }
80
81    pub(crate) fn append_borrowed(&mut self, values: &[&Value]) -> ExecResult<()> {
82        self.append_size(
83            encoded_borrowed_row_record_size(values, self.physical_width)?,
84            false,
85        )
86    }
87
88    fn append_size(&mut self, mut additional: usize, has_origins: bool) -> ExecResult<()> {
89        if has_origins {
90            if self.lock_origin_rows == 0 {
91                let preceding_metadata = self.origin_free_rows.checked_mul(8).ok_or_else(|| {
92                    spill_error("incremental spill batch lock-origin metadata overflow")
93                })?;
94                additional = additional.checked_add(preceding_metadata).ok_or_else(|| {
95                    spill_error("incremental spill batch lock-origin size overflow")
96                })?;
97            }
98            self.lock_origin_rows = self.lock_origin_rows.checked_add(1).ok_or_else(|| {
99                spill_error("incremental spill batch lock-origin row count overflow")
100            })?;
101        } else {
102            self.origin_free_rows = self.origin_free_rows.checked_add(1).ok_or_else(|| {
103                spill_error("incremental spill batch origin-free row count overflow")
104            })?;
105            if self.lock_origin_rows != 0 {
106                additional = additional.checked_add(8).ok_or_else(|| {
107                    spill_error("incremental spill batch lock-origin size overflow")
108                })?;
109            }
110        }
111        self.bytes = self
112            .bytes
113            .checked_add(additional)
114            .ok_or_else(|| spill_error("incremental spill batch size overflow"))?;
115        Ok(())
116    }
117
118    pub(crate) fn bytes(self) -> usize {
119        self.bytes
120    }
121
122    /// Remove one retained row, including the origin-count fields that disappear when the last row carrying origins leaves the batch.
123    pub(crate) fn remove(&mut self, row: &PhysicalRow) -> ExecResult<()> {
124        let mut removed = encoded_physical_row_record_size(row, self.physical_width)?;
125        if row.lock_origins().is_empty() {
126            self.origin_free_rows = self
127                .origin_free_rows
128                .checked_sub(1)
129                .ok_or_else(|| spill_error("spill batch origin-free row count underflow"))?;
130            if self.lock_origin_rows != 0 {
131                removed = removed
132                    .checked_add(8)
133                    .ok_or_else(|| spill_error("spill batch lock-origin size overflow"))?;
134            }
135        } else {
136            self.lock_origin_rows = self
137                .lock_origin_rows
138                .checked_sub(1)
139                .ok_or_else(|| spill_error("spill batch lock-origin row count underflow"))?;
140            if self.lock_origin_rows == 0 {
141                removed = self
142                    .origin_free_rows
143                    .checked_mul(8)
144                    .and_then(|bytes| removed.checked_add(bytes))
145                    .ok_or_else(|| spill_error("spill batch lock-origin size overflow"))?;
146            }
147        }
148        self.bytes = self
149            .bytes
150            .checked_sub(removed)
151            .ok_or_else(|| spill_error("spill batch size underflow"))?;
152        Ok(())
153    }
154}
155
156struct ArenaSegment {
157    arena: SpillFileArena,
158    range: Option<Range<u64>>,
159}
160
161/// Append-only batch buffer with an encoded-byte memory budget.
162///
163/// The budget is exact for the serialized representation and does not claim to
164/// be the Rust allocator's resident-byte accounting. At most one incoming or
165/// decoded batch can itself be larger than the budget; successful pushes do not
166/// retain such an oversized batch in memory.
167pub struct SpillBuffer {
168    schema: Option<RowSchema>,
169    batches: Vec<Batch>,
170    rows: usize,
171    in_memory_rows: usize,
172    in_memory_bytes: usize,
173    max_in_memory_record_bytes: usize,
174    /// Encoded-byte budget. Set to `usize::MAX` to disable spilling.
175    budget_bytes: usize,
176    spill_directory: Option<PathBuf>,
177    spill_file: Option<NamedTempFile>,
178    spill_segment: Option<Box<ArenaSegment>>,
179    spilled_batches: usize,
180    spilled_rows: usize,
181    spilled_bytes: usize,
182    max_spilled_record_bytes: usize,
183}
184
185impl SpillBuffer {
186    pub fn new(budget_bytes: usize) -> Self {
187        Self {
188            schema: None,
189            batches: Vec::new(),
190            rows: 0,
191            in_memory_rows: 0,
192            in_memory_bytes: 0,
193            max_in_memory_record_bytes: 0,
194            budget_bytes,
195            spill_directory: None,
196            spill_file: None,
197            spill_segment: None,
198            spilled_batches: 0,
199            spilled_rows: 0,
200            spilled_bytes: 0,
201            max_spilled_record_bytes: 0,
202        }
203    }
204
205    /// Create a buffer whose temporary spill file will be placed in `directory`.
206    ///
207    /// File creation is deferred until the first spill. This is primarily useful
208    /// when an engine has a dedicated temporary-data volume.
209    pub fn new_in(budget_bytes: usize, directory: impl Into<PathBuf>) -> Self {
210        let mut buffer = Self::new(budget_bytes);
211        buffer.spill_directory = Some(directory.into());
212        buffer
213    }
214
215    pub fn unbounded() -> Self {
216        Self::new(usize::MAX)
217    }
218
219    /// A caller appends each segment completely before beginning the next one.
220    pub(crate) fn with_arena(mut self, arena: &SpillFileArena) -> Self {
221        self.spill_segment = Some(Box::new(ArenaSegment {
222            arena: arena.clone(),
223            range: None,
224        }));
225        self
226    }
227
228    /// Append a batch, spilling automatically when required by the byte budget.
229    ///
230    /// Returns `true` if this push wrote one or more batches to disk. If disk
231    /// creation, encoding, or writing fails, the new batch and all earlier
232    /// batches remain owned by the buffer and the error is returned.
233    pub fn push(&mut self, batch: Batch) -> ExecResult<bool> {
234        if let Some(schema) = self.schema.as_ref() {
235            if schema != &batch.schema {
236                return Err(spill_error(format!(
237                    "spill buffer schema mismatch: expected {:?}, got {:?}",
238                    schema.columns(),
239                    batch.schema.columns()
240                )));
241            }
242        } else {
243            self.schema = Some(batch.schema.clone());
244        }
245        let batch_rows = batch.rows.len();
246        let next_rows = self
247            .rows
248            .checked_add(batch_rows)
249            .ok_or_else(|| spill_error("spill buffer row count overflow"))?;
250        let batch_bytes = match Self::encoded_size(&batch) {
251            Ok(bytes) => bytes,
252            Err(error) => {
253                // Preserve ownership even when an exotic value or an encoded
254                // size overflow prevents budget accounting. The failed
255                // operator will abort, but no row silently disappears.
256                self.retain_batch(batch, usize::MAX);
257                return Err(error);
258            }
259        };
260        let would_exceed = self
261            .in_memory_bytes
262            .checked_add(batch_bytes)
263            .is_none_or(|bytes| bytes > self.budget_bytes);
264
265        let mut spilled = false;
266        if would_exceed && !self.batches.is_empty() {
267            if let Err(error) = self.spill_pending() {
268                self.retain_batch(batch, batch_bytes);
269                return Err(error);
270            }
271            spilled = true;
272        }
273
274        let next_in_memory_rows = self
275            .in_memory_rows
276            .checked_add(batch_rows)
277            .ok_or_else(|| spill_error("spill buffer in-memory row count overflow"))?;
278        let next_in_memory_bytes = self
279            .in_memory_bytes
280            .checked_add(batch_bytes)
281            .ok_or_else(|| spill_error("spill buffer in-memory byte count overflow"))?;
282        self.rows = next_rows;
283        self.in_memory_rows = next_in_memory_rows;
284        self.in_memory_bytes = next_in_memory_bytes;
285        self.max_in_memory_record_bytes = self.max_in_memory_record_bytes.max(batch_bytes);
286        self.batches.push(batch);
287
288        // A single encoded batch may exceed work_mem. It must pass through
289        // memory once, but a successful push never retains it there.
290        if self.in_memory_bytes > self.budget_bytes {
291            self.spill_pending()?;
292            spilled = true;
293        }
294        Ok(spilled)
295    }
296
297    /// Exact byte count used for budget accounting, including the record
298    /// length prefix written to disk.
299    pub fn encoded_size(batch: &Batch) -> ExecResult<usize> {
300        encoded_batch_size(batch)
301    }
302
303    /// Total buffered rows, including rows already written to disk.
304    pub fn rows(&self) -> usize {
305        self.rows
306    }
307
308    /// Rows currently retained in memory.
309    pub fn in_memory_rows(&self) -> usize {
310        self.in_memory_rows
311    }
312
313    /// Exact encoded bytes currently retained in memory.
314    pub fn in_memory_bytes(&self) -> usize {
315        self.in_memory_bytes
316    }
317
318    pub fn budget_bytes(&self) -> usize {
319        self.budget_bytes
320    }
321
322    pub fn over_budget(&self) -> bool {
323        self.in_memory_bytes > self.budget_bytes
324    }
325
326    pub fn has_spilled(&self) -> bool {
327        self.spill_file.is_some()
328    }
329
330    pub fn spilled_rows(&self) -> usize {
331        self.spilled_rows
332    }
333
334    pub fn spilled_batches(&self) -> usize {
335        self.spilled_batches
336    }
337
338    pub fn spilled_bytes(&self) -> usize {
339        self.spilled_bytes
340    }
341
342    /// Path of the live spill file, if one has been created.
343    ///
344    /// The path is diagnostic only and becomes invalid as soon as the buffer or
345    /// the drain iterator that owns the file is dropped.
346    pub fn spill_path(&self) -> Option<&Path> {
347        self.spill_file.as_ref().map(NamedTempFile::path)
348    }
349
350    /// Flush all pending in-memory batches when the byte budget is exceeded.
351    ///
352    /// Returns `true` when batches were written. A failed append is rolled back
353    /// to the previous file length and the pending batches remain in memory, so
354    /// callers never observe a silent partial spill.
355    pub fn spill_if_over_budget(&mut self) -> ExecResult<bool> {
356        if !self.over_budget() || self.batches.is_empty() {
357            return Ok(false);
358        }
359        self.spill_pending()
360    }
361
362    /// Force all pending in-memory batches to disk regardless of the budget.
363    ///
364    /// This is useful at a blocking-operator phase boundary. It returns `false`
365    /// when there is nothing pending.
366    pub fn spill_pending(&mut self) -> ExecResult<bool> {
367        if self.batches.is_empty() {
368            return Ok(false);
369        }
370
371        // Reject metadata overflow before writing. An error after a successful
372        // append would leave a retry able to duplicate records while the
373        // published counters disagreed with the file.
374        let next_spilled_batches = self
375            .spilled_batches
376            .checked_add(self.batches.len())
377            .ok_or_else(|| spill_error("spill batch count overflow"))?;
378        let next_spilled_rows = self
379            .spilled_rows
380            .checked_add(self.in_memory_rows)
381            .ok_or_else(|| spill_error("spill row count overflow"))?;
382        let next_spilled_bytes = self
383            .spilled_bytes
384            .checked_add(self.in_memory_bytes)
385            .ok_or_else(|| spill_error("spill byte count overflow"))?;
386        let next_max_spilled_record_bytes = self
387            .max_spilled_record_bytes
388            .max(self.max_in_memory_record_bytes);
389
390        if let Some(segment) = &mut self.spill_segment {
391            self.spill_file = Some(segment.arena.append(&self.batches, &mut segment.range)?);
392        } else if let Some(file) = self.spill_file.as_mut() {
393            append_batches(file.as_file_mut(), &self.batches)?;
394        } else {
395            let mut file = self.create_spill_file()?;
396            append_batches(file.as_file_mut(), &self.batches)?;
397            self.spill_file = Some(file);
398        }
399
400        self.spilled_batches = next_spilled_batches;
401        self.spilled_rows = next_spilled_rows;
402        self.spilled_bytes = next_spilled_bytes;
403        self.max_spilled_record_bytes = next_max_spilled_record_bytes;
404        self.batches.clear();
405        self.in_memory_rows = 0;
406        self.in_memory_bytes = 0;
407        self.max_in_memory_record_bytes = 0;
408        Ok(true)
409    }
410
411    /// Open a repeatable streaming reader without consuming this buffer.
412    ///
413    /// Spilled batches are decoded one at a time. The in-memory tail is cloned
414    /// one batch at a time only when the reader reaches it.
415    pub fn reader(&self) -> ExecResult<SpillReader<'_>> {
416        let reader = self
417            .spill_file
418            .as_ref()
419            .map(|file| open_spill_reader(file, self.spill_range()))
420            .transpose()?;
421        let disk_finished = reader.is_none();
422        Ok(SpillReader {
423            reader,
424            memory: self.batches.iter(),
425            disk_finished,
426            failed: false,
427            max_record_bytes: self.max_spilled_record_bytes,
428            expected_schema: self.schema.clone(),
429        })
430    }
431
432    /// Open a repeatable physical-row stream without collecting all batches.
433    pub fn read_rows(&self) -> ExecResult<SpillRows<SpillReader<'_>>> {
434        self.reader().map(SpillRows::new)
435    }
436
437    /// Drain buffered batches in their original input order.
438    ///
439    /// The returned iterator owns the temporary file. Each disk read or decode
440    /// failure is returned as a [`crate::physical::ExecError`], and dropping the
441    /// iterator early still removes the temporary file.
442    pub fn drain(&mut self) -> ExecResult<SpillDrain> {
443        let reader = self
444            .spill_file
445            .as_ref()
446            .map(|file| open_spill_reader(file, self.spill_range()))
447            .transpose()?;
448        let spill_file = self.spill_file.take();
449        if let Some(segment) = &mut self.spill_segment {
450            segment.range = None;
451        }
452        let memory = std::mem::take(&mut self.batches).into_iter();
453        let expected_schema = self.schema.take();
454
455        self.rows = 0;
456        self.in_memory_rows = 0;
457        self.in_memory_bytes = 0;
458        self.max_in_memory_record_bytes = 0;
459        self.spilled_batches = 0;
460        self.spilled_rows = 0;
461        self.spilled_bytes = 0;
462        let max_record_bytes = std::mem::take(&mut self.max_spilled_record_bytes);
463
464        let disk_finished = reader.is_none();
465        Ok(SpillDrain {
466            reader,
467            spill_file,
468            memory,
469            disk_finished,
470            failed: false,
471            max_record_bytes,
472            expected_schema,
473        })
474    }
475
476    /// Drain and materialize every restored batch.
477    pub fn drain_all(&mut self) -> ExecResult<Vec<Batch>> {
478        self.drain()?.collect()
479    }
480
481    /// Consume the buffer as a physical-row stream without collecting batches.
482    pub fn drain_rows(&mut self) -> ExecResult<SpillRows<SpillDrain>> {
483        self.drain().map(SpillRows::new)
484    }
485
486    /// Discard all buffered data and remove any spill file.
487    pub fn clear(&mut self) {
488        self.schema = None;
489        self.batches.clear();
490        self.spill_file = None;
491        if let Some(segment) = &mut self.spill_segment {
492            segment.range = None;
493        }
494        self.rows = 0;
495        self.in_memory_rows = 0;
496        self.in_memory_bytes = 0;
497        self.max_in_memory_record_bytes = 0;
498        self.spilled_batches = 0;
499        self.spilled_rows = 0;
500        self.spilled_bytes = 0;
501        self.max_spilled_record_bytes = 0;
502    }
503
504    /// Seal this buffer as an immutable, cheaply cloneable materialization.
505    /// Batches that fit within the configured byte budget remain in memory;
506    /// once spilling has started, every pending batch is flushed and readers
507    /// reopen the file independently. Both forms support repeatable scans
508    /// without collecting the complete input again.
509    pub fn into_shared(mut self, schema: impl Into<RowSchema>) -> ExecResult<SharedSpill> {
510        let schema = schema.into();
511        if let Some(actual) = self.schema.as_ref() {
512            if actual != &schema {
513                return Err(spill_error(format!(
514                    "shared spill schema mismatch: expected {:?}, got {:?}",
515                    schema.columns(),
516                    actual.columns()
517                )));
518            }
519        }
520        let rows = self.rows;
521        let storage = if self.spill_file.is_none() {
522            let batches = std::mem::take(&mut self.batches);
523            SharedSpillStorage::Memory(batches)
524        } else {
525            self.spill_pending()?;
526            SharedSpillStorage::Disk(
527                self.spill_file
528                    .take()
529                    .expect("spill file exists after flushing shared materialization"),
530            )
531        };
532        Ok(SharedSpill {
533            inner: Arc::new(SharedSpillInner {
534                storage,
535                schema,
536                rows,
537                max_record_bytes: self.max_spilled_record_bytes,
538                range: self.spill_segment.and_then(|segment| segment.range),
539            }),
540        })
541    }
542
543    fn create_spill_file(&self) -> ExecResult<NamedTempFile> {
544        file::create(self.spill_directory.as_deref())
545    }
546
547    fn spill_range(&self) -> Option<&Range<u64>> {
548        self.spill_segment
549            .as_ref()
550            .and_then(|segment| segment.range.as_ref())
551    }
552
553    fn retain_batch(&mut self, batch: Batch, encoded_bytes: usize) {
554        self.rows = self.rows.saturating_add(batch.rows.len());
555        self.in_memory_rows = self.in_memory_rows.saturating_add(batch.rows.len());
556        self.in_memory_bytes = self.in_memory_bytes.saturating_add(encoded_bytes);
557        self.max_in_memory_record_bytes = self.max_in_memory_record_bytes.max(encoded_bytes);
558        self.batches.push(batch);
559    }
560}
561
562enum SharedSpillStorage {
563    Memory(Vec<Batch>),
564    Disk(NamedTempFile),
565}
566
567struct SharedSpillInner {
568    storage: SharedSpillStorage,
569    schema: RowSchema,
570    rows: usize,
571    max_record_bytes: usize,
572    range: Option<Range<u64>>,
573}
574
575/// Immutable repeatable row materialization bounded by the source buffer's
576/// memory budget and backed by a temporary file after that budget is exceeded.
577#[derive(Clone)]
578pub struct SharedSpill {
579    inner: Arc<SharedSpillInner>,
580}
581
582impl SharedSpill {
583    pub fn schema(&self) -> &[String] {
584        self.inner.schema.columns()
585    }
586
587    pub fn row_schema(&self) -> &RowSchema {
588        &self.inner.schema
589    }
590
591    pub fn rows(&self) -> usize {
592        self.inner.rows
593    }
594
595    /// Whether this materialization crossed its memory budget and uses disk.
596    pub fn has_spilled(&self) -> bool {
597        matches!(self.inner.storage, SharedSpillStorage::Disk(_))
598    }
599
600    pub fn reader(&self) -> ExecResult<SharedSpillReader> {
601        let source = Arc::clone(&self.inner);
602        Self::reader_from_source(source)
603    }
604
605    /// Consume this materialization into a one-shot reader.
606    ///
607    /// When the in-memory materialization has no other owners, batches move
608    /// directly into the reader instead of being deep-cloned. Shared and disk
609    /// materializations retain the independent-reader behavior of [`Self::reader`].
610    pub fn into_reader(self) -> ExecResult<SharedSpillReader> {
611        match Arc::try_unwrap(self.inner) {
612            Ok(SharedSpillInner {
613                storage: SharedSpillStorage::Memory(batches),
614                schema,
615                max_record_bytes,
616                ..
617            }) => Ok(SharedSpillReader {
618                reader: SharedSpillReaderSource::OwnedMemory(batches.into_iter()),
619                source: None,
620                failed: false,
621                max_record_bytes,
622                expected_schema: Some(schema),
623            }),
624            Ok(inner) => Self::reader_from_source(Arc::new(inner)),
625            Err(source) => Self::reader_from_source(source),
626        }
627    }
628
629    fn reader_from_source(source: Arc<SharedSpillInner>) -> ExecResult<SharedSpillReader> {
630        let reader = match &source.storage {
631            SharedSpillStorage::Memory(_) => SharedSpillReaderSource::Memory { next_batch: 0 },
632            SharedSpillStorage::Disk(file) => {
633                SharedSpillReaderSource::Disk(open_spill_reader(file, source.range.as_ref())?)
634            }
635        };
636        let max_record_bytes = source.max_record_bytes;
637        let expected_schema = Some(source.schema.clone());
638        Ok(SharedSpillReader {
639            reader,
640            source: Some(source),
641            failed: false,
642            max_record_bytes,
643            expected_schema,
644        })
645    }
646
647    /// Open an independent physical-row reader without collecting the spill's
648    /// batches or row count in memory.
649    pub fn read_rows(&self) -> ExecResult<SpillRows<SharedSpillReader>> {
650        self.reader().map(SpillRows::new)
651    }
652}
653
654enum SharedSpillReaderSource {
655    Memory { next_batch: usize },
656    OwnedMemory(std::vec::IntoIter<Batch>),
657    Disk(BufReader<Take<File>>),
658}
659
660fn validate_decoded_schema(batch: Batch, expected_schema: Option<&RowSchema>) -> ExecResult<Batch> {
661    if expected_schema.is_none_or(|expected| expected == &batch.schema) {
662        return Ok(batch);
663    }
664    let expected = expected_schema.expect("schema presence checked above");
665    Err(spill_error(format!(
666        "spill batch schema mismatch: expected {:?}, got {:?}",
667        expected.columns(),
668        batch.schema.columns()
669    )))
670}
671
672fn restore_decoded_schema(
673    mut batch: Batch,
674    expected_schema: Option<&RowSchema>,
675) -> ExecResult<Batch> {
676    // The file carries physical bindings; the buffer owns the original immutable SQL scope, including named rows and record descriptors that are not repeated in each record.
677    if let Some(expected) = expected_schema {
678        if expected.has_same_physical_layout(&batch.schema) {
679            batch.schema = expected.clone();
680        }
681    }
682    validate_decoded_schema(batch, expected_schema)
683}
684
685/// Reader for a [`SharedSpill`]. Independent readers retain the shared source;
686/// a consuming reader may instead own unique in-memory batches directly.
687pub struct SharedSpillReader {
688    reader: SharedSpillReaderSource,
689    source: Option<Arc<SharedSpillInner>>,
690    failed: bool,
691    max_record_bytes: usize,
692    expected_schema: Option<RowSchema>,
693}
694
695impl Iterator for SharedSpillReader {
696    type Item = ExecResult<Batch>;
697
698    fn next(&mut self) -> Option<Self::Item> {
699        if self.failed {
700            return None;
701        }
702        match &mut self.reader {
703            SharedSpillReaderSource::Memory { next_batch } => {
704                let source = self
705                    .source
706                    .as_ref()
707                    .expect("shared memory reader retains its source");
708                let SharedSpillStorage::Memory(batches) = &source.storage else {
709                    unreachable!("shared materialization reader/storage mismatch")
710                };
711                let batch = batches.get(*next_batch)?.clone();
712                *next_batch += 1;
713                Some(validate_decoded_schema(
714                    batch,
715                    self.expected_schema.as_ref(),
716                ))
717            }
718            SharedSpillReaderSource::OwnedMemory(batches) => batches
719                .next()
720                .map(|batch| validate_decoded_schema(batch, self.expected_schema.as_ref())),
721            SharedSpillReaderSource::Disk(reader) => {
722                match read_bounded_spill_record(reader, self.max_record_bytes, "shared spill batch")
723                {
724                    Ok(None) => None,
725                    Ok(Some(record)) => {
726                        let decoded = decode_batch(&record).and_then(|batch| {
727                            restore_decoded_schema(batch, self.expected_schema.as_ref())
728                        });
729                        if decoded.is_err() {
730                            self.failed = true;
731                        }
732                        Some(decoded)
733                    }
734                    Err(error) => {
735                        self.failed = true;
736                        Some(Err(spill_error(format!(
737                            "failed to read shared spill batch: {error}"
738                        ))))
739                    }
740                }
741            }
742        }
743    }
744}
745
746/// Restoring iterator returned by [`SpillBuffer::drain`].
747pub struct SpillDrain {
748    reader: Option<BufReader<Take<File>>>,
749    // Keep the named file alive until disk iteration finishes or the iterator
750    // is dropped. Its Drop implementation unlinks the temporary file.
751    spill_file: Option<NamedTempFile>,
752    memory: std::vec::IntoIter<Batch>,
753    disk_finished: bool,
754    failed: bool,
755    max_record_bytes: usize,
756    expected_schema: Option<RowSchema>,
757}
758
759impl Iterator for SpillDrain {
760    type Item = ExecResult<Batch>;
761
762    fn next(&mut self) -> Option<Self::Item> {
763        if self.failed {
764            return None;
765        }
766
767        if !self.disk_finished {
768            let Some(reader) = self.reader.as_mut() else {
769                self.failed = true;
770                self.disk_finished = true;
771                return Some(Err(spill_error(
772                    "spill drain entered disk phase without a reader",
773                )));
774            };
775            match read_bounded_spill_record(reader, self.max_record_bytes, "spill batch") {
776                Ok(None) => {
777                    self.disk_finished = true;
778                    self.reader = None;
779                    self.spill_file = None;
780                }
781                Ok(Some(record)) => {
782                    let decoded = decode_batch(&record).and_then(|batch| {
783                        restore_decoded_schema(batch, self.expected_schema.as_ref())
784                    });
785                    if decoded.is_err() {
786                        self.failed = true;
787                    }
788                    return Some(decoded);
789                }
790                Err(error) => {
791                    self.failed = true;
792                    return Some(Err(spill_error(format!(
793                        "failed to read spill batch: {error}"
794                    ))));
795                }
796            }
797        }
798
799        self.memory
800            .next()
801            .map(|batch| validate_decoded_schema(batch, self.expected_schema.as_ref()))
802    }
803
804    fn size_hint(&self) -> (usize, Option<usize>) {
805        let lower = if self.disk_finished {
806            self.memory.len()
807        } else {
808            0
809        };
810        (lower, None)
811    }
812}
813
814/// Repeatable, non-consuming batch reader returned by [`SpillBuffer::reader`].
815pub struct SpillReader<'a> {
816    reader: Option<BufReader<Take<File>>>,
817    memory: std::slice::Iter<'a, Batch>,
818    disk_finished: bool,
819    failed: bool,
820    max_record_bytes: usize,
821    expected_schema: Option<RowSchema>,
822}
823
824impl Iterator for SpillReader<'_> {
825    type Item = ExecResult<Batch>;
826
827    fn next(&mut self) -> Option<Self::Item> {
828        if self.failed {
829            return None;
830        }
831
832        if !self.disk_finished {
833            let Some(reader) = self.reader.as_mut() else {
834                self.failed = true;
835                self.disk_finished = true;
836                return Some(Err(spill_error(
837                    "spill reader entered disk phase without a file reader",
838                )));
839            };
840            match read_bounded_spill_record(reader, self.max_record_bytes, "spill batch") {
841                Ok(None) => {
842                    self.disk_finished = true;
843                    self.reader = None;
844                }
845                Ok(Some(record)) => {
846                    let decoded = decode_batch(&record).and_then(|batch| {
847                        restore_decoded_schema(batch, self.expected_schema.as_ref())
848                    });
849                    if decoded.is_err() {
850                        self.failed = true;
851                    }
852                    return Some(decoded);
853                }
854                Err(error) => {
855                    self.failed = true;
856                    return Some(Err(spill_error(format!(
857                        "failed to read spill batch: {error}"
858                    ))));
859                }
860            }
861        }
862
863        self.memory
864            .next()
865            .cloned()
866            .map(|batch| validate_decoded_schema(batch, self.expected_schema.as_ref()))
867    }
868
869    fn size_hint(&self) -> (usize, Option<usize>) {
870        let lower = if self.disk_finished {
871            self.memory.len()
872        } else {
873            0
874        };
875        (lower, None)
876    }
877}
878
879/// Physical-row flattening adapter for [`SpillReader`] and [`SpillDrain`].
880pub struct SpillRows<I> {
881    batches: I,
882    current_schema: Option<RowSchema>,
883    current: std::vec::IntoIter<PhysicalRow>,
884}
885
886impl<I> SpillRows<I> {
887    pub(crate) fn new(batches: I) -> Self {
888        Self {
889            batches,
890            current_schema: None,
891            current: Vec::new().into_iter(),
892        }
893    }
894}
895
896impl<I> Iterator for SpillRows<I>
897where
898    I: Iterator<Item = ExecResult<Batch>>,
899{
900    type Item = ExecResult<OwnedPhysicalRow>;
901
902    fn next(&mut self) -> Option<Self::Item> {
903        loop {
904            if let Some(row) = self.current.next() {
905                let schema = self
906                    .current_schema
907                    .as_ref()
908                    .expect("spill row iterator retains the current batch schema")
909                    .clone();
910                return Some(Ok(OwnedPhysicalRow::new(schema, row)));
911            }
912            match self.batches.next()? {
913                Ok(batch) => {
914                    self.current_schema = Some(batch.schema);
915                    self.current = batch.rows.into_iter();
916                }
917                Err(error) => return Some(Err(error)),
918            }
919        }
920    }
921}
922
923#[cfg(test)]
924mod tests;