linkrs_query/executor/streaming/
chunk.rs

1//! DataChunk: Basic unit of streaming execution
2//!
3//! A DataChunk represents a fixed-size batch of rows processed in streaming mode.
4//! Typical size: 2048 rows (~8MB, OLAP vectorized batch)
5//!
6//! # Ownership & Memory Accounting Rules
7//!
8//! - **`rows`**: Owned `Vec<Vec<Value>>`. `Clone` performs an explicit deep
9//!   copy (there is no shallow-clone path).
10//! - **`schema`**: `Arc<Schema>` — shared reference, cheap `Arc::clone` on `Clone`.
11//! - **`layout`**: `Arc<SlotLayout>` — always present; shared reference, cheap on `Clone`.
12//! - **`memory_reservation`**: `Option<MemoryReservation>` / `Option<MemoryPoolReservation>` —
13//!   ownership stays with the original chunk on `Clone`; the clone gets `None`.
14//!   Use `take_memory_reservation()` to transfer ownership explicitly.
15//!
16//! ## Move-first discipline
17//!
18//! New code moves rows instead of cloning them:
19//! - **`expand_visible_rows()`**: the single terminal move (visible rows out,
20//!   expanded by `multiplicity`); used by `LocalChunkCollector` and every
21//!   `collect` path.
22//! - **`visible_rows()`**: the single borrow of visible rows (selection aware).
23//! - **`take_indices`/`slice`**: move a subset of rows into a new chunk
24//!   (efficient, uses `std::mem::take` per row), preserving `multiplicity`.
25//! - **`Clone`**: the single explicit deep copy. It drops memory reservations
26//!   and the deferred-build flag; the new owner must re-account memory via its
27//!   own pool/tracker.
28//!
29//! # Construction Paths
30//!
31//! - `new_with_layout(rows, layout)` — **Production path**. Schema is derived from
32//!   layout slot metadata. Always preferred when a `SlotLayout` is available.
33//! - `new(rows, schema)` — Schema-driven path. Layout auto-created from column names.
34//! - `from_rows(rows)` / `from_rows_with_col_names(rows, col_names)` — Convenience
35//!   constructors for tests. Always produce a layout (auto-created).
36//! - `project_columns(columns, layout)` — Designated `Project` output path; rebuilds
37//!   the typed layout unconditionally because the output is chunk-bounded.
38//!   `from_columns` is the crate-private row-major fixture behind it.
39//!
40//! # Columnar gating
41//!
42//! Fresh column sets from storage scans and join outputs honor
43//! `use_columnar_path`; small 1:1 rebuilds (project, assign) rebuild
44//! unconditionally by design. Blocking accumulation uses `ColumnarBatch`
45//! (typed) for sort/topn and `MaterializedBatch` (spill form) elsewhere.
46
47mod collector;
48mod columnar_batch;
49mod columnar_common;
50mod core;
51mod eval;
52mod kind;
53mod policy;
54mod pool;
55mod schema;
56mod selection;
57mod typed;
58
59// Re-export public API
60pub use collector::{LocalChunkCollector, COLLECTOR_BLOCK_ROWS};
61pub use columnar_batch::{BatchColumn, ColumnarBatch};
62pub use columnar_common::EdgeHeaderColumn;
63pub use core::DataChunk;
64pub(crate) use policy::use_columnar_path;
65pub use policy::{ColumnarPolicy, QueryColumnarOverride};
66pub use pool::{RowBufferPool, MAX_POOLED_ROWS, MAX_POOLED_ROW_CAPACITY};
67pub use schema::{ColumnInfo, Schema};
68pub(crate) use typed::gather_typed_column;
69pub use typed::{TypedColumn, TypedKind};
70
71/// OLAP vectorized batch size.
72pub const VECTORIZED_BATCH_SIZE: usize = 2048;
73/// Alias for the default chunk size (kept in sync with `ExecutionContext::DEFAULT_CHUNK_SIZE`).
74pub const DEFAULT_CHUNK_SIZE: usize = VECTORIZED_BATCH_SIZE;
75
76#[cfg(test)]
77mod tests;
linkrs_query/executor/streaming/chunk.rs

linkrs_query/executor/streaming/
chunk.rs
