pub struct ExecutionPlan {
pub field_map: FxHashMap<String, String>,
pub drop_fields: FxHashSet<String>,
pub field_types: FxHashMap<String, FieldType>,
pub dictionary_columns: FxHashSet<String>,
pub filter: Option<FilterPredicate>,
pub schema_order: Vec<String>,
pub auto_dict: bool,
}Expand description
A compiled execution plan that controls field renaming, dropping, type assignment, dictionary encoding, row filtering, and column ordering. Default (empty) is a no-op.
Fields§
§field_map: FxHashMap<String, String>Map from raw field name to output column name.
drop_fields: FxHashSet<String>Set of raw field names to drop entirely.
field_types: FxHashMap<String, FieldType>Explicit type overrides per output column name.
dictionary_columns: FxHashSet<String>Set of output column names to dict-encode.
filter: Option<FilterPredicate>Optional row filter predicate.
schema_order: Vec<String>Desired output column order (names in order). Columns not listed here appear after all listed columns in first-appearance order. If empty, first-appearance order is used.
auto_dict: boolWhen true, string columns with low cardinality are automatically upgraded to dictionary encoding during parse.
Implementations§
Source§impl ExecutionPlan
impl ExecutionPlan
Sourcepub fn new() -> Self
pub fn new() -> Self
Examples found in repository?
103fn main() -> Result<()> {
104 let data = generate_data(ROWS);
105 let bytes = data.len();
106 let mb = bytes as f64 / 1_000_000.0;
107 println!("Generated {ROWS} rows ({mb:.1} MB)");
108
109 let mut path = std::env::temp_dir();
110 path.push("rypipe_bench_throughput.tsv");
111 File::create(&path)?.write_all(&data)?;
112
113 let pipeline = Pipeline::new(TsvSplitter, TsvParser).with_plan(
114 rypipe_core::ExecutionPlan::new()
115 .type_as("id", FieldType::Int64)
116 .type_as("amount", FieldType::Float64)
117 .type_as("count", FieldType::Int64),
118 );
119
120 run("single-thread", || {
121 let batch = pipeline.read_path(&path, false, false)?;
122 Ok(batch.num_rows())
123 });
124
125 run("parallel (4 chunks)", || {
126 let batches = pipeline.read_path_par(&path, 4, false, false)?;
127 Ok(batches.iter().map(|b| b.num_rows()).sum::<usize>())
128 });
129
130 run("parallel (8 chunks)", || {
131 let batches = pipeline.read_path_par(&path, 8, false, false)?;
132 Ok(batches.iter().map(|b| b.num_rows()).sum::<usize>())
133 });
134
135 run("bounded (64 MiB)", || {
136 let batches =
137 pipeline.read_path_stream(&path, MemoryBudget::new(64 * 1024 * 1024), false)?;
138 Ok(batches.iter().map(|b| b.num_rows()).sum::<usize>())
139 });
140
141 std::fs::remove_file(&path)?;
142 Ok(())
143}Sourcepub fn rename(self, raw: impl Into<String>, output: impl Into<String>) -> Self
pub fn rename(self, raw: impl Into<String>, output: impl Into<String>) -> Self
Rename a raw field to an output column name.
Sourcepub fn type_as(self, field: impl Into<String>, field_type: FieldType) -> Self
pub fn type_as(self, field: impl Into<String>, field_type: FieldType) -> Self
Set the storage type for an output column.
Examples found in repository?
103fn main() -> Result<()> {
104 let data = generate_data(ROWS);
105 let bytes = data.len();
106 let mb = bytes as f64 / 1_000_000.0;
107 println!("Generated {ROWS} rows ({mb:.1} MB)");
108
109 let mut path = std::env::temp_dir();
110 path.push("rypipe_bench_throughput.tsv");
111 File::create(&path)?.write_all(&data)?;
112
113 let pipeline = Pipeline::new(TsvSplitter, TsvParser).with_plan(
114 rypipe_core::ExecutionPlan::new()
115 .type_as("id", FieldType::Int64)
116 .type_as("amount", FieldType::Float64)
117 .type_as("count", FieldType::Int64),
118 );
119
120 run("single-thread", || {
121 let batch = pipeline.read_path(&path, false, false)?;
122 Ok(batch.num_rows())
123 });
124
125 run("parallel (4 chunks)", || {
126 let batches = pipeline.read_path_par(&path, 4, false, false)?;
127 Ok(batches.iter().map(|b| b.num_rows()).sum::<usize>())
128 });
129
130 run("parallel (8 chunks)", || {
131 let batches = pipeline.read_path_par(&path, 8, false, false)?;
132 Ok(batches.iter().map(|b| b.num_rows()).sum::<usize>())
133 });
134
135 run("bounded (64 MiB)", || {
136 let batches =
137 pipeline.read_path_stream(&path, MemoryBudget::new(64 * 1024 * 1024), false)?;
138 Ok(batches.iter().map(|b| b.num_rows()).sum::<usize>())
139 });
140
141 std::fs::remove_file(&path)?;
142 Ok(())
143}Sourcepub fn dictionary(self, field: impl Into<String>) -> Self
pub fn dictionary(self, field: impl Into<String>) -> Self
Dict-encode an output column.
Sourcepub fn filter_eq(
self,
field: impl Into<String>,
value: impl Into<String>,
) -> Self
pub fn filter_eq( self, field: impl Into<String>, value: impl Into<String>, ) -> Self
Keep only rows where field == value (string comparison, per-row).
Sourcepub fn filter_ne(
self,
field: impl Into<String>,
value: impl Into<String>,
) -> Self
pub fn filter_ne( self, field: impl Into<String>, value: impl Into<String>, ) -> Self
Keep only rows where field != value (string comparison, per-row).
Sourcepub fn filter_compare(
self,
field_a: impl Into<String>,
op: CompareOp,
field_b: impl Into<String>,
) -> Self
pub fn filter_compare( self, field_a: impl Into<String>, op: CompareOp, field_b: impl Into<String>, ) -> Self
Keep only rows where field_a op field_b (post-reduce via Arrow compute).
Sourcepub fn schema_order<I, S>(self, order: I) -> Self
pub fn schema_order<I, S>(self, order: I) -> Self
Set the output column order. Columns not listed appear after the listed ones in first-appearance order.
Sourcepub fn with_auto_dict(self, yes: bool) -> Self
pub fn with_auto_dict(self, yes: bool) -> Self
Enable or disable automatic dictionary upgrade for low-cardinality string columns.
Sourcepub fn column_type(&self, name: &str) -> FieldType
pub fn column_type(&self, name: &str) -> FieldType
Determine the storage type for an output column name.
Sourcepub fn resolve_field<'a>(&'a self, raw: &'a str) -> Option<&'a str>
pub fn resolve_field<'a>(&'a self, raw: &'a str) -> Option<&'a str>
Resolve a raw field name to its output column name.
Returns None if the field should be dropped.
Application order: rename first, then drop, matching left-to-right pipeline semantics.
Trait Implementations§
Source§impl Clone for ExecutionPlan
impl Clone for ExecutionPlan
Source§fn clone(&self) -> ExecutionPlan
fn clone(&self) -> ExecutionPlan
1.0.0 (const: unstable) · Source§fn clone_from(&mut self, source: &Self)
fn clone_from(&mut self, source: &Self)
source. Read moreSource§impl Debug for ExecutionPlan
impl Debug for ExecutionPlan
Source§impl Default for ExecutionPlan
impl Default for ExecutionPlan
Source§fn default() -> ExecutionPlan
fn default() -> ExecutionPlan
Auto Trait Implementations§
impl Freeze for ExecutionPlan
impl RefUnwindSafe for ExecutionPlan
impl Send for ExecutionPlan
impl Sync for ExecutionPlan
impl Unpin for ExecutionPlan
impl UnsafeUnpin for ExecutionPlan
impl UnwindSafe for ExecutionPlan
Blanket Implementations§
impl<T> Allocation for T
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
impl<ST, DT> CastableFrom<ST, Initialized, Initialized> for DT
impl<ST, DT> CastableFrom<ST, Uninit, Uninit> for DT
Source§impl<T> CloneToUninit for Twhere
T: Clone,
impl<T> CloneToUninit for Twhere
T: Clone,
Source§impl<T> IntoEither for T
impl<T> IntoEither for T
Source§fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ
fn into_either(self, into_left: bool) -> Either<Self, Self> ⓘ
self into a Left variant of Either<Self, Self>
if into_left is true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read moreSource§fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
fn into_either_with<F>(self, into_left: F) -> Either<Self, Self> ⓘ
self into a Left variant of Either<Self, Self>
if into_left(&self) returns true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read more