1use std::sync::Arc;
2
3use arrow::datatypes::{Field as ArrowField, Schema};
4use arrow::record_batch::RecordBatch;
5use rustc_hash::FxHashMap as HashMap;
6
7use crate::columnar::ColumnBuilder;
8use crate::decoder::ColumnarSink;
9use crate::plan::ExecutionPlan;
10use crate::value::Value;
11use crate::Result;
12
13pub struct TableBuilder {
17 pub(crate) columns: HashMap<String, ColumnBuilder>,
18 pub(crate) column_order: Vec<String>,
19 pub(crate) row_count: usize,
20 pub(crate) estimated_rows: usize,
21 pub(crate) plan: ExecutionPlan,
22}
23
24impl TableBuilder {
25 pub fn new() -> Self {
26 Self {
27 columns: HashMap::default(),
28 column_order: Vec::new(),
29 row_count: 0,
30 estimated_rows: 0,
31 plan: ExecutionPlan::new(),
32 }
33 }
34
35 pub fn with_capacity(cap: usize) -> Self {
36 Self {
37 columns: HashMap::default(),
38 column_order: Vec::new(),
39 row_count: 0,
40 estimated_rows: cap,
41 plan: ExecutionPlan::new(),
42 }
43 }
44
45 pub fn with_plan(cap: usize, plan: ExecutionPlan) -> Self {
46 Self {
47 columns: HashMap::default(),
48 column_order: Vec::new(),
49 row_count: 0,
50 estimated_rows: cap,
51 plan,
52 }
53 }
54
55 pub fn num_rows(&self) -> usize {
56 self.row_count
57 }
58
59 pub fn num_columns(&self) -> usize {
60 self.column_order.len()
61 }
62
63 pub fn column_names(&self) -> &[String] {
64 &self.column_order
65 }
66
67 pub fn finish(&mut self) -> Result<RecordBatch> {
71 ColumnarSink::finish(self)
72 }
73
74 pub fn reset(&mut self) {
76 self.columns.clear();
77 self.column_order.clear();
78 self.row_count = 0;
79 }
80
81 pub fn normalize(&mut self) {
84 for b in self.columns.values_mut() {
85 while b.len() > self.row_count {
86 b.pop();
87 }
88 }
89 }
90
91 pub fn auto_dict_upgrade(&mut self) {
93 if self.plan.auto_dict {
94 for b in self.columns.values_mut() {
95 b.try_upgrade_to_dict(512);
96 }
97 }
98 }
99
100 pub fn sort_columns(&mut self) {
104 if self.plan.schema_order.is_empty() {
105 return;
106 }
107 let order = &self.plan.schema_order;
108 let rank = |name: &String| order.iter().position(|n| n == name).unwrap_or(usize::MAX);
109 self.column_order.sort_by_key(rank);
110 }
111
112 pub(crate) fn schema_insert_index(&self, name: &str) -> usize {
113 let order = &self.plan.schema_order;
114 if order.is_empty() {
115 return self.column_order.len();
116 }
117 let pos = order.iter().position(|n| n == name);
118 match pos {
119 Some(p) => self
120 .column_order
121 .iter()
122 .position(|existing| {
123 order
124 .iter()
125 .position(|n| n == existing)
126 .is_some_and(|ep| ep > p)
127 })
128 .unwrap_or(self.column_order.len()),
129 None => self.column_order.len(),
130 }
131 }
132
133 fn ensure_column(&mut self, name: &str) {
134 if !self.columns.contains_key(name) {
135 let est = self.estimated_rows.max(64);
136 let col_type = self.plan.column_type(name);
137 let mut b = ColumnBuilder::with_capacity(est, &col_type);
138 for _ in 0..self.row_count {
139 b.push(None);
140 }
141 self.columns.insert(name.to_owned(), b);
142 let idx = self.schema_insert_index(name);
143 self.column_order.insert(idx, name.to_owned());
144 }
145 }
146
147 fn push_field(&mut self, name: &str, value: Value<'_>) {
150 let owned;
152 let resolved: &str = if self.plan.field_map.is_empty() && self.plan.drop_fields.is_empty() {
153 name
154 } else {
155 match self.plan.resolve_field(name) {
156 Some(n) => {
157 owned = n.to_owned();
158 &owned
159 }
160 None => return,
161 }
162 };
163
164 self.ensure_column(resolved);
165 let row_count = self.row_count;
166 if let Some(b) = self.columns.get_mut(resolved) {
167 if b.len() > row_count {
168 b.pop();
169 }
170 b.push_value(value);
171 }
172 }
173
174 fn finish_row(&mut self) {
177 let target = self.row_count + 1;
178 for b in self.columns.values_mut() {
179 while b.len() < target {
180 b.push(None);
181 }
182 }
183
184 if let Some(ref filter) = self.plan.filter {
185 if !filter.check(&self.columns, self.row_count, &self.plan) {
186 for b in self.columns.values_mut() {
187 b.pop();
188 }
189 return;
190 }
191 }
192
193 self.row_count += 1;
194 }
195}
196
197impl Default for TableBuilder {
198 fn default() -> Self {
199 Self::new()
200 }
201}
202
203impl ColumnarSink for TableBuilder {
204 fn begin_row(&mut self) {
205 }
207
208 fn put_field(&mut self, name: &str, value: Value<'_>) {
209 self.push_field(name, value);
210 }
211
212 fn end_row(&mut self) {
213 self.finish_row();
214 }
215
216 fn wants(&self, name: &str) -> bool {
217 self.plan.resolve_field(name).is_some()
218 }
219
220 fn finish(&mut self) -> Result<RecordBatch> {
221 self.normalize();
222
223 if self.column_order.is_empty() {
224 let schema = Arc::new(Schema::empty());
225 return Ok(RecordBatch::try_new(schema, Vec::new())?);
226 }
227
228 self.auto_dict_upgrade();
229 self.sort_columns();
230
231 let mut fields = Vec::with_capacity(self.column_order.len());
232 let mut arrays = Vec::with_capacity(self.column_order.len());
233 for name in &self.column_order {
234 if let Some(b) = self.columns.get(name) {
235 fields.push(ArrowField::new(name.as_str(), b.arrow_datatype(), true));
236 arrays.push(b.to_arrow_array()?);
237 }
238 }
239
240 let schema = Arc::new(Schema::new(fields));
241 Ok(RecordBatch::try_new(schema, arrays)?)
242 }
243}
244
245#[cfg(test)]
246mod tests {
247 use super::*;
248 use crate::columnar::ColumnBuilder;
249 use crate::decoder::{ColumnarSink, RecordParser, Splitter};
250 use crate::plan::{CompareOp, ExecutionPlan, FieldType, FilterPredicate};
251 use crate::value::Value;
252 use crate::Result;
253 use arrow::array::AsArray;
254
255 struct LineParser;
258
259 impl RecordParser for LineParser {
260 fn validate(&self, bytes: &[u8]) -> Result<()> {
261 simdutf8::basic::from_utf8(bytes)?;
262 Ok(())
263 }
264
265 fn parse_chunk(&self, bytes: &[u8], sink: &mut dyn ColumnarSink) -> Result<()> {
266 let text = std::str::from_utf8(bytes).map_err(|e| crate::Error::Plan(e.to_string()))?;
267 for line in text.lines() {
268 if line.is_empty() {
269 continue;
270 }
271 sink.begin_row();
272 for token in line.split_whitespace() {
273 if let Some((k, v)) = token.split_once('=') {
274 sink.put_field(k, Value::Str(v));
275 }
276 }
277 sink.end_row();
278 }
279 Ok(())
280 }
281 }
282
283 struct LineSplitter;
284
285 impl Splitter for LineSplitter {
286 fn find_split_points(&self, bytes: &[u8], max_chunks: usize) -> Vec<usize> {
287 if max_chunks <= 1 || bytes.is_empty() {
288 return vec![0, bytes.len()];
289 }
290 let mut points = vec![0usize];
291 let mut last = 0;
292 for (i, &b) in bytes.iter().enumerate() {
293 if b == b'\n' {
294 let next = i + 1;
295 if next > last && points.len() < max_chunks {
296 points.push(next);
297 last = next;
298 }
299 }
300 }
301 if *points.last().unwrap() != bytes.len() {
302 points.push(bytes.len());
303 }
304 points
305 }
306
307 fn estimate_bytes_per_row(&self, sample: &[u8]) -> usize {
308 let newline_count = sample.iter().filter(|&&b| b == b'\n').count().max(1);
309 (sample.len() / newline_count).max(1)
310 }
311 }
312
313 fn parse_bytes(bytes: &[u8], plan: ExecutionPlan) -> TableBuilder {
314 let mut sink = TableBuilder::with_plan((bytes.len() / 16).max(4), plan);
315 LineParser.parse_chunk(bytes, &mut sink).unwrap();
316 sink
317 }
318
319 #[test]
320 fn test_extend_no_duplicates() {
321 let e1 = parse_bytes(b"A=1 B=2\n", ExecutionPlan::new());
322 let mut merged = TableBuilder::new();
323 merged.extend(e1).unwrap();
324 assert_eq!(merged.num_rows(), 1);
325 for (name, col) in &merged.columns {
326 assert_eq!(
327 col.len(),
328 merged.num_rows(),
329 "column {} length mismatch",
330 name
331 );
332 }
333 }
334
335 #[test]
336 fn test_multi_chunk_same_as_single() {
337 let data = b"A=1\nA=2\nA=3\n";
338 let single = parse_bytes(data, ExecutionPlan::new());
339 assert_eq!(single.num_rows(), 3);
340
341 let splitter = LineSplitter;
342 let points = splitter.find_split_points(data, 2);
343 let mut merged = TableBuilder::new();
344 for w in points.windows(2) {
345 let chunk = &data[w[0]..w[1]];
346 let engine = parse_bytes(chunk, ExecutionPlan::new());
347 merged.extend(engine).unwrap();
348 }
349 assert_eq!(merged.num_rows(), single.num_rows());
350 }
351
352 #[test]
353 fn test_last_write_wins_duplicate_field() {
354 let engine = parse_bytes(b"X=10 X=20\n", ExecutionPlan::new());
355 assert_eq!(engine.num_rows(), 1);
356 let col = engine.columns.get("X").unwrap();
357 assert_eq!(col.as_str_vec(), vec![Some("20".into())]);
358 }
359
360 #[test]
361 fn test_build_plan_rename() {
362 let mut plan = ExecutionPlan::new();
363 plan.field_map.insert("X".to_string(), "Y".to_string());
364 let engine = parse_bytes(b"X=hello\n", plan);
365 assert_eq!(engine.num_rows(), 1);
366 assert!(engine.columns.contains_key("Y"));
367 assert!(!engine.columns.contains_key("X"));
368 assert_eq!(
369 engine.columns.get("Y").unwrap().as_str_vec(),
370 vec![Some("hello".into())]
371 );
372 }
373
374 #[test]
375 fn test_build_plan_drop() {
376 let mut plan = ExecutionPlan::new();
377 plan.drop_fields.insert("X".to_string());
378 let engine = parse_bytes(b"X=hello Y=world\n", plan);
379 assert_eq!(engine.num_rows(), 1);
380 assert!(!engine.columns.contains_key("X"));
381 assert!(engine.columns.contains_key("Y"));
382 }
383
384 #[test]
385 fn test_build_plan_filter_ne() {
386 let mut plan = ExecutionPlan::new();
387 plan.filter = Some(FilterPredicate::NotEqual {
388 field: "X".to_string(),
389 value: "42".to_string(),
390 });
391 let engine = parse_bytes(b"X=10\nX=42\nX=30\n", plan);
392 assert_eq!(engine.num_rows(), 2);
393 let col = engine.columns.get("X").unwrap();
394 assert_eq!(col.as_str_vec(), vec![Some("10".into()), Some("30".into())]);
395 }
396
397 #[test]
398 fn test_build_plan_filter_eq() {
399 let mut plan = ExecutionPlan::new();
400 plan.filter = Some(FilterPredicate::Equal {
401 field: "X".to_string(),
402 value: "10".to_string(),
403 });
404 let engine = parse_bytes(b"X=10\nX=20\nX=10\n", plan);
405 assert_eq!(engine.num_rows(), 2);
406 let col = engine.columns.get("X").unwrap();
407 assert_eq!(col.as_str_vec(), vec![Some("10".into()), Some("10".into())]);
408 }
409
410 #[test]
411 fn test_build_plan_filter_missing_field() {
412 let mut plan = ExecutionPlan::new();
413 plan.filter = Some(FilterPredicate::NotEqual {
414 field: "X".to_string(),
415 value: "10".to_string(),
416 });
417 let engine = parse_bytes(b"X=10\nY=99\n", plan);
418 assert_eq!(engine.num_rows(), 1);
419 let col = engine.columns.get("Y").unwrap();
420 assert_eq!(col.as_str_vec(), vec![Some("99".into())]);
421 }
422
423 #[test]
424 fn test_typed_int64_column() {
425 let mut plan = ExecutionPlan::new();
426 plan.field_types.insert("X".to_string(), FieldType::Int64);
427 let engine = parse_bytes(b"X=42\nX=bad\nX=100\n", plan);
428 assert_eq!(engine.num_rows(), 3);
429 if let ColumnBuilder::Int64(v) = &engine.columns["X"] {
430 assert_eq!(v, &vec![Some(42), None, Some(100)]);
431 } else {
432 panic!("expected Int64 builder");
433 }
434 }
435
436 #[test]
437 fn test_typed_float64_column() {
438 let mut plan = ExecutionPlan::new();
439 plan.field_types.insert("X".to_string(), FieldType::Float64);
440 let engine = parse_bytes(b"X=1.5\n", plan);
441 if let ColumnBuilder::Float64(v) = &engine.columns["X"] {
442 assert!((v[0].unwrap() - 1.5).abs() < 1e-9);
443 } else {
444 panic!("expected Float64 builder");
445 }
446 }
447
448 #[test]
449 fn test_dictionary_column() {
450 let mut plan = ExecutionPlan::new();
451 plan.dictionary_columns.insert("P".to_string());
452 let engine = parse_bytes(b"P=Widget\nP=Gadget\nP=Widget\n", plan);
453 assert_eq!(engine.num_rows(), 3);
454 if let ColumnBuilder::Dictionary { codes, dict, .. } = &engine.columns["P"] {
455 assert_eq!(dict.len(), 2);
456 assert_eq!(codes, &vec![Some(0), Some(1), Some(0)]);
457 } else {
458 panic!("expected Dictionary builder");
459 }
460 }
461
462 #[test]
463 fn test_ragged_late_chunk_column_debut() {
464 let e1 = parse_bytes(b"A=1 B=2\nA=3\n", ExecutionPlan::new());
465 let e2 = parse_bytes(b"B=4 C=5\n", ExecutionPlan::new());
466 let mut merged = TableBuilder::new();
467 merged.extend(e1).unwrap();
468 merged.extend(e2).unwrap();
469
470 assert_eq!(
471 merged.columns["A"].as_str_vec(),
472 vec![Some("1".into()), Some("3".into()), None]
473 );
474 assert_eq!(
475 merged.columns["B"].as_str_vec(),
476 vec![Some("2".into()), None, Some("4".into())]
477 );
478 assert_eq!(
479 merged.columns["C"].as_str_vec(),
480 vec![None, None, Some("5".into())]
481 );
482 }
483
484 #[test]
485 fn test_auto_dict_upgrade_only_post_merge() {
486 let mut plan = ExecutionPlan::new();
487 plan.auto_dict = true;
488 let a = parse_bytes(b"P=x\nP=y\n", plan.clone());
489 let b = parse_bytes(b"P=x\nP=y\n", plan.clone());
490 let mut merged = TableBuilder::with_plan(64, plan);
491 merged.extend(a).unwrap();
492 merged.extend(b).unwrap();
493 merged.auto_dict_upgrade();
494 assert_eq!(merged.num_rows(), 4);
495 }
496
497 #[test]
498 fn test_extend_variant_mismatch_errors_not_panics() {
499 let mut e1 = parse_bytes(b"P=x\n", ExecutionPlan::new());
500 let mut plan = ExecutionPlan::new();
501 plan.dictionary_columns.insert("P".to_string());
502 let e2 = parse_bytes(b"P=x\n", plan);
503 let result = e1.extend(e2);
504 assert!(
505 result.is_err(),
506 "String/Dictionary mismatch must return Err"
507 );
508 }
509
510 #[test]
511 fn test_compare_filter_via_arrow_compute() {
512 use crate::arrow_export::apply_compare_filter;
513
514 let mut plan = ExecutionPlan::new();
515 plan.filter = Some(FilterPredicate::Compare {
516 field_a: "A".to_string(),
517 op: CompareOp::Gt,
518 field_b: "B".to_string(),
519 });
520 let mut engine = parse_bytes(b"A=3 B=1\nA=2 B=2\nA=5 B=4\n", plan);
521 let batch = engine.finish().unwrap();
522 let filtered = apply_compare_filter(
523 batch,
524 &FilterPredicate::Compare {
525 field_a: "A".to_string(),
526 op: CompareOp::Gt,
527 field_b: "B".to_string(),
528 },
529 )
530 .unwrap();
531 assert_eq!(filtered.num_rows(), 2);
532 let a = filtered.column_by_name("A").unwrap().as_string::<i32>();
533 assert_eq!(a.value(0), "3");
534 assert_eq!(a.value(1), "5");
535 }
536}