1use crate::arrow_store;
2use crate::arrow_store::IngestBuilder;
3use crate::engine::CancelToken;
4use crate::stripes::NumericChunk;
5use arrow_array::Array;
6use arrow_schema::DataType;
7use formualizer_common::{CoercionPolicy, DateSystem, ExcelError, LiteralValue};
8use std::sync::Arc;
9
10#[derive(Clone)]
11pub enum RangeBacking<'a> {
12 Borrowed(&'a arrow_store::ArrowSheet),
13 Owned(Arc<arrow_store::ArrowSheet>),
14}
15
16#[derive(Clone)]
19pub struct RangeView<'a> {
20 backing: RangeBacking<'a>,
21 sr: usize,
22 sc: usize,
23 er: usize,
24 ec: usize,
25 rows: usize,
26 cols: usize,
27 cancel_token: Option<CancelToken>,
28}
29
30impl<'a> core::fmt::Debug for RangeView<'a> {
31 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
32 f.debug_struct("RangeView")
33 .field("rows", &self.rows)
34 .field("cols", &self.cols)
35 .field("kind", &self.kind_probe())
36 .finish()
37 }
38}
39
40#[derive(Copy, Clone, Debug, Eq, PartialEq)]
41pub enum RangeKind {
42 Empty,
43 NumericOnly,
44 TextOnly,
45 Mixed,
46}
47
48pub struct ChunkCol {
49 pub numbers: Option<arrow_array::ArrayRef>,
50 pub booleans: Option<arrow_array::ArrayRef>,
51 pub text: Option<arrow_array::ArrayRef>,
52 pub errors: Option<arrow_array::ArrayRef>,
53 pub type_tag: arrow_array::ArrayRef,
54}
55
56pub struct ChunkSlice {
57 pub row_start: usize, pub row_len: usize,
59 pub cols: Vec<ChunkCol>,
60}
61
62pub struct RowChunkIterator<'a> {
63 view: &'a RangeView<'a>,
64 current_chunk_idx: usize,
65}
66
67impl<'a> Iterator for RowChunkIterator<'a> {
68 type Item = Result<ChunkSlice, ExcelError>;
69
70 fn next(&mut self) -> Option<Self::Item> {
71 if self
72 .view
73 .cancel_token
74 .as_ref()
75 .is_some_and(CancelToken::is_cancelled)
76 {
77 return Some(Err(ExcelError::new(
78 formualizer_common::ExcelErrorKind::Cancelled,
79 )));
80 }
81
82 let sheet = self.view.sheet();
83 let chunk_starts = &sheet.chunk_starts;
84 let sheet_rows = sheet.nrows as usize;
85 let row_end = self.view.er.min(sheet_rows.saturating_sub(1));
86
87 while self.current_chunk_idx < chunk_starts.len() {
88 let ci = self.current_chunk_idx;
89 let start = chunk_starts[ci];
90 self.current_chunk_idx += 1;
91
92 let end = if ci + 1 < chunk_starts.len() {
93 chunk_starts[ci + 1]
94 } else {
95 sheet_rows
96 };
97 let len = end.saturating_sub(start);
98 if len == 0 {
99 continue;
100 }
101 let chunk_end_abs = start + len - 1;
102 let is = start.max(self.view.sr);
103 let ie = chunk_end_abs.min(row_end);
104 if is > ie {
105 continue;
106 }
107 let seg_len = ie - is + 1;
108 let rel_off = is - start;
109
110 let mut cols = Vec::with_capacity(self.view.cols);
111 for col_idx in self.view.sc..=self.view.ec {
112 if col_idx >= sheet.columns.len() {
113 let numbers = Some(arrow_array::new_null_array(&DataType::Float64, seg_len));
114 let booleans = Some(arrow_array::new_null_array(&DataType::Boolean, seg_len));
115 let text = Some(arrow_array::new_null_array(&DataType::Utf8, seg_len));
116 let errors = Some(arrow_array::new_null_array(&DataType::UInt8, seg_len));
117 let type_tag: arrow_array::ArrayRef =
118 Arc::new(arrow_array::UInt8Array::from(vec![
119 arrow_store::TypeTag::Empty
120 as u8;
121 seg_len
122 ]));
123 cols.push(ChunkCol {
124 numbers,
125 booleans,
126 text,
127 errors,
128 type_tag,
129 });
130 } else {
131 let col = &sheet.columns[col_idx];
132 let Some(ch) = col.chunk(ci) else {
133 let numbers =
134 Some(arrow_array::new_null_array(&DataType::Float64, seg_len));
135 let booleans =
136 Some(arrow_array::new_null_array(&DataType::Boolean, seg_len));
137 let text = Some(arrow_array::new_null_array(&DataType::Utf8, seg_len));
138 let errors = Some(arrow_array::new_null_array(&DataType::UInt8, seg_len));
139 let type_tag: arrow_array::ArrayRef =
140 Arc::new(arrow_array::UInt8Array::from(vec![
141 arrow_store::TypeTag::Empty
142 as u8;
143 seg_len
144 ]));
145 cols.push(ChunkCol {
146 numbers,
147 booleans,
148 text,
149 errors,
150 type_tag,
151 });
152 continue;
153 };
154
155 let numbers_base: arrow_array::ArrayRef = ch.numbers_or_null();
156 let booleans_base: arrow_array::ArrayRef = ch.booleans_or_null();
157 let text_base: arrow_array::ArrayRef = ch.text_or_null();
158 let errors_base: arrow_array::ArrayRef = ch.errors_or_null();
159
160 let numbers = Some(numbers_base.slice(rel_off, seg_len));
161 let booleans = Some(booleans_base.slice(rel_off, seg_len));
162 let text = Some(text_base.slice(rel_off, seg_len));
163 let errors = Some(errors_base.slice(rel_off, seg_len));
164 let type_tag: arrow_array::ArrayRef =
165 Arc::new(ch.type_tag.slice(rel_off, seg_len));
166 cols.push(ChunkCol {
167 numbers,
168 booleans,
169 text,
170 errors,
171 type_tag,
172 });
173 }
174 }
175 return Some(Ok(ChunkSlice {
176 row_start: is - self.view.sr,
177 row_len: seg_len,
178 cols,
179 }));
180 }
181 None
182 }
183}
184
185impl<'a> RangeView<'a> {
186 pub(crate) fn new(
187 backing: RangeBacking<'a>,
188 sr: usize,
189 sc: usize,
190 er: usize,
191 ec: usize,
192 rows: usize,
193 cols: usize,
194 ) -> Self {
195 Self {
196 backing,
197 sr,
198 sc,
199 er,
200 ec,
201 rows,
202 cols,
203 cancel_token: None,
204 }
205 }
206
207 #[must_use]
213 pub fn with_cancel_token(mut self, token: Option<CancelToken>) -> Self {
214 self.cancel_token = token;
215 self
216 }
217
218 #[inline]
219 pub fn sheet(&self) -> &arrow_store::ArrowSheet {
220 match &self.backing {
221 RangeBacking::Borrowed(s) => s,
222 RangeBacking::Owned(s) => s,
223 }
224 }
225
226 pub fn from_owned_rows(
227 rows: Vec<Vec<LiteralValue>>,
228 date_system: DateSystem,
229 ) -> RangeView<'static> {
230 Self::try_from_owned_rows(rows, date_system, None)
231 .expect("uncancelled RangeView conversion")
232 }
233
234 pub(crate) fn try_from_owned_rows(
235 rows: Vec<Vec<LiteralValue>>,
236 date_system: DateSystem,
237 cancel_token: Option<CancelToken>,
238 ) -> Result<RangeView<'static>, ExcelError> {
239 let nrows = rows.len();
240 let ncols = rows.iter().map(|r| r.len()).max().unwrap_or(0);
241
242 let chunk_rows = 32 * 1024;
243 let mut ib = IngestBuilder::new("__tmp", ncols, chunk_rows, date_system);
244
245 for mut r in rows {
246 if cancel_token.as_ref().is_some_and(CancelToken::is_cancelled) {
247 return Err(ExcelError::new(
248 formualizer_common::ExcelErrorKind::Cancelled,
249 ));
250 }
251 r.resize(ncols, LiteralValue::Empty);
252 ib.append_row(&r).expect("append_row for RangeView");
253 }
254
255 let sheet = Arc::new(ib.finish());
256
257 if nrows == 0 || ncols == 0 {
258 return Ok(RangeView {
259 backing: RangeBacking::Owned(sheet),
260 sr: 1,
261 sc: 1,
262 er: 0,
263 ec: 0,
264 rows: 0,
265 cols: 0,
266 cancel_token,
267 });
268 }
269
270 Ok(RangeView {
271 backing: RangeBacking::Owned(sheet),
272 sr: 0,
273 sc: 0,
274 er: nrows - 1,
275 ec: ncols - 1,
276 rows: nrows,
277 cols: ncols,
278 cancel_token,
279 })
280 }
281
282 pub fn dims(&self) -> (usize, usize) {
283 (self.rows, self.cols)
284 }
285
286 pub fn expand_to(&self, rows: usize, cols: usize) -> RangeView<'a> {
287 let er = self.sr + rows.saturating_sub(1);
288 let ec = self.sc + cols.saturating_sub(1);
289 RangeView {
290 backing: match &self.backing {
291 RangeBacking::Borrowed(s) => RangeBacking::Borrowed(s),
292 RangeBacking::Owned(s) => RangeBacking::Owned(s.clone()),
293 },
294 sr: self.sr,
295 sc: self.sc,
296 er,
297 ec,
298 rows,
299 cols,
300 cancel_token: self.cancel_token.clone(),
301 }
302 }
303
304 pub fn sub_view(&self, rs: usize, cs: usize, rows: usize, cols: usize) -> RangeView<'a> {
305 let abs_sr = self.sr + rs;
306 let abs_sc = self.sc + cs;
307 let er = abs_sr + rows.saturating_sub(1);
308 let ec = abs_sc + cols.saturating_sub(1);
309 RangeView {
310 backing: match &self.backing {
311 RangeBacking::Borrowed(s) => RangeBacking::Borrowed(s),
312 RangeBacking::Owned(s) => RangeBacking::Owned(s.clone()),
313 },
314 sr: abs_sr,
315 sc: abs_sc,
316 er,
317 ec,
318 rows,
319 cols,
320 cancel_token: self.cancel_token.clone(),
321 }
322 }
323
324 #[inline]
325 pub fn is_empty(&self) -> bool {
326 self.rows == 0 || self.cols == 0
327 }
328
329 pub fn start_row(&self) -> usize {
331 self.sr
332 }
333 pub fn end_row(&self) -> usize {
335 self.er
336 }
337 pub fn start_col(&self) -> usize {
339 self.sc
340 }
341 pub fn end_col(&self) -> usize {
343 self.ec
344 }
345 pub fn sheet_name(&self) -> &str {
347 &self.sheet().name
348 }
349
350 pub fn kind_probe(&self) -> RangeKind {
351 if self.is_empty() {
352 return RangeKind::Empty;
353 }
354
355 let mut has_num = false;
356 let mut has_text = false;
357
358 for r in 0..self.rows {
359 for c in 0..self.cols {
360 match self.get_cell(r, c) {
361 LiteralValue::Empty => {}
362 LiteralValue::Number(_) | LiteralValue::Int(_) => has_num = true,
363 LiteralValue::Text(_) => has_text = true,
364 _ => return RangeKind::Mixed,
365 }
366 if has_num && has_text {
367 return RangeKind::Mixed;
368 }
369 }
370 }
371
372 match (has_num, has_text) {
373 (false, false) => RangeKind::Empty,
374 (true, false) => RangeKind::NumericOnly,
375 (false, true) => RangeKind::TextOnly,
376 (true, true) => RangeKind::Mixed,
377 }
378 }
379
380 pub fn as_1x1(&self) -> Option<LiteralValue> {
381 if self.rows == 1 && self.cols == 1 {
382 Some(self.get_cell(0, 0))
383 } else {
384 None
385 }
386 }
387
388 pub fn get_cell(&self, row: usize, col: usize) -> LiteralValue {
391 if row >= self.rows || col >= self.cols {
392 return LiteralValue::Empty;
393 }
394 let abs_row = self.sr + row;
395 let abs_col = self.sc + col;
396 let sheet = self.sheet();
397 let sheet_rows = sheet.nrows as usize;
398 if abs_row >= sheet_rows {
399 return LiteralValue::Empty;
400 }
401 if abs_col >= sheet.columns.len() {
402 return LiteralValue::Empty;
403 }
404 let col_ref = &sheet.columns[abs_col];
405 let chunk_starts = &sheet.chunk_starts;
407 let ch_idx = match chunk_starts.binary_search(&abs_row) {
408 Ok(i) => i,
409 Err(0) => 0,
410 Err(i) => i - 1,
411 };
412 let Some(ch) = col_ref.chunk(ch_idx) else {
413 return LiteralValue::Empty;
414 };
415 let row_start = chunk_starts[ch_idx];
416 let in_off = abs_row - row_start;
417 let cascade = arrow_store::OverlayCascade::new(&ch.overlay, &ch.computed_overlay);
419 if let Some(ov) = cascade.get_scalar(in_off) {
420 return ov.to_literal_for(sheet.date_system);
421 }
422 let tag_u8 = ch.type_tag.value(in_off);
424 match arrow_store::TypeTag::from_u8(tag_u8) {
425 arrow_store::TypeTag::Empty => LiteralValue::Empty,
426 arrow_store::TypeTag::Number => {
427 if let Some(arr) = &ch.numbers {
428 if arr.is_null(in_off) {
429 return LiteralValue::Empty;
430 }
431 LiteralValue::Number(arr.value(in_off))
432 } else {
433 LiteralValue::Empty
434 }
435 }
436 arrow_store::TypeTag::DateTime => {
437 if let Some(arr) = &ch.numbers {
438 if arr.is_null(in_off) {
439 return LiteralValue::Empty;
440 }
441 LiteralValue::try_from_serial_number_for(sheet.date_system, arr.value(in_off))
442 .unwrap_or_else(LiteralValue::Error)
443 } else {
444 LiteralValue::Empty
445 }
446 }
447 arrow_store::TypeTag::Duration => {
448 if let Some(arr) = &ch.numbers {
449 if arr.is_null(in_off) {
450 return LiteralValue::Empty;
451 }
452 let serial = arr.value(in_off);
453 let nanos_f = serial * 86_400.0 * 1_000_000_000.0;
454 let nanos = nanos_f.round().clamp(i64::MIN as f64, i64::MAX as f64) as i64;
455 LiteralValue::Duration(chrono::Duration::nanoseconds(nanos))
456 } else {
457 LiteralValue::Empty
458 }
459 }
460 arrow_store::TypeTag::Boolean => {
461 if let Some(arr) = &ch.booleans {
462 if arr.is_null(in_off) {
463 return LiteralValue::Empty;
464 }
465 LiteralValue::Boolean(arr.value(in_off))
466 } else {
467 LiteralValue::Empty
468 }
469 }
470 arrow_store::TypeTag::Text => {
471 if let Some(arr) = &ch.text {
472 if arr.is_null(in_off) {
473 return LiteralValue::Empty;
474 }
475 let sa = arr
476 .as_any()
477 .downcast_ref::<arrow_array::StringArray>()
478 .unwrap();
479 LiteralValue::Text(sa.value(in_off).to_string())
480 } else {
481 LiteralValue::Empty
482 }
483 }
484 arrow_store::TypeTag::Error => {
485 if let Some(arr) = &ch.errors {
486 if arr.is_null(in_off) {
487 return LiteralValue::Empty;
488 }
489 let kind = arrow_store::unmap_error_code(arr.value(in_off));
490 LiteralValue::Error(ExcelError::new(kind))
491 } else {
492 LiteralValue::Empty
493 }
494 }
495 arrow_store::TypeTag::Pending => LiteralValue::Pending,
496 }
497 }
498
499 pub fn iter_row_chunks(&self) -> RowChunkIterator<'_> {
501 RowChunkIterator {
502 view: self,
503 current_chunk_idx: 0,
504 }
505 }
506
507 pub fn for_each_cell(
509 &self,
510 f: &mut dyn FnMut(&LiteralValue) -> Result<(), ExcelError>,
511 ) -> Result<(), ExcelError> {
512 for res in self.iter_row_chunks() {
513 let cs = res?;
514 for r in 0..cs.row_len {
515 for c in 0..self.cols {
516 let tmp = self.get_cell(cs.row_start + r, c);
517 f(&tmp)?;
518 }
519 }
520 }
521 Ok(())
522 }
523
524 pub fn for_each_row(
526 &self,
527 f: &mut dyn FnMut(&[LiteralValue]) -> Result<(), ExcelError>,
528 ) -> Result<(), ExcelError> {
529 let mut buf: Vec<LiteralValue> = Vec::with_capacity(self.cols);
530 for r in 0..self.rows {
531 buf.clear();
532 for c in 0..self.cols {
533 buf.push(self.get_cell(r, c));
534 }
535 f(&buf[..])?;
536 }
537 Ok(())
538 }
539
540 pub fn for_each_col(
542 &self,
543 f: &mut dyn FnMut(&[LiteralValue]) -> Result<(), ExcelError>,
544 ) -> Result<(), ExcelError> {
545 let mut col_buf: Vec<LiteralValue> = Vec::with_capacity(self.rows);
546 for c in 0..self.cols {
547 col_buf.clear();
548 for r in 0..self.rows {
549 col_buf.push(self.get_cell(r, c));
550 }
551 f(&col_buf[..])?;
552 }
553 Ok(())
554 }
555
556 pub fn get_cell_numeric(&self, row: usize, col: usize, policy: CoercionPolicy) -> Option<f64> {
559 if row >= self.rows || col >= self.cols {
560 return None;
561 }
562
563 let val = self.get_cell(row, col);
564 pack_numeric(&val, policy).ok().flatten()
565 }
566
567 pub fn numbers_chunked(
569 &self,
570 policy: CoercionPolicy,
571 min_chunk: usize,
572 f: &mut dyn FnMut(NumericChunk) -> Result<(), ExcelError>,
573 ) -> Result<(), ExcelError> {
574 if matches!(policy, CoercionPolicy::NumberStrict) {
576 for res in self.numbers_slices() {
577 let (_, _, cols) = res?;
578 for col in cols {
579 if col.null_count() < col.len() {
580 let data = col.values();
581 let validity = if col.null_count() > 0 {
587 None } else {
591 None
592 };
593
594 if col.null_count() == 0 {
595 f(NumericChunk { data, validity })?;
596 } else {
597 let mut buf = Vec::with_capacity(col.len());
599 for i in 0..col.len() {
600 if !col.is_null(i) {
601 buf.push(col.value(i));
602 }
603 }
604 if !buf.is_empty() {
605 f(NumericChunk {
606 data: &buf,
607 validity: None,
608 })?;
609 }
610 }
611 }
612 }
613 }
614 return Ok(());
615 }
616
617 let min_chunk = min_chunk.max(1);
618 let mut buf: Vec<f64> = Vec::with_capacity(min_chunk);
619 let mut flush = |buf: &mut Vec<f64>| -> Result<(), ExcelError> {
620 if buf.is_empty() {
621 return Ok(());
622 }
623 let ptr = buf.as_ptr();
625 let len = buf.len();
626 let slice = unsafe { std::slice::from_raw_parts(ptr, len) };
627 let chunk = NumericChunk {
628 data: slice,
629 validity: None,
630 };
631 f(chunk)?;
632 buf.clear();
633 Ok(())
634 };
635
636 self.for_each_cell(&mut |v| {
637 if let Some(n) = pack_numeric(v, policy)? {
638 buf.push(n);
639 if buf.len() >= min_chunk {
640 flush(&mut buf)?;
641 }
642 }
643 Ok(())
644 })?;
645 flush(&mut buf)?;
646
647 Ok(())
648 }
649
650 pub fn numbers_slices(
652 &self,
653 ) -> impl Iterator<Item = Result<(usize, usize, Vec<Arc<arrow_array::Float64Array>>), ExcelError>> + '_
654 {
655 self.iter_row_chunks().map(move |res| {
656 let cs = res?;
657 let mut out_cols: Vec<Arc<arrow_array::Float64Array>> =
658 Vec::with_capacity(cs.cols.len());
659 let sheet = self.sheet();
660 let chunk_starts = &sheet.chunk_starts;
661
662 for (local_c, col_idx) in (self.sc..=self.ec).enumerate() {
663 let base = cs.cols[local_c]
664 .numbers
665 .as_ref()
666 .expect("numbers lane exists")
667 .clone();
668 let base_fa = base
669 .as_any()
670 .downcast_ref::<arrow_array::Float64Array>()
671 .unwrap()
672 .clone();
673 let base_arc = Arc::new(base_fa);
674
675 let abs_seg_start = self.sr + cs.row_start;
677 let ch_idx = match chunk_starts.binary_search(&abs_seg_start) {
678 Ok(i) => i,
679 Err(0) => 0,
680 Err(i) => i - 1,
681 };
682 if col_idx >= sheet.columns.len() {
683 out_cols.push(base_arc);
684 continue;
685 }
686 let col = &sheet.columns[col_idx];
687 let Some(ch) = col.chunk(ch_idx) else {
688 out_cols.push(base_arc);
689 continue;
690 };
691 let rel_off = (self.sr + cs.row_start) - chunk_starts[ch_idx];
692 let seg_range = rel_off..(rel_off + cs.row_len);
693 let cascade = arrow_store::OverlayCascade::new(&ch.overlay, &ch.computed_overlay);
694 if cascade.has_any_in_range(seg_range.clone()) {
695 let base_fa = base
696 .as_any()
697 .downcast_ref::<arrow_array::Float64Array>()
698 .unwrap();
699 out_cols.push(cascade.select_numbers(seg_range, base_fa));
700 } else {
701 out_cols.push(base_arc);
702 }
703 }
704 Ok((cs.row_start, cs.row_len, out_cols))
705 })
706 }
707
708 pub fn booleans_slices(
710 &self,
711 ) -> impl Iterator<Item = Result<(usize, usize, Vec<Arc<arrow_array::BooleanArray>>), ExcelError>> + '_
712 {
713 self.iter_row_chunks().map(move |res| {
714 let cs = res?;
715 let mut out_cols: Vec<Arc<arrow_array::BooleanArray>> =
716 Vec::with_capacity(cs.cols.len());
717 let sheet = self.sheet();
718 let chunk_starts = &sheet.chunk_starts;
719
720 for (local_c, col_idx) in (self.sc..=self.ec).enumerate() {
721 let base = cs.cols[local_c]
722 .booleans
723 .as_ref()
724 .expect("booleans lane exists")
725 .clone();
726 let base_ba = base
727 .as_any()
728 .downcast_ref::<arrow_array::BooleanArray>()
729 .unwrap()
730 .clone();
731 let base_arc = Arc::new(base_ba);
732
733 let abs_seg_start = self.sr + cs.row_start;
735 let ch_idx = match chunk_starts.binary_search(&abs_seg_start) {
736 Ok(i) => i,
737 Err(0) => 0,
738 Err(i) => i - 1,
739 };
740 if col_idx >= sheet.columns.len() {
741 out_cols.push(base_arc);
742 continue;
743 }
744 let col = &sheet.columns[col_idx];
745 let Some(ch) = col.chunk(ch_idx) else {
746 out_cols.push(base_arc);
747 continue;
748 };
749 let rel_off = (self.sr + cs.row_start) - chunk_starts[ch_idx];
750 let seg_range = rel_off..(rel_off + cs.row_len);
751 let cascade = arrow_store::OverlayCascade::new(&ch.overlay, &ch.computed_overlay);
752 if cascade.has_any_in_range(seg_range.clone()) {
753 let base_ba = base
754 .as_any()
755 .downcast_ref::<arrow_array::BooleanArray>()
756 .unwrap();
757 out_cols.push(cascade.select_booleans(seg_range, base_ba));
758 } else {
759 out_cols.push(base_arc);
760 }
761 }
762 Ok((cs.row_start, cs.row_len, out_cols))
763 })
764 }
765
766 pub fn text_slices(
768 &self,
769 ) -> impl Iterator<Item = Result<(usize, usize, Vec<arrow_array::ArrayRef>), ExcelError>> + '_
770 {
771 self.iter_row_chunks().map(move |res| {
772 let cs = res?;
773 let mut out_cols: Vec<arrow_array::ArrayRef> = Vec::with_capacity(cs.cols.len());
774 let sheet = self.sheet();
775 let chunk_starts = &sheet.chunk_starts;
776
777 for (local_c, col_idx) in (self.sc..=self.ec).enumerate() {
778 let base = cs.cols[local_c]
779 .text
780 .as_ref()
781 .expect("text lane exists")
782 .clone();
783 let abs_seg_start = self.sr + cs.row_start;
784 let ch_idx = match chunk_starts.binary_search(&abs_seg_start) {
785 Ok(i) => i,
786 Err(0) => 0,
787 Err(i) => i - 1,
788 };
789 if col_idx >= sheet.columns.len() {
790 out_cols.push(base.clone());
791 continue;
792 }
793 let col = &sheet.columns[col_idx];
794 let Some(ch) = col.chunk(ch_idx) else {
795 out_cols.push(base.clone());
796 continue;
797 };
798 let rel_off = (self.sr + cs.row_start) - chunk_starts[ch_idx];
799 let seg_range = rel_off..(rel_off + cs.row_len);
800 let cascade = arrow_store::OverlayCascade::new(&ch.overlay, &ch.computed_overlay);
801 if cascade.has_any_in_range(seg_range.clone()) {
802 let base_sa = base
803 .as_any()
804 .downcast_ref::<arrow_array::StringArray>()
805 .unwrap();
806 out_cols.push(cascade.select_text(seg_range, base_sa));
807 } else {
808 out_cols.push(base.clone());
809 }
810 }
811 Ok((cs.row_start, cs.row_len, out_cols))
812 })
813 }
814
815 pub fn lowered_text_slices(
817 &self,
818 ) -> impl Iterator<Item = Result<(usize, usize, Vec<Arc<arrow_array::StringArray>>), ExcelError>> + '_
819 {
820 self.iter_row_chunks().map(move |res| {
821 let cs = res?;
822 let mut out_cols: Vec<Arc<arrow_array::StringArray>> =
823 Vec::with_capacity(cs.cols.len());
824 let sheet = self.sheet();
825 let chunk_starts = &sheet.chunk_starts;
826
827 for (local_c, col_idx) in (self.sc..=self.ec).enumerate() {
828 let abs_seg_start = self.sr + cs.row_start;
830 let ch_idx = match chunk_starts.binary_search(&abs_seg_start) {
831 Ok(i) => i,
832 Err(0) => 0,
833 Err(i) => i - 1,
834 };
835 if col_idx >= sheet.columns.len() {
836 out_cols.push(Arc::new(arrow_array::StringArray::new_null(cs.row_len)));
837 continue;
838 }
839 let col = &sheet.columns[col_idx];
840 let Some(ch) = col.chunk(ch_idx) else {
841 out_cols.push(Arc::new(arrow_array::StringArray::new_null(cs.row_len)));
842 continue;
843 };
844 let rel_off = (self.sr + cs.row_start) - chunk_starts[ch_idx];
845 let seg_range = rel_off..(rel_off + cs.row_len);
846
847 let base_lowered = ch.text_lower_or_null();
848 let base_seg = base_lowered.slice(rel_off, cs.row_len);
849 let base_sa = base_seg
850 .as_any()
851 .downcast_ref::<arrow_array::StringArray>()
852 .expect("lowered slice downcast");
853
854 let cascade = arrow_store::OverlayCascade::new(&ch.overlay, &ch.computed_overlay);
855 if cascade.has_any_in_range(seg_range.clone()) {
856 out_cols.push(cascade.select_lowered_text(seg_range, base_sa));
857 } else {
858 out_cols.push(Arc::new(base_sa.clone()));
859 }
860 }
861 Ok((cs.row_start, cs.row_len, out_cols))
862 })
863 }
864
865 pub fn errors_slices(
867 &self,
868 ) -> impl Iterator<Item = Result<(usize, usize, Vec<Arc<arrow_array::UInt8Array>>), ExcelError>> + '_
869 {
870 self.iter_row_chunks().map(move |res| {
871 let cs = res?;
872 let mut out_cols: Vec<Arc<arrow_array::UInt8Array>> = Vec::with_capacity(cs.cols.len());
873 let sheet = self.sheet();
874 let chunk_starts = &sheet.chunk_starts;
875
876 for (local_c, col_idx) in (self.sc..=self.ec).enumerate() {
877 let base = cs.cols[local_c]
878 .errors
879 .as_ref()
880 .expect("errors lane exists")
881 .clone();
882 let base_e = base
883 .as_any()
884 .downcast_ref::<arrow_array::UInt8Array>()
885 .unwrap()
886 .clone();
887 let base_arc: Arc<arrow_array::UInt8Array> = Arc::new(base_e);
888 let abs_seg_start = self.sr + cs.row_start;
889 let ch_idx = match chunk_starts.binary_search(&abs_seg_start) {
890 Ok(i) => i,
891 Err(0) => 0,
892 Err(i) => i - 1,
893 };
894 if col_idx >= sheet.columns.len() {
895 out_cols.push(base_arc);
896 continue;
897 }
898 let col = &sheet.columns[col_idx];
899 let Some(ch) = col.chunk(ch_idx) else {
900 out_cols.push(base_arc);
901 continue;
902 };
903 let rel_off = (self.sr + cs.row_start) - chunk_starts[ch_idx];
904 let seg_range = rel_off..(rel_off + cs.row_len);
905 let cascade = arrow_store::OverlayCascade::new(&ch.overlay, &ch.computed_overlay);
906 if cascade.has_any_in_range(seg_range.clone()) {
907 let base_ea = base
908 .as_any()
909 .downcast_ref::<arrow_array::UInt8Array>()
910 .unwrap();
911 out_cols.push(cascade.select_errors(seg_range, base_ea));
912 } else {
913 out_cols.push(base_arc);
914 }
915 }
916 Ok((cs.row_start, cs.row_len, out_cols))
917 })
918 }
919
920 pub fn type_tags_slices(
922 &self,
923 ) -> impl Iterator<Item = Result<(usize, usize, Vec<Arc<arrow_array::UInt8Array>>), ExcelError>> + '_
924 {
925 self.iter_row_chunks().map(move |res| {
926 let cs = res?;
927 let mut out_cols: Vec<Arc<arrow_array::UInt8Array>> = Vec::with_capacity(cs.cols.len());
928 let sheet = self.sheet();
929 let chunk_starts = &sheet.chunk_starts;
930
931 for (local_c, col_idx) in (self.sc..=self.ec).enumerate() {
932 let base = cs.cols[local_c].type_tag.clone();
933 let base_ta = base
934 .as_any()
935 .downcast_ref::<arrow_array::UInt8Array>()
936 .unwrap()
937 .clone();
938 let base_arc = Arc::new(base_ta);
939
940 let abs_seg_start = self.sr + cs.row_start;
941 let ch_idx = match chunk_starts.binary_search(&abs_seg_start) {
942 Ok(i) => i,
943 Err(0) => 0,
944 Err(i) => i - 1,
945 };
946 if col_idx >= sheet.columns.len() {
947 out_cols.push(base_arc);
948 continue;
949 }
950 let col = &sheet.columns[col_idx];
951 let Some(ch) = col.chunk(ch_idx) else {
952 out_cols.push(base_arc);
953 continue;
954 };
955 let rel_off = (self.sr + cs.row_start) - chunk_starts[ch_idx];
956 let seg_range = rel_off..(rel_off + cs.row_len);
957 let cascade = arrow_store::OverlayCascade::new(&ch.overlay, &ch.computed_overlay);
958 if cascade.has_any_in_range(seg_range.clone()) {
959 let base_ta = base
960 .as_any()
961 .downcast_ref::<arrow_array::UInt8Array>()
962 .unwrap();
963 out_cols.push(cascade.select_type_tags(seg_range, base_ta));
964 } else {
965 out_cols.push(base_arc);
966 }
967 }
968 Ok((cs.row_start, cs.row_len, out_cols))
969 })
970 }
971
972 pub fn lowered_text_columns(&self) -> Vec<arrow_array::ArrayRef> {
975 use crate::compute_prelude::concat_arrays;
976
977 let mut out: Vec<arrow_array::ArrayRef> = Vec::with_capacity(self.cols);
978 if self.rows == 0 || self.cols == 0 {
979 return out;
980 }
981 let sheet = self.sheet();
982 let chunk_starts = &sheet.chunk_starts;
983 let sheet_rows = sheet.nrows as usize;
985 if sheet_rows == 0 || self.sr >= sheet_rows {
986 for _ in 0..self.cols {
987 out.push(arrow_array::new_null_array(&DataType::Utf8, 0));
988 }
989 return out;
990 }
991 let row_end = self.er.min(sheet_rows.saturating_sub(1));
992 let physical_len = row_end.saturating_sub(self.sr) + 1;
993 for col_idx in self.sc..=self.ec {
994 let mut segs: Vec<arrow_array::ArrayRef> = Vec::new();
995 if col_idx >= sheet.columns.len() {
996 segs.push(arrow_array::new_null_array(&DataType::Utf8, physical_len));
998 } else {
999 let col_ref = &sheet.columns[col_idx];
1000 for (ci, &start) in chunk_starts.iter().enumerate() {
1001 let chunk_end = chunk_starts
1002 .get(ci + 1)
1003 .copied()
1004 .unwrap_or(sheet.nrows as usize);
1005 let len = chunk_end.saturating_sub(start);
1006 if len == 0 {
1007 continue;
1008 }
1009 let end = start + len - 1;
1010 let is = start.max(self.sr);
1011 let ie = end.min(row_end);
1012 if is > ie {
1013 continue;
1014 }
1015 let seg_len = ie - is + 1;
1016 let rel_off = is - start;
1017 if let Some(ch) = col_ref.chunk(ci) {
1018 let seg_range = rel_off..(rel_off + seg_len);
1020 let cascade =
1021 arrow_store::OverlayCascade::new(&ch.overlay, &ch.computed_overlay);
1022 if cascade.has_any_in_range(seg_range.clone()) {
1023 let base_lowered = ch.text_lower_or_null();
1024 let base_seg = base_lowered.slice(rel_off, seg_len);
1025 let base_sa = base_seg
1026 .as_any()
1027 .downcast_ref::<arrow_array::StringArray>()
1028 .expect("lowered slice downcast");
1029 segs.push(cascade.select_lowered_text(seg_range, base_sa));
1030 } else {
1031 let lowered = ch.text_lower_or_null();
1033 segs.push(lowered.slice(rel_off, seg_len));
1034 }
1035 } else {
1036 segs.push(arrow_array::new_null_array(&DataType::Utf8, seg_len));
1037 }
1038 }
1039 }
1040 if segs.is_empty() {
1042 segs.push(arrow_array::new_null_array(&DataType::Utf8, physical_len));
1043 }
1044 let anys: Vec<&dyn arrow_array::Array> = segs
1046 .iter()
1047 .map(|a| a.as_ref() as &dyn arrow_array::Array)
1048 .collect();
1049 let conc = concat_arrays(&anys).expect("concat lowered segments");
1050 out.push(conc);
1051 }
1052 out
1053 }
1054
1055 pub fn slice_numbers(
1057 &self,
1058 rel_start: usize,
1059 len: usize,
1060 ) -> Vec<Option<Arc<arrow_array::Float64Array>>> {
1061 let abs_start = self.sr + rel_start;
1062 let abs_end = abs_start + len;
1063 let sheet = self.sheet();
1064 let chunk_starts = &sheet.chunk_starts;
1065
1066 let mut out_cols = Vec::with_capacity(self.cols);
1067 for col_idx in self.sc..=self.ec {
1068 if col_idx >= sheet.columns.len() {
1069 out_cols.push(None);
1070 continue;
1071 }
1072 let col = &sheet.columns[col_idx];
1073
1074 let start_ch_idx = match chunk_starts.binary_search(&abs_start) {
1075 Ok(i) => i,
1076 Err(0) => 0,
1077 Err(i) => i - 1,
1078 };
1079
1080 let mut segments: Vec<Arc<arrow_array::Float64Array>> = Vec::new();
1081 let mut null_only = true;
1082
1083 let mut curr = abs_start;
1084 let mut remaining = len;
1085 let mut ch_idx = start_ch_idx;
1086
1087 while remaining > 0 && ch_idx < chunk_starts.len() {
1088 let ch_start = chunk_starts[ch_idx];
1089 let ch_end = chunk_starts
1090 .get(ch_idx + 1)
1091 .copied()
1092 .unwrap_or(sheet.nrows as usize);
1093 let ch_len = ch_end.saturating_sub(ch_start);
1094 if ch_len == 0 {
1095 ch_idx += 1;
1096 continue;
1097 }
1098
1099 let overlap_start = curr.max(ch_start);
1100 let overlap_end = ch_end.min(abs_end);
1101
1102 if overlap_start < overlap_end {
1103 let seg_len = overlap_end - overlap_start;
1104 let rel_off_in_chunk = overlap_start - ch_start;
1105
1106 if let Some(ch) = col.chunk(ch_idx) {
1107 let base_nums_arc = ch.numbers_or_null();
1108 let base_nums = base_nums_arc.as_ref();
1109
1110 let seg_range = rel_off_in_chunk..(rel_off_in_chunk + seg_len);
1111 let cascade =
1112 arrow_store::OverlayCascade::new(&ch.overlay, &ch.computed_overlay);
1113
1114 let final_arr = if cascade.has_any_in_range(seg_range.clone()) {
1115 let base_slice = base_nums.slice(rel_off_in_chunk, seg_len);
1116 let base_fa = base_slice
1117 .as_any()
1118 .downcast_ref::<arrow_array::Float64Array>()
1119 .unwrap();
1120 cascade.select_numbers(seg_range, base_fa).as_ref().clone()
1121 } else {
1122 let sl = base_nums.slice(rel_off_in_chunk, seg_len);
1123 sl.as_any()
1124 .downcast_ref::<arrow_array::Float64Array>()
1125 .unwrap()
1126 .clone()
1127 };
1128
1129 if final_arr.null_count() < final_arr.len() {
1130 null_only = false;
1131 }
1132 segments.push(Arc::new(final_arr));
1133 } else {
1134 segments.push(Arc::new(arrow_array::Float64Array::new_null(seg_len)));
1135 }
1136 curr += seg_len;
1137 remaining -= seg_len;
1138 }
1139 ch_idx += 1;
1140 }
1141
1142 if remaining > 0 {
1143 segments.push(Arc::new(arrow_array::Float64Array::new_null(remaining)));
1144 }
1145
1146 if segments.len() == 1 {
1147 if null_only && segments[0].null_count() == segments[0].len() {
1148 out_cols.push(None);
1149 } else {
1150 out_cols.push(Some(segments.pop().unwrap()));
1151 }
1152 } else {
1153 let refs: Vec<&dyn Array> =
1154 segments.iter().map(|a| a.as_ref() as &dyn Array).collect();
1155 let c = crate::compute_prelude::concat_arrays(&refs).expect("concat slice");
1156 let fa = c
1157 .as_any()
1158 .downcast_ref::<arrow_array::Float64Array>()
1159 .unwrap()
1160 .clone();
1161 out_cols.push(Some(Arc::new(fa)));
1162 }
1163 }
1164 out_cols
1165 }
1166
1167 pub fn slice_lowered_text(
1169 &self,
1170 rel_start: usize,
1171 len: usize,
1172 ) -> Vec<Option<Arc<arrow_array::StringArray>>> {
1173 let abs_start = self.sr + rel_start;
1174 let abs_end = abs_start + len;
1175 let sheet = self.sheet();
1176 let chunk_starts = &sheet.chunk_starts;
1177
1178 let mut out_cols = Vec::with_capacity(self.cols);
1179 for col_idx in self.sc..=self.ec {
1180 if col_idx >= sheet.columns.len() {
1181 out_cols.push(None);
1182 continue;
1183 }
1184 let col = &sheet.columns[col_idx];
1185 let start_ch_idx = match chunk_starts.binary_search(&abs_start) {
1186 Ok(i) => i,
1187 Err(0) => 0,
1188 Err(i) => i - 1,
1189 };
1190
1191 let mut segments: Vec<Arc<arrow_array::StringArray>> = Vec::new();
1192 let mut null_only = true;
1193
1194 let mut curr = abs_start;
1195 let mut remaining = len;
1196 let mut ch_idx = start_ch_idx;
1197
1198 while remaining > 0 && ch_idx < chunk_starts.len() {
1199 let ch_start = chunk_starts[ch_idx];
1200 let ch_end = chunk_starts
1201 .get(ch_idx + 1)
1202 .copied()
1203 .unwrap_or(sheet.nrows as usize);
1204 let ch_len = ch_end.saturating_sub(ch_start);
1205 if ch_len == 0 {
1206 ch_idx += 1;
1207 continue;
1208 }
1209
1210 let overlap_start = curr.max(ch_start);
1211 let overlap_end = ch_end.min(abs_end);
1212
1213 if overlap_start < overlap_end {
1214 let seg_len = overlap_end - overlap_start;
1215 let rel_off_in_chunk = overlap_start - ch_start;
1216
1217 if let Some(ch) = col.chunk(ch_idx) {
1218 let base_lowered = ch.text_lower_or_null();
1219 let seg_range = rel_off_in_chunk..(rel_off_in_chunk + seg_len);
1220 let cascade =
1221 arrow_store::OverlayCascade::new(&ch.overlay, &ch.computed_overlay);
1222
1223 let final_arr = if cascade.has_any_in_range(seg_range.clone()) {
1224 let base_slice = base_lowered.slice(rel_off_in_chunk, seg_len);
1225 let base_sa = base_slice
1226 .as_any()
1227 .downcast_ref::<arrow_array::StringArray>()
1228 .unwrap();
1229 cascade
1230 .select_lowered_text(seg_range, base_sa)
1231 .as_ref()
1232 .clone()
1233 } else {
1234 let sl = base_lowered.slice(rel_off_in_chunk, seg_len);
1235 sl.as_any()
1236 .downcast_ref::<arrow_array::StringArray>()
1237 .unwrap()
1238 .clone()
1239 };
1240
1241 if final_arr.null_count() < final_arr.len() {
1242 null_only = false;
1243 }
1244 segments.push(Arc::new(final_arr));
1245 } else {
1246 segments.push(Arc::new(arrow_array::StringArray::new_null(seg_len)));
1247 }
1248 curr += seg_len;
1249 remaining -= seg_len;
1250 }
1251 ch_idx += 1;
1252 }
1253
1254 if remaining > 0 {
1255 segments.push(Arc::new(arrow_array::StringArray::new_null(remaining)));
1256 }
1257
1258 if segments.len() == 1 {
1259 if null_only && segments[0].null_count() == segments[0].len() {
1260 out_cols.push(None);
1261 } else {
1262 out_cols.push(Some(segments.pop().unwrap()));
1263 }
1264 } else {
1265 let refs: Vec<&dyn Array> =
1266 segments.iter().map(|a| a.as_ref() as &dyn Array).collect();
1267 let c = crate::compute_prelude::concat_arrays(&refs).expect("concat text");
1268 let sa = c
1269 .as_any()
1270 .downcast_ref::<arrow_array::StringArray>()
1271 .unwrap()
1272 .clone();
1273 out_cols.push(Some(Arc::new(sa)));
1274 }
1275 }
1276 out_cols
1277 }
1278}
1279
1280#[inline]
1281fn pack_numeric(v: &LiteralValue, policy: CoercionPolicy) -> Result<Option<f64>, ExcelError> {
1282 match policy {
1283 CoercionPolicy::NumberLenientText => match v {
1284 LiteralValue::Error(e) => Err(e.clone()),
1285 LiteralValue::Empty => Ok(None),
1286 other => Ok(crate::coercion::to_number_lenient(other).ok()),
1287 },
1288 CoercionPolicy::NumberStrict => match v {
1289 LiteralValue::Error(e) => Err(e.clone()),
1290 LiteralValue::Empty => Ok(None),
1291 other => Ok(crate::coercion::to_number_strict(other).ok()),
1292 },
1293 _ => match v {
1294 LiteralValue::Error(e) => Err(e.clone()),
1295 _ => Ok(None),
1296 },
1297 }
1298}
1299
1300#[cfg(test)]
1301mod tests {
1302 use super::*;
1303
1304 #[test]
1305 fn owned_rows_numeric_chunking() {
1306 let data: Vec<Vec<LiteralValue>> = vec![
1307 vec![
1308 LiteralValue::Number(1.0),
1309 LiteralValue::Text("x".into()),
1310 LiteralValue::Number(3.0),
1311 ],
1312 vec![
1313 LiteralValue::Boolean(true),
1314 LiteralValue::Empty,
1315 LiteralValue::Number(2.5),
1316 ],
1317 ];
1318 let view = RangeView::from_owned_rows(data, DateSystem::Excel1900);
1319 let mut sum = 0.0f64;
1320 view.numbers_chunked(CoercionPolicy::NumberLenientText, 2, &mut |chunk| {
1321 for &n in chunk.data {
1322 sum += n;
1323 }
1324 Ok(())
1325 })
1326 .unwrap();
1327 assert!((sum - 7.5).abs() < 1e-9);
1328 }
1329
1330 #[test]
1331 fn as_1x1_works() {
1332 let view = RangeView::from_owned_rows(
1333 vec![vec![LiteralValue::Number(7.0)]],
1334 DateSystem::Excel1900,
1335 );
1336 assert_eq!(view.as_1x1(), Some(LiteralValue::Number(7.0)));
1337 }
1338
1339 #[test]
1340 fn pre_cancelled_token_stops_owned_row_construction() {
1341 let token = CancelToken::new();
1342 token.cancel();
1343
1344 let error = RangeView::try_from_owned_rows(
1345 vec![vec![LiteralValue::Number(1.0)]],
1346 DateSystem::Excel1900,
1347 Some(token),
1348 )
1349 .unwrap_err();
1350
1351 assert_eq!(error.kind, formualizer_common::ExcelErrorKind::Cancelled);
1352 }
1353
1354 #[test]
1355 fn pre_cancelled_token_stops_row_chunk_iteration() {
1356 let token = CancelToken::new();
1357 token.cancel();
1358 let view = RangeView::from_owned_rows(
1359 vec![vec![LiteralValue::Number(1.0)]],
1360 DateSystem::Excel1900,
1361 )
1362 .with_cancel_token(Some(token));
1363
1364 let Some(Err(error)) = view.iter_row_chunks().next() else {
1365 panic!("pre-cancelled chunk iteration should return cancellation");
1366 };
1367
1368 assert_eq!(error.kind, formualizer_common::ExcelErrorKind::Cancelled);
1369 }
1370}