formualizer_eval/engine/arena/
data_store.rs

1/// Unified data storage for all value types using arenas
2/// Provides conversion between LiteralValue and ValueRef
3use super::array::ArrayArena;
4use super::ast::{AstArena, AstNodeId, CompactRefType, SheetKey};
5use super::error_arena::{ErrorArena, ErrorRef};
6use super::scalar::ScalarArena;
7use super::string_interner::{StringId, StringInterner};
8use super::value_ref::ValueRef;
9use crate::engine::sheet_registry::SheetRegistry;
10use formualizer_common::{ExcelError, ExcelErrorKind, LiteralValue};
11use formualizer_parse::parser::{
12    ASTNode, ASTNodeType, ExternalBookRef, ExternalReference, ReferenceType, TableReference,
13};
14
15/// Centralized data storage using arenas
16#[derive(Debug)]
17pub struct DataStore {
18    /// Scalar values (floats and large integers)
19    scalars: ScalarArena,
20
21    /// String values
22    strings: StringInterner,
23
24    /// Array values
25    arrays: ArrayArena,
26
27    /// AST nodes for formulas
28    asts: AstArena,
29
30    /// Error storage with message preservation
31    errors: ErrorArena,
32}
33
34impl DataStore {
35    pub fn new() -> Self {
36        Self {
37            scalars: ScalarArena::new(),
38            strings: StringInterner::new(),
39            arrays: ArrayArena::new(),
40            asts: AstArena::new(),
41            errors: ErrorArena::new(),
42        }
43    }
44
45    /// Batch store literal values; returns ValueRefs in same order.
46    pub fn store_values_batch<I>(&mut self, values: I) -> Vec<ValueRef>
47    where
48        I: IntoIterator<Item = LiteralValue>,
49    {
50        let iter = values.into_iter();
51        let (lower, _) = iter.size_hint();
52        let mut out = Vec::with_capacity(lower);
53        for v in iter {
54            out.push(self.store_value(v));
55        }
56        out
57    }
58
59    /// Batch store ASTs; returns AstNodeIds in same order.
60    pub fn store_asts_batch<'a, I>(
61        &mut self,
62        asts: I,
63        sheet_registry: &SheetRegistry,
64    ) -> Vec<AstNodeId>
65    where
66        I: IntoIterator<Item = &'a ASTNode>,
67    {
68        let iter = asts.into_iter();
69        let (lower, _) = iter.size_hint();
70        let mut out = Vec::with_capacity(lower);
71        for ast in iter {
72            out.push(self.store_ast(ast, sheet_registry));
73        }
74        out
75    }
76
77    pub fn with_capacity(estimated_cells: usize) -> Self {
78        Self {
79            scalars: ScalarArena::with_capacity(estimated_cells),
80            strings: StringInterner::with_capacity(estimated_cells / 10),
81            arrays: ArrayArena::with_capacity(estimated_cells / 100),
82            asts: AstArena::with_capacity(estimated_cells / 2),
83            errors: ErrorArena::with_capacity(estimated_cells / 20),
84        }
85    }
86
87    /// Store a LiteralValue and return a ValueRef
88    pub fn store_value(&mut self, value: LiteralValue) -> ValueRef {
89        match value {
90            LiteralValue::Empty => ValueRef::empty(),
91
92            LiteralValue::Number(n) => {
93                // Store as float in scalar arena
94                let idx = self.scalars.insert_float(n);
95                ValueRef::number(idx.as_u32())
96            }
97
98            LiteralValue::Text(s) => {
99                let id = self.strings.intern(&s);
100                ValueRef::string(id.as_u32())
101            }
102
103            LiteralValue::Boolean(b) => ValueRef::boolean(b),
104
105            LiteralValue::Error(err) => self.store_error(&err),
106
107            LiteralValue::Array(array) => {
108                // Convert nested array to ValueRefs
109                let rows = array.len() as u32;
110                let cols = array.first().map(|r| r.len()).unwrap_or(0) as u32;
111
112                let elements: Vec<ValueRef> = array
113                    .into_iter()
114                    .flatten()
115                    .map(|v| self.store_value(v))
116                    .collect();
117
118                let array_ref = self.arrays.insert(rows, cols, elements);
119                ValueRef::array(array_ref.as_u32())
120            }
121
122            LiteralValue::DateTime(dt) => {
123                // The arena uses Excel1900 only as a private, symmetric binary
124                // encoding for chrono values. This is not workbook serial
125                // state and the raw float is never exposed to formula logic.
126                let serial = formualizer_common::datetime_to_serial_for(
127                    formualizer_common::DateSystem::Excel1900,
128                    &dt,
129                );
130                let idx = self.scalars.insert_float(serial);
131                ValueRef::date_time(idx.as_u32())
132            }
133
134            LiteralValue::Date(d) => {
135                // Use the same private arena encoding as DateTime values.
136                let dt = d.and_hms_opt(0, 0, 0).unwrap();
137                let serial = formualizer_common::datetime_to_serial_for(
138                    formualizer_common::DateSystem::Excel1900,
139                    &dt,
140                );
141                let idx = self.scalars.insert_float(serial);
142                ValueRef::date_time(idx.as_u32())
143            }
144
145            LiteralValue::Time(t) => {
146                // Store time as fractional day.
147                let fraction = formualizer_common::time_to_fraction(&t);
148                let idx = self.scalars.insert_float(fraction);
149                ValueRef::date_time(idx.as_u32())
150            }
151
152            LiteralValue::Duration(dur) => {
153                // Store as integer seconds (chrono::Duration has num_seconds())
154                let secs = dur.num_seconds();
155                let idx = self.scalars.insert_integer(secs);
156                let raw_index = idx.as_u32() & 0x7FFF_FFFF;
157                ValueRef::duration(raw_index)
158            }
159
160            LiteralValue::Int(i) => {
161                // Try to use small int optimization
162                if let Some(vref) = ValueRef::small_int(i as i32) {
163                    vref
164                } else {
165                    // Store as large integer
166                    let idx = self.scalars.insert_integer(i);
167                    ValueRef::large_int(idx.as_u32())
168                }
169            }
170
171            LiteralValue::Pending => ValueRef::pending(),
172        }
173    }
174
175    /// Retrieve a LiteralValue from a ValueRef
176    pub fn retrieve_value(&self, value_ref: ValueRef) -> LiteralValue {
177        use super::value_ref::ValueType;
178
179        match value_ref.value_type() {
180            ValueType::Empty => LiteralValue::Empty,
181
182            ValueType::SmallInt => {
183                // Small integers are inlined
184                if let Some(i) = value_ref.as_small_int() {
185                    LiteralValue::Int(i as i64)
186                } else {
187                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
188                }
189            }
190
191            ValueType::LargeInt => {
192                if let Some(idx) = value_ref.arena_index() {
193                    let scalar_ref = super::scalar::ScalarRef::from_raw(idx | (1 << 31));
194                    if let Some(i) = self.scalars.get_integer(scalar_ref) {
195                        LiteralValue::Int(i)
196                    } else {
197                        LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
198                    }
199                } else {
200                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
201                }
202            }
203
204            ValueType::Number => {
205                if let Some(idx) = value_ref.arena_index() {
206                    let scalar_ref = super::scalar::ScalarRef::from_raw(idx);
207                    if let Some(f) = self.scalars.get_float(scalar_ref) {
208                        LiteralValue::Number(f)
209                    } else {
210                        LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
211                    }
212                } else {
213                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
214                }
215            }
216
217            ValueType::String => {
218                if let Some(idx) = value_ref.arena_index() {
219                    let string_id = StringId::from_raw(idx);
220                    let s = self.strings.resolve(string_id);
221                    LiteralValue::Text(s.to_string())
222                } else {
223                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
224                }
225            }
226
227            ValueType::Boolean => {
228                if let Some(b) = value_ref.as_boolean() {
229                    LiteralValue::Boolean(b)
230                } else {
231                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
232                }
233            }
234
235            ValueType::Error => {
236                if let Some(error_ref_raw) = value_ref.as_error_ref() {
237                    let error_ref = ErrorRef::from_raw(error_ref_raw);
238                    if let Some(error) = self.errors.get(error_ref) {
239                        LiteralValue::Error(error)
240                    } else {
241                        LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
242                    }
243                } else {
244                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
245                }
246            }
247
248            ValueType::Array => {
249                if let Some(idx) = value_ref.arena_index() {
250                    let array_ref = super::array::ArrayRef::from_raw(idx);
251                    if let Some(array_2d) = self.arrays.get_2d(array_ref) {
252                        // Convert back to LiteralValue array
253                        let result: Vec<Vec<LiteralValue>> = array_2d
254                            .into_iter()
255                            .map(|row| row.into_iter().map(|v| self.retrieve_value(v)).collect())
256                            .collect();
257                        LiteralValue::Array(result)
258                    } else {
259                        LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
260                    }
261                } else {
262                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
263                }
264            }
265
266            ValueType::DateTime => {
267                if let Some(idx) = value_ref.arena_index() {
268                    let scalar_ref = super::scalar::ScalarRef::from_raw(idx);
269                    if let Some(serial) = self.scalars.get_float(scalar_ref) {
270                        // This is a trusted private encoding, not an untrusted
271                        // workbook serial boundary. Use the compatibility
272                        // decoder so the arena retains its historical
273                        // pre-epoch behavior.
274                        LiteralValue::DateTime(formualizer_common::serial_to_datetime(serial))
275                    } else {
276                        LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
277                    }
278                } else {
279                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
280                }
281            }
282
283            ValueType::Duration => {
284                if let Some(idx) = value_ref.arena_index() {
285                    let scalar_ref = super::scalar::ScalarRef::from_raw(idx | (1 << 31));
286                    if let Some(secs) = self.scalars.get_integer(scalar_ref) {
287                        let dur = chrono::Duration::seconds(secs);
288                        LiteralValue::Duration(dur)
289                    } else {
290                        LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
291                    }
292                } else {
293                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
294                }
295            }
296
297            ValueType::Pending => LiteralValue::Pending,
298
299            ValueType::FormulaAst => {
300                // Formula ASTs shouldn't be returned as values
301                LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
302            }
303        }
304    }
305
306    /// Store an AST node and return its ID
307    pub fn store_ast(&mut self, ast: &ASTNode, sheet_registry: &SheetRegistry) -> AstNodeId {
308        self.convert_ast_node(ast, sheet_registry)
309    }
310
311    /// Retrieve an AST node from its ID
312    pub fn retrieve_ast(&self, id: AstNodeId, sheet_registry: &SheetRegistry) -> Option<ASTNode> {
313        self.reconstruct_ast_node(id, sheet_registry)
314    }
315
316    pub fn resolve_ast_string(&self, id: StringId) -> &str {
317        self.asts.resolve_string(id)
318    }
319
320    pub(crate) fn sheet_names(&self) -> impl Iterator<Item = StringId> + '_ {
321        self.asts.sheet_names()
322    }
323
324    pub(crate) fn function_names(&self) -> impl Iterator<Item = StringId> + '_ {
325        self.asts.function_names()
326    }
327
328    pub(crate) fn resolution_revision(&self) -> u64 {
329        self.asts.resolution_revision()
330    }
331
332    pub(crate) fn ast_strings(&self) -> &StringInterner {
333        self.asts.strings()
334    }
335
336    pub fn reconstruct_reference_type_for_eval(
337        &self,
338        ref_type: &CompactRefType,
339        sheet_registry: &SheetRegistry,
340    ) -> ReferenceType {
341        self.reconstruct_reference_type(ref_type, sheet_registry)
342    }
343
344    /// See [`super::ast::AstArena::compact`].
345    pub(crate) fn compact_asts(
346        &mut self,
347        roots: impl IntoIterator<Item = AstNodeId>,
348    ) -> (Vec<u32>, super::string_interner::StringGarbage) {
349        self.asts.compact(roots)
350    }
351
352    pub(crate) fn ast_node_count(&self) -> usize {
353        self.asts.node_count()
354    }
355
356    pub(crate) fn expanded_node(&self, id: AstNodeId) -> Option<super::ast::AstNodeData> {
357        self.asts.get(id).map(|node| self.asts.expand_node(node))
358    }
359
360    /// See [`super::ast::AstArena::get_compact_ref`].
361    pub fn get_compact_ref(&self, id: super::ast::RefId) -> Option<&CompactRefType> {
362        self.asts.get_compact_ref(id)
363    }
364
365    pub(crate) fn compact_ref(&self, id: super::ast::RefId) -> &CompactRefType {
366        self.asts.compact_ref(id)
367    }
368
369    pub fn get_node(&self, id: AstNodeId) -> Option<&super::ast::AstNodeData<super::ast::RefId>> {
370        self.asts.get(id)
371    }
372
373    /// SUBTOTAL/AGGREGATE call bits of the formula rooted at `id` (see
374    /// `AstNodeEntry::subtotal_calls`).
375    pub(crate) fn ast_subtotal_calls(&self, id: AstNodeId) -> u8 {
376        self.asts.subtotal_calls(id)
377    }
378
379    pub fn get_args(&self, id: AstNodeId) -> Option<&[AstNodeId]> {
380        self.asts.get_function_args(id)
381    }
382
383    pub fn get_array_elems(&self, id: AstNodeId) -> Option<(u16, u16, &[AstNodeId])> {
384        self.asts.get_array_elements_info(id)
385    }
386
387    pub fn ast_needs_structural_rewrite(&self, id: AstNodeId) -> bool {
388        let mut stack = vec![id];
389        while let Some(node_id) = stack.pop() {
390            let Some(node) = self.get_node(node_id) else {
391                continue;
392            };
393            match node {
394                super::ast::AstNodeData::Reference { ref_type, .. } => {
395                    let ref_type = self.compact_ref(*ref_type);
396                    if let CompactRefType::Table { name_id, .. } = ref_type
397                        && self.resolve_ast_string(*name_id).is_empty()
398                    {
399                        return true;
400                    }
401                }
402                super::ast::AstNodeData::UnaryOp { expr_id, .. } => stack.push(*expr_id),
403                super::ast::AstNodeData::BinaryOp {
404                    left_id, right_id, ..
405                } => {
406                    stack.push(*right_id);
407                    stack.push(*left_id);
408                }
409                super::ast::AstNodeData::Function { .. } => {
410                    if let Some(args) = self.get_args(node_id) {
411                        stack.extend(args.iter().rev().copied());
412                    }
413                }
414                super::ast::AstNodeData::Array { .. } => {
415                    if let Some((_, _, elems)) = self.get_array_elems(node_id) {
416                        stack.extend(elems.iter().rev().copied());
417                    }
418                }
419                super::ast::AstNodeData::Literal(_) | super::ast::AstNodeData::Omitted => {}
420            }
421        }
422        false
423    }
424
425    /// Convert ASTNode to arena representation
426    fn convert_ast_node(&mut self, node: &ASTNode, sheet_registry: &SheetRegistry) -> AstNodeId {
427        match &node.node_type {
428            ASTNodeType::Literal(lit) => {
429                let value_ref = self.store_value(lit.clone());
430                self.asts.insert_literal(value_ref)
431            }
432
433            ASTNodeType::Omitted => self.asts.insert_omitted(),
434
435            ASTNodeType::Reference {
436                original,
437                reference,
438            } => {
439                let ref_type = self.convert_reference_type(reference, sheet_registry);
440                self.asts.insert_reference(original, ref_type)
441            }
442
443            ASTNodeType::UnaryOp { op, expr } => {
444                let expr_id = self.convert_ast_node(expr, sheet_registry);
445                self.asts.insert_unary_op(*op, expr_id)
446            }
447
448            ASTNodeType::BinaryOp { op, left, right } => {
449                let left_id = self.convert_ast_node(left, sheet_registry);
450                let right_id = self.convert_ast_node(right, sheet_registry);
451                self.asts.insert_binary_op(*op, left_id, right_id)
452            }
453
454            ASTNodeType::Function { name, args } => {
455                let arg_ids: Vec<AstNodeId> = args
456                    .iter()
457                    .map(|arg| self.convert_ast_node(arg, sheet_registry))
458                    .collect();
459                self.asts.insert_function(name, arg_ids)
460            }
461
462            ASTNodeType::Array(rows) => {
463                let total_elements = rows.iter().map(|r| r.len()).sum();
464                let mut elements = Vec::with_capacity(total_elements);
465
466                let rows_count = rows.len() as u16;
467                let cols_count = rows.first().map(|r| r.len()).unwrap_or(0) as u16;
468
469                for row in rows {
470                    for elem in row {
471                        elements.push(self.convert_ast_node(elem, sheet_registry));
472                    }
473                }
474
475                self.asts.insert_array(rows_count, cols_count, elements)
476            }
477
478            // Postfix call (e.g. LAMBDA immediate-invocation), stored as a
479            // reserved-name function node whose first argument is the callee.
480            ASTNodeType::Call { callee, args } => {
481                let mut arg_ids = Vec::with_capacity(args.len() + 1);
482                arg_ids.push(self.convert_ast_node(callee, sheet_registry));
483                arg_ids.extend(
484                    args.iter()
485                        .map(|arg| self.convert_ast_node(arg, sheet_registry)),
486                );
487                self.asts
488                    .insert_function(super::ast::CALL_NODE_NAME, arg_ids)
489            }
490        }
491    }
492
493    /// Convert ReferenceType to CompactRefType
494    fn convert_reference_type(
495        &mut self,
496        ref_type: &ReferenceType,
497        sheet_registry: &SheetRegistry,
498    ) -> CompactRefType {
499        match ref_type {
500            ReferenceType::Cell {
501                sheet,
502                row,
503                col,
504                row_abs,
505                col_abs,
506            } => {
507                let sheet = match sheet.as_ref() {
508                    Some(s) => match sheet_registry.get_id(s) {
509                        Some(id) => Some(SheetKey::Id(id)),
510                        None => Some(SheetKey::Name(self.asts.strings_mut().intern(s))),
511                    },
512                    None => None,
513                };
514                CompactRefType::Cell {
515                    sheet,
516                    row: *row,
517                    col: *col,
518                    row_abs: *row_abs,
519                    col_abs: *col_abs,
520                }
521            }
522
523            ReferenceType::Range {
524                sheet,
525                start_row,
526                start_col,
527                end_row,
528                end_col,
529                start_row_abs,
530                start_col_abs,
531                end_row_abs,
532                end_col_abs,
533            } => {
534                let sheet = match sheet.as_ref() {
535                    Some(s) => match sheet_registry.get_id(s) {
536                        Some(id) => Some(SheetKey::Id(id)),
537                        None => Some(SheetKey::Name(self.asts.strings_mut().intern(s))),
538                    },
539                    None => None,
540                };
541                // For optional range bounds, use 0/u32::MAX as sentinels for unbounded
542                CompactRefType::Range {
543                    sheet,
544                    start_row: start_row.unwrap_or(0),
545                    start_col: start_col.unwrap_or(0),
546                    end_row: end_row.unwrap_or(u32::MAX),
547                    end_col: end_col.unwrap_or(u32::MAX),
548                    start_row_abs: *start_row_abs,
549                    start_col_abs: *start_col_abs,
550                    end_row_abs: *end_row_abs,
551                    end_col_abs: *end_col_abs,
552                }
553            }
554
555            ReferenceType::External(ext) => {
556                let raw_id = self.asts.strings_mut().intern(&ext.raw);
557                let book_id = self.asts.strings_mut().intern(ext.book.token());
558                let sheet_id = self.asts.strings_mut().intern(&ext.sheet);
559                CompactRefType::External {
560                    raw_id,
561                    book_id,
562                    sheet_id,
563                    kind: ext.kind,
564                }
565            }
566
567            ReferenceType::NamedRange(name) => {
568                let string_id = self.asts.strings_mut().intern(name);
569                CompactRefType::NamedRange(string_id)
570            }
571
572            ReferenceType::Table(table_ref) => {
573                let name_id = self.asts.strings_mut().intern(&table_ref.name);
574                let specifier_id = table_ref
575                    .specifier
576                    .as_ref()
577                    .map(|specifier| self.asts.intern_table_specifier(specifier));
578                CompactRefType::Table {
579                    name_id,
580                    specifier_id,
581                }
582            }
583
584            ReferenceType::Cell3D {
585                sheet_first,
586                sheet_last,
587                row,
588                col,
589                row_abs,
590                col_abs,
591            } => {
592                let sheet_first = self.asts.strings_mut().intern(sheet_first);
593                let sheet_last = self.asts.strings_mut().intern(sheet_last);
594                CompactRefType::Cell3D {
595                    sheet_first,
596                    sheet_last,
597                    row: *row,
598                    col: *col,
599                    row_abs: *row_abs,
600                    col_abs: *col_abs,
601                }
602            }
603
604            ReferenceType::Range3D {
605                sheet_first,
606                sheet_last,
607                start_row,
608                start_col,
609                end_row,
610                end_col,
611                start_row_abs,
612                start_col_abs,
613                end_row_abs,
614                end_col_abs,
615            } => {
616                let sheet_first = self.asts.strings_mut().intern(sheet_first);
617                let sheet_last = self.asts.strings_mut().intern(sheet_last);
618                CompactRefType::Range3D {
619                    sheet_first,
620                    sheet_last,
621                    start_row: start_row.unwrap_or(0),
622                    start_col: start_col.unwrap_or(0),
623                    end_row: end_row.unwrap_or(u32::MAX),
624                    end_col: end_col.unwrap_or(u32::MAX),
625                    start_row_abs: *start_row_abs,
626                    start_col_abs: *start_col_abs,
627                    end_row_abs: *end_row_abs,
628                    end_col_abs: *end_col_abs,
629                }
630            }
631        }
632    }
633
634    /// Reconstruct an ASTNode from arena representation
635    pub(crate) fn reconstruct_ast_node(
636        &self,
637        id: AstNodeId,
638        sheet_registry: &SheetRegistry,
639    ) -> Option<ASTNode> {
640        use super::ast::AstNodeData;
641
642        let node_data = self.asts.get(id)?;
643
644        let node_type = match node_data {
645            AstNodeData::Literal(value_ref) => {
646                let lit = self.retrieve_value(*value_ref);
647                ASTNodeType::Literal(lit)
648            }
649
650            AstNodeData::Omitted => ASTNodeType::Omitted,
651
652            AstNodeData::Reference {
653                original_id,
654                ref_type,
655            } => {
656                let original = self.asts.resolve_string(*original_id).to_string();
657                let reference =
658                    self.reconstruct_reference_type(self.compact_ref(*ref_type), sheet_registry);
659                ASTNodeType::Reference {
660                    original,
661                    reference,
662                }
663            }
664
665            AstNodeData::UnaryOp { operator, expr_id } => {
666                let op = *operator;
667                let expr = Box::new(self.reconstruct_ast_node(*expr_id, sheet_registry)?);
668                ASTNodeType::UnaryOp { op, expr }
669            }
670
671            AstNodeData::BinaryOp {
672                operator,
673                left_id,
674                right_id,
675            } => {
676                let op = *operator;
677                let left = Box::new(self.reconstruct_ast_node(*left_id, sheet_registry)?);
678                let right = Box::new(self.reconstruct_ast_node(*right_id, sheet_registry)?);
679                ASTNodeType::BinaryOp { op, left, right }
680            }
681
682            AstNodeData::Function { name_id, .. } => {
683                let name = self.asts.resolve_string(*name_id).to_string();
684                let arg_ids = self.asts.get_function_args(id)?;
685                let mut args: Vec<ASTNode> = arg_ids
686                    .iter()
687                    .filter_map(|&arg_id| self.reconstruct_ast_node(arg_id, sheet_registry))
688                    .collect();
689                if name == super::ast::CALL_NODE_NAME && !args.is_empty() {
690                    let callee = Box::new(args.remove(0));
691                    ASTNodeType::Call { callee, args }
692                } else {
693                    ASTNodeType::Function { name, args }
694                }
695            }
696
697            AstNodeData::Array { rows, cols, .. } => {
698                let elements = self.asts.get_array_elements(id)?;
699                let mut result = Vec::with_capacity(*rows as usize);
700
701                for r in 0..*rows {
702                    let mut row = Vec::with_capacity(*cols as usize);
703                    for c in 0..*cols {
704                        let idx = (r * *cols + c) as usize;
705                        if let Some(&elem_id) = elements.get(idx)
706                            && let Some(node) = self.reconstruct_ast_node(elem_id, sheet_registry)
707                        {
708                            row.push(node);
709                        }
710                    }
711                    result.push(row);
712                }
713
714                ASTNodeType::Array(result)
715            }
716        };
717
718        Some(ASTNode {
719            node_type,
720            source_token: None, // Token information is not preserved in arena
721            contains_volatile: false,
722        })
723    }
724
725    /// Reconstruct a ReferenceType from CompactRefType
726    fn reconstruct_reference_type(
727        &self,
728        ref_type: &CompactRefType,
729        sheet_registry: &SheetRegistry,
730    ) -> ReferenceType {
731        match ref_type {
732            CompactRefType::Cell {
733                sheet,
734                row,
735                col,
736                row_abs,
737                col_abs,
738            } => {
739                let sheet = match sheet {
740                    Some(SheetKey::Id(id)) => Some(sheet_registry.name(*id).to_string()),
741                    Some(SheetKey::Name(name_id)) => {
742                        Some(self.asts.resolve_string(*name_id).to_string())
743                    }
744                    None => None,
745                };
746                ReferenceType::Cell {
747                    sheet,
748                    row: *row,
749                    col: *col,
750                    row_abs: *row_abs,
751                    col_abs: *col_abs,
752                }
753            }
754
755            CompactRefType::Range {
756                sheet,
757                start_row,
758                start_col,
759                end_row,
760                end_col,
761                start_row_abs,
762                start_col_abs,
763                end_row_abs,
764                end_col_abs,
765            } => {
766                let sheet = match sheet {
767                    Some(SheetKey::Id(id)) => Some(sheet_registry.name(*id).to_string()),
768                    Some(SheetKey::Name(name_id)) => {
769                        Some(self.asts.resolve_string(*name_id).to_string())
770                    }
771                    None => None,
772                };
773                // Convert sentinel values back to None
774                ReferenceType::Range {
775                    sheet,
776                    start_row: if *start_row == 0 {
777                        None
778                    } else {
779                        Some(*start_row)
780                    },
781                    start_col: if *start_col == 0 {
782                        None
783                    } else {
784                        Some(*start_col)
785                    },
786                    end_row: if *end_row == u32::MAX {
787                        None
788                    } else {
789                        Some(*end_row)
790                    },
791                    end_col: if *end_col == u32::MAX {
792                        None
793                    } else {
794                        Some(*end_col)
795                    },
796                    start_row_abs: *start_row_abs,
797                    start_col_abs: *start_col_abs,
798                    end_row_abs: *end_row_abs,
799                    end_col_abs: *end_col_abs,
800                }
801            }
802
803            CompactRefType::External {
804                raw_id,
805                book_id,
806                sheet_id,
807                kind,
808            } => {
809                let raw = self.asts.resolve_string(*raw_id).to_string();
810                let book = self.asts.resolve_string(*book_id).to_string();
811                let sheet = self.asts.resolve_string(*sheet_id).to_string();
812                ReferenceType::External(ExternalReference {
813                    raw,
814                    book: ExternalBookRef::Token(book),
815                    sheet,
816                    kind: *kind,
817                })
818            }
819
820            CompactRefType::NamedRange(string_id) => {
821                let name = self.asts.resolve_string(*string_id).to_string();
822                ReferenceType::NamedRange(name)
823            }
824
825            CompactRefType::Table {
826                name_id,
827                specifier_id,
828            } => {
829                let name = self.asts.resolve_string(*name_id).to_string();
830                let specifier = specifier_id
831                    .and_then(|id| self.asts.resolve_table_specifier(id))
832                    .cloned();
833                ReferenceType::Table(TableReference { name, specifier })
834            }
835
836            CompactRefType::Cell3D {
837                sheet_first,
838                sheet_last,
839                row,
840                col,
841                row_abs,
842                col_abs,
843            } => ReferenceType::Cell3D {
844                sheet_first: self.asts.resolve_string(*sheet_first).to_string(),
845                sheet_last: self.asts.resolve_string(*sheet_last).to_string(),
846                row: *row,
847                col: *col,
848                row_abs: *row_abs,
849                col_abs: *col_abs,
850            },
851
852            CompactRefType::Range3D {
853                sheet_first,
854                sheet_last,
855                start_row,
856                start_col,
857                end_row,
858                end_col,
859                start_row_abs,
860                start_col_abs,
861                end_row_abs,
862                end_col_abs,
863            } => ReferenceType::Range3D {
864                sheet_first: self.asts.resolve_string(*sheet_first).to_string(),
865                sheet_last: self.asts.resolve_string(*sheet_last).to_string(),
866                start_row: if *start_row == 0 {
867                    None
868                } else {
869                    Some(*start_row)
870                },
871                start_col: if *start_col == 0 {
872                    None
873                } else {
874                    Some(*start_col)
875                },
876                end_row: if *end_row == u32::MAX {
877                    None
878                } else {
879                    Some(*end_row)
880                },
881                end_col: if *end_col == u32::MAX {
882                    None
883                } else {
884                    Some(*end_col)
885                },
886                start_row_abs: *start_row_abs,
887                start_col_abs: *start_col_abs,
888                end_row_abs: *end_row_abs,
889                end_col_abs: *end_col_abs,
890            },
891        }
892    }
893
894    /// Store an error with message preservation
895    fn store_error(&mut self, error: &ExcelError) -> ValueRef {
896        let error_ref = self.errors.insert(error);
897        ValueRef::error(error_ref.as_u32())
898    }
899
900    /// Get memory usage statistics
901    pub fn memory_usage(&self) -> DataStoreStats {
902        DataStoreStats {
903            scalar_bytes: self.scalars.memory_usage(),
904            string_bytes: self.strings.memory_usage(),
905            array_bytes: self.arrays.memory_usage(),
906            ast_bytes: self.asts.memory_usage(),
907            error_bytes: self.errors.memory_usage(),
908            total_scalars: self.scalars.len(),
909            total_strings: self.strings.len(),
910            total_arrays: self.arrays.len(),
911            total_ast_nodes: self.asts.stats().node_count,
912            total_errors: self.errors.len(),
913        }
914    }
915
916    /// Clear all data from the store
917    pub fn clear(&mut self) {
918        self.scalars.clear();
919        self.strings.clear();
920        self.arrays.clear();
921        self.asts.clear();
922        self.errors.clear();
923    }
924}
925
926impl Default for DataStore {
927    fn default() -> Self {
928        Self::new()
929    }
930}
931
932/// Statistics about data store memory usage
933#[derive(Debug, Clone)]
934pub struct DataStoreStats {
935    pub scalar_bytes: usize,
936    pub string_bytes: usize,
937    pub array_bytes: usize,
938    pub ast_bytes: usize,
939    pub error_bytes: usize,
940    pub total_scalars: usize,
941    pub total_strings: usize,
942    pub total_arrays: usize,
943    pub total_ast_nodes: usize,
944    pub total_errors: usize,
945}
946
947impl DataStoreStats {
948    pub fn total_bytes(&self) -> usize {
949        self.scalar_bytes + self.string_bytes + self.array_bytes + self.ast_bytes + self.error_bytes
950    }
951}
952
953// Helper trait implementations for ArrayRef and ScalarRef
954impl super::array::ArrayRef {
955    pub fn from_raw(raw: u32) -> Self {
956        super::array::ArrayRef(raw)
957    }
958}
959
960impl super::scalar::ScalarRef {
961    pub fn from_raw(raw: u32) -> Self {
962        Self { raw }
963    }
964
965    pub fn as_u32(self) -> u32 {
966        self.raw
967    }
968}
969
970#[cfg(test)]
971mod tests {
972    use super::*;
973
974    #[test]
975    fn test_data_store_empty_value() {
976        let mut store = DataStore::new();
977        let value_ref = store.store_value(LiteralValue::Empty);
978        assert!(value_ref.is_empty());
979
980        let retrieved = store.retrieve_value(value_ref);
981        assert_eq!(retrieved, LiteralValue::Empty);
982    }
983
984    #[test]
985    fn test_data_store_number() {
986        let mut store = DataStore::new();
987        let value_ref = store.store_value(LiteralValue::Number(42.5));
988
989        let retrieved = store.retrieve_value(value_ref);
990        assert_eq!(retrieved, LiteralValue::Number(42.5));
991    }
992
993    #[test]
994    fn test_data_store_datetime_private_encoding_round_trips() {
995        let mut store = DataStore::new();
996        for datetime in [
997            chrono::NaiveDate::from_ymd_opt(2024, 1, 15)
998                .unwrap()
999                .and_hms_opt(12, 30, 0)
1000                .unwrap(),
1001            chrono::NaiveDate::from_ymd_opt(1899, 12, 30)
1002                .unwrap()
1003                .and_hms_opt(0, 0, 0)
1004                .unwrap(),
1005        ] {
1006            let value_ref = store.store_value(LiteralValue::DateTime(datetime));
1007            assert_eq!(
1008                store.retrieve_value(value_ref),
1009                LiteralValue::DateTime(datetime)
1010            );
1011        }
1012    }
1013
1014    #[test]
1015    fn test_data_store_text() {
1016        let mut store = DataStore::new();
1017        let value_ref = store.store_value(LiteralValue::Text("Hello".to_string()));
1018
1019        let retrieved = store.retrieve_value(value_ref);
1020        assert_eq!(retrieved, LiteralValue::Text("Hello".to_string()));
1021    }
1022
1023    #[test]
1024    fn test_data_store_boolean() {
1025        let mut store = DataStore::new();
1026
1027        let true_ref = store.store_value(LiteralValue::Boolean(true));
1028        let false_ref = store.store_value(LiteralValue::Boolean(false));
1029
1030        assert_eq!(store.retrieve_value(true_ref), LiteralValue::Boolean(true));
1031        assert_eq!(
1032            store.retrieve_value(false_ref),
1033            LiteralValue::Boolean(false)
1034        );
1035    }
1036
1037    #[test]
1038    fn test_data_store_error() {
1039        let mut store = DataStore::new();
1040
1041        let error = ExcelError::new(ExcelErrorKind::Div);
1042        let value_ref = store.store_value(LiteralValue::Error(error.clone()));
1043
1044        let retrieved = store.retrieve_value(value_ref);
1045        match retrieved {
1046            LiteralValue::Error(e) => assert_eq!(e.kind, ExcelErrorKind::Div),
1047            _ => panic!("Expected error"),
1048        }
1049    }
1050
1051    #[test]
1052    fn test_data_store_array() {
1053        let mut store = DataStore::new();
1054
1055        let array = vec![
1056            vec![LiteralValue::Number(1.0), LiteralValue::Number(2.0)],
1057            vec![LiteralValue::Number(3.0), LiteralValue::Number(4.0)],
1058        ];
1059
1060        let value_ref = store.store_value(LiteralValue::Array(array.clone()));
1061        let retrieved = store.retrieve_value(value_ref);
1062
1063        assert_eq!(retrieved, LiteralValue::Array(array));
1064    }
1065
1066    #[test]
1067    fn test_data_store_ast_literal() {
1068        let mut store = DataStore::new();
1069        let mut sheet_registry = SheetRegistry::new();
1070        sheet_registry.id_for("Sheet1");
1071
1072        let ast = ASTNode {
1073            node_type: ASTNodeType::Literal(LiteralValue::Number(42.0)),
1074            source_token: None,
1075            contains_volatile: false,
1076        };
1077
1078        let ast_id = store.store_ast(&ast, &sheet_registry);
1079        let retrieved = store.retrieve_ast(ast_id, &sheet_registry).unwrap();
1080
1081        match retrieved.node_type {
1082            ASTNodeType::Literal(lit) => assert_eq!(lit, LiteralValue::Number(42.0)),
1083            _ => panic!("Expected literal"),
1084        }
1085    }
1086
1087    #[test]
1088    fn test_data_store_ast_binary_op() {
1089        let mut store = DataStore::new();
1090        let mut sheet_registry = SheetRegistry::new();
1091        sheet_registry.id_for("Sheet1");
1092
1093        let ast = ASTNode {
1094            node_type: ASTNodeType::BinaryOp {
1095                op: formualizer_parse::BinaryOperator::Add,
1096                left: Box::new(ASTNode {
1097                    node_type: ASTNodeType::Literal(LiteralValue::Number(1.0)),
1098                    source_token: None,
1099                    contains_volatile: false,
1100                }),
1101                right: Box::new(ASTNode {
1102                    node_type: ASTNodeType::Literal(LiteralValue::Number(2.0)),
1103                    source_token: None,
1104                    contains_volatile: false,
1105                }),
1106            },
1107            source_token: None,
1108            contains_volatile: false,
1109        };
1110
1111        let ast_id = store.store_ast(&ast, &sheet_registry);
1112        let retrieved = store.retrieve_ast(ast_id, &sheet_registry).unwrap();
1113
1114        match retrieved.node_type {
1115            ASTNodeType::BinaryOp { op, left, right } => {
1116                assert_eq!(op.as_str(), "+");
1117                match left.node_type {
1118                    ASTNodeType::Literal(lit) => assert_eq!(lit, LiteralValue::Number(1.0)),
1119                    _ => panic!("Expected literal"),
1120                }
1121                match right.node_type {
1122                    ASTNodeType::Literal(lit) => assert_eq!(lit, LiteralValue::Number(2.0)),
1123                    _ => panic!("Expected literal"),
1124                }
1125            }
1126            _ => panic!("Expected binary op"),
1127        }
1128    }
1129
1130    #[test]
1131    fn test_data_store_ast_function() {
1132        let mut store = DataStore::new();
1133        let mut sheet_registry = SheetRegistry::new();
1134        sheet_registry.id_for("Sheet1");
1135
1136        let ast = ASTNode {
1137            node_type: ASTNodeType::Function {
1138                name: "SUM".to_string(),
1139                args: vec![
1140                    ASTNode {
1141                        node_type: ASTNodeType::Literal(LiteralValue::Number(1.0)),
1142                        source_token: None,
1143                        contains_volatile: false,
1144                    },
1145                    ASTNode {
1146                        node_type: ASTNodeType::Literal(LiteralValue::Number(2.0)),
1147                        source_token: None,
1148                        contains_volatile: false,
1149                    },
1150                ],
1151            },
1152            source_token: None,
1153            contains_volatile: false,
1154        };
1155
1156        let ast_id = store.store_ast(&ast, &sheet_registry);
1157        let retrieved = store.retrieve_ast(ast_id, &sheet_registry).unwrap();
1158
1159        match retrieved.node_type {
1160            ASTNodeType::Function { name, args } => {
1161                assert_eq!(name, "SUM");
1162                assert_eq!(args.len(), 2);
1163            }
1164            _ => panic!("Expected function"),
1165        }
1166    }
1167
1168    #[test]
1169    fn test_data_store_memory_stats() {
1170        let mut store = DataStore::new();
1171
1172        // Add some data
1173        store.store_value(LiteralValue::Number(42.0));
1174        store.store_value(LiteralValue::Text("Hello".to_string()));
1175        store.store_value(LiteralValue::Array(vec![vec![LiteralValue::Number(1.0)]]));
1176
1177        let stats = store.memory_usage();
1178        assert!(stats.total_bytes() > 0);
1179        assert_eq!(stats.total_scalars, 2); // 42.0 and 1.0
1180        assert_eq!(stats.total_strings, 1); // "Hello"
1181        assert_eq!(stats.total_arrays, 1);
1182    }
1183
1184    #[test]
1185    fn test_data_store_clear() {
1186        let mut store = DataStore::new();
1187
1188        store.store_value(LiteralValue::Number(42.0));
1189        store.store_value(LiteralValue::Text("Hello".to_string()));
1190
1191        let stats = store.memory_usage();
1192        assert!(stats.total_scalars > 0);
1193        assert!(stats.total_strings > 0);
1194
1195        store.clear();
1196
1197        let stats = store.memory_usage();
1198        assert_eq!(stats.total_scalars, 0);
1199        assert_eq!(stats.total_strings, 0);
1200    }
1201}
formualizer_eval/engine/arena/data_store.rs

formualizer_eval/engine/arena/
data_store.rs
