formualizer_eval/engine/arena/
data_store.rs

1/// Unified data storage for all value types using arenas
2/// Provides conversion between LiteralValue and ValueRef
3use super::array::ArrayArena;
4use super::ast::{AstArena, AstNodeId, CompactRefType, SheetKey};
5use super::error_arena::{ErrorArena, ErrorRef};
6use super::scalar::ScalarArena;
7use super::string_interner::{StringId, StringInterner};
8use super::value_ref::ValueRef;
9use crate::engine::sheet_registry::SheetRegistry;
10use formualizer_common::{ExcelError, ExcelErrorKind, LiteralValue};
11use formualizer_parse::parser::{
12    ASTNode, ASTNodeType, ExternalBookRef, ExternalReference, ReferenceType, TableReference,
13};
14
15/// Centralized data storage using arenas
16#[derive(Debug)]
17pub struct DataStore {
18    /// Scalar values (floats and large integers)
19    scalars: ScalarArena,
20
21    /// String values
22    strings: StringInterner,
23
24    /// Array values
25    arrays: ArrayArena,
26
27    /// AST nodes for formulas
28    asts: AstArena,
29
30    /// Error storage with message preservation
31    errors: ErrorArena,
32}
33
34impl DataStore {
35    pub fn new() -> Self {
36        Self {
37            scalars: ScalarArena::new(),
38            strings: StringInterner::new(),
39            arrays: ArrayArena::new(),
40            asts: AstArena::new(),
41            errors: ErrorArena::new(),
42        }
43    }
44
45    /// Batch store literal values; returns ValueRefs in same order.
46    pub fn store_values_batch<I>(&mut self, values: I) -> Vec<ValueRef>
47    where
48        I: IntoIterator<Item = LiteralValue>,
49    {
50        let iter = values.into_iter();
51        let (lower, _) = iter.size_hint();
52        let mut out = Vec::with_capacity(lower);
53        for v in iter {
54            out.push(self.store_value(v));
55        }
56        out
57    }
58
59    /// Batch store ASTs; returns AstNodeIds in same order.
60    pub fn store_asts_batch<'a, I>(
61        &mut self,
62        asts: I,
63        sheet_registry: &SheetRegistry,
64    ) -> Vec<AstNodeId>
65    where
66        I: IntoIterator<Item = &'a ASTNode>,
67    {
68        let iter = asts.into_iter();
69        let (lower, _) = iter.size_hint();
70        let mut out = Vec::with_capacity(lower);
71        for ast in iter {
72            out.push(self.store_ast(ast, sheet_registry));
73        }
74        out
75    }
76
77    pub fn with_capacity(estimated_cells: usize) -> Self {
78        Self {
79            scalars: ScalarArena::with_capacity(estimated_cells),
80            strings: StringInterner::with_capacity(estimated_cells / 10),
81            arrays: ArrayArena::with_capacity(estimated_cells / 100),
82            asts: AstArena::with_capacity(estimated_cells / 2),
83            errors: ErrorArena::with_capacity(estimated_cells / 20),
84        }
85    }
86
87    /// Store a LiteralValue and return a ValueRef
88    pub fn store_value(&mut self, value: LiteralValue) -> ValueRef {
89        match value {
90            LiteralValue::Empty => ValueRef::empty(),
91
92            LiteralValue::Number(n) => {
93                // Store as float in scalar arena
94                let idx = self.scalars.insert_float(n);
95                ValueRef::number(idx.as_u32())
96            }
97
98            LiteralValue::Text(s) => {
99                let id = self.strings.intern(&s);
100                ValueRef::string(id.as_u32())
101            }
102
103            LiteralValue::Boolean(b) => ValueRef::boolean(b),
104
105            LiteralValue::Error(err) => self.store_error(&err),
106
107            LiteralValue::Array(array) => {
108                // Convert nested array to ValueRefs
109                let rows = array.len() as u32;
110                let cols = array.first().map(|r| r.len()).unwrap_or(0) as u32;
111
112                let elements: Vec<ValueRef> = array
113                    .into_iter()
114                    .flatten()
115                    .map(|v| self.store_value(v))
116                    .collect();
117
118                let array_ref = self.arrays.insert(rows, cols, elements);
119                ValueRef::array(array_ref.as_u32())
120            }
121
122            LiteralValue::DateTime(dt) => {
123                // The arena uses Excel1900 only as a private, symmetric binary
124                // encoding for chrono values. This is not workbook serial
125                // state and the raw float is never exposed to formula logic.
126                let serial = formualizer_common::datetime_to_serial_for(
127                    formualizer_common::DateSystem::Excel1900,
128                    &dt,
129                );
130                let idx = self.scalars.insert_float(serial);
131                ValueRef::date_time(idx.as_u32())
132            }
133
134            LiteralValue::Date(d) => {
135                // Use the same private arena encoding as DateTime values.
136                let dt = d.and_hms_opt(0, 0, 0).unwrap();
137                let serial = formualizer_common::datetime_to_serial_for(
138                    formualizer_common::DateSystem::Excel1900,
139                    &dt,
140                );
141                let idx = self.scalars.insert_float(serial);
142                ValueRef::date_time(idx.as_u32())
143            }
144
145            LiteralValue::Time(t) => {
146                // Store time as fractional day.
147                let fraction = formualizer_common::time_to_fraction(&t);
148                let idx = self.scalars.insert_float(fraction);
149                ValueRef::date_time(idx.as_u32())
150            }
151
152            LiteralValue::Duration(dur) => {
153                // Store as integer seconds (chrono::Duration has num_seconds())
154                let secs = dur.num_seconds();
155                let idx = self.scalars.insert_integer(secs);
156                let raw_index = idx.as_u32() & 0x7FFF_FFFF;
157                ValueRef::duration(raw_index)
158            }
159
160            LiteralValue::Int(i) => {
161                // Try to use small int optimization
162                if let Some(vref) = ValueRef::small_int(i as i32) {
163                    vref
164                } else {
165                    // Store as large integer
166                    let idx = self.scalars.insert_integer(i);
167                    ValueRef::large_int(idx.as_u32())
168                }
169            }
170
171            LiteralValue::Pending => ValueRef::pending(),
172        }
173    }
174
175    /// Retrieve a LiteralValue from a ValueRef
176    pub fn retrieve_value(&self, value_ref: ValueRef) -> LiteralValue {
177        use super::value_ref::ValueType;
178
179        match value_ref.value_type() {
180            ValueType::Empty => LiteralValue::Empty,
181
182            ValueType::SmallInt => {
183                // Small integers are inlined
184                if let Some(i) = value_ref.as_small_int() {
185                    LiteralValue::Int(i as i64)
186                } else {
187                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
188                }
189            }
190
191            ValueType::LargeInt => {
192                if let Some(idx) = value_ref.arena_index() {
193                    let scalar_ref = super::scalar::ScalarRef::from_raw(idx | (1 << 31));
194                    if let Some(i) = self.scalars.get_integer(scalar_ref) {
195                        LiteralValue::Int(i)
196                    } else {
197                        LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
198                    }
199                } else {
200                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
201                }
202            }
203
204            ValueType::Number => {
205                if let Some(idx) = value_ref.arena_index() {
206                    let scalar_ref = super::scalar::ScalarRef::from_raw(idx);
207                    if let Some(f) = self.scalars.get_float(scalar_ref) {
208                        LiteralValue::Number(f)
209                    } else {
210                        LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
211                    }
212                } else {
213                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
214                }
215            }
216
217            ValueType::String => {
218                if let Some(idx) = value_ref.arena_index() {
219                    let string_id = StringId::from_raw(idx);
220                    let s = self.strings.resolve(string_id);
221                    LiteralValue::Text(s.to_string())
222                } else {
223                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
224                }
225            }
226
227            ValueType::Boolean => {
228                if let Some(b) = value_ref.as_boolean() {
229                    LiteralValue::Boolean(b)
230                } else {
231                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
232                }
233            }
234
235            ValueType::Error => {
236                if let Some(error_ref_raw) = value_ref.as_error_ref() {
237                    let error_ref = ErrorRef::from_raw(error_ref_raw);
238                    if let Some(error) = self.errors.get(error_ref) {
239                        LiteralValue::Error(error)
240                    } else {
241                        LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
242                    }
243                } else {
244                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
245                }
246            }
247
248            ValueType::Array => {
249                if let Some(idx) = value_ref.arena_index() {
250                    let array_ref = super::array::ArrayRef::from_raw(idx);
251                    if let Some(array_2d) = self.arrays.get_2d(array_ref) {
252                        // Convert back to LiteralValue array
253                        let result: Vec<Vec<LiteralValue>> = array_2d
254                            .into_iter()
255                            .map(|row| row.into_iter().map(|v| self.retrieve_value(v)).collect())
256                            .collect();
257                        LiteralValue::Array(result)
258                    } else {
259                        LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
260                    }
261                } else {
262                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
263                }
264            }
265
266            ValueType::DateTime => {
267                if let Some(idx) = value_ref.arena_index() {
268                    let scalar_ref = super::scalar::ScalarRef::from_raw(idx);
269                    if let Some(serial) = self.scalars.get_float(scalar_ref) {
270                        // This is a trusted private encoding, not an untrusted
271                        // workbook serial boundary. Use the compatibility
272                        // decoder so the arena retains its historical
273                        // pre-epoch behavior.
274                        LiteralValue::DateTime(formualizer_common::serial_to_datetime(serial))
275                    } else {
276                        LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
277                    }
278                } else {
279                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
280                }
281            }
282
283            ValueType::Duration => {
284                if let Some(idx) = value_ref.arena_index() {
285                    let scalar_ref = super::scalar::ScalarRef::from_raw(idx | (1 << 31));
286                    if let Some(secs) = self.scalars.get_integer(scalar_ref) {
287                        let dur = chrono::Duration::seconds(secs);
288                        LiteralValue::Duration(dur)
289                    } else {
290                        LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
291                    }
292                } else {
293                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
294                }
295            }
296
297            ValueType::Pending => LiteralValue::Pending,
298
299            ValueType::FormulaAst => {
300                // Formula ASTs shouldn't be returned as values
301                LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
302            }
303        }
304    }
305
306    /// Store an AST node and return its ID
307    pub fn store_ast(&mut self, ast: &ASTNode, sheet_registry: &SheetRegistry) -> AstNodeId {
308        self.convert_ast_node(ast, sheet_registry)
309    }
310
311    /// Retrieve an AST node from its ID
312    pub fn retrieve_ast(&self, id: AstNodeId, sheet_registry: &SheetRegistry) -> Option<ASTNode> {
313        self.reconstruct_ast_node(id, sheet_registry)
314    }
315
316    pub fn resolve_ast_string(&self, id: StringId) -> &str {
317        self.asts.resolve_string(id)
318    }
319
320    pub(crate) fn ast_strings(&self) -> &StringInterner {
321        self.asts.strings()
322    }
323
324    pub fn reconstruct_reference_type_for_eval(
325        &self,
326        ref_type: &CompactRefType,
327        sheet_registry: &SheetRegistry,
328    ) -> ReferenceType {
329        self.reconstruct_reference_type(ref_type, sheet_registry)
330    }
331
332    /// See [`super::ast::AstArena::compact`].
333    pub(crate) fn compact_asts(
334        &mut self,
335        roots: impl IntoIterator<Item = AstNodeId>,
336    ) -> (Vec<u32>, super::string_interner::StringGarbage) {
337        self.asts.compact(roots)
338    }
339
340    pub(crate) fn ast_node_count(&self) -> usize {
341        self.asts.node_count()
342    }
343
344    pub fn get_node(&self, id: AstNodeId) -> Option<&super::ast::AstNodeData> {
345        self.asts.get(id)
346    }
347
348    pub fn get_args(&self, id: AstNodeId) -> Option<&[AstNodeId]> {
349        self.asts.get_function_args(id)
350    }
351
352    pub fn get_array_elems(&self, id: AstNodeId) -> Option<(u16, u16, &[AstNodeId])> {
353        self.asts.get_array_elements_info(id)
354    }
355
356    pub fn ast_needs_structural_rewrite(&self, id: AstNodeId) -> bool {
357        let mut stack = vec![id];
358        while let Some(node_id) = stack.pop() {
359            let Some(node) = self.get_node(node_id) else {
360                continue;
361            };
362            match node {
363                super::ast::AstNodeData::Reference { ref_type, .. } => {
364                    if let CompactRefType::Table { name_id, .. } = ref_type
365                        && self.resolve_ast_string(*name_id).is_empty()
366                    {
367                        return true;
368                    }
369                }
370                super::ast::AstNodeData::UnaryOp { expr_id, .. } => stack.push(*expr_id),
371                super::ast::AstNodeData::BinaryOp {
372                    left_id, right_id, ..
373                } => {
374                    stack.push(*right_id);
375                    stack.push(*left_id);
376                }
377                super::ast::AstNodeData::Function { .. } => {
378                    if let Some(args) = self.get_args(node_id) {
379                        stack.extend(args.iter().rev().copied());
380                    }
381                }
382                super::ast::AstNodeData::Array { .. } => {
383                    if let Some((_, _, elems)) = self.get_array_elems(node_id) {
384                        stack.extend(elems.iter().rev().copied());
385                    }
386                }
387                super::ast::AstNodeData::Literal(_) | super::ast::AstNodeData::Omitted => {}
388            }
389        }
390        false
391    }
392
393    /// Convert ASTNode to arena representation
394    fn convert_ast_node(&mut self, node: &ASTNode, sheet_registry: &SheetRegistry) -> AstNodeId {
395        match &node.node_type {
396            ASTNodeType::Literal(lit) => {
397                let value_ref = self.store_value(lit.clone());
398                self.asts.insert_literal(value_ref)
399            }
400
401            ASTNodeType::Omitted => self.asts.insert_omitted(),
402
403            ASTNodeType::Reference {
404                original,
405                reference,
406            } => {
407                let ref_type = self.convert_reference_type(reference, sheet_registry);
408                self.asts.insert_reference(original, ref_type)
409            }
410
411            ASTNodeType::UnaryOp { op, expr } => {
412                let expr_id = self.convert_ast_node(expr, sheet_registry);
413                self.asts.insert_unary_op(op, expr_id)
414            }
415
416            ASTNodeType::BinaryOp { op, left, right } => {
417                let left_id = self.convert_ast_node(left, sheet_registry);
418                let right_id = self.convert_ast_node(right, sheet_registry);
419                self.asts.insert_binary_op(op, left_id, right_id)
420            }
421
422            ASTNodeType::Function { name, args } => {
423                let arg_ids: Vec<AstNodeId> = args
424                    .iter()
425                    .map(|arg| self.convert_ast_node(arg, sheet_registry))
426                    .collect();
427                self.asts.insert_function(name, arg_ids)
428            }
429
430            ASTNodeType::Array(rows) => {
431                let total_elements = rows.iter().map(|r| r.len()).sum();
432                let mut elements = Vec::with_capacity(total_elements);
433
434                let rows_count = rows.len() as u16;
435                let cols_count = rows.first().map(|r| r.len()).unwrap_or(0) as u16;
436
437                for row in rows {
438                    for elem in row {
439                        elements.push(self.convert_ast_node(elem, sheet_registry));
440                    }
441                }
442
443                self.asts.insert_array(rows_count, cols_count, elements)
444            }
445
446            // Postfix call (e.g. LAMBDA immediate-invocation), stored as a
447            // reserved-name function node whose first argument is the callee.
448            ASTNodeType::Call { callee, args } => {
449                let mut arg_ids = Vec::with_capacity(args.len() + 1);
450                arg_ids.push(self.convert_ast_node(callee, sheet_registry));
451                arg_ids.extend(
452                    args.iter()
453                        .map(|arg| self.convert_ast_node(arg, sheet_registry)),
454                );
455                self.asts
456                    .insert_function(super::ast::CALL_NODE_NAME, arg_ids)
457            }
458        }
459    }
460
461    /// Convert ReferenceType to CompactRefType
462    fn convert_reference_type(
463        &mut self,
464        ref_type: &ReferenceType,
465        sheet_registry: &SheetRegistry,
466    ) -> CompactRefType {
467        match ref_type {
468            ReferenceType::Cell {
469                sheet,
470                row,
471                col,
472                row_abs,
473                col_abs,
474            } => {
475                let sheet = match sheet.as_ref() {
476                    Some(s) => match sheet_registry.get_id(s) {
477                        Some(id) => Some(SheetKey::Id(id)),
478                        None => Some(SheetKey::Name(self.asts.strings_mut().intern(s))),
479                    },
480                    None => None,
481                };
482                CompactRefType::Cell {
483                    sheet,
484                    row: *row,
485                    col: *col,
486                    row_abs: *row_abs,
487                    col_abs: *col_abs,
488                }
489            }
490
491            ReferenceType::Range {
492                sheet,
493                start_row,
494                start_col,
495                end_row,
496                end_col,
497                start_row_abs,
498                start_col_abs,
499                end_row_abs,
500                end_col_abs,
501            } => {
502                let sheet = match sheet.as_ref() {
503                    Some(s) => match sheet_registry.get_id(s) {
504                        Some(id) => Some(SheetKey::Id(id)),
505                        None => Some(SheetKey::Name(self.asts.strings_mut().intern(s))),
506                    },
507                    None => None,
508                };
509                // For optional range bounds, use 0/u32::MAX as sentinels for unbounded
510                CompactRefType::Range {
511                    sheet,
512                    start_row: start_row.unwrap_or(0),
513                    start_col: start_col.unwrap_or(0),
514                    end_row: end_row.unwrap_or(u32::MAX),
515                    end_col: end_col.unwrap_or(u32::MAX),
516                    start_row_abs: *start_row_abs,
517                    start_col_abs: *start_col_abs,
518                    end_row_abs: *end_row_abs,
519                    end_col_abs: *end_col_abs,
520                }
521            }
522
523            ReferenceType::External(ext) => {
524                let raw_id = self.asts.strings_mut().intern(&ext.raw);
525                let book_id = self.asts.strings_mut().intern(ext.book.token());
526                let sheet_id = self.asts.strings_mut().intern(&ext.sheet);
527                CompactRefType::External {
528                    raw_id,
529                    book_id,
530                    sheet_id,
531                    kind: ext.kind,
532                }
533            }
534
535            ReferenceType::NamedRange(name) => {
536                let string_id = self.asts.strings_mut().intern(name);
537                CompactRefType::NamedRange(string_id)
538            }
539
540            ReferenceType::Table(table_ref) => {
541                let name_id = self.asts.strings_mut().intern(&table_ref.name);
542                let specifier_id = table_ref
543                    .specifier
544                    .as_ref()
545                    .map(|specifier| self.asts.intern_table_specifier(specifier));
546                CompactRefType::Table {
547                    name_id,
548                    specifier_id,
549                }
550            }
551
552            ReferenceType::Cell3D {
553                sheet_first,
554                sheet_last,
555                row,
556                col,
557                row_abs,
558                col_abs,
559            } => {
560                let sheet_first = self.asts.strings_mut().intern(sheet_first);
561                let sheet_last = self.asts.strings_mut().intern(sheet_last);
562                CompactRefType::Cell3D {
563                    sheet_first,
564                    sheet_last,
565                    row: *row,
566                    col: *col,
567                    row_abs: *row_abs,
568                    col_abs: *col_abs,
569                }
570            }
571
572            ReferenceType::Range3D {
573                sheet_first,
574                sheet_last,
575                start_row,
576                start_col,
577                end_row,
578                end_col,
579                start_row_abs,
580                start_col_abs,
581                end_row_abs,
582                end_col_abs,
583            } => {
584                let sheet_first = self.asts.strings_mut().intern(sheet_first);
585                let sheet_last = self.asts.strings_mut().intern(sheet_last);
586                CompactRefType::Range3D {
587                    sheet_first,
588                    sheet_last,
589                    start_row: start_row.unwrap_or(0),
590                    start_col: start_col.unwrap_or(0),
591                    end_row: end_row.unwrap_or(u32::MAX),
592                    end_col: end_col.unwrap_or(u32::MAX),
593                    start_row_abs: *start_row_abs,
594                    start_col_abs: *start_col_abs,
595                    end_row_abs: *end_row_abs,
596                    end_col_abs: *end_col_abs,
597                }
598            }
599        }
600    }
601
602    /// Reconstruct an ASTNode from arena representation
603    pub(crate) fn reconstruct_ast_node(
604        &self,
605        id: AstNodeId,
606        sheet_registry: &SheetRegistry,
607    ) -> Option<ASTNode> {
608        use super::ast::AstNodeData;
609
610        let node_data = self.asts.get(id)?;
611
612        let node_type = match node_data {
613            AstNodeData::Literal(value_ref) => {
614                let lit = self.retrieve_value(*value_ref);
615                ASTNodeType::Literal(lit)
616            }
617
618            AstNodeData::Omitted => ASTNodeType::Omitted,
619
620            AstNodeData::Reference {
621                original_id,
622                ref_type,
623            } => {
624                let original = self.asts.resolve_string(*original_id).to_string();
625                let reference = self.reconstruct_reference_type(ref_type, sheet_registry);
626                ASTNodeType::Reference {
627                    original,
628                    reference,
629                }
630            }
631
632            AstNodeData::UnaryOp { op_id, expr_id } => {
633                let op = self.asts.resolve_string(*op_id).to_string();
634                let expr = Box::new(self.reconstruct_ast_node(*expr_id, sheet_registry)?);
635                ASTNodeType::UnaryOp { op, expr }
636            }
637
638            AstNodeData::BinaryOp {
639                op_id,
640                left_id,
641                right_id,
642            } => {
643                let op = self.asts.resolve_string(*op_id).to_string();
644                let left = Box::new(self.reconstruct_ast_node(*left_id, sheet_registry)?);
645                let right = Box::new(self.reconstruct_ast_node(*right_id, sheet_registry)?);
646                ASTNodeType::BinaryOp { op, left, right }
647            }
648
649            AstNodeData::Function { name_id, .. } => {
650                let name = self.asts.resolve_string(*name_id).to_string();
651                let arg_ids = self.asts.get_function_args(id)?;
652                let mut args: Vec<ASTNode> = arg_ids
653                    .iter()
654                    .filter_map(|&arg_id| self.reconstruct_ast_node(arg_id, sheet_registry))
655                    .collect();
656                if name == super::ast::CALL_NODE_NAME && !args.is_empty() {
657                    let callee = Box::new(args.remove(0));
658                    ASTNodeType::Call { callee, args }
659                } else {
660                    ASTNodeType::Function { name, args }
661                }
662            }
663
664            AstNodeData::Array { rows, cols, .. } => {
665                let elements = self.asts.get_array_elements(id)?;
666                let mut result = Vec::with_capacity(*rows as usize);
667
668                for r in 0..*rows {
669                    let mut row = Vec::with_capacity(*cols as usize);
670                    for c in 0..*cols {
671                        let idx = (r * *cols + c) as usize;
672                        if let Some(&elem_id) = elements.get(idx)
673                            && let Some(node) = self.reconstruct_ast_node(elem_id, sheet_registry)
674                        {
675                            row.push(node);
676                        }
677                    }
678                    result.push(row);
679                }
680
681                ASTNodeType::Array(result)
682            }
683        };
684
685        Some(ASTNode {
686            node_type,
687            source_token: None, // Token information is not preserved in arena
688            contains_volatile: false,
689        })
690    }
691
692    /// Reconstruct a ReferenceType from CompactRefType
693    fn reconstruct_reference_type(
694        &self,
695        ref_type: &CompactRefType,
696        sheet_registry: &SheetRegistry,
697    ) -> ReferenceType {
698        match ref_type {
699            CompactRefType::Cell {
700                sheet,
701                row,
702                col,
703                row_abs,
704                col_abs,
705            } => {
706                let sheet = match sheet {
707                    Some(SheetKey::Id(id)) => Some(sheet_registry.name(*id).to_string()),
708                    Some(SheetKey::Name(name_id)) => {
709                        Some(self.asts.resolve_string(*name_id).to_string())
710                    }
711                    None => None,
712                };
713                ReferenceType::Cell {
714                    sheet,
715                    row: *row,
716                    col: *col,
717                    row_abs: *row_abs,
718                    col_abs: *col_abs,
719                }
720            }
721
722            CompactRefType::Range {
723                sheet,
724                start_row,
725                start_col,
726                end_row,
727                end_col,
728                start_row_abs,
729                start_col_abs,
730                end_row_abs,
731                end_col_abs,
732            } => {
733                let sheet = match sheet {
734                    Some(SheetKey::Id(id)) => Some(sheet_registry.name(*id).to_string()),
735                    Some(SheetKey::Name(name_id)) => {
736                        Some(self.asts.resolve_string(*name_id).to_string())
737                    }
738                    None => None,
739                };
740                // Convert sentinel values back to None
741                ReferenceType::Range {
742                    sheet,
743                    start_row: if *start_row == 0 {
744                        None
745                    } else {
746                        Some(*start_row)
747                    },
748                    start_col: if *start_col == 0 {
749                        None
750                    } else {
751                        Some(*start_col)
752                    },
753                    end_row: if *end_row == u32::MAX {
754                        None
755                    } else {
756                        Some(*end_row)
757                    },
758                    end_col: if *end_col == u32::MAX {
759                        None
760                    } else {
761                        Some(*end_col)
762                    },
763                    start_row_abs: *start_row_abs,
764                    start_col_abs: *start_col_abs,
765                    end_row_abs: *end_row_abs,
766                    end_col_abs: *end_col_abs,
767                }
768            }
769
770            CompactRefType::External {
771                raw_id,
772                book_id,
773                sheet_id,
774                kind,
775            } => {
776                let raw = self.asts.resolve_string(*raw_id).to_string();
777                let book = self.asts.resolve_string(*book_id).to_string();
778                let sheet = self.asts.resolve_string(*sheet_id).to_string();
779                ReferenceType::External(ExternalReference {
780                    raw,
781                    book: ExternalBookRef::Token(book),
782                    sheet,
783                    kind: *kind,
784                })
785            }
786
787            CompactRefType::NamedRange(string_id) => {
788                let name = self.asts.resolve_string(*string_id).to_string();
789                ReferenceType::NamedRange(name)
790            }
791
792            CompactRefType::Table {
793                name_id,
794                specifier_id,
795            } => {
796                let name = self.asts.resolve_string(*name_id).to_string();
797                let specifier = specifier_id
798                    .and_then(|id| self.asts.resolve_table_specifier(id))
799                    .cloned();
800                ReferenceType::Table(TableReference { name, specifier })
801            }
802
803            CompactRefType::Cell3D {
804                sheet_first,
805                sheet_last,
806                row,
807                col,
808                row_abs,
809                col_abs,
810            } => ReferenceType::Cell3D {
811                sheet_first: self.asts.resolve_string(*sheet_first).to_string(),
812                sheet_last: self.asts.resolve_string(*sheet_last).to_string(),
813                row: *row,
814                col: *col,
815                row_abs: *row_abs,
816                col_abs: *col_abs,
817            },
818
819            CompactRefType::Range3D {
820                sheet_first,
821                sheet_last,
822                start_row,
823                start_col,
824                end_row,
825                end_col,
826                start_row_abs,
827                start_col_abs,
828                end_row_abs,
829                end_col_abs,
830            } => ReferenceType::Range3D {
831                sheet_first: self.asts.resolve_string(*sheet_first).to_string(),
832                sheet_last: self.asts.resolve_string(*sheet_last).to_string(),
833                start_row: if *start_row == 0 {
834                    None
835                } else {
836                    Some(*start_row)
837                },
838                start_col: if *start_col == 0 {
839                    None
840                } else {
841                    Some(*start_col)
842                },
843                end_row: if *end_row == u32::MAX {
844                    None
845                } else {
846                    Some(*end_row)
847                },
848                end_col: if *end_col == u32::MAX {
849                    None
850                } else {
851                    Some(*end_col)
852                },
853                start_row_abs: *start_row_abs,
854                start_col_abs: *start_col_abs,
855                end_row_abs: *end_row_abs,
856                end_col_abs: *end_col_abs,
857            },
858        }
859    }
860
861    /// Store an error with message preservation
862    fn store_error(&mut self, error: &ExcelError) -> ValueRef {
863        let error_ref = self.errors.insert(error);
864        ValueRef::error(error_ref.as_u32())
865    }
866
867    /// Get memory usage statistics
868    pub fn memory_usage(&self) -> DataStoreStats {
869        DataStoreStats {
870            scalar_bytes: self.scalars.memory_usage(),
871            string_bytes: self.strings.memory_usage(),
872            array_bytes: self.arrays.memory_usage(),
873            ast_bytes: self.asts.memory_usage(),
874            error_bytes: self.errors.memory_usage(),
875            total_scalars: self.scalars.len(),
876            total_strings: self.strings.len(),
877            total_arrays: self.arrays.len(),
878            total_ast_nodes: self.asts.stats().node_count,
879            total_errors: self.errors.len(),
880        }
881    }
882
883    /// Clear all data from the store
884    pub fn clear(&mut self) {
885        self.scalars.clear();
886        self.strings.clear();
887        self.arrays.clear();
888        self.asts.clear();
889        self.errors.clear();
890    }
891}
892
893impl Default for DataStore {
894    fn default() -> Self {
895        Self::new()
896    }
897}
898
899/// Statistics about data store memory usage
900#[derive(Debug, Clone)]
901pub struct DataStoreStats {
902    pub scalar_bytes: usize,
903    pub string_bytes: usize,
904    pub array_bytes: usize,
905    pub ast_bytes: usize,
906    pub error_bytes: usize,
907    pub total_scalars: usize,
908    pub total_strings: usize,
909    pub total_arrays: usize,
910    pub total_ast_nodes: usize,
911    pub total_errors: usize,
912}
913
914impl DataStoreStats {
915    pub fn total_bytes(&self) -> usize {
916        self.scalar_bytes + self.string_bytes + self.array_bytes + self.ast_bytes + self.error_bytes
917    }
918}
919
920// Helper trait implementations for ArrayRef and ScalarRef
921impl super::array::ArrayRef {
922    pub fn from_raw(raw: u32) -> Self {
923        super::array::ArrayRef(raw)
924    }
925}
926
927impl super::scalar::ScalarRef {
928    pub fn from_raw(raw: u32) -> Self {
929        Self { raw }
930    }
931
932    pub fn as_u32(self) -> u32 {
933        self.raw
934    }
935}
936
937#[cfg(test)]
938mod tests {
939    use super::*;
940
941    #[test]
942    fn test_data_store_empty_value() {
943        let mut store = DataStore::new();
944        let value_ref = store.store_value(LiteralValue::Empty);
945        assert!(value_ref.is_empty());
946
947        let retrieved = store.retrieve_value(value_ref);
948        assert_eq!(retrieved, LiteralValue::Empty);
949    }
950
951    #[test]
952    fn test_data_store_number() {
953        let mut store = DataStore::new();
954        let value_ref = store.store_value(LiteralValue::Number(42.5));
955
956        let retrieved = store.retrieve_value(value_ref);
957        assert_eq!(retrieved, LiteralValue::Number(42.5));
958    }
959
960    #[test]
961    fn test_data_store_datetime_private_encoding_round_trips() {
962        let mut store = DataStore::new();
963        for datetime in [
964            chrono::NaiveDate::from_ymd_opt(2024, 1, 15)
965                .unwrap()
966                .and_hms_opt(12, 30, 0)
967                .unwrap(),
968            chrono::NaiveDate::from_ymd_opt(1899, 12, 30)
969                .unwrap()
970                .and_hms_opt(0, 0, 0)
971                .unwrap(),
972        ] {
973            let value_ref = store.store_value(LiteralValue::DateTime(datetime));
974            assert_eq!(
975                store.retrieve_value(value_ref),
976                LiteralValue::DateTime(datetime)
977            );
978        }
979    }
980
981    #[test]
982    fn test_data_store_text() {
983        let mut store = DataStore::new();
984        let value_ref = store.store_value(LiteralValue::Text("Hello".to_string()));
985
986        let retrieved = store.retrieve_value(value_ref);
987        assert_eq!(retrieved, LiteralValue::Text("Hello".to_string()));
988    }
989
990    #[test]
991    fn test_data_store_boolean() {
992        let mut store = DataStore::new();
993
994        let true_ref = store.store_value(LiteralValue::Boolean(true));
995        let false_ref = store.store_value(LiteralValue::Boolean(false));
996
997        assert_eq!(store.retrieve_value(true_ref), LiteralValue::Boolean(true));
998        assert_eq!(
999            store.retrieve_value(false_ref),
1000            LiteralValue::Boolean(false)
1001        );
1002    }
1003
1004    #[test]
1005    fn test_data_store_error() {
1006        let mut store = DataStore::new();
1007
1008        let error = ExcelError::new(ExcelErrorKind::Div);
1009        let value_ref = store.store_value(LiteralValue::Error(error.clone()));
1010
1011        let retrieved = store.retrieve_value(value_ref);
1012        match retrieved {
1013            LiteralValue::Error(e) => assert_eq!(e.kind, ExcelErrorKind::Div),
1014            _ => panic!("Expected error"),
1015        }
1016    }
1017
1018    #[test]
1019    fn test_data_store_array() {
1020        let mut store = DataStore::new();
1021
1022        let array = vec![
1023            vec![LiteralValue::Number(1.0), LiteralValue::Number(2.0)],
1024            vec![LiteralValue::Number(3.0), LiteralValue::Number(4.0)],
1025        ];
1026
1027        let value_ref = store.store_value(LiteralValue::Array(array.clone()));
1028        let retrieved = store.retrieve_value(value_ref);
1029
1030        assert_eq!(retrieved, LiteralValue::Array(array));
1031    }
1032
1033    #[test]
1034    fn test_data_store_ast_literal() {
1035        let mut store = DataStore::new();
1036        let mut sheet_registry = SheetRegistry::new();
1037        sheet_registry.id_for("Sheet1");
1038
1039        let ast = ASTNode {
1040            node_type: ASTNodeType::Literal(LiteralValue::Number(42.0)),
1041            source_token: None,
1042            contains_volatile: false,
1043        };
1044
1045        let ast_id = store.store_ast(&ast, &sheet_registry);
1046        let retrieved = store.retrieve_ast(ast_id, &sheet_registry).unwrap();
1047
1048        match retrieved.node_type {
1049            ASTNodeType::Literal(lit) => assert_eq!(lit, LiteralValue::Number(42.0)),
1050            _ => panic!("Expected literal"),
1051        }
1052    }
1053
1054    #[test]
1055    fn test_data_store_ast_binary_op() {
1056        let mut store = DataStore::new();
1057        let mut sheet_registry = SheetRegistry::new();
1058        sheet_registry.id_for("Sheet1");
1059
1060        let ast = ASTNode {
1061            node_type: ASTNodeType::BinaryOp {
1062                op: "+".to_string(),
1063                left: Box::new(ASTNode {
1064                    node_type: ASTNodeType::Literal(LiteralValue::Number(1.0)),
1065                    source_token: None,
1066                    contains_volatile: false,
1067                }),
1068                right: Box::new(ASTNode {
1069                    node_type: ASTNodeType::Literal(LiteralValue::Number(2.0)),
1070                    source_token: None,
1071                    contains_volatile: false,
1072                }),
1073            },
1074            source_token: None,
1075            contains_volatile: false,
1076        };
1077
1078        let ast_id = store.store_ast(&ast, &sheet_registry);
1079        let retrieved = store.retrieve_ast(ast_id, &sheet_registry).unwrap();
1080
1081        match retrieved.node_type {
1082            ASTNodeType::BinaryOp { op, left, right } => {
1083                assert_eq!(op, "+");
1084                match left.node_type {
1085                    ASTNodeType::Literal(lit) => assert_eq!(lit, LiteralValue::Number(1.0)),
1086                    _ => panic!("Expected literal"),
1087                }
1088                match right.node_type {
1089                    ASTNodeType::Literal(lit) => assert_eq!(lit, LiteralValue::Number(2.0)),
1090                    _ => panic!("Expected literal"),
1091                }
1092            }
1093            _ => panic!("Expected binary op"),
1094        }
1095    }
1096
1097    #[test]
1098    fn test_data_store_ast_function() {
1099        let mut store = DataStore::new();
1100        let mut sheet_registry = SheetRegistry::new();
1101        sheet_registry.id_for("Sheet1");
1102
1103        let ast = ASTNode {
1104            node_type: ASTNodeType::Function {
1105                name: "SUM".to_string(),
1106                args: vec![
1107                    ASTNode {
1108                        node_type: ASTNodeType::Literal(LiteralValue::Number(1.0)),
1109                        source_token: None,
1110                        contains_volatile: false,
1111                    },
1112                    ASTNode {
1113                        node_type: ASTNodeType::Literal(LiteralValue::Number(2.0)),
1114                        source_token: None,
1115                        contains_volatile: false,
1116                    },
1117                ],
1118            },
1119            source_token: None,
1120            contains_volatile: false,
1121        };
1122
1123        let ast_id = store.store_ast(&ast, &sheet_registry);
1124        let retrieved = store.retrieve_ast(ast_id, &sheet_registry).unwrap();
1125
1126        match retrieved.node_type {
1127            ASTNodeType::Function { name, args } => {
1128                assert_eq!(name, "SUM");
1129                assert_eq!(args.len(), 2);
1130            }
1131            _ => panic!("Expected function"),
1132        }
1133    }
1134
1135    #[test]
1136    fn test_data_store_memory_stats() {
1137        let mut store = DataStore::new();
1138
1139        // Add some data
1140        store.store_value(LiteralValue::Number(42.0));
1141        store.store_value(LiteralValue::Text("Hello".to_string()));
1142        store.store_value(LiteralValue::Array(vec![vec![LiteralValue::Number(1.0)]]));
1143
1144        let stats = store.memory_usage();
1145        assert!(stats.total_bytes() > 0);
1146        assert_eq!(stats.total_scalars, 2); // 42.0 and 1.0
1147        assert_eq!(stats.total_strings, 1); // "Hello"
1148        assert_eq!(stats.total_arrays, 1);
1149    }
1150
1151    #[test]
1152    fn test_data_store_clear() {
1153        let mut store = DataStore::new();
1154
1155        store.store_value(LiteralValue::Number(42.0));
1156        store.store_value(LiteralValue::Text("Hello".to_string()));
1157
1158        let stats = store.memory_usage();
1159        assert!(stats.total_scalars > 0);
1160        assert!(stats.total_strings > 0);
1161
1162        store.clear();
1163
1164        let stats = store.memory_usage();
1165        assert_eq!(stats.total_scalars, 0);
1166        assert_eq!(stats.total_strings, 0);
1167    }
1168}
formualizer_eval/engine/arena/data_store.rs

formualizer_eval/engine/arena/
data_store.rs
