formualizer_eval/engine/arena/
data_store.rs

1/// Unified data storage for all value types using arenas
2/// Provides conversion between LiteralValue and ValueRef
3use super::array::ArrayArena;
4use super::ast::{AstArena, AstNodeId, CompactRefType, SheetKey};
5use super::error_arena::{ErrorArena, ErrorRef};
6use super::scalar::ScalarArena;
7use super::string_interner::{StringId, StringInterner};
8use super::value_ref::ValueRef;
9use crate::engine::sheet_registry::SheetRegistry;
10use formualizer_common::{ExcelError, ExcelErrorKind, LiteralValue};
11use formualizer_parse::parser::{
12    ASTNode, ASTNodeType, ExternalBookRef, ExternalReference, ReferenceType, TableReference,
13};
14
15/// Centralized data storage using arenas
16#[derive(Debug)]
17pub struct DataStore {
18    /// Scalar values (floats and large integers)
19    scalars: ScalarArena,
20
21    /// String values
22    strings: StringInterner,
23
24    /// Array values
25    arrays: ArrayArena,
26
27    /// AST nodes for formulas
28    asts: AstArena,
29
30    /// Error storage with message preservation
31    errors: ErrorArena,
32}
33
34impl DataStore {
35    pub fn new() -> Self {
36        Self {
37            scalars: ScalarArena::new(),
38            strings: StringInterner::new(),
39            arrays: ArrayArena::new(),
40            asts: AstArena::new(),
41            errors: ErrorArena::new(),
42        }
43    }
44
45    /// Batch store literal values; returns ValueRefs in same order.
46    pub fn store_values_batch<I>(&mut self, values: I) -> Vec<ValueRef>
47    where
48        I: IntoIterator<Item = LiteralValue>,
49    {
50        let iter = values.into_iter();
51        let (lower, _) = iter.size_hint();
52        let mut out = Vec::with_capacity(lower);
53        for v in iter {
54            out.push(self.store_value(v));
55        }
56        out
57    }
58
59    /// Batch store ASTs; returns AstNodeIds in same order.
60    pub fn store_asts_batch<'a, I>(
61        &mut self,
62        asts: I,
63        sheet_registry: &SheetRegistry,
64    ) -> Vec<AstNodeId>
65    where
66        I: IntoIterator<Item = &'a ASTNode>,
67    {
68        let iter = asts.into_iter();
69        let (lower, _) = iter.size_hint();
70        let mut out = Vec::with_capacity(lower);
71        for ast in iter {
72            out.push(self.store_ast(ast, sheet_registry));
73        }
74        out
75    }
76
77    pub fn with_capacity(estimated_cells: usize) -> Self {
78        Self {
79            scalars: ScalarArena::with_capacity(estimated_cells),
80            strings: StringInterner::with_capacity(estimated_cells / 10),
81            arrays: ArrayArena::with_capacity(estimated_cells / 100),
82            asts: AstArena::with_capacity(estimated_cells / 2),
83            errors: ErrorArena::with_capacity(estimated_cells / 20),
84        }
85    }
86
87    /// Store a LiteralValue and return a ValueRef
88    pub fn store_value(&mut self, value: LiteralValue) -> ValueRef {
89        match value {
90            LiteralValue::Empty => ValueRef::empty(),
91
92            LiteralValue::Number(n) => {
93                // Store as float in scalar arena
94                let idx = self.scalars.insert_float(n);
95                ValueRef::number(idx.as_u32())
96            }
97
98            LiteralValue::Text(s) => {
99                let id = self.strings.intern(&s);
100                ValueRef::string(id.as_u32())
101            }
102
103            LiteralValue::Boolean(b) => ValueRef::boolean(b),
104
105            LiteralValue::Error(err) => self.store_error(&err),
106
107            LiteralValue::Array(array) => {
108                // Convert nested array to ValueRefs
109                let rows = array.len() as u32;
110                let cols = array.first().map(|r| r.len()).unwrap_or(0) as u32;
111
112                let elements: Vec<ValueRef> = array
113                    .into_iter()
114                    .flatten()
115                    .map(|v| self.store_value(v))
116                    .collect();
117
118                let array_ref = self.arrays.insert(rows, cols, elements);
119                ValueRef::array(array_ref.as_u32())
120            }
121
122            LiteralValue::DateTime(dt) => {
123                // The arena uses Excel1900 only as a private, symmetric binary
124                // encoding for chrono values. This is not workbook serial
125                // state and the raw float is never exposed to formula logic.
126                let serial = formualizer_common::datetime_to_serial_for(
127                    formualizer_common::DateSystem::Excel1900,
128                    &dt,
129                );
130                let idx = self.scalars.insert_float(serial);
131                ValueRef::date_time(idx.as_u32())
132            }
133
134            LiteralValue::Date(d) => {
135                // Use the same private arena encoding as DateTime values.
136                let dt = d.and_hms_opt(0, 0, 0).unwrap();
137                let serial = formualizer_common::datetime_to_serial_for(
138                    formualizer_common::DateSystem::Excel1900,
139                    &dt,
140                );
141                let idx = self.scalars.insert_float(serial);
142                ValueRef::date_time(idx.as_u32())
143            }
144
145            LiteralValue::Time(t) => {
146                // Store time as fractional day.
147                let fraction = formualizer_common::time_to_fraction(&t);
148                let idx = self.scalars.insert_float(fraction);
149                ValueRef::date_time(idx.as_u32())
150            }
151
152            LiteralValue::Duration(dur) => {
153                // Store as integer seconds (chrono::Duration has num_seconds())
154                let secs = dur.num_seconds();
155                let idx = self.scalars.insert_integer(secs);
156                let raw_index = idx.as_u32() & 0x7FFF_FFFF;
157                ValueRef::duration(raw_index)
158            }
159
160            LiteralValue::Int(i) => {
161                // Try to use small int optimization
162                if let Some(vref) = ValueRef::small_int(i as i32) {
163                    vref
164                } else {
165                    // Store as large integer
166                    let idx = self.scalars.insert_integer(i);
167                    ValueRef::large_int(idx.as_u32())
168                }
169            }
170
171            LiteralValue::Pending => ValueRef::pending(),
172        }
173    }
174
175    /// Retrieve a LiteralValue from a ValueRef
176    pub fn retrieve_value(&self, value_ref: ValueRef) -> LiteralValue {
177        use super::value_ref::ValueType;
178
179        match value_ref.value_type() {
180            ValueType::Empty => LiteralValue::Empty,
181
182            ValueType::SmallInt => {
183                // Small integers are inlined
184                if let Some(i) = value_ref.as_small_int() {
185                    LiteralValue::Int(i as i64)
186                } else {
187                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
188                }
189            }
190
191            ValueType::LargeInt => {
192                if let Some(idx) = value_ref.arena_index() {
193                    let scalar_ref = super::scalar::ScalarRef::from_raw(idx | (1 << 31));
194                    if let Some(i) = self.scalars.get_integer(scalar_ref) {
195                        LiteralValue::Int(i)
196                    } else {
197                        LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
198                    }
199                } else {
200                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
201                }
202            }
203
204            ValueType::Number => {
205                if let Some(idx) = value_ref.arena_index() {
206                    let scalar_ref = super::scalar::ScalarRef::from_raw(idx);
207                    if let Some(f) = self.scalars.get_float(scalar_ref) {
208                        LiteralValue::Number(f)
209                    } else {
210                        LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
211                    }
212                } else {
213                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
214                }
215            }
216
217            ValueType::String => {
218                if let Some(idx) = value_ref.arena_index() {
219                    let string_id = StringId::from_raw(idx);
220                    let s = self.strings.resolve(string_id);
221                    LiteralValue::Text(s.to_string())
222                } else {
223                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
224                }
225            }
226
227            ValueType::Boolean => {
228                if let Some(b) = value_ref.as_boolean() {
229                    LiteralValue::Boolean(b)
230                } else {
231                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
232                }
233            }
234
235            ValueType::Error => {
236                if let Some(error_ref_raw) = value_ref.as_error_ref() {
237                    let error_ref = ErrorRef::from_raw(error_ref_raw);
238                    if let Some(error) = self.errors.get(error_ref) {
239                        LiteralValue::Error(error)
240                    } else {
241                        LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
242                    }
243                } else {
244                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
245                }
246            }
247
248            ValueType::Array => {
249                if let Some(idx) = value_ref.arena_index() {
250                    let array_ref = super::array::ArrayRef::from_raw(idx);
251                    if let Some(array_2d) = self.arrays.get_2d(array_ref) {
252                        // Convert back to LiteralValue array
253                        let result: Vec<Vec<LiteralValue>> = array_2d
254                            .into_iter()
255                            .map(|row| row.into_iter().map(|v| self.retrieve_value(v)).collect())
256                            .collect();
257                        LiteralValue::Array(result)
258                    } else {
259                        LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
260                    }
261                } else {
262                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
263                }
264            }
265
266            ValueType::DateTime => {
267                if let Some(idx) = value_ref.arena_index() {
268                    let scalar_ref = super::scalar::ScalarRef::from_raw(idx);
269                    if let Some(serial) = self.scalars.get_float(scalar_ref) {
270                        // This is a trusted private encoding, not an untrusted
271                        // workbook serial boundary. Use the compatibility
272                        // decoder so the arena retains its historical
273                        // pre-epoch behavior.
274                        LiteralValue::DateTime(formualizer_common::serial_to_datetime(serial))
275                    } else {
276                        LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
277                    }
278                } else {
279                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
280                }
281            }
282
283            ValueType::Duration => {
284                if let Some(idx) = value_ref.arena_index() {
285                    let scalar_ref = super::scalar::ScalarRef::from_raw(idx | (1 << 31));
286                    if let Some(secs) = self.scalars.get_integer(scalar_ref) {
287                        let dur = chrono::Duration::seconds(secs);
288                        LiteralValue::Duration(dur)
289                    } else {
290                        LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
291                    }
292                } else {
293                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
294                }
295            }
296
297            ValueType::Pending => LiteralValue::Pending,
298
299            ValueType::FormulaAst => {
300                // Formula ASTs shouldn't be returned as values
301                LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
302            }
303        }
304    }
305
306    /// Store an AST node and return its ID
307    pub fn store_ast(&mut self, ast: &ASTNode, sheet_registry: &SheetRegistry) -> AstNodeId {
308        self.convert_ast_node(ast, sheet_registry)
309    }
310
311    /// Retrieve an AST node from its ID
312    pub fn retrieve_ast(&self, id: AstNodeId, sheet_registry: &SheetRegistry) -> Option<ASTNode> {
313        self.reconstruct_ast_node(id, sheet_registry)
314    }
315
316    pub fn resolve_ast_string(&self, id: StringId) -> &str {
317        self.asts.resolve_string(id)
318    }
319
320    pub(crate) fn ast_strings(&self) -> &StringInterner {
321        self.asts.strings()
322    }
323
324    pub fn reconstruct_reference_type_for_eval(
325        &self,
326        ref_type: &CompactRefType,
327        sheet_registry: &SheetRegistry,
328    ) -> ReferenceType {
329        self.reconstruct_reference_type(ref_type, sheet_registry)
330    }
331
332    /// See [`super::ast::AstArena::compact`].
333    pub(crate) fn compact_asts(
334        &mut self,
335        roots: impl IntoIterator<Item = AstNodeId>,
336    ) -> (Vec<u32>, super::string_interner::StringGarbage) {
337        self.asts.compact(roots)
338    }
339
340    pub(crate) fn ast_node_count(&self) -> usize {
341        self.asts.node_count()
342    }
343
344    pub fn get_node(&self, id: AstNodeId) -> Option<&super::ast::AstNodeData> {
345        self.asts.get(id)
346    }
347
348    /// SUBTOTAL/AGGREGATE call bits of the formula rooted at `id` (see
349    /// `AstNodeEntry::subtotal_calls`).
350    pub(crate) fn ast_subtotal_calls(&self, id: AstNodeId) -> u8 {
351        self.asts.subtotal_calls(id)
352    }
353
354    pub fn get_args(&self, id: AstNodeId) -> Option<&[AstNodeId]> {
355        self.asts.get_function_args(id)
356    }
357
358    pub fn get_array_elems(&self, id: AstNodeId) -> Option<(u16, u16, &[AstNodeId])> {
359        self.asts.get_array_elements_info(id)
360    }
361
362    pub fn ast_needs_structural_rewrite(&self, id: AstNodeId) -> bool {
363        let mut stack = vec![id];
364        while let Some(node_id) = stack.pop() {
365            let Some(node) = self.get_node(node_id) else {
366                continue;
367            };
368            match node {
369                super::ast::AstNodeData::Reference { ref_type, .. } => {
370                    if let CompactRefType::Table { name_id, .. } = ref_type
371                        && self.resolve_ast_string(*name_id).is_empty()
372                    {
373                        return true;
374                    }
375                }
376                super::ast::AstNodeData::UnaryOp { expr_id, .. } => stack.push(*expr_id),
377                super::ast::AstNodeData::BinaryOp {
378                    left_id, right_id, ..
379                } => {
380                    stack.push(*right_id);
381                    stack.push(*left_id);
382                }
383                super::ast::AstNodeData::Function { .. } => {
384                    if let Some(args) = self.get_args(node_id) {
385                        stack.extend(args.iter().rev().copied());
386                    }
387                }
388                super::ast::AstNodeData::Array { .. } => {
389                    if let Some((_, _, elems)) = self.get_array_elems(node_id) {
390                        stack.extend(elems.iter().rev().copied());
391                    }
392                }
393                super::ast::AstNodeData::Literal(_) | super::ast::AstNodeData::Omitted => {}
394            }
395        }
396        false
397    }
398
399    /// Convert ASTNode to arena representation
400    fn convert_ast_node(&mut self, node: &ASTNode, sheet_registry: &SheetRegistry) -> AstNodeId {
401        match &node.node_type {
402            ASTNodeType::Literal(lit) => {
403                let value_ref = self.store_value(lit.clone());
404                self.asts.insert_literal(value_ref)
405            }
406
407            ASTNodeType::Omitted => self.asts.insert_omitted(),
408
409            ASTNodeType::Reference {
410                original,
411                reference,
412            } => {
413                let ref_type = self.convert_reference_type(reference, sheet_registry);
414                self.asts.insert_reference(original, ref_type)
415            }
416
417            ASTNodeType::UnaryOp { op, expr } => {
418                let expr_id = self.convert_ast_node(expr, sheet_registry);
419                self.asts.insert_unary_op(op, expr_id)
420            }
421
422            ASTNodeType::BinaryOp { op, left, right } => {
423                let left_id = self.convert_ast_node(left, sheet_registry);
424                let right_id = self.convert_ast_node(right, sheet_registry);
425                self.asts.insert_binary_op(op, left_id, right_id)
426            }
427
428            ASTNodeType::Function { name, args } => {
429                let arg_ids: Vec<AstNodeId> = args
430                    .iter()
431                    .map(|arg| self.convert_ast_node(arg, sheet_registry))
432                    .collect();
433                self.asts.insert_function(name, arg_ids)
434            }
435
436            ASTNodeType::Array(rows) => {
437                let total_elements = rows.iter().map(|r| r.len()).sum();
438                let mut elements = Vec::with_capacity(total_elements);
439
440                let rows_count = rows.len() as u16;
441                let cols_count = rows.first().map(|r| r.len()).unwrap_or(0) as u16;
442
443                for row in rows {
444                    for elem in row {
445                        elements.push(self.convert_ast_node(elem, sheet_registry));
446                    }
447                }
448
449                self.asts.insert_array(rows_count, cols_count, elements)
450            }
451
452            // Postfix call (e.g. LAMBDA immediate-invocation), stored as a
453            // reserved-name function node whose first argument is the callee.
454            ASTNodeType::Call { callee, args } => {
455                let mut arg_ids = Vec::with_capacity(args.len() + 1);
456                arg_ids.push(self.convert_ast_node(callee, sheet_registry));
457                arg_ids.extend(
458                    args.iter()
459                        .map(|arg| self.convert_ast_node(arg, sheet_registry)),
460                );
461                self.asts
462                    .insert_function(super::ast::CALL_NODE_NAME, arg_ids)
463            }
464        }
465    }
466
467    /// Convert ReferenceType to CompactRefType
468    fn convert_reference_type(
469        &mut self,
470        ref_type: &ReferenceType,
471        sheet_registry: &SheetRegistry,
472    ) -> CompactRefType {
473        match ref_type {
474            ReferenceType::Cell {
475                sheet,
476                row,
477                col,
478                row_abs,
479                col_abs,
480            } => {
481                let sheet = match sheet.as_ref() {
482                    Some(s) => match sheet_registry.get_id(s) {
483                        Some(id) => Some(SheetKey::Id(id)),
484                        None => Some(SheetKey::Name(self.asts.strings_mut().intern(s))),
485                    },
486                    None => None,
487                };
488                CompactRefType::Cell {
489                    sheet,
490                    row: *row,
491                    col: *col,
492                    row_abs: *row_abs,
493                    col_abs: *col_abs,
494                }
495            }
496
497            ReferenceType::Range {
498                sheet,
499                start_row,
500                start_col,
501                end_row,
502                end_col,
503                start_row_abs,
504                start_col_abs,
505                end_row_abs,
506                end_col_abs,
507            } => {
508                let sheet = match sheet.as_ref() {
509                    Some(s) => match sheet_registry.get_id(s) {
510                        Some(id) => Some(SheetKey::Id(id)),
511                        None => Some(SheetKey::Name(self.asts.strings_mut().intern(s))),
512                    },
513                    None => None,
514                };
515                // For optional range bounds, use 0/u32::MAX as sentinels for unbounded
516                CompactRefType::Range {
517                    sheet,
518                    start_row: start_row.unwrap_or(0),
519                    start_col: start_col.unwrap_or(0),
520                    end_row: end_row.unwrap_or(u32::MAX),
521                    end_col: end_col.unwrap_or(u32::MAX),
522                    start_row_abs: *start_row_abs,
523                    start_col_abs: *start_col_abs,
524                    end_row_abs: *end_row_abs,
525                    end_col_abs: *end_col_abs,
526                }
527            }
528
529            ReferenceType::External(ext) => {
530                let raw_id = self.asts.strings_mut().intern(&ext.raw);
531                let book_id = self.asts.strings_mut().intern(ext.book.token());
532                let sheet_id = self.asts.strings_mut().intern(&ext.sheet);
533                CompactRefType::External {
534                    raw_id,
535                    book_id,
536                    sheet_id,
537                    kind: ext.kind,
538                }
539            }
540
541            ReferenceType::NamedRange(name) => {
542                let string_id = self.asts.strings_mut().intern(name);
543                CompactRefType::NamedRange(string_id)
544            }
545
546            ReferenceType::Table(table_ref) => {
547                let name_id = self.asts.strings_mut().intern(&table_ref.name);
548                let specifier_id = table_ref
549                    .specifier
550                    .as_ref()
551                    .map(|specifier| self.asts.intern_table_specifier(specifier));
552                CompactRefType::Table {
553                    name_id,
554                    specifier_id,
555                }
556            }
557
558            ReferenceType::Cell3D {
559                sheet_first,
560                sheet_last,
561                row,
562                col,
563                row_abs,
564                col_abs,
565            } => {
566                let sheet_first = self.asts.strings_mut().intern(sheet_first);
567                let sheet_last = self.asts.strings_mut().intern(sheet_last);
568                CompactRefType::Cell3D {
569                    sheet_first,
570                    sheet_last,
571                    row: *row,
572                    col: *col,
573                    row_abs: *row_abs,
574                    col_abs: *col_abs,
575                }
576            }
577
578            ReferenceType::Range3D {
579                sheet_first,
580                sheet_last,
581                start_row,
582                start_col,
583                end_row,
584                end_col,
585                start_row_abs,
586                start_col_abs,
587                end_row_abs,
588                end_col_abs,
589            } => {
590                let sheet_first = self.asts.strings_mut().intern(sheet_first);
591                let sheet_last = self.asts.strings_mut().intern(sheet_last);
592                CompactRefType::Range3D {
593                    sheet_first,
594                    sheet_last,
595                    start_row: start_row.unwrap_or(0),
596                    start_col: start_col.unwrap_or(0),
597                    end_row: end_row.unwrap_or(u32::MAX),
598                    end_col: end_col.unwrap_or(u32::MAX),
599                    start_row_abs: *start_row_abs,
600                    start_col_abs: *start_col_abs,
601                    end_row_abs: *end_row_abs,
602                    end_col_abs: *end_col_abs,
603                }
604            }
605        }
606    }
607
608    /// Reconstruct an ASTNode from arena representation
609    pub(crate) fn reconstruct_ast_node(
610        &self,
611        id: AstNodeId,
612        sheet_registry: &SheetRegistry,
613    ) -> Option<ASTNode> {
614        use super::ast::AstNodeData;
615
616        let node_data = self.asts.get(id)?;
617
618        let node_type = match node_data {
619            AstNodeData::Literal(value_ref) => {
620                let lit = self.retrieve_value(*value_ref);
621                ASTNodeType::Literal(lit)
622            }
623
624            AstNodeData::Omitted => ASTNodeType::Omitted,
625
626            AstNodeData::Reference {
627                original_id,
628                ref_type,
629            } => {
630                let original = self.asts.resolve_string(*original_id).to_string();
631                let reference = self.reconstruct_reference_type(ref_type, sheet_registry);
632                ASTNodeType::Reference {
633                    original,
634                    reference,
635                }
636            }
637
638            AstNodeData::UnaryOp { op_id, expr_id } => {
639                let op = self.asts.resolve_string(*op_id).to_string();
640                let expr = Box::new(self.reconstruct_ast_node(*expr_id, sheet_registry)?);
641                ASTNodeType::UnaryOp { op, expr }
642            }
643
644            AstNodeData::BinaryOp {
645                op_id,
646                left_id,
647                right_id,
648            } => {
649                let op = self.asts.resolve_string(*op_id).to_string();
650                let left = Box::new(self.reconstruct_ast_node(*left_id, sheet_registry)?);
651                let right = Box::new(self.reconstruct_ast_node(*right_id, sheet_registry)?);
652                ASTNodeType::BinaryOp { op, left, right }
653            }
654
655            AstNodeData::Function { name_id, .. } => {
656                let name = self.asts.resolve_string(*name_id).to_string();
657                let arg_ids = self.asts.get_function_args(id)?;
658                let mut args: Vec<ASTNode> = arg_ids
659                    .iter()
660                    .filter_map(|&arg_id| self.reconstruct_ast_node(arg_id, sheet_registry))
661                    .collect();
662                if name == super::ast::CALL_NODE_NAME && !args.is_empty() {
663                    let callee = Box::new(args.remove(0));
664                    ASTNodeType::Call { callee, args }
665                } else {
666                    ASTNodeType::Function { name, args }
667                }
668            }
669
670            AstNodeData::Array { rows, cols, .. } => {
671                let elements = self.asts.get_array_elements(id)?;
672                let mut result = Vec::with_capacity(*rows as usize);
673
674                for r in 0..*rows {
675                    let mut row = Vec::with_capacity(*cols as usize);
676                    for c in 0..*cols {
677                        let idx = (r * *cols + c) as usize;
678                        if let Some(&elem_id) = elements.get(idx)
679                            && let Some(node) = self.reconstruct_ast_node(elem_id, sheet_registry)
680                        {
681                            row.push(node);
682                        }
683                    }
684                    result.push(row);
685                }
686
687                ASTNodeType::Array(result)
688            }
689        };
690
691        Some(ASTNode {
692            node_type,
693            source_token: None, // Token information is not preserved in arena
694            contains_volatile: false,
695        })
696    }
697
698    /// Reconstruct a ReferenceType from CompactRefType
699    fn reconstruct_reference_type(
700        &self,
701        ref_type: &CompactRefType,
702        sheet_registry: &SheetRegistry,
703    ) -> ReferenceType {
704        match ref_type {
705            CompactRefType::Cell {
706                sheet,
707                row,
708                col,
709                row_abs,
710                col_abs,
711            } => {
712                let sheet = match sheet {
713                    Some(SheetKey::Id(id)) => Some(sheet_registry.name(*id).to_string()),
714                    Some(SheetKey::Name(name_id)) => {
715                        Some(self.asts.resolve_string(*name_id).to_string())
716                    }
717                    None => None,
718                };
719                ReferenceType::Cell {
720                    sheet,
721                    row: *row,
722                    col: *col,
723                    row_abs: *row_abs,
724                    col_abs: *col_abs,
725                }
726            }
727
728            CompactRefType::Range {
729                sheet,
730                start_row,
731                start_col,
732                end_row,
733                end_col,
734                start_row_abs,
735                start_col_abs,
736                end_row_abs,
737                end_col_abs,
738            } => {
739                let sheet = match sheet {
740                    Some(SheetKey::Id(id)) => Some(sheet_registry.name(*id).to_string()),
741                    Some(SheetKey::Name(name_id)) => {
742                        Some(self.asts.resolve_string(*name_id).to_string())
743                    }
744                    None => None,
745                };
746                // Convert sentinel values back to None
747                ReferenceType::Range {
748                    sheet,
749                    start_row: if *start_row == 0 {
750                        None
751                    } else {
752                        Some(*start_row)
753                    },
754                    start_col: if *start_col == 0 {
755                        None
756                    } else {
757                        Some(*start_col)
758                    },
759                    end_row: if *end_row == u32::MAX {
760                        None
761                    } else {
762                        Some(*end_row)
763                    },
764                    end_col: if *end_col == u32::MAX {
765                        None
766                    } else {
767                        Some(*end_col)
768                    },
769                    start_row_abs: *start_row_abs,
770                    start_col_abs: *start_col_abs,
771                    end_row_abs: *end_row_abs,
772                    end_col_abs: *end_col_abs,
773                }
774            }
775
776            CompactRefType::External {
777                raw_id,
778                book_id,
779                sheet_id,
780                kind,
781            } => {
782                let raw = self.asts.resolve_string(*raw_id).to_string();
783                let book = self.asts.resolve_string(*book_id).to_string();
784                let sheet = self.asts.resolve_string(*sheet_id).to_string();
785                ReferenceType::External(ExternalReference {
786                    raw,
787                    book: ExternalBookRef::Token(book),
788                    sheet,
789                    kind: *kind,
790                })
791            }
792
793            CompactRefType::NamedRange(string_id) => {
794                let name = self.asts.resolve_string(*string_id).to_string();
795                ReferenceType::NamedRange(name)
796            }
797
798            CompactRefType::Table {
799                name_id,
800                specifier_id,
801            } => {
802                let name = self.asts.resolve_string(*name_id).to_string();
803                let specifier = specifier_id
804                    .and_then(|id| self.asts.resolve_table_specifier(id))
805                    .cloned();
806                ReferenceType::Table(TableReference { name, specifier })
807            }
808
809            CompactRefType::Cell3D {
810                sheet_first,
811                sheet_last,
812                row,
813                col,
814                row_abs,
815                col_abs,
816            } => ReferenceType::Cell3D {
817                sheet_first: self.asts.resolve_string(*sheet_first).to_string(),
818                sheet_last: self.asts.resolve_string(*sheet_last).to_string(),
819                row: *row,
820                col: *col,
821                row_abs: *row_abs,
822                col_abs: *col_abs,
823            },
824
825            CompactRefType::Range3D {
826                sheet_first,
827                sheet_last,
828                start_row,
829                start_col,
830                end_row,
831                end_col,
832                start_row_abs,
833                start_col_abs,
834                end_row_abs,
835                end_col_abs,
836            } => ReferenceType::Range3D {
837                sheet_first: self.asts.resolve_string(*sheet_first).to_string(),
838                sheet_last: self.asts.resolve_string(*sheet_last).to_string(),
839                start_row: if *start_row == 0 {
840                    None
841                } else {
842                    Some(*start_row)
843                },
844                start_col: if *start_col == 0 {
845                    None
846                } else {
847                    Some(*start_col)
848                },
849                end_row: if *end_row == u32::MAX {
850                    None
851                } else {
852                    Some(*end_row)
853                },
854                end_col: if *end_col == u32::MAX {
855                    None
856                } else {
857                    Some(*end_col)
858                },
859                start_row_abs: *start_row_abs,
860                start_col_abs: *start_col_abs,
861                end_row_abs: *end_row_abs,
862                end_col_abs: *end_col_abs,
863            },
864        }
865    }
866
867    /// Store an error with message preservation
868    fn store_error(&mut self, error: &ExcelError) -> ValueRef {
869        let error_ref = self.errors.insert(error);
870        ValueRef::error(error_ref.as_u32())
871    }
872
873    /// Get memory usage statistics
874    pub fn memory_usage(&self) -> DataStoreStats {
875        DataStoreStats {
876            scalar_bytes: self.scalars.memory_usage(),
877            string_bytes: self.strings.memory_usage(),
878            array_bytes: self.arrays.memory_usage(),
879            ast_bytes: self.asts.memory_usage(),
880            error_bytes: self.errors.memory_usage(),
881            total_scalars: self.scalars.len(),
882            total_strings: self.strings.len(),
883            total_arrays: self.arrays.len(),
884            total_ast_nodes: self.asts.stats().node_count,
885            total_errors: self.errors.len(),
886        }
887    }
888
889    /// Clear all data from the store
890    pub fn clear(&mut self) {
891        self.scalars.clear();
892        self.strings.clear();
893        self.arrays.clear();
894        self.asts.clear();
895        self.errors.clear();
896    }
897}
898
899impl Default for DataStore {
900    fn default() -> Self {
901        Self::new()
902    }
903}
904
905/// Statistics about data store memory usage
906#[derive(Debug, Clone)]
907pub struct DataStoreStats {
908    pub scalar_bytes: usize,
909    pub string_bytes: usize,
910    pub array_bytes: usize,
911    pub ast_bytes: usize,
912    pub error_bytes: usize,
913    pub total_scalars: usize,
914    pub total_strings: usize,
915    pub total_arrays: usize,
916    pub total_ast_nodes: usize,
917    pub total_errors: usize,
918}
919
920impl DataStoreStats {
921    pub fn total_bytes(&self) -> usize {
922        self.scalar_bytes + self.string_bytes + self.array_bytes + self.ast_bytes + self.error_bytes
923    }
924}
925
926// Helper trait implementations for ArrayRef and ScalarRef
927impl super::array::ArrayRef {
928    pub fn from_raw(raw: u32) -> Self {
929        super::array::ArrayRef(raw)
930    }
931}
932
933impl super::scalar::ScalarRef {
934    pub fn from_raw(raw: u32) -> Self {
935        Self { raw }
936    }
937
938    pub fn as_u32(self) -> u32 {
939        self.raw
940    }
941}
942
943#[cfg(test)]
944mod tests {
945    use super::*;
946
947    #[test]
948    fn test_data_store_empty_value() {
949        let mut store = DataStore::new();
950        let value_ref = store.store_value(LiteralValue::Empty);
951        assert!(value_ref.is_empty());
952
953        let retrieved = store.retrieve_value(value_ref);
954        assert_eq!(retrieved, LiteralValue::Empty);
955    }
956
957    #[test]
958    fn test_data_store_number() {
959        let mut store = DataStore::new();
960        let value_ref = store.store_value(LiteralValue::Number(42.5));
961
962        let retrieved = store.retrieve_value(value_ref);
963        assert_eq!(retrieved, LiteralValue::Number(42.5));
964    }
965
966    #[test]
967    fn test_data_store_datetime_private_encoding_round_trips() {
968        let mut store = DataStore::new();
969        for datetime in [
970            chrono::NaiveDate::from_ymd_opt(2024, 1, 15)
971                .unwrap()
972                .and_hms_opt(12, 30, 0)
973                .unwrap(),
974            chrono::NaiveDate::from_ymd_opt(1899, 12, 30)
975                .unwrap()
976                .and_hms_opt(0, 0, 0)
977                .unwrap(),
978        ] {
979            let value_ref = store.store_value(LiteralValue::DateTime(datetime));
980            assert_eq!(
981                store.retrieve_value(value_ref),
982                LiteralValue::DateTime(datetime)
983            );
984        }
985    }
986
987    #[test]
988    fn test_data_store_text() {
989        let mut store = DataStore::new();
990        let value_ref = store.store_value(LiteralValue::Text("Hello".to_string()));
991
992        let retrieved = store.retrieve_value(value_ref);
993        assert_eq!(retrieved, LiteralValue::Text("Hello".to_string()));
994    }
995
996    #[test]
997    fn test_data_store_boolean() {
998        let mut store = DataStore::new();
999
1000        let true_ref = store.store_value(LiteralValue::Boolean(true));
1001        let false_ref = store.store_value(LiteralValue::Boolean(false));
1002
1003        assert_eq!(store.retrieve_value(true_ref), LiteralValue::Boolean(true));
1004        assert_eq!(
1005            store.retrieve_value(false_ref),
1006            LiteralValue::Boolean(false)
1007        );
1008    }
1009
1010    #[test]
1011    fn test_data_store_error() {
1012        let mut store = DataStore::new();
1013
1014        let error = ExcelError::new(ExcelErrorKind::Div);
1015        let value_ref = store.store_value(LiteralValue::Error(error.clone()));
1016
1017        let retrieved = store.retrieve_value(value_ref);
1018        match retrieved {
1019            LiteralValue::Error(e) => assert_eq!(e.kind, ExcelErrorKind::Div),
1020            _ => panic!("Expected error"),
1021        }
1022    }
1023
1024    #[test]
1025    fn test_data_store_array() {
1026        let mut store = DataStore::new();
1027
1028        let array = vec![
1029            vec![LiteralValue::Number(1.0), LiteralValue::Number(2.0)],
1030            vec![LiteralValue::Number(3.0), LiteralValue::Number(4.0)],
1031        ];
1032
1033        let value_ref = store.store_value(LiteralValue::Array(array.clone()));
1034        let retrieved = store.retrieve_value(value_ref);
1035
1036        assert_eq!(retrieved, LiteralValue::Array(array));
1037    }
1038
1039    #[test]
1040    fn test_data_store_ast_literal() {
1041        let mut store = DataStore::new();
1042        let mut sheet_registry = SheetRegistry::new();
1043        sheet_registry.id_for("Sheet1");
1044
1045        let ast = ASTNode {
1046            node_type: ASTNodeType::Literal(LiteralValue::Number(42.0)),
1047            source_token: None,
1048            contains_volatile: false,
1049        };
1050
1051        let ast_id = store.store_ast(&ast, &sheet_registry);
1052        let retrieved = store.retrieve_ast(ast_id, &sheet_registry).unwrap();
1053
1054        match retrieved.node_type {
1055            ASTNodeType::Literal(lit) => assert_eq!(lit, LiteralValue::Number(42.0)),
1056            _ => panic!("Expected literal"),
1057        }
1058    }
1059
1060    #[test]
1061    fn test_data_store_ast_binary_op() {
1062        let mut store = DataStore::new();
1063        let mut sheet_registry = SheetRegistry::new();
1064        sheet_registry.id_for("Sheet1");
1065
1066        let ast = ASTNode {
1067            node_type: ASTNodeType::BinaryOp {
1068                op: "+".to_string(),
1069                left: Box::new(ASTNode {
1070                    node_type: ASTNodeType::Literal(LiteralValue::Number(1.0)),
1071                    source_token: None,
1072                    contains_volatile: false,
1073                }),
1074                right: Box::new(ASTNode {
1075                    node_type: ASTNodeType::Literal(LiteralValue::Number(2.0)),
1076                    source_token: None,
1077                    contains_volatile: false,
1078                }),
1079            },
1080            source_token: None,
1081            contains_volatile: false,
1082        };
1083
1084        let ast_id = store.store_ast(&ast, &sheet_registry);
1085        let retrieved = store.retrieve_ast(ast_id, &sheet_registry).unwrap();
1086
1087        match retrieved.node_type {
1088            ASTNodeType::BinaryOp { op, left, right } => {
1089                assert_eq!(op, "+");
1090                match left.node_type {
1091                    ASTNodeType::Literal(lit) => assert_eq!(lit, LiteralValue::Number(1.0)),
1092                    _ => panic!("Expected literal"),
1093                }
1094                match right.node_type {
1095                    ASTNodeType::Literal(lit) => assert_eq!(lit, LiteralValue::Number(2.0)),
1096                    _ => panic!("Expected literal"),
1097                }
1098            }
1099            _ => panic!("Expected binary op"),
1100        }
1101    }
1102
1103    #[test]
1104    fn test_data_store_ast_function() {
1105        let mut store = DataStore::new();
1106        let mut sheet_registry = SheetRegistry::new();
1107        sheet_registry.id_for("Sheet1");
1108
1109        let ast = ASTNode {
1110            node_type: ASTNodeType::Function {
1111                name: "SUM".to_string(),
1112                args: vec![
1113                    ASTNode {
1114                        node_type: ASTNodeType::Literal(LiteralValue::Number(1.0)),
1115                        source_token: None,
1116                        contains_volatile: false,
1117                    },
1118                    ASTNode {
1119                        node_type: ASTNodeType::Literal(LiteralValue::Number(2.0)),
1120                        source_token: None,
1121                        contains_volatile: false,
1122                    },
1123                ],
1124            },
1125            source_token: None,
1126            contains_volatile: false,
1127        };
1128
1129        let ast_id = store.store_ast(&ast, &sheet_registry);
1130        let retrieved = store.retrieve_ast(ast_id, &sheet_registry).unwrap();
1131
1132        match retrieved.node_type {
1133            ASTNodeType::Function { name, args } => {
1134                assert_eq!(name, "SUM");
1135                assert_eq!(args.len(), 2);
1136            }
1137            _ => panic!("Expected function"),
1138        }
1139    }
1140
1141    #[test]
1142    fn test_data_store_memory_stats() {
1143        let mut store = DataStore::new();
1144
1145        // Add some data
1146        store.store_value(LiteralValue::Number(42.0));
1147        store.store_value(LiteralValue::Text("Hello".to_string()));
1148        store.store_value(LiteralValue::Array(vec![vec![LiteralValue::Number(1.0)]]));
1149
1150        let stats = store.memory_usage();
1151        assert!(stats.total_bytes() > 0);
1152        assert_eq!(stats.total_scalars, 2); // 42.0 and 1.0
1153        assert_eq!(stats.total_strings, 1); // "Hello"
1154        assert_eq!(stats.total_arrays, 1);
1155    }
1156
1157    #[test]
1158    fn test_data_store_clear() {
1159        let mut store = DataStore::new();
1160
1161        store.store_value(LiteralValue::Number(42.0));
1162        store.store_value(LiteralValue::Text("Hello".to_string()));
1163
1164        let stats = store.memory_usage();
1165        assert!(stats.total_scalars > 0);
1166        assert!(stats.total_strings > 0);
1167
1168        store.clear();
1169
1170        let stats = store.memory_usage();
1171        assert_eq!(stats.total_scalars, 0);
1172        assert_eq!(stats.total_strings, 0);
1173    }
1174}
formualizer_eval/engine/arena/data_store.rs

formualizer_eval/engine/arena/
data_store.rs
