formualizer_eval/engine/arena/
data_store.rs

1/// Unified data storage for all value types using arenas
2/// Provides conversion between LiteralValue and ValueRef
3use super::array::ArrayArena;
4use super::ast::{AstArena, AstNodeId, CompactRefType, SheetKey};
5use super::error_arena::{ErrorArena, ErrorRef};
6use super::scalar::ScalarArena;
7use super::string_interner::{StringId, StringInterner};
8use super::value_ref::ValueRef;
9use crate::engine::sheet_registry::SheetRegistry;
10use formualizer_common::{ExcelError, ExcelErrorKind, LiteralValue};
11use formualizer_parse::parser::{
12    ASTNode, ASTNodeType, ExternalBookRef, ExternalReference, ReferenceType, TableReference,
13};
14
15/// Centralized data storage using arenas
16#[derive(Debug)]
17pub struct DataStore {
18    /// Scalar values (floats and large integers)
19    scalars: ScalarArena,
20
21    /// String values
22    strings: StringInterner,
23
24    /// Array values
25    arrays: ArrayArena,
26
27    /// AST nodes for formulas
28    asts: AstArena,
29
30    /// Error storage with message preservation
31    errors: ErrorArena,
32}
33
34impl DataStore {
35    pub fn new() -> Self {
36        Self {
37            scalars: ScalarArena::new(),
38            strings: StringInterner::new(),
39            arrays: ArrayArena::new(),
40            asts: AstArena::new(),
41            errors: ErrorArena::new(),
42        }
43    }
44
45    /// Batch store literal values; returns ValueRefs in same order.
46    pub fn store_values_batch<I>(&mut self, values: I) -> Vec<ValueRef>
47    where
48        I: IntoIterator<Item = LiteralValue>,
49    {
50        let iter = values.into_iter();
51        let (lower, _) = iter.size_hint();
52        let mut out = Vec::with_capacity(lower);
53        for v in iter {
54            out.push(self.store_value(v));
55        }
56        out
57    }
58
59    /// Batch store ASTs; returns AstNodeIds in same order.
60    pub fn store_asts_batch<'a, I>(
61        &mut self,
62        asts: I,
63        sheet_registry: &SheetRegistry,
64    ) -> Vec<AstNodeId>
65    where
66        I: IntoIterator<Item = &'a ASTNode>,
67    {
68        let iter = asts.into_iter();
69        let (lower, _) = iter.size_hint();
70        let mut out = Vec::with_capacity(lower);
71        for ast in iter {
72            out.push(self.store_ast(ast, sheet_registry));
73        }
74        out
75    }
76
77    pub fn with_capacity(estimated_cells: usize) -> Self {
78        Self {
79            scalars: ScalarArena::with_capacity(estimated_cells),
80            strings: StringInterner::with_capacity(estimated_cells / 10),
81            arrays: ArrayArena::with_capacity(estimated_cells / 100),
82            asts: AstArena::with_capacity(estimated_cells / 2),
83            errors: ErrorArena::with_capacity(estimated_cells / 20),
84        }
85    }
86
87    /// Store a LiteralValue and return a ValueRef
88    pub fn store_value(&mut self, value: LiteralValue) -> ValueRef {
89        match value {
90            LiteralValue::Empty => ValueRef::empty(),
91
92            LiteralValue::Number(n) => {
93                // Store as float in scalar arena
94                let idx = self.scalars.insert_float(n);
95                ValueRef::number(idx.as_u32())
96            }
97
98            LiteralValue::Text(s) => {
99                let id = self.strings.intern(&s);
100                ValueRef::string(id.as_u32())
101            }
102
103            LiteralValue::Boolean(b) => ValueRef::boolean(b),
104
105            LiteralValue::Error(err) => self.store_error(&err),
106
107            LiteralValue::Array(array) => {
108                // Convert nested array to ValueRefs
109                let rows = array.len() as u32;
110                let cols = array.first().map(|r| r.len()).unwrap_or(0) as u32;
111
112                let elements: Vec<ValueRef> = array
113                    .into_iter()
114                    .flatten()
115                    .map(|v| self.store_value(v))
116                    .collect();
117
118                let array_ref = self.arrays.insert(rows, cols, elements);
119                ValueRef::array(array_ref.as_u32())
120            }
121
122            LiteralValue::DateTime(dt) => {
123                // The arena uses Excel1900 only as a private, symmetric binary
124                // encoding for chrono values. This is not workbook serial
125                // state and the raw float is never exposed to formula logic.
126                let serial = formualizer_common::datetime_to_serial_for(
127                    formualizer_common::DateSystem::Excel1900,
128                    &dt,
129                );
130                let idx = self.scalars.insert_float(serial);
131                ValueRef::date_time(idx.as_u32())
132            }
133
134            LiteralValue::Date(d) => {
135                // Use the same private arena encoding as DateTime values.
136                let dt = d.and_hms_opt(0, 0, 0).unwrap();
137                let serial = formualizer_common::datetime_to_serial_for(
138                    formualizer_common::DateSystem::Excel1900,
139                    &dt,
140                );
141                let idx = self.scalars.insert_float(serial);
142                ValueRef::date_time(idx.as_u32())
143            }
144
145            LiteralValue::Time(t) => {
146                // Store time as fractional day.
147                let fraction = formualizer_common::time_to_fraction(&t);
148                let idx = self.scalars.insert_float(fraction);
149                ValueRef::date_time(idx.as_u32())
150            }
151
152            LiteralValue::Duration(dur) => {
153                // Store as integer seconds (chrono::Duration has num_seconds())
154                let secs = dur.num_seconds();
155                let idx = self.scalars.insert_integer(secs);
156                let raw_index = idx.as_u32() & 0x7FFF_FFFF;
157                ValueRef::duration(raw_index)
158            }
159
160            LiteralValue::Int(i) => {
161                // Try to use small int optimization
162                if let Some(vref) = ValueRef::small_int(i as i32) {
163                    vref
164                } else {
165                    // Store as large integer
166                    let idx = self.scalars.insert_integer(i);
167                    ValueRef::large_int(idx.as_u32())
168                }
169            }
170
171            LiteralValue::Pending => ValueRef::pending(),
172        }
173    }
174
175    /// Retrieve a LiteralValue from a ValueRef
176    pub fn retrieve_value(&self, value_ref: ValueRef) -> LiteralValue {
177        use super::value_ref::ValueType;
178
179        match value_ref.value_type() {
180            ValueType::Empty => LiteralValue::Empty,
181
182            ValueType::SmallInt => {
183                // Small integers are inlined
184                if let Some(i) = value_ref.as_small_int() {
185                    LiteralValue::Int(i as i64)
186                } else {
187                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
188                }
189            }
190
191            ValueType::LargeInt => {
192                if let Some(idx) = value_ref.arena_index() {
193                    let scalar_ref = super::scalar::ScalarRef::from_raw(idx | (1 << 31));
194                    if let Some(i) = self.scalars.get_integer(scalar_ref) {
195                        LiteralValue::Int(i)
196                    } else {
197                        LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
198                    }
199                } else {
200                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
201                }
202            }
203
204            ValueType::Number => {
205                if let Some(idx) = value_ref.arena_index() {
206                    let scalar_ref = super::scalar::ScalarRef::from_raw(idx);
207                    if let Some(f) = self.scalars.get_float(scalar_ref) {
208                        LiteralValue::Number(f)
209                    } else {
210                        LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
211                    }
212                } else {
213                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
214                }
215            }
216
217            ValueType::String => {
218                if let Some(idx) = value_ref.arena_index() {
219                    let string_id = StringId::from_raw(idx);
220                    let s = self.strings.resolve(string_id);
221                    LiteralValue::Text(s.to_string())
222                } else {
223                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
224                }
225            }
226
227            ValueType::Boolean => {
228                if let Some(b) = value_ref.as_boolean() {
229                    LiteralValue::Boolean(b)
230                } else {
231                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
232                }
233            }
234
235            ValueType::Error => {
236                if let Some(error_ref_raw) = value_ref.as_error_ref() {
237                    let error_ref = ErrorRef::from_raw(error_ref_raw);
238                    if let Some(error) = self.errors.get(error_ref) {
239                        LiteralValue::Error(error)
240                    } else {
241                        LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
242                    }
243                } else {
244                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
245                }
246            }
247
248            ValueType::Array => {
249                if let Some(idx) = value_ref.arena_index() {
250                    let array_ref = super::array::ArrayRef::from_raw(idx);
251                    if let Some(array_2d) = self.arrays.get_2d(array_ref) {
252                        // Convert back to LiteralValue array
253                        let result: Vec<Vec<LiteralValue>> = array_2d
254                            .into_iter()
255                            .map(|row| row.into_iter().map(|v| self.retrieve_value(v)).collect())
256                            .collect();
257                        LiteralValue::Array(result)
258                    } else {
259                        LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
260                    }
261                } else {
262                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
263                }
264            }
265
266            ValueType::DateTime => {
267                if let Some(idx) = value_ref.arena_index() {
268                    let scalar_ref = super::scalar::ScalarRef::from_raw(idx);
269                    if let Some(serial) = self.scalars.get_float(scalar_ref) {
270                        // This is a trusted private encoding, not an untrusted
271                        // workbook serial boundary. Use the compatibility
272                        // decoder so the arena retains its historical
273                        // pre-epoch behavior.
274                        LiteralValue::DateTime(formualizer_common::serial_to_datetime(serial))
275                    } else {
276                        LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
277                    }
278                } else {
279                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
280                }
281            }
282
283            ValueType::Duration => {
284                if let Some(idx) = value_ref.arena_index() {
285                    let scalar_ref = super::scalar::ScalarRef::from_raw(idx | (1 << 31));
286                    if let Some(secs) = self.scalars.get_integer(scalar_ref) {
287                        let dur = chrono::Duration::seconds(secs);
288                        LiteralValue::Duration(dur)
289                    } else {
290                        LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
291                    }
292                } else {
293                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
294                }
295            }
296
297            ValueType::Pending => LiteralValue::Pending,
298
299            ValueType::FormulaAst => {
300                // Formula ASTs shouldn't be returned as values
301                LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
302            }
303        }
304    }
305
306    /// Store an AST node and return its ID
307    pub fn store_ast(&mut self, ast: &ASTNode, sheet_registry: &SheetRegistry) -> AstNodeId {
308        self.convert_ast_node(ast, sheet_registry)
309    }
310
311    /// Retrieve an AST node from its ID
312    pub fn retrieve_ast(&self, id: AstNodeId, sheet_registry: &SheetRegistry) -> Option<ASTNode> {
313        self.reconstruct_ast_node(id, sheet_registry)
314    }
315
316    pub fn resolve_ast_string(&self, id: StringId) -> &str {
317        self.asts.resolve_string(id)
318    }
319
320    pub(crate) fn ast_strings(&self) -> &StringInterner {
321        self.asts.strings()
322    }
323
324    pub fn reconstruct_reference_type_for_eval(
325        &self,
326        ref_type: &CompactRefType,
327        sheet_registry: &SheetRegistry,
328    ) -> ReferenceType {
329        self.reconstruct_reference_type(ref_type, sheet_registry)
330    }
331
332    pub fn get_node(&self, id: AstNodeId) -> Option<&super::ast::AstNodeData> {
333        self.asts.get(id)
334    }
335
336    pub fn get_args(&self, id: AstNodeId) -> Option<&[AstNodeId]> {
337        self.asts.get_function_args(id)
338    }
339
340    pub fn get_array_elems(&self, id: AstNodeId) -> Option<(u16, u16, &[AstNodeId])> {
341        self.asts.get_array_elements_info(id)
342    }
343
344    pub fn ast_needs_structural_rewrite(&self, id: AstNodeId) -> bool {
345        let mut stack = vec![id];
346        while let Some(node_id) = stack.pop() {
347            let Some(node) = self.get_node(node_id) else {
348                continue;
349            };
350            match node {
351                super::ast::AstNodeData::Reference { ref_type, .. } => {
352                    if let CompactRefType::Table { name_id, .. } = ref_type
353                        && self.resolve_ast_string(*name_id).is_empty()
354                    {
355                        return true;
356                    }
357                }
358                super::ast::AstNodeData::UnaryOp { expr_id, .. } => stack.push(*expr_id),
359                super::ast::AstNodeData::BinaryOp {
360                    left_id, right_id, ..
361                } => {
362                    stack.push(*right_id);
363                    stack.push(*left_id);
364                }
365                super::ast::AstNodeData::Function { .. } => {
366                    if let Some(args) = self.get_args(node_id) {
367                        stack.extend(args.iter().rev().copied());
368                    }
369                }
370                super::ast::AstNodeData::Array { .. } => {
371                    if let Some((_, _, elems)) = self.get_array_elems(node_id) {
372                        stack.extend(elems.iter().rev().copied());
373                    }
374                }
375                super::ast::AstNodeData::Literal(_) | super::ast::AstNodeData::Omitted => {}
376            }
377        }
378        false
379    }
380
381    /// Convert ASTNode to arena representation
382    fn convert_ast_node(&mut self, node: &ASTNode, sheet_registry: &SheetRegistry) -> AstNodeId {
383        match &node.node_type {
384            ASTNodeType::Literal(lit) => {
385                let value_ref = self.store_value(lit.clone());
386                self.asts.insert_literal(value_ref)
387            }
388
389            ASTNodeType::Omitted => self.asts.insert_omitted(),
390
391            ASTNodeType::Reference {
392                original,
393                reference,
394            } => {
395                let ref_type = self.convert_reference_type(reference, sheet_registry);
396                self.asts.insert_reference(original, ref_type)
397            }
398
399            ASTNodeType::UnaryOp { op, expr } => {
400                let expr_id = self.convert_ast_node(expr, sheet_registry);
401                self.asts.insert_unary_op(op, expr_id)
402            }
403
404            ASTNodeType::BinaryOp { op, left, right } => {
405                let left_id = self.convert_ast_node(left, sheet_registry);
406                let right_id = self.convert_ast_node(right, sheet_registry);
407                self.asts.insert_binary_op(op, left_id, right_id)
408            }
409
410            ASTNodeType::Function { name, args } => {
411                let arg_ids: Vec<AstNodeId> = args
412                    .iter()
413                    .map(|arg| self.convert_ast_node(arg, sheet_registry))
414                    .collect();
415                self.asts.insert_function(name, arg_ids)
416            }
417
418            ASTNodeType::Array(rows) => {
419                let total_elements = rows.iter().map(|r| r.len()).sum();
420                let mut elements = Vec::with_capacity(total_elements);
421
422                let rows_count = rows.len() as u16;
423                let cols_count = rows.first().map(|r| r.len()).unwrap_or(0) as u16;
424
425                for row in rows {
426                    for elem in row {
427                        elements.push(self.convert_ast_node(elem, sheet_registry));
428                    }
429                }
430
431                self.asts.insert_array(rows_count, cols_count, elements)
432            }
433
434            // Postfix call (e.g. LAMBDA immediate-invocation). The arena does
435            // not yet have a dedicated node kind for this, and full evaluator
436            // semantics are out of scope for the parser-side change. Store an
437            // unsupported-formula error literal so that downstream evaluation
438            // surfaces a clear #N/A!-style error instead of silently producing
439            // a wrong result.
440            ASTNodeType::Call { .. } => {
441                let value_ref = self.store_value(LiteralValue::Error(
442                    ExcelError::new(ExcelErrorKind::NImpl)
443                        .with_message("Immediate-invocation calls are not yet supported"),
444                ));
445                self.asts.insert_literal(value_ref)
446            }
447        }
448    }
449
450    /// Convert ReferenceType to CompactRefType
451    fn convert_reference_type(
452        &mut self,
453        ref_type: &ReferenceType,
454        sheet_registry: &SheetRegistry,
455    ) -> CompactRefType {
456        match ref_type {
457            ReferenceType::Cell {
458                sheet,
459                row,
460                col,
461                row_abs,
462                col_abs,
463            } => {
464                let sheet = match sheet.as_ref() {
465                    Some(s) => match sheet_registry.get_id(s) {
466                        Some(id) => Some(SheetKey::Id(id)),
467                        None => Some(SheetKey::Name(self.asts.strings_mut().intern(s))),
468                    },
469                    None => None,
470                };
471                CompactRefType::Cell {
472                    sheet,
473                    row: *row,
474                    col: *col,
475                    row_abs: *row_abs,
476                    col_abs: *col_abs,
477                }
478            }
479
480            ReferenceType::Range {
481                sheet,
482                start_row,
483                start_col,
484                end_row,
485                end_col,
486                start_row_abs,
487                start_col_abs,
488                end_row_abs,
489                end_col_abs,
490            } => {
491                let sheet = match sheet.as_ref() {
492                    Some(s) => match sheet_registry.get_id(s) {
493                        Some(id) => Some(SheetKey::Id(id)),
494                        None => Some(SheetKey::Name(self.asts.strings_mut().intern(s))),
495                    },
496                    None => None,
497                };
498                // For optional range bounds, use 0/u32::MAX as sentinels for unbounded
499                CompactRefType::Range {
500                    sheet,
501                    start_row: start_row.unwrap_or(0),
502                    start_col: start_col.unwrap_or(0),
503                    end_row: end_row.unwrap_or(u32::MAX),
504                    end_col: end_col.unwrap_or(u32::MAX),
505                    start_row_abs: *start_row_abs,
506                    start_col_abs: *start_col_abs,
507                    end_row_abs: *end_row_abs,
508                    end_col_abs: *end_col_abs,
509                }
510            }
511
512            ReferenceType::External(ext) => {
513                let raw_id = self.asts.strings_mut().intern(&ext.raw);
514                let book_id = self.asts.strings_mut().intern(ext.book.token());
515                let sheet_id = self.asts.strings_mut().intern(&ext.sheet);
516                CompactRefType::External {
517                    raw_id,
518                    book_id,
519                    sheet_id,
520                    kind: ext.kind,
521                }
522            }
523
524            ReferenceType::NamedRange(name) => {
525                let string_id = self.asts.strings_mut().intern(name);
526                CompactRefType::NamedRange(string_id)
527            }
528
529            ReferenceType::Table(table_ref) => {
530                let name_id = self.asts.strings_mut().intern(&table_ref.name);
531                let specifier_id = table_ref
532                    .specifier
533                    .as_ref()
534                    .map(|specifier| self.asts.intern_table_specifier(specifier));
535                CompactRefType::Table {
536                    name_id,
537                    specifier_id,
538                }
539            }
540
541            ReferenceType::Cell3D {
542                sheet_first,
543                sheet_last,
544                row,
545                col,
546                row_abs,
547                col_abs,
548            } => {
549                let sheet_first = self.asts.strings_mut().intern(sheet_first);
550                let sheet_last = self.asts.strings_mut().intern(sheet_last);
551                CompactRefType::Cell3D {
552                    sheet_first,
553                    sheet_last,
554                    row: *row,
555                    col: *col,
556                    row_abs: *row_abs,
557                    col_abs: *col_abs,
558                }
559            }
560
561            ReferenceType::Range3D {
562                sheet_first,
563                sheet_last,
564                start_row,
565                start_col,
566                end_row,
567                end_col,
568                start_row_abs,
569                start_col_abs,
570                end_row_abs,
571                end_col_abs,
572            } => {
573                let sheet_first = self.asts.strings_mut().intern(sheet_first);
574                let sheet_last = self.asts.strings_mut().intern(sheet_last);
575                CompactRefType::Range3D {
576                    sheet_first,
577                    sheet_last,
578                    start_row: start_row.unwrap_or(0),
579                    start_col: start_col.unwrap_or(0),
580                    end_row: end_row.unwrap_or(u32::MAX),
581                    end_col: end_col.unwrap_or(u32::MAX),
582                    start_row_abs: *start_row_abs,
583                    start_col_abs: *start_col_abs,
584                    end_row_abs: *end_row_abs,
585                    end_col_abs: *end_col_abs,
586                }
587            }
588        }
589    }
590
591    /// Reconstruct an ASTNode from arena representation
592    pub(crate) fn reconstruct_ast_node(
593        &self,
594        id: AstNodeId,
595        sheet_registry: &SheetRegistry,
596    ) -> Option<ASTNode> {
597        use super::ast::AstNodeData;
598
599        let node_data = self.asts.get(id)?;
600
601        let node_type = match node_data {
602            AstNodeData::Literal(value_ref) => {
603                let lit = self.retrieve_value(*value_ref);
604                ASTNodeType::Literal(lit)
605            }
606
607            AstNodeData::Omitted => ASTNodeType::Omitted,
608
609            AstNodeData::Reference {
610                original_id,
611                ref_type,
612            } => {
613                let original = self.asts.resolve_string(*original_id).to_string();
614                let reference = self.reconstruct_reference_type(ref_type, sheet_registry);
615                ASTNodeType::Reference {
616                    original,
617                    reference,
618                }
619            }
620
621            AstNodeData::UnaryOp { op_id, expr_id } => {
622                let op = self.asts.resolve_string(*op_id).to_string();
623                let expr = Box::new(self.reconstruct_ast_node(*expr_id, sheet_registry)?);
624                ASTNodeType::UnaryOp { op, expr }
625            }
626
627            AstNodeData::BinaryOp {
628                op_id,
629                left_id,
630                right_id,
631            } => {
632                let op = self.asts.resolve_string(*op_id).to_string();
633                let left = Box::new(self.reconstruct_ast_node(*left_id, sheet_registry)?);
634                let right = Box::new(self.reconstruct_ast_node(*right_id, sheet_registry)?);
635                ASTNodeType::BinaryOp { op, left, right }
636            }
637
638            AstNodeData::Function { name_id, .. } => {
639                let name = self.asts.resolve_string(*name_id).to_string();
640                let arg_ids = self.asts.get_function_args(id)?;
641                let args: Vec<ASTNode> = arg_ids
642                    .iter()
643                    .filter_map(|&arg_id| self.reconstruct_ast_node(arg_id, sheet_registry))
644                    .collect();
645                ASTNodeType::Function { name, args }
646            }
647
648            AstNodeData::Array { rows, cols, .. } => {
649                let elements = self.asts.get_array_elements(id)?;
650                let mut result = Vec::with_capacity(*rows as usize);
651
652                for r in 0..*rows {
653                    let mut row = Vec::with_capacity(*cols as usize);
654                    for c in 0..*cols {
655                        let idx = (r * *cols + c) as usize;
656                        if let Some(&elem_id) = elements.get(idx)
657                            && let Some(node) = self.reconstruct_ast_node(elem_id, sheet_registry)
658                        {
659                            row.push(node);
660                        }
661                    }
662                    result.push(row);
663                }
664
665                ASTNodeType::Array(result)
666            }
667        };
668
669        Some(ASTNode {
670            node_type,
671            source_token: None, // Token information is not preserved in arena
672            contains_volatile: false,
673        })
674    }
675
676    /// Reconstruct a ReferenceType from CompactRefType
677    fn reconstruct_reference_type(
678        &self,
679        ref_type: &CompactRefType,
680        sheet_registry: &SheetRegistry,
681    ) -> ReferenceType {
682        match ref_type {
683            CompactRefType::Cell {
684                sheet,
685                row,
686                col,
687                row_abs,
688                col_abs,
689            } => {
690                let sheet = match sheet {
691                    Some(SheetKey::Id(id)) => Some(sheet_registry.name(*id).to_string()),
692                    Some(SheetKey::Name(name_id)) => {
693                        Some(self.asts.resolve_string(*name_id).to_string())
694                    }
695                    None => None,
696                };
697                ReferenceType::Cell {
698                    sheet,
699                    row: *row,
700                    col: *col,
701                    row_abs: *row_abs,
702                    col_abs: *col_abs,
703                }
704            }
705
706            CompactRefType::Range {
707                sheet,
708                start_row,
709                start_col,
710                end_row,
711                end_col,
712                start_row_abs,
713                start_col_abs,
714                end_row_abs,
715                end_col_abs,
716            } => {
717                let sheet = match sheet {
718                    Some(SheetKey::Id(id)) => Some(sheet_registry.name(*id).to_string()),
719                    Some(SheetKey::Name(name_id)) => {
720                        Some(self.asts.resolve_string(*name_id).to_string())
721                    }
722                    None => None,
723                };
724                // Convert sentinel values back to None
725                ReferenceType::Range {
726                    sheet,
727                    start_row: if *start_row == 0 {
728                        None
729                    } else {
730                        Some(*start_row)
731                    },
732                    start_col: if *start_col == 0 {
733                        None
734                    } else {
735                        Some(*start_col)
736                    },
737                    end_row: if *end_row == u32::MAX {
738                        None
739                    } else {
740                        Some(*end_row)
741                    },
742                    end_col: if *end_col == u32::MAX {
743                        None
744                    } else {
745                        Some(*end_col)
746                    },
747                    start_row_abs: *start_row_abs,
748                    start_col_abs: *start_col_abs,
749                    end_row_abs: *end_row_abs,
750                    end_col_abs: *end_col_abs,
751                }
752            }
753
754            CompactRefType::External {
755                raw_id,
756                book_id,
757                sheet_id,
758                kind,
759            } => {
760                let raw = self.asts.resolve_string(*raw_id).to_string();
761                let book = self.asts.resolve_string(*book_id).to_string();
762                let sheet = self.asts.resolve_string(*sheet_id).to_string();
763                ReferenceType::External(ExternalReference {
764                    raw,
765                    book: ExternalBookRef::Token(book),
766                    sheet,
767                    kind: *kind,
768                })
769            }
770
771            CompactRefType::NamedRange(string_id) => {
772                let name = self.asts.resolve_string(*string_id).to_string();
773                ReferenceType::NamedRange(name)
774            }
775
776            CompactRefType::Table {
777                name_id,
778                specifier_id,
779            } => {
780                let name = self.asts.resolve_string(*name_id).to_string();
781                let specifier = specifier_id
782                    .and_then(|id| self.asts.resolve_table_specifier(id))
783                    .cloned();
784                ReferenceType::Table(TableReference { name, specifier })
785            }
786
787            CompactRefType::Cell3D {
788                sheet_first,
789                sheet_last,
790                row,
791                col,
792                row_abs,
793                col_abs,
794            } => ReferenceType::Cell3D {
795                sheet_first: self.asts.resolve_string(*sheet_first).to_string(),
796                sheet_last: self.asts.resolve_string(*sheet_last).to_string(),
797                row: *row,
798                col: *col,
799                row_abs: *row_abs,
800                col_abs: *col_abs,
801            },
802
803            CompactRefType::Range3D {
804                sheet_first,
805                sheet_last,
806                start_row,
807                start_col,
808                end_row,
809                end_col,
810                start_row_abs,
811                start_col_abs,
812                end_row_abs,
813                end_col_abs,
814            } => ReferenceType::Range3D {
815                sheet_first: self.asts.resolve_string(*sheet_first).to_string(),
816                sheet_last: self.asts.resolve_string(*sheet_last).to_string(),
817                start_row: if *start_row == 0 {
818                    None
819                } else {
820                    Some(*start_row)
821                },
822                start_col: if *start_col == 0 {
823                    None
824                } else {
825                    Some(*start_col)
826                },
827                end_row: if *end_row == u32::MAX {
828                    None
829                } else {
830                    Some(*end_row)
831                },
832                end_col: if *end_col == u32::MAX {
833                    None
834                } else {
835                    Some(*end_col)
836                },
837                start_row_abs: *start_row_abs,
838                start_col_abs: *start_col_abs,
839                end_row_abs: *end_row_abs,
840                end_col_abs: *end_col_abs,
841            },
842        }
843    }
844
845    /// Store an error with message preservation
846    fn store_error(&mut self, error: &ExcelError) -> ValueRef {
847        let error_ref = self.errors.insert(error);
848        ValueRef::error(error_ref.as_u32())
849    }
850
851    /// Get memory usage statistics
852    pub fn memory_usage(&self) -> DataStoreStats {
853        DataStoreStats {
854            scalar_bytes: self.scalars.memory_usage(),
855            string_bytes: self.strings.memory_usage(),
856            array_bytes: self.arrays.memory_usage(),
857            ast_bytes: self.asts.memory_usage(),
858            error_bytes: self.errors.memory_usage(),
859            total_scalars: self.scalars.len(),
860            total_strings: self.strings.len(),
861            total_arrays: self.arrays.len(),
862            total_ast_nodes: self.asts.stats().node_count,
863            total_errors: self.errors.len(),
864        }
865    }
866
867    /// Clear all data from the store
868    pub fn clear(&mut self) {
869        self.scalars.clear();
870        self.strings.clear();
871        self.arrays.clear();
872        self.asts.clear();
873        self.errors.clear();
874    }
875}
876
877impl Default for DataStore {
878    fn default() -> Self {
879        Self::new()
880    }
881}
882
883/// Statistics about data store memory usage
884#[derive(Debug, Clone)]
885pub struct DataStoreStats {
886    pub scalar_bytes: usize,
887    pub string_bytes: usize,
888    pub array_bytes: usize,
889    pub ast_bytes: usize,
890    pub error_bytes: usize,
891    pub total_scalars: usize,
892    pub total_strings: usize,
893    pub total_arrays: usize,
894    pub total_ast_nodes: usize,
895    pub total_errors: usize,
896}
897
898impl DataStoreStats {
899    pub fn total_bytes(&self) -> usize {
900        self.scalar_bytes + self.string_bytes + self.array_bytes + self.ast_bytes + self.error_bytes
901    }
902}
903
904// Helper trait implementations for ArrayRef and ScalarRef
905impl super::array::ArrayRef {
906    pub fn from_raw(raw: u32) -> Self {
907        super::array::ArrayRef(raw)
908    }
909}
910
911impl super::scalar::ScalarRef {
912    pub fn from_raw(raw: u32) -> Self {
913        Self { raw }
914    }
915
916    pub fn as_u32(self) -> u32 {
917        self.raw
918    }
919}
920
921#[cfg(test)]
922mod tests {
923    use super::*;
924
925    #[test]
926    fn test_data_store_empty_value() {
927        let mut store = DataStore::new();
928        let value_ref = store.store_value(LiteralValue::Empty);
929        assert!(value_ref.is_empty());
930
931        let retrieved = store.retrieve_value(value_ref);
932        assert_eq!(retrieved, LiteralValue::Empty);
933    }
934
935    #[test]
936    fn test_data_store_number() {
937        let mut store = DataStore::new();
938        let value_ref = store.store_value(LiteralValue::Number(42.5));
939
940        let retrieved = store.retrieve_value(value_ref);
941        assert_eq!(retrieved, LiteralValue::Number(42.5));
942    }
943
944    #[test]
945    fn test_data_store_datetime_private_encoding_round_trips() {
946        let mut store = DataStore::new();
947        for datetime in [
948            chrono::NaiveDate::from_ymd_opt(2024, 1, 15)
949                .unwrap()
950                .and_hms_opt(12, 30, 0)
951                .unwrap(),
952            chrono::NaiveDate::from_ymd_opt(1899, 12, 30)
953                .unwrap()
954                .and_hms_opt(0, 0, 0)
955                .unwrap(),
956        ] {
957            let value_ref = store.store_value(LiteralValue::DateTime(datetime));
958            assert_eq!(
959                store.retrieve_value(value_ref),
960                LiteralValue::DateTime(datetime)
961            );
962        }
963    }
964
965    #[test]
966    fn test_data_store_text() {
967        let mut store = DataStore::new();
968        let value_ref = store.store_value(LiteralValue::Text("Hello".to_string()));
969
970        let retrieved = store.retrieve_value(value_ref);
971        assert_eq!(retrieved, LiteralValue::Text("Hello".to_string()));
972    }
973
974    #[test]
975    fn test_data_store_boolean() {
976        let mut store = DataStore::new();
977
978        let true_ref = store.store_value(LiteralValue::Boolean(true));
979        let false_ref = store.store_value(LiteralValue::Boolean(false));
980
981        assert_eq!(store.retrieve_value(true_ref), LiteralValue::Boolean(true));
982        assert_eq!(
983            store.retrieve_value(false_ref),
984            LiteralValue::Boolean(false)
985        );
986    }
987
988    #[test]
989    fn test_data_store_error() {
990        let mut store = DataStore::new();
991
992        let error = ExcelError::new(ExcelErrorKind::Div);
993        let value_ref = store.store_value(LiteralValue::Error(error.clone()));
994
995        let retrieved = store.retrieve_value(value_ref);
996        match retrieved {
997            LiteralValue::Error(e) => assert_eq!(e.kind, ExcelErrorKind::Div),
998            _ => panic!("Expected error"),
999        }
1000    }
1001
1002    #[test]
1003    fn test_data_store_array() {
1004        let mut store = DataStore::new();
1005
1006        let array = vec![
1007            vec![LiteralValue::Number(1.0), LiteralValue::Number(2.0)],
1008            vec![LiteralValue::Number(3.0), LiteralValue::Number(4.0)],
1009        ];
1010
1011        let value_ref = store.store_value(LiteralValue::Array(array.clone()));
1012        let retrieved = store.retrieve_value(value_ref);
1013
1014        assert_eq!(retrieved, LiteralValue::Array(array));
1015    }
1016
1017    #[test]
1018    fn test_data_store_ast_literal() {
1019        let mut store = DataStore::new();
1020        let mut sheet_registry = SheetRegistry::new();
1021        sheet_registry.id_for("Sheet1");
1022
1023        let ast = ASTNode {
1024            node_type: ASTNodeType::Literal(LiteralValue::Number(42.0)),
1025            source_token: None,
1026            contains_volatile: false,
1027        };
1028
1029        let ast_id = store.store_ast(&ast, &sheet_registry);
1030        let retrieved = store.retrieve_ast(ast_id, &sheet_registry).unwrap();
1031
1032        match retrieved.node_type {
1033            ASTNodeType::Literal(lit) => assert_eq!(lit, LiteralValue::Number(42.0)),
1034            _ => panic!("Expected literal"),
1035        }
1036    }
1037
1038    #[test]
1039    fn test_data_store_ast_binary_op() {
1040        let mut store = DataStore::new();
1041        let mut sheet_registry = SheetRegistry::new();
1042        sheet_registry.id_for("Sheet1");
1043
1044        let ast = ASTNode {
1045            node_type: ASTNodeType::BinaryOp {
1046                op: "+".to_string(),
1047                left: Box::new(ASTNode {
1048                    node_type: ASTNodeType::Literal(LiteralValue::Number(1.0)),
1049                    source_token: None,
1050                    contains_volatile: false,
1051                }),
1052                right: Box::new(ASTNode {
1053                    node_type: ASTNodeType::Literal(LiteralValue::Number(2.0)),
1054                    source_token: None,
1055                    contains_volatile: false,
1056                }),
1057            },
1058            source_token: None,
1059            contains_volatile: false,
1060        };
1061
1062        let ast_id = store.store_ast(&ast, &sheet_registry);
1063        let retrieved = store.retrieve_ast(ast_id, &sheet_registry).unwrap();
1064
1065        match retrieved.node_type {
1066            ASTNodeType::BinaryOp { op, left, right } => {
1067                assert_eq!(op, "+");
1068                match left.node_type {
1069                    ASTNodeType::Literal(lit) => assert_eq!(lit, LiteralValue::Number(1.0)),
1070                    _ => panic!("Expected literal"),
1071                }
1072                match right.node_type {
1073                    ASTNodeType::Literal(lit) => assert_eq!(lit, LiteralValue::Number(2.0)),
1074                    _ => panic!("Expected literal"),
1075                }
1076            }
1077            _ => panic!("Expected binary op"),
1078        }
1079    }
1080
1081    #[test]
1082    fn test_data_store_ast_function() {
1083        let mut store = DataStore::new();
1084        let mut sheet_registry = SheetRegistry::new();
1085        sheet_registry.id_for("Sheet1");
1086
1087        let ast = ASTNode {
1088            node_type: ASTNodeType::Function {
1089                name: "SUM".to_string(),
1090                args: vec![
1091                    ASTNode {
1092                        node_type: ASTNodeType::Literal(LiteralValue::Number(1.0)),
1093                        source_token: None,
1094                        contains_volatile: false,
1095                    },
1096                    ASTNode {
1097                        node_type: ASTNodeType::Literal(LiteralValue::Number(2.0)),
1098                        source_token: None,
1099                        contains_volatile: false,
1100                    },
1101                ],
1102            },
1103            source_token: None,
1104            contains_volatile: false,
1105        };
1106
1107        let ast_id = store.store_ast(&ast, &sheet_registry);
1108        let retrieved = store.retrieve_ast(ast_id, &sheet_registry).unwrap();
1109
1110        match retrieved.node_type {
1111            ASTNodeType::Function { name, args } => {
1112                assert_eq!(name, "SUM");
1113                assert_eq!(args.len(), 2);
1114            }
1115            _ => panic!("Expected function"),
1116        }
1117    }
1118
1119    #[test]
1120    fn test_data_store_memory_stats() {
1121        let mut store = DataStore::new();
1122
1123        // Add some data
1124        store.store_value(LiteralValue::Number(42.0));
1125        store.store_value(LiteralValue::Text("Hello".to_string()));
1126        store.store_value(LiteralValue::Array(vec![vec![LiteralValue::Number(1.0)]]));
1127
1128        let stats = store.memory_usage();
1129        assert!(stats.total_bytes() > 0);
1130        assert_eq!(stats.total_scalars, 2); // 42.0 and 1.0
1131        assert_eq!(stats.total_strings, 1); // "Hello"
1132        assert_eq!(stats.total_arrays, 1);
1133    }
1134
1135    #[test]
1136    fn test_data_store_clear() {
1137        let mut store = DataStore::new();
1138
1139        store.store_value(LiteralValue::Number(42.0));
1140        store.store_value(LiteralValue::Text("Hello".to_string()));
1141
1142        let stats = store.memory_usage();
1143        assert!(stats.total_scalars > 0);
1144        assert!(stats.total_strings > 0);
1145
1146        store.clear();
1147
1148        let stats = store.memory_usage();
1149        assert_eq!(stats.total_scalars, 0);
1150        assert_eq!(stats.total_strings, 0);
1151    }
1152}
formualizer_eval/engine/arena/data_store.rs

formualizer_eval/engine/arena/
data_store.rs
