formualizer_eval/engine/arena/
data_store.rs

1/// Unified data storage for all value types using arenas
2/// Provides conversion between LiteralValue and ValueRef
3use super::array::ArrayArena;
4use super::ast::{AstArena, AstNodeId, CompactRefType, SheetKey};
5use super::error_arena::{ErrorArena, ErrorRef};
6use super::scalar::ScalarArena;
7use super::string_interner::{StringId, StringInterner};
8use super::value_ref::ValueRef;
9use crate::engine::sheet_registry::SheetRegistry;
10use formualizer_common::{ExcelError, ExcelErrorKind, LiteralValue};
11use formualizer_parse::parser::{
12    ASTNode, ASTNodeType, ExternalBookRef, ExternalReference, ReferenceType, TableReference,
13};
14
15/// Centralized data storage using arenas
16#[derive(Debug)]
17pub struct DataStore {
18    /// Scalar values (floats and large integers)
19    scalars: ScalarArena,
20
21    /// String values
22    strings: StringInterner,
23
24    /// Array values
25    arrays: ArrayArena,
26
27    /// AST nodes for formulas
28    asts: AstArena,
29
30    /// Error storage with message preservation
31    errors: ErrorArena,
32}
33
34impl DataStore {
35    pub fn new() -> Self {
36        Self {
37            scalars: ScalarArena::new(),
38            strings: StringInterner::new(),
39            arrays: ArrayArena::new(),
40            asts: AstArena::new(),
41            errors: ErrorArena::new(),
42        }
43    }
44
45    /// Batch store literal values; returns ValueRefs in same order.
46    pub fn store_values_batch<I>(&mut self, values: I) -> Vec<ValueRef>
47    where
48        I: IntoIterator<Item = LiteralValue>,
49    {
50        let iter = values.into_iter();
51        let (lower, _) = iter.size_hint();
52        let mut out = Vec::with_capacity(lower);
53        for v in iter {
54            out.push(self.store_value(v));
55        }
56        out
57    }
58
59    /// Batch store ASTs; returns AstNodeIds in same order.
60    pub fn store_asts_batch<'a, I>(
61        &mut self,
62        asts: I,
63        sheet_registry: &SheetRegistry,
64    ) -> Vec<AstNodeId>
65    where
66        I: IntoIterator<Item = &'a ASTNode>,
67    {
68        let iter = asts.into_iter();
69        let (lower, _) = iter.size_hint();
70        let mut out = Vec::with_capacity(lower);
71        for ast in iter {
72            out.push(self.store_ast(ast, sheet_registry));
73        }
74        out
75    }
76
77    pub fn with_capacity(estimated_cells: usize) -> Self {
78        Self {
79            scalars: ScalarArena::with_capacity(estimated_cells),
80            strings: StringInterner::with_capacity(estimated_cells / 10),
81            arrays: ArrayArena::with_capacity(estimated_cells / 100),
82            asts: AstArena::with_capacity(estimated_cells / 2),
83            errors: ErrorArena::with_capacity(estimated_cells / 20),
84        }
85    }
86
87    /// Store a LiteralValue and return a ValueRef
88    pub fn store_value(&mut self, value: LiteralValue) -> ValueRef {
89        match value {
90            LiteralValue::Empty => ValueRef::empty(),
91
92            LiteralValue::Number(n) => {
93                // Store as float in scalar arena
94                let idx = self.scalars.insert_float(n);
95                ValueRef::number(idx.as_u32())
96            }
97
98            LiteralValue::Text(s) => {
99                let id = self.strings.intern(&s);
100                ValueRef::string(id.as_u32())
101            }
102
103            LiteralValue::Boolean(b) => ValueRef::boolean(b),
104
105            LiteralValue::Error(err) => self.store_error(&err),
106
107            LiteralValue::Array(array) => {
108                // Convert nested array to ValueRefs
109                let rows = array.len() as u32;
110                let cols = array.first().map(|r| r.len()).unwrap_or(0) as u32;
111
112                let elements: Vec<ValueRef> = array
113                    .into_iter()
114                    .flatten()
115                    .map(|v| self.store_value(v))
116                    .collect();
117
118                let array_ref = self.arrays.insert(rows, cols, elements);
119                ValueRef::array(array_ref.as_u32())
120            }
121
122            LiteralValue::DateTime(dt) => {
123                // Store serial number as float
124                let serial = formualizer_common::datetime_to_serial(&dt);
125                let idx = self.scalars.insert_float(serial);
126                ValueRef::date_time(idx.as_u32())
127            }
128
129            LiteralValue::Date(d) => {
130                // Convert date to datetime at midnight
131                let dt = d.and_hms_opt(0, 0, 0).unwrap();
132                let serial = formualizer_common::datetime_to_serial(&dt);
133                let idx = self.scalars.insert_float(serial);
134                ValueRef::date_time(idx.as_u32())
135            }
136
137            LiteralValue::Time(t) => {
138                // Store time as fractional day
139                use chrono::Timelike;
140                let seconds = (t.hour() * 3600 + t.minute() * 60 + t.second()) as f64;
141                let fraction = seconds / 86400.0;
142                let idx = self.scalars.insert_float(fraction);
143                ValueRef::date_time(idx.as_u32())
144            }
145
146            LiteralValue::Duration(dur) => {
147                // Store as integer seconds (chrono::Duration has num_seconds())
148                let secs = dur.num_seconds();
149                let idx = self.scalars.insert_integer(secs);
150                let raw_index = idx.as_u32() & 0x7FFF_FFFF;
151                ValueRef::duration(raw_index)
152            }
153
154            LiteralValue::Int(i) => {
155                // Try to use small int optimization
156                if let Some(vref) = ValueRef::small_int(i as i32) {
157                    vref
158                } else {
159                    // Store as large integer
160                    let idx = self.scalars.insert_integer(i);
161                    ValueRef::large_int(idx.as_u32())
162                }
163            }
164
165            LiteralValue::Pending => ValueRef::pending(),
166        }
167    }
168
169    /// Retrieve a LiteralValue from a ValueRef
170    pub fn retrieve_value(&self, value_ref: ValueRef) -> LiteralValue {
171        use super::value_ref::ValueType;
172
173        match value_ref.value_type() {
174            ValueType::Empty => LiteralValue::Empty,
175
176            ValueType::SmallInt => {
177                // Small integers are inlined
178                if let Some(i) = value_ref.as_small_int() {
179                    LiteralValue::Int(i as i64)
180                } else {
181                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
182                }
183            }
184
185            ValueType::LargeInt => {
186                if let Some(idx) = value_ref.arena_index() {
187                    let scalar_ref = super::scalar::ScalarRef::from_raw(idx | (1 << 31));
188                    if let Some(i) = self.scalars.get_integer(scalar_ref) {
189                        LiteralValue::Int(i)
190                    } else {
191                        LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
192                    }
193                } else {
194                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
195                }
196            }
197
198            ValueType::Number => {
199                if let Some(idx) = value_ref.arena_index() {
200                    let scalar_ref = super::scalar::ScalarRef::from_raw(idx);
201                    if let Some(f) = self.scalars.get_float(scalar_ref) {
202                        LiteralValue::Number(f)
203                    } else {
204                        LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
205                    }
206                } else {
207                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
208                }
209            }
210
211            ValueType::String => {
212                if let Some(idx) = value_ref.arena_index() {
213                    let string_id = StringId::from_raw(idx);
214                    let s = self.strings.resolve(string_id);
215                    LiteralValue::Text(s.to_string())
216                } else {
217                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
218                }
219            }
220
221            ValueType::Boolean => {
222                if let Some(b) = value_ref.as_boolean() {
223                    LiteralValue::Boolean(b)
224                } else {
225                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
226                }
227            }
228
229            ValueType::Error => {
230                if let Some(error_ref_raw) = value_ref.as_error_ref() {
231                    let error_ref = ErrorRef::from_raw(error_ref_raw);
232                    if let Some(error) = self.errors.get(error_ref) {
233                        LiteralValue::Error(error)
234                    } else {
235                        LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
236                    }
237                } else {
238                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
239                }
240            }
241
242            ValueType::Array => {
243                if let Some(idx) = value_ref.arena_index() {
244                    let array_ref = super::array::ArrayRef::from_raw(idx);
245                    if let Some(array_2d) = self.arrays.get_2d(array_ref) {
246                        // Convert back to LiteralValue array
247                        let result: Vec<Vec<LiteralValue>> = array_2d
248                            .into_iter()
249                            .map(|row| row.into_iter().map(|v| self.retrieve_value(v)).collect())
250                            .collect();
251                        LiteralValue::Array(result)
252                    } else {
253                        LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
254                    }
255                } else {
256                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
257                }
258            }
259
260            ValueType::DateTime => {
261                if let Some(idx) = value_ref.arena_index() {
262                    let scalar_ref = super::scalar::ScalarRef::from_raw(idx);
263                    if let Some(serial) = self.scalars.get_float(scalar_ref) {
264                        let dt = formualizer_common::serial_to_datetime(serial);
265                        LiteralValue::DateTime(dt)
266                    } else {
267                        LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
268                    }
269                } else {
270                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
271                }
272            }
273
274            ValueType::Duration => {
275                if let Some(idx) = value_ref.arena_index() {
276                    let scalar_ref = super::scalar::ScalarRef::from_raw(idx | (1 << 31));
277                    if let Some(secs) = self.scalars.get_integer(scalar_ref) {
278                        let dur = chrono::Duration::seconds(secs);
279                        LiteralValue::Duration(dur)
280                    } else {
281                        LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
282                    }
283                } else {
284                    LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
285                }
286            }
287
288            ValueType::Pending => LiteralValue::Pending,
289
290            ValueType::FormulaAst => {
291                // Formula ASTs shouldn't be returned as values
292                LiteralValue::Error(ExcelError::new(ExcelErrorKind::Value))
293            }
294        }
295    }
296
297    /// Store an AST node and return its ID
298    pub fn store_ast(&mut self, ast: &ASTNode, sheet_registry: &SheetRegistry) -> AstNodeId {
299        self.convert_ast_node(ast, sheet_registry)
300    }
301
302    /// Retrieve an AST node from its ID
303    pub fn retrieve_ast(&self, id: AstNodeId, sheet_registry: &SheetRegistry) -> Option<ASTNode> {
304        self.reconstruct_ast_node(id, sheet_registry)
305    }
306
307    pub fn resolve_ast_string(&self, id: StringId) -> &str {
308        self.asts.resolve_string(id)
309    }
310
311    pub(crate) fn ast_strings(&self) -> &StringInterner {
312        self.asts.strings()
313    }
314
315    pub fn reconstruct_reference_type_for_eval(
316        &self,
317        ref_type: &CompactRefType,
318        sheet_registry: &SheetRegistry,
319    ) -> ReferenceType {
320        self.reconstruct_reference_type(ref_type, sheet_registry)
321    }
322
323    pub fn get_node(&self, id: AstNodeId) -> Option<&super::ast::AstNodeData> {
324        self.asts.get(id)
325    }
326
327    pub fn get_args(&self, id: AstNodeId) -> Option<&[AstNodeId]> {
328        self.asts.get_function_args(id)
329    }
330
331    pub fn get_array_elems(&self, id: AstNodeId) -> Option<(u16, u16, &[AstNodeId])> {
332        self.asts.get_array_elements_info(id)
333    }
334
335    pub fn ast_needs_structural_rewrite(&self, id: AstNodeId) -> bool {
336        let mut stack = vec![id];
337        while let Some(node_id) = stack.pop() {
338            let Some(node) = self.get_node(node_id) else {
339                continue;
340            };
341            match node {
342                super::ast::AstNodeData::Reference { ref_type, .. } => {
343                    if let CompactRefType::Table { name_id, .. } = ref_type
344                        && self.resolve_ast_string(*name_id).is_empty()
345                    {
346                        return true;
347                    }
348                }
349                super::ast::AstNodeData::UnaryOp { expr_id, .. } => stack.push(*expr_id),
350                super::ast::AstNodeData::BinaryOp {
351                    left_id, right_id, ..
352                } => {
353                    stack.push(*right_id);
354                    stack.push(*left_id);
355                }
356                super::ast::AstNodeData::Function { .. } => {
357                    if let Some(args) = self.get_args(node_id) {
358                        stack.extend(args.iter().rev().copied());
359                    }
360                }
361                super::ast::AstNodeData::Array { .. } => {
362                    if let Some((_, _, elems)) = self.get_array_elems(node_id) {
363                        stack.extend(elems.iter().rev().copied());
364                    }
365                }
366                super::ast::AstNodeData::Literal(_) => {}
367            }
368        }
369        false
370    }
371
372    /// Convert ASTNode to arena representation
373    fn convert_ast_node(&mut self, node: &ASTNode, sheet_registry: &SheetRegistry) -> AstNodeId {
374        match &node.node_type {
375            ASTNodeType::Literal(lit) => {
376                let value_ref = self.store_value(lit.clone());
377                self.asts.insert_literal(value_ref)
378            }
379
380            ASTNodeType::Reference {
381                original,
382                reference,
383            } => {
384                let ref_type = self.convert_reference_type(reference, sheet_registry);
385                self.asts.insert_reference(original, ref_type)
386            }
387
388            ASTNodeType::UnaryOp { op, expr } => {
389                let expr_id = self.convert_ast_node(expr, sheet_registry);
390                self.asts.insert_unary_op(op, expr_id)
391            }
392
393            ASTNodeType::BinaryOp { op, left, right } => {
394                let left_id = self.convert_ast_node(left, sheet_registry);
395                let right_id = self.convert_ast_node(right, sheet_registry);
396                self.asts.insert_binary_op(op, left_id, right_id)
397            }
398
399            ASTNodeType::Function { name, args } => {
400                let arg_ids: Vec<AstNodeId> = args
401                    .iter()
402                    .map(|arg| self.convert_ast_node(arg, sheet_registry))
403                    .collect();
404                self.asts.insert_function(name, arg_ids)
405            }
406
407            ASTNodeType::Array(rows) => {
408                let total_elements = rows.iter().map(|r| r.len()).sum();
409                let mut elements = Vec::with_capacity(total_elements);
410
411                let rows_count = rows.len() as u16;
412                let cols_count = rows.first().map(|r| r.len()).unwrap_or(0) as u16;
413
414                for row in rows {
415                    for elem in row {
416                        elements.push(self.convert_ast_node(elem, sheet_registry));
417                    }
418                }
419
420                self.asts.insert_array(rows_count, cols_count, elements)
421            }
422
423            // Postfix call (e.g. LAMBDA immediate-invocation). The arena does
424            // not yet have a dedicated node kind for this, and full evaluator
425            // semantics are out of scope for the parser-side change. Store an
426            // unsupported-formula error literal so that downstream evaluation
427            // surfaces a clear #N/A!-style error instead of silently producing
428            // a wrong result.
429            ASTNodeType::Call { .. } => {
430                let value_ref = self.store_value(LiteralValue::Error(
431                    ExcelError::new(ExcelErrorKind::NImpl)
432                        .with_message("Immediate-invocation calls are not yet supported"),
433                ));
434                self.asts.insert_literal(value_ref)
435            }
436        }
437    }
438
439    /// Convert ReferenceType to CompactRefType
440    fn convert_reference_type(
441        &mut self,
442        ref_type: &ReferenceType,
443        sheet_registry: &SheetRegistry,
444    ) -> CompactRefType {
445        match ref_type {
446            ReferenceType::Cell {
447                sheet,
448                row,
449                col,
450                row_abs,
451                col_abs,
452            } => {
453                let sheet = match sheet.as_ref() {
454                    Some(s) => match sheet_registry.get_id(s) {
455                        Some(id) => Some(SheetKey::Id(id)),
456                        None => Some(SheetKey::Name(self.asts.strings_mut().intern(s))),
457                    },
458                    None => None,
459                };
460                CompactRefType::Cell {
461                    sheet,
462                    row: *row,
463                    col: *col,
464                    row_abs: *row_abs,
465                    col_abs: *col_abs,
466                }
467            }
468
469            ReferenceType::Range {
470                sheet,
471                start_row,
472                start_col,
473                end_row,
474                end_col,
475                start_row_abs,
476                start_col_abs,
477                end_row_abs,
478                end_col_abs,
479            } => {
480                let sheet = match sheet.as_ref() {
481                    Some(s) => match sheet_registry.get_id(s) {
482                        Some(id) => Some(SheetKey::Id(id)),
483                        None => Some(SheetKey::Name(self.asts.strings_mut().intern(s))),
484                    },
485                    None => None,
486                };
487                // For optional range bounds, use 0/u32::MAX as sentinels for unbounded
488                CompactRefType::Range {
489                    sheet,
490                    start_row: start_row.unwrap_or(0),
491                    start_col: start_col.unwrap_or(0),
492                    end_row: end_row.unwrap_or(u32::MAX),
493                    end_col: end_col.unwrap_or(u32::MAX),
494                    start_row_abs: *start_row_abs,
495                    start_col_abs: *start_col_abs,
496                    end_row_abs: *end_row_abs,
497                    end_col_abs: *end_col_abs,
498                }
499            }
500
501            ReferenceType::External(ext) => {
502                let raw_id = self.asts.strings_mut().intern(&ext.raw);
503                let book_id = self.asts.strings_mut().intern(ext.book.token());
504                let sheet_id = self.asts.strings_mut().intern(&ext.sheet);
505                CompactRefType::External {
506                    raw_id,
507                    book_id,
508                    sheet_id,
509                    kind: ext.kind,
510                }
511            }
512
513            ReferenceType::NamedRange(name) => {
514                let string_id = self.asts.strings_mut().intern(name);
515                CompactRefType::NamedRange(string_id)
516            }
517
518            ReferenceType::Table(table_ref) => {
519                let name_id = self.asts.strings_mut().intern(&table_ref.name);
520                let specifier_id = table_ref
521                    .specifier
522                    .as_ref()
523                    .map(|specifier| self.asts.intern_table_specifier(specifier));
524                CompactRefType::Table {
525                    name_id,
526                    specifier_id,
527                }
528            }
529
530            ReferenceType::Cell3D {
531                sheet_first,
532                sheet_last,
533                row,
534                col,
535                row_abs,
536                col_abs,
537            } => {
538                let sheet_first = self.asts.strings_mut().intern(sheet_first);
539                let sheet_last = self.asts.strings_mut().intern(sheet_last);
540                CompactRefType::Cell3D {
541                    sheet_first,
542                    sheet_last,
543                    row: *row,
544                    col: *col,
545                    row_abs: *row_abs,
546                    col_abs: *col_abs,
547                }
548            }
549
550            ReferenceType::Range3D {
551                sheet_first,
552                sheet_last,
553                start_row,
554                start_col,
555                end_row,
556                end_col,
557                start_row_abs,
558                start_col_abs,
559                end_row_abs,
560                end_col_abs,
561            } => {
562                let sheet_first = self.asts.strings_mut().intern(sheet_first);
563                let sheet_last = self.asts.strings_mut().intern(sheet_last);
564                CompactRefType::Range3D {
565                    sheet_first,
566                    sheet_last,
567                    start_row: start_row.unwrap_or(0),
568                    start_col: start_col.unwrap_or(0),
569                    end_row: end_row.unwrap_or(u32::MAX),
570                    end_col: end_col.unwrap_or(u32::MAX),
571                    start_row_abs: *start_row_abs,
572                    start_col_abs: *start_col_abs,
573                    end_row_abs: *end_row_abs,
574                    end_col_abs: *end_col_abs,
575                }
576            }
577        }
578    }
579
580    /// Reconstruct an ASTNode from arena representation
581    fn reconstruct_ast_node(
582        &self,
583        id: AstNodeId,
584        sheet_registry: &SheetRegistry,
585    ) -> Option<ASTNode> {
586        use super::ast::AstNodeData;
587
588        let node_data = self.asts.get(id)?;
589
590        let node_type = match node_data {
591            AstNodeData::Literal(value_ref) => {
592                let lit = self.retrieve_value(*value_ref);
593                ASTNodeType::Literal(lit)
594            }
595
596            AstNodeData::Reference {
597                original_id,
598                ref_type,
599            } => {
600                let original = self.asts.resolve_string(*original_id).to_string();
601                let reference = self.reconstruct_reference_type(ref_type, sheet_registry);
602                ASTNodeType::Reference {
603                    original,
604                    reference,
605                }
606            }
607
608            AstNodeData::UnaryOp { op_id, expr_id } => {
609                let op = self.asts.resolve_string(*op_id).to_string();
610                let expr = Box::new(self.reconstruct_ast_node(*expr_id, sheet_registry)?);
611                ASTNodeType::UnaryOp { op, expr }
612            }
613
614            AstNodeData::BinaryOp {
615                op_id,
616                left_id,
617                right_id,
618            } => {
619                let op = self.asts.resolve_string(*op_id).to_string();
620                let left = Box::new(self.reconstruct_ast_node(*left_id, sheet_registry)?);
621                let right = Box::new(self.reconstruct_ast_node(*right_id, sheet_registry)?);
622                ASTNodeType::BinaryOp { op, left, right }
623            }
624
625            AstNodeData::Function { name_id, .. } => {
626                let name = self.asts.resolve_string(*name_id).to_string();
627                let arg_ids = self.asts.get_function_args(id)?;
628                let args: Vec<ASTNode> = arg_ids
629                    .iter()
630                    .filter_map(|&arg_id| self.reconstruct_ast_node(arg_id, sheet_registry))
631                    .collect();
632                ASTNodeType::Function { name, args }
633            }
634
635            AstNodeData::Array { rows, cols, .. } => {
636                let elements = self.asts.get_array_elements(id)?;
637                let mut result = Vec::with_capacity(*rows as usize);
638
639                for r in 0..*rows {
640                    let mut row = Vec::with_capacity(*cols as usize);
641                    for c in 0..*cols {
642                        let idx = (r * *cols + c) as usize;
643                        if let Some(&elem_id) = elements.get(idx)
644                            && let Some(node) = self.reconstruct_ast_node(elem_id, sheet_registry)
645                        {
646                            row.push(node);
647                        }
648                    }
649                    result.push(row);
650                }
651
652                ASTNodeType::Array(result)
653            }
654        };
655
656        Some(ASTNode {
657            node_type,
658            source_token: None, // Token information is not preserved in arena
659            contains_volatile: false,
660        })
661    }
662
663    /// Reconstruct a ReferenceType from CompactRefType
664    fn reconstruct_reference_type(
665        &self,
666        ref_type: &CompactRefType,
667        sheet_registry: &SheetRegistry,
668    ) -> ReferenceType {
669        match ref_type {
670            CompactRefType::Cell {
671                sheet,
672                row,
673                col,
674                row_abs,
675                col_abs,
676            } => {
677                let sheet = match sheet {
678                    Some(SheetKey::Id(id)) => Some(sheet_registry.name(*id).to_string()),
679                    Some(SheetKey::Name(name_id)) => {
680                        Some(self.asts.resolve_string(*name_id).to_string())
681                    }
682                    None => None,
683                };
684                ReferenceType::Cell {
685                    sheet,
686                    row: *row,
687                    col: *col,
688                    row_abs: *row_abs,
689                    col_abs: *col_abs,
690                }
691            }
692
693            CompactRefType::Range {
694                sheet,
695                start_row,
696                start_col,
697                end_row,
698                end_col,
699                start_row_abs,
700                start_col_abs,
701                end_row_abs,
702                end_col_abs,
703            } => {
704                let sheet = match sheet {
705                    Some(SheetKey::Id(id)) => Some(sheet_registry.name(*id).to_string()),
706                    Some(SheetKey::Name(name_id)) => {
707                        Some(self.asts.resolve_string(*name_id).to_string())
708                    }
709                    None => None,
710                };
711                // Convert sentinel values back to None
712                ReferenceType::Range {
713                    sheet,
714                    start_row: if *start_row == 0 {
715                        None
716                    } else {
717                        Some(*start_row)
718                    },
719                    start_col: if *start_col == 0 {
720                        None
721                    } else {
722                        Some(*start_col)
723                    },
724                    end_row: if *end_row == u32::MAX {
725                        None
726                    } else {
727                        Some(*end_row)
728                    },
729                    end_col: if *end_col == u32::MAX {
730                        None
731                    } else {
732                        Some(*end_col)
733                    },
734                    start_row_abs: *start_row_abs,
735                    start_col_abs: *start_col_abs,
736                    end_row_abs: *end_row_abs,
737                    end_col_abs: *end_col_abs,
738                }
739            }
740
741            CompactRefType::External {
742                raw_id,
743                book_id,
744                sheet_id,
745                kind,
746            } => {
747                let raw = self.asts.resolve_string(*raw_id).to_string();
748                let book = self.asts.resolve_string(*book_id).to_string();
749                let sheet = self.asts.resolve_string(*sheet_id).to_string();
750                ReferenceType::External(ExternalReference {
751                    raw,
752                    book: ExternalBookRef::Token(book),
753                    sheet,
754                    kind: *kind,
755                })
756            }
757
758            CompactRefType::NamedRange(string_id) => {
759                let name = self.asts.resolve_string(*string_id).to_string();
760                ReferenceType::NamedRange(name)
761            }
762
763            CompactRefType::Table {
764                name_id,
765                specifier_id,
766            } => {
767                let name = self.asts.resolve_string(*name_id).to_string();
768                let specifier = specifier_id
769                    .and_then(|id| self.asts.resolve_table_specifier(id))
770                    .cloned();
771                ReferenceType::Table(TableReference { name, specifier })
772            }
773
774            CompactRefType::Cell3D {
775                sheet_first,
776                sheet_last,
777                row,
778                col,
779                row_abs,
780                col_abs,
781            } => ReferenceType::Cell3D {
782                sheet_first: self.asts.resolve_string(*sheet_first).to_string(),
783                sheet_last: self.asts.resolve_string(*sheet_last).to_string(),
784                row: *row,
785                col: *col,
786                row_abs: *row_abs,
787                col_abs: *col_abs,
788            },
789
790            CompactRefType::Range3D {
791                sheet_first,
792                sheet_last,
793                start_row,
794                start_col,
795                end_row,
796                end_col,
797                start_row_abs,
798                start_col_abs,
799                end_row_abs,
800                end_col_abs,
801            } => ReferenceType::Range3D {
802                sheet_first: self.asts.resolve_string(*sheet_first).to_string(),
803                sheet_last: self.asts.resolve_string(*sheet_last).to_string(),
804                start_row: if *start_row == 0 {
805                    None
806                } else {
807                    Some(*start_row)
808                },
809                start_col: if *start_col == 0 {
810                    None
811                } else {
812                    Some(*start_col)
813                },
814                end_row: if *end_row == u32::MAX {
815                    None
816                } else {
817                    Some(*end_row)
818                },
819                end_col: if *end_col == u32::MAX {
820                    None
821                } else {
822                    Some(*end_col)
823                },
824                start_row_abs: *start_row_abs,
825                start_col_abs: *start_col_abs,
826                end_row_abs: *end_row_abs,
827                end_col_abs: *end_col_abs,
828            },
829        }
830    }
831
832    /// Store an error with message preservation
833    fn store_error(&mut self, error: &ExcelError) -> ValueRef {
834        let error_ref = self.errors.insert(error);
835        ValueRef::error(error_ref.as_u32())
836    }
837
838    /// Get memory usage statistics
839    pub fn memory_usage(&self) -> DataStoreStats {
840        DataStoreStats {
841            scalar_bytes: self.scalars.memory_usage(),
842            string_bytes: self.strings.memory_usage(),
843            array_bytes: self.arrays.memory_usage(),
844            ast_bytes: self.asts.memory_usage(),
845            error_bytes: self.errors.memory_usage(),
846            total_scalars: self.scalars.len(),
847            total_strings: self.strings.len(),
848            total_arrays: self.arrays.len(),
849            total_ast_nodes: self.asts.stats().node_count,
850            total_errors: self.errors.len(),
851        }
852    }
853
854    /// Clear all data from the store
855    pub fn clear(&mut self) {
856        self.scalars.clear();
857        self.strings.clear();
858        self.arrays.clear();
859        self.asts.clear();
860        self.errors.clear();
861    }
862}
863
864impl Default for DataStore {
865    fn default() -> Self {
866        Self::new()
867    }
868}
869
870/// Statistics about data store memory usage
871#[derive(Debug, Clone)]
872pub struct DataStoreStats {
873    pub scalar_bytes: usize,
874    pub string_bytes: usize,
875    pub array_bytes: usize,
876    pub ast_bytes: usize,
877    pub error_bytes: usize,
878    pub total_scalars: usize,
879    pub total_strings: usize,
880    pub total_arrays: usize,
881    pub total_ast_nodes: usize,
882    pub total_errors: usize,
883}
884
885impl DataStoreStats {
886    pub fn total_bytes(&self) -> usize {
887        self.scalar_bytes + self.string_bytes + self.array_bytes + self.ast_bytes + self.error_bytes
888    }
889}
890
891// Helper trait implementations for ArrayRef and ScalarRef
892impl super::array::ArrayRef {
893    pub fn from_raw(raw: u32) -> Self {
894        super::array::ArrayRef(raw)
895    }
896}
897
898impl super::scalar::ScalarRef {
899    pub fn from_raw(raw: u32) -> Self {
900        Self { raw }
901    }
902
903    pub fn as_u32(self) -> u32 {
904        self.raw
905    }
906}
907
908#[cfg(test)]
909mod tests {
910    use super::*;
911
912    #[test]
913    fn test_data_store_empty_value() {
914        let mut store = DataStore::new();
915        let value_ref = store.store_value(LiteralValue::Empty);
916        assert!(value_ref.is_empty());
917
918        let retrieved = store.retrieve_value(value_ref);
919        assert_eq!(retrieved, LiteralValue::Empty);
920    }
921
922    #[test]
923    fn test_data_store_number() {
924        let mut store = DataStore::new();
925        let value_ref = store.store_value(LiteralValue::Number(42.5));
926
927        let retrieved = store.retrieve_value(value_ref);
928        assert_eq!(retrieved, LiteralValue::Number(42.5));
929    }
930
931    #[test]
932    fn test_data_store_text() {
933        let mut store = DataStore::new();
934        let value_ref = store.store_value(LiteralValue::Text("Hello".to_string()));
935
936        let retrieved = store.retrieve_value(value_ref);
937        assert_eq!(retrieved, LiteralValue::Text("Hello".to_string()));
938    }
939
940    #[test]
941    fn test_data_store_boolean() {
942        let mut store = DataStore::new();
943
944        let true_ref = store.store_value(LiteralValue::Boolean(true));
945        let false_ref = store.store_value(LiteralValue::Boolean(false));
946
947        assert_eq!(store.retrieve_value(true_ref), LiteralValue::Boolean(true));
948        assert_eq!(
949            store.retrieve_value(false_ref),
950            LiteralValue::Boolean(false)
951        );
952    }
953
954    #[test]
955    fn test_data_store_error() {
956        let mut store = DataStore::new();
957
958        let error = ExcelError::new(ExcelErrorKind::Div);
959        let value_ref = store.store_value(LiteralValue::Error(error.clone()));
960
961        let retrieved = store.retrieve_value(value_ref);
962        match retrieved {
963            LiteralValue::Error(e) => assert_eq!(e.kind, ExcelErrorKind::Div),
964            _ => panic!("Expected error"),
965        }
966    }
967
968    #[test]
969    fn test_data_store_array() {
970        let mut store = DataStore::new();
971
972        let array = vec![
973            vec![LiteralValue::Number(1.0), LiteralValue::Number(2.0)],
974            vec![LiteralValue::Number(3.0), LiteralValue::Number(4.0)],
975        ];
976
977        let value_ref = store.store_value(LiteralValue::Array(array.clone()));
978        let retrieved = store.retrieve_value(value_ref);
979
980        assert_eq!(retrieved, LiteralValue::Array(array));
981    }
982
983    #[test]
984    fn test_data_store_ast_literal() {
985        let mut store = DataStore::new();
986        let mut sheet_registry = SheetRegistry::new();
987        sheet_registry.id_for("Sheet1");
988
989        let ast = ASTNode {
990            node_type: ASTNodeType::Literal(LiteralValue::Number(42.0)),
991            source_token: None,
992            contains_volatile: false,
993        };
994
995        let ast_id = store.store_ast(&ast, &sheet_registry);
996        let retrieved = store.retrieve_ast(ast_id, &sheet_registry).unwrap();
997
998        match retrieved.node_type {
999            ASTNodeType::Literal(lit) => assert_eq!(lit, LiteralValue::Number(42.0)),
1000            _ => panic!("Expected literal"),
1001        }
1002    }
1003
1004    #[test]
1005    fn test_data_store_ast_binary_op() {
1006        let mut store = DataStore::new();
1007        let mut sheet_registry = SheetRegistry::new();
1008        sheet_registry.id_for("Sheet1");
1009
1010        let ast = ASTNode {
1011            node_type: ASTNodeType::BinaryOp {
1012                op: "+".to_string(),
1013                left: Box::new(ASTNode {
1014                    node_type: ASTNodeType::Literal(LiteralValue::Number(1.0)),
1015                    source_token: None,
1016                    contains_volatile: false,
1017                }),
1018                right: Box::new(ASTNode {
1019                    node_type: ASTNodeType::Literal(LiteralValue::Number(2.0)),
1020                    source_token: None,
1021                    contains_volatile: false,
1022                }),
1023            },
1024            source_token: None,
1025            contains_volatile: false,
1026        };
1027
1028        let ast_id = store.store_ast(&ast, &sheet_registry);
1029        let retrieved = store.retrieve_ast(ast_id, &sheet_registry).unwrap();
1030
1031        match retrieved.node_type {
1032            ASTNodeType::BinaryOp { op, left, right } => {
1033                assert_eq!(op, "+");
1034                match left.node_type {
1035                    ASTNodeType::Literal(lit) => assert_eq!(lit, LiteralValue::Number(1.0)),
1036                    _ => panic!("Expected literal"),
1037                }
1038                match right.node_type {
1039                    ASTNodeType::Literal(lit) => assert_eq!(lit, LiteralValue::Number(2.0)),
1040                    _ => panic!("Expected literal"),
1041                }
1042            }
1043            _ => panic!("Expected binary op"),
1044        }
1045    }
1046
1047    #[test]
1048    fn test_data_store_ast_function() {
1049        let mut store = DataStore::new();
1050        let mut sheet_registry = SheetRegistry::new();
1051        sheet_registry.id_for("Sheet1");
1052
1053        let ast = ASTNode {
1054            node_type: ASTNodeType::Function {
1055                name: "SUM".to_string(),
1056                args: vec![
1057                    ASTNode {
1058                        node_type: ASTNodeType::Literal(LiteralValue::Number(1.0)),
1059                        source_token: None,
1060                        contains_volatile: false,
1061                    },
1062                    ASTNode {
1063                        node_type: ASTNodeType::Literal(LiteralValue::Number(2.0)),
1064                        source_token: None,
1065                        contains_volatile: false,
1066                    },
1067                ],
1068            },
1069            source_token: None,
1070            contains_volatile: false,
1071        };
1072
1073        let ast_id = store.store_ast(&ast, &sheet_registry);
1074        let retrieved = store.retrieve_ast(ast_id, &sheet_registry).unwrap();
1075
1076        match retrieved.node_type {
1077            ASTNodeType::Function { name, args } => {
1078                assert_eq!(name, "SUM");
1079                assert_eq!(args.len(), 2);
1080            }
1081            _ => panic!("Expected function"),
1082        }
1083    }
1084
1085    #[test]
1086    fn test_data_store_memory_stats() {
1087        let mut store = DataStore::new();
1088
1089        // Add some data
1090        store.store_value(LiteralValue::Number(42.0));
1091        store.store_value(LiteralValue::Text("Hello".to_string()));
1092        store.store_value(LiteralValue::Array(vec![vec![LiteralValue::Number(1.0)]]));
1093
1094        let stats = store.memory_usage();
1095        assert!(stats.total_bytes() > 0);
1096        assert_eq!(stats.total_scalars, 2); // 42.0 and 1.0
1097        assert_eq!(stats.total_strings, 1); // "Hello"
1098        assert_eq!(stats.total_arrays, 1);
1099    }
1100
1101    #[test]
1102    fn test_data_store_clear() {
1103        let mut store = DataStore::new();
1104
1105        store.store_value(LiteralValue::Number(42.0));
1106        store.store_value(LiteralValue::Text("Hello".to_string()));
1107
1108        let stats = store.memory_usage();
1109        assert!(stats.total_scalars > 0);
1110        assert!(stats.total_strings > 0);
1111
1112        store.clear();
1113
1114        let stats = store.memory_usage();
1115        assert_eq!(stats.total_scalars, 0);
1116        assert_eq!(stats.total_strings, 0);
1117    }
1118}
formualizer_eval/engine/arena/data_store.rs

formualizer_eval/engine/arena/
data_store.rs
