formualizer_eval/engine/
formula_ingest.rs

1use std::collections::BTreeMap;
2use std::sync::Arc;
3
4use super::FormulaPlaneMode;
5use super::arena::AstNodeId;
6use super::formula_source::{SourceFamilyId, SourceFormulaOrder};
7
8#[derive(Clone, Debug)]
9pub struct FormulaIngestRecord {
10    pub row: u32,
11    pub col: u32,
12    pub ast_id: AstNodeId,
13    pub formula_text: Option<Arc<str>>,
14    pub(crate) source_order: Option<SourceFormulaOrder>,
15    pub(crate) source_family: Option<SourceFamilyId>,
16    pub(crate) partition_owner: Option<SourceFamilyId>,
17    /// Load-time family grouping (Program 2): `Some(anchor)` (0-based)
18    /// when this cell is a relative copy of the formula at `anchor`, whose
19    /// arena AST is `ast_id`; the member's own AST was never interned.
20    pub(crate) member_anchor: Option<(u32, u32)>,
21}
22
23impl FormulaIngestRecord {
24    pub fn new(row: u32, col: u32, ast_id: AstNodeId, formula_text: Option<Arc<str>>) -> Self {
25        Self {
26            row,
27            col,
28            ast_id,
29            formula_text,
30            source_order: None,
31            source_family: None,
32            partition_owner: None,
33            member_anchor: None,
34        }
35    }
36
37    /// A relative copy of `template` (the formula at 0-based `anchor`).
38    pub(crate) fn member(row: u32, col: u32, template: AstNodeId, anchor: (u32, u32)) -> Self {
39        Self {
40            member_anchor: Some(anchor),
41            ..Self::new(row, col, template, None)
42        }
43    }
44
45    /// Whether this record is a grouped family member (its `ast_id` is the
46    /// family template, valid at the anchor cell, not at this cell).
47    pub fn is_family_member(&self) -> bool {
48        self.member_anchor.is_some()
49    }
50
51    pub(crate) fn with_source_proof(
52        mut self,
53        source_order: SourceFormulaOrder,
54        source_family: Option<SourceFamilyId>,
55        partition_owner: Option<SourceFamilyId>,
56    ) -> Self {
57        self.source_order = Some(source_order);
58        self.source_family = source_family;
59        self.partition_owner = partition_owner;
60        self
61    }
62}
63
64/// Load-time family grouping state for one sheet (Program 2, P2-M2):
65/// the last formula staged in each column and the one before the current
66/// cell in its row. A formula that is exactly the formula above it (or to
67/// its left) relocated becomes a member of that formula's family and is
68/// never interned; see `Engine::stage_formula_ast`.
69#[derive(Debug, Default)]
70pub struct FormulaFamilyGrouper {
71    /// col0 -> (row0 of the last formula staged in the column, its family)
72    pub(crate) by_col: rustc_hash::FxHashMap<u32, (u32, GroupedFamily)>,
73    /// (row0, col0, family) of the last formula staged.
74    pub(crate) last: Option<(u32, u32, GroupedFamily)>,
75    /// Members grouped so far.
76    pub(crate) members: u64,
77    /// Staging counters, folded into the engine's by
78    /// `Engine::finish_family_grouper`.
79    pub(crate) counters: super::SourceFamilyCounters,
80    /// Shared-formula identity -> the certified family its first parsed
81    /// formula belongs to.
82    pub(crate) shared_roots: rustc_hash::FxHashMap<u64, GroupedFamily>,
83}
84
85impl FormulaFamilyGrouper {
86    pub fn new() -> Self {
87        Self::default()
88    }
89
90    /// Number of formulas staged as family members.
91    pub fn members(&self) -> u64 {
92        self.members
93    }
94
95    /// Count one formula text staged.
96    #[doc(hidden)]
97    pub fn note_formula(&mut self) {
98        self.counters.formulas += 1;
99    }
100
101    /// Count one formula parse of `bytes` bytes.
102    #[doc(hidden)]
103    pub fn note_parse(&mut self, bytes: usize) {
104        self.counters.parse_calls += 1;
105        self.counters.parse_bytes += bytes as u64;
106    }
107
108    /// Count one formula answered by a parse cache.
109    #[doc(hidden)]
110    pub fn note_parse_cache_hit(&mut self) {
111        self.counters.parse_cache_hits += 1;
112    }
113
114    /// Attach a lexical certificate to the family just noted at
115    /// `(row0, col0)` (the last formula staged).
116    pub(crate) fn attach_lexical(
117        &mut self,
118        row0: u32,
119        col0: u32,
120        lexical: Arc<super::source_family::LexicalTemplate>,
121    ) {
122        if let Some((row, family)) = self.by_col.get_mut(&col0)
123            && *row == row0
124        {
125            family.lexical = Some(Arc::clone(&lexical));
126        }
127        if let Some((row, col, family)) = self.last.as_mut()
128            && (*row, *col) == (row0, col0)
129        {
130            family.lexical = Some(lexical);
131        }
132    }
133
134    pub(crate) fn note(&mut self, row0: u32, col0: u32, family: GroupedFamily) {
135        self.by_col.insert(col0, (row0, family.clone()));
136        self.last = Some((row0, col0, family));
137    }
138}
139
140/// A family template candidate: the template's arena root, its 0-based
141/// anchor cell, and (computed on first use) per reference whether the
142/// template's text is its rendering; `Some(None)` when the template
143/// cannot have members (it needs a structural rewrite).
144#[derive(Clone, Debug)]
145pub(crate) struct GroupedFamily {
146    pub(crate) template: AstNodeId,
147    pub(crate) anchor: (u32, u32),
148    pub(crate) rendered: Option<Option<Arc<[bool]>>>,
149    /// The template's lexical certificate, when it has one: unparsed
150    /// copies of it can join the family.
151    pub(crate) lexical: Option<Arc<super::source_family::LexicalTemplate>>,
152}
153
154#[derive(Clone, Debug)]
155pub struct FormulaIngestBatch {
156    pub sheet_name: String,
157    pub formulas: Vec<FormulaIngestRecord>,
158}
159
160impl FormulaIngestBatch {
161    pub fn new(sheet_name: impl Into<String>, formulas: Vec<FormulaIngestRecord>) -> Self {
162        Self {
163            sheet_name: sheet_name.into(),
164            formulas,
165        }
166    }
167
168    pub fn len(&self) -> usize {
169        self.formulas.len()
170    }
171
172    pub fn is_empty(&self) -> bool {
173        self.formulas.is_empty()
174    }
175}
176
177#[derive(Debug, Clone, PartialEq, Eq)]
178pub struct FormulaIngestReport {
179    pub mode: FormulaPlaneMode,
180    pub formula_cells_seen: u64,
181    pub graph_formula_cells_materialized: u64,
182
183    pub shadow_candidate_cells: u64,
184    pub shadow_accepted_span_cells: u64,
185    pub shadow_fallback_cells: u64,
186    pub shadow_templates_interned: u64,
187    pub shadow_spans_created: u64,
188
189    pub graph_formula_vertices_avoided_shadow: u64,
190    pub ast_roots_avoided_shadow: u64,
191    pub edge_rows_avoided_shadow: u64,
192
193    pub graph_vertices_created: u64,
194    pub graph_edges_created: u64,
195
196    pub source_formula_events: u64,
197    pub source_formula_records_spooled: u64,
198    pub source_spool_encoded_bytes: u64,
199    pub source_spool_peak_memory_bytes: u64,
200    pub source_spool_spilled_bytes: u64,
201    pub source_spool_spill_files: u64,
202    pub source_spool_replays: u64,
203    pub source_ordinary_events: u64,
204    pub source_shared_anchor_events: u64,
205    pub source_shared_descendant_events: u64,
206    pub source_unknown_events: u64,
207    pub source_families_seen: u64,
208    pub source_family_cells_seen: u64,
209    pub source_family_shadow_eligible: u64,
210    pub source_family_shadow_eligible_cells: u64,
211    pub source_family_promoted: u64,
212    pub source_family_promoted_cells: u64,
213    pub source_family_fallback: u64,
214    pub source_family_fallback_cells: u64,
215    pub source_forward_descendants: u64,
216    pub source_evidence_limit_fallbacks: u64,
217    pub source_evidence_peak_bytes: u64,
218    pub source_anchor_parses: u64,
219    pub source_anchor_asts: u64,
220    pub source_anchor_analyses: u64,
221    pub source_descendant_strings_avoided: u64,
222    pub source_descendant_events_avoided: u64,
223    pub source_descendant_analyses_avoided: u64,
224    pub source_compressed_families_prepared: u64,
225    pub source_compressed_cells_prepared: u64,
226    pub source_partitioned_families_seen: u64,
227    pub source_partitioned_families_prepared: u64,
228    pub source_partitioned_families_rejected: u64,
229    pub source_partition_fragments_prepared: u64,
230    pub source_partition_span_cells_prepared: u64,
231    pub source_partition_fallback_cells: u64,
232    pub source_partition_analyses_reused: u64,
233    pub source_partition_function_semantics: u64,
234    pub source_partition_holes: u64,
235    pub source_partition_ordinary_exceptions: u64,
236    pub source_partition_failures: u64,
237    pub source_partition_surviving_cells: u64,
238
239    pub fallback_reasons: BTreeMap<String, u64>,
240}
241
242impl Default for FormulaIngestReport {
243    fn default() -> Self {
244        Self {
245            mode: FormulaPlaneMode::Off,
246            formula_cells_seen: 0,
247            graph_formula_cells_materialized: 0,
248            shadow_candidate_cells: 0,
249            shadow_accepted_span_cells: 0,
250            shadow_fallback_cells: 0,
251            shadow_templates_interned: 0,
252            shadow_spans_created: 0,
253            graph_formula_vertices_avoided_shadow: 0,
254            ast_roots_avoided_shadow: 0,
255            edge_rows_avoided_shadow: 0,
256            graph_vertices_created: 0,
257            graph_edges_created: 0,
258            source_formula_events: 0,
259            source_formula_records_spooled: 0,
260            source_spool_encoded_bytes: 0,
261            source_spool_peak_memory_bytes: 0,
262            source_spool_spilled_bytes: 0,
263            source_spool_spill_files: 0,
264            source_spool_replays: 0,
265            source_ordinary_events: 0,
266            source_shared_anchor_events: 0,
267            source_shared_descendant_events: 0,
268            source_unknown_events: 0,
269            source_families_seen: 0,
270            source_family_cells_seen: 0,
271            source_family_shadow_eligible: 0,
272            source_family_shadow_eligible_cells: 0,
273            source_family_promoted: 0,
274            source_family_promoted_cells: 0,
275            source_family_fallback: 0,
276            source_family_fallback_cells: 0,
277            source_forward_descendants: 0,
278            source_evidence_limit_fallbacks: 0,
279            source_evidence_peak_bytes: 0,
280            source_anchor_parses: 0,
281            source_anchor_asts: 0,
282            source_anchor_analyses: 0,
283            source_descendant_strings_avoided: 0,
284            source_descendant_events_avoided: 0,
285            source_descendant_analyses_avoided: 0,
286            source_compressed_families_prepared: 0,
287            source_compressed_cells_prepared: 0,
288            source_partitioned_families_seen: 0,
289            source_partitioned_families_prepared: 0,
290            source_partitioned_families_rejected: 0,
291            source_partition_fragments_prepared: 0,
292            source_partition_span_cells_prepared: 0,
293            source_partition_fallback_cells: 0,
294            source_partition_analyses_reused: 0,
295            source_partition_function_semantics: 0,
296            source_partition_holes: 0,
297            source_partition_ordinary_exceptions: 0,
298            source_partition_failures: 0,
299            source_partition_surviving_cells: 0,
300            fallback_reasons: BTreeMap::new(),
301        }
302    }
303}
304
305impl FormulaIngestReport {
306    pub(crate) fn with_mode(mode: FormulaPlaneMode) -> Self {
307        Self {
308            mode,
309            ..Self::default()
310        }
311    }
312
313    pub(crate) fn accumulate(&mut self, other: &Self) {
314        self.formula_cells_seen = self
315            .formula_cells_seen
316            .saturating_add(other.formula_cells_seen);
317        self.graph_formula_cells_materialized = self
318            .graph_formula_cells_materialized
319            .saturating_add(other.graph_formula_cells_materialized);
320        self.shadow_candidate_cells = self
321            .shadow_candidate_cells
322            .saturating_add(other.shadow_candidate_cells);
323        self.shadow_accepted_span_cells = self
324            .shadow_accepted_span_cells
325            .saturating_add(other.shadow_accepted_span_cells);
326        self.shadow_fallback_cells = self
327            .shadow_fallback_cells
328            .saturating_add(other.shadow_fallback_cells);
329        self.shadow_templates_interned = self
330            .shadow_templates_interned
331            .saturating_add(other.shadow_templates_interned);
332        self.shadow_spans_created = self
333            .shadow_spans_created
334            .saturating_add(other.shadow_spans_created);
335        self.graph_formula_vertices_avoided_shadow = self
336            .graph_formula_vertices_avoided_shadow
337            .saturating_add(other.graph_formula_vertices_avoided_shadow);
338        self.ast_roots_avoided_shadow = self
339            .ast_roots_avoided_shadow
340            .saturating_add(other.ast_roots_avoided_shadow);
341        self.edge_rows_avoided_shadow = self
342            .edge_rows_avoided_shadow
343            .saturating_add(other.edge_rows_avoided_shadow);
344        self.graph_vertices_created = self
345            .graph_vertices_created
346            .saturating_add(other.graph_vertices_created);
347        self.graph_edges_created = self
348            .graph_edges_created
349            .saturating_add(other.graph_edges_created);
350        self.source_formula_events = self
351            .source_formula_events
352            .saturating_add(other.source_formula_events);
353        self.source_formula_records_spooled = self
354            .source_formula_records_spooled
355            .saturating_add(other.source_formula_records_spooled);
356        self.source_spool_encoded_bytes = self
357            .source_spool_encoded_bytes
358            .saturating_add(other.source_spool_encoded_bytes);
359        self.source_spool_peak_memory_bytes = self
360            .source_spool_peak_memory_bytes
361            .max(other.source_spool_peak_memory_bytes);
362        self.source_spool_spilled_bytes = self
363            .source_spool_spilled_bytes
364            .saturating_add(other.source_spool_spilled_bytes);
365        self.source_spool_spill_files = self
366            .source_spool_spill_files
367            .saturating_add(other.source_spool_spill_files);
368        self.source_spool_replays = self
369            .source_spool_replays
370            .saturating_add(other.source_spool_replays);
371        self.source_ordinary_events = self
372            .source_ordinary_events
373            .saturating_add(other.source_ordinary_events);
374        self.source_shared_anchor_events = self
375            .source_shared_anchor_events
376            .saturating_add(other.source_shared_anchor_events);
377        self.source_shared_descendant_events = self
378            .source_shared_descendant_events
379            .saturating_add(other.source_shared_descendant_events);
380        self.source_unknown_events = self
381            .source_unknown_events
382            .saturating_add(other.source_unknown_events);
383        self.source_families_seen = self
384            .source_families_seen
385            .saturating_add(other.source_families_seen);
386        self.source_family_cells_seen = self
387            .source_family_cells_seen
388            .saturating_add(other.source_family_cells_seen);
389        self.source_family_shadow_eligible = self
390            .source_family_shadow_eligible
391            .saturating_add(other.source_family_shadow_eligible);
392        self.source_family_shadow_eligible_cells = self
393            .source_family_shadow_eligible_cells
394            .saturating_add(other.source_family_shadow_eligible_cells);
395        self.source_family_promoted = self
396            .source_family_promoted
397            .saturating_add(other.source_family_promoted);
398        self.source_family_promoted_cells = self
399            .source_family_promoted_cells
400            .saturating_add(other.source_family_promoted_cells);
401        self.source_family_fallback = self
402            .source_family_fallback
403            .saturating_add(other.source_family_fallback);
404        self.source_family_fallback_cells = self
405            .source_family_fallback_cells
406            .saturating_add(other.source_family_fallback_cells);
407        self.source_forward_descendants = self
408            .source_forward_descendants
409            .saturating_add(other.source_forward_descendants);
410        self.source_evidence_limit_fallbacks = self
411            .source_evidence_limit_fallbacks
412            .saturating_add(other.source_evidence_limit_fallbacks);
413        self.source_evidence_peak_bytes = self
414            .source_evidence_peak_bytes
415            .max(other.source_evidence_peak_bytes);
416        self.source_anchor_parses = self
417            .source_anchor_parses
418            .saturating_add(other.source_anchor_parses);
419        self.source_anchor_asts = self
420            .source_anchor_asts
421            .saturating_add(other.source_anchor_asts);
422        self.source_anchor_analyses = self
423            .source_anchor_analyses
424            .saturating_add(other.source_anchor_analyses);
425        self.source_descendant_strings_avoided = self
426            .source_descendant_strings_avoided
427            .saturating_add(other.source_descendant_strings_avoided);
428        self.source_descendant_events_avoided = self
429            .source_descendant_events_avoided
430            .saturating_add(other.source_descendant_events_avoided);
431        self.source_descendant_analyses_avoided = self
432            .source_descendant_analyses_avoided
433            .saturating_add(other.source_descendant_analyses_avoided);
434        self.source_compressed_families_prepared = self
435            .source_compressed_families_prepared
436            .saturating_add(other.source_compressed_families_prepared);
437        self.source_compressed_cells_prepared = self
438            .source_compressed_cells_prepared
439            .saturating_add(other.source_compressed_cells_prepared);
440        self.source_partitioned_families_seen = self
441            .source_partitioned_families_seen
442            .saturating_add(other.source_partitioned_families_seen);
443        self.source_partitioned_families_prepared = self
444            .source_partitioned_families_prepared
445            .saturating_add(other.source_partitioned_families_prepared);
446        self.source_partitioned_families_rejected = self
447            .source_partitioned_families_rejected
448            .saturating_add(other.source_partitioned_families_rejected);
449        self.source_partition_fragments_prepared = self
450            .source_partition_fragments_prepared
451            .saturating_add(other.source_partition_fragments_prepared);
452        self.source_partition_span_cells_prepared = self
453            .source_partition_span_cells_prepared
454            .saturating_add(other.source_partition_span_cells_prepared);
455        self.source_partition_fallback_cells = self
456            .source_partition_fallback_cells
457            .saturating_add(other.source_partition_fallback_cells);
458        self.source_partition_analyses_reused = self
459            .source_partition_analyses_reused
460            .saturating_add(other.source_partition_analyses_reused);
461        self.source_partition_function_semantics = self
462            .source_partition_function_semantics
463            .saturating_add(other.source_partition_function_semantics);
464        self.source_partition_holes = self
465            .source_partition_holes
466            .saturating_add(other.source_partition_holes);
467        self.source_partition_ordinary_exceptions = self
468            .source_partition_ordinary_exceptions
469            .saturating_add(other.source_partition_ordinary_exceptions);
470        self.source_partition_failures = self
471            .source_partition_failures
472            .saturating_add(other.source_partition_failures);
473        self.source_partition_surviving_cells = self
474            .source_partition_surviving_cells
475            .saturating_add(other.source_partition_surviving_cells);
476        for (reason, count) in &other.fallback_reasons {
477            let total = self.fallback_reasons.entry(reason.clone()).or_default();
478            *total = total.saturating_add(*count);
479        }
480    }
481}
formualizer_eval/engine/formula_ingest.rs

formualizer_eval/engine/
formula_ingest.rs
