1use crate::SheetId;
2
3mod criteria;
4mod criteria_mask_cache;
5#[cfg(test)]
6mod criteria_mask_cache_tests;
7mod exact;
8mod family;
9mod kernels;
10mod lift;
11mod memo;
12use crate::arrow_store::{OverlayFragment, OverlayValue, SheetStore};
13#[cfg(test)]
14use crate::engine::Scheduler;
15use crate::engine::arena::AstNodeId;
16use crate::engine::eval_delta::{
17 DeltaCollector, DeltaMode, EvalDelta, EvalDeltaCompatibilityPolicy,
18};
19use crate::engine::graph::editor::change_log::MutationCapture;
20use crate::engine::graph::prepared_legacy_graph::PreparedLegacyGraphPlan;
21use crate::engine::ingest_pipeline::{DependencyPlanRow, FormulaAstInput};
22use crate::engine::live_edges::{LiveEdgeCollector, RecordingContext};
23use crate::engine::live_graph::analyze_live_graph;
24use crate::engine::lookup_index_cache::{
25 BuildOutcome, LookupAxis, LookupIndex, LookupIndexCache, LookupIndexCacheReport,
26 LookupIndexKey, estimate_bytes,
27};
28use crate::engine::named_range::{NameScope, NamedDefinition};
29use crate::engine::range_view::RangeView;
30use crate::engine::row_visibility::RowVisibilityState;
31use crate::engine::spill::{RegionLockManager, SpillMeta, SpillShape};
32use crate::engine::target_preparation::{
33 StagedFormulaIndex, StagedFormulaLease, StagedPackageLease,
34};
35use crate::engine::used_extent::{
36 ExtentPolicy, OpenRangeBounds, resolve_used_extent_with_fallback,
37};
38use crate::engine::virtual_deps::VirtualDepBuilder;
39use family::{LayerUnit, layer_units, unit_members};
40
41#[path = "freshness.rs"]
42mod freshness;
43use crate::engine::template::region::Region;
44use crate::engine::{
45 ChangeLogger, CycleDetection, CyclePolicy, DependencyGraph, EvalConfig, EvaluationRequestKind,
46 EvaluationRequestOutcome, EvaluationResourceBaselineStats, EvaluationResourceRequestStats,
47 FormulaDirtyLeaseOutcome, FormulaIngestBatch, FormulaIngestRecord, FormulaIngestReport,
48 FormulaParseDiagnostic, FormulaParsePolicy, FormulaPlaneMode, ResourceLedger,
49 RowVisibilitySource, ScheduleUnit, VertexId, VertexKind, VisibilityMaskMode,
50};
51use crate::function::FnCaps;
52use crate::interpreter::Interpreter;
53
54#[cfg(test)]
55mod fn_snapshot_tests;
56use crate::reference::{CellRef, Coord, RangeRef};
57use crate::traits::FunctionProvider;
58use crate::traits::{EvaluationContext, ReferenceInfo, Resolver};
59use formualizer_common::{
60 CoordBuildHasher, LiteralValue, col_letters_from_1based, parse_a1_1based,
61};
62use formualizer_parse::parser::ReferenceType;
63use formualizer_parse::{ASTNode, ASTNodeType, ExcelError, ExcelErrorKind};
64use rayon::ThreadPoolBuilder;
65use rustc_hash::{FxHashMap, FxHashSet};
66use std::collections::{BTreeMap, BTreeSet, VecDeque};
67use std::sync::Arc;
68use std::sync::atomic::{AtomicBool, Ordering};
69use std::time::Instant;
70
71type StagedFormulaEntry = (u32, u32, String);
72type StagedSheetParts = (
73 Vec<StagedFormulaEntry>,
74 Option<crate::engine::DeferredFormulaPackage>,
75);
76
77#[derive(Default)]
87pub(crate) struct StagedSheet {
88 entries: Vec<StagedFormulaEntry>,
89 index: FxHashMap<(u32, u32), usize>,
90 deferred_package: Option<crate::engine::DeferredFormulaPackage>,
91}
92
93impl StagedSheet {
94 fn invalidate_all_deferred_families(package: &mut crate::engine::DeferredFormulaPackage) {
95 package
96 .invalidated
97 .extend(package.families.iter().map(|family| family.source_id));
98 package.invalidated.extend(
99 package
100 .partitioned_families
101 .iter()
102 .map(|family| family.source_id),
103 );
104 }
105
106 fn invalidate_deferred_at(&mut self, row: u32, col: u32) {
107 let Some(package) = self.deferred_package.as_mut() else {
108 return;
109 };
110 if package.suppressed.contains(&(row, col)) {
111 return;
114 }
115 if package.source_geometry_complete
116 && package.families.is_empty()
117 && package.partitioned_families.is_empty()
118 {
119 if let Some((row0, col0)) = row.checked_sub(1).zip(col.checked_sub(1))
120 && package
121 .source_coordinates
122 .binary_search(&crate::engine::SourceCoord {
123 row: row0,
124 col: col0,
125 })
126 .is_ok()
127 {
128 package.suppressed.insert((row, col));
129 }
130 return;
131 }
132 let family = package
133 .replay
134 .lock()
135 .ok()
136 .and_then(|mut replay| replay.formula_at(row, col).ok())
137 .flatten()
138 .and_then(|record| record.partition_owner.or(record.family));
139 if let Some(family) = family {
140 package.invalidated.insert(family);
141 } else {
142 Self::invalidate_all_deferred_families(package);
146 }
147 package.suppressed.insert((row, col));
148 }
149
150 fn reconcile_attached_deferred_package(&mut self) {
151 let Some(package) = self.deferred_package.as_mut() else {
152 return;
153 };
154 if self.entries.is_empty() {
155 return;
156 }
157
158 let replayed = (|| {
159 let mut disposition = crate::engine::FormulaReplayDisposition::default();
160 for partition in &package.partitioned_families {
161 disposition.register_partition(partition, false)?;
162 }
163 package
164 .replay
165 .lock()
166 .map_err(|_| "deferred formula spool lock poisoned".to_string())?
167 .replay_partitioned(&disposition, &package.partitioned_families)
168 })();
169
170 let Ok(records) = replayed else {
171 Self::invalidate_all_deferred_families(package);
172 package
173 .suppressed
174 .extend(self.entries.iter().map(|(row, col, _)| (*row, *col)));
175 return;
176 };
177 let mut owners = FxHashMap::default();
178 for record in &records {
179 owners
180 .entry((record.row, record.col))
181 .or_insert(record.partition_owner.or(record.family));
182 }
183 for (row, col, _) in &self.entries {
184 if let Some(family) = owners.get(&(*row, *col)).copied().flatten() {
185 package.invalidated.insert(family);
186 } else {
187 Self::invalidate_all_deferred_families(package);
188 }
189 package.suppressed.insert((*row, *col));
190 }
191 package.reconciliation_replay = Some(records);
192 }
193
194 fn stage(&mut self, row: u32, col: u32, text: String) {
195 self.invalidate_deferred_at(row, col);
196 match self.index.entry((row, col)) {
197 std::collections::hash_map::Entry::Occupied(slot) => {
198 self.entries[*slot.get()].2 = text;
199 }
200 std::collections::hash_map::Entry::Vacant(slot) => {
201 slot.insert(self.entries.len());
202 self.entries.push((row, col, text));
203 }
204 }
205 }
206
207 fn remove(&mut self, row: u32, col: u32) -> Option<String> {
208 let deferred_text = self.get(row, col);
209 self.invalidate_deferred_at(row, col);
210 let Some(idx) = self.index.remove(&(row, col)) else {
211 return deferred_text;
212 };
213 let (_, _, text) = self.entries.remove(idx);
214 for slot in self.index.values_mut() {
216 if *slot > idx {
217 *slot -= 1;
218 }
219 }
220 Some(text)
221 }
222
223 fn get_ordinary(&self, row: u32, col: u32) -> Option<&str> {
224 self.index
225 .get(&(row, col))
226 .map(|&i| self.entries[i].2.as_str())
227 }
228
229 fn remove_ordinary(&mut self, row: u32, col: u32) -> Option<String> {
230 let idx = self.index.remove(&(row, col))?;
231 let (_, _, text) = self.entries.remove(idx);
232 for slot in self.index.values_mut() {
233 if *slot > idx {
234 *slot -= 1;
235 }
236 }
237 Some(text)
238 }
239
240 fn get(&self, row: u32, col: u32) -> Option<String> {
241 if let Some(text) = self.get_ordinary(row, col) {
242 return Some(text.to_string());
243 }
244 let package = self.deferred_package.as_ref()?;
245 if package.suppressed.contains(&(row, col)) {
246 return None;
247 }
248 package
249 .replay
250 .lock()
251 .ok()?
252 .formula_at(row, col)
253 .ok()?
254 .map(|record| record.text)
255 }
256
257 fn len(&self) -> usize {
258 self.deferred_package
259 .as_ref()
260 .map_or(self.entries.len(), |package| {
261 (if package.source_geometry_complete
262 && (package.coordinates_cover_families
263 || (package.families.is_empty() && package.partitioned_families.is_empty()))
264 {
265 package.source_coordinates.len()
266 } else {
267 usize::try_from(package.report.source_formula_records_spooled)
268 .unwrap_or(usize::MAX)
269 })
270 .saturating_sub(package.suppressed.len())
271 .saturating_add(self.entries.len())
272 })
273 }
274
275 fn is_empty(&self) -> bool {
276 self.entries.is_empty() && self.deferred_package.is_none()
277 }
278
279 fn into_parts(self) -> StagedSheetParts {
280 (self.entries, self.deferred_package)
281 }
282}
283
284type StagedFormulaMap = std::collections::HashMap<String, StagedSheet>;
285
286#[derive(Clone, Debug, PartialEq, Eq, Hash)]
287struct PreparationRegion {
288 sheet: String,
289 sheet_id: SheetId,
290 start_row: u32,
291 start_col: u32,
292 end_row: u32,
293 end_col: u32,
294}
295
296struct OrderedTargetProducers {
298 ordered: Vec<crate::engine::target_preparation::TargetProducer>,
299 seen: FxHashSet<crate::engine::target_preparation::TargetProducer>,
300}
301
302impl OrderedTargetProducers {
303 fn with_capacity(capacity: usize) -> Result<Self, std::collections::TryReserveError> {
304 let mut ordered = Vec::new();
305 ordered.try_reserve(capacity)?;
306 let mut seen = FxHashSet::default();
307 seen.try_reserve(capacity)?;
308 Ok(Self { ordered, seen })
309 }
310
311 fn from_ordered(
312 ordered: Vec<crate::engine::target_preparation::TargetProducer>,
313 ) -> Result<Self, std::collections::TryReserveError> {
314 let mut seen = FxHashSet::default();
315 seen.try_reserve(ordered.len())?;
316 seen.extend(ordered.iter().copied());
317 Ok(Self { ordered, seen })
318 }
319
320 fn push(
321 &mut self,
322 producer: crate::engine::target_preparation::TargetProducer,
323 ) -> Result<bool, std::collections::TryReserveError> {
324 #[cfg(test)]
325 TARGET_ROOT_DEDUP_PROBES.with(|probes| probes.set(probes.get().saturating_add(1)));
326 if self.seen.contains(&producer) {
327 return Ok(false);
328 }
329 self.seen.try_reserve(1)?;
330 self.ordered.try_reserve(1)?;
331 self.seen.insert(producer);
332 self.ordered.push(producer);
333 Ok(true)
334 }
335
336 fn len(&self) -> usize {
337 self.ordered.len()
338 }
339
340 fn into_vec(self) -> Vec<crate::engine::target_preparation::TargetProducer> {
341 self.ordered
342 }
343}
344
345#[cfg(test)]
346thread_local! {
347 static TARGET_ROOT_DEDUP_PROBES: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
348}
349
350fn target_root_allocation_error(observed: usize, request_id: Option<u64>) -> ExcelError {
351 crate::engine::ResourceLedgerError::Exhausted(formualizer_common::ResourceExhaustionDetail {
352 reason: formualizer_common::ResourceExhaustionReason::ScratchMemory,
353 limit: u64::MAX,
354 observed: observed as u64,
355 request_id,
356 })
357 .into_excel_error()
358}
359
360#[derive(Debug)]
361struct PreparedOrdinaryStagedFormula {
362 sheet: String,
363 sheet_id: SheetId,
364 lease: StagedFormulaLease,
365 ast_id: Option<AstNodeId>,
366 plan: Option<DependencyPlanRow>,
367}
368
369struct PreparedTargetSourcePackage {
370 sheet: String,
371 sheet_id: SheetId,
372 lease: StagedPackageLease,
373 selected_points: Option<BTreeSet<(u32, u32)>>,
374 complete_selections: BTreeSet<crate::engine::SourceFamilyId>,
375 deferred_shared: bool,
376 direct_domains: Vec<(
377 crate::engine::SourceFamilyId,
378 crate::engine::PlacementDomainTransport,
379 )>,
380 source_report: crate::engine::FormulaCompressedSourceReport,
381 replay_records: Vec<crate::engine::DeferredReplayFormula>,
382 spool_replays: u64,
383 disposition: crate::engine::FormulaReplayDisposition,
384 legacy: Vec<(u32, u32, AstNodeId, DependencyPlanRow)>,
385 direct_families: usize,
386 direct_cells: u64,
387 direct_fragments: u64,
388 direct_complete_families: u64,
389 direct_complete_cells: u64,
390 direct_partition_families: u64,
391 direct_partition_cells: u64,
392 anchor_parses: u64,
393 anchor_asts: u64,
394 anchor_analyses: u64,
395}
396
397impl PreparedTargetSourcePackage {
398 fn empty_selection(sheet: &str, sheet_id: SheetId, lease: StagedPackageLease) -> Self {
399 Self {
400 sheet: sheet.to_owned(),
401 sheet_id,
402 lease,
403 selected_points: Some(BTreeSet::new()),
404 complete_selections: Default::default(),
405 deferred_shared: false,
406 direct_domains: Vec::new(),
407 source_report: Default::default(),
408 replay_records: Vec::new(),
409 spool_replays: 0,
410 disposition: Default::default(),
411 legacy: Vec::new(),
412 direct_families: 0,
413 direct_cells: 0,
414 direct_fragments: 0,
415 direct_complete_families: 0,
416 direct_complete_cells: 0,
417 direct_partition_families: 0,
418 direct_partition_cells: 0,
419 anchor_parses: 0,
420 anchor_asts: 0,
421 anchor_analyses: 0,
422 }
423 }
424
425 fn direct_contains(&self, row: u32, col: u32) -> bool {
426 self.direct_domains.iter().any(|(_, domain)| {
427 let rect = domain.rect();
428 row > rect.start.row
429 && row <= rect.end.row + 1
430 && col > rect.start.col
431 && col <= rect.end.col + 1
432 })
433 }
434
435 fn fallback_records(&self) -> impl Iterator<Item = &crate::engine::DeferredReplayFormula> {
436 self.replay_records.iter().filter(|record| {
437 let Some((row, col)) = record.row.checked_sub(1).zip(record.col.checked_sub(1)) else {
438 return true;
439 };
440 let coord = crate::engine::SourceCoord { row, col };
441 let disposition = match record.family {
442 Some(family) => self.disposition.shared_disposition(family, coord),
443 None => self.disposition.ordinary_disposition(coord).0,
444 };
445 !matches!(
446 disposition,
447 crate::engine::FormulaReplayCoordinateDisposition::Direct
448 | crate::engine::FormulaReplayCoordinateDisposition::Suppressed
449 )
450 })
451 }
452}
453
454type PreparedFormulaBatches = Vec<FormulaIngestBatch>;
455type StagedFormulaBatches = Vec<(String, StagedSheet)>;
456
457type CompressedReplayBatch = (
458 FormulaIngestBatch,
459 crate::engine::FormulaCompressedSourceBatch,
460);
461type PreparedSourceBatch = (
462 FormulaIngestBatch,
463 crate::engine::FormulaCompressedSourceReport,
464 crate::engine::FormulaCompressedPreparation,
465);
466type PreparedStagedFormulaBatches = (
467 PreparedFormulaBatches,
468 Vec<CompressedReplayBatch>,
469 Vec<PreparedSourceBatch>,
470 FxHashSet<crate::engine::arena::AstNodeId>,
472);
473
474#[doc(hidden)]
477pub struct SourceFormulaIngress<'a, R> {
478 engine: &'a mut Engine<R>,
479}
480
481impl<R> SourceFormulaIngress<'_, R>
482where
483 R: EvaluationContext,
484{
485 pub fn prepare_families(
486 &mut self,
487 sheet_name: &str,
488 families: &[crate::engine::SourceFormulaFamily],
489 ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
490 self.engine.observe_function_semantic_epoch()?;
491 Ok(self
492 .engine
493 .prepare_source_formula_families(sheet_name, families))
494 }
495
496 pub fn prepare_eager_proposals(
497 &mut self,
498 sheet_name: &str,
499 families: &[crate::engine::SourceFormulaFamily],
500 partitions: &[crate::engine::PartitionedSourceFormulaFamily],
501 formula_record_count: u64,
502 replay: Box<dyn crate::engine::DeferredFormulaReplay>,
503 ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
504 self.engine.observe_function_semantic_epoch()?;
505 let replay = Arc::new(std::sync::Mutex::new(replay));
506 self.engine.prepare_source_formula_proposals(
507 sheet_name,
508 families,
509 partitions,
510 partitions,
511 formula_record_count,
512 replay,
513 &BTreeSet::new(),
514 &Default::default(),
515 None,
516 )
517 }
518
519 pub fn stage_deferred(&mut self, package: crate::engine::DeferredFormulaPackage) {
520 self.engine.stage_deferred_formula_package(package);
521 }
522
523 pub fn ingest_replay_batches(
524 &mut self,
525 batches: Vec<(
526 FormulaIngestBatch,
527 crate::engine::FormulaCompressedSourceBatch,
528 )>,
529 ) -> Result<FormulaIngestReport, ExcelError> {
530 self.engine
531 .ingest_compressed_formula_source_batches(batches)
532 }
533
534 pub fn finish_prepared(
535 &mut self,
536 batches: Vec<(
537 FormulaIngestBatch,
538 crate::engine::FormulaCompressedSourceReport,
539 crate::engine::FormulaCompressedPreparation,
540 )>,
541 ) -> Result<FormulaIngestReport, ExcelError> {
542 self.engine.finish_compressed_formula_sources(batches)
543 }
544}
545
546const COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH: usize = 8;
550
551#[cfg(debug_assertions)]
553fn same_value_bits(a: &LiteralValue, b: &LiteralValue) -> bool {
554 match (a, b) {
555 (LiteralValue::Number(x), LiteralValue::Number(y)) => x.to_bits() == y.to_bits(),
556 _ => a == b,
557 }
558}
559
560fn buffer_layer_writes(layer: &crate::engine::scheduler::Layer) -> bool {
563 !layer.sequential && layer.vertices.len() >= COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH
564}
565const PARALLEL_SCHEDULE_MIN_CANDIDATES: usize = 16 * 1024;
573const BASE_SCHEDULE_MIN_VERTICES: usize = 64;
577const BASE_SCHEDULE_RATIO: usize = 256;
578const BASE_SCHEDULE_MIN_REQUEST: usize = 128;
583const PARALLEL_LAYER_PROBE: std::time::Duration = std::time::Duration::from_micros(300);
584const PARALLEL_LAYER_WORTH: std::time::Duration = std::time::Duration::from_micros(150);
585const EXPENSIVE_VERTEX_NS: u128 = 10_000;
587const UNIT_CANCELLED: &str = "Evaluation cancelled; the evaluated unit was not committed";
589const GROUP_CANCELLED: &str =
590 "Parallel evaluation cancelled; the evaluated group was not committed";
591const SCC_CANCELLED: &str = "Evaluation cancelled; the evaluated cycle was not completed";
592
593#[derive(Debug, Clone, PartialEq)]
594pub(crate) enum ComputedWrite {
595 Cell {
596 seq: u64,
597 sheet_id: SheetId,
598 row0: u32,
599 col0: u32,
600 value: OverlayValue,
601 format_id: Option<crate::format::FormatId>,
602 },
603 Rect {
604 seq: u64,
605 sheet_id: SheetId,
606 sr0: u32,
607 sc0: u32,
608 values: Vec<Vec<OverlayValue>>,
609 },
610 Run {
613 seq: u64,
614 sheet_id: SheetId,
615 row0: u32,
616 col0: u32,
617 entries: Vec<(OverlayValue, Option<crate::format::FormatId>)>,
618 },
619}
620
621impl ComputedWrite {
622 #[inline]
623 pub(crate) fn seq(&self) -> u64 {
624 match self {
625 ComputedWrite::Cell { seq, .. }
626 | ComputedWrite::Rect { seq, .. }
627 | ComputedWrite::Run { seq, .. } => *seq,
628 }
629 }
630}
631
632#[derive(Debug, Default)]
633pub(crate) struct ComputedWriteBuffer {
634 writes: Vec<ComputedWrite>,
635 next_seq: u64,
636 estimated_bytes: usize,
637 formats_present: bool,
638}
639
640impl ComputedWriteBuffer {
641 const ENTRY_BASE_BYTES: usize = 32;
642
643 #[inline]
644 pub(crate) fn is_empty(&self) -> bool {
645 self.writes.is_empty()
646 }
647
648 #[inline]
649 pub(crate) fn len(&self) -> usize {
650 self.writes.len()
651 }
652
653 #[inline]
654 pub(crate) fn writes(&self) -> &[ComputedWrite] {
655 &self.writes
656 }
657
658 #[inline]
659 pub(crate) fn estimated_bytes(&self) -> usize {
660 self.estimated_bytes
661 }
662
663 #[cfg(feature = "tracing")]
664 fn traced_cell_count(&self) -> usize {
665 self.writes
666 .iter()
667 .map(|write| match write {
668 ComputedWrite::Cell { .. } => 1,
669 ComputedWrite::Rect { values, .. } => values.iter().map(Vec::len).sum(),
670 ComputedWrite::Run { entries, .. } => entries.len(),
671 })
672 .sum()
673 }
674
675 pub(crate) fn push_cell(
676 &mut self,
677 sheet_id: SheetId,
678 row0: u32,
679 col0: u32,
680 value: OverlayValue,
681 ) {
682 self.push_cell_with_format(sheet_id, row0, col0, value, None);
683 }
684
685 pub(crate) fn push_cell_with_format(
686 &mut self,
687 sheet_id: SheetId,
688 row0: u32,
689 col0: u32,
690 value: OverlayValue,
691 format_id: Option<crate::format::FormatId>,
692 ) {
693 let format_id = format_id.filter(|id| *id != crate::format::FormatId::GENERAL);
694 self.formats_present |= format_id.is_some();
695 let seq = self.next_sequence();
696 self.estimated_bytes = self
697 .estimated_bytes
698 .saturating_add(Self::estimate_value_bytes(&value));
699 self.writes.push(ComputedWrite::Cell {
700 seq,
701 sheet_id,
702 row0,
703 col0,
704 value,
705 format_id,
706 });
707 }
708
709 pub(crate) fn push_column_run(
710 &mut self,
711 sheet_id: SheetId,
712 row0: u32,
713 col0: u32,
714 mut entries: Vec<(OverlayValue, Option<crate::format::FormatId>)>,
715 ) {
716 let seq = self.next_sequence();
717 let mut added = 0usize;
718 for (value, format_id) in entries.iter_mut() {
719 *format_id = format_id.filter(|id| *id != crate::format::FormatId::GENERAL);
720 self.formats_present |= format_id.is_some();
721 added = added.saturating_add(Self::estimate_value_bytes(value));
722 }
723 self.estimated_bytes = self.estimated_bytes.saturating_add(added);
724 self.writes.push(ComputedWrite::Run {
725 seq,
726 sheet_id,
727 row0,
728 col0,
729 entries,
730 });
731 }
732
733 pub(crate) fn push_rect(
734 &mut self,
735 sheet_id: SheetId,
736 sr0: u32,
737 sc0: u32,
738 values: Vec<Vec<OverlayValue>>,
739 ) {
740 let seq = self.next_sequence();
741 let added = values
742 .iter()
743 .flat_map(|row| row.iter())
744 .map(Self::estimate_value_bytes)
745 .fold(0usize, usize::saturating_add);
746 self.estimated_bytes = self.estimated_bytes.saturating_add(added);
747 self.writes.push(ComputedWrite::Rect {
748 seq,
749 sheet_id,
750 sr0,
751 sc0,
752 values,
753 });
754 }
755
756 pub(crate) fn clear(&mut self) {
757 self.writes.clear();
758 self.estimated_bytes = 0;
759 self.formats_present = false;
760 }
761
762 fn take_writes(&mut self) -> (Vec<ComputedWrite>, bool) {
763 self.estimated_bytes = 0;
764 let formats_present = std::mem::take(&mut self.formats_present);
765 (std::mem::take(&mut self.writes), formats_present)
766 }
767
768 fn next_sequence(&mut self) -> u64 {
769 let seq = self.next_seq;
770 self.next_seq = self.next_seq.wrapping_add(1);
771 seq
772 }
773
774 #[inline]
775 fn estimate_value_bytes(value: &OverlayValue) -> usize {
776 Self::ENTRY_BASE_BYTES.saturating_add(value.estimated_payload_bytes())
777 }
778}
779
780#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
781struct ComputedWriteChunkKey {
782 sheet_id: SheetId,
783 col0: u32,
784 chunk_idx: usize,
785 chunk_start_row0: u32,
786}
787
788#[derive(Debug, Clone, PartialEq)]
789pub(crate) struct ComputedWriteChunkEntryPlan {
790 pub(crate) row_in_chunk: usize,
791 pub(crate) seq: u64,
792 pub(crate) value: OverlayValue,
793 pub(crate) format_id: Option<crate::format::FormatId>,
794}
795
796#[derive(Debug, Clone, PartialEq, Eq)]
797pub(crate) enum ComputedWriteChunkPlanShape {
798 Point,
799 SparseOffsets {
800 entries: usize,
801 span_len: usize,
802 },
803 DenseRange {
804 start: usize,
805 len: usize,
806 },
807 RunRange {
808 start: usize,
809 len: usize,
810 runs: usize,
811 },
812}
813
814#[derive(Debug, Clone, PartialEq, Eq)]
815pub(crate) enum ComputedWriteFormatClear {
816 Range { start: usize, end: usize },
817 Offsets(Vec<usize>),
818}
819
820#[derive(Debug, Clone, PartialEq)]
821pub(crate) enum ComputedWriteChunkFormatEffect {
822 NoFormatWork,
823 ClearStale(ComputedWriteFormatClear),
824 SetExplicit(Vec<(usize, Option<crate::format::FormatId>)>),
825}
826
827#[derive(Debug, Clone, PartialEq)]
828pub(crate) struct ComputedWriteChunkPlan {
829 pub(crate) sheet_id: SheetId,
830 pub(crate) col0: u32,
831 pub(crate) chunk_idx: usize,
832 pub(crate) chunk_start_row0: u32,
833 pub(crate) entries: Vec<ComputedWriteChunkEntryPlan>,
834 pub(crate) shape: ComputedWriteChunkPlanShape,
835 pub(crate) format_effect: ComputedWriteChunkFormatEffect,
836}
837
838#[derive(Debug, Clone, Default, PartialEq)]
839pub(crate) struct ComputedWriteCoalescingPlan {
840 pub(crate) chunks: Vec<ComputedWriteChunkPlan>,
841 pub(crate) input_cells: usize,
842 pub(crate) coalesced_cells: usize,
843 pub(crate) overwritten_cells: usize,
844}
845
846impl ComputedWriteCoalescingPlan {
847 #[inline]
848 pub(crate) fn is_empty(&self) -> bool {
849 self.chunks.is_empty()
850 }
851}
852
853impl ComputedWriteChunkPlan {
854 fn from_group(
855 key: ComputedWriteChunkKey,
856 mut entries: Vec<ComputedWriteChunkEntryPlan>,
857 formats_present: bool,
858 computed_lane_has_formats: bool,
859 ) -> (Self, usize) {
860 entries.sort_by_key(|entry| (entry.row_in_chunk, entry.seq));
861 let input_len = entries.len();
862 let mut coalesced: Vec<ComputedWriteChunkEntryPlan> = Vec::with_capacity(input_len);
863 for entry in entries {
864 if let Some(prev) = coalesced.last_mut()
865 && prev.row_in_chunk == entry.row_in_chunk
866 {
867 *prev = entry;
868 continue;
869 }
870 coalesced.push(entry);
871 }
872 let overwritten = input_len.saturating_sub(coalesced.len());
873 let shape = Self::classify_shape(&coalesced);
874 let format_effect = Self::classify_format_effect(
875 &coalesced,
876 &shape,
877 formats_present,
878 computed_lane_has_formats,
879 );
880 (
881 Self {
882 sheet_id: key.sheet_id,
883 col0: key.col0,
884 chunk_idx: key.chunk_idx,
885 chunk_start_row0: key.chunk_start_row0,
886 entries: coalesced,
887 shape,
888 format_effect,
889 },
890 overwritten,
891 )
892 }
893
894 fn classify_format_effect(
895 entries: &[ComputedWriteChunkEntryPlan],
896 shape: &ComputedWriteChunkPlanShape,
897 formats_present: bool,
898 computed_lane_has_formats: bool,
899 ) -> ComputedWriteChunkFormatEffect {
900 if !formats_present {
901 return if computed_lane_has_formats {
902 ComputedWriteChunkFormatEffect::ClearStale(Self::format_clear_spec(entries, shape))
903 } else {
904 ComputedWriteChunkFormatEffect::NoFormatWork
905 };
906 }
907
908 if entries.iter().all(|entry| entry.format_id.is_none()) {
909 return if computed_lane_has_formats {
910 ComputedWriteChunkFormatEffect::ClearStale(Self::format_clear_spec(entries, shape))
911 } else {
912 ComputedWriteChunkFormatEffect::NoFormatWork
913 };
914 }
915
916 ComputedWriteChunkFormatEffect::SetExplicit(
917 entries
918 .iter()
919 .map(|entry| (entry.row_in_chunk, entry.format_id))
920 .collect(),
921 )
922 }
923
924 fn format_clear_spec(
925 entries: &[ComputedWriteChunkEntryPlan],
926 shape: &ComputedWriteChunkPlanShape,
927 ) -> ComputedWriteFormatClear {
928 match shape {
929 ComputedWriteChunkPlanShape::DenseRange { start, len }
930 | ComputedWriteChunkPlanShape::RunRange { start, len, .. } => {
931 ComputedWriteFormatClear::Range {
932 start: *start,
933 end: start.saturating_add(*len),
934 }
935 }
936 ComputedWriteChunkPlanShape::Point
937 | ComputedWriteChunkPlanShape::SparseOffsets { .. } => {
938 ComputedWriteFormatClear::Offsets(
939 entries.iter().map(|entry| entry.row_in_chunk).collect(),
940 )
941 }
942 }
943 }
944
945 fn classify_shape(entries: &[ComputedWriteChunkEntryPlan]) -> ComputedWriteChunkPlanShape {
946 debug_assert!(!entries.is_empty());
947 if entries.len() == 1 {
948 return ComputedWriteChunkPlanShape::Point;
949 }
950
951 let start = entries[0].row_in_chunk;
952 let end = entries[entries.len() - 1].row_in_chunk;
953 let span_len = end.saturating_sub(start).saturating_add(1);
954 if span_len != entries.len() {
955 return ComputedWriteChunkPlanShape::SparseOffsets {
956 entries: entries.len(),
957 span_len,
958 };
959 }
960
961 let runs = Self::run_count(entries);
962 if runs < entries.len() {
963 ComputedWriteChunkPlanShape::RunRange {
964 start,
965 len: entries.len(),
966 runs,
967 }
968 } else {
969 ComputedWriteChunkPlanShape::DenseRange {
970 start,
971 len: entries.len(),
972 }
973 }
974 }
975
976 fn run_count(entries: &[ComputedWriteChunkEntryPlan]) -> usize {
977 let mut runs = 0usize;
978 let mut prev: Option<&OverlayValue> = None;
979 for entry in entries {
980 if prev != Some(&entry.value) {
981 runs = runs.saturating_add(1);
982 prev = Some(&entry.value);
983 }
984 }
985 runs
986 }
987}
988
989#[cfg(feature = "tracing")]
990#[derive(Default)]
991struct TraceEvaluationCounters {
992 computed_vertices: usize,
993 cycles: usize,
994}
995
996pub struct Engine<R> {
997 criteria_mask_cache: criteria_mask_cache::CriteriaMaskCache,
998 pub(crate) graph: DependencyGraph,
999 resolver: R,
1000 pub(super) resolved_functions: crate::interpreter::ResolvedFunctions,
1001 resolved_sheet_names:
1002 FxHashMap<crate::engine::arena::StringId, (crate::SheetId, Option<usize>)>,
1003 resolution_sheet_revision: Option<(u64, u64, usize)>,
1004 resolution_arrow_sheet_names: Vec<Arc<str>>,
1005 resolved_arrow_sheets: Vec<Option<usize>>,
1006 arrow_sheet_index: std::sync::OnceLock<Vec<Option<usize>>>,
1008 pub config: EvalConfig,
1009 workbook_load_limits: crate::engine::WorkbookLoadLimits,
1010 clock: crate::timezone::SnapshotClock,
1015 thread_pool: Option<Arc<rayon::ThreadPool>>,
1016 pub recalc_epoch: u64,
1017 snapshot_id: std::sync::atomic::AtomicU64,
1018 topology_epoch: u64,
1019 cached_static_schedule: Option<CachedScheduleEntry>,
1020 recent_schedules: Vec<CachedScheduleEntry>,
1025 base_schedule: Option<CachedScheduleEntry>,
1030 #[cfg(any(test, feature = "benchmark_internal"))]
1031 recalc_reuse_probe: std::sync::Mutex<RecalcReuseProbe>,
1032 spill_mgr: ShimSpillManager,
1033 arrow_sheets: SheetStore,
1035 format_registry: crate::format::FormatRegistry,
1037 derived_formats: crate::engine::derived_formats::DerivedFormats,
1039 #[cfg(test)]
1040 derived_format_operations_for_test: std::sync::atomic::AtomicU64,
1041 #[cfg(test)]
1042 family_members_for_test: std::sync::atomic::AtomicU64,
1043 #[cfg(test)]
1044 invariant_bound_members_for_test: std::sync::atomic::AtomicU64,
1045 #[cfg(test)]
1046 lifted_members_for_test: std::sync::atomic::AtomicU64,
1047 #[cfg(test)]
1048 chained_members_for_test: std::sync::atomic::AtomicU64,
1049 #[cfg(test)]
1050 lane_clean_reads_for_test: std::sync::atomic::AtomicU64,
1051 #[cfg(test)]
1052 criteria_kernel_members_for_test: std::sync::atomic::AtomicU64,
1053 #[cfg(test)]
1054 memo_hits_for_test: std::sync::atomic::AtomicU64,
1055 compressed_at_build: Option<u64>,
1057 #[cfg(test)]
1058 computed_overlay_set_explicit_entry_operations_for_test: u64,
1059 #[cfg(test)]
1060 computed_overlay_stale_clear_range_effects_for_test: u64,
1061 #[cfg(test)]
1062 computed_overlay_stale_clear_offset_attempts_for_test: u64,
1063 #[cfg(test)]
1064 computed_format_vector_allocations_for_test: std::sync::atomic::AtomicU64,
1065 has_edited: bool,
1067 overlay_compactions: u64,
1069
1070 computed_overlay_bytes_estimate: usize,
1072 computed_overlay_mirroring_disabled: bool,
1073 pub(crate) force_materialize_range_views: bool,
1076 row_bounds_cache: std::sync::RwLock<Option<RowBoundsCache>>,
1078 used_axis_bounds_cache: std::sync::RwLock<Option<UsedAxisBoundsCache>>,
1080 lookup_index_cache: LookupIndexCache,
1081 source_cache: Arc<std::sync::RwLock<SourceCache>>,
1082 source_formula_token: Arc<()>,
1084 pub(crate) source_family_mode: crate::engine::SourceFamilyMode,
1086 pub(crate) source_family_counters: crate::engine::SourceFamilyCounters,
1087 recalc_plan_token: Arc<()>,
1089 staged_formulas: StagedFormulaMap,
1091 staged_formula_index: StagedFormulaIndex,
1093 blocked_pending_spills: Vec<(VertexId, CellRef, Region)>,
1095 spill_blocker_waiters: SpillBlockerWaiters,
1099 row_visibility: FxHashMap<SheetId, RowVisibilityState>,
1101 row_visibility_mask_cache: std::sync::RwLock<
1103 FxHashMap<VisibilityMaskCacheKey, std::sync::Arc<arrow_array::BooleanArray>>,
1104 >,
1105 formula_parse_diagnostics: Vec<FormulaParseDiagnostic>,
1107 last_formula_ingest_report: Option<FormulaIngestReport>,
1109 formula_ingest_report_total: FormulaIngestReport,
1111 active_cancel_flag: Option<crate::engine::CancelToken>,
1113 active_evaluation_deadline: Option<Instant>,
1115
1116 action_depth: u32,
1121
1122 last_virtual_dep_telemetry: VirtualDepTelemetry,
1124 virtual_dep_fallback_activations: u64,
1125
1126 last_cycle_telemetry: CycleTelemetry,
1128
1129 next_evaluation_resource_request_id: u64,
1131 evaluation_resource_request_depth: usize,
1132 active_evaluation_resource_request: Option<EvaluationResourceRequestStats>,
1133 last_evaluation_resource_request: Option<EvaluationResourceRequestStats>,
1134 evaluation_resource_baseline: EvaluationResourceBaselineStats,
1135 evaluation_resource_request_started_at: Option<crate::instant::FzInstant>,
1136 evaluation_resource_budgets: crate::engine::EvaluationBudgets,
1137 evaluation_resource_config_diagnostic:
1138 Option<crate::engine::EvaluationResourceConfigDiagnostic>,
1139 active_resource_ledger: Option<ResourceLedger>,
1140 source_cache_footprints: Vec<std::sync::Weak<std::sync::atomic::AtomicU64>>,
1141 source_cache_accounted: u64,
1142
1143 pending_iterative_redirty: Vec<VertexId>,
1162 snapshot_evaluation_active: bool,
1164 snapshot_volatile_redirty_pending: bool,
1165 retained_scc_members: FxHashMap<VertexId, u64>,
1183 next_retained_scc_id: u64,
1184 retained_scc_config_fingerprint: u64,
1191 retained_scc_function_epoch_seen: u64,
1196 retained_scc_provider_revision_seen: Option<u64>,
1197 retained_scc_dirty_at_begin: Vec<(VertexId, u64)>,
1204
1205 iterative_state_values: FxHashMap<VertexId, LiteralValue>,
1218
1219 function_semantic_epoch_seen: u64,
1221 function_provider_revision_seen: Option<u64>,
1223
1224 #[cfg(feature = "tracing")]
1225 trace_evaluation_counters: TraceEvaluationCounters,
1226 #[cfg(test)]
1227 evaluation_request_begin_count_for_test: u64,
1228 #[cfg(any(test, feature = "test-support"))]
1229 before_prepared_span_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1230 #[cfg(test)]
1231 before_target_preparation_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1232 #[cfg(test)]
1233 before_target_planning_snapshot_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1234 #[cfg(test)]
1235 inject_target_semantic_stale_once_for_test: bool,
1236 #[cfg(test)]
1237 force_virtual_dep_changes_remaining_for_test: usize,
1238 freshness: freshness::Freshness,
1240 #[cfg(test)]
1241 fail_evaluation_commit_preflight_once_for_test: bool,
1242 #[cfg(test)]
1243 target_preparation_fault_for_test:
1244 Option<crate::engine::target_preparation::TargetPreparationFault>,
1245 #[cfg(test)]
1246 force_non_cycle_schedule_fallback_for_test: bool,
1247 #[cfg(test)]
1248 before_legacy_fallback_final_provider_sample_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1249 #[cfg(test)]
1250 after_eager_proposal_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1251}
1252
1253impl<R: EvaluationContext> Engine<R> {
1256 pub(crate) fn ingest_pipeline(&mut self) -> crate::engine::ingest_pipeline::IngestPipeline<'_> {
1257 self.graph.ingest_pipeline(&self.resolver)
1258 }
1259}
1260
1261pub struct EngineAction<'a, R>
1262where
1263 R: EvaluationContext,
1264{
1265 engine: &'a mut Engine<R>,
1266 name: String,
1267 capture: Option<*mut MutationCapture>,
1270 arrow_undo: Option<*mut crate::engine::ArrowUndoBatch>,
1273 atomic_policy: bool,
1275}
1276
1277impl<'a, R> EngineAction<'a, R>
1278where
1279 R: EvaluationContext,
1280{
1281 #[inline]
1282 fn addr_for(&mut self, sheet: &str, row: u32, col: u32) -> crate::reference::CellRef {
1283 let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1284 let coord = crate::reference::Coord::from_excel(row, col, true, true);
1285 crate::reference::CellRef::new(sheet_id, coord)
1286 }
1287
1288 #[inline]
1289 pub fn name(&self) -> &str {
1290 &self.name
1291 }
1292
1293 #[inline]
1294 pub fn set_cell_value(
1295 &mut self,
1296 sheet: &str,
1297 row: u32,
1298 col: u32,
1299 value: LiteralValue,
1300 ) -> Result<(), crate::engine::EditorError> {
1301 if self.capture.is_some() {
1302 let old_value = self.engine.read_cell_value(sheet, row, col);
1303 let mut old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1304 let addr = self.addr_for(sheet, row, col);
1305 let Some(capture_ptr) = self.capture else {
1306 return Err(crate::engine::EditorError::TransactionFailed {
1307 reason: "action_with_logger: missing mutation capture".to_string(),
1308 });
1309 };
1310
1311 let old_comp = if self.arrow_undo.is_some() {
1314 self.engine.read_computed_overlay_cell(sheet, row, col)
1315 } else {
1316 None
1317 };
1318
1319 if self.engine.graph_admission_enabled() {
1320 let admission =
1321 self.engine
1322 .graph
1323 .preview_value_mutation(addr.sheet_id, row, col)?;
1324 self.engine.preflight_graph_admission(admission)?;
1325 }
1326 if old_formula.is_none() {
1327 old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1328 }
1329
1330 let delta_old_sem = if old_formula.is_some() {
1331 None
1332 } else {
1333 Some(old_value.clone().unwrap_or(LiteralValue::Empty))
1334 };
1335
1336 let start_len = unsafe { (&*capture_ptr).len() };
1337
1338 let capture = unsafe { &mut *capture_ptr };
1340 self.engine.edit_with_capture(capture, |editor| {
1341 editor.set_cell_value_with_old_state(
1342 addr,
1343 value.clone(),
1344 old_value.clone(),
1345 old_formula.clone(),
1346 );
1347 })?;
1348 self.engine.record_structural_change(StructuralScope::Cell {
1349 sheet: addr.sheet_id,
1350 row: addr.coord.row(),
1351 col: addr.coord.col(),
1352 });
1353
1354 if let Some(undo_ptr) = self.arrow_undo {
1355 let new_events = &unsafe { (&*capture_ptr).events() }[start_len..];
1357 let undo = unsafe { &mut *undo_ptr };
1358 self.engine
1359 .record_spill_ops_into_arrow_undo(undo, new_events);
1360
1361 let new_comp = self.engine.read_computed_overlay_cell(sheet, row, col);
1363 let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1364 let row0 = row.saturating_sub(1);
1365 let col0 = col.saturating_sub(1);
1366 let delta_new_sem = Some(value.clone());
1367 undo.record_delta_cell(sheet_id, row0, col0, delta_old_sem, delta_new_sem);
1368 undo.record_computed_cell(sheet_id, row0, col0, old_comp, new_comp);
1369 }
1370 Ok(())
1371 } else {
1372 self.engine
1373 .set_cell_value(sheet, row, col, value)
1374 .map_err(crate::engine::EditorError::from)
1375 }
1376 }
1377
1378 #[inline]
1379 pub fn set_cell_formula(
1380 &mut self,
1381 sheet: &str,
1382 row: u32,
1383 col: u32,
1384 ast: ASTNode,
1385 ) -> Result<(), crate::engine::EditorError> {
1386 if self.capture.is_some() {
1387 let old_value = self.engine.read_cell_value(sheet, row, col);
1388 let mut old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1389 let addr = self.addr_for(sheet, row, col);
1390 let Some(capture_ptr) = self.capture else {
1391 return Err(crate::engine::EditorError::TransactionFailed {
1392 reason: "action_with_logger: missing mutation capture".to_string(),
1393 });
1394 };
1395
1396 let admitted_formula = if self.engine.graph_admission_enabled() {
1397 let placement =
1398 CellRef::new(addr.sheet_id, Coord::from_excel(row, col, true, true));
1399 let ingested = self.engine.ingest_pipeline().ingest_formula(
1400 FormulaAstInput::Tree(ast.clone()),
1401 placement,
1402 None,
1403 )?;
1404 let admission = self.engine.graph.preview_formula_mutations(&[(
1405 addr.sheet_id,
1406 row,
1407 col,
1408 ingested.dep_plan.clone(),
1409 )])?;
1410 self.engine.preflight_graph_admission(admission)?;
1411 Some((ingested.ast_id, ingested.dep_plan))
1412 } else {
1413 None
1414 };
1415 if old_formula.is_none() {
1416 old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1417 }
1418 let delta_old = if self.arrow_undo.is_some() {
1419 if old_formula.is_some() {
1420 None
1421 } else {
1422 Some(old_value.clone().unwrap_or(LiteralValue::Empty))
1423 }
1424 } else {
1425 None
1426 };
1427 let start_len = unsafe { (&*capture_ptr).len() };
1428
1429 let capture = unsafe { &mut *capture_ptr };
1431 self.engine.edit_with_capture(capture, |editor| {
1432 if let Some((ast_id, plan)) = admitted_formula {
1433 editor.set_cell_formula_with_prepared_plan(
1434 addr,
1435 ast.clone(),
1436 old_value,
1437 old_formula,
1438 ast_id,
1439 plan,
1440 );
1441 } else {
1442 editor.set_cell_formula_with_old_state(
1443 addr,
1444 ast.clone(),
1445 old_value,
1446 old_formula,
1447 );
1448 }
1449 })?;
1450 self.engine.record_structural_change(StructuralScope::Cell {
1451 sheet: addr.sheet_id,
1452 row: addr.coord.row(),
1453 col: addr.coord.col(),
1454 });
1455
1456 if let Some(undo_ptr) = self.arrow_undo {
1457 let new_events = &unsafe { (&*capture_ptr).events() }[start_len..];
1458 let undo = unsafe { &mut *undo_ptr };
1459 self.engine
1460 .record_spill_ops_into_arrow_undo(undo, new_events);
1461 let delta_new: Option<LiteralValue> = None;
1462 let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1463 let row0 = row.saturating_sub(1);
1464 let col0 = col.saturating_sub(1);
1465 undo.record_delta_cell(sheet_id, row0, col0, delta_old, delta_new);
1466 }
1467 Ok(())
1468 } else {
1469 self.engine
1470 .set_cell_formula(sheet, row, col, ast)
1471 .map_err(crate::engine::EditorError::from)
1472 }
1473 }
1474
1475 #[inline]
1476 pub fn set_row_hidden(
1477 &mut self,
1478 sheet: &str,
1479 row_1based: u32,
1480 hidden: bool,
1481 source: RowVisibilitySource,
1482 ) -> Result<(), crate::engine::EditorError> {
1483 if self.capture.is_some() {
1484 let sheet_id = self.engine.ensure_known_sheet_id(sheet)?;
1485 let row0 = Engine::<R>::normalize_row_1based(row_1based)?;
1486 let old_hidden = self
1487 .engine
1488 .row_visibility
1489 .get(&sheet_id)
1490 .map(|state| state.is_row_hidden(row0, Some(source)))
1491 .unwrap_or(false);
1492 if old_hidden == hidden {
1493 return Ok(());
1494 }
1495
1496 let _ = self
1497 .engine
1498 .set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source);
1499
1500 let Some(capture_ptr) = self.capture else {
1501 return Err(crate::engine::EditorError::TransactionFailed {
1502 reason: "action_with_logger: missing mutation capture".to_string(),
1503 });
1504 };
1505 unsafe { &mut *capture_ptr }.record(crate::engine::ChangeEvent::SetRowVisibility {
1506 sheet_id,
1507 row0,
1508 source,
1509 old_hidden,
1510 new_hidden: hidden,
1511 });
1512
1513 Ok(())
1514 } else {
1515 self.engine
1516 .set_row_hidden(sheet, row_1based, hidden, source)
1517 }
1518 }
1519
1520 #[inline]
1521 pub fn set_rows_hidden(
1522 &mut self,
1523 sheet: &str,
1524 start_row_1based: u32,
1525 end_row_1based: u32,
1526 hidden: bool,
1527 source: RowVisibilitySource,
1528 ) -> Result<(), crate::engine::EditorError> {
1529 if self.capture.is_some() {
1530 let sheet_id = self.engine.ensure_known_sheet_id(sheet)?;
1531 let (start_row0, end_row0) =
1532 Engine::<R>::normalize_row_range_1based(start_row_1based, end_row_1based)?;
1533
1534 let Some(capture_ptr) = self.capture else {
1535 return Err(crate::engine::EditorError::TransactionFailed {
1536 reason: "action_with_logger: missing mutation capture".to_string(),
1537 });
1538 };
1539 let capture = unsafe { &mut *capture_ptr };
1540
1541 for row0 in start_row0..=end_row0 {
1542 let old_hidden = self
1543 .engine
1544 .row_visibility
1545 .get(&sheet_id)
1546 .map(|state| state.is_row_hidden(row0, Some(source)))
1547 .unwrap_or(false);
1548 if old_hidden == hidden {
1549 continue;
1550 }
1551
1552 let _ = self
1553 .engine
1554 .set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source);
1555
1556 capture.record(crate::engine::ChangeEvent::SetRowVisibility {
1557 sheet_id,
1558 row0,
1559 source,
1560 old_hidden,
1561 new_hidden: hidden,
1562 });
1563 }
1564
1565 Ok(())
1566 } else {
1567 self.engine
1568 .set_rows_hidden(sheet, start_row_1based, end_row_1based, hidden, source)
1569 }
1570 }
1571
1572 #[inline]
1573 pub fn insert_rows(
1574 &mut self,
1575 sheet: &str,
1576 before: u32,
1577 count: u32,
1578 ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1579 if count == 0 {
1580 return Ok(crate::engine::ShiftSummary::default());
1581 }
1582 if self.capture.is_some() {
1583 let Some(capture_ptr) = self.capture else {
1584 return Err(crate::engine::EditorError::TransactionFailed {
1585 reason: "action_atomic: missing mutation capture".to_string(),
1586 });
1587 };
1588
1589 let before0 = before.saturating_sub(1);
1590 self.engine.check_insert_room(sheet, before0, count, true)?;
1591 let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1592 let occupancy = self.engine.structural_row_occupancy(sheet, sheet_id);
1593 let affected_region = Engine::<R>::structural_row_region(sheet_id, before0);
1594
1595 let summary = {
1597 let capture = unsafe { &mut *capture_ptr };
1598 let mut out: Result<crate::engine::ShiftSummary, crate::engine::EditorError> =
1599 Ok(crate::engine::ShiftSummary::default());
1600 self.engine.edit_with_capture(capture, |editor| {
1601 editor.set_structural_occupancy(occupancy);
1602 out = editor.insert_rows(sheet_id, before0, count);
1603 })?;
1604 out?
1605 };
1606
1607 self.engine.ensure_arrow_sheet(sheet);
1609 if let Some(asheet) = self.engine.arrow_sheets.sheet_mut(sheet) {
1610 asheet.insert_rows(before0 as usize, count as usize);
1611 }
1612 self.engine
1613 .purge_derived_formats_after_row(sheet_id, before0);
1614 self.engine
1615 .shift_row_visibility_insert(sheet_id, before0, count);
1616 self.engine.mark_moved_formula_vertices_dirty(&summary);
1617 self.engine
1618 .clear_computed_overlay_after_row(sheet, before0 as usize);
1619 self.engine
1620 .record_structural_change(StructuralScope::Region(affected_region));
1621 if let Some(undo_ptr) = self.arrow_undo {
1622 unsafe { &mut *undo_ptr }.record_insert_rows(sheet_id, before0, count);
1623 }
1624 Ok(summary)
1625 } else {
1626 self.engine.insert_rows(sheet, before, count)
1627 }
1628 }
1629
1630 #[inline]
1631 pub fn delete_rows(
1632 &mut self,
1633 sheet: &str,
1634 start: u32,
1635 count: u32,
1636 ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1637 if count == 0 {
1638 return Ok(crate::engine::ShiftSummary::default());
1639 }
1640 if self.atomic_policy {
1641 return Err(crate::engine::EditorError::TransactionUnsupported {
1642 reason:
1643 "delete_rows is not supported inside atomic actions (conservative rollback policy)"
1644 .to_string(),
1645 });
1646 }
1647 self.engine.delete_rows(sheet, start, count)
1648 }
1649
1650 #[inline]
1651 pub fn insert_columns(
1652 &mut self,
1653 sheet: &str,
1654 before: u32,
1655 count: u32,
1656 ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1657 if count == 0 {
1658 return Ok(crate::engine::ShiftSummary::default());
1659 }
1660 if self.capture.is_some() {
1661 let Some(capture_ptr) = self.capture else {
1662 return Err(crate::engine::EditorError::TransactionFailed {
1663 reason: "action_atomic: missing mutation capture".to_string(),
1664 });
1665 };
1666
1667 let before0 = before.saturating_sub(1);
1668 self.engine
1669 .check_insert_room(sheet, before0, count, false)?;
1670 let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1671 let occupancy = self.engine.structural_column_occupancy();
1672 let affected_region = Engine::<R>::structural_col_region(sheet_id, before0);
1673
1674 let summary = {
1675 let capture = unsafe { &mut *capture_ptr };
1676 let mut out: Result<crate::engine::ShiftSummary, crate::engine::EditorError> =
1677 Ok(crate::engine::ShiftSummary::default());
1678 self.engine.edit_with_capture(capture, |editor| {
1679 editor.set_structural_occupancy(occupancy);
1680 out = editor.insert_columns(sheet_id, before0, count);
1681 })?;
1682 out?
1683 };
1684
1685 self.engine.ensure_arrow_sheet(sheet);
1686 if let Some(asheet) = self.engine.arrow_sheets.sheet_mut(sheet) {
1687 asheet.insert_columns(before0 as usize, count as usize);
1688 }
1689 self.engine
1690 .purge_derived_formats_after_col(sheet_id, before0);
1691 self.engine.mark_moved_formula_vertices_dirty(&summary);
1692 self.engine
1693 .clear_computed_overlay_after_col(sheet, before0 as usize);
1694 self.engine
1695 .record_structural_change(StructuralScope::Region(affected_region));
1696 if let Some(undo_ptr) = self.arrow_undo {
1697 unsafe { &mut *undo_ptr }.record_insert_cols(sheet_id, before0, count);
1698 }
1699 Ok(summary)
1700 } else {
1701 self.engine.insert_columns(sheet, before, count)
1702 }
1703 }
1704
1705 #[inline]
1706 pub fn delete_columns(
1707 &mut self,
1708 sheet: &str,
1709 start: u32,
1710 count: u32,
1711 ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1712 if count == 0 {
1713 return Ok(crate::engine::ShiftSummary::default());
1714 }
1715 if self.atomic_policy {
1716 return Err(crate::engine::EditorError::TransactionUnsupported {
1717 reason:
1718 "delete_columns is not supported inside atomic actions (conservative rollback policy)"
1719 .to_string(),
1720 });
1721 }
1722 self.engine.delete_columns(sheet, start, count)
1723 }
1724
1725 #[inline]
1730 pub fn action<T>(
1731 &mut self,
1732 name: impl AsRef<str>,
1733 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
1734 ) -> Result<T, crate::engine::EditorError> {
1735 self.engine.action(name, f)
1736 }
1737}
1738
1739struct ActionDepthGuard<'a, R> {
1740 engine: *mut Engine<R>,
1741 _marker: std::marker::PhantomData<&'a mut Engine<R>>,
1742}
1743
1744impl<'a, R> Drop for ActionDepthGuard<'a, R> {
1745 fn drop(&mut self) {
1746 unsafe {
1749 let e = &mut *self.engine;
1750 e.action_depth = e.action_depth.saturating_sub(1);
1751 }
1752 }
1753}
1754
1755#[derive(Default)]
1756struct SourceCache {
1757 scalars: FxHashMap<(String, Option<u64>), LiteralValue>,
1758 tables: FxHashMap<(String, Option<u64>), Arc<dyn crate::traits::Table>>,
1759 external_views: FxHashMap<(String, Option<u64>), RangeView<'static>>,
1762}
1763
1764#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1765struct VisibilityMaskCacheKey {
1766 sheet_id: SheetId,
1767 start_row0: u32,
1768 end_row0: u32,
1769 mode: VisibilityMaskMode,
1770 version: u64,
1771}
1772
1773#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1774enum StructuralScope {
1775 Cell { sheet: SheetId, row: u32, col: u32 },
1776 Region(Region),
1777 Sheet(SheetId),
1778 RemovedSheet(SheetId),
1779 OpaqueGlobal,
1780 AllSheets,
1781}
1782
1783#[derive(Default)]
1786struct SpillBlockerWaiters {
1787 by_cell: FxHashMap<CellRef, Vec<VertexId>>,
1788 by_anchor: FxHashMap<VertexId, CellRef>,
1789}
1790
1791impl SpillBlockerWaiters {
1792 const REGION_PROBE_CELLS: u64 = 4096;
1794
1795 fn is_empty(&self) -> bool {
1796 self.by_anchor.is_empty()
1797 }
1798
1799 fn wait(&mut self, anchor: VertexId, blocker: CellRef) {
1800 match self.by_anchor.insert(anchor, blocker) {
1801 Some(old) if old == blocker => return,
1802 Some(old) => self.unlink(anchor, old),
1803 None => {}
1804 }
1805 self.by_cell.entry(blocker).or_default().push(anchor);
1806 }
1807
1808 fn forget(&mut self, anchor: VertexId) {
1809 if self.by_anchor.is_empty() {
1810 return;
1811 }
1812 if let Some(old) = self.by_anchor.remove(&anchor) {
1813 self.unlink(anchor, old);
1814 }
1815 }
1816
1817 fn unlink(&mut self, anchor: VertexId, cell: CellRef) {
1818 if let Some(waiters) = self.by_cell.get_mut(&cell) {
1819 waiters.retain(|&v| v != anchor);
1820 if waiters.is_empty() {
1821 self.by_cell.remove(&cell);
1822 }
1823 }
1824 }
1825
1826 fn take_affected(&mut self, scope: StructuralScope) -> Vec<VertexId> {
1828 let cells: Vec<CellRef> = match scope {
1829 StructuralScope::Cell { sheet, row, col } => {
1830 match Coord::try_new(row, col, true, true) {
1831 Ok(coord) if self.by_cell.contains_key(&CellRef::new(sheet, coord)) => {
1832 vec![CellRef::new(sheet, coord)]
1833 }
1834 _ => Vec::new(),
1835 }
1836 }
1837 StructuralScope::Region(region) => {
1838 let (rows, cols) = region.axis_ranges();
1839 let (r0, r1) = rows.query_bounds();
1840 let (c0, c1) = cols.query_bounds();
1841 let area = (u64::from(r1.saturating_sub(r0)) + 1)
1845 .saturating_mul(u64::from(c1.saturating_sub(c0)) + 1);
1846 if area <= Self::REGION_PROBE_CELLS {
1847 let sheet = region.sheet_id();
1848 (r0..=r1)
1851 .flat_map(|r| (c0..=c1).map(move |c| (r, c)))
1852 .filter_map(|(r, c)| Coord::try_new(r, c, true, true).ok())
1853 .map(|coord| CellRef::new(sheet, coord))
1854 .filter(|cell| self.by_cell.contains_key(cell))
1855 .collect()
1856 } else {
1857 self.by_cell
1858 .keys()
1859 .filter(|cell| {
1860 region.intersects(&Region::point(
1861 cell.sheet_id,
1862 cell.coord.row(),
1863 cell.coord.col(),
1864 ))
1865 })
1866 .copied()
1867 .collect()
1868 }
1869 }
1870 StructuralScope::Sheet(sheet) | StructuralScope::RemovedSheet(sheet) => self
1871 .by_cell
1872 .keys()
1873 .filter(|cell| cell.sheet_id == sheet)
1874 .copied()
1875 .collect(),
1876 StructuralScope::OpaqueGlobal | StructuralScope::AllSheets => {
1877 self.by_cell.keys().copied().collect()
1878 }
1879 };
1880 let mut woken = Vec::new();
1881 for cell in cells {
1882 if let Some(waiters) = self.by_cell.remove(&cell) {
1883 for anchor in waiters {
1884 self.by_anchor.remove(&anchor);
1885 woken.push(anchor);
1886 }
1887 }
1888 }
1889 woken
1890 }
1891}
1892
1893#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
1894enum LoggedEditImpact {
1895 NoOp,
1896 DataOnly,
1897 Topology,
1898}
1899
1900#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1901enum LoggedEditDirection {
1902 Original,
1903 InverseReplay,
1904 ForwardReplay,
1905}
1906
1907#[derive(Clone, Copy)]
1908struct InvalidationBaseline {
1909 snapshot_id: u64,
1910 topology_epoch: u64,
1911}
1912
1913struct SourceCacheSession {
1914 cache: Arc<std::sync::RwLock<SourceCache>>,
1915}
1916
1917impl Drop for SourceCacheSession {
1918 fn drop(&mut self) {
1919 if let Ok(mut g) = self.cache.write() {
1920 *g = SourceCache::default();
1921 }
1922 }
1923}
1924
1925#[derive(Debug)]
1926#[non_exhaustive]
1927pub struct EvalResult {
1928 pub computed_vertices: usize,
1929 pub cycle_errors: usize,
1930 pub elapsed: std::time::Duration,
1931}
1932
1933#[derive(Clone, Debug, PartialEq, Eq)]
1934#[non_exhaustive]
1935pub struct TableMetadata {
1936 pub name: String,
1937 pub sheet: String,
1938 pub start_row: u32,
1939 pub start_col: u32,
1940 pub end_row: u32,
1941 pub end_col: u32,
1942 pub header_row: bool,
1943 pub headers: Vec<String>,
1944 pub totals_row: bool,
1945}
1946
1947#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
1955#[non_exhaustive]
1956pub struct EngineBaselineStats {
1957 pub graph_vertex_count: usize,
1958 pub graph_formula_vertex_count: usize,
1959 pub graph_edge_count: usize,
1960 pub dirty_vertex_count: usize,
1961 pub evaluation_vertex_count: usize,
1962 pub formula_ast_root_count: usize,
1963 pub formula_ast_node_count: usize,
1964 pub staged_formula_count: usize,
1965 pub formula_plane_active_span_count: usize,
1966 pub formula_plane_producer_result_entries: usize,
1967 pub formula_plane_consumer_read_entries: usize,
1968 pub formula_plane_mixed_topology_cache_builds: u64,
1969 pub formula_plane_mixed_topology_cache_hits: u64,
1970 pub formula_plane_mixed_topology_cache_overflows: u64,
1971 pub formula_plane_dirty_pending_events: usize,
1972 pub formula_plane_dirty_region_events_recorded: u64,
1973 pub formula_plane_dirty_span_region_events_recorded: u64,
1974 pub formula_plane_dirty_whole_span_seeds_recorded: u64,
1975 pub formula_plane_dirty_global_invalidations: u64,
1976 pub formula_plane_structural_span_candidates: u64,
1977 pub formula_plane_cycle_member_span_demotions: u64,
1978 pub formula_plane_array_result_span_demotions: u64,
1979 pub retained_scc_members: usize,
1982}
1983
1984#[derive(Debug, Clone, Default)]
1985#[non_exhaustive]
1986pub struct VirtualDepTelemetry {
1987 pub candidate_vertices_total: usize,
1988 pub vdeps_vertices_total: usize,
1989 pub vdeps_edges_total: usize,
1990 pub builder_elapsed_ms_total: u128,
1991 pub schedule_virtual_passes: usize,
1992 pub schedule_static_passes: usize,
1993 pub schedule_cache_hits: usize,
1994 pub schedule_cache_misses: usize,
1995 pub reused_schedule_vertices_total: usize,
1996 pub replan_iterations: usize,
1997 pub changed_vdeps_total: usize,
1998 pub bailout_reason: Option<&'static str>,
1999 pub fallback_mode_activations: u64,
2000}
2001
2002#[derive(Debug, Clone, Default, PartialEq)]
2011#[non_exhaustive]
2012pub struct CycleTelemetry {
2013 pub static_sccs: usize,
2015 pub phantom_sccs: usize,
2017 pub live_cycles_witnessed: usize,
2019 pub circ_cells_stamped: usize,
2021 pub settle_passes_total: usize,
2024 pub max_passes_single_scc: usize,
2026 pub iterated_sccs: usize,
2029 pub converged_sccs: usize,
2032 pub capped_sccs: usize,
2038 pub max_abs_delta_at_stop: f64,
2042 pub nan_converged: usize,
2045 pub reused_sccs: usize,
2050 pub reused_scc_members: usize,
2052 pub elapsed_ms: u128,
2054}
2055
2056#[derive(Debug, Clone, Copy)]
2057struct ScheduleBuildMeta {
2058 candidate_vertices: usize,
2059 vdeps_vertices: usize,
2060 vdeps_edges: usize,
2061 builder_elapsed_ms: u128,
2062 used_virtual_schedule: bool,
2063 schedule_cache_hit: bool,
2064 schedule_cache_eligible: bool,
2065}
2066
2067#[cfg(any(test, feature = "benchmark_internal"))]
2068#[doc(hidden)]
2069#[derive(Debug, Clone, Default)]
2070pub struct RecalcReuseProbe {
2071 pub schedule_requests: usize,
2072 pub schedule_cache_hits: usize,
2073 pub schedule_cache_misses: usize,
2074 pub schedule_cache_ineligible: usize,
2075 pub schedule_builds: usize,
2076 pub schedule_base_restrictions: usize,
2078 pub schedule_shared_handles: usize,
2079 pub schedule_retained_bytes: usize,
2080 pub legacy_target_requests: usize,
2081 pub target_schedule_builds: usize,
2082 pub demand_builds: usize,
2083 pub demand_vertices: usize,
2084 pub demand_clean_formulas: usize,
2085 pub demand_explicit_edges: usize,
2086 pub demand_virtual_builder_calls: usize,
2087}
2088
2089#[cfg(any(test, feature = "benchmark_internal"))]
2090fn schedule_probe_retained_bytes(schedule: &crate::engine::Schedule) -> usize {
2091 fn vector_bytes<T>(values: &Vec<T>) -> usize {
2092 values.capacity() * std::mem::size_of::<T>()
2093 }
2094
2095 [
2096 vector_bytes(&schedule.units),
2097 vector_bytes(&schedule.layers),
2098 vector_bytes(&schedule.cycles),
2099 ]
2100 .into_iter()
2101 .chain(
2102 schedule
2103 .layers
2104 .iter()
2105 .map(|layer| vector_bytes(&layer.vertices)),
2106 )
2107 .chain(schedule.cycles.iter().map(vector_bytes))
2108 .sum()
2109}
2110
2111#[derive(Debug, Clone)]
2112struct CachedScheduleEntry {
2113 topology_epoch: u64,
2114 authority_revision: (u64, u64),
2117 candidate_vertices: VertexIdRuns,
2120 schedule: Arc<crate::engine::scheduler::Schedule>,
2121}
2122
2123#[derive(Debug, Clone, Default)]
2126struct VertexIdRuns(Vec<(u32, u32)>);
2127
2128impl VertexIdRuns {
2129 fn from_slice(ids: &[VertexId]) -> Self {
2130 let mut runs: Vec<(u32, u32)> = Vec::new();
2131 for v in ids {
2132 match runs.last_mut() {
2133 Some((first, len)) if first.checked_add(*len) == Some(v.0) => *len += 1,
2134 _ => runs.push((v.0, 1)),
2135 }
2136 }
2137 runs.shrink_to_fit();
2138 Self(runs)
2139 }
2140
2141 fn equals(&self, ids: &[VertexId]) -> bool {
2142 let mut rest = ids;
2143 for &(first, len) in &self.0 {
2144 let len = len as usize;
2145 if rest.len() < len {
2146 return false;
2147 }
2148 let (head, tail) = rest.split_at(len);
2149 if head
2150 .iter()
2151 .enumerate()
2152 .any(|(i, v)| v.0 != first.wrapping_add(i as u32))
2153 {
2154 return false;
2155 }
2156 rest = tail;
2157 }
2158 rest.is_empty()
2159 }
2160
2161 fn heap_bytes(&self) -> usize {
2162 self.0.capacity() * std::mem::size_of::<(u32, u32)>()
2163 }
2164
2165 fn len(&self) -> usize {
2167 self.0.iter().map(|&(_, len)| len as usize).sum()
2168 }
2169}
2170
2171#[cfg(test)]
2172mod vertex_id_runs_tests {
2173 use super::{VertexId, VertexIdRuns};
2174
2175 #[test]
2176 fn runs_compare_like_the_list() {
2177 let ids = |v: &[u32]| v.iter().map(|&i| VertexId(i)).collect::<Vec<_>>();
2178 let list = ids(&[5, 6, 7, 2, 3, 9, 10, 10]);
2179 let runs = VertexIdRuns::from_slice(&list);
2180 assert_eq!(runs.0, vec![(5, 3), (2, 2), (9, 2), (10, 1)]);
2181 assert!(runs.equals(&list));
2182 assert!(!runs.equals(&list[..7]));
2183 assert!(!runs.equals(&ids(&[5, 6, 7, 2, 3, 9, 10, 11])));
2184 assert!(!runs.equals(&ids(&[5, 6, 7, 2, 3, 9, 10, 10, 11])));
2185 assert!(VertexIdRuns::from_slice(&[]).equals(&[]));
2186 assert!(!VertexIdRuns::from_slice(&[]).equals(&list));
2187 let edge = ids(&[u32::MAX - 1, u32::MAX, 0]);
2188 assert!(VertexIdRuns::from_slice(&edge).equals(&edge));
2189 }
2190}
2191
2192enum EvaluationSchedule {
2194 Owned(crate::engine::scheduler::Schedule),
2195 Shared(Arc<crate::engine::scheduler::Schedule>),
2196}
2197
2198impl std::ops::Deref for EvaluationSchedule {
2199 type Target = crate::engine::scheduler::Schedule;
2200
2201 fn deref(&self) -> &Self::Target {
2202 match self {
2203 Self::Owned(schedule) => schedule,
2204 Self::Shared(schedule) => schedule,
2205 }
2206 }
2207}
2208
2209type ScheduleBuildOutput = (
2210 crate::engine::scheduler::Schedule,
2211 FxHashMap<VertexId, Vec<VertexId>>,
2212 ScheduleBuildMeta,
2213);
2214
2215type EvaluationScheduleBuildOutput = (
2216 EvaluationSchedule,
2217 FxHashMap<VertexId, Vec<VertexId>>,
2218 ScheduleBuildMeta,
2219);
2220
2221#[derive(Debug)]
2223pub struct RecalcPlan {
2224 key: RecalcPlanKey,
2225 kind: RecalcPlanKind,
2226}
2227
2228#[derive(Debug)]
2229struct RecalcPlanKey {
2230 engine_token: Arc<()>,
2231 revisions: PlanningRevisionSnapshot,
2232}
2233
2234#[derive(Clone, Debug, PartialEq, Eq)]
2235struct PlanningRevisionSnapshot {
2236 engine_topology_epoch: u64,
2237 graph_topology_revision: u64,
2238 staged: u64,
2239 symbols: u64,
2240 semantic: u64,
2241 provider: Option<u64>,
2242 deterministic_mode: crate::engine::DeterministicMode,
2243 budgets: crate::engine::EvaluationBudgets,
2244}
2245
2246#[derive(Debug)]
2247enum RecalcPlanKind {
2248 CompatibilityFull {
2249 schedule: crate::engine::Schedule,
2250 has_dynamic_refs: bool,
2251 },
2252 Target {
2253 targets: Vec<crate::engine::EvaluationTarget>,
2254 scope: crate::engine::PrepareScope,
2255 topology: RecalcTopology,
2256 dynamic_policy: DynamicPlanPolicy,
2257 },
2258}
2259
2260#[derive(Debug)]
2261enum RecalcTopology {
2262 RunLocalRecipe,
2263 Workbook,
2264}
2265
2266#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2267enum DynamicPlanPolicy {
2268 BoundedTargetReplan,
2269}
2270
2271impl RecalcPlan {
2272 pub fn layer_count(&self) -> usize {
2275 match &self.kind {
2276 RecalcPlanKind::CompatibilityFull { schedule, .. } => schedule.layers.len(),
2277 RecalcPlanKind::Target { .. } => 0,
2278 }
2279 }
2280
2281 pub fn has_dynamic_refs(&self) -> bool {
2282 match &self.kind {
2283 RecalcPlanKind::CompatibilityFull {
2284 has_dynamic_refs, ..
2285 } => *has_dynamic_refs,
2286 RecalcPlanKind::Target { .. } => false,
2287 }
2288 }
2289
2290 #[cfg(test)]
2291 pub(crate) fn force_stale_reasons_for_test(
2292 &mut self,
2293 reasons: &[formualizer_common::PlanStaleReason],
2294 ) {
2295 use formualizer_common::PlanStaleReason;
2296 for reason in reasons {
2297 match reason {
2298 PlanStaleReason::Engine => {
2299 self.key.engine_token = Arc::new(());
2300 }
2301 PlanStaleReason::Provider => {
2302 self.key.revisions.provider = Some(
2303 self.key
2304 .revisions
2305 .provider
2306 .unwrap_or_default()
2307 .wrapping_add(1),
2308 );
2309 }
2310 PlanStaleReason::Semantic => {
2311 self.key.revisions.semantic = self.key.revisions.semantic.wrapping_add(1);
2312 }
2313 PlanStaleReason::Budget => {
2314 let current = self.key.revisions.budgets.work.max_work_units;
2315 self.key.revisions.budgets.work.max_work_units =
2316 Some(current.unwrap_or_default().wrapping_add(1));
2317 }
2318 PlanStaleReason::Staged => {
2319 self.key.revisions.staged = self.key.revisions.staged.wrapping_add(1);
2320 }
2321 PlanStaleReason::Symbols => {
2322 self.key.revisions.symbols = self.key.revisions.symbols.wrapping_add(1);
2323 }
2324 PlanStaleReason::Graph => {
2325 self.key.revisions.graph_topology_revision =
2326 self.key.revisions.graph_topology_revision.wrapping_add(1);
2327 }
2328 _ => {}
2329 }
2330 }
2331 }
2332}
2333
2334#[cfg(any(test, feature = "test-support"))]
2335pub(crate) mod criteria_mask_test_hooks {
2336 use std::cell::Cell;
2337
2338 thread_local! {
2339 static MASK_CALLS_ROWS: Cell<(usize, usize)> = const { Cell::new((0, 0)) };
2340 static TEXT_SEGMENTS_TOTAL: Cell<usize> = const { Cell::new(0) };
2341 static TEXT_SEGMENTS_ALL_NULL: Cell<usize> = const { Cell::new(0) };
2342 }
2343
2344 pub(crate) fn take_mask_work() -> (usize, usize) {
2345 MASK_CALLS_ROWS.with(|c| c.replace((0, 0)))
2346 }
2347
2348 pub(crate) fn note_mask(rows: usize) {
2349 MASK_CALLS_ROWS.with(|c| {
2350 let (calls, work) = c.get();
2351 c.set((calls + 1, work + rows));
2352 });
2353 }
2354
2355 pub fn reset_text_segment_counters() {
2356 TEXT_SEGMENTS_TOTAL.with(|c| c.set(0));
2357 TEXT_SEGMENTS_ALL_NULL.with(|c| c.set(0));
2358 }
2359
2360 pub fn text_segment_counters() -> (usize, usize) {
2361 let a = TEXT_SEGMENTS_TOTAL.with(|c| c.get());
2362 let b = TEXT_SEGMENTS_ALL_NULL.with(|c| c.get());
2363 (a, b)
2364 }
2365
2366 pub(crate) fn inc_total() {
2367 TEXT_SEGMENTS_TOTAL.with(|c| c.set(c.get() + 1));
2368 }
2369 pub(crate) fn inc_all_null() {
2370 TEXT_SEGMENTS_ALL_NULL.with(|c| c.set(c.get() + 1));
2371 }
2372}
2373
2374#[cfg(test)]
2375pub(crate) mod visibility_mask_test_hooks {
2376 use std::cell::Cell;
2377
2378 thread_local! {
2379 static HITS: Cell<usize> = const { Cell::new(0) };
2380 static MISSES: Cell<usize> = const { Cell::new(0) };
2381 static EVICTIONS: Cell<usize> = const { Cell::new(0) };
2382 }
2383
2384 pub fn reset() {
2385 HITS.with(|c| c.set(0));
2386 MISSES.with(|c| c.set(0));
2387 EVICTIONS.with(|c| c.set(0));
2388 }
2389
2390 pub fn counters() -> (usize, usize, usize) {
2391 let hits = HITS.with(|c| c.get());
2392 let misses = MISSES.with(|c| c.get());
2393 let evictions = EVICTIONS.with(|c| c.get());
2394 (hits, misses, evictions)
2395 }
2396
2397 pub(crate) fn inc_hit() {
2398 HITS.with(|c| c.set(c.get() + 1));
2399 }
2400
2401 pub(crate) fn inc_miss() {
2402 MISSES.with(|c| c.set(c.get() + 1));
2403 }
2404
2405 pub(crate) fn inc_eviction() {
2406 EVICTIONS.with(|c| c.set(c.get() + 1));
2407 }
2408}
2409
2410fn is_numeric_text_equality(pred: &crate::args::CriteriaPredicate) -> bool {
2411 match pred {
2412 crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Text(text)) => {
2413 text.trim().parse::<f64>().is_ok_and(f64::is_finite)
2414 }
2415 _ => false,
2416 }
2417}
2418
2419fn fill_numeric_equality_nulls(
2424 view: &RangeView<'_>,
2425 row_start: usize,
2426 row_len: usize,
2427 col_in_view: usize,
2428 mask: &arrow_array::BooleanArray,
2429 n: f64,
2430 ne: bool,
2431) -> Option<arrow_array::BooleanArray> {
2432 use arrow_array::Array as _;
2433 let texts = view.slice_lowered_text(row_start, row_len);
2434 let text = texts.get(col_in_view).and_then(|t| t.as_ref());
2435 match text {
2436 None => {
2437 if !ne {
2438 return Some(mask.clone());
2440 }
2441 let nulls = arrow::compute::is_null(mask).ok()?;
2442 crate::compute_prelude::boolean::or_kleene(mask, &nulls).ok()
2443 }
2444 Some(text) => {
2445 if text.len() != mask.len() {
2446 return None;
2447 }
2448 let mut out = arrow_array::builder::BooleanBuilder::with_capacity(mask.len());
2449 for i in 0..mask.len() {
2450 if mask.is_valid(i) {
2451 out.append_value(mask.value(i));
2452 } else if text.is_valid(i) {
2453 let equal = crate::locale::Locale::invariant()
2454 .parse_number_invariant(text.value(i))
2455 .is_some_and(|x| (x - n).abs() < 1e-12);
2456 out.append_value(equal != ne);
2457 } else {
2458 out.append_value(ne);
2459 }
2460 }
2461 Some(out.finish())
2462 }
2463 }
2464}
2465
2466fn compute_criteria_mask(
2467 view: &RangeView<'_>,
2468 col_in_view: usize,
2469 pred: &crate::args::CriteriaPredicate,
2470) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
2471 use crate::compute_prelude::{boolean, cmp, concat_arrays};
2472 use arrow::compute::kernels::comparison::{ilike, nilike};
2473 use arrow_array::{
2474 Array as _, ArrayRef, BooleanArray, Float64Array, StringArray, builder::BooleanBuilder,
2475 };
2476
2477 fn apply_numeric_pred(
2479 chunk: &Float64Array,
2480 pred: &crate::args::CriteriaPredicate,
2481 ) -> Option<BooleanArray> {
2482 match pred {
2483 crate::args::CriteriaPredicate::Gt(n) => {
2484 cmp::gt(chunk, &Float64Array::new_scalar(*n)).ok()
2485 }
2486 crate::args::CriteriaPredicate::Ge(n) => {
2487 cmp::gt_eq(chunk, &Float64Array::new_scalar(*n)).ok()
2488 }
2489 crate::args::CriteriaPredicate::Lt(n) => {
2490 cmp::lt(chunk, &Float64Array::new_scalar(*n)).ok()
2491 }
2492 crate::args::CriteriaPredicate::Le(n) => {
2493 cmp::lt_eq(chunk, &Float64Array::new_scalar(*n)).ok()
2494 }
2495 crate::args::CriteriaPredicate::Eq(v) => match v {
2496 formualizer_common::LiteralValue::Number(x) => {
2497 cmp::eq(chunk, &Float64Array::new_scalar(*x)).ok()
2498 }
2499 formualizer_common::LiteralValue::Int(i) => {
2500 cmp::eq(chunk, &Float64Array::new_scalar(*i as f64)).ok()
2501 }
2502 _ => None,
2503 },
2504 crate::args::CriteriaPredicate::Ne(v) => match v {
2505 formualizer_common::LiteralValue::Number(x) => {
2506 cmp::neq(chunk, &Float64Array::new_scalar(*x)).ok()
2507 }
2508 formualizer_common::LiteralValue::Int(i) => {
2509 cmp::neq(chunk, &Float64Array::new_scalar(*i as f64)).ok()
2510 }
2511 _ => None,
2512 },
2513 _ => None,
2514 }
2515 }
2516
2517 let is_numeric_pred = matches!(
2519 pred,
2520 crate::args::CriteriaPredicate::Gt(_)
2521 | crate::args::CriteriaPredicate::Ge(_)
2522 | crate::args::CriteriaPredicate::Lt(_)
2523 | crate::args::CriteriaPredicate::Le(_)
2524 | crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Number(_))
2525 | crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Int(_))
2526 | crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Number(_))
2527 | crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Int(_))
2528 );
2529
2530 if is_numeric_pred {
2534 let mut bool_parts: Vec<BooleanArray> = Vec::new();
2535 let number = |v: &formualizer_common::LiteralValue| match v {
2539 formualizer_common::LiteralValue::Number(n) => Some(*n),
2540 formualizer_common::LiteralValue::Int(i) => Some(*i as f64),
2541 _ => None,
2542 };
2543 let equality = match pred {
2544 crate::args::CriteriaPredicate::Eq(v) => number(v).map(|n| (n, false)),
2545 crate::args::CriteriaPredicate::Ne(v) => number(v).map(|n| (n, true)),
2546 _ => None,
2547 };
2548 for res in view.numbers_slices() {
2549 let (rs, rl, cols_seg) = res.ok()?;
2550 if col_in_view < cols_seg.len() {
2551 let chunk = cols_seg[col_in_view].as_ref();
2552 let mut mask = apply_numeric_pred(chunk, pred)?;
2553 if let Some((n, ne)) = equality
2554 && mask.null_count() > 0
2555 {
2556 mask = fill_numeric_equality_nulls(view, rs, rl, col_in_view, &mask, n, ne)?;
2557 }
2558 bool_parts.push(mask);
2559 }
2560 }
2561
2562 if bool_parts.is_empty() {
2563 return None;
2564 } else if bool_parts.len() == 1 {
2565 return Some(std::sync::Arc::new(bool_parts.remove(0)));
2566 } else {
2567 let anys: Vec<&dyn arrow_array::Array> = bool_parts
2569 .iter()
2570 .map(|a| a as &dyn arrow_array::Array)
2571 .collect();
2572 let conc: ArrayRef = concat_arrays(&anys).ok()?;
2573 let ba = conc.as_any().downcast_ref::<BooleanArray>()?.clone();
2574 return Some(std::sync::Arc::new(ba));
2575 }
2576 }
2577
2578 if is_numeric_text_equality(pred)
2582 || matches!(pred, crate::args::CriteriaPredicate::TextLike { .. })
2583 {
2584 for tags in view.type_tags_slices() {
2585 let (_, _, cols) = tags.ok()?;
2586 let tags = cols.get(col_in_view)?;
2587 if tags.values().iter().any(|tag| {
2588 *tag == crate::arrow_store::TypeTag::Empty as u8
2589 || *tag == crate::arrow_store::TypeTag::Number as u8
2590 || *tag == crate::arrow_store::TypeTag::Boolean as u8
2591 }) {
2592 let mut mask = BooleanBuilder::new();
2593 for chunk in view.iter_row_chunks() {
2594 let chunk = chunk.ok()?;
2595 for row in chunk.row_start..chunk.row_start + chunk.row_len {
2596 mask.append_value(crate::builtins::criteria_match(
2597 pred,
2598 &view.get_cell(row, col_in_view),
2599 ));
2600 }
2601 }
2602 return Some(std::sync::Arc::new(mask.finish()));
2603 }
2604 }
2605 }
2606
2607 if matches!(pred, crate::args::CriteriaPredicate::TextLike { pattern, .. }
2609 if pattern.contains('~'))
2610 {
2611 return None;
2612 }
2613
2614 if matches!(
2618 pred,
2619 crate::args::CriteriaPredicate::IsBlank | crate::args::CriteriaPredicate::NotBlank
2620 ) {
2621 let want_blank = matches!(pred, crate::args::CriteriaPredicate::IsBlank);
2622 let mut bool_parts: Vec<BooleanArray> = Vec::new();
2623 for tags in view.type_tags_slices() {
2624 let (_, _, cols) = tags.ok()?;
2625 let tags = cols.get(col_in_view)?;
2626 let mut bb = BooleanBuilder::with_capacity(tags.len());
2627 for i in 0..tags.len() {
2628 let blank = tags.value(i) == crate::arrow_store::TypeTag::Empty as u8;
2629 bb.append_value(blank == want_blank);
2630 }
2631 bool_parts.push(bb.finish());
2632 }
2633 return match bool_parts.len() {
2634 0 => None,
2635 1 => Some(std::sync::Arc::new(bool_parts.remove(0))),
2636 _ => {
2637 let anys: Vec<&dyn arrow_array::Array> = bool_parts
2638 .iter()
2639 .map(|a| a as &dyn arrow_array::Array)
2640 .collect();
2641 let conc: ArrayRef = concat_arrays(&anys).ok()?;
2642 let ba = conc.as_any().downcast_ref::<BooleanArray>()?.clone();
2643 Some(std::sync::Arc::new(ba))
2644 }
2645 };
2646 }
2647
2648 let like_pattern = |text: &str, wildcards: bool| {
2650 let mut pattern = String::with_capacity(text.len());
2651 for ch in text.chars() {
2652 match ch {
2653 '%' | '_' | '\\' => {
2654 pattern.push('\\');
2655 pattern.push(ch);
2656 }
2657 '*' if wildcards => pattern.push('%'),
2658 '?' if wildcards => pattern.push('_'),
2659 _ => pattern.push(ch),
2660 }
2661 }
2662 pattern
2663 };
2664 let (text_kind, text_pat, empty_special) = match pred {
2665 crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Text(t)) => {
2666 (0u8, like_pattern(&t.to_lowercase(), false), t.is_empty())
2667 }
2668 crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Text(t)) => {
2669 (1u8, like_pattern(&t.to_lowercase(), false), false)
2670 }
2671 crate::args::CriteriaPredicate::TextLike {
2672 pattern,
2673 case_insensitive,
2674 } => {
2675 let p = if *case_insensitive {
2676 pattern.to_lowercase()
2677 } else {
2678 pattern.clone()
2679 };
2680 (2u8, like_pattern(&p, true), false)
2681 }
2682 _ => return None,
2683 };
2684
2685 let text_pat_is_empty = text_pat.is_empty();
2686 let ne_matches_blank = text_kind == 1 && !text_pat_is_empty;
2687 let pat = StringArray::new_scalar(text_pat);
2688 let mut bool_parts: Vec<BooleanArray> = Vec::new();
2689
2690 let mut tag_slices = view.type_tags_slices();
2691 for res in view.iter_row_chunks() {
2692 let cs = res.ok()?;
2693 if cs.row_len == 0 {
2694 continue;
2695 }
2696 #[cfg(test)]
2697 criteria_mask_test_hooks::inc_total();
2698
2699 let slices = view.slice_lowered_text(cs.row_start, cs.row_len);
2700 if col_in_view >= slices.len() {
2701 return None;
2702 }
2703
2704 let seg_opt = slices[col_in_view].as_ref().map(|a| a.as_ref());
2705 if empty_special || (text_kind == 1 && text_pat_is_empty) {
2706 let (tag_start, tag_len, tags) = tag_slices.next()?.ok()?;
2707 if tag_start != cs.row_start || tag_len != cs.row_len {
2708 return None;
2709 }
2710 let tags = tags.get(col_in_view)?;
2711 let strings = seg_opt.and_then(|a| a.as_any().downcast_ref::<StringArray>());
2715 let mut bb = BooleanBuilder::with_capacity(cs.row_len);
2716 for i in 0..cs.row_len {
2717 let blank = tags.value(i) == crate::arrow_store::TypeTag::Empty as u8
2718 || (tags.value(i) == crate::arrow_store::TypeTag::Text as u8
2719 && strings.is_some_and(|s| s.is_valid(i) && s.value(i).is_empty()));
2720 bb.append_value(if text_kind == 0 { blank } else { !blank });
2721 }
2722 #[cfg(test)]
2723 if seg_opt.is_none() {
2724 criteria_mask_test_hooks::inc_all_null();
2725 }
2726 bool_parts.push(bb.finish());
2727 continue;
2728 }
2729 let seg = match seg_opt {
2730 Some(s) => s,
2731 None => {
2732 #[cfg(test)]
2733 criteria_mask_test_hooks::inc_all_null();
2734 if (text_kind == 0 && empty_special) || ne_matches_blank {
2735 let mut bb = BooleanBuilder::with_capacity(cs.row_len);
2737 bb.append_n(cs.row_len, true);
2738 bool_parts.push(bb.finish());
2739 } else {
2740 bool_parts.push(BooleanArray::new_null(cs.row_len));
2742 }
2743 continue;
2744 }
2745 };
2746
2747 let seg_sa = seg.as_any().downcast_ref::<StringArray>()?;
2748 let mut m = match text_kind {
2749 0 => ilike(seg_sa, &pat).ok()?,
2750 1 => nilike(seg_sa, &pat).ok()?,
2751 2 => ilike(seg_sa, &pat).ok()?,
2752 _ => return None,
2753 };
2754
2755 if ((text_kind == 0 && empty_special) || ne_matches_blank) && seg_sa.null_count() > 0 {
2761 let mut bb = BooleanBuilder::with_capacity(seg_sa.len());
2763 for i in 0..seg_sa.len() {
2764 bb.append_value(seg_sa.is_null(i));
2765 }
2766 let nulls = bb.finish();
2767 m = boolean::or_kleene(&m, &nulls).ok()?;
2768 }
2769
2770 bool_parts.push(m);
2771 }
2772
2773 if bool_parts.is_empty() {
2774 None
2775 } else if bool_parts.len() == 1 {
2776 Some(std::sync::Arc::new(bool_parts.remove(0)))
2777 } else {
2778 let anys: Vec<&dyn arrow_array::Array> = bool_parts
2779 .iter()
2780 .map(|a| a as &dyn arrow_array::Array)
2781 .collect();
2782 let conc: ArrayRef = concat_arrays(&anys).ok()?;
2783 let ba = conc.as_any().downcast_ref::<BooleanArray>()?.clone();
2784 Some(std::sync::Arc::new(ba))
2785 }
2786}
2787
2788#[derive(Debug, Clone)]
2789pub struct LayerInfo {
2790 pub vertex_count: usize,
2791 pub parallel_eligible: bool,
2792 pub sample_cells: Vec<String>, }
2794
2795#[derive(Debug, Clone)]
2796pub struct EvalPlan {
2797 pub total_vertices_to_evaluate: usize,
2798 pub layers: Vec<LayerInfo>,
2799 pub cycles_detected: usize,
2800 pub dirty_count: usize,
2801 pub volatile_count: usize,
2802 pub parallel_enabled: bool,
2803 pub estimated_parallel_layers: usize,
2804 pub target_cells: Vec<String>,
2805}
2806
2807#[inline]
2810fn plane_mode_ignored() -> bool {
2811 true
2812}
2813
2814#[cfg(feature = "test-support")]
2817#[doc(hidden)]
2818pub fn formula_plane_mode_ignored_for_test() -> bool {
2819 plane_mode_ignored()
2820}
2821
2822impl<R> Engine<R>
2823where
2824 R: EvaluationContext,
2825{
2826 pub fn new(resolver: R, config: EvalConfig) -> Self {
2833 let config = if plane_mode_ignored() {
2838 EvalConfig {
2839 formula_plane_mode: FormulaPlaneMode::Off,
2840 ..config
2841 }
2842 } else {
2843 config
2844 };
2845 if let Err(msg) = config.cycle.validate() {
2846 panic!("invalid CycleConfig: {msg}");
2847 }
2848 crate::builtins::load_builtins();
2849 let resolved_resources = crate::engine::resource_ledger::resolve_evaluation_budgets(
2850 &config.evaluation_budgets,
2851 config.max_vertices,
2852 config.max_memory_mb,
2853 config.max_eval_time,
2854 );
2855
2856 let clock = config.deterministic_mode.build_clock().unwrap_or_else(|_| {
2857 #[cfg(feature = "system-clock")]
2858 {
2859 Arc::new(crate::timezone::SystemClock::new(
2860 crate::timezone::TimeZoneSpec::default(),
2861 ))
2862 }
2863 #[cfg(not(feature = "system-clock"))]
2864 {
2865 Arc::new(crate::timezone::FixedClock::new(
2866 chrono::DateTime::UNIX_EPOCH,
2867 crate::timezone::TimeZoneSpec::Utc,
2868 ))
2869 }
2870 });
2871
2872 let thread_pool = if config.enable_parallel {
2874 let mut builder = ThreadPoolBuilder::new();
2875 if let Some(max_threads) = config.max_threads {
2876 builder = builder.num_threads(max_threads);
2877 }
2878
2879 match builder.build() {
2880 Ok(pool) => Some(Arc::new(pool)),
2881 Err(_) => {
2882 None
2884 }
2885 }
2886 } else {
2887 None
2888 };
2889
2890 let lookup_cache_max_bytes = config.lookup_index_cache_max_bytes;
2892 let function_provider_revision_seen = resolver.planning_semantic_revision();
2893 let mut engine = Self {
2894 criteria_mask_cache: Default::default(),
2895 graph: DependencyGraph::new_with_config(config.clone()),
2896 resolver,
2897 resolved_functions: Default::default(),
2898 resolved_sheet_names: FxHashMap::default(),
2899 resolution_sheet_revision: None,
2900 resolution_arrow_sheet_names: Vec::new(),
2901 resolved_arrow_sheets: Vec::new(),
2902 arrow_sheet_index: std::sync::OnceLock::new(),
2903 config,
2904 workbook_load_limits: crate::engine::WorkbookLoadLimits::default(),
2905 clock: crate::timezone::SnapshotClock::new(clock),
2906 thread_pool,
2907 recalc_epoch: 0,
2908 snapshot_id: std::sync::atomic::AtomicU64::new(1),
2909 topology_epoch: 0,
2910 cached_static_schedule: None,
2911 recent_schedules: Vec::new(),
2912 base_schedule: None,
2913 #[cfg(any(test, feature = "benchmark_internal"))]
2914 recalc_reuse_probe: std::sync::Mutex::new(RecalcReuseProbe::default()),
2915 spill_mgr: ShimSpillManager::default(),
2916 arrow_sheets: SheetStore::default(),
2917 format_registry: crate::format::FormatRegistry::default(),
2918 derived_formats: Default::default(),
2919 #[cfg(test)]
2920 derived_format_operations_for_test: std::sync::atomic::AtomicU64::new(0),
2921 #[cfg(test)]
2922 family_members_for_test: std::sync::atomic::AtomicU64::new(0),
2923 #[cfg(test)]
2924 invariant_bound_members_for_test: std::sync::atomic::AtomicU64::new(0),
2925 #[cfg(test)]
2926 lifted_members_for_test: std::sync::atomic::AtomicU64::new(0),
2927 #[cfg(test)]
2928 chained_members_for_test: std::sync::atomic::AtomicU64::new(0),
2929 #[cfg(test)]
2930 lane_clean_reads_for_test: std::sync::atomic::AtomicU64::new(0),
2931 #[cfg(test)]
2932 criteria_kernel_members_for_test: std::sync::atomic::AtomicU64::new(0),
2933 #[cfg(test)]
2934 memo_hits_for_test: std::sync::atomic::AtomicU64::new(0),
2935 compressed_at_build: None,
2936 #[cfg(test)]
2937 computed_overlay_set_explicit_entry_operations_for_test: 0,
2938 #[cfg(test)]
2939 computed_overlay_stale_clear_range_effects_for_test: 0,
2940 #[cfg(test)]
2941 computed_overlay_stale_clear_offset_attempts_for_test: 0,
2942 #[cfg(test)]
2943 computed_format_vector_allocations_for_test: std::sync::atomic::AtomicU64::new(0),
2944 has_edited: false,
2945 overlay_compactions: 0,
2946 computed_overlay_bytes_estimate: 0,
2947 computed_overlay_mirroring_disabled: false,
2948 force_materialize_range_views: false,
2949 row_bounds_cache: std::sync::RwLock::new(None),
2950 used_axis_bounds_cache: std::sync::RwLock::new(None),
2951 lookup_index_cache: LookupIndexCache::new(lookup_cache_max_bytes),
2952 source_cache: Arc::new(std::sync::RwLock::new(SourceCache::default())),
2953 source_formula_token: Arc::new(()),
2954 source_family_mode: crate::engine::SourceFamilyMode::from_env(),
2955 source_family_counters: crate::engine::SourceFamilyCounters::default(),
2956 recalc_plan_token: Arc::new(()),
2957 staged_formulas: std::collections::HashMap::new(),
2958 staged_formula_index: StagedFormulaIndex::default(),
2959 blocked_pending_spills: Vec::new(),
2960 spill_blocker_waiters: SpillBlockerWaiters::default(),
2961 row_visibility: FxHashMap::default(),
2962 row_visibility_mask_cache: std::sync::RwLock::new(FxHashMap::default()),
2963 formula_parse_diagnostics: Vec::new(),
2964 last_formula_ingest_report: None,
2965 formula_ingest_report_total: FormulaIngestReport::default(),
2966 active_cancel_flag: None,
2967 active_evaluation_deadline: None,
2968 action_depth: 0,
2969 last_virtual_dep_telemetry: VirtualDepTelemetry::default(),
2970 virtual_dep_fallback_activations: 0,
2971 last_cycle_telemetry: CycleTelemetry::default(),
2972 next_evaluation_resource_request_id: 1,
2973 evaluation_resource_request_depth: 0,
2974 active_evaluation_resource_request: None,
2975 last_evaluation_resource_request: None,
2976 evaluation_resource_baseline: EvaluationResourceBaselineStats::default(),
2977 evaluation_resource_request_started_at: None,
2978 evaluation_resource_budgets: resolved_resources.budgets,
2979 evaluation_resource_config_diagnostic: resolved_resources.diagnostic,
2980 active_resource_ledger: None,
2981 source_cache_footprints: Vec::new(),
2982 source_cache_accounted: 0,
2983 pending_iterative_redirty: Vec::new(),
2984 snapshot_evaluation_active: false,
2985 snapshot_volatile_redirty_pending: false,
2986 retained_scc_members: FxHashMap::default(),
2987 next_retained_scc_id: 0,
2988 retained_scc_config_fingerprint: 0,
2989 retained_scc_function_epoch_seen: 0,
2990 retained_scc_provider_revision_seen: None,
2991 retained_scc_dirty_at_begin: Vec::new(),
2992 iterative_state_values: FxHashMap::default(),
2993 function_semantic_epoch_seen: crate::function_registry::semantic_epoch(),
2994 function_provider_revision_seen,
2995 #[cfg(feature = "tracing")]
2996 trace_evaluation_counters: TraceEvaluationCounters::default(),
2997 #[cfg(test)]
2998 evaluation_request_begin_count_for_test: 0,
2999 #[cfg(any(test, feature = "test-support"))]
3000 before_prepared_span_commit_hook: None,
3001 #[cfg(test)]
3002 before_target_preparation_commit_hook: None,
3003 #[cfg(test)]
3004 before_target_planning_snapshot_hook: None,
3005 #[cfg(test)]
3006 inject_target_semantic_stale_once_for_test: false,
3007 #[cfg(test)]
3008 force_virtual_dep_changes_remaining_for_test: 0,
3009 freshness: Default::default(),
3010 #[cfg(test)]
3011 fail_evaluation_commit_preflight_once_for_test: false,
3012 #[cfg(test)]
3013 target_preparation_fault_for_test: None,
3014 #[cfg(test)]
3015 force_non_cycle_schedule_fallback_for_test: false,
3016 #[cfg(test)]
3017 before_legacy_fallback_final_provider_sample_hook: None,
3018 #[cfg(test)]
3019 after_eager_proposal_commit_hook: None,
3020 };
3021 engine.config.arrow_storage_enabled = true;
3023 engine.config.delta_overlay_enabled = true;
3024 engine.config.write_formula_overlay_enabled = true;
3025 let default_sheet = engine.graph.default_sheet_name().to_string();
3026 engine.ensure_arrow_sheet(&default_sheet);
3027 engine
3028 }
3029
3030 pub fn with_thread_pool(
3035 resolver: R,
3036 config: EvalConfig,
3037 thread_pool: Arc<rayon::ThreadPool>,
3038 ) -> Self {
3039 if let Err(msg) = config.cycle.validate() {
3040 panic!("invalid CycleConfig: {msg}");
3041 }
3042 crate::builtins::load_builtins();
3043 let resolved_resources = crate::engine::resource_ledger::resolve_evaluation_budgets(
3044 &config.evaluation_budgets,
3045 config.max_vertices,
3046 config.max_memory_mb,
3047 config.max_eval_time,
3048 );
3049 let clock = config.deterministic_mode.build_clock().unwrap_or_else(|_| {
3050 #[cfg(feature = "system-clock")]
3051 {
3052 Arc::new(crate::timezone::SystemClock::new(
3053 crate::timezone::TimeZoneSpec::default(),
3054 ))
3055 }
3056 #[cfg(not(feature = "system-clock"))]
3057 {
3058 Arc::new(crate::timezone::FixedClock::new(
3059 chrono::DateTime::UNIX_EPOCH,
3060 crate::timezone::TimeZoneSpec::Utc,
3061 ))
3062 }
3063 });
3064 let lookup_cache_max_bytes = config.lookup_index_cache_max_bytes;
3066 let function_provider_revision_seen = resolver.planning_semantic_revision();
3067 let mut engine = Self {
3068 criteria_mask_cache: Default::default(),
3069 graph: DependencyGraph::new_with_config(config.clone()),
3070 resolver,
3071 resolved_functions: Default::default(),
3072 resolved_sheet_names: FxHashMap::default(),
3073 resolution_sheet_revision: None,
3074 resolution_arrow_sheet_names: Vec::new(),
3075 resolved_arrow_sheets: Vec::new(),
3076 arrow_sheet_index: std::sync::OnceLock::new(),
3077 config,
3078 workbook_load_limits: crate::engine::WorkbookLoadLimits::default(),
3079 clock: crate::timezone::SnapshotClock::new(clock),
3080 thread_pool: Some(thread_pool),
3081 recalc_epoch: 0,
3082 snapshot_id: std::sync::atomic::AtomicU64::new(1),
3083 topology_epoch: 0,
3084 cached_static_schedule: None,
3085 recent_schedules: Vec::new(),
3086 base_schedule: None,
3087 #[cfg(any(test, feature = "benchmark_internal"))]
3088 recalc_reuse_probe: std::sync::Mutex::new(RecalcReuseProbe::default()),
3089 spill_mgr: ShimSpillManager::default(),
3090 arrow_sheets: SheetStore::default(),
3091 format_registry: crate::format::FormatRegistry::default(),
3092 derived_formats: Default::default(),
3093 #[cfg(test)]
3094 derived_format_operations_for_test: std::sync::atomic::AtomicU64::new(0),
3095 #[cfg(test)]
3096 family_members_for_test: std::sync::atomic::AtomicU64::new(0),
3097 #[cfg(test)]
3098 invariant_bound_members_for_test: std::sync::atomic::AtomicU64::new(0),
3099 #[cfg(test)]
3100 lifted_members_for_test: std::sync::atomic::AtomicU64::new(0),
3101 #[cfg(test)]
3102 chained_members_for_test: std::sync::atomic::AtomicU64::new(0),
3103 #[cfg(test)]
3104 lane_clean_reads_for_test: std::sync::atomic::AtomicU64::new(0),
3105 #[cfg(test)]
3106 criteria_kernel_members_for_test: std::sync::atomic::AtomicU64::new(0),
3107 #[cfg(test)]
3108 memo_hits_for_test: std::sync::atomic::AtomicU64::new(0),
3109 compressed_at_build: None,
3110 #[cfg(test)]
3111 computed_overlay_set_explicit_entry_operations_for_test: 0,
3112 #[cfg(test)]
3113 computed_overlay_stale_clear_range_effects_for_test: 0,
3114 #[cfg(test)]
3115 computed_overlay_stale_clear_offset_attempts_for_test: 0,
3116 #[cfg(test)]
3117 computed_format_vector_allocations_for_test: std::sync::atomic::AtomicU64::new(0),
3118 has_edited: false,
3119 overlay_compactions: 0,
3120 computed_overlay_bytes_estimate: 0,
3121 computed_overlay_mirroring_disabled: false,
3122 force_materialize_range_views: false,
3123 row_bounds_cache: std::sync::RwLock::new(None),
3124 used_axis_bounds_cache: std::sync::RwLock::new(None),
3125 lookup_index_cache: LookupIndexCache::new(lookup_cache_max_bytes),
3126 source_cache: Arc::new(std::sync::RwLock::new(SourceCache::default())),
3127 source_formula_token: Arc::new(()),
3128 source_family_mode: crate::engine::SourceFamilyMode::from_env(),
3129 source_family_counters: crate::engine::SourceFamilyCounters::default(),
3130 recalc_plan_token: Arc::new(()),
3131 staged_formulas: std::collections::HashMap::new(),
3132 staged_formula_index: StagedFormulaIndex::default(),
3133 blocked_pending_spills: Vec::new(),
3134 spill_blocker_waiters: SpillBlockerWaiters::default(),
3135 row_visibility: FxHashMap::default(),
3136 row_visibility_mask_cache: std::sync::RwLock::new(FxHashMap::default()),
3137 formula_parse_diagnostics: Vec::new(),
3138 last_formula_ingest_report: None,
3139 formula_ingest_report_total: FormulaIngestReport::default(),
3140 active_cancel_flag: None,
3141 active_evaluation_deadline: None,
3142 action_depth: 0,
3143 last_virtual_dep_telemetry: VirtualDepTelemetry::default(),
3144 virtual_dep_fallback_activations: 0,
3145 last_cycle_telemetry: CycleTelemetry::default(),
3146 next_evaluation_resource_request_id: 1,
3147 evaluation_resource_request_depth: 0,
3148 active_evaluation_resource_request: None,
3149 last_evaluation_resource_request: None,
3150 evaluation_resource_baseline: EvaluationResourceBaselineStats::default(),
3151 evaluation_resource_request_started_at: None,
3152 evaluation_resource_budgets: resolved_resources.budgets,
3153 evaluation_resource_config_diagnostic: resolved_resources.diagnostic,
3154 active_resource_ledger: None,
3155 source_cache_footprints: Vec::new(),
3156 source_cache_accounted: 0,
3157 pending_iterative_redirty: Vec::new(),
3158 snapshot_evaluation_active: false,
3159 snapshot_volatile_redirty_pending: false,
3160 retained_scc_members: FxHashMap::default(),
3161 next_retained_scc_id: 0,
3162 retained_scc_config_fingerprint: 0,
3163 retained_scc_function_epoch_seen: 0,
3164 retained_scc_provider_revision_seen: None,
3165 retained_scc_dirty_at_begin: Vec::new(),
3166 iterative_state_values: FxHashMap::default(),
3167 function_semantic_epoch_seen: crate::function_registry::semantic_epoch(),
3168 function_provider_revision_seen,
3169 #[cfg(feature = "tracing")]
3170 trace_evaluation_counters: TraceEvaluationCounters::default(),
3171 #[cfg(test)]
3172 evaluation_request_begin_count_for_test: 0,
3173 #[cfg(any(test, feature = "test-support"))]
3174 before_prepared_span_commit_hook: None,
3175 #[cfg(test)]
3176 before_target_preparation_commit_hook: None,
3177 #[cfg(test)]
3178 before_target_planning_snapshot_hook: None,
3179 #[cfg(test)]
3180 inject_target_semantic_stale_once_for_test: false,
3181 #[cfg(test)]
3182 force_virtual_dep_changes_remaining_for_test: 0,
3183 freshness: Default::default(),
3184 #[cfg(test)]
3185 fail_evaluation_commit_preflight_once_for_test: false,
3186 #[cfg(test)]
3187 target_preparation_fault_for_test: None,
3188 #[cfg(test)]
3189 force_non_cycle_schedule_fallback_for_test: false,
3190 #[cfg(test)]
3191 before_legacy_fallback_final_provider_sample_hook: None,
3192 #[cfg(test)]
3193 after_eager_proposal_commit_hook: None,
3194 };
3195 engine.config.arrow_storage_enabled = true;
3197 engine.config.delta_overlay_enabled = true;
3198 engine.config.write_formula_overlay_enabled = true;
3199 let default_sheet = engine.graph.default_sheet_name().to_string();
3200 engine.ensure_arrow_sheet(&default_sheet);
3201 engine
3202 }
3203
3204 pub fn workbook_load_limits(&self) -> &crate::engine::WorkbookLoadLimits {
3205 &self.workbook_load_limits
3206 }
3207
3208 pub fn set_workbook_load_limits(&mut self, limits: crate::engine::WorkbookLoadLimits) {
3209 self.workbook_load_limits = limits;
3210 }
3211
3212 fn clear_source_cache(&self) {
3213 if let Ok(mut g) = self.source_cache.write() {
3214 *g = SourceCache::default();
3215 }
3216 }
3217
3218 pub fn last_virtual_dep_telemetry(&self) -> &VirtualDepTelemetry {
3219 &self.last_virtual_dep_telemetry
3220 }
3221
3222 pub fn last_cycle_telemetry(&self) -> &CycleTelemetry {
3226 &self.last_cycle_telemetry
3227 }
3228
3229 pub fn last_evaluation_resource_request_stats(
3231 &self,
3232 ) -> Option<&EvaluationResourceRequestStats> {
3233 self.last_evaluation_resource_request.as_ref()
3234 }
3235
3236 pub fn evaluation_resource_baseline_stats(&self) -> EvaluationResourceBaselineStats {
3238 self.evaluation_resource_baseline
3239 }
3240
3241 pub fn evaluation_resource_budgets(&self) -> &crate::engine::EvaluationBudgets {
3242 &self.evaluation_resource_budgets
3243 }
3244
3245 pub fn evaluation_resource_config_diagnostic(
3247 &self,
3248 ) -> Option<&crate::engine::EvaluationResourceConfigDiagnostic> {
3249 self.evaluation_resource_config_diagnostic.as_ref()
3250 }
3251
3252 pub fn reset_evaluation_resource_telemetry(&mut self) {
3254 self.evaluation_resource_baseline = EvaluationResourceBaselineStats::default();
3255 self.last_evaluation_resource_request = None;
3256 }
3257
3258 fn reconcile_source_cache_footprints(&mut self) -> Result<(), ExcelError> {
3261 self.blocked_pending_spills.retain(|&(vertex, anchor, _)| {
3262 self.graph.vertex_exists(vertex)
3263 && self.graph.get_cell_ref(vertex) == Some(anchor)
3264 && matches!(
3265 self.graph.get_vertex_kind(vertex),
3266 VertexKind::FormulaScalar | VertexKind::FormulaArray
3267 )
3268 });
3269 if self.blocked_pending_spills.is_empty() {
3270 self.blocked_pending_spills = Vec::new();
3271 }
3272 let mut bytes = (self.blocked_pending_spills.capacity()
3273 * std::mem::size_of::<(VertexId, CellRef, Region)>()) as u64;
3274 self.source_cache_footprints.retain(|weak| {
3275 let Some(footprint) = weak.upgrade() else {
3276 return false;
3277 };
3278 bytes = bytes.saturating_add(footprint.load(std::sync::atomic::Ordering::Acquire));
3279 true
3280 });
3281 if let Some(ledger) = self.active_resource_ledger.as_mut() {
3282 ledger
3283 .release_retained(self.source_cache_accounted)
3284 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
3285 ledger.observe_retained(bytes);
3287 self.source_cache_accounted = bytes;
3288 ledger
3289 .reserve_retained(0)
3290 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
3291 }
3292 Ok(())
3293 }
3294
3295 fn duration_ns(duration: std::time::Duration) -> u64 {
3296 u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX)
3297 }
3298
3299 fn observe_evaluation_resource_request<T>(
3300 &mut self,
3301 kind: EvaluationRequestKind,
3302 evaluate: impl FnOnce(&mut Self) -> Result<T, ExcelError>,
3303 ) -> Result<T, ExcelError> {
3304 let outermost = self.evaluation_resource_request_depth == 0;
3305 if outermost && self.snapshot_volatile_redirty_pending {
3306 self.snapshot_volatile_redirty_pending = false;
3307 self.graph.redirty_volatiles();
3308 }
3309 #[cfg(feature = "tracing")]
3310 if outermost {
3311 self.trace_evaluation_counters = TraceEvaluationCounters::default();
3312 }
3313 #[cfg(feature = "tracing")]
3314 let request_kind = if matches!(
3315 kind,
3316 EvaluationRequestKind::Full
3317 | EvaluationRequestKind::FullWithDelta
3318 | EvaluationRequestKind::FullCancellable
3319 | EvaluationRequestKind::FullLogged
3320 ) {
3321 "full"
3322 } else {
3323 "targeted"
3324 };
3325 #[cfg(feature = "tracing")]
3326 let _request_span = outermost.then(|| {
3327 crate::engine::trace::fz_span!(
3328 tracing::Level::INFO,
3329 "evaluate",
3330 "evaluate.request",
3331 kind = request_kind,
3332 mode = ?FormulaPlaneMode::Off
3333 )
3334 });
3335 if outermost {
3336 let request_id = self.next_evaluation_resource_request_id;
3337 self.next_evaluation_resource_request_id = request_id
3338 .checked_add(1)
3339 .expect("evaluation resource request ID exhausted");
3340 self.active_evaluation_resource_request = Some(EvaluationResourceRequestStats::new(
3341 request_id,
3342 kind,
3343 FormulaPlaneMode::Off,
3344 self.staged_formula_count(),
3345 ));
3346 self.evaluation_resource_baseline.record_started(request_id);
3347 self.evaluation_resource_request_started_at = Some(crate::instant::FzInstant::now());
3348 self.source_cache_accounted = 0;
3349 self.active_resource_ledger = Some(ResourceLedger::new(
3350 Some(request_id),
3351 self.evaluation_resource_budgets.clone(),
3352 ));
3353 }
3354 self.evaluation_resource_request_depth =
3355 self.evaluation_resource_request_depth.saturating_add(1);
3356 let result = if outermost {
3357 self.reconcile_source_cache_footprints()
3358 .and_then(|()| self.resource_checkpoint(0))
3359 .and_then(|()| evaluate(self))
3360 } else {
3361 evaluate(self)
3362 };
3363 if outermost && result.is_err() {
3364 self.freshness_abort_pass();
3365 }
3366 self.evaluation_resource_request_depth =
3367 self.evaluation_resource_request_depth.saturating_sub(1);
3368
3369 let reconciliation = if outermost {
3370 self.reconcile_source_cache_footprints()
3371 } else {
3372 Ok(())
3373 };
3374 let result = result.and_then(|value| reconciliation.map(|()| value));
3375 if outermost {
3376 let total_ns = self
3377 .evaluation_resource_request_started_at
3378 .take()
3379 .map(|start| Self::duration_ns(start.elapsed()))
3380 .unwrap_or(0);
3381 let mut stats = self
3382 .active_evaluation_resource_request
3383 .take()
3384 .expect("outer evaluation resource request has active stats");
3385 let mut ledger = self
3386 .active_resource_ledger
3387 .take()
3388 .expect("outer evaluation resource request has active ledger");
3389 ledger.release_all_scratch();
3390 stats.ledger.update(ledger.snapshot());
3391 stats.outcome = match &result {
3392 Ok(_) => EvaluationRequestOutcome::Success,
3393 Err(error) if error.kind == ExcelErrorKind::Cancelled => {
3394 EvaluationRequestOutcome::Cancelled
3395 }
3396 Err(_) => EvaluationRequestOutcome::Error,
3397 };
3398 if stats.dirty_lease == FormulaDirtyLeaseOutcome::Acquired {
3399 stats.dirty_lease = if stats.outcome == EvaluationRequestOutcome::Cancelled {
3400 FormulaDirtyLeaseOutcome::RetainedOnCancellation
3401 } else {
3402 FormulaDirtyLeaseOutcome::RetainedOnError
3403 };
3404 }
3405 stats.phases.total_ns = total_ns;
3406 let attributed = stats
3407 .phases
3408 .staged_prepare_ns
3409 .saturating_add(stats.phases.topology_ns)
3410 .saturating_add(stats.phases.materialization_ns);
3411 stats.phases.evaluation_ns = total_ns.saturating_sub(attributed);
3412 self.evaluation_resource_baseline.record_finished(&stats);
3413 self.last_evaluation_resource_request = Some(stats);
3414 crate::engine::trace::fz_event!(
3415 tracing::Level::INFO,
3416 "evaluate",
3417 "evaluate.summary",
3418 computed_vertices = self.trace_evaluation_counters.computed_vertices,
3419 cycles = self.trace_evaluation_counters.cycles,
3420 cancelled = matches!(
3421 &result,
3422 Err(error) if error.kind == ExcelErrorKind::Cancelled
3423 )
3424 );
3425 }
3426 result
3427 }
3428
3429 pub fn set_evaluation_resource_budgets(&mut self, budgets: crate::engine::EvaluationBudgets) {
3430 self.evaluation_resource_budgets = budgets.clone();
3431 self.config.evaluation_budgets = budgets.clone();
3432 self.graph.set_evaluation_budgets(budgets);
3433 }
3434
3435 #[cfg(test)]
3436 pub(crate) fn set_evaluation_budgets_for_test(
3437 &mut self,
3438 budgets: crate::engine::EvaluationBudgets,
3439 ) {
3440 self.set_evaluation_resource_budgets(budgets);
3441 }
3442
3443 fn preflight_evaluation_commit_window(
3444 &mut self,
3445 bounded_writes: usize,
3446 ) -> Result<crate::instant::FzInstant, ExcelError> {
3447 #[cfg(test)]
3448 if std::mem::take(&mut self.fail_evaluation_commit_preflight_once_for_test) {
3449 return Err(crate::engine::ResourceLedgerError::Exhausted(
3450 formualizer_common::ResourceExhaustionDetail {
3451 reason: formualizer_common::ResourceExhaustionReason::Deadline,
3452 limit: 0,
3453 observed: 1,
3454 request_id: self
3455 .active_evaluation_resource_request
3456 .as_ref()
3457 .map(|stats| stats.request_id),
3458 },
3459 )
3460 .into_excel_error());
3461 }
3462 let estimate = std::time::Duration::from_nanos(
3463 u64::try_from(bounded_writes)
3464 .unwrap_or(u64::MAX)
3465 .saturating_mul(100),
3466 );
3467 if let Some(ledger) = self.active_resource_ledger.as_mut() {
3468 ledger
3469 .preflight_commit_window(estimate)
3470 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
3471 }
3472 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3473 stats.evaluation_commit_preflight_count =
3474 stats.evaluation_commit_preflight_count.saturating_add(1);
3475 stats.evaluation_commit_estimated_ns = stats
3476 .evaluation_commit_estimated_ns
3477 .saturating_add(Self::duration_ns(estimate));
3478 }
3479 Ok(crate::instant::FzInstant::now())
3480 }
3481
3482 fn observe_evaluation_commit_window(&mut self, started: crate::instant::FzInstant) {
3483 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3484 stats.evaluation_commit_actual_ns = stats
3485 .evaluation_commit_actual_ns
3486 .saturating_add(Self::duration_ns(started.elapsed()));
3487 }
3488 }
3489
3490 fn live_cancellation_after_work(&self, message: &'static str) -> Result<(), ExcelError> {
3502 if self
3503 .active_cancel_flag
3504 .as_ref()
3505 .is_some_and(|cancel| cancel.is_cancelled())
3506 {
3507 return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(message));
3508 }
3509 Ok(())
3510 }
3511
3512 fn cancellation_checkpoint(&self, message: &'static str) -> Result<(), ExcelError> {
3513 if self
3514 .active_cancel_flag
3515 .as_ref()
3516 .is_some_and(|cancel| cancel.is_cancelled())
3517 {
3518 return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(message));
3519 }
3520 if self
3521 .active_evaluation_deadline
3522 .is_some_and(|deadline| Instant::now() >= deadline)
3523 {
3524 return Err(crate::engine::ResourceLedgerError::Exhausted(
3525 formualizer_common::ResourceExhaustionDetail {
3526 reason: formualizer_common::ResourceExhaustionReason::Deadline,
3527 limit: 0,
3528 observed: 1,
3529 request_id: self
3530 .active_evaluation_resource_request
3531 .as_ref()
3532 .map(|request| request.request_id),
3533 },
3534 )
3535 .into_excel_error()
3536 .with_message(message));
3537 }
3538 Ok(())
3539 }
3540
3541 fn resource_checkpoint(&mut self, work_units: u64) -> Result<(), ExcelError> {
3542 let Some(ledger) = self.active_resource_ledger.as_mut() else {
3543 return Ok(());
3544 };
3545 ledger
3546 .charge_work(work_units)
3547 .and_then(|()| ledger.checkpoint_deadline())
3548 .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3549 }
3550
3551 fn charge_bounded_work(&mut self, mut work_units: u64) -> Result<(), ExcelError> {
3552 if work_units == 0 {
3553 return self.resource_checkpoint(0);
3554 }
3555 while work_units > 0 {
3556 let chunk = work_units.min(256);
3557 self.resource_checkpoint(chunk)?;
3558 work_units -= chunk;
3559 }
3560 Ok(())
3561 }
3562
3563 fn reserve_request_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3564 if let Some(ledger) = self.active_resource_ledger.as_mut() {
3565 ledger.observe_scratch(bytes);
3568 }
3569 Ok(())
3570 }
3571
3572 fn release_request_scratch(&mut self, bytes: u64) {
3573 if let Some(ledger) = self.active_resource_ledger.as_mut() {
3574 let released = ledger.release_scratch(bytes);
3575 debug_assert!(
3576 released.is_ok(),
3577 "request scratch release exceeded the outstanding reservation"
3578 );
3579 }
3580 }
3581
3582 fn reserve_topology_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3583 self.active_resource_ledger
3584 .as_mut()
3585 .map_or(Ok(()), |ledger| ledger.reserve_schedule_discovery(bytes))
3586 .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3587 }
3588
3589 fn reserve_graph_source_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3590 self.active_resource_ledger
3591 .as_mut()
3592 .map_or(Ok(()), |ledger| ledger.reserve_graph_source(bytes))
3593 .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3594 }
3595
3596 fn with_request_scratch<T>(
3597 &mut self,
3598 bytes: u64,
3599 work: impl FnOnce(&mut Self) -> Result<T, ExcelError>,
3600 ) -> Result<T, ExcelError> {
3601 self.reserve_request_scratch(bytes)?;
3602 let result = work(self);
3603 self.release_request_scratch(bytes);
3604 result
3605 }
3606
3607 fn observe_staged_preparation(
3608 &mut self,
3609 selected: usize,
3610 retained: usize,
3611 elapsed: std::time::Duration,
3612 ) {
3613 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3614 stats.staged_selected = stats.staged_selected.saturating_add(selected as u64);
3615 stats.staged_retained = retained as u64;
3616 stats.phases.staged_prepare_ns = stats
3617 .phases
3618 .staged_prepare_ns
3619 .saturating_add(Self::duration_ns(elapsed));
3620 }
3621 }
3622
3623 fn graph_admission_enabled(&self) -> bool {
3624 crate::engine::resource_ledger::graph_admission_enabled(&self.evaluation_resource_budgets)
3625 }
3626
3627 fn preflight_graph_admission(
3628 &mut self,
3629 usage: crate::engine::resource_ledger::GraphAdmission,
3630 ) -> Result<(), ExcelError> {
3631 let request_id = self
3632 .active_evaluation_resource_request
3633 .as_ref()
3634 .map(|stats| stats.request_id);
3635 crate::engine::resource_ledger::preflight_graph_admission(
3636 &self.evaluation_resource_budgets,
3637 usage,
3638 request_id,
3639 )
3640 .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3641 }
3642
3643 fn prepared_legacy_admission(
3644 &mut self,
3645 plan: &PreparedLegacyGraphPlan,
3646 materialization_cells: u64,
3647 ) -> Result<(), ExcelError> {
3648 if !self.graph_admission_enabled() {
3649 return Ok(());
3650 }
3651 let stats = self.graph.baseline_stats();
3652 let added_edges = plan.planned_edge_count().ok_or_else(|| {
3653 ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3654 })?;
3655 let removed_edges = plan.removed_edge_count().ok_or_else(|| {
3656 ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3657 })?;
3658 self.preflight_graph_admission(crate::engine::resource_ledger::GraphAdmission {
3659 final_vertices: stats
3660 .graph_vertex_count
3661 .checked_add(plan.new_vertex_count())
3662 .ok_or_else(|| {
3663 ExcelError::new(ExcelErrorKind::NImpl)
3664 .with_message("graph vertex count overflow")
3665 })?,
3666 final_edges: stats
3667 .graph_edge_count
3668 .checked_sub(removed_edges)
3669 .and_then(|count| count.checked_add(added_edges))
3670 .ok_or_else(|| {
3671 ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3672 })?,
3673 materialization_cells,
3674 added_vertices: plan.new_vertex_count(),
3675 added_edges,
3676 })
3677 }
3678
3679 fn observe_target_admission_failure(
3680 &mut self,
3681 reason: formualizer_common::ResourceExhaustionReason,
3682 ) {
3683 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3684 stats.target_admission_failure = Some(reason);
3685 }
3686 }
3687
3688 fn observe_target_preparation_report(
3689 &mut self,
3690 report: &crate::engine::PreparedTargetGraphReport,
3691 ) {
3692 let reason_bit = |reason: crate::engine::OpaqueReason| -> u64 {
3693 let index = match reason {
3694 crate::engine::OpaqueReason::DynamicReference => 0,
3695 crate::engine::OpaqueReason::RuntimeTextReference => 1,
3696 crate::engine::OpaqueReason::UnknownFunction => 2,
3697 crate::engine::OpaqueReason::UnknownCustomFunction => 3,
3698 crate::engine::OpaqueReason::UnresolvedCrossSheetBinding => 4,
3699 crate::engine::OpaqueReason::UnresolvedName => 5,
3700 crate::engine::OpaqueReason::UnresolvedTable => 6,
3701 crate::engine::OpaqueReason::FormulaName => 7,
3702 crate::engine::OpaqueReason::DeferredSourcePackage => 8,
3703 crate::engine::OpaqueReason::UnsupportedSourceSemantics => 9,
3704 crate::engine::OpaqueReason::UncertainDefaultSheetBinding => 10,
3705 };
3706 1u64 << index
3707 };
3708 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3709 stats.staged_selected = report.selected_staged_cells as u64;
3710 stats.staged_retained = report.retained_staged_cells as u64;
3711 stats.target_requested = report.requested_targets as u64;
3712 stats.target_normalized_regions = report.normalized_regions as u64;
3713 stats.target_scope_level = match &report.widened_scope {
3714 crate::engine::PrepareScope::Exact => 0,
3715 crate::engine::PrepareScope::Sheets(_) => 1,
3716 crate::engine::PrepareScope::Workbook => 2,
3717 };
3718 stats.target_widening_reason_bits = report
3719 .widening_reasons
3720 .iter()
3721 .copied()
3722 .fold(0, |bits, reason| bits | reason_bit(reason));
3723 stats.graph_source_scratch_estimated = report.estimated_scratch_bytes;
3724 stats.graph_source_scratch_observed = report.observed_scratch_bytes;
3725 stats.target_commit_estimated_work = report.estimated_commit_work;
3726 stats.target_commit_actual_work = report.actual_commit_work;
3727 stats.target_commit_window_ns = Self::duration_ns(report.commit_window);
3728 stats.phases.staged_prepare_ns = stats
3729 .phases
3730 .staged_prepare_ns
3731 .saturating_add(Self::duration_ns(report.commit_window));
3732 }
3733 }
3734
3735 pub(crate) fn current_resolved_functions(
3741 &self,
3742 ) -> Option<&crate::interpreter::ResolvedFunctions> {
3743 self.resolved_functions
3744 .is_current(self.graph.data_store(), &self.resolver)
3745 .then_some(&self.resolved_functions)
3746 }
3747
3748 fn refresh_resolution_snapshots(&mut self) {
3749 self.resolved_functions
3750 .refresh(self.graph.data_store(), &self.resolver);
3751 let named_asts = self
3752 .graph
3753 .named_ranges_iter()
3754 .map(|(_, named)| named)
3755 .chain(self.graph.sheet_named_ranges_iter().map(|(_, named)| named))
3756 .filter_map(|named| match &named.definition {
3757 NamedDefinition::Formula { ast, .. } => Some(ast),
3758 _ => None,
3759 });
3760 self.resolved_functions.refresh_ast_functions(
3761 self.graph.data_store(),
3762 &self.resolver,
3763 named_asts,
3764 self.graph.symbol_revision(),
3765 );
3766 let revision = (
3770 self.graph.sheet_reg().revision(),
3771 self.graph.data_store().resolution_revision(),
3772 self.graph.data_store().sheet_names().count(),
3773 );
3774 let arrow_names_unchanged = self
3775 .resolution_arrow_sheet_names
3776 .iter()
3777 .map(|name| name.as_ref())
3778 .eq(self
3779 .arrow_sheets
3780 .sheets
3781 .iter()
3782 .map(|sheet| sheet.name.as_ref()));
3783 if self.resolution_sheet_revision == Some(revision) && arrow_names_unchanged {
3784 return;
3785 }
3786 self.resolution_arrow_sheet_names.clear();
3787 self.resolution_arrow_sheet_names.extend(
3788 self.arrow_sheets
3789 .sheets
3790 .iter()
3791 .map(|sheet| Arc::clone(&sheet.name)),
3792 );
3793 self.resolved_sheet_names.clear();
3794 for id in self.graph.data_store().sheet_names() {
3795 let name = self.graph.data_store().resolve_ast_string(id);
3796 if let Some(sheet_id) = self.graph.sheet_id(name) {
3797 self.resolved_sheet_names.insert(
3798 id,
3799 (
3800 sheet_id,
3801 self.arrow_sheets
3802 .sheets
3803 .iter()
3804 .position(|sheet| sheet.name.as_ref() == name),
3805 ),
3806 );
3807 }
3808 }
3809 self.resolved_arrow_sheets.clear();
3810 for (index, sheet) in self.arrow_sheets.sheets.iter().enumerate() {
3811 if let Some(id) = self.graph.sheet_id(&sheet.name) {
3812 let len = usize::from(id).saturating_add(1);
3813 if self.resolved_arrow_sheets.len() < len {
3814 self.resolved_arrow_sheets.resize(len, None);
3815 }
3816 if let Some(slot) = self.resolved_arrow_sheets.get_mut(usize::from(id)) {
3817 *slot = Some(index);
3818 }
3819 }
3820 }
3821 self.resolution_sheet_revision = Some(revision);
3822 }
3823
3824 fn begin_evaluation_request(&mut self) {
3825 self.refresh_resolution_snapshots();
3826 self.freshness_begin_request();
3827 #[cfg(test)]
3828 {
3829 self.evaluation_request_begin_count_for_test = self
3830 .evaluation_request_begin_count_for_test
3831 .saturating_add(1);
3832 }
3833 self.last_cycle_telemetry = CycleTelemetry::default();
3834 self.graph.authority_sync();
3835 self.pending_iterative_redirty.clear();
3838 self.reconcile_retained_sccs_at_request_begin();
3839 self.clock.refresh();
3843 }
3844
3845 fn redirty_for_next_recalc(&mut self) {
3854 if self.snapshot_evaluation_active {
3855 self.snapshot_volatile_redirty_pending = true;
3856 } else {
3857 self.graph.redirty_volatiles();
3858 }
3859 let pending = std::mem::take(&mut self.pending_iterative_redirty);
3860 let dirty_at_begin = std::mem::take(&mut self.retained_scc_dirty_at_begin);
3861 for (vertex, scc) in dirty_at_begin {
3862 if self.retained_scc_members.get(&vertex) == Some(&scc) {
3863 self.retained_scc_members.remove(&vertex);
3869 if !self.graph.is_dirty(vertex) {
3870 self.iterative_state_values.remove(&vertex);
3871 }
3872 }
3873 }
3874 if !self.iterative_state_values.is_empty() || !self.retained_scc_members.is_empty() {
3875 let graph = &self.graph;
3876 self.iterative_state_values
3877 .retain(|vertex, _| graph.is_live_formula_vertex(*vertex));
3878 self.retained_scc_members
3879 .retain(|vertex, _| graph.is_live_formula_vertex(*vertex));
3880 }
3881 for &vertex in &pending {
3886 if !self.graph.is_live_formula_vertex(vertex) {
3887 continue;
3888 }
3889 if let Some(cell) = self.graph.get_cell_ref(vertex) {
3890 let sheet_name = self.graph.sheet_name(cell.sheet_id);
3891 match self.read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1) {
3892 Some(value) if !matches!(value, LiteralValue::Empty) => {
3893 self.iterative_state_values.insert(vertex, value);
3894 }
3895 _ => {
3896 self.iterative_state_values.remove(&vertex);
3897 }
3898 }
3899 }
3900 }
3901 if !pending.is_empty() {
3902 self.graph.redirty_iterative_members(&pending);
3903 }
3904 }
3905
3906 fn retained_scc_config_fingerprint(&self) -> u64 {
3911 use std::hash::{Hash, Hasher};
3912 let mut hasher = rustc_hash::FxHasher::default();
3913 let config = &self.config;
3914 std::mem::discriminant(&config.cycle.detection).hash(&mut hasher);
3915 match config.cycle.policy {
3916 CyclePolicy::Error => 0u8.hash(&mut hasher),
3917 CyclePolicy::Iterate {
3918 max_iterations,
3919 max_change,
3920 } => {
3921 1u8.hash(&mut hasher);
3922 max_iterations.hash(&mut hasher);
3923 max_change.to_bits().hash(&mut hasher);
3924 }
3925 }
3926 config.date_system.hash(&mut hasher);
3927 config.workbook_seed.hash(&mut hasher);
3928 std::mem::discriminant(&config.volatile_level).hash(&mut hasher);
3929 format!("{:?}", config.deterministic_mode).hash(&mut hasher);
3932 config.range_expansion_limit.hash(&mut hasher);
3933 config.max_open_ended_rows.hash(&mut hasher);
3934 config.max_open_ended_cols.hash(&mut hasher);
3935 hasher.finish()
3936 }
3937
3938 fn reconcile_retained_sccs_at_request_begin(&mut self) {
3944 self.retained_scc_dirty_at_begin.clear();
3945 if self.retained_scc_members.is_empty() {
3946 return;
3947 }
3948 let graph = &self.graph;
3951 self.retained_scc_members
3952 .retain(|vertex, _| graph.is_live_formula_vertex(*vertex));
3953 if self.retained_scc_config_fingerprint() != self.retained_scc_config_fingerprint {
3954 let members: Vec<VertexId> = self.retained_scc_members.keys().copied().collect();
3955 self.retained_scc_members.clear();
3956 self.graph.mark_dirty_many(&members);
3957 return;
3958 }
3959 let changes =
3960 crate::function_registry::semantic_changes_since(self.retained_scc_function_epoch_seen);
3961 let global_changed = changes.epoch != self.retained_scc_function_epoch_seen;
3962 let provider_revision = self.resolver.planning_semantic_revision();
3963 let provider_changed = provider_revision != self.retained_scc_provider_revision_seen;
3964 if global_changed || provider_changed {
3965 let changed: BTreeSet<(String, String)> = changes.keys.into_iter().collect();
3966 let affected: Vec<VertexId> = self
3967 .retained_scc_members
3968 .keys()
3969 .copied()
3970 .filter(|&vertex| {
3971 let Some(ast) = self.graph.get_formula(vertex) else {
3972 return true;
3973 };
3974 (global_changed
3975 && (!changes.complete || Self::ast_uses_changed_function(&ast, &changed)))
3976 || (provider_changed && Self::ast_contains_function(&ast))
3977 })
3978 .collect();
3979 for vertex in &affected {
3980 self.retained_scc_members.remove(vertex);
3981 }
3982 if !affected.is_empty() {
3983 self.graph.mark_dirty_many(&affected);
3984 }
3985 self.retained_scc_function_epoch_seen = changes.epoch;
3986 self.retained_scc_provider_revision_seen = provider_revision;
3987 }
3988 let mut dirty_sccs: FxHashSet<u64> = FxHashSet::default();
3989 let mut all_sccs: FxHashSet<u64> = FxHashSet::default();
3990 for (&vertex, &scc) in &self.retained_scc_members {
3991 all_sccs.insert(scc);
3992 if self.graph.is_dirty(vertex) {
3993 dirty_sccs.insert(scc);
3994 self.retained_scc_dirty_at_begin.push((vertex, scc));
3995 }
3996 }
3997 let reused_members = self
3998 .retained_scc_members
3999 .values()
4000 .filter(|scc| !dirty_sccs.contains(scc))
4001 .count();
4002 let t = &mut self.last_cycle_telemetry;
4003 t.reused_sccs = all_sccs.len() - dirty_sccs.len();
4004 t.reused_scc_members = reused_members;
4005 }
4006
4007 pub fn virtual_dep_fallback_activations(&self) -> u64 {
4008 self.virtual_dep_fallback_activations
4009 }
4010
4011 #[cfg(test)]
4012 pub(crate) fn lookup_index_flights_built_for_test(&self) -> usize {
4013 self.lookup_index_cache
4014 .flights_built
4015 .load(std::sync::atomic::Ordering::Relaxed)
4016 }
4017
4018 pub(crate) fn last_lookup_index_cache_report(&self) -> LookupIndexCacheReport {
4019 self.lookup_index_cache.report()
4020 }
4021
4022 fn lookup_view_contains_volatile(&self, view: &RangeView<'_>, sheet_id: SheetId) -> bool {
4023 let start_row = view.start_row();
4024 let end_row = view.end_row();
4025 let start_col = view.start_col();
4026 let end_col = view.end_col();
4027 for row in start_row..=end_row {
4028 let Ok(row_u32) = u32::try_from(row) else {
4029 return true;
4030 };
4031 for col in start_col..=end_col {
4032 let Ok(col_u32) = u32::try_from(col) else {
4033 return true;
4034 };
4035 let cell_ref = self
4036 .graph
4037 .make_cell_ref_internal(sheet_id, row_u32, col_u32);
4038 if let Some(vertex_id) = self.graph.get_vertex_id_for_address(&cell_ref)
4039 && self.graph.is_volatile(vertex_id)
4040 {
4041 return true;
4042 }
4043 }
4044 }
4045 false
4046 }
4047
4048 fn build_lookup_index_impl(
4049 &self,
4050 view: &RangeView<'_>,
4051 axis: LookupAxis,
4052 ) -> Option<Arc<LookupIndex>> {
4053 let (rows, cols) = view.dims();
4054 if rows == 0 || cols == 0 {
4055 self.lookup_index_cache.note_skipped_tiny();
4056 return None;
4057 }
4058 let len = match axis {
4059 LookupAxis::ColumnInView(col) => {
4060 if col >= cols {
4061 self.lookup_index_cache.note_skipped_tiny();
4062 return None;
4063 }
4064 rows
4065 }
4066 LookupAxis::RowInView(row) => {
4067 if row >= rows {
4068 self.lookup_index_cache.note_skipped_tiny();
4069 return None;
4070 }
4071 cols
4072 }
4073 };
4074 if len < 64 {
4075 self.lookup_index_cache.note_skipped_tiny();
4076 return None;
4077 }
4078
4079 let sheet_id = self.graph.sheet_id(view.sheet_name())?;
4080 let key = LookupIndexKey {
4081 sheet_id,
4082 start_row: u32::try_from(view.start_row()).ok()?,
4083 start_col: u32::try_from(view.start_col()).ok()?,
4084 end_row: u32::try_from(view.end_row()).ok()?,
4085 end_col: u32::try_from(view.end_col()).ok()?,
4086 axis,
4087 snapshot_id: self.data_snapshot_id(),
4088 };
4089 if let Some(index) = self.lookup_index_cache.get(&key) {
4090 return Some(index);
4091 }
4092 if self
4093 .lookup_index_cache
4094 .would_exceed_cap(estimate_bytes(len, 0))
4095 {
4096 self.lookup_index_cache.note_skipped_cap();
4097 return None;
4098 }
4099 if !self.lookup_index_cache.should_build(key) {
4100 return None;
4101 }
4102 self.lookup_index_cache.single_flight(key, || {
4104 if let Some(index) = self.lookup_index_cache.recheck(&key) {
4105 return Some(index);
4106 }
4107 if self.lookup_index_cache.is_known_volatile(&key) {
4108 self.lookup_index_cache.note_skipped_volatile();
4109 return None;
4110 }
4111 if self.lookup_view_contains_volatile(view, sheet_id) {
4112 self.lookup_index_cache.note_volatile_key(key);
4113 self.lookup_index_cache.note_skipped_volatile();
4114 return None;
4115 }
4116 #[cfg(test)]
4117 self.lookup_index_cache
4118 .flights_built
4119 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
4120 match LookupIndex::build(view, axis, self.config.date_system).ok()? {
4121 BuildOutcome::Built(index) => self.lookup_index_cache.insert_if_room(key, index),
4122 BuildOutcome::ErrorInLookupAxis => {
4123 self.lookup_index_cache.note_skipped_error();
4124 None
4125 }
4126 BuildOutcome::Degenerate => {
4127 self.lookup_index_cache.note_skipped_tiny();
4128 None
4129 }
4130 }
4131 })
4132 }
4133
4134 fn reset_virtual_dep_telemetry_if_disabled(&mut self) {
4135 if !self.config.enable_virtual_dep_telemetry {
4136 self.last_virtual_dep_telemetry = VirtualDepTelemetry {
4137 fallback_mode_activations: self.virtual_dep_fallback_activations,
4138 ..VirtualDepTelemetry::default()
4139 };
4140 }
4141 }
4142
4143 fn source_cache_session(&self) -> SourceCacheSession {
4144 self.clear_source_cache();
4145 SourceCacheSession {
4146 cache: self.source_cache.clone(),
4147 }
4148 }
4149
4150 fn resolve_source_scalar_cached(
4151 &self,
4152 name: &str,
4153 version: Option<u64>,
4154 ) -> Result<LiteralValue, ExcelError> {
4155 let key = (name.to_string(), version);
4156 if let Ok(mut g) = self.source_cache.write() {
4157 if let Some(v) = g.scalars.get(&key) {
4158 return Ok(v.clone());
4159 }
4160
4161 let v = self.resolver.resolve_source_scalar(name).map_err(|err| {
4162 if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
4163 ExcelError::new(ExcelErrorKind::Ref)
4164 .with_message(format!("Unresolved source scalar: {name}"))
4165 } else {
4166 err
4167 }
4168 })?;
4169 g.scalars.insert(key, v.clone());
4170 Ok(v)
4171 } else {
4172 self.resolver.resolve_source_scalar(name).map_err(|err| {
4173 if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
4174 ExcelError::new(ExcelErrorKind::Ref)
4175 .with_message(format!("Unresolved source scalar: {name}"))
4176 } else {
4177 err
4178 }
4179 })
4180 }
4181 }
4182
4183 fn resolve_source_table_cached(
4184 &self,
4185 name: &str,
4186 version: Option<u64>,
4187 ) -> Result<Arc<dyn crate::traits::Table>, ExcelError> {
4188 let key = (name.to_string(), version);
4189 if let Ok(mut g) = self.source_cache.write() {
4190 if let Some(t) = g.tables.get(&key) {
4191 return Ok(t.clone());
4192 }
4193
4194 let t = self.resolver.resolve_source_table(name).map_err(|err| {
4195 if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
4196 ExcelError::new(ExcelErrorKind::Ref)
4197 .with_message(format!("Unresolved source table: {name}"))
4198 } else {
4199 err
4200 }
4201 })?;
4202 let t: Arc<dyn crate::traits::Table> = Arc::from(t);
4203 g.tables.insert(key, t.clone());
4204 Ok(t)
4205 } else {
4206 self.resolver
4207 .resolve_source_table(name)
4208 .map_err(|err| {
4209 if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
4210 ExcelError::new(ExcelErrorKind::Ref)
4211 .with_message(format!("Unresolved source table: {name}"))
4212 } else {
4213 err
4214 }
4215 })
4216 .map(Arc::from)
4217 }
4218 }
4219
4220 fn source_table_to_range_view(
4221 &self,
4222 table: &dyn crate::traits::Table,
4223 spec: &Option<formualizer_parse::parser::TableSpecifier>,
4224 ) -> Result<RangeView<'static>, ExcelError> {
4225 use formualizer_parse::parser::{SpecialItem, TableSpecifier};
4226
4227 let owned = match spec {
4228 Some(TableSpecifier::Column(c)) => {
4229 let c = c.trim();
4230 if c == "@" || c.contains('[') || c.contains(']') || c.contains(',') {
4231 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
4232 "Complex structured references not yet supported".to_string(),
4233 ));
4234 }
4235 table.get_column(c)?.materialise().into_owned()
4236 }
4237 Some(TableSpecifier::ColumnRange(start, end)) => {
4238 let cols = table.columns();
4239 let start = start.trim();
4240 let end = end.trim();
4241 let start_key = start.to_lowercase();
4242 let end_key = end.to_lowercase();
4243 let start_idx = cols.iter().position(|n| n.to_lowercase() == start_key);
4244 let end_idx = cols.iter().position(|n| n.to_lowercase() == end_key);
4245 if let (Some(mut si), Some(mut ei)) = (start_idx, end_idx) {
4246 if si > ei {
4247 std::mem::swap(&mut si, &mut ei);
4248 }
4249 let h = table.data_height();
4250 let w = ei - si + 1;
4251 let mut rows = vec![vec![LiteralValue::Empty; w]; h];
4252 for (offset, ci) in (si..=ei).enumerate() {
4253 let cname = &cols[ci];
4254 let col_range = table.get_column(cname)?;
4255 let (rh, _) = col_range.dimensions();
4256 for (r, row) in rows.iter_mut().enumerate().take(h.min(rh)) {
4257 row[offset] = col_range.get(r, 0)?;
4258 }
4259 }
4260 rows
4261 } else {
4262 return Err(ExcelError::new(ExcelErrorKind::Ref)
4263 .with_message("Column range refers to unknown column(s)".to_string()));
4264 }
4265 }
4266 Some(TableSpecifier::SpecialItem(SpecialItem::Headers))
4267 | Some(TableSpecifier::Headers) => table
4268 .headers_row()
4269 .map(|r| r.materialise().into_owned())
4270 .unwrap_or_default(),
4271 Some(TableSpecifier::SpecialItem(SpecialItem::Totals))
4272 | Some(TableSpecifier::Totals) => table
4273 .totals_row()
4274 .map(|r| r.materialise().into_owned())
4275 .unwrap_or_default(),
4276 Some(TableSpecifier::SpecialItem(SpecialItem::Data)) | Some(TableSpecifier::Data) => {
4277 table
4278 .data_body()
4279 .map(|r| r.materialise().into_owned())
4280 .unwrap_or_default()
4281 }
4282 Some(TableSpecifier::SpecialItem(SpecialItem::All)) | Some(TableSpecifier::All) => {
4283 let mut out: Vec<Vec<LiteralValue>> = Vec::new();
4284 if let Some(h) = table.headers_row() {
4285 out.extend(h.iter_rows());
4286 }
4287 if let Some(body) = table.data_body() {
4288 out.extend(body.iter_rows());
4289 }
4290 if let Some(tr) = table.totals_row() {
4291 out.extend(tr.iter_rows());
4292 }
4293 out
4294 }
4295 Some(TableSpecifier::SpecialItem(SpecialItem::ThisRow)) => {
4296 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
4297 "@ (This Row) requires table-aware context; not yet supported".to_string(),
4298 ));
4299 }
4300 Some(TableSpecifier::Row(_)) | Some(TableSpecifier::Combination(_)) => {
4301 return Err(ExcelError::new(ExcelErrorKind::NImpl)
4302 .with_message("Complex structured references not yet supported".to_string()));
4303 }
4304 None => {
4305 return Err(ExcelError::new(ExcelErrorKind::NImpl)
4306 .with_message("Table reference without specifier is unsupported".to_string()));
4307 }
4308 };
4309
4310 Ok(RangeView::from_owned_rows(owned, self.config.date_system))
4311 }
4312
4313 pub fn default_sheet_id(&self) -> SheetId {
4314 self.graph.default_sheet_id()
4315 }
4316
4317 pub fn default_sheet_name(&self) -> &str {
4318 self.graph.default_sheet_name()
4319 }
4320
4321 pub fn set_workbook_seed(&mut self, seed: u64) {
4323 self.config.workbook_seed = seed;
4324 }
4325
4326 pub fn set_volatile_level(&mut self, level: crate::traits::VolatileLevel) {
4328 self.config.volatile_level = level;
4329 }
4330
4331 pub fn set_temporal_egress(&mut self, policy: crate::engine::TemporalEgress) {
4333 self.config.temporal_egress = policy;
4334 }
4335
4336 pub fn temporal_egress(&self) -> crate::engine::TemporalEgress {
4337 self.config.temporal_egress
4338 }
4339
4340 pub fn set_deterministic_mode(
4342 &mut self,
4343 mode: crate::engine::DeterministicMode,
4344 ) -> Result<(), ExcelError> {
4345 let clock = mode.build_clock()?;
4346 self.config.deterministic_mode = mode;
4347 self.clock = crate::timezone::SnapshotClock::new(clock);
4348 Ok(())
4349 }
4350
4351 pub fn set_clock(&mut self, clock: Arc<dyn crate::timezone::ClockProvider>) {
4359 self.clock = crate::timezone::SnapshotClock::new(clock);
4360 }
4361
4362 fn validate_deterministic_mode(&self) -> Result<(), ExcelError> {
4363 self.config.deterministic_mode.validate()
4364 }
4365
4366 pub fn sheet_id(&self, name: &str) -> Option<SheetId> {
4367 self.graph.sheet_id(name)
4368 }
4369
4370 pub fn sheet_id_mut(&mut self, name: &str) -> SheetId {
4371 self.add_sheet(name)
4372 .unwrap_or_else(|_| self.graph.sheet_id_mut(name))
4373 }
4374
4375 pub fn sheet_name(&self, id: SheetId) -> &str {
4376 self.graph.sheet_name(id)
4377 }
4378
4379 fn arrow_sheet_position(&self, id: SheetId) -> Option<usize> {
4380 let index = self.arrow_sheet_index.get_or_init(|| {
4381 let mut index = Vec::new();
4382 for (position, sheet) in self.arrow_sheets.sheets.iter().enumerate() {
4383 if let Some(id) = self.graph.sheet_id(&sheet.name) {
4384 let needed = usize::from(id).saturating_add(1);
4385 if index.len() < needed {
4386 index.resize(needed, None);
4387 }
4388 if let Some(slot) = index.get_mut(usize::from(id)) {
4389 *slot = Some(position);
4390 }
4391 }
4392 }
4393 index
4394 });
4395 index.get(usize::from(id)).copied().flatten()
4396 }
4397
4398 pub(crate) fn arrow_sheet_by_id(&self, id: SheetId) -> Option<&crate::arrow_store::ArrowSheet> {
4400 self.arrow_sheets.sheets.get(self.arrow_sheet_position(id)?)
4401 }
4402
4403 pub(crate) fn arrow_sheet_by_id_mut(
4405 &mut self,
4406 id: SheetId,
4407 ) -> Option<&mut crate::arrow_store::ArrowSheet> {
4408 let position = self.arrow_sheet_position(id)?;
4409 self.arrow_sheets.sheets.get_mut(position)
4410 }
4411
4412 fn rebuild_arrow_sheet_index(&mut self) {
4413 self.arrow_sheet_index.take();
4414 self.resolved_arrow_sheets.clear();
4415 for (index, sheet) in self.arrow_sheets.sheets.iter().enumerate() {
4416 if let Some(id) = self.graph.sheet_id(&sheet.name) {
4417 let len = usize::from(id).saturating_add(1);
4418 if self.resolved_arrow_sheets.len() < len {
4419 self.resolved_arrow_sheets.resize(len, None);
4420 }
4421 if let Some(slot) = self.resolved_arrow_sheets.get_mut(usize::from(id)) {
4422 *slot = Some(index);
4423 }
4424 }
4425 }
4426 }
4427
4428 pub fn add_sheet(&mut self, name: &str) -> Result<SheetId, ExcelError> {
4429 let id = self.graph.add_sheet(name)?;
4430 self.ensure_arrow_sheet(name);
4431 self.mark_topology_edited();
4432 Ok(id)
4433 }
4434
4435 pub fn duplicate_sheet(&mut self, source: &str, new_name: &str) -> Result<SheetId, ExcelError> {
4436 let source_id = self.graph.sheet_id(source).ok_or_else(|| {
4437 ExcelError::new(ExcelErrorKind::Value).with_message("Source sheet does not exist")
4438 })?;
4439 if new_name.is_empty() || new_name.len() > 255 {
4440 return Err(ExcelError::new(ExcelErrorKind::Value).with_message("Invalid sheet name"));
4441 }
4442 if self.graph.sheet_id(new_name).is_some() {
4443 return Err(ExcelError::new(ExcelErrorKind::Value)
4444 .with_message(format!("Sheet '{new_name}' already exists")));
4445 }
4446 let new_id = self.graph.duplicate_sheet(source_id, new_name)?;
4447
4448 if let Some(source_sheet) = self.arrow_sheets.sheet(source).cloned() {
4449 let mut copied_sheet = source_sheet;
4450 copied_sheet.name = Arc::<str>::from(new_name);
4451 self.arrow_sheets.sheets.push(copied_sheet);
4452 self.rebuild_arrow_sheet_index();
4453 } else {
4454 self.ensure_arrow_sheet(new_name);
4455 }
4456
4457 let duplicated_formulas = self
4458 .graph
4459 .formula_vertices()
4460 .into_iter()
4461 .filter(|vertex| {
4462 self.graph
4463 .get_cell_ref(*vertex)
4464 .is_some_and(|cell| cell.sheet_id == new_id)
4465 })
4466 .collect::<Vec<_>>();
4467 self.graph.mark_vertices_dirty_batch(&duplicated_formulas);
4468 self.mark_topology_edited();
4469 Ok(new_id)
4470 }
4471
4472 fn ensure_arrow_sheet(&mut self, name: &str) {
4473 if self.arrow_sheets.sheet(name).is_some() {
4474 return;
4475 }
4476 self.arrow_sheets
4477 .sheets
4478 .push(crate::arrow_store::ArrowSheet {
4479 name: std::sync::Arc::<str>::from(name),
4480 date_system: self.config.date_system,
4481 columns: Vec::new(),
4482 nrows: 0,
4483 chunk_starts: Vec::new(),
4484 chunk_rows: 32 * 1024,
4485 });
4486 self.rebuild_arrow_sheet_index();
4487 }
4488
4489 pub fn remove_sheet(&mut self, sheet_id: SheetId) -> Result<(), ExcelError> {
4490 let name = self.graph.sheet_name(sheet_id).to_string();
4491 self.purge_derived_formats_for_sheet(sheet_id);
4492 self.graph.remove_sheet(sheet_id)?;
4493 self.arrow_sheets.sheets.retain(|s| s.name.as_ref() != name);
4494 self.rebuild_arrow_sheet_index();
4495 self.clear_all_computed_overlays();
4499 self.mark_all_formula_vertices_dirty();
4500 self.clear_staged_formulas_for_sheet(&name);
4501 if self.row_visibility.remove(&sheet_id).is_some() {
4502 self.invalidate_row_visibility_mask_cache();
4503 }
4504 self.record_structural_change(StructuralScope::RemovedSheet(sheet_id));
4505 self.mark_topology_edited();
4506 Ok(())
4507 }
4508
4509 fn rename_sheet_in_arrow_store(&mut self, target_name: &str, new_name: &str) -> bool {
4511 if let Some(asheet) = self
4512 .arrow_sheets
4513 .sheets
4514 .iter_mut()
4515 .find(|s| s.name.as_ref() == target_name)
4516 {
4517 asheet.name = std::sync::Arc::<str>::from(new_name);
4518 return true;
4519 }
4520 false
4521 }
4522
4523 pub fn rename_sheet(&mut self, sheet_id: SheetId, new_name: &str) -> Result<(), ExcelError> {
4524 let old_name = self.graph.sheet_name(sheet_id).to_string();
4525
4526 self.rename_sheet_in_arrow_store(&old_name, new_name);
4529
4530 match self.graph.rename_sheet(sheet_id, new_name) {
4532 Ok(_) => {
4533 self.rename_staged_formula_sheet(&old_name, new_name);
4534 let sheet_vertices: Vec<VertexId> = self
4536 .graph
4537 .grid_vertices_in_sheet(sheet_id)
4538 .map(|(id, _)| id)
4539 .collect();
4540 for v_id in sheet_vertices {
4541 self.graph.mark_vertex_dirty(v_id);
4542 }
4543 self.mark_topology_edited();
4545 Ok(())
4546 }
4547 Err(e) => {
4548 self.rename_sheet_in_arrow_store(new_name, &old_name);
4550 Err(e)
4551 }
4552 }
4553 }
4554
4555 pub fn named_ranges_iter(
4556 &self,
4557 ) -> impl Iterator<Item = (&String, &crate::engine::named_range::NamedRange)> {
4558 self.graph.named_ranges_iter()
4559 }
4560
4561 pub fn sheet_named_ranges_iter(
4562 &self,
4563 ) -> impl Iterator<Item = (&(SheetId, String), &crate::engine::named_range::NamedRange)> {
4564 self.graph.sheet_named_ranges_iter()
4565 }
4566
4567 pub fn resolve_name_entry(
4568 &self,
4569 name: &str,
4570 current_sheet: SheetId,
4571 ) -> Option<&crate::engine::named_range::NamedRange> {
4572 self.graph.resolve_name_entry(name, current_sheet)
4573 }
4574
4575 pub(crate) fn name_query_scope(
4585 &self,
4586 scope_sheet: Option<&str>,
4587 ) -> Result<NameScope, ExcelError> {
4588 match scope_sheet {
4589 None => Ok(NameScope::Workbook),
4590 Some(sheet) => self
4591 .graph
4592 .sheet_id(sheet)
4593 .map(NameScope::Sheet)
4594 .ok_or_else(|| {
4595 ExcelError::new(ExcelErrorKind::Ref)
4596 .with_message(format!("name scope sheet not found: {sheet}"))
4597 }),
4598 }
4599 }
4600
4601 fn resolve_sheet_locator(
4614 &self,
4615 locator: &crate::reference::SharedSheetLocator<'_>,
4616 context_sheet: SheetId,
4617 ) -> Result<SheetId, ExcelError> {
4618 self.graph
4619 .sheet_reg()
4620 .resolve_locator(locator, context_sheet)
4621 }
4622
4623 pub fn has_name(&self, name: &str, scope_sheet: Option<&str>) -> bool {
4624 let Ok(scope) = self.name_query_scope(scope_sheet) else {
4625 return false;
4626 };
4627 self.graph
4628 .resolve_name_entry_in_scope(name, scope)
4629 .is_some()
4630 }
4631
4632 pub fn resolved_name_value(
4636 &self,
4637 name: &str,
4638 scope_sheet: Option<&str>,
4639 ) -> Option<LiteralValue> {
4640 let scope = self.name_query_scope(scope_sheet).ok()?;
4641 let entry = self.graph.resolve_name_entry_in_scope(name, scope)?;
4642 self.graph.get_value(entry.vertex)
4643 }
4644
4645 pub fn table_metadata(&self, name: &str) -> Option<TableMetadata> {
4646 let entry = self.graph.resolve_table_entry(name)?;
4647 Some(TableMetadata {
4648 name: entry.name.clone(),
4649 sheet: self.graph.sheet_name(entry.sheet_id()).to_string(),
4650 start_row: entry.range.start.coord.row() + 1,
4651 start_col: entry.range.start.coord.col() + 1,
4652 end_row: entry.range.end.coord.row() + 1,
4653 end_col: entry.range.end.coord.col() + 1,
4654 header_row: entry.header_row,
4655 headers: entry.headers.clone(),
4656 totals_row: entry.totals_row,
4657 })
4658 }
4659
4660 pub fn tables(&self) -> Vec<TableMetadata> {
4662 self.graph
4663 .table_names()
4664 .into_iter()
4665 .filter_map(|name| self.table_metadata(&name))
4666 .collect()
4667 }
4668
4669 pub fn named_ranges_snapshot(&self) -> Vec<crate::engine::named_range::NamedRangeSnapshot> {
4670 let mut out: Vec<crate::engine::named_range::NamedRangeSnapshot> = Vec::new();
4671
4672 for (name, named) in self.graph.named_ranges_iter() {
4673 out.push(crate::engine::named_range::NamedRangeSnapshot {
4674 name: name.clone(),
4675 scope: NameScope::Workbook,
4676 definition: named.definition.clone(),
4677 });
4678 }
4679
4680 for ((sheet_id, name), named) in self.graph.sheet_named_ranges_iter() {
4681 out.push(crate::engine::named_range::NamedRangeSnapshot {
4682 name: name.clone(),
4683 scope: NameScope::Sheet(*sheet_id),
4684 definition: named.definition.clone(),
4685 });
4686 }
4687
4688 out.sort_by(|a, b| {
4689 let a_scope = match a.scope {
4690 NameScope::Workbook => (0u8, 0u32),
4691 NameScope::Sheet(id) => (1u8, u32::from(id)),
4692 };
4693 let b_scope = match b.scope {
4694 NameScope::Workbook => (0u8, 0u32),
4695 NameScope::Sheet(id) => (1u8, u32::from(id)),
4696 };
4697 a_scope.cmp(&b_scope).then_with(|| a.name.cmp(&b.name))
4698 });
4699
4700 out
4701 }
4702
4703 pub fn named_ranges_snapshot_for_sheet(
4704 &self,
4705 sheet_id: SheetId,
4706 ) -> Vec<crate::engine::named_range::NamedRangeSnapshot> {
4707 self.named_ranges_snapshot()
4708 .into_iter()
4709 .filter(|entry| match entry.scope {
4710 NameScope::Workbook => true,
4711 NameScope::Sheet(id) => id == sheet_id,
4712 })
4713 .collect()
4714 }
4715
4716 pub fn define_name(
4717 &mut self,
4718 name: &str,
4719 definition: NamedDefinition,
4720 scope: NameScope,
4721 ) -> Result<(), ExcelError> {
4722 self.graph.validate_define_name(name, scope)?;
4723 self.graph.define_name(name, definition, scope)?;
4724 self.record_structural_change(StructuralScope::AllSheets);
4725
4726 self.mark_topology_edited();
4727
4728 Ok(())
4729 }
4730
4731 pub fn update_name(
4732 &mut self,
4733 name: &str,
4734 definition: NamedDefinition,
4735 scope: NameScope,
4736 ) -> Result<(), ExcelError> {
4737 self.graph.validate_existing_name(name, scope)?;
4738 self.graph.update_name(name, definition, scope)?;
4739 self.record_structural_change(StructuralScope::AllSheets);
4740
4741 self.mark_topology_edited();
4742
4743 Ok(())
4744 }
4745
4746 pub fn delete_name(&mut self, name: &str, scope: NameScope) -> Result<(), ExcelError> {
4747 self.graph.validate_existing_name(name, scope)?;
4748 self.graph.delete_name(name, scope)?;
4749 self.record_structural_change(StructuralScope::AllSheets);
4750
4751 self.mark_topology_edited();
4752
4753 Ok(())
4754 }
4755
4756 pub fn define_table(
4757 &mut self,
4758 name: &str,
4759 range: crate::reference::RangeRef,
4760 header_row: bool,
4761 headers: Vec<String>,
4762 totals_row: bool,
4763 ) -> Result<(), ExcelError> {
4764 self.graph
4765 .define_table(name, range, header_row, headers, totals_row)?;
4766 self.record_structural_change(StructuralScope::AllSheets);
4767 self.mark_topology_edited();
4768 Ok(())
4769 }
4770
4771 pub fn define_source_scalar(
4772 &mut self,
4773 name: &str,
4774 version: Option<u64>,
4775 ) -> Result<(), ExcelError> {
4776 self.graph.define_source_scalar(name, version)?;
4777 self.record_structural_change(StructuralScope::OpaqueGlobal);
4778 self.mark_topology_edited();
4779 Ok(())
4780 }
4781
4782 pub fn define_source_table(
4783 &mut self,
4784 name: &str,
4785 version: Option<u64>,
4786 ) -> Result<(), ExcelError> {
4787 self.graph.define_source_table(name, version)?;
4788 self.record_structural_change(StructuralScope::OpaqueGlobal);
4789 self.mark_topology_edited();
4790 Ok(())
4791 }
4792
4793 pub fn set_source_scalar_version(
4794 &mut self,
4795 name: &str,
4796 version: Option<u64>,
4797 ) -> Result<(), ExcelError> {
4798 self.graph.set_source_scalar_version(name, version)?;
4799 Ok(())
4800 }
4801
4802 pub fn set_source_table_version(
4803 &mut self,
4804 name: &str,
4805 version: Option<u64>,
4806 ) -> Result<(), ExcelError> {
4807 self.graph.set_source_table_version(name, version)?;
4808 Ok(())
4809 }
4810
4811 pub fn invalidate_source(&mut self, name: &str) -> Result<(), ExcelError> {
4812 self.graph.invalidate_source(name)?;
4813 Ok(())
4814 }
4815
4816 pub fn vertex_value(&self, vertex: VertexId) -> Option<LiteralValue> {
4817 self.graph.get_value(vertex)
4818 }
4819
4820 pub fn graph_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
4821 self.graph.get_cell_value(sheet, row, col)
4822 }
4823
4824 pub fn vertex_for_cell(&self, cell: &CellRef) -> Option<VertexId> {
4825 self.graph.get_vertex_for_cell(cell)
4826 }
4827
4828 pub fn evaluation_vertices(&self) -> Vec<VertexId> {
4829 self.graph.get_evaluation_vertices()
4830 }
4831
4832 pub fn baseline_stats(&self) -> EngineBaselineStats {
4834 let graph = self.graph.baseline_stats();
4835 EngineBaselineStats {
4836 graph_vertex_count: graph.graph_vertex_count,
4837 graph_formula_vertex_count: graph.graph_formula_vertex_count,
4838 graph_edge_count: graph.graph_edge_count,
4839 dirty_vertex_count: graph.dirty_vertex_count,
4840 evaluation_vertex_count: graph.evaluation_vertex_count,
4841 formula_ast_root_count: graph.formula_ast_root_count,
4842 formula_ast_node_count: graph.formula_ast_node_count,
4843 staged_formula_count: self.staged_formula_count(),
4844 formula_plane_active_span_count: 0,
4845 formula_plane_producer_result_entries: 0,
4846 formula_plane_consumer_read_entries: 0,
4847 formula_plane_mixed_topology_cache_builds: 0,
4848 formula_plane_mixed_topology_cache_hits: 0,
4849 formula_plane_mixed_topology_cache_overflows: 0,
4850 formula_plane_dirty_pending_events: 0,
4851 formula_plane_dirty_region_events_recorded: 0,
4852 formula_plane_dirty_span_region_events_recorded: 0,
4853 formula_plane_dirty_whole_span_seeds_recorded: 0,
4854 formula_plane_dirty_global_invalidations: 0,
4855 formula_plane_structural_span_candidates: 0,
4856 formula_plane_cycle_member_span_demotions: 0,
4857 formula_plane_array_result_span_demotions: 0,
4858 retained_scc_members: self.retained_scc_members.len(),
4859 }
4860 }
4861
4862 pub(crate) fn inspection_mutation_revision(&self) -> u64 {
4864 self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed)
4865 }
4866
4867 #[cfg(test)]
4868 pub(crate) fn used_axis_bounds_cache_stats(&self) -> (usize, usize, usize, usize) {
4869 self.used_axis_bounds_cache
4870 .read()
4871 .ok()
4872 .and_then(|guard| {
4873 guard.as_ref().map(|cache| {
4874 (
4875 cache.row_hits.load(Ordering::Relaxed),
4876 cache.row_misses.load(Ordering::Relaxed),
4877 cache.col_hits.load(Ordering::Relaxed),
4878 cache.col_misses.load(Ordering::Relaxed),
4879 )
4880 })
4881 })
4882 .unwrap_or((0, 0, 0, 0))
4883 }
4884
4885 pub fn set_first_load_assume_new(&mut self, enabled: bool) {
4886 self.graph.set_first_load_assume_new(enabled);
4887 }
4888
4889 pub fn first_load_assume_new(&self) -> bool {
4890 self.graph.first_load_assume_new()
4891 }
4892
4893 pub fn reset_ensure_touched(&mut self) {
4894 self.graph.reset_ensure_touched();
4895 }
4896
4897 pub fn finalize_sheet_index(&mut self, sheet: &str) {
4898 self.graph.finalize_sheet_index(sheet);
4899 }
4900
4901 #[doc(hidden)]
4903 pub fn finalize_all_sheet_indexes<'a>(&mut self, sheets: impl IntoIterator<Item = &'a str>) {
4904 self.graph.finalize_all_sheet_indexes(sheets);
4905 }
4906
4907 pub fn action<T>(
4916 &mut self,
4917 name: impl AsRef<str>,
4918 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4919 ) -> Result<T, crate::engine::EditorError> {
4920 if self.action_depth != 0 {
4921 return Err(crate::engine::EditorError::TransactionFailed {
4922 reason: "Nested Engine::action calls are not supported (ticket 614: commit-only surface)"
4923 .to_string(),
4924 });
4925 }
4926
4927 self.action_depth = 1;
4928 let engine_ptr: *mut Engine<R> = self;
4929 let _guard = ActionDepthGuard {
4930 engine: engine_ptr,
4931 _marker: std::marker::PhantomData,
4932 };
4933
4934 let mut tx = EngineAction {
4935 engine: self,
4936 name: name.as_ref().to_string(),
4937 capture: None,
4938 arrow_undo: None,
4939 atomic_policy: false,
4940 };
4941 f(&mut tx)
4942 }
4943
4944 pub fn action_atomic<T>(
4948 &mut self,
4949 name: impl Into<String>,
4950 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4951 ) -> Result<T, crate::engine::EditorError> {
4952 let (v, _j) = self.action_atomic_journal(name, f)?;
4953 Ok(v)
4954 }
4955
4956 pub fn action_atomic_journal<T>(
4958 &mut self,
4959 name: impl Into<String>,
4960 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4961 ) -> Result<(T, crate::engine::ActionJournal), crate::engine::EditorError> {
4962 if self.action_depth != 0 {
4963 return Err(crate::engine::EditorError::TransactionFailed {
4964 reason: "Nested Engine::action calls are not supported (deterministic rule)"
4965 .to_string(),
4966 });
4967 }
4968
4969 self.action_depth = 1;
4970 let engine_ptr: *mut Engine<R> = self;
4971 let _guard = ActionDepthGuard {
4972 engine: engine_ptr,
4973 _marker: std::marker::PhantomData,
4974 };
4975
4976 let name_str = name.into();
4977 let mut capture = MutationCapture::new(Default::default());
4978 let start_len = capture.len();
4979 self.action_atomic_impl(&mut capture, start_len, true, name_str, f)
4980 }
4981
4982 fn action_atomic_impl<T>(
4983 &mut self,
4984 capture: &mut MutationCapture,
4985 start_len: usize,
4986 expand_runs: bool,
4987 name: String,
4988 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4989 ) -> Result<(T, crate::engine::ActionJournal), crate::engine::EditorError> {
4990 let invalidation_baseline = self.invalidation_baseline();
4991 let mut arrow_undo = crate::engine::ArrowUndoBatch::default();
4992 let arrow_ptr: *mut crate::engine::ArrowUndoBatch = &mut arrow_undo;
4993
4994 let capture_ptr: *mut MutationCapture = capture;
4995 let mut tx = EngineAction {
4996 engine: self,
4997 name: name.clone(),
4998 capture: Some(capture_ptr),
4999 arrow_undo: Some(arrow_ptr),
5000 atomic_policy: true,
5001 };
5002
5003 let res = f(&mut tx);
5004
5005 let capture_ref = unsafe { &*capture_ptr };
5012 let has_runs = capture_ref.lazy_len() > 0;
5013 let graph_events: Vec<crate::engine::ChangeEvent> = if expand_runs || res.is_err() {
5014 capture_ref.expanded_events_from(start_len, 0)
5015 } else {
5016 capture_ref.events()[start_len..].to_vec()
5017 };
5018 let graph_batch = crate::engine::GraphUndoBatch {
5019 events: graph_events,
5020 };
5021 let affected_cells = arrow_undo.ops.len();
5022 let journal = crate::engine::ActionJournal {
5023 name,
5024 graph: graph_batch,
5025 arrow: arrow_undo,
5026 affected_cells,
5027 };
5028
5029 match res {
5030 Ok(v) => {
5031 if !journal.graph.is_empty() || !journal.arrow.is_empty() || has_runs {
5032 for event in &journal.graph.events {
5033 self.record_change_for_event(event);
5034 }
5035 let mut impact = Self::classify_change_events(
5036 &journal.graph.events,
5037 LoggedEditDirection::Original,
5038 )
5039 .max(Self::classify_arrow_undo(&journal.arrow));
5040 if has_runs {
5041 impact = impact.max(LoggedEditImpact::Topology);
5042 }
5043 self.apply_logged_edit_impact(impact, invalidation_baseline);
5044 }
5045 Ok((v, journal))
5046 }
5047 Err(e) => {
5048 if let Err(rb) = self.rollback_from_action_journal(&journal, invalidation_baseline)
5049 {
5050 return Err(crate::engine::EditorError::TransactionFailed {
5051 reason: format!(
5052 "Engine::action_atomic rollback failed after error '{e}': {rb}"
5053 ),
5054 });
5055 }
5056 if !journal.graph.is_empty() || !journal.arrow.is_empty() {
5057 for event in &journal.graph.events {
5058 self.record_change_for_event(event);
5059 }
5060 }
5061 Err(e)
5062 }
5063 }
5064 }
5065
5066 pub fn action_with_logger<T>(
5073 &mut self,
5074 log: &mut crate::engine::ChangeLog,
5075 name: impl AsRef<str>,
5076 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
5077 ) -> Result<T, crate::engine::EditorError> {
5078 if self.action_depth != 0 {
5079 return Err(crate::engine::EditorError::TransactionFailed {
5080 reason: "Nested Engine::action calls are not supported (deterministic rule)"
5081 .to_string(),
5082 });
5083 }
5084
5085 self.action_depth = 1;
5086 let engine_ptr: *mut Engine<R> = self;
5087 let _guard = ActionDepthGuard {
5088 engine: engine_ptr,
5089 _marker: std::marker::PhantomData,
5090 };
5091
5092 let name_str = name.as_ref().to_string();
5093 let mut capture = MutationCapture::new(log.current_meta());
5094 let start_len = capture.len();
5095 capture.begin_compound(name_str.clone());
5096
5097 let res = self.action_atomic_impl(&mut capture, start_len, false, name_str, f);
5100 capture.close_compounds();
5101
5102 match res {
5103 Ok((v, _journal)) => {
5104 log.publish_capture(capture);
5105 Ok(v)
5106 }
5107 Err(e) => {
5108 log.discard_capture(capture);
5110 Err(e)
5111 }
5112 }
5113 }
5114
5115 fn rollback_from_action_journal(
5116 &mut self,
5117 journal: &crate::engine::ActionJournal,
5118 invalidation_baseline: InvalidationBaseline,
5119 ) -> Result<(), crate::engine::EditorError> {
5120 self.invalidate_for_action_journal(
5123 journal,
5124 LoggedEditDirection::InverseReplay,
5125 invalidation_baseline,
5126 );
5127 journal.graph.undo(&mut self.graph)?;
5129 self.apply_inverse_row_visibility_events(&journal.graph.events);
5131 self.apply_arrow_undo_batch(&journal.arrow, true);
5133 Ok(())
5134 }
5135
5136 fn rollback_from_change_events(
5137 &mut self,
5138 events: &[crate::engine::ChangeEvent],
5139 invalidation_baseline: InvalidationBaseline,
5140 ) -> Result<(), crate::engine::EditorError> {
5141 use crate::engine::ChangeEvent;
5142
5143 self.invalidate_for_change_events(
5146 events,
5147 LoggedEditDirection::InverseReplay,
5148 invalidation_baseline,
5149 );
5150
5151 self.graph
5153 .authority_set_replay(crate::engine::authority::history::Replay::Undo);
5154 let rolled_back = (|| {
5155 let mut editor = crate::engine::VertexEditor::new(&mut self.graph);
5156 let mut compound_stack: Vec<usize> = Vec::new();
5157 for (i, ev) in events.iter().enumerate().rev() {
5158 match ev {
5159 ChangeEvent::CompoundEnd { depth } => {
5160 compound_stack.push(*depth);
5161 if let Some(description) =
5162 crate::engine::graph::editor::change_log::compound_start_description(
5163 i,
5164 |j| &events[j],
5165 )
5166 {
5167 editor.inverse_compound_end(description);
5168 }
5169 }
5170 ChangeEvent::CompoundStart { depth, .. } => {
5171 if compound_stack.last() == Some(depth) {
5172 compound_stack.pop();
5173 }
5174 editor.apply_inverse(ev.clone())?;
5175 }
5176 ChangeEvent::SetRowVisibility { .. } => {
5177 }
5179 _ => {
5180 editor.apply_inverse(ev.clone())?;
5181 }
5182 }
5183 }
5184 Ok::<_, crate::engine::EditorError>(())
5185 })();
5186 self.graph
5187 .authority_set_replay(crate::engine::authority::history::Replay::Forward);
5188 rolled_back?;
5189
5190 for ev in events.iter().rev() {
5192 self.apply_inverse_row_visibility_event(ev);
5193 }
5194
5195 for ev in events.iter().rev() {
5197 self.mirror_inverse_change_to_arrow(ev);
5198 }
5199
5200 Ok(())
5201 }
5202
5203 fn read_cell_formula_ast(&self, sheet: &str, row: u32, col: u32) -> Option<ASTNode> {
5204 let sheet_id = self.graph.sheet_id(sheet)?;
5205 let coord = Coord::from_excel(row, col, true, true);
5206 let cell = CellRef::new(sheet_id, coord);
5207 let vid = self.graph.get_vertex_for_cell(&cell)?;
5208 self.graph.get_formula(vid)
5209 }
5210
5211 pub fn define_name_with_logger(
5212 &mut self,
5213 log: &mut crate::engine::ChangeLog,
5214 name: &str,
5215 definition: NamedDefinition,
5216 scope: NameScope,
5217 ) -> Result<(), crate::engine::EditorError> {
5218 self.graph
5219 .validate_define_name(name, scope)
5220 .map_err(crate::engine::EditorError::Excel)?;
5221
5222 {
5223 let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
5224 editor.define_name(name, definition, scope)?;
5225 }
5226 self.record_structural_change(StructuralScope::AllSheets);
5227
5228 self.mark_topology_edited();
5229
5230 Ok(())
5231 }
5232
5233 pub fn update_name_with_logger(
5234 &mut self,
5235 log: &mut crate::engine::ChangeLog,
5236 name: &str,
5237 definition: NamedDefinition,
5238 scope: NameScope,
5239 ) -> Result<(), crate::engine::EditorError> {
5240 self.graph
5241 .validate_existing_name(name, scope)
5242 .map_err(crate::engine::EditorError::Excel)?;
5243 {
5244 let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
5245 editor.update_name(name, definition, scope)?;
5246 }
5247 self.record_structural_change(StructuralScope::AllSheets);
5248
5249 self.mark_topology_edited();
5250
5251 Ok(())
5252 }
5253
5254 pub fn delete_name_with_logger(
5255 &mut self,
5256 log: &mut crate::engine::ChangeLog,
5257 name: &str,
5258 scope: NameScope,
5259 ) -> Result<(), crate::engine::EditorError> {
5260 self.graph
5261 .validate_existing_name(name, scope)
5262 .map_err(crate::engine::EditorError::Excel)?;
5263 {
5264 let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
5265 editor.delete_name(name, scope)?;
5266 }
5267 self.record_structural_change(StructuralScope::AllSheets);
5268
5269 self.mark_topology_edited();
5270
5271 Ok(())
5272 }
5273
5274 pub fn edit_with_logger<T>(
5275 &mut self,
5276 log: &mut crate::engine::ChangeLog,
5277 f: impl FnOnce(&mut crate::engine::VertexEditor) -> T,
5278 ) -> Result<T, crate::engine::EditorError> {
5279 let mut capture = MutationCapture::new(log.current_meta());
5280 let result = self.edit_with_capture(&mut capture, f);
5281 capture.close_compounds();
5282 match result {
5283 Ok(value) => {
5284 log.publish_capture(capture);
5285 Ok(value)
5286 }
5287 Err(error) => {
5288 log.discard_capture(capture);
5289 Err(error)
5290 }
5291 }
5292 }
5293
5294 fn edit_with_capture<T>(
5295 &mut self,
5296 capture: &mut MutationCapture,
5297 f: impl FnOnce(&mut crate::engine::VertexEditor) -> T,
5298 ) -> Result<T, crate::engine::EditorError> {
5299 let invalidation_baseline = self.invalidation_baseline();
5300 let start_len = capture.len();
5301 let lazy_start = capture.lazy_len();
5302
5303 struct ArrowSpillReader<'a> {
5306 sheets: &'a crate::arrow_store::SheetStore,
5307 }
5308 impl crate::engine::graph::editor::vertex_editor::SpillValueReader for ArrowSpillReader<'_> {
5309 fn read_cell_value(
5310 &self,
5311 sheet: &str,
5312 row: u32,
5313 col: u32,
5314 ) -> Option<formualizer_common::LiteralValue> {
5315 use formualizer_common::LiteralValue;
5316 let asheet = self.sheets.sheet(sheet)?;
5317 let r0 = row.saturating_sub(1) as usize;
5318 let c0 = col.saturating_sub(1) as usize;
5319 let v = asheet.get_cell_value(r0, c0);
5320 if matches!(v, LiteralValue::Empty) {
5321 None
5322 } else {
5323 Some(v)
5324 }
5325 }
5326 }
5327
5328 let ret = {
5329 let spill_reader = ArrowSpillReader {
5330 sheets: &self.arrow_sheets,
5331 };
5332 let mut editor = crate::engine::VertexEditor::with_capture_and_spill_reader(
5333 &mut self.graph,
5334 capture,
5335 &spill_reader,
5336 );
5337 f(&mut editor)
5338 };
5339
5340 let new_events = capture.events()[start_len..].to_vec();
5344 let new_runs = capture.lazy_len() > lazy_start;
5345 if new_events.iter().any(|event| {
5346 matches!(
5347 event,
5348 ChangeEvent::DefineName { .. }
5349 | ChangeEvent::UpdateName { .. }
5350 | ChangeEvent::DeleteName { .. }
5351 )
5352 }) {
5353 let all = capture.expanded_events_from(start_len, lazy_start);
5354 self.rollback_from_change_events(&all, invalidation_baseline)?;
5355 return Err(crate::engine::EditorError::TransactionUnsupported {
5356 reason: "name mutations must use Engine's prepared logged-name APIs".to_string(),
5357 });
5358 }
5359
5360 self.clear_logged_cell_format_states(&new_events);
5363 for ev in &new_events {
5364 self.mirror_forward_change_to_arrow(ev);
5365 }
5366 for ev in &new_events {
5367 self.record_change_for_event(ev);
5368 }
5369
5370 if self.action_depth == 0 {
5373 let mut impact =
5374 Self::classify_change_events(&new_events, LoggedEditDirection::Original);
5375 if new_runs {
5376 impact = impact.max(LoggedEditImpact::Topology);
5377 }
5378 self.apply_logged_edit_impact(impact, invalidation_baseline);
5379 }
5380
5381 Ok(ret)
5382 }
5383
5384 pub(crate) fn preflight_replay_admission(
5385 &mut self,
5386 events: &[ChangeEvent],
5387 forward: bool,
5388 ) -> Result<(), crate::engine::EditorError> {
5389 if !self.graph_admission_enabled() {
5390 return Ok(());
5391 }
5392 let mut vertex_delta = 0i128;
5393 let mut edge_delta = 0i128;
5394 let mut added_vertices = 0usize;
5395 let mut added_edges = 0usize;
5396 let mut formula_cells = BTreeSet::new();
5397 for event in events {
5398 match event {
5399 ChangeEvent::AddVertex {
5400 formula,
5401 coord,
5402 sheet_id,
5403 ..
5404 } => {
5405 let delta = if forward { 1 } else { -1 };
5406 vertex_delta += delta;
5407 if forward {
5408 added_vertices = added_vertices.saturating_add(1);
5409 if formula.is_some() {
5410 formula_cells.insert((*sheet_id, coord.row(), coord.col()));
5411 }
5412 }
5413 }
5414 ChangeEvent::RemoveVertex {
5415 old_formula,
5416 coord,
5417 sheet_id,
5418 ..
5419 } => {
5420 let delta = if forward { -1 } else { 1 };
5421 vertex_delta += delta;
5422 if !forward {
5423 added_vertices = added_vertices.saturating_add(1);
5424 if old_formula.is_some()
5425 && let (Some(sheet_id), Some(coord)) = (sheet_id, coord)
5426 {
5427 formula_cells.insert((*sheet_id, coord.row(), coord.col()));
5428 }
5429 }
5430 }
5431 ChangeEvent::EdgeAdded { .. } => {
5432 let delta = if forward { 1 } else { -1 };
5433 edge_delta += delta;
5434 if forward {
5435 added_edges = added_edges.saturating_add(1);
5436 }
5437 }
5438 ChangeEvent::EdgeRemoved { .. } => {
5439 let delta = if forward { -1 } else { 1 };
5440 edge_delta += delta;
5441 if !forward {
5442 added_edges = added_edges.saturating_add(1);
5443 }
5444 }
5445 ChangeEvent::SetFormula {
5446 addr, old_formula, ..
5447 } if (forward || old_formula.is_some()) => {
5448 formula_cells.insert((addr.sheet_id, addr.coord.row(), addr.coord.col()));
5449 }
5450 _ => {}
5451 }
5452 }
5453 let stats = self.graph.baseline_stats();
5454 let final_vertices = i128::try_from(stats.graph_vertex_count)
5455 .ok()
5456 .and_then(|count| count.checked_add(vertex_delta))
5457 .and_then(|count| usize::try_from(count).ok())
5458 .ok_or_else(|| {
5459 crate::engine::EditorError::Excel(
5460 ExcelError::new(ExcelErrorKind::NImpl)
5461 .with_message("replay vertex count overflow"),
5462 )
5463 })?;
5464 let final_edges = i128::try_from(stats.graph_edge_count)
5465 .ok()
5466 .and_then(|count| count.checked_add(edge_delta))
5467 .and_then(|count| usize::try_from(count).ok())
5468 .ok_or_else(|| {
5469 crate::engine::EditorError::Excel(
5470 ExcelError::new(ExcelErrorKind::NImpl)
5471 .with_message("replay edge count overflow"),
5472 )
5473 })?;
5474 self.preflight_graph_admission(crate::engine::resource_ledger::GraphAdmission {
5475 final_vertices,
5476 final_edges,
5477 materialization_cells: formula_cells.len() as u64,
5478 added_vertices,
5479 added_edges,
5480 })
5481 .map_err(crate::engine::EditorError::Excel)
5482 }
5483
5484 pub fn undo_logged(
5489 &mut self,
5490 undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5491 log: &mut crate::engine::ChangeLog,
5492 ) -> Result<(), crate::engine::EditorError> {
5493 let pending_events = log
5494 .last_group_indices()
5495 .into_iter()
5496 .map(|index| log.events()[index].clone())
5497 .collect::<Vec<_>>();
5498 self.preflight_replay_admission(&pending_events, false)?;
5499 let invalidation_baseline = self.invalidation_baseline();
5500 self.invalidate_for_change_events(
5503 &pending_events,
5504 LoggedEditDirection::InverseReplay,
5505 invalidation_baseline,
5506 );
5507 let batch = undo.undo(&mut self.graph, log)?;
5508 for item in batch.iter().rev() {
5509 self.apply_inverse_row_visibility_event(&item.event);
5510 self.apply_inverse_staged_formula_event(&item.event);
5511 }
5512 if !batch.is_empty() {
5513 let events = batch
5514 .iter()
5515 .map(|item| item.event.clone())
5516 .collect::<Vec<_>>();
5517 self.clear_logged_cell_format_states(&events);
5518 }
5519 self.mirror_undo_batch_to_arrow(&batch);
5520 if !batch.is_empty() {
5521 for item in &batch {
5522 self.record_change_for_event(&item.event);
5523 }
5524 }
5525 crate::engine::trace::fz_event!(
5526 tracing::Level::INFO,
5527 "history",
5528 "history.replay",
5529 op = "undo",
5530 events_replayed = batch.len(),
5531 ownership_resyncs = batch.len()
5532 );
5533 Ok(())
5534 }
5535
5536 pub fn redo_logged(
5537 &mut self,
5538 undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5539 log: &mut crate::engine::ChangeLog,
5540 ) -> Result<(), crate::engine::EditorError> {
5541 let pending_events = undo.pending_redo_events();
5542 self.preflight_replay_admission(&pending_events, true)?;
5543 let invalidation_baseline = self.invalidation_baseline();
5544 self.invalidate_for_change_events(
5545 &pending_events,
5546 LoggedEditDirection::ForwardReplay,
5547 invalidation_baseline,
5548 );
5549 let batch = undo.redo(&mut self.graph, log)?;
5550 for item in &batch {
5551 self.apply_forward_row_visibility_event(&item.event);
5552 self.apply_forward_staged_formula_event(&item.event);
5553 }
5554 if !batch.is_empty() {
5555 let events = batch
5556 .iter()
5557 .map(|item| item.event.clone())
5558 .collect::<Vec<_>>();
5559 self.clear_logged_cell_format_states(&events);
5560 }
5561 self.mirror_redo_batch_to_arrow(&batch);
5562 if !batch.is_empty() {
5563 for item in &batch {
5564 self.record_change_for_event(&item.event);
5565 }
5566 }
5567 crate::engine::trace::fz_event!(
5568 tracing::Level::INFO,
5569 "history",
5570 "history.replay",
5571 op = "redo",
5572 events_replayed = batch.len(),
5573 ownership_resyncs = batch.len()
5574 );
5575 Ok(())
5576 }
5577
5578 pub fn undo_action(
5582 &mut self,
5583 undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5584 ) -> Result<(), crate::engine::EditorError> {
5585 let Some(journal) = undo.pop_undo_action() else {
5586 return Ok(());
5587 };
5588 if let Err(error) = self.preflight_replay_admission(&journal.graph.events, false) {
5589 undo.push_done_action(journal);
5590 return Err(error);
5591 }
5592 let invalidation_baseline = self.invalidation_baseline();
5593
5594 self.invalidate_for_action_journal(
5595 &journal,
5596 LoggedEditDirection::InverseReplay,
5597 invalidation_baseline,
5598 );
5599 journal.graph.undo(&mut self.graph)?;
5600 self.apply_inverse_row_visibility_events(&journal.graph.events);
5601 self.apply_arrow_undo_batch(&journal.arrow, true);
5602 if !journal.graph.is_empty() || !journal.arrow.is_empty() {
5603 for event in &journal.graph.events {
5604 self.record_change_for_event(event);
5605 }
5606 }
5607
5608 #[cfg(feature = "tracing")]
5609 let events_replayed = journal.graph.events.len();
5610 crate::engine::trace::fz_event!(
5611 tracing::Level::INFO,
5612 "history",
5613 "history.replay",
5614 op = "undo",
5615 events_replayed,
5616 ownership_resyncs = events_replayed
5617 );
5618 undo.push_redo_action(journal);
5619 Ok(())
5620 }
5621
5622 pub fn redo_action(
5626 &mut self,
5627 undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5628 ) -> Result<(), crate::engine::EditorError> {
5629 let Some(journal) = undo.pop_redo_action() else {
5630 return Ok(());
5631 };
5632 if let Err(error) = self.preflight_replay_admission(&journal.graph.events, true) {
5633 undo.push_redo_action(journal);
5634 return Err(error);
5635 }
5636 let invalidation_baseline = self.invalidation_baseline();
5637 self.invalidate_for_action_journal(
5638 &journal,
5639 LoggedEditDirection::ForwardReplay,
5640 invalidation_baseline,
5641 );
5642 journal.graph.redo(&mut self.graph)?;
5643 self.apply_forward_row_visibility_events(&journal.graph.events);
5644 self.apply_arrow_undo_batch(&journal.arrow, false);
5645 if !journal.graph.is_empty() || !journal.arrow.is_empty() {
5646 for event in &journal.graph.events {
5647 self.record_change_for_event(event);
5648 }
5649 }
5650
5651 #[cfg(feature = "tracing")]
5652 let events_replayed = journal.graph.events.len();
5653 crate::engine::trace::fz_event!(
5654 tracing::Level::INFO,
5655 "history",
5656 "history.replay",
5657 op = "redo",
5658 events_replayed,
5659 ownership_resyncs = events_replayed
5660 );
5661 undo.push_done_action(journal);
5662 Ok(())
5663 }
5664
5665 fn cellref_to_sheet_row_col(&self, addr: &crate::reference::CellRef) -> (String, u32, u32) {
5666 let sheet = self.graph.sheet_name(addr.sheet_id).to_string();
5667 let row = addr.coord.row() + 1;
5669 let col = addr.coord.col() + 1;
5670 (sheet, row, col)
5671 }
5672
5673 fn mirror_undo_batch_to_arrow(
5674 &mut self,
5675 batch: &[crate::engine::graph::editor::undo_engine::UndoBatchItem],
5676 ) {
5677 for item in batch.iter().rev() {
5679 self.mirror_inverse_change_to_arrow(&item.event);
5680 }
5681 }
5682
5683 fn mirror_redo_batch_to_arrow(
5684 &mut self,
5685 batch: &[crate::engine::graph::editor::undo_engine::UndoBatchItem],
5686 ) {
5687 for item in batch.iter() {
5689 self.mirror_forward_change_to_arrow(&item.event);
5690 }
5691 }
5692
5693 fn mirror_inverse_change_to_arrow(&mut self, ev: &crate::engine::ChangeEvent) {
5694 use crate::engine::ChangeEvent;
5695 use formualizer_common::LiteralValue;
5696
5697 match ev {
5698 ChangeEvent::SetValue {
5699 addr,
5700 old_value,
5701 old_formula,
5702 ..
5703 } => {
5704 let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5705 if old_formula.is_some() {
5706 self.clear_delta_overlay_cell(&sheet, row, col);
5707 } else {
5708 let v = old_value.clone().unwrap_or(LiteralValue::Empty);
5709 self.mirror_value_to_overlay(&sheet, row, col, &v);
5710 }
5711 }
5712 ChangeEvent::SetFormula {
5713 addr,
5714 old_value,
5715 old_formula,
5716 ..
5717 } => {
5718 let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5719 if old_formula.is_some() {
5720 self.clear_delta_overlay_cell(&sheet, row, col);
5721 } else {
5722 let v = old_value.clone().unwrap_or(LiteralValue::Empty);
5723 self.mirror_value_to_overlay(&sheet, row, col, &v);
5724 }
5725 }
5726 ChangeEvent::SpillCommitted { old, new, .. } => {
5727 self.mirror_spill_snapshot(new, true);
5729 if let Some(snap) = old {
5730 self.mirror_spill_snapshot(snap, false);
5731 }
5732 }
5733 ChangeEvent::SpillCleared { old, .. } => {
5734 self.mirror_spill_snapshot(old, false);
5736 }
5737 ChangeEvent::SetRowVisibility { .. } => {
5738 }
5740 _ => {}
5741 }
5742 }
5743
5744 fn mirror_forward_change_to_arrow(&mut self, ev: &crate::engine::ChangeEvent) {
5745 use crate::engine::ChangeEvent;
5746
5747 match ev {
5748 ChangeEvent::SetValue { addr, new, .. } => {
5749 let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5750 self.mirror_value_to_overlay(&sheet, row, col, new);
5751 }
5752 ChangeEvent::SetFormula { addr, .. } => {
5753 let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5754 self.clear_delta_overlay_cell(&sheet, row, col);
5755 }
5757 ChangeEvent::SpillCommitted { old, new, .. } => {
5758 if let Some(snap) = old {
5759 self.mirror_spill_snapshot(snap, true);
5760 }
5761 self.mirror_spill_snapshot(new, false);
5762 }
5763 ChangeEvent::SpillCleared { old, .. } => {
5764 self.mirror_spill_snapshot(old, true);
5765 }
5766 ChangeEvent::SetRowVisibility { .. } => {
5767 }
5769 _ => {
5770 }
5772 }
5773 }
5774
5775 fn mirror_spill_snapshot(
5776 &mut self,
5777 snap: &crate::engine::graph::editor::change_log::SpillSnapshot,
5778 clear_only: bool,
5779 ) {
5780 use formualizer_common::LiteralValue;
5781
5782 let mut i = 0usize;
5783 for row in &snap.values {
5784 for v in row {
5785 if let Some(cell) = snap.target_cells.get(i) {
5786 let (sheet, r, c) = self.cellref_to_sheet_row_col(cell);
5787 let out = if clear_only {
5788 LiteralValue::Empty
5789 } else {
5790 v.clone()
5791 };
5792 self.mirror_value_to_computed_overlay(&sheet, r, c, &out);
5793 }
5794 i += 1;
5795 }
5796 }
5797 if clear_only {
5799 for cell in snap.target_cells.iter().skip(i) {
5800 let (sheet, r, c) = self.cellref_to_sheet_row_col(cell);
5801 self.mirror_value_to_computed_overlay(&sheet, r, c, &LiteralValue::Empty);
5802 }
5803 }
5804 }
5805
5806 pub fn set_default_sheet_by_name(&mut self, name: &str) {
5807 self.graph.set_default_sheet_by_name(name);
5808 }
5809
5810 pub fn set_default_sheet_by_id(&mut self, id: SheetId) {
5811 self.graph.set_default_sheet_by_id(id);
5812 }
5813
5814 pub fn set_sheet_index_mode(&mut self, mode: crate::engine::SheetIndexMode) {
5815 self.graph.set_sheet_index_mode(mode);
5816 }
5817
5818 #[cfg(feature = "test-support")]
5819 #[doc(hidden)]
5820 pub fn mark_all_formulas_dirty_for_test(&mut self) {
5821 self.mark_all_formula_vertices_dirty();
5822 }
5823
5824 #[cfg(feature = "test-support")]
5825 #[doc(hidden)]
5826 pub fn take_criteria_mask_work_for_test() -> (usize, usize) {
5827 criteria_mask_test_hooks::take_mask_work()
5828 }
5829
5830 #[cfg(feature = "test-support")]
5831 #[doc(hidden)]
5832 pub fn lookup_index_cache_report_for_test(&self) -> LookupIndexCacheReport {
5833 self.lookup_index_cache.report()
5834 }
5835
5836 #[cfg(any(test, feature = "benchmark_internal"))]
5837 #[doc(hidden)]
5838 pub fn reset_recalc_reuse_probe(&mut self) {
5839 *self.recalc_reuse_probe.get_mut().unwrap() = RecalcReuseProbe::default();
5840 }
5841
5842 #[cfg(any(test, feature = "benchmark_internal"))]
5843 #[doc(hidden)]
5844 pub fn recalc_reuse_probe(&self) -> RecalcReuseProbe {
5845 let mut probe = self.recalc_reuse_probe.lock().unwrap().clone();
5846 if let Some(cached) = self.cached_static_schedule.as_ref() {
5847 let entry_bytes = |e: &CachedScheduleEntry| {
5848 std::mem::size_of::<CachedScheduleEntry>()
5849 + e.candidate_vertices.heap_bytes()
5850 + std::mem::size_of::<crate::engine::Schedule>()
5851 + 2 * std::mem::size_of::<usize>()
5852 + schedule_probe_retained_bytes(&e.schedule)
5853 };
5854 probe.schedule_retained_bytes = entry_bytes(cached)
5855 + self.recent_schedules.iter().map(entry_bytes).sum::<usize>()
5856 + self.base_schedule.as_ref().map_or(0, entry_bytes)
5857 + self.recent_schedules.capacity() * std::mem::size_of::<CachedScheduleEntry>();
5858 }
5859 probe
5860 }
5861
5862 #[cfg(test)]
5863 pub(crate) fn cached_static_schedule_for_test(&self) -> Option<Arc<crate::engine::Schedule>> {
5864 self.cached_static_schedule
5865 .as_ref()
5866 .map(|cached| Arc::clone(&cached.schedule))
5867 }
5868
5869 fn clear_cached_static_schedule(&mut self) {
5870 self.cached_static_schedule = None;
5871 self.recent_schedules.clear();
5872 self.base_schedule = None;
5873 }
5874
5875 fn retain_recent_schedule(&mut self, entry: CachedScheduleEntry) {
5878 const RECENT_SCHEDULES: usize = 8;
5879 const RECENT_SCHEDULE_VERTICES: usize = 65_536;
5880 let revision = self.schedule_cache_authority_revision();
5881 let epoch = self.topology_epoch;
5882 self.recent_schedules
5883 .retain(|e| e.topology_epoch == epoch && e.authority_revision == revision);
5884 let current =
5885 |e: &CachedScheduleEntry| e.topology_epoch == epoch && e.authority_revision == revision;
5886 if self.base_schedule.as_ref().is_some_and(|b| !current(b)) {
5887 self.base_schedule = None;
5888 }
5889 let mut entry = entry;
5892 if current(&entry)
5893 && entry.candidate_vertices.len() > BASE_SCHEDULE_MIN_VERTICES
5894 && self
5895 .base_schedule
5896 .as_ref()
5897 .is_none_or(|b| entry.candidate_vertices.len() > b.candidate_vertices.len())
5898 {
5899 match self.base_schedule.replace(entry) {
5900 Some(previous) => entry = previous,
5901 None => return,
5902 }
5903 }
5904 if entry.topology_epoch != epoch
5905 || entry.authority_revision != revision
5906 || entry.candidate_vertices.len() > RECENT_SCHEDULE_VERTICES / 4
5907 {
5908 return;
5909 }
5910 self.recent_schedules.push(entry);
5911 let mut total: usize = self
5912 .recent_schedules
5913 .iter()
5914 .map(|e| e.candidate_vertices.len())
5915 .sum();
5916 while self.recent_schedules.len() > RECENT_SCHEDULES || total > RECENT_SCHEDULE_VERTICES {
5917 let oldest = self.recent_schedules.remove(0);
5918 total -= oldest.candidate_vertices.len();
5919 }
5920 }
5921
5922 fn invalidation_baseline(&self) -> InvalidationBaseline {
5923 InvalidationBaseline {
5924 snapshot_id: self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed),
5925 topology_epoch: self.topology_epoch,
5926 }
5927 }
5928
5929 fn classify_change_events(
5930 events: &[crate::engine::ChangeEvent],
5931 direction: LoggedEditDirection,
5932 ) -> LoggedEditImpact {
5933 use crate::engine::ChangeEvent;
5934
5935 events
5936 .iter()
5937 .map(|event| match event {
5938 ChangeEvent::CompoundStart { .. } | ChangeEvent::CompoundEnd { .. } => {
5939 LoggedEditImpact::NoOp
5940 }
5941 ChangeEvent::SetRowVisibility { .. }
5942 | ChangeEvent::SetValue {
5943 old_formula: None,
5944 old_value: Some(_),
5945 ..
5946 } => LoggedEditImpact::DataOnly,
5947 ChangeEvent::SetValue {
5951 old_formula: None,
5952 old_value: None,
5953 ..
5954 } if direction == LoggedEditDirection::Original => LoggedEditImpact::DataOnly,
5955 ChangeEvent::SetValue { .. }
5956 | ChangeEvent::SetFormula { .. }
5957 | ChangeEvent::AddVertex { .. }
5958 | ChangeEvent::RemoveVertex { .. }
5959 | ChangeEvent::VertexMoved { .. }
5960 | ChangeEvent::FormulaAdjusted { .. }
5961 | ChangeEvent::NamedRangeAdjusted { .. }
5962 | ChangeEvent::EdgeAdded { .. }
5963 | ChangeEvent::EdgeRemoved { .. }
5964 | ChangeEvent::DefineName { .. }
5965 | ChangeEvent::UpdateName { .. }
5966 | ChangeEvent::DeleteName { .. }
5967 | ChangeEvent::SpillCommitted { .. }
5968 | ChangeEvent::SpillCleared { .. }
5969 | ChangeEvent::StagedFormulaCellChanged { .. } => LoggedEditImpact::Topology,
5970 })
5971 .max()
5972 .unwrap_or(LoggedEditImpact::NoOp)
5973 }
5974
5975 fn classify_arrow_undo(arrow: &crate::engine::ArrowUndoBatch) -> LoggedEditImpact {
5976 use crate::engine::ArrowOp;
5977
5978 arrow
5979 .ops
5980 .iter()
5981 .map(|op| match op {
5982 ArrowOp::SetDeltaCell { .. } | ArrowOp::SetComputedCell { .. } => {
5983 LoggedEditImpact::DataOnly
5984 }
5985 ArrowOp::RestoreComputedRect { .. }
5986 | ArrowOp::InsertRows { .. }
5987 | ArrowOp::InsertCols { .. } => LoggedEditImpact::Topology,
5988 })
5989 .max()
5990 .unwrap_or(LoggedEditImpact::NoOp)
5991 }
5992
5993 fn apply_logged_edit_impact(
5994 &mut self,
5995 impact: LoggedEditImpact,
5996 baseline: InvalidationBaseline,
5997 ) {
5998 match impact {
5999 LoggedEditImpact::NoOp => {}
6000 LoggedEditImpact::DataOnly => {
6001 if self.topology_epoch == baseline.topology_epoch
6002 && self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed)
6003 == baseline.snapshot_id
6004 {
6005 self.mark_data_edited();
6006 }
6007 }
6008 LoggedEditImpact::Topology => {
6009 if self.topology_epoch == baseline.topology_epoch {
6012 self.mark_topology_edited();
6013 }
6014 }
6015 }
6016 }
6017
6018 fn invalidate_for_change_events(
6019 &mut self,
6020 events: &[crate::engine::ChangeEvent],
6021 direction: LoggedEditDirection,
6022 baseline: InvalidationBaseline,
6023 ) {
6024 self.apply_logged_edit_impact(Self::classify_change_events(events, direction), baseline);
6025 }
6026
6027 fn invalidate_for_action_journal(
6028 &mut self,
6029 journal: &crate::engine::ActionJournal,
6030 direction: LoggedEditDirection,
6031 baseline: InvalidationBaseline,
6032 ) {
6033 let impact = Self::classify_change_events(&journal.graph.events, direction)
6034 .max(Self::classify_arrow_undo(&journal.arrow));
6035 self.apply_logged_edit_impact(impact, baseline);
6036 }
6037
6038 pub fn mark_data_edited(&mut self) {
6041 self.lookup_index_cache.clear();
6042 self.snapshot_id
6043 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
6044 self.has_edited = true;
6045 }
6046
6047 pub fn mark_topology_edited(&mut self) {
6049 self.arrow_sheet_index.take();
6050 self.lookup_index_cache.clear();
6051 self.snapshot_id
6052 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
6053 self.topology_epoch = self.topology_epoch.wrapping_add(1);
6054 self.graph.bump_topology_revision();
6055 self.clear_cached_static_schedule();
6056 if let Some(ledger) = self.active_resource_ledger.as_mut() {
6057 let released = ledger.account_mixed_cache(0);
6058 debug_assert!(released.is_ok());
6059 }
6060 self.has_edited = true;
6061 self.graph.authority_sync_eager();
6065 }
6066
6067 fn mark_all_formula_vertices_dirty(&mut self) {
6068 let vertices: Vec<VertexId> = self.graph.vertices_with_formulas().collect();
6069 for vertex in vertices {
6070 self.graph.mark_vertex_dirty(vertex);
6071 }
6072 }
6073
6074 fn mark_moved_formula_vertices_dirty(
6075 &mut self,
6076 summary: &crate::engine::graph::editor::vertex_editor::ShiftSummary,
6077 ) {
6078 for vertex in &summary.vertices_moved {
6079 if self.graph.has_formula(*vertex) {
6080 self.graph.mark_vertex_dirty(*vertex);
6081 }
6082 }
6083 }
6084
6085 pub fn sheet_store(&self) -> &SheetStore {
6087 &self.arrow_sheets
6088 }
6089
6090 pub(crate) fn has_row_visibility_state(&self) -> bool {
6093 !self.row_visibility.is_empty()
6094 }
6095
6096 pub fn sheet_store_mut(&mut self) -> &mut SheetStore {
6098 self.arrow_sheet_index.take();
6099 &mut self.arrow_sheets
6100 }
6101
6102 pub fn has_staged_formulas(&self) -> bool {
6103 !self.staged_formulas.is_empty()
6104 }
6105
6106 pub fn staged_formula_count(&self) -> usize {
6107 self.staged_formulas.values().map(StagedSheet::len).sum()
6108 }
6109
6110 pub fn stage_formula_text(&mut self, sheet: &str, row: u32, col: u32, text: String) {
6112 self.staged_formulas
6113 .entry(sheet.to_string())
6114 .or_default()
6115 .stage(row, col, text);
6116 self.staged_formula_index.stage(sheet, row, col);
6117 if let Some(sheet) = self.graph.sheet_id(sheet) {
6118 self.invalidate_pending_spills(StructuralScope::Cell {
6119 sheet,
6120 row: row.saturating_sub(1),
6121 col: col.saturating_sub(1),
6122 });
6123 }
6124 }
6125
6126 fn index_removed_staged_sheet(&mut self, sheet: &str, staged: &StagedSheet) {
6127 for (row, col, _) in &staged.entries {
6128 self.staged_formula_index.remove(sheet, *row, *col);
6129 }
6130 if staged.deferred_package.is_some() {
6131 self.staged_formula_index.set_package(sheet, None);
6132 }
6133 }
6134
6135 fn restore_staged_sheet(&mut self, sheet: String, staged: StagedSheet) {
6136 self.staged_formulas.insert(sheet, staged);
6137 }
6138
6139 #[doc(hidden)]
6140 pub fn source_formula_ingress(&mut self) -> SourceFormulaIngress<'_, R> {
6141 SourceFormulaIngress { engine: self }
6142 }
6143
6144 #[doc(hidden)]
6145 #[doc(hidden)]
6148 #[cfg(any(test, feature = "test-support"))]
6149 pub fn set_before_prepared_span_commit_hook(
6150 &mut self,
6151 hook: impl FnOnce() + Send + Sync + 'static,
6152 ) {
6153 self.before_prepared_span_commit_hook = Some(Box::new(hook));
6154 }
6155
6156 #[doc(hidden)]
6157 #[cfg(test)]
6159 pub(crate) fn set_before_target_preparation_commit_hook(
6160 &mut self,
6161 hook: impl FnOnce() + Send + Sync + 'static,
6162 ) {
6163 self.before_target_preparation_commit_hook = Some(Box::new(hook));
6164 }
6165
6166 fn stage_deferred_formula_package(&mut self, package: crate::engine::DeferredFormulaPackage) {
6167 if let Ok(replay) = package.replay.lock()
6168 && let Some(footprint) = replay.selection_cache_footprint()
6169 && !self
6170 .source_cache_footprints
6171 .iter()
6172 .any(|known| known.ptr_eq(&footprint))
6173 {
6174 self.source_cache_footprints.push(footprint);
6175 }
6176 let sheet = package.sheet_name.clone();
6177 let staged = self.staged_formulas.entry(sheet.clone()).or_default();
6178 debug_assert!(staged.deferred_package.is_none());
6179 staged.deferred_package = Some(package);
6180 staged.reconcile_attached_deferred_package();
6181 self.staged_formula_index
6182 .set_package(&sheet, staged.deferred_package.as_ref());
6183 if let Some(sheet_id) = self.graph.sheet_id(&sheet) {
6184 let package = self
6185 .staged_formulas
6186 .get(&sheet)
6187 .and_then(|staged| staged.deferred_package.as_ref());
6188 for &(vertex, anchor, region) in &self.blocked_pending_spills {
6189 if anchor.sheet_id == sheet_id
6190 && self.graph.vertex_exists(vertex)
6191 && self.graph.get_cell_ref(vertex) == Some(anchor)
6192 && self.staged_formula_index.package_occupies_spill(
6193 &sheet,
6194 (anchor.coord.row() + 1, anchor.coord.col() + 1),
6195 (
6196 region.rows.query_bounds().1 + 1,
6197 region.cols.query_bounds().1 + 1,
6198 ),
6199 |point| package.is_some_and(|package| package.suppressed.contains(&point)),
6200 )
6201 {
6202 self.graph.mark_vertex_dirty(vertex);
6203 }
6204 }
6205 }
6206 }
6207
6208 pub fn clear_staged_formula_text(&mut self, sheet: &str, row: u32, col: u32) -> Option<String> {
6209 let mut removed = None;
6210 let mut remove_sheet = false;
6211 let mut had_package = false;
6212 if let Some(entries) = self.staged_formulas.get_mut(sheet) {
6213 had_package = entries.deferred_package.is_some();
6214 removed = entries.remove(row, col);
6215 remove_sheet = entries.is_empty();
6216 }
6217 let ordinary_removed = self.staged_formula_index.remove(sheet, row, col);
6218 if !ordinary_removed && had_package {
6219 self.staged_formula_index.touch_package(sheet);
6220 }
6221 if remove_sheet {
6222 self.staged_formulas.remove(sheet);
6223 self.staged_formula_index.set_package(sheet, None);
6224 }
6225 if (ordinary_removed || had_package)
6226 && let Some(sheet) = self.graph.sheet_id(sheet)
6227 {
6228 self.invalidate_pending_spills(StructuralScope::Cell {
6229 sheet,
6230 row: row.saturating_sub(1),
6231 col: col.saturating_sub(1),
6232 });
6233 }
6234 removed
6235 }
6236
6237 pub fn clear_staged_formulas_for_sheet(&mut self, sheet: &str) {
6238 if self.staged_formulas.remove(sheet).is_some() {
6239 self.staged_formula_index.clear_sheet(sheet);
6240 if let Some(sheet_id) = self.graph.sheet_id(sheet) {
6241 self.invalidate_pending_spills(StructuralScope::Sheet(sheet_id));
6242 }
6243 }
6244 }
6245
6246 pub fn rename_staged_formula_sheet(&mut self, old: &str, new: &str) {
6247 let Some(entries) = self.staged_formulas.remove(old) else {
6248 return;
6249 };
6250 self.staged_formula_index.clear_sheet(old);
6251 let (formulas, mut package) = entries.into_parts();
6252 for (row, col, text) in formulas {
6253 self.stage_formula_text(new, row, col, text);
6254 }
6255 if let Some(package) = package.as_mut() {
6256 package.sheet_name = new.to_string();
6257 }
6258 if let Some(package) = package {
6259 self.stage_deferred_formula_package(package);
6260 }
6261 }
6262
6263 pub fn get_staged_formula_text(&self, sheet: &str, row: u32, col: u32) -> Option<String> {
6265 self.staged_formulas
6266 .get(sheet)
6267 .and_then(|v| v.get(row, col))
6268 }
6269
6270 pub fn formula_parse_diagnostics(&self) -> &[FormulaParseDiagnostic] {
6271 &self.formula_parse_diagnostics
6272 }
6273
6274 pub fn take_formula_parse_diagnostics(&mut self) -> Vec<FormulaParseDiagnostic> {
6275 std::mem::take(&mut self.formula_parse_diagnostics)
6276 }
6277
6278 pub fn clear_formula_parse_diagnostics(&mut self) {
6279 self.formula_parse_diagnostics.clear();
6280 }
6281
6282 pub fn last_formula_ingest_report(&self) -> Option<&FormulaIngestReport> {
6283 self.last_formula_ingest_report.as_ref()
6284 }
6285
6286 pub fn formula_ingest_report_total(&self) -> &FormulaIngestReport {
6287 &self.formula_ingest_report_total
6288 }
6289
6290 #[cfg(test)]
6291 pub(crate) fn set_before_target_planning_snapshot_hook_for_test(
6292 &mut self,
6293 hook: impl FnOnce() + Send + Sync + 'static,
6294 ) {
6295 self.before_target_planning_snapshot_hook = Some(Box::new(hook));
6296 }
6297
6298 #[cfg(test)]
6299 pub(crate) fn inject_target_semantic_stale_once_for_test(&mut self) {
6300 self.inject_target_semantic_stale_once_for_test = true;
6301 }
6302
6303 #[cfg(test)]
6304 pub(crate) fn force_virtual_dep_changes_for_test(&mut self, rounds: usize) {
6305 self.force_virtual_dep_changes_remaining_for_test = rounds;
6306 }
6307
6308 #[cfg(test)]
6309 pub(crate) fn fail_evaluation_commit_preflight_once_for_test(&mut self) {
6310 self.fail_evaluation_commit_preflight_once_for_test = true;
6311 }
6312
6313 #[cfg(test)]
6314 pub(crate) fn set_target_preparation_fault_for_test(
6315 &mut self,
6316 fault: crate::engine::target_preparation::TargetPreparationFault,
6317 ) {
6318 self.target_preparation_fault_for_test = Some(fault);
6319 }
6320
6321 #[cfg(test)]
6322 pub(crate) fn staged_formula_index_revision_for_test(&self) -> u64 {
6323 self.staged_formula_index.revision()
6324 }
6325
6326 #[cfg(test)]
6327 pub(crate) fn deferred_package_for_test(
6328 &self,
6329 sheet: &str,
6330 ) -> &crate::engine::DeferredFormulaPackage {
6331 self.staged_formulas
6332 .get(sheet)
6333 .unwrap()
6334 .deferred_package
6335 .as_ref()
6336 .unwrap()
6337 }
6338
6339 #[cfg(test)]
6340 pub(crate) fn staged_formula_index_is_consistent_for_test(&self) -> bool {
6341 let ordinary_storage = self
6342 .staged_formulas
6343 .values()
6344 .map(|sheet| sheet.entries.len())
6345 .sum::<usize>();
6346 let package_storage = self
6347 .staged_formulas
6348 .values()
6349 .filter(|sheet| sheet.deferred_package.is_some())
6350 .count();
6351 ordinary_storage == self.staged_formula_index.ordinary_count()
6352 && package_storage == self.staged_formula_index.package_count()
6353 && self.staged_formulas.iter().all(|(name, sheet)| {
6354 sheet.entries.iter().all(|(row, col, _)| {
6355 let leases = self
6356 .staged_formula_index
6357 .leases_in_region(name, *row, *col, *row, *col);
6358 leases.len() == 1 && leases[0].row == *row && leases[0].col == *col
6359 })
6360 })
6361 }
6362
6363 #[cfg(test)]
6364 pub(crate) fn evaluation_request_begin_count_for_test(&self) -> u64 {
6365 self.evaluation_request_begin_count_for_test
6366 }
6367
6368 #[cfg(test)]
6369 pub(crate) fn set_before_legacy_fallback_final_provider_sample_hook(
6370 &mut self,
6371 hook: impl FnOnce() + Send + Sync + 'static,
6372 ) {
6373 self.before_legacy_fallback_final_provider_sample_hook = Some(Box::new(hook));
6374 }
6375
6376 #[cfg(test)]
6377 pub(crate) fn set_after_eager_proposal_commit_hook(
6378 &mut self,
6379 hook: impl FnOnce() + Send + Sync + 'static,
6380 ) {
6381 self.after_eager_proposal_commit_hook = Some(Box::new(hook));
6382 }
6383
6384 #[cfg(test)]
6385 pub(crate) fn topology_epoch_for_test(&self) -> u64 {
6386 self.topology_epoch
6387 }
6388
6389 #[cfg(test)]
6390 pub(crate) fn graph_topology_revision_for_test(&self) -> u64 {
6391 self.graph.topology_revision()
6392 }
6393
6394 fn record_formula_ingest_report(&mut self, report: FormulaIngestReport) {
6395 self.formula_ingest_report_total.mode = report.mode;
6396 self.formula_ingest_report_total.accumulate(&report);
6397 self.last_formula_ingest_report = Some(report);
6398 }
6399
6400 fn collect_planning_function_requests(
6401 ast: &ASTNode,
6402 requests: &mut Vec<(String, String, usize)>,
6403 ) {
6404 match &ast.node_type {
6405 ASTNodeType::Function { name, args } => {
6406 requests.push((String::new(), name.clone(), args.len()));
6407 for arg in args {
6408 Self::collect_planning_function_requests(arg, requests);
6409 }
6410 }
6411 ASTNodeType::BinaryOp { left, right, .. } => {
6412 Self::collect_planning_function_requests(left, requests);
6413 Self::collect_planning_function_requests(right, requests);
6414 }
6415 ASTNodeType::UnaryOp { expr, .. } => {
6416 Self::collect_planning_function_requests(expr, requests);
6417 }
6418 ASTNodeType::Call { callee, args } => {
6419 Self::collect_planning_function_requests(callee, requests);
6420 for arg in args {
6421 Self::collect_planning_function_requests(arg, requests);
6422 }
6423 }
6424 ASTNodeType::Array(rows) => {
6425 for cell in rows.iter().flatten() {
6426 Self::collect_planning_function_requests(cell, requests);
6427 }
6428 }
6429 ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => {}
6430 }
6431 }
6432
6433 fn prepared_function_semantics_changed(
6434 &self,
6435 preparation: &crate::engine::FormulaCompressedPreparation,
6436 guard: &crate::function_registry::SemanticEpochReadGuard,
6437 ) -> bool {
6438 if preparation.function_semantic_epoch == guard.epoch() {
6439 return false;
6440 }
6441
6442 guard
6443 .semantic_changes_affect_requests_since(preparation.function_semantic_epoch, Vec::new())
6444 }
6445
6446 pub(crate) fn prepare_source_formula_families(
6447 &mut self,
6448 sheet_name: &str,
6449 families: &[crate::engine::SourceFormulaFamily],
6450 ) -> crate::engine::FormulaCompressedPreparation {
6451 crate::engine::FormulaCompressedPreparation {
6452 engine_token: Arc::clone(&self.source_formula_token),
6453 function_semantic_epoch: crate::function_registry::semantic_epoch(),
6454 function_provider_revision: None,
6455 function_semantics_used: false,
6456 sheet_name: Arc::from(sheet_name),
6457 rejected: BTreeMap::new(),
6458 eager_replay: Vec::new(),
6459 preparation_spool_replays: 0,
6460 clean_rejected_anchor_counts: [0; 3],
6461 fragmented_rejected_anchor_counts: [0; 3],
6462 exact_replay: None,
6463 replay_disposition: crate::engine::FormulaReplayDisposition::default(),
6464 }
6465 }
6466
6467 fn prepare_source_formula_proposals(
6468 &mut self,
6469 sheet_name: &str,
6470 families: &[crate::engine::SourceFormulaFamily],
6471 authority_partitions: &[crate::engine::PartitionedSourceFormulaFamily],
6472 replay_partitions: &[crate::engine::PartitionedSourceFormulaFamily],
6473 formula_record_count: u64,
6474 replay: Arc<std::sync::Mutex<Box<dyn crate::engine::DeferredFormulaReplay>>>,
6475 suppressed: &BTreeSet<(u32, u32)>,
6476 consumed: &FxHashSet<(crate::engine::SourceFamilyId, crate::engine::SourceCoord)>,
6477 consumed_engine: Option<&Arc<()>>,
6478 ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
6479 let mut preparation = self.prepare_source_formula_families(sheet_name, families);
6480 preparation.exact_replay = Some(Arc::clone(&replay));
6481 preparation
6482 .replay_disposition
6483 .register_consumed_members(consumed_engine, consumed.iter().copied());
6484 if !preparation
6485 .replay_disposition
6486 .consumed_engine_matches(&self.source_formula_token)
6487 {
6488 return Err(ExcelError::new(ExcelErrorKind::Value)
6489 .with_message("ResidualConsumedEngineMismatch"));
6490 }
6491 preparation
6492 .replay_disposition
6493 .extend_suppressed_excel_coords(suppressed.iter().copied());
6494 Ok(preparation)
6495 }
6496
6497 fn formula_batch_from_exact_replay(
6498 &mut self,
6499 sheet_name: &str,
6500 replayed: impl IntoIterator<Item = crate::engine::DeferredReplayFormula>,
6501 ) -> Result<FormulaIngestBatch, ExcelError> {
6502 let mut cache = rustc_hash::FxHashMap::default();
6503 let mut formulas = Vec::new();
6504 for record in replayed {
6505 let key = if record.text.starts_with('=') {
6506 record.text
6507 } else {
6508 format!("={}", record.text)
6509 };
6510 let ast_id = if let Some(cached) = cache.get(&key) {
6511 *cached
6512 } else {
6513 let parsed = match formualizer_parse::parser::parse(&key) {
6514 Ok(parsed) => parsed,
6515 Err(error) => {
6516 let Some(parsed) = self.handle_formula_parse_error(
6517 sheet_name,
6518 record.row,
6519 record.col,
6520 &key,
6521 error.to_string(),
6522 )?
6523 else {
6524 continue;
6525 };
6526 parsed
6527 }
6528 };
6529 let ast_id = self.intern_formula_ast(&parsed);
6530 cache.insert(key.clone(), ast_id);
6531 ast_id
6532 };
6533 formulas.push(
6534 FormulaIngestRecord::new(record.row, record.col, ast_id, Some(Arc::from(key)))
6535 .with_source_proof(record.source_order, record.family, record.partition_owner),
6536 );
6537 }
6538 Ok(FormulaIngestBatch::new(sheet_name.to_string(), formulas))
6539 }
6540
6541 fn prepare_target_combined_legacy_graph(
6542 &self,
6543 packages: &[PreparedTargetSourcePackage],
6544 ordinary: &[PreparedOrdinaryStagedFormula],
6545 ) -> Result<(PreparedLegacyGraphPlan, usize), ExcelError> {
6546 let mut planned_by_coord = BTreeMap::new();
6547 for package in packages {
6548 for (row, col, ast_id, plan) in &package.legacy {
6549 planned_by_coord.insert((package.sheet_id, *row, *col), (*ast_id, plan.clone()));
6550 }
6551 }
6552 for formula in ordinary {
6553 if let Some((ast_id, plan)) = formula.ast_id.zip(formula.plan.clone()) {
6554 planned_by_coord.insert(
6555 (formula.sheet_id, formula.lease.row, formula.lease.col),
6556 (ast_id, plan),
6557 );
6558 }
6559 }
6560 let planned = planned_by_coord
6561 .into_iter()
6562 .map(|((sheet_id, row, col), (ast_id, plan))| (sheet_id, row, col, ast_id, plan))
6563 .collect::<Vec<_>>();
6564 let formula_count = planned.len();
6565 let graph = self
6566 .graph
6567 .prepare_legacy_graph_plan_multi_sheet(planned)
6568 .map_err(|error| {
6569 ExcelError::new(ExcelErrorKind::Value)
6570 .with_message(format!("target graph preparation failed: {error}"))
6571 })?;
6572 Ok((graph, formula_count))
6573 }
6574
6575 fn replay_target_coordinates(
6576 &mut self,
6577 replay: &Arc<std::sync::Mutex<Box<dyn crate::engine::DeferredFormulaReplay>>>,
6578 coordinates: &[(u32, u32)],
6579 deadline: Option<std::time::Instant>,
6580 scratch: &mut u64,
6581 ) -> Result<Option<Vec<crate::engine::DeferredReplayFormula>>, ExcelError> {
6582 if coordinates.is_empty() {
6583 return Ok(Some(Vec::new()));
6584 }
6585 let mut guard = replay.lock().map_err(|_| {
6586 ExcelError::new(ExcelErrorKind::Value)
6587 .with_message("deferred formula spool lock poisoned")
6588 })?;
6589 let retained = guard.selection_cache_footprint().is_some();
6590 let records = guard.replay_selected_exact(coordinates, &mut |work, bytes| {
6591 self.target_preparation_checkpoint(deadline, work)?;
6592 if retained && let Some(ledger) = self.active_resource_ledger.as_mut() {
6593 ledger
6594 .reserve_retained(bytes)
6595 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
6596 self.source_cache_accounted = self.source_cache_accounted.saturating_add(bytes);
6597 }
6598 self.reserve_graph_source_scratch(bytes)?;
6599 *scratch = scratch.saturating_add(bytes);
6600 Ok(())
6601 });
6602 drop(guard);
6603 let reconciled = self.reconcile_source_cache_footprints();
6604 let records = records?;
6605 reconciled?;
6606 Ok(records)
6607 }
6608
6609 fn prepare_target_exact_source_selection(
6610 &mut self,
6611 sheet: &str,
6612 lease: StagedPackageLease,
6613 coordinates: Vec<(u32, u32)>,
6614 previous: &BTreeSet<(u32, u32)>,
6615 allow_partial_shared: bool,
6616 deadline: Option<std::time::Instant>,
6617 scratch: &mut u64,
6618 ) -> Result<Option<PreparedTargetSourcePackage>, ExcelError> {
6619 let package = self
6620 .staged_formulas
6621 .get(sheet)
6622 .and_then(|s| s.deferred_package.as_ref())
6623 .unwrap();
6624 if !package.source_geometry_complete
6625 || ((!package.families.is_empty() || !package.partitioned_families.is_empty())
6626 && !package.coordinates_cover_families)
6627 || package.reconciliation_replay.is_some()
6628 {
6629 return Ok(None);
6630 }
6631 let selected_points: BTreeSet<_> = coordinates
6632 .into_iter()
6633 .filter(|point| !package.suppressed.contains(point))
6634 .collect();
6635 let mut prepared = PreparedTargetSourcePackage::empty_selection(
6636 sheet,
6637 self.graph.sheet_id(sheet).unwrap(),
6638 lease,
6639 );
6640 if selected_points.is_empty() {
6641 return Ok(Some(prepared));
6642 }
6643 let mut routing = crate::engine::FormulaReplayDisposition::default();
6644 for partition in &package.partitioned_families {
6645 routing
6646 .register_partition(partition, false)
6647 .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?;
6648 }
6649 routing.extend_suppressed_excel_coords(package.suppressed.iter().copied());
6650 routing.register_consumed_members(
6651 package.consumed_engine.as_ref(),
6652 package.consumed_members.iter().copied(),
6653 );
6654 let replay = Arc::clone(&package.replay);
6655 let source_report = package.accounting_report();
6656 prepared.source_report = source_report;
6657 prepared.disposition = routing.clone();
6658
6659 let replay_points: BTreeSet<_> = selected_points
6660 .iter()
6661 .copied()
6662 .filter(|&(row, col)| !prepared.direct_contains(row, col))
6663 .collect();
6664 let coordinates: Vec<_> = replay_points.iter().copied().collect();
6665 let records = self.replay_target_coordinates(&replay, &coordinates, deadline, scratch)?;
6666 prepared.spool_replays = u64::from(!coordinates.is_empty());
6667 let Some(mut replay_records) = records else {
6668 return Ok(None);
6669 };
6670 for record in &mut replay_records {
6671 if record.family.is_none() {
6672 record.partition_owner = routing
6673 .ordinary_disposition(crate::engine::SourceCoord {
6674 row: record.row.saturating_sub(1),
6675 col: record.col.saturating_sub(1),
6676 })
6677 .1;
6678 }
6679 }
6680 replay_records.sort_by_key(|record| record.source_order);
6681 if replay_records
6682 .iter()
6683 .any(|record| !selected_points.contains(&(record.row, record.col)))
6684 || replay_records
6685 .windows(2)
6686 .any(|records| records[0].source_order == records[1].source_order)
6687 {
6688 return Err(ExcelError::new(ExcelErrorKind::Value)
6689 .with_message("invalid indexed exact source selection"));
6690 }
6691 let represented: BTreeSet<_> = replay_records.iter().map(|r| (r.row, r.col)).collect();
6692 if represented != replay_points {
6693 return Err(ExcelError::new(ExcelErrorKind::Value)
6694 .with_message("incomplete indexed exact source selection"));
6695 }
6696 let source = self
6699 .staged_formulas
6700 .get(sheet)
6701 .unwrap()
6702 .deferred_package
6703 .as_ref()
6704 .unwrap();
6705 let contains = |rect: crate::engine::SourceRect, coord: crate::engine::SourceCoord| {
6706 coord.row >= rect.start.row
6707 && coord.row <= rect.end.row
6708 && coord.col >= rect.start.col
6709 && coord.col <= rect.end.col
6710 };
6711 let mut checked = BTreeSet::new();
6712 for record in replay_records.iter().rev() {
6713 if !checked.insert((record.row, record.col)) {
6714 continue;
6715 }
6716 let coord = crate::engine::SourceCoord {
6717 row: record.row - 1,
6718 col: record.col - 1,
6719 };
6720 let agrees = |owner, shared: bool| {
6721 record.family == shared.then_some(owner)
6722 && record.partition_owner.or(record.family) == Some(owner)
6723 };
6724 let mut valid = true;
6725 for family in &source.families {
6726 let owns = match &family.members {
6727 crate::engine::SourceFamilyMembers::CompleteDomain(domain) => {
6728 contains(domain.rect(), coord)
6729 }
6730 crate::engine::SourceFamilyMembers::ExplicitMembers(members) => {
6731 members.as_slice().binary_search(&coord).is_ok()
6732 }
6733 };
6734 if owns && !agrees(family.source_id, true) {
6735 valid = false;
6736 }
6737 }
6738 for family in &source.partitioned_families {
6739 if family
6740 .fragments
6741 .iter()
6742 .any(|fragment| contains(fragment.rect(), coord))
6743 && !agrees(family.source_id, true)
6744 {
6745 valid = false;
6746 }
6747 if let Some(member) = family
6748 .legacy_members
6749 .as_slice()
6750 .iter()
6751 .find(|member| member.coord == coord)
6752 && !agrees(
6753 family.source_id,
6754 member.kind == crate::engine::PartitionLegacyMemberKind::SharedFamilyMember,
6755 )
6756 {
6757 valid = false;
6758 }
6759 }
6760 if !valid {
6761 return Err(ExcelError::new(ExcelErrorKind::Value)
6762 .with_message("indexed source ownership mismatch"));
6763 }
6764 }
6765 prepared.selected_points = Some(selected_points);
6766 prepared.replay_records = replay_records;
6767 Ok(Some(prepared))
6768 }
6769
6770 fn prepare_target_source_package(
6771 &mut self,
6772 sheet: &str,
6773 lease: StagedPackageLease,
6774 deadline: Option<std::time::Instant>,
6775 ) -> Result<PreparedTargetSourcePackage, ExcelError> {
6776 let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
6777 ExcelError::new(ExcelErrorKind::Ref)
6778 .with_message(format!("deferred source sheet not found: {sheet}"))
6779 })?;
6780 let (
6781 source_report,
6782 families,
6783 partitions,
6784 replay,
6785 invalidated,
6786 suppressed,
6787 consumed_members,
6788 consumed_engine,
6789 reconciliation_replay,
6790 ) = {
6791 let package = self
6792 .staged_formulas
6793 .get(sheet)
6794 .and_then(|staged| staged.deferred_package.as_ref())
6795 .ok_or_else(|| {
6796 ExcelError::new(ExcelErrorKind::Value)
6797 .with_message("staged deferred source package is unavailable")
6798 })?;
6799 if package.sheet_name != sheet {
6800 return Err(ExcelError::new(ExcelErrorKind::Value)
6801 .with_message("deferred formula package sheet mismatch"));
6802 }
6803 (
6804 package.accounting_report(),
6805 package.families.clone(),
6806 package.partitioned_families.clone(),
6807 Arc::clone(&package.replay),
6808 package.invalidated.clone(),
6809 package.suppressed.clone(),
6810 package.consumed_members.clone(),
6811 package.consumed_engine.clone(),
6812 package.reconciliation_replay.clone(),
6813 )
6814 };
6815
6816 let mut replay_disposition = crate::engine::FormulaReplayDisposition::default();
6817 for partition in &partitions {
6818 replay_disposition
6819 .register_partition(partition, false)
6820 .map_err(|reason| ExcelError::new(ExcelErrorKind::Value).with_message(reason))?;
6821 }
6822 replay_disposition.extend_suppressed_excel_coords(suppressed.iter().copied());
6823 replay_disposition
6824 .register_consumed_members(consumed_engine.as_ref(), consumed_members.iter().copied());
6825 if !replay_disposition.consumed_engine_matches(&self.source_formula_token) {
6826 return Err(ExcelError::new(ExcelErrorKind::Value)
6827 .with_message("ResidualConsumedEngineMismatch"));
6828 }
6829 self.target_preparation_checkpoint(deadline, 1)?;
6830 let mut replay_records = if let Some(mut records) = reconciliation_replay {
6831 records.retain(|record| {
6832 let Some((row, col)) = record.row.checked_sub(1).zip(record.col.checked_sub(1))
6833 else {
6834 return true;
6835 };
6836 let coord = crate::engine::SourceCoord { row, col };
6837 let disposition = record.family.map_or_else(
6838 || replay_disposition.ordinary_disposition(coord).0,
6839 |family| replay_disposition.shared_disposition(family, coord),
6840 );
6841 !matches!(
6842 disposition,
6843 crate::engine::FormulaReplayCoordinateDisposition::Direct
6844 | crate::engine::FormulaReplayCoordinateDisposition::Suppressed
6845 )
6846 });
6847 records
6848 } else {
6849 replay
6850 .lock()
6851 .map_err(|_| {
6852 ExcelError::new(ExcelErrorKind::Value)
6853 .with_message("deferred formula spool lock poisoned")
6854 })?
6855 .replay_partitioned(&replay_disposition, &partitions)
6856 .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?
6857 };
6858 replay_records.sort_by_key(|record| record.source_order);
6859 for chunk in replay_records.chunks(256) {
6860 self.target_preparation_checkpoint(deadline, chunk.len() as u64)?;
6861 }
6862 if replay_records
6863 .windows(2)
6864 .any(|records| records[0].source_order == records[1].source_order)
6865 {
6866 return Err(ExcelError::new(ExcelErrorKind::Value)
6867 .with_message("duplicate deferred source-order proof"));
6868 }
6869
6870 let disposition = replay_disposition;
6871 let direct_families = 0usize;
6872 let direct_cells = 0u64;
6873 let direct_fragments = 0u64;
6874 let direct_complete_families = 0u64;
6875 let direct_complete_cells = 0u64;
6876 let direct_partition_families = 0u64;
6877 let direct_partition_cells = 0u64;
6878 let anchor_parses = 0u64;
6879 let anchor_asts = 0u64;
6880 let anchor_analyses = 0u64;
6881
6882 Ok(PreparedTargetSourcePackage {
6883 sheet: sheet.to_string(),
6884 sheet_id,
6885 lease,
6886 selected_points: None,
6887 complete_selections: Default::default(),
6888 deferred_shared: false,
6889 direct_domains: Vec::new(),
6890 source_report,
6891 replay_records,
6892 spool_replays: 1,
6893 disposition,
6894 legacy: Vec::new(),
6895 direct_families,
6896 direct_cells,
6897 direct_fragments,
6898 direct_complete_families,
6899 direct_complete_cells,
6900 direct_partition_families,
6901 direct_partition_cells,
6902 anchor_parses,
6903 anchor_asts,
6904 anchor_analyses,
6905 })
6906 }
6907
6908 fn fallback_planning_snapshot(
6909 &self,
6910 batch: &FormulaIngestBatch,
6911 ) -> Result<crate::function_registry::RegistryPlanningSnapshot, ExcelError> {
6912 let mut requests = Vec::new();
6913 for formula in &batch.formulas {
6914 let ast = self
6915 .graph
6916 .data_store()
6917 .retrieve_ast(formula.ast_id, self.graph.sheet_reg())
6918 .ok_or_else(|| {
6919 ExcelError::new(ExcelErrorKind::Value)
6920 .with_message("ordered fallback AST is unavailable")
6921 })?;
6922 Self::collect_planning_function_requests(&ast, &mut requests);
6923 }
6924 requests.sort();
6925 requests.dedup();
6926 crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
6927 &self.resolver,
6928 requests,
6929 )
6930 .map_err(|error| ExcelError::new(ExcelErrorKind::Value).with_message(format!("{error:?}")))
6931 }
6932
6933 fn prepare_legacy_batch_fallback(
6934 &mut self,
6935 batch: FormulaIngestBatch,
6936 function_provider: &dyn crate::traits::FunctionProvider,
6937 ) -> Result<
6938 (
6939 crate::engine::graph::prepared_legacy_graph::PreparedLegacyGraphPlan,
6940 u64,
6941 ),
6942 ExcelError,
6943 > {
6944 let formula_count = batch.formulas.len() as u64;
6945 let sheet_id = self.graph.sheet_id_mut(&batch.sheet_name);
6946 let mut planned = Vec::with_capacity(batch.formulas.len());
6947 for record in batch.formulas {
6948 let placement = CellRef::new(
6949 sheet_id,
6950 Coord::from_excel(record.row, record.col, true, true),
6951 );
6952 let ingested = self
6953 .graph
6954 .ingest_pipeline(function_provider)
6955 .enable_function_semantics()
6956 .ingest_formula(
6957 FormulaAstInput::RawArena(record.ast_id),
6958 placement,
6959 record.formula_text,
6960 )
6961 .map_err(|error| {
6962 ExcelError::new(ExcelErrorKind::Value).with_message(format!("{error:?}"))
6963 })?;
6964 planned.push((record.row, record.col, ingested.ast_id, ingested.dep_plan));
6965 }
6966 let plan = self
6967 .graph
6968 .prepare_legacy_graph_plan(sheet_id, planned)
6969 .map_err(|error| {
6970 ExcelError::new(ExcelErrorKind::Value).with_message(error.to_string())
6971 })?;
6972 Ok((plan, formula_count))
6973 }
6974
6975 fn publish_compressed_partial_report(
6976 &mut self,
6977 report: &FormulaIngestReport,
6978 direct_report: &FormulaIngestReport,
6979 ) {
6980 if direct_report.source_family_promoted == 0
6981 && direct_report.graph_formula_cells_materialized == 0
6982 {
6983 return;
6984 }
6985 let mut published = report.clone();
6986 published.accumulate(direct_report);
6987 self.record_formula_ingest_report(published);
6988 }
6989
6990 fn finish_compressed_formula_sources(
6991 &mut self,
6992 batches: Vec<(
6993 FormulaIngestBatch,
6994 crate::engine::FormulaCompressedSourceReport,
6995 crate::engine::FormulaCompressedPreparation,
6996 )>,
6997 ) -> Result<FormulaIngestReport, ExcelError> {
6998 self.observe_function_semantic_epoch()?;
6999 if batches.iter().any(|(_, _, preparation)| {
7000 !Arc::ptr_eq(&preparation.engine_token, &self.source_formula_token)
7001 }) {
7002 return Err(ExcelError::new(ExcelErrorKind::Value)
7003 .with_message("compressed source preparation belongs to another engine"));
7004 }
7005 if batches.iter().any(|(fallback, _, preparation)| {
7006 preparation.sheet_name.as_ref() != fallback.sheet_name
7007 }) {
7008 return Err(ExcelError::new(ExcelErrorKind::Value)
7009 .with_message("compressed source preparation sheet mismatch"));
7010 }
7011 let initial_guard = crate::function_registry::semantic_epoch_read_guard();
7012 let initial_provider_revision = self.resolver.planning_semantic_revision();
7013 let mut fallback_batches = Vec::with_capacity(batches.len());
7014 let mut stale_fallback_batches = Vec::new();
7015 let mut pending_preparations = Vec::new();
7016 for (mut fallback, mut source, mut preparation) in batches {
7017 for formula in fallback.formulas.drain(..) {
7018 let source_order = formula.source_order.ok_or_else(|| {
7019 ExcelError::new(ExcelErrorKind::Value).with_message(
7020 "compressed source supplied formulas without source-order proof",
7021 )
7022 })?;
7023 let text = formula.formula_text.ok_or_else(|| {
7024 ExcelError::new(ExcelErrorKind::Value).with_message(
7025 "ordered compressed fallback formula has no exact source text",
7026 )
7027 })?;
7028 preparation
7029 .eager_replay
7030 .push(crate::engine::DeferredReplayFormula {
7031 source_order,
7032 row: formula.row,
7033 col: formula.col,
7034 text: text.to_string(),
7035 family: formula.source_family,
7036 partition_owner: formula.partition_owner,
7037 });
7038 }
7039 preparation
7040 .eager_replay
7041 .sort_by_key(|record| record.source_order);
7042 source.source_spool_replays = source
7043 .source_spool_replays
7044 .saturating_add(preparation.preparation_spool_replays);
7045 let stale_reason = preparation
7046 .function_semantics_used
7047 .then(|| {
7048 if preparation.function_provider_revision != initial_provider_revision {
7049 Some("FunctionProviderRevisionChanged")
7050 } else if self.prepared_function_semantics_changed(&preparation, &initial_guard)
7051 {
7052 Some("FunctionSemanticEpochChanged")
7053 } else {
7054 None
7055 }
7056 })
7057 .flatten();
7058 let stale_semantics = stale_reason.is_some();
7059 for reason in preparation.rejected.values() {
7060 *source.fallback_reasons.entry(reason.clone()).or_default() += 1;
7061 }
7062 source.replay_families = source.families_seen;
7064 source.replay_cells = source.family_cells_seen;
7065 let compressed = crate::engine::FormulaCompressedSourceBatch::new(
7066 fallback.sheet_name.clone(),
7067 source,
7068 );
7069 if stale_semantics {
7070 stale_fallback_batches.push((fallback, compressed));
7071 } else {
7072 fallback_batches.push((fallback, compressed));
7073 }
7074 pending_preparations.push((preparation, stale_semantics));
7075 }
7076 drop(initial_guard);
7077
7078 let configured_mode = self.config.formula_plane_mode;
7080 self.config.formula_plane_mode = FormulaPlaneMode::Off;
7081 let stale_result =
7082 self.ingest_compressed_formula_source_batches_inner(stale_fallback_batches, false);
7083 self.config.formula_plane_mode = configured_mode;
7084 let mut report = stale_result?;
7085 report.mode = configured_mode;
7086
7087 self.config.formula_plane_mode = FormulaPlaneMode::Off;
7088 let fallback_result =
7089 self.ingest_compressed_formula_source_batches_inner(fallback_batches, false);
7090 self.config.formula_plane_mode = configured_mode;
7091 match fallback_result {
7092 Ok(fallback_report) => report.accumulate(&fallback_report),
7093 Err(error) => {
7094 self.record_formula_ingest_report(report);
7095 return Err(error);
7096 }
7097 }
7098
7099 let mut direct_report =
7100 FormulaIngestReport::with_mode(FormulaPlaneMode::AuthoritativeExperimental);
7101 for (preparation, _) in &pending_preparations {
7102 direct_report.source_anchor_parses = direct_report.source_anchor_parses.saturating_add(
7103 preparation.clean_rejected_anchor_counts[0]
7104 .saturating_add(preparation.fragmented_rejected_anchor_counts[0]),
7105 );
7106 direct_report.source_anchor_asts = direct_report.source_anchor_asts.saturating_add(
7107 preparation.clean_rejected_anchor_counts[1]
7108 .saturating_add(preparation.fragmented_rejected_anchor_counts[1]),
7109 );
7110 direct_report.source_anchor_analyses =
7111 direct_report.source_anchor_analyses.saturating_add(
7112 preparation.clean_rejected_anchor_counts[2]
7113 .saturating_add(preparation.fragmented_rejected_anchor_counts[2]),
7114 );
7115 }
7116 loop {
7117 #[cfg(any(test, feature = "test-support"))]
7118 if let Some(hook) = self.before_prepared_span_commit_hook.take() {
7119 hook();
7120 }
7121 let commit_guard = crate::function_registry::semantic_epoch_read_guard();
7122 let commit_provider_revision = self.resolver.planning_semantic_revision();
7123 let mut newly_stale = Vec::new();
7124 let mut current = Vec::new();
7125 for pending in pending_preparations.drain(..) {
7126 let stale_reason = pending
7127 .0
7128 .function_semantics_used
7129 .then(|| {
7130 if pending.0.function_provider_revision != commit_provider_revision {
7131 Some("FunctionProviderRevisionChanged")
7132 } else if self
7133 .prepared_function_semantics_changed(&pending.0, &commit_guard)
7134 {
7135 Some("FunctionSemanticEpochChanged")
7136 } else {
7137 None
7138 }
7139 })
7140 .flatten();
7141 if let Some(reason) = stale_reason {
7142 newly_stale.push((pending, reason));
7143 } else {
7144 current.push(pending);
7145 }
7146 }
7147 if !newly_stale.is_empty() {
7148 drop(commit_guard);
7149 for ((mut preparation, was_initially_stale), reason) in newly_stale {
7150 preparation
7151 .eager_replay
7152 .sort_by_key(|record| record.source_order);
7153 direct_report.source_spool_replays =
7154 direct_report.source_spool_replays.saturating_add(1);
7155 if !was_initially_stale {
7156 direct_report
7157 .fallback_reasons
7158 .entry(reason.to_string())
7159 .or_default();
7160 }
7161 preparation.function_semantics_used = false;
7162 if !preparation.eager_replay.is_empty() {
7163 current.push((preparation, false));
7164 }
7165 }
7166 pending_preparations = current;
7167 continue;
7168 }
7169 drop(commit_guard);
7170
7171 enum SourceProposal {
7173 KnownFallback {
7174 source_order: crate::engine::SourceFormulaOrder,
7175 records: Vec<crate::engine::DeferredReplayFormula>,
7176 },
7177 }
7178
7179 impl SourceProposal {
7180 fn source_order(&self) -> crate::engine::SourceFormulaOrder {
7181 match self {
7182 Self::KnownFallback { source_order, .. } => *source_order,
7183 }
7184 }
7185 }
7186
7187 for (mut preparation, _) in current {
7188 let mut proposals = Vec::with_capacity(preparation.eager_replay.len());
7189 let mut family_fallbacks: BTreeMap<_, Vec<_>> = BTreeMap::new();
7190 for record in preparation.eager_replay.drain(..) {
7191 if let Some(owner) = record.partition_owner.or(record.family) {
7192 family_fallbacks.entry(owner).or_default().push(record);
7193 } else {
7194 proposals.push(SourceProposal::KnownFallback {
7195 source_order: record.source_order,
7196 records: vec![record],
7197 });
7198 }
7199 }
7200 for (_, mut records) in family_fallbacks {
7201 records.sort_by_key(|record| record.source_order);
7202 if records
7203 .windows(2)
7204 .any(|window| window[0].source_order == window[1].source_order)
7205 {
7206 self.publish_compressed_partial_report(&report, &direct_report);
7207 return Err(ExcelError::new(ExcelErrorKind::Value)
7208 .with_message("duplicate exact-replay source-order proof"));
7209 }
7210 let Some(source_order) = records.first().map(|record| record.source_order)
7211 else {
7212 self.publish_compressed_partial_report(&report, &direct_report);
7213 return Err(ExcelError::new(ExcelErrorKind::Value)
7214 .with_message("empty exact-replay fallback family"));
7215 };
7216 proposals.push(SourceProposal::KnownFallback {
7217 source_order,
7218 records,
7219 });
7220 }
7221 proposals.sort_by_key(SourceProposal::source_order);
7222 if proposals
7223 .windows(2)
7224 .any(|window| window[0].source_order() == window[1].source_order())
7225 {
7226 self.publish_compressed_partial_report(&report, &direct_report);
7227 return Err(ExcelError::new(ExcelErrorKind::Value)
7228 .with_message("ambiguous compressed source-order proof"));
7229 }
7230
7231 for proposal in proposals {
7232 match proposal {
7233 SourceProposal::KnownFallback { records, .. } => {
7234 let batch = match self.formula_batch_from_exact_replay(
7235 preparation.sheet_name.as_ref(),
7236 records,
7237 ) {
7238 Ok(batch) => batch,
7239 Err(error) => {
7240 self.publish_compressed_partial_report(&report, &direct_report);
7241 return Err(error);
7242 }
7243 };
7244 if batch.is_empty() {
7245 continue;
7246 }
7247 let snapshot = match self.fallback_planning_snapshot(&batch) {
7248 Ok(snapshot) => snapshot,
7249 Err(error) => {
7250 self.publish_compressed_partial_report(&report, &direct_report);
7251 return Err(error);
7252 }
7253 };
7254 let commit_guard =
7255 crate::function_registry::semantic_epoch_read_guard();
7256 let provider_revision_initial =
7257 self.resolver.planning_semantic_revision();
7258 if (commit_guard.epoch() != snapshot.epoch()
7259 && snapshot.semantic_changes_affect_requests_since_guarded(
7260 &commit_guard,
7261 snapshot.epoch(),
7262 ))
7263 || snapshot.provider_revision().is_some_and(|revision| {
7264 Some(revision) != provider_revision_initial
7265 })
7266 {
7267 drop(commit_guard);
7268 self.publish_compressed_partial_report(&report, &direct_report);
7269 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
7270 "ordered fallback planning snapshot became stale",
7271 ));
7272 }
7273 let (plan, formula_count) = match self
7274 .prepare_legacy_batch_fallback(batch, &snapshot)
7275 {
7276 Ok(plan) => plan,
7277 Err(error) => {
7278 drop(commit_guard);
7279 self.publish_compressed_partial_report(&report, &direct_report);
7280 return Err(error);
7281 }
7282 };
7283 let provider_revision_after =
7284 self.resolver.planning_semantic_revision();
7285 if provider_revision_after != provider_revision_initial {
7286 drop(commit_guard);
7287 self.publish_compressed_partial_report(&report, &direct_report);
7288 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
7289 "function provider changed while preparing ordered fallback",
7290 ));
7291 }
7292 let graph_vertices = plan.new_vertex_count();
7293 let Some(graph_edges) = plan.planned_edge_count() else {
7294 drop(commit_guard);
7295 self.publish_compressed_partial_report(&report, &direct_report);
7296 return Err(ExcelError::new(ExcelErrorKind::Value)
7297 .with_message("prepared ordered fallback size overflow"));
7298 };
7299 if let Err(error) = self.prepared_legacy_admission(&plan, formula_count)
7300 {
7301 drop(commit_guard);
7302 self.publish_compressed_partial_report(&report, &direct_report);
7303 return Err(error);
7304 }
7305 if let Err(error) =
7306 self.graph.validate_prepared_legacy_graph_plan(&plan)
7307 {
7308 drop(commit_guard);
7309 self.publish_compressed_partial_report(&report, &direct_report);
7310 return Err(ExcelError::new(ExcelErrorKind::Value)
7311 .with_message(error.to_string()));
7312 }
7313 #[cfg(test)]
7314 if let Some(hook) = self
7315 .before_legacy_fallback_final_provider_sample_hook
7316 .take()
7317 {
7318 hook();
7319 }
7320 let provider_revision_final =
7321 self.resolver.planning_semantic_revision();
7322 if provider_revision_final != provider_revision_initial {
7323 drop(commit_guard);
7324 self.publish_compressed_partial_report(&report, &direct_report);
7325 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
7326 "function provider changed after ordered fallback validation",
7327 ));
7328 }
7329 let graph_formulas =
7330 self.graph.apply_prevalidated_legacy_graph_plan(plan);
7331 direct_report.formula_cells_seen = direct_report
7332 .formula_cells_seen
7333 .saturating_add(formula_count);
7334 direct_report.graph_formula_cells_materialized = direct_report
7335 .graph_formula_cells_materialized
7336 .saturating_add(graph_formulas as u64);
7337 direct_report.graph_vertices_created = direct_report
7338 .graph_vertices_created
7339 .saturating_add(graph_vertices as u64);
7340 direct_report.graph_edges_created = direct_report
7341 .graph_edges_created
7342 .saturating_add(graph_edges as u64);
7343 }
7344 }
7345 #[cfg(test)]
7346 if let Some(hook) = self.after_eager_proposal_commit_hook.take() {
7347 hook();
7348 }
7349 }
7350 }
7351 break;
7352 }
7353 report.accumulate(&direct_report);
7354 self.record_formula_ingest_report(report.clone());
7355 Ok(report)
7356 }
7357 pub(crate) fn ingest_compressed_formula_source_batches(
7359 &mut self,
7360 batches: Vec<(
7361 FormulaIngestBatch,
7362 crate::engine::FormulaCompressedSourceBatch,
7363 )>,
7364 ) -> Result<FormulaIngestReport, ExcelError> {
7365 self.ingest_compressed_formula_source_batches_inner(batches, true)
7366 }
7367
7368 fn ingest_compressed_formula_source_batches_inner(
7369 &mut self,
7370 batches: Vec<(
7371 FormulaIngestBatch,
7372 crate::engine::FormulaCompressedSourceBatch,
7373 )>,
7374 publish_report: bool,
7375 ) -> Result<FormulaIngestReport, ExcelError> {
7376 let mut source_counts = [0_u64; 11];
7377 let mut source_report = crate::engine::FormulaCompressedSourceReport::default();
7378 let mut formula_batches = Vec::with_capacity(batches.len());
7379 let mut compressed_families = Vec::new();
7380 let mut partitioned_families = Vec::new();
7381 for (batch, compressed_batch) in batches {
7382 let (sheet_name, compressed, families, partitions) = compressed_batch.into_parts();
7383 if sheet_name.as_ref() != batch.sheet_name {
7384 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
7385 "compressed formula source sheet does not match its replay batch",
7386 ));
7387 }
7388 compressed_families.push((batch.sheet_name.clone(), families));
7389 partitioned_families.push((batch.sheet_name.clone(), partitions));
7390 source_counts[0] = source_counts[0].saturating_add(compressed.source_formula_events);
7391 source_counts[1] = source_counts[1].saturating_add(compressed.source_ordinary_events);
7392 source_counts[2] =
7393 source_counts[2].saturating_add(compressed.source_shared_anchor_events);
7394 source_counts[3] =
7395 source_counts[3].saturating_add(compressed.source_shared_descendant_events);
7396 source_counts[4] = source_counts[4].saturating_add(compressed.source_unknown_events);
7397 source_counts[5] =
7398 source_counts[5].saturating_add(compressed.source_formula_records_spooled);
7399 source_counts[6] =
7400 source_counts[6].saturating_add(compressed.source_spool_encoded_bytes);
7401 source_counts[7] = source_counts[7].max(compressed.source_spool_peak_memory_bytes);
7402 source_counts[8] =
7403 source_counts[8].saturating_add(compressed.source_spool_spilled_bytes);
7404 source_counts[9] = source_counts[9].saturating_add(compressed.source_spool_spill_files);
7405 source_counts[10] = source_counts[10].saturating_add(compressed.source_spool_replays);
7406 source_report.families_seen = source_report
7407 .families_seen
7408 .saturating_add(compressed.families_seen);
7409 source_report.family_cells_seen = source_report
7410 .family_cells_seen
7411 .saturating_add(compressed.family_cells_seen);
7412 source_report.source_clean_families = source_report
7413 .source_clean_families
7414 .saturating_add(compressed.source_clean_families);
7415 source_report.source_clean_cells = source_report
7416 .source_clean_cells
7417 .saturating_add(compressed.source_clean_cells);
7418 source_report.source_fragmentable_families = source_report
7419 .source_fragmentable_families
7420 .saturating_add(compressed.source_fragmentable_families);
7421 source_report.source_fragmentable_cells = source_report
7422 .source_fragmentable_cells
7423 .saturating_add(compressed.source_fragmentable_cells);
7424 source_report.source_fragment_count = source_report
7425 .source_fragment_count
7426 .saturating_add(compressed.source_fragment_count);
7427 source_report.source_isolated_fallback_cells = source_report
7428 .source_isolated_fallback_cells
7429 .saturating_add(compressed.source_isolated_fallback_cells);
7430 source_report.source_hole_exclusions = source_report
7431 .source_hole_exclusions
7432 .saturating_add(compressed.source_hole_exclusions);
7433 source_report.source_ordinary_exclusions = source_report
7434 .source_ordinary_exclusions
7435 .saturating_add(compressed.source_ordinary_exclusions);
7436 source_report.source_partition_failures = source_report
7437 .source_partition_failures
7438 .saturating_add(compressed.source_partition_failures);
7439 source_report.replay_families = source_report
7440 .replay_families
7441 .saturating_add(compressed.replay_families);
7442 source_report.replay_cells = source_report
7443 .replay_cells
7444 .saturating_add(compressed.replay_cells);
7445 source_report.forward_descendants = source_report
7446 .forward_descendants
7447 .saturating_add(compressed.forward_descendants);
7448 source_report.evidence_limit_fallbacks = source_report
7449 .evidence_limit_fallbacks
7450 .saturating_add(compressed.evidence_limit_fallbacks);
7451 source_report.evidence_peak_bytes = source_report
7452 .evidence_peak_bytes
7453 .max(compressed.evidence_peak_bytes);
7454 for (reason, count) in compressed.fallback_reasons {
7455 *source_report.fallback_reasons.entry(reason).or_default() += count;
7456 }
7457 formula_batches.push(batch);
7458 }
7459 self.ingest_formula_batches_inner(
7460 formula_batches,
7461 source_counts,
7462 Some(source_report),
7463 compressed_families,
7464 partitioned_families,
7465 publish_report,
7466 )
7467 }
7468
7469 pub fn ingest_formula_batches(
7470 &mut self,
7471 batches: Vec<FormulaIngestBatch>,
7472 ) -> Result<FormulaIngestReport, ExcelError> {
7473 let has_formulas = batches.iter().any(|batch| !batch.formulas.is_empty());
7474 let report = self.ingest_formula_batches_inner(
7475 batches,
7476 [0; 11],
7477 None,
7478 Vec::new(),
7479 Vec::new(),
7480 true,
7481 )?;
7482 if has_formulas {
7483 self.mark_topology_edited();
7484 }
7485 Ok(report)
7486 }
7487
7488 fn ingest_formula_batches_unpublished(
7489 &mut self,
7490 batches: Vec<FormulaIngestBatch>,
7491 ) -> Result<FormulaIngestReport, ExcelError> {
7492 self.ingest_formula_batches_inner(batches, [0; 11], None, Vec::new(), Vec::new(), false)
7493 }
7494
7495 fn ingest_formula_batches_inner(
7496 &mut self,
7497 batches: Vec<FormulaIngestBatch>,
7498 source_counts: [u64; 11],
7499 source_report: Option<crate::engine::FormulaCompressedSourceReport>,
7500 compressed_families: Vec<(String, Vec<crate::engine::SourceFormulaFamily>)>,
7501 partitioned_families: Vec<(String, Vec<crate::engine::PartitionedSourceFormulaFamily>)>,
7502 publish_report: bool,
7503 ) -> Result<FormulaIngestReport, ExcelError> {
7504 self.observe_function_semantic_epoch()?;
7505 let formula_cells_seen = batches.iter().map(|batch| batch.len() as u64).sum();
7506 #[cfg(feature = "tracing")]
7507 let arena_nodes_before = self.graph.data_store().memory_usage().total_ast_nodes;
7508 #[cfg(feature = "tracing")]
7509 let route = if !partitioned_families.is_empty() {
7510 "partitioned"
7511 } else if !compressed_families.is_empty() {
7512 "compressed_source"
7513 } else if source_report.is_some() {
7514 "replay"
7515 } else {
7516 "ordinary"
7517 };
7518 let _ingest_span = crate::engine::trace::fz_span!(
7519 tracing::Level::INFO,
7520 "ingest",
7521 "ingest.batch",
7522 mode = ?FormulaPlaneMode::Off,
7523 route,
7524 formula_cells = formula_cells_seen
7525 );
7526 let mut report = FormulaIngestReport::with_mode(FormulaPlaneMode::Off);
7527 let materialize_batches = batches;
7528 report.formula_cells_seen = formula_cells_seen;
7529 report.source_formula_events = source_counts[0];
7530 report.source_ordinary_events = source_counts[1];
7531 report.source_shared_anchor_events = source_counts[2];
7532 report.source_shared_descendant_events = source_counts[3];
7533 report.source_unknown_events = source_counts[4];
7534 report.source_formula_records_spooled = source_counts[5];
7535 report.source_spool_encoded_bytes = source_counts[6];
7536 report.source_spool_peak_memory_bytes = source_counts[7];
7537 report.source_spool_spilled_bytes = source_counts[8];
7538 report.source_spool_spill_files = source_counts[9];
7539 report.source_spool_replays = source_counts[10];
7540 if let Some(source) = source_report {
7541 report.source_families_seen = source.families_seen;
7542 report.source_family_cells_seen = source.family_cells_seen;
7543 report.source_family_shadow_eligible = source.source_clean_families;
7544 report.source_family_shadow_eligible_cells = source.source_clean_cells;
7545 report.source_partitioned_families_seen = source.source_fragmentable_families;
7546 report.source_partition_holes = source.source_hole_exclusions;
7547 report.source_partition_ordinary_exceptions = source.source_ordinary_exclusions;
7548 report.source_partition_failures = source.source_partition_failures;
7549 report.source_partition_surviving_cells = source.source_fragmentable_cells;
7550 report.source_family_fallback = report
7551 .source_family_fallback
7552 .saturating_add(source.replay_families);
7553 report.source_family_fallback_cells = report
7554 .source_family_fallback_cells
7555 .saturating_add(source.replay_cells);
7556 report.source_forward_descendants = source.forward_descendants;
7557 report.source_evidence_limit_fallbacks = source.evidence_limit_fallbacks;
7558 report.source_evidence_peak_bytes = source.evidence_peak_bytes;
7559 for (reason, count) in source.fallback_reasons {
7560 let total = report.fallback_reasons.entry(reason).or_default();
7561 *total = total.saturating_add(count);
7562 }
7563 }
7564
7565 if self.graph.first_load_assume_new()
7569 && !self.graph_admission_enabled()
7570 && !materialize_batches.iter().all(FormulaIngestBatch::is_empty)
7571 {
7572 let mut builder =
7573 crate::engine::ingest_builder::BulkIngestBuilder::new(&mut self.graph);
7574 for batch in materialize_batches {
7575 if batch.is_empty() {
7576 continue;
7577 }
7578 let sheet_id = builder
7579 .add_sheet_checked(&batch.sheet_name)
7580 .ok_or_else(|| {
7581 ExcelError::new(ExcelErrorKind::Ref)
7582 .with_message(format!("unknown ingest sheet: {}", batch.sheet_name))
7583 })?;
7584 builder.add_formula_refs(
7585 sheet_id,
7586 batch.formulas.into_iter().map(|record| {
7587 (
7588 record.row,
7589 record.col,
7590 crate::engine::graph::FormulaRef::of_ingested(
7591 record.ast_id,
7592 record.member_anchor,
7593 ),
7594 )
7595 }),
7596 );
7597 }
7598 let summary = builder.finish_with_provider(&self.resolver)?;
7599 report.graph_formula_cells_materialized = summary.formulas as u64;
7600 report.graph_vertices_created = summary.vertices as u64;
7601 report.graph_edges_created = summary.edges as u64;
7602 } else if !materialize_batches.iter().all(FormulaIngestBatch::is_empty) {
7603 let mut prepared_by_sheet: BTreeMap<String, Vec<_>> = BTreeMap::new();
7604 for batch in materialize_batches {
7605 if batch.is_empty() {
7606 continue;
7607 }
7608 let sheet_id = self.graph.sheet_id(&batch.sheet_name).ok_or_else(|| {
7609 ExcelError::new(ExcelErrorKind::Ref)
7610 .with_message(format!("unknown ingest sheet: {}", batch.sheet_name))
7611 })?;
7612 let mut pipeline = self.ingest_pipeline();
7613 let prepared = prepared_by_sheet.entry(batch.sheet_name).or_default();
7616 prepared.reserve(batch.formulas.len());
7617 for record in batch.formulas {
7618 let placement = CellRef::new(
7619 sheet_id,
7620 Coord::from_excel(record.row, record.col, true, true),
7621 );
7622 let input = match record.member_anchor {
7623 Some(anchor) => FormulaAstInput::Member {
7624 template: record.ast_id,
7625 anchor,
7626 },
7627 None => FormulaAstInput::RawArena(record.ast_id),
7628 };
7629 let formula = pipeline.ingest_formula(input, placement, None)?;
7630 prepared.push((
7631 record.row,
7632 record.col,
7633 crate::engine::graph::FormulaRef::of_ingested(
7634 formula.ast_id,
7635 formula.member_anchor,
7636 ),
7637 formula.dep_plan,
7638 ));
7639 }
7640 }
7641 let admission_preflighted = self.graph_admission_enabled();
7642 if admission_preflighted {
7643 let mut preview = Vec::new();
7644 for (sheet_name, formulas) in &prepared_by_sheet {
7645 let sheet_id = self.graph.sheet_id(sheet_name).ok_or_else(|| {
7646 ExcelError::new(ExcelErrorKind::Ref)
7647 .with_message(format!("unknown ingest sheet: {sheet_name}"))
7648 })?;
7649 preview.extend(
7650 formulas
7651 .iter()
7652 .map(|(row, col, _, plan)| (sheet_id, *row, *col, plan.clone())),
7653 );
7654 }
7655 let admission = self.graph.preview_formula_mutations(&preview)?;
7656 self.preflight_graph_admission(admission)?;
7657 }
7658
7659 let mut builder = self.begin_bulk_ingest();
7660 if admission_preflighted {
7661 builder.mark_admission_preflighted();
7662 }
7663 for (sheet_name, formulas) in prepared_by_sheet {
7664 if formulas.is_empty() {
7665 continue;
7666 }
7667 let sheet_id = builder.add_sheet(&sheet_name);
7668 builder.add_formula_plans(sheet_id, formulas);
7669 }
7670 let summary = builder.finish()?;
7671 report.graph_formula_cells_materialized = summary.formulas as u64;
7672 report.graph_vertices_created = summary.vertices as u64;
7673 report.graph_edges_created = summary.edges as u64;
7674 }
7675
7676 crate::engine::trace::fz_event!(
7677 tracing::Level::INFO,
7678 "ingest",
7679 "ingest.summary",
7680 candidate_cells = report.shadow_candidate_cells,
7681 accepted_span_cells = report.shadow_accepted_span_cells,
7682 fallback_cells = report.shadow_fallback_cells,
7683 spans_created = report.shadow_spans_created,
7684 templates_interned = report.shadow_templates_interned,
7685 graph_vertices_created = report.graph_vertices_created,
7686 graph_edges_created = report.graph_edges_created,
7687 arena_nodes_delta = self
7688 .graph
7689 .data_store()
7690 .memory_usage()
7691 .total_ast_nodes
7692 .saturating_sub(arena_nodes_before)
7693 );
7694 if publish_report {
7695 self.record_formula_ingest_report(report.clone());
7696 }
7697 Ok(report)
7698 }
7699
7700 fn dedup_formula_parse_diagnostics_since(&mut self, start: usize) {
7701 let mut unique = Vec::new();
7702 for diagnostic in self.formula_parse_diagnostics.drain(start..) {
7703 let duplicate = unique.iter().any(|prior: &FormulaParseDiagnostic| {
7704 prior.sheet == diagnostic.sheet
7705 && prior.row == diagnostic.row
7706 && prior.col == diagnostic.col
7707 && prior.formula == diagnostic.formula
7708 && prior.policy == diagnostic.policy
7709 });
7710 if !duplicate {
7711 unique.push(diagnostic);
7712 }
7713 }
7714 self.formula_parse_diagnostics.extend(unique);
7715 }
7716
7717 pub fn handle_formula_parse_error(
7718 &mut self,
7719 sheet: &str,
7720 row: u32,
7721 col: u32,
7722 formula: &str,
7723 message: String,
7724 ) -> Result<Option<ASTNode>, ExcelError> {
7725 let policy = self.config.formula_parse_policy;
7726
7727 if policy == FormulaParsePolicy::Strict {
7728 let col_a1 = col_letters_from_1based(col).unwrap_or_else(|_| "?".to_string());
7729 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(format!(
7730 "Formula parse error at {sheet}!{col_a1}{row}: {message}"
7731 )));
7732 }
7733
7734 self.formula_parse_diagnostics.push(FormulaParseDiagnostic {
7735 sheet: sheet.to_string(),
7736 row,
7737 col,
7738 formula: formula.to_string(),
7739 message: message.clone(),
7740 policy,
7741 });
7742
7743 match policy {
7744 FormulaParsePolicy::Strict => unreachable!(),
7745 FormulaParsePolicy::KeepCachedValue => Ok(None),
7746 FormulaParsePolicy::AsText => Ok(Some(ASTNode::new(
7747 ASTNodeType::Literal(LiteralValue::Text(formula.to_string())),
7748 None,
7749 ))),
7750 FormulaParsePolicy::CoerceToError => {
7751 let err = ExcelError::new(ExcelErrorKind::Error)
7752 .with_message(format!("Malformed formula: {message}"));
7753 Ok(Some(ASTNode::new(
7754 ASTNodeType::Literal(LiteralValue::Error(err)),
7755 None,
7756 )))
7757 }
7758 }
7759 }
7760
7761 #[cfg(test)]
7762 fn target_preparation_fault(
7763 &mut self,
7764 seam: crate::engine::target_preparation::TargetPreparationFault,
7765 ) -> Result<(), ExcelError> {
7766 if self.target_preparation_fault_for_test == Some(seam) {
7767 self.target_preparation_fault_for_test = None;
7768 Err(ExcelError::new(ExcelErrorKind::Value)
7769 .with_message(format!("injected target preparation fault: {seam:?}")))
7770 } else {
7771 Ok(())
7772 }
7773 }
7774
7775 fn preparation_stale(
7776 reason: formualizer_common::PreparationStaleReason,
7777 message: impl Into<String>,
7778 ) -> ExcelError {
7779 ExcelError::new(ExcelErrorKind::Value)
7780 .with_message(message)
7781 .with_extra(formualizer_common::ExcelErrorExtra::PreparationStale { reason })
7782 }
7783
7784 fn preparation_revision_stale_reason(
7785 assumptions: &crate::engine::PreparationRevision,
7786 current: &crate::engine::PreparationRevision,
7787 planning_requests: &BTreeSet<(String, String, usize)>,
7788 staged_leases_match: bool,
7789 ) -> Option<formualizer_common::PreparationStaleReason> {
7790 if assumptions.graph != current.graph {
7791 Some(formualizer_common::PreparationStaleReason::Graph)
7792 } else if assumptions.staged != current.staged || !staged_leases_match {
7793 Some(formualizer_common::PreparationStaleReason::Staged)
7794 } else if assumptions.symbols != current.symbols {
7795 Some(formualizer_common::PreparationStaleReason::Symbols)
7796 } else if assumptions.provider != current.provider {
7797 Some(formualizer_common::PreparationStaleReason::Provider)
7798 } else if assumptions.semantic != current.semantic
7799 && crate::function_registry::semantic_changes_affect_requests_since(
7800 assumptions.semantic,
7801 planning_requests.iter().cloned(),
7802 )
7803 {
7804 Some(formualizer_common::PreparationStaleReason::Semantic)
7805 } else {
7806 None
7807 }
7808 }
7809
7810 fn preparation_revisions(&self) -> crate::engine::PreparationRevision {
7811 crate::engine::PreparationRevision {
7812 graph: self.graph.topology_revision(),
7813 authority: 0,
7814 authority_indexes: 0,
7815 authority_indexed_plane: 0,
7816 staged: self.staged_formula_index.revision(),
7817 symbols: self.graph.symbol_revision(),
7818 semantic: crate::function_registry::semantic_epoch(),
7819 provider: self.resolver.planning_semantic_revision(),
7820 }
7821 }
7822
7823 fn planning_revision_snapshot(&self) -> PlanningRevisionSnapshot {
7824 let registry_guard = crate::function_registry::semantic_epoch_read_guard();
7825 let provider = self.resolver.planning_semantic_revision();
7826 let semantic = registry_guard.epoch();
7827 PlanningRevisionSnapshot {
7828 engine_topology_epoch: self.topology_epoch,
7829 graph_topology_revision: self.graph.topology_revision(),
7830 staged: self.staged_formula_index.revision(),
7831 symbols: self.graph.symbol_revision(),
7832 semantic,
7833 provider,
7834 deterministic_mode: self.config.deterministic_mode.clone(),
7835 budgets: self.evaluation_resource_budgets.clone(),
7836 }
7837 }
7838
7839 fn recalc_plan_key(&self) -> RecalcPlanKey {
7840 RecalcPlanKey {
7841 engine_token: Arc::clone(&self.recalc_plan_token),
7842 revisions: self.planning_revision_snapshot(),
7843 }
7844 }
7845
7846 fn plan_stale(reason: formualizer_common::PlanStaleReason) -> ExcelError {
7847 ExcelError::new(ExcelErrorKind::Value)
7848 .with_message(format!("recalculation plan is stale: {}", reason.as_str()))
7849 .with_extra(formualizer_common::ExcelErrorExtra::PlanStale { reason })
7850 }
7851
7852 fn validate_recalc_plan_key(&self, key: &RecalcPlanKey) -> Result<(), ExcelError> {
7853 use formualizer_common::PlanStaleReason;
7854
7855 if !Arc::ptr_eq(&key.engine_token, &self.recalc_plan_token) {
7856 return Err(Self::plan_stale(PlanStaleReason::Engine));
7857 }
7858
7859 let current = self.planning_revision_snapshot();
7860 let expected = &key.revisions;
7861 let stale = if expected.provider != current.provider {
7862 Some(PlanStaleReason::Provider)
7863 } else if expected.semantic != current.semantic {
7864 Some(PlanStaleReason::Semantic)
7865 } else if expected.budgets != current.budgets
7866 || expected.deterministic_mode != current.deterministic_mode
7867 {
7868 Some(PlanStaleReason::Budget)
7869 } else if expected.staged != current.staged {
7870 Some(PlanStaleReason::Staged)
7871 } else if expected.symbols != current.symbols {
7872 Some(PlanStaleReason::Symbols)
7873 } else if expected.graph_topology_revision != current.graph_topology_revision
7874 || expected.engine_topology_epoch != current.engine_topology_epoch
7875 {
7876 Some(PlanStaleReason::Graph)
7877 } else {
7878 None
7879 };
7880 stale.map_or(Ok(()), |reason| Err(Self::plan_stale(reason)))
7881 }
7882
7883 fn target_preparation_checkpoint(
7884 &mut self,
7885 deadline: Option<std::time::Instant>,
7886 work: u64,
7887 ) -> Result<(), ExcelError> {
7888 if self
7889 .active_cancel_flag
7890 .as_ref()
7891 .is_some_and(|cancel| cancel.is_cancelled())
7892 {
7893 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
7894 .with_message("target graph preparation cancelled"));
7895 }
7896 if deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline) {
7897 return Err(crate::engine::ResourceLedgerError::Exhausted(
7898 formualizer_common::ResourceExhaustionDetail {
7899 reason: formualizer_common::ResourceExhaustionReason::Deadline,
7900 limit: 0,
7901 observed: 1,
7902 request_id: self
7903 .active_evaluation_resource_request
7904 .as_ref()
7905 .map(|stats| stats.request_id),
7906 },
7907 )
7908 .into_excel_error());
7909 }
7910 self.charge_bounded_work(work)
7911 }
7912
7913 fn opaque_reason_in_ast(
7914 &self,
7915 ast: &ASTNode,
7916 provider: &crate::function_registry::RegistryPlanningSnapshot,
7917 ) -> Option<crate::engine::OpaqueReason> {
7918 match &ast.node_type {
7919 ASTNodeType::Function { name, args } => {
7920 let canonical = name.rsplit('.').next().unwrap_or(name).to_ascii_uppercase();
7921 if canonical == "INDIRECT" {
7922 return Some(crate::engine::OpaqueReason::RuntimeTextReference);
7923 }
7924 let Some(caps) = provider.function_capabilities("", name) else {
7925 return Some(crate::engine::OpaqueReason::UnknownFunction);
7926 };
7927 if caps.contains(FnCaps::DYNAMIC_DEPENDENCY)
7928 || caps.contains(FnCaps::RETURNS_REFERENCE)
7929 {
7930 return Some(crate::engine::OpaqueReason::DynamicReference);
7931 }
7932 args.iter()
7933 .find_map(|arg| self.opaque_reason_in_ast(arg, provider))
7934 }
7935 ASTNodeType::Call { .. } => Some(crate::engine::OpaqueReason::UnknownCustomFunction),
7936 ASTNodeType::UnaryOp { expr, .. } => self.opaque_reason_in_ast(expr, provider),
7937 ASTNodeType::BinaryOp { left, right, .. } => self
7938 .opaque_reason_in_ast(left, provider)
7939 .or_else(|| self.opaque_reason_in_ast(right, provider)),
7940 ASTNodeType::Array(rows) => rows
7941 .iter()
7942 .flat_map(|row| row.iter())
7943 .find_map(|item| self.opaque_reason_in_ast(item, provider)),
7944 ASTNodeType::Reference {
7945 reference:
7946 ReferenceType::Cell {
7947 sheet: Some(sheet), ..
7948 }
7949 | ReferenceType::Range {
7950 sheet: Some(sheet), ..
7951 },
7952 ..
7953 } if self.graph.sheet_id(sheet).is_none() => {
7954 Some(crate::engine::OpaqueReason::UnresolvedCrossSheetBinding)
7955 }
7956 ASTNodeType::Reference {
7957 reference:
7958 ReferenceType::External(_)
7959 | ReferenceType::Cell3D { .. }
7960 | ReferenceType::Range3D { .. },
7961 ..
7962 } => Some(crate::engine::OpaqueReason::UnresolvedCrossSheetBinding),
7963 ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => None,
7964 }
7965 }
7966
7967 fn target_planning_snapshot(
7968 &mut self,
7969 ast: &ASTNode,
7970 planning_requests: &mut BTreeSet<(String, String, usize)>,
7971 ) -> Result<crate::function_registry::RegistryPlanningSnapshot, ExcelError> {
7972 #[cfg(test)]
7973 if std::mem::take(&mut self.inject_target_semantic_stale_once_for_test) {
7974 return Err(Self::preparation_stale(
7975 formualizer_common::PreparationStaleReason::Semantic,
7976 "injected target semantic preparation movement",
7977 ));
7978 }
7979 #[cfg(test)]
7980 if let Some(hook) = self.before_target_planning_snapshot_hook.take() {
7981 hook();
7982 }
7983 let mut requests = Vec::new();
7984 Self::collect_planning_function_requests(ast, &mut requests);
7985 requests.sort();
7986 requests.dedup();
7987 planning_requests.extend(requests.iter().cloned());
7988 crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
7989 &self.resolver,
7990 requests,
7991 )
7992 .map_err(|error| {
7993 ExcelError::new(ExcelErrorKind::Value)
7994 .with_message(format!("target planning snapshot unavailable: {error:?}"))
7995 })
7996 }
7997
7998 fn target_planning_snapshot_stale_reason(
7999 snapshot: &crate::function_registry::RegistryPlanningSnapshot,
8000 assumptions: &crate::engine::PreparationRevision,
8001 ) -> Option<formualizer_common::PreparationStaleReason> {
8002 if snapshot
8003 .provider_revision()
8004 .is_some_and(|revision| Some(revision) != assumptions.provider)
8005 {
8006 Some(formualizer_common::PreparationStaleReason::Provider)
8007 } else if snapshot.epoch() != assumptions.semantic
8008 && snapshot.semantic_changes_affect_requests_since(assumptions.semantic)
8009 {
8010 Some(formualizer_common::PreparationStaleReason::Semantic)
8011 } else {
8012 None
8013 }
8014 }
8015
8016 fn widen_target_preparation(
8017 policy: crate::engine::OpaquePreparePolicy,
8018 scope: &mut crate::engine::PrepareScope,
8019 reasons: &mut Vec<crate::engine::OpaqueReason>,
8020 reason: crate::engine::OpaqueReason,
8021 ) -> Result<bool, ExcelError> {
8022 if policy == crate::engine::OpaquePreparePolicy::Error {
8023 return Err(ExcelError::new(ExcelErrorKind::NImpl)
8024 .with_message(format!("opaque target preparation semantics: {reason:?}")));
8025 }
8026 if !reasons.contains(&reason) {
8027 reasons.push(reason);
8028 }
8029 if !matches!(scope, crate::engine::PrepareScope::Workbook) {
8030 *scope = crate::engine::PrepareScope::Workbook;
8031 Ok(true)
8032 } else {
8033 Ok(false)
8034 }
8035 }
8036
8037 fn widen_target_preparation_to_sheet(
8038 policy: crate::engine::OpaquePreparePolicy,
8039 scope: &mut crate::engine::PrepareScope,
8040 reasons: &mut Vec<crate::engine::OpaqueReason>,
8041 reason: crate::engine::OpaqueReason,
8042 sheet: &str,
8043 ) -> Result<bool, ExcelError> {
8044 if policy == crate::engine::OpaquePreparePolicy::Error {
8045 return Err(ExcelError::new(ExcelErrorKind::NImpl)
8046 .with_message(format!("opaque target preparation semantics: {reason:?}")));
8047 }
8048 if !reasons.contains(&reason) {
8049 reasons.push(reason);
8050 }
8051 match scope {
8052 crate::engine::PrepareScope::Exact => {
8053 *scope = crate::engine::PrepareScope::Sheets(vec![sheet.to_string()]);
8054 Ok(true)
8055 }
8056 crate::engine::PrepareScope::Sheets(sheets) => {
8057 if sheets.iter().any(|candidate| candidate == sheet) {
8058 Ok(false)
8059 } else {
8060 sheets.push(sheet.to_string());
8061 sheets.sort();
8062 Ok(true)
8063 }
8064 }
8065 crate::engine::PrepareScope::Workbook => Ok(false),
8066 }
8067 }
8068
8069 fn ast_has_proven_sheet_local_dynamic(
8070 ast: &ASTNode,
8071 provider: &crate::function_registry::RegistryPlanningSnapshot,
8072 ) -> bool {
8073 fn classify(
8074 ast: &ASTNode,
8075 provider: &crate::function_registry::RegistryPlanningSnapshot,
8076 ) -> (bool, bool) {
8077 match &ast.node_type {
8078 ASTNodeType::Function { name, args } => {
8079 let Some(caps) = provider.function_capabilities("", name) else {
8080 return (false, false);
8081 };
8082 let dynamic = caps.contains(FnCaps::DYNAMIC_DEPENDENCY)
8083 || caps.contains(FnCaps::RETURNS_REFERENCE);
8084 let canonical = name.rsplit('.').next().unwrap_or(name);
8085 if dynamic
8086 && !canonical.eq_ignore_ascii_case("OFFSET")
8087 && !canonical.eq_ignore_ascii_case("INDEX")
8088 {
8089 return (false, true);
8090 }
8091 let mut has_dynamic = dynamic;
8092 for arg in args {
8093 let (safe, child_dynamic) = classify(arg, provider);
8094 if !safe {
8095 return (false, has_dynamic || child_dynamic);
8096 }
8097 has_dynamic |= child_dynamic;
8098 }
8099 (true, has_dynamic)
8100 }
8101 ASTNodeType::UnaryOp { expr, .. } => classify(expr, provider),
8102 ASTNodeType::BinaryOp { left, right, .. } => {
8103 let (left_safe, left_dynamic) = classify(left, provider);
8104 let (right_safe, right_dynamic) = classify(right, provider);
8105 (left_safe && right_safe, left_dynamic || right_dynamic)
8106 }
8107 ASTNodeType::Array(rows) => {
8108 let mut has_dynamic = false;
8109 for item in rows.iter().flatten() {
8110 let (safe, child_dynamic) = classify(item, provider);
8111 if !safe {
8112 return (false, has_dynamic || child_dynamic);
8113 }
8114 has_dynamic |= child_dynamic;
8115 }
8116 (true, has_dynamic)
8117 }
8118 ASTNodeType::Reference {
8119 reference:
8120 ReferenceType::Cell { sheet: None, .. }
8121 | ReferenceType::Range { sheet: None, .. },
8122 ..
8123 }
8124 | ASTNodeType::Literal(_)
8125 | ASTNodeType::Omitted => (true, false),
8126 ASTNodeType::Call { .. } | ASTNodeType::Reference { .. } => (false, false),
8127 }
8128 }
8129
8130 let (safe, dynamic) = classify(ast, provider);
8131 safe && dynamic
8132 }
8133
8134 fn table_selection_region(
8135 &self,
8136 entry: &crate::engine::graph::TableEntry,
8137 selection: &crate::engine::TableSelection,
8138 ) -> Result<PreparationRegion, ExcelError> {
8139 let mut start_row = entry.range.start.coord.row() + 1;
8140 let mut end_row = entry.range.end.coord.row() + 1;
8141 let mut start_col = entry.range.start.coord.col() + 1;
8142 let mut end_col = entry.range.end.coord.col() + 1;
8143 match selection {
8144 crate::engine::TableSelection::Whole => {}
8145 crate::engine::TableSelection::Headers => {
8146 if !entry.header_row {
8147 return Err(ExcelError::new(ExcelErrorKind::Value)
8148 .with_message(format!("table {} has no header row", entry.name)));
8149 }
8150 end_row = start_row;
8151 }
8152 crate::engine::TableSelection::Data => {
8153 if entry.header_row {
8154 start_row = start_row.saturating_add(1);
8155 }
8156 if entry.totals_row {
8157 end_row = end_row.saturating_sub(1);
8158 }
8159 }
8160 crate::engine::TableSelection::Totals => {
8161 if !entry.totals_row {
8162 return Err(ExcelError::new(ExcelErrorKind::Value)
8163 .with_message(format!("table {} has no totals row", entry.name)));
8164 }
8165 start_row = end_row;
8166 }
8167 crate::engine::TableSelection::Column(column) => {
8168 let index = entry.col_index(column).ok_or_else(|| {
8169 ExcelError::new(ExcelErrorKind::Name)
8170 .with_message(format!("unknown table column: {column}"))
8171 })?;
8172 start_col = start_col.saturating_add(index as u32);
8173 end_col = start_col;
8174 }
8175 crate::engine::TableSelection::Columns { start, end } => {
8176 let first = entry.col_index(start).ok_or_else(|| {
8177 ExcelError::new(ExcelErrorKind::Name)
8178 .with_message(format!("unknown table column: {start}"))
8179 })?;
8180 let last = entry.col_index(end).ok_or_else(|| {
8181 ExcelError::new(ExcelErrorKind::Name)
8182 .with_message(format!("unknown table column: {end}"))
8183 })?;
8184 if first > last {
8185 return Err(ExcelError::new(ExcelErrorKind::Value)
8186 .with_message("table column selection is reversed"));
8187 }
8188 start_col = start_col.saturating_add(first as u32);
8189 end_col = entry
8190 .range
8191 .start
8192 .coord
8193 .col()
8194 .saturating_add(last as u32)
8195 .saturating_add(1);
8196 }
8197 }
8198 if start_row > end_row && matches!(selection, crate::engine::TableSelection::Data) {
8199 start_row = entry.range.start.coord.row() + 1;
8200 end_row = start_row;
8201 }
8202 if start_row > end_row || start_col > end_col {
8203 return Err(
8204 ExcelError::new(ExcelErrorKind::Value).with_message("table selection is empty")
8205 );
8206 }
8207 Ok(PreparationRegion {
8208 sheet: self.graph.sheet_name(entry.sheet_id()).to_string(),
8209 sheet_id: entry.sheet_id(),
8210 start_row,
8211 start_col,
8212 end_row,
8213 end_col,
8214 })
8215 }
8216
8217 #[cfg(test)]
8218 pub(crate) fn reset_target_root_dedup_probes_for_test() {
8219 TARGET_ROOT_DEDUP_PROBES.with(|probes| probes.set(0));
8220 }
8221
8222 #[cfg(test)]
8223 pub(crate) fn target_root_dedup_probes_for_test() -> usize {
8224 TARGET_ROOT_DEDUP_PROBES.with(std::cell::Cell::get)
8225 }
8226
8227 pub(crate) fn resolve_target_producers(
8228 &mut self,
8229 targets: &[crate::engine::EvaluationTarget],
8230 ) -> Result<Vec<crate::engine::target_preparation::TargetProducer>, ExcelError> {
8231 use crate::engine::target_preparation::TargetProducer;
8232
8233 let request_id = self
8234 .active_evaluation_resource_request
8235 .as_ref()
8236 .map(|request| request.request_id);
8237 let mut roots = OrderedTargetProducers::with_capacity(targets.len())
8238 .map_err(|_| target_root_allocation_error(targets.len(), request_id))?;
8239 let resolve_region = |engine: &mut Self,
8240 region: Region,
8241 value_only: Option<CellRef>,
8242 roots: &mut OrderedTargetProducers|
8243 -> Result<(), ExcelError> {
8244 let before = roots.len();
8245 for anchor in engine.graph.spill_anchors_in_region(
8246 region.sheet_id(),
8247 region.axis_ranges().0.query_bounds().0,
8248 region.axis_ranges().1.query_bounds().0,
8249 region.axis_ranges().0.query_bounds().1,
8250 region.axis_ranges().1.query_bounds().1,
8251 ) {
8252 roots
8253 .push(TargetProducer::Legacy(anchor))
8254 .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
8255 }
8256 for vertex in engine.graph.vertices_in_region(
8257 region.sheet_id(),
8258 region.axis_ranges().0.query_bounds().0,
8259 region.axis_ranges().0.query_bounds().1,
8260 region.axis_ranges().1.query_bounds().0,
8261 region.axis_ranges().1.query_bounds().1,
8262 ) {
8263 let vertex = engine
8264 .graph
8265 .get_cell_ref(vertex)
8266 .and_then(|cell| engine.graph.spill_registry_anchor_for_cell(cell))
8267 .unwrap_or(vertex);
8268 if matches!(
8273 engine.graph.get_vertex_kind(vertex),
8274 VertexKind::FormulaScalar | VertexKind::FormulaArray
8275 ) {
8276 roots
8277 .push(TargetProducer::Legacy(vertex))
8278 .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
8279 }
8280 }
8281 if roots.len() == before
8282 && let Some(cell) = value_only
8283 {
8284 roots
8285 .push(TargetProducer::ValueOnly(cell))
8286 .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
8287 }
8288 Ok(())
8289 };
8290
8291 for target in targets {
8292 match target {
8293 crate::engine::EvaluationTarget::Cell { sheet, row, col } => {
8294 if *row == 0 || *col == 0 {
8295 return Err(ExcelError::new(ExcelErrorKind::Ref)
8296 .with_message("target cell coordinates are one-based"));
8297 }
8298 let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
8299 ExcelError::new(ExcelErrorKind::Ref)
8300 .with_message(format!("target sheet not found: {sheet}"))
8301 })?;
8302 let cell = CellRef::new(sheet_id, Coord::from_excel(*row, *col, true, true));
8303 resolve_region(
8304 self,
8305 Region::point(sheet_id, *row - 1, *col - 1),
8306 Some(cell),
8307 &mut roots,
8308 )?;
8309 }
8310 crate::engine::EvaluationTarget::Range(range) => {
8311 let sheet_id = self.graph.sheet_id(&range.sheet).ok_or_else(|| {
8312 ExcelError::new(ExcelErrorKind::Ref)
8313 .with_message(format!("target sheet not found: {}", range.sheet))
8314 })?;
8315 resolve_region(
8316 self,
8317 Region::rect(
8318 sheet_id,
8319 range.start_row - 1,
8320 range.end_row - 1,
8321 range.start_col - 1,
8322 range.end_col - 1,
8323 ),
8324 None,
8325 &mut roots,
8326 )?;
8327 }
8328 crate::engine::EvaluationTarget::Name { name, scope_sheet } => {
8329 let scope = self.name_query_scope(scope_sheet.as_deref())?;
8330 if let Some(entry) = self.graph.resolve_name_entry_in_scope(name, scope) {
8331 roots
8332 .push(TargetProducer::Symbol(entry.vertex))
8333 .map_err(|_| {
8334 target_root_allocation_error(roots.len() + 1, request_id)
8335 })?;
8336 }
8337 }
8338 crate::engine::EvaluationTarget::Table { name, .. } => {
8339 if let Some(entry) = self.graph.resolve_table_entry(name) {
8340 roots
8341 .push(TargetProducer::Symbol(entry.vertex))
8342 .map_err(|_| {
8343 target_root_allocation_error(roots.len() + 1, request_id)
8344 })?;
8345 }
8346 }
8347 }
8348 }
8349 Ok(roots.into_vec())
8350 }
8351
8352 pub fn prepare_graph_for_targets(
8356 &mut self,
8357 targets: &[crate::engine::EvaluationTarget],
8358 options: crate::engine::TargetEvalOptions<'_>,
8359 ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
8360 let previous_budgets = self.evaluation_resource_budgets.clone();
8361 let diagnostics_len = self.formula_parse_diagnostics.len();
8362 let previous_report = self.last_formula_ingest_report.clone();
8363 if let Some(budgets) = options.budgets {
8364 self.evaluation_resource_budgets = budgets.clone();
8365 }
8366 let previous_graph_budget_override = self
8367 .graph
8368 .set_admission_budget_override(Some(self.evaluation_resource_budgets.clone()));
8369 let previous_cancel = self.active_cancel_flag.take();
8373 self.active_cancel_flag = options.cancel.clone();
8374 let result = self.observe_evaluation_resource_request(
8375 EvaluationRequestKind::TargetPreparation,
8376 |engine| engine.prepare_graph_for_targets_unobserved(targets, &options),
8377 );
8378 self.active_cancel_flag = previous_cancel;
8379 self.graph
8380 .set_admission_budget_override(previous_graph_budget_override);
8381 self.evaluation_resource_budgets = previous_budgets;
8382 if result.is_err() {
8383 self.formula_parse_diagnostics.truncate(diagnostics_len);
8384 self.last_formula_ingest_report = previous_report;
8385 }
8386 result
8387 }
8388
8389 fn prepare_graph_for_targets_unobserved(
8390 &mut self,
8391 targets: &[crate::engine::EvaluationTarget],
8392 options: &crate::engine::TargetEvalOptions<'_>,
8393 ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
8394 let scratch_checkpoint = self
8395 .active_resource_ledger
8396 .as_ref()
8397 .map_or(0, crate::engine::ResourceLedger::scratch_checkpoint);
8398 let result = self.prepare_graph_for_targets_transaction(targets, options);
8399 let release = self
8400 .active_resource_ledger
8401 .as_mut()
8402 .map_or(Ok(()), |ledger| {
8403 ledger.release_scratch_to(scratch_checkpoint)
8404 });
8405 match (result, release) {
8406 (result, Ok(())) => result,
8407 (Ok(_), Err(error)) | (Err(_), Err(error)) => Err(error.into_excel_error()),
8408 }
8409 }
8410
8411 fn prepare_graph_for_targets_transaction(
8412 &mut self,
8413 targets: &[crate::engine::EvaluationTarget],
8414 options: &crate::engine::TargetEvalOptions<'_>,
8415 ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
8416 use crate::engine::{
8417 OpaqueReason, PreparationOutcome, PrepareScope, PreparedTargetGraphReport,
8418 TableSelection,
8419 };
8420
8421 self.target_preparation_checkpoint(options.deadline, 0)?;
8422 self.observe_function_semantic_epoch()?;
8423 let assumptions = self.preparation_revisions();
8424 let ledger_at_start = self
8425 .active_resource_ledger
8426 .as_ref()
8427 .map(|ledger| ledger.snapshot());
8428 let diagnostics_len = self.formula_parse_diagnostics.len();
8429 let report_len = self.last_formula_ingest_report.clone();
8430 let request_id = options.request_id.or_else(|| {
8431 self.active_evaluation_resource_request
8432 .as_ref()
8433 .map(|stats| stats.request_id)
8434 });
8435
8436 let mut scope = PrepareScope::Exact;
8437 let mut reasons = Vec::new();
8438 let mut regions = VecDeque::new();
8439 let mut deferred_shared_regions = VecDeque::new();
8440 let mut normalized = Vec::with_capacity(targets.len());
8441 let mut symbol_vertices = VecDeque::new();
8442 for target in targets {
8443 self.target_preparation_checkpoint(options.deadline, 1)?;
8444 match target {
8445 crate::engine::EvaluationTarget::Cell { sheet, row, col } => {
8446 if *row == 0 || *col == 0 {
8447 return Err(ExcelError::new(ExcelErrorKind::Ref)
8448 .with_message("target cell coordinates are one-based"));
8449 }
8450 let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
8451 ExcelError::new(ExcelErrorKind::Ref)
8452 .with_message(format!("target sheet not found: {sheet}"))
8453 })?;
8454 regions.push_back(PreparationRegion {
8455 sheet: sheet.clone(),
8456 sheet_id,
8457 start_row: *row,
8458 start_col: *col,
8459 end_row: *row,
8460 end_col: *col,
8461 });
8462 normalized.push(target.clone());
8463 }
8464 crate::engine::EvaluationTarget::Range(range) => {
8465 if range.start_row == 0
8466 || range.start_col == 0
8467 || range.end_row < range.start_row
8468 || range.end_col < range.start_col
8469 {
8470 return Err(ExcelError::new(ExcelErrorKind::Ref)
8471 .with_message("invalid target range"));
8472 }
8473 let sheet_id = self.graph.sheet_id(&range.sheet).ok_or_else(|| {
8474 ExcelError::new(ExcelErrorKind::Ref)
8475 .with_message(format!("target sheet not found: {}", range.sheet))
8476 })?;
8477 regions.push_back(PreparationRegion {
8478 sheet: range.sheet.clone(),
8479 sheet_id,
8480 start_row: range.start_row,
8481 start_col: range.start_col,
8482 end_row: range.end_row,
8483 end_col: range.end_col,
8484 });
8485 normalized.push(target.clone());
8486 }
8487 crate::engine::EvaluationTarget::Name { name, scope_sheet } => {
8488 let name_scope = self.name_query_scope(scope_sheet.as_deref())?;
8489 if let Some(entry) = self.graph.resolve_name_entry_in_scope(name, name_scope) {
8490 symbol_vertices.push_back(entry.vertex);
8491 } else {
8492 Self::widen_target_preparation(
8493 options.opaque_policy,
8494 &mut scope,
8495 &mut reasons,
8496 OpaqueReason::UnresolvedName,
8497 )?;
8498 }
8499 normalized.push(target.clone());
8500 }
8501 crate::engine::EvaluationTarget::Table { name, selection } => {
8502 if let Some(entry) = self.graph.resolve_table_entry(name) {
8503 let region = self.table_selection_region(entry, selection)?;
8504 symbol_vertices.push_back(entry.vertex);
8505 regions.push_back(region);
8506 } else {
8507 Self::widen_target_preparation(
8508 options.opaque_policy,
8509 &mut scope,
8510 &mut reasons,
8511 OpaqueReason::UnresolvedTable,
8512 )?;
8513 }
8514 normalized.push(target.clone());
8515 }
8516 }
8517 }
8518
8519 let mut visited_regions = FxHashSet::default();
8520 let mut visited_vertices = FxHashSet::default();
8521 let mut selected = FxHashSet::default();
8522 let mut prepared = Vec::new();
8523 let mut pending_diagnostics = Vec::new();
8524 let mut planning_requests = BTreeSet::new();
8525 let mut selected_cells = Vec::new();
8526 let mut workbook_seeded = false;
8527 let mut sheet_scope_seeded = BTreeSet::new();
8528 let mut indexed_query_sheets = FxHashSet::default();
8529 let mut discovery_scratch_reserved = 0u64;
8530 let mut package_encountered = false;
8531 let mut selected_package_sheets = FxHashSet::default();
8532 let mut selected_package_points: BTreeMap<String, BTreeSet<(u32, u32)>> = BTreeMap::new();
8533 let mut prepared_packages: Vec<PreparedTargetSourcePackage> = Vec::new();
8534 let authoritative_with_ordinary = false;
8535 let has_unknown_package_sheet = self
8536 .staged_formula_index
8537 .package_sheets()
8538 .any(|sheet| self.graph.sheet_id(sheet).is_none());
8539
8540 loop {
8541 if let PrepareScope::Sheets(sheets) = &scope {
8542 for sheet in sheets.clone() {
8543 if !sheet_scope_seeded.insert(sheet.clone()) {
8544 continue;
8545 }
8546 let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
8547 package_encountered = true;
8548 Self::widen_target_preparation(
8549 options.opaque_policy,
8550 &mut scope,
8551 &mut reasons,
8552 OpaqueReason::UnsupportedSourceSemantics,
8553 )?;
8554 continue;
8555 };
8556 for lease in self.staged_formula_index.leases_for_sheet(&sheet) {
8557 self.target_preparation_checkpoint(options.deadline, 1)?;
8558 regions.push_back(PreparationRegion {
8559 sheet: sheet.clone(),
8560 sheet_id,
8561 start_row: lease.row,
8562 start_col: lease.col,
8563 end_row: lease.row,
8564 end_col: lease.col,
8565 });
8566 }
8567 if self
8568 .staged_formula_index
8569 .package_lease_for_sheet(&sheet)
8570 .is_some()
8571 {
8572 regions.push_back(PreparationRegion {
8573 sheet: sheet.clone(),
8574 sheet_id,
8575 start_row: 1,
8576 start_col: 1,
8577 end_row: self.workbook_load_limits.max_sheet_rows,
8578 end_col: self.workbook_load_limits.max_sheet_cols,
8579 });
8580 }
8581 }
8582 }
8583 if matches!(scope, PrepareScope::Workbook) && !workbook_seeded {
8584 workbook_seeded = true;
8585 for (sheet, lease) in self.staged_formula_index.all_leases() {
8586 self.target_preparation_checkpoint(options.deadline, 1)?;
8587 let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
8588 package_encountered = true;
8589 Self::widen_target_preparation(
8590 options.opaque_policy,
8591 &mut scope,
8592 &mut reasons,
8593 OpaqueReason::UnsupportedSourceSemantics,
8594 )?;
8595 continue;
8596 };
8597 regions.push_back(PreparationRegion {
8598 sheet,
8599 sheet_id,
8600 start_row: lease.row,
8601 start_col: lease.col,
8602 end_row: lease.row,
8603 end_col: lease.col,
8604 });
8605 }
8606 let package_sheets = self
8607 .staged_formula_index
8608 .package_sheets()
8609 .map(str::to_string)
8610 .collect::<Vec<_>>();
8611 for sheet in package_sheets {
8612 let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
8613 package_encountered = true;
8614 Self::widen_target_preparation(
8615 options.opaque_policy,
8616 &mut scope,
8617 &mut reasons,
8618 OpaqueReason::UnsupportedSourceSemantics,
8619 )?;
8620 continue;
8621 };
8622 regions.push_back(PreparationRegion {
8623 sheet,
8624 sheet_id,
8625 start_row: 1,
8626 start_col: 1,
8627 end_row: self.workbook_load_limits.max_sheet_rows,
8628 end_col: self.workbook_load_limits.max_sheet_cols,
8629 });
8630 }
8631 }
8632
8633 let allow_partial_shared = regions.is_empty() && symbol_vertices.is_empty();
8636 let next_region = if allow_partial_shared {
8637 deferred_shared_regions.pop_front()
8638 } else {
8639 regions.pop_front()
8640 };
8641 let Some(region) = next_region else {
8642 if let Some(vertex) = symbol_vertices.pop_front() {
8643 self.target_preparation_checkpoint(options.deadline, 1)?;
8644 if !visited_vertices.insert(vertex) || !self.graph.vertex_exists(vertex) {
8645 continue;
8646 }
8647 let vertex_is_dynamic = self.graph.is_dynamic(vertex);
8648 if let Some(ast) = self.graph.get_formula(vertex) {
8649 let snapshot =
8650 self.target_planning_snapshot(&ast, &mut planning_requests)?;
8651 if let Some(reason) =
8652 Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
8653 {
8654 return Err(Self::preparation_stale(
8655 reason,
8656 "target planning snapshot became stale during discovery",
8657 ));
8658 }
8659 let opaque = self.opaque_reason_in_ast(&ast, &snapshot);
8660 if let Some(reason) =
8661 opaque.or(vertex_is_dynamic.then_some(OpaqueReason::DynamicReference))
8662 {
8663 if reason == OpaqueReason::DynamicReference
8664 && Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot)
8665 {
8666 let sheet = self.graph.get_vertex_sheet_id(vertex);
8667 let sheet = self.graph.sheet_name(sheet).to_string();
8668 Self::widen_target_preparation_to_sheet(
8669 options.opaque_policy,
8670 &mut scope,
8671 &mut reasons,
8672 reason,
8673 &sheet,
8674 )?;
8675 } else {
8676 Self::widen_target_preparation(
8677 options.opaque_policy,
8678 &mut scope,
8679 &mut reasons,
8680 reason,
8681 )?;
8682 }
8683 }
8684 } else if vertex_is_dynamic {
8685 Self::widen_target_preparation(
8686 options.opaque_policy,
8687 &mut scope,
8688 &mut reasons,
8689 OpaqueReason::DynamicReference,
8690 )?;
8691 }
8692 if let Some(anchor) = self
8693 .graph
8694 .get_cell_ref(vertex)
8695 .and_then(|cell| self.graph.spill_registry_anchor_for_cell(cell))
8696 {
8697 symbol_vertices.push_back(anchor);
8698 }
8699 if let Some(cell) = self.graph.get_cell_ref(vertex) {
8700 let sheet = self.graph.sheet_name(cell.sheet_id).to_string();
8701 regions.push_back(PreparationRegion {
8702 sheet,
8703 sheet_id: cell.sheet_id,
8704 start_row: cell.coord.row() + 1,
8705 start_col: cell.coord.col() + 1,
8706 end_row: cell.coord.row() + 1,
8707 end_col: cell.coord.col() + 1,
8708 });
8709 }
8710 match self.graph.authority_vertex_precedents(vertex) {
8714 Some(precedents) => {
8715 for (sheet_id, rect) in precedents {
8716 self.target_preparation_checkpoint(options.deadline, 1)?;
8717 if sheet_id == crate::engine::authority::geom::SYMBOL_SHEET {
8718 for slot in rect.r0..=rect.r1 {
8719 if let Some(symbol) =
8720 self.graph.authority_host().symbols().vertex(slot)
8721 {
8722 symbol_vertices.push_back(symbol);
8723 }
8724 }
8725 continue;
8726 }
8727 let sheet = self.graph.sheet_name(sheet_id).to_string();
8728 regions.push_back(PreparationRegion {
8729 sheet,
8730 sheet_id,
8731 start_row: rect.r0 + 1,
8732 start_col: rect.c0 + 1,
8733 end_row: (rect.r1 + 1)
8734 .min(self.workbook_load_limits.max_sheet_rows),
8735 end_col: (rect.c1 + 1)
8736 .min(self.workbook_load_limits.max_sheet_cols),
8737 });
8738 }
8739 }
8740 None => {
8742 if Self::widen_target_preparation(
8743 options.opaque_policy,
8744 &mut scope,
8745 &mut reasons,
8746 OpaqueReason::UnresolvedCrossSheetBinding,
8747 )? {
8748 continue;
8749 }
8750 }
8751 }
8752 if let Some(name) = self.graph.named_range_by_vertex(vertex).cloned() {
8753 match &name.definition {
8754 NamedDefinition::Cell(cell) => regions.push_back(PreparationRegion {
8755 sheet: self.graph.sheet_name(cell.sheet_id).to_string(),
8756 sheet_id: cell.sheet_id,
8757 start_row: cell.coord.row() + 1,
8758 start_col: cell.coord.col() + 1,
8759 end_row: cell.coord.row() + 1,
8760 end_col: cell.coord.col() + 1,
8761 }),
8762 NamedDefinition::Range(range) => regions.push_back(PreparationRegion {
8763 sheet: self.graph.sheet_name(range.start.sheet_id).to_string(),
8764 sheet_id: range.start.sheet_id,
8765 start_row: range.start.coord.row() + 1,
8766 start_col: range.start.coord.col() + 1,
8767 end_row: range.end.coord.row() + 1,
8768 end_col: range.end.coord.col() + 1,
8769 }),
8770 NamedDefinition::Formula {
8771 ast,
8772 dependencies,
8773 range_deps,
8774 } => {
8775 let snapshot =
8776 self.target_planning_snapshot(ast, &mut planning_requests)?;
8777 if let Some(reason) = Self::target_planning_snapshot_stale_reason(
8778 &snapshot,
8779 &assumptions,
8780 ) {
8781 return Err(Self::preparation_stale(
8782 reason,
8783 "target planning snapshot became stale during discovery",
8784 ));
8785 }
8786 if let Some(reason) = self.opaque_reason_in_ast(ast, &snapshot) {
8787 Self::widen_target_preparation(
8788 options.opaque_policy,
8789 &mut scope,
8790 &mut reasons,
8791 reason,
8792 )?;
8793 }
8794 for dependency in dependencies {
8795 self.target_preparation_checkpoint(options.deadline, 1)?;
8796 symbol_vertices.push_back(*dependency);
8797 }
8798 for range in range_deps {
8799 self.target_preparation_checkpoint(options.deadline, 1)?;
8800 let context_sheet = self.graph.get_vertex_sheet_id(vertex);
8808 let Ok(sheet_id) =
8809 self.resolve_sheet_locator(&range.sheet, context_sheet)
8810 else {
8811 if Self::widen_target_preparation(
8812 options.opaque_policy,
8813 &mut scope,
8814 &mut reasons,
8815 OpaqueReason::UnresolvedCrossSheetBinding,
8816 )? {
8817 break;
8818 }
8819 continue;
8820 };
8821 regions.push_back(PreparationRegion {
8822 sheet: self.graph.sheet_name(sheet_id).to_string(),
8823 sheet_id,
8824 start_row: range
8825 .start_row
8826 .map_or(1, |bound| bound.index + 1),
8827 start_col: range
8828 .start_col
8829 .map_or(1, |bound| bound.index + 1),
8830 end_row: range.end_row.map_or(
8831 self.workbook_load_limits.max_sheet_rows,
8832 |bound| bound.index + 1,
8833 ),
8834 end_col: range.end_col.map_or(
8835 self.workbook_load_limits.max_sheet_cols,
8836 |bound| bound.index + 1,
8837 ),
8838 });
8839 }
8840 }
8841 NamedDefinition::Literal(_) => {}
8842 }
8843 }
8844 if let Some(table) = self.graph.table_by_vertex(vertex) {
8845 regions
8846 .push_back(self.table_selection_region(table, &TableSelection::Whole)?);
8847 }
8848 continue;
8849 }
8850 break;
8851 };
8852
8853 self.target_preparation_checkpoint(options.deadline, 1)?;
8854 if !visited_regions.insert(region.clone()) && !allow_partial_shared {
8855 continue;
8856 }
8857 if let PrepareScope::Sheets(sheets) = &mut scope
8858 && !sheets.iter().any(|sheet| sheet == ®ion.sheet)
8859 {
8860 sheets.push(region.sheet.clone());
8861 sheets.sort();
8862 }
8863 let package_match = self.staged_formula_index.package_for_region(
8864 ®ion.sheet,
8865 region.start_row,
8866 region.start_col,
8867 region.end_row,
8868 region.end_col,
8869 );
8870 let package_lease = match package_match {
8871 Some(Ok(lease)) => Some(lease),
8872 Some(Err(()))
8873 if region.start_row == 1
8874 && region.start_col == 1
8875 && region.end_row == self.workbook_load_limits.max_sheet_rows
8876 && region.end_col == self.workbook_load_limits.max_sheet_cols =>
8877 {
8878 self.staged_formula_index
8879 .package_lease_for_sheet(®ion.sheet)
8880 }
8881 Some(Err(())) => {
8882 Self::widen_target_preparation(
8883 options.opaque_policy,
8884 &mut scope,
8885 &mut reasons,
8886 OpaqueReason::DeferredSourcePackage,
8887 )?;
8888 None
8889 }
8890 None => None,
8891 };
8892 let compatibility_before_package_replay = package_lease.is_some()
8893 && (authoritative_with_ordinary || has_unknown_package_sheet);
8894 let package_lease = if compatibility_before_package_replay {
8895 package_encountered = true;
8896 Self::widen_target_preparation(
8897 options.opaque_policy,
8898 &mut scope,
8899 &mut reasons,
8900 OpaqueReason::UnsupportedSourceSemantics,
8901 )?;
8902 None
8903 } else {
8904 package_lease
8905 };
8906 if let Some(package_lease) = package_lease
8907 && !selected_package_sheets.contains(®ion.sheet)
8908 {
8909 self.target_preparation_checkpoint(options.deadline, 1)?;
8910 let mut points = self.staged_formula_index.package_points_in_region(
8911 ®ion.sheet,
8912 region.start_row,
8913 region.start_col,
8914 region.end_row,
8915 region.end_col,
8916 );
8917 if let Some(selected) = selected_package_points.get(®ion.sheet) {
8918 points.retain(|point| !selected.contains(point));
8919 }
8920
8921 if let Some(selected) = selected_package_points.get(®ion.sheet) {
8922 points.retain(|point| !selected.contains(point));
8923 }
8924 let permit_partial = allow_partial_shared
8925 || (points.len() == 1
8926 && regions.is_empty()
8927 && symbol_vertices.is_empty()
8928 && deferred_shared_regions.is_empty());
8929 let partial = self.prepare_target_exact_source_selection(
8930 ®ion.sheet,
8931 package_lease,
8932 points,
8933 selected_package_points
8934 .get(®ion.sheet)
8935 .unwrap_or(&BTreeSet::new()),
8936 permit_partial,
8937 options.deadline,
8938 &mut discovery_scratch_reserved,
8939 )?;
8940 let mut package = if let Some(mut package) = partial {
8941 if package.deferred_shared {
8942 deferred_shared_regions.push_back(region.clone());
8943 }
8944 if !package.direct_domains.is_empty() {
8948 for prior in prepared_packages
8949 .iter_mut()
8950 .filter(|prior| prior.sheet == region.sheet)
8951 {
8952 prior
8953 .replay_records
8954 .retain(|record| !package.direct_contains(record.row, record.col));
8955 prior
8956 .legacy
8957 .retain(|(row, col, _, _)| !package.direct_contains(*row, *col));
8958 if let Some(points) = prior.selected_points.as_mut() {
8959 points.retain(|&(row, col)| !package.direct_contains(row, col));
8960 }
8961 }
8962 }
8963 if selected_package_points.contains_key(®ion.sheet) {
8964 package.source_report = Default::default();
8965 }
8966 let points = package.selected_points.as_ref().unwrap();
8967 if !points.is_empty() {
8968 selected_package_points
8969 .entry(region.sheet.clone())
8970 .or_default()
8971 .extend(points.iter().copied());
8972 }
8973 package
8974 } else {
8975 selected_package_sheets.insert(region.sheet.clone());
8976 self.prepare_target_source_package(
8977 ®ion.sheet,
8978 package_lease,
8979 options.deadline,
8980 )?
8981 };
8982 if package
8983 .selected_points
8984 .as_ref()
8985 .is_none_or(|points| !points.is_empty())
8986 {
8987 let mut final_fallback = BTreeMap::new();
8988 for record in package.fallback_records() {
8989 final_fallback.insert((record.row, record.col), record.clone());
8990 }
8991 let batch = self.formula_batch_from_exact_replay(
8992 ®ion.sheet,
8993 final_fallback.into_values(),
8994 )?;
8995 for record in batch.formulas {
8996 self.target_preparation_checkpoint(options.deadline, 1)?;
8997 let ast = self
8998 .graph
8999 .data_store()
9000 .retrieve_ast(record.ast_id, self.graph.sheet_reg())
9001 .ok_or_else(|| {
9002 ExcelError::new(ExcelErrorKind::Value)
9003 .with_message("target fallback AST is unavailable")
9004 })?;
9005 let snapshot =
9006 self.target_planning_snapshot(&ast, &mut planning_requests)?;
9007 if let Some(reason) =
9008 Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
9009 {
9010 return Err(Self::preparation_stale(
9011 reason,
9012 "target fallback planning snapshot became stale during discovery",
9013 ));
9014 }
9015 let proven_sheet_local_dynamic =
9016 Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot);
9017 if let Some(reason) = self.opaque_reason_in_ast(&ast, &snapshot) {
9018 if reason == OpaqueReason::DynamicReference
9019 && proven_sheet_local_dynamic
9020 {
9021 Self::widen_target_preparation_to_sheet(
9022 options.opaque_policy,
9023 &mut scope,
9024 &mut reasons,
9025 reason,
9026 ®ion.sheet,
9027 )?;
9028 } else {
9029 Self::widen_target_preparation(
9030 options.opaque_policy,
9031 &mut scope,
9032 &mut reasons,
9033 reason,
9034 )?;
9035 }
9036 }
9037 let placement = CellRef::new(
9038 package.sheet_id,
9039 Coord::from_excel(record.row, record.col, true, true),
9040 );
9041 let ingested = self
9042 .graph
9043 .ingest_pipeline(&snapshot)
9044 .enable_function_semantics()
9045 .ingest_formula(
9046 FormulaAstInput::RawArena(record.ast_id),
9047 placement,
9048 record.formula_text,
9049 )?;
9050 if ingested.dep_plan.dynamic {
9051 if proven_sheet_local_dynamic {
9052 Self::widen_target_preparation_to_sheet(
9053 options.opaque_policy,
9054 &mut scope,
9055 &mut reasons,
9056 OpaqueReason::DynamicReference,
9057 ®ion.sheet,
9058 )?;
9059 } else {
9060 Self::widen_target_preparation(
9061 options.opaque_policy,
9062 &mut scope,
9063 &mut reasons,
9064 OpaqueReason::DynamicReference,
9065 )?;
9066 }
9067 }
9068 for dep in &ingested.dep_plan.direct_cell_deps {
9069 self.target_preparation_checkpoint(options.deadline, 1)?;
9070 regions.push_back(PreparationRegion {
9071 sheet: self.graph.sheet_name(dep.sheet_id).to_string(),
9072 sheet_id: dep.sheet_id,
9073 start_row: dep.coord.row().saturating_add(1),
9074 start_col: dep.coord.col().saturating_add(1),
9075 end_row: dep.coord.row().saturating_add(1),
9076 end_col: dep.coord.col().saturating_add(1),
9077 });
9078 }
9079 for range in &ingested.dep_plan.range_deps {
9080 self.target_preparation_checkpoint(options.deadline, 1)?;
9081 let Ok(dependency_sheet) =
9084 self.resolve_sheet_locator(&range.sheet, package.sheet_id)
9085 else {
9086 Self::widen_target_preparation(
9087 options.opaque_policy,
9088 &mut scope,
9089 &mut reasons,
9090 OpaqueReason::UnresolvedCrossSheetBinding,
9091 )?;
9092 continue;
9093 };
9094 regions.push_back(PreparationRegion {
9095 sheet: self.graph.sheet_name(dependency_sheet).to_string(),
9096 sheet_id: dependency_sheet,
9097 start_row: range.start_row.map_or(1, |bound| bound.index + 1),
9098 start_col: range.start_col.map_or(1, |bound| bound.index + 1),
9099 end_row: range
9100 .end_row
9101 .map_or(self.workbook_load_limits.max_sheet_rows, |bound| {
9102 bound.index + 1
9103 }),
9104 end_col: range
9105 .end_col
9106 .map_or(self.workbook_load_limits.max_sheet_cols, |bound| {
9107 bound.index + 1
9108 }),
9109 });
9110 }
9111 for name in ingested
9112 .dep_plan
9113 .resolved_named_refs
9114 .iter()
9115 .chain(&ingested.dep_plan.named_refs)
9116 {
9117 self.target_preparation_checkpoint(options.deadline, 1)?;
9118 if let Some(entry) =
9119 self.graph.resolve_name_entry(name, package.sheet_id)
9120 {
9121 symbol_vertices.push_back(entry.vertex);
9122 } else if self.graph.resolve_source_scalar_entry(name).is_none()
9123 && self.graph.resolve_source_table_entry(name).is_none()
9124 {
9125 Self::widen_target_preparation(
9126 options.opaque_policy,
9127 &mut scope,
9128 &mut reasons,
9129 OpaqueReason::UnresolvedName,
9130 )?;
9131 }
9132 }
9133 for table in &ingested.dep_plan.table_refs {
9134 self.target_preparation_checkpoint(options.deadline, 1)?;
9135 if let Some(entry) = self.graph.resolve_table_entry(table) {
9136 symbol_vertices.push_back(entry.vertex);
9137 } else if self.graph.resolve_source_table_entry(table).is_none() {
9138 Self::widen_target_preparation(
9139 options.opaque_policy,
9140 &mut scope,
9141 &mut reasons,
9142 OpaqueReason::UnresolvedTable,
9143 )?;
9144 }
9145 }
9146 package.legacy.push((
9147 record.row,
9148 record.col,
9149 ingested.ast_id,
9150 ingested.dep_plan,
9151 ));
9152 }
9153 prepared_packages.push(package);
9154 }
9155 }
9156 let leases = self.staged_formula_index.leases_in_region(
9157 ®ion.sheet,
9158 region.start_row,
9159 region.start_col,
9160 region.end_row,
9161 region.end_col,
9162 );
9163 for lease in leases {
9164 self.target_preparation_checkpoint(options.deadline, 1)?;
9165 let sheet_id = self.graph.sheet_id(®ion.sheet).ok_or_else(|| {
9166 ExcelError::new(ExcelErrorKind::Ref)
9167 .with_message(format!("staged formula sheet not found: {}", region.sheet))
9168 })?;
9169 let key = (region.sheet.clone(), lease.row, lease.col, lease.generation);
9170 if !selected.insert(key) {
9171 continue;
9172 }
9173 let text = self
9174 .staged_formulas
9175 .get(®ion.sheet)
9176 .and_then(|sheet| sheet.get_ordinary(lease.row, lease.col))
9177 .ok_or_else(|| {
9178 ExcelError::new(ExcelErrorKind::Value)
9179 .with_message("staged formula index is stale")
9180 })?
9181 .to_string();
9182 let formula = if text.starts_with('=') {
9183 text.clone()
9184 } else {
9185 format!("={text}")
9186 };
9187 self.target_preparation_checkpoint(options.deadline, 1)?;
9188 let ast = match formualizer_parse::parser::parse(&formula) {
9189 Ok(ast) => ast,
9190 Err(error) => {
9191 if self.config.formula_parse_policy == FormulaParsePolicy::Strict {
9192 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
9193 format!(
9194 "Formula parse error at {}!{}{}: {error}",
9195 region.sheet,
9196 col_letters_from_1based(lease.col)
9197 .unwrap_or_else(|_| "?".to_string()),
9198 lease.row
9199 ),
9200 ));
9201 }
9202 pending_diagnostics.push(FormulaParseDiagnostic {
9203 sheet: region.sheet.clone(),
9204 row: lease.row,
9205 col: lease.col,
9206 formula: formula.clone(),
9207 message: error.to_string(),
9208 policy: self.config.formula_parse_policy,
9209 });
9210 match self.config.formula_parse_policy {
9211 FormulaParsePolicy::KeepCachedValue => {
9212 selected_cells.push(
9213 formualizer_common::RangeAddress::new(
9214 region.sheet.clone(),
9215 lease.row,
9216 lease.col,
9217 lease.row,
9218 lease.col,
9219 )
9220 .expect("selected staged coordinates are valid"),
9221 );
9222 prepared.push(PreparedOrdinaryStagedFormula {
9223 sheet: region.sheet.clone(),
9224 sheet_id,
9225 lease,
9226 ast_id: None,
9227 plan: None,
9228 });
9229 continue;
9230 }
9231 FormulaParsePolicy::AsText => ASTNode::new(
9232 ASTNodeType::Literal(LiteralValue::Text(formula.clone())),
9233 None,
9234 ),
9235 FormulaParsePolicy::CoerceToError => ASTNode::new(
9236 ASTNodeType::Literal(LiteralValue::Error(
9237 ExcelError::new(ExcelErrorKind::Error)
9238 .with_message(format!("Malformed formula: {error}")),
9239 )),
9240 None,
9241 ),
9242 FormulaParsePolicy::Strict => unreachable!(),
9243 }
9244 }
9245 };
9246 self.target_preparation_checkpoint(options.deadline, 1)?;
9247 let snapshot = self.target_planning_snapshot(&ast, &mut planning_requests)?;
9248 self.target_preparation_checkpoint(options.deadline, 1)?;
9249 if let Some(reason) =
9250 Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
9251 {
9252 return Err(Self::preparation_stale(
9253 reason,
9254 "target planning snapshot became stale during discovery",
9255 ));
9256 }
9257 if let Some(reason) = self.opaque_reason_in_ast(&ast, &snapshot) {
9258 if reason == OpaqueReason::DynamicReference
9259 && Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot)
9260 {
9261 Self::widen_target_preparation_to_sheet(
9262 options.opaque_policy,
9263 &mut scope,
9264 &mut reasons,
9265 reason,
9266 ®ion.sheet,
9267 )?;
9268 } else {
9269 Self::widen_target_preparation(
9270 options.opaque_policy,
9271 &mut scope,
9272 &mut reasons,
9273 reason,
9274 )?;
9275 }
9276 }
9277 let proven_sheet_local_dynamic =
9278 Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot);
9279 let placement = CellRef::new(
9280 sheet_id,
9281 Coord::from_excel(lease.row, lease.col, true, true),
9282 );
9283 let ingested = self.graph.ingest_pipeline(&snapshot).ingest_formula(
9284 FormulaAstInput::Tree(ast),
9285 placement,
9286 Some(Arc::from(formula)),
9287 )?;
9288 self.target_preparation_checkpoint(options.deadline, 1)?;
9289 if ingested.dep_plan.dynamic {
9290 if proven_sheet_local_dynamic {
9291 Self::widen_target_preparation_to_sheet(
9292 options.opaque_policy,
9293 &mut scope,
9294 &mut reasons,
9295 OpaqueReason::DynamicReference,
9296 ®ion.sheet,
9297 )?;
9298 } else {
9299 Self::widen_target_preparation(
9300 options.opaque_policy,
9301 &mut scope,
9302 &mut reasons,
9303 OpaqueReason::DynamicReference,
9304 )?;
9305 }
9306 }
9307 for dep in &ingested.dep_plan.direct_cell_deps {
9308 self.target_preparation_checkpoint(options.deadline, 1)?;
9309 regions.push_back(PreparationRegion {
9310 sheet: self.graph.sheet_name(dep.sheet_id).to_string(),
9311 sheet_id: dep.sheet_id,
9312 start_row: dep.coord.row() + 1,
9313 start_col: dep.coord.col() + 1,
9314 end_row: dep.coord.row() + 1,
9315 end_col: dep.coord.col() + 1,
9316 });
9317 }
9318 for range in &ingested.dep_plan.range_deps {
9319 self.target_preparation_checkpoint(options.deadline, 1)?;
9320 let Ok(dependency_sheet) = self.resolve_sheet_locator(&range.sheet, sheet_id)
9322 else {
9323 Self::widen_target_preparation(
9324 options.opaque_policy,
9325 &mut scope,
9326 &mut reasons,
9327 OpaqueReason::UnresolvedCrossSheetBinding,
9328 )?;
9329 continue;
9330 };
9331 regions.push_back(PreparationRegion {
9332 sheet: self.graph.sheet_name(dependency_sheet).to_string(),
9333 sheet_id: dependency_sheet,
9334 start_row: range.start_row.map_or(1, |bound| bound.index + 1),
9335 start_col: range.start_col.map_or(1, |bound| bound.index + 1),
9336 end_row: range
9337 .end_row
9338 .map_or(self.workbook_load_limits.max_sheet_rows, |bound| {
9339 bound.index + 1
9340 }),
9341 end_col: range
9342 .end_col
9343 .map_or(self.workbook_load_limits.max_sheet_cols, |bound| {
9344 bound.index + 1
9345 }),
9346 });
9347 }
9348 for name in ingested
9349 .dep_plan
9350 .resolved_named_refs
9351 .iter()
9352 .chain(&ingested.dep_plan.named_refs)
9353 {
9354 self.target_preparation_checkpoint(options.deadline, 1)?;
9355 if let Some(entry) = self.graph.resolve_name_entry(name, sheet_id) {
9356 symbol_vertices.push_back(entry.vertex);
9357 } else if self.graph.resolve_source_scalar_entry(name).is_none()
9358 && self.graph.resolve_source_table_entry(name).is_none()
9359 {
9360 Self::widen_target_preparation(
9361 options.opaque_policy,
9362 &mut scope,
9363 &mut reasons,
9364 OpaqueReason::UnresolvedName,
9365 )?;
9366 }
9367 }
9368 for table in &ingested.dep_plan.table_refs {
9369 self.target_preparation_checkpoint(options.deadline, 1)?;
9370 if let Some(entry) = self.graph.resolve_table_entry(table) {
9371 symbol_vertices.push_back(entry.vertex);
9372 } else if self.graph.resolve_source_table_entry(table).is_none() {
9373 Self::widen_target_preparation(
9374 options.opaque_policy,
9375 &mut scope,
9376 &mut reasons,
9377 OpaqueReason::UnresolvedTable,
9378 )?;
9379 }
9380 }
9381 selected_cells.push(
9382 formualizer_common::RangeAddress::new(
9383 region.sheet.clone(),
9384 lease.row,
9385 lease.col,
9386 lease.row,
9387 lease.col,
9388 )
9389 .expect("selected staged coordinates are valid"),
9390 );
9391 prepared.push(PreparedOrdinaryStagedFormula {
9392 sheet: region.sheet.clone(),
9393 sheet_id,
9394 lease,
9395 ast_id: Some(ingested.ast_id),
9396 plan: Some(ingested.dep_plan),
9397 });
9398 }
9399
9400 if indexed_query_sheets.insert(region.sheet_id) {
9401 self.graph.prepare_sheet_index_for_query(region.sheet_id);
9402 let bytes = (self.graph.sheet_index_vertex_count(region.sheet_id) as u64)
9403 .saturating_mul(32);
9404 self.reserve_graph_source_scratch(bytes)?;
9405 discovery_scratch_reserved = discovery_scratch_reserved.saturating_add(bytes);
9406 }
9407 let spill_anchors = self.graph.spill_anchors_in_region(
9408 region.sheet_id,
9409 region.start_row - 1,
9410 region.start_col - 1,
9411 region.end_row - 1,
9412 region.end_col - 1,
9413 );
9414 for anchor in spill_anchors {
9415 self.target_preparation_checkpoint(options.deadline, 1)?;
9416 symbol_vertices.push_back(anchor);
9417 }
9418 let vertices = self.graph.vertices_in_region(
9419 region.sheet_id,
9420 region.start_row - 1,
9421 region.end_row - 1,
9422 region.start_col - 1,
9423 region.end_col - 1,
9424 );
9425 for vertex in vertices {
9426 self.target_preparation_checkpoint(options.deadline, 1)?;
9427 symbol_vertices.push_back(vertex);
9428 }
9429 }
9430
9431 #[cfg(test)]
9432 self.target_preparation_fault(
9433 crate::engine::target_preparation::TargetPreparationFault::AfterDiscovery,
9434 )?;
9435
9436 if package_encountered {
9437 Self::widen_target_preparation(
9438 options.opaque_policy,
9439 &mut scope,
9440 &mut reasons,
9441 OpaqueReason::UnsupportedSourceSemantics,
9442 )?;
9443 self.target_preparation_checkpoint(options.deadline, 0)?;
9444 let selected_count = self.staged_formula_count();
9445 let selected_packages = self
9446 .staged_formulas
9447 .values()
9448 .filter_map(|staged| staged.deferred_package.as_ref())
9449 .map(|package| package.families.len() + package.partitioned_families.len())
9450 .sum();
9451 self.formula_parse_diagnostics.truncate(diagnostics_len);
9452 self.last_formula_ingest_report = report_len;
9453 #[cfg(test)]
9454 if let Some(hook) = self.before_target_preparation_commit_hook.take() {
9455 hook();
9456 }
9457 self.target_preparation_checkpoint(options.deadline, 0)?;
9458 #[cfg(test)]
9459 self.target_preparation_fault(
9460 crate::engine::target_preparation::TargetPreparationFault::FinalRevisionValidation,
9461 )?;
9462 let current_revisions = self.preparation_revisions();
9463 if let Some(reason) = Self::preparation_revision_stale_reason(
9464 &assumptions,
9465 ¤t_revisions,
9466 &planning_requests,
9467 true,
9468 ) {
9469 return Err(Self::preparation_stale(
9470 reason,
9471 "target compatibility preparation plan is stale",
9472 ));
9473 }
9474 #[cfg(test)]
9475 self.target_preparation_fault(
9476 crate::engine::target_preparation::TargetPreparationFault::FinalGraphValidation,
9477 )?;
9478 let commit_work_before = self
9479 .active_resource_ledger
9480 .as_ref()
9481 .map_or(0, |ledger| ledger.snapshot().work_charged);
9482 let commit_started = crate::instant::FzInstant::now();
9483 self.build_graph_all_unobserved()?;
9484 let commit_window = commit_started.elapsed();
9485 let ledger_after = self
9486 .active_resource_ledger
9487 .as_ref()
9488 .map(|ledger| ledger.snapshot());
9489 let actual_commit_work = ledger_after
9490 .map_or(0, |snapshot| snapshot.work_charged)
9491 .saturating_sub(commit_work_before);
9492 let observed_scratch_bytes = ledger_after
9493 .map_or(0, |snapshot| snapshot.scratch_peak)
9494 .saturating_sub(ledger_at_start.map_or(0, |snapshot| snapshot.scratch_current));
9495 let revisions = assumptions.clone();
9496 let report = PreparedTargetGraphReport {
9497 request_id: request_id.unwrap_or_default(),
9498 requested_targets: targets.len(),
9499 normalized_regions: visited_regions.len(),
9500 normalized_target_list: normalized,
9501 selected_staged_cells: selected_count,
9502 selected_source_families: selected_packages,
9503 retained_staged_cells: self.staged_formula_count(),
9504 selected_cells,
9505 retained_cells: Vec::new(),
9506 widened_scope: PrepareScope::Workbook,
9507 widening_reasons: reasons,
9508 revisions,
9509 commit_window,
9510 estimated_scratch_bytes: discovery_scratch_reserved
9511 .saturating_add((selected_count as u64).saturating_mul(256)),
9512 observed_scratch_bytes,
9513 estimated_commit_work: selected_count as u64,
9514 actual_commit_work,
9515 outcome: PreparationOutcome::CompatibilityPrepared,
9516 };
9517 self.observe_target_preparation_report(&report);
9518 return Ok(report);
9519 }
9520
9521 prepared.sort_by_key(|formula| formula.lease.insertion_order);
9522 for package in &prepared_packages {
9523 if let Some(points) = &package.selected_points {
9524 selected_cells.extend(points.iter().filter_map(|&(row, col)| {
9525 formualizer_common::RangeAddress::new(&package.sheet, row, col, row, col).ok()
9526 }));
9527 continue;
9528 }
9529 selected_cells.extend(package.replay_records.iter().filter_map(|record| {
9530 formualizer_common::RangeAddress::new(
9531 package.sheet.clone(),
9532 record.row,
9533 record.col,
9534 record.row,
9535 record.col,
9536 )
9537 .ok()
9538 }));
9539 }
9540 let (legacy_graph, planned_formula_count) =
9541 self.prepare_target_combined_legacy_graph(&prepared_packages, &prepared)?;
9542 let new_vertices = legacy_graph.new_vertex_count();
9543 let new_edges = legacy_graph.planned_edge_count().ok_or_else(|| {
9544 ExcelError::new(ExcelErrorKind::NImpl).with_message("target graph edge count overflow")
9545 })?;
9546 let removed_edges = legacy_graph.removed_edge_count().ok_or_else(|| {
9547 ExcelError::new(ExcelErrorKind::NImpl).with_message("target graph edge count overflow")
9548 })?;
9549 let current = self.graph.baseline_stats();
9550 let final_vertices = current
9551 .graph_vertex_count
9552 .checked_add(new_vertices)
9553 .ok_or_else(|| {
9554 crate::engine::ResourceLedgerError::Exhausted(
9555 formualizer_common::ResourceExhaustionDetail {
9556 reason: formualizer_common::ResourceExhaustionReason::ArithmeticOverflow,
9557 limit: u64::MAX,
9558 observed: u64::MAX,
9559 request_id,
9560 },
9561 )
9562 .into_excel_error()
9563 })?;
9564 let final_edges = current
9565 .graph_edge_count
9566 .checked_sub(removed_edges)
9567 .and_then(|count| count.checked_add(new_edges))
9568 .ok_or_else(|| {
9569 crate::engine::ResourceLedgerError::Exhausted(
9570 formualizer_common::ResourceExhaustionDetail {
9571 reason: formualizer_common::ResourceExhaustionReason::ArithmeticOverflow,
9572 limit: u64::MAX,
9573 observed: u64::MAX,
9574 request_id,
9575 },
9576 )
9577 .into_excel_error()
9578 })?;
9579 #[cfg(test)]
9580 self.target_preparation_fault(
9581 crate::engine::target_preparation::TargetPreparationFault::Admission,
9582 )?;
9583 let resource = |reason, limit: u64, observed: u64| {
9584 crate::engine::ResourceLedgerError::Exhausted(
9585 formualizer_common::ResourceExhaustionDetail {
9586 reason,
9587 limit,
9588 observed,
9589 request_id,
9590 },
9591 )
9592 .into_excel_error()
9593 };
9594 let admission = crate::engine::resource_ledger::GraphAdmission {
9595 final_vertices,
9596 final_edges,
9597 materialization_cells: planned_formula_count as u64,
9598 added_vertices: new_vertices,
9599 added_edges: new_edges,
9600 };
9601 let materialized_bytes = admission
9602 .materialized_graph_bytes()
9603 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
9604 if let Err(error) = self.preflight_graph_admission(admission) {
9605 if let formualizer_common::ExcelErrorExtra::Resource { detail } = error.extra() {
9606 self.observe_target_admission_failure(detail.reason);
9607 }
9608 return Err(error);
9609 }
9610 let selected_package_records = prepared_packages
9611 .iter()
9612 .map(|package| package.replay_records.len() as u64)
9613 .sum::<u64>();
9614 let planned_working_bytes = (prepared.len() as u64)
9615 .saturating_add(selected_package_records)
9616 .saturating_mul(256)
9617 .saturating_add((visited_regions.len() as u64).saturating_mul(128))
9618 .saturating_add((visited_vertices.len() as u64).saturating_mul(32))
9619 .saturating_add((selected.len() as u64).saturating_mul(96))
9620 .saturating_add(materialized_bytes);
9621 let residual_scratch = selected_package_points
9622 .iter()
9623 .map(|(sheet, points)| {
9624 let source = self
9625 .staged_formulas
9626 .get(sheet)
9627 .unwrap()
9628 .deferred_package
9629 .as_ref()
9630 .unwrap();
9631 (source.families.len() as u64)
9632 .saturating_mul(256)
9633 .saturating_add(
9634 source
9635 .partitioned_families
9636 .iter()
9637 .map(|family| {
9638 256u64
9639 .saturating_add(family.fragments.len() as u64 * 32)
9640 .saturating_add(family.legacy_members.len() as u64 * 32)
9641 })
9642 .sum::<u64>(),
9643 )
9644 .saturating_add(points.len() as u64 * 32)
9647 .saturating_add(
9648 prepared_packages
9649 .iter()
9650 .filter(|package| &package.sheet == sheet)
9651 .map(|package| package.replay_records.len() as u64 * 480)
9652 .sum::<u64>(),
9653 )
9654 })
9655 .sum::<u64>();
9656 let scratch_bytes = discovery_scratch_reserved
9657 .saturating_add(planned_working_bytes)
9658 .saturating_add(residual_scratch);
9659 let remaining_scratch = scratch_bytes.saturating_sub(discovery_scratch_reserved);
9660 if let Err(error) = self.reserve_graph_source_scratch(remaining_scratch) {
9661 self.observe_target_admission_failure(
9662 formualizer_common::ResourceExhaustionReason::ScratchMemory,
9663 );
9664 return Err(error);
9665 }
9666
9667 let mut residual_sources = BTreeMap::new();
9668 let mut residual_owners = BTreeMap::new();
9669 for (sheet, points) in &selected_package_points {
9670 let selected: BTreeMap<_, BTreeSet<_>> = prepared_packages
9671 .iter()
9672 .filter(|package| &package.sheet == sheet)
9673 .flat_map(|package| &package.replay_records)
9674 .filter_map(|record| {
9675 record.partition_owner.or(record.family).map(|owner| {
9676 (
9677 owner,
9678 crate::engine::SourceCoord {
9679 row: record.row - 1,
9680 col: record.col - 1,
9681 },
9682 )
9683 })
9684 })
9685 .fold(BTreeMap::new(), |mut map, (owner, coord)| {
9686 map.entry(owner).or_default().insert(coord);
9687 map
9688 });
9689 let metadata_work = self
9690 .staged_formulas
9691 .get(sheet)
9692 .unwrap()
9693 .deferred_package
9694 .as_ref()
9695 .map_or(0, |source| {
9696 source.families.len() as u64
9697 + source
9698 .partitioned_families
9699 .iter()
9700 .map(|family| {
9701 1 + family.fragments.len() as u64 * family.fragments.len() as u64
9702 + family.legacy_members.len() as u64
9703 })
9704 .sum::<u64>()
9705 });
9706 let split_work = selected
9707 .values()
9708 .map(|points| points.len() as u64 * 128)
9709 .sum::<u64>();
9710 self.target_preparation_checkpoint(
9711 options.deadline,
9712 metadata_work
9713 .saturating_add(split_work)
9714 .saturating_add(points.len() as u64),
9715 )?;
9716 let source = self
9717 .staged_formulas
9718 .get(sheet)
9719 .unwrap()
9720 .deferred_package
9721 .as_ref()
9722 .unwrap();
9723 let complete: BTreeSet<_> = prepared_packages
9724 .iter()
9725 .filter(|package| &package.sheet == sheet)
9726 .flat_map(|package| package.complete_selections.iter().copied())
9727 .collect();
9728 let residual = source
9729 .residual_sources(&selected, &complete, &self.workbook_load_limits)
9730 .map_err(|reason| ExcelError::new(ExcelErrorKind::Value).with_message(reason))?;
9731 residual_owners.insert(
9732 sheet.clone(),
9733 residual
9734 .1
9735 .iter()
9736 .map(|family| family.source_id)
9737 .collect::<BTreeSet<_>>(),
9738 );
9739 residual_sources.insert(sheet.clone(), residual);
9740 }
9741
9742 for (sheet, points) in &selected_package_points {
9745 let package = self
9746 .staged_formulas
9747 .get_mut(sheet)
9748 .unwrap()
9749 .deferred_package
9750 .as_mut()
9751 .unwrap();
9752 let owner_count = prepared_packages
9753 .iter()
9754 .filter(|p| &p.sheet == sheet)
9755 .flat_map(|p| &p.replay_records)
9756 .filter_map(|record| record.partition_owner.or(record.family))
9757 .filter(|owner| residual_owners[sheet].contains(owner))
9758 .count();
9759 package
9760 .consumed_members
9761 .try_reserve(owner_count)
9762 .map_err(|_| {
9763 resource(
9764 formualizer_common::ResourceExhaustionReason::ScratchMemory,
9765 0,
9766 owner_count as u64 * 24,
9767 )
9768 })?;
9769 package.suppressed.try_reserve(points.len()).map_err(|_| {
9770 resource(
9771 formualizer_common::ResourceExhaustionReason::ScratchMemory,
9772 0,
9773 points.len() as u64 * 16,
9774 )
9775 })?;
9776 }
9777
9778 let estimated_commit_duration = std::time::Duration::from_nanos(
9779 (new_vertices as u64)
9780 .saturating_add(new_edges as u64)
9781 .saturating_add(prepared.len() as u64)
9782 .saturating_add(selected_package_records)
9783 .max(1)
9784 .saturating_mul(100),
9785 );
9786 if options.deadline.is_some_and(|deadline| {
9787 std::time::Instant::now()
9788 .checked_add(estimated_commit_duration)
9789 .is_none_or(|finish| finish > deadline)
9790 }) {
9791 self.observe_target_admission_failure(
9792 formualizer_common::ResourceExhaustionReason::Deadline,
9793 );
9794 return Err(resource(
9795 formualizer_common::ResourceExhaustionReason::Deadline,
9796 0,
9797 1,
9798 ));
9799 }
9800 #[cfg(test)]
9801 if let Some(hook) = self.before_target_preparation_commit_hook.take() {
9802 hook();
9803 }
9804 self.target_preparation_checkpoint(options.deadline, 0)?;
9805 #[cfg(test)]
9806 self.target_preparation_fault(
9807 crate::engine::target_preparation::TargetPreparationFault::FinalRevisionValidation,
9808 )?;
9809 let current_revisions = self.preparation_revisions();
9810 let staged_leases_match = prepared.iter().all(|formula| {
9811 self.staged_formula_index
9812 .lease_matches(&formula.sheet, formula.lease)
9813 }) && prepared_packages.iter().all(|package| {
9814 self.staged_formula_index
9815 .package_lease_matches(&package.sheet, package.lease)
9816 });
9817 let stale_reason = Self::preparation_revision_stale_reason(
9818 &assumptions,
9819 ¤t_revisions,
9820 &planning_requests,
9821 staged_leases_match,
9822 );
9823 if let Some(reason) = stale_reason {
9824 return Err(Self::preparation_stale(
9825 reason,
9826 "target graph preparation plan is stale",
9827 ));
9828 }
9829 #[cfg(test)]
9830 self.target_preparation_fault(
9831 crate::engine::target_preparation::TargetPreparationFault::FinalGraphValidation,
9832 )?;
9833 self.graph
9834 .validate_prepared_legacy_graph_plan(&legacy_graph)
9835 .map_err(|error| {
9836 Self::preparation_stale(
9837 formualizer_common::PreparationStaleReason::Graph,
9838 format!("target graph preparation plan is stale: {error}"),
9839 )
9840 })?;
9841 #[cfg(test)]
9842 self.target_preparation_fault(
9843 crate::engine::target_preparation::TargetPreparationFault::Reservation,
9844 )?;
9845 self.graph.reserve_prepared_legacy_graph_plan(&legacy_graph);
9846 self.formula_parse_diagnostics
9847 .try_reserve(pending_diagnostics.len())
9848 .map_err(|_| {
9849 resource(
9850 formualizer_common::ResourceExhaustionReason::Admission,
9851 pending_diagnostics.len() as u64,
9852 pending_diagnostics.len() as u64,
9853 )
9854 })?;
9855 self.target_preparation_checkpoint(options.deadline, 0)?;
9856 #[cfg(test)]
9857 self.target_preparation_fault(
9858 crate::engine::target_preparation::TargetPreparationFault::BeforeFirstMutation,
9859 )?;
9860
9861 let commit_started = crate::instant::FzInstant::now();
9862 let committed = self
9863 .graph
9864 .apply_prevalidated_legacy_graph_plan(legacy_graph);
9865 for formula in &prepared {
9866 let removed = self
9867 .staged_formulas
9868 .get_mut(&formula.sheet)
9869 .and_then(|sheet| sheet.remove_ordinary(formula.lease.row, formula.lease.col));
9870 debug_assert!(removed.is_some());
9871 let index_removed = self.staged_formula_index.remove(
9872 &formula.sheet,
9873 formula.lease.row,
9874 formula.lease.col,
9875 );
9876 debug_assert!(index_removed);
9877 }
9878 for package in &prepared_packages {
9879 if let Some(points) = &package.selected_points {
9880 let staged = self.staged_formulas.get_mut(&package.sheet).unwrap();
9881 let source = staged.deferred_package.as_mut().unwrap();
9882 source.consumed_engine = Some(Arc::clone(&self.source_formula_token));
9883 source
9886 .consumed_members
9887 .extend(package.replay_records.iter().filter_map(|record| {
9888 record
9889 .partition_owner
9890 .or(record.family)
9891 .filter(|owner| residual_owners[&package.sheet].contains(owner))
9892 .map(|owner| {
9893 (
9894 owner,
9895 crate::engine::SourceCoord {
9896 row: record.row - 1,
9897 col: record.col - 1,
9898 },
9899 )
9900 })
9901 }));
9902 source.suppressed.extend(points.iter().copied());
9903 source.source_accounted = true;
9904 self.staged_formula_index
9905 .consume_package_points(&package.sheet, points);
9906 if source.suppressed.len() >= source.source_coordinates.len() {
9907 staged.deferred_package = None;
9908 self.staged_formula_index.set_package(&package.sheet, None);
9909 }
9910 } else {
9911 let removed = self
9912 .staged_formulas
9913 .get_mut(&package.sheet)
9914 .and_then(|staged| staged.deferred_package.take());
9915 debug_assert!(removed.is_some());
9916 self.staged_formula_index.set_package(&package.sheet, None);
9917 }
9918 }
9919 for (sheet, (families, partitions)) in residual_sources {
9920 if let Some(source) = self
9921 .staged_formulas
9922 .get_mut(&sheet)
9923 .and_then(|staged| staged.deferred_package.as_mut())
9924 {
9925 source.families = families;
9926 source.partitioned_families = partitions;
9927 source
9928 .consumed_members
9929 .retain(|(owner, _)| residual_owners[&sheet].contains(owner));
9930 self.staged_formula_index.update_package_family_count(
9931 &sheet,
9932 source.families.len() + source.partitioned_families.len(),
9933 );
9934 }
9935 }
9936 let empty_sheets = self
9937 .staged_formulas
9938 .iter()
9939 .filter_map(|(sheet, staged)| staged.is_empty().then_some(sheet.clone()))
9940 .collect::<Vec<_>>();
9941 for sheet in empty_sheets {
9942 self.staged_formulas.remove(&sheet);
9943 }
9944 if committed > 0 {
9945 self.mark_topology_edited();
9946 }
9947 self.formula_parse_diagnostics.extend(pending_diagnostics);
9948 if !prepared.is_empty() || !prepared_packages.is_empty() {
9949 let mut ingest_delta = FormulaIngestReport::with_mode(FormulaPlaneMode::Off);
9950 ingest_delta.formula_cells_seen = (prepared.len() as u64).saturating_add(
9951 prepared_packages
9952 .iter()
9953 .map(|package| {
9954 (package.replay_records.len() as u64).saturating_add(
9955 if package.selected_points.is_some() {
9956 package.direct_cells
9957 } else {
9958 0
9959 },
9960 )
9961 })
9962 .sum::<u64>(),
9963 );
9964 ingest_delta.graph_formula_cells_materialized = committed as u64;
9965 ingest_delta.graph_vertices_created = new_vertices as u64;
9966 ingest_delta.graph_edges_created = new_edges as u64;
9967 for package in &prepared_packages {
9968 let source = &package.source_report;
9969 ingest_delta.source_formula_events = ingest_delta
9970 .source_formula_events
9971 .saturating_add(source.source_formula_events);
9972 ingest_delta.source_formula_records_spooled = ingest_delta
9973 .source_formula_records_spooled
9974 .saturating_add(source.source_formula_records_spooled);
9975 ingest_delta.source_spool_encoded_bytes = ingest_delta
9976 .source_spool_encoded_bytes
9977 .saturating_add(source.source_spool_encoded_bytes);
9978 ingest_delta.source_spool_peak_memory_bytes = ingest_delta
9979 .source_spool_peak_memory_bytes
9980 .max(source.source_spool_peak_memory_bytes);
9981 ingest_delta.source_spool_spilled_bytes = ingest_delta
9982 .source_spool_spilled_bytes
9983 .saturating_add(source.source_spool_spilled_bytes);
9984 ingest_delta.source_spool_spill_files = ingest_delta
9985 .source_spool_spill_files
9986 .saturating_add(source.source_spool_spill_files);
9987 ingest_delta.source_spool_replays = ingest_delta
9988 .source_spool_replays
9989 .saturating_add(source.source_spool_replays)
9990 .saturating_add(package.spool_replays);
9991 ingest_delta.source_families_seen = ingest_delta
9992 .source_families_seen
9993 .saturating_add(source.families_seen);
9994 ingest_delta.source_family_cells_seen = ingest_delta
9995 .source_family_cells_seen
9996 .saturating_add(source.family_cells_seen);
9997 ingest_delta.source_family_shadow_eligible = ingest_delta
9998 .source_family_shadow_eligible
9999 .saturating_add(source.source_clean_families);
10000 ingest_delta.source_family_shadow_eligible_cells = ingest_delta
10001 .source_family_shadow_eligible_cells
10002 .saturating_add(source.source_clean_cells);
10003 ingest_delta.source_partitioned_families_seen = ingest_delta
10004 .source_partitioned_families_seen
10005 .saturating_add(source.source_fragmentable_families);
10006 ingest_delta.source_partition_holes = ingest_delta
10007 .source_partition_holes
10008 .saturating_add(source.source_hole_exclusions);
10009 ingest_delta.source_partition_ordinary_exceptions = ingest_delta
10010 .source_partition_ordinary_exceptions
10011 .saturating_add(source.source_ordinary_exclusions);
10012 ingest_delta.source_partition_surviving_cells = ingest_delta
10013 .source_partition_surviving_cells
10014 .saturating_add(source.source_fragmentable_cells);
10015 for (reason, count) in &source.fallback_reasons {
10016 let total = ingest_delta
10017 .fallback_reasons
10018 .entry(reason.clone())
10019 .or_default();
10020 *total = total.saturating_add(*count);
10021 }
10022
10023 ingest_delta.source_family_fallback = ingest_delta
10024 .source_family_fallback
10025 .saturating_add(source.families_seen);
10026 ingest_delta.source_family_fallback_cells = ingest_delta
10027 .source_family_fallback_cells
10028 .saturating_add(source.family_cells_seen);
10029 }
10030 self.record_formula_ingest_report(ingest_delta);
10031 }
10032 let commit_window = commit_started.elapsed();
10033 let retained_cells = self
10034 .staged_formula_index
10035 .all_leases()
10036 .into_iter()
10037 .filter_map(|(sheet, lease)| {
10038 formualizer_common::RangeAddress::new(
10039 sheet, lease.row, lease.col, lease.row, lease.col,
10040 )
10041 .ok()
10042 })
10043 .collect::<Vec<_>>();
10044 let observed_scratch_bytes = self
10045 .active_resource_ledger
10046 .as_ref()
10047 .map_or(0, |ledger| ledger.snapshot().scratch_peak)
10048 .saturating_sub(ledger_at_start.map_or(0, |snapshot| snapshot.scratch_current));
10049 let committed_spans = 0u64;
10050 let selected_source_families = prepared_packages
10051 .iter()
10052 .filter(|package| package.selected_points.is_none())
10053 .map(|package| package.lease.family_count)
10054 .sum::<usize>()
10055 + prepared_packages
10056 .iter()
10057 .filter(|package| package.selected_points.is_some())
10058 .flat_map(|package| package.replay_records.iter())
10059 .filter_map(|record| record.partition_owner.or(record.family))
10060 .chain(
10061 prepared_packages
10062 .iter()
10063 .flat_map(|package| package.complete_selections.iter().copied()),
10064 )
10065 .collect::<BTreeSet<_>>()
10066 .len();
10067 let selected_staged_cells = prepared.len().saturating_add(
10068 prepared_packages
10069 .iter()
10070 .map(|package| {
10071 package
10072 .selected_points
10073 .as_ref()
10074 .map_or(package.replay_records.len(), BTreeSet::len)
10075 })
10076 .sum::<usize>(),
10077 );
10078 let actual_commit_work = (new_vertices as u64)
10079 .saturating_add(new_edges as u64)
10080 .saturating_add(committed as u64)
10081 .saturating_add(committed_spans)
10082 .saturating_add(prepared.len() as u64)
10083 .saturating_add(
10084 prepared_packages
10085 .iter()
10086 .map(|package| {
10087 package
10088 .selected_points
10089 .as_ref()
10090 .map_or(1, |points| points.len() as u64)
10091 })
10092 .sum::<u64>(),
10093 );
10094 let report = PreparedTargetGraphReport {
10095 request_id: request_id.unwrap_or_default(),
10096 requested_targets: targets.len(),
10097 normalized_regions: visited_regions.len(),
10098 normalized_target_list: normalized,
10099 selected_staged_cells,
10100 selected_source_families,
10101 retained_staged_cells: self.staged_formula_count(),
10102 selected_cells,
10103 retained_cells,
10104 widened_scope: scope,
10105 widening_reasons: reasons,
10106 revisions: assumptions,
10107 commit_window,
10108 estimated_scratch_bytes: scratch_bytes,
10109 observed_scratch_bytes,
10110 estimated_commit_work: (new_vertices as u64)
10111 .saturating_add(new_edges as u64)
10112 .saturating_add(prepared.len() as u64)
10113 .saturating_add(selected_package_records),
10114 actual_commit_work,
10115 outcome: PreparationOutcome::Prepared,
10116 };
10117 self.observe_target_preparation_report(&report);
10118 Ok(report)
10119 }
10120
10121 pub fn build_graph_all(&mut self) -> Result<(), formualizer_parse::ExcelError> {
10123 self.observe_evaluation_resource_request(EvaluationRequestKind::Full, |engine| {
10124 engine.build_graph_all_unobserved()
10125 })
10126 }
10127
10128 fn build_graph_all_unobserved(&mut self) -> Result<(), formualizer_parse::ExcelError> {
10129 let selected = self.staged_formula_count();
10130 let started = crate::instant::FzInstant::now();
10131 self.resource_checkpoint(selected as u64)?;
10132 let scratch_bytes = (selected as u64).saturating_mul(256);
10133 let result = self.with_request_scratch(scratch_bytes, |engine| {
10134 let index_snapshot = engine.staged_formula_index.clone();
10135 let collected = std::mem::take(&mut engine.staged_formulas)
10136 .into_iter()
10137 .collect();
10138 engine.staged_formula_index.clear_all();
10139 engine.build_graph_from_staged_batches(collected, false, index_snapshot)
10140 });
10141 self.observe_staged_preparation(selected, self.staged_formula_count(), started.elapsed());
10142 result
10143 }
10144
10145 pub fn build_graph_for_sheets<'a, I: IntoIterator<Item = &'a str>>(
10147 &mut self,
10148 sheets: I,
10149 ) -> Result<(), formualizer_parse::ExcelError> {
10150 let mut sheets = sheets.into_iter();
10151 self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, move |engine| {
10152 let name_scratch = (sheets.size_hint().0 as u64).saturating_mul(64);
10153 engine.with_request_scratch(name_scratch, |engine| {
10154 let names = sheets.by_ref().map(str::to_string).collect::<Vec<_>>();
10157 engine.charge_bounded_work(names.len() as u64)?;
10158 engine.build_graph_for_sheet_names_unobserved(names)
10159 })
10160 })
10161 }
10162
10163 fn build_graph_for_sheet_names_unobserved(
10164 &mut self,
10165 sheets: Vec<String>,
10166 ) -> Result<(), formualizer_parse::ExcelError> {
10167 let started = crate::instant::FzInstant::now();
10168 let selected = sheets
10169 .iter()
10170 .filter_map(|sheet| self.staged_formulas.get(sheet))
10171 .map(StagedSheet::len)
10172 .sum::<usize>();
10173 self.resource_checkpoint(selected as u64)?;
10174 let scratch_bytes = (selected as u64).saturating_mul(256);
10175 self.reserve_request_scratch(scratch_bytes)?;
10176 let index_snapshot = self.staged_formula_index.clone();
10177 let mut collected = Vec::new();
10178 for sheet in sheets {
10179 if let Some(staged) = self.staged_formulas.remove(&sheet) {
10180 self.index_removed_staged_sheet(&sheet, &staged);
10181 collected.push((sheet, staged));
10182 }
10183 }
10184 let result = self.build_graph_from_staged_batches(collected, true, index_snapshot);
10185 self.release_request_scratch(scratch_bytes);
10186 self.observe_staged_preparation(selected, self.staged_formula_count(), started.elapsed());
10187 result
10188 }
10189
10190 fn build_graph_from_staged_batches(
10191 &mut self,
10192 collected: StagedFormulaBatches,
10193 share_parse_cache_across_sheets: bool,
10194 staged_index_snapshot: StagedFormulaIndex,
10195 ) -> Result<(), formualizer_parse::ExcelError> {
10196 if collected.is_empty() {
10197 return Ok(());
10198 }
10199 for (sheet, _) in &collected {
10200 let _ = self.add_sheet(sheet);
10201 }
10202
10203 let diagnostics_len = self.formula_parse_diagnostics.len();
10204 let mut collected = collected;
10205 let prepared = match self
10206 .prepare_staged_formula_batches(&mut collected, share_parse_cache_across_sheets)
10207 {
10208 Ok(prepared) => prepared,
10209 Err(error) => {
10210 self.formula_parse_diagnostics.truncate(diagnostics_len);
10211 for (sheet, staged) in collected {
10212 self.restore_staged_sheet(sheet, staged);
10213 }
10214 self.staged_formula_index = staged_index_snapshot;
10215 return Err(error);
10216 }
10217 };
10218 let (ordinary, compressed, direct, may_fail) = prepared;
10219
10220 let first_build = if self.deferred_build_can_be_first_load() {
10230 if let Err(error) = self.check_staged_formula_plans(
10231 ordinary
10232 .iter()
10233 .chain(compressed.iter().map(|(batch, _)| batch)),
10234 &may_fail,
10235 ) {
10236 self.formula_parse_diagnostics.truncate(diagnostics_len);
10237 for (sheet, staged) in collected {
10238 self.restore_staged_sheet(sheet, staged);
10239 }
10240 self.staged_formula_index = staged_index_snapshot;
10241 return Err(error);
10242 }
10243 self.deferred_build_as_first_load()
10244 } else {
10245 None
10246 };
10247 let built_sheets: Vec<String> = if first_build.is_some() {
10248 collected.iter().map(|(sheet, _)| sheet.clone()).collect()
10249 } else {
10250 Vec::new()
10251 };
10252
10253 let result = (|| {
10257 if !ordinary.is_empty() {
10258 self.ingest_formula_batches(ordinary)?;
10259 }
10260 if !compressed.is_empty() {
10261 self.ingest_compressed_formula_source_batches(compressed)?;
10262 }
10263 if !direct.is_empty() {
10264 self.finish_compressed_formula_sources(direct)?;
10265 }
10266 Ok(())
10267 })();
10268 if let Some(saved) = first_build {
10269 self.leave_deferred_first_load(saved, &built_sheets);
10270 }
10271 if let Err(error) = result {
10272 self.formula_parse_diagnostics.truncate(diagnostics_len);
10273 for (sheet, staged) in collected {
10274 self.restore_staged_sheet(sheet, staged);
10275 }
10276 self.staged_formula_index = staged_index_snapshot;
10277 return Err(error);
10278 }
10279 self.dedup_formula_parse_diagnostics_since(diagnostics_len);
10280 Ok(())
10281 }
10282
10283 fn deferred_build_can_be_first_load(&self) -> bool {
10289 !self.graph.first_load_assume_new()
10290 && !self.graph_admission_enabled()
10291 && self.graph.formula_vertex_count() == 0
10292 }
10293
10294 fn check_staged_formula_plans<'b>(
10299 &mut self,
10300 batches: impl Iterator<Item = &'b FormulaIngestBatch>,
10301 may_fail: &FxHashSet<crate::engine::arena::AstNodeId>,
10302 ) -> Result<(), ExcelError> {
10303 if may_fail.is_empty() {
10304 return Ok(());
10305 }
10306 let mut seen: FxHashSet<(SheetId, crate::engine::arena::AstNodeId)> = FxHashSet::default();
10307 for batch in batches {
10308 let sheet_id = self.graph.sheet_id(&batch.sheet_name).ok_or_else(|| {
10309 ExcelError::new(ExcelErrorKind::Ref)
10310 .with_message(format!("unknown ingest sheet: {}", batch.sheet_name))
10311 })?;
10312 let mut pipeline = self.ingest_pipeline();
10313 for record in &batch.formulas {
10314 if record.member_anchor.is_some()
10315 || !may_fail.contains(&record.ast_id)
10316 || !seen.insert((sheet_id, record.ast_id))
10317 {
10318 continue;
10319 }
10320 let placement = CellRef::new(
10321 sheet_id,
10322 Coord::from_excel(record.row, record.col, true, true),
10323 );
10324 pipeline.ingest_formula(
10325 FormulaAstInput::RawArena(record.ast_id),
10326 placement,
10327 None,
10328 )?;
10329 }
10330 }
10331 Ok(())
10332 }
10333
10334 fn deferred_build_as_first_load(&mut self) -> Option<(crate::engine::SheetIndexMode, usize)> {
10335 if !self.deferred_build_can_be_first_load() {
10336 return None;
10337 }
10338 let saved = (
10339 self.graph.get_config().sheet_index_mode,
10340 self.config.range_expansion_limit,
10341 );
10342 self.graph
10343 .set_sheet_index_mode(crate::engine::SheetIndexMode::Lazy);
10344 self.config.range_expansion_limit = 0;
10345 self.graph.set_first_load_assume_new(true);
10346 self.graph.reset_ensure_touched();
10347 Some(saved)
10348 }
10349
10350 fn leave_deferred_first_load(
10353 &mut self,
10354 (index_mode, range_limit): (crate::engine::SheetIndexMode, usize),
10355 sheets: &[String],
10356 ) {
10357 self.graph.set_first_load_assume_new(false);
10358 self.graph.reset_ensure_touched();
10359 self.graph.set_sheet_index_mode(index_mode);
10360 self.config.range_expansion_limit = range_limit;
10361 for sheet in sheets {
10362 self.graph.finalize_sheet_index(sheet);
10363 }
10364 }
10365
10366 fn prepare_staged_formula_batches(
10367 &mut self,
10368 collected: &mut StagedFormulaBatches,
10369 share_parse_cache_across_sheets: bool,
10370 ) -> Result<PreparedStagedFormulaBatches, formualizer_parse::ExcelError> {
10371 let mut ordinary = Vec::new();
10372 let mut compressed = Vec::new();
10373 let mut direct = Vec::new();
10374 let mut may_fail = FxHashSet::default();
10375 let mut cache: rustc_hash::FxHashMap<String, Option<crate::engine::arena::AstNodeId>> =
10376 rustc_hash::FxHashMap::default();
10377 cache.reserve(4096);
10378
10379 for (sheet, staged) in collected {
10380 if !share_parse_cache_across_sheets {
10381 cache.clear();
10382 }
10383 let mut grouper = crate::engine::FormulaFamilyGrouper::new();
10387 let mut replayed_records = Vec::new();
10391 let deferred_source = None;
10392 let mut deferred_fallback = None;
10393 if let Some(package) = staged.deferred_package.as_mut() {
10394 if package.sheet_name != *sheet {
10395 return Err(ExcelError::new(ExcelErrorKind::Value)
10396 .with_message("deferred formula package sheet mismatch"));
10397 }
10398 let eligible: Vec<_> = package
10399 .families
10400 .iter()
10401 .filter(|family| !package.invalidated.contains(&family.source_id))
10402 .cloned()
10403 .collect();
10404 let eligible_partitions: Vec<_> = package
10405 .partitioned_families
10406 .iter()
10407 .filter(|family| !package.invalidated.contains(&family.source_id))
10408 .cloned()
10409 .collect();
10410
10411 let mut replay_disposition = crate::engine::FormulaReplayDisposition::default();
10412 for partition in &eligible_partitions {
10413 replay_disposition
10414 .register_partition(partition, false)
10415 .map_err(|reason| {
10416 ExcelError::new(ExcelErrorKind::Value).with_message(reason)
10417 })?;
10418 }
10419 replay_disposition
10420 .extend_suppressed_excel_coords(package.suppressed.iter().copied());
10421 let replayed = package
10422 .replay
10423 .lock()
10424 .map_err(|_| {
10425 ExcelError::new(ExcelErrorKind::Value)
10426 .with_message("deferred formula spool lock poisoned")
10427 })?
10428 .replay(&replay_disposition)
10429 .map_err(|message| {
10430 ExcelError::new(ExcelErrorKind::Value).with_message(message)
10431 })?;
10432 replayed_records = replayed;
10433 let mut report = package.accounting_report();
10434 report.source_spool_replays = report.source_spool_replays.saturating_add(1);
10435 deferred_fallback = Some((report, package.families.clone(), eligible_partitions));
10436 }
10437
10438 let n_entries = staged.entries.len() + replayed_records.len();
10439 let entries = staged
10440 .entries
10441 .iter()
10442 .cloned()
10443 .map(|(row, col, text)| (row, col, text, None))
10444 .chain(replayed_records.into_iter().map(|record| {
10445 (
10446 record.row,
10447 record.col,
10448 record.text,
10449 Some((record.source_order, record.family, record.partition_owner)),
10450 )
10451 }));
10452 let mut formulas = Vec::with_capacity(n_entries);
10453 let staged_order_base = u64::MAX.saturating_sub(n_entries as u64);
10454 for (entry_index, (row, col, txt, source_proof)) in entries.enumerate() {
10455 let key = if txt.starts_with('=') {
10456 txt
10457 } else {
10458 format!("={txt}")
10459 };
10460 grouper.note_formula();
10461 let shared_key = source_proof
10463 .and_then(|(_, family, _)| family)
10464 .map(|family| u64::from(family.source_index));
10465 let staged_record = if let Some(cached) = cache.get(&key) {
10466 grouper.note_parse_cache_hit();
10467 cached.map(|ast_id| {
10468 self.note_staged_formula(&mut grouper, row, col, ast_id);
10469 FormulaIngestRecord::new(row, col, ast_id, Some(Arc::<str>::from(key)))
10470 })
10471 } else if let Some(record) =
10472 self.stage_relocated_text(&mut grouper, row, col, &key, shared_key)
10473 {
10474 Some(record)
10475 } else {
10476 grouper.note_parse(key.len());
10477 let parsed = formualizer_parse::parser::parse(&key);
10478 let parsed_text = parsed.is_ok();
10479 let parsed = match parsed {
10480 Ok(parsed) => Some(parsed),
10481 Err(error) => self.handle_formula_parse_error(
10482 sheet,
10483 row,
10484 col,
10485 &key,
10486 error.to_string(),
10487 )?,
10488 };
10489 match parsed {
10490 Some(ast) => {
10491 let record = self.stage_formula_ast(&mut grouper, row, col, &ast, None);
10492 if parsed_text {
10493 self.note_parsed_text(
10494 &mut grouper,
10495 row,
10496 col,
10497 &key,
10498 &ast,
10499 &record,
10500 shared_key,
10501 );
10502 }
10503 if !record.is_family_member() && self.formula_may_fail_planning(&ast) {
10504 may_fail.insert(record.ast_id);
10505 }
10506 if record.is_family_member() {
10509 Some(record)
10510 } else {
10511 let ast_id = record.ast_id;
10512 cache.insert(key.clone(), Some(ast_id));
10513 Some(FormulaIngestRecord::new(
10514 row,
10515 col,
10516 ast_id,
10517 Some(Arc::<str>::from(key)),
10518 ))
10519 }
10520 }
10521 None => {
10522 cache.insert(key, None);
10523 None
10524 }
10525 }
10526 };
10527
10528 if let Some(mut formula) = staged_record {
10529 if let Some((order, family, owner)) = source_proof {
10530 formula = formula.with_source_proof(order, family, owner);
10531 } else if deferred_source.is_some() {
10532 formula = formula.with_source_proof(
10533 crate::engine::SourceFormulaOrder::new(
10534 staged_order_base.saturating_add(entry_index as u64),
10535 ),
10536 None,
10537 None,
10538 );
10539 }
10540 formulas.push(formula);
10541 }
10542 }
10543
10544 self.finish_family_grouper(&mut grouper);
10545 let batch = FormulaIngestBatch::new(sheet.clone(), formulas);
10546 if let Some((report, preparation)) = deferred_source {
10547 direct.push((batch, report, preparation));
10548 } else if let Some((report, families, partitions)) = deferred_fallback {
10549 let source_batch = crate::engine::FormulaCompressedSourceBatch::with_proposals(
10550 batch.sheet_name.clone(),
10551 report,
10552 families,
10553 partitions,
10554 );
10555 compressed.push((batch, source_batch));
10556 } else if !batch.is_empty() {
10557 ordinary.push(batch);
10558 }
10559 }
10560 if std::env::var("FZ_DEBUG_LOAD")
10561 .ok()
10562 .is_some_and(|v| v != "0")
10563 {
10564 eprintln!(
10565 "[fz][families] deferred build: {}",
10566 self.source_family_counters.debug_line()
10567 );
10568 }
10569 Ok((ordinary, compressed, direct, may_fail))
10570 }
10571
10572 fn formula_may_fail_planning(&self, ast: &formualizer_parse::parser::ASTNode) -> bool {
10577 use formualizer_parse::parser::{ASTNodeType, ReferenceType};
10578 let sheet_missing = |sheet: &Option<String>| {
10579 sheet
10580 .as_deref()
10581 .is_some_and(|s| self.graph.sheet_id(s).is_none())
10582 };
10583 match &ast.node_type {
10584 ASTNodeType::Literal(_) | ASTNodeType::Omitted => false,
10585 ASTNodeType::Reference { reference, .. } => match reference {
10586 ReferenceType::Cell { sheet, .. } => sheet_missing(sheet),
10587 ReferenceType::Range {
10588 sheet,
10589 start_row,
10590 start_col,
10591 end_row,
10592 end_col,
10593 ..
10594 } => {
10595 sheet_missing(sheet)
10596 || matches!((start_row, end_row), (Some(a), Some(b)) if a > b)
10597 || matches!((start_col, end_col), (Some(a), Some(b)) if a > b)
10598 }
10599 ReferenceType::NamedRange(_) => false,
10600 _ => true,
10601 },
10602 ASTNodeType::UnaryOp { expr, .. } => self.formula_may_fail_planning(expr),
10603 ASTNodeType::BinaryOp { left, right, .. } => {
10604 self.formula_may_fail_planning(left) || self.formula_may_fail_planning(right)
10605 }
10606 ASTNodeType::Function { args, .. } => {
10607 args.iter().any(|a| self.formula_may_fail_planning(a))
10608 }
10609 ASTNodeType::Call { callee, args } => {
10610 self.formula_may_fail_planning(callee)
10611 || args.iter().any(|a| self.formula_may_fail_planning(a))
10612 }
10613 ASTNodeType::Array(rows) => rows
10614 .iter()
10615 .flatten()
10616 .any(|a| self.formula_may_fail_planning(a)),
10617 }
10618 }
10619
10620 pub fn begin_bulk_ingest_arrow(
10622 &mut self,
10623 ) -> crate::engine::arrow_ingest::ArrowBulkIngestBuilder<'_, R> {
10624 crate::engine::arrow_ingest::ArrowBulkIngestBuilder::new(self)
10625 }
10626
10627 pub fn begin_bulk_update_arrow(
10629 &mut self,
10630 ) -> crate::engine::arrow_ingest::ArrowBulkUpdateBuilder<'_, R> {
10631 crate::engine::arrow_ingest::ArrowBulkUpdateBuilder::new(self)
10632 }
10633
10634 fn ensure_known_sheet_id(&self, sheet: &str) -> Result<SheetId, crate::engine::EditorError> {
10635 self.graph.sheet_id(sheet).ok_or(
10636 crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
10637 name: sheet.to_string(),
10638 reason: "Unknown sheet".to_string(),
10639 },
10640 )
10641 }
10642
10643 fn normalize_row_1based(row_1based: u32) -> Result<u32, crate::engine::EditorError> {
10644 if row_1based == 0 {
10645 return Err(crate::engine::EditorError::OutOfBounds { row: 0, col: 0 });
10646 }
10647 Ok(row_1based - 1)
10648 }
10649
10650 fn normalize_row_range_1based(
10651 start_row_1based: u32,
10652 end_row_1based: u32,
10653 ) -> Result<(u32, u32), crate::engine::EditorError> {
10654 if start_row_1based == 0 || end_row_1based == 0 {
10655 return Err(crate::engine::EditorError::OutOfBounds { row: 0, col: 0 });
10656 }
10657 if start_row_1based > end_row_1based {
10658 return Err(crate::engine::EditorError::TransactionFailed {
10659 reason: "Row range start is greater than end".to_string(),
10660 });
10661 }
10662 Ok((start_row_1based - 1, end_row_1based - 1))
10663 }
10664
10665 fn invalidate_row_visibility_mask_cache(&self) {
10666 if let Ok(mut cache) = self.row_visibility_mask_cache.write() {
10667 cache.clear();
10668 }
10669 }
10670
10671 fn row_visibility_changed(&mut self) {
10674 self.invalidate_row_visibility_mask_cache();
10675 self.graph.mark_row_visibility_readers_dirty();
10676 }
10677
10678 fn set_row_hidden_by_sheet_id(
10679 &mut self,
10680 sheet_id: SheetId,
10681 row0: u32,
10682 hidden: bool,
10683 source: RowVisibilitySource,
10684 ) -> bool {
10685 let changed = {
10686 let state = self.row_visibility.entry(sheet_id).or_default();
10687 state.set_row_hidden(row0, hidden, source)
10688 };
10689
10690 let remove_entry = self
10691 .row_visibility
10692 .get(&sheet_id)
10693 .map(|state| state.is_empty())
10694 .unwrap_or(false);
10695 if remove_entry {
10696 self.row_visibility.remove(&sheet_id);
10697 }
10698
10699 if changed {
10700 self.row_visibility_changed();
10701 }
10702
10703 changed
10704 }
10705
10706 fn set_rows_hidden_by_sheet_id(
10707 &mut self,
10708 sheet_id: SheetId,
10709 start_row0: u32,
10710 end_row0: u32,
10711 hidden: bool,
10712 source: RowVisibilitySource,
10713 ) -> bool {
10714 let changed = {
10715 let state = self.row_visibility.entry(sheet_id).or_default();
10716 state.set_rows_hidden(start_row0, end_row0, hidden, source)
10717 };
10718
10719 let remove_entry = self
10720 .row_visibility
10721 .get(&sheet_id)
10722 .map(|state| state.is_empty())
10723 .unwrap_or(false);
10724 if remove_entry {
10725 self.row_visibility.remove(&sheet_id);
10726 }
10727
10728 if changed {
10729 self.row_visibility_changed();
10730 }
10731
10732 changed
10733 }
10734
10735 fn shift_row_visibility_insert(&mut self, sheet_id: SheetId, before0: u32, count: u32) {
10736 if count == 0 {
10737 return;
10738 }
10739 let mut changed = false;
10740 let remove_entry = if let Some(state) = self.row_visibility.get_mut(&sheet_id) {
10741 changed = state.insert_rows(before0, count);
10742 state.is_empty()
10743 } else {
10744 false
10745 };
10746 if remove_entry {
10747 self.row_visibility.remove(&sheet_id);
10748 }
10749 if changed {
10750 self.row_visibility_changed();
10751 }
10752 }
10753
10754 fn shift_row_visibility_delete(&mut self, sheet_id: SheetId, start0: u32, count: u32) {
10755 if count == 0 {
10756 return;
10757 }
10758 let mut changed = false;
10759 let remove_entry = if let Some(state) = self.row_visibility.get_mut(&sheet_id) {
10760 changed = state.delete_rows(start0, count);
10761 state.is_empty()
10762 } else {
10763 false
10764 };
10765 if remove_entry {
10766 self.row_visibility.remove(&sheet_id);
10767 }
10768 if changed {
10769 self.row_visibility_changed();
10770 }
10771 }
10772
10773 fn apply_inverse_row_visibility_event(&mut self, event: &crate::engine::ChangeEvent) {
10774 if let crate::engine::ChangeEvent::SetRowVisibility {
10775 sheet_id,
10776 row0,
10777 source,
10778 old_hidden,
10779 ..
10780 } = event
10781 {
10782 let _ = self.set_row_hidden_by_sheet_id(*sheet_id, *row0, *old_hidden, *source);
10783 }
10784 }
10785
10786 fn apply_forward_row_visibility_event(&mut self, event: &crate::engine::ChangeEvent) {
10787 if let crate::engine::ChangeEvent::SetRowVisibility {
10788 sheet_id,
10789 row0,
10790 source,
10791 new_hidden,
10792 ..
10793 } = event
10794 {
10795 let _ = self.set_row_hidden_by_sheet_id(*sheet_id, *row0, *new_hidden, *source);
10796 }
10797 }
10798
10799 fn apply_inverse_row_visibility_events(&mut self, events: &[crate::engine::ChangeEvent]) {
10800 for event in events.iter().rev() {
10801 self.apply_inverse_row_visibility_event(event);
10802 }
10803 }
10804
10805 fn apply_forward_row_visibility_events(&mut self, events: &[crate::engine::ChangeEvent]) {
10806 for event in events {
10807 self.apply_forward_row_visibility_event(event);
10808 }
10809 }
10810
10811 fn apply_inverse_staged_formula_event(&mut self, event: &crate::engine::ChangeEvent) {
10812 if let crate::engine::ChangeEvent::StagedFormulaCellChanged {
10813 sheet,
10814 row,
10815 col,
10816 old,
10817 ..
10818 } = event
10819 {
10820 self.apply_staged_formula_cell(sheet, *row, *col, old.as_deref());
10821 }
10822 }
10823
10824 fn apply_forward_staged_formula_event(&mut self, event: &crate::engine::ChangeEvent) {
10825 if let crate::engine::ChangeEvent::StagedFormulaCellChanged {
10826 sheet,
10827 row,
10828 col,
10829 new,
10830 ..
10831 } = event
10832 {
10833 self.apply_staged_formula_cell(sheet, *row, *col, new.as_deref());
10834 }
10835 }
10836
10837 fn apply_staged_formula_cell(&mut self, sheet: &str, row: u32, col: u32, target: Option<&str>) {
10840 match target {
10841 Some(text) => self.stage_formula_text(sheet, row, col, text.to_string()),
10842 None => {
10843 self.clear_staged_formula_text(sheet, row, col);
10844 }
10845 }
10846 }
10847
10848 pub fn set_row_hidden(
10849 &mut self,
10850 sheet: &str,
10851 row_1based: u32,
10852 hidden: bool,
10853 source: RowVisibilitySource,
10854 ) -> Result<(), crate::engine::EditorError> {
10855 self.observe_function_semantic_epoch()
10856 .map_err(crate::engine::EditorError::Excel)?;
10857 self.observe_function_semantic_epoch()
10858 .map_err(crate::engine::EditorError::Excel)?;
10859 let sheet_id = self.ensure_known_sheet_id(sheet)?;
10860 let row0 = Self::normalize_row_1based(row_1based)?;
10861 if self.set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source) {
10862 self.record_structural_change(StructuralScope::Region(Region::whole_row(
10863 sheet_id, row0,
10864 )));
10865 self.mark_data_edited();
10866 }
10867 Ok(())
10868 }
10869
10870 pub fn set_rows_hidden(
10871 &mut self,
10872 sheet: &str,
10873 start_row_1based: u32,
10874 end_row_1based: u32,
10875 hidden: bool,
10876 source: RowVisibilitySource,
10877 ) -> Result<(), crate::engine::EditorError> {
10878 let sheet_id = self.ensure_known_sheet_id(sheet)?;
10879 let (start_row0, end_row0) =
10880 Self::normalize_row_range_1based(start_row_1based, end_row_1based)?;
10881 if self.set_rows_hidden_by_sheet_id(sheet_id, start_row0, end_row0, hidden, source) {
10882 if start_row0 == end_row0 {
10883 self.record_structural_change(StructuralScope::Region(Region::whole_row(
10884 sheet_id, start_row0,
10885 )));
10886 } else {
10887 self.record_structural_change(StructuralScope::Sheet(sheet_id));
10888 }
10889 self.mark_data_edited();
10890 }
10891 Ok(())
10892 }
10893
10894 pub fn is_row_hidden(
10895 &self,
10896 sheet: &str,
10897 row_1based: u32,
10898 source: Option<RowVisibilitySource>,
10899 ) -> Option<bool> {
10900 let sheet_id = self.graph.sheet_id(sheet)?;
10901 let row0 = row_1based.checked_sub(1)?;
10902 Some(
10903 self.row_visibility
10904 .get(&sheet_id)
10905 .map(|state| state.is_row_hidden(row0, source))
10906 .unwrap_or(false),
10907 )
10908 }
10909
10910 pub fn row_visibility_version(&self, sheet: &str) -> Option<u64> {
10911 let sheet_id = self.graph.sheet_id(sheet)?;
10912 Some(
10913 self.row_visibility
10914 .get(&sheet_id)
10915 .map(|state| state.version())
10916 .unwrap_or(0),
10917 )
10918 }
10919
10920 fn nested_subtotal_cells_for_view(
10926 &self,
10927 view: &RangeView<'_>,
10928 include_aggregate: bool,
10929 ) -> Option<Vec<(usize, usize, usize)>> {
10930 if !view.is_sheet_backed() {
10931 return None;
10932 }
10933 let (rows, cols) = view.dims();
10934 if rows == 0 || cols == 0 {
10935 return Some(Vec::new());
10936 }
10937 let sheet_id = self.graph.sheet_id(view.sheet_name())?;
10938 let (r0, c0) = (view.start_row(), view.start_col());
10939 let mask = if include_aggregate {
10940 crate::engine::arena::SUBTOTAL_CALL | crate::engine::arena::AGGREGATE_CALL
10941 } else {
10942 crate::engine::arena::SUBTOTAL_CALL
10943 };
10944 let ds = self.graph.data_store();
10945 let mut found: Vec<(usize, usize, usize)> = self
10946 .graph
10947 .formula_intervals_in_region(
10948 sheet_id,
10949 r0 as u32,
10950 (r0 + rows - 1) as u32,
10951 c0 as u32,
10952 (c0 + cols - 1) as u32,
10953 |template| ds.ast_subtotal_calls(template) & mask != 0,
10954 )
10955 .into_iter()
10956 .map(|(col, lo, hi)| (col as usize - c0, lo as usize - r0, hi as usize - r0))
10957 .collect();
10958 found.sort_unstable();
10959 let mut out: Vec<(usize, usize, usize)> = Vec::with_capacity(found.len());
10962 for (col, lo, hi) in found {
10963 match out.last_mut() {
10964 Some(last) if last.0 == col && lo <= last.2 + 1 => last.2 = last.2.max(hi),
10965 _ => out.push((col, lo, hi)),
10966 }
10967 }
10968 Some(out)
10969 }
10970
10971 fn build_row_visibility_mask_for_view(
10972 &self,
10973 view: &RangeView<'_>,
10974 mode: VisibilityMaskMode,
10975 ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
10976 let sheet_rows = view.sheet().nrows as usize;
10977 if sheet_rows == 0 || view.start_row() >= sheet_rows {
10978 return Some(std::sync::Arc::new(arrow_array::BooleanArray::new_null(0)));
10979 }
10980
10981 let sheet_id = self.graph.sheet_id(view.sheet_name())?;
10982 let start_row0 = view.start_row() as u32;
10983 let end_row0 = view.end_row().min(sheet_rows.saturating_sub(1)) as u32;
10984 let version = self
10985 .row_visibility
10986 .get(&sheet_id)
10987 .map(|state| state.version())
10988 .unwrap_or(0);
10989 let key = VisibilityMaskCacheKey {
10990 sheet_id,
10991 start_row0,
10992 end_row0,
10993 mode,
10994 version,
10995 };
10996
10997 if let Ok(cache) = self.row_visibility_mask_cache.read()
10998 && let Some(mask) = cache.get(&key)
10999 {
11000 #[cfg(test)]
11001 visibility_mask_test_hooks::inc_hit();
11002 return Some(mask.clone());
11003 }
11004
11005 #[cfg(test)]
11006 visibility_mask_test_hooks::inc_miss();
11007
11008 let state = self.row_visibility.get(&sheet_id);
11009 let mut out = Vec::with_capacity((end_row0 - start_row0 + 1) as usize);
11010 for row0 in start_row0..=end_row0 {
11011 let manual_hidden = state
11012 .map(|s| s.is_row_hidden(row0, Some(RowVisibilitySource::Manual)))
11013 .unwrap_or(false);
11014 let filter_hidden = state
11015 .map(|s| s.is_row_hidden(row0, Some(RowVisibilitySource::Filter)))
11016 .unwrap_or(false);
11017
11018 let include = match mode {
11019 VisibilityMaskMode::IncludeAll => true,
11020 VisibilityMaskMode::ExcludeManualHidden => !manual_hidden,
11021 VisibilityMaskMode::ExcludeFilterHidden => !filter_hidden,
11022 VisibilityMaskMode::ExcludeManualOrFilterHidden => {
11023 !(manual_hidden || filter_hidden)
11024 }
11025 };
11026 out.push(include);
11027 }
11028
11029 let mask = std::sync::Arc::new(arrow_array::BooleanArray::from(out));
11030 if let Ok(mut cache) = self.row_visibility_mask_cache.write() {
11031 const MAX_CACHE_ENTRIES: usize = 4096;
11032 if cache.len() >= MAX_CACHE_ENTRIES {
11033 cache.clear();
11034 #[cfg(test)]
11035 visibility_mask_test_hooks::inc_eviction();
11036 }
11037 cache.insert(key, mask.clone());
11038 }
11039
11040 Some(mask)
11041 }
11042
11043 fn observe_function_semantic_epoch(&mut self) -> Result<bool, ExcelError> {
11044 let changes =
11045 crate::function_registry::semantic_changes_since(self.function_semantic_epoch_seen);
11046 let global_changed = changes.epoch != self.function_semantic_epoch_seen;
11047 let provider_revision = self.resolver.planning_semantic_revision();
11048 let provider_changed = provider_revision != self.function_provider_revision_seen;
11049 if !global_changed && !provider_changed {
11050 return Ok(false);
11051 }
11052
11053 let changed = !changes.keys.is_empty();
11054 if global_changed && changed || provider_changed {
11055 self.cached_static_schedule = None;
11056 self.recent_schedules.clear();
11057 self.base_schedule = None;
11058 }
11059 self.function_semantic_epoch_seen = changes.epoch;
11060 self.function_provider_revision_seen = provider_revision;
11061 Ok(false)
11062 }
11063
11064 pub(crate) fn ast_uses_changed_function(
11065 ast: &ASTNode,
11066 changed: &BTreeSet<(String, String)>,
11067 ) -> bool {
11068 match &ast.node_type {
11069 ASTNodeType::Function { name, args } => {
11070 let normalized = name.to_uppercase();
11071 let mut spellings = vec![(String::new(), normalized.clone())];
11072 let mut stripped = normalized.as_str();
11073 while let Some(rest) = ["_XLFN.", "_XLL.", "_XLWS."]
11074 .iter()
11075 .find_map(|prefix| stripped.strip_prefix(prefix))
11076 {
11077 stripped = rest;
11078 spellings.push((String::new(), stripped.to_string()));
11079 }
11080 let resolved = crate::function_registry::resolve("", name);
11081 let directly_changed = spellings.iter().any(|spelling| changed.contains(spelling))
11082 || resolved.as_ref().is_some_and(|resolved| {
11083 changed.contains(&(
11084 resolved.namespace.clone(),
11085 resolved.canonical_name.clone(),
11086 ))
11087 });
11088 directly_changed
11089 || resolved.is_none()
11090 || args
11091 .iter()
11092 .any(|arg| Self::ast_uses_changed_function(arg, changed))
11093 }
11094 ASTNodeType::Call { callee, args } => {
11095 Self::ast_uses_changed_function(callee, changed)
11096 || args
11097 .iter()
11098 .any(|arg| Self::ast_uses_changed_function(arg, changed))
11099 }
11100 ASTNodeType::UnaryOp { expr, .. } => Self::ast_uses_changed_function(expr, changed),
11101 ASTNodeType::BinaryOp { left, right, .. } => {
11102 Self::ast_uses_changed_function(left, changed)
11103 || Self::ast_uses_changed_function(right, changed)
11104 }
11105 ASTNodeType::Array(rows) => rows
11106 .iter()
11107 .flatten()
11108 .any(|node| Self::ast_uses_changed_function(node, changed)),
11109 _ => false,
11110 }
11111 }
11112
11113 fn ast_contains_function(ast: &ASTNode) -> bool {
11114 match &ast.node_type {
11115 ASTNodeType::Function { .. } => true,
11116 ASTNodeType::Call { callee, args } => {
11117 Self::ast_contains_function(callee) || args.iter().any(Self::ast_contains_function)
11118 }
11119 ASTNodeType::UnaryOp { expr, .. } => Self::ast_contains_function(expr),
11120 ASTNodeType::BinaryOp { left, right, .. } => {
11121 Self::ast_contains_function(left) || Self::ast_contains_function(right)
11122 }
11123 ASTNodeType::Array(rows) => rows.iter().flatten().any(Self::ast_contains_function),
11124 ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => false,
11125 }
11126 }
11127
11128 fn structural_row_region(sheet_id: SheetId, start_row0: u32) -> Region {
11129 Region::rows_from(sheet_id, start_row0)
11130 }
11131
11132 fn structural_col_region(sheet_id: SheetId, start_col0: u32) -> Region {
11133 Region::cols_from(sheet_id, start_col0)
11134 }
11135
11136 #[cfg(test)]
11137 pub(crate) fn force_non_cycle_schedule_fallback_for_test(&mut self) {
11138 self.force_non_cycle_schedule_fallback_for_test = true;
11139 }
11140
11141 fn materialize_deferred_sheet_before_structural_edit(
11142 &mut self,
11143 sheet: &str,
11144 ) -> Result<(), crate::engine::EditorError> {
11145 if self.staged_formulas.contains_key(sheet) {
11146 self.build_graph_for_sheets([sheet])?;
11147 }
11148 Ok(())
11149 }
11150
11151 fn check_insert_room(
11156 &self,
11157 sheet: &str,
11158 before0: u32,
11159 count: u32,
11160 rows: bool,
11161 ) -> Result<(), crate::engine::EditorError> {
11162 let limit = if rows {
11163 crate::engine::authority::geom::MAX_ROW
11164 } else {
11165 crate::engine::authority::geom::MAX_COL
11166 } + 1;
11167 let arrow = self.arrow_sheets.sheet(sheet).map_or(0, |a| {
11168 if rows {
11169 a.nrows
11170 } else {
11171 u32::try_from(a.columns.len()).unwrap_or(u32::MAX)
11172 }
11173 });
11174 let formulas = self.graph.sheet_id(sheet).map_or(0, |id| {
11175 self.graph
11176 .grid_vertices_in_sheet(id)
11177 .map(|(_, at)| if rows { at.row() } else { at.col() } + 1)
11178 .max()
11179 .unwrap_or(0)
11180 });
11181 let extent = arrow.max(formulas);
11182 let pushed_off = extent > before0 && extent.saturating_add(count) > limit;
11183 if count > limit || pushed_off {
11184 return Err(crate::engine::EditorError::Excel(
11185 ExcelError::new(ExcelErrorKind::Ref).with_message(format!(
11186 "inserting {count} {} would push non-empty cells off the sheet",
11187 if rows { "rows" } else { "columns" }
11188 )),
11189 ));
11190 }
11191 Ok(())
11192 }
11193
11194 fn structural_row_occupancy(
11195 &self,
11196 sheet: &str,
11197 sheet_id: SheetId,
11198 ) -> crate::engine::graph::StructuralOccupancy {
11199 if !self.graph.has_compressed_range_readers() {
11200 return crate::engine::graph::StructuralOccupancy::default();
11201 }
11202 let mut occupancy = self.graph.structural_occupancy(sheet_id);
11203 if let Some(arrow_sheet) = self.arrow_sheets.sheet(sheet) {
11204 occupancy.include_arrow_sheet(arrow_sheet);
11205 occupancy
11206 } else {
11207 crate::engine::graph::StructuralOccupancy::conservative()
11209 }
11210 }
11211
11212 fn structural_column_occupancy(&self) -> crate::engine::graph::StructuralOccupancy {
11213 crate::engine::graph::StructuralOccupancy::conservative()
11217 }
11218
11219 pub fn insert_rows(
11221 &mut self,
11222 sheet: &str,
11223 before: u32,
11224 count: u32,
11225 ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
11226 {
11227 if count == 0 {
11228 return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
11229 }
11230 self.observe_function_semantic_epoch()
11231 .map_err(crate::engine::EditorError::Excel)?;
11232 use crate::engine::graph::editor::vertex_editor::VertexEditor;
11233 self.materialize_deferred_sheet_before_structural_edit(sheet)?;
11234 let sheet_id = self.ensure_known_sheet_id(sheet)?;
11235 let before0 = before.saturating_sub(1);
11236 self.check_insert_room(sheet, before0, count, true)?;
11237 let affected_region = Self::structural_row_region(sheet_id, before0);
11238 let occupancy = self.structural_row_occupancy(sheet, sheet_id);
11239 let summary = {
11240 let mut editor =
11241 VertexEditor::new(&mut self.graph).with_structural_occupancy(occupancy);
11242 editor.insert_rows(sheet_id, before0, count)?
11243 };
11244 if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
11245 let before0 = before0 as usize;
11246 asheet.insert_rows(before0, count as usize);
11247 }
11248 self.purge_derived_formats_after_row(sheet_id, before0);
11249 self.mark_moved_formula_vertices_dirty(&summary);
11250 self.clear_computed_overlay_after_row(sheet, before0 as usize);
11251 self.shift_row_visibility_insert(sheet_id, before0, count);
11252 self.record_structural_change(StructuralScope::Region(affected_region));
11253 self.mark_topology_edited();
11254 Ok(summary)
11255 }
11256
11257 pub fn delete_rows(
11259 &mut self,
11260 sheet: &str,
11261 start: u32,
11262 count: u32,
11263 ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
11264 {
11265 if count == 0 {
11266 return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
11267 }
11268 self.observe_function_semantic_epoch()
11269 .map_err(crate::engine::EditorError::Excel)?;
11270 use crate::engine::graph::editor::vertex_editor::VertexEditor;
11271 self.materialize_deferred_sheet_before_structural_edit(sheet)?;
11272 let sheet_id = self.ensure_known_sheet_id(sheet)?;
11273 let start0 = start.saturating_sub(1);
11274 let affected_region = Self::structural_row_region(sheet_id, start0);
11275 let occupancy = self.structural_row_occupancy(sheet, sheet_id);
11276 let summary = {
11277 let mut editor =
11278 VertexEditor::new(&mut self.graph).with_structural_occupancy(occupancy);
11279 editor.delete_rows(sheet_id, start0, count)?
11280 };
11281 if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
11282 let start0 = start0 as usize;
11283 asheet.delete_rows(start0, count as usize);
11284 }
11285 self.purge_derived_formats_after_row(sheet_id, start0);
11286 self.mark_moved_formula_vertices_dirty(&summary);
11287 self.clear_computed_overlay_after_row(sheet, start0 as usize);
11288 self.shift_row_visibility_delete(sheet_id, start0, count);
11289 self.record_structural_change(StructuralScope::Region(affected_region));
11290 self.mark_topology_edited();
11291 Ok(summary)
11292 }
11293
11294 pub fn insert_columns(
11296 &mut self,
11297 sheet: &str,
11298 before: u32,
11299 count: u32,
11300 ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
11301 {
11302 if count == 0 {
11303 return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
11304 }
11305 self.observe_function_semantic_epoch()
11306 .map_err(crate::engine::EditorError::Excel)?;
11307 use crate::engine::graph::editor::vertex_editor::VertexEditor;
11308 self.materialize_deferred_sheet_before_structural_edit(sheet)?;
11309 let sheet_id = self.graph.sheet_id(sheet).ok_or(
11310 crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
11311 name: sheet.to_string(),
11312 reason: "Unknown sheet".to_string(),
11313 },
11314 )?;
11315 let before0 = before.saturating_sub(1);
11316 self.check_insert_room(sheet, before0, count, false)?;
11317 let affected_region = Self::structural_col_region(sheet_id, before0);
11318 let occupancy = self.structural_column_occupancy();
11319 let summary = {
11320 let mut editor =
11321 VertexEditor::new(&mut self.graph).with_structural_occupancy(occupancy);
11322 editor.insert_columns(sheet_id, before0, count)?
11323 };
11324 if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
11325 let before0 = before0 as usize;
11326 asheet.insert_columns(before0, count as usize);
11327 }
11328 self.purge_derived_formats_after_col(sheet_id, before0);
11329 self.mark_moved_formula_vertices_dirty(&summary);
11330 self.clear_computed_overlay_after_col(sheet, before0 as usize);
11331 self.record_structural_change(StructuralScope::Region(affected_region));
11332 self.mark_topology_edited();
11333 Ok(summary)
11334 }
11335
11336 pub fn delete_columns(
11338 &mut self,
11339 sheet: &str,
11340 start: u32,
11341 count: u32,
11342 ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
11343 {
11344 if count == 0 {
11345 return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
11346 }
11347 self.observe_function_semantic_epoch()
11348 .map_err(crate::engine::EditorError::Excel)?;
11349 use crate::engine::graph::editor::vertex_editor::VertexEditor;
11350 self.materialize_deferred_sheet_before_structural_edit(sheet)?;
11351 let sheet_id = self.graph.sheet_id(sheet).ok_or(
11352 crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
11353 name: sheet.to_string(),
11354 reason: "Unknown sheet".to_string(),
11355 },
11356 )?;
11357 let start0 = start.saturating_sub(1);
11358 let affected_region = Self::structural_col_region(sheet_id, start0);
11359 let occupancy = self.structural_column_occupancy();
11360 let summary = {
11361 let mut editor =
11362 VertexEditor::new(&mut self.graph).with_structural_occupancy(occupancy);
11363 editor.delete_columns(sheet_id, start0, count)?
11364 };
11365 if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
11366 let start0 = start0 as usize;
11367 asheet.delete_columns(start0, count as usize);
11368 }
11369 self.purge_derived_formats_after_col(sheet_id, start0);
11370 self.mark_moved_formula_vertices_dirty(&summary);
11371 self.clear_computed_overlay_after_col(sheet, start0 as usize);
11372 self.record_structural_change(StructuralScope::Region(affected_region));
11373 self.mark_topology_edited();
11374 Ok(summary)
11375 }
11376 fn arrow_used_row_bounds(
11378 &self,
11379 sheet: &str,
11380 start_col: u32,
11381 end_col: u32,
11382 ) -> Option<(u32, u32)> {
11383 let a = self.sheet_store().sheet(sheet)?;
11384 if a.columns.is_empty() {
11385 return None;
11386 }
11387 let sc0 = start_col.saturating_sub(1) as usize;
11388 let ec0 = end_col.saturating_sub(1) as usize;
11389 let col_hi = a.columns.len().saturating_sub(1);
11390 if sc0 > col_hi {
11391 return None;
11392 }
11393 let ec0 = ec0.min(col_hi);
11394 let snap = self.data_snapshot_id();
11396 let mut min_r0: Option<usize> = None;
11397 for ci in sc0..=ec0 {
11398 let sheet_id = self.graph.sheet_id(sheet)?;
11399 if let Some((Some(mv), _)) = self.row_bounds_cache.read().ok().and_then(|g| {
11400 g.as_ref()
11401 .and_then(|c| c.get_row_bounds(sheet_id, ci, snap))
11402 }) {
11403 let mv = mv as usize;
11404 min_r0 = Some(min_r0.map(|m| m.min(mv)).unwrap_or(mv));
11405 continue;
11406 }
11407 let (min_c, max_c) = Self::scan_column_used_bounds(a, ci);
11409 if let Ok(mut g) = self.row_bounds_cache.write() {
11410 g.get_or_insert_with(|| RowBoundsCache::new(snap))
11411 .put_row_bounds(sheet_id, ci, snap, (min_c, max_c));
11412 }
11413 if let Some(m) = min_c {
11414 min_r0 = Some(min_r0.map(|mm| mm.min(m as usize)).unwrap_or(m as usize));
11415 }
11416 }
11417 min_r0?;
11418 let mut max_r0: Option<usize> = None;
11419 for ci in sc0..=ec0 {
11420 let sheet_id = self.graph.sheet_id(sheet)?;
11421 if let Some((_, Some(mv))) = self.row_bounds_cache.read().ok().and_then(|g| {
11422 g.as_ref()
11423 .and_then(|c| c.get_row_bounds(sheet_id, ci, snap))
11424 }) {
11425 let mv = mv as usize;
11426 max_r0 = Some(max_r0.map(|m| m.max(mv)).unwrap_or(mv));
11427 continue;
11428 }
11429 let (_min_c, max_c) = Self::scan_column_used_bounds(a, ci);
11430 if let Ok(mut g) = self.row_bounds_cache.write() {
11431 g.get_or_insert_with(|| RowBoundsCache::new(snap))
11432 .put_row_bounds(sheet_id, ci, snap, (_min_c, max_c));
11433 }
11434 if let Some(m) = max_c {
11435 max_r0 = Some(max_r0.map(|mm| mm.max(m as usize)).unwrap_or(m as usize));
11436 }
11437 }
11438 match (min_r0, max_r0) {
11439 (Some(a0), Some(b0)) => Some(((a0 as u32) + 1, (b0 as u32) + 1)),
11440 _ => None,
11441 }
11442 }
11443
11444 fn scan_column_used_bounds(
11445 a: &crate::arrow_store::ArrowSheet,
11446 ci: usize,
11447 ) -> (Option<u32>, Option<u32>) {
11448 let col = &a.columns[ci];
11449
11450 let mut min_r0: Option<u32> = None;
11452 for (chunk_idx, chunk) in col.chunks.iter().enumerate() {
11453 let tags = chunk.type_tag.values();
11454 for (off, &t) in tags.iter().enumerate() {
11455 let overlay_non_empty = chunk
11456 .overlay
11457 .get(off)
11458 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11459 .unwrap_or(false)
11460 || chunk
11461 .computed_overlay
11462 .get(off)
11463 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11464 .unwrap_or(false);
11465 if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
11466 let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
11467 break;
11468 };
11469 let row0 = chunk_start + off;
11470 min_r0 = Some(row0 as u32);
11471 break;
11472 }
11473 }
11474 if min_r0.is_some() {
11475 break;
11476 }
11477 }
11478 if min_r0.is_none() && !col.sparse_chunks.is_empty() {
11479 let mut sparse_idxs: Vec<usize> = col.sparse_chunks.keys().copied().collect();
11480 sparse_idxs.sort_unstable();
11481 for chunk_idx in sparse_idxs {
11482 let Some(chunk) = col.sparse_chunks.get(&chunk_idx) else {
11483 continue;
11484 };
11485 let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
11486 continue;
11487 };
11488 let tags = chunk.type_tag.values();
11489 for (off, &t) in tags.iter().enumerate() {
11490 let overlay_non_empty = chunk
11491 .overlay
11492 .get(off)
11493 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11494 .unwrap_or(false)
11495 || chunk
11496 .computed_overlay
11497 .get(off)
11498 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11499 .unwrap_or(false);
11500 if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
11501 let row0 = chunk_start + off;
11502 min_r0 = Some(row0 as u32);
11503 break;
11504 }
11505 }
11506 if min_r0.is_some() {
11507 break;
11508 }
11509 }
11510 }
11511
11512 let mut max_r0: Option<u32> = None;
11514 if !col.sparse_chunks.is_empty() {
11515 let mut sparse_idxs: Vec<usize> = col.sparse_chunks.keys().copied().collect();
11516 sparse_idxs.sort_unstable_by(|a, b| b.cmp(a));
11517 for chunk_idx in sparse_idxs {
11518 let Some(chunk) = col.sparse_chunks.get(&chunk_idx) else {
11519 continue;
11520 };
11521 let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
11522 continue;
11523 };
11524 let tags = chunk.type_tag.values();
11525 for (rev_idx, &t) in tags.iter().enumerate().rev() {
11526 let overlay_non_empty = chunk
11527 .overlay
11528 .get(rev_idx)
11529 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11530 .unwrap_or(false)
11531 || chunk
11532 .computed_overlay
11533 .get(rev_idx)
11534 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11535 .unwrap_or(false);
11536 if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
11537 let row0 = chunk_start + rev_idx;
11538 max_r0 = Some(row0 as u32);
11539 break;
11540 }
11541 }
11542 if max_r0.is_some() {
11543 break;
11544 }
11545 }
11546 }
11547 if max_r0.is_none() {
11548 for (chunk_idx, chunk) in col.chunks.iter().enumerate().rev() {
11549 let tags = chunk.type_tag.values();
11550 for (rev_idx, &t) in tags.iter().enumerate().rev() {
11551 let overlay_non_empty = chunk
11552 .overlay
11553 .get(rev_idx)
11554 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11555 .unwrap_or(false)
11556 || chunk
11557 .computed_overlay
11558 .get(rev_idx)
11559 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11560 .unwrap_or(false);
11561 if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
11562 let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
11563 break;
11564 };
11565 let row0 = chunk_start + rev_idx;
11566 max_r0 = Some(row0 as u32);
11567 break;
11568 }
11569 }
11570 if max_r0.is_some() {
11571 break;
11572 }
11573 }
11574 }
11575
11576 (min_r0, max_r0)
11577 }
11578
11579 fn arrow_used_col_bounds(
11581 &self,
11582 sheet: &str,
11583 start_row: u32,
11584 end_row: u32,
11585 ) -> Option<(u32, u32)> {
11586 let a = self.sheet_store().sheet(sheet)?;
11587 if a.columns.is_empty() {
11588 return None;
11589 }
11590 let sr0 = start_row.saturating_sub(1) as usize;
11591 let er0 = end_row.saturating_sub(1) as usize;
11592 if sr0 > er0 {
11593 return None;
11594 }
11595 let mut min_c0: Option<usize> = None;
11598 let mut max_c0: Option<usize> = None;
11599 for (ci, col) in a.columns.iter().enumerate() {
11601 let mut any_in_range = false;
11602
11603 let scan_chunk = |chunk_idx: usize, chunk: &crate::arrow_store::ColumnChunk| -> bool {
11604 let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
11605 return false;
11606 };
11607 let chunk_len = chunk.type_tag.len();
11608 if chunk_len == 0 {
11609 return false;
11610 }
11611 let chunk_end = chunk_start + chunk_len.saturating_sub(1);
11612 if sr0 > chunk_end || er0 < chunk_start {
11614 return false;
11615 }
11616 let start_off = sr0.max(chunk_start) - chunk_start;
11617 let end_off = er0.min(chunk_end) - chunk_start;
11618 let tags = chunk.type_tag.values();
11619 for off in start_off..=end_off {
11620 let overlay_non_empty = chunk
11621 .overlay
11622 .get(off)
11623 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11624 .unwrap_or(false)
11625 || chunk
11626 .computed_overlay
11627 .get(off)
11628 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11629 .unwrap_or(false);
11630 if overlay_non_empty || tags[off] != crate::arrow_store::TypeTag::Empty as u8 {
11631 return true;
11632 }
11633 }
11634 false
11635 };
11636
11637 for (chunk_idx, chunk) in col.chunks.iter().enumerate() {
11638 if scan_chunk(chunk_idx, chunk) {
11639 any_in_range = true;
11640 break;
11641 }
11642 }
11643
11644 if !any_in_range && !col.sparse_chunks.is_empty() {
11645 for (&chunk_idx, chunk) in col.sparse_chunks.iter() {
11646 if scan_chunk(chunk_idx, chunk) {
11647 any_in_range = true;
11648 break;
11649 }
11650 }
11651 }
11652
11653 if any_in_range {
11654 min_c0 = Some(min_c0.map(|m| m.min(ci)).unwrap_or(ci));
11655 max_c0 = Some(max_c0.map(|m| m.max(ci)).unwrap_or(ci));
11656 }
11657 }
11658 match (min_c0, max_c0) {
11659 (Some(a0), Some(b0)) => Some(((a0 as u32) + 1, (b0 as u32) + 1)),
11660 _ => None,
11661 }
11662 }
11663
11664 fn formula_row_bounds_for_columns(
11665 &self,
11666 sheet: &str,
11667 start_col: u32,
11668 end_col: u32,
11669 ) -> Option<(u32, u32)> {
11670 let sheet_id = self.graph.sheet_id(sheet)?;
11671 let sc0 = start_col.saturating_sub(1);
11672 let ec0 = end_col.saturating_sub(1);
11673 let mut min_r0: Option<u32> = None;
11674 let mut max_r0: Option<u32> = None;
11675
11676 if self.graph.sheet_index(sheet_id).is_some() {
11677 for vid in self.graph.vertices_in_cols(sheet_id, sc0, ec0) {
11678 if !matches!(
11679 self.graph.get_vertex_kind(vid),
11680 VertexKind::FormulaScalar | VertexKind::FormulaArray
11681 ) {
11682 continue;
11683 }
11684 let Some(row0) = self.graph.vertex_grid_addr(vid).map(|addr| addr.row()) else {
11685 continue;
11686 };
11687 min_r0 = Some(min_r0.map(|m| m.min(row0)).unwrap_or(row0));
11688 max_r0 = Some(max_r0.map(|m| m.max(row0)).unwrap_or(row0));
11689 }
11690 } else {
11691 for (vid, coord) in self.graph.grid_vertices_in_sheet(sheet_id) {
11692 if !matches!(
11693 self.graph.get_vertex_kind(vid),
11694 VertexKind::FormulaScalar | VertexKind::FormulaArray
11695 ) {
11696 continue;
11697 }
11698 let col0 = coord.col();
11699 if col0 < sc0 || col0 > ec0 {
11700 continue;
11701 }
11702 let row0 = coord.row();
11703 min_r0 = Some(min_r0.map(|m| m.min(row0)).unwrap_or(row0));
11704 max_r0 = Some(max_r0.map(|m| m.max(row0)).unwrap_or(row0));
11705 }
11706 }
11707
11708 match (min_r0, max_r0) {
11709 (Some(a0), Some(b0)) => Some((a0 + 1, b0 + 1)),
11710 _ => None,
11711 }
11712 }
11713
11714 fn formula_col_bounds_for_rows(
11715 &self,
11716 sheet: &str,
11717 start_row: u32,
11718 end_row: u32,
11719 ) -> Option<(u32, u32)> {
11720 let sheet_id = self.graph.sheet_id(sheet)?;
11721 let sr0 = start_row.saturating_sub(1);
11722 let er0 = end_row.saturating_sub(1);
11723 let mut min_c0: Option<u32> = None;
11724 let mut max_c0: Option<u32> = None;
11725
11726 if self.graph.sheet_index(sheet_id).is_some() {
11727 for vid in self.graph.vertices_in_rows(sheet_id, sr0, er0) {
11728 if !matches!(
11729 self.graph.get_vertex_kind(vid),
11730 VertexKind::FormulaScalar | VertexKind::FormulaArray
11731 ) {
11732 continue;
11733 }
11734 let Some(col0) = self.graph.vertex_grid_addr(vid).map(|addr| addr.col()) else {
11735 continue;
11736 };
11737 min_c0 = Some(min_c0.map(|m| m.min(col0)).unwrap_or(col0));
11738 max_c0 = Some(max_c0.map(|m| m.max(col0)).unwrap_or(col0));
11739 }
11740 } else {
11741 for (vid, coord) in self.graph.grid_vertices_in_sheet(sheet_id) {
11742 if !matches!(
11743 self.graph.get_vertex_kind(vid),
11744 VertexKind::FormulaScalar | VertexKind::FormulaArray
11745 ) {
11746 continue;
11747 }
11748 let row0 = coord.row();
11749 if row0 < sr0 || row0 > er0 {
11750 continue;
11751 }
11752 let col0 = coord.col();
11753 min_c0 = Some(min_c0.map(|m| m.min(col0)).unwrap_or(col0));
11754 max_c0 = Some(max_c0.map(|m| m.max(col0)).unwrap_or(col0));
11755 }
11756 }
11757
11758 match (min_c0, max_c0) {
11759 (Some(a0), Some(b0)) => Some((a0 + 1, b0 + 1)),
11760 _ => None,
11761 }
11762 }
11763
11764 fn union_used_bounds(
11765 first: Option<(u32, u32)>,
11766 second: Option<(u32, u32)>,
11767 ) -> Option<(u32, u32)> {
11768 match (first, second) {
11769 (Some((a0, b0)), Some((a1, b1))) => Some((a0.min(a1), b0.max(b1))),
11770 (Some(bounds), None) | (None, Some(bounds)) => Some(bounds),
11771 (None, None) => None,
11772 }
11773 }
11774
11775 fn mirror_value_to_overlay(&mut self, sheet: &str, row: u32, col: u32, value: &LiteralValue) {
11778 if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
11779 return;
11780 }
11781 if self.arrow_sheets.sheet(sheet).is_none() {
11782 self.arrow_sheets
11783 .sheets
11784 .push(crate::arrow_store::ArrowSheet {
11785 name: std::sync::Arc::<str>::from(sheet),
11786 date_system: self.config.date_system,
11787 columns: Vec::new(),
11788 nrows: 0,
11789 chunk_starts: Vec::new(),
11790 chunk_rows: 32 * 1024,
11791 });
11792 }
11793
11794 let row0 = row.saturating_sub(1) as usize;
11795 let col0 = col.saturating_sub(1) as usize;
11796
11797 let asheet = self
11798 .arrow_sheets
11799 .sheet_mut(sheet)
11800 .expect("ArrowSheet must exist");
11801
11802 let cur_cols = asheet.columns.len();
11803 if col0 >= cur_cols {
11804 asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
11805 }
11806
11807 if row0 >= asheet.nrows as usize {
11808 if asheet.columns.is_empty() {
11809 asheet.insert_columns(0, 1);
11810 }
11811 asheet.ensure_row_capacity(row0 + 1);
11812 }
11813 if let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) {
11814 let ov =
11815 crate::arrow_store::OverlayValue::from_literal_value(value, asheet.date_system);
11816 let computed_delta = if let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) {
11817 let _ = ch.overlay.set(in_off, ov);
11818 let format = match value {
11819 LiteralValue::Date(_) => Some(crate::format::FormatId::DATE),
11820 LiteralValue::DateTime(_) => Some(crate::format::FormatId::DATETIME),
11821 LiteralValue::Time(_) => Some(crate::format::FormatId::TIME),
11822 LiteralValue::Duration(_) => Some(crate::format::FormatId::DURATION),
11823 _ => None,
11824 };
11825 ch.overlay.set_format(in_off, format);
11826 ch.computed_overlay.remove(in_off)
11831 } else {
11832 return;
11833 };
11834 let abs_threshold = 1024usize;
11836 let frac_den = 50usize;
11837 let freed = asheet.maybe_compact_chunk(col0, ch_idx, abs_threshold, frac_den);
11838 if freed > 0 {
11839 self.overlay_compactions = self.overlay_compactions.saturating_add(1);
11840 }
11841 self.adjust_computed_overlay_bytes(computed_delta);
11842 }
11843 }
11844
11845 fn clear_delta_overlay_cell(&mut self, sheet: &str, row: u32, col: u32) {
11850 if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
11851 return;
11852 }
11853 let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
11854 return;
11855 };
11856 let row0 = row.saturating_sub(1) as usize;
11857 let col0 = col.saturating_sub(1) as usize;
11858 if row0 >= asheet.nrows as usize {
11859 return;
11860 }
11861 if col0 >= asheet.columns.len() {
11862 return;
11863 }
11864 let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
11865 return;
11866 };
11867 if let Some(ch) = asheet.columns[col0].chunk_mut(ch_idx) {
11868 let _ = ch.overlay.remove(in_off);
11869 }
11870 }
11871
11872 fn clear_computed_overlay_after_row(&mut self, sheet: &str, start_row0: usize) {
11873 if !(self.config.arrow_storage_enabled && self.config.write_formula_overlay_enabled) {
11874 return;
11875 }
11876
11877 let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
11878 return;
11879 };
11880 if start_row0 >= asheet.nrows as usize {
11881 return;
11882 }
11883
11884 let starts = asheet.chunk_starts.clone();
11885 let nrows = asheet.nrows as usize;
11886 let mut delta = 0isize;
11887 for col in &mut asheet.columns {
11888 for (chunk_idx, ch) in col.chunks.iter_mut().enumerate() {
11889 let Some(&chunk_start) = starts.get(chunk_idx) else {
11890 continue;
11891 };
11892 let chunk_end = starts
11893 .get(chunk_idx + 1)
11894 .copied()
11895 .unwrap_or(nrows)
11896 .min(chunk_start.saturating_add(ch.len()));
11897 if chunk_end <= start_row0 {
11898 continue;
11899 }
11900 if chunk_start >= start_row0 {
11901 delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
11902 } else {
11903 let start_in_chunk = start_row0.saturating_sub(chunk_start).min(ch.len());
11904 delta = delta
11905 .saturating_add(ch.computed_overlay.remove_range(start_in_chunk..ch.len()));
11906 }
11907 }
11908
11909 for (chunk_idx, ch) in &mut col.sparse_chunks {
11910 let Some(&chunk_start) = starts.get(*chunk_idx) else {
11911 continue;
11912 };
11913 let chunk_end = starts
11914 .get(*chunk_idx + 1)
11915 .copied()
11916 .unwrap_or(nrows)
11917 .min(chunk_start.saturating_add(ch.len()));
11918 if chunk_end <= start_row0 {
11919 continue;
11920 }
11921 if chunk_start >= start_row0 {
11922 delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
11923 } else {
11924 let start_in_chunk = start_row0.saturating_sub(chunk_start).min(ch.len());
11925 delta = delta
11926 .saturating_add(ch.computed_overlay.remove_range(start_in_chunk..ch.len()));
11927 }
11928 }
11929 }
11930 self.adjust_computed_overlay_bytes(delta);
11931 }
11932
11933 fn clear_computed_overlay_after_col(&mut self, sheet: &str, start_col0: usize) {
11934 if !(self.config.arrow_storage_enabled && self.config.write_formula_overlay_enabled) {
11935 return;
11936 }
11937
11938 let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
11939 return;
11940 };
11941 if start_col0 >= asheet.columns.len() {
11942 return;
11943 }
11944
11945 let mut delta = 0isize;
11946 for col in asheet.columns.iter_mut().skip(start_col0) {
11947 for ch in &mut col.chunks {
11948 delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
11949 }
11950 for ch in col.sparse_chunks.values_mut() {
11951 delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
11952 }
11953 }
11954 self.adjust_computed_overlay_bytes(delta);
11955 }
11956
11957 #[inline]
11958 fn literal_to_overlay_value(
11959 value: &LiteralValue,
11960 date_system: crate::engine::DateSystem,
11961 ) -> crate::arrow_store::OverlayValue {
11962 crate::arrow_store::OverlayValue::from_literal_value(value, date_system)
11963 }
11964
11965 fn arrow_sheet_date_system(&self, sheet: &str) -> crate::engine::DateSystem {
11966 self.arrow_sheets
11967 .sheet(sheet)
11968 .map(|sheet| sheet.date_system)
11969 .unwrap_or(self.config.date_system)
11970 }
11971
11972 fn read_delta_overlay_cell(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
11975 if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
11976 return None;
11977 }
11978 let asheet = self.arrow_sheets.sheet(sheet)?;
11979 let row0 = row.saturating_sub(1) as usize;
11980 let col0 = col.saturating_sub(1) as usize;
11981 if row0 >= asheet.nrows as usize || col0 >= asheet.columns.len() {
11982 return None;
11983 }
11984 let (ch_idx, in_off) = asheet.chunk_of_row(row0)?;
11985 let ch = asheet.columns[col0].chunk(ch_idx)?;
11986 ch.overlay
11987 .get_scalar(in_off)
11988 .map(|ov| ov.to_literal_for(asheet.date_system))
11989 }
11990
11991 fn read_computed_overlay_cell(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
11994 if !(self.config.arrow_storage_enabled
11995 && self.config.delta_overlay_enabled
11996 && self.config.write_formula_overlay_enabled)
11997 {
11998 return None;
11999 }
12000 let asheet = self.arrow_sheets.sheet(sheet)?;
12001 let row0 = row.saturating_sub(1) as usize;
12002 let col0 = col.saturating_sub(1) as usize;
12003 if row0 >= asheet.nrows as usize || col0 >= asheet.columns.len() {
12004 return None;
12005 }
12006 let (ch_idx, in_off) = asheet.chunk_of_row(row0)?;
12007 let ch = asheet.columns[col0].chunk(ch_idx)?;
12008 ch.computed_overlay
12009 .get_scalar(in_off)
12010 .map(|ov| ov.to_literal_for(asheet.date_system))
12011 }
12012
12013 fn set_delta_overlay_cell_raw(
12014 &mut self,
12015 sheet: &str,
12016 row: u32,
12017 col: u32,
12018 value: Option<LiteralValue>,
12019 ) {
12020 if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
12021 return;
12022 }
12023
12024 self.ensure_arrow_sheet(sheet);
12025 let date_system = self.arrow_sheet_date_system(sheet);
12026 let ov_opt = value
12027 .as_ref()
12028 .map(|value| Self::literal_to_overlay_value(value, date_system));
12029 let row0 = row.saturating_sub(1) as usize;
12030 let col0 = col.saturating_sub(1) as usize;
12031 let asheet = self
12032 .arrow_sheets
12033 .sheet_mut(sheet)
12034 .expect("ArrowSheet must exist");
12035
12036 let cur_cols = asheet.columns.len();
12037 if col0 >= cur_cols {
12038 asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
12039 }
12040 if row0 >= asheet.nrows as usize {
12041 if asheet.columns.is_empty() {
12042 asheet.insert_columns(0, 1);
12043 }
12044 asheet.ensure_row_capacity(row0 + 1);
12045 }
12046
12047 let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
12048 return;
12049 };
12050 let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
12051 return;
12052 };
12053
12054 if let Some(ov) = ov_opt {
12055 let _ = ch.overlay.set(in_off, ov);
12056 } else {
12057 let _ = ch.overlay.remove(in_off);
12058 }
12059 }
12060
12061 fn set_computed_overlay_cell_raw(
12062 &mut self,
12063 sheet: &str,
12064 row: u32,
12065 col: u32,
12066 value: Option<LiteralValue>,
12067 ) {
12068 if !(self.config.arrow_storage_enabled
12069 && self.config.delta_overlay_enabled
12070 && self.config.write_formula_overlay_enabled)
12071 {
12072 return;
12073 }
12074
12075 self.ensure_arrow_sheet(sheet);
12076 let date_system = self.arrow_sheet_date_system(sheet);
12077 let ov_opt = value
12078 .as_ref()
12079 .map(|value| Self::literal_to_overlay_value(value, date_system));
12080 let row0 = row.saturating_sub(1) as usize;
12081 let col0 = col.saturating_sub(1) as usize;
12082 let asheet = self
12083 .arrow_sheets
12084 .sheet_mut(sheet)
12085 .expect("ArrowSheet must exist");
12086
12087 let cur_cols = asheet.columns.len();
12088 if col0 >= cur_cols {
12089 asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
12090 }
12091 if row0 >= asheet.nrows as usize {
12092 if asheet.columns.is_empty() {
12093 asheet.insert_columns(0, 1);
12094 }
12095 asheet.ensure_row_capacity(row0 + 1);
12096 }
12097
12098 let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
12099 return;
12100 };
12101 let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
12102 return;
12103 };
12104
12105 let delta = if let Some(ov) = ov_opt {
12106 ch.computed_overlay.set(in_off, ov)
12107 } else {
12108 ch.computed_overlay.remove(in_off)
12109 };
12110 self.adjust_computed_overlay_bytes(delta);
12111 }
12112
12113 fn apply_arrow_undo_batch(&mut self, batch: &crate::engine::ArrowUndoBatch, undo: bool) {
12114 use crate::engine::ArrowOp;
12115
12116 let iter: Box<dyn Iterator<Item = &ArrowOp>> = if undo {
12117 Box::new(batch.ops.iter().rev())
12118 } else {
12119 Box::new(batch.ops.iter())
12120 };
12121
12122 for op in iter {
12123 match op {
12124 ArrowOp::SetDeltaCell {
12125 sheet_id,
12126 row0,
12127 col0,
12128 old,
12129 new,
12130 } => {
12131 let sheet = self.graph.sheet_name(*sheet_id).to_string();
12132 let v = if undo { old.clone() } else { new.clone() };
12133 self.set_delta_overlay_cell_raw(&sheet, row0 + 1, col0 + 1, v);
12134 }
12135 ArrowOp::SetComputedCell {
12136 sheet_id,
12137 row0,
12138 col0,
12139 old,
12140 new,
12141 } => {
12142 let sheet = self.graph.sheet_name(*sheet_id).to_string();
12143 let v = if undo { old.clone() } else { new.clone() };
12144 self.set_computed_overlay_cell_raw(&sheet, row0 + 1, col0 + 1, v);
12145 }
12146 ArrowOp::RestoreComputedRect {
12147 sheet_id,
12148 sr0,
12149 sc0,
12150 er0,
12151 ec0,
12152 old,
12153 new,
12154 } => {
12155 let sheet = self.graph.sheet_name(*sheet_id).to_string();
12156 let vals = if undo { old } else { new };
12157 let height = (*er0).saturating_sub(*sr0) as usize + 1;
12158 let width = (*ec0).saturating_sub(*sc0) as usize + 1;
12159 for r in 0..height {
12160 for c in 0..width {
12161 let v = vals
12162 .get(r)
12163 .and_then(|row| row.get(c))
12164 .cloned()
12165 .unwrap_or(LiteralValue::Empty);
12166 self.set_computed_overlay_cell_raw(
12167 &sheet,
12168 *sr0 + 1 + r as u32,
12169 *sc0 + 1 + c as u32,
12170 Some(v),
12171 );
12172 }
12173 }
12174 }
12175 ArrowOp::InsertRows {
12176 sheet_id,
12177 before0,
12178 count,
12179 } => {
12180 let sheet = self.graph.sheet_name(*sheet_id).to_string();
12181 self.ensure_arrow_sheet(&sheet);
12182 if let Some(asheet) = self.arrow_sheets.sheet_mut(&sheet) {
12183 if undo {
12184 asheet.delete_rows(*before0 as usize, *count as usize);
12185 } else {
12186 asheet.insert_rows(*before0 as usize, *count as usize);
12187 }
12188 }
12189 self.purge_derived_formats_after_row(*sheet_id, *before0);
12190 }
12191 ArrowOp::InsertCols {
12192 sheet_id,
12193 before0,
12194 count,
12195 } => {
12196 let sheet = self.graph.sheet_name(*sheet_id).to_string();
12197 self.ensure_arrow_sheet(&sheet);
12198 if let Some(asheet) = self.arrow_sheets.sheet_mut(&sheet) {
12199 if undo {
12200 asheet.delete_columns(*before0 as usize, *count as usize);
12201 } else {
12202 asheet.insert_columns(*before0 as usize, *count as usize);
12203 }
12204 }
12205 self.purge_derived_formats_after_col(*sheet_id, *before0);
12206 }
12207 }
12208 }
12209 }
12210
12211 fn record_spill_ops_into_arrow_undo(
12212 &mut self,
12213 undo: &mut crate::engine::ArrowUndoBatch,
12214 events: &[crate::engine::ChangeEvent],
12215 ) {
12216 use crate::engine::ChangeEvent;
12217 use formualizer_common::LiteralValue;
12218
12219 #[allow(clippy::type_complexity)]
12220 let rect_from_snapshot =
12221 |snap: &crate::engine::graph::editor::change_log::SpillSnapshot|
12222 -> Option<(SheetId, u32, u32, u32, u32, Vec<Vec<LiteralValue>>)> {
12223 if snap.target_cells.is_empty() {
12224 return None;
12225 }
12226 let sheet_id = snap.target_cells[0].sheet_id;
12227 let sr0 = snap.target_cells[0].coord.row();
12228 let sc0 = snap.target_cells[0].coord.col();
12229 if snap.values.is_empty() || snap.values[0].is_empty() {
12230 return None;
12231 }
12232 let h = snap.values.len() as u32;
12233 let w = snap.values[0].len() as u32;
12234 let er0 = sr0.saturating_add(h.saturating_sub(1));
12235 let ec0 = sc0.saturating_add(w.saturating_sub(1));
12236 Some((sheet_id, sr0, sc0, er0, ec0, snap.values.clone()))
12237 };
12238
12239 for ev in events {
12240 match ev {
12241 ChangeEvent::SpillCommitted { old, new, .. } => {
12242 if let Some((sid, sr0, sc0, er0, ec0, new_vals)) = rect_from_snapshot(new) {
12243 let old_vals = if let Some(old_snap) = old {
12244 rect_from_snapshot(old_snap)
12245 .map(|(_, _, _, _, _, v)| v)
12246 .unwrap_or_else(|| {
12247 vec![
12248 vec![LiteralValue::Empty; new_vals[0].len()];
12249 new_vals.len()
12250 ]
12251 })
12252 } else {
12253 vec![vec![LiteralValue::Empty; new_vals[0].len()]; new_vals.len()]
12254 };
12255 undo.record_restore_computed_rect(
12256 sid, sr0, sc0, er0, ec0, old_vals, new_vals,
12257 );
12258 }
12259 }
12260 ChangeEvent::SpillCleared { old, .. } => {
12261 if let Some((sid, sr0, sc0, er0, ec0, old_vals)) = rect_from_snapshot(old) {
12262 let new_vals =
12263 vec![vec![LiteralValue::Empty; old_vals[0].len()]; old_vals.len()];
12264 undo.record_restore_computed_rect(
12265 sid, sr0, sc0, er0, ec0, old_vals, new_vals,
12266 );
12267 }
12268 }
12269 _ => {}
12270 }
12271 }
12272 }
12273
12274 fn mirror_value_to_computed_overlay(
12279 &mut self,
12280 sheet: &str,
12281 row: u32,
12282 col: u32,
12283 value: &LiteralValue,
12284 ) {
12285 if !(self.config.arrow_storage_enabled
12286 && self.config.delta_overlay_enabled
12287 && self.config.write_formula_overlay_enabled)
12288 {
12289 return;
12290 }
12291 if self.computed_overlay_mirroring_disabled {
12292 return;
12293 }
12294
12295 let date_system = self.arrow_sheet_date_system(sheet);
12296 let ov = Self::literal_to_overlay_value(value, date_system);
12297 self.write_computed_overlay_value_0based(
12298 sheet,
12299 row.saturating_sub(1),
12300 col.saturating_sub(1),
12301 ov,
12302 );
12303 }
12304
12305 fn record_derived_format(&self, vertex_id: VertexId, format: Option<crate::format::FormatId>) {
12306 if let Some(cell) = self.graph.get_cell_ref(vertex_id) {
12307 self.record_derived_format_at(cell, format);
12308 }
12309 }
12310
12311 fn record_derived_format_at(&self, cell: CellRef, format: Option<crate::format::FormatId>) {
12312 #[cfg(test)]
12313 self.derived_format_operations_for_test
12314 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
12315 let format = format.filter(|id| *id != crate::format::FormatId::GENERAL);
12316 self.derived_formats.set(cell, format);
12317 }
12318
12319 fn clear_cell_format_state(&mut self, sheet: &str, cell: CellRef) {
12320 self.derived_formats.set(cell, None);
12321 if let Some(arrow) = self.arrow_sheets.sheet_mut(sheet) {
12322 arrow.clear_format(cell.coord.row() as usize, cell.coord.col() as usize);
12323 }
12324 }
12325
12326 fn clear_logged_cell_format_states(&mut self, events: &[ChangeEvent]) {
12327 let cells = events
12328 .iter()
12329 .filter_map(|event| match event {
12330 ChangeEvent::SetValue { addr, .. } | ChangeEvent::SetFormula { addr, .. } => {
12331 Some(*addr)
12332 }
12333 _ => None,
12334 })
12335 .collect::<FxHashSet<_>>();
12336 for cell in cells {
12337 let sheet = self.graph.sheet_name(cell.sheet_id).to_string();
12338 self.clear_cell_format_state(&sheet, cell);
12339 }
12340 }
12341
12342 fn purge_derived_formats_after_row(&mut self, sheet_id: SheetId, start0: u32) {
12343 self.derived_formats
12344 .retain(|cell| cell.sheet_id != sheet_id || cell.coord.row() < start0);
12345 }
12346
12347 fn purge_derived_formats_after_col(&mut self, sheet_id: SheetId, start0: u32) {
12348 self.derived_formats
12349 .retain(|cell| cell.sheet_id != sheet_id || cell.coord.col() < start0);
12350 }
12351
12352 fn purge_derived_formats_for_sheet(&mut self, sheet_id: SheetId) {
12353 self.derived_formats
12354 .retain(|cell| cell.sheet_id != sheet_id);
12355 }
12356
12357 #[cfg(test)]
12358 pub(crate) fn debug_computed_overlay_format_0based(
12359 &self,
12360 sheet: &str,
12361 row0: u32,
12362 col0: u32,
12363 ) -> Option<crate::format::FormatId> {
12364 let sheet = self.arrow_sheets.sheet(sheet)?;
12365 let (chunk_idx, row_in_chunk) = sheet.chunk_of_row(row0 as usize)?;
12366 sheet
12367 .columns
12368 .get(col0 as usize)?
12369 .chunk(chunk_idx)?
12370 .computed_overlay
12371 .get_format(row_in_chunk)
12372 }
12373
12374 #[cfg(test)]
12375 pub(crate) fn debug_computed_overlay_chunk_has_formats_0based(
12376 &self,
12377 sheet: &str,
12378 row0: u32,
12379 col0: u32,
12380 ) -> bool {
12381 let Some(sheet) = self.arrow_sheets.sheet(sheet) else {
12382 return false;
12383 };
12384 let Some((chunk_idx, _)) = sheet.chunk_of_row(row0 as usize) else {
12385 return false;
12386 };
12387 sheet
12388 .columns
12389 .get(col0 as usize)
12390 .and_then(|column| column.chunk(chunk_idx))
12391 .is_some_and(|chunk| chunk.computed_overlay.has_formats())
12392 }
12393
12394 #[cfg(test)]
12395 pub(crate) fn debug_clear_derived_format_0based(&mut self, sheet: &str, row0: u32, col0: u32) {
12396 if let Some(sheet_id) = self.graph.sheet_id(sheet) {
12397 self.derived_formats
12398 .set(CellRef::new_absolute(sheet_id, row0, col0), None);
12399 }
12400 }
12401
12402 #[cfg(test)]
12403 pub(crate) fn debug_record_derived_format_0based(
12404 &self,
12405 sheet: &str,
12406 row0: u32,
12407 col0: u32,
12408 format: Option<crate::format::FormatId>,
12409 ) {
12410 if let Some(sheet_id) = self.graph.sheet_id(sheet) {
12411 self.record_derived_format_at(CellRef::new_absolute(sheet_id, row0, col0), format);
12412 }
12413 }
12414
12415 #[cfg(test)]
12416 pub(crate) fn debug_derived_format_0based(
12417 &self,
12418 sheet: &str,
12419 row0: u32,
12420 col0: u32,
12421 ) -> Option<crate::format::FormatId> {
12422 let sheet_id = self.graph.sheet_id(sheet)?;
12423 self.derived_formats
12424 .get(&CellRef::new_absolute(sheet_id, row0, col0))
12425 }
12426
12427 #[cfg(test)]
12428 pub(crate) fn debug_reset_format_write_operation_counts(&mut self) {
12429 self.derived_format_operations_for_test
12430 .store(0, std::sync::atomic::Ordering::Relaxed);
12431 self.computed_overlay_set_explicit_entry_operations_for_test = 0;
12432 self.computed_overlay_stale_clear_range_effects_for_test = 0;
12433 self.computed_overlay_stale_clear_offset_attempts_for_test = 0;
12434 self.computed_format_vector_allocations_for_test
12435 .store(0, std::sync::atomic::Ordering::Relaxed);
12436 }
12437
12438 #[cfg(test)]
12439 pub(crate) fn debug_format_write_operation_counts(&self) -> (u64, u64, u64, u64, u64) {
12440 (
12441 self.derived_format_operations_for_test
12442 .load(std::sync::atomic::Ordering::Relaxed),
12443 self.computed_overlay_set_explicit_entry_operations_for_test,
12444 self.computed_format_vector_allocations_for_test
12445 .load(std::sync::atomic::Ordering::Relaxed),
12446 self.computed_overlay_stale_clear_range_effects_for_test,
12447 self.computed_overlay_stale_clear_offset_attempts_for_test,
12448 )
12449 }
12450
12451 fn write_computed_overlay_format_0based(
12452 &mut self,
12453 sheet: &str,
12454 row0: u32,
12455 col0: u32,
12456 format: Option<crate::format::FormatId>,
12457 ) {
12458 self.ensure_arrow_sheet(sheet);
12459 let (row0, col0) = (row0 as usize, col0 as usize);
12460 let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
12461 return;
12462 };
12463 if col0 >= asheet.columns.len() {
12464 asheet.insert_columns(asheet.columns.len(), col0 + 1 - asheet.columns.len());
12465 }
12466 if row0 >= asheet.nrows as usize {
12467 asheet.ensure_row_capacity(row0 + 1);
12468 }
12469 let Some((chunk, offset)) = asheet.chunk_of_row(row0) else {
12470 return;
12471 };
12472 if let Some(chunk) = asheet.ensure_column_chunk_mut(col0, chunk) {
12473 chunk.computed_overlay.set_format(offset, format);
12474 }
12475 }
12476
12477 fn write_computed_cell_0based(
12482 &mut self,
12483 cell: CellRef,
12484 value: &LiteralValue,
12485 format: Option<crate::format::FormatId>,
12486 ) {
12487 if !(self.config.arrow_storage_enabled
12488 && self.config.delta_overlay_enabled
12489 && self.config.write_formula_overlay_enabled)
12490 || self.computed_overlay_mirroring_disabled
12491 {
12492 return;
12493 }
12494 if self.arrow_sheet_by_id(cell.sheet_id).is_none() {
12495 let sheet = self.graph.sheet_name(cell.sheet_id).to_string();
12496 self.ensure_arrow_sheet(&sheet);
12497 }
12498 let (row0, col0) = (cell.coord.row() as usize, cell.coord.col() as usize);
12499 let Some(asheet) = self.arrow_sheet_by_id_mut(cell.sheet_id) else {
12500 return;
12501 };
12502 let ov = Self::literal_to_overlay_value(value, asheet.date_system);
12503 let cur_cols = asheet.columns.len();
12504 if col0 >= cur_cols {
12505 asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
12506 }
12507 if row0 >= asheet.nrows as usize {
12508 asheet.ensure_row_capacity(row0 + 1);
12509 }
12510 let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
12511 return;
12512 };
12513 let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
12514 return;
12515 };
12516 let delta = ch.computed_overlay.set_scalar(in_off, ov);
12517 ch.computed_overlay.set_format(in_off, format);
12518 self.adjust_computed_overlay_bytes(delta);
12519 if let Some(cap) = self.config.max_overlay_memory_bytes
12520 && self.computed_overlay_bytes_estimate > cap
12521 {
12522 self.disable_computed_overlay_mirroring_due_to_budget(cap);
12523 }
12524 }
12525
12526 fn write_computed_overlay_value_0based(
12527 &mut self,
12528 sheet: &str,
12529 row0: u32,
12530 col0: u32,
12531 value: OverlayValue,
12532 ) {
12533 if !(self.config.arrow_storage_enabled
12534 && self.config.delta_overlay_enabled
12535 && self.config.write_formula_overlay_enabled)
12536 {
12537 return;
12538 }
12539 if self.computed_overlay_mirroring_disabled {
12540 return;
12541 }
12542
12543 self.ensure_arrow_sheet(sheet);
12544
12545 let row0 = row0 as usize;
12546 let col0 = col0 as usize;
12547 let asheet = self
12548 .arrow_sheets
12549 .sheet_mut(sheet)
12550 .expect("ArrowSheet must exist");
12551
12552 let cur_cols = asheet.columns.len();
12553 if col0 >= cur_cols {
12554 asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
12555 }
12556
12557 if row0 >= asheet.nrows as usize {
12558 if asheet.columns.is_empty() {
12559 asheet.insert_columns(0, 1);
12560 }
12561 asheet.ensure_row_capacity(row0 + 1);
12562 }
12563
12564 let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
12565 return;
12566 };
12567 let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
12568 return;
12569 };
12570
12571 let delta = ch.computed_overlay.set_scalar(in_off, value);
12572 self.adjust_computed_overlay_bytes(delta);
12573
12574 if let Some(cap) = self.config.max_overlay_memory_bytes
12575 && self.computed_overlay_bytes_estimate > cap
12576 {
12577 self.disable_computed_overlay_mirroring_due_to_budget(cap);
12578 }
12579 }
12580
12581 pub(crate) fn plan_computed_write_coalescing(
12582 &self,
12583 buffer: &ComputedWriteBuffer,
12584 ) -> ComputedWriteCoalescingPlan {
12585 self.plan_computed_write_coalescing_from_writes(
12586 buffer.writes().iter().cloned(),
12587 buffer.formats_present,
12588 )
12589 }
12590
12591 fn plan_owned_computed_write_coalescing(
12592 &self,
12593 writes: Vec<ComputedWrite>,
12594 formats_present: bool,
12595 ) -> ComputedWriteCoalescingPlan {
12596 self.plan_computed_write_coalescing_from_writes(writes, formats_present)
12597 }
12598
12599 fn plan_computed_write_coalescing_from_writes(
12600 &self,
12601 writes: impl IntoIterator<Item = ComputedWrite>,
12602 formats_present: bool,
12603 ) -> ComputedWriteCoalescingPlan {
12604 let mut groups: BTreeMap<ComputedWriteChunkKey, Vec<ComputedWriteChunkEntryPlan>> =
12605 BTreeMap::new();
12606 let mut input_cells = 0usize;
12607 let mut located: Option<(SheetId, Option<&crate::arrow_store::ArrowSheet>)> = None;
12609
12610 for write in writes {
12611 match write {
12612 ComputedWrite::Cell {
12613 seq,
12614 sheet_id,
12615 row0,
12616 col0,
12617 value,
12618 format_id,
12619 } => {
12620 input_cells = input_cells.saturating_add(1);
12621 let sheet = match located {
12622 Some((id, sheet)) if id == sheet_id => sheet,
12623 _ => {
12624 let sheet = self.arrow_sheets.sheet(self.graph.sheet_name(sheet_id));
12625 located = Some((sheet_id, sheet));
12626 sheet
12627 }
12628 };
12629 let (chunk_idx, chunk_start_row0, row_in_chunk) = match sheet {
12630 Some(sheet) => {
12631 Self::locate_row_in_sheet_for_computed_write_plan(sheet, row0 as usize)
12632 }
12633 None => Self::locate_row_in_empty_sheet_for_computed_write_plan(
12634 row0 as usize,
12635 32 * 1024,
12636 ),
12637 };
12638 groups
12639 .entry(ComputedWriteChunkKey {
12640 sheet_id,
12641 col0,
12642 chunk_idx,
12643 chunk_start_row0,
12644 })
12645 .or_default()
12646 .push(ComputedWriteChunkEntryPlan {
12647 row_in_chunk,
12648 seq,
12649 value,
12650 format_id,
12651 });
12652 }
12653 ComputedWrite::Run {
12654 seq,
12655 sheet_id,
12656 row0,
12657 col0,
12658 entries,
12659 } => {
12660 input_cells = input_cells.saturating_add(entries.len());
12661 let sheet = match located {
12662 Some((id, sheet)) if id == sheet_id => sheet,
12663 _ => {
12664 let sheet = self.arrow_sheets.sheet(self.graph.sheet_name(sheet_id));
12665 located = Some((sheet_id, sheet));
12666 sheet
12667 }
12668 };
12669 let mut segment: Vec<ComputedWriteChunkEntryPlan> = Vec::new();
12673 let mut segment_key: Option<ComputedWriteChunkKey> = None;
12674 for (k, (value, format_id)) in entries.into_iter().enumerate() {
12675 let row = row0.saturating_add(k as u32) as usize;
12676 let (chunk_idx, chunk_start_row0, row_in_chunk) = match sheet {
12677 Some(sheet) => {
12678 Self::locate_row_in_sheet_for_computed_write_plan(sheet, row)
12679 }
12680 None => Self::locate_row_in_empty_sheet_for_computed_write_plan(
12681 row,
12682 32 * 1024,
12683 ),
12684 };
12685 let key = ComputedWriteChunkKey {
12686 sheet_id,
12687 col0,
12688 chunk_idx,
12689 chunk_start_row0,
12690 };
12691 if segment_key != Some(key)
12692 && let Some(done) = segment_key.replace(key)
12693 {
12694 groups.entry(done).or_default().append(&mut segment);
12695 }
12696 segment.push(ComputedWriteChunkEntryPlan {
12697 row_in_chunk,
12698 seq,
12699 value,
12700 format_id,
12701 });
12702 }
12703 if let Some(done) = segment_key {
12704 groups.entry(done).or_default().append(&mut segment);
12705 }
12706 }
12707 ComputedWrite::Rect {
12708 seq,
12709 sheet_id,
12710 sr0,
12711 sc0,
12712 values,
12713 } => {
12714 for (r_off, row) in values.into_iter().enumerate() {
12715 for (c_off, value) in row.into_iter().enumerate() {
12716 input_cells = input_cells.saturating_add(1);
12717 self.push_computed_write_plan_entry(
12718 &mut groups,
12719 seq,
12720 sheet_id,
12721 sr0.saturating_add(r_off as u32),
12722 sc0.saturating_add(c_off as u32),
12723 value,
12724 None,
12725 );
12726 }
12727 }
12728 }
12729 }
12730 }
12731
12732 let mut plan = ComputedWriteCoalescingPlan {
12733 chunks: Vec::with_capacity(groups.len()),
12734 input_cells,
12735 coalesced_cells: 0,
12736 overwritten_cells: 0,
12737 };
12738 for (key, entries) in groups {
12739 let computed_lane_has_formats =
12740 self.computed_overlay_chunk_has_formats(key.sheet_id, key.col0, key.chunk_idx);
12741 let (chunk_plan, overwritten) = ComputedWriteChunkPlan::from_group(
12742 key,
12743 entries,
12744 formats_present,
12745 computed_lane_has_formats,
12746 );
12747 #[cfg(test)]
12748 if matches!(
12749 &chunk_plan.format_effect,
12750 ComputedWriteChunkFormatEffect::SetExplicit(_)
12751 ) {
12752 self.computed_format_vector_allocations_for_test
12753 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
12754 }
12755 plan.coalesced_cells = plan
12756 .coalesced_cells
12757 .saturating_add(chunk_plan.entries.len());
12758 plan.overwritten_cells = plan.overwritten_cells.saturating_add(overwritten);
12759 plan.chunks.push(chunk_plan);
12760 }
12761 debug_assert_eq!(
12762 plan.input_cells,
12763 plan.coalesced_cells.saturating_add(plan.overwritten_cells)
12764 );
12765 plan
12766 }
12767
12768 fn push_computed_write_plan_entry(
12769 &self,
12770 groups: &mut BTreeMap<ComputedWriteChunkKey, Vec<ComputedWriteChunkEntryPlan>>,
12771 seq: u64,
12772 sheet_id: SheetId,
12773 row0: u32,
12774 col0: u32,
12775 value: OverlayValue,
12776 format_id: Option<crate::format::FormatId>,
12777 ) {
12778 let (chunk_idx, chunk_start_row0, row_in_chunk) =
12779 self.locate_computed_write_chunk(sheet_id, row0);
12780 let key = ComputedWriteChunkKey {
12781 sheet_id,
12782 col0,
12783 chunk_idx,
12784 chunk_start_row0,
12785 };
12786 groups
12787 .entry(key)
12788 .or_default()
12789 .push(ComputedWriteChunkEntryPlan {
12790 row_in_chunk,
12791 seq,
12792 value,
12793 format_id,
12794 });
12795 }
12796
12797 fn computed_overlay_chunk_has_formats(
12798 &self,
12799 sheet_id: SheetId,
12800 col0: u32,
12801 chunk_idx: usize,
12802 ) -> bool {
12803 let sheet_name = self.graph.sheet_name(sheet_id);
12804 self.arrow_sheets
12805 .sheet(sheet_name)
12806 .and_then(|sheet| sheet.columns.get(col0 as usize))
12807 .and_then(|column| column.chunk(chunk_idx))
12808 .is_some_and(|chunk| chunk.computed_overlay.has_formats())
12809 }
12810
12811 fn locate_computed_write_chunk(&self, sheet_id: SheetId, row0: u32) -> (usize, u32, usize) {
12812 let sheet_name = self.graph.sheet_name(sheet_id);
12813 if let Some(sheet) = self.arrow_sheets.sheet(sheet_name) {
12814 return Self::locate_row_in_sheet_for_computed_write_plan(sheet, row0 as usize);
12815 }
12816 Self::locate_row_in_empty_sheet_for_computed_write_plan(row0 as usize, 32 * 1024)
12817 }
12818
12819 fn locate_row_in_sheet_for_computed_write_plan(
12820 sheet: &crate::arrow_store::ArrowSheet,
12821 row0: usize,
12822 ) -> (usize, u32, usize) {
12823 if row0 < sheet.nrows as usize
12824 && let Some((chunk_idx, row_in_chunk)) = sheet.chunk_of_row(row0)
12825 {
12826 let chunk_start = sheet.chunk_starts.get(chunk_idx).copied().unwrap_or(0);
12827 return (chunk_idx, chunk_start as u32, row_in_chunk);
12828 }
12829
12830 let chunk_rows = sheet.chunk_rows.max(1);
12831 if sheet.chunk_starts.is_empty() {
12832 return Self::locate_row_in_empty_sheet_for_computed_write_plan(row0, chunk_rows);
12833 }
12834
12835 let mut chunk_idx = sheet.chunk_starts.len().saturating_sub(1);
12836 let mut chunk_start = sheet.chunk_starts[chunk_idx];
12837 while chunk_start.saturating_add(chunk_rows) <= row0 {
12838 chunk_idx = chunk_idx.saturating_add(1);
12839 chunk_start = chunk_start.saturating_add(chunk_rows);
12840 }
12841 (
12842 chunk_idx,
12843 chunk_start as u32,
12844 row0.saturating_sub(chunk_start),
12845 )
12846 }
12847
12848 fn locate_row_in_empty_sheet_for_computed_write_plan(
12849 row0: usize,
12850 chunk_rows: usize,
12851 ) -> (usize, u32, usize) {
12852 let chunk_rows = chunk_rows.max(1);
12853 let chunk_idx = row0 / chunk_rows;
12854 let chunk_start = chunk_idx.saturating_mul(chunk_rows);
12855 (
12856 chunk_idx,
12857 chunk_start as u32,
12858 row0.saturating_sub(chunk_start),
12859 )
12860 }
12861
12862 #[cfg(test)]
12863 pub(crate) fn debug_plan_computed_write_coalescing(
12864 &self,
12865 buffer: &ComputedWriteBuffer,
12866 ) -> ComputedWriteCoalescingPlan {
12867 self.plan_computed_write_coalescing(buffer)
12868 }
12869
12870 pub(crate) fn flush_computed_write_buffer(
12871 &mut self,
12872 buffer: &mut ComputedWriteBuffer,
12873 ) -> Result<(), ExcelError> {
12874 if buffer.is_empty() {
12875 return Ok(());
12876 }
12877
12878 self.resource_checkpoint(0)?;
12883 let commit_started = self.preflight_evaluation_commit_window(buffer.len())?;
12884 let (writes, formats_present) = buffer.take_writes();
12885 let plan = self.plan_owned_computed_write_coalescing(writes, formats_present);
12886 self.flush_computed_write_plan(plan);
12887 self.observe_evaluation_commit_window(commit_started);
12888
12889 Ok(())
12890 }
12891
12892 fn flush_computed_write_plan(&mut self, plan: ComputedWriteCoalescingPlan) {
12893 for chunk in plan.chunks {
12894 self.flush_computed_write_chunk_plan(chunk);
12895 }
12896 }
12897
12898 fn flush_computed_write_chunk_plan(&mut self, chunk: ComputedWriteChunkPlan) {
12899 match &chunk.shape {
12900 ComputedWriteChunkPlanShape::Point => {
12901 self.flush_computed_write_chunk_plan_as_points(chunk);
12902 }
12903 ComputedWriteChunkPlanShape::SparseOffsets { .. } => {
12904 self.flush_computed_write_chunk_plan_as_sparse_fragment_or_points(chunk);
12905 }
12906 ComputedWriteChunkPlanShape::DenseRange { .. } => {
12907 self.flush_computed_write_chunk_plan_as_dense_fragment(chunk);
12908 }
12909 ComputedWriteChunkPlanShape::RunRange { len, runs, .. } => {
12910 if Self::should_emit_computed_run_fragment(*len, *runs) {
12911 self.flush_computed_write_chunk_plan_as_run_fragment(chunk);
12912 } else {
12913 self.flush_computed_write_chunk_plan_as_dense_fragment(chunk);
12914 }
12915 }
12916 }
12917 }
12918
12919 #[inline]
12920 fn should_emit_computed_run_fragment(len: usize, runs: usize) -> bool {
12921 runs <= len / 2
12922 }
12923
12924 fn flush_computed_write_chunk_plan_as_points(&mut self, chunk: ComputedWriteChunkPlan) {
12925 let sheet_name = self.graph.sheet_name(chunk.sheet_id).to_string();
12926 for entry in chunk.entries {
12927 let row0 = chunk
12928 .chunk_start_row0
12929 .saturating_add(entry.row_in_chunk as u32);
12930 self.write_computed_overlay_value_0based(&sheet_name, row0, chunk.col0, entry.value);
12931 }
12932 self.apply_computed_overlay_format_effect(
12933 chunk.sheet_id,
12934 chunk.col0,
12935 chunk.chunk_idx,
12936 chunk.format_effect,
12937 );
12938 }
12939
12940 fn flush_computed_write_chunk_plan_as_sparse_fragment_or_points(
12941 &mut self,
12942 chunk: ComputedWriteChunkPlan,
12943 ) {
12944 let point_estimate = Self::computed_write_chunk_plan_point_estimate(&chunk);
12945 let sheet_id = chunk.sheet_id;
12946 let col0 = chunk.col0;
12947 let chunk_idx = chunk.chunk_idx;
12948 let chunk_start_row0 = chunk.chunk_start_row0;
12949 let format_effect = chunk.format_effect;
12950 let items: Vec<(usize, OverlayValue)> = chunk
12951 .entries
12952 .into_iter()
12953 .map(|entry| (entry.row_in_chunk, entry.value))
12954 .collect();
12955 match OverlayFragment::sparse_offsets_if_estimated_smaller_than_points(
12956 items,
12957 point_estimate,
12958 ) {
12959 Some(Ok(fragment)) => {
12960 self.apply_computed_overlay_fragment(sheet_id, col0, chunk_idx, fragment);
12961 }
12962 Some(Err(cells)) => {
12963 self.flush_computed_overlay_cells_as_points(
12964 sheet_id,
12965 col0,
12966 chunk_start_row0,
12967 cells,
12968 );
12969 }
12970 None => {}
12971 }
12972 self.apply_computed_overlay_format_effect(sheet_id, col0, chunk_idx, format_effect);
12973 }
12974
12975 #[inline]
12976 fn computed_write_chunk_plan_point_estimate(chunk: &ComputedWriteChunkPlan) -> usize {
12977 chunk
12978 .entries
12979 .iter()
12980 .map(|entry| ComputedWriteBuffer::estimate_value_bytes(&entry.value))
12981 .fold(0usize, usize::saturating_add)
12982 }
12983
12984 fn flush_computed_overlay_cells_as_points(
12985 &mut self,
12986 sheet_id: SheetId,
12987 col0: u32,
12988 chunk_start_row0: u32,
12989 cells: Vec<(usize, OverlayValue)>,
12990 ) {
12991 let sheet_name = self.graph.sheet_name(sheet_id).to_string();
12992 for (row_in_chunk, value) in cells {
12993 let row0 = chunk_start_row0.saturating_add(row_in_chunk as u32);
12994 self.write_computed_overlay_value_0based(&sheet_name, row0, col0, value);
12995 }
12996 }
12997
12998 fn flush_computed_write_chunk_plan_as_dense_fragment(&mut self, chunk: ComputedWriteChunkPlan) {
12999 if chunk.entries.is_empty() {
13000 return;
13001 }
13002 let start = chunk.entries[0].row_in_chunk;
13003 let values: Vec<OverlayValue> =
13004 chunk.entries.into_iter().map(|entry| entry.value).collect();
13005 if let Some(fragment) = OverlayFragment::dense_range(start, values) {
13006 self.apply_computed_overlay_fragment(
13007 chunk.sheet_id,
13008 chunk.col0,
13009 chunk.chunk_idx,
13010 fragment,
13011 );
13012 }
13013 self.apply_computed_overlay_format_effect(
13014 chunk.sheet_id,
13015 chunk.col0,
13016 chunk.chunk_idx,
13017 chunk.format_effect,
13018 );
13019 }
13020
13021 fn flush_computed_write_chunk_plan_as_run_fragment(&mut self, chunk: ComputedWriteChunkPlan) {
13022 if chunk.entries.is_empty() {
13023 return;
13024 }
13025 let start = chunk.entries[0].row_in_chunk;
13026 let values: Vec<OverlayValue> =
13027 chunk.entries.into_iter().map(|entry| entry.value).collect();
13028 if let Some(fragment) = OverlayFragment::run_range(start, values) {
13029 self.apply_computed_overlay_fragment(
13030 chunk.sheet_id,
13031 chunk.col0,
13032 chunk.chunk_idx,
13033 fragment,
13034 );
13035 }
13036 self.apply_computed_overlay_format_effect(
13037 chunk.sheet_id,
13038 chunk.col0,
13039 chunk.chunk_idx,
13040 chunk.format_effect,
13041 );
13042 }
13043
13044 fn apply_computed_overlay_format_effect(
13045 &mut self,
13046 sheet_id: SheetId,
13047 col0: u32,
13048 chunk_idx: usize,
13049 effect: ComputedWriteChunkFormatEffect,
13050 ) {
13051 if !(self.config.arrow_storage_enabled
13052 && self.config.delta_overlay_enabled
13053 && self.config.write_formula_overlay_enabled)
13054 || self.computed_overlay_mirroring_disabled
13055 {
13056 return;
13057 }
13058
13059 let sheet_name = self.graph.sheet_name(sheet_id);
13060 let Some(sheet) = self.arrow_sheets.sheet_mut(sheet_name) else {
13061 return;
13062 };
13063 let Some(chunk) = sheet
13064 .columns
13065 .get_mut(col0 as usize)
13066 .and_then(|column| column.chunk_mut(chunk_idx))
13067 else {
13068 return;
13069 };
13070 match effect {
13071 ComputedWriteChunkFormatEffect::NoFormatWork => {}
13072 ComputedWriteChunkFormatEffect::ClearStale(ComputedWriteFormatClear::Range {
13073 start,
13074 end,
13075 }) => {
13076 #[cfg(test)]
13077 {
13078 self.computed_overlay_stale_clear_range_effects_for_test = self
13079 .computed_overlay_stale_clear_range_effects_for_test
13080 .saturating_add(1);
13081 }
13082 chunk.computed_overlay.clear_format_range(start, end);
13083 }
13084 ComputedWriteChunkFormatEffect::ClearStale(ComputedWriteFormatClear::Offsets(
13085 offsets,
13086 )) => {
13087 #[cfg(test)]
13088 {
13089 self.computed_overlay_stale_clear_offset_attempts_for_test = self
13090 .computed_overlay_stale_clear_offset_attempts_for_test
13091 .saturating_add(offsets.len() as u64);
13092 }
13093 chunk.computed_overlay.clear_format_offsets(&offsets);
13094 }
13095 ComputedWriteChunkFormatEffect::SetExplicit(formats) => {
13096 #[cfg(test)]
13097 {
13098 self.computed_overlay_set_explicit_entry_operations_for_test = self
13099 .computed_overlay_set_explicit_entry_operations_for_test
13100 .saturating_add(formats.len() as u64);
13101 }
13102 for (row_in_chunk, format_id) in formats {
13103 chunk.computed_overlay.set_format(row_in_chunk, format_id);
13104 }
13105 }
13106 }
13107 }
13108
13109 fn apply_computed_overlay_fragment(
13110 &mut self,
13111 sheet_id: SheetId,
13112 col0: u32,
13113 chunk_idx: usize,
13114 fragment: OverlayFragment,
13115 ) {
13116 if !(self.config.arrow_storage_enabled
13117 && self.config.delta_overlay_enabled
13118 && self.config.write_formula_overlay_enabled)
13119 {
13120 return;
13121 }
13122 if self.computed_overlay_mirroring_disabled {
13123 return;
13124 }
13125
13126 let sheet_name = self.graph.sheet_name(sheet_id).to_string();
13127 self.ensure_arrow_sheet(&sheet_name);
13128
13129 let col0 = col0 as usize;
13130 let asheet = self
13131 .arrow_sheets
13132 .sheet_mut(&sheet_name)
13133 .expect("ArrowSheet must exist");
13134
13135 let cur_cols = asheet.columns.len();
13136 if col0 >= cur_cols {
13137 asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
13138 }
13139
13140 let start_row0 = asheet
13141 .chunk_starts
13142 .get(chunk_idx)
13143 .copied()
13144 .unwrap_or_else(|| chunk_idx.saturating_mul(asheet.chunk_rows.max(1)));
13145 let required_rows =
13146 start_row0.saturating_add(fragment.max_covered_offset().saturating_add(1));
13147 if required_rows > asheet.nrows as usize {
13148 if asheet.columns.is_empty() {
13149 asheet.insert_columns(0, 1);
13150 }
13151 asheet.ensure_row_capacity(required_rows);
13152 }
13153
13154 let Some(ch) = asheet.ensure_column_chunk_mut(col0, chunk_idx) else {
13155 return;
13156 };
13157 let delta = ch.computed_overlay.apply_fragment(fragment);
13158 self.adjust_computed_overlay_bytes(delta);
13159
13160 if let Some(cap) = self.config.max_overlay_memory_bytes
13161 && self.computed_overlay_bytes_estimate > cap
13162 {
13163 self.disable_computed_overlay_mirroring_due_to_budget(cap);
13164 }
13165 }
13166
13167 #[inline]
13168 fn adjust_computed_overlay_bytes(&mut self, delta: isize) {
13169 if delta >= 0 {
13170 self.computed_overlay_bytes_estimate = self
13171 .computed_overlay_bytes_estimate
13172 .saturating_add(delta as usize);
13173 } else {
13174 self.computed_overlay_bytes_estimate = self
13175 .computed_overlay_bytes_estimate
13176 .saturating_sub((-delta) as usize);
13177 }
13178 }
13179
13180 fn clear_all_computed_overlays(&mut self) {
13181 let mut freed_total = 0usize;
13182 for sh in self.arrow_sheets.sheets.iter_mut() {
13183 for col in sh.columns.iter_mut() {
13184 for ch in col.chunks.iter_mut() {
13185 freed_total = freed_total.saturating_add(ch.computed_overlay.clear());
13186 }
13187 for ch in col.sparse_chunks.values_mut() {
13188 freed_total = freed_total.saturating_add(ch.computed_overlay.clear());
13189 }
13190 }
13191 }
13192 self.computed_overlay_bytes_estimate = self
13193 .computed_overlay_bytes_estimate
13194 .saturating_sub(freed_total);
13195 }
13196
13197 fn disable_computed_overlay_mirroring_due_to_budget(&mut self, _cap: usize) {
13198 self.compact_all_computed_overlays();
13201 }
13202
13203 fn compact_all_computed_overlays(&mut self) {
13206 let mut freed_total = 0usize;
13207 for sheet in self.arrow_sheets.sheets.iter_mut() {
13208 for col_idx in 0..sheet.columns.len() {
13209 let num_dense = sheet.columns[col_idx].chunks.len();
13211 for ch_idx in 0..num_dense {
13212 freed_total += sheet.compact_computed_overlay_chunk(col_idx, ch_idx);
13213 }
13214 let sparse_keys: Vec<usize> = sheet.columns[col_idx]
13216 .sparse_chunks
13217 .keys()
13218 .copied()
13219 .collect();
13220 for ch_idx in sparse_keys {
13221 freed_total += sheet.compact_computed_overlay_sparse_chunk(col_idx, ch_idx);
13222 }
13223 }
13224 }
13225 self.computed_overlay_bytes_estimate = self
13226 .computed_overlay_bytes_estimate
13227 .saturating_sub(freed_total);
13228 self.overlay_compactions = self.overlay_compactions.saturating_add(1);
13229 }
13230
13231 fn mirror_vertex_value_to_overlay(&mut self, vertex_id: VertexId, value: &LiteralValue) {
13232 let _ = self.record_vertex_value_to_overlay(vertex_id, value, None);
13233 }
13234
13235 fn record_vertex_value_to_overlay(
13236 &mut self,
13237 vertex_id: VertexId,
13238 value: &LiteralValue,
13239 computed_writes: Option<&mut ComputedWriteBuffer>,
13240 ) -> Result<(), ExcelError> {
13241 if !(self.config.arrow_storage_enabled
13242 && self.config.delta_overlay_enabled
13243 && self.config.write_formula_overlay_enabled)
13244 {
13245 return Ok(());
13246 }
13247 if self.computed_overlay_mirroring_disabled {
13248 return Ok(());
13249 }
13250 if !matches!(
13251 self.graph.get_vertex_kind(vertex_id),
13252 VertexKind::FormulaScalar | VertexKind::FormulaArray
13253 ) {
13254 return Ok(());
13255 }
13256 let Some(cell) = self.graph.get_cell_ref(vertex_id) else {
13257 return Ok(());
13258 };
13259 let Some(buffer) = computed_writes else {
13260 let format_id = self.derived_formats.get(&cell);
13262 self.write_computed_cell_0based(cell, value, format_id);
13263 return Ok(());
13264 };
13265 let date_system = self
13266 .arrow_sheet_by_id(cell.sheet_id)
13267 .map(|sheet| sheet.date_system)
13268 .unwrap_or(self.config.date_system);
13269 let ov = Self::literal_to_overlay_value(value, date_system);
13270 {
13271 let format_id = self.derived_formats.get(&cell);
13272 buffer.push_cell_with_format(
13273 cell.sheet_id,
13274 cell.coord.row(),
13275 cell.coord.col(),
13276 ov,
13277 format_id,
13278 );
13279 if self.should_flush_computed_write_buffer(buffer) {
13280 self.flush_computed_write_buffer(buffer)?;
13281 }
13282 }
13283 Ok(())
13284 }
13285
13286 #[inline]
13287 fn should_flush_computed_write_buffer(&self, buffer: &ComputedWriteBuffer) -> bool {
13288 self.config.max_overlay_memory_bytes.is_some_and(|cap| {
13289 if cap == 0 {
13290 return false;
13291 }
13292 self.computed_overlay_bytes_estimate
13293 .saturating_add(buffer.estimated_bytes())
13294 > cap
13295 })
13296 }
13297
13298 pub fn overlay_memory_usage(&self) -> usize {
13300 self.computed_overlay_bytes_estimate
13301 }
13302
13303 #[cfg(test)]
13304 pub(crate) fn debug_overlay_compactions(&self) -> u64 {
13305 self.overlay_compactions
13306 }
13307
13308 #[cfg(test)]
13309 pub(crate) fn debug_recompute_computed_overlay_bytes(&mut self) -> usize {
13310 let mut total = 0usize;
13311 for sheet in &self.arrow_sheets.sheets {
13312 for column in &sheet.columns {
13313 for chunk in &column.chunks {
13314 total = total.saturating_add(chunk.computed_overlay.estimated_bytes());
13315 }
13316 for chunk in column.sparse_chunks.values() {
13317 total = total.saturating_add(chunk.computed_overlay.estimated_bytes());
13318 }
13319 }
13320 }
13321 self.computed_overlay_bytes_estimate = total;
13322 total
13323 }
13324
13325 fn resolve_sheet_locator_for_write(
13326 &mut self,
13327 loc: formualizer_common::SheetLocator<'_>,
13328 current_sheet: &str,
13329 ) -> Result<SheetId, ExcelError> {
13330 Ok(match loc {
13331 formualizer_common::SheetLocator::Id(id) => id,
13332 formualizer_common::SheetLocator::Name(name) => self.graph.sheet_id_mut(name.as_ref()),
13333 formualizer_common::SheetLocator::Current => self.graph.sheet_id_mut(current_sheet),
13334 })
13335 }
13336
13337 fn resolve_sheet_locator_for_read(
13338 &self,
13339 loc: formualizer_common::SheetLocator<'_>,
13340 current_sheet: &str,
13341 ) -> Result<SheetId, ExcelError> {
13342 match loc {
13343 formualizer_common::SheetLocator::Id(id) => Ok(id),
13344 formualizer_common::SheetLocator::Name(name) => self
13345 .graph
13346 .sheet_id(name.as_ref())
13347 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref)),
13348 formualizer_common::SheetLocator::Current => self
13349 .graph
13350 .sheet_id(current_sheet)
13351 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref)),
13352 }
13353 }
13354
13355 pub fn set_cell_value(
13357 &mut self,
13358 sheet: &str,
13359 row: u32,
13360 col: u32,
13361 value: LiteralValue,
13362 ) -> Result<(), ExcelError> {
13363 self.observe_function_semantic_epoch()?;
13364 let sheet_existed = self.graph.sheet_id(sheet).is_some();
13365 let sheet_id = self.graph.sheet_id_mut(sheet);
13366 let cell_ref = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13367 let replaced = self
13368 .graph
13369 .get_vertex_id_for_address(&cell_ref)
13370 .filter(|&vertex| {
13371 matches!(
13372 self.graph.get_vertex_kind(vertex),
13373 VertexKind::FormulaScalar | VertexKind::FormulaArray
13374 )
13375 });
13376 let replaced_formula = replaced.is_some();
13377 if let Some(anchor) = replaced
13381 && self.graph.spill_registry_has_anchor(anchor)
13382 {
13383 self.clear_spill_projection_and_mirror(anchor, None);
13384 }
13385 self.graph.set_cell_value(sheet, row, col, value.clone())?;
13386 self.clear_cell_format_state(sheet, cell_ref);
13387 self.record_changed_cell(sheet, row, col);
13388 if !sheet_existed || replaced_formula {
13389 self.mark_topology_edited();
13390 }
13391 self.mirror_value_to_overlay(sheet, row, col, &value);
13393 self.mark_data_edited();
13395 Ok(())
13396 }
13397
13398 fn record_changed_cell(&mut self, sheet: &str, row: u32, col: u32) {
13400 let sheet_id = self.graph.sheet_id_mut(sheet);
13401 self.record_structural_change(StructuralScope::Cell {
13402 sheet: sheet_id,
13403 row: row.saturating_sub(1),
13404 col: col.saturating_sub(1),
13405 });
13406 }
13407
13408 fn record_change_for_event(&mut self, event: &ChangeEvent) {
13409 match event {
13410 ChangeEvent::SetValue { addr, .. } | ChangeEvent::SetFormula { addr, .. } => {
13411 self.record_structural_change(StructuralScope::Cell {
13412 sheet: addr.sheet_id,
13413 row: addr.coord.row(),
13414 col: addr.coord.col(),
13415 });
13416 }
13417 ChangeEvent::SpillCommitted { new, .. } => {
13418 if let Some(scope) = Self::structural_scope_from_cells(&new.target_cells) {
13419 self.record_structural_change(scope);
13420 }
13421 }
13422 ChangeEvent::SpillCleared { old, .. } => {
13423 if let Some(scope) = Self::structural_scope_from_cells(&old.target_cells) {
13424 self.record_structural_change(scope);
13425 }
13426 }
13427 ChangeEvent::DefineName { .. }
13428 | ChangeEvent::UpdateName { .. }
13429 | ChangeEvent::DeleteName { .. }
13430 | ChangeEvent::NamedRangeAdjusted { .. } => {
13431 self.record_structural_change(StructuralScope::AllSheets);
13435 }
13436 ChangeEvent::VertexMoved { .. } | ChangeEvent::FormulaAdjusted { .. } => {
13437 }
13440 ChangeEvent::SetRowVisibility { sheet_id, row0, .. } => {
13441 self.record_structural_change(StructuralScope::Region(Region::whole_row(
13442 *sheet_id, *row0,
13443 )));
13444 }
13445 ChangeEvent::AddVertex { .. }
13446 | ChangeEvent::RemoveVertex { .. }
13447 | ChangeEvent::EdgeAdded { .. }
13448 | ChangeEvent::EdgeRemoved { .. }
13449 | ChangeEvent::CompoundStart { .. }
13450 | ChangeEvent::CompoundEnd { .. }
13451 | ChangeEvent::StagedFormulaCellChanged { .. } => {}
13452 }
13453 }
13454
13455 fn record_structural_change(&mut self, scope: StructuralScope) {
13456 if let StructuralScope::Cell { sheet, row, col } = scope {
13457 self.wake_spill_owner_of_entered_formula(sheet, row, col);
13458 }
13459 self.wake_spill_blocker_waiters(scope);
13460 self.invalidate_pending_spills(scope);
13461 }
13462
13463 fn wake_spill_blocker_waiters(&mut self, scope: StructuralScope) {
13467 if self.spill_blocker_waiters.is_empty() {
13468 return;
13469 }
13470 let woken = self.spill_blocker_waiters.take_affected(scope);
13471 let woken: Vec<VertexId> = woken
13472 .into_iter()
13473 .filter(|&vertex| {
13474 self.graph.vertex_exists(vertex)
13475 && matches!(
13476 self.graph.get_vertex_kind(vertex),
13477 VertexKind::FormulaScalar | VertexKind::FormulaArray
13478 )
13479 })
13480 .collect();
13481 if !woken.is_empty() {
13482 self.graph.mark_dirty_many(&woken);
13483 }
13484 }
13485
13486 fn wake_spill_owner_of_entered_formula(&mut self, sheet: SheetId, row: u32, col: u32) {
13490 let cell = self.graph.make_cell_ref_internal(sheet, row, col);
13491 if let Some(owner) = self.graph.spill_registry_anchor_for_cell(cell)
13492 && self.graph.is_foreign_formula_cell(&cell, owner)
13493 {
13494 self.graph.mark_dirty_many(&[owner]);
13495 }
13496 }
13497
13498 fn structural_scope_from_cells(cells: &[CellRef]) -> Option<StructuralScope> {
13499 let first = cells.first()?;
13500 let sheet_id = first.sheet_id;
13501 if cells.iter().any(|cell| cell.sheet_id != sheet_id) {
13502 return Some(StructuralScope::OpaqueGlobal);
13503 }
13504 let mut row_start = first.coord.row();
13505 let mut row_end = row_start;
13506 let mut col_start = first.coord.col();
13507 let mut col_end = col_start;
13508 for cell in cells.iter().skip(1) {
13509 row_start = row_start.min(cell.coord.row());
13510 row_end = row_end.max(cell.coord.row());
13511 col_start = col_start.min(cell.coord.col());
13512 col_end = col_end.max(cell.coord.col());
13513 }
13514 Some(StructuralScope::Region(Region::rect(
13515 sheet_id, row_start, row_end, col_start, col_end,
13516 )))
13517 }
13518
13519 pub fn set_cell_value_ref(
13520 &mut self,
13521 cell: formualizer_common::SheetCellRef<'_>,
13522 current_sheet: &str,
13523 value: LiteralValue,
13524 ) -> Result<(), ExcelError> {
13525 let owned = cell.into_owned();
13526 let sheet_id = self.resolve_sheet_locator_for_write(owned.sheet, current_sheet)?;
13527 let sheet_name = self.graph.sheet_name(sheet_id).to_string();
13528 self.set_cell_value(
13529 &sheet_name,
13530 owned.coord.row() + 1,
13531 owned.coord.col() + 1,
13532 value,
13533 )
13534 }
13535
13536 pub fn set_cell_formula_ref(
13537 &mut self,
13538 cell: formualizer_common::SheetCellRef<'_>,
13539 current_sheet: &str,
13540 ast: ASTNode,
13541 ) -> Result<(), ExcelError> {
13542 let owned = cell.into_owned();
13543 let sheet_id = self.resolve_sheet_locator_for_write(owned.sheet, current_sheet)?;
13544 let sheet_name = self.graph.sheet_name(sheet_id).to_string();
13545 self.set_cell_formula(
13546 &sheet_name,
13547 owned.coord.row() + 1,
13548 owned.coord.col() + 1,
13549 ast,
13550 )
13551 }
13552
13553 pub fn get_cell_value_ref(
13554 &self,
13555 cell: formualizer_common::SheetCellRef<'_>,
13556 current_sheet: &str,
13557 ) -> Result<Option<LiteralValue>, ExcelError> {
13558 let owned = cell.into_owned();
13559 let sheet_id = self.resolve_sheet_locator_for_read(owned.sheet, current_sheet)?;
13560 let sheet_name = self.graph.sheet_name(sheet_id);
13561 Ok(self.get_cell_value(sheet_name, owned.coord.row() + 1, owned.coord.col() + 1))
13562 }
13563
13564 pub fn resolve_range_view_sheet_ref<'c>(
13565 &'c self,
13566 r: &formualizer_common::SheetRef<'_>,
13567 current_sheet: &str,
13568 ) -> Result<RangeView<'c>, ExcelError> {
13569 use formualizer_common::SheetLocator;
13570
13571 let sheet_to_opt_name = |loc: SheetLocator<'_>| -> Result<Option<String>, ExcelError> {
13572 match loc {
13573 SheetLocator::Current => Ok(None),
13574 SheetLocator::Name(name) => Ok(Some(name.as_ref().to_string())),
13575 SheetLocator::Id(id) => Ok(Some(self.graph.sheet_name(id).to_string())),
13576 }
13577 };
13578
13579 let rt = match r {
13580 formualizer_common::SheetRef::Cell(cell) => ReferenceType::Cell {
13581 sheet: sheet_to_opt_name(cell.sheet.clone())?,
13582 row: cell.coord.row() + 1,
13583 col: cell.coord.col() + 1,
13584 row_abs: cell.coord.row_abs(),
13585 col_abs: cell.coord.col_abs(),
13586 },
13587 formualizer_common::SheetRef::Range(range) => ReferenceType::Range {
13588 sheet: sheet_to_opt_name(range.sheet.clone())?,
13589 start_row: range.start_row.map(|b| b.index + 1),
13590 start_col: range.start_col.map(|b| b.index + 1),
13591 end_row: range.end_row.map(|b| b.index + 1),
13592 end_col: range.end_col.map(|b| b.index + 1),
13593 start_row_abs: range.start_row.map(|b| b.abs).unwrap_or(false),
13594 start_col_abs: range.start_col.map(|b| b.abs).unwrap_or(false),
13595 end_row_abs: range.end_row.map(|b| b.abs).unwrap_or(false),
13596 end_col_abs: range.end_col.map(|b| b.abs).unwrap_or(false),
13597 },
13598 };
13599
13600 crate::traits::EvaluationContext::resolve_range_view(self, &rt, current_sheet)
13601 }
13602
13603 pub fn set_cell_formula(
13605 &mut self,
13606 sheet: &str,
13607 row: u32,
13608 col: u32,
13609 ast: ASTNode,
13610 ) -> Result<(), ExcelError> {
13611 self.observe_function_semantic_epoch()?;
13612 let sheet_id = self.graph.sheet_id_mut(sheet);
13613 let placement = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13614 let ingested = {
13615 let mut pipeline = self.ingest_pipeline();
13616 pipeline.ingest_formula(FormulaAstInput::Tree(ast), placement, None)?
13617 };
13618 self.graph.set_cell_formula_with_plan(
13619 sheet,
13620 row,
13621 col,
13622 ingested.ast_id,
13623 &ingested.dep_plan,
13624 ingested.dep_plan.volatile,
13625 ingested.dep_plan.dynamic,
13626 )?;
13627 self.clear_cell_format_state(sheet, placement);
13628 self.record_changed_cell(sheet, row, col);
13629
13630 self.clear_delta_overlay_cell(sheet, row, col);
13635
13636 self.mark_topology_edited();
13638 Ok(())
13639 }
13640
13641 pub fn bulk_set_formulas<I>(&mut self, sheet: &str, items: I) -> Result<usize, ExcelError>
13643 where
13644 I: IntoIterator<Item = (u32, u32, ASTNode)>,
13645 {
13646 let collected: Vec<(u32, u32, ASTNode)> = items.into_iter().collect();
13647 let edited_cells: Vec<(u32, u32)> = collected.iter().map(|(r, c, _)| (*r, *c)).collect();
13648 let sheet_id = self.graph.sheet_id_mut(sheet);
13649 let ingested = {
13650 let mut pipeline = self.ingest_pipeline();
13651 let inputs = collected.into_iter().map(|(row, col, ast)| {
13652 let placement = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13653 (FormulaAstInput::Tree(ast), placement, None)
13654 });
13655 pipeline.ingest_batch(inputs)?
13656 };
13657 let planned: Vec<(u32, u32, AstNodeId, DependencyPlanRow)> = ingested
13658 .into_iter()
13659 .map(|formula| {
13660 (
13661 formula.placement.coord.row() + 1,
13662 formula.placement.coord.col() + 1,
13663 formula.ast_id,
13664 formula.dep_plan,
13665 )
13666 })
13667 .collect();
13668 let n = self.graph.bulk_set_formulas_with_plans(sheet, planned)?;
13669 for (row, col) in edited_cells {
13670 let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13671 self.clear_cell_format_state(sheet, cell);
13672 self.record_changed_cell(sheet, row, col);
13673 }
13674 if n > 0 {
13676 self.mark_topology_edited();
13677 }
13678 Ok(n)
13679 }
13680
13681 pub fn declare_dynamic_array_anchor(
13710 &mut self,
13711 sheet: &str,
13712 row: u32,
13713 col: u32,
13714 ) -> Result<(), ExcelError> {
13715 let not_a_formula = || {
13716 ExcelError::new(ExcelErrorKind::Ref)
13717 .with_message("Declared dynamic array anchor must be a formula cell")
13718 };
13719 if row == 0 || col == 0 {
13720 return Err(not_a_formula());
13721 }
13722 if self.get_staged_formula_text(sheet, row, col).is_some() {
13723 return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
13724 "Declared dynamic array anchor is still staged; build the graph first",
13725 ));
13726 }
13727 let sheet_id = self.graph.sheet_id(sheet).ok_or_else(not_a_formula)?;
13728 let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13729 let vertex = self
13730 .graph
13731 .cell_vertex_mut(&cell)
13732 .filter(|v| self.graph.vertex_has_formula(*v))
13733 .ok_or_else(not_a_formula)?;
13734 if self.graph.declare_dynamic_anchor(vertex, cell) {
13735 self.graph.mark_dirty_many(&[vertex]);
13736 self.mark_topology_edited();
13737 }
13738 Ok(())
13739 }
13740
13741 pub fn declare_fixed_array_formula(
13747 &mut self,
13748 sheet: &str,
13749 row: u32,
13750 col: u32,
13751 rows: u32,
13752 cols: u32,
13753 ) -> Result<(), ExcelError> {
13754 if self.config.family_execution {
13755 return Err(ExcelError::new(ExcelErrorKind::Ref)
13756 .with_message("Fixed arrays require family_execution = false"));
13757 }
13758 if row == 0
13759 || col == 0
13760 || rows == 0
13761 || cols == 0
13762 || row.checked_add(rows - 1).is_none_or(|r| r > 1_048_576)
13763 || col.checked_add(cols - 1).is_none_or(|c| c > 16_384)
13764 || u64::from(rows) * u64::from(cols) > u64::from(self.config.spill.max_spill_cells)
13765 {
13766 return Err(ExcelError::new(ExcelErrorKind::Ref)
13767 .with_message("Invalid or over-cap fixed array extent"));
13768 }
13769 if self.get_staged_formula_text(sheet, row, col).is_some() {
13770 return Err(ExcelError::new(ExcelErrorKind::Ref)
13771 .with_message("Fixed array formula is still staged; build the graph first"));
13772 }
13773 let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
13774 ExcelError::new(ExcelErrorKind::Ref).with_message("Unknown fixed array sheet")
13775 })?;
13776 let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13777 let vertex = self
13778 .graph
13779 .cell_vertex_mut(&cell)
13780 .filter(|v| self.graph.vertex_has_formula(*v))
13781 .ok_or_else(|| {
13782 ExcelError::new(ExcelErrorKind::Ref)
13783 .with_message("Fixed array anchor must be a formula cell")
13784 })?;
13785 if (rows == 1 && cols == 1 && self.graph.fixed_single_arrays.contains(&vertex))
13786 || self.graph.fixed_array_shapes.get(&vertex) == Some(&(rows, cols))
13787 {
13788 return Ok(());
13789 }
13790 if self.graph.fixed_single_arrays.contains(&vertex)
13791 || self.graph.spill_registry_has_anchor(vertex)
13792 || self.graph.is_current_declared_dynamic_anchor(vertex)
13793 {
13794 return Err(ExcelError::new(ExcelErrorKind::Ref)
13795 .with_message("Array declaration already exists; replace the formula first"));
13796 }
13797 if rows == 1 && cols == 1 {
13798 self.graph.fixed_single_arrays.insert(vertex);
13799 } else {
13800 let mut targets = Vec::with_capacity((rows * cols) as usize);
13801 for r in 0..rows {
13802 for c in 0..cols {
13803 targets.push(self.graph.make_cell_ref(sheet, row + r, col + c));
13804 }
13805 }
13806 self.graph.plan_spill_region(vertex, &targets)?;
13807 self.graph
13809 .commit_spill_region_atomic_with_fault(vertex, targets, vec![], None)?;
13810 self.graph.fixed_array_shapes.insert(vertex, (rows, cols));
13811 }
13812 self.graph.mark_dirty_many(&[vertex]);
13813 self.mark_topology_edited();
13814 Ok(())
13815 }
13816
13817 #[inline]
13818 fn normalize_public_cell_read(v: LiteralValue) -> Option<LiteralValue> {
13819 match v {
13820 LiteralValue::Empty => None,
13821 LiteralValue::Int(i) => Some(LiteralValue::Number(i as f64)),
13822 other => Some(other),
13823 }
13824 }
13825
13826 fn materialize_temporal_egress(
13827 value: LiteralValue,
13828 class: Option<&formualizer_common::numfmt::FormatClass>,
13829 policy: crate::engine::TemporalEgress,
13830 date_system: crate::engine::DateSystem,
13831 ) -> LiteralValue {
13832 use formualizer_common::numfmt::FormatClass;
13833 if policy == crate::engine::TemporalEgress::Serial {
13834 return value;
13835 }
13836 let LiteralValue::Number(serial) = value else {
13837 return value;
13838 };
13839 match class {
13840 Some(FormatClass::Date) if serial.fract() != 0.0 => {
13841 formualizer_common::try_serial_to_datetime_for(date_system, serial)
13842 .map(LiteralValue::DateTime)
13843 .unwrap_or(LiteralValue::Number(serial))
13844 }
13845 Some(FormatClass::Date) => {
13846 formualizer_common::try_serial_to_date_for(date_system, serial)
13847 .map(LiteralValue::Date)
13848 .unwrap_or(LiteralValue::Number(serial))
13849 }
13850 Some(FormatClass::DateTime) => {
13851 formualizer_common::try_serial_to_datetime_for(date_system, serial)
13852 .map(LiteralValue::DateTime)
13853 .unwrap_or(LiteralValue::Number(serial))
13854 }
13855 Some(FormatClass::Time) if (0.0..1.0).contains(&serial) => {
13856 let nanos = (serial * 86_400_000_000_000.0).round() as u64;
13857 let nanos = nanos.min(86_400_000_000_000 - 1);
13858 chrono::NaiveTime::from_num_seconds_from_midnight_opt(
13859 (nanos / 1_000_000_000) as u32,
13860 (nanos % 1_000_000_000) as u32,
13861 )
13862 .map(LiteralValue::Time)
13863 .unwrap_or(LiteralValue::Number(serial))
13864 }
13865 Some(FormatClass::Time | FormatClass::Duration) => {
13866 let nanos = (serial * 86_400.0 * 1_000_000_000.0).round();
13867 if nanos.is_finite() && nanos >= i64::MIN as f64 && nanos <= i64::MAX as f64 {
13868 LiteralValue::Duration(chrono::Duration::nanoseconds(nanos as i64))
13869 } else {
13870 LiteralValue::Number(serial)
13871 }
13872 }
13873 _ => LiteralValue::Number(serial),
13874 }
13875 }
13876
13877 pub(crate) fn effective_format_id(
13878 &self,
13879 sheet: &str,
13880 row: u32,
13881 col: u32,
13882 ) -> Option<crate::format::FormatId> {
13883 let arrow = self.arrow_sheets.sheet(sheet).and_then(|arrow| {
13884 arrow.format_id(
13885 row.saturating_sub(1) as usize,
13886 col.saturating_sub(1) as usize,
13887 )
13888 });
13889 arrow.or_else(|| {
13890 let sheet_id = self.graph.sheet_id(sheet)?;
13891 let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13892 self.derived_formats.get(&cell)
13893 })
13894 }
13895
13896 pub fn get_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
13898 let raw = self.read_cell_value(sheet, row, col)?;
13899 let format = self.effective_format_id(sheet, row, col);
13900 let class = format.and_then(|id| self.format_registry.class(id));
13901 Self::normalize_public_cell_read(Self::materialize_temporal_egress(
13902 raw,
13903 class,
13904 self.config.temporal_egress,
13905 self.config.date_system,
13906 ))
13907 }
13908
13909 pub fn get_range_values(
13911 &self,
13912 sheet: &str,
13913 sr: u32,
13914 sc: u32,
13915 er: u32,
13916 ec: u32,
13917 ) -> Vec<Vec<LiteralValue>> {
13918 let height = er.saturating_sub(sr).saturating_add(1) as usize;
13919 let width = ec.saturating_sub(sc).saturating_add(1) as usize;
13920 let Some(asheet) = self.sheet_store().sheet(sheet) else {
13921 return vec![vec![LiteralValue::Empty; width]; height];
13922 };
13923 let view = asheet.range_view(
13924 sr.saturating_sub(1) as usize,
13925 sc.saturating_sub(1) as usize,
13926 er.saturating_sub(1) as usize,
13927 ec.saturating_sub(1) as usize,
13928 );
13929 let sheet_id = self.graph.sheet_id(sheet);
13930 let derived_formats = &self.derived_formats;
13931 let has_derived_formats =
13932 sheet_id.is_some_and(|sheet_id| derived_formats.any(|cell| cell.sheet_id == sheet_id));
13933 let mut out = Vec::with_capacity(height);
13934 if !asheet.has_formats() && !has_derived_formats {
13935 for rr in 0..height {
13936 let mut row = Vec::with_capacity(width);
13937 for cc in 0..width {
13938 row.push(view.get_cell(rr, cc));
13939 }
13940 out.push(row);
13941 }
13942 return out;
13943 }
13944 let format_registry = &self.format_registry;
13945 for rr in 0..height {
13946 let mut row = Vec::with_capacity(width);
13947 for cc in 0..width {
13948 let raw = view.get_cell(rr, cc);
13949 let row0 = sr.saturating_sub(1).saturating_add(rr as u32);
13950 let col0 = sc.saturating_sub(1).saturating_add(cc as u32);
13951 let format = asheet.format_id(row0 as usize, col0 as usize).or_else(|| {
13952 let cell = CellRef::new(sheet_id?, Coord::new(row0, col0, true, true));
13953 derived_formats.get(&cell)
13954 });
13955 let class = format.and_then(|id| format_registry.class(id));
13956 row.push(Self::materialize_temporal_egress(
13957 raw,
13958 class,
13959 self.config.temporal_egress,
13960 self.config.date_system,
13961 ));
13962 }
13963 out.push(row);
13964 }
13965 out
13966 }
13967
13968 fn declared_anchor_holds_scalar(&self, anchor: CellRef) -> bool {
13974 let sheet = self.graph.sheet_name(anchor.sheet_id);
13975 if sheet.is_empty() {
13976 return false;
13977 }
13978 matches!(
13979 self.read_cell_value(sheet, anchor.coord.row() + 1, anchor.coord.col() + 1),
13980 Some(value) if !matches!(
13981 value,
13982 LiteralValue::Error(_)
13983 | LiteralValue::Array(_)
13984 | LiteralValue::Pending
13985 | LiteralValue::Empty
13986 )
13987 )
13988 }
13989
13990 pub(crate) fn read_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
13991 let asheet = self.sheet_store().sheet(sheet)?;
13992 let r0 = row.saturating_sub(1) as usize;
13993 let c0 = col.saturating_sub(1) as usize;
13994 let v = asheet.get_cell_value(r0, c0);
13995 if matches!(v, LiteralValue::Empty) {
13996 None
13997 } else {
13998 Some(v)
13999 }
14000 }
14001
14002 pub(crate) fn read_range_values(
14004 &self,
14005 sheet: &str,
14006 sr: u32,
14007 sc: u32,
14008 er: u32,
14009 ec: u32,
14010 ) -> RangeView<'_> {
14011 let Some(asheet) = self.sheet_store().sheet(sheet) else {
14012 return RangeView::from_owned_rows(Vec::new(), self.config.date_system);
14013 };
14014 if er < sr || ec < sc {
14015 return asheet.range_view(1, 1, 0, 0);
14016 }
14017 let sr0 = sr.saturating_sub(1) as usize;
14018 let sc0 = sc.saturating_sub(1) as usize;
14019 let er0 = er.saturating_sub(1) as usize;
14020 let ec0 = ec.saturating_sub(1) as usize;
14021 asheet.range_view(sr0, sc0, er0, ec0)
14022 }
14023
14024 pub fn get_cell(
14026 &self,
14027 sheet: &str,
14028 row: u32,
14029 col: u32,
14030 ) -> Option<(Option<formualizer_parse::ASTNode>, Option<LiteralValue>)> {
14031 let v = self.get_cell_value(sheet, row, col);
14032 let sheet_id = self.graph.sheet_id(sheet)?;
14033 let coord = Coord::from_excel(row, col, true, true);
14034 let cell = CellRef::new(sheet_id, coord);
14035 if let Some(vid) = self.graph.get_vertex_for_cell(&cell) {
14036 let ast = self.graph.get_formula(vid);
14037 Some((ast, v))
14038 } else if v.is_some() || self.graph.had_legacy_cell_vertex(&cell) {
14039 Some((None, v))
14043 } else {
14044 None
14045 }
14046 }
14047
14048 pub fn begin_batch(&mut self) {
14050 self.graph.begin_batch();
14051 }
14052
14053 pub fn end_batch(&mut self) {
14055 self.graph.end_batch();
14056 }
14057
14058 pub fn begin_deferred_dirty(&mut self) {
14067 self.graph.begin_deferred_dirty();
14068 }
14069
14070 pub fn end_deferred_dirty(&mut self) {
14073 let _ = self.graph.end_deferred_dirty();
14074 }
14075
14076 pub fn dirty_propagation_visits(&self) -> u64 {
14080 self.graph.dirty_propagation_visits()
14081 }
14082
14083 #[inline]
14086 fn record_cell_if_changed(
14087 delta: &mut DeltaCollector,
14088 cell: &CellRef,
14089 old: &LiteralValue,
14090 new: &LiteralValue,
14091 ) {
14092 if old != new {
14093 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
14094 }
14095 }
14096
14097 pub fn evaluate_vertex(&mut self, vertex_id: VertexId) -> Result<LiteralValue, ExcelError> {
14098 self.observe_evaluation_resource_request(EvaluationRequestKind::Vertex, |engine| {
14099 engine.observe_function_semantic_epoch()?;
14100 engine.resource_checkpoint(1)?;
14102 if !engine.graph.vertex_exists(vertex_id) {
14103 return engine.evaluate_vertex_impl(vertex_id, None);
14104 }
14105 let is_formula = matches!(
14106 engine.graph.get_vertex_kind(vertex_id),
14107 VertexKind::FormulaScalar | VertexKind::FormulaArray
14108 );
14109 if is_formula {
14110 engine.begin_evaluation_request();
14111 #[cfg(any(test, feature = "legacy_oracle"))]
14112 engine.graph.flush_pending_edge_deltas();
14113 let roots = [crate::engine::target_preparation::TargetProducer::Legacy(
14114 vertex_id,
14115 )];
14116 engine.evaluate_legacy_target_roots(&roots, None)?;
14117 }
14118 if !is_formula {
14119 engine.refresh_resolution_snapshots();
14120 }
14121 engine.evaluate_vertex_impl(vertex_id, None)
14122 })
14123 }
14124
14125 fn publish_oversized_spill(
14127 &mut self,
14128 vertex_id: VertexId,
14129 error: ExcelError,
14130 mut delta: Option<&mut DeltaCollector>,
14131 ) -> LiteralValue {
14132 self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
14133 let anchor = self
14134 .graph
14135 .get_cell_ref(vertex_id)
14136 .expect("cell ref for vertex");
14137 let spill_val = LiteralValue::Error(error);
14138 if let Some(d) = delta {
14139 let old = self
14140 .read_cell_value(
14141 self.graph.sheet_name(anchor.sheet_id),
14142 anchor.coord.row() + 1,
14143 anchor.coord.col() + 1,
14144 )
14145 .unwrap_or(LiteralValue::Empty);
14146 if old != spill_val {
14147 d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
14148 }
14149 }
14150 self.graph.update_vertex_value_ref(vertex_id, &spill_val);
14151 if self.config.arrow_storage_enabled
14152 && self.config.delta_overlay_enabled
14153 && self.config.write_formula_overlay_enabled
14154 {
14155 let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
14156 self.mirror_value_to_computed_overlay(
14157 &sheet_name,
14158 anchor.coord.row() + 1,
14159 anchor.coord.col() + 1,
14160 &spill_val,
14161 );
14162 }
14163 spill_val
14164 }
14165
14166 fn evaluate_vertex_impl(
14167 &mut self,
14168 vertex_id: VertexId,
14169 delta: Option<&mut DeltaCollector>,
14170 ) -> Result<LiteralValue, ExcelError> {
14171 if !self.graph.vertex_exists(vertex_id) {
14175 return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
14176 .with_message(format!("Vertex not found: {vertex_id:?}")));
14177 }
14178 if (self.active_resource_ledger.is_some()
14179 || self.graph.fixed_array_shapes.contains_key(&vertex_id))
14180 && matches!(
14181 self.graph.get_vertex_kind(vertex_id),
14182 VertexKind::FormulaScalar | VertexKind::FormulaArray
14183 )
14184 {
14185 let value = self
14186 .evaluate_vertex_immutable(vertex_id)
14187 .unwrap_or_else(LiteralValue::Error);
14188 let effects = self.plan_vertex_effects(vertex_id, value.clone())?;
14189 self.resource_checkpoint(0)?;
14191 let mut delta = delta;
14192 for effect in &effects {
14193 self.apply_effect_with_computed_writes(effect, delta.as_deref_mut(), None, None)?;
14194 }
14195 return Ok(value);
14196 }
14197
14198 let mut delta = delta;
14199
14200 let kind = self.graph.get_vertex_kind(vertex_id);
14202 let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
14203
14204 let view = match kind {
14205 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
14206 if let Some(view) = self.graph.formula_view(vertex_id) {
14207 view
14208 } else {
14209 return Ok(LiteralValue::Number(0.0));
14210 }
14211 }
14212 VertexKind::Empty | VertexKind::Cell => {
14213 if let Some(cell_ref) = self.graph.get_cell_ref(vertex_id) {
14214 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
14215 let row = cell_ref.coord.row() + 1;
14216 let col = cell_ref.coord.col() + 1;
14217 if let Some(v) = self.read_cell_value(sheet_name, row, col) {
14218 return Ok(v);
14219 }
14220 }
14221 return Ok(LiteralValue::Number(0.0));
14222 }
14223 VertexKind::NamedScalar => {
14224 let value = self.evaluate_named_scalar(vertex_id, sheet_id)?;
14225 return Ok(value);
14226 }
14227 VertexKind::NamedArray => {
14228 let value = self.evaluate_named_array(vertex_id, sheet_id)?;
14229 return Ok(value);
14230 }
14231 VertexKind::InfiniteRange
14232 | VertexKind::Range
14233 | VertexKind::External
14234 | VertexKind::Table => {
14235 return Ok(LiteralValue::Number(0.0));
14237 }
14238 };
14239
14240 let sheet_name = self.graph.sheet_name(sheet_id);
14242 let cell_ref = self
14243 .graph
14244 .get_cell_ref(vertex_id)
14245 .expect("cell ref for vertex");
14246 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref)
14247 .with_resolved_functions(&self.resolved_functions)
14248 .with_arena_cells(self);
14249
14250 let result = interpreter.evaluate_formula_view(
14251 view,
14252 self.graph.data_store(),
14253 self.graph.sheet_reg(),
14254 );
14255
14256 match result {
14258 Ok(cv) => {
14259 let derived_format = cv.format_id();
14260 self.record_derived_format(vertex_id, derived_format);
14261 let oversized_range = if matches!(cv, crate::traits::CalcValue::Range(_))
14262 && self.graph.fixed_single_arrays.contains(&vertex_id)
14263 {
14264 None
14265 } else {
14266 crate::engine::result_finalization::range_spill_error(
14267 &cv,
14268 self.config.spill.max_spill_cells,
14269 )
14270 };
14271 let is_oversized_range = oversized_range.is_some();
14272 let result_literal = if let Some(error) = oversized_range {
14273 drop(cv);
14274 LiteralValue::Error(error)
14275 } else {
14276 self.materialize_formula_result(vertex_id, cv)
14277 };
14278 let output_sheet_name = sheet_name.to_string();
14279 self.write_computed_overlay_format_0based(
14280 &output_sheet_name,
14281 cell_ref.coord.row(),
14282 cell_ref.coord.col(),
14283 derived_format,
14284 );
14285 if is_oversized_range {
14286 let LiteralValue::Error(error) = result_literal else {
14287 unreachable!()
14288 };
14289 self.graph.set_kind(vertex_id, VertexKind::FormulaArray);
14290 return Ok(self.publish_oversized_spill(
14291 vertex_id,
14292 error,
14293 delta.as_deref_mut(),
14294 ));
14295 }
14296 match result_literal {
14297 LiteralValue::Array(rows) => {
14298 self.graph
14300 .set_kind(vertex_id, crate::engine::vertex::VertexKind::FormulaArray);
14301 let anchor = self
14303 .graph
14304 .get_cell_ref(vertex_id)
14305 .expect("cell ref for vertex");
14306 let sheet_id = anchor.sheet_id;
14307 let h = rows.len() as u32;
14308 let w = rows.first().map(|r| r.len()).unwrap_or(0) as u32;
14309
14310 let spill_cells = (h as u64).saturating_mul(w as u64);
14312 if spill_cells > self.config.spill.max_spill_cells as u64 {
14313 let spill_err = ExcelError::new(ExcelErrorKind::Spill)
14314 .with_message("SpillTooLarge")
14315 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
14316 expected_rows: h,
14317 expected_cols: w,
14318 });
14319 return Ok(self.publish_oversized_spill(
14320 vertex_id,
14321 spill_err,
14322 delta.as_deref_mut(),
14323 ));
14324 }
14325 const PACKED_MAX_ROW: u32 = 1_048_575; const PACKED_MAX_COL: u32 = 16_383; let end_row = anchor.coord.row().saturating_add(h).saturating_sub(1);
14329 let end_col = anchor.coord.col().saturating_add(w).saturating_sub(1);
14330 if end_row > PACKED_MAX_ROW || end_col > PACKED_MAX_COL {
14331 self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
14332 let spill_err = ExcelError::new(ExcelErrorKind::Spill)
14333 .with_message("Spill exceeds sheet bounds")
14334 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
14335 expected_rows: h,
14336 expected_cols: w,
14337 });
14338 let spill_val = LiteralValue::Error(spill_err.clone());
14339 if let Some(d) = delta.as_deref_mut() {
14340 let old = self
14341 .read_cell_value(
14342 self.graph.sheet_name(anchor.sheet_id),
14343 anchor.coord.row() + 1,
14344 anchor.coord.col() + 1,
14345 )
14346 .unwrap_or(LiteralValue::Empty);
14347 if old != spill_val {
14348 d.record_cell(
14349 anchor.sheet_id,
14350 anchor.coord.row(),
14351 anchor.coord.col(),
14352 );
14353 }
14354 }
14355 self.graph.update_vertex_value_ref(vertex_id, &spill_val);
14356 if self.config.arrow_storage_enabled
14357 && self.config.delta_overlay_enabled
14358 && self.config.write_formula_overlay_enabled
14359 {
14360 let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
14361 self.mirror_value_to_computed_overlay(
14362 &sheet_name,
14363 anchor.coord.row() + 1,
14364 anchor.coord.col() + 1,
14365 &spill_val,
14366 );
14367 }
14368 return Ok(spill_val);
14369 }
14370 let mut targets = Vec::new();
14371 for r in 0..h {
14372 for c in 0..w {
14373 targets.push(self.graph.make_cell_ref_internal(
14374 sheet_id,
14375 anchor.coord.row() + r,
14376 anchor.coord.col() + c,
14377 ));
14378 }
14379 }
14380
14381 match self.spill_mgr.reserve(
14383 vertex_id,
14384 anchor,
14385 SpillShape { rows: h, cols: w },
14386 SpillMeta {
14387 epoch: self.recalc_epoch,
14388 config: self.config.spill,
14389 },
14390 ) {
14391 Ok(()) => {
14392 if let Err(e) = self.commit_spill_and_mirror(
14396 vertex_id,
14397 &targets,
14398 rows.clone(),
14399 delta.as_deref_mut(),
14400 ) {
14401 if e.kind != ExcelErrorKind::Spill {
14402 return Err(e);
14403 }
14404 self.clear_spill_projection_and_mirror(
14406 vertex_id,
14407 delta.as_deref_mut(),
14408 );
14409 if let Some(d) = delta.as_deref_mut() {
14410 let old = self
14411 .read_cell_value(
14412 self.graph.sheet_name(anchor.sheet_id),
14413 anchor.coord.row() + 1,
14414 anchor.coord.col() + 1,
14415 )
14416 .unwrap_or(LiteralValue::Empty);
14417 let new = LiteralValue::Error(e.clone());
14418 if old != new {
14419 d.record_cell(
14420 anchor.sheet_id,
14421 anchor.coord.row(),
14422 anchor.coord.col(),
14423 );
14424 }
14425 }
14426 let err_val = LiteralValue::Error(e.clone());
14427 self.graph.update_vertex_value_ref(vertex_id, &err_val);
14428 if self.config.arrow_storage_enabled
14429 && self.config.delta_overlay_enabled
14430 && self.config.write_formula_overlay_enabled
14431 {
14432 let sheet_name =
14433 self.graph.sheet_name(anchor.sheet_id).to_string();
14434 self.mirror_value_to_computed_overlay(
14435 &sheet_name,
14436 anchor.coord.row() + 1,
14437 anchor.coord.col() + 1,
14438 &err_val,
14439 );
14440 }
14441 return Ok(err_val);
14442 }
14443 let top_left = rows
14445 .first()
14446 .and_then(|r| r.first())
14447 .cloned()
14448 .unwrap_or(LiteralValue::Empty);
14449 self.graph.update_vertex_value_ref(vertex_id, &top_left);
14450 Ok(top_left)
14451 }
14452 Err(e) => {
14453 self.clear_spill_projection_and_mirror(
14454 vertex_id,
14455 delta.as_deref_mut(),
14456 );
14457 let spill_err = ExcelError::new(ExcelErrorKind::Spill)
14458 .with_message(e.message().unwrap_or("Spill blocked"))
14459 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
14460 expected_rows: h,
14461 expected_cols: w,
14462 });
14463 let spill_val = LiteralValue::Error(spill_err.clone());
14464 if let Some(d) = delta.as_deref_mut() {
14465 let old = self
14466 .read_cell_value(
14467 self.graph.sheet_name(anchor.sheet_id),
14468 anchor.coord.row() + 1,
14469 anchor.coord.col() + 1,
14470 )
14471 .unwrap_or(LiteralValue::Empty);
14472 if old != spill_val {
14473 d.record_cell(
14474 anchor.sheet_id,
14475 anchor.coord.row(),
14476 anchor.coord.col(),
14477 );
14478 }
14479 }
14480 self.graph.update_vertex_value_ref(vertex_id, &spill_val);
14481 if self.config.arrow_storage_enabled
14482 && self.config.delta_overlay_enabled
14483 && self.config.write_formula_overlay_enabled
14484 {
14485 let sheet_name =
14486 self.graph.sheet_name(anchor.sheet_id).to_string();
14487 self.mirror_value_to_computed_overlay(
14488 &sheet_name,
14489 anchor.coord.row() + 1,
14490 anchor.coord.col() + 1,
14491 &spill_val,
14492 );
14493 }
14494 Ok(spill_val)
14495 }
14496 }
14497 }
14498 other => {
14499 let spill_cells = self.graph.spill_cells_to_clear(vertex_id);
14501 if let Some(d) = delta.as_deref_mut()
14502 && let Some(anchor) = self.graph.get_cell_ref_for_vertex(vertex_id)
14503 {
14504 if spill_cells.is_empty() {
14505 let old = self
14506 .read_cell_value(
14507 self.graph.sheet_name(anchor.sheet_id),
14508 anchor.coord.row() + 1,
14509 anchor.coord.col() + 1,
14510 )
14511 .unwrap_or(LiteralValue::Empty);
14512 if old != other {
14513 d.record_cell(
14514 anchor.sheet_id,
14515 anchor.coord.row(),
14516 anchor.coord.col(),
14517 );
14518 }
14519 } else {
14520 for cell in spill_cells.iter() {
14521 let sheet_name = self.graph.sheet_name(cell.sheet_id);
14522 let old = self
14523 .read_cell_value(
14524 sheet_name,
14525 cell.coord.row() + 1,
14526 cell.coord.col() + 1,
14527 )
14528 .unwrap_or(LiteralValue::Empty);
14529 let new = if cell.sheet_id == anchor.sheet_id
14530 && cell.coord.row() == anchor.coord.row()
14531 && cell.coord.col() == anchor.coord.col()
14532 {
14533 other.clone()
14534 } else {
14535 LiteralValue::Empty
14536 };
14537 Self::record_cell_if_changed(d, cell, &old, &new);
14538 }
14539 }
14540 }
14541 self.graph.clear_spill_region(vertex_id);
14542 if let Some(scope) = Self::structural_scope_from_cells(&spill_cells) {
14543 self.record_structural_change(scope);
14544 }
14545 if self.config.arrow_storage_enabled
14546 && self.config.delta_overlay_enabled
14547 && self.config.write_formula_overlay_enabled
14548 {
14549 let empty = LiteralValue::Empty;
14550 for cell in spill_cells.iter() {
14551 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
14552 self.mirror_value_to_computed_overlay(
14553 &sheet_name,
14554 cell.coord.row() + 1,
14555 cell.coord.col() + 1,
14556 &empty,
14557 );
14558 }
14559 }
14560 self.graph.update_vertex_value_ref(vertex_id, &other);
14561 if self.config.arrow_storage_enabled
14563 && self.config.delta_overlay_enabled
14564 && self.config.write_formula_overlay_enabled
14565 {
14566 let anchor = self
14567 .graph
14568 .get_cell_ref(vertex_id)
14569 .expect("cell ref for vertex");
14570 let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
14571 self.mirror_value_to_computed_overlay(
14572 &sheet_name,
14573 anchor.coord.row() + 1,
14574 anchor.coord.col() + 1,
14575 &other,
14576 );
14577 }
14578 Ok(other)
14579 }
14580 }
14581 }
14582 Err(e) => {
14583 let spill_cells = self.graph.spill_cells_to_clear(vertex_id);
14586 let err_val = LiteralValue::Error(e.clone());
14587 if let Some(d) = delta
14588 && let Some(anchor) = self.graph.get_cell_ref_for_vertex(vertex_id)
14589 {
14590 if spill_cells.is_empty() {
14591 let old = self
14592 .read_cell_value(
14593 self.graph.sheet_name(anchor.sheet_id),
14594 anchor.coord.row() + 1,
14595 anchor.coord.col() + 1,
14596 )
14597 .unwrap_or(LiteralValue::Empty);
14598 if old != err_val {
14599 d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
14600 }
14601 } else {
14602 for cell in spill_cells.iter() {
14603 let sheet_name = self.graph.sheet_name(cell.sheet_id);
14604 let old = self
14605 .read_cell_value(
14606 sheet_name,
14607 cell.coord.row() + 1,
14608 cell.coord.col() + 1,
14609 )
14610 .unwrap_or(LiteralValue::Empty);
14611 let new = if cell.sheet_id == anchor.sheet_id
14612 && cell.coord.row() == anchor.coord.row()
14613 && cell.coord.col() == anchor.coord.col()
14614 {
14615 err_val.clone()
14616 } else {
14617 LiteralValue::Empty
14618 };
14619 Self::record_cell_if_changed(d, cell, &old, &new);
14620 }
14621 }
14622 }
14623 self.graph.clear_spill_region(vertex_id);
14624 if let Some(scope) = Self::structural_scope_from_cells(&spill_cells) {
14625 self.record_structural_change(scope);
14626 }
14627 if self.config.arrow_storage_enabled
14628 && self.config.delta_overlay_enabled
14629 && self.config.write_formula_overlay_enabled
14630 {
14631 let empty = LiteralValue::Empty;
14632 for cell in spill_cells.iter() {
14633 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
14634 self.mirror_value_to_computed_overlay(
14635 &sheet_name,
14636 cell.coord.row() + 1,
14637 cell.coord.col() + 1,
14638 &empty,
14639 );
14640 }
14641 }
14642 self.graph.update_vertex_value_ref(vertex_id, &err_val);
14643 if self.config.arrow_storage_enabled
14644 && self.config.delta_overlay_enabled
14645 && self.config.write_formula_overlay_enabled
14646 {
14647 let anchor = self
14648 .graph
14649 .get_cell_ref(vertex_id)
14650 .expect("cell ref for vertex");
14651 let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
14652 self.mirror_value_to_computed_overlay(
14653 &sheet_name,
14654 anchor.coord.row() + 1,
14655 anchor.coord.col() + 1,
14656 &err_val,
14657 );
14658 }
14659 Ok(err_val)
14660 }
14661 }
14662 }
14663
14664 fn evaluate_named_scalar(
14665 &mut self,
14666 vertex_id: VertexId,
14667 sheet_id: SheetId,
14668 ) -> Result<LiteralValue, ExcelError> {
14669 let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
14670 ExcelError::new(ExcelErrorKind::Name)
14671 .with_message("Named range metadata missing".to_string())
14672 })?;
14673
14674 match &named_range.definition {
14675 NamedDefinition::Cell(cell_ref) => {
14676 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
14677 let row = cell_ref.coord.row() + 1;
14678 let col = cell_ref.coord.col() + 1;
14679
14680 if let Some(dep_vertex) = self.graph.get_vertex_for_cell(cell_ref)
14681 && matches!(
14682 self.graph.get_vertex_kind(dep_vertex),
14683 VertexKind::FormulaScalar | VertexKind::FormulaArray
14684 )
14685 {
14686 let value = self.evaluate_vertex(dep_vertex)?;
14688 self.graph.update_vertex_value_ref(vertex_id, &value);
14689 Ok(value)
14690 } else {
14691 let value = self
14692 .read_cell_value(sheet_name, row, col)
14693 .unwrap_or(LiteralValue::Empty);
14694 self.graph.update_vertex_value_ref(vertex_id, &value);
14695 Ok(value)
14696 }
14697 }
14698 NamedDefinition::Literal(v) => {
14699 let out = v.clone();
14700 self.graph.update_vertex_value_ref(vertex_id, &out);
14701 Ok(out)
14702 }
14703 NamedDefinition::Formula { ast, .. } => {
14704 let context_sheet = match named_range.scope {
14705 NameScope::Sheet(id) => id,
14706 NameScope::Workbook => sheet_id,
14707 };
14708 let sheet_name = self.graph.sheet_name(context_sheet);
14709 let cell_ref = self
14710 .graph
14711 .get_cell_ref(vertex_id)
14712 .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
14713 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref)
14714 .with_resolved_functions(&self.resolved_functions)
14715 .with_arena_cells(self);
14716 match interpreter.evaluate_ast(ast) {
14717 Ok(cv) => {
14718 let value = cv.into_literal();
14719 match value {
14720 LiteralValue::Array(_) => {
14721 let err = ExcelError::new(ExcelErrorKind::NImpl)
14722 .with_message("Array result in scalar named range".to_string());
14723 let err_val = LiteralValue::Error(err.clone());
14724 self.graph.update_vertex_value_ref(vertex_id, &err_val);
14725 Ok(err_val)
14726 }
14727 other => {
14728 self.graph.update_vertex_value_ref(vertex_id, &other);
14729 Ok(other)
14730 }
14731 }
14732 }
14733 Err(err) => {
14734 let err_val = LiteralValue::Error(err.clone());
14735 self.graph.update_vertex_value_ref(vertex_id, &err_val);
14736 Ok(err_val)
14737 }
14738 }
14739 }
14740 NamedDefinition::Range(_) => Err(ExcelError::new(ExcelErrorKind::Value)
14741 .with_message("Range-valued name evaluated as scalar".to_string())),
14742 }
14743 }
14744
14745 fn evaluate_named_array(
14746 &mut self,
14747 vertex_id: VertexId,
14748 sheet_id: SheetId,
14749 ) -> Result<LiteralValue, ExcelError> {
14750 let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
14751 ExcelError::new(ExcelErrorKind::Name)
14752 .with_message("Named range metadata missing".to_string())
14753 })?;
14754
14755 let out = match &named_range.definition {
14756 NamedDefinition::Range(range_ref) => {
14757 if range_ref.start.sheet_id != range_ref.end.sheet_id {
14758 return Err(ExcelError::new(ExcelErrorKind::Ref)
14759 .with_message("Named range cannot span sheets".to_string()));
14760 }
14761
14762 let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
14763 let sr0 = range_ref.start.coord.row();
14764 let sc0 = range_ref.start.coord.col();
14765 let er0 = range_ref.end.coord.row();
14766 let ec0 = range_ref.end.coord.col();
14767 if sr0 > er0 || sc0 > ec0 {
14768 return Err(ExcelError::new(ExcelErrorKind::Ref)
14769 .with_message("Invalid named range bounds".to_string()));
14770 }
14771
14772 let h = (er0 - sr0 + 1) as usize;
14773 let w = (ec0 - sc0 + 1) as usize;
14774 let cell_count = (h as u64).saturating_mul(w as u64);
14775 if cell_count > self.config.spill.max_spill_cells as u64 {
14776 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
14777 "Named range too large to materialize as an array".to_string(),
14778 ));
14779 }
14780
14781 let mut rows = Vec::with_capacity(h);
14782 for r0 in sr0..=er0 {
14783 let mut row = Vec::with_capacity(w);
14784 for c0 in sc0..=ec0 {
14785 let v = self
14786 .read_cell_value(sheet_name, r0 + 1, c0 + 1)
14787 .unwrap_or(LiteralValue::Empty);
14788 row.push(v);
14789 }
14790 rows.push(row);
14791 }
14792 LiteralValue::Array(rows)
14793 }
14794 NamedDefinition::Cell(cell_ref) => {
14795 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
14796 let row = cell_ref.coord.row() + 1;
14797 let col = cell_ref.coord.col() + 1;
14798 let v = self
14799 .read_cell_value(sheet_name, row, col)
14800 .unwrap_or(LiteralValue::Empty);
14801 LiteralValue::Array(vec![vec![v]])
14802 }
14803 NamedDefinition::Literal(v) => LiteralValue::Array(vec![vec![v.clone()]]),
14804 NamedDefinition::Formula { ast, .. } => {
14805 let context_sheet = match named_range.scope {
14806 NameScope::Sheet(id) => id,
14807 NameScope::Workbook => sheet_id,
14808 };
14809 let sheet_name = self.graph.sheet_name(context_sheet);
14810 let cell_ref = self
14811 .graph
14812 .get_cell_ref(vertex_id)
14813 .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
14814 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref)
14815 .with_resolved_functions(&self.resolved_functions)
14816 .with_arena_cells(self);
14817 match interpreter.evaluate_ast(ast) {
14818 Ok(cv) => {
14819 let v = cv.into_literal();
14820 match v {
14821 LiteralValue::Array(_) => v,
14822 other => LiteralValue::Array(vec![vec![other]]),
14823 }
14824 }
14825 Err(err) => LiteralValue::Error(err),
14826 }
14827 }
14828 };
14829
14830 self.graph.update_vertex_value_ref(vertex_id, &out);
14831 Ok(out)
14832 }
14833
14834 fn replan_exhausted_error(&self, limit: usize, context: &str) -> ExcelError {
14835 crate::engine::ResourceLedgerError::Exhausted(
14836 formualizer_common::ResourceExhaustionDetail {
14837 reason: formualizer_common::ResourceExhaustionReason::WorkUnits,
14838 limit: limit as u64,
14839 observed: limit.saturating_add(1) as u64,
14840 request_id: self
14841 .active_evaluation_resource_request
14842 .as_ref()
14843 .map(|stats| stats.request_id),
14844 },
14845 )
14846 .into_excel_error()
14847 .with_message(format!("{context} did not converge after {limit} replans"))
14848 }
14849
14850 fn transient_target_preparation_stale(error: &ExcelError) -> bool {
14851 matches!(
14852 error.extra(),
14853 formualizer_common::ExcelErrorExtra::PreparationStale {
14854 reason: formualizer_common::PreparationStaleReason::Semantic
14855 | formualizer_common::PreparationStaleReason::Provider
14856 }
14857 )
14858 }
14859
14860 fn prepare_graph_for_routed_evaluation(
14861 &mut self,
14862 targets: &[crate::engine::EvaluationTarget],
14863 options: &crate::engine::TargetEvalOptions<'_>,
14864 ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
14865 const MAX_TRANSIENT_PREPARATION_RETRIES: usize = 2;
14866 let mut retries = 0usize;
14867 loop {
14868 match self.prepare_graph_for_targets_unobserved(targets, options) {
14869 Err(error)
14870 if Self::transient_target_preparation_stale(&error)
14871 && retries < MAX_TRANSIENT_PREPARATION_RETRIES =>
14872 {
14873 retries = retries.saturating_add(1);
14874 }
14875 result => return result,
14876 }
14877 }
14878 }
14879
14880 fn evaluate_mixed_targets(
14881 &mut self,
14882 targets: &[crate::engine::EvaluationTarget],
14883 delta: Option<&mut DeltaCollector>,
14884 ) -> Result<EvalResult, ExcelError> {
14885 let _source_cache = self.source_cache_session();
14886 let cancel = self.active_cancel_flag.clone();
14887 let options = crate::engine::TargetEvalOptions {
14888 request_id: self
14889 .active_evaluation_resource_request
14890 .as_ref()
14891 .map(|stats| stats.request_id),
14892 cancel,
14893 deadline: None,
14894 budgets: None,
14895 opaque_policy: crate::engine::OpaquePreparePolicy::Widen,
14896 };
14897 self.prepare_and_execute_target_recipe(targets, &options, delta)
14898 }
14899
14900 fn prepare_and_execute_target_recipe(
14901 &mut self,
14902 targets: &[crate::engine::EvaluationTarget],
14903 options: &crate::engine::TargetEvalOptions<'_>,
14904 delta: Option<&mut DeltaCollector>,
14905 ) -> Result<EvalResult, ExcelError> {
14906 let preparation = self.prepare_graph_for_routed_evaluation(targets, options)?;
14907 self.execute_prepared_target_recipe(targets, &preparation.widened_scope, delta)
14908 }
14909
14910 fn execute_prepared_target_recipe(
14911 &mut self,
14912 targets: &[crate::engine::EvaluationTarget],
14913 scope: &crate::engine::PrepareScope,
14914 delta: Option<&mut DeltaCollector>,
14915 ) -> Result<EvalResult, ExcelError> {
14916 self.require_unified_authority()?;
14917 if matches!(scope, crate::engine::PrepareScope::Workbook)
14918 && let Some(stats) = self.active_evaluation_resource_request.as_mut()
14919 {
14920 stats.workbook_exact_attempts = stats.workbook_exact_attempts.max(1);
14921 }
14922 let mut roots = self.resolve_target_producers(targets)?;
14923 if let crate::engine::PrepareScope::Sheets(sheets) = scope {
14924 let request_id = self
14925 .active_evaluation_resource_request
14926 .as_ref()
14927 .map(|request| request.request_id);
14928 let root_count = roots.len();
14929 let mut widened_roots =
14930 OrderedTargetProducers::from_ordered(std::mem::take(&mut roots))
14931 .map_err(|_| target_root_allocation_error(root_count, request_id))?;
14932 let sheet_ids = sheets
14933 .iter()
14934 .filter_map(|sheet| self.graph.sheet_id(sheet))
14935 .collect::<FxHashSet<_>>();
14936 for vertex in self.graph.formula_vertices() {
14937 if sheet_ids.contains(&self.graph.get_vertex_sheet_id(vertex)) {
14938 widened_roots
14939 .push(crate::engine::target_preparation::TargetProducer::Legacy(
14940 vertex,
14941 ))
14942 .map_err(|_| {
14943 target_root_allocation_error(widened_roots.len() + 1, request_id)
14944 })?;
14945 }
14946 }
14947 roots = widened_roots.into_vec();
14948 }
14949 self.begin_evaluation_request();
14950 #[cfg(any(test, feature = "legacy_oracle"))]
14951 self.graph.flush_pending_edge_deltas();
14952 let workbook_scope = matches!(scope, crate::engine::PrepareScope::Workbook);
14953 if workbook_scope {
14954 if let Some(delta) = delta {
14955 self.evaluate_all_with_delta_collector(delta)
14956 } else {
14957 self.evaluate_all_legacy_impl()
14958 }
14959 } else {
14960 self.evaluate_legacy_target_roots(&roots, delta)
14961 }
14962 }
14963
14964 fn legacy_coordinate_targets(
14965 &mut self,
14966 targets: &[(&str, u32, u32)],
14967 ) -> Vec<crate::engine::EvaluationTarget> {
14968 targets
14969 .iter()
14970 .map(|(sheet, row, col)| {
14971 self.graph.sheet_id_mut(sheet);
14974 crate::engine::EvaluationTarget::Cell {
14975 sheet: (*sheet).to_string(),
14976 row: *row,
14977 col: *col,
14978 }
14979 })
14980 .collect()
14981 }
14982
14983 pub fn evaluate_targets(
14985 &mut self,
14986 targets: &[crate::engine::EvaluationTarget],
14987 ) -> Result<EvalResult, ExcelError> {
14988 self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
14989 engine.observe_function_semantic_epoch()?;
14990 engine.validate_deterministic_mode()?;
14991 engine.evaluate_mixed_targets(targets, None)
14992 })
14993 }
14994
14995 pub fn evaluate_targets_with_options(
14997 &mut self,
14998 targets: &[crate::engine::EvaluationTarget],
14999 options: crate::engine::TargetEvalOptions<'_>,
15000 ) -> Result<EvalResult, ExcelError> {
15001 self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
15002 engine.active_cancel_flag = options.cancel.clone();
15003 engine.active_evaluation_deadline = options.deadline;
15004 let result = (|| {
15005 engine.cancellation_checkpoint("Evaluation cancelled before target preparation")?;
15006 engine.observe_function_semantic_epoch()?;
15007 engine.validate_deterministic_mode()?;
15008 let _source_cache = engine.source_cache_session();
15009 engine.prepare_and_execute_target_recipe(targets, &options, None)
15010 })();
15011 engine.active_cancel_flag = None;
15012 engine.active_evaluation_deadline = None;
15013 result
15014 })
15015 }
15016
15017 pub fn evaluate_targets_with_delta(
15019 &mut self,
15020 targets: &[crate::engine::EvaluationTarget],
15021 ) -> Result<(EvalResult, crate::engine::TargetEvalDelta), ExcelError> {
15022 self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
15023 engine.observe_function_semantic_epoch()?;
15024 engine.validate_deterministic_mode()?;
15025 let mut collector = DeltaCollector::new(DeltaMode::Cells);
15026 let result = engine.evaluate_mixed_targets(targets, Some(&mut collector))?;
15027 Ok((result, collector.finish_target()))
15028 })
15029 }
15030
15031 pub fn evaluate_until(
15033 &mut self,
15034 targets: &[(&str, u32, u32)],
15035 ) -> Result<EvalResult, ExcelError> {
15036 self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
15037 engine.evaluate_until_unobserved(targets)
15038 })
15039 }
15040
15041 fn evaluate_until_unobserved(
15042 &mut self,
15043 targets: &[(&str, u32, u32)],
15044 ) -> Result<EvalResult, ExcelError> {
15045 self.observe_function_semantic_epoch()?;
15046 let targets = self.legacy_coordinate_targets(targets);
15047 self.evaluate_mixed_targets(&targets, None)
15048 }
15049
15050 fn evaluate_until_with_delta_collector(
15051 &mut self,
15052 targets: &[(&str, u32, u32)],
15053 delta: &mut DeltaCollector,
15054 ) -> Result<EvalResult, ExcelError> {
15055 let targets = self.legacy_coordinate_targets(targets);
15056 self.evaluate_mixed_targets(&targets, Some(delta))
15057 }
15058
15059 fn evaluate_legacy_target_roots(
15060 &mut self,
15061 roots: &[crate::engine::target_preparation::TargetProducer],
15062 mut delta: Option<&mut DeltaCollector>,
15063 ) -> Result<EvalResult, ExcelError> {
15064 use crate::engine::target_preparation::TargetProducer;
15065 #[cfg(any(test, feature = "benchmark_internal"))]
15066 {
15067 self.recalc_reuse_probe
15068 .get_mut()
15069 .unwrap()
15070 .legacy_target_requests += 1;
15071 }
15072 let start = crate::instant::FzInstant::now();
15073 let root_vertices = roots
15074 .iter()
15075 .filter_map(|root| match root {
15076 TargetProducer::Legacy(vertex) | TargetProducer::Symbol(vertex) => Some(*vertex),
15077 TargetProducer::ValueOnly(_) => None,
15078 })
15079 .collect::<Vec<_>>();
15080 let mut computed_vertices = 0usize;
15081 let mut cycle_errors = 0usize;
15082 let mut replans = 0usize;
15083 const MAX_REPLAN: usize = 5;
15084 self.graph.authority_sync();
15085 loop {
15086 let (precedents_to_eval, old_vdeps) = self.demand_subgraph(&root_vertices)?;
15087 if precedents_to_eval.is_empty() {
15088 break;
15089 }
15090 #[cfg(any(test, feature = "benchmark_internal"))]
15091 {
15092 self.recalc_reuse_probe
15093 .get_mut()
15094 .unwrap()
15095 .target_schedule_builds += 1;
15096 }
15097 let schedule = {
15098 self.graph.authority_sync();
15099 let mut ledger = self.active_resource_ledger.take();
15100 let result = self.create_authority_schedule(
15101 &precedents_to_eval,
15102 &old_vdeps,
15103 ledger.as_mut(),
15104 );
15105 self.active_resource_ledger = ledger;
15106 result?
15107 };
15108 self.begin_pass(&schedule);
15109 for (unit_index, &unit) in schedule.units.iter().enumerate() {
15110 self.cancellation_checkpoint("Evaluation cancelled before target schedule unit")?;
15111 match unit {
15112 ScheduleUnit::Cycle(index) => {
15113 if self.handle_cycle_unit(
15114 schedule.unit_cycle(index),
15115 delta.as_deref_mut(),
15116 None,
15117 None,
15118 )? > 0
15119 {
15120 cycle_errors = cycle_errors.saturating_add(1);
15121 }
15122 }
15123 ScheduleUnit::Layer(index) => {
15124 let layer = schedule.unit_layer(index);
15125 let evaluated = if let Some(delta) = delta.as_deref_mut() {
15126 if self.thread_pool.is_some() && layer.vertices.len() > 1 {
15127 self.evaluate_layer_parallel_with_delta(layer, delta)?
15128 } else {
15129 self.evaluate_layer_sequential_with_delta(layer, delta)?
15130 }
15131 } else if self.thread_pool.is_some() && layer.vertices.len() > 1 {
15132 self.evaluate_layer_parallel(layer)?
15133 } else {
15134 self.evaluate_layer_sequential(layer)?
15135 };
15136 computed_vertices = computed_vertices.saturating_add(evaluated);
15137 }
15138 }
15139 if self.stop_after_unit(&schedule, unit_index) {
15140 break;
15141 }
15142 }
15143 let changed = self.changed_virtual_dep_vertices(&precedents_to_eval, &old_vdeps);
15144 self.resource_checkpoint(0)?;
15145 if !self.finish_target_pass_dirty(&precedents_to_eval, &changed) {
15146 break;
15147 }
15148 if replans >= MAX_REPLAN {
15149 return Err(self.replan_exhausted_error(
15150 MAX_REPLAN,
15151 "targeted legacy dynamic dependency evaluation",
15152 ));
15153 }
15154 replans = replans.saturating_add(1);
15155 }
15156 self.redirty_for_next_recalc();
15157 Ok(EvalResult {
15158 computed_vertices,
15159 cycle_errors,
15160 elapsed: start.elapsed(),
15161 })
15162 }
15163
15164 pub fn build_recalc_plan(&self) -> Result<RecalcPlan, ExcelError> {
15166 if self.has_staged_formulas() || self.staged_formula_index.has_packages() {
15167 return Err(
15168 Self::plan_stale(formualizer_common::PlanStaleReason::Staged).with_message(
15169 "compatibility recalculation plans require all staged formulas to be prepared",
15170 ),
15171 );
15172 }
15173 let key = self.recalc_plan_key();
15174 let mut vertices: Vec<VertexId> = self.graph.vertices_with_formulas().collect();
15175 vertices.sort_unstable();
15176 let has_dynamic_refs = vertices.iter().copied().any(|v| self.graph.is_dynamic(v));
15177 let schedule = if vertices.is_empty() {
15178 crate::engine::Schedule {
15179 units: Vec::new(),
15180 layers: Vec::new(),
15181 cycles: Vec::new(),
15182 }
15183 } else {
15184 self.create_evaluation_schedule_uncached(&vertices, None)?.0
15185 };
15186 self.validate_recalc_plan_key(&key)?;
15187 Ok(RecalcPlan {
15188 key,
15189 kind: RecalcPlanKind::CompatibilityFull {
15190 schedule,
15191 has_dynamic_refs,
15192 },
15193 })
15194 }
15195
15196 pub fn build_recalc_plan_for_targets(
15198 &mut self,
15199 targets: &[crate::engine::EvaluationTarget],
15200 ) -> Result<RecalcPlan, ExcelError> {
15201 self.build_recalc_plan_for_targets_with_options(
15202 targets,
15203 crate::engine::TargetEvalOptions::default(),
15204 )
15205 }
15206
15207 pub fn build_recalc_plan_for_targets_with_options(
15208 &mut self,
15209 targets: &[crate::engine::EvaluationTarget],
15210 options: crate::engine::TargetEvalOptions<'_>,
15211 ) -> Result<RecalcPlan, ExcelError> {
15212 self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
15213 engine.observe_function_semantic_epoch()?;
15214 engine.validate_deterministic_mode()?;
15215 let _source_cache = engine.source_cache_session();
15216 let preparation = engine.prepare_graph_for_routed_evaluation(targets, &options)?;
15217 #[cfg(any(test, feature = "legacy_oracle"))]
15218 engine.graph.flush_pending_edge_deltas();
15219 let topology = if matches!(
15220 preparation.widened_scope,
15221 crate::engine::PrepareScope::Workbook
15222 ) {
15223 RecalcTopology::Workbook
15224 } else {
15225 RecalcTopology::RunLocalRecipe
15226 };
15227 Ok(RecalcPlan {
15228 key: engine.recalc_plan_key(),
15229 kind: RecalcPlanKind::Target {
15230 targets: targets.to_vec(),
15231 scope: preparation.widened_scope,
15232 topology,
15233 dynamic_policy: DynamicPlanPolicy::BoundedTargetReplan,
15234 },
15235 })
15236 })
15237 }
15238
15239 pub fn evaluate_recalc_plan(&mut self, plan: &RecalcPlan) -> Result<EvalResult, ExcelError> {
15241 self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
15242 engine.evaluate_recalc_plan_unobserved(plan)
15243 })
15244 }
15245
15246 pub fn evaluate_recalc_plan_with_controls(
15247 &mut self,
15248 plan: &RecalcPlan,
15249 cancel: Option<crate::engine::CancelToken>,
15250 deadline: Option<Instant>,
15251 ) -> Result<EvalResult, ExcelError> {
15252 self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
15253 engine.active_cancel_flag = cancel.clone();
15254 engine.active_evaluation_deadline = deadline;
15255 let result = engine.evaluate_recalc_plan_unobserved(plan);
15256 engine.active_cancel_flag = None;
15257 engine.active_evaluation_deadline = None;
15258 result
15259 })
15260 }
15261
15262 fn evaluate_recalc_plan_unobserved(
15263 &mut self,
15264 plan: &RecalcPlan,
15265 ) -> Result<EvalResult, ExcelError> {
15266 #[cfg(any(test, feature = "legacy_oracle"))]
15267 self.graph.flush_pending_edge_deltas();
15268 self.validate_recalc_plan_key(&plan.key)?;
15269 self.cancellation_checkpoint("Evaluation cancelled before recalculation plan execution")?;
15270 self.validate_deterministic_mode()?;
15271
15272 match &plan.kind {
15273 RecalcPlanKind::Target {
15274 targets,
15275 scope,
15276 topology,
15277 dynamic_policy,
15278 } => {
15279 debug_assert_eq!(*dynamic_policy, DynamicPlanPolicy::BoundedTargetReplan);
15280 debug_assert_eq!(
15281 matches!(topology, RecalcTopology::Workbook),
15282 matches!(scope, crate::engine::PrepareScope::Workbook)
15283 );
15284 let _source_cache = self.source_cache_session();
15285 self.execute_prepared_target_recipe(targets, scope, None)
15286 }
15287 RecalcPlanKind::CompatibilityFull {
15288 schedule,
15289 has_dynamic_refs,
15290 } => {
15291 let _source_cache = self.source_cache_session();
15292 self.begin_evaluation_request();
15293 if *has_dynamic_refs {
15294 self.virtual_dep_fallback_activations =
15295 self.virtual_dep_fallback_activations.saturating_add(1);
15296 return self.evaluate_all_coordinator();
15297 }
15298
15299 let start = crate::instant::FzInstant::now();
15300 let dirty_vertices = self.graph.get_evaluation_vertices();
15301 if dirty_vertices.is_empty() {
15302 return Ok(EvalResult {
15303 computed_vertices: 0,
15304 cycle_errors: 0,
15305 elapsed: start.elapsed(),
15306 });
15307 }
15308
15309 let dirty_set: FxHashSet<VertexId> = dirty_vertices.iter().copied().collect();
15310 let mut computed_vertices = 0;
15311 let mut cycle_errors = 0;
15312 for &unit in &schedule.units {
15313 self.cancellation_checkpoint(
15314 "Evaluation cancelled before recalculation plan schedule unit",
15315 )?;
15316 match unit {
15317 ScheduleUnit::Cycle(i) => {
15318 let stamped = self.handle_cycle_unit(
15319 schedule.unit_cycle(i),
15320 None,
15321 Some(&dirty_set),
15322 None,
15323 )?;
15324 if stamped > 0 {
15325 cycle_errors += 1;
15326 }
15327 }
15328 ScheduleUnit::Layer(i) => {
15329 let work: Vec<VertexId> = schedule
15330 .unit_layer(i)
15331 .vertices
15332 .iter()
15333 .copied()
15334 .filter(|v| dirty_set.contains(v))
15335 .collect();
15336 if work.is_empty() {
15337 continue;
15338 }
15339 let temp_layer = crate::engine::scheduler::Layer::new(work);
15340 if self.thread_pool.is_some() && temp_layer.vertices.len() > 1 {
15341 computed_vertices += self.evaluate_layer_parallel(&temp_layer)?;
15342 } else {
15343 computed_vertices += self.evaluate_layer_sequential(&temp_layer)?;
15344 }
15345 }
15346 }
15347 }
15348
15349 self.resource_checkpoint(0)?;
15350 self.graph.clear_dirty_flags(&dirty_vertices);
15351 self.redirty_for_next_recalc();
15352 Ok(EvalResult {
15353 computed_vertices,
15354 cycle_errors,
15355 elapsed: start.elapsed(),
15356 })
15357 }
15358 }
15359 }
15360}
15361
15362impl<R> Engine<R>
15363where
15364 R: EvaluationContext,
15365{
15366 fn require_unified_authority(&mut self) -> Result<(), ExcelError> {
15371 self.graph
15372 .authority()
15373 .map(|_| ())
15374 .map_err(Self::authority_excel_error)
15375 }
15376
15377 fn authority_excel_error(error: crate::engine::authority::store::AuthorityError) -> ExcelError {
15378 use crate::engine::authority::store::AuthorityError;
15379 #[cfg(test)]
15383 if std::env::var_os("FZ_AUTHORITY_DEFERRED_TRACE").is_some() {
15384 eprintln!("M1B_AUTHORITY_ERROR {error:?}");
15385 }
15386 let kind = match error {
15387 AuthorityError::Unsupported { .. } => ExcelErrorKind::NImpl,
15388 _ => ExcelErrorKind::Error,
15389 };
15390 ExcelError::new(kind).with_message(format!("unified_authority: {error:?}"))
15391 }
15392
15393 pub fn evaluate_all_for_snapshot(
15407 &mut self,
15408 cancel: Option<crate::engine::CancelToken>,
15409 ) -> Result<EvalResult, ExcelError> {
15410 struct SnapshotScope<'e, R: EvaluationContext>(&'e mut Engine<R>);
15414 impl<R: EvaluationContext> Drop for SnapshotScope<'_, R> {
15415 fn drop(&mut self) {
15416 self.0.snapshot_evaluation_active = false;
15417 if std::thread::panicking() {
15418 self.0.snapshot_volatile_redirty_pending = true;
15419 }
15420 }
15421 }
15422 self.snapshot_evaluation_active = true;
15423 let result = {
15424 let scope = SnapshotScope(self);
15425 match cancel {
15426 Some(cancel) => scope.0.evaluate_all_cancellable(cancel),
15427 None => scope.0.evaluate_all(),
15428 }
15429 };
15430 if result.is_err() {
15431 self.snapshot_volatile_redirty_pending = false;
15432 self.graph.redirty_volatiles();
15433 }
15434 result
15435 }
15436
15437 pub fn evaluate_all(&mut self) -> Result<EvalResult, ExcelError> {
15439 self.observe_evaluation_resource_request(EvaluationRequestKind::Full, |engine| {
15441 engine.evaluate_all_unobserved()
15442 })
15443 }
15444
15445 fn evaluate_all_unobserved(&mut self) -> Result<EvalResult, ExcelError> {
15446 debug_assert!(
15447 !self.graph.deferred_dirty_active(),
15448 "deferred-dirty scope leaked into evaluate_all: a begin_deferred_dirty \
15449 was not balanced by end_deferred_dirty"
15450 );
15451 self.observe_function_semantic_epoch()?;
15452 self.lookup_index_cache.reset_counters();
15453 let _source_cache = self.source_cache_session();
15454 self.validate_deterministic_mode()?;
15455 if self.config.defer_graph_building {
15456 self.build_graph_all()?;
15458 }
15459 self.evaluate_all_coordinator()
15460 }
15461
15462 fn evaluate_all_coordinator(&mut self) -> Result<EvalResult, ExcelError> {
15465 self.require_unified_authority()?;
15466 self.begin_evaluation_request();
15467 self.evaluate_all_legacy_impl()
15468 }
15469
15470 fn legacy_pass_run_units(
15476 &mut self,
15477 schedule: &crate::engine::scheduler::Schedule,
15478 ) -> Result<(usize, usize), ExcelError> {
15479 let mut computed_vertices = 0;
15480 let mut cycle_count = 0;
15481 self.begin_pass(schedule);
15482 for (unit_index, &unit) in schedule.units.iter().enumerate() {
15483 match unit {
15484 ScheduleUnit::Cycle(i) => {
15485 if self.handle_cycle_unit(schedule.unit_cycle(i), None, None, None)? > 0 {
15486 cycle_count += 1;
15487 }
15488 }
15489 ScheduleUnit::Layer(i) => {
15490 let layer = schedule.unit_layer(i);
15491 if self.thread_pool.is_some() && layer.vertices.len() > 1 {
15492 computed_vertices += self.evaluate_layer_parallel(layer)?;
15493 } else {
15494 computed_vertices += self.evaluate_layer_sequential(layer)?;
15495 }
15496 }
15497 }
15498 if self.stop_after_unit(schedule, unit_index) {
15499 break;
15500 }
15501 }
15502 Ok((computed_vertices, cycle_count))
15503 }
15504
15505 fn evaluate_all_legacy_impl(&mut self) -> Result<EvalResult, ExcelError> {
15512 self.reset_virtual_dep_telemetry_if_disabled();
15513 let _span_eval =
15514 crate::engine::trace::fz_span!(tracing::Level::INFO, "evaluate", "evaluate.legacy");
15515 let start = crate::instant::FzInstant::now();
15516 let mut computed_vertices = 0;
15517 let mut cycle_errors = 0;
15518 let mut replan_iterations = 0;
15519 const MAX_REPLAN: usize = 5;
15520 let mut telemetry = self
15521 .config
15522 .enable_virtual_dep_telemetry
15523 .then(|| self.start_virtual_dep_telemetry());
15524
15525 loop {
15526 let to_evaluate = self.graph.get_evaluation_vertices();
15527 if to_evaluate.is_empty() {
15528 if let Some(t) = telemetry.as_mut()
15529 && t.bailout_reason.is_none()
15530 {
15531 t.bailout_reason = Some("no_work");
15532 }
15533 break;
15534 }
15535
15536 let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
15537 if let Some(t) = telemetry.as_mut() {
15538 Self::accumulate_schedule_meta(t, &meta);
15539 }
15540
15541 let (pass_computed, pass_cycles) = self.legacy_pass_run_units(&schedule)?;
15542 computed_vertices += pass_computed;
15543 cycle_errors += pass_cycles;
15544
15545 let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
15547 if let Some(t) = telemetry.as_mut() {
15548 t.changed_vdeps_total += changed_vertices.len();
15549 }
15550
15551 self.resource_checkpoint(0)?;
15552 if !self.finish_pass_dirty(&to_evaluate, &changed_vertices) {
15553 if let Some(t) = telemetry.as_mut() {
15554 t.bailout_reason = Some("converged");
15555 }
15556 break;
15557 }
15558 if replan_iterations >= MAX_REPLAN {
15559 if let Some(mut t) = telemetry.take() {
15560 t.bailout_reason = Some("max_replan");
15561 t.replan_iterations = replan_iterations;
15562 self.last_virtual_dep_telemetry = t;
15563 }
15564 return Err(
15565 self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
15566 );
15567 }
15568
15569 replan_iterations += 1;
15570 }
15571
15572 if let Some(mut t) = telemetry {
15573 t.replan_iterations = replan_iterations;
15574 self.last_virtual_dep_telemetry = t;
15575 }
15576
15577 self.redirty_for_next_recalc();
15579
15580 self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
15582
15583 Ok(EvalResult {
15584 computed_vertices,
15585 cycle_errors,
15586 elapsed: start.elapsed(),
15587 })
15588 }
15589
15590 pub fn evaluate_all_with_target_delta(
15591 &mut self,
15592 ) -> Result<(EvalResult, crate::engine::TargetEvalDelta), ExcelError> {
15593 self.observe_evaluation_resource_request(EvaluationRequestKind::FullWithDelta, |engine| {
15594 engine.observe_function_semantic_epoch()?;
15595 let mut collector = DeltaCollector::new(DeltaMode::Cells);
15596 let result = engine.evaluate_all_with_delta_collector(&mut collector)?;
15597 Ok((result, collector.finish_target()))
15598 })
15599 }
15600
15601 pub fn evaluate_all_with_delta(&mut self) -> Result<(EvalResult, EvalDelta), ExcelError> {
15602 self.evaluate_all_with_delta_policy(EvalDeltaCompatibilityPolicy::Unlimited)
15603 }
15604
15605 pub fn evaluate_all_with_delta_policy(
15606 &mut self,
15607 policy: EvalDeltaCompatibilityPolicy,
15608 ) -> Result<(EvalResult, EvalDelta), ExcelError> {
15609 self.observe_evaluation_resource_request(EvaluationRequestKind::FullWithDelta, |engine| {
15610 engine.observe_function_semantic_epoch()?;
15611 let mut collector = DeltaCollector::new(DeltaMode::Cells);
15612 let result = engine.evaluate_all_with_delta_collector(&mut collector)?;
15613 Ok((result, collector.finish_with_policy(policy)?))
15614 })
15615 }
15616
15617 fn evaluate_all_with_delta_collector(
15618 &mut self,
15619 delta: &mut DeltaCollector,
15620 ) -> Result<EvalResult, ExcelError> {
15621 let _source_cache = self.source_cache_session();
15622 if self.config.defer_graph_building {
15623 self.build_graph_all()?;
15624 }
15625 self.require_unified_authority()?;
15626 self.begin_evaluation_request();
15627 self.reset_virtual_dep_telemetry_if_disabled();
15628 let _span_eval = crate::engine::trace::fz_span!(
15629 tracing::Level::INFO,
15630 "evaluate",
15631 "evaluate.legacy_delta"
15632 );
15633 let start = crate::instant::FzInstant::now();
15634 let mut computed_vertices = 0;
15635 let mut cycle_errors = 0;
15636
15637 let mut replan_iterations = 0;
15638 const MAX_REPLAN: usize = 5;
15639 let mut telemetry = self
15640 .config
15641 .enable_virtual_dep_telemetry
15642 .then(|| self.start_virtual_dep_telemetry());
15643
15644 loop {
15645 let to_evaluate = self.graph.get_evaluation_vertices();
15646 if to_evaluate.is_empty() {
15647 if let Some(t) = telemetry.as_mut()
15648 && t.bailout_reason.is_none()
15649 {
15650 t.bailout_reason = Some("no_work");
15651 }
15652 break;
15653 }
15654
15655 let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
15656 if let Some(t) = telemetry.as_mut() {
15657 Self::accumulate_schedule_meta(t, &meta);
15658 }
15659
15660 self.begin_pass(&schedule);
15661 for (unit_index, &unit) in schedule.units.iter().enumerate() {
15662 match unit {
15663 ScheduleUnit::Cycle(i) => {
15664 if self.handle_cycle_unit(
15665 schedule.unit_cycle(i),
15666 Some(delta),
15667 None,
15668 None,
15669 )? > 0
15670 {
15671 cycle_errors += 1;
15672 }
15673 }
15674 ScheduleUnit::Layer(i) => {
15675 let layer = schedule.unit_layer(i);
15676 if self.thread_pool.is_some() && layer.vertices.len() > 1 {
15677 computed_vertices +=
15678 self.evaluate_layer_parallel_with_delta(layer, delta)?;
15679 } else {
15680 computed_vertices +=
15681 self.evaluate_layer_sequential_with_delta(layer, delta)?;
15682 }
15683 }
15684 }
15685 if self.stop_after_unit(&schedule, unit_index) {
15686 break;
15687 }
15688 }
15689
15690 let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
15691 if let Some(t) = telemetry.as_mut() {
15692 t.changed_vdeps_total += changed_vertices.len();
15693 }
15694 self.resource_checkpoint(0)?;
15695 if !self.finish_pass_dirty(&to_evaluate, &changed_vertices) {
15696 if let Some(t) = telemetry.as_mut() {
15697 t.bailout_reason = Some("converged");
15698 }
15699 break;
15700 }
15701 if replan_iterations >= MAX_REPLAN {
15702 if let Some(mut t) = telemetry.take() {
15703 t.bailout_reason = Some("max_replan");
15704 t.replan_iterations = replan_iterations;
15705 self.last_virtual_dep_telemetry = t;
15706 }
15707 return Err(
15708 self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
15709 );
15710 }
15711 replan_iterations += 1;
15712 }
15713
15714 if let Some(mut t) = telemetry {
15715 t.replan_iterations = replan_iterations;
15716 self.last_virtual_dep_telemetry = t;
15717 }
15718
15719 self.redirty_for_next_recalc();
15720 self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
15721
15722 Ok(EvalResult {
15723 computed_vertices,
15724 cycle_errors,
15725 elapsed: start.elapsed(),
15726 })
15727 }
15728
15729 pub fn evaluate_cell(
15739 &mut self,
15740 sheet: &str,
15741 row: u32,
15742 col: u32,
15743 ) -> Result<Option<LiteralValue>, ExcelError> {
15744 self.observe_evaluation_resource_request(EvaluationRequestKind::Cell, |engine| {
15745 engine.evaluate_cell_unobserved(sheet, row, col)
15746 })
15747 }
15748
15749 fn evaluate_cell_unobserved(
15750 &mut self,
15751 sheet: &str,
15752 row: u32,
15753 col: u32,
15754 ) -> Result<Option<LiteralValue>, ExcelError> {
15755 if row == 0 || col == 0 {
15756 return Err(ExcelError::new(ExcelErrorKind::Ref)
15757 .with_message("Row and column must be >= 1".to_string()));
15758 }
15759
15760 let result = self.evaluate_cells(&[(sheet, row, col)])?;
15761
15762 match result.len() {
15763 0 => Ok(None),
15764 1 => {
15765 let v = result.into_iter().next().unwrap();
15766 Ok(v)
15767 }
15768 _ => unreachable!("evaluate_cells returned unexpected length"),
15769 }
15770 }
15771
15772 pub fn evaluate_cells(
15779 &mut self,
15780 targets: &[(&str, u32, u32)],
15781 ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
15782 self.observe_evaluation_resource_request(EvaluationRequestKind::Cells, |engine| {
15783 engine.evaluate_cells_unobserved(targets)
15784 })
15785 }
15786
15787 fn evaluate_cells_unobserved(
15788 &mut self,
15789 targets: &[(&str, u32, u32)],
15790 ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
15791 self.observe_function_semantic_epoch()?;
15792 debug_assert!(
15793 !self.graph.deferred_dirty_active(),
15794 "deferred-dirty scope leaked into evaluate_cells: a begin_deferred_dirty \
15795 was not balanced by end_deferred_dirty"
15796 );
15797 self.validate_deterministic_mode()?;
15798 if targets.is_empty() {
15799 return Ok(Vec::new());
15800 }
15801 let typed_targets = self.legacy_coordinate_targets(targets);
15802 self.evaluate_mixed_targets(&typed_targets, None)?;
15803 Ok(targets
15804 .iter()
15805 .map(|(s, r, c)| self.get_cell_value(s, *r, *c))
15806 .collect())
15807 }
15808
15809 pub fn evaluate_cells_cancellable(
15810 &mut self,
15811 targets: &[(&str, u32, u32)],
15812 cancel: crate::engine::CancelToken,
15813 ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
15814 self.observe_evaluation_resource_request(
15815 EvaluationRequestKind::CellsCancellable,
15816 |engine| {
15817 engine.observe_function_semantic_epoch()?;
15818 engine.active_cancel_flag = Some(cancel.clone());
15819 let res = engine.evaluate_cells_cancellable_impl(targets, cancel.as_flag());
15820 engine.active_cancel_flag = None;
15821 res
15822 },
15823 )
15824 }
15825
15826 fn evaluate_cells_cancellable_impl(
15827 &mut self,
15828 targets: &[(&str, u32, u32)],
15829 cancel_flag: &AtomicBool,
15830 ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
15831 self.validate_deterministic_mode()?;
15832 if targets.is_empty() {
15833 return Ok(Vec::new());
15834 }
15835 if cancel_flag.load(Ordering::Relaxed) {
15836 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
15837 .with_message("Evaluation cancelled before target preparation"));
15838 }
15839 let typed_targets = self.legacy_coordinate_targets(targets);
15840 self.evaluate_mixed_targets(&typed_targets, None)?;
15841 Ok(targets
15842 .iter()
15843 .map(|(sheet, row, col)| self.get_cell_value(sheet, *row, *col))
15844 .collect())
15845 }
15846
15847 pub fn evaluate_cells_with_target_delta(
15848 &mut self,
15849 targets: &[(&str, u32, u32)],
15850 ) -> Result<(Vec<Option<LiteralValue>>, crate::engine::TargetEvalDelta), ExcelError> {
15851 self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
15852 engine.observe_function_semantic_epoch()?;
15853 engine.validate_deterministic_mode()?;
15854 if targets.is_empty() {
15855 return Ok((Vec::new(), crate::engine::TargetEvalDelta::default()));
15856 }
15857 let mut collector = DeltaCollector::new(DeltaMode::Cells);
15858 engine.evaluate_until_with_delta_collector(targets, &mut collector)?;
15859 let values = targets
15860 .iter()
15861 .map(|(sheet, row, col)| engine.get_cell_value(sheet, *row, *col))
15862 .collect();
15863 Ok((values, collector.finish_target()))
15864 })
15865 }
15866
15867 pub fn evaluate_cells_with_delta(
15868 &mut self,
15869 targets: &[(&str, u32, u32)],
15870 ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
15871 self.evaluate_cells_with_delta_policy(targets, EvalDeltaCompatibilityPolicy::Unlimited)
15872 }
15873
15874 pub fn evaluate_cells_with_delta_policy(
15875 &mut self,
15876 targets: &[(&str, u32, u32)],
15877 policy: EvalDeltaCompatibilityPolicy,
15878 ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
15879 self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
15880 engine.evaluate_cells_with_delta_unobserved(targets, policy)
15881 })
15882 }
15883
15884 fn evaluate_cells_with_delta_unobserved(
15885 &mut self,
15886 targets: &[(&str, u32, u32)],
15887 policy: EvalDeltaCompatibilityPolicy,
15888 ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
15889 self.observe_function_semantic_epoch()?;
15890 self.validate_deterministic_mode()?;
15891 if targets.is_empty() {
15892 return Ok((Vec::new(), EvalDelta::default()));
15893 }
15894 let mut collector = DeltaCollector::new(DeltaMode::Cells);
15895 self.evaluate_until_with_delta_collector(targets, &mut collector)?;
15896 let values = targets
15897 .iter()
15898 .map(|(s, r, c)| self.get_cell_value(s, *r, *c))
15899 .collect();
15900 Ok((values, collector.finish_with_policy(policy)?))
15901 }
15902
15903 pub fn get_eval_plan(&self, targets: &[(&str, u32, u32)]) -> Result<EvalPlan, ExcelError> {
15905 if targets.is_empty() {
15906 return Ok(EvalPlan {
15907 total_vertices_to_evaluate: 0,
15908 layers: Vec::new(),
15909 cycles_detected: 0,
15910 dirty_count: 0,
15911 volatile_count: 0,
15912 parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
15913 estimated_parallel_layers: 0,
15914 target_cells: Vec::new(),
15915 });
15916 }
15917 if self.config.defer_graph_building && self.has_staged_formulas() {
15918 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
15919 "Evaluation plan requested with deferred graph; build first or call evaluate_*",
15920 ));
15921 }
15922
15923 let addresses: Vec<String> = targets
15925 .iter()
15926 .map(|(s, r, c)| format!("{}!{}{}", s, Self::col_to_letters(*c), r))
15927 .collect();
15928
15929 let mut target_addrs = Vec::new();
15931 for (sheet, row, col) in targets {
15932 if let Some(sheet_id) = self.graph.sheet_id(sheet) {
15933 let coord = Coord::from_excel(*row, *col, true, true);
15934 target_addrs.push(CellRef::new(sheet_id, coord));
15935 }
15936 }
15937
15938 let mut target_vertex_ids = Vec::new();
15940 for addr in &target_addrs {
15941 if let Some(vertex_id) = self.graph.get_vertex_id_for_address(addr) {
15942 target_vertex_ids.push(vertex_id);
15943 }
15944 }
15945
15946 if target_vertex_ids.is_empty() {
15947 return Ok(EvalPlan {
15948 total_vertices_to_evaluate: 0,
15949 layers: Vec::new(),
15950 cycles_detected: 0,
15951 dirty_count: 0,
15952 volatile_count: 0,
15953 parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
15954 estimated_parallel_layers: 0,
15955 target_cells: addresses,
15956 });
15957 }
15958
15959 let (precedents_to_eval, vdeps) = self.demand_subgraph(&target_vertex_ids)?;
15961
15962 if precedents_to_eval.is_empty() {
15963 return Ok(EvalPlan {
15964 total_vertices_to_evaluate: 0,
15965 layers: Vec::new(),
15966 cycles_detected: 0,
15967 dirty_count: 0,
15968 volatile_count: 0,
15969 parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
15970 estimated_parallel_layers: 0,
15971 target_cells: addresses,
15972 });
15973 }
15974
15975 let mut dirty_count = 0;
15977 let mut volatile_count = 0;
15978 for &vertex_id in &precedents_to_eval {
15979 if self.graph.is_dirty(vertex_id) {
15980 dirty_count += 1;
15981 }
15982 if self.graph.is_volatile(vertex_id) {
15983 volatile_count += 1;
15984 }
15985 }
15986
15987 let schedule = self.create_authority_schedule(&precedents_to_eval, &vdeps, None)?;
15989
15990 let mut layers = Vec::new();
15992 let mut estimated_parallel_layers = 0;
15993 let parallel_enabled = self.config.enable_parallel && self.thread_pool.is_some();
15994
15995 for layer in &schedule.layers {
15996 let parallel_eligible = parallel_enabled && layer.vertices.len() > 1;
15997 if parallel_eligible {
15998 estimated_parallel_layers += 1;
15999 }
16000
16001 let sample_cells: Vec<String> = layer
16003 .vertices
16004 .iter()
16005 .take(5)
16006 .filter_map(|&vertex_id| {
16007 self.graph
16008 .get_cell_ref_for_vertex(vertex_id)
16009 .map(|cell_ref| {
16010 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
16011 format!(
16012 "{}!{}{}",
16013 sheet_name,
16014 Self::col_to_letters(cell_ref.coord.col().saturating_add(1)),
16015 cell_ref.coord.row() + 1
16016 )
16017 })
16018 })
16019 .collect();
16020
16021 layers.push(LayerInfo {
16022 vertex_count: layer.vertices.len(),
16023 parallel_eligible,
16024 sample_cells,
16025 });
16026 }
16027
16028 Ok(EvalPlan {
16029 total_vertices_to_evaluate: precedents_to_eval.len(),
16030 layers,
16031 cycles_detected: schedule.cycles.len(),
16032 dirty_count,
16033 volatile_count,
16034 parallel_enabled,
16035 estimated_parallel_layers,
16036 target_cells: addresses,
16037 })
16038 }
16039 fn create_evaluation_schedule(
16041 &mut self,
16042 to_evaluate: &[VertexId],
16043 ) -> Result<EvaluationScheduleBuildOutput, ExcelError> {
16044 #[cfg(any(test, feature = "benchmark_internal"))]
16045 {
16046 self.recalc_reuse_probe.get_mut().unwrap().schedule_requests += 1;
16047 }
16048 #[cfg(any(test, feature = "legacy_oracle"))]
16051 self.graph.flush_pending_edge_deltas();
16052 self.graph.authority_sync();
16054 if self.can_use_static_schedule_cache(to_evaluate) {
16055 let revision = self.schedule_cache_authority_revision();
16057 let current = |e: &CachedScheduleEntry| {
16058 e.topology_epoch == self.topology_epoch && e.authority_revision == revision
16059 };
16060 if !self
16061 .cached_static_schedule
16062 .as_ref()
16063 .is_some_and(|c| current(c) && c.candidate_vertices.equals(to_evaluate))
16064 && let Some(i) = self.recent_schedules.iter().position(|e| {
16065 current(e)
16066 && e.candidate_vertices.len() == to_evaluate.len()
16067 && e.candidate_vertices.equals(to_evaluate)
16068 })
16069 {
16070 let hit = self.recent_schedules.remove(i);
16071 if let Some(previous) = self.cached_static_schedule.replace(hit) {
16072 self.retain_recent_schedule(previous);
16073 }
16074 }
16075 if let Some(cached) = self.cached_static_schedule.as_ref()
16076 && cached.topology_epoch == self.topology_epoch
16077 && cached.authority_revision == self.schedule_cache_authority_revision()
16078 && cached.candidate_vertices.equals(to_evaluate)
16079 {
16080 let meta = ScheduleBuildMeta {
16081 candidate_vertices: to_evaluate.len(),
16082 vdeps_vertices: 0,
16083 vdeps_edges: 0,
16084 builder_elapsed_ms: 0,
16085 used_virtual_schedule: false,
16086 schedule_cache_hit: true,
16087 schedule_cache_eligible: true,
16088 };
16089 #[cfg(any(test, feature = "benchmark_internal"))]
16090 {
16091 let mut probe = self.recalc_reuse_probe.lock().unwrap();
16092 probe.schedule_cache_hits += 1;
16093 probe.schedule_shared_handles += 1;
16094 }
16095 return Ok((
16096 EvaluationSchedule::Shared(Arc::clone(&cached.schedule)),
16097 FxHashMap::default(),
16098 meta,
16099 ));
16100 }
16101
16102 let (schedule, vdeps, mut meta) = match self.schedule_from_base(to_evaluate)? {
16103 Some(schedule) => (
16104 schedule,
16105 FxHashMap::default(),
16106 ScheduleBuildMeta {
16107 candidate_vertices: to_evaluate.len(),
16108 vdeps_vertices: 0,
16109 vdeps_edges: 0,
16110 builder_elapsed_ms: 0,
16111 used_virtual_schedule: false,
16112 schedule_cache_hit: false,
16113 schedule_cache_eligible: true,
16114 },
16115 ),
16116 None => self.create_evaluation_schedule_active(to_evaluate)?,
16117 };
16118 meta.schedule_cache_hit = false;
16119 meta.schedule_cache_eligible = true;
16120 #[cfg(any(test, feature = "benchmark_internal"))]
16121 {
16122 self.recalc_reuse_probe
16123 .get_mut()
16124 .unwrap()
16125 .schedule_cache_misses += 1;
16126 }
16127 let schedule = if vdeps.is_empty() {
16128 let mut schedule = schedule;
16131 schedule.units.shrink_to_fit();
16132 for layer in &mut schedule.layers {
16133 layer.vertices.shrink_to_fit();
16134 }
16135 schedule.layers.shrink_to_fit();
16136 for cycle in &mut schedule.cycles {
16137 cycle.shrink_to_fit();
16138 }
16139 schedule.cycles.shrink_to_fit();
16140 let schedule = Arc::new(schedule);
16141 #[cfg(any(test, feature = "benchmark_internal"))]
16142 {
16143 self.recalc_reuse_probe
16144 .get_mut()
16145 .unwrap()
16146 .schedule_shared_handles += 1;
16147 }
16148 let entry = CachedScheduleEntry {
16149 topology_epoch: self.topology_epoch,
16150 authority_revision: self.schedule_cache_authority_revision(),
16151 candidate_vertices: VertexIdRuns::from_slice(to_evaluate),
16152 schedule: Arc::clone(&schedule),
16153 };
16154 if let Some(previous) = self.cached_static_schedule.replace(entry) {
16155 self.retain_recent_schedule(previous);
16156 }
16157 EvaluationSchedule::Shared(schedule)
16158 } else {
16159 EvaluationSchedule::Owned(schedule)
16160 };
16161 return Ok((schedule, vdeps, meta));
16162 }
16163
16164 let (schedule, vdeps, mut meta) = self.create_evaluation_schedule_active(to_evaluate)?;
16165 meta.schedule_cache_hit = false;
16166 meta.schedule_cache_eligible = false;
16167 #[cfg(any(test, feature = "benchmark_internal"))]
16168 {
16169 self.recalc_reuse_probe
16170 .get_mut()
16171 .unwrap()
16172 .schedule_cache_ineligible += 1;
16173 }
16174 Ok((EvaluationSchedule::Owned(schedule), vdeps, meta))
16175 }
16176
16177 fn schedule_from_base(
16183 &mut self,
16184 to_evaluate: &[VertexId],
16185 ) -> Result<Option<crate::engine::scheduler::Schedule>, ExcelError> {
16186 let revision = self.schedule_cache_authority_revision();
16187 let current = |e: &&CachedScheduleEntry| {
16188 e.topology_epoch == self.topology_epoch && e.authority_revision == revision
16189 };
16190 let Some(base) = [
16193 self.base_schedule.as_ref(),
16194 self.cached_static_schedule.as_ref(),
16195 ]
16196 .into_iter()
16197 .flatten()
16198 .filter(current)
16199 .max_by_key(|e| e.candidate_vertices.len()) else {
16200 return Ok(None);
16201 };
16202 let base_len = base.candidate_vertices.len();
16203 if base_len <= BASE_SCHEDULE_MIN_VERTICES
16204 || to_evaluate.len() < BASE_SCHEDULE_MIN_REQUEST
16205 || to_evaluate.len() > base_len
16206 || base_len > to_evaluate.len().saturating_mul(BASE_SCHEDULE_RATIO)
16207 {
16208 return Ok(None);
16209 }
16210 let mut keep = crate::engine::idset::DenseIdSet::default();
16211 keep.extend(to_evaluate.iter().copied());
16212 let Some((schedule, kept)) = base.schedule.restrict(&keep) else {
16213 return Ok(None);
16214 };
16215 if kept != keep.len() {
16217 return Ok(None);
16218 }
16219 if let Some(ledger) = self.active_resource_ledger.as_mut() {
16220 let layers: usize = schedule
16221 .layers
16222 .iter()
16223 .map(|l| {
16224 l.vertices.capacity() * std::mem::size_of::<VertexId>()
16225 + l.runs.capacity()
16226 * std::mem::size_of::<crate::engine::scheduler::LayerRun>()
16227 })
16228 .sum();
16229 let bytes = (keep.heap_bytes()
16230 + layers
16231 + schedule.layers.capacity()
16232 * std::mem::size_of::<crate::engine::scheduler::Layer>()
16233 + schedule.units.capacity()
16234 * std::mem::size_of::<crate::engine::scheduler::ScheduleUnit>()
16235 + schedule
16236 .cycles
16237 .iter()
16238 .map(|c| c.capacity() * 4)
16239 .sum::<usize>()) as u64;
16240 ledger
16241 .reserve_schedule_discovery(bytes)
16242 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16243 ledger
16244 .release_scratch(bytes)
16245 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16246 }
16247 #[cfg(debug_assertions)]
16248 self.debug_check_restricted_schedule(to_evaluate, &schedule);
16249 #[cfg(any(test, feature = "benchmark_internal"))]
16250 {
16251 self.recalc_reuse_probe
16252 .get_mut()
16253 .unwrap()
16254 .schedule_base_restrictions += 1;
16255 }
16256 Ok(Some(schedule))
16257 }
16258
16259 #[cfg(debug_assertions)]
16263 fn debug_check_restricted_schedule(
16264 &self,
16265 to_evaluate: &[VertexId],
16266 schedule: &crate::engine::scheduler::Schedule,
16267 ) {
16268 let mut position: FxHashMap<VertexId, usize> = FxHashMap::default();
16269 for (u, unit) in schedule.units.iter().enumerate() {
16270 let vertices: &[VertexId] = match *unit {
16271 crate::engine::scheduler::ScheduleUnit::Layer(i) => {
16272 &schedule.unit_layer(i).vertices
16273 }
16274 crate::engine::scheduler::ScheduleUnit::Cycle(i) => schedule.unit_cycle(i),
16275 };
16276 for &v in vertices {
16277 assert!(
16278 position.insert(v, u).is_none(),
16279 "vertex twice in a restricted schedule"
16280 );
16281 }
16282 }
16283 assert_eq!(position.len(), to_evaluate.len());
16284 if to_evaluate.len() > 512 {
16285 return;
16286 }
16287 let Ok(store) = self.graph.authority_plan_store() else {
16290 return;
16291 };
16292 let cells: Vec<(VertexId, (u16, u32, u32))> = to_evaluate
16293 .iter()
16294 .filter_map(|&v| self.graph.authority_cell_of_vertex(v).map(|c| (v, c)))
16295 .collect();
16296 for &(v, (sheet, row, col)) in &cells {
16297 let Some(owner) = store.owner_at((sheet, row, col)) else {
16298 continue;
16299 };
16300 let Ok(refined) = store.refine_owner_column(owner, col, row, row, None) else {
16301 continue;
16302 };
16303 for piece in &refined.pieces {
16304 for edge in &refined.edges[piece.edge_start..piece.edge_end] {
16305 let Some(image) = edge.proj.forward(&piece.domain) else {
16306 continue;
16307 };
16308 for &(d, (s2, r2, c2)) in &cells {
16309 if d == v
16310 || edge.proj.sheet != s2
16311 || !(image.r0 <= r2
16312 && r2 <= image.r1
16313 && image.c0 <= c2
16314 && c2 <= image.c1)
16315 {
16316 continue;
16317 }
16318 let (pd, pv) = (position[&d], position[&v]);
16319 let same_ok = pd == pv
16320 && match schedule.units[pv] {
16321 crate::engine::scheduler::ScheduleUnit::Layer(i) => {
16322 schedule.unit_layer(i).sequential
16323 }
16324 crate::engine::scheduler::ScheduleUnit::Cycle(_) => true,
16325 };
16326 assert!(
16327 pd < pv || same_ok,
16328 "restricted schedule orders {v:?} before its precedent {d:?}"
16329 );
16330 }
16331 }
16332 }
16333 }
16334 }
16335
16336 fn maybe_compress_formulas(&mut self) {
16340 if !self.config.formula_compression {
16341 return;
16342 }
16343 let builds = self.graph.authority_host().builds;
16344 if self.compressed_at_build == Some(builds) {
16345 return;
16346 }
16347 self.compressed_at_build = Some(builds);
16348 if self.has_staged_formulas() {
16349 self.graph.virtualize_family_members();
16352 return;
16353 }
16354 let pool = self.thread_pool.clone();
16355 let (_, garbage) = self.graph.compress_family_formulas(pool.as_deref());
16356 self.graph.virtualize_family_members();
16357 match pool {
16361 Some(pool) if garbage.len() >= 1024 => pool.spawn(move || drop(garbage)),
16362 _ => drop(garbage),
16363 }
16364 }
16365
16366 fn create_evaluation_schedule_active(
16367 &mut self,
16368 to_evaluate: &[VertexId],
16369 ) -> Result<ScheduleBuildOutput, ExcelError> {
16370 self.graph.authority_sync();
16371 self.maybe_compress_formulas();
16372 let mut ledger = self.active_resource_ledger.take();
16373 let result = self.create_evaluation_schedule_uncached(to_evaluate, ledger.as_mut());
16374 self.active_resource_ledger = ledger;
16375 result
16376 }
16377
16378 fn create_evaluation_schedule_uncached(
16379 &self,
16380 to_evaluate: &[VertexId],
16381 #[allow(unused_variables)] ledger: Option<&mut ResourceLedger>,
16382 ) -> Result<ScheduleBuildOutput, ExcelError> {
16383 #[cfg(any(test, feature = "benchmark_internal"))]
16384 {
16385 self.recalc_reuse_probe.lock().unwrap().schedule_builds += 1;
16386 }
16387 let builder = VirtualDepBuilder::new(self);
16388 #[allow(unused_mut)]
16389 let (mut vdeps, augmented, builder_elapsed_ms, vdeps_edges) =
16390 if self.config.enable_virtual_dep_telemetry {
16391 let build_started = crate::instant::FzInstant::now();
16392 let (vdeps, augmented) = builder.build(to_evaluate);
16393 let builder_elapsed_ms = build_started.elapsed().as_millis();
16394 let vdeps_edges = vdeps.values().map(|deps| deps.len()).sum::<usize>();
16395 (vdeps, augmented, builder_elapsed_ms, vdeps_edges)
16396 } else {
16397 let (vdeps, augmented) = builder.build(to_evaluate);
16398 (vdeps, augmented, 0, 0)
16399 };
16400
16401 {
16403 self.freshness_merge_hints(to_evaluate, &mut vdeps);
16404 self.freshness_extent_hints(to_evaluate, &mut vdeps);
16405 }
16406 let mut final_evaluate = to_evaluate.to_vec();
16407 if !augmented.is_empty() {
16408 final_evaluate.extend(augmented);
16409 final_evaluate.sort_unstable();
16410 final_evaluate.dedup();
16411 }
16412
16413 let use_virtual = !vdeps.is_empty();
16414
16415 let schedule = self.create_authority_schedule(&final_evaluate, &vdeps, ledger)?;
16416
16417 let meta = ScheduleBuildMeta {
16418 candidate_vertices: to_evaluate.len(),
16419 vdeps_vertices: vdeps.len(),
16420 vdeps_edges,
16421 builder_elapsed_ms,
16422 used_virtual_schedule: use_virtual,
16423 schedule_cache_hit: false,
16424 schedule_cache_eligible: false,
16425 };
16426
16427 Ok((schedule, vdeps, meta))
16428 }
16429
16430 fn create_authority_schedule(
16431 &self,
16432 candidates: &[VertexId],
16433 vdeps: &FxHashMap<VertexId, Vec<VertexId>>,
16434 mut ledger: Option<&mut ResourceLedger>,
16435 ) -> Result<crate::engine::scheduler::Schedule, ExcelError> {
16436 use crate::engine::authority::{
16437 geom::{Cover, Rect},
16438 plan_schedule, planner,
16439 proj::{AxisMap, RefProj},
16440 store::{EdgeKey, Tag},
16441 };
16442 let failure = |message: String| {
16443 ExcelError::new(ExcelErrorKind::Error)
16444 .with_message(format!("unified_authority planner: {message}"))
16445 };
16446 self.cancellation_checkpoint("Evaluation cancelled before authority planning")?;
16447 let store = self
16448 .graph
16449 .authority_plan_store()
16450 .map_err(Self::authority_excel_error)?;
16451 if let [only] = candidates
16454 && vdeps.is_empty()
16455 && self.graph.authority_host().observed(*only).is_none()
16456 && let Some(cell) = self.graph.authority_cell_of_vertex(*only)
16457 && cell.0 != crate::engine::authority::geom::SYMBOL_SHEET
16458 && let Some(single) = planner::plan_single(store, cell)
16459 {
16460 #[cfg(debug_assertions)]
16461 {
16462 let mut cover = Cover::new();
16463 cover.insert_rect(cell.0, &Rect::new(cell.1, cell.2, cell.1, cell.2));
16464 let general = planner::plan_with_hints(store, &cover, &[], None, None, None)
16465 .expect("general plan of one cell");
16466 assert_eq!(
16467 general.cells.as_slice(),
16468 &[single],
16469 "single-cell plan differs from the planner at {cell:?}"
16470 );
16471 }
16472 let adapted = plan_schedule::schedule(
16473 &[single],
16474 0,
16475 None,
16476 |cell| {
16477 self.graph
16478 .authority_vertex_of_formula(cell.id, (cell.sheet, cell.row, cell.col))
16479 .ok_or_else(|| failure("missing executor identity".to_owned()))
16480 },
16481 |_work| Ok(()),
16482 )
16483 .map_err(|error| match error {
16484 plan_schedule::ScheduleError::Runtime(error) => error,
16485 other => failure(format!("{other:?}")),
16486 })?;
16487 if let Some(ledger) = ledger {
16488 ledger
16489 .reserve_schedule_discovery(adapted.peak_heap_bytes)
16490 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16491 ledger
16492 .release_scratch(adapted.peak_heap_bytes)
16493 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16494 }
16495 return Ok(adapted.schedule);
16496 }
16497 if (2..=planner::SMALL_PLAN_MAX).contains(&candidates.len())
16500 && vdeps.is_empty()
16501 && candidates
16502 .iter()
16503 .all(|&v| self.graph.authority_host().observed(v).is_none())
16504 {
16505 let cells: Option<Vec<(u16, u32, u32)>> = candidates
16506 .iter()
16507 .map(|&v| self.graph.authority_cell_of_vertex(v))
16508 .collect();
16509 if let Some(small) = cells.as_deref().and_then(|c| planner::plan_small(store, c)) {
16510 #[cfg(debug_assertions)]
16511 {
16512 let mut cover = Cover::new();
16513 for c in cells.as_deref().unwrap_or_default() {
16514 cover.insert_rect(c.0, &Rect::new(c.1, c.2, c.1, c.2));
16515 }
16516 let general = planner::plan_with_hints(store, &cover, &[], None, None, None)
16517 .expect("general plan of a small request");
16518 let key = |c: &planner::OrderedCell| (c.sheet, c.row, c.col, c.id, c.owner);
16519 let mut a: Vec<_> = general.cells.iter().map(key).collect();
16520 let mut b: Vec<_> = small.iter().map(key).collect();
16521 a.sort_unstable();
16522 b.sort_unstable();
16523 assert_eq!(a, b, "small plan cells differ from the planner");
16524 assert!(
16525 general.cells.iter().all(|c| c.cycle.is_none()),
16526 "small plan of a cyclic request"
16527 );
16528 }
16529 let adapted = plan_schedule::schedule(
16530 &small,
16531 0,
16532 None,
16533 |cell| {
16534 self.graph
16535 .authority_vertex_of_formula(cell.id, (cell.sheet, cell.row, cell.col))
16536 .ok_or_else(|| failure("missing executor identity".to_owned()))
16537 },
16538 |_work| Ok(()),
16539 )
16540 .map_err(|error| match error {
16541 plan_schedule::ScheduleError::Runtime(error) => error,
16542 other => failure(format!("{other:?}")),
16543 })?;
16544 if let Some(ledger) = ledger {
16545 ledger
16546 .reserve_schedule_discovery(adapted.peak_heap_bytes)
16547 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16548 ledger
16549 .release_scratch(adapted.peak_heap_bytes)
16550 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16551 }
16552 return Ok(adapted.schedule);
16553 }
16554 }
16555 let mut cover = Cover::new();
16560 {
16561 let cell_of = |&id: &VertexId| {
16562 self.graph
16563 .authority_cell_of_vertex(id)
16564 .map(|(sheet, row, col)| (sheet, col, row))
16565 };
16566 let cells: Vec<(u16, u32, u32)> = match self.thread_pool.as_deref() {
16569 Some(pool) if candidates.len() >= PARALLEL_SCHEDULE_MIN_CANDIDATES => {
16570 use rayon::prelude::*;
16571 pool.install(|| {
16572 let mut cells: Vec<_> = candidates.par_iter().filter_map(cell_of).collect();
16573 cells.par_sort_unstable();
16574 cells
16575 })
16576 }
16577 _ => {
16578 let mut cells: Vec<_> = candidates.iter().filter_map(cell_of).collect();
16579 cells.sort_unstable();
16580 cells
16581 }
16582 };
16583 let mut i = 0;
16584 while i < cells.len() {
16585 let (sheet, col, r0) = cells[i];
16586 let mut r1 = r0;
16587 let mut j = i + 1;
16588 while j < cells.len() && cells[j].0 == sheet && cells[j].1 == col {
16589 if cells[j].2 > r1 + 1 {
16590 break;
16591 }
16592 r1 = r1.max(cells[j].2);
16593 j += 1;
16594 }
16595 cover.insert_rect(sheet, &Rect::new(r0, col, r1, col));
16596 i = j;
16597 }
16598 }
16599 let mut hints = Vec::new();
16600 for (&reader, deps) in vdeps {
16601 let Some(reader) = self.graph.authority_cell_of_vertex(reader) else {
16602 continue;
16603 };
16604 for &dependency in deps {
16605 let Some(dep) = self.graph.authority_cell_of_vertex(dependency) else {
16606 continue;
16607 };
16608 hints.push(planner::PlanHint {
16609 reader: (reader.0, reader.2, reader.1),
16610 edge: EdgeKey {
16611 dep_sheet: reader.0,
16612 tag: Tag::X,
16613 lk: u32::MAX,
16614 proj: RefProj {
16615 sheet: dep.0,
16616 rows: AxisMap::fixed(dep.1, dep.1),
16617 cols: AxisMap::fixed(dep.2, dep.2),
16618 },
16619 },
16620 });
16621 }
16622 }
16623 let host = self.graph.authority_host();
16625 for &id in candidates.iter().filter(|_| host.has_observed()) {
16626 let Some(reads) = host.observed(id) else {
16627 continue;
16628 };
16629 let Some(reader) = self.graph.authority_cell_of_vertex(id) else {
16630 continue;
16631 };
16632 for &(sheet, r0, c0, r1, c1) in reads {
16633 hints.push(planner::PlanHint {
16634 reader: (reader.0, reader.2, reader.1),
16635 edge: EdgeKey {
16636 dep_sheet: reader.0,
16637 tag: Tag::X,
16638 lk: u32::MAX,
16639 proj: RefProj {
16640 sheet,
16641 rows: AxisMap::fixed(r0, r1),
16642 cols: AxisMap::fixed(c0, c1),
16643 },
16644 },
16645 });
16646 }
16647 }
16648 hints.sort_unstable_by_key(|hint| hint.reader);
16649 let checkpoint = ledger
16650 .as_ref()
16651 .map_or(0, |ledger| ledger.scratch_checkpoint());
16652 let scratch_limit = ledger
16653 .as_ref()
16654 .and_then(|ledger| ledger.schedule_discovery_limit())
16655 .map(|limit| limit.saturating_sub(checkpoint));
16656 let ordered = planner::plan_with_hints(store, &cover, &hints, scratch_limit, None, None)
16657 .map_err(|error| failure(format!("{error:?}")))?;
16658 let adapted = plan_schedule::schedule(
16662 &ordered.cells,
16663 ordered.heap_bytes(),
16664 scratch_limit,
16665 |cell| {
16666 self.graph
16667 .authority_vertex_of_formula(cell.id, (cell.sheet, cell.row, cell.col))
16668 .ok_or_else(|| failure("missing executor identity".to_owned()))
16669 },
16670 |_work| {
16671 self.cancellation_checkpoint("Evaluation cancelled during authority planning")?;
16672 if let Some(ledger) = ledger.as_deref_mut() {
16673 ledger
16674 .checkpoint_deadline()
16675 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16676 }
16677 Ok(())
16678 },
16679 )
16680 .map_err(|error| match error {
16681 plan_schedule::ScheduleError::Runtime(error) => error,
16682 other => failure(format!("{other:?}")),
16683 })?;
16684 if let Some(ledger) = ledger {
16685 let peak = ordered.peak_heap_bytes.max(adapted.peak_heap_bytes);
16686 ledger
16687 .reserve_schedule_discovery(peak)
16688 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16689 ledger
16690 .release_scratch(peak)
16691 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16692 }
16693 match self.thread_pool.as_deref() {
16695 Some(pool) if ordered.cells.len() >= PARALLEL_SCHEDULE_MIN_CANDIDATES => {
16696 pool.spawn(move || drop(ordered))
16697 }
16698 _ => drop(ordered),
16699 }
16700 Ok(adapted.schedule)
16701 }
16702
16703 fn can_use_static_schedule_cache(&self, to_evaluate: &[VertexId]) -> bool {
16709 {
16710 let host = self.graph.authority_host();
16714 !to_evaluate.is_empty()
16715 && !self.freshness_has_hints()
16716 && to_evaluate
16717 .iter()
16718 .all(|&v| !self.graph.is_dynamic(v) || host.observed(v).is_some())
16719 }
16720 }
16721
16722 fn schedule_cache_authority_revision(&self) -> (u64, u64) {
16723 {
16724 let host = self.graph.authority_host();
16727 (host.revision(), host.rev_dyn())
16728 }
16729 }
16730
16731 fn start_virtual_dep_telemetry(&self) -> VirtualDepTelemetry {
16732 VirtualDepTelemetry {
16733 fallback_mode_activations: self.virtual_dep_fallback_activations,
16734 ..VirtualDepTelemetry::default()
16735 }
16736 }
16737
16738 fn accumulate_schedule_meta(telemetry: &mut VirtualDepTelemetry, meta: &ScheduleBuildMeta) {
16739 telemetry.candidate_vertices_total += meta.candidate_vertices;
16740 telemetry.vdeps_vertices_total += meta.vdeps_vertices;
16741 telemetry.vdeps_edges_total += meta.vdeps_edges;
16742 telemetry.builder_elapsed_ms_total += meta.builder_elapsed_ms;
16743 if meta.schedule_cache_eligible {
16744 if meta.schedule_cache_hit {
16745 telemetry.schedule_cache_hits += 1;
16746 telemetry.reused_schedule_vertices_total += meta.candidate_vertices;
16747 } else {
16748 telemetry.schedule_cache_misses += 1;
16749 }
16750 }
16751 if meta.used_virtual_schedule {
16752 telemetry.schedule_virtual_passes += 1;
16753 } else {
16754 telemetry.schedule_static_passes += 1;
16755 }
16756 }
16757
16758 fn finish_pass_dirty(&mut self, to_evaluate: &[VertexId], changed: &[VertexId]) -> bool {
16763 self.finish_pass_dirty_scoped(to_evaluate, changed, true)
16764 }
16765
16766 fn finish_target_pass_dirty(&mut self, to_evaluate: &[VertexId], changed: &[VertexId]) -> bool {
16769 self.finish_pass_dirty_scoped(to_evaluate, changed, false)
16770 }
16771
16772 fn finish_pass_dirty_scoped(
16773 &mut self,
16774 to_evaluate: &[VertexId],
16775 changed: &[VertexId],
16776 #[allow(unused_variables)] whole_workbook: bool,
16777 ) -> bool {
16778 self.freshness_finish_pass(to_evaluate, changed, whole_workbook)
16779 }
16780
16781 fn begin_pass(
16783 &mut self,
16784 #[allow(unused_variables)] schedule: &crate::engine::scheduler::Schedule,
16785 ) {
16786 self.freshness_begin_pass(schedule);
16787 }
16788
16789 fn stop_after_unit(
16791 &mut self,
16792 #[allow(unused_variables)] schedule: &crate::engine::scheduler::Schedule,
16793 #[allow(unused_variables)] index: usize,
16794 ) -> bool {
16795 self.freshness_stop_after_unit(schedule, index)
16796 }
16797
16798 fn changed_virtual_dep_vertices(
16799 &mut self,
16800 to_evaluate: &[VertexId],
16801 old_vdeps: &FxHashMap<VertexId, Vec<VertexId>>,
16802 ) -> Vec<VertexId> {
16803 #[cfg(test)]
16804 if self.force_virtual_dep_changes_remaining_for_test > 0
16805 && let Some(vertex) = to_evaluate.first().copied()
16806 {
16807 self.force_virtual_dep_changes_remaining_for_test -= 1;
16808 return vec![vertex];
16809 }
16810 if self.freshness_armed() {
16813 return Vec::new();
16814 }
16815 if !to_evaluate
16816 .iter()
16817 .copied()
16818 .any(|v| self.graph.is_dynamic(v))
16819 {
16820 return Vec::new();
16821 }
16822
16823 let builder = VirtualDepBuilder::new(self);
16824 let (new_vdeps, _) = builder.build(to_evaluate);
16825
16826 let mut candidates = FxHashSet::default();
16827 candidates.extend(old_vdeps.keys().copied());
16828 candidates.extend(new_vdeps.keys().copied());
16829
16830 let mut changed = Vec::new();
16831 for v in candidates {
16832 if old_vdeps.get(&v) != new_vdeps.get(&v) {
16833 changed.push(v);
16834 }
16835 }
16836 changed
16837 }
16838
16839 #[allow(clippy::type_complexity)]
16845 fn demand_subgraph(
16846 &self,
16847 targets: &[VertexId],
16848 ) -> Result<
16849 (
16850 Vec<VertexId>,
16851 rustc_hash::FxHashMap<VertexId, Vec<VertexId>>,
16852 ),
16853 ExcelError,
16854 > {
16855 self.authority_demand_subgraph(targets)
16856 }
16857
16858 #[allow(clippy::type_complexity)]
16864 fn authority_demand_subgraph(
16865 &self,
16866 targets: &[VertexId],
16867 ) -> Result<
16868 (
16869 Vec<VertexId>,
16870 rustc_hash::FxHashMap<VertexId, Vec<VertexId>>,
16871 ),
16872 ExcelError,
16873 > {
16874 use crate::engine::authority::geom::{Cell, Rect, SYMBOL_SHEET};
16875 use crate::engine::authority::store::TagFilter;
16876 use rustc_hash::{FxHashMap, FxHashSet};
16877 let store = self
16878 .graph
16879 .authority_plan_store()
16880 .map_err(Self::authority_excel_error)?;
16881 let ids = store.ids();
16882 let mut to_evaluate: FxHashSet<VertexId> = FxHashSet::default();
16883 let mut vdeps: FxHashMap<VertexId, Vec<VertexId>> = FxHashMap::default();
16884 let mut visited: FxHashSet<Cell> = FxHashSet::default();
16885 let mut stack: Vec<(Cell, u32)> = Vec::new();
16888 let push_formulas = |stack: &mut Vec<(Cell, u32)>,
16890 visited: &FxHashSet<Cell>,
16891 sheet: u16,
16892 rect: Rect| {
16893 for col in rect.c0..=rect.c1 {
16894 ids.visit_runs_in(sheet, col, rect.r0, rect.r1, &mut |h| {
16895 let run = ids.run(h);
16896 let r0 = run.row_start.max(rect.r0);
16897 let r1 = (run.row_start + run.len - 1).min(rect.r1);
16898 for row in r0..=r1 {
16899 if !visited.contains(&(sheet, row, col)) {
16900 stack.push(((sheet, row, col), run.first_id + (row - run.row_start)));
16901 }
16902 }
16903 });
16904 }
16905 };
16906 for &v in targets {
16907 if let Some(table) = self.graph.table_by_vertex(v) {
16908 let (s, e) = (table.range.start, table.range.end);
16912 push_formulas(
16913 &mut stack,
16914 &visited,
16915 s.sheet_id,
16916 Rect::new(s.coord.row(), s.coord.col(), e.coord.row(), e.coord.col()),
16917 );
16918 } else if let Some(cell) = self.graph.authority_cell_of_vertex(v) {
16919 stack.push((cell, crate::engine::authority::identity::NO_VID));
16920 }
16921 }
16922 let mut hits = Vec::new();
16923 #[cfg(any(test, feature = "benchmark_internal"))]
16924 let (mut probe_vertices, mut probe_clean_formulas, mut probe_edges, mut probe_dynamic) =
16925 (0, 0, 0, 0);
16926 while let Some((cell, id)) = stack.pop() {
16927 if !visited.insert(cell) {
16928 continue;
16929 }
16930 let Some(v) = self.graph.authority_vertex_of_formula(id, cell) else {
16931 continue;
16932 };
16933 if !self.graph.vertex_exists(v) {
16934 continue;
16935 }
16936 #[cfg(any(test, feature = "benchmark_internal"))]
16937 {
16938 probe_vertices += 1;
16939 }
16940 match self.graph.get_vertex_kind(v) {
16941 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
16942 if self.graph.is_dirty(v) || self.graph.is_volatile(v) {
16943 to_evaluate.insert(v);
16944 } else {
16945 #[cfg(any(test, feature = "benchmark_internal"))]
16946 {
16947 probe_clean_formulas += 1;
16948 }
16949 }
16950 }
16951 VertexKind::NamedScalar | VertexKind::NamedArray => {
16952 to_evaluate.insert(v);
16953 }
16954 _ => {}
16955 }
16956 hits.clear();
16957 store.direct_precedents(cell, TagFilter::All, &mut hits);
16958 #[cfg(any(test, feature = "benchmark_internal"))]
16959 {
16960 probe_edges += hits.len();
16961 }
16962 for &(_, sheet, rect) in &hits {
16963 if sheet != SYMBOL_SHEET {
16966 for anchor in self
16967 .graph
16968 .spill_anchors_in_region(sheet, rect.r0, rect.c0, rect.r1, rect.c1)
16969 {
16970 if anchor != v && self.graph.is_dirty(anchor) {
16971 vdeps.entry(v).or_default().push(anchor);
16972 if let Some(c) = self.graph.authority_cell_of_vertex(anchor) {
16973 stack.push((c, crate::engine::authority::identity::NO_VID));
16974 }
16975 }
16976 }
16977 }
16978 if sheet == SYMBOL_SHEET {
16979 stack.extend((rect.r0..=rect.r1).map(|slot| {
16980 (
16981 (SYMBOL_SHEET, slot, 0),
16982 crate::engine::authority::identity::NO_VID,
16983 )
16984 }));
16985 continue;
16986 }
16987 push_formulas(&mut stack, &visited, sheet, rect);
16988 }
16989 if self.graph.is_dynamic(v) {
16990 #[cfg(any(test, feature = "benchmark_internal"))]
16991 {
16992 probe_dynamic += 1;
16993 }
16994 if let Some(reads) = self.graph.authority_host().observed(v) {
16996 for &(sheet, r0, c0, r1, c1) in reads {
16997 push_formulas(&mut stack, &visited, sheet, Rect::new(r0, c0, r1, c1));
16998 }
16999 }
17000 let (vdeps_map, _) = VirtualDepBuilder::new(self).build(&[v]);
17001 let hinted = self.freshness_hints(v).unwrap_or(&[]);
17004 if let Some(deps) = vdeps_map
17005 .get(&v)
17006 .map(|deps| deps.iter().chain(hinted))
17007 .or(Some([].iter().chain(hinted)))
17008 {
17009 for &u in deps {
17010 vdeps.entry(v).or_default().push(u);
17011 if let Some(c) = self.graph.authority_cell_of_vertex(u) {
17012 stack.push((c, crate::engine::authority::identity::NO_VID));
17013 }
17014 }
17015 }
17016 }
17017 }
17018 let mut result: Vec<VertexId> = to_evaluate.into_iter().collect();
17019 result.sort_unstable();
17020 for deps in vdeps.values_mut() {
17021 deps.sort_unstable();
17022 deps.dedup();
17023 }
17024 #[cfg(any(test, feature = "benchmark_internal"))]
17025 {
17026 let mut probe = self.recalc_reuse_probe.lock().unwrap();
17027 probe.demand_builds += 1;
17028 probe.demand_vertices += probe_vertices;
17029 probe.demand_clean_formulas += probe_clean_formulas;
17030 probe.demand_explicit_edges += probe_edges;
17031 probe.demand_virtual_builder_calls += probe_dynamic;
17032 }
17033 Ok((result, vdeps))
17034 }
17035
17036 fn col_to_letters(col: u32) -> String {
17038 col_letters_from_1based(col).expect("column index must be >= 1")
17039 }
17040
17041 pub fn evaluate_all_cancellable(
17043 &mut self,
17044 cancel: crate::engine::CancelToken,
17045 ) -> Result<EvalResult, ExcelError> {
17046 self.observe_evaluation_resource_request(EvaluationRequestKind::FullCancellable, |engine| {
17047 engine.observe_function_semantic_epoch()?;
17048 engine.active_cancel_flag = Some(cancel.clone());
17049 let res = engine.evaluate_all_cancellable_impl(cancel.as_flag());
17050 engine.active_cancel_flag = None;
17051 res
17052 })
17053 }
17054
17055 fn evaluate_all_cancellable_impl(
17056 &mut self,
17057 cancel_flag: &AtomicBool,
17058 ) -> Result<EvalResult, ExcelError> {
17059 let _source_cache = self.source_cache_session();
17060 self.validate_deterministic_mode()?;
17061 if self.config.defer_graph_building {
17062 self.build_graph_all()?;
17063 }
17064 if cancel_flag.load(Ordering::Relaxed) {
17065 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
17066 .with_message("Evaluation cancelled before scheduling".to_string()));
17067 }
17068 self.require_unified_authority()?;
17069 self.begin_evaluation_request();
17070 self.reset_virtual_dep_telemetry_if_disabled();
17071 let start = crate::instant::FzInstant::now();
17072 let mut computed_vertices = 0;
17073 let mut cycle_errors = 0;
17074
17075 let mut replan_iterations = 0;
17076 const MAX_REPLAN: usize = 5;
17077 let mut telemetry = self
17078 .config
17079 .enable_virtual_dep_telemetry
17080 .then(|| self.start_virtual_dep_telemetry());
17081
17082 loop {
17083 if cancel_flag.load(Ordering::Relaxed) {
17084 if let Some(mut t) = telemetry {
17085 t.bailout_reason = Some("cancelled");
17086 t.replan_iterations = replan_iterations;
17087 self.last_virtual_dep_telemetry = t;
17088 }
17089 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
17090 .with_message("Evaluation cancelled before scheduling".to_string()));
17091 }
17092
17093 let to_evaluate = self.graph.get_evaluation_vertices();
17094 if to_evaluate.is_empty() {
17095 if let Some(t) = telemetry.as_mut()
17096 && t.bailout_reason.is_none()
17097 {
17098 t.bailout_reason = Some("no_work");
17099 }
17100 break;
17101 }
17102
17103 let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
17104 if let Some(t) = telemetry.as_mut() {
17105 Self::accumulate_schedule_meta(t, &meta);
17106 }
17107
17108 self.begin_pass(&schedule);
17111 for (unit_index, &unit) in schedule.units.iter().enumerate() {
17112 match unit {
17113 ScheduleUnit::Cycle(i) => {
17114 if cancel_flag.load(Ordering::Relaxed) {
17116 if let Some(mut t) = telemetry {
17117 t.bailout_reason = Some("cancelled");
17118 t.replan_iterations = replan_iterations;
17119 self.last_virtual_dep_telemetry = t;
17120 }
17121 return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(
17122 "Evaluation cancelled during cycle handling".to_string(),
17123 ));
17124 }
17125
17126 if self.handle_cycle_unit(
17127 schedule.unit_cycle(i),
17128 None,
17129 None,
17130 Some(cancel_flag),
17131 )? > 0
17132 {
17133 cycle_errors += 1;
17134 }
17135 }
17136 ScheduleUnit::Layer(i) => {
17137 let layer = schedule.unit_layer(i);
17138 if cancel_flag.load(Ordering::Relaxed) {
17140 if let Some(mut t) = telemetry {
17141 t.bailout_reason = Some("cancelled");
17142 t.replan_iterations = replan_iterations;
17143 self.last_virtual_dep_telemetry = t;
17144 }
17145 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
17146 .with_message("Evaluation cancelled between layers".to_string()));
17147 }
17148
17149 if self.thread_pool.is_some() && layer.vertices.len() > 1 {
17151 computed_vertices +=
17152 self.evaluate_layer_parallel_cancellable(layer, cancel_flag)?;
17153 } else {
17154 computed_vertices +=
17155 self.evaluate_layer_sequential_cancellable(layer, cancel_flag)?;
17156 }
17157 }
17158 }
17159 if self.stop_after_unit(&schedule, unit_index) {
17160 break;
17161 }
17162 }
17163
17164 let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
17165 if let Some(t) = telemetry.as_mut() {
17166 t.changed_vdeps_total += changed_vertices.len();
17167 }
17168 self.resource_checkpoint(0)?;
17169 if !self.finish_pass_dirty(&to_evaluate, &changed_vertices) {
17170 if let Some(t) = telemetry.as_mut() {
17171 t.bailout_reason = Some("converged");
17172 }
17173 break;
17174 }
17175 if replan_iterations >= MAX_REPLAN {
17176 if let Some(mut t) = telemetry.take() {
17177 t.bailout_reason = Some("max_replan");
17178 t.replan_iterations = replan_iterations;
17179 self.last_virtual_dep_telemetry = t;
17180 }
17181 return Err(
17182 self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
17183 );
17184 }
17185 replan_iterations += 1;
17186 }
17187
17188 if let Some(mut t) = telemetry {
17189 t.replan_iterations = replan_iterations;
17190 self.last_virtual_dep_telemetry = t;
17191 }
17192
17193 self.redirty_for_next_recalc();
17195 self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
17196
17197 Ok(EvalResult {
17198 computed_vertices,
17199 cycle_errors,
17200 elapsed: start.elapsed(),
17201 })
17202 }
17203
17204 pub fn evaluate_until_cancellable(
17206 &mut self,
17207 targets: &[&str],
17208 cancel: crate::engine::CancelToken,
17209 ) -> Result<EvalResult, ExcelError> {
17210 self.observe_evaluation_resource_request(
17211 EvaluationRequestKind::TargetedCancellable,
17212 |engine| {
17213 engine.observe_function_semantic_epoch()?;
17214 engine.active_cancel_flag = Some(cancel.clone());
17215 let res = engine.evaluate_until_cancellable_impl(targets, cancel.as_flag());
17216 engine.active_cancel_flag = None;
17217 res
17218 },
17219 )
17220 }
17221
17222 fn evaluate_until_cancellable_impl(
17223 &mut self,
17224 targets: &[&str],
17225 cancel_flag: &AtomicBool,
17226 ) -> Result<EvalResult, ExcelError> {
17227 if cancel_flag.load(Ordering::Relaxed) {
17228 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
17229 .with_message("Evaluation cancelled before target preparation"));
17230 }
17231 let mut typed_targets = Vec::with_capacity(targets.len());
17232 for target in targets {
17233 let (sheet, row, col) = self.parse_a1_notation(target)?;
17234 self.graph.sheet_id_mut(&sheet);
17235 typed_targets.push(crate::engine::EvaluationTarget::Cell { sheet, row, col });
17236 }
17237 self.evaluate_mixed_targets(&typed_targets, None)
17238 }
17239
17240 fn parse_a1_notation(&self, address: &str) -> Result<(String, u32, u32), ExcelError> {
17241 let mut quoted = false;
17242 let mut separator = None;
17243 let bytes = address.as_bytes();
17244 let mut index = 0usize;
17245 while index < bytes.len() {
17246 match bytes[index] {
17247 b'\'' => {
17248 if quoted && bytes.get(index + 1) == Some(&b'\'') {
17249 index = index.saturating_add(1);
17250 } else {
17251 quoted = !quoted;
17252 }
17253 }
17254 b'!' if !quoted => separator = Some(index),
17255 _ => {}
17256 }
17257 index = index.saturating_add(1);
17258 }
17259 if quoted {
17260 return Err(ExcelError::new(ExcelErrorKind::Ref)
17261 .with_message(format!("Invalid quoted sheet reference `{address}`")));
17262 }
17263 let (sheet, cell_part) = match separator {
17264 Some(separator) => {
17265 let raw_sheet = &address[..separator];
17266 let sheet = if raw_sheet.starts_with('\'') && raw_sheet.ends_with('\'') {
17267 raw_sheet[1..raw_sheet.len().saturating_sub(1)].replace("''", "'")
17268 } else {
17269 raw_sheet.to_string()
17270 };
17271 (sheet, &address[separator + 1..])
17272 }
17273 None => (self.default_sheet_name().to_string(), address),
17274 };
17275
17276 let (row, col, _, _) = parse_a1_1based(cell_part).map_err(|err| {
17277 ExcelError::new(ExcelErrorKind::Ref)
17278 .with_message(format!("Invalid cell reference `{cell_part}`: {err}"))
17279 })?;
17280
17281 Ok((sheet, row, col))
17282 }
17283
17284 fn is_ast_volatile_with_provider(&self, ast: &ASTNode) -> bool {
17286 use formualizer_parse::parser::ASTNodeType;
17287 match &ast.node_type {
17288 ASTNodeType::Function { name, args, .. } => {
17289 if let Some(func) = self
17290 .get_function("", name)
17291 .or_else(|| crate::function_registry::get("", name))
17292 && func.caps().contains(crate::function::FnCaps::VOLATILE)
17293 {
17294 return true;
17295 }
17296 args.iter()
17297 .any(|arg| self.is_ast_volatile_with_provider(arg))
17298 }
17299 ASTNodeType::BinaryOp { left, right, .. } => {
17300 self.is_ast_volatile_with_provider(left)
17301 || self.is_ast_volatile_with_provider(right)
17302 }
17303 ASTNodeType::UnaryOp { expr, .. } => self.is_ast_volatile_with_provider(expr),
17304 ASTNodeType::Array(rows) => rows.iter().any(|row| {
17305 row.iter()
17306 .any(|cell| self.is_ast_volatile_with_provider(cell))
17307 }),
17308 _ => false,
17309 }
17310 }
17311
17312 fn evaluate_layer_sequential(
17314 &mut self,
17315 layer: &super::scheduler::Layer,
17316 ) -> Result<usize, ExcelError> {
17317 self.resource_checkpoint(layer.vertices.len() as u64)?;
17318 self.evaluate_layer_sequential_effects(layer)
17319 }
17320
17321 fn update_vertex_value_with_delta(
17322 &mut self,
17323 vertex_id: VertexId,
17324 new_value: LiteralValue,
17325 delta: &mut DeltaCollector,
17326 ) {
17327 if delta.mode != DeltaMode::Off
17328 && let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id)
17329 {
17330 let sheet_name = self.graph.sheet_name(cell.sheet_id);
17331 let old = self
17332 .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
17333 .unwrap_or(LiteralValue::Empty);
17334 if old != new_value {
17335 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
17336 }
17337 }
17338 self.graph.update_vertex_value_ref(vertex_id, &new_value);
17339 self.mirror_vertex_value_to_overlay(vertex_id, &new_value);
17340 }
17341
17342 fn evaluate_layer_sequential_with_delta(
17343 &mut self,
17344 layer: &super::scheduler::Layer,
17345 delta: &mut DeltaCollector,
17346 ) -> Result<usize, ExcelError> {
17347 self.resource_checkpoint(layer.vertices.len() as u64)?;
17348 self.evaluate_layer_sequential_with_delta_effects(layer, delta)
17349 }
17350
17351 fn evaluate_layer_sequential_cancellable(
17353 &mut self,
17354 layer: &super::scheduler::Layer,
17355 cancel_flag: &AtomicBool,
17356 ) -> Result<usize, ExcelError> {
17357 self.resource_checkpoint(layer.vertices.len() as u64)?;
17358 self.evaluate_layer_sequential_cancellable_effects(layer, cancel_flag)
17359 }
17360
17361 fn evaluate_layer_sequential_cancellable_demand_driven(
17363 &mut self,
17364 layer: &super::scheduler::Layer,
17365 cancel_flag: &AtomicBool,
17366 ) -> Result<usize, ExcelError> {
17367 self.resource_checkpoint(layer.vertices.len() as u64)?;
17368 self.evaluate_layer_sequential_cancellable_demand_driven_effects(layer, cancel_flag)
17369 }
17370
17371 fn evaluate_layer_parallel(
17383 &mut self,
17384 layer: &super::scheduler::Layer,
17385 ) -> Result<usize, ExcelError> {
17386 if layer.sequential {
17387 return self.evaluate_layer_sequential(layer);
17388 }
17389 self.resource_checkpoint(layer.vertices.len() as u64)?;
17390 let len = layer.vertices.len();
17391 let buffered = buffer_layer_writes(layer);
17392 let (probe, worth) = (PARALLEL_LAYER_PROBE, PARALLEL_LAYER_WORTH);
17393 let start = crate::instant::FzInstant::now();
17394 let mut pos = 0usize;
17395 let mut step = 1usize;
17396 while pos < len {
17397 if pos > 0 {
17398 let elapsed = start.elapsed();
17399 let per_vertex = elapsed.as_nanos() / pos as u128 + 1;
17401 let rest_estimate = per_vertex * (len - pos) as u128;
17402 if len - pos >= 2 && (elapsed >= probe || rest_estimate >= worth.as_nanos()) {
17403 let rest = layer.sub_layer(pos, len);
17404 let min_chunk = if per_vertex >= EXPENSIVE_VERTEX_NS {
17407 1
17408 } else {
17409 8
17410 };
17411 return Ok(pos + self.evaluate_layer_parallel_effects(&rest, min_chunk)?);
17412 }
17413 let fit = (probe.saturating_sub(elapsed).as_nanos() / per_vertex) as usize + 1;
17416 step = step.saturating_mul(2).min(fit);
17417 }
17418 let end = (pos + step).min(len);
17419 let slice = layer.sub_layer(pos, end);
17420 pos += self.evaluate_layer_units_until(
17423 &slice,
17424 None,
17425 None,
17426 None,
17427 buffered,
17428 Some(start + probe),
17429 )?;
17430 }
17431 Ok(len)
17432 }
17433
17434 fn evaluate_layer_parallel_with_delta(
17435 &mut self,
17436 layer: &super::scheduler::Layer,
17437 delta: &mut DeltaCollector,
17438 ) -> Result<usize, ExcelError> {
17439 if layer.sequential {
17440 return self.evaluate_layer_sequential_with_delta(layer, delta);
17441 }
17442 self.resource_checkpoint(layer.vertices.len() as u64)?;
17443 self.evaluate_layer_parallel_with_delta_effects(layer, delta)
17444 }
17445
17446 fn evaluate_layer_parallel_cancellable(
17448 &mut self,
17449 layer: &super::scheduler::Layer,
17450 cancel_flag: &AtomicBool,
17451 ) -> Result<usize, ExcelError> {
17452 if layer.sequential {
17453 return self.evaluate_layer_sequential_cancellable(layer, cancel_flag);
17454 }
17455 self.resource_checkpoint(layer.vertices.len() as u64)?;
17456 self.evaluate_layer_parallel_cancellable_effects(layer, cancel_flag)
17457 }
17458
17459 fn evaluate_vertex_immutable(&self, vertex_id: VertexId) -> Result<LiteralValue, ExcelError> {
17461 if !self.graph.vertex_exists(vertex_id) {
17463 return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
17464 .with_message(format!("Vertex not found: {vertex_id:?}")));
17465 }
17466
17467 let kind = self.graph.get_vertex_kind(vertex_id);
17469 let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
17470
17471 let view = match kind {
17472 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
17473 if let Some(view) = self.graph.formula_view(vertex_id) {
17474 view
17475 } else {
17476 return Ok(LiteralValue::Number(0.0));
17477 }
17478 }
17479 VertexKind::Empty | VertexKind::Cell => {
17480 if let Some(cell_ref) = self.graph.get_cell_ref(vertex_id) {
17481 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
17482 let row = cell_ref.coord.row() + 1;
17483 let col = cell_ref.coord.col() + 1;
17484 if let Some(v) = self.read_cell_value(sheet_name, row, col) {
17485 return Ok(v);
17486 }
17487 }
17488 return Ok(LiteralValue::Number(0.0));
17489 }
17490 VertexKind::NamedScalar => {
17491 let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
17492 ExcelError::new(ExcelErrorKind::Name)
17493 .with_message("Named range metadata missing".to_string())
17494 })?;
17495
17496 return match &named_range.definition {
17497 NamedDefinition::Cell(cell_ref) => {
17498 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
17499 Ok(self
17500 .read_cell_value(
17501 sheet_name,
17502 cell_ref.coord.row() + 1,
17503 cell_ref.coord.col() + 1,
17504 )
17505 .unwrap_or(LiteralValue::Empty))
17506 }
17507 NamedDefinition::Literal(v) => Ok(v.clone()),
17508 NamedDefinition::Formula { ast, .. } => {
17509 let context_sheet = match named_range.scope {
17510 NameScope::Sheet(id) => id,
17511 NameScope::Workbook => sheet_id,
17512 };
17513 let sheet_name = self.graph.sheet_name(context_sheet);
17514 let cell_ref = self
17515 .graph
17516 .get_cell_ref(vertex_id)
17517 .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
17518 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref)
17519 .with_resolved_functions(&self.resolved_functions)
17520 .with_arena_cells(self);
17521 interpreter.evaluate_ast(ast).map(|cv| cv.into_literal())
17522 }
17523 NamedDefinition::Range(_) => Err(ExcelError::new(ExcelErrorKind::Value)
17524 .with_message("Range-valued name evaluated as scalar".to_string())),
17525 };
17526 }
17527 VertexKind::NamedArray => {
17528 let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
17529 ExcelError::new(ExcelErrorKind::Name)
17530 .with_message("Named range metadata missing".to_string())
17531 })?;
17532
17533 return match &named_range.definition {
17534 NamedDefinition::Range(range_ref) => {
17535 if range_ref.start.sheet_id != range_ref.end.sheet_id {
17536 return Err(ExcelError::new(ExcelErrorKind::Ref)
17537 .with_message("Named range cannot span sheets".to_string()));
17538 }
17539 let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
17540 let sr0 = range_ref.start.coord.row();
17541 let sc0 = range_ref.start.coord.col();
17542 let er0 = range_ref.end.coord.row();
17543 let ec0 = range_ref.end.coord.col();
17544 if sr0 > er0 || sc0 > ec0 {
17545 return Err(ExcelError::new(ExcelErrorKind::Ref)
17546 .with_message("Invalid named range bounds".to_string()));
17547 }
17548
17549 let h = (er0 - sr0 + 1) as usize;
17550 let w = (ec0 - sc0 + 1) as usize;
17551 let cell_count = (h as u64).saturating_mul(w as u64);
17552 if cell_count > self.config.spill.max_spill_cells as u64 {
17553 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
17554 "Named range too large to materialize as an array".to_string(),
17555 ));
17556 }
17557
17558 let sheet_id = range_ref.start.sheet_id;
17560 let asheet = self.arrow_sheets.sheet(sheet_name);
17561 let mut rows = Vec::with_capacity(h);
17562 for r0 in sr0..=er0 {
17563 let mut row = Vec::with_capacity(w);
17564 for c0 in sc0..=ec0 {
17565 row.push(
17566 self.read_cell_formatted_in(sheet_id, asheet, r0 + 1, c0 + 1)
17567 .0,
17568 );
17569 }
17570 rows.push(row);
17571 }
17572 Ok(LiteralValue::Array(rows))
17573 }
17574 NamedDefinition::Cell(cell_ref) => {
17575 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
17576 let row = cell_ref.coord.row() + 1;
17577 let col = cell_ref.coord.col() + 1;
17578 let v = self
17579 .read_cell_value(sheet_name, row, col)
17580 .unwrap_or(LiteralValue::Empty);
17581 Ok(LiteralValue::Array(vec![vec![v]]))
17582 }
17583 NamedDefinition::Literal(v) => Ok(LiteralValue::Array(vec![vec![v.clone()]])),
17584 NamedDefinition::Formula { ast, .. } => {
17585 let context_sheet = match named_range.scope {
17586 NameScope::Sheet(id) => id,
17587 NameScope::Workbook => sheet_id,
17588 };
17589 let sheet_name = self.graph.sheet_name(context_sheet);
17590 let cell_ref = self
17591 .graph
17592 .get_cell_ref(vertex_id)
17593 .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
17594 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref)
17595 .with_resolved_functions(&self.resolved_functions)
17596 .with_arena_cells(self);
17597 match interpreter.evaluate_ast(ast) {
17598 Ok(cv) => {
17599 let v = cv.into_literal();
17600 match v {
17601 LiteralValue::Array(_) => Ok(v),
17602 other => Ok(LiteralValue::Array(vec![vec![other]])),
17603 }
17604 }
17605 Err(err) => Ok(LiteralValue::Error(err)),
17606 }
17607 }
17608 };
17609 }
17610 VertexKind::InfiniteRange
17611 | VertexKind::Range
17612 | VertexKind::External
17613 | VertexKind::Table => {
17614 return Ok(LiteralValue::Number(0.0));
17616 }
17617 };
17618
17619 let sheet_name = self.graph.sheet_name(sheet_id);
17621 let cell_ref = self
17622 .graph
17623 .get_cell_ref(vertex_id)
17624 .expect("cell ref for vertex");
17625 if let Some(result) =
17626 self.freshness_evaluate_recorded(vertex_id, sheet_name, cell_ref, view)
17627 {
17628 return result;
17629 }
17630 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref)
17631 .with_resolved_functions(&self.resolved_functions)
17632 .with_arena_cells(self);
17633
17634 interpreter
17635 .evaluate_formula_view(view, self.graph.data_store(), self.graph.sheet_reg())
17636 .map(|cv| {
17637 let format = cv.format_id();
17638 self.record_derived_format(vertex_id, format);
17639 self.materialize_formula_result(vertex_id, cv)
17640 })
17641 .or_else(|error| self.fit_formula_error(vertex_id, error))
17642 }
17643
17644 pub fn thread_pool(&self) -> Option<&Arc<rayon::ThreadPool>> {
17646 self.thread_pool.as_ref()
17647 }
17648}
17649
17650#[derive(Default)]
17651struct RowBoundsCache {
17652 snapshot: u64,
17653 map: rustc_hash::FxHashMap<(u32, usize), (Option<u32>, Option<u32>)>,
17655}
17656
17657impl RowBoundsCache {
17658 fn new(snapshot: u64) -> Self {
17659 Self {
17660 snapshot,
17661 map: Default::default(),
17662 }
17663 }
17664 fn get_row_bounds(
17665 &self,
17666 sheet_id: SheetId,
17667 col_idx: usize,
17668 snapshot: u64,
17669 ) -> Option<(Option<u32>, Option<u32>)> {
17670 if self.snapshot != snapshot {
17671 return None;
17672 }
17673 self.map.get(&(sheet_id as u32, col_idx)).copied()
17674 }
17675 fn put_row_bounds(
17676 &mut self,
17677 sheet_id: SheetId,
17678 col_idx: usize,
17679 snapshot: u64,
17680 bounds: (Option<u32>, Option<u32>),
17681 ) {
17682 if self.snapshot != snapshot {
17683 self.snapshot = snapshot;
17684 self.map.clear();
17685 }
17686 self.map.insert((sheet_id as u32, col_idx), bounds);
17687 }
17688}
17689
17690struct UsedAxisBoundsCache {
17691 snapshot: u64,
17692 row_bounds_by_col_span: rustc_hash::FxHashMap<(SheetId, u32, u32), Option<(u32, u32)>>,
17693 col_bounds_by_row_span: rustc_hash::FxHashMap<(SheetId, u32, u32), Option<(u32, u32)>>,
17694 #[cfg(test)]
17695 row_hits: std::sync::atomic::AtomicUsize,
17696 #[cfg(test)]
17697 row_misses: std::sync::atomic::AtomicUsize,
17698 #[cfg(test)]
17699 col_hits: std::sync::atomic::AtomicUsize,
17700 #[cfg(test)]
17701 col_misses: std::sync::atomic::AtomicUsize,
17702}
17703
17704impl UsedAxisBoundsCache {
17705 fn new(snapshot: u64) -> Self {
17706 Self {
17707 snapshot,
17708 row_bounds_by_col_span: Default::default(),
17709 col_bounds_by_row_span: Default::default(),
17710 #[cfg(test)]
17711 row_hits: std::sync::atomic::AtomicUsize::new(0),
17712 #[cfg(test)]
17713 row_misses: std::sync::atomic::AtomicUsize::new(0),
17714 #[cfg(test)]
17715 col_hits: std::sync::atomic::AtomicUsize::new(0),
17716 #[cfg(test)]
17717 col_misses: std::sync::atomic::AtomicUsize::new(0),
17718 }
17719 }
17720
17721 fn reset_for_snapshot(&mut self, snapshot: u64) {
17722 if self.snapshot != snapshot {
17723 self.snapshot = snapshot;
17724 self.row_bounds_by_col_span.clear();
17725 self.col_bounds_by_row_span.clear();
17726 }
17727 }
17728
17729 fn get_row_bounds(
17730 &self,
17731 sheet_id: SheetId,
17732 start_col: u32,
17733 end_col: u32,
17734 snapshot: u64,
17735 ) -> Option<Option<(u32, u32)>> {
17736 if self.snapshot != snapshot {
17737 return None;
17738 }
17739 let cached = self
17740 .row_bounds_by_col_span
17741 .get(&(sheet_id, start_col, end_col))
17742 .copied();
17743 #[cfg(test)]
17744 if cached.is_some() {
17745 self.row_hits.fetch_add(1, Ordering::Relaxed);
17746 }
17747 cached
17748 }
17749
17750 fn put_row_bounds(
17751 &mut self,
17752 sheet_id: SheetId,
17753 start_col: u32,
17754 end_col: u32,
17755 snapshot: u64,
17756 bounds: Option<(u32, u32)>,
17757 ) {
17758 self.reset_for_snapshot(snapshot);
17759 self.row_bounds_by_col_span
17760 .insert((sheet_id, start_col, end_col), bounds);
17761 #[cfg(test)]
17762 self.row_misses.fetch_add(1, Ordering::Relaxed);
17763 }
17764
17765 fn get_col_bounds(
17766 &self,
17767 sheet_id: SheetId,
17768 start_row: u32,
17769 end_row: u32,
17770 snapshot: u64,
17771 ) -> Option<Option<(u32, u32)>> {
17772 if self.snapshot != snapshot {
17773 return None;
17774 }
17775 let cached = self
17776 .col_bounds_by_row_span
17777 .get(&(sheet_id, start_row, end_row))
17778 .copied();
17779 #[cfg(test)]
17780 if cached.is_some() {
17781 self.col_hits.fetch_add(1, Ordering::Relaxed);
17782 }
17783 cached
17784 }
17785
17786 fn put_col_bounds(
17787 &mut self,
17788 sheet_id: SheetId,
17789 start_row: u32,
17790 end_row: u32,
17791 snapshot: u64,
17792 bounds: Option<(u32, u32)>,
17793 ) {
17794 self.reset_for_snapshot(snapshot);
17795 self.col_bounds_by_row_span
17796 .insert((sheet_id, start_row, end_row), bounds);
17797 #[cfg(test)]
17798 self.col_misses.fetch_add(1, Ordering::Relaxed);
17799 }
17800}
17801
17802#[derive(Default)]
17804pub struct ShimSpillManager {
17805 region_locks: RegionLockManager,
17806 pub(crate) active_locks: rustc_hash::FxHashMap<VertexId, u64>,
17807}
17808
17809impl ShimSpillManager {
17810 pub(crate) fn reserve(
17811 &mut self,
17812 owner: VertexId,
17813 anchor_cell: CellRef,
17814 shape: SpillShape,
17815 _meta: SpillMeta,
17816 ) -> Result<(), ExcelError> {
17817 let region = crate::engine::spill::Region {
17819 sheet_id: anchor_cell.sheet_id as u32,
17820 row_start: anchor_cell.coord.row(),
17821 row_end: anchor_cell
17822 .coord
17823 .row()
17824 .saturating_add(shape.rows)
17825 .saturating_sub(1),
17826 col_start: anchor_cell.coord.col(),
17827 col_end: anchor_cell
17828 .coord
17829 .col()
17830 .saturating_add(shape.cols)
17831 .saturating_sub(1),
17832 };
17833 match self.region_locks.reserve(region, owner) {
17834 Ok(id) => {
17835 if id != 0 {
17836 self.active_locks.insert(owner, id);
17837 }
17838 Ok(())
17839 }
17840 Err(e) => Err(e),
17841 }
17842 }
17843
17844 pub(crate) fn release_owner(&mut self, owner: VertexId) {
17850 if let Some(id) = self.active_locks.remove(&owner) {
17851 self.region_locks.release(id);
17852 }
17853 }
17854
17855 pub(crate) fn commit_array_with_value_probe<F>(
17856 &mut self,
17857 graph: &mut DependencyGraph,
17858 anchor_vertex: VertexId,
17859 targets: &[CellRef],
17860 rows: Vec<Vec<LiteralValue>>,
17861 mut value_probe: F,
17862 ) -> Result<(), ExcelError>
17863 where
17864 F: FnMut(&DependencyGraph, &CellRef) -> Option<LiteralValue>,
17865 {
17866 use formualizer_common::{ExcelErrorExtra, ExcelErrorKind};
17867
17868 let plan_res = graph.plan_spill_region(anchor_vertex, targets);
17872 if let Err(e) = plan_res {
17873 if let Some(id) = self.active_locks.remove(&anchor_vertex) {
17874 self.region_locks.release(id);
17875 }
17876 return Err(e);
17877 }
17878
17879 if !graph.value_cache_enabled() {
17880 let (expected_rows, expected_cols) = if targets.is_empty() {
17882 (0u32, 0u32)
17883 } else {
17884 let mut min_r = u32::MAX;
17885 let mut max_r = 0u32;
17886 let mut min_c = u32::MAX;
17887 let mut max_c = 0u32;
17888 for cell in targets {
17889 let r = cell.coord.row();
17890 let c = cell.coord.col();
17891 min_r = min_r.min(r);
17892 max_r = max_r.max(r);
17893 min_c = min_c.min(c);
17894 max_c = max_c.max(c);
17895 }
17896 (
17897 max_r.saturating_sub(min_r).saturating_add(1),
17898 max_c.saturating_sub(min_c).saturating_add(1),
17899 )
17900 };
17901
17902 let anchor_cell = graph
17903 .get_cell_ref(anchor_vertex)
17904 .expect("anchor cell ref for spill commit");
17905
17906 for cell in targets {
17907 if *cell == anchor_cell {
17909 continue;
17910 }
17911 if graph.spill_registry_anchor_for_cell(*cell).is_some() {
17913 continue;
17914 }
17915 if let Some(vid) = graph.get_vertex_id_for_address(cell)
17917 && vid != anchor_vertex
17918 {
17919 match graph.get_vertex_kind(vid) {
17920 crate::engine::vertex::VertexKind::FormulaScalar
17921 | crate::engine::vertex::VertexKind::FormulaArray => {
17922 continue;
17924 }
17925 _ => {}
17926 }
17927 }
17928
17929 if let Some(v) = value_probe(graph, cell)
17930 && !matches!(v, LiteralValue::Empty)
17931 {
17932 if let Some(id) = self.active_locks.remove(&anchor_vertex) {
17933 self.region_locks.release(id);
17934 }
17935 return Err(ExcelError::new(ExcelErrorKind::Spill)
17936 .with_message("BlockedByValue")
17937 .with_extra(ExcelErrorExtra::Spill {
17938 expected_rows,
17939 expected_cols,
17940 }));
17941 }
17942 }
17943 }
17944
17945 let commit_res = graph.commit_spill_region_atomic_with_fault(
17946 anchor_vertex,
17947 targets.to_vec(),
17948 rows,
17949 None,
17950 );
17951 if let Some(id) = self.active_locks.remove(&anchor_vertex) {
17952 self.region_locks.release(id);
17953 }
17954 commit_res.map(|_| ())
17955 }
17956
17957 pub(crate) fn commit_array_with_overlay<R: EvaluationContext>(
17959 &mut self,
17960 engine: &mut Engine<R>,
17961 anchor_vertex: VertexId,
17962 targets: &[CellRef],
17963 rows: Vec<Vec<LiteralValue>>,
17964 ) -> Result<(), ExcelError> {
17965 if let Err(error) = engine.guard_pending_spill_commit(anchor_vertex, targets) {
17966 self.release_owner(anchor_vertex);
17967 return Err(error);
17968 }
17969 let plan_res = engine.graph.plan_spill_region(anchor_vertex, targets);
17971 if let Err(e) = plan_res {
17972 if let Some(id) = self.active_locks.remove(&anchor_vertex) {
17973 self.region_locks.release(id);
17974 }
17975 return Err(e);
17976 }
17977
17978 let commit_res = engine.graph.commit_spill_region_atomic_with_fault(
17979 anchor_vertex,
17980 targets.to_vec(),
17981 rows.clone(),
17982 None,
17983 );
17984 if let Some(id) = self.active_locks.remove(&anchor_vertex) {
17985 self.region_locks.release(id);
17986 }
17987 commit_res.map(|_| ())?;
17988 engine
17989 .blocked_pending_spills
17990 .retain(|entry| entry.0 != anchor_vertex);
17991 engine.spill_blocker_waiters.forget(anchor_vertex);
17992
17993 if engine.config.arrow_storage_enabled
17995 && engine.config.delta_overlay_enabled
17996 && engine.config.write_formula_overlay_enabled
17997 {
17998 for (idx, cell) in targets.iter().enumerate() {
18000 let (r_off, c_off) = {
18001 if rows.is_empty() || rows[0].is_empty() {
18002 (0usize, 0usize)
18003 } else {
18004 let width = rows[0].len();
18005 (idx / width, idx % width)
18006 }
18007 };
18008 let v = rows
18009 .get(r_off)
18010 .and_then(|r| r.get(c_off))
18011 .cloned()
18012 .unwrap_or(LiteralValue::Empty);
18013 let sheet_name = engine.graph.sheet_name(cell.sheet_id).to_string();
18014 engine.mirror_value_to_computed_overlay(
18015 &sheet_name,
18016 cell.coord.row() + 1,
18017 cell.coord.col() + 1,
18018 &v,
18019 );
18020 }
18021 }
18022 Ok(())
18023 }
18024}
18025
18026impl<R> Engine<R>
18027where
18028 R: EvaluationContext,
18029{
18030 fn resolve_shared_ref(
18031 &self,
18032 reference: &ReferenceType,
18033 current_sheet: &str,
18034 ) -> Result<formualizer_common::SheetRef<'static>, ExcelError> {
18035 use formualizer_common::{
18036 SheetCellRef as SharedCellRef, SheetLocator, SheetRangeRef as SharedRangeRef,
18037 SheetRef as SharedRef,
18038 };
18039
18040 let sr = match reference {
18042 ReferenceType::Cell {
18043 sheet,
18044 row,
18045 col,
18046 row_abs,
18047 col_abs,
18048 } => {
18049 let row0 = row
18050 .checked_sub(1)
18051 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18052 let col0 = col
18053 .checked_sub(1)
18054 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18055 let sheet_loc = match sheet.as_deref() {
18056 Some(name) => SheetLocator::from_name(name),
18057 None => SheetLocator::Current,
18058 };
18059 let coord = formualizer_common::RelativeCoord::new(row0, col0, *row_abs, *col_abs);
18060 SharedRef::Cell(SharedCellRef::new(sheet_loc, coord))
18061 }
18062 ReferenceType::Range {
18063 sheet,
18064 start_row,
18065 start_col,
18066 end_row,
18067 end_col,
18068 start_row_abs,
18069 start_col_abs,
18070 end_row_abs,
18071 end_col_abs,
18072 } => {
18073 let sheet_loc = match sheet.as_deref() {
18074 Some(name) => SheetLocator::from_name(name),
18075 None => SheetLocator::Current,
18076 };
18077 let sr = start_row
18078 .map(|r| {
18079 r.checked_sub(1)
18080 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
18081 })
18082 .transpose()?;
18083 let sc = start_col
18084 .map(|c| {
18085 c.checked_sub(1)
18086 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
18087 })
18088 .transpose()?;
18089 let er = end_row
18090 .map(|r| {
18091 r.checked_sub(1)
18092 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
18093 })
18094 .transpose()?;
18095 let ec = end_col
18096 .map(|c| {
18097 c.checked_sub(1)
18098 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
18099 })
18100 .transpose()?;
18101 let range = SharedRangeRef::from_parts(
18102 sheet_loc,
18103 sr.map(|idx| formualizer_common::AxisBound::new(idx, *start_row_abs)),
18104 sc.map(|idx| formualizer_common::AxisBound::new(idx, *start_col_abs)),
18105 er.map(|idx| formualizer_common::AxisBound::new(idx, *end_row_abs)),
18106 ec.map(|idx| formualizer_common::AxisBound::new(idx, *end_col_abs)),
18107 )
18108 .map_err(|_| ExcelError::new(ExcelErrorKind::Ref))?;
18109 SharedRef::Range(range)
18110 }
18111 _ => return Err(ExcelError::new(ExcelErrorKind::Ref)),
18112 };
18113
18114 let current_id = self
18115 .graph
18116 .sheet_id(current_sheet)
18117 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18118
18119 let resolve_loc = |loc: SheetLocator<'_>| -> Result<SheetLocator<'static>, ExcelError> {
18120 match loc {
18121 SheetLocator::Current => Ok(SheetLocator::Id(current_id)),
18122 SheetLocator::Id(id) => Ok(SheetLocator::Id(id)),
18123 SheetLocator::Name(name) => {
18124 let n = name.as_ref();
18125 self.graph
18126 .sheet_id(n)
18127 .map(SheetLocator::Id)
18128 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
18129 }
18130 }
18131 };
18132
18133 match sr {
18134 SharedRef::Cell(cell) => {
18135 let owned = cell.into_owned();
18136 let sheet = resolve_loc(owned.sheet)?;
18137 Ok(SharedRef::Cell(SharedCellRef::new(sheet, owned.coord)))
18138 }
18139 SharedRef::Range(range) => {
18140 let owned = range.into_owned();
18141 let sheet = resolve_loc(owned.sheet)?;
18142 Ok(SharedRef::Range(SharedRangeRef {
18143 sheet,
18144 start_row: owned.start_row,
18145 start_col: owned.start_col,
18146 end_row: owned.end_row,
18147 end_col: owned.end_col,
18148 }))
18149 }
18150 }
18151 }
18152}
18153
18154impl<R> crate::traits::ReferenceResolver for Engine<R>
18157where
18158 R: EvaluationContext,
18159{
18160 fn resolve_cell_reference(
18161 &self,
18162 sheet: Option<&str>,
18163 row: u32,
18164 col: u32,
18165 ) -> Result<LiteralValue, ExcelError> {
18166 let Some(sheet_name) = sheet else {
18177 return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
18178 "Unqualified cell reference resolved without sheet context".to_string(),
18179 ));
18180 };
18181 if let Some(v) = self.read_cell_value(sheet_name, row, col) {
18184 Ok(v)
18185 } else {
18186 Ok(LiteralValue::Number(0.0))
18188 }
18189 }
18190}
18191
18192impl<R> crate::traits::RangeResolver for Engine<R>
18193where
18194 R: EvaluationContext,
18195{
18196 fn resolve_range_reference(
18197 &self,
18198 sheet: Option<&str>,
18199 sr: Option<u32>,
18200 sc: Option<u32>,
18201 er: Option<u32>,
18202 ec: Option<u32>,
18203 ) -> Result<Box<dyn crate::traits::Range>, ExcelError> {
18204 self.resolver.resolve_range_reference(sheet, sr, sc, er, ec)
18207 }
18208}
18209
18210impl<R> crate::traits::NamedRangeResolver for Engine<R>
18211where
18212 R: EvaluationContext,
18213{
18214 fn resolve_named_range_reference(
18215 &self,
18216 name: &str,
18217 ) -> Result<Vec<Vec<LiteralValue>>, ExcelError> {
18218 self.resolver.resolve_named_range_reference(name)
18219 }
18220}
18221
18222impl<R> crate::traits::TableResolver for Engine<R>
18223where
18224 R: EvaluationContext,
18225{
18226 fn resolve_table_reference(
18227 &self,
18228 tref: &formualizer_parse::parser::TableReference,
18229 ) -> Result<Box<dyn crate::traits::Table>, ExcelError> {
18230 self.resolver.resolve_table_reference(tref)
18231 }
18232}
18233
18234impl<R> crate::traits::SourceResolver for Engine<R>
18235where
18236 R: EvaluationContext,
18237{
18238 fn source_scalar_version(&self, name: &str) -> Option<u64> {
18239 self.resolver.source_scalar_version(name)
18240 }
18241
18242 fn resolve_source_scalar(&self, name: &str) -> Result<LiteralValue, ExcelError> {
18243 self.resolver.resolve_source_scalar(name)
18244 }
18245
18246 fn source_table_version(&self, name: &str) -> Option<u64> {
18247 self.resolver.source_table_version(name)
18248 }
18249
18250 fn resolve_source_table(
18251 &self,
18252 name: &str,
18253 ) -> Result<Box<dyn crate::traits::Table>, ExcelError> {
18254 self.resolver.resolve_source_table(name)
18255 }
18256}
18257
18258impl<R> crate::traits::Resolver for Engine<R> where R: EvaluationContext {}
18260
18261impl<R: EvaluationContext> crate::interpreter::ArenaCellResolver for Engine<R> {
18263 fn resolve_arena_range(
18264 &self,
18265 reference: &crate::engine::arena::CompactRefType,
18266 current: SheetId,
18267 row_delta: i64,
18268 col_delta: i64,
18269 ) -> Result<RangeView<'_>, ExcelError> {
18270 use crate::engine::arena::{CompactRefType, SheetKey};
18271 let CompactRefType::Range {
18272 sheet,
18273 start_row,
18274 start_col,
18275 end_row,
18276 end_col,
18277 start_row_abs,
18278 start_col_abs,
18279 end_row_abs,
18280 end_col_abs,
18281 } = reference
18282 else {
18283 return Err(ExcelError::new(ExcelErrorKind::Ref));
18284 };
18285 let sheet_id = match sheet {
18286 None => current,
18287 Some(SheetKey::Id(id)) => *id,
18288 Some(SheetKey::Name(id)) => self
18289 .graph
18290 .sheet_id(self.graph.data_store().resolve_ast_string(*id))
18291 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?,
18292 };
18293 if self.graph.sheet_reg().name(sheet_id).is_empty() {
18294 return Err(ExcelError::new(ExcelErrorKind::Ref));
18295 }
18296 let axis = |value, sentinel, delta, absolute| {
18297 if value == sentinel {
18298 Ok(None)
18299 } else {
18300 crate::interpreter::shift_axis_for_offset(value, delta, absolute).map(Some)
18301 }
18302 };
18303 let sr = axis(*start_row, 0, row_delta, *start_row_abs)?;
18304 let sc = axis(*start_col, 0, col_delta, *start_col_abs)?;
18305 let er = axis(*end_row, u32::MAX, row_delta, *end_row_abs)?;
18306 let ec = axis(*end_col, u32::MAX, col_delta, *end_col_abs)?;
18307 if sr.zip(er).is_some_and(|(s, e)| s > e) || sc.zip(ec).is_some_and(|(s, e)| s > e) {
18308 return Err(ExcelError::new(ExcelErrorKind::Ref));
18309 }
18310 self.resolve_range_coordinates(sheet_id, sr, sc, er, ec)
18311 }
18312
18313 fn resolve_arena_cell(
18314 &self,
18315 sheet: Option<crate::engine::arena::SheetKey>,
18316 current: crate::SheetId,
18317 row: u32,
18318 col: u32,
18319 ) -> Result<(LiteralValue, Option<crate::format::FormatId>), ExcelError> {
18320 use crate::engine::arena::SheetKey;
18321 let (sheet_id, named_arrow) = match sheet {
18322 None => (current, None),
18323 Some(SheetKey::Id(id)) => {
18324 if self.graph.sheet_reg().name(id).is_empty() {
18325 return Err(ExcelError::new(ExcelErrorKind::Ref));
18326 }
18327 (id, None)
18328 }
18329 Some(SheetKey::Name(id)) => {
18330 let (sheet_id, arrow) = self
18331 .resolved_sheet_names
18332 .get(&id)
18333 .copied()
18334 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18335 (sheet_id, Some(arrow))
18336 }
18337 };
18338 let arrow = named_arrow.unwrap_or_else(|| {
18339 self.resolved_arrow_sheets
18340 .get(usize::from(sheet_id))
18341 .copied()
18342 .flatten()
18343 });
18344 let asheet = arrow
18345 .and_then(|index| self.arrow_sheets.sheets.get(index))
18346 .or_else(|| {
18347 let name = match sheet {
18351 Some(SheetKey::Name(id)) => self.graph.data_store().resolve_ast_string(id),
18352 _ => self.graph.sheet_name(sheet_id),
18353 };
18354 self.arrow_sheets.sheet(name)
18355 });
18356 Ok(self.read_cell_formatted_in(sheet_id, asheet, row, col))
18357 }
18358}
18359
18360impl<R> crate::traits::FunctionProvider for Engine<R>
18361where
18362 R: EvaluationContext,
18363{
18364 fn planning_semantic_revision(&self) -> Option<u64> {
18365 self.resolver.planning_semantic_revision()
18366 }
18367
18368 fn get_function(
18369 &self,
18370 prefix: &str,
18371 name: &str,
18372 ) -> Option<std::sync::Arc<dyn crate::function::Function>> {
18373 self.resolver.get_function(prefix, name)
18374 }
18375
18376 fn get_function_for_planning(
18377 &self,
18378 prefix: &str,
18379 name: &str,
18380 ) -> Option<std::sync::Arc<dyn crate::function::Function>> {
18381 self.resolver.get_function_for_planning(prefix, name)
18382 }
18383}
18384
18385impl<R> Engine<R>
18386where
18387 R: EvaluationContext,
18388{
18389 fn resolve_range_coordinates(
18390 &self,
18391 sheet_id: SheetId,
18392 sr: Option<u32>,
18393 sc: Option<u32>,
18394 er: Option<u32>,
18395 ec: Option<u32>,
18396 ) -> Result<RangeView<'_>, ExcelError> {
18397 let sheet_name = self.graph.sheet_name(sheet_id);
18398 let extent = resolve_used_extent_with_fallback(
18399 OpenRangeBounds {
18400 start_row: sr,
18401 start_column: sc,
18402 end_row: er,
18403 end_column: ec,
18404 },
18405 ExtentPolicy::EvaluationCompat {
18406 fallback_row: None,
18407 fallback_column: None,
18408 },
18409 || {
18410 self.sheet_bounds(sheet_name)
18411 .map(|_| self.config.max_open_ended_rows)
18412 },
18413 || {
18414 self.sheet_bounds(sheet_name)
18415 .map(|_| self.config.max_open_ended_cols)
18416 },
18417 |first, last| self.used_rows_for_columns(sheet_name, first, last),
18418 |first, last| self.used_cols_for_rows(sheet_name, first, last),
18419 );
18420 let (sr, sc, er, ec) = extent
18421 .map(|e| (e.start_row, e.start_column, e.end_row, e.end_column))
18422 .unwrap_or((1, 1, 0, 0));
18423 if self.force_materialize_range_views {
18424 if er < sr || ec < sc {
18425 return Ok(RangeView::from_owned_rows(
18426 Vec::new(),
18427 self.config.date_system,
18428 ));
18429 }
18430 let h = (er - sr + 1) as u64;
18431 let w = (ec - sc + 1) as u64;
18432 if h.saturating_mul(w) <= self.config.spill.max_spill_cells as u64 {
18433 let mut rows = Vec::with_capacity(h as usize);
18434 for r in sr..=er {
18435 let mut row = Vec::with_capacity(w as usize);
18436 for c in sc..=ec {
18437 row.push(
18438 self.read_cell_value(sheet_name, r, c)
18439 .unwrap_or(LiteralValue::Empty),
18440 );
18441 }
18442 rows.push(row);
18443 }
18444 return Ok(RangeView::from_owned_rows(rows, self.config.date_system));
18445 }
18446 }
18447 let Some(asheet) = self.arrow_sheet_by_id(sheet_id) else {
18448 return Ok(RangeView::from_owned_rows(
18449 Vec::new(),
18450 self.config.date_system,
18451 ));
18452 };
18453 Ok(if er < sr || ec < sc {
18454 asheet.range_view(1, 1, 0, 0)
18455 } else {
18456 asheet.range_view(
18457 sr.saturating_sub(1) as usize,
18458 sc.saturating_sub(1) as usize,
18459 er.saturating_sub(1) as usize,
18460 ec.saturating_sub(1) as usize,
18461 )
18462 })
18463 }
18464
18465 pub(crate) fn semantic_used_rows_for_columns(
18473 &self,
18474 sheet: &str,
18475 start_col: u32,
18476 end_col: u32,
18477 ) -> Option<(u32, u32)> {
18478 let arrow_bounds = self
18479 .sheet_store()
18480 .sheet(sheet)
18481 .and_then(|_| self.arrow_used_row_bounds(sheet, start_col, end_col));
18482 let formula_bounds = self.formula_row_bounds_for_columns(sheet, start_col, end_col);
18483 Self::union_used_bounds(arrow_bounds, formula_bounds)
18484 }
18485
18486 pub(crate) fn semantic_used_cols_for_rows(
18487 &self,
18488 sheet: &str,
18489 start_row: u32,
18490 end_row: u32,
18491 ) -> Option<(u32, u32)> {
18492 let arrow_bounds = self
18493 .sheet_store()
18494 .sheet(sheet)
18495 .and_then(|_| self.arrow_used_col_bounds(sheet, start_row, end_row));
18496 let formula_bounds = self.formula_col_bounds_for_rows(sheet, start_row, end_row);
18497 Self::union_used_bounds(arrow_bounds, formula_bounds)
18498 }
18499}
18500
18501impl<R> crate::traits::EvaluationContext for Engine<R>
18503where
18504 R: EvaluationContext,
18505{
18506 fn clock(&self) -> &dyn crate::timezone::ClockProvider {
18507 &self.clock
18508 }
18509
18510 fn thread_pool(&self) -> Option<&Arc<rayon::ThreadPool>> {
18511 self.thread_pool.as_ref()
18512 }
18513
18514 fn cancellation_token(&self) -> Option<crate::engine::CancelToken> {
18515 self.active_cancel_flag.clone()
18516 }
18517
18518 fn chunk_hint(&self) -> Option<usize> {
18519 let hint =
18521 (self.config.stripe_height as usize).saturating_mul(self.config.stripe_width as usize);
18522 Some(hint.clamp(1024, 1 << 20)) }
18524
18525 fn materialized_array_cells(&self) -> u64 {
18526 self.config
18527 .evaluation_budgets
18528 .scratch
18529 .materialized_array_cells
18530 .unwrap_or(crate::engine::resource_ledger::DEFAULT_MATERIALIZED_ARRAY_CELLS)
18531 }
18532
18533 fn volatile_level(&self) -> crate::traits::VolatileLevel {
18534 self.config.volatile_level
18535 }
18536
18537 fn workbook_seed(&self) -> u64 {
18538 self.config.workbook_seed
18539 }
18540
18541 fn recalc_epoch(&self) -> u64 {
18542 self.recalc_epoch
18543 }
18544
18545 fn workbook_sheet_count(&self) -> Option<usize> {
18546 Some(self.graph.sheet_reg().active_len())
18547 }
18548
18549 fn sheet_index_by_name(&self, sheet: &str) -> Option<usize> {
18550 self.graph.sheet_reg().active_position(sheet)
18551 }
18552
18553 fn current_sheet_index(&self, current_sheet: &str) -> Option<usize> {
18554 self.sheet_index_by_name(current_sheet)
18555 }
18556
18557 fn inspect_reference(
18558 &self,
18559 reference: &ReferenceType,
18560 current_sheet: &str,
18561 ) -> Result<Option<ReferenceInfo>, ExcelError> {
18562 let sheet_info = |sheet_name: &str| -> Result<(SheetId, usize), ExcelError> {
18563 let sheet_id = self
18564 .graph
18565 .sheet_id(sheet_name)
18566 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18567 let sheet_index = self
18568 .graph
18569 .sheet_reg()
18570 .active_position_by_id(sheet_id)
18571 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18572 Ok((sheet_id, sheet_index))
18573 };
18574
18575 let cell_info =
18576 |sheet_name: &str, row: u32, col: u32| -> Result<ReferenceInfo, ExcelError> {
18577 let (sheet_id, sheet_index) = sheet_info(sheet_name)?;
18578 let row0 = row
18579 .checked_sub(1)
18580 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18581 let col0 = col
18582 .checked_sub(1)
18583 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18584 Ok(ReferenceInfo {
18585 first_sheet_index: Some(sheet_index),
18586 sheet_count: Some(1),
18587 first_cell: Some(CellRef::new(sheet_id, Coord::new(row0, col0, true, true))),
18588 })
18589 };
18590
18591 let range_info = |sheet_name: &str,
18592 start_row: Option<u32>,
18593 start_col: Option<u32>|
18594 -> Result<ReferenceInfo, ExcelError> {
18595 let (sheet_id, sheet_index) = sheet_info(sheet_name)?;
18596 let row = start_row.unwrap_or(1);
18597 let col = start_col.unwrap_or(1);
18598 let row0 = row
18599 .checked_sub(1)
18600 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18601 let col0 = col
18602 .checked_sub(1)
18603 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18604 Ok(ReferenceInfo {
18605 first_sheet_index: Some(sheet_index),
18606 sheet_count: Some(1),
18607 first_cell: Some(CellRef::new(sheet_id, Coord::new(row0, col0, true, true))),
18608 })
18609 };
18610
18611 let info = match reference {
18612 ReferenceType::Cell {
18613 sheet, row, col, ..
18614 } => {
18615 let sheet_name = sheet.as_deref().unwrap_or(current_sheet);
18616 cell_info(sheet_name, *row, *col)?
18617 }
18618 ReferenceType::Range {
18619 sheet,
18620 start_row,
18621 start_col,
18622 ..
18623 } => {
18624 let sheet_name = sheet.as_deref().unwrap_or(current_sheet);
18625 range_info(sheet_name, *start_row, *start_col)?
18626 }
18627 ReferenceType::Cell3D {
18628 sheet_first,
18629 sheet_last,
18630 row,
18631 col,
18632 ..
18633 } => {
18634 let first = cell_info(sheet_first, *row, *col)?;
18635 ReferenceInfo {
18636 first_sheet_index: first.first_sheet_index,
18637 sheet_count: self
18638 .graph
18639 .sheet_reg()
18640 .active_span_len(sheet_first, sheet_last),
18641 first_cell: first.first_cell,
18642 }
18643 }
18644 ReferenceType::Range3D {
18645 sheet_first,
18646 sheet_last,
18647 start_row,
18648 start_col,
18649 ..
18650 } => {
18651 let first = range_info(sheet_first, *start_row, *start_col)?;
18652 ReferenceInfo {
18653 first_sheet_index: first.first_sheet_index,
18654 sheet_count: self
18655 .graph
18656 .sheet_reg()
18657 .active_span_len(sheet_first, sheet_last),
18658 first_cell: first.first_cell,
18659 }
18660 }
18661 ReferenceType::NamedRange(name) => {
18662 let current_id = self
18663 .graph
18664 .sheet_id(current_sheet)
18665 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18666 let named = self
18667 .graph
18668 .resolve_name_entry(name, current_id)
18669 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18670 match &named.definition {
18671 NamedDefinition::Cell(cell) => ReferenceInfo {
18672 first_sheet_index: self
18673 .graph
18674 .sheet_reg()
18675 .active_position_by_id(cell.sheet_id),
18676 sheet_count: Some(1),
18677 first_cell: Some(*cell),
18678 },
18679 NamedDefinition::Range(range) => ReferenceInfo {
18680 first_sheet_index: self
18681 .graph
18682 .sheet_reg()
18683 .active_position_by_id(range.start.sheet_id),
18684 sheet_count: Some(1),
18685 first_cell: Some(range.start),
18686 },
18687 NamedDefinition::Literal(_) | NamedDefinition::Formula { .. } => {
18688 ReferenceInfo {
18689 first_sheet_index: None,
18690 sheet_count: None,
18691 first_cell: None,
18692 }
18693 }
18694 }
18695 }
18696 ReferenceType::Table(tref) => {
18697 let table = self
18698 .graph
18699 .resolve_table_entry(&tref.name)
18700 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18701 ReferenceInfo {
18702 first_sheet_index: self
18703 .graph
18704 .sheet_reg()
18705 .active_position_by_id(table.range.start.sheet_id),
18706 sheet_count: Some(1),
18707 first_cell: Some(table.range.start),
18708 }
18709 }
18710 ReferenceType::External(_) => return Err(ExcelError::new(ExcelErrorKind::Ref)),
18711 };
18712
18713 Ok(Some(info))
18714 }
18715
18716 fn resolve_spill_reference(
18717 &self,
18718 anchor: &ReferenceType,
18719 current_sheet: &str,
18720 ) -> Result<ReferenceType, ExcelError> {
18721 let no_spill = || {
18722 ExcelError::new(ExcelErrorKind::Ref)
18723 .with_message("Spill reference anchor has no current spill")
18724 };
18725 let mut qualify = true;
18728 let anchor_cell = match anchor {
18729 ReferenceType::Cell {
18730 sheet, row, col, ..
18731 } => {
18732 qualify = sheet.is_some();
18733 let sheet_name = sheet.as_deref().unwrap_or(current_sheet);
18734 let sheet_id = self.graph.sheet_id(sheet_name).ok_or_else(no_spill)?;
18735 if *row == 0 || *col == 0 {
18736 return Err(no_spill());
18737 }
18738 CellRef::new(sheet_id, Coord::from_excel(*row, *col, true, true))
18739 }
18740 ReferenceType::NamedRange(name) => {
18741 let current_id = self.graph.sheet_id(current_sheet).ok_or_else(no_spill)?;
18742 let named = self
18743 .graph
18744 .resolve_name_entry(name, current_id)
18745 .ok_or_else(no_spill)?;
18746 match &named.definition {
18747 NamedDefinition::Cell(cell) => *cell,
18748 NamedDefinition::Range(range)
18749 if range.start.sheet_id == range.end.sheet_id
18750 && range.start.coord.row() == range.end.coord.row()
18751 && range.start.coord.col() == range.end.coord.col() =>
18752 {
18753 range.start
18754 }
18755 _ => {
18756 return Err(ExcelError::new(ExcelErrorKind::Ref)
18757 .with_message("Spill reference name must refer to a single cell"));
18758 }
18759 }
18760 }
18761 _ => {
18762 return Err(ExcelError::new(ExcelErrorKind::Ref)
18763 .with_message("Spill reference operand must be a single cell"));
18764 }
18765 };
18766 let vertex = self
18767 .graph
18768 .get_vertex_id_for_address(&anchor_cell)
18769 .ok_or_else(no_spill)?;
18770 if self.graph.fixed_single_arrays.contains(&vertex)
18773 || self.graph.fixed_array_shapes.contains_key(&vertex)
18774 {
18775 return Err(ExcelError::new(ExcelErrorKind::Ref)
18776 .with_message("Spill reference to a fixed array formula"));
18777 }
18778 let (first, last) = match self.graph.spill_extent_for_anchor(vertex) {
18779 Some(extent) => extent,
18780 None if !self.graph.spill_registry_has_anchor(vertex)
18784 && self.graph.is_current_declared_dynamic_anchor(vertex)
18785 && self.declared_anchor_holds_scalar(anchor_cell) =>
18786 {
18787 (anchor_cell, anchor_cell)
18788 }
18789 None => return Err(no_spill()),
18790 };
18791 let sheet_name = qualify.then(|| self.graph.sheet_name(first.sheet_id).to_string());
18792 let (sr, sc) = (first.coord.row() + 1, first.coord.col() + 1);
18793 let (er, ec) = (last.coord.row() + 1, last.coord.col() + 1);
18794 Ok(if sr == er && sc == ec {
18796 ReferenceType::cell_with_abs(sheet_name, sr, sc, true, true)
18797 } else {
18798 ReferenceType::Range {
18799 sheet: sheet_name,
18800 start_row: Some(sr),
18801 start_col: Some(sc),
18802 end_row: Some(er),
18803 end_col: Some(ec),
18804 start_row_abs: true,
18805 start_col_abs: true,
18806 end_row_abs: true,
18807 end_col_abs: true,
18808 }
18809 })
18810 }
18811
18812 fn formula_text_at_cell(&self, cell: CellRef) -> Result<Option<String>, ExcelError> {
18813 let sheet_name = self.graph.sheet_name(cell.sheet_id);
18814 if sheet_name.is_empty() {
18815 return Err(ExcelError::new(ExcelErrorKind::Ref));
18816 }
18817 let row = cell.coord.row() + 1;
18818 let col = cell.coord.col() + 1;
18819
18820 if let Some(entries) = self.staged_formulas.get(sheet_name)
18821 && let Some(text) = entries.get(row, col)
18822 {
18823 return Ok(Some(if text.starts_with('=') {
18824 text.to_owned()
18825 } else {
18826 format!("={text}")
18827 }));
18828 }
18829
18830 let Some((Some(ast), _)) = self.get_cell(sheet_name, row, col) else {
18831 return Ok(None);
18832 };
18833 Ok(Some(formualizer_parse::pretty::canonical_formula(&ast)))
18834 }
18835
18836 fn used_rows_for_columns(
18837 &self,
18838 sheet: &str,
18839 start_col: u32,
18840 end_col: u32,
18841 ) -> Option<(u32, u32)> {
18842 let sheet_id = self.graph.sheet_id(sheet)?;
18844 let snap = self.data_snapshot_id();
18845 if let Some(cached) = self.used_axis_bounds_cache.read().ok().and_then(|guard| {
18846 guard
18847 .as_ref()
18848 .and_then(|cache| cache.get_row_bounds(sheet_id, start_col, end_col, snap))
18849 }) {
18850 return cached;
18851 }
18852
18853 let arrow_bounds = self
18854 .sheet_store()
18855 .sheet(sheet)
18856 .and_then(|_| self.arrow_used_row_bounds(sheet, start_col, end_col));
18857 let formula_bounds = self.formula_row_bounds_for_columns(sheet, start_col, end_col);
18858 let computed = if let Some(bounds) = Self::union_used_bounds(arrow_bounds, formula_bounds) {
18859 Some(bounds)
18860 } else {
18861 let sc0 = start_col.saturating_sub(1);
18862 let ec0 = end_col.saturating_sub(1);
18863 self.graph
18864 .used_row_bounds_for_columns(sheet_id, sc0, ec0)
18865 .map(|(a0, b0)| (a0 + 1, b0 + 1))
18866 };
18867
18868 if let Ok(mut guard) = self.used_axis_bounds_cache.write() {
18869 guard
18870 .get_or_insert_with(|| UsedAxisBoundsCache::new(snap))
18871 .put_row_bounds(sheet_id, start_col, end_col, snap, computed);
18872 }
18873
18874 computed
18875 }
18876
18877 fn used_cols_for_rows(&self, sheet: &str, start_row: u32, end_row: u32) -> Option<(u32, u32)> {
18878 let sheet_id = self.graph.sheet_id(sheet)?;
18880 let snap = self.data_snapshot_id();
18881 if let Some(cached) = self.used_axis_bounds_cache.read().ok().and_then(|guard| {
18882 guard
18883 .as_ref()
18884 .and_then(|cache| cache.get_col_bounds(sheet_id, start_row, end_row, snap))
18885 }) {
18886 return cached;
18887 }
18888
18889 let arrow_bounds = self
18890 .sheet_store()
18891 .sheet(sheet)
18892 .and_then(|_| self.arrow_used_col_bounds(sheet, start_row, end_row));
18893 let formula_bounds = self.formula_col_bounds_for_rows(sheet, start_row, end_row);
18894 let computed = if let Some(bounds) = Self::union_used_bounds(arrow_bounds, formula_bounds) {
18895 Some(bounds)
18896 } else {
18897 let sr0 = start_row.saturating_sub(1);
18898 let er0 = end_row.saturating_sub(1);
18899 self.graph
18900 .used_col_bounds_for_rows(sheet_id, sr0, er0)
18901 .map(|(a0, b0)| (a0 + 1, b0 + 1))
18902 };
18903
18904 if let Ok(mut guard) = self.used_axis_bounds_cache.write() {
18905 guard
18906 .get_or_insert_with(|| UsedAxisBoundsCache::new(snap))
18907 .put_col_bounds(sheet_id, start_row, end_row, snap, computed);
18908 }
18909
18910 computed
18911 }
18912
18913 fn sheet_bounds(&self, sheet: &str) -> Option<(u32, u32)> {
18914 let _ = self.graph.sheet_id(sheet)?;
18915 Some((1_048_576, 16_384)) }
18919
18920 fn data_snapshot_id(&self) -> u64 {
18921 self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed)
18922 }
18923
18924 fn backend_caps(&self) -> crate::traits::BackendCaps {
18925 crate::traits::BackendCaps {
18926 streaming: true,
18927 used_region: true,
18928 write: false,
18929 tables: false,
18930 async_stream: false,
18931 }
18932 }
18933
18934 fn build_lookup_index(
18935 &self,
18936 view: &RangeView<'_>,
18937 axis: LookupAxis,
18938 ) -> Option<Arc<LookupIndex>> {
18939 self.build_lookup_index_impl(view, axis)
18940 }
18941
18942 fn date_system(&self) -> crate::engine::DateSystem {
18945 self.config.date_system
18946 }
18947 fn resolve_range_view<'c>(
18949 &'c self,
18950 reference: &ReferenceType,
18951 current_sheet: &str,
18952 ) -> Result<RangeView<'c>, ExcelError> {
18953 match reference {
18954 ReferenceType::External(ext) => {
18955 let name = ext.raw.as_str();
18956 match ext.kind {
18957 formualizer_parse::parser::ExternalRefKind::Cell { .. } => {
18958 let Some(source) = self.graph.resolve_source_scalar_entry(name) else {
18959 return Err(ExcelError::new(ExcelErrorKind::Name)
18960 .with_message(format!("Undefined name: {name}")));
18961 };
18962 let version = source
18963 .version
18964 .or_else(|| self.resolver.source_scalar_version(name));
18965 let v = self.resolve_source_scalar_cached(name, version)?;
18966 Ok(RangeView::from_owned_rows(
18967 vec![vec![v]],
18968 self.config.date_system,
18969 ))
18970 }
18971 formualizer_parse::parser::ExternalRefKind::Range { .. } => {
18972 let Some(source) = self.graph.resolve_source_table_entry(name) else {
18973 let kind =
18977 if crate::engine::refs::unbound_external_range_defers(&ext.kind) {
18978 ExcelErrorKind::Ref
18979 } else {
18980 ExcelErrorKind::Name
18981 };
18982 return Err(ExcelError::new(kind)
18983 .with_message(format!("Undefined table: {name}")));
18984 };
18985 let version = source
18986 .version
18987 .or_else(|| self.resolver.source_table_version(name));
18988 let key = (name.to_string(), version);
18989 if let Ok(g) = self.source_cache.read()
18990 && let Some(view) = g.external_views.get(&key)
18991 {
18992 return Ok(view.clone());
18993 }
18994 let table = self.resolve_source_table_cached(name, version)?;
18995 let spec = Some(formualizer_parse::parser::TableSpecifier::Data);
18996 let Ok(mut g) = self.source_cache.write() else {
18999 return self.source_table_to_range_view(table.as_ref(), &spec);
19000 };
19001 if let Some(view) = g.external_views.get(&key) {
19002 return Ok(view.clone());
19003 }
19004 let view = self.source_table_to_range_view(table.as_ref(), &spec)?;
19005 g.external_views.insert(key, view.clone());
19006 Ok(view)
19007 }
19008 }
19009 }
19010 ReferenceType::Range { .. } => {
19011 let shared = self.resolve_shared_ref(reference, current_sheet)?;
19012 let formualizer_common::SheetRef::Range(range) = shared else {
19013 return Err(ExcelError::new(ExcelErrorKind::Ref));
19014 };
19015 let sheet_id = match range.sheet {
19018 formualizer_common::SheetLocator::Id(id) => id,
19019 formualizer_common::SheetLocator::Current
19020 | formualizer_common::SheetLocator::Name(_) => {
19021 return Err(ExcelError::new(ExcelErrorKind::Ref));
19022 }
19023 };
19024 let bounded_range = if range.start_row.is_some()
19025 && range.start_col.is_some()
19026 && range.end_row.is_some()
19027 && range.end_col.is_some()
19028 {
19029 Some(RangeRef::try_from_shared(range.as_ref())?)
19030 } else {
19031 None
19032 };
19033
19034 let sr = bounded_range
19035 .as_ref()
19036 .map(|r| r.start.coord.row() + 1)
19037 .or_else(|| range.start_row.map(|b| b.index + 1));
19038 let sc = bounded_range
19039 .as_ref()
19040 .map(|r| r.start.coord.col() + 1)
19041 .or_else(|| range.start_col.map(|b| b.index + 1));
19042 let er = bounded_range
19043 .as_ref()
19044 .map(|r| r.end.coord.row() + 1)
19045 .or_else(|| range.end_row.map(|b| b.index + 1));
19046 let ec = bounded_range
19047 .as_ref()
19048 .map(|r| r.end.coord.col() + 1)
19049 .or_else(|| range.end_col.map(|b| b.index + 1));
19050
19051 self.resolve_range_coordinates(sheet_id, sr, sc, er, ec)
19052 }
19053 ReferenceType::Cell { .. } => {
19054 let shared = self.resolve_shared_ref(reference, current_sheet)?;
19055 let formualizer_common::SheetRef::Cell(cell) = shared else {
19056 return Err(ExcelError::new(ExcelErrorKind::Ref));
19057 };
19058 let addr = CellRef::try_from_shared(cell)?;
19059 let sheet_id = addr.sheet_id;
19060 let sheet_name = self.graph.sheet_name(sheet_id);
19061 let row = addr.coord.row() + 1;
19062 let col = addr.coord.col() + 1;
19063
19064 if self.force_materialize_range_views {
19065 let v = self
19066 .read_cell_value(sheet_name, row, col)
19067 .unwrap_or(LiteralValue::Empty);
19068 return Ok(RangeView::from_owned_rows(
19069 vec![vec![v]],
19070 self.config.date_system,
19071 ));
19072 }
19073
19074 if let Some(asheet) = self.sheet_store().sheet(sheet_name) {
19075 let r0 = row.saturating_sub(1) as usize;
19076 let c0 = col.saturating_sub(1) as usize;
19077 let rv = asheet.range_view(r0, c0, r0, c0);
19078 Ok(rv)
19079 } else {
19080 let v = self
19081 .read_cell_value(sheet_name, row, col)
19082 .unwrap_or(LiteralValue::Empty);
19083 Ok(RangeView::from_owned_rows(
19084 vec![vec![v]],
19085 self.config.date_system,
19086 ))
19087 }
19088 }
19089 ReferenceType::NamedRange(name) => {
19090 if let Some(current_id) = self.graph.sheet_id(current_sheet)
19091 && let Some(named) = self.graph.resolve_name_entry(name, current_id)
19092 {
19093 match &named.definition {
19094 NamedDefinition::Cell(cell_ref) => {
19095 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
19096 if self.force_materialize_range_views {
19097 let v = self
19098 .read_cell_value(
19099 sheet_name,
19100 cell_ref.coord.row() + 1,
19101 cell_ref.coord.col() + 1,
19102 )
19103 .unwrap_or(LiteralValue::Empty);
19104 return Ok(RangeView::from_owned_rows(
19105 vec![vec![v]],
19106 self.config.date_system,
19107 ));
19108 } else {
19109 let asheet = self
19110 .sheet_store()
19111 .sheet(sheet_name)
19112 .expect("Arrow sheet missing for named cell");
19113 let r0 = cell_ref.coord.row() as usize;
19114 let c0 = cell_ref.coord.col() as usize;
19115 let rv = asheet.range_view(r0, c0, r0, c0);
19116 return Ok(rv);
19117 }
19118 }
19119 NamedDefinition::Range(range_ref) => {
19120 let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
19121 let sr = range_ref.start.coord.row() + 1;
19122 let sc = range_ref.start.coord.col() + 1;
19123 let er = range_ref.end.coord.row() + 1;
19124 let ec = range_ref.end.coord.col() + 1;
19125 if self.force_materialize_range_views {
19126 let h = (er.saturating_sub(sr) + 1) as u64;
19127 let w = (ec.saturating_sub(sc) + 1) as u64;
19128 let cell_count = h.saturating_mul(w);
19129 if cell_count <= self.config.spill.max_spill_cells as u64 {
19130 let mut rows: Vec<Vec<LiteralValue>> =
19131 Vec::with_capacity(h as usize);
19132 for r in sr..=er {
19133 let mut rowv: Vec<LiteralValue> =
19134 Vec::with_capacity(w as usize);
19135 for c in sc..=ec {
19136 rowv.push(
19137 self.read_cell_value(sheet_name, r, c)
19138 .unwrap_or(LiteralValue::Empty),
19139 );
19140 }
19141 rows.push(rowv);
19142 }
19143 return Ok(RangeView::from_owned_rows(
19144 rows,
19145 self.config.date_system,
19146 ));
19147 }
19148 }
19149 let asheet = self
19150 .sheet_store()
19151 .sheet(sheet_name)
19152 .expect("Arrow sheet missing for named range");
19153 let sr0 = range_ref.start.coord.row() as usize;
19154 let sc0 = range_ref.start.coord.col() as usize;
19155 let er0 = range_ref.end.coord.row() as usize;
19156 let ec0 = range_ref.end.coord.col() as usize;
19157 let rv = asheet.range_view(sr0, sc0, er0, ec0);
19158 return Ok(rv);
19159 }
19160 NamedDefinition::Literal(v) => {
19161 return Ok(RangeView::from_owned_rows(
19162 vec![vec![v.clone()]],
19163 self.config.date_system,
19164 ));
19165 }
19166 NamedDefinition::Formula { ast, .. } => {
19167 if let ASTNodeType::Reference {
19170 reference: external @ ReferenceType::External(_),
19171 ..
19172 } = &ast.node_type
19173 {
19174 return self.resolve_range_view(external, current_sheet);
19175 }
19176 if let Some(value) = self.graph.get_value(named.vertex) {
19177 return Ok(RangeView::from_owned_rows(
19178 vec![vec![value]],
19179 self.config.date_system,
19180 ));
19181 }
19182 }
19183 }
19184 }
19185
19186 if let Some(source) = self.graph.resolve_source_scalar_entry(name) {
19187 let version = source
19188 .version
19189 .or_else(|| self.resolver.source_scalar_version(name));
19190 let v = self.resolve_source_scalar_cached(name, version)?;
19191 return Ok(RangeView::from_owned_rows(
19192 vec![vec![v]],
19193 self.config.date_system,
19194 ));
19195 }
19196
19197 let data = self.resolver.resolve_named_range_reference(name)?;
19198 Ok(RangeView::from_owned_rows(data, self.config.date_system))
19199 }
19200 ReferenceType::Table(tref) => {
19201 if let Some(table) = self.graph.resolve_table_entry(&tref.name) {
19202 let sheet_name = self.graph.sheet_name(table.range.start.sheet_id);
19203 let asheet = self
19204 .sheet_store()
19205 .sheet(sheet_name)
19206 .expect("Arrow sheet missing for table reference");
19207
19208 let sr0 = table.range.start.coord.row() as usize;
19209 let sc0 = table.range.start.coord.col() as usize;
19210 let er0 = table.range.end.coord.row() as usize;
19211 let ec0 = table.range.end.coord.col() as usize;
19212
19213 let has_totals = table.totals_row;
19214 let has_headers = table.header_row;
19215 let data_sr = if has_headers {
19216 sr0.saturating_add(1)
19217 } else {
19218 sr0
19219 };
19220 let data_er = if has_totals {
19221 er0.saturating_sub(1)
19222 } else {
19223 er0
19224 };
19225
19226 let select = |sr: usize, sc: usize, er: usize, ec: usize| {
19227 if sr > er || sc > ec {
19228 asheet.range_view(1, 1, 0, 0)
19229 } else {
19230 asheet.range_view(sr, sc, er, ec)
19231 }
19232 };
19233
19234 let av = match &tref.specifier {
19235 None => {
19236 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
19237 "Table reference without specifier is unsupported".to_string(),
19238 ));
19239 }
19240 Some(formualizer_parse::parser::TableSpecifier::Column(col)) => {
19241 let Some(idx) = table.col_index(col) else {
19242 return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
19243 "Column refers to unknown table column".to_string(),
19244 ));
19245 };
19246 let c0 = sc0 + idx;
19247 select(data_sr, c0, data_er, c0)
19248 }
19249 Some(formualizer_parse::parser::TableSpecifier::ColumnRange(
19250 start,
19251 end,
19252 )) => {
19253 let Some(si) = table.col_index(start) else {
19254 return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
19255 "Column range refers to unknown column(s)".to_string(),
19256 ));
19257 };
19258 let Some(ei) = table.col_index(end) else {
19259 return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
19260 "Column range refers to unknown column(s)".to_string(),
19261 ));
19262 };
19263 let (mut a, mut b) = (si, ei);
19264 if a > b {
19265 std::mem::swap(&mut a, &mut b);
19266 }
19267 let c_start = sc0 + a;
19268 let c_end = sc0 + b;
19269 select(data_sr, c_start, data_er, c_end)
19270 }
19271 Some(formualizer_parse::parser::TableSpecifier::All)
19272 | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
19273 formualizer_parse::parser::SpecialItem::All,
19274 )) => select(sr0, sc0, er0, ec0),
19275 Some(formualizer_parse::parser::TableSpecifier::Data)
19276 | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
19277 formualizer_parse::parser::SpecialItem::Data,
19278 )) => select(data_sr, sc0, data_er, ec0),
19279 Some(formualizer_parse::parser::TableSpecifier::Headers)
19280 | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
19281 formualizer_parse::parser::SpecialItem::Headers,
19282 )) => {
19283 if !has_headers {
19284 asheet.range_view(1, 1, 0, 0)
19285 } else {
19286 select(sr0, sc0, sr0, ec0)
19287 }
19288 }
19289 Some(formualizer_parse::parser::TableSpecifier::Totals)
19290 | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
19291 formualizer_parse::parser::SpecialItem::Totals,
19292 )) => {
19293 if !has_totals {
19294 asheet.range_view(1, 1, 0, 0)
19295 } else {
19296 select(er0, sc0, er0, ec0)
19297 }
19298 }
19299 Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
19300 formualizer_parse::parser::SpecialItem::ThisRow,
19301 )) => {
19302 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
19303 "@ (This Row) requires table-aware context; not yet supported"
19304 .to_string(),
19305 ));
19306 }
19307 Some(formualizer_parse::parser::TableSpecifier::Row(_))
19308 | Some(formualizer_parse::parser::TableSpecifier::Combination(_)) => {
19309 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
19310 "Complex structured references not yet supported".to_string(),
19311 ));
19312 }
19313 };
19314
19315 return Ok(av);
19316 }
19317
19318 if let Some(source) = self.graph.resolve_source_table_entry(&tref.name) {
19319 let version = source
19320 .version
19321 .or_else(|| self.resolver.source_table_version(&tref.name));
19322 let table = self.resolve_source_table_cached(&tref.name, version)?;
19323 return self.source_table_to_range_view(table.as_ref(), &tref.specifier);
19324 }
19325
19326 let boxed = self
19331 .resolve_range_like(&ReferenceType::Table(tref.clone()))
19332 .map_err(|e| {
19333 if e.kind == ExcelErrorKind::NImpl {
19334 ExcelError::new(ExcelErrorKind::Name)
19335 .with_message(format!("Unknown table: {}", tref.name))
19336 } else {
19337 e
19338 }
19339 })?;
19340 let owned = boxed.materialise().into_owned();
19341 Ok(RangeView::from_owned_rows(owned, self.config.date_system))
19342 }
19343 ReferenceType::Cell3D { .. } | ReferenceType::Range3D { .. } => {
19344 Err(ExcelError::new(ExcelErrorKind::NImpl)
19345 .with_message("3D references are not yet supported".to_string()))
19346 }
19347 }
19348 }
19349
19350 fn resolve_cell_format(
19351 &self,
19352 sheet: Option<&str>,
19353 row: u32,
19354 col: u32,
19355 current_sheet: &str,
19356 ) -> Option<crate::format::FormatId> {
19357 self.effective_format_id(sheet.unwrap_or(current_sheet), row, col)
19358 }
19359
19360 fn format_class(
19361 &self,
19362 format: crate::format::FormatId,
19363 ) -> Option<formualizer_common::numfmt::FormatClass> {
19364 self.format_registry.class(format).cloned()
19365 }
19366
19367 fn record_cell_derived_format(
19368 &self,
19369 sheet: &str,
19370 row: u32,
19371 col: u32,
19372 format: Option<crate::format::FormatId>,
19373 ) {
19374 if let Some(sheet_id) = self.graph.sheet_id(sheet) {
19375 let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
19376 self.record_derived_format_at(cell, format);
19377 }
19378 }
19379
19380 fn resolve_cell_reference_value(
19381 &self,
19382 sheet: Option<&str>,
19383 row: u32,
19384 col: u32,
19385 current_sheet: &str,
19386 ) -> Result<LiteralValue, ExcelError> {
19387 let sheet_name = sheet.unwrap_or(current_sheet);
19388 if self.graph.sheet_id(sheet_name).is_none() {
19389 return Err(ExcelError::new(ExcelErrorKind::Ref));
19390 }
19391 Ok(self
19392 .read_cell_value(sheet_name, row, col)
19393 .unwrap_or(LiteralValue::Empty))
19394 }
19395
19396 fn resolve_cell_reference_value_formatted(
19397 &self,
19398 sheet: Option<&str>,
19399 row: u32,
19400 col: u32,
19401 current_sheet: &str,
19402 ) -> Result<(LiteralValue, Option<crate::format::FormatId>), ExcelError> {
19403 let sheet_name = sheet.unwrap_or(current_sheet);
19406 let Some(sheet_id) = self.graph.sheet_id(sheet_name) else {
19407 return Err(ExcelError::new(ExcelErrorKind::Ref));
19408 };
19409 let asheet = self.arrow_sheets.sheet(sheet_name);
19410 Ok(self.read_cell_formatted_in(sheet_id, asheet, row, col))
19411 }
19412
19413 fn build_criteria_mask(
19414 &self,
19415 view: &RangeView<'_>,
19416 col_in_view: usize,
19417 pred: &crate::args::CriteriaPredicate,
19418 ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
19419 #[cfg(any(test, feature = "test-support"))]
19420 criteria_mask_test_hooks::note_mask(view.dims().0);
19421 if view.dims().1 == 0 {
19422 return None;
19423 }
19424 let sheet_rows = view.sheet().nrows as usize;
19427 if sheet_rows == 0 || view.start_row() >= sheet_rows {
19428 return Some(std::sync::Arc::new(arrow_array::BooleanArray::new_null(0)));
19429 }
19430 self.criteria_mask_cache.get_or_build(
19431 view,
19432 col_in_view,
19433 pred,
19434 (
19435 self.recalc_epoch,
19436 self.topology_epoch,
19437 self.graph.topology_revision(),
19438 ),
19439 || compute_criteria_mask(view, col_in_view, pred),
19440 )
19441 }
19442
19443 fn build_row_visibility_mask(
19444 &self,
19445 view: &RangeView<'_>,
19446 mode: VisibilityMaskMode,
19447 ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
19448 self.build_row_visibility_mask_for_view(view, mode)
19449 }
19450
19451 fn nested_subtotal_cells(
19452 &self,
19453 view: &RangeView<'_>,
19454 include_aggregate: bool,
19455 ) -> Option<Vec<(usize, usize, usize)>> {
19456 self.nested_subtotal_cells_for_view(view, include_aggregate)
19457 }
19458}
19459
19460impl<R> Engine<R>
19461where
19462 R: EvaluationContext,
19463{
19464 fn clear_spill_projection_and_mirror(
19465 &mut self,
19466 anchor_vertex: VertexId,
19467 delta: Option<&mut DeltaCollector>,
19468 ) {
19469 let spill_cells = self.graph.spill_cells_to_clear(anchor_vertex);
19470 if spill_cells.is_empty() {
19471 return;
19472 }
19473
19474 if let Some(delta) = delta
19475 && delta.mode != DeltaMode::Off
19476 {
19477 let empty = LiteralValue::Empty;
19478 for cell in spill_cells.iter() {
19479 let sheet_name = self.graph.sheet_name(cell.sheet_id);
19480 let old = self
19481 .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
19482 .unwrap_or(LiteralValue::Empty);
19483 if old != empty {
19484 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
19485 }
19486 }
19487 }
19488
19489 self.graph.clear_spill_region(anchor_vertex);
19490 if let Some(scope) = Self::structural_scope_from_cells(&spill_cells) {
19491 self.record_structural_change(scope);
19492 }
19493
19494 if self.config.arrow_storage_enabled
19495 && self.config.delta_overlay_enabled
19496 && self.config.write_formula_overlay_enabled
19497 {
19498 let empty = LiteralValue::Empty;
19499 for cell in spill_cells.iter() {
19500 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
19501 self.mirror_value_to_computed_overlay(
19502 &sheet_name,
19503 cell.coord.row() + 1,
19504 cell.coord.col() + 1,
19505 &empty,
19506 );
19507 }
19508 }
19509 }
19510
19511 fn apply_cycle_outcome(
19525 &mut self,
19526 cycle: &[VertexId],
19527 mut delta: Option<&mut DeltaCollector>,
19528 dirty_filter: Option<&FxHashSet<VertexId>>,
19529 ) -> usize {
19530 let circ_error = LiteralValue::Error(
19531 ExcelError::new(ExcelErrorKind::Circ)
19532 .with_message("Circular dependency detected".to_string()),
19533 );
19534 let mut stamped = 0usize;
19535 for &vertex_id in cycle {
19536 if let Some(filter) = dirty_filter
19537 && !filter.contains(&vertex_id)
19538 {
19539 continue;
19540 }
19541 self.stamp_cycle_error(vertex_id, &circ_error, delta.as_deref_mut());
19542 stamped += 1;
19543 }
19544 stamped
19545 }
19546
19547 fn stamp_cycle_error(
19562 &mut self,
19563 vertex_id: VertexId,
19564 circ_error: &LiteralValue,
19565 mut delta: Option<&mut DeltaCollector>,
19566 ) {
19567 if self.graph.spill_registry_has_anchor(vertex_id) {
19569 self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
19570 }
19571 self.spill_mgr.release_owner(vertex_id);
19574
19575 if let Some(d) = delta
19577 && d.mode != DeltaMode::Off
19578 && let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id)
19579 {
19580 let sheet_name = self.graph.sheet_name(cell.sheet_id);
19581 let old = self
19582 .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
19583 .unwrap_or(LiteralValue::Empty);
19584 if old != *circ_error {
19585 d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
19586 }
19587 }
19588
19589 self.graph.update_vertex_value_ref(vertex_id, circ_error);
19590 self.mirror_vertex_value_to_overlay(vertex_id, circ_error);
19591 }
19592
19593 fn handle_cycle_unit(
19606 &mut self,
19607 cycle: &[VertexId],
19608 mut delta: Option<&mut DeltaCollector>,
19609 dirty_filter: Option<&FxHashSet<VertexId>>,
19610 cancel_flag: Option<&AtomicBool>,
19611 ) -> Result<usize, ExcelError> {
19612 self.resource_checkpoint(cycle.len() as u64)?;
19613 match self.config.cycle.detection {
19614 CycleDetection::Static => {
19615 Ok(self.apply_cycle_outcome(cycle, delta.as_deref_mut(), dirty_filter))
19616 }
19617 CycleDetection::Runtime => {
19618 if let Some(filter) = dirty_filter
19619 && !cycle.iter().any(|v| filter.contains(v))
19620 {
19621 return Ok(0);
19622 }
19623 self.evaluate_scc_unit(cycle, delta, cancel_flag)
19627 }
19628 }
19629 }
19630
19631 pub(crate) fn evaluate_scc_unit(
19651 &mut self,
19652 cycle: &[VertexId],
19653 mut delta: Option<&mut DeltaCollector>,
19654 cancel_flag: Option<&AtomicBool>,
19655 ) -> Result<usize, ExcelError> {
19656 struct SccMember {
19657 vertex: VertexId,
19658 cell: Option<CellRef>,
19659 }
19660
19661 let task_start = crate::instant::FzInstant::now();
19662
19663 let mut cell_members: Vec<(VertexId, CellRef)> = Vec::new();
19668 let mut name_members: Vec<(VertexId, String)> = Vec::new();
19669 let mut other_members: Vec<VertexId> = Vec::new();
19670 for &v in cycle {
19671 match self.graph.get_vertex_kind(v) {
19672 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
19673 match self.graph.get_cell_ref(v) {
19674 Some(cell) => cell_members.push((v, cell)),
19675 None => other_members.push(v),
19676 }
19677 }
19678 VertexKind::NamedScalar | VertexKind::NamedArray => {
19679 match self.graph.name_key_for_vertex(v) {
19680 Some(key) => name_members.push((v, key)),
19681 None => other_members.push(v),
19682 }
19683 }
19684 _ => other_members.push(v),
19685 }
19686 }
19687 cell_members.sort_unstable_by_key(|(_, c)| (c.sheet_id, c.coord.row(), c.coord.col()));
19688 name_members.sort_unstable_by(|(av, ak), (bv, bk)| ak.cmp(bk).then(av.cmp(bv)));
19689 other_members.sort_unstable();
19690
19691 let cell_refs: Vec<CellRef> = cell_members.iter().map(|(_, c)| *c).collect();
19692 let name_keys: Vec<String> = name_members.iter().map(|(_, k)| k.clone()).collect();
19693 let mut members: Vec<SccMember> = Vec::with_capacity(cycle.len());
19694 for (v, c) in &cell_members {
19695 members.push(SccMember {
19696 vertex: *v,
19697 cell: Some(*c),
19698 });
19699 }
19700 for (v, _) in &name_members {
19701 members.push(SccMember {
19702 vertex: *v,
19703 cell: None,
19704 });
19705 }
19706 for v in &other_members {
19707 members.push(SccMember {
19708 vertex: *v,
19709 cell: None,
19710 });
19711 }
19712 let n = members.len();
19713 let recordable = cell_refs.len() + name_keys.len();
19716
19717 let circ_error = LiteralValue::Error(
19718 ExcelError::new(ExcelErrorKind::Circ)
19719 .with_message("Circular dependency detected".to_string()),
19720 );
19721
19722 if !self.iterative_state_values.is_empty() {
19733 let restore: Vec<(VertexId, LiteralValue)> = members
19734 .iter()
19735 .filter_map(|m| {
19736 let cell = m.cell?;
19737 let persisted = self.iterative_state_values.get(&m.vertex)?;
19738 let sheet_name = self.graph.sheet_name(cell.sheet_id);
19739 let overlay = self
19740 .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
19741 .unwrap_or(LiteralValue::Empty);
19742 if matches!(overlay, LiteralValue::Empty) {
19743 Some((m.vertex, persisted.clone()))
19744 } else {
19745 None
19746 }
19747 })
19748 .collect();
19749 for (vertex, value) in restore {
19750 self.mirror_vertex_value_to_overlay(vertex, &value);
19751 }
19752 }
19753
19754 let snapshot: Vec<LiteralValue> = members
19757 .iter()
19758 .map(|m| match m.cell {
19759 Some(cell) => {
19760 let sheet_name = self.graph.sheet_name(cell.sheet_id);
19761 self.read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
19762 .unwrap_or(LiteralValue::Empty)
19763 }
19764 None => self
19765 .graph
19766 .get_value(m.vertex)
19767 .unwrap_or(LiteralValue::Empty),
19768 })
19769 .collect();
19770
19771 let mut excluded = vec![false; n];
19776 let mut last_value = snapshot.clone();
19777 let mut stamped = 0usize;
19778 for (i, m) in members.iter().enumerate() {
19779 match self.graph.get_vertex_kind(m.vertex) {
19780 VertexKind::FormulaArray => {
19781 self.stamp_cycle_error(m.vertex, &circ_error, delta.as_deref_mut());
19785 excluded[i] = true;
19786 last_value[i] = circ_error.clone();
19787 stamped += 1;
19788 }
19789 VertexKind::FormulaScalar | VertexKind::NamedScalar | VertexKind::NamedArray => {}
19790 _ => excluded[i] = true,
19791 }
19792 }
19793
19794 let collector = LiveEdgeCollector::new_with_names(&cell_refs, &name_keys);
19795
19796 let mut out_edges: Vec<Vec<u32>> = vec![Vec::new(); n];
19798 let mut pos: Vec<i64> = vec![-1; n];
19800 let mut changed = vec![false; n];
19802
19803 macro_rules! run_member {
19806 ($i:expr) => {{
19807 let i: usize = $i;
19808 let m = &members[i];
19809 if i < recordable {
19810 collector.set_current(i as u32);
19811 }
19812 let value = {
19813 let ctx = RecordingContext::new(&*self, &collector);
19814 match self.evaluate_vertex_recorded(m.vertex, &ctx, &collector) {
19815 Ok(v) => v,
19816 Err(e) => LiteralValue::Error(e),
19817 }
19818 };
19819 let is_cell_formula = m.cell.is_some();
19820 if is_cell_formula && matches!(value, LiteralValue::Array(_)) {
19821 self.stamp_cycle_error(m.vertex, &circ_error, None);
19826 excluded[i] = true;
19827 stamped += 1;
19828 changed[i] = last_value[i] != circ_error;
19829 last_value[i] = circ_error.clone();
19830 } else {
19831 self.graph.update_vertex_value_ref(m.vertex, &value);
19832 self.mirror_vertex_value_to_overlay(m.vertex, &value);
19833 #[cfg(debug_assertions)]
19837 if let Some(cell) = m.cell {
19838 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
19839 debug_assert!(
19840 self.read_delta_overlay_cell(
19841 &sheet_name,
19842 cell.coord.row() + 1,
19843 cell.coord.col() + 1
19844 )
19845 .is_none(),
19846 "user overlay must never shadow a formula SCC member ({sheet_name}!r{}c{})",
19847 cell.coord.row() + 1,
19848 cell.coord.col() + 1
19849 );
19850 }
19851 changed[i] = last_value[i] != value;
19852 last_value[i] = value;
19853 }
19854 }};
19855 }
19856
19857 let check_cancel = |flag: Option<&AtomicBool>| -> Result<(), ExcelError> {
19858 if let Some(flag) = flag
19859 && flag.load(Ordering::Relaxed)
19860 {
19861 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
19862 .with_message("Evaluation cancelled during SCC evaluation".to_string()));
19863 }
19864 Ok(())
19865 };
19866
19867 check_cancel(cancel_flag)?;
19869 let mut passes = 1usize;
19870 {
19871 let mut p = 0i64;
19872 for i in 0..n {
19873 if excluded[i] {
19874 continue;
19875 }
19876 run_member!(i);
19877 pos[i] = p;
19878 p += 1;
19879 }
19880 }
19881
19882 let policy = self.config.cycle.policy;
19898 let cap = n + 2;
19899 let mut witnessed_cycles = 0usize;
19900 let mut capped = false;
19901 let mut iterating = false;
19903 let mut converged = false;
19904 let mut exact_fixed_point = false;
19905 let mut prev_pass: Option<Vec<LiteralValue>> = None;
19909 let mut iter_max_delta = 0f64;
19912 let mut iter_nan_converged = 0usize;
19913 let mut settle_passes = 0usize;
19917 loop {
19918 let drained = collector.take_edges();
19921 for i in 0..n {
19922 if pos[i] >= 0 {
19923 out_edges[i].clear();
19924 }
19925 }
19926 for (from, to) in drained {
19927 debug_assert!(
19928 pos[from as usize] >= 0,
19929 "edge from a member that did not run"
19930 );
19931 out_edges[from as usize].push(to);
19932 }
19933 let mut edges: Vec<(u32, u32)> = Vec::new();
19934 for (i, outs) in out_edges.iter().enumerate() {
19935 if excluded[i] {
19936 continue;
19937 }
19938 for &t in outs {
19939 edges.push((i as u32, t));
19940 }
19941 }
19942 edges.sort_unstable();
19943 edges.dedup();
19944
19945 let analysis = analyze_live_graph(n, &edges);
19946
19947 if analysis.cycle_count > 0 {
19948 witnessed_cycles = witnessed_cycles.max(analysis.cycle_count);
19952 match policy {
19953 CyclePolicy::Error => {
19954 for i in 0..n {
19960 if analysis.in_cycle[i] && !excluded[i] {
19961 self.stamp_cycle_error(members[i].vertex, &circ_error, None);
19962 excluded[i] = true;
19963 last_value[i] = circ_error.clone();
19964 stamped += 1;
19965 }
19966 }
19967 check_cancel(cancel_flag)?;
19968 let order: Vec<usize> = analysis
19969 .topo
19970 .iter()
19971 .map(|&i| i as usize)
19972 .filter(|&i| !excluded[i])
19973 .collect();
19974 if !order.is_empty() {
19975 passes += 1;
19976 for i in order {
19977 run_member!(i);
19978 }
19979 }
19980 break;
19981 }
19982 CyclePolicy::Iterate {
19983 max_iterations,
19984 max_change,
19985 } => {
19986 iterating = true;
19988
19989 if let Some(prev) = &prev_pass {
19995 let mut round_max_delta = 0f64;
19996 let mut round_nan = 0usize;
19997 let mut all_converged = true;
19998 let mut round_exact = true;
19999 for i in 0..n {
20000 if excluded[i] {
20001 continue;
20005 }
20006 let out = crate::engine::convergence::values_converged(
20007 &prev[i],
20008 &last_value[i],
20009 max_change,
20010 self.config.date_system,
20011 );
20012 if out.nan_converged {
20013 round_nan += 1;
20014 }
20015 if let Some(d) = out.abs_delta {
20016 round_max_delta = round_max_delta.max(d);
20017 if d != 0.0 {
20018 round_exact = false;
20019 }
20020 }
20021 if !out.converged {
20022 all_converged = false;
20023 }
20024 }
20025 iter_max_delta = round_max_delta;
20028 iter_nan_converged = round_nan;
20029 if all_converged {
20030 exact_fixed_point = round_exact;
20031 converged = true;
20032 break;
20033 }
20034 }
20035
20036 if passes >= max_iterations as usize {
20048 capped = true;
20049 break;
20050 }
20051
20052 check_cancel(cancel_flag)?;
20053 prev_pass = Some(last_value.clone());
20061 for x in pos.iter_mut() {
20062 *x = -1;
20063 }
20064 changed.fill(false);
20065 passes += 1;
20066 let mut p = 0i64;
20067 for i in 0..n {
20068 if excluded[i] {
20069 continue;
20070 }
20071 run_member!(i);
20072 pos[i] = p;
20073 p += 1;
20074 }
20075 continue;
20076 }
20077 }
20078 }
20079
20080 let mut stale: Vec<usize> = Vec::new();
20083 for i in 0..n {
20084 if excluded[i] {
20085 continue;
20086 }
20087 let is_stale = out_edges[i].iter().any(|&t| {
20088 let t = t as usize;
20089 changed[t] && (pos[i] < 0 || (pos[t] >= 0 && pos[i] < pos[t]))
20090 });
20091 if is_stale {
20092 stale.push(i);
20093 }
20094 }
20095 if stale.is_empty() {
20096 break; }
20098 if 1 + settle_passes >= cap {
20099 capped = true;
20101 for (i, m) in members.iter().enumerate() {
20102 if !excluded[i] {
20103 self.stamp_cycle_error(m.vertex, &circ_error, None);
20104 excluded[i] = true;
20105 last_value[i] = circ_error.clone();
20106 stamped += 1;
20107 }
20108 }
20109 break;
20110 }
20111
20112 check_cancel(cancel_flag)?;
20113 prev_pass = None;
20120 let topo_pos = analysis.topo_positions();
20121 stale.sort_unstable_by_key(|&i| topo_pos[i]);
20122 for x in pos.iter_mut() {
20123 *x = -1;
20124 }
20125 changed.fill(false);
20126 passes += 1;
20127 settle_passes += 1;
20128 for (p, i) in stale.into_iter().enumerate() {
20129 run_member!(i);
20130 pos[i] = p as i64;
20131 }
20132 }
20133
20134 self.live_cancellation_after_work(SCC_CANCELLED)?;
20140
20141 if iterating && !converged && !capped {
20146 converged = true;
20147 }
20148
20149 collector.clear_current();
20152 if let Some(d) = delta
20153 && d.mode != DeltaMode::Off
20154 {
20155 for (i, m) in members.iter().enumerate() {
20156 if let Some(cell) = m.cell
20157 && last_value[i] != snapshot[i]
20158 {
20159 d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
20160 }
20161 }
20162 }
20163
20164 if !self.retained_scc_members.is_empty() {
20182 for m in members.iter() {
20183 self.retained_scc_members.remove(&m.vertex);
20184 }
20185 }
20186 if iterating {
20187 let retain = converged
20188 && !capped
20189 && exact_fixed_point
20190 && iter_nan_converged == 0
20191 && members
20192 .iter()
20193 .all(|m| !self.graph.is_volatile(m.vertex) && !self.graph.is_dynamic(m.vertex));
20194 if retain {
20195 if self.retained_scc_members.is_empty() {
20196 self.retained_scc_config_fingerprint = self.retained_scc_config_fingerprint();
20197 self.retained_scc_function_epoch_seen =
20198 crate::function_registry::semantic_epoch();
20199 self.retained_scc_provider_revision_seen =
20200 self.resolver.planning_semantic_revision();
20201 }
20202 let scc_id = self.next_retained_scc_id;
20203 self.next_retained_scc_id = self.next_retained_scc_id.wrapping_add(1);
20204 for (i, m) in members.iter().enumerate() {
20205 self.retained_scc_members.insert(m.vertex, scc_id);
20206 if matches!(last_value[i], LiteralValue::Empty) {
20209 self.iterative_state_values.remove(&m.vertex);
20210 } else {
20211 self.iterative_state_values
20212 .insert(m.vertex, last_value[i].clone());
20213 }
20214 }
20215 } else {
20216 self.pending_iterative_redirty
20217 .extend(members.iter().map(|m| m.vertex));
20218 }
20219 } else if !self.iterative_state_values.is_empty() {
20220 for m in members.iter() {
20224 self.iterative_state_values.remove(&m.vertex);
20225 }
20226 }
20227
20228 {
20229 let t = &mut self.last_cycle_telemetry;
20230 t.static_sccs += 1;
20231 if witnessed_cycles == 0 && stamped == 0 && !capped {
20232 t.phantom_sccs += 1;
20233 }
20234 t.live_cycles_witnessed += witnessed_cycles;
20235 t.circ_cells_stamped += stamped;
20236 t.settle_passes_total += passes;
20237 t.max_passes_single_scc = t.max_passes_single_scc.max(passes);
20238 if iterating {
20239 t.iterated_sccs += 1;
20240 if converged {
20241 t.converged_sccs += 1;
20242 }
20243 t.max_abs_delta_at_stop = t.max_abs_delta_at_stop.max(iter_max_delta);
20244 t.nan_converged += iter_nan_converged;
20245 }
20246 if capped {
20247 t.capped_sccs += 1;
20248 }
20249 t.elapsed_ms += task_start.elapsed().as_millis();
20250 }
20251
20252 Ok(stamped)
20253 }
20254
20255 fn evaluate_vertex_recorded(
20263 &self,
20264 vertex_id: VertexId,
20265 ctx: &RecordingContext<'_, R>,
20266 collector: &LiveEdgeCollector,
20267 ) -> Result<LiteralValue, ExcelError> {
20268 if !self.graph.vertex_exists(vertex_id) {
20269 return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
20270 .with_message(format!("Vertex not found: {vertex_id:?}")));
20271 }
20272
20273 let kind = self.graph.get_vertex_kind(vertex_id);
20274 let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
20275
20276 match kind {
20277 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
20278 let Some(view) = self.graph.formula_view(vertex_id) else {
20279 return Ok(LiteralValue::Number(0.0)); };
20281 let sheet_name = self.graph.sheet_name(sheet_id);
20282 let cell_ref = self
20283 .graph
20284 .get_cell_ref(vertex_id)
20285 .expect("cell ref for vertex");
20286 let interpreter = Interpreter::new_with_cell(ctx, sheet_name, cell_ref)
20287 .with_resolved_functions(&self.resolved_functions);
20288 interpreter
20289 .evaluate_formula_view(view, self.graph.data_store(), self.graph.sheet_reg())
20290 .map(|cv| {
20291 let format = cv.format_id();
20292 self.record_derived_format(vertex_id, format);
20293 self.materialize_formula_result(vertex_id, cv)
20294 })
20295 .or_else(|error| self.fit_formula_error(vertex_id, error))
20296 }
20297 VertexKind::NamedScalar | VertexKind::NamedArray => {
20298 let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
20299 ExcelError::new(ExcelErrorKind::Name)
20300 .with_message("Named range metadata missing".to_string())
20301 })?;
20302
20303 match &named_range.definition {
20304 NamedDefinition::Formula { ast, .. } => {
20305 let context_sheet = match named_range.scope {
20306 NameScope::Sheet(id) => id,
20307 NameScope::Workbook => sheet_id,
20308 };
20309 let sheet_name = self.graph.sheet_name(context_sheet);
20310 let cell_ref = self
20311 .graph
20312 .get_cell_ref(vertex_id)
20313 .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
20314 let interpreter = Interpreter::new_with_cell(ctx, sheet_name, cell_ref)
20315 .with_resolved_functions(&self.resolved_functions);
20316 if kind == VertexKind::NamedScalar {
20317 interpreter.evaluate_ast(ast).map(|cv| cv.into_literal())
20318 } else {
20319 match interpreter.evaluate_ast(ast) {
20320 Ok(cv) => match cv.into_literal() {
20321 v @ LiteralValue::Array(_) => Ok(v),
20322 other => Ok(LiteralValue::Array(vec![vec![other]])),
20323 },
20324 Err(err) => Ok(LiteralValue::Error(err)),
20325 }
20326 }
20327 }
20328 NamedDefinition::Cell(cell_ref) => {
20329 collector.record_scalar(
20333 cell_ref.sheet_id,
20334 cell_ref.coord.row(),
20335 cell_ref.coord.col(),
20336 );
20337 self.evaluate_vertex_immutable(vertex_id)
20338 }
20339 NamedDefinition::Range(range_ref) => {
20340 if range_ref.start.sheet_id == range_ref.end.sheet_id {
20341 collector.record_rect(
20342 range_ref.start.sheet_id,
20343 range_ref.start.coord.row(),
20344 range_ref.start.coord.col(),
20345 range_ref.end.coord.row(),
20346 range_ref.end.coord.col(),
20347 );
20348 }
20349 self.evaluate_vertex_immutable(vertex_id)
20350 }
20351 NamedDefinition::Literal(_) => self.evaluate_vertex_immutable(vertex_id),
20352 }
20353 }
20354 _ => self.evaluate_vertex_immutable(vertex_id),
20355 }
20356 }
20357
20358 fn pending_spill_occupied(&self, anchor: CellRef, end_row: u32, end_col: u32) -> bool {
20360 let sheet = self.graph.sheet_name(anchor.sheet_id);
20361 let package = self
20362 .staged_formulas
20363 .get(sheet)
20364 .and_then(|staged| staged.deferred_package.as_ref());
20365 self.staged_formula_index.occupies_spill(
20366 sheet,
20367 (anchor.coord.row() + 1, anchor.coord.col() + 1),
20368 (end_row + 1, end_col + 1),
20369 |point| package.is_some_and(|package| package.suppressed.contains(&point)),
20370 )
20371 }
20372
20373 fn remember_pending_spill(
20374 &mut self,
20375 vertex: VertexId,
20376 anchor: CellRef,
20377 region: Region,
20378 ) -> Result<(), ExcelError> {
20379 self.cancellation_checkpoint("pending spill occupancy")?;
20380 self.resource_checkpoint(1)?;
20381 if let Some(entry) = self
20382 .blocked_pending_spills
20383 .iter_mut()
20384 .find(|entry| entry.0 == vertex)
20385 {
20386 *entry = (vertex, anchor, region);
20387 return Ok(());
20388 }
20389 if self.blocked_pending_spills.len() == self.blocked_pending_spills.capacity() {
20390 let additional = self.blocked_pending_spills.capacity().max(1);
20393 let bytes =
20394 (additional as u64)
20395 .saturating_mul(std::mem::size_of::<(VertexId, CellRef, Region)>() as u64);
20396 if let Some(ledger) = self.active_resource_ledger.as_mut() {
20397 ledger
20398 .reserve_retained(bytes)
20399 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
20400 self.source_cache_accounted = self.source_cache_accounted.saturating_add(bytes);
20401 }
20402 if self
20403 .blocked_pending_spills
20404 .try_reserve_exact(additional)
20405 .is_err()
20406 {
20407 if let Some(ledger) = self.active_resource_ledger.as_mut() {
20408 ledger
20409 .release_retained(bytes)
20410 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
20411 self.source_cache_accounted -= bytes;
20412 }
20413 return Err(crate::engine::ResourceLedgerError::Exhausted(
20414 formualizer_common::ResourceExhaustionDetail {
20415 reason: formualizer_common::ResourceExhaustionReason::RetainedMemory,
20416 limit: u64::MAX,
20417 observed: bytes,
20418 request_id: None,
20419 },
20420 )
20421 .into_excel_error());
20422 }
20423 }
20424 self.blocked_pending_spills.push((vertex, anchor, region));
20425 Ok(())
20426 }
20427
20428 fn invalidate_pending_spills(&mut self, scope: StructuralScope) {
20432 if let StructuralScope::RemovedSheet(sheet) = scope {
20433 self.blocked_pending_spills
20434 .retain(|entry| entry.1.sheet_id != sheet);
20435 return;
20436 }
20437 for &(vertex, anchor, region) in &self.blocked_pending_spills {
20438 let affected = match scope {
20439 StructuralScope::Cell { sheet, row, col } => {
20440 region.intersects(&Region::point(sheet, row, col))
20441 }
20442 StructuralScope::Region(changed) => region.intersects(&changed),
20443 StructuralScope::Sheet(sheet) | StructuralScope::RemovedSheet(sheet) => {
20444 region.sheet_id() == sheet
20445 }
20446 StructuralScope::OpaqueGlobal | StructuralScope::AllSheets => true,
20447 };
20448 if affected
20449 && self.graph.vertex_exists(vertex)
20450 && self.graph.get_cell_ref(vertex) == Some(anchor)
20451 && matches!(
20452 self.graph.get_vertex_kind(vertex),
20453 VertexKind::FormulaScalar | VertexKind::FormulaArray
20454 )
20455 {
20456 self.graph.mark_vertex_dirty(vertex);
20457 }
20458 }
20459 }
20460
20461 fn guard_pending_spill_commit(
20462 &mut self,
20463 anchor_vertex: VertexId,
20464 targets: &[CellRef],
20465 ) -> Result<(), ExcelError> {
20466 let Some(anchor) = self.graph.get_cell_ref(anchor_vertex) else {
20467 return Ok(());
20468 };
20469 let Some(last) = targets.last() else {
20470 return Ok(());
20471 };
20472 let occupied = self.pending_spill_occupied(anchor, last.coord.row(), last.coord.col())
20473 || self
20474 .graph
20475 .table_intersects_spill(anchor, last.coord.row(), last.coord.col());
20476 if (occupied
20477 || self
20478 .blocked_pending_spills
20479 .iter()
20480 .any(|entry| entry.0 == anchor_vertex))
20481 && let Err(error) = self.remember_pending_spill(
20482 anchor_vertex,
20483 anchor,
20484 Region::rect(
20485 anchor.sheet_id,
20486 anchor.coord.row(),
20487 last.coord.row(),
20488 anchor.coord.col(),
20489 last.coord.col(),
20490 ),
20491 )
20492 {
20493 self.spill_mgr.release_owner(anchor_vertex);
20494 return Err(error);
20495 }
20496 if occupied {
20497 self.spill_mgr.release_owner(anchor_vertex);
20498 return Err(ExcelError::new(ExcelErrorKind::Spill)
20499 .with_message("Spill blocked")
20500 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
20501 expected_rows: last.coord.row() - anchor.coord.row() + 1,
20502 expected_cols: last.coord.col() - anchor.coord.col() + 1,
20503 }));
20504 }
20505 Ok(())
20506 }
20507
20508 fn commit_spill_and_mirror(
20510 &mut self,
20511 anchor_vertex: VertexId,
20512 targets: &[CellRef],
20513 rows: Vec<Vec<LiteralValue>>,
20514 delta: Option<&mut DeltaCollector>,
20515 ) -> Result<(), ExcelError> {
20516 self.guard_pending_spill_commit(anchor_vertex, targets)?;
20517 let prev_spill_cells = self.graph.spill_cells_to_clear(anchor_vertex);
20518
20519 if let Some(delta) = delta
20520 && delta.mode != DeltaMode::Off
20521 {
20522 let target_set: std::collections::HashSet<CellRef, CoordBuildHasher> =
20523 targets.iter().copied().collect();
20524 let empty = LiteralValue::Empty;
20525
20526 for cell in prev_spill_cells.iter() {
20528 if target_set.contains(cell) {
20529 continue;
20530 }
20531 let sheet_name = self.graph.sheet_name(cell.sheet_id);
20532 let old = self
20533 .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
20534 .unwrap_or(LiteralValue::Empty);
20535 if old != empty {
20536 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
20537 }
20538 }
20539
20540 if !targets.is_empty() && !rows.is_empty() && !rows[0].is_empty() {
20542 let width = rows[0].len();
20543 for (idx, cell) in targets.iter().enumerate() {
20544 let r_off = idx / width;
20545 let c_off = idx % width;
20546 let new = rows
20547 .get(r_off)
20548 .and_then(|r| r.get(c_off))
20549 .cloned()
20550 .unwrap_or(LiteralValue::Empty);
20551 let sheet_name = self.graph.sheet_name(cell.sheet_id);
20552 let old = self
20553 .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
20554 .unwrap_or(LiteralValue::Empty);
20555 if old != new {
20556 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
20557 }
20558 }
20559 } else {
20560 for cell in targets.iter() {
20562 let sheet_name = self.graph.sheet_name(cell.sheet_id);
20563 let old = self
20564 .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
20565 .unwrap_or(LiteralValue::Empty);
20566 if !matches!(old, LiteralValue::Empty) {
20567 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
20568 }
20569 }
20570 }
20571 }
20572
20573 let arrow_sheets = &self.arrow_sheets;
20576 self.spill_mgr.commit_array_with_value_probe(
20577 &mut self.graph,
20578 anchor_vertex,
20579 targets,
20580 rows.clone(),
20581 |g, cell| {
20582 let sheet_name = g.sheet_name(cell.sheet_id);
20583 let asheet = arrow_sheets.sheet(sheet_name)?;
20584 let r0 = cell.coord.row() as usize;
20585 let c0 = cell.coord.col() as usize;
20586 let v = asheet.get_cell_value(r0, c0);
20587 if matches!(v, LiteralValue::Empty) {
20588 None
20589 } else {
20590 Some(v)
20591 }
20592 },
20593 )?;
20594
20595 self.blocked_pending_spills
20596 .retain(|entry| entry.0 != anchor_vertex);
20597 self.spill_blocker_waiters.forget(anchor_vertex);
20598 if let Some(scope) = Self::structural_scope_from_cells(&prev_spill_cells) {
20599 self.record_structural_change(scope);
20600 }
20601 if let Some(scope) = Self::structural_scope_from_cells(targets) {
20602 self.record_structural_change(scope);
20603 }
20604
20605 if self.config.arrow_storage_enabled
20606 && self.config.delta_overlay_enabled
20607 && self.config.write_formula_overlay_enabled
20608 {
20609 if !prev_spill_cells.is_empty() {
20610 let target_set: std::collections::HashSet<CellRef, CoordBuildHasher> =
20611 targets.iter().copied().collect();
20612 let empty = LiteralValue::Empty;
20613 for cell in prev_spill_cells.iter() {
20614 if !target_set.contains(cell) {
20615 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
20616 self.mirror_value_to_computed_overlay(
20617 &sheet_name,
20618 cell.coord.row() + 1,
20619 cell.coord.col() + 1,
20620 &empty,
20621 );
20622 }
20623 }
20624 }
20625
20626 for (idx, cell) in targets.iter().enumerate() {
20627 if rows.is_empty() || rows[0].is_empty() {
20628 break;
20629 }
20630 let width = rows[0].len();
20631 let r_off = idx / width;
20632 let c_off = idx % width;
20633 let v = rows[r_off][c_off].clone();
20634 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
20635 self.mirror_value_to_computed_overlay(
20636 &sheet_name,
20637 cell.coord.row() + 1,
20638 cell.coord.col() + 1,
20639 &v,
20640 );
20641 }
20642 }
20643 Ok(())
20644 }
20645}
20646
20647#[cfg(test)]
20648#[path = "tests/authority_schedule_execution.rs"]
20649mod authority_schedule_execution;
20650
20651#[cfg(test)]
20652#[path = "tests/scalar_commit.rs"]
20653mod scalar_commit_tests;
20654
20655#[cfg(test)]
20656#[path = "tests/pending_spill.rs"]
20657mod pending_spill_tests;
20658
20659#[cfg(test)]
20660#[path = "tests/spill_batch_abort_192.rs"]
20661mod spill_batch_abort_192;
20662
20663use crate::engine::effects::Effect;
20668
20669type PlannedEffects = smallvec::SmallVec<[Effect; 2]>;
20671use crate::engine::graph::editor::change_log::{ChangeEvent, ChangeLog, SpillSnapshot};
20672
20673impl<R> Engine<R>
20674where
20675 R: EvaluationContext,
20676{
20677 pub(crate) fn plan_vertex_effects(
20684 &mut self,
20685 vertex_id: VertexId,
20686 computed_value: LiteralValue,
20687 ) -> Result<PlannedEffects, ExcelError> {
20688 if self.freshness_armed() {
20691 if self.freshness_drop_stale(vertex_id) {
20692 return Ok(PlannedEffects::new());
20693 }
20694 let effects = self.plan_vertex_effects_unrecorded(vertex_id, computed_value)?;
20695 self.freshness_mark_committed(vertex_id);
20696 return Ok(effects);
20697 }
20698 self.plan_vertex_effects_unrecorded(vertex_id, computed_value)
20699 }
20700
20701 fn plan_vertex_effects_unrecorded(
20702 &mut self,
20703 vertex_id: VertexId,
20704 computed_value: LiteralValue,
20705 ) -> Result<PlannedEffects, ExcelError> {
20706 let kind = self.graph.get_vertex_kind(vertex_id);
20707 let is_formula = matches!(kind, VertexKind::FormulaScalar | VertexKind::FormulaArray);
20708
20709 if !is_formula {
20713 if let Some(cell) = self.graph.get_cell_ref(vertex_id)
20714 && let Some(owner) = self.graph.spill_registry_anchor_for_cell(cell)
20715 && owner != vertex_id
20716 {
20717 return Ok(PlannedEffects::new());
20718 }
20719 return Ok(smallvec::smallvec![Effect::WriteCell {
20721 vertex_id,
20722 value: computed_value,
20723 }]);
20724 }
20725
20726 match computed_value {
20727 LiteralValue::Array(rows) => self.plan_array_effects(vertex_id, rows),
20728 other => self.plan_scalar_effects(vertex_id, other),
20729 }
20730 }
20731
20732 fn plan_scalar_effects(
20734 &mut self,
20735 vertex_id: VertexId,
20736 value: LiteralValue,
20737 ) -> Result<PlannedEffects, ExcelError> {
20738 if let LiteralValue::Error(error) = &value
20741 && error.kind == ExcelErrorKind::Spill
20742 && error.message() == Some("SpillTooLarge")
20743 && let formualizer_common::ExcelErrorExtra::Spill {
20744 expected_rows,
20745 expected_cols,
20746 } = error.extra()
20747 && u64::from(*expected_rows).saturating_mul(u64::from(*expected_cols))
20748 > u64::from(self.config.spill.max_spill_cells)
20749 {
20750 self.graph.set_kind(vertex_id, VertexKind::FormulaArray);
20751 return self.plan_spill_error_effects(
20752 vertex_id,
20753 "SpillTooLarge",
20754 *expected_rows,
20755 *expected_cols,
20756 );
20757 }
20758 if !matches!(&value, LiteralValue::Error(e) if e.kind == ExcelErrorKind::Spill) {
20759 self.blocked_pending_spills
20760 .retain(|entry| entry.0 != vertex_id);
20761 self.spill_blocker_waiters.forget(vertex_id);
20762 }
20763 let has_spill = self
20764 .graph
20765 .spill_cells_for_anchor(vertex_id)
20766 .is_some_and(|c| !c.is_empty());
20767
20768 let mut effects = PlannedEffects::new();
20769 if has_spill {
20770 effects.push(Effect::SpillClear {
20771 anchor_vertex: vertex_id,
20772 });
20773 }
20774 effects.push(Effect::WriteCell { vertex_id, value });
20775 Ok(effects)
20776 }
20777
20778 fn plan_array_effects(
20780 &mut self,
20781 vertex_id: VertexId,
20782 rows: Vec<Vec<LiteralValue>>,
20783 ) -> Result<PlannedEffects, ExcelError> {
20784 self.graph.set_kind(vertex_id, VertexKind::FormulaArray);
20786
20787 let anchor = self
20788 .graph
20789 .get_cell_ref(vertex_id)
20790 .expect("cell ref for vertex");
20791 let sheet_id = anchor.sheet_id;
20792 let h = rows.len() as u32;
20793 let w = rows.first().map(|r| r.len()).unwrap_or(0) as u32;
20794
20795 let spill_cells = (h as u64).saturating_mul(w as u64);
20797 if spill_cells > self.config.spill.max_spill_cells as u64 {
20798 return self.plan_spill_error_effects(vertex_id, "SpillTooLarge", h, w);
20799 }
20800
20801 const PACKED_MAX_ROW: u32 = 1_048_575;
20803 const PACKED_MAX_COL: u32 = 16_383;
20804 let end_row = anchor.coord.row().saturating_add(h).saturating_sub(1);
20805 let end_col = anchor.coord.col().saturating_add(w).saturating_sub(1);
20806 if end_row > PACKED_MAX_ROW || end_col > PACKED_MAX_COL {
20807 return self.plan_spill_error_effects(vertex_id, "Spill exceeds sheet bounds", h, w);
20808 }
20809
20810 if h != 0 && w != 0 && self.graph.table_intersects_spill(anchor, end_row, end_col) {
20811 return self.plan_spill_error_effects(vertex_id, "Spill blocked by table", h, w);
20812 }
20813 let mut targets = Vec::new();
20814 for r in 0..h {
20815 for c in 0..w {
20816 targets.push(self.graph.make_cell_ref_internal(
20817 sheet_id,
20818 anchor.coord.row() + r,
20819 anchor.coord.col() + c,
20820 ));
20821 }
20822 }
20823
20824 if h != 0 && w != 0 {
20825 let occupied = self.pending_spill_occupied(anchor, end_row, end_col);
20826 if occupied
20827 || self
20828 .blocked_pending_spills
20829 .iter()
20830 .any(|entry| entry.0 == vertex_id)
20831 {
20832 self.spill_mgr.release_owner(vertex_id);
20833 self.remember_pending_spill(
20834 vertex_id,
20835 anchor,
20836 Region::rect(
20837 sheet_id,
20838 anchor.coord.row(),
20839 end_row,
20840 anchor.coord.col(),
20841 end_col,
20842 ),
20843 )?;
20844 }
20845 if occupied {
20846 return self.plan_spill_error_effects(vertex_id, "Spill blocked", h, w);
20847 }
20848 }
20849
20850 match self.spill_mgr.reserve(
20852 vertex_id,
20853 anchor,
20854 SpillShape { rows: h, cols: w },
20855 SpillMeta {
20856 epoch: self.recalc_epoch,
20857 config: self.config.spill,
20858 },
20859 ) {
20860 Ok(()) => {
20861 let mut yield_effects = PlannedEffects::new();
20863 if let Err((e, blocker)) = self
20864 .graph
20865 .plan_spill_region_with_blocker(vertex_id, &targets)
20866 {
20867 let yielding = if e.message() == Some("BlockedBySpill") {
20868 self.graph.spills_yielding_to(vertex_id, &targets)
20869 } else {
20870 None
20871 };
20872 if let Some(yielding) = yielding {
20873 for &other in &yielding {
20877 let (rows, cols) = self
20878 .graph
20879 .spill_extent_for_anchor(other)
20880 .map(|(first, last)| {
20881 (
20882 last.coord.row() - first.coord.row() + 1,
20883 last.coord.col() - first.coord.col() + 1,
20884 )
20885 })
20886 .unwrap_or((0, 0));
20887 yield_effects.push(Effect::SpillClear {
20888 anchor_vertex: other,
20889 });
20890 yield_effects.push(Effect::WriteCell {
20891 vertex_id: other,
20892 value: Self::spill_error_value("Spill blocked", rows, cols),
20893 });
20894 }
20895 self.graph.mark_dirty_many(&yielding);
20896 } else {
20897 if matches!(e.message(), Some("BlockedByFormula" | "BlockedBySpill")) {
20901 self.spill_blocker_waiters.wait(vertex_id, blocker);
20902 }
20903 return self.plan_spill_error_effects(vertex_id, "Spill blocked", h, w);
20904 }
20905 }
20906
20907 if !self.graph.value_cache_enabled() {
20911 let sheet_name = self.graph.sheet_name(sheet_id);
20912 if let Some(asheet) = self.sheet_store().sheet(sheet_name) {
20913 for cell in targets.iter() {
20914 if *cell == anchor {
20916 continue;
20917 }
20918 if self.graph.spill_registry_anchor_for_cell(*cell).is_some() {
20920 continue;
20921 }
20922 if let Some(vid) = self.graph.get_vertex_id_for_address(cell)
20924 && vid != vertex_id
20925 {
20926 match self.graph.get_vertex_kind(vid) {
20927 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
20928 continue;
20929 }
20930 _ => {}
20931 }
20932 }
20933
20934 let v = asheet.get_cell_value(
20935 cell.coord.row() as usize,
20936 cell.coord.col() as usize,
20937 );
20938 if !matches!(v, LiteralValue::Empty) {
20939 return self.plan_spill_error_effects(
20940 vertex_id,
20941 "BlockedByValue",
20942 h,
20943 w,
20944 );
20945 }
20946 }
20947 }
20948 }
20949
20950 let top_left = rows
20951 .first()
20952 .and_then(|r| r.first())
20953 .cloned()
20954 .unwrap_or(LiteralValue::Empty);
20955
20956 let mut effects = yield_effects;
20957 let has_prev = self
20959 .graph
20960 .spill_cells_for_anchor(vertex_id)
20961 .is_some_and(|c| !c.is_empty());
20962 if has_prev {
20963 effects.push(Effect::SpillClear {
20964 anchor_vertex: vertex_id,
20965 });
20966 }
20967 effects.push(Effect::SpillCommit {
20968 anchor_vertex: vertex_id,
20969 anchor_cell: anchor,
20970 target_cells: targets,
20971 values: rows,
20972 });
20973 effects.push(Effect::WriteCell {
20974 vertex_id,
20975 value: top_left,
20976 });
20977 Ok(effects)
20978 }
20979 Err(e) => {
20980 let msg = e.message().unwrap_or("Spill blocked");
20981 self.plan_spill_error_effects(vertex_id, msg, h, w)
20982 }
20983 }
20984 }
20985
20986 fn spill_error_value(message: &str, expected_rows: u32, expected_cols: u32) -> LiteralValue {
20988 LiteralValue::Error(
20989 ExcelError::new(ExcelErrorKind::Spill)
20990 .with_message(message)
20991 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
20992 expected_rows,
20993 expected_cols,
20994 }),
20995 )
20996 }
20997
20998 fn plan_spill_error_effects(
21000 &mut self,
21001 vertex_id: VertexId,
21002 message: &str,
21003 expected_rows: u32,
21004 expected_cols: u32,
21005 ) -> Result<PlannedEffects, ExcelError> {
21006 self.spill_mgr.release_owner(vertex_id);
21007 let spill_val = Self::spill_error_value(message, expected_rows, expected_cols);
21008
21009 let effects = smallvec::smallvec![
21010 Effect::SpillClear {
21011 anchor_vertex: vertex_id,
21012 },
21013 Effect::WriteCell {
21014 vertex_id,
21015 value: spill_val,
21016 },
21017 ];
21018 Ok(effects)
21019 }
21020
21021 pub(crate) fn apply_effect(
21023 &mut self,
21024 effect: &Effect,
21025 delta: Option<&mut DeltaCollector>,
21026 log: Option<&mut ChangeLog>,
21027 ) -> Result<(), ExcelError> {
21028 self.apply_effect_with_computed_writes(effect, delta, log, None)
21029 }
21030
21031 fn apply_effect_with_computed_writes(
21032 &mut self,
21033 effect: &Effect,
21034 delta: Option<&mut DeltaCollector>,
21035 log: Option<&mut ChangeLog>,
21036 computed_writes: Option<&mut ComputedWriteBuffer>,
21037 ) -> Result<(), ExcelError> {
21038 match effect {
21039 Effect::WriteCell { vertex_id, value } => {
21040 self.apply_write_cell(*vertex_id, value, delta, computed_writes)?;
21041 }
21042 Effect::SpillClear { anchor_vertex } => {
21043 self.apply_spill_clear(*anchor_vertex, delta, log, computed_writes)?;
21044 }
21045 Effect::SpillCommit {
21046 anchor_vertex,
21047 anchor_cell: _,
21048 target_cells,
21049 values,
21050 } => {
21051 self.apply_spill_commit(
21052 *anchor_vertex,
21053 target_cells,
21054 values.clone(),
21055 delta,
21056 log,
21057 computed_writes,
21058 )?;
21059 }
21060 }
21061 Ok(())
21062 }
21063
21064 fn apply_write_cell(
21066 &mut self,
21067 vertex_id: VertexId,
21068 value: &LiteralValue,
21069 delta: Option<&mut DeltaCollector>,
21070 mut computed_writes: Option<&mut ComputedWriteBuffer>,
21071 ) -> Result<(), ExcelError> {
21072 if let Some(d) = delta
21073 && d.mode != DeltaMode::Off
21074 {
21075 if let Some(buffer) = computed_writes.as_deref_mut() {
21076 self.flush_computed_write_buffer(buffer)?;
21077 }
21078 if let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id) {
21079 let sheet_name = self.graph.sheet_name(cell.sheet_id);
21080 let old = self
21081 .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
21082 .unwrap_or(LiteralValue::Empty);
21083 if old != *value {
21084 d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
21085 }
21086 }
21087 }
21088 self.graph.update_vertex_value_ref(vertex_id, value);
21089 self.record_vertex_value_to_overlay(vertex_id, value, computed_writes)?;
21090 Ok(())
21091 }
21092
21093 fn apply_spill_clear(
21095 &mut self,
21096 anchor_vertex: VertexId,
21097 delta: Option<&mut DeltaCollector>,
21098 log: Option<&mut ChangeLog>,
21099 computed_writes: Option<&mut ComputedWriteBuffer>,
21100 ) -> Result<(), ExcelError> {
21101 if let Some(buffer) = computed_writes {
21102 self.flush_computed_write_buffer(buffer)?;
21103 }
21104
21105 let spill_cells = self.graph.spill_cells_to_clear(anchor_vertex);
21106 if spill_cells.is_empty() {
21107 return Ok(());
21108 }
21109
21110 let snapshot = if log.is_some() {
21112 self.snapshot_spill_for_anchor(anchor_vertex)
21113 } else {
21114 None
21115 };
21116
21117 if let Some(d) = delta
21119 && d.mode != DeltaMode::Off
21120 {
21121 let empty = LiteralValue::Empty;
21122 for cell in spill_cells.iter() {
21123 let sheet_name = self.graph.sheet_name(cell.sheet_id);
21124 let old = self
21125 .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
21126 .unwrap_or(LiteralValue::Empty);
21127 if old != empty {
21128 d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
21129 }
21130 }
21131 }
21132
21133 self.graph.clear_spill_region(anchor_vertex);
21134 if let Some(scope) = Self::structural_scope_from_cells(&spill_cells) {
21135 self.record_structural_change(scope);
21136 }
21137
21138 if self.config.arrow_storage_enabled
21140 && self.config.delta_overlay_enabled
21141 && self.config.write_formula_overlay_enabled
21142 {
21143 let empty = LiteralValue::Empty;
21144 for cell in spill_cells.iter() {
21145 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
21146 self.mirror_value_to_computed_overlay(
21147 &sheet_name,
21148 cell.coord.row() + 1,
21149 cell.coord.col() + 1,
21150 &empty,
21151 );
21152 }
21153 }
21154
21155 if let Some(log) = log
21157 && let Some(old) = snapshot
21158 {
21159 log.record(ChangeEvent::SpillCleared {
21160 anchor: anchor_vertex,
21161 old,
21162 });
21163 }
21164 Ok(())
21165 }
21166
21167 fn apply_spill_commit(
21169 &mut self,
21170 anchor_vertex: VertexId,
21171 target_cells: &[CellRef],
21172 values: Vec<Vec<LiteralValue>>,
21173 delta: Option<&mut DeltaCollector>,
21174 log: Option<&mut ChangeLog>,
21175 computed_writes: Option<&mut ComputedWriteBuffer>,
21176 ) -> Result<(), ExcelError> {
21177 self.guard_pending_spill_commit(anchor_vertex, target_cells)?;
21178 if let Some(buffer) = computed_writes {
21179 self.flush_computed_write_buffer(buffer)?;
21180 }
21181
21182 let old_snapshot = if log.is_some() {
21184 self.snapshot_spill_for_anchor(anchor_vertex)
21185 } else {
21186 None
21187 };
21188
21189 self.commit_spill_and_mirror(anchor_vertex, target_cells, values.clone(), delta)?;
21191
21192 if let Some(log) = log {
21194 log.record(ChangeEvent::SpillCommitted {
21195 anchor: anchor_vertex,
21196 old: old_snapshot,
21197 new: SpillSnapshot {
21198 target_cells: target_cells.to_vec(),
21199 values,
21200 },
21201 });
21202 }
21203 Ok(())
21204 }
21205
21206 fn snapshot_spill_for_anchor(&self, anchor: VertexId) -> Option<SpillSnapshot> {
21211 let cells = self.graph.spill_cells_for_anchor(anchor)?.to_vec();
21212 if cells.is_empty() {
21213 return None;
21214 }
21215
21216 let max = self.config.spill.max_spill_cells as usize;
21217 let mut cells = cells;
21218 if cells.len() > max {
21219 cells.truncate(max);
21220 }
21221
21222 let first = *cells.first().expect("non-empty spill cells");
21223 let sheet_name = self.graph.sheet_name(first.sheet_id).to_string();
21224 let row0 = first.coord.row();
21225 let col0 = first.coord.col();
21226
21227 let mut max_row = row0;
21228 let mut max_col = col0;
21229 let mut by_coord: FxHashMap<(u32, u32), LiteralValue> = FxHashMap::default();
21230 for cell in &cells {
21231 max_row = max_row.max(cell.coord.row());
21232 max_col = max_col.max(cell.coord.col());
21233 let v = self
21234 .read_cell_value(&sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
21235 .unwrap_or(LiteralValue::Empty);
21236 by_coord.insert((cell.coord.row(), cell.coord.col()), v);
21237 }
21238
21239 let rows = (max_row - row0 + 1) as usize;
21240 let cols = (max_col - col0 + 1) as usize;
21241 let mut values: Vec<Vec<LiteralValue>> = Vec::with_capacity(rows);
21242 for r in 0..rows {
21243 let mut row: Vec<LiteralValue> = Vec::with_capacity(cols);
21244 for c in 0..cols {
21245 row.push(
21246 by_coord
21247 .get(&(row0 + r as u32, col0 + c as u32))
21248 .cloned()
21249 .unwrap_or(LiteralValue::Empty),
21250 );
21251 }
21252 values.push(row);
21253 }
21254
21255 Some(SpillSnapshot {
21256 target_cells: cells,
21257 values,
21258 })
21259 }
21260
21261 fn flush_before_range_dependent_vertex(
21262 &mut self,
21263 vertex_id: VertexId,
21264 computed_writes: &mut ComputedWriteBuffer,
21265 ) -> Result<(), ExcelError> {
21266 if self.graph.reads_compressed_range(vertex_id) {
21267 self.flush_computed_write_buffer(computed_writes)?;
21268 }
21269 Ok(())
21270 }
21271
21272 fn plan_vertex_effects_with_computed_flush(
21273 &mut self,
21274 vertex_id: VertexId,
21275 computed_value: LiteralValue,
21276 computed_writes: &mut ComputedWriteBuffer,
21277 ) -> Result<PlannedEffects, ExcelError> {
21278 if matches!(&computed_value, LiteralValue::Array(_)) {
21279 self.flush_computed_write_buffer(computed_writes)?;
21280 }
21281 self.plan_vertex_effects(vertex_id, computed_value)
21282 }
21283
21284 fn evaluate_small_layer_direct_effects(
21287 &mut self,
21288 layer: &super::scheduler::Layer,
21289 delta: Option<&mut DeltaCollector>,
21290 log: Option<&mut ChangeLog>,
21291 cancel_flag: Option<&AtomicBool>,
21292 cancel_check_every: usize,
21293 cancel_message: &'static str,
21294 ) -> Result<usize, ExcelError> {
21295 let cancel = cancel_flag.map(|flag| (flag, cancel_check_every, cancel_message));
21296 self.evaluate_layer_units(layer, delta, log, cancel, false)
21297 }
21298
21299 fn evaluate_layer_units(
21306 &mut self,
21307 layer: &super::scheduler::Layer,
21308 delta: Option<&mut DeltaCollector>,
21309 log: Option<&mut ChangeLog>,
21310 cancel: Option<(&AtomicBool, usize, &'static str)>,
21311 buffered: bool,
21312 ) -> Result<usize, ExcelError> {
21313 self.evaluate_layer_units_until(layer, delta, log, cancel, buffered, None)
21314 }
21315
21316 fn evaluate_layer_units_until(
21320 &mut self,
21321 layer: &super::scheduler::Layer,
21322 mut delta: Option<&mut DeltaCollector>,
21323 mut log: Option<&mut ChangeLog>,
21324 cancel: Option<(&AtomicBool, usize, &'static str)>,
21325 buffered: bool,
21326 stop_at: Option<crate::instant::FzInstant>,
21327 ) -> Result<usize, ExcelError> {
21328 let mut chain_values = None;
21331 if layer.sequential && !layer.runs.is_empty() {
21332 chain_values = match layer.runs.as_slice() {
21333 [run] if run.start == 0 && run.len as usize == layer.vertices.len() => {
21334 self.try_chain_lift(*run, &layer.vertices)
21335 }
21336 _ => None,
21337 };
21338 if chain_values.is_none() {
21339 let cells = super::scheduler::Layer {
21340 vertices: layer.vertices.clone(),
21341 runs: Vec::new(),
21342 sequential: true,
21343 };
21344 return self.evaluate_layer_units_until(&cells, delta, log, cancel, false, stop_at);
21345 }
21346 }
21347 let chained = chain_values.is_some();
21348 let buffered = buffered || chained;
21350 let track_unflushed = buffered
21357 && self.freshness_armed()
21358 && layer.vertices.iter().any(|&v| self.graph.is_dynamic(v));
21359 let mut committed_unit: &[VertexId] = &[];
21360 let mut computed_writes = ComputedWriteBuffer::default();
21361 let mut next_check = 0usize;
21362 let mut done = 0usize;
21363 for unit in layer_units(layer) {
21364 if done > 0
21365 && let Some(stop_at) = stop_at
21366 && crate::instant::FzInstant::now() >= stop_at
21367 {
21368 self.flush_computed_write_buffer(&mut computed_writes)?;
21369 if track_unflushed {
21370 self.freshness_flushed();
21371 }
21372 return Ok(done);
21373 }
21374 if let Some((flag, every, message)) = cancel
21375 && every > 0
21376 && done >= next_check
21377 {
21378 next_check = (done / every + 1) * every;
21379 if flag.load(Ordering::Relaxed) {
21380 if buffered {
21381 self.flush_computed_write_buffer(&mut computed_writes)?;
21382 }
21383 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
21384 .with_message(message.to_string()));
21385 }
21386 }
21387 if buffered && self.unit_reads_compressed_range(layer, unit) {
21388 self.flush_computed_write_buffer(&mut computed_writes)?;
21389 }
21390 if track_unflushed {
21394 if computed_writes.is_empty() {
21395 self.freshness_flushed();
21396 } else {
21397 self.freshness_note_unflushed(committed_unit);
21398 }
21399 committed_unit = unit_members(layer, unit);
21400 }
21401 let values = match (chain_values.take(), unit) {
21402 (Some(chain), LayerUnit::Run(run)) => {
21403 if let Err(e) = self.live_cancellation_after_work(UNIT_CANCELLED) {
21404 self.flush_computed_write_buffer(&mut computed_writes)?;
21405 return Err(e);
21406 }
21407 let members =
21408 &layer.vertices[run.start as usize..(run.start + run.len) as usize];
21409 let delta_active = delta.as_deref().is_some_and(|d| d.mode != DeltaMode::Off);
21410 match self.commit_run_numbers(
21411 run,
21412 members,
21413 &chain,
21414 delta_active,
21415 Some(&mut computed_writes),
21416 ) {
21417 Ok(true) => {
21418 done += chain.len();
21419 continue;
21420 }
21421 Ok(false) => {}
21422 Err(e) => {
21423 self.flush_computed_write_buffer(&mut computed_writes)?;
21424 return Err(e);
21425 }
21426 }
21427 members
21428 .iter()
21429 .copied()
21430 .zip(chain.into_iter().map(LiteralValue::Number))
21431 .collect()
21432 }
21433 (_, unit) => self.evaluate_unit_immutable(layer, unit),
21434 };
21435 if let Err(e) = self.live_cancellation_after_work(UNIT_CANCELLED) {
21438 self.flush_computed_write_buffer(&mut computed_writes)?;
21439 return Err(e);
21440 }
21441 done += values.len();
21442 if let LayerUnit::Run(run) = unit {
21443 let members = &layer.vertices[run.start as usize..(run.start + run.len) as usize];
21444 let delta_active = delta.as_deref().is_some_and(|d| d.mode != DeltaMode::Off);
21445 let committed = self.commit_run_scalars(
21446 run,
21447 members,
21448 &values,
21449 delta_active,
21450 buffered.then_some(&mut computed_writes),
21451 );
21452 match committed {
21453 Ok(true) => continue,
21454 Ok(false) => {}
21455 Err(e) => {
21456 self.flush_computed_write_buffer(&mut computed_writes)?;
21457 return Err(e);
21458 }
21459 }
21460 }
21461 let guarded = self.freshness_begin_batch_commit(&values);
21464 for (vertex_id, value) in values {
21465 let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21466 let effects = if buffered {
21467 self.plan_vertex_effects_with_computed_flush(
21468 vertex_id,
21469 value,
21470 &mut computed_writes,
21471 )
21472 } else {
21473 self.plan_vertex_effects(vertex_id, value)
21474 };
21475 let effects = match effects {
21476 Ok(effects) => effects,
21477 Err(e) => {
21478 self.flush_computed_write_buffer(&mut computed_writes)?;
21479 return Err(e);
21480 }
21481 };
21482 for effect in &effects {
21483 if let Err(e) = self.apply_effect_with_computed_writes(
21484 effect,
21485 delta.as_deref_mut(),
21486 log.as_deref_mut(),
21487 buffered.then_some(&mut computed_writes),
21488 ) {
21489 self.flush_computed_write_buffer(&mut computed_writes)?;
21490 return Err(e);
21491 }
21492 }
21493 self.freshness_keep_redirtied(redirtied, vertex_id);
21494 }
21495 }
21496 self.flush_computed_write_buffer(&mut computed_writes)?;
21497 if track_unflushed {
21498 self.freshness_flushed();
21499 }
21500 #[cfg(debug_assertions)]
21503 if chained {
21504 for &v in &layer.vertices {
21505 let cell = self.graph.get_cell_ref(v);
21506 let (sheet, row, col) = cell
21507 .map(|c| {
21508 (
21509 self.graph.sheet_name(c.sheet_id).to_string(),
21510 c.coord.row() + 1,
21511 c.coord.col() + 1,
21512 )
21513 })
21514 .expect("chain member cell");
21515 let written = self.read_cell_value(&sheet, row, col);
21516 let oracle = self
21517 .evaluate_vertex_immutable(v)
21518 .unwrap_or_else(LiteralValue::Error);
21519 assert!(
21520 written
21521 .as_ref()
21522 .is_some_and(|w| same_value_bits(w, &oracle)),
21523 "chain member {sheet}!R{row}C{col}: {written:?} vs per-cell {oracle:?}"
21524 );
21525 }
21526 }
21527 #[cfg(not(debug_assertions))]
21528 let _ = chained;
21529 Ok(layer.vertices.len())
21530 }
21531
21532 fn evaluate_layer_sequential_effects(
21534 &mut self,
21535 layer: &super::scheduler::Layer,
21536 ) -> Result<usize, ExcelError> {
21537 let buffered = buffer_layer_writes(layer);
21538 self.evaluate_layer_units(layer, None, None, None, buffered)
21539 }
21540
21541 fn evaluate_layer_sequential_with_delta_effects(
21543 &mut self,
21544 layer: &super::scheduler::Layer,
21545 delta: &mut DeltaCollector,
21546 ) -> Result<usize, ExcelError> {
21547 let buffered = buffer_layer_writes(layer);
21548 self.evaluate_layer_units(layer, Some(delta), None, None, buffered)
21549 }
21550
21551 fn evaluate_layer_sequential_cancellable_effects(
21553 &mut self,
21554 layer: &super::scheduler::Layer,
21555 cancel_flag: &AtomicBool,
21556 ) -> Result<usize, ExcelError> {
21557 let buffered = buffer_layer_writes(layer);
21558 let cancel = (cancel_flag, 256, "Evaluation cancelled within layer");
21559 self.evaluate_layer_units(layer, None, None, Some(cancel), buffered)
21560 }
21561
21562 fn evaluate_layer_sequential_cancellable_demand_driven_effects(
21564 &mut self,
21565 layer: &super::scheduler::Layer,
21566 cancel_flag: &AtomicBool,
21567 ) -> Result<usize, ExcelError> {
21568 let buffered = buffer_layer_writes(layer);
21569 let cancel = (
21570 cancel_flag,
21571 128,
21572 "Demand-driven evaluation cancelled within layer",
21573 );
21574 self.evaluate_layer_units(layer, None, None, Some(cancel), buffered)
21575 }
21576
21577 fn evaluate_layer_parallel_effects(
21579 &mut self,
21580 layer: &super::scheduler::Layer,
21581 min_chunk: u32,
21582 ) -> Result<usize, ExcelError> {
21583 let thread_pool = self.thread_pool.as_ref().unwrap().clone();
21584
21585 let phases = self.parallel_phases(layer);
21586 let mut applied = 0usize;
21587
21588 for (units, group) in &phases {
21589 let group = &group[..];
21590 if group.is_empty() {
21591 continue;
21592 }
21593 let mut computed_writes = ComputedWriteBuffer::default();
21594
21595 let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> =
21596 thread_pool.install(|| self.evaluate_units_parallel(layer, units, None, min_chunk));
21597
21598 let results = results.and_then(|results| {
21601 self.live_cancellation_after_work(GROUP_CANCELLED)
21602 .map(|()| results)
21603 });
21604 self.freshness_gate_group(group);
21607 match results {
21608 Ok(vertex_results) => {
21609 let (vertex_results, committed) = self.commit_parallel_runs(
21610 layer,
21611 units,
21612 vertex_results,
21613 false,
21614 &mut computed_writes,
21615 )?;
21616 applied = applied.saturating_add(committed);
21617 let guarded = self.freshness_begin_batch_commit(&vertex_results);
21622 let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
21623 let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
21624 for (vertex_id, result) in vertex_results {
21625 if matches!(result, LiteralValue::Array(_)) {
21626 arrays.push((vertex_id, result));
21627 } else {
21628 others.push((vertex_id, result));
21629 }
21630 }
21631 for (vertex_id, result) in arrays {
21632 let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21633 let effects = match self.plan_vertex_effects_with_computed_flush(
21634 vertex_id,
21635 result,
21636 &mut computed_writes,
21637 ) {
21638 Ok(effects) => effects,
21639 Err(e) => {
21640 self.flush_computed_write_buffer(&mut computed_writes)?;
21641 return Err(e);
21642 }
21643 };
21644 for effect in &effects {
21645 if let Err(e) = self.apply_effect_with_computed_writes(
21646 effect,
21647 None,
21648 None,
21649 Some(&mut computed_writes),
21650 ) {
21651 self.flush_computed_write_buffer(&mut computed_writes)?;
21652 return Err(e);
21653 }
21654 }
21655 self.freshness_keep_redirtied(redirtied, vertex_id);
21656 applied = applied.saturating_add(1);
21657 }
21658 self.flush_computed_write_buffer(&mut computed_writes)?;
21660 for (vertex_id, result) in others {
21661 let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21662 let effects = match self.plan_vertex_effects_with_computed_flush(
21663 vertex_id,
21664 result,
21665 &mut computed_writes,
21666 ) {
21667 Ok(effects) => effects,
21668 Err(e) => {
21669 self.flush_computed_write_buffer(&mut computed_writes)?;
21670 return Err(e);
21671 }
21672 };
21673 for effect in &effects {
21674 if let Err(e) = self.apply_effect_with_computed_writes(
21675 effect,
21676 None,
21677 None,
21678 Some(&mut computed_writes),
21679 ) {
21680 self.flush_computed_write_buffer(&mut computed_writes)?;
21681 return Err(e);
21682 }
21683 }
21684 self.freshness_keep_redirtied(redirtied, vertex_id);
21685 applied = applied.saturating_add(1);
21686 }
21687 self.flush_computed_write_buffer(&mut computed_writes)?;
21689 }
21690 Err(e) => {
21691 self.flush_computed_write_buffer(&mut computed_writes)?;
21692 return Err(e);
21693 }
21694 }
21695 }
21696
21697 Ok(applied)
21698 }
21699
21700 fn evaluate_layer_parallel_with_delta_effects(
21702 &mut self,
21703 layer: &super::scheduler::Layer,
21704 delta: &mut DeltaCollector,
21705 ) -> Result<usize, ExcelError> {
21706 let thread_pool = self.thread_pool.as_ref().unwrap().clone();
21707
21708 let phases = self.parallel_phases(layer);
21709 let mut applied = 0usize;
21710
21711 for (units, group) in &phases {
21712 let group = &group[..];
21713 if group.is_empty() {
21714 continue;
21715 }
21716 let mut computed_writes = ComputedWriteBuffer::default();
21717 let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> =
21718 thread_pool.install(|| self.evaluate_units_parallel(layer, units, None, 8));
21719
21720 let results = results.and_then(|results| {
21723 self.live_cancellation_after_work(GROUP_CANCELLED)
21724 .map(|()| results)
21725 });
21726 self.freshness_gate_group(group);
21729 match results {
21730 Ok(vertex_results) => {
21731 let (vertex_results, committed) = self.commit_parallel_runs(
21732 layer,
21733 units,
21734 vertex_results,
21735 delta.mode != DeltaMode::Off,
21736 &mut computed_writes,
21737 )?;
21738 applied = applied.saturating_add(committed);
21739 let guarded = self.freshness_begin_batch_commit(&vertex_results);
21742 let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
21743 let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
21744 for (vertex_id, result) in vertex_results {
21745 if matches!(result, LiteralValue::Array(_)) {
21746 arrays.push((vertex_id, result));
21747 } else {
21748 others.push((vertex_id, result));
21749 }
21750 }
21751 for (vertex_id, result) in arrays {
21752 let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21753 let effects = match self.plan_vertex_effects_with_computed_flush(
21754 vertex_id,
21755 result,
21756 &mut computed_writes,
21757 ) {
21758 Ok(effects) => effects,
21759 Err(e) => {
21760 self.flush_computed_write_buffer(&mut computed_writes)?;
21761 return Err(e);
21762 }
21763 };
21764 for effect in &effects {
21765 if let Err(e) = self.apply_effect_with_computed_writes(
21766 effect,
21767 Some(delta),
21768 None,
21769 Some(&mut computed_writes),
21770 ) {
21771 self.flush_computed_write_buffer(&mut computed_writes)?;
21772 return Err(e);
21773 }
21774 }
21775 self.freshness_keep_redirtied(redirtied, vertex_id);
21776 applied = applied.saturating_add(1);
21777 }
21778 self.flush_computed_write_buffer(&mut computed_writes)?;
21779 for (vertex_id, result) in others {
21780 let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21781 let effects = match self.plan_vertex_effects_with_computed_flush(
21782 vertex_id,
21783 result,
21784 &mut computed_writes,
21785 ) {
21786 Ok(effects) => effects,
21787 Err(e) => {
21788 self.flush_computed_write_buffer(&mut computed_writes)?;
21789 return Err(e);
21790 }
21791 };
21792 for effect in &effects {
21793 if let Err(e) = self.apply_effect_with_computed_writes(
21794 effect,
21795 Some(delta),
21796 None,
21797 Some(&mut computed_writes),
21798 ) {
21799 self.flush_computed_write_buffer(&mut computed_writes)?;
21800 return Err(e);
21801 }
21802 }
21803 self.freshness_keep_redirtied(redirtied, vertex_id);
21804 applied = applied.saturating_add(1);
21805 }
21806 self.flush_computed_write_buffer(&mut computed_writes)?;
21807 }
21808 Err(e) => {
21809 self.flush_computed_write_buffer(&mut computed_writes)?;
21810 return Err(e);
21811 }
21812 }
21813 }
21814
21815 Ok(applied)
21816 }
21817
21818 fn evaluate_layer_parallel_cancellable_effects(
21820 &mut self,
21821 layer: &super::scheduler::Layer,
21822 cancel_flag: &AtomicBool,
21823 ) -> Result<usize, ExcelError> {
21824 let thread_pool = self.thread_pool.as_ref().unwrap().clone();
21825
21826 if cancel_flag.load(Ordering::Relaxed) {
21827 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
21828 .with_message("Parallel evaluation cancelled before starting".to_string()));
21829 }
21830
21831 let phases = self.parallel_phases(layer);
21832 let mut applied = 0usize;
21833
21834 for (units, group) in &phases {
21835 let group = &group[..];
21836 if group.is_empty() {
21837 continue;
21838 }
21839 let mut computed_writes = ComputedWriteBuffer::default();
21840
21841 let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> = thread_pool
21842 .install(|| self.evaluate_units_parallel(layer, units, Some(cancel_flag), 8));
21843
21844 let results = results.and_then(|results| {
21847 self.live_cancellation_after_work(GROUP_CANCELLED)
21848 .map(|()| results)
21849 });
21850 self.freshness_gate_group(group);
21853 match results {
21854 Ok(vertex_results) => {
21855 let (vertex_results, committed) = self.commit_parallel_runs(
21856 layer,
21857 units,
21858 vertex_results,
21859 false,
21860 &mut computed_writes,
21861 )?;
21862 applied = applied.saturating_add(committed);
21863 let guarded = self.freshness_begin_batch_commit(&vertex_results);
21866 let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
21867 let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
21868 for (vertex_id, result) in vertex_results {
21869 if matches!(result, LiteralValue::Array(_)) {
21870 arrays.push((vertex_id, result));
21871 } else {
21872 others.push((vertex_id, result));
21873 }
21874 }
21875 for (vertex_id, result) in arrays {
21876 let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21877 let effects = match self.plan_vertex_effects_with_computed_flush(
21878 vertex_id,
21879 result,
21880 &mut computed_writes,
21881 ) {
21882 Ok(effects) => effects,
21883 Err(e) => {
21884 self.flush_computed_write_buffer(&mut computed_writes)?;
21885 return Err(e);
21886 }
21887 };
21888 for effect in &effects {
21889 if let Err(e) = self.apply_effect_with_computed_writes(
21890 effect,
21891 None,
21892 None,
21893 Some(&mut computed_writes),
21894 ) {
21895 self.flush_computed_write_buffer(&mut computed_writes)?;
21896 return Err(e);
21897 }
21898 }
21899 self.freshness_keep_redirtied(redirtied, vertex_id);
21900 applied = applied.saturating_add(1);
21901 }
21902 self.flush_computed_write_buffer(&mut computed_writes)?;
21903 for (vertex_id, result) in others {
21904 let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21905 let effects = match self.plan_vertex_effects_with_computed_flush(
21906 vertex_id,
21907 result,
21908 &mut computed_writes,
21909 ) {
21910 Ok(effects) => effects,
21911 Err(e) => {
21912 self.flush_computed_write_buffer(&mut computed_writes)?;
21913 return Err(e);
21914 }
21915 };
21916 for effect in &effects {
21917 if let Err(e) = self.apply_effect_with_computed_writes(
21918 effect,
21919 None,
21920 None,
21921 Some(&mut computed_writes),
21922 ) {
21923 self.flush_computed_write_buffer(&mut computed_writes)?;
21924 return Err(e);
21925 }
21926 }
21927 self.freshness_keep_redirtied(redirtied, vertex_id);
21928 applied = applied.saturating_add(1);
21929 }
21930 self.flush_computed_write_buffer(&mut computed_writes)?;
21931 }
21932 Err(e) => {
21933 self.flush_computed_write_buffer(&mut computed_writes)?;
21934 return Err(e);
21935 }
21936 }
21937 }
21938
21939 Ok(applied)
21940 }
21941
21942 pub fn evaluate_all_logged(&mut self, log: &mut ChangeLog) -> Result<EvalResult, ExcelError> {
21949 self.observe_evaluation_resource_request(EvaluationRequestKind::FullLogged, |engine| {
21950 engine.evaluate_all_logged_unobserved(log)
21951 })
21952 }
21953
21954 fn evaluate_all_logged_unobserved(
21955 &mut self,
21956 log: &mut ChangeLog,
21957 ) -> Result<EvalResult, ExcelError> {
21958 self.observe_function_semantic_epoch()?;
21959 let _source_cache = self.source_cache_session();
21960 self.validate_deterministic_mode()?;
21961 if self.config.defer_graph_building {
21962 self.build_graph_all()?;
21963 }
21964 self.require_unified_authority()?;
21965 self.begin_evaluation_request();
21966 self.reset_virtual_dep_telemetry_if_disabled();
21967 let start = crate::instant::FzInstant::now();
21968 let mut computed_vertices = 0;
21969 let mut cycle_errors = 0;
21970
21971 let mut replan_iterations = 0;
21972 const MAX_REPLAN: usize = 5;
21973 let mut telemetry = self
21974 .config
21975 .enable_virtual_dep_telemetry
21976 .then(|| self.start_virtual_dep_telemetry());
21977
21978 log.begin_compound(format!("evaluate_all(epoch={})", self.recalc_epoch));
21979
21980 let result = (|| -> Result<EvalResult, ExcelError> {
21981 loop {
21982 let to_evaluate = self.graph.get_evaluation_vertices();
21983 if to_evaluate.is_empty() {
21984 if let Some(t) = telemetry.as_mut()
21985 && t.bailout_reason.is_none()
21986 {
21987 t.bailout_reason = Some("no_work");
21988 }
21989 break;
21990 }
21991
21992 let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
21993 if let Some(t) = telemetry.as_mut() {
21994 Self::accumulate_schedule_meta(t, &meta);
21995 }
21996
21997 self.begin_pass(&schedule);
22000 for (unit_index, &unit) in schedule.units.iter().enumerate() {
22001 match unit {
22002 ScheduleUnit::Cycle(i) => {
22003 if self.handle_cycle_unit(schedule.unit_cycle(i), None, None, None)? > 0
22014 {
22015 cycle_errors += 1;
22016 }
22017 }
22018 ScheduleUnit::Layer(i) => {
22019 computed_vertices +=
22020 self.evaluate_layer_logged(schedule.unit_layer(i), log)?;
22021 }
22022 }
22023 if self.stop_after_unit(&schedule, unit_index) {
22024 break;
22025 }
22026 }
22027
22028 let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
22029 if let Some(t) = telemetry.as_mut() {
22030 t.changed_vdeps_total += changed_vertices.len();
22031 }
22032 self.resource_checkpoint(0)?;
22033 if !self.finish_pass_dirty(&to_evaluate, &changed_vertices) {
22034 if let Some(t) = telemetry.as_mut() {
22035 t.bailout_reason = Some("converged");
22036 }
22037 break;
22038 }
22039 if replan_iterations >= MAX_REPLAN {
22040 if let Some(mut t) = telemetry.take() {
22041 t.bailout_reason = Some("max_replan");
22042 t.replan_iterations = replan_iterations;
22043 self.last_virtual_dep_telemetry = t;
22044 }
22045 return Err(
22046 self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
22047 );
22048 }
22049 replan_iterations += 1;
22050 }
22051
22052 if let Some(mut t) = telemetry {
22053 t.replan_iterations = replan_iterations;
22054 self.last_virtual_dep_telemetry = t;
22055 }
22056
22057 self.redirty_for_next_recalc();
22058 self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
22059
22060 Ok(EvalResult {
22061 computed_vertices,
22062 cycle_errors,
22063 elapsed: start.elapsed(),
22064 })
22065 })();
22066 log.end_compound();
22067 result
22068 }
22069
22070 fn evaluate_layer_logged(
22072 &mut self,
22073 layer: &super::scheduler::Layer,
22074 log: &mut ChangeLog,
22075 ) -> Result<usize, ExcelError> {
22076 self.resource_checkpoint(layer.vertices.len() as u64)?;
22077 self.evaluate_layer_units(layer, None, Some(log), None, true)
22078 }
22079}