1use crate::SheetId;
2
3mod criteria;
4mod criteria_mask_cache;
5#[cfg(test)]
6mod criteria_mask_cache_tests;
7mod exact;
8mod family;
9mod kernels;
10mod lift;
11mod memo;
12use crate::arrow_store::{OverlayFragment, OverlayValue, SheetStore};
13#[cfg(test)]
14use crate::engine::Scheduler;
15use crate::engine::arena::AstNodeId;
16use crate::engine::eval_delta::{
17 DeltaCollector, DeltaMode, EvalDelta, EvalDeltaCompatibilityPolicy,
18};
19use crate::engine::graph::editor::change_log::MutationCapture;
20use crate::engine::graph::prepared_legacy_graph::PreparedLegacyGraphPlan;
21use crate::engine::ingest_pipeline::{DependencyPlanRow, FormulaAstInput};
22use crate::engine::live_edges::{LiveEdgeCollector, RecordingContext};
23use crate::engine::live_graph::analyze_live_graph;
24use crate::engine::lookup_index_cache::{
25 BuildOutcome, LookupAxis, LookupIndex, LookupIndexCache, LookupIndexCacheReport,
26 LookupIndexKey, estimate_bytes,
27};
28use crate::engine::named_range::{NameScope, NamedDefinition};
29use crate::engine::range_view::RangeView;
30use crate::engine::row_visibility::RowVisibilityState;
31use crate::engine::spill::{RegionLockManager, SpillMeta, SpillShape};
32use crate::engine::target_preparation::{
33 StagedFormulaIndex, StagedFormulaLease, StagedPackageLease,
34};
35use crate::engine::used_extent::{
36 ExtentPolicy, OpenRangeBounds, resolve_used_extent_with_fallback,
37};
38use crate::engine::virtual_deps::VirtualDepBuilder;
39use family::{LayerUnit, layer_units, unit_members};
40
41#[path = "freshness.rs"]
42mod freshness;
43use crate::engine::template::region::Region;
44use crate::engine::{
45 ChangeLogger, CycleDetection, CyclePolicy, DependencyGraph, EvalConfig, EvaluationRequestKind,
46 EvaluationRequestOutcome, EvaluationResourceBaselineStats, EvaluationResourceRequestStats,
47 FormulaDirtyLeaseOutcome, FormulaIngestBatch, FormulaIngestRecord, FormulaIngestReport,
48 FormulaParseDiagnostic, FormulaParsePolicy, FormulaPlaneMode, ResourceLedger,
49 RowVisibilitySource, ScheduleUnit, VertexId, VertexKind, VisibilityMaskMode,
50};
51use crate::function::FnCaps;
52use crate::interpreter::Interpreter;
53
54#[cfg(test)]
55mod fn_snapshot_tests;
56use crate::reference::{CellRef, Coord, RangeRef};
57use crate::traits::FunctionProvider;
58use crate::traits::{EvaluationContext, ReferenceInfo, Resolver};
59use formualizer_common::{
60 CoordBuildHasher, LiteralValue, col_letters_from_1based, parse_a1_1based,
61};
62use formualizer_parse::parser::ReferenceType;
63use formualizer_parse::{ASTNode, ASTNodeType, ExcelError, ExcelErrorKind};
64use rayon::ThreadPoolBuilder;
65use rustc_hash::{FxHashMap, FxHashSet};
66use std::collections::{BTreeMap, BTreeSet, VecDeque};
67use std::sync::Arc;
68use std::sync::atomic::{AtomicBool, Ordering};
69use std::time::Instant;
70
71type StagedFormulaEntry = (u32, u32, String);
72type StagedSheetParts = (
73 Vec<StagedFormulaEntry>,
74 Option<crate::engine::DeferredFormulaPackage>,
75);
76
77#[derive(Default)]
87pub(crate) struct StagedSheet {
88 entries: Vec<StagedFormulaEntry>,
89 index: FxHashMap<(u32, u32), usize>,
90 deferred_package: Option<crate::engine::DeferredFormulaPackage>,
91}
92
93impl StagedSheet {
94 fn invalidate_all_deferred_families(package: &mut crate::engine::DeferredFormulaPackage) {
95 package
96 .invalidated
97 .extend(package.families.iter().map(|family| family.source_id));
98 package.invalidated.extend(
99 package
100 .partitioned_families
101 .iter()
102 .map(|family| family.source_id),
103 );
104 }
105
106 fn invalidate_deferred_at(&mut self, row: u32, col: u32) {
107 let Some(package) = self.deferred_package.as_mut() else {
108 return;
109 };
110 if package.suppressed.contains(&(row, col)) {
111 return;
114 }
115 if package.source_geometry_complete
116 && package.families.is_empty()
117 && package.partitioned_families.is_empty()
118 {
119 if let Some((row0, col0)) = row.checked_sub(1).zip(col.checked_sub(1))
120 && package
121 .source_coordinates
122 .binary_search(&crate::engine::SourceCoord {
123 row: row0,
124 col: col0,
125 })
126 .is_ok()
127 {
128 package.suppressed.insert((row, col));
129 }
130 return;
131 }
132 let family = package
133 .replay
134 .lock()
135 .ok()
136 .and_then(|mut replay| replay.formula_at(row, col).ok())
137 .flatten()
138 .and_then(|record| record.partition_owner.or(record.family));
139 if let Some(family) = family {
140 package.invalidated.insert(family);
141 } else {
142 Self::invalidate_all_deferred_families(package);
146 }
147 package.suppressed.insert((row, col));
148 }
149
150 fn reconcile_attached_deferred_package(&mut self) {
151 let Some(package) = self.deferred_package.as_mut() else {
152 return;
153 };
154 if self.entries.is_empty() {
155 return;
156 }
157
158 let replayed = (|| {
159 let mut disposition = crate::engine::FormulaReplayDisposition::default();
160 for partition in &package.partitioned_families {
161 disposition.register_partition(partition, false)?;
162 }
163 package
164 .replay
165 .lock()
166 .map_err(|_| "deferred formula spool lock poisoned".to_string())?
167 .replay_partitioned(&disposition, &package.partitioned_families)
168 })();
169
170 let Ok(records) = replayed else {
171 Self::invalidate_all_deferred_families(package);
172 package
173 .suppressed
174 .extend(self.entries.iter().map(|(row, col, _)| (*row, *col)));
175 return;
176 };
177 let mut owners = FxHashMap::default();
178 for record in &records {
179 owners
180 .entry((record.row, record.col))
181 .or_insert(record.partition_owner.or(record.family));
182 }
183 for (row, col, _) in &self.entries {
184 if let Some(family) = owners.get(&(*row, *col)).copied().flatten() {
185 package.invalidated.insert(family);
186 } else {
187 Self::invalidate_all_deferred_families(package);
188 }
189 package.suppressed.insert((*row, *col));
190 }
191 package.reconciliation_replay = Some(records);
192 }
193
194 fn stage(&mut self, row: u32, col: u32, text: String) {
195 self.invalidate_deferred_at(row, col);
196 match self.index.entry((row, col)) {
197 std::collections::hash_map::Entry::Occupied(slot) => {
198 self.entries[*slot.get()].2 = text;
199 }
200 std::collections::hash_map::Entry::Vacant(slot) => {
201 slot.insert(self.entries.len());
202 self.entries.push((row, col, text));
203 }
204 }
205 }
206
207 fn remove(&mut self, row: u32, col: u32) -> Option<String> {
208 let deferred_text = self.get(row, col);
209 self.invalidate_deferred_at(row, col);
210 let Some(idx) = self.index.remove(&(row, col)) else {
211 return deferred_text;
212 };
213 let (_, _, text) = self.entries.remove(idx);
214 for slot in self.index.values_mut() {
216 if *slot > idx {
217 *slot -= 1;
218 }
219 }
220 Some(text)
221 }
222
223 fn get_ordinary(&self, row: u32, col: u32) -> Option<&str> {
224 self.index
225 .get(&(row, col))
226 .map(|&i| self.entries[i].2.as_str())
227 }
228
229 fn remove_ordinary(&mut self, row: u32, col: u32) -> Option<String> {
230 let idx = self.index.remove(&(row, col))?;
231 let (_, _, text) = self.entries.remove(idx);
232 for slot in self.index.values_mut() {
233 if *slot > idx {
234 *slot -= 1;
235 }
236 }
237 Some(text)
238 }
239
240 fn get(&self, row: u32, col: u32) -> Option<String> {
241 if let Some(text) = self.get_ordinary(row, col) {
242 return Some(text.to_string());
243 }
244 let package = self.deferred_package.as_ref()?;
245 if package.suppressed.contains(&(row, col)) {
246 return None;
247 }
248 package
249 .replay
250 .lock()
251 .ok()?
252 .formula_at(row, col)
253 .ok()?
254 .map(|record| record.text)
255 }
256
257 fn len(&self) -> usize {
258 self.deferred_package
259 .as_ref()
260 .map_or(self.entries.len(), |package| {
261 (if package.source_geometry_complete
262 && (package.coordinates_cover_families
263 || (package.families.is_empty() && package.partitioned_families.is_empty()))
264 {
265 package.source_coordinates.len()
266 } else {
267 usize::try_from(package.report.source_formula_records_spooled)
268 .unwrap_or(usize::MAX)
269 })
270 .saturating_sub(package.suppressed.len())
271 .saturating_add(self.entries.len())
272 })
273 }
274
275 fn is_empty(&self) -> bool {
276 self.entries.is_empty() && self.deferred_package.is_none()
277 }
278
279 fn into_parts(self) -> StagedSheetParts {
280 (self.entries, self.deferred_package)
281 }
282}
283
284type StagedFormulaMap = std::collections::HashMap<String, StagedSheet>;
285
286#[derive(Clone, Debug, PartialEq, Eq, Hash)]
287struct PreparationRegion {
288 sheet: String,
289 sheet_id: SheetId,
290 start_row: u32,
291 start_col: u32,
292 end_row: u32,
293 end_col: u32,
294}
295
296struct OrderedTargetProducers {
298 ordered: Vec<crate::engine::target_preparation::TargetProducer>,
299 seen: FxHashSet<crate::engine::target_preparation::TargetProducer>,
300}
301
302impl OrderedTargetProducers {
303 fn with_capacity(capacity: usize) -> Result<Self, std::collections::TryReserveError> {
304 let mut ordered = Vec::new();
305 ordered.try_reserve(capacity)?;
306 let mut seen = FxHashSet::default();
307 seen.try_reserve(capacity)?;
308 Ok(Self { ordered, seen })
309 }
310
311 fn from_ordered(
312 ordered: Vec<crate::engine::target_preparation::TargetProducer>,
313 ) -> Result<Self, std::collections::TryReserveError> {
314 let mut seen = FxHashSet::default();
315 seen.try_reserve(ordered.len())?;
316 seen.extend(ordered.iter().copied());
317 Ok(Self { ordered, seen })
318 }
319
320 fn push(
321 &mut self,
322 producer: crate::engine::target_preparation::TargetProducer,
323 ) -> Result<bool, std::collections::TryReserveError> {
324 #[cfg(test)]
325 TARGET_ROOT_DEDUP_PROBES.with(|probes| probes.set(probes.get().saturating_add(1)));
326 if self.seen.contains(&producer) {
327 return Ok(false);
328 }
329 self.seen.try_reserve(1)?;
330 self.ordered.try_reserve(1)?;
331 self.seen.insert(producer);
332 self.ordered.push(producer);
333 Ok(true)
334 }
335
336 fn len(&self) -> usize {
337 self.ordered.len()
338 }
339
340 fn into_vec(self) -> Vec<crate::engine::target_preparation::TargetProducer> {
341 self.ordered
342 }
343}
344
345#[cfg(test)]
346thread_local! {
347 static TARGET_ROOT_DEDUP_PROBES: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
348}
349
350fn target_root_allocation_error(observed: usize, request_id: Option<u64>) -> ExcelError {
351 crate::engine::ResourceLedgerError::Exhausted(formualizer_common::ResourceExhaustionDetail {
352 reason: formualizer_common::ResourceExhaustionReason::ScratchMemory,
353 limit: u64::MAX,
354 observed: observed as u64,
355 request_id,
356 })
357 .into_excel_error()
358}
359
360#[derive(Debug)]
361struct PreparedOrdinaryStagedFormula {
362 sheet: String,
363 sheet_id: SheetId,
364 lease: StagedFormulaLease,
365 ast_id: Option<AstNodeId>,
366 plan: Option<DependencyPlanRow>,
367}
368
369struct PreparedTargetSourcePackage {
370 sheet: String,
371 sheet_id: SheetId,
372 lease: StagedPackageLease,
373 selected_points: Option<BTreeSet<(u32, u32)>>,
374 complete_selections: BTreeSet<crate::engine::SourceFamilyId>,
375 deferred_shared: bool,
376 direct_domains: Vec<(
377 crate::engine::SourceFamilyId,
378 crate::engine::PlacementDomainTransport,
379 )>,
380 source_report: crate::engine::FormulaCompressedSourceReport,
381 replay_records: Vec<crate::engine::DeferredReplayFormula>,
382 spool_replays: u64,
383 disposition: crate::engine::FormulaReplayDisposition,
384 legacy: Vec<(u32, u32, AstNodeId, DependencyPlanRow)>,
385 direct_families: usize,
386 direct_cells: u64,
387 direct_fragments: u64,
388 direct_complete_families: u64,
389 direct_complete_cells: u64,
390 direct_partition_families: u64,
391 direct_partition_cells: u64,
392 anchor_parses: u64,
393 anchor_asts: u64,
394 anchor_analyses: u64,
395}
396
397impl PreparedTargetSourcePackage {
398 fn empty_selection(sheet: &str, sheet_id: SheetId, lease: StagedPackageLease) -> Self {
399 Self {
400 sheet: sheet.to_owned(),
401 sheet_id,
402 lease,
403 selected_points: Some(BTreeSet::new()),
404 complete_selections: Default::default(),
405 deferred_shared: false,
406 direct_domains: Vec::new(),
407 source_report: Default::default(),
408 replay_records: Vec::new(),
409 spool_replays: 0,
410 disposition: Default::default(),
411 legacy: Vec::new(),
412 direct_families: 0,
413 direct_cells: 0,
414 direct_fragments: 0,
415 direct_complete_families: 0,
416 direct_complete_cells: 0,
417 direct_partition_families: 0,
418 direct_partition_cells: 0,
419 anchor_parses: 0,
420 anchor_asts: 0,
421 anchor_analyses: 0,
422 }
423 }
424
425 fn direct_contains(&self, row: u32, col: u32) -> bool {
426 self.direct_domains.iter().any(|(_, domain)| {
427 let rect = domain.rect();
428 row > rect.start.row
429 && row <= rect.end.row + 1
430 && col > rect.start.col
431 && col <= rect.end.col + 1
432 })
433 }
434
435 fn fallback_records(&self) -> impl Iterator<Item = &crate::engine::DeferredReplayFormula> {
436 self.replay_records.iter().filter(|record| {
437 let Some((row, col)) = record.row.checked_sub(1).zip(record.col.checked_sub(1)) else {
438 return true;
439 };
440 let coord = crate::engine::SourceCoord { row, col };
441 let disposition = match record.family {
442 Some(family) => self.disposition.shared_disposition(family, coord),
443 None => self.disposition.ordinary_disposition(coord).0,
444 };
445 !matches!(
446 disposition,
447 crate::engine::FormulaReplayCoordinateDisposition::Direct
448 | crate::engine::FormulaReplayCoordinateDisposition::Suppressed
449 )
450 })
451 }
452}
453
454type PreparedFormulaBatches = Vec<FormulaIngestBatch>;
455type StagedFormulaBatches = Vec<(String, StagedSheet)>;
456
457type CompressedReplayBatch = (
458 FormulaIngestBatch,
459 crate::engine::FormulaCompressedSourceBatch,
460);
461type PreparedSourceBatch = (
462 FormulaIngestBatch,
463 crate::engine::FormulaCompressedSourceReport,
464 crate::engine::FormulaCompressedPreparation,
465);
466type PreparedStagedFormulaBatches = (
467 PreparedFormulaBatches,
468 Vec<CompressedReplayBatch>,
469 Vec<PreparedSourceBatch>,
470 FxHashSet<crate::engine::arena::AstNodeId>,
472);
473
474#[doc(hidden)]
477pub struct SourceFormulaIngress<'a, R> {
478 engine: &'a mut Engine<R>,
479}
480
481impl<R> SourceFormulaIngress<'_, R>
482where
483 R: EvaluationContext,
484{
485 pub fn prepare_families(
486 &mut self,
487 sheet_name: &str,
488 families: &[crate::engine::SourceFormulaFamily],
489 ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
490 self.engine.observe_function_semantic_epoch()?;
491 Ok(self
492 .engine
493 .prepare_source_formula_families(sheet_name, families))
494 }
495
496 pub fn prepare_eager_proposals(
497 &mut self,
498 sheet_name: &str,
499 families: &[crate::engine::SourceFormulaFamily],
500 partitions: &[crate::engine::PartitionedSourceFormulaFamily],
501 formula_record_count: u64,
502 replay: Box<dyn crate::engine::DeferredFormulaReplay>,
503 ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
504 self.engine.observe_function_semantic_epoch()?;
505 let replay = Arc::new(std::sync::Mutex::new(replay));
506 self.engine.prepare_source_formula_proposals(
507 sheet_name,
508 families,
509 partitions,
510 partitions,
511 formula_record_count,
512 replay,
513 &BTreeSet::new(),
514 &Default::default(),
515 None,
516 )
517 }
518
519 pub fn stage_deferred(&mut self, package: crate::engine::DeferredFormulaPackage) {
520 self.engine.stage_deferred_formula_package(package);
521 }
522
523 pub fn ingest_replay_batches(
524 &mut self,
525 batches: Vec<(
526 FormulaIngestBatch,
527 crate::engine::FormulaCompressedSourceBatch,
528 )>,
529 ) -> Result<FormulaIngestReport, ExcelError> {
530 self.engine
531 .ingest_compressed_formula_source_batches(batches)
532 }
533
534 pub fn finish_prepared(
535 &mut self,
536 batches: Vec<(
537 FormulaIngestBatch,
538 crate::engine::FormulaCompressedSourceReport,
539 crate::engine::FormulaCompressedPreparation,
540 )>,
541 ) -> Result<FormulaIngestReport, ExcelError> {
542 self.engine.finish_compressed_formula_sources(batches)
543 }
544}
545
546const COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH: usize = 8;
550
551#[cfg(debug_assertions)]
553fn same_value_bits(a: &LiteralValue, b: &LiteralValue) -> bool {
554 match (a, b) {
555 (LiteralValue::Number(x), LiteralValue::Number(y)) => x.to_bits() == y.to_bits(),
556 _ => a == b,
557 }
558}
559
560fn buffer_layer_writes(layer: &crate::engine::scheduler::Layer) -> bool {
563 !layer.sequential && layer.vertices.len() >= COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH
564}
565const PARALLEL_SCHEDULE_MIN_CANDIDATES: usize = 16 * 1024;
573const BASE_SCHEDULE_MIN_VERTICES: usize = 64;
577const BASE_SCHEDULE_RATIO: usize = 256;
578const BASE_SCHEDULE_MIN_REQUEST: usize = 128;
583const PARALLEL_LAYER_PROBE: std::time::Duration = std::time::Duration::from_micros(300);
584const PARALLEL_LAYER_WORTH: std::time::Duration = std::time::Duration::from_micros(150);
585const EXPENSIVE_VERTEX_NS: u128 = 10_000;
587const UNIT_CANCELLED: &str = "Evaluation cancelled; the evaluated unit was not committed";
589const GROUP_CANCELLED: &str =
590 "Parallel evaluation cancelled; the evaluated group was not committed";
591const SCC_CANCELLED: &str = "Evaluation cancelled; the evaluated cycle was not completed";
592
593#[derive(Debug, Clone, PartialEq)]
594pub(crate) enum ComputedWrite {
595 Cell {
596 seq: u64,
597 sheet_id: SheetId,
598 row0: u32,
599 col0: u32,
600 value: OverlayValue,
601 format_id: Option<crate::format::FormatId>,
602 },
603 Rect {
604 seq: u64,
605 sheet_id: SheetId,
606 sr0: u32,
607 sc0: u32,
608 values: Vec<Vec<OverlayValue>>,
609 },
610 Run {
613 seq: u64,
614 sheet_id: SheetId,
615 row0: u32,
616 col0: u32,
617 entries: Vec<(OverlayValue, Option<crate::format::FormatId>)>,
618 },
619}
620
621impl ComputedWrite {
622 #[inline]
623 pub(crate) fn seq(&self) -> u64 {
624 match self {
625 ComputedWrite::Cell { seq, .. }
626 | ComputedWrite::Rect { seq, .. }
627 | ComputedWrite::Run { seq, .. } => *seq,
628 }
629 }
630}
631
632#[derive(Debug, Default)]
633pub(crate) struct ComputedWriteBuffer {
634 writes: Vec<ComputedWrite>,
635 next_seq: u64,
636 estimated_bytes: usize,
637 formats_present: bool,
638}
639
640impl ComputedWriteBuffer {
641 const ENTRY_BASE_BYTES: usize = 32;
642
643 #[inline]
644 pub(crate) fn is_empty(&self) -> bool {
645 self.writes.is_empty()
646 }
647
648 #[inline]
649 pub(crate) fn len(&self) -> usize {
650 self.writes.len()
651 }
652
653 #[inline]
654 pub(crate) fn writes(&self) -> &[ComputedWrite] {
655 &self.writes
656 }
657
658 #[inline]
659 pub(crate) fn estimated_bytes(&self) -> usize {
660 self.estimated_bytes
661 }
662
663 #[cfg(feature = "tracing")]
664 fn traced_cell_count(&self) -> usize {
665 self.writes
666 .iter()
667 .map(|write| match write {
668 ComputedWrite::Cell { .. } => 1,
669 ComputedWrite::Rect { values, .. } => values.iter().map(Vec::len).sum(),
670 ComputedWrite::Run { entries, .. } => entries.len(),
671 })
672 .sum()
673 }
674
675 pub(crate) fn push_cell(
676 &mut self,
677 sheet_id: SheetId,
678 row0: u32,
679 col0: u32,
680 value: OverlayValue,
681 ) {
682 self.push_cell_with_format(sheet_id, row0, col0, value, None);
683 }
684
685 pub(crate) fn push_cell_with_format(
686 &mut self,
687 sheet_id: SheetId,
688 row0: u32,
689 col0: u32,
690 value: OverlayValue,
691 format_id: Option<crate::format::FormatId>,
692 ) {
693 let format_id = format_id.filter(|id| *id != crate::format::FormatId::GENERAL);
694 self.formats_present |= format_id.is_some();
695 let seq = self.next_sequence();
696 self.estimated_bytes = self
697 .estimated_bytes
698 .saturating_add(Self::estimate_value_bytes(&value));
699 self.writes.push(ComputedWrite::Cell {
700 seq,
701 sheet_id,
702 row0,
703 col0,
704 value,
705 format_id,
706 });
707 }
708
709 pub(crate) fn push_column_run(
710 &mut self,
711 sheet_id: SheetId,
712 row0: u32,
713 col0: u32,
714 mut entries: Vec<(OverlayValue, Option<crate::format::FormatId>)>,
715 ) {
716 let seq = self.next_sequence();
717 let mut added = 0usize;
718 for (value, format_id) in entries.iter_mut() {
719 *format_id = format_id.filter(|id| *id != crate::format::FormatId::GENERAL);
720 self.formats_present |= format_id.is_some();
721 added = added.saturating_add(Self::estimate_value_bytes(value));
722 }
723 self.estimated_bytes = self.estimated_bytes.saturating_add(added);
724 self.writes.push(ComputedWrite::Run {
725 seq,
726 sheet_id,
727 row0,
728 col0,
729 entries,
730 });
731 }
732
733 pub(crate) fn push_rect(
734 &mut self,
735 sheet_id: SheetId,
736 sr0: u32,
737 sc0: u32,
738 values: Vec<Vec<OverlayValue>>,
739 ) {
740 let seq = self.next_sequence();
741 let added = values
742 .iter()
743 .flat_map(|row| row.iter())
744 .map(Self::estimate_value_bytes)
745 .fold(0usize, usize::saturating_add);
746 self.estimated_bytes = self.estimated_bytes.saturating_add(added);
747 self.writes.push(ComputedWrite::Rect {
748 seq,
749 sheet_id,
750 sr0,
751 sc0,
752 values,
753 });
754 }
755
756 pub(crate) fn clear(&mut self) {
757 self.writes.clear();
758 self.estimated_bytes = 0;
759 self.formats_present = false;
760 }
761
762 fn take_writes(&mut self) -> (Vec<ComputedWrite>, bool) {
763 self.estimated_bytes = 0;
764 let formats_present = std::mem::take(&mut self.formats_present);
765 (std::mem::take(&mut self.writes), formats_present)
766 }
767
768 fn next_sequence(&mut self) -> u64 {
769 let seq = self.next_seq;
770 self.next_seq = self.next_seq.wrapping_add(1);
771 seq
772 }
773
774 #[inline]
775 fn estimate_value_bytes(value: &OverlayValue) -> usize {
776 Self::ENTRY_BASE_BYTES.saturating_add(value.estimated_payload_bytes())
777 }
778}
779
780#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
781struct ComputedWriteChunkKey {
782 sheet_id: SheetId,
783 col0: u32,
784 chunk_idx: usize,
785 chunk_start_row0: u32,
786}
787
788#[derive(Debug, Clone, PartialEq)]
789pub(crate) struct ComputedWriteChunkEntryPlan {
790 pub(crate) row_in_chunk: usize,
791 pub(crate) seq: u64,
792 pub(crate) value: OverlayValue,
793 pub(crate) format_id: Option<crate::format::FormatId>,
794}
795
796#[derive(Debug, Clone, PartialEq, Eq)]
797pub(crate) enum ComputedWriteChunkPlanShape {
798 Point,
799 SparseOffsets {
800 entries: usize,
801 span_len: usize,
802 },
803 DenseRange {
804 start: usize,
805 len: usize,
806 },
807 RunRange {
808 start: usize,
809 len: usize,
810 runs: usize,
811 },
812}
813
814#[derive(Debug, Clone, PartialEq, Eq)]
815pub(crate) enum ComputedWriteFormatClear {
816 Range { start: usize, end: usize },
817 Offsets(Vec<usize>),
818}
819
820#[derive(Debug, Clone, PartialEq)]
821pub(crate) enum ComputedWriteChunkFormatEffect {
822 NoFormatWork,
823 ClearStale(ComputedWriteFormatClear),
824 SetExplicit(Vec<(usize, Option<crate::format::FormatId>)>),
825}
826
827#[derive(Debug, Clone, PartialEq)]
828pub(crate) struct ComputedWriteChunkPlan {
829 pub(crate) sheet_id: SheetId,
830 pub(crate) col0: u32,
831 pub(crate) chunk_idx: usize,
832 pub(crate) chunk_start_row0: u32,
833 pub(crate) entries: Vec<ComputedWriteChunkEntryPlan>,
834 pub(crate) shape: ComputedWriteChunkPlanShape,
835 pub(crate) format_effect: ComputedWriteChunkFormatEffect,
836}
837
838#[derive(Debug, Clone, Default, PartialEq)]
839pub(crate) struct ComputedWriteCoalescingPlan {
840 pub(crate) chunks: Vec<ComputedWriteChunkPlan>,
841 pub(crate) input_cells: usize,
842 pub(crate) coalesced_cells: usize,
843 pub(crate) overwritten_cells: usize,
844}
845
846impl ComputedWriteCoalescingPlan {
847 #[inline]
848 pub(crate) fn is_empty(&self) -> bool {
849 self.chunks.is_empty()
850 }
851}
852
853impl ComputedWriteChunkPlan {
854 fn from_group(
855 key: ComputedWriteChunkKey,
856 mut entries: Vec<ComputedWriteChunkEntryPlan>,
857 formats_present: bool,
858 computed_lane_has_formats: bool,
859 ) -> (Self, usize) {
860 entries.sort_by_key(|entry| (entry.row_in_chunk, entry.seq));
861 let input_len = entries.len();
862 let mut coalesced: Vec<ComputedWriteChunkEntryPlan> = Vec::with_capacity(input_len);
863 for entry in entries {
864 if let Some(prev) = coalesced.last_mut()
865 && prev.row_in_chunk == entry.row_in_chunk
866 {
867 *prev = entry;
868 continue;
869 }
870 coalesced.push(entry);
871 }
872 let overwritten = input_len.saturating_sub(coalesced.len());
873 let shape = Self::classify_shape(&coalesced);
874 let format_effect = Self::classify_format_effect(
875 &coalesced,
876 &shape,
877 formats_present,
878 computed_lane_has_formats,
879 );
880 (
881 Self {
882 sheet_id: key.sheet_id,
883 col0: key.col0,
884 chunk_idx: key.chunk_idx,
885 chunk_start_row0: key.chunk_start_row0,
886 entries: coalesced,
887 shape,
888 format_effect,
889 },
890 overwritten,
891 )
892 }
893
894 fn classify_format_effect(
895 entries: &[ComputedWriteChunkEntryPlan],
896 shape: &ComputedWriteChunkPlanShape,
897 formats_present: bool,
898 computed_lane_has_formats: bool,
899 ) -> ComputedWriteChunkFormatEffect {
900 if !formats_present {
901 return if computed_lane_has_formats {
902 ComputedWriteChunkFormatEffect::ClearStale(Self::format_clear_spec(entries, shape))
903 } else {
904 ComputedWriteChunkFormatEffect::NoFormatWork
905 };
906 }
907
908 if entries.iter().all(|entry| entry.format_id.is_none()) {
909 return if computed_lane_has_formats {
910 ComputedWriteChunkFormatEffect::ClearStale(Self::format_clear_spec(entries, shape))
911 } else {
912 ComputedWriteChunkFormatEffect::NoFormatWork
913 };
914 }
915
916 ComputedWriteChunkFormatEffect::SetExplicit(
917 entries
918 .iter()
919 .map(|entry| (entry.row_in_chunk, entry.format_id))
920 .collect(),
921 )
922 }
923
924 fn format_clear_spec(
925 entries: &[ComputedWriteChunkEntryPlan],
926 shape: &ComputedWriteChunkPlanShape,
927 ) -> ComputedWriteFormatClear {
928 match shape {
929 ComputedWriteChunkPlanShape::DenseRange { start, len }
930 | ComputedWriteChunkPlanShape::RunRange { start, len, .. } => {
931 ComputedWriteFormatClear::Range {
932 start: *start,
933 end: start.saturating_add(*len),
934 }
935 }
936 ComputedWriteChunkPlanShape::Point
937 | ComputedWriteChunkPlanShape::SparseOffsets { .. } => {
938 ComputedWriteFormatClear::Offsets(
939 entries.iter().map(|entry| entry.row_in_chunk).collect(),
940 )
941 }
942 }
943 }
944
945 fn classify_shape(entries: &[ComputedWriteChunkEntryPlan]) -> ComputedWriteChunkPlanShape {
946 debug_assert!(!entries.is_empty());
947 if entries.len() == 1 {
948 return ComputedWriteChunkPlanShape::Point;
949 }
950
951 let start = entries[0].row_in_chunk;
952 let end = entries[entries.len() - 1].row_in_chunk;
953 let span_len = end.saturating_sub(start).saturating_add(1);
954 if span_len != entries.len() {
955 return ComputedWriteChunkPlanShape::SparseOffsets {
956 entries: entries.len(),
957 span_len,
958 };
959 }
960
961 let runs = Self::run_count(entries);
962 if runs < entries.len() {
963 ComputedWriteChunkPlanShape::RunRange {
964 start,
965 len: entries.len(),
966 runs,
967 }
968 } else {
969 ComputedWriteChunkPlanShape::DenseRange {
970 start,
971 len: entries.len(),
972 }
973 }
974 }
975
976 fn run_count(entries: &[ComputedWriteChunkEntryPlan]) -> usize {
977 let mut runs = 0usize;
978 let mut prev: Option<&OverlayValue> = None;
979 for entry in entries {
980 if prev != Some(&entry.value) {
981 runs = runs.saturating_add(1);
982 prev = Some(&entry.value);
983 }
984 }
985 runs
986 }
987}
988
989#[cfg(feature = "tracing")]
990#[derive(Default)]
991struct TraceEvaluationCounters {
992 computed_vertices: usize,
993 cycles: usize,
994}
995
996pub struct Engine<R> {
997 criteria_mask_cache: criteria_mask_cache::CriteriaMaskCache,
998 pub(crate) graph: DependencyGraph,
999 resolver: R,
1000 pub(super) resolved_functions: crate::interpreter::ResolvedFunctions,
1001 resolved_sheet_names:
1002 FxHashMap<crate::engine::arena::StringId, (crate::SheetId, Option<usize>)>,
1003 resolution_sheet_revision: Option<(u64, u64, usize)>,
1004 resolution_arrow_sheet_names: Vec<Arc<str>>,
1005 resolved_arrow_sheets: Vec<Option<usize>>,
1006 arrow_sheet_index: std::sync::OnceLock<Vec<Option<usize>>>,
1008 pub config: EvalConfig,
1009 workbook_load_limits: crate::engine::WorkbookLoadLimits,
1010 clock: crate::timezone::SnapshotClock,
1015 thread_pool: Option<Arc<rayon::ThreadPool>>,
1016 pub recalc_epoch: u64,
1017 snapshot_id: std::sync::atomic::AtomicU64,
1018 topology_epoch: u64,
1019 cached_static_schedule: Option<CachedScheduleEntry>,
1020 recent_schedules: Vec<CachedScheduleEntry>,
1025 base_schedule: Option<CachedScheduleEntry>,
1030 #[cfg(any(test, feature = "benchmark_internal"))]
1031 recalc_reuse_probe: std::sync::Mutex<RecalcReuseProbe>,
1032 spill_mgr: ShimSpillManager,
1033 arrow_sheets: SheetStore,
1035 format_registry: crate::format::FormatRegistry,
1037 derived_formats: crate::engine::derived_formats::DerivedFormats,
1039 #[cfg(test)]
1040 derived_format_operations_for_test: std::sync::atomic::AtomicU64,
1041 #[cfg(test)]
1042 family_members_for_test: std::sync::atomic::AtomicU64,
1043 #[cfg(test)]
1044 invariant_bound_members_for_test: std::sync::atomic::AtomicU64,
1045 #[cfg(test)]
1046 lifted_members_for_test: std::sync::atomic::AtomicU64,
1047 #[cfg(test)]
1048 chained_members_for_test: std::sync::atomic::AtomicU64,
1049 #[cfg(test)]
1050 lane_clean_reads_for_test: std::sync::atomic::AtomicU64,
1051 #[cfg(test)]
1052 criteria_kernel_members_for_test: std::sync::atomic::AtomicU64,
1053 #[cfg(test)]
1054 memo_hits_for_test: std::sync::atomic::AtomicU64,
1055 compressed_at_build: Option<u64>,
1057 #[cfg(test)]
1058 computed_overlay_set_explicit_entry_operations_for_test: u64,
1059 #[cfg(test)]
1060 computed_overlay_stale_clear_range_effects_for_test: u64,
1061 #[cfg(test)]
1062 computed_overlay_stale_clear_offset_attempts_for_test: u64,
1063 #[cfg(test)]
1064 computed_format_vector_allocations_for_test: std::sync::atomic::AtomicU64,
1065 has_edited: bool,
1067 overlay_compactions: u64,
1069
1070 computed_overlay_bytes_estimate: usize,
1072 computed_overlay_mirroring_disabled: bool,
1073 pub(crate) force_materialize_range_views: bool,
1076 row_bounds_cache: std::sync::RwLock<Option<RowBoundsCache>>,
1078 used_axis_bounds_cache: std::sync::RwLock<Option<UsedAxisBoundsCache>>,
1080 lookup_index_cache: LookupIndexCache,
1081 source_cache: Arc<std::sync::RwLock<SourceCache>>,
1082 source_formula_token: Arc<()>,
1084 pub(crate) source_family_mode: crate::engine::SourceFamilyMode,
1086 pub(crate) source_family_counters: crate::engine::SourceFamilyCounters,
1087 recalc_plan_token: Arc<()>,
1089 staged_formulas: StagedFormulaMap,
1091 staged_formula_index: StagedFormulaIndex,
1093 blocked_pending_spills: Vec<(VertexId, CellRef, Region)>,
1095 spill_blocker_waiters: SpillBlockerWaiters,
1099 row_visibility: FxHashMap<SheetId, RowVisibilityState>,
1101 row_visibility_mask_cache: std::sync::RwLock<
1103 FxHashMap<VisibilityMaskCacheKey, std::sync::Arc<arrow_array::BooleanArray>>,
1104 >,
1105 formula_parse_diagnostics: Vec<FormulaParseDiagnostic>,
1107 last_formula_ingest_report: Option<FormulaIngestReport>,
1109 formula_ingest_report_total: FormulaIngestReport,
1111 active_cancel_flag: Option<crate::engine::CancelToken>,
1113 active_evaluation_deadline: Option<Instant>,
1115
1116 action_depth: u32,
1121
1122 last_virtual_dep_telemetry: VirtualDepTelemetry,
1124 virtual_dep_fallback_activations: u64,
1125
1126 last_cycle_telemetry: CycleTelemetry,
1128
1129 next_evaluation_resource_request_id: u64,
1131 evaluation_resource_request_depth: usize,
1132 active_evaluation_resource_request: Option<EvaluationResourceRequestStats>,
1133 last_evaluation_resource_request: Option<EvaluationResourceRequestStats>,
1134 evaluation_resource_baseline: EvaluationResourceBaselineStats,
1135 evaluation_resource_request_started_at: Option<crate::instant::FzInstant>,
1136 evaluation_resource_budgets: crate::engine::EvaluationBudgets,
1137 evaluation_resource_config_diagnostic:
1138 Option<crate::engine::EvaluationResourceConfigDiagnostic>,
1139 active_resource_ledger: Option<ResourceLedger>,
1140 source_cache_footprints: Vec<std::sync::Weak<std::sync::atomic::AtomicU64>>,
1141 source_cache_accounted: u64,
1142
1143 pending_iterative_redirty: Vec<VertexId>,
1162 snapshot_evaluation_active: bool,
1164 snapshot_volatile_redirty_pending: bool,
1165 retained_scc_members: FxHashMap<VertexId, u64>,
1183 next_retained_scc_id: u64,
1184 retained_scc_config_fingerprint: u64,
1191 retained_scc_function_epoch_seen: u64,
1196 retained_scc_provider_revision_seen: Option<u64>,
1197 retained_scc_dirty_at_begin: Vec<(VertexId, u64)>,
1204
1205 iterative_state_values: FxHashMap<VertexId, LiteralValue>,
1218
1219 function_semantic_epoch_seen: u64,
1221 function_provider_revision_seen: Option<u64>,
1223
1224 #[cfg(feature = "tracing")]
1225 trace_evaluation_counters: TraceEvaluationCounters,
1226 #[cfg(test)]
1227 evaluation_request_begin_count_for_test: u64,
1228 #[cfg(any(test, feature = "test-support"))]
1229 before_prepared_span_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1230 #[cfg(test)]
1231 before_target_preparation_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1232 #[cfg(test)]
1233 before_target_planning_snapshot_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1234 #[cfg(test)]
1235 inject_target_semantic_stale_once_for_test: bool,
1236 #[cfg(test)]
1237 force_virtual_dep_changes_remaining_for_test: usize,
1238 freshness: freshness::Freshness,
1240 #[cfg(test)]
1241 fail_evaluation_commit_preflight_once_for_test: bool,
1242 #[cfg(test)]
1243 target_preparation_fault_for_test:
1244 Option<crate::engine::target_preparation::TargetPreparationFault>,
1245 #[cfg(test)]
1246 force_non_cycle_schedule_fallback_for_test: bool,
1247 #[cfg(test)]
1248 before_legacy_fallback_final_provider_sample_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1249 #[cfg(test)]
1250 after_eager_proposal_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1251}
1252
1253impl<R: EvaluationContext> Engine<R> {
1256 pub(crate) fn ingest_pipeline(&mut self) -> crate::engine::ingest_pipeline::IngestPipeline<'_> {
1257 self.graph.ingest_pipeline(&self.resolver)
1258 }
1259}
1260
1261pub struct EngineAction<'a, R>
1262where
1263 R: EvaluationContext,
1264{
1265 engine: &'a mut Engine<R>,
1266 name: String,
1267 capture: Option<*mut MutationCapture>,
1270 arrow_undo: Option<*mut crate::engine::ArrowUndoBatch>,
1273 atomic_policy: bool,
1275}
1276
1277impl<'a, R> EngineAction<'a, R>
1278where
1279 R: EvaluationContext,
1280{
1281 #[inline]
1282 fn addr_for(&mut self, sheet: &str, row: u32, col: u32) -> crate::reference::CellRef {
1283 let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1284 let coord = crate::reference::Coord::from_excel(row, col, true, true);
1285 crate::reference::CellRef::new(sheet_id, coord)
1286 }
1287
1288 #[inline]
1289 pub fn name(&self) -> &str {
1290 &self.name
1291 }
1292
1293 #[inline]
1294 pub fn set_cell_value(
1295 &mut self,
1296 sheet: &str,
1297 row: u32,
1298 col: u32,
1299 value: LiteralValue,
1300 ) -> Result<(), crate::engine::EditorError> {
1301 if self.capture.is_some() {
1302 let old_value = self.engine.read_cell_value(sheet, row, col);
1303 let mut old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1304 let addr = self.addr_for(sheet, row, col);
1305 let Some(capture_ptr) = self.capture else {
1306 return Err(crate::engine::EditorError::TransactionFailed {
1307 reason: "action_with_logger: missing mutation capture".to_string(),
1308 });
1309 };
1310
1311 let old_comp = if self.arrow_undo.is_some() {
1314 self.engine.read_computed_overlay_cell(sheet, row, col)
1315 } else {
1316 None
1317 };
1318
1319 if self.engine.graph_admission_enabled() {
1320 let admission =
1321 self.engine
1322 .graph
1323 .preview_value_mutation(addr.sheet_id, row, col)?;
1324 self.engine.preflight_graph_admission(admission)?;
1325 }
1326 if old_formula.is_none() {
1327 old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1328 }
1329
1330 let delta_old_sem = if old_formula.is_some() {
1331 None
1332 } else {
1333 Some(old_value.clone().unwrap_or(LiteralValue::Empty))
1334 };
1335
1336 let start_len = unsafe { (&*capture_ptr).len() };
1337
1338 let capture = unsafe { &mut *capture_ptr };
1340 self.engine.edit_with_capture(capture, |editor| {
1341 editor.set_cell_value_with_old_state(
1342 addr,
1343 value.clone(),
1344 old_value.clone(),
1345 old_formula.clone(),
1346 );
1347 })?;
1348 self.engine.record_structural_change(StructuralScope::Cell {
1349 sheet: addr.sheet_id,
1350 row: addr.coord.row(),
1351 col: addr.coord.col(),
1352 });
1353
1354 if let Some(undo_ptr) = self.arrow_undo {
1355 let new_events = &unsafe { (&*capture_ptr).events() }[start_len..];
1357 let undo = unsafe { &mut *undo_ptr };
1358 self.engine
1359 .record_spill_ops_into_arrow_undo(undo, new_events);
1360
1361 let new_comp = self.engine.read_computed_overlay_cell(sheet, row, col);
1363 let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1364 let row0 = row.saturating_sub(1);
1365 let col0 = col.saturating_sub(1);
1366 let delta_new_sem = Some(value.clone());
1367 undo.record_delta_cell(sheet_id, row0, col0, delta_old_sem, delta_new_sem);
1368 undo.record_computed_cell(sheet_id, row0, col0, old_comp, new_comp);
1369 }
1370 Ok(())
1371 } else {
1372 self.engine
1373 .set_cell_value(sheet, row, col, value)
1374 .map_err(crate::engine::EditorError::from)
1375 }
1376 }
1377
1378 #[inline]
1379 pub fn set_cell_formula(
1380 &mut self,
1381 sheet: &str,
1382 row: u32,
1383 col: u32,
1384 ast: ASTNode,
1385 ) -> Result<(), crate::engine::EditorError> {
1386 if self.capture.is_some() {
1387 let old_value = self.engine.read_cell_value(sheet, row, col);
1388 let mut old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1389 let addr = self.addr_for(sheet, row, col);
1390 let Some(capture_ptr) = self.capture else {
1391 return Err(crate::engine::EditorError::TransactionFailed {
1392 reason: "action_with_logger: missing mutation capture".to_string(),
1393 });
1394 };
1395
1396 let admitted_formula = if self.engine.graph_admission_enabled() {
1397 let placement =
1398 CellRef::new(addr.sheet_id, Coord::from_excel(row, col, true, true));
1399 let ingested = self.engine.ingest_pipeline().ingest_formula(
1400 FormulaAstInput::Tree(ast.clone()),
1401 placement,
1402 None,
1403 )?;
1404 let admission = self.engine.graph.preview_formula_mutations(&[(
1405 addr.sheet_id,
1406 row,
1407 col,
1408 ingested.dep_plan.clone(),
1409 )])?;
1410 self.engine.preflight_graph_admission(admission)?;
1411 Some((ingested.ast_id, ingested.dep_plan))
1412 } else {
1413 None
1414 };
1415 if old_formula.is_none() {
1416 old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1417 }
1418 let delta_old = if self.arrow_undo.is_some() {
1419 if old_formula.is_some() {
1420 None
1421 } else {
1422 Some(old_value.clone().unwrap_or(LiteralValue::Empty))
1423 }
1424 } else {
1425 None
1426 };
1427 let start_len = unsafe { (&*capture_ptr).len() };
1428
1429 let capture = unsafe { &mut *capture_ptr };
1431 self.engine.edit_with_capture(capture, |editor| {
1432 if let Some((ast_id, plan)) = admitted_formula {
1433 editor.set_cell_formula_with_prepared_plan(
1434 addr,
1435 ast.clone(),
1436 old_value,
1437 old_formula,
1438 ast_id,
1439 plan,
1440 );
1441 } else {
1442 editor.set_cell_formula_with_old_state(
1443 addr,
1444 ast.clone(),
1445 old_value,
1446 old_formula,
1447 );
1448 }
1449 })?;
1450 self.engine.record_structural_change(StructuralScope::Cell {
1451 sheet: addr.sheet_id,
1452 row: addr.coord.row(),
1453 col: addr.coord.col(),
1454 });
1455
1456 if let Some(undo_ptr) = self.arrow_undo {
1457 let new_events = &unsafe { (&*capture_ptr).events() }[start_len..];
1458 let undo = unsafe { &mut *undo_ptr };
1459 self.engine
1460 .record_spill_ops_into_arrow_undo(undo, new_events);
1461 let delta_new: Option<LiteralValue> = None;
1462 let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1463 let row0 = row.saturating_sub(1);
1464 let col0 = col.saturating_sub(1);
1465 undo.record_delta_cell(sheet_id, row0, col0, delta_old, delta_new);
1466 }
1467 Ok(())
1468 } else {
1469 self.engine
1470 .set_cell_formula(sheet, row, col, ast)
1471 .map_err(crate::engine::EditorError::from)
1472 }
1473 }
1474
1475 #[inline]
1476 pub fn set_row_hidden(
1477 &mut self,
1478 sheet: &str,
1479 row_1based: u32,
1480 hidden: bool,
1481 source: RowVisibilitySource,
1482 ) -> Result<(), crate::engine::EditorError> {
1483 if self.capture.is_some() {
1484 let sheet_id = self.engine.ensure_known_sheet_id(sheet)?;
1485 let row0 = Engine::<R>::normalize_row_1based(row_1based)?;
1486 let old_hidden = self
1487 .engine
1488 .row_visibility
1489 .get(&sheet_id)
1490 .map(|state| state.is_row_hidden(row0, Some(source)))
1491 .unwrap_or(false);
1492 if old_hidden == hidden {
1493 return Ok(());
1494 }
1495
1496 let _ = self
1497 .engine
1498 .set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source);
1499
1500 let Some(capture_ptr) = self.capture else {
1501 return Err(crate::engine::EditorError::TransactionFailed {
1502 reason: "action_with_logger: missing mutation capture".to_string(),
1503 });
1504 };
1505 unsafe { &mut *capture_ptr }.record(crate::engine::ChangeEvent::SetRowVisibility {
1506 sheet_id,
1507 row0,
1508 source,
1509 old_hidden,
1510 new_hidden: hidden,
1511 });
1512
1513 Ok(())
1514 } else {
1515 self.engine
1516 .set_row_hidden(sheet, row_1based, hidden, source)
1517 }
1518 }
1519
1520 #[inline]
1521 pub fn set_rows_hidden(
1522 &mut self,
1523 sheet: &str,
1524 start_row_1based: u32,
1525 end_row_1based: u32,
1526 hidden: bool,
1527 source: RowVisibilitySource,
1528 ) -> Result<(), crate::engine::EditorError> {
1529 if self.capture.is_some() {
1530 let sheet_id = self.engine.ensure_known_sheet_id(sheet)?;
1531 let (start_row0, end_row0) =
1532 Engine::<R>::normalize_row_range_1based(start_row_1based, end_row_1based)?;
1533
1534 let Some(capture_ptr) = self.capture else {
1535 return Err(crate::engine::EditorError::TransactionFailed {
1536 reason: "action_with_logger: missing mutation capture".to_string(),
1537 });
1538 };
1539 let capture = unsafe { &mut *capture_ptr };
1540
1541 for row0 in start_row0..=end_row0 {
1542 let old_hidden = self
1543 .engine
1544 .row_visibility
1545 .get(&sheet_id)
1546 .map(|state| state.is_row_hidden(row0, Some(source)))
1547 .unwrap_or(false);
1548 if old_hidden == hidden {
1549 continue;
1550 }
1551
1552 let _ = self
1553 .engine
1554 .set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source);
1555
1556 capture.record(crate::engine::ChangeEvent::SetRowVisibility {
1557 sheet_id,
1558 row0,
1559 source,
1560 old_hidden,
1561 new_hidden: hidden,
1562 });
1563 }
1564
1565 Ok(())
1566 } else {
1567 self.engine
1568 .set_rows_hidden(sheet, start_row_1based, end_row_1based, hidden, source)
1569 }
1570 }
1571
1572 #[inline]
1573 pub fn insert_rows(
1574 &mut self,
1575 sheet: &str,
1576 before: u32,
1577 count: u32,
1578 ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1579 if count == 0 {
1580 return Ok(crate::engine::ShiftSummary::default());
1581 }
1582 if self.capture.is_some() {
1583 let Some(capture_ptr) = self.capture else {
1584 return Err(crate::engine::EditorError::TransactionFailed {
1585 reason: "action_atomic: missing mutation capture".to_string(),
1586 });
1587 };
1588
1589 let before0 = before.saturating_sub(1);
1590 self.engine.check_insert_room(sheet, before0, count, true)?;
1591 let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1592 let occupancy = self.engine.structural_row_occupancy(sheet, sheet_id);
1593 let affected_region = Engine::<R>::structural_row_region(sheet_id, before0);
1594
1595 let summary = {
1597 let capture = unsafe { &mut *capture_ptr };
1598 let mut out: Result<crate::engine::ShiftSummary, crate::engine::EditorError> =
1599 Ok(crate::engine::ShiftSummary::default());
1600 self.engine.edit_with_capture(capture, |editor| {
1601 editor.set_structural_occupancy(occupancy);
1602 out = editor.insert_rows(sheet_id, before0, count);
1603 })?;
1604 out?
1605 };
1606
1607 self.engine.ensure_arrow_sheet(sheet);
1609 if let Some(asheet) = self.engine.arrow_sheets.sheet_mut(sheet) {
1610 asheet.insert_rows(before0 as usize, count as usize);
1611 }
1612 self.engine
1613 .purge_derived_formats_after_row(sheet_id, before0);
1614 self.engine
1615 .shift_row_visibility_insert(sheet_id, before0, count);
1616 self.engine.mark_moved_formula_vertices_dirty(&summary);
1617 self.engine
1618 .clear_computed_overlay_after_row(sheet, before0 as usize);
1619 self.engine
1620 .record_structural_change(StructuralScope::Region(affected_region));
1621 if let Some(undo_ptr) = self.arrow_undo {
1622 unsafe { &mut *undo_ptr }.record_insert_rows(sheet_id, before0, count);
1623 }
1624 Ok(summary)
1625 } else {
1626 self.engine.insert_rows(sheet, before, count)
1627 }
1628 }
1629
1630 #[inline]
1631 pub fn delete_rows(
1632 &mut self,
1633 sheet: &str,
1634 start: u32,
1635 count: u32,
1636 ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1637 if count == 0 {
1638 return Ok(crate::engine::ShiftSummary::default());
1639 }
1640 if self.atomic_policy {
1641 return Err(crate::engine::EditorError::TransactionUnsupported {
1642 reason:
1643 "delete_rows is not supported inside atomic actions (conservative rollback policy)"
1644 .to_string(),
1645 });
1646 }
1647 self.engine.delete_rows(sheet, start, count)
1648 }
1649
1650 #[inline]
1651 pub fn insert_columns(
1652 &mut self,
1653 sheet: &str,
1654 before: u32,
1655 count: u32,
1656 ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1657 if count == 0 {
1658 return Ok(crate::engine::ShiftSummary::default());
1659 }
1660 if self.capture.is_some() {
1661 let Some(capture_ptr) = self.capture else {
1662 return Err(crate::engine::EditorError::TransactionFailed {
1663 reason: "action_atomic: missing mutation capture".to_string(),
1664 });
1665 };
1666
1667 let before0 = before.saturating_sub(1);
1668 self.engine
1669 .check_insert_room(sheet, before0, count, false)?;
1670 let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1671 let occupancy = self.engine.structural_column_occupancy();
1672 let affected_region = Engine::<R>::structural_col_region(sheet_id, before0);
1673
1674 let summary = {
1675 let capture = unsafe { &mut *capture_ptr };
1676 let mut out: Result<crate::engine::ShiftSummary, crate::engine::EditorError> =
1677 Ok(crate::engine::ShiftSummary::default());
1678 self.engine.edit_with_capture(capture, |editor| {
1679 editor.set_structural_occupancy(occupancy);
1680 out = editor.insert_columns(sheet_id, before0, count);
1681 })?;
1682 out?
1683 };
1684
1685 self.engine.ensure_arrow_sheet(sheet);
1686 if let Some(asheet) = self.engine.arrow_sheets.sheet_mut(sheet) {
1687 asheet.insert_columns(before0 as usize, count as usize);
1688 }
1689 self.engine
1690 .purge_derived_formats_after_col(sheet_id, before0);
1691 self.engine.mark_moved_formula_vertices_dirty(&summary);
1692 self.engine
1693 .clear_computed_overlay_after_col(sheet, before0 as usize);
1694 self.engine
1695 .record_structural_change(StructuralScope::Region(affected_region));
1696 if let Some(undo_ptr) = self.arrow_undo {
1697 unsafe { &mut *undo_ptr }.record_insert_cols(sheet_id, before0, count);
1698 }
1699 Ok(summary)
1700 } else {
1701 self.engine.insert_columns(sheet, before, count)
1702 }
1703 }
1704
1705 #[inline]
1706 pub fn delete_columns(
1707 &mut self,
1708 sheet: &str,
1709 start: u32,
1710 count: u32,
1711 ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1712 if count == 0 {
1713 return Ok(crate::engine::ShiftSummary::default());
1714 }
1715 if self.atomic_policy {
1716 return Err(crate::engine::EditorError::TransactionUnsupported {
1717 reason:
1718 "delete_columns is not supported inside atomic actions (conservative rollback policy)"
1719 .to_string(),
1720 });
1721 }
1722 self.engine.delete_columns(sheet, start, count)
1723 }
1724
1725 #[inline]
1730 pub fn action<T>(
1731 &mut self,
1732 name: impl AsRef<str>,
1733 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
1734 ) -> Result<T, crate::engine::EditorError> {
1735 self.engine.action(name, f)
1736 }
1737}
1738
1739struct ActionDepthGuard<'a, R> {
1740 engine: *mut Engine<R>,
1741 _marker: std::marker::PhantomData<&'a mut Engine<R>>,
1742}
1743
1744impl<'a, R> Drop for ActionDepthGuard<'a, R> {
1745 fn drop(&mut self) {
1746 unsafe {
1749 let e = &mut *self.engine;
1750 e.action_depth = e.action_depth.saturating_sub(1);
1751 }
1752 }
1753}
1754
1755#[derive(Default)]
1756struct SourceCache {
1757 scalars: FxHashMap<(String, Option<u64>), LiteralValue>,
1758 tables: FxHashMap<(String, Option<u64>), Arc<dyn crate::traits::Table>>,
1759 external_views: FxHashMap<(String, Option<u64>), RangeView<'static>>,
1762}
1763
1764#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1765struct VisibilityMaskCacheKey {
1766 sheet_id: SheetId,
1767 start_row0: u32,
1768 end_row0: u32,
1769 mode: VisibilityMaskMode,
1770 version: u64,
1771}
1772
1773#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1774enum StructuralScope {
1775 Cell { sheet: SheetId, row: u32, col: u32 },
1776 Region(Region),
1777 Sheet(SheetId),
1778 RemovedSheet(SheetId),
1779 OpaqueGlobal,
1780 AllSheets,
1781}
1782
1783#[derive(Default)]
1786struct SpillBlockerWaiters {
1787 by_cell: FxHashMap<CellRef, Vec<VertexId>>,
1788 by_anchor: FxHashMap<VertexId, CellRef>,
1789}
1790
1791impl SpillBlockerWaiters {
1792 const REGION_PROBE_CELLS: u64 = 4096;
1794
1795 fn is_empty(&self) -> bool {
1796 self.by_anchor.is_empty()
1797 }
1798
1799 fn wait(&mut self, anchor: VertexId, blocker: CellRef) {
1800 match self.by_anchor.insert(anchor, blocker) {
1801 Some(old) if old == blocker => return,
1802 Some(old) => self.unlink(anchor, old),
1803 None => {}
1804 }
1805 self.by_cell.entry(blocker).or_default().push(anchor);
1806 }
1807
1808 fn forget(&mut self, anchor: VertexId) {
1809 if self.by_anchor.is_empty() {
1810 return;
1811 }
1812 if let Some(old) = self.by_anchor.remove(&anchor) {
1813 self.unlink(anchor, old);
1814 }
1815 }
1816
1817 fn unlink(&mut self, anchor: VertexId, cell: CellRef) {
1818 if let Some(waiters) = self.by_cell.get_mut(&cell) {
1819 waiters.retain(|&v| v != anchor);
1820 if waiters.is_empty() {
1821 self.by_cell.remove(&cell);
1822 }
1823 }
1824 }
1825
1826 fn take_affected(&mut self, scope: StructuralScope) -> Vec<VertexId> {
1828 let cells: Vec<CellRef> = match scope {
1829 StructuralScope::Cell { sheet, row, col } => {
1830 match Coord::try_new(row, col, true, true) {
1831 Ok(coord) if self.by_cell.contains_key(&CellRef::new(sheet, coord)) => {
1832 vec![CellRef::new(sheet, coord)]
1833 }
1834 _ => Vec::new(),
1835 }
1836 }
1837 StructuralScope::Region(region) => {
1838 let (rows, cols) = region.axis_ranges();
1839 let (r0, r1) = rows.query_bounds();
1840 let (c0, c1) = cols.query_bounds();
1841 let area = (u64::from(r1.saturating_sub(r0)) + 1)
1845 .saturating_mul(u64::from(c1.saturating_sub(c0)) + 1);
1846 if area <= Self::REGION_PROBE_CELLS {
1847 let sheet = region.sheet_id();
1848 (r0..=r1)
1851 .flat_map(|r| (c0..=c1).map(move |c| (r, c)))
1852 .filter_map(|(r, c)| Coord::try_new(r, c, true, true).ok())
1853 .map(|coord| CellRef::new(sheet, coord))
1854 .filter(|cell| self.by_cell.contains_key(cell))
1855 .collect()
1856 } else {
1857 self.by_cell
1858 .keys()
1859 .filter(|cell| {
1860 region.intersects(&Region::point(
1861 cell.sheet_id,
1862 cell.coord.row(),
1863 cell.coord.col(),
1864 ))
1865 })
1866 .copied()
1867 .collect()
1868 }
1869 }
1870 StructuralScope::Sheet(sheet) | StructuralScope::RemovedSheet(sheet) => self
1871 .by_cell
1872 .keys()
1873 .filter(|cell| cell.sheet_id == sheet)
1874 .copied()
1875 .collect(),
1876 StructuralScope::OpaqueGlobal | StructuralScope::AllSheets => {
1877 self.by_cell.keys().copied().collect()
1878 }
1879 };
1880 let mut woken = Vec::new();
1881 for cell in cells {
1882 if let Some(waiters) = self.by_cell.remove(&cell) {
1883 for anchor in waiters {
1884 self.by_anchor.remove(&anchor);
1885 woken.push(anchor);
1886 }
1887 }
1888 }
1889 woken
1890 }
1891}
1892
1893#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
1894enum LoggedEditImpact {
1895 NoOp,
1896 DataOnly,
1897 Topology,
1898}
1899
1900#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1901enum LoggedEditDirection {
1902 Original,
1903 InverseReplay,
1904 ForwardReplay,
1905}
1906
1907#[derive(Clone, Copy)]
1908struct InvalidationBaseline {
1909 snapshot_id: u64,
1910 topology_epoch: u64,
1911}
1912
1913struct SourceCacheSession {
1914 cache: Arc<std::sync::RwLock<SourceCache>>,
1915}
1916
1917impl Drop for SourceCacheSession {
1918 fn drop(&mut self) {
1919 if let Ok(mut g) = self.cache.write() {
1920 *g = SourceCache::default();
1921 }
1922 }
1923}
1924
1925#[derive(Debug)]
1926#[non_exhaustive]
1927pub struct EvalResult {
1928 pub computed_vertices: usize,
1929 pub cycle_errors: usize,
1930 pub elapsed: std::time::Duration,
1931}
1932
1933#[derive(Clone, Debug, PartialEq, Eq)]
1934#[non_exhaustive]
1935pub struct TableMetadata {
1936 pub name: String,
1937 pub sheet: String,
1938 pub start_row: u32,
1939 pub start_col: u32,
1940 pub end_row: u32,
1941 pub end_col: u32,
1942 pub header_row: bool,
1943 pub headers: Vec<String>,
1944 pub totals_row: bool,
1945}
1946
1947#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
1955#[non_exhaustive]
1956pub struct EngineBaselineStats {
1957 pub graph_vertex_count: usize,
1958 pub graph_formula_vertex_count: usize,
1959 pub graph_edge_count: usize,
1960 pub dirty_vertex_count: usize,
1961 pub evaluation_vertex_count: usize,
1962 pub formula_ast_root_count: usize,
1963 pub formula_ast_node_count: usize,
1964 pub staged_formula_count: usize,
1965 pub formula_plane_active_span_count: usize,
1966 pub formula_plane_producer_result_entries: usize,
1967 pub formula_plane_consumer_read_entries: usize,
1968 pub formula_plane_mixed_topology_cache_builds: u64,
1969 pub formula_plane_mixed_topology_cache_hits: u64,
1970 pub formula_plane_mixed_topology_cache_overflows: u64,
1971 pub formula_plane_dirty_pending_events: usize,
1972 pub formula_plane_dirty_region_events_recorded: u64,
1973 pub formula_plane_dirty_span_region_events_recorded: u64,
1974 pub formula_plane_dirty_whole_span_seeds_recorded: u64,
1975 pub formula_plane_dirty_global_invalidations: u64,
1976 pub formula_plane_structural_span_candidates: u64,
1977 pub formula_plane_cycle_member_span_demotions: u64,
1978 pub formula_plane_array_result_span_demotions: u64,
1979 pub retained_scc_members: usize,
1982}
1983
1984#[derive(Debug, Clone, Default)]
1985#[non_exhaustive]
1986pub struct VirtualDepTelemetry {
1987 pub candidate_vertices_total: usize,
1988 pub vdeps_vertices_total: usize,
1989 pub vdeps_edges_total: usize,
1990 pub builder_elapsed_ms_total: u128,
1991 pub schedule_virtual_passes: usize,
1992 pub schedule_static_passes: usize,
1993 pub schedule_cache_hits: usize,
1994 pub schedule_cache_misses: usize,
1995 pub reused_schedule_vertices_total: usize,
1996 pub replan_iterations: usize,
1997 pub changed_vdeps_total: usize,
1998 pub bailout_reason: Option<&'static str>,
1999 pub fallback_mode_activations: u64,
2000}
2001
2002#[derive(Debug, Clone, Default, PartialEq)]
2011#[non_exhaustive]
2012pub struct CycleTelemetry {
2013 pub static_sccs: usize,
2015 pub phantom_sccs: usize,
2017 pub live_cycles_witnessed: usize,
2019 pub circ_cells_stamped: usize,
2021 pub settle_passes_total: usize,
2024 pub max_passes_single_scc: usize,
2026 pub iterated_sccs: usize,
2029 pub converged_sccs: usize,
2032 pub capped_sccs: usize,
2038 pub max_abs_delta_at_stop: f64,
2042 pub nan_converged: usize,
2045 pub reused_sccs: usize,
2050 pub reused_scc_members: usize,
2052 pub elapsed_ms: u128,
2054}
2055
2056#[derive(Debug, Clone, Copy)]
2057struct ScheduleBuildMeta {
2058 candidate_vertices: usize,
2059 vdeps_vertices: usize,
2060 vdeps_edges: usize,
2061 builder_elapsed_ms: u128,
2062 used_virtual_schedule: bool,
2063 schedule_cache_hit: bool,
2064 schedule_cache_eligible: bool,
2065}
2066
2067#[cfg(any(test, feature = "benchmark_internal"))]
2068#[doc(hidden)]
2069#[derive(Debug, Clone, Default)]
2070pub struct RecalcReuseProbe {
2071 pub schedule_requests: usize,
2072 pub schedule_cache_hits: usize,
2073 pub schedule_cache_misses: usize,
2074 pub schedule_cache_ineligible: usize,
2075 pub schedule_builds: usize,
2076 pub schedule_base_restrictions: usize,
2078 pub schedule_shared_handles: usize,
2079 pub schedule_retained_bytes: usize,
2080 pub legacy_target_requests: usize,
2081 pub target_schedule_builds: usize,
2082 pub demand_builds: usize,
2083 pub demand_vertices: usize,
2084 pub demand_clean_formulas: usize,
2085 pub demand_explicit_edges: usize,
2086 pub demand_virtual_builder_calls: usize,
2087}
2088
2089#[cfg(any(test, feature = "benchmark_internal"))]
2090fn schedule_probe_retained_bytes(schedule: &crate::engine::Schedule) -> usize {
2091 fn vector_bytes<T>(values: &Vec<T>) -> usize {
2092 values.capacity() * std::mem::size_of::<T>()
2093 }
2094
2095 [
2096 vector_bytes(&schedule.units),
2097 vector_bytes(&schedule.layers),
2098 vector_bytes(&schedule.cycles),
2099 ]
2100 .into_iter()
2101 .chain(
2102 schedule
2103 .layers
2104 .iter()
2105 .map(|layer| vector_bytes(&layer.vertices)),
2106 )
2107 .chain(schedule.cycles.iter().map(vector_bytes))
2108 .sum()
2109}
2110
2111#[derive(Debug, Clone)]
2112struct CachedScheduleEntry {
2113 topology_epoch: u64,
2114 authority_revision: (u64, u64),
2117 candidate_vertices: VertexIdRuns,
2120 schedule: Arc<crate::engine::scheduler::Schedule>,
2121}
2122
2123#[derive(Debug, Clone, Default)]
2126struct VertexIdRuns(Vec<(u32, u32)>);
2127
2128impl VertexIdRuns {
2129 fn from_slice(ids: &[VertexId]) -> Self {
2130 let mut runs: Vec<(u32, u32)> = Vec::new();
2131 for v in ids {
2132 match runs.last_mut() {
2133 Some((first, len)) if first.checked_add(*len) == Some(v.0) => *len += 1,
2134 _ => runs.push((v.0, 1)),
2135 }
2136 }
2137 runs.shrink_to_fit();
2138 Self(runs)
2139 }
2140
2141 fn equals(&self, ids: &[VertexId]) -> bool {
2142 let mut rest = ids;
2143 for &(first, len) in &self.0 {
2144 let len = len as usize;
2145 if rest.len() < len {
2146 return false;
2147 }
2148 let (head, tail) = rest.split_at(len);
2149 if head
2150 .iter()
2151 .enumerate()
2152 .any(|(i, v)| v.0 != first.wrapping_add(i as u32))
2153 {
2154 return false;
2155 }
2156 rest = tail;
2157 }
2158 rest.is_empty()
2159 }
2160
2161 fn heap_bytes(&self) -> usize {
2162 self.0.capacity() * std::mem::size_of::<(u32, u32)>()
2163 }
2164
2165 fn len(&self) -> usize {
2167 self.0.iter().map(|&(_, len)| len as usize).sum()
2168 }
2169}
2170
2171#[cfg(test)]
2172mod vertex_id_runs_tests {
2173 use super::{VertexId, VertexIdRuns};
2174
2175 #[test]
2176 fn runs_compare_like_the_list() {
2177 let ids = |v: &[u32]| v.iter().map(|&i| VertexId(i)).collect::<Vec<_>>();
2178 let list = ids(&[5, 6, 7, 2, 3, 9, 10, 10]);
2179 let runs = VertexIdRuns::from_slice(&list);
2180 assert_eq!(runs.0, vec![(5, 3), (2, 2), (9, 2), (10, 1)]);
2181 assert!(runs.equals(&list));
2182 assert!(!runs.equals(&list[..7]));
2183 assert!(!runs.equals(&ids(&[5, 6, 7, 2, 3, 9, 10, 11])));
2184 assert!(!runs.equals(&ids(&[5, 6, 7, 2, 3, 9, 10, 10, 11])));
2185 assert!(VertexIdRuns::from_slice(&[]).equals(&[]));
2186 assert!(!VertexIdRuns::from_slice(&[]).equals(&list));
2187 let edge = ids(&[u32::MAX - 1, u32::MAX, 0]);
2188 assert!(VertexIdRuns::from_slice(&edge).equals(&edge));
2189 }
2190}
2191
2192enum EvaluationSchedule {
2194 Owned(crate::engine::scheduler::Schedule),
2195 Shared(Arc<crate::engine::scheduler::Schedule>),
2196}
2197
2198impl std::ops::Deref for EvaluationSchedule {
2199 type Target = crate::engine::scheduler::Schedule;
2200
2201 fn deref(&self) -> &Self::Target {
2202 match self {
2203 Self::Owned(schedule) => schedule,
2204 Self::Shared(schedule) => schedule,
2205 }
2206 }
2207}
2208
2209type ScheduleBuildOutput = (
2210 crate::engine::scheduler::Schedule,
2211 FxHashMap<VertexId, Vec<VertexId>>,
2212 ScheduleBuildMeta,
2213);
2214
2215type EvaluationScheduleBuildOutput = (
2216 EvaluationSchedule,
2217 FxHashMap<VertexId, Vec<VertexId>>,
2218 ScheduleBuildMeta,
2219);
2220
2221#[derive(Debug)]
2223pub struct RecalcPlan {
2224 key: RecalcPlanKey,
2225 kind: RecalcPlanKind,
2226}
2227
2228#[derive(Debug)]
2229struct RecalcPlanKey {
2230 engine_token: Arc<()>,
2231 revisions: PlanningRevisionSnapshot,
2232}
2233
2234#[derive(Clone, Debug, PartialEq, Eq)]
2235struct PlanningRevisionSnapshot {
2236 engine_topology_epoch: u64,
2237 graph_topology_revision: u64,
2238 staged: u64,
2239 symbols: u64,
2240 semantic: u64,
2241 provider: Option<u64>,
2242 deterministic_mode: crate::engine::DeterministicMode,
2243 budgets: crate::engine::EvaluationBudgets,
2244}
2245
2246#[derive(Debug)]
2247enum RecalcPlanKind {
2248 CompatibilityFull {
2249 schedule: crate::engine::Schedule,
2250 has_dynamic_refs: bool,
2251 },
2252 Target {
2253 targets: Vec<crate::engine::EvaluationTarget>,
2254 scope: crate::engine::PrepareScope,
2255 topology: RecalcTopology,
2256 dynamic_policy: DynamicPlanPolicy,
2257 },
2258}
2259
2260#[derive(Debug)]
2261enum RecalcTopology {
2262 RunLocalRecipe,
2263 Workbook,
2264}
2265
2266#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2267enum DynamicPlanPolicy {
2268 BoundedTargetReplan,
2269}
2270
2271impl RecalcPlan {
2272 pub fn layer_count(&self) -> usize {
2275 match &self.kind {
2276 RecalcPlanKind::CompatibilityFull { schedule, .. } => schedule.layers.len(),
2277 RecalcPlanKind::Target { .. } => 0,
2278 }
2279 }
2280
2281 pub fn has_dynamic_refs(&self) -> bool {
2282 match &self.kind {
2283 RecalcPlanKind::CompatibilityFull {
2284 has_dynamic_refs, ..
2285 } => *has_dynamic_refs,
2286 RecalcPlanKind::Target { .. } => false,
2287 }
2288 }
2289
2290 #[cfg(test)]
2291 pub(crate) fn force_stale_reasons_for_test(
2292 &mut self,
2293 reasons: &[formualizer_common::PlanStaleReason],
2294 ) {
2295 use formualizer_common::PlanStaleReason;
2296 for reason in reasons {
2297 match reason {
2298 PlanStaleReason::Engine => {
2299 self.key.engine_token = Arc::new(());
2300 }
2301 PlanStaleReason::Provider => {
2302 self.key.revisions.provider = Some(
2303 self.key
2304 .revisions
2305 .provider
2306 .unwrap_or_default()
2307 .wrapping_add(1),
2308 );
2309 }
2310 PlanStaleReason::Semantic => {
2311 self.key.revisions.semantic = self.key.revisions.semantic.wrapping_add(1);
2312 }
2313 PlanStaleReason::Budget => {
2314 let current = self.key.revisions.budgets.work.max_work_units;
2315 self.key.revisions.budgets.work.max_work_units =
2316 Some(current.unwrap_or_default().wrapping_add(1));
2317 }
2318 PlanStaleReason::Staged => {
2319 self.key.revisions.staged = self.key.revisions.staged.wrapping_add(1);
2320 }
2321 PlanStaleReason::Symbols => {
2322 self.key.revisions.symbols = self.key.revisions.symbols.wrapping_add(1);
2323 }
2324 PlanStaleReason::Graph => {
2325 self.key.revisions.graph_topology_revision =
2326 self.key.revisions.graph_topology_revision.wrapping_add(1);
2327 }
2328 _ => {}
2329 }
2330 }
2331 }
2332}
2333
2334#[cfg(any(test, feature = "test-support"))]
2335pub(crate) mod criteria_mask_test_hooks {
2336 use std::cell::Cell;
2337
2338 thread_local! {
2339 static MASK_CALLS_ROWS: Cell<(usize, usize)> = const { Cell::new((0, 0)) };
2340 static TEXT_SEGMENTS_TOTAL: Cell<usize> = const { Cell::new(0) };
2341 static TEXT_SEGMENTS_ALL_NULL: Cell<usize> = const { Cell::new(0) };
2342 }
2343
2344 pub(crate) fn take_mask_work() -> (usize, usize) {
2345 MASK_CALLS_ROWS.with(|c| c.replace((0, 0)))
2346 }
2347
2348 pub(crate) fn note_mask(rows: usize) {
2349 MASK_CALLS_ROWS.with(|c| {
2350 let (calls, work) = c.get();
2351 c.set((calls + 1, work + rows));
2352 });
2353 }
2354
2355 pub fn reset_text_segment_counters() {
2356 TEXT_SEGMENTS_TOTAL.with(|c| c.set(0));
2357 TEXT_SEGMENTS_ALL_NULL.with(|c| c.set(0));
2358 }
2359
2360 pub fn text_segment_counters() -> (usize, usize) {
2361 let a = TEXT_SEGMENTS_TOTAL.with(|c| c.get());
2362 let b = TEXT_SEGMENTS_ALL_NULL.with(|c| c.get());
2363 (a, b)
2364 }
2365
2366 pub(crate) fn inc_total() {
2367 TEXT_SEGMENTS_TOTAL.with(|c| c.set(c.get() + 1));
2368 }
2369 pub(crate) fn inc_all_null() {
2370 TEXT_SEGMENTS_ALL_NULL.with(|c| c.set(c.get() + 1));
2371 }
2372}
2373
2374#[cfg(test)]
2375pub(crate) mod visibility_mask_test_hooks {
2376 use std::cell::Cell;
2377
2378 thread_local! {
2379 static HITS: Cell<usize> = const { Cell::new(0) };
2380 static MISSES: Cell<usize> = const { Cell::new(0) };
2381 static EVICTIONS: Cell<usize> = const { Cell::new(0) };
2382 }
2383
2384 pub fn reset() {
2385 HITS.with(|c| c.set(0));
2386 MISSES.with(|c| c.set(0));
2387 EVICTIONS.with(|c| c.set(0));
2388 }
2389
2390 pub fn counters() -> (usize, usize, usize) {
2391 let hits = HITS.with(|c| c.get());
2392 let misses = MISSES.with(|c| c.get());
2393 let evictions = EVICTIONS.with(|c| c.get());
2394 (hits, misses, evictions)
2395 }
2396
2397 pub(crate) fn inc_hit() {
2398 HITS.with(|c| c.set(c.get() + 1));
2399 }
2400
2401 pub(crate) fn inc_miss() {
2402 MISSES.with(|c| c.set(c.get() + 1));
2403 }
2404
2405 pub(crate) fn inc_eviction() {
2406 EVICTIONS.with(|c| c.set(c.get() + 1));
2407 }
2408}
2409
2410fn is_numeric_text_equality(pred: &crate::args::CriteriaPredicate) -> bool {
2411 match pred {
2412 crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Text(text)) => {
2413 text.trim().parse::<f64>().is_ok_and(f64::is_finite)
2414 }
2415 _ => false,
2416 }
2417}
2418
2419fn fill_numeric_equality_nulls(
2424 view: &RangeView<'_>,
2425 row_start: usize,
2426 row_len: usize,
2427 col_in_view: usize,
2428 mask: &arrow_array::BooleanArray,
2429 n: f64,
2430 ne: bool,
2431) -> Option<arrow_array::BooleanArray> {
2432 use arrow_array::Array as _;
2433 let texts = view.slice_lowered_text(row_start, row_len);
2434 let text = texts.get(col_in_view).and_then(|t| t.as_ref());
2435 match text {
2436 None => {
2437 if !ne {
2438 return Some(mask.clone());
2440 }
2441 let nulls = arrow::compute::is_null(mask).ok()?;
2442 crate::compute_prelude::boolean::or_kleene(mask, &nulls).ok()
2443 }
2444 Some(text) => {
2445 if text.len() != mask.len() {
2446 return None;
2447 }
2448 let mut out = arrow_array::builder::BooleanBuilder::with_capacity(mask.len());
2449 for i in 0..mask.len() {
2450 if mask.is_valid(i) {
2451 out.append_value(mask.value(i));
2452 } else if text.is_valid(i) {
2453 let equal = crate::locale::Locale::invariant()
2454 .parse_number_invariant(text.value(i))
2455 .is_some_and(|x| (x - n).abs() < 1e-12);
2456 out.append_value(equal != ne);
2457 } else {
2458 out.append_value(ne);
2459 }
2460 }
2461 Some(out.finish())
2462 }
2463 }
2464}
2465
2466fn compute_criteria_mask(
2467 view: &RangeView<'_>,
2468 col_in_view: usize,
2469 pred: &crate::args::CriteriaPredicate,
2470) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
2471 use crate::compute_prelude::{boolean, cmp, concat_arrays};
2472 use arrow::compute::kernels::comparison::{ilike, nilike};
2473 use arrow_array::{
2474 Array as _, ArrayRef, BooleanArray, Float64Array, StringArray, builder::BooleanBuilder,
2475 };
2476
2477 fn apply_numeric_pred(
2479 chunk: &Float64Array,
2480 pred: &crate::args::CriteriaPredicate,
2481 ) -> Option<BooleanArray> {
2482 match pred {
2483 crate::args::CriteriaPredicate::Gt(n) => {
2484 cmp::gt(chunk, &Float64Array::new_scalar(*n)).ok()
2485 }
2486 crate::args::CriteriaPredicate::Ge(n) => {
2487 cmp::gt_eq(chunk, &Float64Array::new_scalar(*n)).ok()
2488 }
2489 crate::args::CriteriaPredicate::Lt(n) => {
2490 cmp::lt(chunk, &Float64Array::new_scalar(*n)).ok()
2491 }
2492 crate::args::CriteriaPredicate::Le(n) => {
2493 cmp::lt_eq(chunk, &Float64Array::new_scalar(*n)).ok()
2494 }
2495 crate::args::CriteriaPredicate::Eq(v) => match v {
2496 formualizer_common::LiteralValue::Number(x) => {
2497 cmp::eq(chunk, &Float64Array::new_scalar(*x)).ok()
2498 }
2499 formualizer_common::LiteralValue::Int(i) => {
2500 cmp::eq(chunk, &Float64Array::new_scalar(*i as f64)).ok()
2501 }
2502 _ => None,
2503 },
2504 crate::args::CriteriaPredicate::Ne(v) => match v {
2505 formualizer_common::LiteralValue::Number(x) => {
2506 cmp::neq(chunk, &Float64Array::new_scalar(*x)).ok()
2507 }
2508 formualizer_common::LiteralValue::Int(i) => {
2509 cmp::neq(chunk, &Float64Array::new_scalar(*i as f64)).ok()
2510 }
2511 _ => None,
2512 },
2513 _ => None,
2514 }
2515 }
2516
2517 let is_numeric_pred = matches!(
2519 pred,
2520 crate::args::CriteriaPredicate::Gt(_)
2521 | crate::args::CriteriaPredicate::Ge(_)
2522 | crate::args::CriteriaPredicate::Lt(_)
2523 | crate::args::CriteriaPredicate::Le(_)
2524 | crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Number(_))
2525 | crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Int(_))
2526 | crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Number(_))
2527 | crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Int(_))
2528 );
2529
2530 if is_numeric_pred {
2534 let mut bool_parts: Vec<BooleanArray> = Vec::new();
2535 let number = |v: &formualizer_common::LiteralValue| match v {
2539 formualizer_common::LiteralValue::Number(n) => Some(*n),
2540 formualizer_common::LiteralValue::Int(i) => Some(*i as f64),
2541 _ => None,
2542 };
2543 let equality = match pred {
2544 crate::args::CriteriaPredicate::Eq(v) => number(v).map(|n| (n, false)),
2545 crate::args::CriteriaPredicate::Ne(v) => number(v).map(|n| (n, true)),
2546 _ => None,
2547 };
2548 for res in view.numbers_slices() {
2549 let (rs, rl, cols_seg) = res.ok()?;
2550 if col_in_view < cols_seg.len() {
2551 let chunk = cols_seg[col_in_view].as_ref();
2552 let mut mask = apply_numeric_pred(chunk, pred)?;
2553 if let Some((n, ne)) = equality
2554 && mask.null_count() > 0
2555 {
2556 mask = fill_numeric_equality_nulls(view, rs, rl, col_in_view, &mask, n, ne)?;
2557 }
2558 bool_parts.push(mask);
2559 }
2560 }
2561
2562 if bool_parts.is_empty() {
2563 return None;
2564 } else if bool_parts.len() == 1 {
2565 return Some(std::sync::Arc::new(bool_parts.remove(0)));
2566 } else {
2567 let anys: Vec<&dyn arrow_array::Array> = bool_parts
2569 .iter()
2570 .map(|a| a as &dyn arrow_array::Array)
2571 .collect();
2572 let conc: ArrayRef = concat_arrays(&anys).ok()?;
2573 let ba = conc.as_any().downcast_ref::<BooleanArray>()?.clone();
2574 return Some(std::sync::Arc::new(ba));
2575 }
2576 }
2577
2578 if is_numeric_text_equality(pred)
2582 || matches!(pred, crate::args::CriteriaPredicate::TextLike { .. })
2583 {
2584 for tags in view.type_tags_slices() {
2585 let (_, _, cols) = tags.ok()?;
2586 let tags = cols.get(col_in_view)?;
2587 if tags.values().iter().any(|tag| {
2588 *tag == crate::arrow_store::TypeTag::Empty as u8
2589 || *tag == crate::arrow_store::TypeTag::Number as u8
2590 || *tag == crate::arrow_store::TypeTag::Boolean as u8
2591 }) {
2592 let mut mask = BooleanBuilder::new();
2593 for chunk in view.iter_row_chunks() {
2594 let chunk = chunk.ok()?;
2595 for row in chunk.row_start..chunk.row_start + chunk.row_len {
2596 mask.append_value(crate::builtins::criteria_match(
2597 pred,
2598 &view.get_cell(row, col_in_view),
2599 ));
2600 }
2601 }
2602 return Some(std::sync::Arc::new(mask.finish()));
2603 }
2604 }
2605 }
2606
2607 if matches!(pred, crate::args::CriteriaPredicate::TextLike { pattern, .. }
2609 if pattern.contains('~'))
2610 {
2611 return None;
2612 }
2613
2614 if matches!(
2618 pred,
2619 crate::args::CriteriaPredicate::IsBlank | crate::args::CriteriaPredicate::NotBlank
2620 ) {
2621 let want_blank = matches!(pred, crate::args::CriteriaPredicate::IsBlank);
2622 let mut bool_parts: Vec<BooleanArray> = Vec::new();
2623 for tags in view.type_tags_slices() {
2624 let (_, _, cols) = tags.ok()?;
2625 let tags = cols.get(col_in_view)?;
2626 let mut bb = BooleanBuilder::with_capacity(tags.len());
2627 for i in 0..tags.len() {
2628 let blank = tags.value(i) == crate::arrow_store::TypeTag::Empty as u8;
2629 bb.append_value(blank == want_blank);
2630 }
2631 bool_parts.push(bb.finish());
2632 }
2633 return match bool_parts.len() {
2634 0 => None,
2635 1 => Some(std::sync::Arc::new(bool_parts.remove(0))),
2636 _ => {
2637 let anys: Vec<&dyn arrow_array::Array> = bool_parts
2638 .iter()
2639 .map(|a| a as &dyn arrow_array::Array)
2640 .collect();
2641 let conc: ArrayRef = concat_arrays(&anys).ok()?;
2642 let ba = conc.as_any().downcast_ref::<BooleanArray>()?.clone();
2643 Some(std::sync::Arc::new(ba))
2644 }
2645 };
2646 }
2647
2648 let like_pattern = |text: &str, wildcards: bool| {
2650 let mut pattern = String::with_capacity(text.len());
2651 for ch in text.chars() {
2652 match ch {
2653 '%' | '_' | '\\' => {
2654 pattern.push('\\');
2655 pattern.push(ch);
2656 }
2657 '*' if wildcards => pattern.push('%'),
2658 '?' if wildcards => pattern.push('_'),
2659 _ => pattern.push(ch),
2660 }
2661 }
2662 pattern
2663 };
2664 let (text_kind, text_pat, empty_special) = match pred {
2665 crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Text(t)) => {
2666 (0u8, like_pattern(&t.to_lowercase(), false), t.is_empty())
2667 }
2668 crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Text(t)) => {
2669 (1u8, like_pattern(&t.to_lowercase(), false), false)
2670 }
2671 crate::args::CriteriaPredicate::TextLike {
2672 pattern,
2673 case_insensitive,
2674 } => {
2675 let p = if *case_insensitive {
2676 pattern.to_lowercase()
2677 } else {
2678 pattern.clone()
2679 };
2680 (2u8, like_pattern(&p, true), false)
2681 }
2682 _ => return None,
2683 };
2684
2685 let text_pat_is_empty = text_pat.is_empty();
2686 let ne_matches_blank = text_kind == 1 && !text_pat_is_empty;
2687 let pat = StringArray::new_scalar(text_pat);
2688 let mut bool_parts: Vec<BooleanArray> = Vec::new();
2689
2690 let mut tag_slices = view.type_tags_slices();
2691 for res in view.iter_row_chunks() {
2692 let cs = res.ok()?;
2693 if cs.row_len == 0 {
2694 continue;
2695 }
2696 #[cfg(test)]
2697 criteria_mask_test_hooks::inc_total();
2698
2699 let slices = view.slice_lowered_text(cs.row_start, cs.row_len);
2700 if col_in_view >= slices.len() {
2701 return None;
2702 }
2703
2704 let seg_opt = slices[col_in_view].as_ref().map(|a| a.as_ref());
2705 if empty_special || (text_kind == 1 && text_pat_is_empty) {
2706 let (tag_start, tag_len, tags) = tag_slices.next()?.ok()?;
2707 if tag_start != cs.row_start || tag_len != cs.row_len {
2708 return None;
2709 }
2710 let tags = tags.get(col_in_view)?;
2711 let strings = seg_opt.and_then(|a| a.as_any().downcast_ref::<StringArray>());
2715 let mut bb = BooleanBuilder::with_capacity(cs.row_len);
2716 for i in 0..cs.row_len {
2717 let blank = tags.value(i) == crate::arrow_store::TypeTag::Empty as u8
2718 || (tags.value(i) == crate::arrow_store::TypeTag::Text as u8
2719 && strings.is_some_and(|s| s.is_valid(i) && s.value(i).is_empty()));
2720 bb.append_value(if text_kind == 0 { blank } else { !blank });
2721 }
2722 #[cfg(test)]
2723 if seg_opt.is_none() {
2724 criteria_mask_test_hooks::inc_all_null();
2725 }
2726 bool_parts.push(bb.finish());
2727 continue;
2728 }
2729 let seg = match seg_opt {
2730 Some(s) => s,
2731 None => {
2732 #[cfg(test)]
2733 criteria_mask_test_hooks::inc_all_null();
2734 if (text_kind == 0 && empty_special) || ne_matches_blank {
2735 let mut bb = BooleanBuilder::with_capacity(cs.row_len);
2737 bb.append_n(cs.row_len, true);
2738 bool_parts.push(bb.finish());
2739 } else {
2740 bool_parts.push(BooleanArray::new_null(cs.row_len));
2742 }
2743 continue;
2744 }
2745 };
2746
2747 let seg_sa = seg.as_any().downcast_ref::<StringArray>()?;
2748 let mut m = match text_kind {
2749 0 => ilike(seg_sa, &pat).ok()?,
2750 1 => nilike(seg_sa, &pat).ok()?,
2751 2 => ilike(seg_sa, &pat).ok()?,
2752 _ => return None,
2753 };
2754
2755 if ((text_kind == 0 && empty_special) || ne_matches_blank) && seg_sa.null_count() > 0 {
2761 let mut bb = BooleanBuilder::with_capacity(seg_sa.len());
2763 for i in 0..seg_sa.len() {
2764 bb.append_value(seg_sa.is_null(i));
2765 }
2766 let nulls = bb.finish();
2767 m = boolean::or_kleene(&m, &nulls).ok()?;
2768 }
2769
2770 bool_parts.push(m);
2771 }
2772
2773 if bool_parts.is_empty() {
2774 None
2775 } else if bool_parts.len() == 1 {
2776 Some(std::sync::Arc::new(bool_parts.remove(0)))
2777 } else {
2778 let anys: Vec<&dyn arrow_array::Array> = bool_parts
2779 .iter()
2780 .map(|a| a as &dyn arrow_array::Array)
2781 .collect();
2782 let conc: ArrayRef = concat_arrays(&anys).ok()?;
2783 let ba = conc.as_any().downcast_ref::<BooleanArray>()?.clone();
2784 Some(std::sync::Arc::new(ba))
2785 }
2786}
2787
2788#[derive(Debug, Clone)]
2789pub struct LayerInfo {
2790 pub vertex_count: usize,
2791 pub parallel_eligible: bool,
2792 pub sample_cells: Vec<String>, }
2794
2795#[derive(Debug, Clone)]
2796pub struct EvalPlan {
2797 pub total_vertices_to_evaluate: usize,
2798 pub layers: Vec<LayerInfo>,
2799 pub cycles_detected: usize,
2800 pub dirty_count: usize,
2801 pub volatile_count: usize,
2802 pub parallel_enabled: bool,
2803 pub estimated_parallel_layers: usize,
2804 pub target_cells: Vec<String>,
2805}
2806
2807#[inline]
2810fn plane_mode_ignored() -> bool {
2811 true
2812}
2813
2814#[cfg(feature = "test-support")]
2817#[doc(hidden)]
2818pub fn formula_plane_mode_ignored_for_test() -> bool {
2819 plane_mode_ignored()
2820}
2821
2822impl<R> Engine<R>
2823where
2824 R: EvaluationContext,
2825{
2826 pub fn new(resolver: R, config: EvalConfig) -> Self {
2833 let config = if plane_mode_ignored() {
2838 EvalConfig {
2839 formula_plane_mode: FormulaPlaneMode::Off,
2840 ..config
2841 }
2842 } else {
2843 config
2844 };
2845 if let Err(msg) = config.cycle.validate() {
2846 panic!("invalid CycleConfig: {msg}");
2847 }
2848 crate::builtins::load_builtins();
2849 let resolved_resources = crate::engine::resource_ledger::resolve_evaluation_budgets(
2850 &config.evaluation_budgets,
2851 config.max_vertices,
2852 config.max_memory_mb,
2853 config.max_eval_time,
2854 );
2855
2856 let clock = config.deterministic_mode.build_clock().unwrap_or_else(|_| {
2857 #[cfg(feature = "system-clock")]
2858 {
2859 Arc::new(crate::timezone::SystemClock::new(
2860 crate::timezone::TimeZoneSpec::default(),
2861 ))
2862 }
2863 #[cfg(not(feature = "system-clock"))]
2864 {
2865 Arc::new(crate::timezone::FixedClock::new(
2866 chrono::DateTime::UNIX_EPOCH,
2867 crate::timezone::TimeZoneSpec::Utc,
2868 ))
2869 }
2870 });
2871
2872 let thread_pool = if config.enable_parallel {
2874 let mut builder = ThreadPoolBuilder::new();
2875 if let Some(max_threads) = config.max_threads {
2876 builder = builder.num_threads(max_threads);
2877 }
2878
2879 match builder.build() {
2880 Ok(pool) => Some(Arc::new(pool)),
2881 Err(_) => {
2882 None
2884 }
2885 }
2886 } else {
2887 None
2888 };
2889
2890 let lookup_cache_max_bytes = config.lookup_index_cache_max_bytes;
2892 let function_provider_revision_seen = resolver.planning_semantic_revision();
2893 let mut engine = Self {
2894 criteria_mask_cache: Default::default(),
2895 graph: DependencyGraph::new_with_config(config.clone()),
2896 resolver,
2897 resolved_functions: Default::default(),
2898 resolved_sheet_names: FxHashMap::default(),
2899 resolution_sheet_revision: None,
2900 resolution_arrow_sheet_names: Vec::new(),
2901 resolved_arrow_sheets: Vec::new(),
2902 arrow_sheet_index: std::sync::OnceLock::new(),
2903 config,
2904 workbook_load_limits: crate::engine::WorkbookLoadLimits::default(),
2905 clock: crate::timezone::SnapshotClock::new(clock),
2906 thread_pool,
2907 recalc_epoch: 0,
2908 snapshot_id: std::sync::atomic::AtomicU64::new(1),
2909 topology_epoch: 0,
2910 cached_static_schedule: None,
2911 recent_schedules: Vec::new(),
2912 base_schedule: None,
2913 #[cfg(any(test, feature = "benchmark_internal"))]
2914 recalc_reuse_probe: std::sync::Mutex::new(RecalcReuseProbe::default()),
2915 spill_mgr: ShimSpillManager::default(),
2916 arrow_sheets: SheetStore::default(),
2917 format_registry: crate::format::FormatRegistry::default(),
2918 derived_formats: Default::default(),
2919 #[cfg(test)]
2920 derived_format_operations_for_test: std::sync::atomic::AtomicU64::new(0),
2921 #[cfg(test)]
2922 family_members_for_test: std::sync::atomic::AtomicU64::new(0),
2923 #[cfg(test)]
2924 invariant_bound_members_for_test: std::sync::atomic::AtomicU64::new(0),
2925 #[cfg(test)]
2926 lifted_members_for_test: std::sync::atomic::AtomicU64::new(0),
2927 #[cfg(test)]
2928 chained_members_for_test: std::sync::atomic::AtomicU64::new(0),
2929 #[cfg(test)]
2930 lane_clean_reads_for_test: std::sync::atomic::AtomicU64::new(0),
2931 #[cfg(test)]
2932 criteria_kernel_members_for_test: std::sync::atomic::AtomicU64::new(0),
2933 #[cfg(test)]
2934 memo_hits_for_test: std::sync::atomic::AtomicU64::new(0),
2935 compressed_at_build: None,
2936 #[cfg(test)]
2937 computed_overlay_set_explicit_entry_operations_for_test: 0,
2938 #[cfg(test)]
2939 computed_overlay_stale_clear_range_effects_for_test: 0,
2940 #[cfg(test)]
2941 computed_overlay_stale_clear_offset_attempts_for_test: 0,
2942 #[cfg(test)]
2943 computed_format_vector_allocations_for_test: std::sync::atomic::AtomicU64::new(0),
2944 has_edited: false,
2945 overlay_compactions: 0,
2946 computed_overlay_bytes_estimate: 0,
2947 computed_overlay_mirroring_disabled: false,
2948 force_materialize_range_views: false,
2949 row_bounds_cache: std::sync::RwLock::new(None),
2950 used_axis_bounds_cache: std::sync::RwLock::new(None),
2951 lookup_index_cache: LookupIndexCache::new(lookup_cache_max_bytes),
2952 source_cache: Arc::new(std::sync::RwLock::new(SourceCache::default())),
2953 source_formula_token: Arc::new(()),
2954 source_family_mode: crate::engine::SourceFamilyMode::from_env(),
2955 source_family_counters: crate::engine::SourceFamilyCounters::default(),
2956 recalc_plan_token: Arc::new(()),
2957 staged_formulas: std::collections::HashMap::new(),
2958 staged_formula_index: StagedFormulaIndex::default(),
2959 blocked_pending_spills: Vec::new(),
2960 spill_blocker_waiters: SpillBlockerWaiters::default(),
2961 row_visibility: FxHashMap::default(),
2962 row_visibility_mask_cache: std::sync::RwLock::new(FxHashMap::default()),
2963 formula_parse_diagnostics: Vec::new(),
2964 last_formula_ingest_report: None,
2965 formula_ingest_report_total: FormulaIngestReport::default(),
2966 active_cancel_flag: None,
2967 active_evaluation_deadline: None,
2968 action_depth: 0,
2969 last_virtual_dep_telemetry: VirtualDepTelemetry::default(),
2970 virtual_dep_fallback_activations: 0,
2971 last_cycle_telemetry: CycleTelemetry::default(),
2972 next_evaluation_resource_request_id: 1,
2973 evaluation_resource_request_depth: 0,
2974 active_evaluation_resource_request: None,
2975 last_evaluation_resource_request: None,
2976 evaluation_resource_baseline: EvaluationResourceBaselineStats::default(),
2977 evaluation_resource_request_started_at: None,
2978 evaluation_resource_budgets: resolved_resources.budgets,
2979 evaluation_resource_config_diagnostic: resolved_resources.diagnostic,
2980 active_resource_ledger: None,
2981 source_cache_footprints: Vec::new(),
2982 source_cache_accounted: 0,
2983 pending_iterative_redirty: Vec::new(),
2984 snapshot_evaluation_active: false,
2985 snapshot_volatile_redirty_pending: false,
2986 retained_scc_members: FxHashMap::default(),
2987 next_retained_scc_id: 0,
2988 retained_scc_config_fingerprint: 0,
2989 retained_scc_function_epoch_seen: 0,
2990 retained_scc_provider_revision_seen: None,
2991 retained_scc_dirty_at_begin: Vec::new(),
2992 iterative_state_values: FxHashMap::default(),
2993 function_semantic_epoch_seen: crate::function_registry::semantic_epoch(),
2994 function_provider_revision_seen,
2995 #[cfg(feature = "tracing")]
2996 trace_evaluation_counters: TraceEvaluationCounters::default(),
2997 #[cfg(test)]
2998 evaluation_request_begin_count_for_test: 0,
2999 #[cfg(any(test, feature = "test-support"))]
3000 before_prepared_span_commit_hook: None,
3001 #[cfg(test)]
3002 before_target_preparation_commit_hook: None,
3003 #[cfg(test)]
3004 before_target_planning_snapshot_hook: None,
3005 #[cfg(test)]
3006 inject_target_semantic_stale_once_for_test: false,
3007 #[cfg(test)]
3008 force_virtual_dep_changes_remaining_for_test: 0,
3009 freshness: Default::default(),
3010 #[cfg(test)]
3011 fail_evaluation_commit_preflight_once_for_test: false,
3012 #[cfg(test)]
3013 target_preparation_fault_for_test: None,
3014 #[cfg(test)]
3015 force_non_cycle_schedule_fallback_for_test: false,
3016 #[cfg(test)]
3017 before_legacy_fallback_final_provider_sample_hook: None,
3018 #[cfg(test)]
3019 after_eager_proposal_commit_hook: None,
3020 };
3021 engine.config.arrow_storage_enabled = true;
3023 engine.config.delta_overlay_enabled = true;
3024 engine.config.write_formula_overlay_enabled = true;
3025 let default_sheet = engine.graph.default_sheet_name().to_string();
3026 engine.ensure_arrow_sheet(&default_sheet);
3027 engine
3028 }
3029
3030 pub fn with_thread_pool(
3035 resolver: R,
3036 config: EvalConfig,
3037 thread_pool: Arc<rayon::ThreadPool>,
3038 ) -> Self {
3039 if let Err(msg) = config.cycle.validate() {
3040 panic!("invalid CycleConfig: {msg}");
3041 }
3042 crate::builtins::load_builtins();
3043 let resolved_resources = crate::engine::resource_ledger::resolve_evaluation_budgets(
3044 &config.evaluation_budgets,
3045 config.max_vertices,
3046 config.max_memory_mb,
3047 config.max_eval_time,
3048 );
3049 let clock = config.deterministic_mode.build_clock().unwrap_or_else(|_| {
3050 #[cfg(feature = "system-clock")]
3051 {
3052 Arc::new(crate::timezone::SystemClock::new(
3053 crate::timezone::TimeZoneSpec::default(),
3054 ))
3055 }
3056 #[cfg(not(feature = "system-clock"))]
3057 {
3058 Arc::new(crate::timezone::FixedClock::new(
3059 chrono::DateTime::UNIX_EPOCH,
3060 crate::timezone::TimeZoneSpec::Utc,
3061 ))
3062 }
3063 });
3064 let lookup_cache_max_bytes = config.lookup_index_cache_max_bytes;
3066 let function_provider_revision_seen = resolver.planning_semantic_revision();
3067 let mut engine = Self {
3068 criteria_mask_cache: Default::default(),
3069 graph: DependencyGraph::new_with_config(config.clone()),
3070 resolver,
3071 resolved_functions: Default::default(),
3072 resolved_sheet_names: FxHashMap::default(),
3073 resolution_sheet_revision: None,
3074 resolution_arrow_sheet_names: Vec::new(),
3075 resolved_arrow_sheets: Vec::new(),
3076 arrow_sheet_index: std::sync::OnceLock::new(),
3077 config,
3078 workbook_load_limits: crate::engine::WorkbookLoadLimits::default(),
3079 clock: crate::timezone::SnapshotClock::new(clock),
3080 thread_pool: Some(thread_pool),
3081 recalc_epoch: 0,
3082 snapshot_id: std::sync::atomic::AtomicU64::new(1),
3083 topology_epoch: 0,
3084 cached_static_schedule: None,
3085 recent_schedules: Vec::new(),
3086 base_schedule: None,
3087 #[cfg(any(test, feature = "benchmark_internal"))]
3088 recalc_reuse_probe: std::sync::Mutex::new(RecalcReuseProbe::default()),
3089 spill_mgr: ShimSpillManager::default(),
3090 arrow_sheets: SheetStore::default(),
3091 format_registry: crate::format::FormatRegistry::default(),
3092 derived_formats: Default::default(),
3093 #[cfg(test)]
3094 derived_format_operations_for_test: std::sync::atomic::AtomicU64::new(0),
3095 #[cfg(test)]
3096 family_members_for_test: std::sync::atomic::AtomicU64::new(0),
3097 #[cfg(test)]
3098 invariant_bound_members_for_test: std::sync::atomic::AtomicU64::new(0),
3099 #[cfg(test)]
3100 lifted_members_for_test: std::sync::atomic::AtomicU64::new(0),
3101 #[cfg(test)]
3102 chained_members_for_test: std::sync::atomic::AtomicU64::new(0),
3103 #[cfg(test)]
3104 lane_clean_reads_for_test: std::sync::atomic::AtomicU64::new(0),
3105 #[cfg(test)]
3106 criteria_kernel_members_for_test: std::sync::atomic::AtomicU64::new(0),
3107 #[cfg(test)]
3108 memo_hits_for_test: std::sync::atomic::AtomicU64::new(0),
3109 compressed_at_build: None,
3110 #[cfg(test)]
3111 computed_overlay_set_explicit_entry_operations_for_test: 0,
3112 #[cfg(test)]
3113 computed_overlay_stale_clear_range_effects_for_test: 0,
3114 #[cfg(test)]
3115 computed_overlay_stale_clear_offset_attempts_for_test: 0,
3116 #[cfg(test)]
3117 computed_format_vector_allocations_for_test: std::sync::atomic::AtomicU64::new(0),
3118 has_edited: false,
3119 overlay_compactions: 0,
3120 computed_overlay_bytes_estimate: 0,
3121 computed_overlay_mirroring_disabled: false,
3122 force_materialize_range_views: false,
3123 row_bounds_cache: std::sync::RwLock::new(None),
3124 used_axis_bounds_cache: std::sync::RwLock::new(None),
3125 lookup_index_cache: LookupIndexCache::new(lookup_cache_max_bytes),
3126 source_cache: Arc::new(std::sync::RwLock::new(SourceCache::default())),
3127 source_formula_token: Arc::new(()),
3128 source_family_mode: crate::engine::SourceFamilyMode::from_env(),
3129 source_family_counters: crate::engine::SourceFamilyCounters::default(),
3130 recalc_plan_token: Arc::new(()),
3131 staged_formulas: std::collections::HashMap::new(),
3132 staged_formula_index: StagedFormulaIndex::default(),
3133 blocked_pending_spills: Vec::new(),
3134 spill_blocker_waiters: SpillBlockerWaiters::default(),
3135 row_visibility: FxHashMap::default(),
3136 row_visibility_mask_cache: std::sync::RwLock::new(FxHashMap::default()),
3137 formula_parse_diagnostics: Vec::new(),
3138 last_formula_ingest_report: None,
3139 formula_ingest_report_total: FormulaIngestReport::default(),
3140 active_cancel_flag: None,
3141 active_evaluation_deadline: None,
3142 action_depth: 0,
3143 last_virtual_dep_telemetry: VirtualDepTelemetry::default(),
3144 virtual_dep_fallback_activations: 0,
3145 last_cycle_telemetry: CycleTelemetry::default(),
3146 next_evaluation_resource_request_id: 1,
3147 evaluation_resource_request_depth: 0,
3148 active_evaluation_resource_request: None,
3149 last_evaluation_resource_request: None,
3150 evaluation_resource_baseline: EvaluationResourceBaselineStats::default(),
3151 evaluation_resource_request_started_at: None,
3152 evaluation_resource_budgets: resolved_resources.budgets,
3153 evaluation_resource_config_diagnostic: resolved_resources.diagnostic,
3154 active_resource_ledger: None,
3155 source_cache_footprints: Vec::new(),
3156 source_cache_accounted: 0,
3157 pending_iterative_redirty: Vec::new(),
3158 snapshot_evaluation_active: false,
3159 snapshot_volatile_redirty_pending: false,
3160 retained_scc_members: FxHashMap::default(),
3161 next_retained_scc_id: 0,
3162 retained_scc_config_fingerprint: 0,
3163 retained_scc_function_epoch_seen: 0,
3164 retained_scc_provider_revision_seen: None,
3165 retained_scc_dirty_at_begin: Vec::new(),
3166 iterative_state_values: FxHashMap::default(),
3167 function_semantic_epoch_seen: crate::function_registry::semantic_epoch(),
3168 function_provider_revision_seen,
3169 #[cfg(feature = "tracing")]
3170 trace_evaluation_counters: TraceEvaluationCounters::default(),
3171 #[cfg(test)]
3172 evaluation_request_begin_count_for_test: 0,
3173 #[cfg(any(test, feature = "test-support"))]
3174 before_prepared_span_commit_hook: None,
3175 #[cfg(test)]
3176 before_target_preparation_commit_hook: None,
3177 #[cfg(test)]
3178 before_target_planning_snapshot_hook: None,
3179 #[cfg(test)]
3180 inject_target_semantic_stale_once_for_test: false,
3181 #[cfg(test)]
3182 force_virtual_dep_changes_remaining_for_test: 0,
3183 freshness: Default::default(),
3184 #[cfg(test)]
3185 fail_evaluation_commit_preflight_once_for_test: false,
3186 #[cfg(test)]
3187 target_preparation_fault_for_test: None,
3188 #[cfg(test)]
3189 force_non_cycle_schedule_fallback_for_test: false,
3190 #[cfg(test)]
3191 before_legacy_fallback_final_provider_sample_hook: None,
3192 #[cfg(test)]
3193 after_eager_proposal_commit_hook: None,
3194 };
3195 engine.config.arrow_storage_enabled = true;
3197 engine.config.delta_overlay_enabled = true;
3198 engine.config.write_formula_overlay_enabled = true;
3199 let default_sheet = engine.graph.default_sheet_name().to_string();
3200 engine.ensure_arrow_sheet(&default_sheet);
3201 engine
3202 }
3203
3204 pub fn workbook_load_limits(&self) -> &crate::engine::WorkbookLoadLimits {
3205 &self.workbook_load_limits
3206 }
3207
3208 pub fn set_workbook_load_limits(&mut self, limits: crate::engine::WorkbookLoadLimits) {
3209 self.workbook_load_limits = limits;
3210 }
3211
3212 fn clear_source_cache(&self) {
3213 if let Ok(mut g) = self.source_cache.write() {
3214 *g = SourceCache::default();
3215 }
3216 }
3217
3218 pub fn last_virtual_dep_telemetry(&self) -> &VirtualDepTelemetry {
3219 &self.last_virtual_dep_telemetry
3220 }
3221
3222 pub fn last_cycle_telemetry(&self) -> &CycleTelemetry {
3226 &self.last_cycle_telemetry
3227 }
3228
3229 pub fn last_evaluation_resource_request_stats(
3231 &self,
3232 ) -> Option<&EvaluationResourceRequestStats> {
3233 self.last_evaluation_resource_request.as_ref()
3234 }
3235
3236 pub fn evaluation_resource_baseline_stats(&self) -> EvaluationResourceBaselineStats {
3238 self.evaluation_resource_baseline
3239 }
3240
3241 pub fn evaluation_resource_budgets(&self) -> &crate::engine::EvaluationBudgets {
3242 &self.evaluation_resource_budgets
3243 }
3244
3245 pub fn evaluation_resource_config_diagnostic(
3247 &self,
3248 ) -> Option<&crate::engine::EvaluationResourceConfigDiagnostic> {
3249 self.evaluation_resource_config_diagnostic.as_ref()
3250 }
3251
3252 pub fn reset_evaluation_resource_telemetry(&mut self) {
3254 self.evaluation_resource_baseline = EvaluationResourceBaselineStats::default();
3255 self.last_evaluation_resource_request = None;
3256 }
3257
3258 fn reconcile_source_cache_footprints(&mut self) -> Result<(), ExcelError> {
3261 self.blocked_pending_spills.retain(|&(vertex, anchor, _)| {
3262 self.graph.vertex_exists(vertex)
3263 && self.graph.get_cell_ref(vertex) == Some(anchor)
3264 && matches!(
3265 self.graph.get_vertex_kind(vertex),
3266 VertexKind::FormulaScalar | VertexKind::FormulaArray
3267 )
3268 });
3269 if self.blocked_pending_spills.is_empty() {
3270 self.blocked_pending_spills = Vec::new();
3271 }
3272 let mut bytes = (self.blocked_pending_spills.capacity()
3273 * std::mem::size_of::<(VertexId, CellRef, Region)>()) as u64;
3274 self.source_cache_footprints.retain(|weak| {
3275 let Some(footprint) = weak.upgrade() else {
3276 return false;
3277 };
3278 bytes = bytes.saturating_add(footprint.load(std::sync::atomic::Ordering::Acquire));
3279 true
3280 });
3281 if let Some(ledger) = self.active_resource_ledger.as_mut() {
3282 ledger
3283 .release_retained(self.source_cache_accounted)
3284 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
3285 ledger.observe_retained(bytes);
3287 self.source_cache_accounted = bytes;
3288 ledger
3289 .reserve_retained(0)
3290 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
3291 }
3292 Ok(())
3293 }
3294
3295 fn duration_ns(duration: std::time::Duration) -> u64 {
3296 u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX)
3297 }
3298
3299 fn observe_evaluation_resource_request<T>(
3300 &mut self,
3301 kind: EvaluationRequestKind,
3302 evaluate: impl FnOnce(&mut Self) -> Result<T, ExcelError>,
3303 ) -> Result<T, ExcelError> {
3304 let outermost = self.evaluation_resource_request_depth == 0;
3305 if outermost && self.snapshot_volatile_redirty_pending {
3306 self.snapshot_volatile_redirty_pending = false;
3307 self.graph.redirty_volatiles();
3308 }
3309 #[cfg(feature = "tracing")]
3310 if outermost {
3311 self.trace_evaluation_counters = TraceEvaluationCounters::default();
3312 }
3313 #[cfg(feature = "tracing")]
3314 let request_kind = if matches!(
3315 kind,
3316 EvaluationRequestKind::Full
3317 | EvaluationRequestKind::FullWithDelta
3318 | EvaluationRequestKind::FullCancellable
3319 | EvaluationRequestKind::FullLogged
3320 ) {
3321 "full"
3322 } else {
3323 "targeted"
3324 };
3325 #[cfg(feature = "tracing")]
3326 let _request_span = outermost.then(|| {
3327 crate::engine::trace::fz_span!(
3328 tracing::Level::INFO,
3329 "evaluate",
3330 "evaluate.request",
3331 kind = request_kind,
3332 mode = ?FormulaPlaneMode::Off
3333 )
3334 });
3335 if outermost {
3336 let request_id = self.next_evaluation_resource_request_id;
3337 self.next_evaluation_resource_request_id = request_id
3338 .checked_add(1)
3339 .expect("evaluation resource request ID exhausted");
3340 self.active_evaluation_resource_request = Some(EvaluationResourceRequestStats::new(
3341 request_id,
3342 kind,
3343 FormulaPlaneMode::Off,
3344 self.staged_formula_count(),
3345 ));
3346 self.evaluation_resource_baseline.record_started(request_id);
3347 self.evaluation_resource_request_started_at = Some(crate::instant::FzInstant::now());
3348 self.source_cache_accounted = 0;
3349 self.active_resource_ledger = Some(ResourceLedger::new(
3350 Some(request_id),
3351 self.evaluation_resource_budgets.clone(),
3352 ));
3353 }
3354 self.evaluation_resource_request_depth =
3355 self.evaluation_resource_request_depth.saturating_add(1);
3356 let result = if outermost {
3357 self.reconcile_source_cache_footprints()
3358 .and_then(|()| self.resource_checkpoint(0))
3359 .and_then(|()| evaluate(self))
3360 } else {
3361 evaluate(self)
3362 };
3363 if outermost && result.is_err() {
3364 self.freshness_abort_pass();
3365 }
3366 self.evaluation_resource_request_depth =
3367 self.evaluation_resource_request_depth.saturating_sub(1);
3368
3369 let reconciliation = if outermost {
3370 self.reconcile_source_cache_footprints()
3371 } else {
3372 Ok(())
3373 };
3374 let result = result.and_then(|value| reconciliation.map(|()| value));
3375 if outermost {
3376 let total_ns = self
3377 .evaluation_resource_request_started_at
3378 .take()
3379 .map(|start| Self::duration_ns(start.elapsed()))
3380 .unwrap_or(0);
3381 let mut stats = self
3382 .active_evaluation_resource_request
3383 .take()
3384 .expect("outer evaluation resource request has active stats");
3385 let mut ledger = self
3386 .active_resource_ledger
3387 .take()
3388 .expect("outer evaluation resource request has active ledger");
3389 ledger.release_all_scratch();
3390 stats.ledger.update(ledger.snapshot());
3391 stats.outcome = match &result {
3392 Ok(_) => EvaluationRequestOutcome::Success,
3393 Err(error) if error.kind == ExcelErrorKind::Cancelled => {
3394 EvaluationRequestOutcome::Cancelled
3395 }
3396 Err(_) => EvaluationRequestOutcome::Error,
3397 };
3398 if stats.dirty_lease == FormulaDirtyLeaseOutcome::Acquired {
3399 stats.dirty_lease = if stats.outcome == EvaluationRequestOutcome::Cancelled {
3400 FormulaDirtyLeaseOutcome::RetainedOnCancellation
3401 } else {
3402 FormulaDirtyLeaseOutcome::RetainedOnError
3403 };
3404 }
3405 stats.phases.total_ns = total_ns;
3406 let attributed = stats
3407 .phases
3408 .staged_prepare_ns
3409 .saturating_add(stats.phases.topology_ns)
3410 .saturating_add(stats.phases.materialization_ns);
3411 stats.phases.evaluation_ns = total_ns.saturating_sub(attributed);
3412 self.evaluation_resource_baseline.record_finished(&stats);
3413 self.last_evaluation_resource_request = Some(stats);
3414 crate::engine::trace::fz_event!(
3415 tracing::Level::INFO,
3416 "evaluate",
3417 "evaluate.summary",
3418 computed_vertices = self.trace_evaluation_counters.computed_vertices,
3419 cycles = self.trace_evaluation_counters.cycles,
3420 cancelled = matches!(
3421 &result,
3422 Err(error) if error.kind == ExcelErrorKind::Cancelled
3423 )
3424 );
3425 }
3426 result
3427 }
3428
3429 pub fn set_evaluation_resource_budgets(&mut self, budgets: crate::engine::EvaluationBudgets) {
3430 self.evaluation_resource_budgets = budgets.clone();
3431 self.config.evaluation_budgets = budgets.clone();
3432 self.graph.set_evaluation_budgets(budgets);
3433 }
3434
3435 #[cfg(test)]
3436 pub(crate) fn set_evaluation_budgets_for_test(
3437 &mut self,
3438 budgets: crate::engine::EvaluationBudgets,
3439 ) {
3440 self.set_evaluation_resource_budgets(budgets);
3441 }
3442
3443 fn preflight_evaluation_commit_window(
3444 &mut self,
3445 bounded_writes: usize,
3446 ) -> Result<crate::instant::FzInstant, ExcelError> {
3447 #[cfg(test)]
3448 if std::mem::take(&mut self.fail_evaluation_commit_preflight_once_for_test) {
3449 return Err(crate::engine::ResourceLedgerError::Exhausted(
3450 formualizer_common::ResourceExhaustionDetail {
3451 reason: formualizer_common::ResourceExhaustionReason::Deadline,
3452 limit: 0,
3453 observed: 1,
3454 request_id: self
3455 .active_evaluation_resource_request
3456 .as_ref()
3457 .map(|stats| stats.request_id),
3458 },
3459 )
3460 .into_excel_error());
3461 }
3462 let estimate = std::time::Duration::from_nanos(
3463 u64::try_from(bounded_writes)
3464 .unwrap_or(u64::MAX)
3465 .saturating_mul(100),
3466 );
3467 if let Some(ledger) = self.active_resource_ledger.as_mut() {
3468 ledger
3469 .preflight_commit_window(estimate)
3470 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
3471 }
3472 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3473 stats.evaluation_commit_preflight_count =
3474 stats.evaluation_commit_preflight_count.saturating_add(1);
3475 stats.evaluation_commit_estimated_ns = stats
3476 .evaluation_commit_estimated_ns
3477 .saturating_add(Self::duration_ns(estimate));
3478 }
3479 Ok(crate::instant::FzInstant::now())
3480 }
3481
3482 fn observe_evaluation_commit_window(&mut self, started: crate::instant::FzInstant) {
3483 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3484 stats.evaluation_commit_actual_ns = stats
3485 .evaluation_commit_actual_ns
3486 .saturating_add(Self::duration_ns(started.elapsed()));
3487 }
3488 }
3489
3490 fn live_cancellation_after_work(&self, message: &'static str) -> Result<(), ExcelError> {
3502 if self
3503 .active_cancel_flag
3504 .as_ref()
3505 .is_some_and(|cancel| cancel.is_cancelled())
3506 {
3507 return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(message));
3508 }
3509 Ok(())
3510 }
3511
3512 fn cancellation_checkpoint(&self, message: &'static str) -> Result<(), ExcelError> {
3513 if self
3514 .active_cancel_flag
3515 .as_ref()
3516 .is_some_and(|cancel| cancel.is_cancelled())
3517 {
3518 return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(message));
3519 }
3520 if self
3521 .active_evaluation_deadline
3522 .is_some_and(|deadline| Instant::now() >= deadline)
3523 {
3524 return Err(crate::engine::ResourceLedgerError::Exhausted(
3525 formualizer_common::ResourceExhaustionDetail {
3526 reason: formualizer_common::ResourceExhaustionReason::Deadline,
3527 limit: 0,
3528 observed: 1,
3529 request_id: self
3530 .active_evaluation_resource_request
3531 .as_ref()
3532 .map(|request| request.request_id),
3533 },
3534 )
3535 .into_excel_error()
3536 .with_message(message));
3537 }
3538 Ok(())
3539 }
3540
3541 fn resource_checkpoint(&mut self, work_units: u64) -> Result<(), ExcelError> {
3542 let Some(ledger) = self.active_resource_ledger.as_mut() else {
3543 return Ok(());
3544 };
3545 ledger
3546 .charge_work(work_units)
3547 .and_then(|()| ledger.checkpoint_deadline())
3548 .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3549 }
3550
3551 fn charge_bounded_work(&mut self, mut work_units: u64) -> Result<(), ExcelError> {
3552 if work_units == 0 {
3553 return self.resource_checkpoint(0);
3554 }
3555 while work_units > 0 {
3556 let chunk = work_units.min(256);
3557 self.resource_checkpoint(chunk)?;
3558 work_units -= chunk;
3559 }
3560 Ok(())
3561 }
3562
3563 fn reserve_request_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3564 if let Some(ledger) = self.active_resource_ledger.as_mut() {
3565 ledger.observe_scratch(bytes);
3568 }
3569 Ok(())
3570 }
3571
3572 fn release_request_scratch(&mut self, bytes: u64) {
3573 if let Some(ledger) = self.active_resource_ledger.as_mut() {
3574 let released = ledger.release_scratch(bytes);
3575 debug_assert!(
3576 released.is_ok(),
3577 "request scratch release exceeded the outstanding reservation"
3578 );
3579 }
3580 }
3581
3582 fn reserve_topology_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3583 self.active_resource_ledger
3584 .as_mut()
3585 .map_or(Ok(()), |ledger| ledger.reserve_schedule_discovery(bytes))
3586 .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3587 }
3588
3589 fn reserve_graph_source_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3590 self.active_resource_ledger
3591 .as_mut()
3592 .map_or(Ok(()), |ledger| ledger.reserve_graph_source(bytes))
3593 .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3594 }
3595
3596 fn with_request_scratch<T>(
3597 &mut self,
3598 bytes: u64,
3599 work: impl FnOnce(&mut Self) -> Result<T, ExcelError>,
3600 ) -> Result<T, ExcelError> {
3601 self.reserve_request_scratch(bytes)?;
3602 let result = work(self);
3603 self.release_request_scratch(bytes);
3604 result
3605 }
3606
3607 fn observe_staged_preparation(
3608 &mut self,
3609 selected: usize,
3610 retained: usize,
3611 elapsed: std::time::Duration,
3612 ) {
3613 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3614 stats.staged_selected = stats.staged_selected.saturating_add(selected as u64);
3615 stats.staged_retained = retained as u64;
3616 stats.phases.staged_prepare_ns = stats
3617 .phases
3618 .staged_prepare_ns
3619 .saturating_add(Self::duration_ns(elapsed));
3620 }
3621 }
3622
3623 fn graph_admission_enabled(&self) -> bool {
3624 crate::engine::resource_ledger::graph_admission_enabled(&self.evaluation_resource_budgets)
3625 }
3626
3627 fn preflight_graph_admission(
3628 &mut self,
3629 usage: crate::engine::resource_ledger::GraphAdmission,
3630 ) -> Result<(), ExcelError> {
3631 let request_id = self
3632 .active_evaluation_resource_request
3633 .as_ref()
3634 .map(|stats| stats.request_id);
3635 crate::engine::resource_ledger::preflight_graph_admission(
3636 &self.evaluation_resource_budgets,
3637 usage,
3638 request_id,
3639 )
3640 .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3641 }
3642
3643 fn prepared_legacy_admission(
3644 &mut self,
3645 plan: &PreparedLegacyGraphPlan,
3646 materialization_cells: u64,
3647 ) -> Result<(), ExcelError> {
3648 if !self.graph_admission_enabled() {
3649 return Ok(());
3650 }
3651 let stats = self.graph.baseline_stats();
3652 let added_edges = plan.planned_edge_count().ok_or_else(|| {
3653 ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3654 })?;
3655 let removed_edges = plan.removed_edge_count().ok_or_else(|| {
3656 ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3657 })?;
3658 self.preflight_graph_admission(crate::engine::resource_ledger::GraphAdmission {
3659 final_vertices: stats
3660 .graph_vertex_count
3661 .checked_add(plan.new_vertex_count())
3662 .ok_or_else(|| {
3663 ExcelError::new(ExcelErrorKind::NImpl)
3664 .with_message("graph vertex count overflow")
3665 })?,
3666 final_edges: stats
3667 .graph_edge_count
3668 .checked_sub(removed_edges)
3669 .and_then(|count| count.checked_add(added_edges))
3670 .ok_or_else(|| {
3671 ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3672 })?,
3673 materialization_cells,
3674 added_vertices: plan.new_vertex_count(),
3675 added_edges,
3676 })
3677 }
3678
3679 fn observe_target_admission_failure(
3680 &mut self,
3681 reason: formualizer_common::ResourceExhaustionReason,
3682 ) {
3683 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3684 stats.target_admission_failure = Some(reason);
3685 }
3686 }
3687
3688 fn observe_target_preparation_report(
3689 &mut self,
3690 report: &crate::engine::PreparedTargetGraphReport,
3691 ) {
3692 let reason_bit = |reason: crate::engine::OpaqueReason| -> u64 {
3693 let index = match reason {
3694 crate::engine::OpaqueReason::DynamicReference => 0,
3695 crate::engine::OpaqueReason::RuntimeTextReference => 1,
3696 crate::engine::OpaqueReason::UnknownFunction => 2,
3697 crate::engine::OpaqueReason::UnknownCustomFunction => 3,
3698 crate::engine::OpaqueReason::UnresolvedCrossSheetBinding => 4,
3699 crate::engine::OpaqueReason::UnresolvedName => 5,
3700 crate::engine::OpaqueReason::UnresolvedTable => 6,
3701 crate::engine::OpaqueReason::FormulaName => 7,
3702 crate::engine::OpaqueReason::DeferredSourcePackage => 8,
3703 crate::engine::OpaqueReason::UnsupportedSourceSemantics => 9,
3704 crate::engine::OpaqueReason::UncertainDefaultSheetBinding => 10,
3705 };
3706 1u64 << index
3707 };
3708 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3709 stats.staged_selected = report.selected_staged_cells as u64;
3710 stats.staged_retained = report.retained_staged_cells as u64;
3711 stats.target_requested = report.requested_targets as u64;
3712 stats.target_normalized_regions = report.normalized_regions as u64;
3713 stats.target_scope_level = match &report.widened_scope {
3714 crate::engine::PrepareScope::Exact => 0,
3715 crate::engine::PrepareScope::Sheets(_) => 1,
3716 crate::engine::PrepareScope::Workbook => 2,
3717 };
3718 stats.target_widening_reason_bits = report
3719 .widening_reasons
3720 .iter()
3721 .copied()
3722 .fold(0, |bits, reason| bits | reason_bit(reason));
3723 stats.graph_source_scratch_estimated = report.estimated_scratch_bytes;
3724 stats.graph_source_scratch_observed = report.observed_scratch_bytes;
3725 stats.target_commit_estimated_work = report.estimated_commit_work;
3726 stats.target_commit_actual_work = report.actual_commit_work;
3727 stats.target_commit_window_ns = Self::duration_ns(report.commit_window);
3728 stats.phases.staged_prepare_ns = stats
3729 .phases
3730 .staged_prepare_ns
3731 .saturating_add(Self::duration_ns(report.commit_window));
3732 }
3733 }
3734
3735 pub(crate) fn current_resolved_functions(
3741 &self,
3742 ) -> Option<&crate::interpreter::ResolvedFunctions> {
3743 self.resolved_functions
3744 .is_current(self.graph.data_store(), &self.resolver)
3745 .then_some(&self.resolved_functions)
3746 }
3747
3748 fn refresh_resolution_snapshots(&mut self) {
3749 self.resolved_functions
3750 .refresh(self.graph.data_store(), &self.resolver);
3751 let named_asts = self
3752 .graph
3753 .named_ranges_iter()
3754 .map(|(_, named)| named)
3755 .chain(self.graph.sheet_named_ranges_iter().map(|(_, named)| named))
3756 .filter_map(|named| match &named.definition {
3757 NamedDefinition::Formula { ast, .. } => Some(ast),
3758 _ => None,
3759 });
3760 self.resolved_functions.refresh_ast_functions(
3761 self.graph.data_store(),
3762 &self.resolver,
3763 named_asts,
3764 self.graph.symbol_revision(),
3765 );
3766 let revision = (
3770 self.graph.sheet_reg().revision(),
3771 self.graph.data_store().resolution_revision(),
3772 self.graph.data_store().sheet_names().count(),
3773 );
3774 let arrow_names_unchanged = self
3775 .resolution_arrow_sheet_names
3776 .iter()
3777 .map(|name| name.as_ref())
3778 .eq(self
3779 .arrow_sheets
3780 .sheets
3781 .iter()
3782 .map(|sheet| sheet.name.as_ref()));
3783 if self.resolution_sheet_revision == Some(revision) && arrow_names_unchanged {
3784 return;
3785 }
3786 self.resolution_arrow_sheet_names.clear();
3787 self.resolution_arrow_sheet_names.extend(
3788 self.arrow_sheets
3789 .sheets
3790 .iter()
3791 .map(|sheet| Arc::clone(&sheet.name)),
3792 );
3793 self.resolved_sheet_names.clear();
3794 for id in self.graph.data_store().sheet_names() {
3795 let name = self.graph.data_store().resolve_ast_string(id);
3796 if let Some(sheet_id) = self.graph.sheet_id(name) {
3797 self.resolved_sheet_names.insert(
3798 id,
3799 (
3800 sheet_id,
3801 self.arrow_sheets
3802 .sheets
3803 .iter()
3804 .position(|sheet| sheet.name.as_ref() == name),
3805 ),
3806 );
3807 }
3808 }
3809 self.resolved_arrow_sheets.clear();
3810 for (index, sheet) in self.arrow_sheets.sheets.iter().enumerate() {
3811 if let Some(id) = self.graph.sheet_id(&sheet.name) {
3812 let len = usize::from(id).saturating_add(1);
3813 if self.resolved_arrow_sheets.len() < len {
3814 self.resolved_arrow_sheets.resize(len, None);
3815 }
3816 if let Some(slot) = self.resolved_arrow_sheets.get_mut(usize::from(id)) {
3817 *slot = Some(index);
3818 }
3819 }
3820 }
3821 self.resolution_sheet_revision = Some(revision);
3822 }
3823
3824 fn begin_evaluation_request(&mut self) {
3825 self.refresh_resolution_snapshots();
3826 self.freshness_begin_request();
3827 #[cfg(test)]
3828 {
3829 self.evaluation_request_begin_count_for_test = self
3830 .evaluation_request_begin_count_for_test
3831 .saturating_add(1);
3832 }
3833 self.last_cycle_telemetry = CycleTelemetry::default();
3834 self.graph.authority_sync();
3835 self.pending_iterative_redirty.clear();
3838 self.reconcile_retained_sccs_at_request_begin();
3839 self.clock.refresh();
3843 }
3844
3845 fn redirty_for_next_recalc(&mut self) {
3854 if self.snapshot_evaluation_active {
3855 self.snapshot_volatile_redirty_pending = true;
3856 } else {
3857 self.graph.redirty_volatiles();
3858 }
3859 let pending = std::mem::take(&mut self.pending_iterative_redirty);
3860 let dirty_at_begin = std::mem::take(&mut self.retained_scc_dirty_at_begin);
3861 for (vertex, scc) in dirty_at_begin {
3862 if self.retained_scc_members.get(&vertex) == Some(&scc) {
3863 self.retained_scc_members.remove(&vertex);
3869 if !self.graph.is_dirty(vertex) {
3870 self.iterative_state_values.remove(&vertex);
3871 }
3872 }
3873 }
3874 if !self.iterative_state_values.is_empty() || !self.retained_scc_members.is_empty() {
3875 let graph = &self.graph;
3876 self.iterative_state_values
3877 .retain(|vertex, _| graph.is_live_formula_vertex(*vertex));
3878 self.retained_scc_members
3879 .retain(|vertex, _| graph.is_live_formula_vertex(*vertex));
3880 }
3881 for &vertex in &pending {
3886 if !self.graph.is_live_formula_vertex(vertex) {
3887 continue;
3888 }
3889 if let Some(cell) = self.graph.get_cell_ref(vertex) {
3890 let sheet_name = self.graph.sheet_name(cell.sheet_id);
3891 match self.read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1) {
3892 Some(value) if !matches!(value, LiteralValue::Empty) => {
3893 self.iterative_state_values.insert(vertex, value);
3894 }
3895 _ => {
3896 self.iterative_state_values.remove(&vertex);
3897 }
3898 }
3899 }
3900 }
3901 if !pending.is_empty() {
3902 self.graph.redirty_iterative_members(&pending);
3903 }
3904 }
3905
3906 fn retained_scc_config_fingerprint(&self) -> u64 {
3911 use std::hash::{Hash, Hasher};
3912 let mut hasher = rustc_hash::FxHasher::default();
3913 let config = &self.config;
3914 std::mem::discriminant(&config.cycle.detection).hash(&mut hasher);
3915 match config.cycle.policy {
3916 CyclePolicy::Error => 0u8.hash(&mut hasher),
3917 CyclePolicy::Iterate {
3918 max_iterations,
3919 max_change,
3920 } => {
3921 1u8.hash(&mut hasher);
3922 max_iterations.hash(&mut hasher);
3923 max_change.to_bits().hash(&mut hasher);
3924 }
3925 }
3926 config.date_system.hash(&mut hasher);
3927 config.workbook_seed.hash(&mut hasher);
3928 std::mem::discriminant(&config.volatile_level).hash(&mut hasher);
3929 format!("{:?}", config.deterministic_mode).hash(&mut hasher);
3932 config.range_expansion_limit.hash(&mut hasher);
3933 config.max_open_ended_rows.hash(&mut hasher);
3934 config.max_open_ended_cols.hash(&mut hasher);
3935 hasher.finish()
3936 }
3937
3938 fn reconcile_retained_sccs_at_request_begin(&mut self) {
3944 self.retained_scc_dirty_at_begin.clear();
3945 if self.retained_scc_members.is_empty() {
3946 return;
3947 }
3948 let graph = &self.graph;
3951 self.retained_scc_members
3952 .retain(|vertex, _| graph.is_live_formula_vertex(*vertex));
3953 if self.retained_scc_config_fingerprint() != self.retained_scc_config_fingerprint {
3954 let members: Vec<VertexId> = self.retained_scc_members.keys().copied().collect();
3955 self.retained_scc_members.clear();
3956 self.graph.mark_dirty_many(&members);
3957 return;
3958 }
3959 let changes =
3960 crate::function_registry::semantic_changes_since(self.retained_scc_function_epoch_seen);
3961 let global_changed = changes.epoch != self.retained_scc_function_epoch_seen;
3962 let provider_revision = self.resolver.planning_semantic_revision();
3963 let provider_changed = provider_revision != self.retained_scc_provider_revision_seen;
3964 if global_changed || provider_changed {
3965 let changed: BTreeSet<(String, String)> = changes.keys.into_iter().collect();
3966 let affected: Vec<VertexId> = self
3967 .retained_scc_members
3968 .keys()
3969 .copied()
3970 .filter(|&vertex| {
3971 let Some(ast) = self.graph.get_formula(vertex) else {
3972 return true;
3973 };
3974 (global_changed
3975 && (!changes.complete || Self::ast_uses_changed_function(&ast, &changed)))
3976 || (provider_changed && Self::ast_contains_function(&ast))
3977 })
3978 .collect();
3979 for vertex in &affected {
3980 self.retained_scc_members.remove(vertex);
3981 }
3982 if !affected.is_empty() {
3983 self.graph.mark_dirty_many(&affected);
3984 }
3985 self.retained_scc_function_epoch_seen = changes.epoch;
3986 self.retained_scc_provider_revision_seen = provider_revision;
3987 }
3988 let mut dirty_sccs: FxHashSet<u64> = FxHashSet::default();
3989 let mut all_sccs: FxHashSet<u64> = FxHashSet::default();
3990 for (&vertex, &scc) in &self.retained_scc_members {
3991 all_sccs.insert(scc);
3992 if self.graph.is_dirty(vertex) {
3993 dirty_sccs.insert(scc);
3994 self.retained_scc_dirty_at_begin.push((vertex, scc));
3995 }
3996 }
3997 let reused_members = self
3998 .retained_scc_members
3999 .values()
4000 .filter(|scc| !dirty_sccs.contains(scc))
4001 .count();
4002 let t = &mut self.last_cycle_telemetry;
4003 t.reused_sccs = all_sccs.len() - dirty_sccs.len();
4004 t.reused_scc_members = reused_members;
4005 }
4006
4007 pub fn virtual_dep_fallback_activations(&self) -> u64 {
4008 self.virtual_dep_fallback_activations
4009 }
4010
4011 #[cfg(test)]
4012 pub(crate) fn lookup_index_flights_built_for_test(&self) -> usize {
4013 self.lookup_index_cache
4014 .flights_built
4015 .load(std::sync::atomic::Ordering::Relaxed)
4016 }
4017
4018 pub(crate) fn last_lookup_index_cache_report(&self) -> LookupIndexCacheReport {
4019 self.lookup_index_cache.report()
4020 }
4021
4022 fn lookup_view_contains_volatile(&self, view: &RangeView<'_>, sheet_id: SheetId) -> bool {
4023 let start_row = view.start_row();
4024 let end_row = view.end_row();
4025 let start_col = view.start_col();
4026 let end_col = view.end_col();
4027 for row in start_row..=end_row {
4028 let Ok(row_u32) = u32::try_from(row) else {
4029 return true;
4030 };
4031 for col in start_col..=end_col {
4032 let Ok(col_u32) = u32::try_from(col) else {
4033 return true;
4034 };
4035 let cell_ref = self
4036 .graph
4037 .make_cell_ref_internal(sheet_id, row_u32, col_u32);
4038 if let Some(vertex_id) = self.graph.get_vertex_id_for_address(&cell_ref)
4039 && self.graph.is_volatile(vertex_id)
4040 {
4041 return true;
4042 }
4043 }
4044 }
4045 false
4046 }
4047
4048 fn build_lookup_index_impl(
4049 &self,
4050 view: &RangeView<'_>,
4051 axis: LookupAxis,
4052 ) -> Option<Arc<LookupIndex>> {
4053 let (rows, cols) = view.dims();
4054 if rows == 0 || cols == 0 {
4055 self.lookup_index_cache.note_skipped_tiny();
4056 return None;
4057 }
4058 let len = match axis {
4059 LookupAxis::ColumnInView(col) => {
4060 if col >= cols {
4061 self.lookup_index_cache.note_skipped_tiny();
4062 return None;
4063 }
4064 rows
4065 }
4066 LookupAxis::RowInView(row) => {
4067 if row >= rows {
4068 self.lookup_index_cache.note_skipped_tiny();
4069 return None;
4070 }
4071 cols
4072 }
4073 };
4074 if len < 64 {
4075 self.lookup_index_cache.note_skipped_tiny();
4076 return None;
4077 }
4078
4079 let sheet_id = self.graph.sheet_id(view.sheet_name())?;
4080 let key = LookupIndexKey {
4081 sheet_id,
4082 start_row: u32::try_from(view.start_row()).ok()?,
4083 start_col: u32::try_from(view.start_col()).ok()?,
4084 end_row: u32::try_from(view.end_row()).ok()?,
4085 end_col: u32::try_from(view.end_col()).ok()?,
4086 axis,
4087 snapshot_id: self.data_snapshot_id(),
4088 };
4089 if let Some(index) = self.lookup_index_cache.get(&key) {
4090 return Some(index);
4091 }
4092 if self
4093 .lookup_index_cache
4094 .would_exceed_cap(estimate_bytes(len, 0))
4095 {
4096 self.lookup_index_cache.note_skipped_cap();
4097 return None;
4098 }
4099 if !self.lookup_index_cache.should_build(key) {
4100 return None;
4101 }
4102 self.lookup_index_cache.single_flight(key, || {
4104 if let Some(index) = self.lookup_index_cache.recheck(&key) {
4105 return Some(index);
4106 }
4107 if self.lookup_index_cache.is_known_volatile(&key) {
4108 self.lookup_index_cache.note_skipped_volatile();
4109 return None;
4110 }
4111 if self.lookup_view_contains_volatile(view, sheet_id) {
4112 self.lookup_index_cache.note_volatile_key(key);
4113 self.lookup_index_cache.note_skipped_volatile();
4114 return None;
4115 }
4116 #[cfg(test)]
4117 self.lookup_index_cache
4118 .flights_built
4119 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
4120 match LookupIndex::build(view, axis, self.config.date_system).ok()? {
4121 BuildOutcome::Built(index) => self.lookup_index_cache.insert_if_room(key, index),
4122 BuildOutcome::ErrorInLookupAxis => {
4123 self.lookup_index_cache.note_skipped_error();
4124 None
4125 }
4126 BuildOutcome::Degenerate => {
4127 self.lookup_index_cache.note_skipped_tiny();
4128 None
4129 }
4130 }
4131 })
4132 }
4133
4134 fn reset_virtual_dep_telemetry_if_disabled(&mut self) {
4135 if !self.config.enable_virtual_dep_telemetry {
4136 self.last_virtual_dep_telemetry = VirtualDepTelemetry {
4137 fallback_mode_activations: self.virtual_dep_fallback_activations,
4138 ..VirtualDepTelemetry::default()
4139 };
4140 }
4141 }
4142
4143 fn source_cache_session(&self) -> SourceCacheSession {
4144 self.clear_source_cache();
4145 SourceCacheSession {
4146 cache: self.source_cache.clone(),
4147 }
4148 }
4149
4150 fn resolve_source_scalar_cached(
4151 &self,
4152 name: &str,
4153 version: Option<u64>,
4154 ) -> Result<LiteralValue, ExcelError> {
4155 let key = (name.to_string(), version);
4156 if let Ok(mut g) = self.source_cache.write() {
4157 if let Some(v) = g.scalars.get(&key) {
4158 return Ok(v.clone());
4159 }
4160
4161 let v = self.resolver.resolve_source_scalar(name).map_err(|err| {
4162 if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
4163 ExcelError::new(ExcelErrorKind::Ref)
4164 .with_message(format!("Unresolved source scalar: {name}"))
4165 } else {
4166 err
4167 }
4168 })?;
4169 g.scalars.insert(key, v.clone());
4170 Ok(v)
4171 } else {
4172 self.resolver.resolve_source_scalar(name).map_err(|err| {
4173 if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
4174 ExcelError::new(ExcelErrorKind::Ref)
4175 .with_message(format!("Unresolved source scalar: {name}"))
4176 } else {
4177 err
4178 }
4179 })
4180 }
4181 }
4182
4183 fn resolve_source_table_cached(
4184 &self,
4185 name: &str,
4186 version: Option<u64>,
4187 ) -> Result<Arc<dyn crate::traits::Table>, ExcelError> {
4188 let key = (name.to_string(), version);
4189 if let Ok(mut g) = self.source_cache.write() {
4190 if let Some(t) = g.tables.get(&key) {
4191 return Ok(t.clone());
4192 }
4193
4194 let t = self.resolver.resolve_source_table(name).map_err(|err| {
4195 if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
4196 ExcelError::new(ExcelErrorKind::Ref)
4197 .with_message(format!("Unresolved source table: {name}"))
4198 } else {
4199 err
4200 }
4201 })?;
4202 let t: Arc<dyn crate::traits::Table> = Arc::from(t);
4203 g.tables.insert(key, t.clone());
4204 Ok(t)
4205 } else {
4206 self.resolver
4207 .resolve_source_table(name)
4208 .map_err(|err| {
4209 if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
4210 ExcelError::new(ExcelErrorKind::Ref)
4211 .with_message(format!("Unresolved source table: {name}"))
4212 } else {
4213 err
4214 }
4215 })
4216 .map(Arc::from)
4217 }
4218 }
4219
4220 fn source_table_to_range_view(
4221 &self,
4222 table: &dyn crate::traits::Table,
4223 spec: &Option<formualizer_parse::parser::TableSpecifier>,
4224 ) -> Result<RangeView<'static>, ExcelError> {
4225 use formualizer_parse::parser::{SpecialItem, TableSpecifier};
4226
4227 let owned = match spec {
4228 Some(TableSpecifier::Column(c)) => {
4229 let c = c.trim();
4230 if c == "@" || c.contains('[') || c.contains(']') || c.contains(',') {
4231 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
4232 "Complex structured references not yet supported".to_string(),
4233 ));
4234 }
4235 table.get_column(c)?.materialise().into_owned()
4236 }
4237 Some(TableSpecifier::ColumnRange(start, end)) => {
4238 let cols = table.columns();
4239 let start = start.trim();
4240 let end = end.trim();
4241 let start_key = start.to_lowercase();
4242 let end_key = end.to_lowercase();
4243 let start_idx = cols.iter().position(|n| n.to_lowercase() == start_key);
4244 let end_idx = cols.iter().position(|n| n.to_lowercase() == end_key);
4245 if let (Some(mut si), Some(mut ei)) = (start_idx, end_idx) {
4246 if si > ei {
4247 std::mem::swap(&mut si, &mut ei);
4248 }
4249 let h = table.data_height();
4250 let w = ei - si + 1;
4251 let mut rows = vec![vec![LiteralValue::Empty; w]; h];
4252 for (offset, ci) in (si..=ei).enumerate() {
4253 let cname = &cols[ci];
4254 let col_range = table.get_column(cname)?;
4255 let (rh, _) = col_range.dimensions();
4256 for (r, row) in rows.iter_mut().enumerate().take(h.min(rh)) {
4257 row[offset] = col_range.get(r, 0)?;
4258 }
4259 }
4260 rows
4261 } else {
4262 return Err(ExcelError::new(ExcelErrorKind::Ref)
4263 .with_message("Column range refers to unknown column(s)".to_string()));
4264 }
4265 }
4266 Some(TableSpecifier::SpecialItem(SpecialItem::Headers))
4267 | Some(TableSpecifier::Headers) => table
4268 .headers_row()
4269 .map(|r| r.materialise().into_owned())
4270 .unwrap_or_default(),
4271 Some(TableSpecifier::SpecialItem(SpecialItem::Totals))
4272 | Some(TableSpecifier::Totals) => table
4273 .totals_row()
4274 .map(|r| r.materialise().into_owned())
4275 .unwrap_or_default(),
4276 Some(TableSpecifier::SpecialItem(SpecialItem::Data)) | Some(TableSpecifier::Data) => {
4277 table
4278 .data_body()
4279 .map(|r| r.materialise().into_owned())
4280 .unwrap_or_default()
4281 }
4282 Some(TableSpecifier::SpecialItem(SpecialItem::All)) | Some(TableSpecifier::All) => {
4283 let mut out: Vec<Vec<LiteralValue>> = Vec::new();
4284 if let Some(h) = table.headers_row() {
4285 out.extend(h.iter_rows());
4286 }
4287 if let Some(body) = table.data_body() {
4288 out.extend(body.iter_rows());
4289 }
4290 if let Some(tr) = table.totals_row() {
4291 out.extend(tr.iter_rows());
4292 }
4293 out
4294 }
4295 Some(TableSpecifier::SpecialItem(SpecialItem::ThisRow)) => {
4296 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
4297 "@ (This Row) requires table-aware context; not yet supported".to_string(),
4298 ));
4299 }
4300 Some(TableSpecifier::Row(_)) | Some(TableSpecifier::Combination(_)) => {
4301 return Err(ExcelError::new(ExcelErrorKind::NImpl)
4302 .with_message("Complex structured references not yet supported".to_string()));
4303 }
4304 None => {
4305 return Err(ExcelError::new(ExcelErrorKind::NImpl)
4306 .with_message("Table reference without specifier is unsupported".to_string()));
4307 }
4308 };
4309
4310 Ok(RangeView::from_owned_rows(owned, self.config.date_system))
4311 }
4312
4313 pub fn default_sheet_id(&self) -> SheetId {
4314 self.graph.default_sheet_id()
4315 }
4316
4317 pub fn default_sheet_name(&self) -> &str {
4318 self.graph.default_sheet_name()
4319 }
4320
4321 pub fn set_workbook_seed(&mut self, seed: u64) {
4323 self.config.workbook_seed = seed;
4324 }
4325
4326 pub fn set_volatile_level(&mut self, level: crate::traits::VolatileLevel) {
4328 self.config.volatile_level = level;
4329 }
4330
4331 pub fn set_temporal_egress(&mut self, policy: crate::engine::TemporalEgress) {
4333 self.config.temporal_egress = policy;
4334 }
4335
4336 pub fn temporal_egress(&self) -> crate::engine::TemporalEgress {
4337 self.config.temporal_egress
4338 }
4339
4340 pub fn set_deterministic_mode(
4342 &mut self,
4343 mode: crate::engine::DeterministicMode,
4344 ) -> Result<(), ExcelError> {
4345 let clock = mode.build_clock()?;
4346 self.config.deterministic_mode = mode;
4347 self.clock = crate::timezone::SnapshotClock::new(clock);
4348 Ok(())
4349 }
4350
4351 pub fn set_clock(&mut self, clock: Arc<dyn crate::timezone::ClockProvider>) {
4359 self.clock = crate::timezone::SnapshotClock::new(clock);
4360 }
4361
4362 fn validate_deterministic_mode(&self) -> Result<(), ExcelError> {
4363 self.config.deterministic_mode.validate()
4364 }
4365
4366 pub fn sheet_id(&self, name: &str) -> Option<SheetId> {
4367 self.graph.sheet_id(name)
4368 }
4369
4370 pub fn sheet_id_mut(&mut self, name: &str) -> SheetId {
4371 self.add_sheet(name)
4372 .unwrap_or_else(|_| self.graph.sheet_id_mut(name))
4373 }
4374
4375 pub fn sheet_name(&self, id: SheetId) -> &str {
4376 self.graph.sheet_name(id)
4377 }
4378
4379 fn arrow_sheet_position(&self, id: SheetId) -> Option<usize> {
4380 let index = self.arrow_sheet_index.get_or_init(|| {
4381 let mut index = Vec::new();
4382 for (position, sheet) in self.arrow_sheets.sheets.iter().enumerate() {
4383 if let Some(id) = self.graph.sheet_id(&sheet.name) {
4384 let needed = usize::from(id).saturating_add(1);
4385 if index.len() < needed {
4386 index.resize(needed, None);
4387 }
4388 if let Some(slot) = index.get_mut(usize::from(id)) {
4389 *slot = Some(position);
4390 }
4391 }
4392 }
4393 index
4394 });
4395 index.get(usize::from(id)).copied().flatten()
4396 }
4397
4398 pub(crate) fn arrow_sheet_by_id(&self, id: SheetId) -> Option<&crate::arrow_store::ArrowSheet> {
4400 self.arrow_sheets.sheets.get(self.arrow_sheet_position(id)?)
4401 }
4402
4403 pub(crate) fn arrow_sheet_by_id_mut(
4405 &mut self,
4406 id: SheetId,
4407 ) -> Option<&mut crate::arrow_store::ArrowSheet> {
4408 let position = self.arrow_sheet_position(id)?;
4409 self.arrow_sheets.sheets.get_mut(position)
4410 }
4411
4412 fn rebuild_arrow_sheet_index(&mut self) {
4413 self.arrow_sheet_index.take();
4414 self.resolved_arrow_sheets.clear();
4415 for (index, sheet) in self.arrow_sheets.sheets.iter().enumerate() {
4416 if let Some(id) = self.graph.sheet_id(&sheet.name) {
4417 let len = usize::from(id).saturating_add(1);
4418 if self.resolved_arrow_sheets.len() < len {
4419 self.resolved_arrow_sheets.resize(len, None);
4420 }
4421 if let Some(slot) = self.resolved_arrow_sheets.get_mut(usize::from(id)) {
4422 *slot = Some(index);
4423 }
4424 }
4425 }
4426 }
4427
4428 pub fn add_sheet(&mut self, name: &str) -> Result<SheetId, ExcelError> {
4429 let id = self.graph.add_sheet(name)?;
4430 self.ensure_arrow_sheet(name);
4431 self.mark_topology_edited();
4432 Ok(id)
4433 }
4434
4435 pub fn duplicate_sheet(&mut self, source: &str, new_name: &str) -> Result<SheetId, ExcelError> {
4436 let source_id = self.graph.sheet_id(source).ok_or_else(|| {
4437 ExcelError::new(ExcelErrorKind::Value).with_message("Source sheet does not exist")
4438 })?;
4439 if new_name.is_empty() || new_name.len() > 255 {
4440 return Err(ExcelError::new(ExcelErrorKind::Value).with_message("Invalid sheet name"));
4441 }
4442 if self.graph.sheet_id(new_name).is_some() {
4443 return Err(ExcelError::new(ExcelErrorKind::Value)
4444 .with_message(format!("Sheet '{new_name}' already exists")));
4445 }
4446 let new_id = self.graph.duplicate_sheet(source_id, new_name)?;
4447
4448 if let Some(source_sheet) = self.arrow_sheets.sheet(source).cloned() {
4449 let mut copied_sheet = source_sheet;
4450 copied_sheet.name = Arc::<str>::from(new_name);
4451 self.arrow_sheets.sheets.push(copied_sheet);
4452 self.rebuild_arrow_sheet_index();
4453 } else {
4454 self.ensure_arrow_sheet(new_name);
4455 }
4456
4457 let duplicated_formulas = self
4458 .graph
4459 .formula_vertices()
4460 .into_iter()
4461 .filter(|vertex| {
4462 self.graph
4463 .get_cell_ref(*vertex)
4464 .is_some_and(|cell| cell.sheet_id == new_id)
4465 })
4466 .collect::<Vec<_>>();
4467 self.graph.mark_vertices_dirty_batch(&duplicated_formulas);
4468 self.mark_topology_edited();
4469 Ok(new_id)
4470 }
4471
4472 fn ensure_arrow_sheet(&mut self, name: &str) {
4473 if self.arrow_sheets.sheet(name).is_some() {
4474 return;
4475 }
4476 self.arrow_sheets
4477 .sheets
4478 .push(crate::arrow_store::ArrowSheet {
4479 name: std::sync::Arc::<str>::from(name),
4480 date_system: self.config.date_system,
4481 columns: Vec::new(),
4482 nrows: 0,
4483 chunk_starts: Vec::new(),
4484 chunk_rows: 32 * 1024,
4485 });
4486 self.rebuild_arrow_sheet_index();
4487 }
4488
4489 pub fn remove_sheet(&mut self, sheet_id: SheetId) -> Result<(), ExcelError> {
4490 let name = self.graph.sheet_name(sheet_id).to_string();
4491 self.purge_derived_formats_for_sheet(sheet_id);
4492 self.graph.remove_sheet(sheet_id)?;
4493 self.arrow_sheets.sheets.retain(|s| s.name.as_ref() != name);
4494 self.rebuild_arrow_sheet_index();
4495 self.clear_all_computed_overlays();
4499 self.mark_all_formula_vertices_dirty();
4500 self.clear_staged_formulas_for_sheet(&name);
4501 if self.row_visibility.remove(&sheet_id).is_some() {
4502 self.invalidate_row_visibility_mask_cache();
4503 }
4504 self.record_structural_change(StructuralScope::RemovedSheet(sheet_id));
4505 self.mark_topology_edited();
4506 Ok(())
4507 }
4508
4509 fn rename_sheet_in_arrow_store(&mut self, target_name: &str, new_name: &str) -> bool {
4511 if let Some(asheet) = self
4512 .arrow_sheets
4513 .sheets
4514 .iter_mut()
4515 .find(|s| s.name.as_ref() == target_name)
4516 {
4517 asheet.name = std::sync::Arc::<str>::from(new_name);
4518 return true;
4519 }
4520 false
4521 }
4522
4523 pub fn rename_sheet(&mut self, sheet_id: SheetId, new_name: &str) -> Result<(), ExcelError> {
4524 let old_name = self.graph.sheet_name(sheet_id).to_string();
4525
4526 self.rename_sheet_in_arrow_store(&old_name, new_name);
4529
4530 match self.graph.rename_sheet(sheet_id, new_name) {
4532 Ok(_) => {
4533 self.rename_staged_formula_sheet(&old_name, new_name);
4534 let sheet_vertices: Vec<VertexId> = self
4536 .graph
4537 .grid_vertices_in_sheet(sheet_id)
4538 .map(|(id, _)| id)
4539 .collect();
4540 for v_id in sheet_vertices {
4541 self.graph.mark_vertex_dirty(v_id);
4542 }
4543 self.mark_topology_edited();
4545 Ok(())
4546 }
4547 Err(e) => {
4548 self.rename_sheet_in_arrow_store(new_name, &old_name);
4550 Err(e)
4551 }
4552 }
4553 }
4554
4555 pub fn named_ranges_iter(
4556 &self,
4557 ) -> impl Iterator<Item = (&String, &crate::engine::named_range::NamedRange)> {
4558 self.graph.named_ranges_iter()
4559 }
4560
4561 pub fn sheet_named_ranges_iter(
4562 &self,
4563 ) -> impl Iterator<Item = (&(SheetId, String), &crate::engine::named_range::NamedRange)> {
4564 self.graph.sheet_named_ranges_iter()
4565 }
4566
4567 pub fn resolve_name_entry(
4568 &self,
4569 name: &str,
4570 current_sheet: SheetId,
4571 ) -> Option<&crate::engine::named_range::NamedRange> {
4572 self.graph.resolve_name_entry(name, current_sheet)
4573 }
4574
4575 pub(crate) fn name_query_scope(
4585 &self,
4586 scope_sheet: Option<&str>,
4587 ) -> Result<NameScope, ExcelError> {
4588 match scope_sheet {
4589 None => Ok(NameScope::Workbook),
4590 Some(sheet) => self
4591 .graph
4592 .sheet_id(sheet)
4593 .map(NameScope::Sheet)
4594 .ok_or_else(|| {
4595 ExcelError::new(ExcelErrorKind::Ref)
4596 .with_message(format!("name scope sheet not found: {sheet}"))
4597 }),
4598 }
4599 }
4600
4601 fn resolve_sheet_locator(
4614 &self,
4615 locator: &crate::reference::SharedSheetLocator<'_>,
4616 context_sheet: SheetId,
4617 ) -> Result<SheetId, ExcelError> {
4618 self.graph
4619 .sheet_reg()
4620 .resolve_locator(locator, context_sheet)
4621 }
4622
4623 pub fn has_name(&self, name: &str, scope_sheet: Option<&str>) -> bool {
4624 let Ok(scope) = self.name_query_scope(scope_sheet) else {
4625 return false;
4626 };
4627 self.graph
4628 .resolve_name_entry_in_scope(name, scope)
4629 .is_some()
4630 }
4631
4632 pub fn resolved_name_value(
4636 &self,
4637 name: &str,
4638 scope_sheet: Option<&str>,
4639 ) -> Option<LiteralValue> {
4640 let scope = self.name_query_scope(scope_sheet).ok()?;
4641 let entry = self.graph.resolve_name_entry_in_scope(name, scope)?;
4642 self.graph.get_value(entry.vertex)
4643 }
4644
4645 pub fn table_metadata(&self, name: &str) -> Option<TableMetadata> {
4646 let entry = self.graph.resolve_table_entry(name)?;
4647 Some(TableMetadata {
4648 name: entry.name.clone(),
4649 sheet: self.graph.sheet_name(entry.sheet_id()).to_string(),
4650 start_row: entry.range.start.coord.row() + 1,
4651 start_col: entry.range.start.coord.col() + 1,
4652 end_row: entry.range.end.coord.row() + 1,
4653 end_col: entry.range.end.coord.col() + 1,
4654 header_row: entry.header_row,
4655 headers: entry.headers.clone(),
4656 totals_row: entry.totals_row,
4657 })
4658 }
4659
4660 pub fn tables(&self) -> Vec<TableMetadata> {
4662 self.graph
4663 .table_names()
4664 .into_iter()
4665 .filter_map(|name| self.table_metadata(&name))
4666 .collect()
4667 }
4668
4669 pub fn named_ranges_snapshot(&self) -> Vec<crate::engine::named_range::NamedRangeSnapshot> {
4670 let mut out: Vec<crate::engine::named_range::NamedRangeSnapshot> = Vec::new();
4671
4672 for (name, named) in self.graph.named_ranges_iter() {
4673 out.push(crate::engine::named_range::NamedRangeSnapshot {
4674 name: name.clone(),
4675 scope: NameScope::Workbook,
4676 definition: named.definition.clone(),
4677 });
4678 }
4679
4680 for ((sheet_id, name), named) in self.graph.sheet_named_ranges_iter() {
4681 out.push(crate::engine::named_range::NamedRangeSnapshot {
4682 name: name.clone(),
4683 scope: NameScope::Sheet(*sheet_id),
4684 definition: named.definition.clone(),
4685 });
4686 }
4687
4688 out.sort_by(|a, b| {
4689 let a_scope = match a.scope {
4690 NameScope::Workbook => (0u8, 0u32),
4691 NameScope::Sheet(id) => (1u8, u32::from(id)),
4692 };
4693 let b_scope = match b.scope {
4694 NameScope::Workbook => (0u8, 0u32),
4695 NameScope::Sheet(id) => (1u8, u32::from(id)),
4696 };
4697 a_scope.cmp(&b_scope).then_with(|| a.name.cmp(&b.name))
4698 });
4699
4700 out
4701 }
4702
4703 pub fn named_ranges_snapshot_for_sheet(
4704 &self,
4705 sheet_id: SheetId,
4706 ) -> Vec<crate::engine::named_range::NamedRangeSnapshot> {
4707 self.named_ranges_snapshot()
4708 .into_iter()
4709 .filter(|entry| match entry.scope {
4710 NameScope::Workbook => true,
4711 NameScope::Sheet(id) => id == sheet_id,
4712 })
4713 .collect()
4714 }
4715
4716 pub fn define_name(
4717 &mut self,
4718 name: &str,
4719 definition: NamedDefinition,
4720 scope: NameScope,
4721 ) -> Result<(), ExcelError> {
4722 self.graph.validate_define_name(name, scope)?;
4723 self.graph.define_name(name, definition, scope)?;
4724 self.record_structural_change(StructuralScope::AllSheets);
4725
4726 self.mark_topology_edited();
4727
4728 Ok(())
4729 }
4730
4731 pub fn update_name(
4732 &mut self,
4733 name: &str,
4734 definition: NamedDefinition,
4735 scope: NameScope,
4736 ) -> Result<(), ExcelError> {
4737 self.graph.validate_existing_name(name, scope)?;
4738 self.graph.update_name(name, definition, scope)?;
4739 self.record_structural_change(StructuralScope::AllSheets);
4740
4741 self.mark_topology_edited();
4742
4743 Ok(())
4744 }
4745
4746 pub fn delete_name(&mut self, name: &str, scope: NameScope) -> Result<(), ExcelError> {
4747 self.graph.validate_existing_name(name, scope)?;
4748 self.graph.delete_name(name, scope)?;
4749 self.record_structural_change(StructuralScope::AllSheets);
4750
4751 self.mark_topology_edited();
4752
4753 Ok(())
4754 }
4755
4756 pub fn define_table(
4757 &mut self,
4758 name: &str,
4759 range: crate::reference::RangeRef,
4760 header_row: bool,
4761 headers: Vec<String>,
4762 totals_row: bool,
4763 ) -> Result<(), ExcelError> {
4764 self.graph
4765 .define_table(name, range, header_row, headers, totals_row)?;
4766 self.record_structural_change(StructuralScope::AllSheets);
4767 self.mark_topology_edited();
4768 Ok(())
4769 }
4770
4771 pub fn define_source_scalar(
4772 &mut self,
4773 name: &str,
4774 version: Option<u64>,
4775 ) -> Result<(), ExcelError> {
4776 self.graph.define_source_scalar(name, version)?;
4777 self.record_structural_change(StructuralScope::OpaqueGlobal);
4778 self.mark_topology_edited();
4779 Ok(())
4780 }
4781
4782 pub fn define_source_table(
4783 &mut self,
4784 name: &str,
4785 version: Option<u64>,
4786 ) -> Result<(), ExcelError> {
4787 self.graph.define_source_table(name, version)?;
4788 self.record_structural_change(StructuralScope::OpaqueGlobal);
4789 self.mark_topology_edited();
4790 Ok(())
4791 }
4792
4793 pub fn set_source_scalar_version(
4794 &mut self,
4795 name: &str,
4796 version: Option<u64>,
4797 ) -> Result<(), ExcelError> {
4798 self.graph.set_source_scalar_version(name, version)?;
4799 Ok(())
4800 }
4801
4802 pub fn set_source_table_version(
4803 &mut self,
4804 name: &str,
4805 version: Option<u64>,
4806 ) -> Result<(), ExcelError> {
4807 self.graph.set_source_table_version(name, version)?;
4808 Ok(())
4809 }
4810
4811 pub fn invalidate_source(&mut self, name: &str) -> Result<(), ExcelError> {
4812 self.graph.invalidate_source(name)?;
4813 Ok(())
4814 }
4815
4816 pub fn vertex_value(&self, vertex: VertexId) -> Option<LiteralValue> {
4817 self.graph.get_value(vertex)
4818 }
4819
4820 pub fn graph_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
4821 self.graph.get_cell_value(sheet, row, col)
4822 }
4823
4824 pub fn vertex_for_cell(&self, cell: &CellRef) -> Option<VertexId> {
4825 self.graph.get_vertex_for_cell(cell)
4826 }
4827
4828 pub fn evaluation_vertices(&self) -> Vec<VertexId> {
4829 self.graph.get_evaluation_vertices()
4830 }
4831
4832 pub fn baseline_stats(&self) -> EngineBaselineStats {
4834 let graph = self.graph.baseline_stats();
4835 EngineBaselineStats {
4836 graph_vertex_count: graph.graph_vertex_count,
4837 graph_formula_vertex_count: graph.graph_formula_vertex_count,
4838 graph_edge_count: graph.graph_edge_count,
4839 dirty_vertex_count: graph.dirty_vertex_count,
4840 evaluation_vertex_count: graph.evaluation_vertex_count,
4841 formula_ast_root_count: graph.formula_ast_root_count,
4842 formula_ast_node_count: graph.formula_ast_node_count,
4843 staged_formula_count: self.staged_formula_count(),
4844 formula_plane_active_span_count: 0,
4845 formula_plane_producer_result_entries: 0,
4846 formula_plane_consumer_read_entries: 0,
4847 formula_plane_mixed_topology_cache_builds: 0,
4848 formula_plane_mixed_topology_cache_hits: 0,
4849 formula_plane_mixed_topology_cache_overflows: 0,
4850 formula_plane_dirty_pending_events: 0,
4851 formula_plane_dirty_region_events_recorded: 0,
4852 formula_plane_dirty_span_region_events_recorded: 0,
4853 formula_plane_dirty_whole_span_seeds_recorded: 0,
4854 formula_plane_dirty_global_invalidations: 0,
4855 formula_plane_structural_span_candidates: 0,
4856 formula_plane_cycle_member_span_demotions: 0,
4857 formula_plane_array_result_span_demotions: 0,
4858 retained_scc_members: self.retained_scc_members.len(),
4859 }
4860 }
4861
4862 pub(crate) fn inspection_mutation_revision(&self) -> u64 {
4864 self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed)
4865 }
4866
4867 #[cfg(test)]
4868 pub(crate) fn used_axis_bounds_cache_stats(&self) -> (usize, usize, usize, usize) {
4869 self.used_axis_bounds_cache
4870 .read()
4871 .ok()
4872 .and_then(|guard| {
4873 guard.as_ref().map(|cache| {
4874 (
4875 cache.row_hits.load(Ordering::Relaxed),
4876 cache.row_misses.load(Ordering::Relaxed),
4877 cache.col_hits.load(Ordering::Relaxed),
4878 cache.col_misses.load(Ordering::Relaxed),
4879 )
4880 })
4881 })
4882 .unwrap_or((0, 0, 0, 0))
4883 }
4884
4885 pub fn set_first_load_assume_new(&mut self, enabled: bool) {
4886 self.graph.set_first_load_assume_new(enabled);
4887 }
4888
4889 pub fn first_load_assume_new(&self) -> bool {
4890 self.graph.first_load_assume_new()
4891 }
4892
4893 pub fn reset_ensure_touched(&mut self) {
4894 self.graph.reset_ensure_touched();
4895 }
4896
4897 pub fn finalize_sheet_index(&mut self, sheet: &str) {
4898 self.graph.finalize_sheet_index(sheet);
4899 }
4900
4901 pub fn action<T>(
4910 &mut self,
4911 name: impl AsRef<str>,
4912 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4913 ) -> Result<T, crate::engine::EditorError> {
4914 if self.action_depth != 0 {
4915 return Err(crate::engine::EditorError::TransactionFailed {
4916 reason: "Nested Engine::action calls are not supported (ticket 614: commit-only surface)"
4917 .to_string(),
4918 });
4919 }
4920
4921 self.action_depth = 1;
4922 let engine_ptr: *mut Engine<R> = self;
4923 let _guard = ActionDepthGuard {
4924 engine: engine_ptr,
4925 _marker: std::marker::PhantomData,
4926 };
4927
4928 let mut tx = EngineAction {
4929 engine: self,
4930 name: name.as_ref().to_string(),
4931 capture: None,
4932 arrow_undo: None,
4933 atomic_policy: false,
4934 };
4935 f(&mut tx)
4936 }
4937
4938 pub fn action_atomic<T>(
4942 &mut self,
4943 name: impl Into<String>,
4944 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4945 ) -> Result<T, crate::engine::EditorError> {
4946 let (v, _j) = self.action_atomic_journal(name, f)?;
4947 Ok(v)
4948 }
4949
4950 pub fn action_atomic_journal<T>(
4952 &mut self,
4953 name: impl Into<String>,
4954 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4955 ) -> Result<(T, crate::engine::ActionJournal), crate::engine::EditorError> {
4956 if self.action_depth != 0 {
4957 return Err(crate::engine::EditorError::TransactionFailed {
4958 reason: "Nested Engine::action calls are not supported (deterministic rule)"
4959 .to_string(),
4960 });
4961 }
4962
4963 self.action_depth = 1;
4964 let engine_ptr: *mut Engine<R> = self;
4965 let _guard = ActionDepthGuard {
4966 engine: engine_ptr,
4967 _marker: std::marker::PhantomData,
4968 };
4969
4970 let name_str = name.into();
4971 let mut capture = MutationCapture::new(Default::default());
4972 let start_len = capture.len();
4973 self.action_atomic_impl(&mut capture, start_len, true, name_str, f)
4974 }
4975
4976 fn action_atomic_impl<T>(
4977 &mut self,
4978 capture: &mut MutationCapture,
4979 start_len: usize,
4980 expand_runs: bool,
4981 name: String,
4982 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4983 ) -> Result<(T, crate::engine::ActionJournal), crate::engine::EditorError> {
4984 let invalidation_baseline = self.invalidation_baseline();
4985 let mut arrow_undo = crate::engine::ArrowUndoBatch::default();
4986 let arrow_ptr: *mut crate::engine::ArrowUndoBatch = &mut arrow_undo;
4987
4988 let capture_ptr: *mut MutationCapture = capture;
4989 let mut tx = EngineAction {
4990 engine: self,
4991 name: name.clone(),
4992 capture: Some(capture_ptr),
4993 arrow_undo: Some(arrow_ptr),
4994 atomic_policy: true,
4995 };
4996
4997 let res = f(&mut tx);
4998
4999 let capture_ref = unsafe { &*capture_ptr };
5006 let has_runs = capture_ref.lazy_len() > 0;
5007 let graph_events: Vec<crate::engine::ChangeEvent> = if expand_runs || res.is_err() {
5008 capture_ref.expanded_events_from(start_len, 0)
5009 } else {
5010 capture_ref.events()[start_len..].to_vec()
5011 };
5012 let graph_batch = crate::engine::GraphUndoBatch {
5013 events: graph_events,
5014 };
5015 let affected_cells = arrow_undo.ops.len();
5016 let journal = crate::engine::ActionJournal {
5017 name,
5018 graph: graph_batch,
5019 arrow: arrow_undo,
5020 affected_cells,
5021 };
5022
5023 match res {
5024 Ok(v) => {
5025 if !journal.graph.is_empty() || !journal.arrow.is_empty() || has_runs {
5026 for event in &journal.graph.events {
5027 self.record_change_for_event(event);
5028 }
5029 let mut impact = Self::classify_change_events(
5030 &journal.graph.events,
5031 LoggedEditDirection::Original,
5032 )
5033 .max(Self::classify_arrow_undo(&journal.arrow));
5034 if has_runs {
5035 impact = impact.max(LoggedEditImpact::Topology);
5036 }
5037 self.apply_logged_edit_impact(impact, invalidation_baseline);
5038 }
5039 Ok((v, journal))
5040 }
5041 Err(e) => {
5042 if let Err(rb) = self.rollback_from_action_journal(&journal, invalidation_baseline)
5043 {
5044 return Err(crate::engine::EditorError::TransactionFailed {
5045 reason: format!(
5046 "Engine::action_atomic rollback failed after error '{e}': {rb}"
5047 ),
5048 });
5049 }
5050 if !journal.graph.is_empty() || !journal.arrow.is_empty() {
5051 for event in &journal.graph.events {
5052 self.record_change_for_event(event);
5053 }
5054 }
5055 Err(e)
5056 }
5057 }
5058 }
5059
5060 pub fn action_with_logger<T>(
5067 &mut self,
5068 log: &mut crate::engine::ChangeLog,
5069 name: impl AsRef<str>,
5070 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
5071 ) -> Result<T, crate::engine::EditorError> {
5072 if self.action_depth != 0 {
5073 return Err(crate::engine::EditorError::TransactionFailed {
5074 reason: "Nested Engine::action calls are not supported (deterministic rule)"
5075 .to_string(),
5076 });
5077 }
5078
5079 self.action_depth = 1;
5080 let engine_ptr: *mut Engine<R> = self;
5081 let _guard = ActionDepthGuard {
5082 engine: engine_ptr,
5083 _marker: std::marker::PhantomData,
5084 };
5085
5086 let name_str = name.as_ref().to_string();
5087 let mut capture = MutationCapture::new(log.current_meta());
5088 let start_len = capture.len();
5089 capture.begin_compound(name_str.clone());
5090
5091 let res = self.action_atomic_impl(&mut capture, start_len, false, name_str, f);
5094 capture.close_compounds();
5095
5096 match res {
5097 Ok((v, _journal)) => {
5098 log.publish_capture(capture);
5099 Ok(v)
5100 }
5101 Err(e) => {
5102 log.discard_capture(capture);
5104 Err(e)
5105 }
5106 }
5107 }
5108
5109 fn rollback_from_action_journal(
5110 &mut self,
5111 journal: &crate::engine::ActionJournal,
5112 invalidation_baseline: InvalidationBaseline,
5113 ) -> Result<(), crate::engine::EditorError> {
5114 self.invalidate_for_action_journal(
5117 journal,
5118 LoggedEditDirection::InverseReplay,
5119 invalidation_baseline,
5120 );
5121 journal.graph.undo(&mut self.graph)?;
5123 self.apply_inverse_row_visibility_events(&journal.graph.events);
5125 self.apply_arrow_undo_batch(&journal.arrow, true);
5127 Ok(())
5128 }
5129
5130 fn rollback_from_change_events(
5131 &mut self,
5132 events: &[crate::engine::ChangeEvent],
5133 invalidation_baseline: InvalidationBaseline,
5134 ) -> Result<(), crate::engine::EditorError> {
5135 use crate::engine::ChangeEvent;
5136
5137 self.invalidate_for_change_events(
5140 events,
5141 LoggedEditDirection::InverseReplay,
5142 invalidation_baseline,
5143 );
5144
5145 self.graph
5147 .authority_set_replay(crate::engine::authority::history::Replay::Undo);
5148 let rolled_back = (|| {
5149 let mut editor = crate::engine::VertexEditor::new(&mut self.graph);
5150 let mut compound_stack: Vec<usize> = Vec::new();
5151 for (i, ev) in events.iter().enumerate().rev() {
5152 match ev {
5153 ChangeEvent::CompoundEnd { depth } => {
5154 compound_stack.push(*depth);
5155 if let Some(description) =
5156 crate::engine::graph::editor::change_log::compound_start_description(
5157 i,
5158 |j| &events[j],
5159 )
5160 {
5161 editor.inverse_compound_end(description);
5162 }
5163 }
5164 ChangeEvent::CompoundStart { depth, .. } => {
5165 if compound_stack.last() == Some(depth) {
5166 compound_stack.pop();
5167 }
5168 editor.apply_inverse(ev.clone())?;
5169 }
5170 ChangeEvent::SetRowVisibility { .. } => {
5171 }
5173 _ => {
5174 editor.apply_inverse(ev.clone())?;
5175 }
5176 }
5177 }
5178 Ok::<_, crate::engine::EditorError>(())
5179 })();
5180 self.graph
5181 .authority_set_replay(crate::engine::authority::history::Replay::Forward);
5182 rolled_back?;
5183
5184 for ev in events.iter().rev() {
5186 self.apply_inverse_row_visibility_event(ev);
5187 }
5188
5189 for ev in events.iter().rev() {
5191 self.mirror_inverse_change_to_arrow(ev);
5192 }
5193
5194 Ok(())
5195 }
5196
5197 fn read_cell_formula_ast(&self, sheet: &str, row: u32, col: u32) -> Option<ASTNode> {
5198 let sheet_id = self.graph.sheet_id(sheet)?;
5199 let coord = Coord::from_excel(row, col, true, true);
5200 let cell = CellRef::new(sheet_id, coord);
5201 let vid = self.graph.get_vertex_for_cell(&cell)?;
5202 self.graph.get_formula(vid)
5203 }
5204
5205 pub fn define_name_with_logger(
5206 &mut self,
5207 log: &mut crate::engine::ChangeLog,
5208 name: &str,
5209 definition: NamedDefinition,
5210 scope: NameScope,
5211 ) -> Result<(), crate::engine::EditorError> {
5212 self.graph
5213 .validate_define_name(name, scope)
5214 .map_err(crate::engine::EditorError::Excel)?;
5215
5216 {
5217 let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
5218 editor.define_name(name, definition, scope)?;
5219 }
5220 self.record_structural_change(StructuralScope::AllSheets);
5221
5222 self.mark_topology_edited();
5223
5224 Ok(())
5225 }
5226
5227 pub fn update_name_with_logger(
5228 &mut self,
5229 log: &mut crate::engine::ChangeLog,
5230 name: &str,
5231 definition: NamedDefinition,
5232 scope: NameScope,
5233 ) -> Result<(), crate::engine::EditorError> {
5234 self.graph
5235 .validate_existing_name(name, scope)
5236 .map_err(crate::engine::EditorError::Excel)?;
5237 {
5238 let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
5239 editor.update_name(name, definition, scope)?;
5240 }
5241 self.record_structural_change(StructuralScope::AllSheets);
5242
5243 self.mark_topology_edited();
5244
5245 Ok(())
5246 }
5247
5248 pub fn delete_name_with_logger(
5249 &mut self,
5250 log: &mut crate::engine::ChangeLog,
5251 name: &str,
5252 scope: NameScope,
5253 ) -> Result<(), crate::engine::EditorError> {
5254 self.graph
5255 .validate_existing_name(name, scope)
5256 .map_err(crate::engine::EditorError::Excel)?;
5257 {
5258 let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
5259 editor.delete_name(name, scope)?;
5260 }
5261 self.record_structural_change(StructuralScope::AllSheets);
5262
5263 self.mark_topology_edited();
5264
5265 Ok(())
5266 }
5267
5268 pub fn edit_with_logger<T>(
5269 &mut self,
5270 log: &mut crate::engine::ChangeLog,
5271 f: impl FnOnce(&mut crate::engine::VertexEditor) -> T,
5272 ) -> Result<T, crate::engine::EditorError> {
5273 let mut capture = MutationCapture::new(log.current_meta());
5274 let result = self.edit_with_capture(&mut capture, f);
5275 capture.close_compounds();
5276 match result {
5277 Ok(value) => {
5278 log.publish_capture(capture);
5279 Ok(value)
5280 }
5281 Err(error) => {
5282 log.discard_capture(capture);
5283 Err(error)
5284 }
5285 }
5286 }
5287
5288 fn edit_with_capture<T>(
5289 &mut self,
5290 capture: &mut MutationCapture,
5291 f: impl FnOnce(&mut crate::engine::VertexEditor) -> T,
5292 ) -> Result<T, crate::engine::EditorError> {
5293 let invalidation_baseline = self.invalidation_baseline();
5294 let start_len = capture.len();
5295 let lazy_start = capture.lazy_len();
5296
5297 struct ArrowSpillReader<'a> {
5300 sheets: &'a crate::arrow_store::SheetStore,
5301 }
5302 impl crate::engine::graph::editor::vertex_editor::SpillValueReader for ArrowSpillReader<'_> {
5303 fn read_cell_value(
5304 &self,
5305 sheet: &str,
5306 row: u32,
5307 col: u32,
5308 ) -> Option<formualizer_common::LiteralValue> {
5309 use formualizer_common::LiteralValue;
5310 let asheet = self.sheets.sheet(sheet)?;
5311 let r0 = row.saturating_sub(1) as usize;
5312 let c0 = col.saturating_sub(1) as usize;
5313 let v = asheet.get_cell_value(r0, c0);
5314 if matches!(v, LiteralValue::Empty) {
5315 None
5316 } else {
5317 Some(v)
5318 }
5319 }
5320 }
5321
5322 let ret = {
5323 let spill_reader = ArrowSpillReader {
5324 sheets: &self.arrow_sheets,
5325 };
5326 let mut editor = crate::engine::VertexEditor::with_capture_and_spill_reader(
5327 &mut self.graph,
5328 capture,
5329 &spill_reader,
5330 );
5331 f(&mut editor)
5332 };
5333
5334 let new_events = capture.events()[start_len..].to_vec();
5338 let new_runs = capture.lazy_len() > lazy_start;
5339 if new_events.iter().any(|event| {
5340 matches!(
5341 event,
5342 ChangeEvent::DefineName { .. }
5343 | ChangeEvent::UpdateName { .. }
5344 | ChangeEvent::DeleteName { .. }
5345 )
5346 }) {
5347 let all = capture.expanded_events_from(start_len, lazy_start);
5348 self.rollback_from_change_events(&all, invalidation_baseline)?;
5349 return Err(crate::engine::EditorError::TransactionUnsupported {
5350 reason: "name mutations must use Engine's prepared logged-name APIs".to_string(),
5351 });
5352 }
5353
5354 self.clear_logged_cell_format_states(&new_events);
5357 for ev in &new_events {
5358 self.mirror_forward_change_to_arrow(ev);
5359 }
5360 for ev in &new_events {
5361 self.record_change_for_event(ev);
5362 }
5363
5364 if self.action_depth == 0 {
5367 let mut impact =
5368 Self::classify_change_events(&new_events, LoggedEditDirection::Original);
5369 if new_runs {
5370 impact = impact.max(LoggedEditImpact::Topology);
5371 }
5372 self.apply_logged_edit_impact(impact, invalidation_baseline);
5373 }
5374
5375 Ok(ret)
5376 }
5377
5378 pub(crate) fn preflight_replay_admission(
5379 &mut self,
5380 events: &[ChangeEvent],
5381 forward: bool,
5382 ) -> Result<(), crate::engine::EditorError> {
5383 if !self.graph_admission_enabled() {
5384 return Ok(());
5385 }
5386 let mut vertex_delta = 0i128;
5387 let mut edge_delta = 0i128;
5388 let mut added_vertices = 0usize;
5389 let mut added_edges = 0usize;
5390 let mut formula_cells = BTreeSet::new();
5391 for event in events {
5392 match event {
5393 ChangeEvent::AddVertex {
5394 formula,
5395 coord,
5396 sheet_id,
5397 ..
5398 } => {
5399 let delta = if forward { 1 } else { -1 };
5400 vertex_delta += delta;
5401 if forward {
5402 added_vertices = added_vertices.saturating_add(1);
5403 if formula.is_some() {
5404 formula_cells.insert((*sheet_id, coord.row(), coord.col()));
5405 }
5406 }
5407 }
5408 ChangeEvent::RemoveVertex {
5409 old_formula,
5410 coord,
5411 sheet_id,
5412 ..
5413 } => {
5414 let delta = if forward { -1 } else { 1 };
5415 vertex_delta += delta;
5416 if !forward {
5417 added_vertices = added_vertices.saturating_add(1);
5418 if old_formula.is_some()
5419 && let (Some(sheet_id), Some(coord)) = (sheet_id, coord)
5420 {
5421 formula_cells.insert((*sheet_id, coord.row(), coord.col()));
5422 }
5423 }
5424 }
5425 ChangeEvent::EdgeAdded { .. } => {
5426 let delta = if forward { 1 } else { -1 };
5427 edge_delta += delta;
5428 if forward {
5429 added_edges = added_edges.saturating_add(1);
5430 }
5431 }
5432 ChangeEvent::EdgeRemoved { .. } => {
5433 let delta = if forward { -1 } else { 1 };
5434 edge_delta += delta;
5435 if !forward {
5436 added_edges = added_edges.saturating_add(1);
5437 }
5438 }
5439 ChangeEvent::SetFormula {
5440 addr, old_formula, ..
5441 } if (forward || old_formula.is_some()) => {
5442 formula_cells.insert((addr.sheet_id, addr.coord.row(), addr.coord.col()));
5443 }
5444 _ => {}
5445 }
5446 }
5447 let stats = self.graph.baseline_stats();
5448 let final_vertices = i128::try_from(stats.graph_vertex_count)
5449 .ok()
5450 .and_then(|count| count.checked_add(vertex_delta))
5451 .and_then(|count| usize::try_from(count).ok())
5452 .ok_or_else(|| {
5453 crate::engine::EditorError::Excel(
5454 ExcelError::new(ExcelErrorKind::NImpl)
5455 .with_message("replay vertex count overflow"),
5456 )
5457 })?;
5458 let final_edges = i128::try_from(stats.graph_edge_count)
5459 .ok()
5460 .and_then(|count| count.checked_add(edge_delta))
5461 .and_then(|count| usize::try_from(count).ok())
5462 .ok_or_else(|| {
5463 crate::engine::EditorError::Excel(
5464 ExcelError::new(ExcelErrorKind::NImpl)
5465 .with_message("replay edge count overflow"),
5466 )
5467 })?;
5468 self.preflight_graph_admission(crate::engine::resource_ledger::GraphAdmission {
5469 final_vertices,
5470 final_edges,
5471 materialization_cells: formula_cells.len() as u64,
5472 added_vertices,
5473 added_edges,
5474 })
5475 .map_err(crate::engine::EditorError::Excel)
5476 }
5477
5478 pub fn undo_logged(
5483 &mut self,
5484 undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5485 log: &mut crate::engine::ChangeLog,
5486 ) -> Result<(), crate::engine::EditorError> {
5487 let pending_events = log
5488 .last_group_indices()
5489 .into_iter()
5490 .map(|index| log.events()[index].clone())
5491 .collect::<Vec<_>>();
5492 self.preflight_replay_admission(&pending_events, false)?;
5493 let invalidation_baseline = self.invalidation_baseline();
5494 self.invalidate_for_change_events(
5497 &pending_events,
5498 LoggedEditDirection::InverseReplay,
5499 invalidation_baseline,
5500 );
5501 let batch = undo.undo(&mut self.graph, log)?;
5502 for item in batch.iter().rev() {
5503 self.apply_inverse_row_visibility_event(&item.event);
5504 self.apply_inverse_staged_formula_event(&item.event);
5505 }
5506 if !batch.is_empty() {
5507 let events = batch
5508 .iter()
5509 .map(|item| item.event.clone())
5510 .collect::<Vec<_>>();
5511 self.clear_logged_cell_format_states(&events);
5512 }
5513 self.mirror_undo_batch_to_arrow(&batch);
5514 if !batch.is_empty() {
5515 for item in &batch {
5516 self.record_change_for_event(&item.event);
5517 }
5518 }
5519 crate::engine::trace::fz_event!(
5520 tracing::Level::INFO,
5521 "history",
5522 "history.replay",
5523 op = "undo",
5524 events_replayed = batch.len(),
5525 ownership_resyncs = batch.len()
5526 );
5527 Ok(())
5528 }
5529
5530 pub fn redo_logged(
5531 &mut self,
5532 undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5533 log: &mut crate::engine::ChangeLog,
5534 ) -> Result<(), crate::engine::EditorError> {
5535 let pending_events = undo.pending_redo_events();
5536 self.preflight_replay_admission(&pending_events, true)?;
5537 let invalidation_baseline = self.invalidation_baseline();
5538 self.invalidate_for_change_events(
5539 &pending_events,
5540 LoggedEditDirection::ForwardReplay,
5541 invalidation_baseline,
5542 );
5543 let batch = undo.redo(&mut self.graph, log)?;
5544 for item in &batch {
5545 self.apply_forward_row_visibility_event(&item.event);
5546 self.apply_forward_staged_formula_event(&item.event);
5547 }
5548 if !batch.is_empty() {
5549 let events = batch
5550 .iter()
5551 .map(|item| item.event.clone())
5552 .collect::<Vec<_>>();
5553 self.clear_logged_cell_format_states(&events);
5554 }
5555 self.mirror_redo_batch_to_arrow(&batch);
5556 if !batch.is_empty() {
5557 for item in &batch {
5558 self.record_change_for_event(&item.event);
5559 }
5560 }
5561 crate::engine::trace::fz_event!(
5562 tracing::Level::INFO,
5563 "history",
5564 "history.replay",
5565 op = "redo",
5566 events_replayed = batch.len(),
5567 ownership_resyncs = batch.len()
5568 );
5569 Ok(())
5570 }
5571
5572 pub fn undo_action(
5576 &mut self,
5577 undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5578 ) -> Result<(), crate::engine::EditorError> {
5579 let Some(journal) = undo.pop_undo_action() else {
5580 return Ok(());
5581 };
5582 if let Err(error) = self.preflight_replay_admission(&journal.graph.events, false) {
5583 undo.push_done_action(journal);
5584 return Err(error);
5585 }
5586 let invalidation_baseline = self.invalidation_baseline();
5587
5588 self.invalidate_for_action_journal(
5589 &journal,
5590 LoggedEditDirection::InverseReplay,
5591 invalidation_baseline,
5592 );
5593 journal.graph.undo(&mut self.graph)?;
5594 self.apply_inverse_row_visibility_events(&journal.graph.events);
5595 self.apply_arrow_undo_batch(&journal.arrow, true);
5596 if !journal.graph.is_empty() || !journal.arrow.is_empty() {
5597 for event in &journal.graph.events {
5598 self.record_change_for_event(event);
5599 }
5600 }
5601
5602 #[cfg(feature = "tracing")]
5603 let events_replayed = journal.graph.events.len();
5604 crate::engine::trace::fz_event!(
5605 tracing::Level::INFO,
5606 "history",
5607 "history.replay",
5608 op = "undo",
5609 events_replayed,
5610 ownership_resyncs = events_replayed
5611 );
5612 undo.push_redo_action(journal);
5613 Ok(())
5614 }
5615
5616 pub fn redo_action(
5620 &mut self,
5621 undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5622 ) -> Result<(), crate::engine::EditorError> {
5623 let Some(journal) = undo.pop_redo_action() else {
5624 return Ok(());
5625 };
5626 if let Err(error) = self.preflight_replay_admission(&journal.graph.events, true) {
5627 undo.push_redo_action(journal);
5628 return Err(error);
5629 }
5630 let invalidation_baseline = self.invalidation_baseline();
5631 self.invalidate_for_action_journal(
5632 &journal,
5633 LoggedEditDirection::ForwardReplay,
5634 invalidation_baseline,
5635 );
5636 journal.graph.redo(&mut self.graph)?;
5637 self.apply_forward_row_visibility_events(&journal.graph.events);
5638 self.apply_arrow_undo_batch(&journal.arrow, false);
5639 if !journal.graph.is_empty() || !journal.arrow.is_empty() {
5640 for event in &journal.graph.events {
5641 self.record_change_for_event(event);
5642 }
5643 }
5644
5645 #[cfg(feature = "tracing")]
5646 let events_replayed = journal.graph.events.len();
5647 crate::engine::trace::fz_event!(
5648 tracing::Level::INFO,
5649 "history",
5650 "history.replay",
5651 op = "redo",
5652 events_replayed,
5653 ownership_resyncs = events_replayed
5654 );
5655 undo.push_done_action(journal);
5656 Ok(())
5657 }
5658
5659 fn cellref_to_sheet_row_col(&self, addr: &crate::reference::CellRef) -> (String, u32, u32) {
5660 let sheet = self.graph.sheet_name(addr.sheet_id).to_string();
5661 let row = addr.coord.row() + 1;
5663 let col = addr.coord.col() + 1;
5664 (sheet, row, col)
5665 }
5666
5667 fn mirror_undo_batch_to_arrow(
5668 &mut self,
5669 batch: &[crate::engine::graph::editor::undo_engine::UndoBatchItem],
5670 ) {
5671 for item in batch.iter().rev() {
5673 self.mirror_inverse_change_to_arrow(&item.event);
5674 }
5675 }
5676
5677 fn mirror_redo_batch_to_arrow(
5678 &mut self,
5679 batch: &[crate::engine::graph::editor::undo_engine::UndoBatchItem],
5680 ) {
5681 for item in batch.iter() {
5683 self.mirror_forward_change_to_arrow(&item.event);
5684 }
5685 }
5686
5687 fn mirror_inverse_change_to_arrow(&mut self, ev: &crate::engine::ChangeEvent) {
5688 use crate::engine::ChangeEvent;
5689 use formualizer_common::LiteralValue;
5690
5691 match ev {
5692 ChangeEvent::SetValue {
5693 addr,
5694 old_value,
5695 old_formula,
5696 ..
5697 } => {
5698 let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5699 if old_formula.is_some() {
5700 self.clear_delta_overlay_cell(&sheet, row, col);
5701 } else {
5702 let v = old_value.clone().unwrap_or(LiteralValue::Empty);
5703 self.mirror_value_to_overlay(&sheet, row, col, &v);
5704 }
5705 }
5706 ChangeEvent::SetFormula {
5707 addr,
5708 old_value,
5709 old_formula,
5710 ..
5711 } => {
5712 let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5713 if old_formula.is_some() {
5714 self.clear_delta_overlay_cell(&sheet, row, col);
5715 } else {
5716 let v = old_value.clone().unwrap_or(LiteralValue::Empty);
5717 self.mirror_value_to_overlay(&sheet, row, col, &v);
5718 }
5719 }
5720 ChangeEvent::SpillCommitted { old, new, .. } => {
5721 self.mirror_spill_snapshot(new, true);
5723 if let Some(snap) = old {
5724 self.mirror_spill_snapshot(snap, false);
5725 }
5726 }
5727 ChangeEvent::SpillCleared { old, .. } => {
5728 self.mirror_spill_snapshot(old, false);
5730 }
5731 ChangeEvent::SetRowVisibility { .. } => {
5732 }
5734 _ => {}
5735 }
5736 }
5737
5738 fn mirror_forward_change_to_arrow(&mut self, ev: &crate::engine::ChangeEvent) {
5739 use crate::engine::ChangeEvent;
5740
5741 match ev {
5742 ChangeEvent::SetValue { addr, new, .. } => {
5743 let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5744 self.mirror_value_to_overlay(&sheet, row, col, new);
5745 }
5746 ChangeEvent::SetFormula { addr, .. } => {
5747 let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5748 self.clear_delta_overlay_cell(&sheet, row, col);
5749 }
5751 ChangeEvent::SpillCommitted { old, new, .. } => {
5752 if let Some(snap) = old {
5753 self.mirror_spill_snapshot(snap, true);
5754 }
5755 self.mirror_spill_snapshot(new, false);
5756 }
5757 ChangeEvent::SpillCleared { old, .. } => {
5758 self.mirror_spill_snapshot(old, true);
5759 }
5760 ChangeEvent::SetRowVisibility { .. } => {
5761 }
5763 _ => {
5764 }
5766 }
5767 }
5768
5769 fn mirror_spill_snapshot(
5770 &mut self,
5771 snap: &crate::engine::graph::editor::change_log::SpillSnapshot,
5772 clear_only: bool,
5773 ) {
5774 use formualizer_common::LiteralValue;
5775
5776 let mut i = 0usize;
5777 for row in &snap.values {
5778 for v in row {
5779 if let Some(cell) = snap.target_cells.get(i) {
5780 let (sheet, r, c) = self.cellref_to_sheet_row_col(cell);
5781 let out = if clear_only {
5782 LiteralValue::Empty
5783 } else {
5784 v.clone()
5785 };
5786 self.mirror_value_to_computed_overlay(&sheet, r, c, &out);
5787 }
5788 i += 1;
5789 }
5790 }
5791 if clear_only {
5793 for cell in snap.target_cells.iter().skip(i) {
5794 let (sheet, r, c) = self.cellref_to_sheet_row_col(cell);
5795 self.mirror_value_to_computed_overlay(&sheet, r, c, &LiteralValue::Empty);
5796 }
5797 }
5798 }
5799
5800 pub fn set_default_sheet_by_name(&mut self, name: &str) {
5801 self.graph.set_default_sheet_by_name(name);
5802 }
5803
5804 pub fn set_default_sheet_by_id(&mut self, id: SheetId) {
5805 self.graph.set_default_sheet_by_id(id);
5806 }
5807
5808 pub fn set_sheet_index_mode(&mut self, mode: crate::engine::SheetIndexMode) {
5809 self.graph.set_sheet_index_mode(mode);
5810 }
5811
5812 #[cfg(feature = "test-support")]
5813 #[doc(hidden)]
5814 pub fn mark_all_formulas_dirty_for_test(&mut self) {
5815 self.mark_all_formula_vertices_dirty();
5816 }
5817
5818 #[cfg(feature = "test-support")]
5819 #[doc(hidden)]
5820 pub fn take_criteria_mask_work_for_test() -> (usize, usize) {
5821 criteria_mask_test_hooks::take_mask_work()
5822 }
5823
5824 #[cfg(feature = "test-support")]
5825 #[doc(hidden)]
5826 pub fn lookup_index_cache_report_for_test(&self) -> LookupIndexCacheReport {
5827 self.lookup_index_cache.report()
5828 }
5829
5830 #[cfg(any(test, feature = "benchmark_internal"))]
5831 #[doc(hidden)]
5832 pub fn reset_recalc_reuse_probe(&mut self) {
5833 *self.recalc_reuse_probe.get_mut().unwrap() = RecalcReuseProbe::default();
5834 }
5835
5836 #[cfg(any(test, feature = "benchmark_internal"))]
5837 #[doc(hidden)]
5838 pub fn recalc_reuse_probe(&self) -> RecalcReuseProbe {
5839 let mut probe = self.recalc_reuse_probe.lock().unwrap().clone();
5840 if let Some(cached) = self.cached_static_schedule.as_ref() {
5841 let entry_bytes = |e: &CachedScheduleEntry| {
5842 std::mem::size_of::<CachedScheduleEntry>()
5843 + e.candidate_vertices.heap_bytes()
5844 + std::mem::size_of::<crate::engine::Schedule>()
5845 + 2 * std::mem::size_of::<usize>()
5846 + schedule_probe_retained_bytes(&e.schedule)
5847 };
5848 probe.schedule_retained_bytes = entry_bytes(cached)
5849 + self.recent_schedules.iter().map(entry_bytes).sum::<usize>()
5850 + self.base_schedule.as_ref().map_or(0, entry_bytes)
5851 + self.recent_schedules.capacity() * std::mem::size_of::<CachedScheduleEntry>();
5852 }
5853 probe
5854 }
5855
5856 #[cfg(test)]
5857 pub(crate) fn cached_static_schedule_for_test(&self) -> Option<Arc<crate::engine::Schedule>> {
5858 self.cached_static_schedule
5859 .as_ref()
5860 .map(|cached| Arc::clone(&cached.schedule))
5861 }
5862
5863 fn clear_cached_static_schedule(&mut self) {
5864 self.cached_static_schedule = None;
5865 self.recent_schedules.clear();
5866 self.base_schedule = None;
5867 }
5868
5869 fn retain_recent_schedule(&mut self, entry: CachedScheduleEntry) {
5872 const RECENT_SCHEDULES: usize = 8;
5873 const RECENT_SCHEDULE_VERTICES: usize = 65_536;
5874 let revision = self.schedule_cache_authority_revision();
5875 let epoch = self.topology_epoch;
5876 self.recent_schedules
5877 .retain(|e| e.topology_epoch == epoch && e.authority_revision == revision);
5878 let current =
5879 |e: &CachedScheduleEntry| e.topology_epoch == epoch && e.authority_revision == revision;
5880 if self.base_schedule.as_ref().is_some_and(|b| !current(b)) {
5881 self.base_schedule = None;
5882 }
5883 let mut entry = entry;
5886 if current(&entry)
5887 && entry.candidate_vertices.len() > BASE_SCHEDULE_MIN_VERTICES
5888 && self
5889 .base_schedule
5890 .as_ref()
5891 .is_none_or(|b| entry.candidate_vertices.len() > b.candidate_vertices.len())
5892 {
5893 match self.base_schedule.replace(entry) {
5894 Some(previous) => entry = previous,
5895 None => return,
5896 }
5897 }
5898 if entry.topology_epoch != epoch
5899 || entry.authority_revision != revision
5900 || entry.candidate_vertices.len() > RECENT_SCHEDULE_VERTICES / 4
5901 {
5902 return;
5903 }
5904 self.recent_schedules.push(entry);
5905 let mut total: usize = self
5906 .recent_schedules
5907 .iter()
5908 .map(|e| e.candidate_vertices.len())
5909 .sum();
5910 while self.recent_schedules.len() > RECENT_SCHEDULES || total > RECENT_SCHEDULE_VERTICES {
5911 let oldest = self.recent_schedules.remove(0);
5912 total -= oldest.candidate_vertices.len();
5913 }
5914 }
5915
5916 fn invalidation_baseline(&self) -> InvalidationBaseline {
5917 InvalidationBaseline {
5918 snapshot_id: self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed),
5919 topology_epoch: self.topology_epoch,
5920 }
5921 }
5922
5923 fn classify_change_events(
5924 events: &[crate::engine::ChangeEvent],
5925 direction: LoggedEditDirection,
5926 ) -> LoggedEditImpact {
5927 use crate::engine::ChangeEvent;
5928
5929 events
5930 .iter()
5931 .map(|event| match event {
5932 ChangeEvent::CompoundStart { .. } | ChangeEvent::CompoundEnd { .. } => {
5933 LoggedEditImpact::NoOp
5934 }
5935 ChangeEvent::SetRowVisibility { .. }
5936 | ChangeEvent::SetValue {
5937 old_formula: None,
5938 old_value: Some(_),
5939 ..
5940 } => LoggedEditImpact::DataOnly,
5941 ChangeEvent::SetValue {
5945 old_formula: None,
5946 old_value: None,
5947 ..
5948 } if direction == LoggedEditDirection::Original => LoggedEditImpact::DataOnly,
5949 ChangeEvent::SetValue { .. }
5950 | ChangeEvent::SetFormula { .. }
5951 | ChangeEvent::AddVertex { .. }
5952 | ChangeEvent::RemoveVertex { .. }
5953 | ChangeEvent::VertexMoved { .. }
5954 | ChangeEvent::FormulaAdjusted { .. }
5955 | ChangeEvent::NamedRangeAdjusted { .. }
5956 | ChangeEvent::EdgeAdded { .. }
5957 | ChangeEvent::EdgeRemoved { .. }
5958 | ChangeEvent::DefineName { .. }
5959 | ChangeEvent::UpdateName { .. }
5960 | ChangeEvent::DeleteName { .. }
5961 | ChangeEvent::SpillCommitted { .. }
5962 | ChangeEvent::SpillCleared { .. }
5963 | ChangeEvent::StagedFormulaCellChanged { .. } => LoggedEditImpact::Topology,
5964 })
5965 .max()
5966 .unwrap_or(LoggedEditImpact::NoOp)
5967 }
5968
5969 fn classify_arrow_undo(arrow: &crate::engine::ArrowUndoBatch) -> LoggedEditImpact {
5970 use crate::engine::ArrowOp;
5971
5972 arrow
5973 .ops
5974 .iter()
5975 .map(|op| match op {
5976 ArrowOp::SetDeltaCell { .. } | ArrowOp::SetComputedCell { .. } => {
5977 LoggedEditImpact::DataOnly
5978 }
5979 ArrowOp::RestoreComputedRect { .. }
5980 | ArrowOp::InsertRows { .. }
5981 | ArrowOp::InsertCols { .. } => LoggedEditImpact::Topology,
5982 })
5983 .max()
5984 .unwrap_or(LoggedEditImpact::NoOp)
5985 }
5986
5987 fn apply_logged_edit_impact(
5988 &mut self,
5989 impact: LoggedEditImpact,
5990 baseline: InvalidationBaseline,
5991 ) {
5992 match impact {
5993 LoggedEditImpact::NoOp => {}
5994 LoggedEditImpact::DataOnly => {
5995 if self.topology_epoch == baseline.topology_epoch
5996 && self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed)
5997 == baseline.snapshot_id
5998 {
5999 self.mark_data_edited();
6000 }
6001 }
6002 LoggedEditImpact::Topology => {
6003 if self.topology_epoch == baseline.topology_epoch {
6006 self.mark_topology_edited();
6007 }
6008 }
6009 }
6010 }
6011
6012 fn invalidate_for_change_events(
6013 &mut self,
6014 events: &[crate::engine::ChangeEvent],
6015 direction: LoggedEditDirection,
6016 baseline: InvalidationBaseline,
6017 ) {
6018 self.apply_logged_edit_impact(Self::classify_change_events(events, direction), baseline);
6019 }
6020
6021 fn invalidate_for_action_journal(
6022 &mut self,
6023 journal: &crate::engine::ActionJournal,
6024 direction: LoggedEditDirection,
6025 baseline: InvalidationBaseline,
6026 ) {
6027 let impact = Self::classify_change_events(&journal.graph.events, direction)
6028 .max(Self::classify_arrow_undo(&journal.arrow));
6029 self.apply_logged_edit_impact(impact, baseline);
6030 }
6031
6032 pub fn mark_data_edited(&mut self) {
6035 self.lookup_index_cache.clear();
6036 self.snapshot_id
6037 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
6038 self.has_edited = true;
6039 }
6040
6041 pub fn mark_topology_edited(&mut self) {
6043 self.arrow_sheet_index.take();
6044 self.lookup_index_cache.clear();
6045 self.snapshot_id
6046 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
6047 self.topology_epoch = self.topology_epoch.wrapping_add(1);
6048 self.graph.bump_topology_revision();
6049 self.clear_cached_static_schedule();
6050 if let Some(ledger) = self.active_resource_ledger.as_mut() {
6051 let released = ledger.account_mixed_cache(0);
6052 debug_assert!(released.is_ok());
6053 }
6054 self.has_edited = true;
6055 self.graph.authority_sync_eager();
6059 }
6060
6061 fn mark_all_formula_vertices_dirty(&mut self) {
6062 let vertices: Vec<VertexId> = self.graph.vertices_with_formulas().collect();
6063 for vertex in vertices {
6064 self.graph.mark_vertex_dirty(vertex);
6065 }
6066 }
6067
6068 fn mark_moved_formula_vertices_dirty(
6069 &mut self,
6070 summary: &crate::engine::graph::editor::vertex_editor::ShiftSummary,
6071 ) {
6072 for vertex in &summary.vertices_moved {
6073 if self.graph.has_formula(*vertex) {
6074 self.graph.mark_vertex_dirty(*vertex);
6075 }
6076 }
6077 }
6078
6079 pub fn sheet_store(&self) -> &SheetStore {
6081 &self.arrow_sheets
6082 }
6083
6084 pub(crate) fn has_row_visibility_state(&self) -> bool {
6087 !self.row_visibility.is_empty()
6088 }
6089
6090 pub fn sheet_store_mut(&mut self) -> &mut SheetStore {
6092 self.arrow_sheet_index.take();
6093 &mut self.arrow_sheets
6094 }
6095
6096 pub fn has_staged_formulas(&self) -> bool {
6097 !self.staged_formulas.is_empty()
6098 }
6099
6100 pub fn staged_formula_count(&self) -> usize {
6101 self.staged_formulas.values().map(StagedSheet::len).sum()
6102 }
6103
6104 pub fn stage_formula_text(&mut self, sheet: &str, row: u32, col: u32, text: String) {
6106 self.staged_formulas
6107 .entry(sheet.to_string())
6108 .or_default()
6109 .stage(row, col, text);
6110 self.staged_formula_index.stage(sheet, row, col);
6111 if let Some(sheet) = self.graph.sheet_id(sheet) {
6112 self.invalidate_pending_spills(StructuralScope::Cell {
6113 sheet,
6114 row: row.saturating_sub(1),
6115 col: col.saturating_sub(1),
6116 });
6117 }
6118 }
6119
6120 fn index_removed_staged_sheet(&mut self, sheet: &str, staged: &StagedSheet) {
6121 for (row, col, _) in &staged.entries {
6122 self.staged_formula_index.remove(sheet, *row, *col);
6123 }
6124 if staged.deferred_package.is_some() {
6125 self.staged_formula_index.set_package(sheet, None);
6126 }
6127 }
6128
6129 fn restore_staged_sheet(&mut self, sheet: String, staged: StagedSheet) {
6130 self.staged_formulas.insert(sheet, staged);
6131 }
6132
6133 #[doc(hidden)]
6134 pub fn source_formula_ingress(&mut self) -> SourceFormulaIngress<'_, R> {
6135 SourceFormulaIngress { engine: self }
6136 }
6137
6138 #[doc(hidden)]
6139 #[doc(hidden)]
6142 #[cfg(any(test, feature = "test-support"))]
6143 pub fn set_before_prepared_span_commit_hook(
6144 &mut self,
6145 hook: impl FnOnce() + Send + Sync + 'static,
6146 ) {
6147 self.before_prepared_span_commit_hook = Some(Box::new(hook));
6148 }
6149
6150 #[doc(hidden)]
6151 #[cfg(test)]
6153 pub(crate) fn set_before_target_preparation_commit_hook(
6154 &mut self,
6155 hook: impl FnOnce() + Send + Sync + 'static,
6156 ) {
6157 self.before_target_preparation_commit_hook = Some(Box::new(hook));
6158 }
6159
6160 fn stage_deferred_formula_package(&mut self, package: crate::engine::DeferredFormulaPackage) {
6161 if let Ok(replay) = package.replay.lock()
6162 && let Some(footprint) = replay.selection_cache_footprint()
6163 && !self
6164 .source_cache_footprints
6165 .iter()
6166 .any(|known| known.ptr_eq(&footprint))
6167 {
6168 self.source_cache_footprints.push(footprint);
6169 }
6170 let sheet = package.sheet_name.clone();
6171 let staged = self.staged_formulas.entry(sheet.clone()).or_default();
6172 debug_assert!(staged.deferred_package.is_none());
6173 staged.deferred_package = Some(package);
6174 staged.reconcile_attached_deferred_package();
6175 self.staged_formula_index
6176 .set_package(&sheet, staged.deferred_package.as_ref());
6177 if let Some(sheet_id) = self.graph.sheet_id(&sheet) {
6178 let package = self
6179 .staged_formulas
6180 .get(&sheet)
6181 .and_then(|staged| staged.deferred_package.as_ref());
6182 for &(vertex, anchor, region) in &self.blocked_pending_spills {
6183 if anchor.sheet_id == sheet_id
6184 && self.graph.vertex_exists(vertex)
6185 && self.graph.get_cell_ref(vertex) == Some(anchor)
6186 && self.staged_formula_index.package_occupies_spill(
6187 &sheet,
6188 (anchor.coord.row() + 1, anchor.coord.col() + 1),
6189 (
6190 region.rows.query_bounds().1 + 1,
6191 region.cols.query_bounds().1 + 1,
6192 ),
6193 |point| package.is_some_and(|package| package.suppressed.contains(&point)),
6194 )
6195 {
6196 self.graph.mark_vertex_dirty(vertex);
6197 }
6198 }
6199 }
6200 }
6201
6202 pub fn clear_staged_formula_text(&mut self, sheet: &str, row: u32, col: u32) -> Option<String> {
6203 let mut removed = None;
6204 let mut remove_sheet = false;
6205 let mut had_package = false;
6206 if let Some(entries) = self.staged_formulas.get_mut(sheet) {
6207 had_package = entries.deferred_package.is_some();
6208 removed = entries.remove(row, col);
6209 remove_sheet = entries.is_empty();
6210 }
6211 let ordinary_removed = self.staged_formula_index.remove(sheet, row, col);
6212 if !ordinary_removed && had_package {
6213 self.staged_formula_index.touch_package(sheet);
6214 }
6215 if remove_sheet {
6216 self.staged_formulas.remove(sheet);
6217 self.staged_formula_index.set_package(sheet, None);
6218 }
6219 if (ordinary_removed || had_package)
6220 && let Some(sheet) = self.graph.sheet_id(sheet)
6221 {
6222 self.invalidate_pending_spills(StructuralScope::Cell {
6223 sheet,
6224 row: row.saturating_sub(1),
6225 col: col.saturating_sub(1),
6226 });
6227 }
6228 removed
6229 }
6230
6231 pub fn clear_staged_formulas_for_sheet(&mut self, sheet: &str) {
6232 if self.staged_formulas.remove(sheet).is_some() {
6233 self.staged_formula_index.clear_sheet(sheet);
6234 if let Some(sheet_id) = self.graph.sheet_id(sheet) {
6235 self.invalidate_pending_spills(StructuralScope::Sheet(sheet_id));
6236 }
6237 }
6238 }
6239
6240 pub fn rename_staged_formula_sheet(&mut self, old: &str, new: &str) {
6241 let Some(entries) = self.staged_formulas.remove(old) else {
6242 return;
6243 };
6244 self.staged_formula_index.clear_sheet(old);
6245 let (formulas, mut package) = entries.into_parts();
6246 for (row, col, text) in formulas {
6247 self.stage_formula_text(new, row, col, text);
6248 }
6249 if let Some(package) = package.as_mut() {
6250 package.sheet_name = new.to_string();
6251 }
6252 if let Some(package) = package {
6253 self.stage_deferred_formula_package(package);
6254 }
6255 }
6256
6257 pub fn get_staged_formula_text(&self, sheet: &str, row: u32, col: u32) -> Option<String> {
6259 self.staged_formulas
6260 .get(sheet)
6261 .and_then(|v| v.get(row, col))
6262 }
6263
6264 pub fn formula_parse_diagnostics(&self) -> &[FormulaParseDiagnostic] {
6265 &self.formula_parse_diagnostics
6266 }
6267
6268 pub fn take_formula_parse_diagnostics(&mut self) -> Vec<FormulaParseDiagnostic> {
6269 std::mem::take(&mut self.formula_parse_diagnostics)
6270 }
6271
6272 pub fn clear_formula_parse_diagnostics(&mut self) {
6273 self.formula_parse_diagnostics.clear();
6274 }
6275
6276 pub fn last_formula_ingest_report(&self) -> Option<&FormulaIngestReport> {
6277 self.last_formula_ingest_report.as_ref()
6278 }
6279
6280 pub fn formula_ingest_report_total(&self) -> &FormulaIngestReport {
6281 &self.formula_ingest_report_total
6282 }
6283
6284 #[cfg(test)]
6285 pub(crate) fn set_before_target_planning_snapshot_hook_for_test(
6286 &mut self,
6287 hook: impl FnOnce() + Send + Sync + 'static,
6288 ) {
6289 self.before_target_planning_snapshot_hook = Some(Box::new(hook));
6290 }
6291
6292 #[cfg(test)]
6293 pub(crate) fn inject_target_semantic_stale_once_for_test(&mut self) {
6294 self.inject_target_semantic_stale_once_for_test = true;
6295 }
6296
6297 #[cfg(test)]
6298 pub(crate) fn force_virtual_dep_changes_for_test(&mut self, rounds: usize) {
6299 self.force_virtual_dep_changes_remaining_for_test = rounds;
6300 }
6301
6302 #[cfg(test)]
6303 pub(crate) fn fail_evaluation_commit_preflight_once_for_test(&mut self) {
6304 self.fail_evaluation_commit_preflight_once_for_test = true;
6305 }
6306
6307 #[cfg(test)]
6308 pub(crate) fn set_target_preparation_fault_for_test(
6309 &mut self,
6310 fault: crate::engine::target_preparation::TargetPreparationFault,
6311 ) {
6312 self.target_preparation_fault_for_test = Some(fault);
6313 }
6314
6315 #[cfg(test)]
6316 pub(crate) fn staged_formula_index_revision_for_test(&self) -> u64 {
6317 self.staged_formula_index.revision()
6318 }
6319
6320 #[cfg(test)]
6321 pub(crate) fn deferred_package_for_test(
6322 &self,
6323 sheet: &str,
6324 ) -> &crate::engine::DeferredFormulaPackage {
6325 self.staged_formulas
6326 .get(sheet)
6327 .unwrap()
6328 .deferred_package
6329 .as_ref()
6330 .unwrap()
6331 }
6332
6333 #[cfg(test)]
6334 pub(crate) fn staged_formula_index_is_consistent_for_test(&self) -> bool {
6335 let ordinary_storage = self
6336 .staged_formulas
6337 .values()
6338 .map(|sheet| sheet.entries.len())
6339 .sum::<usize>();
6340 let package_storage = self
6341 .staged_formulas
6342 .values()
6343 .filter(|sheet| sheet.deferred_package.is_some())
6344 .count();
6345 ordinary_storage == self.staged_formula_index.ordinary_count()
6346 && package_storage == self.staged_formula_index.package_count()
6347 && self.staged_formulas.iter().all(|(name, sheet)| {
6348 sheet.entries.iter().all(|(row, col, _)| {
6349 let leases = self
6350 .staged_formula_index
6351 .leases_in_region(name, *row, *col, *row, *col);
6352 leases.len() == 1 && leases[0].row == *row && leases[0].col == *col
6353 })
6354 })
6355 }
6356
6357 #[cfg(test)]
6358 pub(crate) fn evaluation_request_begin_count_for_test(&self) -> u64 {
6359 self.evaluation_request_begin_count_for_test
6360 }
6361
6362 #[cfg(test)]
6363 pub(crate) fn set_before_legacy_fallback_final_provider_sample_hook(
6364 &mut self,
6365 hook: impl FnOnce() + Send + Sync + 'static,
6366 ) {
6367 self.before_legacy_fallback_final_provider_sample_hook = Some(Box::new(hook));
6368 }
6369
6370 #[cfg(test)]
6371 pub(crate) fn set_after_eager_proposal_commit_hook(
6372 &mut self,
6373 hook: impl FnOnce() + Send + Sync + 'static,
6374 ) {
6375 self.after_eager_proposal_commit_hook = Some(Box::new(hook));
6376 }
6377
6378 #[cfg(test)]
6379 pub(crate) fn topology_epoch_for_test(&self) -> u64 {
6380 self.topology_epoch
6381 }
6382
6383 #[cfg(test)]
6384 pub(crate) fn graph_topology_revision_for_test(&self) -> u64 {
6385 self.graph.topology_revision()
6386 }
6387
6388 fn record_formula_ingest_report(&mut self, report: FormulaIngestReport) {
6389 self.formula_ingest_report_total.mode = report.mode;
6390 self.formula_ingest_report_total.accumulate(&report);
6391 self.last_formula_ingest_report = Some(report);
6392 }
6393
6394 fn collect_planning_function_requests(
6395 ast: &ASTNode,
6396 requests: &mut Vec<(String, String, usize)>,
6397 ) {
6398 match &ast.node_type {
6399 ASTNodeType::Function { name, args } => {
6400 requests.push((String::new(), name.clone(), args.len()));
6401 for arg in args {
6402 Self::collect_planning_function_requests(arg, requests);
6403 }
6404 }
6405 ASTNodeType::BinaryOp { left, right, .. } => {
6406 Self::collect_planning_function_requests(left, requests);
6407 Self::collect_planning_function_requests(right, requests);
6408 }
6409 ASTNodeType::UnaryOp { expr, .. } => {
6410 Self::collect_planning_function_requests(expr, requests);
6411 }
6412 ASTNodeType::Call { callee, args } => {
6413 Self::collect_planning_function_requests(callee, requests);
6414 for arg in args {
6415 Self::collect_planning_function_requests(arg, requests);
6416 }
6417 }
6418 ASTNodeType::Array(rows) => {
6419 for cell in rows.iter().flatten() {
6420 Self::collect_planning_function_requests(cell, requests);
6421 }
6422 }
6423 ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => {}
6424 }
6425 }
6426
6427 fn prepared_function_semantics_changed(
6428 &self,
6429 preparation: &crate::engine::FormulaCompressedPreparation,
6430 guard: &crate::function_registry::SemanticEpochReadGuard,
6431 ) -> bool {
6432 if preparation.function_semantic_epoch == guard.epoch() {
6433 return false;
6434 }
6435
6436 guard
6437 .semantic_changes_affect_requests_since(preparation.function_semantic_epoch, Vec::new())
6438 }
6439
6440 pub(crate) fn prepare_source_formula_families(
6441 &mut self,
6442 sheet_name: &str,
6443 families: &[crate::engine::SourceFormulaFamily],
6444 ) -> crate::engine::FormulaCompressedPreparation {
6445 crate::engine::FormulaCompressedPreparation {
6446 engine_token: Arc::clone(&self.source_formula_token),
6447 function_semantic_epoch: crate::function_registry::semantic_epoch(),
6448 function_provider_revision: None,
6449 function_semantics_used: false,
6450 sheet_name: Arc::from(sheet_name),
6451 rejected: BTreeMap::new(),
6452 eager_replay: Vec::new(),
6453 preparation_spool_replays: 0,
6454 clean_rejected_anchor_counts: [0; 3],
6455 fragmented_rejected_anchor_counts: [0; 3],
6456 exact_replay: None,
6457 replay_disposition: crate::engine::FormulaReplayDisposition::default(),
6458 }
6459 }
6460
6461 fn prepare_source_formula_proposals(
6462 &mut self,
6463 sheet_name: &str,
6464 families: &[crate::engine::SourceFormulaFamily],
6465 authority_partitions: &[crate::engine::PartitionedSourceFormulaFamily],
6466 replay_partitions: &[crate::engine::PartitionedSourceFormulaFamily],
6467 formula_record_count: u64,
6468 replay: Arc<std::sync::Mutex<Box<dyn crate::engine::DeferredFormulaReplay>>>,
6469 suppressed: &BTreeSet<(u32, u32)>,
6470 consumed: &FxHashSet<(crate::engine::SourceFamilyId, crate::engine::SourceCoord)>,
6471 consumed_engine: Option<&Arc<()>>,
6472 ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
6473 let mut preparation = self.prepare_source_formula_families(sheet_name, families);
6474 preparation.exact_replay = Some(Arc::clone(&replay));
6475 preparation
6476 .replay_disposition
6477 .register_consumed_members(consumed_engine, consumed.iter().copied());
6478 if !preparation
6479 .replay_disposition
6480 .consumed_engine_matches(&self.source_formula_token)
6481 {
6482 return Err(ExcelError::new(ExcelErrorKind::Value)
6483 .with_message("ResidualConsumedEngineMismatch"));
6484 }
6485 preparation
6486 .replay_disposition
6487 .extend_suppressed_excel_coords(suppressed.iter().copied());
6488 Ok(preparation)
6489 }
6490
6491 fn formula_batch_from_exact_replay(
6492 &mut self,
6493 sheet_name: &str,
6494 replayed: impl IntoIterator<Item = crate::engine::DeferredReplayFormula>,
6495 ) -> Result<FormulaIngestBatch, ExcelError> {
6496 let mut cache = rustc_hash::FxHashMap::default();
6497 let mut formulas = Vec::new();
6498 for record in replayed {
6499 let key = if record.text.starts_with('=') {
6500 record.text
6501 } else {
6502 format!("={}", record.text)
6503 };
6504 let ast_id = if let Some(cached) = cache.get(&key) {
6505 *cached
6506 } else {
6507 let parsed = match formualizer_parse::parser::parse(&key) {
6508 Ok(parsed) => parsed,
6509 Err(error) => {
6510 let Some(parsed) = self.handle_formula_parse_error(
6511 sheet_name,
6512 record.row,
6513 record.col,
6514 &key,
6515 error.to_string(),
6516 )?
6517 else {
6518 continue;
6519 };
6520 parsed
6521 }
6522 };
6523 let ast_id = self.intern_formula_ast(&parsed);
6524 cache.insert(key.clone(), ast_id);
6525 ast_id
6526 };
6527 formulas.push(
6528 FormulaIngestRecord::new(record.row, record.col, ast_id, Some(Arc::from(key)))
6529 .with_source_proof(record.source_order, record.family, record.partition_owner),
6530 );
6531 }
6532 Ok(FormulaIngestBatch::new(sheet_name.to_string(), formulas))
6533 }
6534
6535 fn prepare_target_combined_legacy_graph(
6536 &self,
6537 packages: &[PreparedTargetSourcePackage],
6538 ordinary: &[PreparedOrdinaryStagedFormula],
6539 ) -> Result<(PreparedLegacyGraphPlan, usize), ExcelError> {
6540 let mut planned_by_coord = BTreeMap::new();
6541 for package in packages {
6542 for (row, col, ast_id, plan) in &package.legacy {
6543 planned_by_coord.insert((package.sheet_id, *row, *col), (*ast_id, plan.clone()));
6544 }
6545 }
6546 for formula in ordinary {
6547 if let Some((ast_id, plan)) = formula.ast_id.zip(formula.plan.clone()) {
6548 planned_by_coord.insert(
6549 (formula.sheet_id, formula.lease.row, formula.lease.col),
6550 (ast_id, plan),
6551 );
6552 }
6553 }
6554 let planned = planned_by_coord
6555 .into_iter()
6556 .map(|((sheet_id, row, col), (ast_id, plan))| (sheet_id, row, col, ast_id, plan))
6557 .collect::<Vec<_>>();
6558 let formula_count = planned.len();
6559 let graph = self
6560 .graph
6561 .prepare_legacy_graph_plan_multi_sheet(planned)
6562 .map_err(|error| {
6563 ExcelError::new(ExcelErrorKind::Value)
6564 .with_message(format!("target graph preparation failed: {error}"))
6565 })?;
6566 Ok((graph, formula_count))
6567 }
6568
6569 fn replay_target_coordinates(
6570 &mut self,
6571 replay: &Arc<std::sync::Mutex<Box<dyn crate::engine::DeferredFormulaReplay>>>,
6572 coordinates: &[(u32, u32)],
6573 deadline: Option<std::time::Instant>,
6574 scratch: &mut u64,
6575 ) -> Result<Option<Vec<crate::engine::DeferredReplayFormula>>, ExcelError> {
6576 if coordinates.is_empty() {
6577 return Ok(Some(Vec::new()));
6578 }
6579 let mut guard = replay.lock().map_err(|_| {
6580 ExcelError::new(ExcelErrorKind::Value)
6581 .with_message("deferred formula spool lock poisoned")
6582 })?;
6583 let retained = guard.selection_cache_footprint().is_some();
6584 let records = guard.replay_selected_exact(coordinates, &mut |work, bytes| {
6585 self.target_preparation_checkpoint(deadline, work)?;
6586 if retained && let Some(ledger) = self.active_resource_ledger.as_mut() {
6587 ledger
6588 .reserve_retained(bytes)
6589 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
6590 self.source_cache_accounted = self.source_cache_accounted.saturating_add(bytes);
6591 }
6592 self.reserve_graph_source_scratch(bytes)?;
6593 *scratch = scratch.saturating_add(bytes);
6594 Ok(())
6595 });
6596 drop(guard);
6597 let reconciled = self.reconcile_source_cache_footprints();
6598 let records = records?;
6599 reconciled?;
6600 Ok(records)
6601 }
6602
6603 fn prepare_target_exact_source_selection(
6604 &mut self,
6605 sheet: &str,
6606 lease: StagedPackageLease,
6607 coordinates: Vec<(u32, u32)>,
6608 previous: &BTreeSet<(u32, u32)>,
6609 allow_partial_shared: bool,
6610 deadline: Option<std::time::Instant>,
6611 scratch: &mut u64,
6612 ) -> Result<Option<PreparedTargetSourcePackage>, ExcelError> {
6613 let package = self
6614 .staged_formulas
6615 .get(sheet)
6616 .and_then(|s| s.deferred_package.as_ref())
6617 .unwrap();
6618 if !package.source_geometry_complete
6619 || ((!package.families.is_empty() || !package.partitioned_families.is_empty())
6620 && !package.coordinates_cover_families)
6621 || package.reconciliation_replay.is_some()
6622 {
6623 return Ok(None);
6624 }
6625 let selected_points: BTreeSet<_> = coordinates
6626 .into_iter()
6627 .filter(|point| !package.suppressed.contains(point))
6628 .collect();
6629 let mut prepared = PreparedTargetSourcePackage::empty_selection(
6630 sheet,
6631 self.graph.sheet_id(sheet).unwrap(),
6632 lease,
6633 );
6634 if selected_points.is_empty() {
6635 return Ok(Some(prepared));
6636 }
6637 let mut routing = crate::engine::FormulaReplayDisposition::default();
6638 for partition in &package.partitioned_families {
6639 routing
6640 .register_partition(partition, false)
6641 .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?;
6642 }
6643 routing.extend_suppressed_excel_coords(package.suppressed.iter().copied());
6644 routing.register_consumed_members(
6645 package.consumed_engine.as_ref(),
6646 package.consumed_members.iter().copied(),
6647 );
6648 let replay = Arc::clone(&package.replay);
6649 let source_report = package.accounting_report();
6650 prepared.source_report = source_report;
6651 prepared.disposition = routing.clone();
6652
6653 let replay_points: BTreeSet<_> = selected_points
6654 .iter()
6655 .copied()
6656 .filter(|&(row, col)| !prepared.direct_contains(row, col))
6657 .collect();
6658 let coordinates: Vec<_> = replay_points.iter().copied().collect();
6659 let records = self.replay_target_coordinates(&replay, &coordinates, deadline, scratch)?;
6660 prepared.spool_replays = u64::from(!coordinates.is_empty());
6661 let Some(mut replay_records) = records else {
6662 return Ok(None);
6663 };
6664 for record in &mut replay_records {
6665 if record.family.is_none() {
6666 record.partition_owner = routing
6667 .ordinary_disposition(crate::engine::SourceCoord {
6668 row: record.row.saturating_sub(1),
6669 col: record.col.saturating_sub(1),
6670 })
6671 .1;
6672 }
6673 }
6674 replay_records.sort_by_key(|record| record.source_order);
6675 if replay_records
6676 .iter()
6677 .any(|record| !selected_points.contains(&(record.row, record.col)))
6678 || replay_records
6679 .windows(2)
6680 .any(|records| records[0].source_order == records[1].source_order)
6681 {
6682 return Err(ExcelError::new(ExcelErrorKind::Value)
6683 .with_message("invalid indexed exact source selection"));
6684 }
6685 let represented: BTreeSet<_> = replay_records.iter().map(|r| (r.row, r.col)).collect();
6686 if represented != replay_points {
6687 return Err(ExcelError::new(ExcelErrorKind::Value)
6688 .with_message("incomplete indexed exact source selection"));
6689 }
6690 let source = self
6693 .staged_formulas
6694 .get(sheet)
6695 .unwrap()
6696 .deferred_package
6697 .as_ref()
6698 .unwrap();
6699 let contains = |rect: crate::engine::SourceRect, coord: crate::engine::SourceCoord| {
6700 coord.row >= rect.start.row
6701 && coord.row <= rect.end.row
6702 && coord.col >= rect.start.col
6703 && coord.col <= rect.end.col
6704 };
6705 let mut checked = BTreeSet::new();
6706 for record in replay_records.iter().rev() {
6707 if !checked.insert((record.row, record.col)) {
6708 continue;
6709 }
6710 let coord = crate::engine::SourceCoord {
6711 row: record.row - 1,
6712 col: record.col - 1,
6713 };
6714 let agrees = |owner, shared: bool| {
6715 record.family == shared.then_some(owner)
6716 && record.partition_owner.or(record.family) == Some(owner)
6717 };
6718 let mut valid = true;
6719 for family in &source.families {
6720 let owns = match &family.members {
6721 crate::engine::SourceFamilyMembers::CompleteDomain(domain) => {
6722 contains(domain.rect(), coord)
6723 }
6724 crate::engine::SourceFamilyMembers::ExplicitMembers(members) => {
6725 members.as_slice().binary_search(&coord).is_ok()
6726 }
6727 };
6728 if owns && !agrees(family.source_id, true) {
6729 valid = false;
6730 }
6731 }
6732 for family in &source.partitioned_families {
6733 if family
6734 .fragments
6735 .iter()
6736 .any(|fragment| contains(fragment.rect(), coord))
6737 && !agrees(family.source_id, true)
6738 {
6739 valid = false;
6740 }
6741 if let Some(member) = family
6742 .legacy_members
6743 .as_slice()
6744 .iter()
6745 .find(|member| member.coord == coord)
6746 && !agrees(
6747 family.source_id,
6748 member.kind == crate::engine::PartitionLegacyMemberKind::SharedFamilyMember,
6749 )
6750 {
6751 valid = false;
6752 }
6753 }
6754 if !valid {
6755 return Err(ExcelError::new(ExcelErrorKind::Value)
6756 .with_message("indexed source ownership mismatch"));
6757 }
6758 }
6759 prepared.selected_points = Some(selected_points);
6760 prepared.replay_records = replay_records;
6761 Ok(Some(prepared))
6762 }
6763
6764 fn prepare_target_source_package(
6765 &mut self,
6766 sheet: &str,
6767 lease: StagedPackageLease,
6768 deadline: Option<std::time::Instant>,
6769 ) -> Result<PreparedTargetSourcePackage, ExcelError> {
6770 let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
6771 ExcelError::new(ExcelErrorKind::Ref)
6772 .with_message(format!("deferred source sheet not found: {sheet}"))
6773 })?;
6774 let (
6775 source_report,
6776 families,
6777 partitions,
6778 replay,
6779 invalidated,
6780 suppressed,
6781 consumed_members,
6782 consumed_engine,
6783 reconciliation_replay,
6784 ) = {
6785 let package = self
6786 .staged_formulas
6787 .get(sheet)
6788 .and_then(|staged| staged.deferred_package.as_ref())
6789 .ok_or_else(|| {
6790 ExcelError::new(ExcelErrorKind::Value)
6791 .with_message("staged deferred source package is unavailable")
6792 })?;
6793 if package.sheet_name != sheet {
6794 return Err(ExcelError::new(ExcelErrorKind::Value)
6795 .with_message("deferred formula package sheet mismatch"));
6796 }
6797 (
6798 package.accounting_report(),
6799 package.families.clone(),
6800 package.partitioned_families.clone(),
6801 Arc::clone(&package.replay),
6802 package.invalidated.clone(),
6803 package.suppressed.clone(),
6804 package.consumed_members.clone(),
6805 package.consumed_engine.clone(),
6806 package.reconciliation_replay.clone(),
6807 )
6808 };
6809
6810 let mut replay_disposition = crate::engine::FormulaReplayDisposition::default();
6811 for partition in &partitions {
6812 replay_disposition
6813 .register_partition(partition, false)
6814 .map_err(|reason| ExcelError::new(ExcelErrorKind::Value).with_message(reason))?;
6815 }
6816 replay_disposition.extend_suppressed_excel_coords(suppressed.iter().copied());
6817 replay_disposition
6818 .register_consumed_members(consumed_engine.as_ref(), consumed_members.iter().copied());
6819 if !replay_disposition.consumed_engine_matches(&self.source_formula_token) {
6820 return Err(ExcelError::new(ExcelErrorKind::Value)
6821 .with_message("ResidualConsumedEngineMismatch"));
6822 }
6823 self.target_preparation_checkpoint(deadline, 1)?;
6824 let mut replay_records = if let Some(mut records) = reconciliation_replay {
6825 records.retain(|record| {
6826 let Some((row, col)) = record.row.checked_sub(1).zip(record.col.checked_sub(1))
6827 else {
6828 return true;
6829 };
6830 let coord = crate::engine::SourceCoord { row, col };
6831 let disposition = record.family.map_or_else(
6832 || replay_disposition.ordinary_disposition(coord).0,
6833 |family| replay_disposition.shared_disposition(family, coord),
6834 );
6835 !matches!(
6836 disposition,
6837 crate::engine::FormulaReplayCoordinateDisposition::Direct
6838 | crate::engine::FormulaReplayCoordinateDisposition::Suppressed
6839 )
6840 });
6841 records
6842 } else {
6843 replay
6844 .lock()
6845 .map_err(|_| {
6846 ExcelError::new(ExcelErrorKind::Value)
6847 .with_message("deferred formula spool lock poisoned")
6848 })?
6849 .replay_partitioned(&replay_disposition, &partitions)
6850 .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?
6851 };
6852 replay_records.sort_by_key(|record| record.source_order);
6853 for chunk in replay_records.chunks(256) {
6854 self.target_preparation_checkpoint(deadline, chunk.len() as u64)?;
6855 }
6856 if replay_records
6857 .windows(2)
6858 .any(|records| records[0].source_order == records[1].source_order)
6859 {
6860 return Err(ExcelError::new(ExcelErrorKind::Value)
6861 .with_message("duplicate deferred source-order proof"));
6862 }
6863
6864 let disposition = replay_disposition;
6865 let direct_families = 0usize;
6866 let direct_cells = 0u64;
6867 let direct_fragments = 0u64;
6868 let direct_complete_families = 0u64;
6869 let direct_complete_cells = 0u64;
6870 let direct_partition_families = 0u64;
6871 let direct_partition_cells = 0u64;
6872 let anchor_parses = 0u64;
6873 let anchor_asts = 0u64;
6874 let anchor_analyses = 0u64;
6875
6876 Ok(PreparedTargetSourcePackage {
6877 sheet: sheet.to_string(),
6878 sheet_id,
6879 lease,
6880 selected_points: None,
6881 complete_selections: Default::default(),
6882 deferred_shared: false,
6883 direct_domains: Vec::new(),
6884 source_report,
6885 replay_records,
6886 spool_replays: 1,
6887 disposition,
6888 legacy: Vec::new(),
6889 direct_families,
6890 direct_cells,
6891 direct_fragments,
6892 direct_complete_families,
6893 direct_complete_cells,
6894 direct_partition_families,
6895 direct_partition_cells,
6896 anchor_parses,
6897 anchor_asts,
6898 anchor_analyses,
6899 })
6900 }
6901
6902 fn fallback_planning_snapshot(
6903 &self,
6904 batch: &FormulaIngestBatch,
6905 ) -> Result<crate::function_registry::RegistryPlanningSnapshot, ExcelError> {
6906 let mut requests = Vec::new();
6907 for formula in &batch.formulas {
6908 let ast = self
6909 .graph
6910 .data_store()
6911 .retrieve_ast(formula.ast_id, self.graph.sheet_reg())
6912 .ok_or_else(|| {
6913 ExcelError::new(ExcelErrorKind::Value)
6914 .with_message("ordered fallback AST is unavailable")
6915 })?;
6916 Self::collect_planning_function_requests(&ast, &mut requests);
6917 }
6918 requests.sort();
6919 requests.dedup();
6920 crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
6921 &self.resolver,
6922 requests,
6923 )
6924 .map_err(|error| ExcelError::new(ExcelErrorKind::Value).with_message(format!("{error:?}")))
6925 }
6926
6927 fn prepare_legacy_batch_fallback(
6928 &mut self,
6929 batch: FormulaIngestBatch,
6930 function_provider: &dyn crate::traits::FunctionProvider,
6931 ) -> Result<
6932 (
6933 crate::engine::graph::prepared_legacy_graph::PreparedLegacyGraphPlan,
6934 u64,
6935 ),
6936 ExcelError,
6937 > {
6938 let formula_count = batch.formulas.len() as u64;
6939 let sheet_id = self.graph.sheet_id_mut(&batch.sheet_name);
6940 let mut planned = Vec::with_capacity(batch.formulas.len());
6941 for record in batch.formulas {
6942 let placement = CellRef::new(
6943 sheet_id,
6944 Coord::from_excel(record.row, record.col, true, true),
6945 );
6946 let ingested = self
6947 .graph
6948 .ingest_pipeline(function_provider)
6949 .enable_function_semantics()
6950 .ingest_formula(
6951 FormulaAstInput::RawArena(record.ast_id),
6952 placement,
6953 record.formula_text,
6954 )
6955 .map_err(|error| {
6956 ExcelError::new(ExcelErrorKind::Value).with_message(format!("{error:?}"))
6957 })?;
6958 planned.push((record.row, record.col, ingested.ast_id, ingested.dep_plan));
6959 }
6960 let plan = self
6961 .graph
6962 .prepare_legacy_graph_plan(sheet_id, planned)
6963 .map_err(|error| {
6964 ExcelError::new(ExcelErrorKind::Value).with_message(error.to_string())
6965 })?;
6966 Ok((plan, formula_count))
6967 }
6968
6969 fn publish_compressed_partial_report(
6970 &mut self,
6971 report: &FormulaIngestReport,
6972 direct_report: &FormulaIngestReport,
6973 ) {
6974 if direct_report.source_family_promoted == 0
6975 && direct_report.graph_formula_cells_materialized == 0
6976 {
6977 return;
6978 }
6979 let mut published = report.clone();
6980 published.accumulate(direct_report);
6981 self.record_formula_ingest_report(published);
6982 }
6983
6984 fn finish_compressed_formula_sources(
6985 &mut self,
6986 batches: Vec<(
6987 FormulaIngestBatch,
6988 crate::engine::FormulaCompressedSourceReport,
6989 crate::engine::FormulaCompressedPreparation,
6990 )>,
6991 ) -> Result<FormulaIngestReport, ExcelError> {
6992 self.observe_function_semantic_epoch()?;
6993 if batches.iter().any(|(_, _, preparation)| {
6994 !Arc::ptr_eq(&preparation.engine_token, &self.source_formula_token)
6995 }) {
6996 return Err(ExcelError::new(ExcelErrorKind::Value)
6997 .with_message("compressed source preparation belongs to another engine"));
6998 }
6999 if batches.iter().any(|(fallback, _, preparation)| {
7000 preparation.sheet_name.as_ref() != fallback.sheet_name
7001 }) {
7002 return Err(ExcelError::new(ExcelErrorKind::Value)
7003 .with_message("compressed source preparation sheet mismatch"));
7004 }
7005 let initial_guard = crate::function_registry::semantic_epoch_read_guard();
7006 let initial_provider_revision = self.resolver.planning_semantic_revision();
7007 let mut fallback_batches = Vec::with_capacity(batches.len());
7008 let mut stale_fallback_batches = Vec::new();
7009 let mut pending_preparations = Vec::new();
7010 for (mut fallback, mut source, mut preparation) in batches {
7011 for formula in fallback.formulas.drain(..) {
7012 let source_order = formula.source_order.ok_or_else(|| {
7013 ExcelError::new(ExcelErrorKind::Value).with_message(
7014 "compressed source supplied formulas without source-order proof",
7015 )
7016 })?;
7017 let text = formula.formula_text.ok_or_else(|| {
7018 ExcelError::new(ExcelErrorKind::Value).with_message(
7019 "ordered compressed fallback formula has no exact source text",
7020 )
7021 })?;
7022 preparation
7023 .eager_replay
7024 .push(crate::engine::DeferredReplayFormula {
7025 source_order,
7026 row: formula.row,
7027 col: formula.col,
7028 text: text.to_string(),
7029 family: formula.source_family,
7030 partition_owner: formula.partition_owner,
7031 });
7032 }
7033 preparation
7034 .eager_replay
7035 .sort_by_key(|record| record.source_order);
7036 source.source_spool_replays = source
7037 .source_spool_replays
7038 .saturating_add(preparation.preparation_spool_replays);
7039 let stale_reason = preparation
7040 .function_semantics_used
7041 .then(|| {
7042 if preparation.function_provider_revision != initial_provider_revision {
7043 Some("FunctionProviderRevisionChanged")
7044 } else if self.prepared_function_semantics_changed(&preparation, &initial_guard)
7045 {
7046 Some("FunctionSemanticEpochChanged")
7047 } else {
7048 None
7049 }
7050 })
7051 .flatten();
7052 let stale_semantics = stale_reason.is_some();
7053 for reason in preparation.rejected.values() {
7054 *source.fallback_reasons.entry(reason.clone()).or_default() += 1;
7055 }
7056 source.replay_families = source.families_seen;
7058 source.replay_cells = source.family_cells_seen;
7059 let compressed = crate::engine::FormulaCompressedSourceBatch::new(
7060 fallback.sheet_name.clone(),
7061 source,
7062 );
7063 if stale_semantics {
7064 stale_fallback_batches.push((fallback, compressed));
7065 } else {
7066 fallback_batches.push((fallback, compressed));
7067 }
7068 pending_preparations.push((preparation, stale_semantics));
7069 }
7070 drop(initial_guard);
7071
7072 let configured_mode = self.config.formula_plane_mode;
7074 self.config.formula_plane_mode = FormulaPlaneMode::Off;
7075 let stale_result =
7076 self.ingest_compressed_formula_source_batches_inner(stale_fallback_batches, false);
7077 self.config.formula_plane_mode = configured_mode;
7078 let mut report = stale_result?;
7079 report.mode = configured_mode;
7080
7081 self.config.formula_plane_mode = FormulaPlaneMode::Off;
7082 let fallback_result =
7083 self.ingest_compressed_formula_source_batches_inner(fallback_batches, false);
7084 self.config.formula_plane_mode = configured_mode;
7085 match fallback_result {
7086 Ok(fallback_report) => report.accumulate(&fallback_report),
7087 Err(error) => {
7088 self.record_formula_ingest_report(report);
7089 return Err(error);
7090 }
7091 }
7092
7093 let mut direct_report =
7094 FormulaIngestReport::with_mode(FormulaPlaneMode::AuthoritativeExperimental);
7095 for (preparation, _) in &pending_preparations {
7096 direct_report.source_anchor_parses = direct_report.source_anchor_parses.saturating_add(
7097 preparation.clean_rejected_anchor_counts[0]
7098 .saturating_add(preparation.fragmented_rejected_anchor_counts[0]),
7099 );
7100 direct_report.source_anchor_asts = direct_report.source_anchor_asts.saturating_add(
7101 preparation.clean_rejected_anchor_counts[1]
7102 .saturating_add(preparation.fragmented_rejected_anchor_counts[1]),
7103 );
7104 direct_report.source_anchor_analyses =
7105 direct_report.source_anchor_analyses.saturating_add(
7106 preparation.clean_rejected_anchor_counts[2]
7107 .saturating_add(preparation.fragmented_rejected_anchor_counts[2]),
7108 );
7109 }
7110 loop {
7111 #[cfg(any(test, feature = "test-support"))]
7112 if let Some(hook) = self.before_prepared_span_commit_hook.take() {
7113 hook();
7114 }
7115 let commit_guard = crate::function_registry::semantic_epoch_read_guard();
7116 let commit_provider_revision = self.resolver.planning_semantic_revision();
7117 let mut newly_stale = Vec::new();
7118 let mut current = Vec::new();
7119 for pending in pending_preparations.drain(..) {
7120 let stale_reason = pending
7121 .0
7122 .function_semantics_used
7123 .then(|| {
7124 if pending.0.function_provider_revision != commit_provider_revision {
7125 Some("FunctionProviderRevisionChanged")
7126 } else if self
7127 .prepared_function_semantics_changed(&pending.0, &commit_guard)
7128 {
7129 Some("FunctionSemanticEpochChanged")
7130 } else {
7131 None
7132 }
7133 })
7134 .flatten();
7135 if let Some(reason) = stale_reason {
7136 newly_stale.push((pending, reason));
7137 } else {
7138 current.push(pending);
7139 }
7140 }
7141 if !newly_stale.is_empty() {
7142 drop(commit_guard);
7143 for ((mut preparation, was_initially_stale), reason) in newly_stale {
7144 preparation
7145 .eager_replay
7146 .sort_by_key(|record| record.source_order);
7147 direct_report.source_spool_replays =
7148 direct_report.source_spool_replays.saturating_add(1);
7149 if !was_initially_stale {
7150 direct_report
7151 .fallback_reasons
7152 .entry(reason.to_string())
7153 .or_default();
7154 }
7155 preparation.function_semantics_used = false;
7156 if !preparation.eager_replay.is_empty() {
7157 current.push((preparation, false));
7158 }
7159 }
7160 pending_preparations = current;
7161 continue;
7162 }
7163 drop(commit_guard);
7164
7165 enum SourceProposal {
7167 KnownFallback {
7168 source_order: crate::engine::SourceFormulaOrder,
7169 records: Vec<crate::engine::DeferredReplayFormula>,
7170 },
7171 }
7172
7173 impl SourceProposal {
7174 fn source_order(&self) -> crate::engine::SourceFormulaOrder {
7175 match self {
7176 Self::KnownFallback { source_order, .. } => *source_order,
7177 }
7178 }
7179 }
7180
7181 for (mut preparation, _) in current {
7182 let mut proposals = Vec::with_capacity(preparation.eager_replay.len());
7183 let mut family_fallbacks: BTreeMap<_, Vec<_>> = BTreeMap::new();
7184 for record in preparation.eager_replay.drain(..) {
7185 if let Some(owner) = record.partition_owner.or(record.family) {
7186 family_fallbacks.entry(owner).or_default().push(record);
7187 } else {
7188 proposals.push(SourceProposal::KnownFallback {
7189 source_order: record.source_order,
7190 records: vec![record],
7191 });
7192 }
7193 }
7194 for (_, mut records) in family_fallbacks {
7195 records.sort_by_key(|record| record.source_order);
7196 if records
7197 .windows(2)
7198 .any(|window| window[0].source_order == window[1].source_order)
7199 {
7200 self.publish_compressed_partial_report(&report, &direct_report);
7201 return Err(ExcelError::new(ExcelErrorKind::Value)
7202 .with_message("duplicate exact-replay source-order proof"));
7203 }
7204 let Some(source_order) = records.first().map(|record| record.source_order)
7205 else {
7206 self.publish_compressed_partial_report(&report, &direct_report);
7207 return Err(ExcelError::new(ExcelErrorKind::Value)
7208 .with_message("empty exact-replay fallback family"));
7209 };
7210 proposals.push(SourceProposal::KnownFallback {
7211 source_order,
7212 records,
7213 });
7214 }
7215 proposals.sort_by_key(SourceProposal::source_order);
7216 if proposals
7217 .windows(2)
7218 .any(|window| window[0].source_order() == window[1].source_order())
7219 {
7220 self.publish_compressed_partial_report(&report, &direct_report);
7221 return Err(ExcelError::new(ExcelErrorKind::Value)
7222 .with_message("ambiguous compressed source-order proof"));
7223 }
7224
7225 for proposal in proposals {
7226 match proposal {
7227 SourceProposal::KnownFallback { records, .. } => {
7228 let batch = match self.formula_batch_from_exact_replay(
7229 preparation.sheet_name.as_ref(),
7230 records,
7231 ) {
7232 Ok(batch) => batch,
7233 Err(error) => {
7234 self.publish_compressed_partial_report(&report, &direct_report);
7235 return Err(error);
7236 }
7237 };
7238 if batch.is_empty() {
7239 continue;
7240 }
7241 let snapshot = match self.fallback_planning_snapshot(&batch) {
7242 Ok(snapshot) => snapshot,
7243 Err(error) => {
7244 self.publish_compressed_partial_report(&report, &direct_report);
7245 return Err(error);
7246 }
7247 };
7248 let commit_guard =
7249 crate::function_registry::semantic_epoch_read_guard();
7250 let provider_revision_initial =
7251 self.resolver.planning_semantic_revision();
7252 if (commit_guard.epoch() != snapshot.epoch()
7253 && snapshot.semantic_changes_affect_requests_since_guarded(
7254 &commit_guard,
7255 snapshot.epoch(),
7256 ))
7257 || snapshot.provider_revision().is_some_and(|revision| {
7258 Some(revision) != provider_revision_initial
7259 })
7260 {
7261 drop(commit_guard);
7262 self.publish_compressed_partial_report(&report, &direct_report);
7263 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
7264 "ordered fallback planning snapshot became stale",
7265 ));
7266 }
7267 let (plan, formula_count) = match self
7268 .prepare_legacy_batch_fallback(batch, &snapshot)
7269 {
7270 Ok(plan) => plan,
7271 Err(error) => {
7272 drop(commit_guard);
7273 self.publish_compressed_partial_report(&report, &direct_report);
7274 return Err(error);
7275 }
7276 };
7277 let provider_revision_after =
7278 self.resolver.planning_semantic_revision();
7279 if provider_revision_after != provider_revision_initial {
7280 drop(commit_guard);
7281 self.publish_compressed_partial_report(&report, &direct_report);
7282 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
7283 "function provider changed while preparing ordered fallback",
7284 ));
7285 }
7286 let graph_vertices = plan.new_vertex_count();
7287 let Some(graph_edges) = plan.planned_edge_count() else {
7288 drop(commit_guard);
7289 self.publish_compressed_partial_report(&report, &direct_report);
7290 return Err(ExcelError::new(ExcelErrorKind::Value)
7291 .with_message("prepared ordered fallback size overflow"));
7292 };
7293 if let Err(error) = self.prepared_legacy_admission(&plan, formula_count)
7294 {
7295 drop(commit_guard);
7296 self.publish_compressed_partial_report(&report, &direct_report);
7297 return Err(error);
7298 }
7299 if let Err(error) =
7300 self.graph.validate_prepared_legacy_graph_plan(&plan)
7301 {
7302 drop(commit_guard);
7303 self.publish_compressed_partial_report(&report, &direct_report);
7304 return Err(ExcelError::new(ExcelErrorKind::Value)
7305 .with_message(error.to_string()));
7306 }
7307 #[cfg(test)]
7308 if let Some(hook) = self
7309 .before_legacy_fallback_final_provider_sample_hook
7310 .take()
7311 {
7312 hook();
7313 }
7314 let provider_revision_final =
7315 self.resolver.planning_semantic_revision();
7316 if provider_revision_final != provider_revision_initial {
7317 drop(commit_guard);
7318 self.publish_compressed_partial_report(&report, &direct_report);
7319 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
7320 "function provider changed after ordered fallback validation",
7321 ));
7322 }
7323 let graph_formulas =
7324 self.graph.apply_prevalidated_legacy_graph_plan(plan);
7325 direct_report.formula_cells_seen = direct_report
7326 .formula_cells_seen
7327 .saturating_add(formula_count);
7328 direct_report.graph_formula_cells_materialized = direct_report
7329 .graph_formula_cells_materialized
7330 .saturating_add(graph_formulas as u64);
7331 direct_report.graph_vertices_created = direct_report
7332 .graph_vertices_created
7333 .saturating_add(graph_vertices as u64);
7334 direct_report.graph_edges_created = direct_report
7335 .graph_edges_created
7336 .saturating_add(graph_edges as u64);
7337 }
7338 }
7339 #[cfg(test)]
7340 if let Some(hook) = self.after_eager_proposal_commit_hook.take() {
7341 hook();
7342 }
7343 }
7344 }
7345 break;
7346 }
7347 report.accumulate(&direct_report);
7348 self.record_formula_ingest_report(report.clone());
7349 Ok(report)
7350 }
7351 pub(crate) fn ingest_compressed_formula_source_batches(
7353 &mut self,
7354 batches: Vec<(
7355 FormulaIngestBatch,
7356 crate::engine::FormulaCompressedSourceBatch,
7357 )>,
7358 ) -> Result<FormulaIngestReport, ExcelError> {
7359 self.ingest_compressed_formula_source_batches_inner(batches, true)
7360 }
7361
7362 fn ingest_compressed_formula_source_batches_inner(
7363 &mut self,
7364 batches: Vec<(
7365 FormulaIngestBatch,
7366 crate::engine::FormulaCompressedSourceBatch,
7367 )>,
7368 publish_report: bool,
7369 ) -> Result<FormulaIngestReport, ExcelError> {
7370 let mut source_counts = [0_u64; 11];
7371 let mut source_report = crate::engine::FormulaCompressedSourceReport::default();
7372 let mut formula_batches = Vec::with_capacity(batches.len());
7373 let mut compressed_families = Vec::new();
7374 let mut partitioned_families = Vec::new();
7375 for (batch, compressed_batch) in batches {
7376 let (sheet_name, compressed, families, partitions) = compressed_batch.into_parts();
7377 if sheet_name.as_ref() != batch.sheet_name {
7378 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
7379 "compressed formula source sheet does not match its replay batch",
7380 ));
7381 }
7382 compressed_families.push((batch.sheet_name.clone(), families));
7383 partitioned_families.push((batch.sheet_name.clone(), partitions));
7384 source_counts[0] = source_counts[0].saturating_add(compressed.source_formula_events);
7385 source_counts[1] = source_counts[1].saturating_add(compressed.source_ordinary_events);
7386 source_counts[2] =
7387 source_counts[2].saturating_add(compressed.source_shared_anchor_events);
7388 source_counts[3] =
7389 source_counts[3].saturating_add(compressed.source_shared_descendant_events);
7390 source_counts[4] = source_counts[4].saturating_add(compressed.source_unknown_events);
7391 source_counts[5] =
7392 source_counts[5].saturating_add(compressed.source_formula_records_spooled);
7393 source_counts[6] =
7394 source_counts[6].saturating_add(compressed.source_spool_encoded_bytes);
7395 source_counts[7] = source_counts[7].max(compressed.source_spool_peak_memory_bytes);
7396 source_counts[8] =
7397 source_counts[8].saturating_add(compressed.source_spool_spilled_bytes);
7398 source_counts[9] = source_counts[9].saturating_add(compressed.source_spool_spill_files);
7399 source_counts[10] = source_counts[10].saturating_add(compressed.source_spool_replays);
7400 source_report.families_seen = source_report
7401 .families_seen
7402 .saturating_add(compressed.families_seen);
7403 source_report.family_cells_seen = source_report
7404 .family_cells_seen
7405 .saturating_add(compressed.family_cells_seen);
7406 source_report.source_clean_families = source_report
7407 .source_clean_families
7408 .saturating_add(compressed.source_clean_families);
7409 source_report.source_clean_cells = source_report
7410 .source_clean_cells
7411 .saturating_add(compressed.source_clean_cells);
7412 source_report.source_fragmentable_families = source_report
7413 .source_fragmentable_families
7414 .saturating_add(compressed.source_fragmentable_families);
7415 source_report.source_fragmentable_cells = source_report
7416 .source_fragmentable_cells
7417 .saturating_add(compressed.source_fragmentable_cells);
7418 source_report.source_fragment_count = source_report
7419 .source_fragment_count
7420 .saturating_add(compressed.source_fragment_count);
7421 source_report.source_isolated_fallback_cells = source_report
7422 .source_isolated_fallback_cells
7423 .saturating_add(compressed.source_isolated_fallback_cells);
7424 source_report.source_hole_exclusions = source_report
7425 .source_hole_exclusions
7426 .saturating_add(compressed.source_hole_exclusions);
7427 source_report.source_ordinary_exclusions = source_report
7428 .source_ordinary_exclusions
7429 .saturating_add(compressed.source_ordinary_exclusions);
7430 source_report.source_partition_failures = source_report
7431 .source_partition_failures
7432 .saturating_add(compressed.source_partition_failures);
7433 source_report.replay_families = source_report
7434 .replay_families
7435 .saturating_add(compressed.replay_families);
7436 source_report.replay_cells = source_report
7437 .replay_cells
7438 .saturating_add(compressed.replay_cells);
7439 source_report.forward_descendants = source_report
7440 .forward_descendants
7441 .saturating_add(compressed.forward_descendants);
7442 source_report.evidence_limit_fallbacks = source_report
7443 .evidence_limit_fallbacks
7444 .saturating_add(compressed.evidence_limit_fallbacks);
7445 source_report.evidence_peak_bytes = source_report
7446 .evidence_peak_bytes
7447 .max(compressed.evidence_peak_bytes);
7448 for (reason, count) in compressed.fallback_reasons {
7449 *source_report.fallback_reasons.entry(reason).or_default() += count;
7450 }
7451 formula_batches.push(batch);
7452 }
7453 self.ingest_formula_batches_inner(
7454 formula_batches,
7455 source_counts,
7456 Some(source_report),
7457 compressed_families,
7458 partitioned_families,
7459 publish_report,
7460 )
7461 }
7462
7463 pub fn ingest_formula_batches(
7464 &mut self,
7465 batches: Vec<FormulaIngestBatch>,
7466 ) -> Result<FormulaIngestReport, ExcelError> {
7467 let has_formulas = batches.iter().any(|batch| !batch.formulas.is_empty());
7468 let report = self.ingest_formula_batches_inner(
7469 batches,
7470 [0; 11],
7471 None,
7472 Vec::new(),
7473 Vec::new(),
7474 true,
7475 )?;
7476 if has_formulas {
7477 self.mark_topology_edited();
7478 }
7479 Ok(report)
7480 }
7481
7482 fn ingest_formula_batches_unpublished(
7483 &mut self,
7484 batches: Vec<FormulaIngestBatch>,
7485 ) -> Result<FormulaIngestReport, ExcelError> {
7486 self.ingest_formula_batches_inner(batches, [0; 11], None, Vec::new(), Vec::new(), false)
7487 }
7488
7489 fn ingest_formula_batches_inner(
7490 &mut self,
7491 batches: Vec<FormulaIngestBatch>,
7492 source_counts: [u64; 11],
7493 source_report: Option<crate::engine::FormulaCompressedSourceReport>,
7494 compressed_families: Vec<(String, Vec<crate::engine::SourceFormulaFamily>)>,
7495 partitioned_families: Vec<(String, Vec<crate::engine::PartitionedSourceFormulaFamily>)>,
7496 publish_report: bool,
7497 ) -> Result<FormulaIngestReport, ExcelError> {
7498 self.observe_function_semantic_epoch()?;
7499 let formula_cells_seen = batches.iter().map(|batch| batch.len() as u64).sum();
7500 #[cfg(feature = "tracing")]
7501 let arena_nodes_before = self.graph.data_store().memory_usage().total_ast_nodes;
7502 #[cfg(feature = "tracing")]
7503 let route = if !partitioned_families.is_empty() {
7504 "partitioned"
7505 } else if !compressed_families.is_empty() {
7506 "compressed_source"
7507 } else if source_report.is_some() {
7508 "replay"
7509 } else {
7510 "ordinary"
7511 };
7512 let _ingest_span = crate::engine::trace::fz_span!(
7513 tracing::Level::INFO,
7514 "ingest",
7515 "ingest.batch",
7516 mode = ?FormulaPlaneMode::Off,
7517 route,
7518 formula_cells = formula_cells_seen
7519 );
7520 let mut report = FormulaIngestReport::with_mode(FormulaPlaneMode::Off);
7521 let materialize_batches = batches;
7522 report.formula_cells_seen = formula_cells_seen;
7523 report.source_formula_events = source_counts[0];
7524 report.source_ordinary_events = source_counts[1];
7525 report.source_shared_anchor_events = source_counts[2];
7526 report.source_shared_descendant_events = source_counts[3];
7527 report.source_unknown_events = source_counts[4];
7528 report.source_formula_records_spooled = source_counts[5];
7529 report.source_spool_encoded_bytes = source_counts[6];
7530 report.source_spool_peak_memory_bytes = source_counts[7];
7531 report.source_spool_spilled_bytes = source_counts[8];
7532 report.source_spool_spill_files = source_counts[9];
7533 report.source_spool_replays = source_counts[10];
7534 if let Some(source) = source_report {
7535 report.source_families_seen = source.families_seen;
7536 report.source_family_cells_seen = source.family_cells_seen;
7537 report.source_family_shadow_eligible = source.source_clean_families;
7538 report.source_family_shadow_eligible_cells = source.source_clean_cells;
7539 report.source_partitioned_families_seen = source.source_fragmentable_families;
7540 report.source_partition_holes = source.source_hole_exclusions;
7541 report.source_partition_ordinary_exceptions = source.source_ordinary_exclusions;
7542 report.source_partition_failures = source.source_partition_failures;
7543 report.source_partition_surviving_cells = source.source_fragmentable_cells;
7544 report.source_family_fallback = report
7545 .source_family_fallback
7546 .saturating_add(source.replay_families);
7547 report.source_family_fallback_cells = report
7548 .source_family_fallback_cells
7549 .saturating_add(source.replay_cells);
7550 report.source_forward_descendants = source.forward_descendants;
7551 report.source_evidence_limit_fallbacks = source.evidence_limit_fallbacks;
7552 report.source_evidence_peak_bytes = source.evidence_peak_bytes;
7553 for (reason, count) in source.fallback_reasons {
7554 let total = report.fallback_reasons.entry(reason).or_default();
7555 *total = total.saturating_add(count);
7556 }
7557 }
7558
7559 if self.graph.first_load_assume_new()
7563 && !self.graph_admission_enabled()
7564 && !materialize_batches.iter().all(FormulaIngestBatch::is_empty)
7565 {
7566 let mut builder =
7567 crate::engine::ingest_builder::BulkIngestBuilder::new(&mut self.graph);
7568 for batch in materialize_batches {
7569 if batch.is_empty() {
7570 continue;
7571 }
7572 let sheet_id = builder
7573 .add_sheet_checked(&batch.sheet_name)
7574 .ok_or_else(|| {
7575 ExcelError::new(ExcelErrorKind::Ref)
7576 .with_message(format!("unknown ingest sheet: {}", batch.sheet_name))
7577 })?;
7578 builder.add_formula_refs(
7579 sheet_id,
7580 batch.formulas.into_iter().map(|record| {
7581 (
7582 record.row,
7583 record.col,
7584 crate::engine::graph::FormulaRef::of_ingested(
7585 record.ast_id,
7586 record.member_anchor,
7587 ),
7588 )
7589 }),
7590 );
7591 }
7592 let summary = builder.finish_with_provider(&self.resolver)?;
7593 report.graph_formula_cells_materialized = summary.formulas as u64;
7594 report.graph_vertices_created = summary.vertices as u64;
7595 report.graph_edges_created = summary.edges as u64;
7596 } else if !materialize_batches.iter().all(FormulaIngestBatch::is_empty) {
7597 let mut prepared_by_sheet: BTreeMap<String, Vec<_>> = BTreeMap::new();
7598 for batch in materialize_batches {
7599 if batch.is_empty() {
7600 continue;
7601 }
7602 let sheet_id = self.graph.sheet_id(&batch.sheet_name).ok_or_else(|| {
7603 ExcelError::new(ExcelErrorKind::Ref)
7604 .with_message(format!("unknown ingest sheet: {}", batch.sheet_name))
7605 })?;
7606 let mut pipeline = self.ingest_pipeline();
7607 let prepared = prepared_by_sheet.entry(batch.sheet_name).or_default();
7610 prepared.reserve(batch.formulas.len());
7611 for record in batch.formulas {
7612 let placement = CellRef::new(
7613 sheet_id,
7614 Coord::from_excel(record.row, record.col, true, true),
7615 );
7616 let input = match record.member_anchor {
7617 Some(anchor) => FormulaAstInput::Member {
7618 template: record.ast_id,
7619 anchor,
7620 },
7621 None => FormulaAstInput::RawArena(record.ast_id),
7622 };
7623 let formula = pipeline.ingest_formula(input, placement, None)?;
7624 prepared.push((
7625 record.row,
7626 record.col,
7627 crate::engine::graph::FormulaRef::of_ingested(
7628 formula.ast_id,
7629 formula.member_anchor,
7630 ),
7631 formula.dep_plan,
7632 ));
7633 }
7634 }
7635 let admission_preflighted = self.graph_admission_enabled();
7636 if admission_preflighted {
7637 let mut preview = Vec::new();
7638 for (sheet_name, formulas) in &prepared_by_sheet {
7639 let sheet_id = self.graph.sheet_id(sheet_name).ok_or_else(|| {
7640 ExcelError::new(ExcelErrorKind::Ref)
7641 .with_message(format!("unknown ingest sheet: {sheet_name}"))
7642 })?;
7643 preview.extend(
7644 formulas
7645 .iter()
7646 .map(|(row, col, _, plan)| (sheet_id, *row, *col, plan.clone())),
7647 );
7648 }
7649 let admission = self.graph.preview_formula_mutations(&preview)?;
7650 self.preflight_graph_admission(admission)?;
7651 }
7652
7653 let mut builder = self.begin_bulk_ingest();
7654 if admission_preflighted {
7655 builder.mark_admission_preflighted();
7656 }
7657 for (sheet_name, formulas) in prepared_by_sheet {
7658 if formulas.is_empty() {
7659 continue;
7660 }
7661 let sheet_id = builder.add_sheet(&sheet_name);
7662 builder.add_formula_plans(sheet_id, formulas);
7663 }
7664 let summary = builder.finish()?;
7665 report.graph_formula_cells_materialized = summary.formulas as u64;
7666 report.graph_vertices_created = summary.vertices as u64;
7667 report.graph_edges_created = summary.edges as u64;
7668 }
7669
7670 crate::engine::trace::fz_event!(
7671 tracing::Level::INFO,
7672 "ingest",
7673 "ingest.summary",
7674 candidate_cells = report.shadow_candidate_cells,
7675 accepted_span_cells = report.shadow_accepted_span_cells,
7676 fallback_cells = report.shadow_fallback_cells,
7677 spans_created = report.shadow_spans_created,
7678 templates_interned = report.shadow_templates_interned,
7679 graph_vertices_created = report.graph_vertices_created,
7680 graph_edges_created = report.graph_edges_created,
7681 arena_nodes_delta = self
7682 .graph
7683 .data_store()
7684 .memory_usage()
7685 .total_ast_nodes
7686 .saturating_sub(arena_nodes_before)
7687 );
7688 if publish_report {
7689 self.record_formula_ingest_report(report.clone());
7690 }
7691 Ok(report)
7692 }
7693
7694 fn dedup_formula_parse_diagnostics_since(&mut self, start: usize) {
7695 let mut unique = Vec::new();
7696 for diagnostic in self.formula_parse_diagnostics.drain(start..) {
7697 let duplicate = unique.iter().any(|prior: &FormulaParseDiagnostic| {
7698 prior.sheet == diagnostic.sheet
7699 && prior.row == diagnostic.row
7700 && prior.col == diagnostic.col
7701 && prior.formula == diagnostic.formula
7702 && prior.policy == diagnostic.policy
7703 });
7704 if !duplicate {
7705 unique.push(diagnostic);
7706 }
7707 }
7708 self.formula_parse_diagnostics.extend(unique);
7709 }
7710
7711 pub fn handle_formula_parse_error(
7712 &mut self,
7713 sheet: &str,
7714 row: u32,
7715 col: u32,
7716 formula: &str,
7717 message: String,
7718 ) -> Result<Option<ASTNode>, ExcelError> {
7719 let policy = self.config.formula_parse_policy;
7720
7721 if policy == FormulaParsePolicy::Strict {
7722 let col_a1 = col_letters_from_1based(col).unwrap_or_else(|_| "?".to_string());
7723 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(format!(
7724 "Formula parse error at {sheet}!{col_a1}{row}: {message}"
7725 )));
7726 }
7727
7728 self.formula_parse_diagnostics.push(FormulaParseDiagnostic {
7729 sheet: sheet.to_string(),
7730 row,
7731 col,
7732 formula: formula.to_string(),
7733 message: message.clone(),
7734 policy,
7735 });
7736
7737 match policy {
7738 FormulaParsePolicy::Strict => unreachable!(),
7739 FormulaParsePolicy::KeepCachedValue => Ok(None),
7740 FormulaParsePolicy::AsText => Ok(Some(ASTNode::new(
7741 ASTNodeType::Literal(LiteralValue::Text(formula.to_string())),
7742 None,
7743 ))),
7744 FormulaParsePolicy::CoerceToError => {
7745 let err = ExcelError::new(ExcelErrorKind::Error)
7746 .with_message(format!("Malformed formula: {message}"));
7747 Ok(Some(ASTNode::new(
7748 ASTNodeType::Literal(LiteralValue::Error(err)),
7749 None,
7750 )))
7751 }
7752 }
7753 }
7754
7755 #[cfg(test)]
7756 fn target_preparation_fault(
7757 &mut self,
7758 seam: crate::engine::target_preparation::TargetPreparationFault,
7759 ) -> Result<(), ExcelError> {
7760 if self.target_preparation_fault_for_test == Some(seam) {
7761 self.target_preparation_fault_for_test = None;
7762 Err(ExcelError::new(ExcelErrorKind::Value)
7763 .with_message(format!("injected target preparation fault: {seam:?}")))
7764 } else {
7765 Ok(())
7766 }
7767 }
7768
7769 fn preparation_stale(
7770 reason: formualizer_common::PreparationStaleReason,
7771 message: impl Into<String>,
7772 ) -> ExcelError {
7773 ExcelError::new(ExcelErrorKind::Value)
7774 .with_message(message)
7775 .with_extra(formualizer_common::ExcelErrorExtra::PreparationStale { reason })
7776 }
7777
7778 fn preparation_revision_stale_reason(
7779 assumptions: &crate::engine::PreparationRevision,
7780 current: &crate::engine::PreparationRevision,
7781 planning_requests: &BTreeSet<(String, String, usize)>,
7782 staged_leases_match: bool,
7783 ) -> Option<formualizer_common::PreparationStaleReason> {
7784 if assumptions.graph != current.graph {
7785 Some(formualizer_common::PreparationStaleReason::Graph)
7786 } else if assumptions.staged != current.staged || !staged_leases_match {
7787 Some(formualizer_common::PreparationStaleReason::Staged)
7788 } else if assumptions.symbols != current.symbols {
7789 Some(formualizer_common::PreparationStaleReason::Symbols)
7790 } else if assumptions.provider != current.provider {
7791 Some(formualizer_common::PreparationStaleReason::Provider)
7792 } else if assumptions.semantic != current.semantic
7793 && crate::function_registry::semantic_changes_affect_requests_since(
7794 assumptions.semantic,
7795 planning_requests.iter().cloned(),
7796 )
7797 {
7798 Some(formualizer_common::PreparationStaleReason::Semantic)
7799 } else {
7800 None
7801 }
7802 }
7803
7804 fn preparation_revisions(&self) -> crate::engine::PreparationRevision {
7805 crate::engine::PreparationRevision {
7806 graph: self.graph.topology_revision(),
7807 authority: 0,
7808 authority_indexes: 0,
7809 authority_indexed_plane: 0,
7810 staged: self.staged_formula_index.revision(),
7811 symbols: self.graph.symbol_revision(),
7812 semantic: crate::function_registry::semantic_epoch(),
7813 provider: self.resolver.planning_semantic_revision(),
7814 }
7815 }
7816
7817 fn planning_revision_snapshot(&self) -> PlanningRevisionSnapshot {
7818 let registry_guard = crate::function_registry::semantic_epoch_read_guard();
7819 let provider = self.resolver.planning_semantic_revision();
7820 let semantic = registry_guard.epoch();
7821 PlanningRevisionSnapshot {
7822 engine_topology_epoch: self.topology_epoch,
7823 graph_topology_revision: self.graph.topology_revision(),
7824 staged: self.staged_formula_index.revision(),
7825 symbols: self.graph.symbol_revision(),
7826 semantic,
7827 provider,
7828 deterministic_mode: self.config.deterministic_mode.clone(),
7829 budgets: self.evaluation_resource_budgets.clone(),
7830 }
7831 }
7832
7833 fn recalc_plan_key(&self) -> RecalcPlanKey {
7834 RecalcPlanKey {
7835 engine_token: Arc::clone(&self.recalc_plan_token),
7836 revisions: self.planning_revision_snapshot(),
7837 }
7838 }
7839
7840 fn plan_stale(reason: formualizer_common::PlanStaleReason) -> ExcelError {
7841 ExcelError::new(ExcelErrorKind::Value)
7842 .with_message(format!("recalculation plan is stale: {}", reason.as_str()))
7843 .with_extra(formualizer_common::ExcelErrorExtra::PlanStale { reason })
7844 }
7845
7846 fn validate_recalc_plan_key(&self, key: &RecalcPlanKey) -> Result<(), ExcelError> {
7847 use formualizer_common::PlanStaleReason;
7848
7849 if !Arc::ptr_eq(&key.engine_token, &self.recalc_plan_token) {
7850 return Err(Self::plan_stale(PlanStaleReason::Engine));
7851 }
7852
7853 let current = self.planning_revision_snapshot();
7854 let expected = &key.revisions;
7855 let stale = if expected.provider != current.provider {
7856 Some(PlanStaleReason::Provider)
7857 } else if expected.semantic != current.semantic {
7858 Some(PlanStaleReason::Semantic)
7859 } else if expected.budgets != current.budgets
7860 || expected.deterministic_mode != current.deterministic_mode
7861 {
7862 Some(PlanStaleReason::Budget)
7863 } else if expected.staged != current.staged {
7864 Some(PlanStaleReason::Staged)
7865 } else if expected.symbols != current.symbols {
7866 Some(PlanStaleReason::Symbols)
7867 } else if expected.graph_topology_revision != current.graph_topology_revision
7868 || expected.engine_topology_epoch != current.engine_topology_epoch
7869 {
7870 Some(PlanStaleReason::Graph)
7871 } else {
7872 None
7873 };
7874 stale.map_or(Ok(()), |reason| Err(Self::plan_stale(reason)))
7875 }
7876
7877 fn target_preparation_checkpoint(
7878 &mut self,
7879 deadline: Option<std::time::Instant>,
7880 work: u64,
7881 ) -> Result<(), ExcelError> {
7882 if self
7883 .active_cancel_flag
7884 .as_ref()
7885 .is_some_and(|cancel| cancel.is_cancelled())
7886 {
7887 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
7888 .with_message("target graph preparation cancelled"));
7889 }
7890 if deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline) {
7891 return Err(crate::engine::ResourceLedgerError::Exhausted(
7892 formualizer_common::ResourceExhaustionDetail {
7893 reason: formualizer_common::ResourceExhaustionReason::Deadline,
7894 limit: 0,
7895 observed: 1,
7896 request_id: self
7897 .active_evaluation_resource_request
7898 .as_ref()
7899 .map(|stats| stats.request_id),
7900 },
7901 )
7902 .into_excel_error());
7903 }
7904 self.charge_bounded_work(work)
7905 }
7906
7907 fn opaque_reason_in_ast(
7908 &self,
7909 ast: &ASTNode,
7910 provider: &crate::function_registry::RegistryPlanningSnapshot,
7911 ) -> Option<crate::engine::OpaqueReason> {
7912 match &ast.node_type {
7913 ASTNodeType::Function { name, args } => {
7914 let canonical = name.rsplit('.').next().unwrap_or(name).to_ascii_uppercase();
7915 if canonical == "INDIRECT" {
7916 return Some(crate::engine::OpaqueReason::RuntimeTextReference);
7917 }
7918 let Some(caps) = provider.function_capabilities("", name) else {
7919 return Some(crate::engine::OpaqueReason::UnknownFunction);
7920 };
7921 if caps.contains(FnCaps::DYNAMIC_DEPENDENCY)
7922 || caps.contains(FnCaps::RETURNS_REFERENCE)
7923 {
7924 return Some(crate::engine::OpaqueReason::DynamicReference);
7925 }
7926 args.iter()
7927 .find_map(|arg| self.opaque_reason_in_ast(arg, provider))
7928 }
7929 ASTNodeType::Call { .. } => Some(crate::engine::OpaqueReason::UnknownCustomFunction),
7930 ASTNodeType::UnaryOp { expr, .. } => self.opaque_reason_in_ast(expr, provider),
7931 ASTNodeType::BinaryOp { left, right, .. } => self
7932 .opaque_reason_in_ast(left, provider)
7933 .or_else(|| self.opaque_reason_in_ast(right, provider)),
7934 ASTNodeType::Array(rows) => rows
7935 .iter()
7936 .flat_map(|row| row.iter())
7937 .find_map(|item| self.opaque_reason_in_ast(item, provider)),
7938 ASTNodeType::Reference {
7939 reference:
7940 ReferenceType::Cell {
7941 sheet: Some(sheet), ..
7942 }
7943 | ReferenceType::Range {
7944 sheet: Some(sheet), ..
7945 },
7946 ..
7947 } if self.graph.sheet_id(sheet).is_none() => {
7948 Some(crate::engine::OpaqueReason::UnresolvedCrossSheetBinding)
7949 }
7950 ASTNodeType::Reference {
7951 reference:
7952 ReferenceType::External(_)
7953 | ReferenceType::Cell3D { .. }
7954 | ReferenceType::Range3D { .. },
7955 ..
7956 } => Some(crate::engine::OpaqueReason::UnresolvedCrossSheetBinding),
7957 ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => None,
7958 }
7959 }
7960
7961 fn target_planning_snapshot(
7962 &mut self,
7963 ast: &ASTNode,
7964 planning_requests: &mut BTreeSet<(String, String, usize)>,
7965 ) -> Result<crate::function_registry::RegistryPlanningSnapshot, ExcelError> {
7966 #[cfg(test)]
7967 if std::mem::take(&mut self.inject_target_semantic_stale_once_for_test) {
7968 return Err(Self::preparation_stale(
7969 formualizer_common::PreparationStaleReason::Semantic,
7970 "injected target semantic preparation movement",
7971 ));
7972 }
7973 #[cfg(test)]
7974 if let Some(hook) = self.before_target_planning_snapshot_hook.take() {
7975 hook();
7976 }
7977 let mut requests = Vec::new();
7978 Self::collect_planning_function_requests(ast, &mut requests);
7979 requests.sort();
7980 requests.dedup();
7981 planning_requests.extend(requests.iter().cloned());
7982 crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
7983 &self.resolver,
7984 requests,
7985 )
7986 .map_err(|error| {
7987 ExcelError::new(ExcelErrorKind::Value)
7988 .with_message(format!("target planning snapshot unavailable: {error:?}"))
7989 })
7990 }
7991
7992 fn target_planning_snapshot_stale_reason(
7993 snapshot: &crate::function_registry::RegistryPlanningSnapshot,
7994 assumptions: &crate::engine::PreparationRevision,
7995 ) -> Option<formualizer_common::PreparationStaleReason> {
7996 if snapshot
7997 .provider_revision()
7998 .is_some_and(|revision| Some(revision) != assumptions.provider)
7999 {
8000 Some(formualizer_common::PreparationStaleReason::Provider)
8001 } else if snapshot.epoch() != assumptions.semantic
8002 && snapshot.semantic_changes_affect_requests_since(assumptions.semantic)
8003 {
8004 Some(formualizer_common::PreparationStaleReason::Semantic)
8005 } else {
8006 None
8007 }
8008 }
8009
8010 fn widen_target_preparation(
8011 policy: crate::engine::OpaquePreparePolicy,
8012 scope: &mut crate::engine::PrepareScope,
8013 reasons: &mut Vec<crate::engine::OpaqueReason>,
8014 reason: crate::engine::OpaqueReason,
8015 ) -> Result<bool, ExcelError> {
8016 if policy == crate::engine::OpaquePreparePolicy::Error {
8017 return Err(ExcelError::new(ExcelErrorKind::NImpl)
8018 .with_message(format!("opaque target preparation semantics: {reason:?}")));
8019 }
8020 if !reasons.contains(&reason) {
8021 reasons.push(reason);
8022 }
8023 if !matches!(scope, crate::engine::PrepareScope::Workbook) {
8024 *scope = crate::engine::PrepareScope::Workbook;
8025 Ok(true)
8026 } else {
8027 Ok(false)
8028 }
8029 }
8030
8031 fn widen_target_preparation_to_sheet(
8032 policy: crate::engine::OpaquePreparePolicy,
8033 scope: &mut crate::engine::PrepareScope,
8034 reasons: &mut Vec<crate::engine::OpaqueReason>,
8035 reason: crate::engine::OpaqueReason,
8036 sheet: &str,
8037 ) -> Result<bool, ExcelError> {
8038 if policy == crate::engine::OpaquePreparePolicy::Error {
8039 return Err(ExcelError::new(ExcelErrorKind::NImpl)
8040 .with_message(format!("opaque target preparation semantics: {reason:?}")));
8041 }
8042 if !reasons.contains(&reason) {
8043 reasons.push(reason);
8044 }
8045 match scope {
8046 crate::engine::PrepareScope::Exact => {
8047 *scope = crate::engine::PrepareScope::Sheets(vec![sheet.to_string()]);
8048 Ok(true)
8049 }
8050 crate::engine::PrepareScope::Sheets(sheets) => {
8051 if sheets.iter().any(|candidate| candidate == sheet) {
8052 Ok(false)
8053 } else {
8054 sheets.push(sheet.to_string());
8055 sheets.sort();
8056 Ok(true)
8057 }
8058 }
8059 crate::engine::PrepareScope::Workbook => Ok(false),
8060 }
8061 }
8062
8063 fn ast_has_proven_sheet_local_dynamic(
8064 ast: &ASTNode,
8065 provider: &crate::function_registry::RegistryPlanningSnapshot,
8066 ) -> bool {
8067 fn classify(
8068 ast: &ASTNode,
8069 provider: &crate::function_registry::RegistryPlanningSnapshot,
8070 ) -> (bool, bool) {
8071 match &ast.node_type {
8072 ASTNodeType::Function { name, args } => {
8073 let Some(caps) = provider.function_capabilities("", name) else {
8074 return (false, false);
8075 };
8076 let dynamic = caps.contains(FnCaps::DYNAMIC_DEPENDENCY)
8077 || caps.contains(FnCaps::RETURNS_REFERENCE);
8078 let canonical = name.rsplit('.').next().unwrap_or(name);
8079 if dynamic
8080 && !canonical.eq_ignore_ascii_case("OFFSET")
8081 && !canonical.eq_ignore_ascii_case("INDEX")
8082 {
8083 return (false, true);
8084 }
8085 let mut has_dynamic = dynamic;
8086 for arg in args {
8087 let (safe, child_dynamic) = classify(arg, provider);
8088 if !safe {
8089 return (false, has_dynamic || child_dynamic);
8090 }
8091 has_dynamic |= child_dynamic;
8092 }
8093 (true, has_dynamic)
8094 }
8095 ASTNodeType::UnaryOp { expr, .. } => classify(expr, provider),
8096 ASTNodeType::BinaryOp { left, right, .. } => {
8097 let (left_safe, left_dynamic) = classify(left, provider);
8098 let (right_safe, right_dynamic) = classify(right, provider);
8099 (left_safe && right_safe, left_dynamic || right_dynamic)
8100 }
8101 ASTNodeType::Array(rows) => {
8102 let mut has_dynamic = false;
8103 for item in rows.iter().flatten() {
8104 let (safe, child_dynamic) = classify(item, provider);
8105 if !safe {
8106 return (false, has_dynamic || child_dynamic);
8107 }
8108 has_dynamic |= child_dynamic;
8109 }
8110 (true, has_dynamic)
8111 }
8112 ASTNodeType::Reference {
8113 reference:
8114 ReferenceType::Cell { sheet: None, .. }
8115 | ReferenceType::Range { sheet: None, .. },
8116 ..
8117 }
8118 | ASTNodeType::Literal(_)
8119 | ASTNodeType::Omitted => (true, false),
8120 ASTNodeType::Call { .. } | ASTNodeType::Reference { .. } => (false, false),
8121 }
8122 }
8123
8124 let (safe, dynamic) = classify(ast, provider);
8125 safe && dynamic
8126 }
8127
8128 fn table_selection_region(
8129 &self,
8130 entry: &crate::engine::graph::TableEntry,
8131 selection: &crate::engine::TableSelection,
8132 ) -> Result<PreparationRegion, ExcelError> {
8133 let mut start_row = entry.range.start.coord.row() + 1;
8134 let mut end_row = entry.range.end.coord.row() + 1;
8135 let mut start_col = entry.range.start.coord.col() + 1;
8136 let mut end_col = entry.range.end.coord.col() + 1;
8137 match selection {
8138 crate::engine::TableSelection::Whole => {}
8139 crate::engine::TableSelection::Headers => {
8140 if !entry.header_row {
8141 return Err(ExcelError::new(ExcelErrorKind::Value)
8142 .with_message(format!("table {} has no header row", entry.name)));
8143 }
8144 end_row = start_row;
8145 }
8146 crate::engine::TableSelection::Data => {
8147 if entry.header_row {
8148 start_row = start_row.saturating_add(1);
8149 }
8150 if entry.totals_row {
8151 end_row = end_row.saturating_sub(1);
8152 }
8153 }
8154 crate::engine::TableSelection::Totals => {
8155 if !entry.totals_row {
8156 return Err(ExcelError::new(ExcelErrorKind::Value)
8157 .with_message(format!("table {} has no totals row", entry.name)));
8158 }
8159 start_row = end_row;
8160 }
8161 crate::engine::TableSelection::Column(column) => {
8162 let index = entry.col_index(column).ok_or_else(|| {
8163 ExcelError::new(ExcelErrorKind::Name)
8164 .with_message(format!("unknown table column: {column}"))
8165 })?;
8166 start_col = start_col.saturating_add(index as u32);
8167 end_col = start_col;
8168 }
8169 crate::engine::TableSelection::Columns { start, end } => {
8170 let first = entry.col_index(start).ok_or_else(|| {
8171 ExcelError::new(ExcelErrorKind::Name)
8172 .with_message(format!("unknown table column: {start}"))
8173 })?;
8174 let last = entry.col_index(end).ok_or_else(|| {
8175 ExcelError::new(ExcelErrorKind::Name)
8176 .with_message(format!("unknown table column: {end}"))
8177 })?;
8178 if first > last {
8179 return Err(ExcelError::new(ExcelErrorKind::Value)
8180 .with_message("table column selection is reversed"));
8181 }
8182 start_col = start_col.saturating_add(first as u32);
8183 end_col = entry
8184 .range
8185 .start
8186 .coord
8187 .col()
8188 .saturating_add(last as u32)
8189 .saturating_add(1);
8190 }
8191 }
8192 if start_row > end_row && matches!(selection, crate::engine::TableSelection::Data) {
8193 start_row = entry.range.start.coord.row() + 1;
8194 end_row = start_row;
8195 }
8196 if start_row > end_row || start_col > end_col {
8197 return Err(
8198 ExcelError::new(ExcelErrorKind::Value).with_message("table selection is empty")
8199 );
8200 }
8201 Ok(PreparationRegion {
8202 sheet: self.graph.sheet_name(entry.sheet_id()).to_string(),
8203 sheet_id: entry.sheet_id(),
8204 start_row,
8205 start_col,
8206 end_row,
8207 end_col,
8208 })
8209 }
8210
8211 #[cfg(test)]
8212 pub(crate) fn reset_target_root_dedup_probes_for_test() {
8213 TARGET_ROOT_DEDUP_PROBES.with(|probes| probes.set(0));
8214 }
8215
8216 #[cfg(test)]
8217 pub(crate) fn target_root_dedup_probes_for_test() -> usize {
8218 TARGET_ROOT_DEDUP_PROBES.with(std::cell::Cell::get)
8219 }
8220
8221 pub(crate) fn resolve_target_producers(
8222 &mut self,
8223 targets: &[crate::engine::EvaluationTarget],
8224 ) -> Result<Vec<crate::engine::target_preparation::TargetProducer>, ExcelError> {
8225 use crate::engine::target_preparation::TargetProducer;
8226
8227 let request_id = self
8228 .active_evaluation_resource_request
8229 .as_ref()
8230 .map(|request| request.request_id);
8231 let mut roots = OrderedTargetProducers::with_capacity(targets.len())
8232 .map_err(|_| target_root_allocation_error(targets.len(), request_id))?;
8233 let resolve_region = |engine: &mut Self,
8234 region: Region,
8235 value_only: Option<CellRef>,
8236 roots: &mut OrderedTargetProducers|
8237 -> Result<(), ExcelError> {
8238 let before = roots.len();
8239 for anchor in engine.graph.spill_anchors_in_region(
8240 region.sheet_id(),
8241 region.axis_ranges().0.query_bounds().0,
8242 region.axis_ranges().1.query_bounds().0,
8243 region.axis_ranges().0.query_bounds().1,
8244 region.axis_ranges().1.query_bounds().1,
8245 ) {
8246 roots
8247 .push(TargetProducer::Legacy(anchor))
8248 .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
8249 }
8250 for vertex in engine.graph.vertices_in_region(
8251 region.sheet_id(),
8252 region.axis_ranges().0.query_bounds().0,
8253 region.axis_ranges().0.query_bounds().1,
8254 region.axis_ranges().1.query_bounds().0,
8255 region.axis_ranges().1.query_bounds().1,
8256 ) {
8257 let vertex = engine
8258 .graph
8259 .get_cell_ref(vertex)
8260 .and_then(|cell| engine.graph.spill_registry_anchor_for_cell(cell))
8261 .unwrap_or(vertex);
8262 if matches!(
8267 engine.graph.get_vertex_kind(vertex),
8268 VertexKind::FormulaScalar | VertexKind::FormulaArray
8269 ) {
8270 roots
8271 .push(TargetProducer::Legacy(vertex))
8272 .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
8273 }
8274 }
8275 if roots.len() == before
8276 && let Some(cell) = value_only
8277 {
8278 roots
8279 .push(TargetProducer::ValueOnly(cell))
8280 .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
8281 }
8282 Ok(())
8283 };
8284
8285 for target in targets {
8286 match target {
8287 crate::engine::EvaluationTarget::Cell { sheet, row, col } => {
8288 if *row == 0 || *col == 0 {
8289 return Err(ExcelError::new(ExcelErrorKind::Ref)
8290 .with_message("target cell coordinates are one-based"));
8291 }
8292 let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
8293 ExcelError::new(ExcelErrorKind::Ref)
8294 .with_message(format!("target sheet not found: {sheet}"))
8295 })?;
8296 let cell = CellRef::new(sheet_id, Coord::from_excel(*row, *col, true, true));
8297 resolve_region(
8298 self,
8299 Region::point(sheet_id, *row - 1, *col - 1),
8300 Some(cell),
8301 &mut roots,
8302 )?;
8303 }
8304 crate::engine::EvaluationTarget::Range(range) => {
8305 let sheet_id = self.graph.sheet_id(&range.sheet).ok_or_else(|| {
8306 ExcelError::new(ExcelErrorKind::Ref)
8307 .with_message(format!("target sheet not found: {}", range.sheet))
8308 })?;
8309 resolve_region(
8310 self,
8311 Region::rect(
8312 sheet_id,
8313 range.start_row - 1,
8314 range.end_row - 1,
8315 range.start_col - 1,
8316 range.end_col - 1,
8317 ),
8318 None,
8319 &mut roots,
8320 )?;
8321 }
8322 crate::engine::EvaluationTarget::Name { name, scope_sheet } => {
8323 let scope = self.name_query_scope(scope_sheet.as_deref())?;
8324 if let Some(entry) = self.graph.resolve_name_entry_in_scope(name, scope) {
8325 roots
8326 .push(TargetProducer::Symbol(entry.vertex))
8327 .map_err(|_| {
8328 target_root_allocation_error(roots.len() + 1, request_id)
8329 })?;
8330 }
8331 }
8332 crate::engine::EvaluationTarget::Table { name, .. } => {
8333 if let Some(entry) = self.graph.resolve_table_entry(name) {
8334 roots
8335 .push(TargetProducer::Symbol(entry.vertex))
8336 .map_err(|_| {
8337 target_root_allocation_error(roots.len() + 1, request_id)
8338 })?;
8339 }
8340 }
8341 }
8342 }
8343 Ok(roots.into_vec())
8344 }
8345
8346 pub fn prepare_graph_for_targets(
8350 &mut self,
8351 targets: &[crate::engine::EvaluationTarget],
8352 options: crate::engine::TargetEvalOptions<'_>,
8353 ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
8354 let previous_budgets = self.evaluation_resource_budgets.clone();
8355 let diagnostics_len = self.formula_parse_diagnostics.len();
8356 let previous_report = self.last_formula_ingest_report.clone();
8357 if let Some(budgets) = options.budgets {
8358 self.evaluation_resource_budgets = budgets.clone();
8359 }
8360 let previous_graph_budget_override = self
8361 .graph
8362 .set_admission_budget_override(Some(self.evaluation_resource_budgets.clone()));
8363 let previous_cancel = self.active_cancel_flag.take();
8367 self.active_cancel_flag = options.cancel.clone();
8368 let result = self.observe_evaluation_resource_request(
8369 EvaluationRequestKind::TargetPreparation,
8370 |engine| engine.prepare_graph_for_targets_unobserved(targets, &options),
8371 );
8372 self.active_cancel_flag = previous_cancel;
8373 self.graph
8374 .set_admission_budget_override(previous_graph_budget_override);
8375 self.evaluation_resource_budgets = previous_budgets;
8376 if result.is_err() {
8377 self.formula_parse_diagnostics.truncate(diagnostics_len);
8378 self.last_formula_ingest_report = previous_report;
8379 }
8380 result
8381 }
8382
8383 fn prepare_graph_for_targets_unobserved(
8384 &mut self,
8385 targets: &[crate::engine::EvaluationTarget],
8386 options: &crate::engine::TargetEvalOptions<'_>,
8387 ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
8388 let scratch_checkpoint = self
8389 .active_resource_ledger
8390 .as_ref()
8391 .map_or(0, crate::engine::ResourceLedger::scratch_checkpoint);
8392 let result = self.prepare_graph_for_targets_transaction(targets, options);
8393 let release = self
8394 .active_resource_ledger
8395 .as_mut()
8396 .map_or(Ok(()), |ledger| {
8397 ledger.release_scratch_to(scratch_checkpoint)
8398 });
8399 match (result, release) {
8400 (result, Ok(())) => result,
8401 (Ok(_), Err(error)) | (Err(_), Err(error)) => Err(error.into_excel_error()),
8402 }
8403 }
8404
8405 fn prepare_graph_for_targets_transaction(
8406 &mut self,
8407 targets: &[crate::engine::EvaluationTarget],
8408 options: &crate::engine::TargetEvalOptions<'_>,
8409 ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
8410 use crate::engine::{
8411 OpaqueReason, PreparationOutcome, PrepareScope, PreparedTargetGraphReport,
8412 TableSelection,
8413 };
8414
8415 self.target_preparation_checkpoint(options.deadline, 0)?;
8416 self.observe_function_semantic_epoch()?;
8417 let assumptions = self.preparation_revisions();
8418 let ledger_at_start = self
8419 .active_resource_ledger
8420 .as_ref()
8421 .map(|ledger| ledger.snapshot());
8422 let diagnostics_len = self.formula_parse_diagnostics.len();
8423 let report_len = self.last_formula_ingest_report.clone();
8424 let request_id = options.request_id.or_else(|| {
8425 self.active_evaluation_resource_request
8426 .as_ref()
8427 .map(|stats| stats.request_id)
8428 });
8429
8430 let mut scope = PrepareScope::Exact;
8431 let mut reasons = Vec::new();
8432 let mut regions = VecDeque::new();
8433 let mut deferred_shared_regions = VecDeque::new();
8434 let mut normalized = Vec::with_capacity(targets.len());
8435 let mut symbol_vertices = VecDeque::new();
8436 for target in targets {
8437 self.target_preparation_checkpoint(options.deadline, 1)?;
8438 match target {
8439 crate::engine::EvaluationTarget::Cell { sheet, row, col } => {
8440 if *row == 0 || *col == 0 {
8441 return Err(ExcelError::new(ExcelErrorKind::Ref)
8442 .with_message("target cell coordinates are one-based"));
8443 }
8444 let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
8445 ExcelError::new(ExcelErrorKind::Ref)
8446 .with_message(format!("target sheet not found: {sheet}"))
8447 })?;
8448 regions.push_back(PreparationRegion {
8449 sheet: sheet.clone(),
8450 sheet_id,
8451 start_row: *row,
8452 start_col: *col,
8453 end_row: *row,
8454 end_col: *col,
8455 });
8456 normalized.push(target.clone());
8457 }
8458 crate::engine::EvaluationTarget::Range(range) => {
8459 if range.start_row == 0
8460 || range.start_col == 0
8461 || range.end_row < range.start_row
8462 || range.end_col < range.start_col
8463 {
8464 return Err(ExcelError::new(ExcelErrorKind::Ref)
8465 .with_message("invalid target range"));
8466 }
8467 let sheet_id = self.graph.sheet_id(&range.sheet).ok_or_else(|| {
8468 ExcelError::new(ExcelErrorKind::Ref)
8469 .with_message(format!("target sheet not found: {}", range.sheet))
8470 })?;
8471 regions.push_back(PreparationRegion {
8472 sheet: range.sheet.clone(),
8473 sheet_id,
8474 start_row: range.start_row,
8475 start_col: range.start_col,
8476 end_row: range.end_row,
8477 end_col: range.end_col,
8478 });
8479 normalized.push(target.clone());
8480 }
8481 crate::engine::EvaluationTarget::Name { name, scope_sheet } => {
8482 let name_scope = self.name_query_scope(scope_sheet.as_deref())?;
8483 if let Some(entry) = self.graph.resolve_name_entry_in_scope(name, name_scope) {
8484 symbol_vertices.push_back(entry.vertex);
8485 } else {
8486 Self::widen_target_preparation(
8487 options.opaque_policy,
8488 &mut scope,
8489 &mut reasons,
8490 OpaqueReason::UnresolvedName,
8491 )?;
8492 }
8493 normalized.push(target.clone());
8494 }
8495 crate::engine::EvaluationTarget::Table { name, selection } => {
8496 if let Some(entry) = self.graph.resolve_table_entry(name) {
8497 let region = self.table_selection_region(entry, selection)?;
8498 symbol_vertices.push_back(entry.vertex);
8499 regions.push_back(region);
8500 } else {
8501 Self::widen_target_preparation(
8502 options.opaque_policy,
8503 &mut scope,
8504 &mut reasons,
8505 OpaqueReason::UnresolvedTable,
8506 )?;
8507 }
8508 normalized.push(target.clone());
8509 }
8510 }
8511 }
8512
8513 let mut visited_regions = FxHashSet::default();
8514 let mut visited_vertices = FxHashSet::default();
8515 let mut selected = FxHashSet::default();
8516 let mut prepared = Vec::new();
8517 let mut pending_diagnostics = Vec::new();
8518 let mut planning_requests = BTreeSet::new();
8519 let mut selected_cells = Vec::new();
8520 let mut workbook_seeded = false;
8521 let mut sheet_scope_seeded = BTreeSet::new();
8522 let mut indexed_query_sheets = FxHashSet::default();
8523 let mut discovery_scratch_reserved = 0u64;
8524 let mut package_encountered = false;
8525 let mut selected_package_sheets = FxHashSet::default();
8526 let mut selected_package_points: BTreeMap<String, BTreeSet<(u32, u32)>> = BTreeMap::new();
8527 let mut prepared_packages: Vec<PreparedTargetSourcePackage> = Vec::new();
8528 let authoritative_with_ordinary = false;
8529 let has_unknown_package_sheet = self
8530 .staged_formula_index
8531 .package_sheets()
8532 .any(|sheet| self.graph.sheet_id(sheet).is_none());
8533
8534 loop {
8535 if let PrepareScope::Sheets(sheets) = &scope {
8536 for sheet in sheets.clone() {
8537 if !sheet_scope_seeded.insert(sheet.clone()) {
8538 continue;
8539 }
8540 let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
8541 package_encountered = true;
8542 Self::widen_target_preparation(
8543 options.opaque_policy,
8544 &mut scope,
8545 &mut reasons,
8546 OpaqueReason::UnsupportedSourceSemantics,
8547 )?;
8548 continue;
8549 };
8550 for lease in self.staged_formula_index.leases_for_sheet(&sheet) {
8551 self.target_preparation_checkpoint(options.deadline, 1)?;
8552 regions.push_back(PreparationRegion {
8553 sheet: sheet.clone(),
8554 sheet_id,
8555 start_row: lease.row,
8556 start_col: lease.col,
8557 end_row: lease.row,
8558 end_col: lease.col,
8559 });
8560 }
8561 if self
8562 .staged_formula_index
8563 .package_lease_for_sheet(&sheet)
8564 .is_some()
8565 {
8566 regions.push_back(PreparationRegion {
8567 sheet: sheet.clone(),
8568 sheet_id,
8569 start_row: 1,
8570 start_col: 1,
8571 end_row: self.workbook_load_limits.max_sheet_rows,
8572 end_col: self.workbook_load_limits.max_sheet_cols,
8573 });
8574 }
8575 }
8576 }
8577 if matches!(scope, PrepareScope::Workbook) && !workbook_seeded {
8578 workbook_seeded = true;
8579 for (sheet, lease) in self.staged_formula_index.all_leases() {
8580 self.target_preparation_checkpoint(options.deadline, 1)?;
8581 let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
8582 package_encountered = true;
8583 Self::widen_target_preparation(
8584 options.opaque_policy,
8585 &mut scope,
8586 &mut reasons,
8587 OpaqueReason::UnsupportedSourceSemantics,
8588 )?;
8589 continue;
8590 };
8591 regions.push_back(PreparationRegion {
8592 sheet,
8593 sheet_id,
8594 start_row: lease.row,
8595 start_col: lease.col,
8596 end_row: lease.row,
8597 end_col: lease.col,
8598 });
8599 }
8600 let package_sheets = self
8601 .staged_formula_index
8602 .package_sheets()
8603 .map(str::to_string)
8604 .collect::<Vec<_>>();
8605 for sheet in package_sheets {
8606 let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
8607 package_encountered = true;
8608 Self::widen_target_preparation(
8609 options.opaque_policy,
8610 &mut scope,
8611 &mut reasons,
8612 OpaqueReason::UnsupportedSourceSemantics,
8613 )?;
8614 continue;
8615 };
8616 regions.push_back(PreparationRegion {
8617 sheet,
8618 sheet_id,
8619 start_row: 1,
8620 start_col: 1,
8621 end_row: self.workbook_load_limits.max_sheet_rows,
8622 end_col: self.workbook_load_limits.max_sheet_cols,
8623 });
8624 }
8625 }
8626
8627 let allow_partial_shared = regions.is_empty() && symbol_vertices.is_empty();
8630 let next_region = if allow_partial_shared {
8631 deferred_shared_regions.pop_front()
8632 } else {
8633 regions.pop_front()
8634 };
8635 let Some(region) = next_region else {
8636 if let Some(vertex) = symbol_vertices.pop_front() {
8637 self.target_preparation_checkpoint(options.deadline, 1)?;
8638 if !visited_vertices.insert(vertex) || !self.graph.vertex_exists(vertex) {
8639 continue;
8640 }
8641 let vertex_is_dynamic = self.graph.is_dynamic(vertex);
8642 if let Some(ast) = self.graph.get_formula(vertex) {
8643 let snapshot =
8644 self.target_planning_snapshot(&ast, &mut planning_requests)?;
8645 if let Some(reason) =
8646 Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
8647 {
8648 return Err(Self::preparation_stale(
8649 reason,
8650 "target planning snapshot became stale during discovery",
8651 ));
8652 }
8653 let opaque = self.opaque_reason_in_ast(&ast, &snapshot);
8654 if let Some(reason) =
8655 opaque.or(vertex_is_dynamic.then_some(OpaqueReason::DynamicReference))
8656 {
8657 if reason == OpaqueReason::DynamicReference
8658 && Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot)
8659 {
8660 let sheet = self.graph.get_vertex_sheet_id(vertex);
8661 let sheet = self.graph.sheet_name(sheet).to_string();
8662 Self::widen_target_preparation_to_sheet(
8663 options.opaque_policy,
8664 &mut scope,
8665 &mut reasons,
8666 reason,
8667 &sheet,
8668 )?;
8669 } else {
8670 Self::widen_target_preparation(
8671 options.opaque_policy,
8672 &mut scope,
8673 &mut reasons,
8674 reason,
8675 )?;
8676 }
8677 }
8678 } else if vertex_is_dynamic {
8679 Self::widen_target_preparation(
8680 options.opaque_policy,
8681 &mut scope,
8682 &mut reasons,
8683 OpaqueReason::DynamicReference,
8684 )?;
8685 }
8686 if let Some(anchor) = self
8687 .graph
8688 .get_cell_ref(vertex)
8689 .and_then(|cell| self.graph.spill_registry_anchor_for_cell(cell))
8690 {
8691 symbol_vertices.push_back(anchor);
8692 }
8693 if let Some(cell) = self.graph.get_cell_ref(vertex) {
8694 let sheet = self.graph.sheet_name(cell.sheet_id).to_string();
8695 regions.push_back(PreparationRegion {
8696 sheet,
8697 sheet_id: cell.sheet_id,
8698 start_row: cell.coord.row() + 1,
8699 start_col: cell.coord.col() + 1,
8700 end_row: cell.coord.row() + 1,
8701 end_col: cell.coord.col() + 1,
8702 });
8703 }
8704 match self.graph.authority_vertex_precedents(vertex) {
8708 Some(precedents) => {
8709 for (sheet_id, rect) in precedents {
8710 self.target_preparation_checkpoint(options.deadline, 1)?;
8711 if sheet_id == crate::engine::authority::geom::SYMBOL_SHEET {
8712 for slot in rect.r0..=rect.r1 {
8713 if let Some(symbol) =
8714 self.graph.authority_host().symbols().vertex(slot)
8715 {
8716 symbol_vertices.push_back(symbol);
8717 }
8718 }
8719 continue;
8720 }
8721 let sheet = self.graph.sheet_name(sheet_id).to_string();
8722 regions.push_back(PreparationRegion {
8723 sheet,
8724 sheet_id,
8725 start_row: rect.r0 + 1,
8726 start_col: rect.c0 + 1,
8727 end_row: (rect.r1 + 1)
8728 .min(self.workbook_load_limits.max_sheet_rows),
8729 end_col: (rect.c1 + 1)
8730 .min(self.workbook_load_limits.max_sheet_cols),
8731 });
8732 }
8733 }
8734 None => {
8736 if Self::widen_target_preparation(
8737 options.opaque_policy,
8738 &mut scope,
8739 &mut reasons,
8740 OpaqueReason::UnresolvedCrossSheetBinding,
8741 )? {
8742 continue;
8743 }
8744 }
8745 }
8746 if let Some(name) = self.graph.named_range_by_vertex(vertex).cloned() {
8747 match &name.definition {
8748 NamedDefinition::Cell(cell) => regions.push_back(PreparationRegion {
8749 sheet: self.graph.sheet_name(cell.sheet_id).to_string(),
8750 sheet_id: cell.sheet_id,
8751 start_row: cell.coord.row() + 1,
8752 start_col: cell.coord.col() + 1,
8753 end_row: cell.coord.row() + 1,
8754 end_col: cell.coord.col() + 1,
8755 }),
8756 NamedDefinition::Range(range) => regions.push_back(PreparationRegion {
8757 sheet: self.graph.sheet_name(range.start.sheet_id).to_string(),
8758 sheet_id: range.start.sheet_id,
8759 start_row: range.start.coord.row() + 1,
8760 start_col: range.start.coord.col() + 1,
8761 end_row: range.end.coord.row() + 1,
8762 end_col: range.end.coord.col() + 1,
8763 }),
8764 NamedDefinition::Formula {
8765 ast,
8766 dependencies,
8767 range_deps,
8768 } => {
8769 let snapshot =
8770 self.target_planning_snapshot(ast, &mut planning_requests)?;
8771 if let Some(reason) = Self::target_planning_snapshot_stale_reason(
8772 &snapshot,
8773 &assumptions,
8774 ) {
8775 return Err(Self::preparation_stale(
8776 reason,
8777 "target planning snapshot became stale during discovery",
8778 ));
8779 }
8780 if let Some(reason) = self.opaque_reason_in_ast(ast, &snapshot) {
8781 Self::widen_target_preparation(
8782 options.opaque_policy,
8783 &mut scope,
8784 &mut reasons,
8785 reason,
8786 )?;
8787 }
8788 for dependency in dependencies {
8789 self.target_preparation_checkpoint(options.deadline, 1)?;
8790 symbol_vertices.push_back(*dependency);
8791 }
8792 for range in range_deps {
8793 self.target_preparation_checkpoint(options.deadline, 1)?;
8794 let context_sheet = self.graph.get_vertex_sheet_id(vertex);
8802 let Ok(sheet_id) =
8803 self.resolve_sheet_locator(&range.sheet, context_sheet)
8804 else {
8805 if Self::widen_target_preparation(
8806 options.opaque_policy,
8807 &mut scope,
8808 &mut reasons,
8809 OpaqueReason::UnresolvedCrossSheetBinding,
8810 )? {
8811 break;
8812 }
8813 continue;
8814 };
8815 regions.push_back(PreparationRegion {
8816 sheet: self.graph.sheet_name(sheet_id).to_string(),
8817 sheet_id,
8818 start_row: range
8819 .start_row
8820 .map_or(1, |bound| bound.index + 1),
8821 start_col: range
8822 .start_col
8823 .map_or(1, |bound| bound.index + 1),
8824 end_row: range.end_row.map_or(
8825 self.workbook_load_limits.max_sheet_rows,
8826 |bound| bound.index + 1,
8827 ),
8828 end_col: range.end_col.map_or(
8829 self.workbook_load_limits.max_sheet_cols,
8830 |bound| bound.index + 1,
8831 ),
8832 });
8833 }
8834 }
8835 NamedDefinition::Literal(_) => {}
8836 }
8837 }
8838 if let Some(table) = self.graph.table_by_vertex(vertex) {
8839 regions
8840 .push_back(self.table_selection_region(table, &TableSelection::Whole)?);
8841 }
8842 continue;
8843 }
8844 break;
8845 };
8846
8847 self.target_preparation_checkpoint(options.deadline, 1)?;
8848 if !visited_regions.insert(region.clone()) && !allow_partial_shared {
8849 continue;
8850 }
8851 if let PrepareScope::Sheets(sheets) = &mut scope
8852 && !sheets.iter().any(|sheet| sheet == ®ion.sheet)
8853 {
8854 sheets.push(region.sheet.clone());
8855 sheets.sort();
8856 }
8857 let package_match = self.staged_formula_index.package_for_region(
8858 ®ion.sheet,
8859 region.start_row,
8860 region.start_col,
8861 region.end_row,
8862 region.end_col,
8863 );
8864 let package_lease = match package_match {
8865 Some(Ok(lease)) => Some(lease),
8866 Some(Err(()))
8867 if region.start_row == 1
8868 && region.start_col == 1
8869 && region.end_row == self.workbook_load_limits.max_sheet_rows
8870 && region.end_col == self.workbook_load_limits.max_sheet_cols =>
8871 {
8872 self.staged_formula_index
8873 .package_lease_for_sheet(®ion.sheet)
8874 }
8875 Some(Err(())) => {
8876 Self::widen_target_preparation(
8877 options.opaque_policy,
8878 &mut scope,
8879 &mut reasons,
8880 OpaqueReason::DeferredSourcePackage,
8881 )?;
8882 None
8883 }
8884 None => None,
8885 };
8886 let compatibility_before_package_replay = package_lease.is_some()
8887 && (authoritative_with_ordinary || has_unknown_package_sheet);
8888 let package_lease = if compatibility_before_package_replay {
8889 package_encountered = true;
8890 Self::widen_target_preparation(
8891 options.opaque_policy,
8892 &mut scope,
8893 &mut reasons,
8894 OpaqueReason::UnsupportedSourceSemantics,
8895 )?;
8896 None
8897 } else {
8898 package_lease
8899 };
8900 if let Some(package_lease) = package_lease
8901 && !selected_package_sheets.contains(®ion.sheet)
8902 {
8903 self.target_preparation_checkpoint(options.deadline, 1)?;
8904 let mut points = self.staged_formula_index.package_points_in_region(
8905 ®ion.sheet,
8906 region.start_row,
8907 region.start_col,
8908 region.end_row,
8909 region.end_col,
8910 );
8911 if let Some(selected) = selected_package_points.get(®ion.sheet) {
8912 points.retain(|point| !selected.contains(point));
8913 }
8914
8915 if let Some(selected) = selected_package_points.get(®ion.sheet) {
8916 points.retain(|point| !selected.contains(point));
8917 }
8918 let permit_partial = allow_partial_shared
8919 || (points.len() == 1
8920 && regions.is_empty()
8921 && symbol_vertices.is_empty()
8922 && deferred_shared_regions.is_empty());
8923 let partial = self.prepare_target_exact_source_selection(
8924 ®ion.sheet,
8925 package_lease,
8926 points,
8927 selected_package_points
8928 .get(®ion.sheet)
8929 .unwrap_or(&BTreeSet::new()),
8930 permit_partial,
8931 options.deadline,
8932 &mut discovery_scratch_reserved,
8933 )?;
8934 let mut package = if let Some(mut package) = partial {
8935 if package.deferred_shared {
8936 deferred_shared_regions.push_back(region.clone());
8937 }
8938 if !package.direct_domains.is_empty() {
8942 for prior in prepared_packages
8943 .iter_mut()
8944 .filter(|prior| prior.sheet == region.sheet)
8945 {
8946 prior
8947 .replay_records
8948 .retain(|record| !package.direct_contains(record.row, record.col));
8949 prior
8950 .legacy
8951 .retain(|(row, col, _, _)| !package.direct_contains(*row, *col));
8952 if let Some(points) = prior.selected_points.as_mut() {
8953 points.retain(|&(row, col)| !package.direct_contains(row, col));
8954 }
8955 }
8956 }
8957 if selected_package_points.contains_key(®ion.sheet) {
8958 package.source_report = Default::default();
8959 }
8960 let points = package.selected_points.as_ref().unwrap();
8961 if !points.is_empty() {
8962 selected_package_points
8963 .entry(region.sheet.clone())
8964 .or_default()
8965 .extend(points.iter().copied());
8966 }
8967 package
8968 } else {
8969 selected_package_sheets.insert(region.sheet.clone());
8970 self.prepare_target_source_package(
8971 ®ion.sheet,
8972 package_lease,
8973 options.deadline,
8974 )?
8975 };
8976 if package
8977 .selected_points
8978 .as_ref()
8979 .is_none_or(|points| !points.is_empty())
8980 {
8981 let mut final_fallback = BTreeMap::new();
8982 for record in package.fallback_records() {
8983 final_fallback.insert((record.row, record.col), record.clone());
8984 }
8985 let batch = self.formula_batch_from_exact_replay(
8986 ®ion.sheet,
8987 final_fallback.into_values(),
8988 )?;
8989 for record in batch.formulas {
8990 self.target_preparation_checkpoint(options.deadline, 1)?;
8991 let ast = self
8992 .graph
8993 .data_store()
8994 .retrieve_ast(record.ast_id, self.graph.sheet_reg())
8995 .ok_or_else(|| {
8996 ExcelError::new(ExcelErrorKind::Value)
8997 .with_message("target fallback AST is unavailable")
8998 })?;
8999 let snapshot =
9000 self.target_planning_snapshot(&ast, &mut planning_requests)?;
9001 if let Some(reason) =
9002 Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
9003 {
9004 return Err(Self::preparation_stale(
9005 reason,
9006 "target fallback planning snapshot became stale during discovery",
9007 ));
9008 }
9009 let proven_sheet_local_dynamic =
9010 Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot);
9011 if let Some(reason) = self.opaque_reason_in_ast(&ast, &snapshot) {
9012 if reason == OpaqueReason::DynamicReference
9013 && proven_sheet_local_dynamic
9014 {
9015 Self::widen_target_preparation_to_sheet(
9016 options.opaque_policy,
9017 &mut scope,
9018 &mut reasons,
9019 reason,
9020 ®ion.sheet,
9021 )?;
9022 } else {
9023 Self::widen_target_preparation(
9024 options.opaque_policy,
9025 &mut scope,
9026 &mut reasons,
9027 reason,
9028 )?;
9029 }
9030 }
9031 let placement = CellRef::new(
9032 package.sheet_id,
9033 Coord::from_excel(record.row, record.col, true, true),
9034 );
9035 let ingested = self
9036 .graph
9037 .ingest_pipeline(&snapshot)
9038 .enable_function_semantics()
9039 .ingest_formula(
9040 FormulaAstInput::RawArena(record.ast_id),
9041 placement,
9042 record.formula_text,
9043 )?;
9044 if ingested.dep_plan.dynamic {
9045 if proven_sheet_local_dynamic {
9046 Self::widen_target_preparation_to_sheet(
9047 options.opaque_policy,
9048 &mut scope,
9049 &mut reasons,
9050 OpaqueReason::DynamicReference,
9051 ®ion.sheet,
9052 )?;
9053 } else {
9054 Self::widen_target_preparation(
9055 options.opaque_policy,
9056 &mut scope,
9057 &mut reasons,
9058 OpaqueReason::DynamicReference,
9059 )?;
9060 }
9061 }
9062 for dep in &ingested.dep_plan.direct_cell_deps {
9063 self.target_preparation_checkpoint(options.deadline, 1)?;
9064 regions.push_back(PreparationRegion {
9065 sheet: self.graph.sheet_name(dep.sheet_id).to_string(),
9066 sheet_id: dep.sheet_id,
9067 start_row: dep.coord.row().saturating_add(1),
9068 start_col: dep.coord.col().saturating_add(1),
9069 end_row: dep.coord.row().saturating_add(1),
9070 end_col: dep.coord.col().saturating_add(1),
9071 });
9072 }
9073 for range in &ingested.dep_plan.range_deps {
9074 self.target_preparation_checkpoint(options.deadline, 1)?;
9075 let Ok(dependency_sheet) =
9078 self.resolve_sheet_locator(&range.sheet, package.sheet_id)
9079 else {
9080 Self::widen_target_preparation(
9081 options.opaque_policy,
9082 &mut scope,
9083 &mut reasons,
9084 OpaqueReason::UnresolvedCrossSheetBinding,
9085 )?;
9086 continue;
9087 };
9088 regions.push_back(PreparationRegion {
9089 sheet: self.graph.sheet_name(dependency_sheet).to_string(),
9090 sheet_id: dependency_sheet,
9091 start_row: range.start_row.map_or(1, |bound| bound.index + 1),
9092 start_col: range.start_col.map_or(1, |bound| bound.index + 1),
9093 end_row: range
9094 .end_row
9095 .map_or(self.workbook_load_limits.max_sheet_rows, |bound| {
9096 bound.index + 1
9097 }),
9098 end_col: range
9099 .end_col
9100 .map_or(self.workbook_load_limits.max_sheet_cols, |bound| {
9101 bound.index + 1
9102 }),
9103 });
9104 }
9105 for name in ingested
9106 .dep_plan
9107 .resolved_named_refs
9108 .iter()
9109 .chain(&ingested.dep_plan.named_refs)
9110 {
9111 self.target_preparation_checkpoint(options.deadline, 1)?;
9112 if let Some(entry) =
9113 self.graph.resolve_name_entry(name, package.sheet_id)
9114 {
9115 symbol_vertices.push_back(entry.vertex);
9116 } else if self.graph.resolve_source_scalar_entry(name).is_none()
9117 && self.graph.resolve_source_table_entry(name).is_none()
9118 {
9119 Self::widen_target_preparation(
9120 options.opaque_policy,
9121 &mut scope,
9122 &mut reasons,
9123 OpaqueReason::UnresolvedName,
9124 )?;
9125 }
9126 }
9127 for table in &ingested.dep_plan.table_refs {
9128 self.target_preparation_checkpoint(options.deadline, 1)?;
9129 if let Some(entry) = self.graph.resolve_table_entry(table) {
9130 symbol_vertices.push_back(entry.vertex);
9131 } else if self.graph.resolve_source_table_entry(table).is_none() {
9132 Self::widen_target_preparation(
9133 options.opaque_policy,
9134 &mut scope,
9135 &mut reasons,
9136 OpaqueReason::UnresolvedTable,
9137 )?;
9138 }
9139 }
9140 package.legacy.push((
9141 record.row,
9142 record.col,
9143 ingested.ast_id,
9144 ingested.dep_plan,
9145 ));
9146 }
9147 prepared_packages.push(package);
9148 }
9149 }
9150 let leases = self.staged_formula_index.leases_in_region(
9151 ®ion.sheet,
9152 region.start_row,
9153 region.start_col,
9154 region.end_row,
9155 region.end_col,
9156 );
9157 for lease in leases {
9158 self.target_preparation_checkpoint(options.deadline, 1)?;
9159 let sheet_id = self.graph.sheet_id(®ion.sheet).ok_or_else(|| {
9160 ExcelError::new(ExcelErrorKind::Ref)
9161 .with_message(format!("staged formula sheet not found: {}", region.sheet))
9162 })?;
9163 let key = (region.sheet.clone(), lease.row, lease.col, lease.generation);
9164 if !selected.insert(key) {
9165 continue;
9166 }
9167 let text = self
9168 .staged_formulas
9169 .get(®ion.sheet)
9170 .and_then(|sheet| sheet.get_ordinary(lease.row, lease.col))
9171 .ok_or_else(|| {
9172 ExcelError::new(ExcelErrorKind::Value)
9173 .with_message("staged formula index is stale")
9174 })?
9175 .to_string();
9176 let formula = if text.starts_with('=') {
9177 text.clone()
9178 } else {
9179 format!("={text}")
9180 };
9181 self.target_preparation_checkpoint(options.deadline, 1)?;
9182 let ast = match formualizer_parse::parser::parse(&formula) {
9183 Ok(ast) => ast,
9184 Err(error) => {
9185 if self.config.formula_parse_policy == FormulaParsePolicy::Strict {
9186 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
9187 format!(
9188 "Formula parse error at {}!{}{}: {error}",
9189 region.sheet,
9190 col_letters_from_1based(lease.col)
9191 .unwrap_or_else(|_| "?".to_string()),
9192 lease.row
9193 ),
9194 ));
9195 }
9196 pending_diagnostics.push(FormulaParseDiagnostic {
9197 sheet: region.sheet.clone(),
9198 row: lease.row,
9199 col: lease.col,
9200 formula: formula.clone(),
9201 message: error.to_string(),
9202 policy: self.config.formula_parse_policy,
9203 });
9204 match self.config.formula_parse_policy {
9205 FormulaParsePolicy::KeepCachedValue => {
9206 selected_cells.push(
9207 formualizer_common::RangeAddress::new(
9208 region.sheet.clone(),
9209 lease.row,
9210 lease.col,
9211 lease.row,
9212 lease.col,
9213 )
9214 .expect("selected staged coordinates are valid"),
9215 );
9216 prepared.push(PreparedOrdinaryStagedFormula {
9217 sheet: region.sheet.clone(),
9218 sheet_id,
9219 lease,
9220 ast_id: None,
9221 plan: None,
9222 });
9223 continue;
9224 }
9225 FormulaParsePolicy::AsText => ASTNode::new(
9226 ASTNodeType::Literal(LiteralValue::Text(formula.clone())),
9227 None,
9228 ),
9229 FormulaParsePolicy::CoerceToError => ASTNode::new(
9230 ASTNodeType::Literal(LiteralValue::Error(
9231 ExcelError::new(ExcelErrorKind::Error)
9232 .with_message(format!("Malformed formula: {error}")),
9233 )),
9234 None,
9235 ),
9236 FormulaParsePolicy::Strict => unreachable!(),
9237 }
9238 }
9239 };
9240 self.target_preparation_checkpoint(options.deadline, 1)?;
9241 let snapshot = self.target_planning_snapshot(&ast, &mut planning_requests)?;
9242 self.target_preparation_checkpoint(options.deadline, 1)?;
9243 if let Some(reason) =
9244 Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
9245 {
9246 return Err(Self::preparation_stale(
9247 reason,
9248 "target planning snapshot became stale during discovery",
9249 ));
9250 }
9251 if let Some(reason) = self.opaque_reason_in_ast(&ast, &snapshot) {
9252 if reason == OpaqueReason::DynamicReference
9253 && Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot)
9254 {
9255 Self::widen_target_preparation_to_sheet(
9256 options.opaque_policy,
9257 &mut scope,
9258 &mut reasons,
9259 reason,
9260 ®ion.sheet,
9261 )?;
9262 } else {
9263 Self::widen_target_preparation(
9264 options.opaque_policy,
9265 &mut scope,
9266 &mut reasons,
9267 reason,
9268 )?;
9269 }
9270 }
9271 let proven_sheet_local_dynamic =
9272 Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot);
9273 let placement = CellRef::new(
9274 sheet_id,
9275 Coord::from_excel(lease.row, lease.col, true, true),
9276 );
9277 let ingested = self.graph.ingest_pipeline(&snapshot).ingest_formula(
9278 FormulaAstInput::Tree(ast),
9279 placement,
9280 Some(Arc::from(formula)),
9281 )?;
9282 self.target_preparation_checkpoint(options.deadline, 1)?;
9283 if ingested.dep_plan.dynamic {
9284 if proven_sheet_local_dynamic {
9285 Self::widen_target_preparation_to_sheet(
9286 options.opaque_policy,
9287 &mut scope,
9288 &mut reasons,
9289 OpaqueReason::DynamicReference,
9290 ®ion.sheet,
9291 )?;
9292 } else {
9293 Self::widen_target_preparation(
9294 options.opaque_policy,
9295 &mut scope,
9296 &mut reasons,
9297 OpaqueReason::DynamicReference,
9298 )?;
9299 }
9300 }
9301 for dep in &ingested.dep_plan.direct_cell_deps {
9302 self.target_preparation_checkpoint(options.deadline, 1)?;
9303 regions.push_back(PreparationRegion {
9304 sheet: self.graph.sheet_name(dep.sheet_id).to_string(),
9305 sheet_id: dep.sheet_id,
9306 start_row: dep.coord.row() + 1,
9307 start_col: dep.coord.col() + 1,
9308 end_row: dep.coord.row() + 1,
9309 end_col: dep.coord.col() + 1,
9310 });
9311 }
9312 for range in &ingested.dep_plan.range_deps {
9313 self.target_preparation_checkpoint(options.deadline, 1)?;
9314 let Ok(dependency_sheet) = self.resolve_sheet_locator(&range.sheet, sheet_id)
9316 else {
9317 Self::widen_target_preparation(
9318 options.opaque_policy,
9319 &mut scope,
9320 &mut reasons,
9321 OpaqueReason::UnresolvedCrossSheetBinding,
9322 )?;
9323 continue;
9324 };
9325 regions.push_back(PreparationRegion {
9326 sheet: self.graph.sheet_name(dependency_sheet).to_string(),
9327 sheet_id: dependency_sheet,
9328 start_row: range.start_row.map_or(1, |bound| bound.index + 1),
9329 start_col: range.start_col.map_or(1, |bound| bound.index + 1),
9330 end_row: range
9331 .end_row
9332 .map_or(self.workbook_load_limits.max_sheet_rows, |bound| {
9333 bound.index + 1
9334 }),
9335 end_col: range
9336 .end_col
9337 .map_or(self.workbook_load_limits.max_sheet_cols, |bound| {
9338 bound.index + 1
9339 }),
9340 });
9341 }
9342 for name in ingested
9343 .dep_plan
9344 .resolved_named_refs
9345 .iter()
9346 .chain(&ingested.dep_plan.named_refs)
9347 {
9348 self.target_preparation_checkpoint(options.deadline, 1)?;
9349 if let Some(entry) = self.graph.resolve_name_entry(name, sheet_id) {
9350 symbol_vertices.push_back(entry.vertex);
9351 } else if self.graph.resolve_source_scalar_entry(name).is_none()
9352 && self.graph.resolve_source_table_entry(name).is_none()
9353 {
9354 Self::widen_target_preparation(
9355 options.opaque_policy,
9356 &mut scope,
9357 &mut reasons,
9358 OpaqueReason::UnresolvedName,
9359 )?;
9360 }
9361 }
9362 for table in &ingested.dep_plan.table_refs {
9363 self.target_preparation_checkpoint(options.deadline, 1)?;
9364 if let Some(entry) = self.graph.resolve_table_entry(table) {
9365 symbol_vertices.push_back(entry.vertex);
9366 } else if self.graph.resolve_source_table_entry(table).is_none() {
9367 Self::widen_target_preparation(
9368 options.opaque_policy,
9369 &mut scope,
9370 &mut reasons,
9371 OpaqueReason::UnresolvedTable,
9372 )?;
9373 }
9374 }
9375 selected_cells.push(
9376 formualizer_common::RangeAddress::new(
9377 region.sheet.clone(),
9378 lease.row,
9379 lease.col,
9380 lease.row,
9381 lease.col,
9382 )
9383 .expect("selected staged coordinates are valid"),
9384 );
9385 prepared.push(PreparedOrdinaryStagedFormula {
9386 sheet: region.sheet.clone(),
9387 sheet_id,
9388 lease,
9389 ast_id: Some(ingested.ast_id),
9390 plan: Some(ingested.dep_plan),
9391 });
9392 }
9393
9394 if indexed_query_sheets.insert(region.sheet_id) {
9395 self.graph.prepare_sheet_index_for_query(region.sheet_id);
9396 let bytes = (self.graph.sheet_index_vertex_count(region.sheet_id) as u64)
9397 .saturating_mul(32);
9398 self.reserve_graph_source_scratch(bytes)?;
9399 discovery_scratch_reserved = discovery_scratch_reserved.saturating_add(bytes);
9400 }
9401 let spill_anchors = self.graph.spill_anchors_in_region(
9402 region.sheet_id,
9403 region.start_row - 1,
9404 region.start_col - 1,
9405 region.end_row - 1,
9406 region.end_col - 1,
9407 );
9408 for anchor in spill_anchors {
9409 self.target_preparation_checkpoint(options.deadline, 1)?;
9410 symbol_vertices.push_back(anchor);
9411 }
9412 let vertices = self.graph.vertices_in_region(
9413 region.sheet_id,
9414 region.start_row - 1,
9415 region.end_row - 1,
9416 region.start_col - 1,
9417 region.end_col - 1,
9418 );
9419 for vertex in vertices {
9420 self.target_preparation_checkpoint(options.deadline, 1)?;
9421 symbol_vertices.push_back(vertex);
9422 }
9423 }
9424
9425 #[cfg(test)]
9426 self.target_preparation_fault(
9427 crate::engine::target_preparation::TargetPreparationFault::AfterDiscovery,
9428 )?;
9429
9430 if package_encountered {
9431 Self::widen_target_preparation(
9432 options.opaque_policy,
9433 &mut scope,
9434 &mut reasons,
9435 OpaqueReason::UnsupportedSourceSemantics,
9436 )?;
9437 self.target_preparation_checkpoint(options.deadline, 0)?;
9438 let selected_count = self.staged_formula_count();
9439 let selected_packages = self
9440 .staged_formulas
9441 .values()
9442 .filter_map(|staged| staged.deferred_package.as_ref())
9443 .map(|package| package.families.len() + package.partitioned_families.len())
9444 .sum();
9445 self.formula_parse_diagnostics.truncate(diagnostics_len);
9446 self.last_formula_ingest_report = report_len;
9447 #[cfg(test)]
9448 if let Some(hook) = self.before_target_preparation_commit_hook.take() {
9449 hook();
9450 }
9451 self.target_preparation_checkpoint(options.deadline, 0)?;
9452 #[cfg(test)]
9453 self.target_preparation_fault(
9454 crate::engine::target_preparation::TargetPreparationFault::FinalRevisionValidation,
9455 )?;
9456 let current_revisions = self.preparation_revisions();
9457 if let Some(reason) = Self::preparation_revision_stale_reason(
9458 &assumptions,
9459 ¤t_revisions,
9460 &planning_requests,
9461 true,
9462 ) {
9463 return Err(Self::preparation_stale(
9464 reason,
9465 "target compatibility preparation plan is stale",
9466 ));
9467 }
9468 #[cfg(test)]
9469 self.target_preparation_fault(
9470 crate::engine::target_preparation::TargetPreparationFault::FinalGraphValidation,
9471 )?;
9472 let commit_work_before = self
9473 .active_resource_ledger
9474 .as_ref()
9475 .map_or(0, |ledger| ledger.snapshot().work_charged);
9476 let commit_started = crate::instant::FzInstant::now();
9477 self.build_graph_all_unobserved()?;
9478 let commit_window = commit_started.elapsed();
9479 let ledger_after = self
9480 .active_resource_ledger
9481 .as_ref()
9482 .map(|ledger| ledger.snapshot());
9483 let actual_commit_work = ledger_after
9484 .map_or(0, |snapshot| snapshot.work_charged)
9485 .saturating_sub(commit_work_before);
9486 let observed_scratch_bytes = ledger_after
9487 .map_or(0, |snapshot| snapshot.scratch_peak)
9488 .saturating_sub(ledger_at_start.map_or(0, |snapshot| snapshot.scratch_current));
9489 let revisions = assumptions.clone();
9490 let report = PreparedTargetGraphReport {
9491 request_id: request_id.unwrap_or_default(),
9492 requested_targets: targets.len(),
9493 normalized_regions: visited_regions.len(),
9494 normalized_target_list: normalized,
9495 selected_staged_cells: selected_count,
9496 selected_source_families: selected_packages,
9497 retained_staged_cells: self.staged_formula_count(),
9498 selected_cells,
9499 retained_cells: Vec::new(),
9500 widened_scope: PrepareScope::Workbook,
9501 widening_reasons: reasons,
9502 revisions,
9503 commit_window,
9504 estimated_scratch_bytes: discovery_scratch_reserved
9505 .saturating_add((selected_count as u64).saturating_mul(256)),
9506 observed_scratch_bytes,
9507 estimated_commit_work: selected_count as u64,
9508 actual_commit_work,
9509 outcome: PreparationOutcome::CompatibilityPrepared,
9510 };
9511 self.observe_target_preparation_report(&report);
9512 return Ok(report);
9513 }
9514
9515 prepared.sort_by_key(|formula| formula.lease.insertion_order);
9516 for package in &prepared_packages {
9517 if let Some(points) = &package.selected_points {
9518 selected_cells.extend(points.iter().filter_map(|&(row, col)| {
9519 formualizer_common::RangeAddress::new(&package.sheet, row, col, row, col).ok()
9520 }));
9521 continue;
9522 }
9523 selected_cells.extend(package.replay_records.iter().filter_map(|record| {
9524 formualizer_common::RangeAddress::new(
9525 package.sheet.clone(),
9526 record.row,
9527 record.col,
9528 record.row,
9529 record.col,
9530 )
9531 .ok()
9532 }));
9533 }
9534 let (legacy_graph, planned_formula_count) =
9535 self.prepare_target_combined_legacy_graph(&prepared_packages, &prepared)?;
9536 let new_vertices = legacy_graph.new_vertex_count();
9537 let new_edges = legacy_graph.planned_edge_count().ok_or_else(|| {
9538 ExcelError::new(ExcelErrorKind::NImpl).with_message("target graph edge count overflow")
9539 })?;
9540 let removed_edges = legacy_graph.removed_edge_count().ok_or_else(|| {
9541 ExcelError::new(ExcelErrorKind::NImpl).with_message("target graph edge count overflow")
9542 })?;
9543 let current = self.graph.baseline_stats();
9544 let final_vertices = current
9545 .graph_vertex_count
9546 .checked_add(new_vertices)
9547 .ok_or_else(|| {
9548 crate::engine::ResourceLedgerError::Exhausted(
9549 formualizer_common::ResourceExhaustionDetail {
9550 reason: formualizer_common::ResourceExhaustionReason::ArithmeticOverflow,
9551 limit: u64::MAX,
9552 observed: u64::MAX,
9553 request_id,
9554 },
9555 )
9556 .into_excel_error()
9557 })?;
9558 let final_edges = current
9559 .graph_edge_count
9560 .checked_sub(removed_edges)
9561 .and_then(|count| count.checked_add(new_edges))
9562 .ok_or_else(|| {
9563 crate::engine::ResourceLedgerError::Exhausted(
9564 formualizer_common::ResourceExhaustionDetail {
9565 reason: formualizer_common::ResourceExhaustionReason::ArithmeticOverflow,
9566 limit: u64::MAX,
9567 observed: u64::MAX,
9568 request_id,
9569 },
9570 )
9571 .into_excel_error()
9572 })?;
9573 #[cfg(test)]
9574 self.target_preparation_fault(
9575 crate::engine::target_preparation::TargetPreparationFault::Admission,
9576 )?;
9577 let resource = |reason, limit: u64, observed: u64| {
9578 crate::engine::ResourceLedgerError::Exhausted(
9579 formualizer_common::ResourceExhaustionDetail {
9580 reason,
9581 limit,
9582 observed,
9583 request_id,
9584 },
9585 )
9586 .into_excel_error()
9587 };
9588 let admission = crate::engine::resource_ledger::GraphAdmission {
9589 final_vertices,
9590 final_edges,
9591 materialization_cells: planned_formula_count as u64,
9592 added_vertices: new_vertices,
9593 added_edges: new_edges,
9594 };
9595 let materialized_bytes = admission
9596 .materialized_graph_bytes()
9597 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
9598 if let Err(error) = self.preflight_graph_admission(admission) {
9599 if let formualizer_common::ExcelErrorExtra::Resource { detail } = error.extra() {
9600 self.observe_target_admission_failure(detail.reason);
9601 }
9602 return Err(error);
9603 }
9604 let selected_package_records = prepared_packages
9605 .iter()
9606 .map(|package| package.replay_records.len() as u64)
9607 .sum::<u64>();
9608 let planned_working_bytes = (prepared.len() as u64)
9609 .saturating_add(selected_package_records)
9610 .saturating_mul(256)
9611 .saturating_add((visited_regions.len() as u64).saturating_mul(128))
9612 .saturating_add((visited_vertices.len() as u64).saturating_mul(32))
9613 .saturating_add((selected.len() as u64).saturating_mul(96))
9614 .saturating_add(materialized_bytes);
9615 let residual_scratch = selected_package_points
9616 .iter()
9617 .map(|(sheet, points)| {
9618 let source = self
9619 .staged_formulas
9620 .get(sheet)
9621 .unwrap()
9622 .deferred_package
9623 .as_ref()
9624 .unwrap();
9625 (source.families.len() as u64)
9626 .saturating_mul(256)
9627 .saturating_add(
9628 source
9629 .partitioned_families
9630 .iter()
9631 .map(|family| {
9632 256u64
9633 .saturating_add(family.fragments.len() as u64 * 32)
9634 .saturating_add(family.legacy_members.len() as u64 * 32)
9635 })
9636 .sum::<u64>(),
9637 )
9638 .saturating_add(points.len() as u64 * 32)
9641 .saturating_add(
9642 prepared_packages
9643 .iter()
9644 .filter(|package| &package.sheet == sheet)
9645 .map(|package| package.replay_records.len() as u64 * 480)
9646 .sum::<u64>(),
9647 )
9648 })
9649 .sum::<u64>();
9650 let scratch_bytes = discovery_scratch_reserved
9651 .saturating_add(planned_working_bytes)
9652 .saturating_add(residual_scratch);
9653 let remaining_scratch = scratch_bytes.saturating_sub(discovery_scratch_reserved);
9654 if let Err(error) = self.reserve_graph_source_scratch(remaining_scratch) {
9655 self.observe_target_admission_failure(
9656 formualizer_common::ResourceExhaustionReason::ScratchMemory,
9657 );
9658 return Err(error);
9659 }
9660
9661 let mut residual_sources = BTreeMap::new();
9662 let mut residual_owners = BTreeMap::new();
9663 for (sheet, points) in &selected_package_points {
9664 let selected: BTreeMap<_, BTreeSet<_>> = prepared_packages
9665 .iter()
9666 .filter(|package| &package.sheet == sheet)
9667 .flat_map(|package| &package.replay_records)
9668 .filter_map(|record| {
9669 record.partition_owner.or(record.family).map(|owner| {
9670 (
9671 owner,
9672 crate::engine::SourceCoord {
9673 row: record.row - 1,
9674 col: record.col - 1,
9675 },
9676 )
9677 })
9678 })
9679 .fold(BTreeMap::new(), |mut map, (owner, coord)| {
9680 map.entry(owner).or_default().insert(coord);
9681 map
9682 });
9683 let metadata_work = self
9684 .staged_formulas
9685 .get(sheet)
9686 .unwrap()
9687 .deferred_package
9688 .as_ref()
9689 .map_or(0, |source| {
9690 source.families.len() as u64
9691 + source
9692 .partitioned_families
9693 .iter()
9694 .map(|family| {
9695 1 + family.fragments.len() as u64 * family.fragments.len() as u64
9696 + family.legacy_members.len() as u64
9697 })
9698 .sum::<u64>()
9699 });
9700 let split_work = selected
9701 .values()
9702 .map(|points| points.len() as u64 * 128)
9703 .sum::<u64>();
9704 self.target_preparation_checkpoint(
9705 options.deadline,
9706 metadata_work
9707 .saturating_add(split_work)
9708 .saturating_add(points.len() as u64),
9709 )?;
9710 let source = self
9711 .staged_formulas
9712 .get(sheet)
9713 .unwrap()
9714 .deferred_package
9715 .as_ref()
9716 .unwrap();
9717 let complete: BTreeSet<_> = prepared_packages
9718 .iter()
9719 .filter(|package| &package.sheet == sheet)
9720 .flat_map(|package| package.complete_selections.iter().copied())
9721 .collect();
9722 let residual = source
9723 .residual_sources(&selected, &complete, &self.workbook_load_limits)
9724 .map_err(|reason| ExcelError::new(ExcelErrorKind::Value).with_message(reason))?;
9725 residual_owners.insert(
9726 sheet.clone(),
9727 residual
9728 .1
9729 .iter()
9730 .map(|family| family.source_id)
9731 .collect::<BTreeSet<_>>(),
9732 );
9733 residual_sources.insert(sheet.clone(), residual);
9734 }
9735
9736 for (sheet, points) in &selected_package_points {
9739 let package = self
9740 .staged_formulas
9741 .get_mut(sheet)
9742 .unwrap()
9743 .deferred_package
9744 .as_mut()
9745 .unwrap();
9746 let owner_count = prepared_packages
9747 .iter()
9748 .filter(|p| &p.sheet == sheet)
9749 .flat_map(|p| &p.replay_records)
9750 .filter_map(|record| record.partition_owner.or(record.family))
9751 .filter(|owner| residual_owners[sheet].contains(owner))
9752 .count();
9753 package
9754 .consumed_members
9755 .try_reserve(owner_count)
9756 .map_err(|_| {
9757 resource(
9758 formualizer_common::ResourceExhaustionReason::ScratchMemory,
9759 0,
9760 owner_count as u64 * 24,
9761 )
9762 })?;
9763 package.suppressed.try_reserve(points.len()).map_err(|_| {
9764 resource(
9765 formualizer_common::ResourceExhaustionReason::ScratchMemory,
9766 0,
9767 points.len() as u64 * 16,
9768 )
9769 })?;
9770 }
9771
9772 let estimated_commit_duration = std::time::Duration::from_nanos(
9773 (new_vertices as u64)
9774 .saturating_add(new_edges as u64)
9775 .saturating_add(prepared.len() as u64)
9776 .saturating_add(selected_package_records)
9777 .max(1)
9778 .saturating_mul(100),
9779 );
9780 if options.deadline.is_some_and(|deadline| {
9781 std::time::Instant::now()
9782 .checked_add(estimated_commit_duration)
9783 .is_none_or(|finish| finish > deadline)
9784 }) {
9785 self.observe_target_admission_failure(
9786 formualizer_common::ResourceExhaustionReason::Deadline,
9787 );
9788 return Err(resource(
9789 formualizer_common::ResourceExhaustionReason::Deadline,
9790 0,
9791 1,
9792 ));
9793 }
9794 #[cfg(test)]
9795 if let Some(hook) = self.before_target_preparation_commit_hook.take() {
9796 hook();
9797 }
9798 self.target_preparation_checkpoint(options.deadline, 0)?;
9799 #[cfg(test)]
9800 self.target_preparation_fault(
9801 crate::engine::target_preparation::TargetPreparationFault::FinalRevisionValidation,
9802 )?;
9803 let current_revisions = self.preparation_revisions();
9804 let staged_leases_match = prepared.iter().all(|formula| {
9805 self.staged_formula_index
9806 .lease_matches(&formula.sheet, formula.lease)
9807 }) && prepared_packages.iter().all(|package| {
9808 self.staged_formula_index
9809 .package_lease_matches(&package.sheet, package.lease)
9810 });
9811 let stale_reason = Self::preparation_revision_stale_reason(
9812 &assumptions,
9813 ¤t_revisions,
9814 &planning_requests,
9815 staged_leases_match,
9816 );
9817 if let Some(reason) = stale_reason {
9818 return Err(Self::preparation_stale(
9819 reason,
9820 "target graph preparation plan is stale",
9821 ));
9822 }
9823 #[cfg(test)]
9824 self.target_preparation_fault(
9825 crate::engine::target_preparation::TargetPreparationFault::FinalGraphValidation,
9826 )?;
9827 self.graph
9828 .validate_prepared_legacy_graph_plan(&legacy_graph)
9829 .map_err(|error| {
9830 Self::preparation_stale(
9831 formualizer_common::PreparationStaleReason::Graph,
9832 format!("target graph preparation plan is stale: {error}"),
9833 )
9834 })?;
9835 #[cfg(test)]
9836 self.target_preparation_fault(
9837 crate::engine::target_preparation::TargetPreparationFault::Reservation,
9838 )?;
9839 self.graph.reserve_prepared_legacy_graph_plan(&legacy_graph);
9840 self.formula_parse_diagnostics
9841 .try_reserve(pending_diagnostics.len())
9842 .map_err(|_| {
9843 resource(
9844 formualizer_common::ResourceExhaustionReason::Admission,
9845 pending_diagnostics.len() as u64,
9846 pending_diagnostics.len() as u64,
9847 )
9848 })?;
9849 self.target_preparation_checkpoint(options.deadline, 0)?;
9850 #[cfg(test)]
9851 self.target_preparation_fault(
9852 crate::engine::target_preparation::TargetPreparationFault::BeforeFirstMutation,
9853 )?;
9854
9855 let commit_started = crate::instant::FzInstant::now();
9856 let committed = self
9857 .graph
9858 .apply_prevalidated_legacy_graph_plan(legacy_graph);
9859 for formula in &prepared {
9860 let removed = self
9861 .staged_formulas
9862 .get_mut(&formula.sheet)
9863 .and_then(|sheet| sheet.remove_ordinary(formula.lease.row, formula.lease.col));
9864 debug_assert!(removed.is_some());
9865 let index_removed = self.staged_formula_index.remove(
9866 &formula.sheet,
9867 formula.lease.row,
9868 formula.lease.col,
9869 );
9870 debug_assert!(index_removed);
9871 }
9872 for package in &prepared_packages {
9873 if let Some(points) = &package.selected_points {
9874 let staged = self.staged_formulas.get_mut(&package.sheet).unwrap();
9875 let source = staged.deferred_package.as_mut().unwrap();
9876 source.consumed_engine = Some(Arc::clone(&self.source_formula_token));
9877 source
9880 .consumed_members
9881 .extend(package.replay_records.iter().filter_map(|record| {
9882 record
9883 .partition_owner
9884 .or(record.family)
9885 .filter(|owner| residual_owners[&package.sheet].contains(owner))
9886 .map(|owner| {
9887 (
9888 owner,
9889 crate::engine::SourceCoord {
9890 row: record.row - 1,
9891 col: record.col - 1,
9892 },
9893 )
9894 })
9895 }));
9896 source.suppressed.extend(points.iter().copied());
9897 source.source_accounted = true;
9898 self.staged_formula_index
9899 .consume_package_points(&package.sheet, points);
9900 if source.suppressed.len() >= source.source_coordinates.len() {
9901 staged.deferred_package = None;
9902 self.staged_formula_index.set_package(&package.sheet, None);
9903 }
9904 } else {
9905 let removed = self
9906 .staged_formulas
9907 .get_mut(&package.sheet)
9908 .and_then(|staged| staged.deferred_package.take());
9909 debug_assert!(removed.is_some());
9910 self.staged_formula_index.set_package(&package.sheet, None);
9911 }
9912 }
9913 for (sheet, (families, partitions)) in residual_sources {
9914 if let Some(source) = self
9915 .staged_formulas
9916 .get_mut(&sheet)
9917 .and_then(|staged| staged.deferred_package.as_mut())
9918 {
9919 source.families = families;
9920 source.partitioned_families = partitions;
9921 source
9922 .consumed_members
9923 .retain(|(owner, _)| residual_owners[&sheet].contains(owner));
9924 self.staged_formula_index.update_package_family_count(
9925 &sheet,
9926 source.families.len() + source.partitioned_families.len(),
9927 );
9928 }
9929 }
9930 let empty_sheets = self
9931 .staged_formulas
9932 .iter()
9933 .filter_map(|(sheet, staged)| staged.is_empty().then_some(sheet.clone()))
9934 .collect::<Vec<_>>();
9935 for sheet in empty_sheets {
9936 self.staged_formulas.remove(&sheet);
9937 }
9938 if committed > 0 {
9939 self.mark_topology_edited();
9940 }
9941 self.formula_parse_diagnostics.extend(pending_diagnostics);
9942 if !prepared.is_empty() || !prepared_packages.is_empty() {
9943 let mut ingest_delta = FormulaIngestReport::with_mode(FormulaPlaneMode::Off);
9944 ingest_delta.formula_cells_seen = (prepared.len() as u64).saturating_add(
9945 prepared_packages
9946 .iter()
9947 .map(|package| {
9948 (package.replay_records.len() as u64).saturating_add(
9949 if package.selected_points.is_some() {
9950 package.direct_cells
9951 } else {
9952 0
9953 },
9954 )
9955 })
9956 .sum::<u64>(),
9957 );
9958 ingest_delta.graph_formula_cells_materialized = committed as u64;
9959 ingest_delta.graph_vertices_created = new_vertices as u64;
9960 ingest_delta.graph_edges_created = new_edges as u64;
9961 for package in &prepared_packages {
9962 let source = &package.source_report;
9963 ingest_delta.source_formula_events = ingest_delta
9964 .source_formula_events
9965 .saturating_add(source.source_formula_events);
9966 ingest_delta.source_formula_records_spooled = ingest_delta
9967 .source_formula_records_spooled
9968 .saturating_add(source.source_formula_records_spooled);
9969 ingest_delta.source_spool_encoded_bytes = ingest_delta
9970 .source_spool_encoded_bytes
9971 .saturating_add(source.source_spool_encoded_bytes);
9972 ingest_delta.source_spool_peak_memory_bytes = ingest_delta
9973 .source_spool_peak_memory_bytes
9974 .max(source.source_spool_peak_memory_bytes);
9975 ingest_delta.source_spool_spilled_bytes = ingest_delta
9976 .source_spool_spilled_bytes
9977 .saturating_add(source.source_spool_spilled_bytes);
9978 ingest_delta.source_spool_spill_files = ingest_delta
9979 .source_spool_spill_files
9980 .saturating_add(source.source_spool_spill_files);
9981 ingest_delta.source_spool_replays = ingest_delta
9982 .source_spool_replays
9983 .saturating_add(source.source_spool_replays)
9984 .saturating_add(package.spool_replays);
9985 ingest_delta.source_families_seen = ingest_delta
9986 .source_families_seen
9987 .saturating_add(source.families_seen);
9988 ingest_delta.source_family_cells_seen = ingest_delta
9989 .source_family_cells_seen
9990 .saturating_add(source.family_cells_seen);
9991 ingest_delta.source_family_shadow_eligible = ingest_delta
9992 .source_family_shadow_eligible
9993 .saturating_add(source.source_clean_families);
9994 ingest_delta.source_family_shadow_eligible_cells = ingest_delta
9995 .source_family_shadow_eligible_cells
9996 .saturating_add(source.source_clean_cells);
9997 ingest_delta.source_partitioned_families_seen = ingest_delta
9998 .source_partitioned_families_seen
9999 .saturating_add(source.source_fragmentable_families);
10000 ingest_delta.source_partition_holes = ingest_delta
10001 .source_partition_holes
10002 .saturating_add(source.source_hole_exclusions);
10003 ingest_delta.source_partition_ordinary_exceptions = ingest_delta
10004 .source_partition_ordinary_exceptions
10005 .saturating_add(source.source_ordinary_exclusions);
10006 ingest_delta.source_partition_surviving_cells = ingest_delta
10007 .source_partition_surviving_cells
10008 .saturating_add(source.source_fragmentable_cells);
10009 for (reason, count) in &source.fallback_reasons {
10010 let total = ingest_delta
10011 .fallback_reasons
10012 .entry(reason.clone())
10013 .or_default();
10014 *total = total.saturating_add(*count);
10015 }
10016
10017 ingest_delta.source_family_fallback = ingest_delta
10018 .source_family_fallback
10019 .saturating_add(source.families_seen);
10020 ingest_delta.source_family_fallback_cells = ingest_delta
10021 .source_family_fallback_cells
10022 .saturating_add(source.family_cells_seen);
10023 }
10024 self.record_formula_ingest_report(ingest_delta);
10025 }
10026 let commit_window = commit_started.elapsed();
10027 let retained_cells = self
10028 .staged_formula_index
10029 .all_leases()
10030 .into_iter()
10031 .filter_map(|(sheet, lease)| {
10032 formualizer_common::RangeAddress::new(
10033 sheet, lease.row, lease.col, lease.row, lease.col,
10034 )
10035 .ok()
10036 })
10037 .collect::<Vec<_>>();
10038 let observed_scratch_bytes = self
10039 .active_resource_ledger
10040 .as_ref()
10041 .map_or(0, |ledger| ledger.snapshot().scratch_peak)
10042 .saturating_sub(ledger_at_start.map_or(0, |snapshot| snapshot.scratch_current));
10043 let committed_spans = 0u64;
10044 let selected_source_families = prepared_packages
10045 .iter()
10046 .filter(|package| package.selected_points.is_none())
10047 .map(|package| package.lease.family_count)
10048 .sum::<usize>()
10049 + prepared_packages
10050 .iter()
10051 .filter(|package| package.selected_points.is_some())
10052 .flat_map(|package| package.replay_records.iter())
10053 .filter_map(|record| record.partition_owner.or(record.family))
10054 .chain(
10055 prepared_packages
10056 .iter()
10057 .flat_map(|package| package.complete_selections.iter().copied()),
10058 )
10059 .collect::<BTreeSet<_>>()
10060 .len();
10061 let selected_staged_cells = prepared.len().saturating_add(
10062 prepared_packages
10063 .iter()
10064 .map(|package| {
10065 package
10066 .selected_points
10067 .as_ref()
10068 .map_or(package.replay_records.len(), BTreeSet::len)
10069 })
10070 .sum::<usize>(),
10071 );
10072 let actual_commit_work = (new_vertices as u64)
10073 .saturating_add(new_edges as u64)
10074 .saturating_add(committed as u64)
10075 .saturating_add(committed_spans)
10076 .saturating_add(prepared.len() as u64)
10077 .saturating_add(
10078 prepared_packages
10079 .iter()
10080 .map(|package| {
10081 package
10082 .selected_points
10083 .as_ref()
10084 .map_or(1, |points| points.len() as u64)
10085 })
10086 .sum::<u64>(),
10087 );
10088 let report = PreparedTargetGraphReport {
10089 request_id: request_id.unwrap_or_default(),
10090 requested_targets: targets.len(),
10091 normalized_regions: visited_regions.len(),
10092 normalized_target_list: normalized,
10093 selected_staged_cells,
10094 selected_source_families,
10095 retained_staged_cells: self.staged_formula_count(),
10096 selected_cells,
10097 retained_cells,
10098 widened_scope: scope,
10099 widening_reasons: reasons,
10100 revisions: assumptions,
10101 commit_window,
10102 estimated_scratch_bytes: scratch_bytes,
10103 observed_scratch_bytes,
10104 estimated_commit_work: (new_vertices as u64)
10105 .saturating_add(new_edges as u64)
10106 .saturating_add(prepared.len() as u64)
10107 .saturating_add(selected_package_records),
10108 actual_commit_work,
10109 outcome: PreparationOutcome::Prepared,
10110 };
10111 self.observe_target_preparation_report(&report);
10112 Ok(report)
10113 }
10114
10115 pub fn build_graph_all(&mut self) -> Result<(), formualizer_parse::ExcelError> {
10117 self.observe_evaluation_resource_request(EvaluationRequestKind::Full, |engine| {
10118 engine.build_graph_all_unobserved()
10119 })
10120 }
10121
10122 fn build_graph_all_unobserved(&mut self) -> Result<(), formualizer_parse::ExcelError> {
10123 let selected = self.staged_formula_count();
10124 let started = crate::instant::FzInstant::now();
10125 self.resource_checkpoint(selected as u64)?;
10126 let scratch_bytes = (selected as u64).saturating_mul(256);
10127 let result = self.with_request_scratch(scratch_bytes, |engine| {
10128 let index_snapshot = engine.staged_formula_index.clone();
10129 let collected = std::mem::take(&mut engine.staged_formulas)
10130 .into_iter()
10131 .collect();
10132 engine.staged_formula_index.clear_all();
10133 engine.build_graph_from_staged_batches(collected, false, index_snapshot)
10134 });
10135 self.observe_staged_preparation(selected, self.staged_formula_count(), started.elapsed());
10136 result
10137 }
10138
10139 pub fn build_graph_for_sheets<'a, I: IntoIterator<Item = &'a str>>(
10141 &mut self,
10142 sheets: I,
10143 ) -> Result<(), formualizer_parse::ExcelError> {
10144 let mut sheets = sheets.into_iter();
10145 self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, move |engine| {
10146 let name_scratch = (sheets.size_hint().0 as u64).saturating_mul(64);
10147 engine.with_request_scratch(name_scratch, |engine| {
10148 let names = sheets.by_ref().map(str::to_string).collect::<Vec<_>>();
10151 engine.charge_bounded_work(names.len() as u64)?;
10152 engine.build_graph_for_sheet_names_unobserved(names)
10153 })
10154 })
10155 }
10156
10157 fn build_graph_for_sheet_names_unobserved(
10158 &mut self,
10159 sheets: Vec<String>,
10160 ) -> Result<(), formualizer_parse::ExcelError> {
10161 let started = crate::instant::FzInstant::now();
10162 let selected = sheets
10163 .iter()
10164 .filter_map(|sheet| self.staged_formulas.get(sheet))
10165 .map(StagedSheet::len)
10166 .sum::<usize>();
10167 self.resource_checkpoint(selected as u64)?;
10168 let scratch_bytes = (selected as u64).saturating_mul(256);
10169 self.reserve_request_scratch(scratch_bytes)?;
10170 let index_snapshot = self.staged_formula_index.clone();
10171 let mut collected = Vec::new();
10172 for sheet in sheets {
10173 if let Some(staged) = self.staged_formulas.remove(&sheet) {
10174 self.index_removed_staged_sheet(&sheet, &staged);
10175 collected.push((sheet, staged));
10176 }
10177 }
10178 let result = self.build_graph_from_staged_batches(collected, true, index_snapshot);
10179 self.release_request_scratch(scratch_bytes);
10180 self.observe_staged_preparation(selected, self.staged_formula_count(), started.elapsed());
10181 result
10182 }
10183
10184 fn build_graph_from_staged_batches(
10185 &mut self,
10186 collected: StagedFormulaBatches,
10187 share_parse_cache_across_sheets: bool,
10188 staged_index_snapshot: StagedFormulaIndex,
10189 ) -> Result<(), formualizer_parse::ExcelError> {
10190 if collected.is_empty() {
10191 return Ok(());
10192 }
10193 for (sheet, _) in &collected {
10194 let _ = self.add_sheet(sheet);
10195 }
10196
10197 let diagnostics_len = self.formula_parse_diagnostics.len();
10198 let mut collected = collected;
10199 let prepared = match self
10200 .prepare_staged_formula_batches(&mut collected, share_parse_cache_across_sheets)
10201 {
10202 Ok(prepared) => prepared,
10203 Err(error) => {
10204 self.formula_parse_diagnostics.truncate(diagnostics_len);
10205 for (sheet, staged) in collected {
10206 self.restore_staged_sheet(sheet, staged);
10207 }
10208 self.staged_formula_index = staged_index_snapshot;
10209 return Err(error);
10210 }
10211 };
10212 let (ordinary, compressed, direct, may_fail) = prepared;
10213
10214 let first_build = if self.deferred_build_can_be_first_load() {
10224 if let Err(error) = self.check_staged_formula_plans(
10225 ordinary
10226 .iter()
10227 .chain(compressed.iter().map(|(batch, _)| batch)),
10228 &may_fail,
10229 ) {
10230 self.formula_parse_diagnostics.truncate(diagnostics_len);
10231 for (sheet, staged) in collected {
10232 self.restore_staged_sheet(sheet, staged);
10233 }
10234 self.staged_formula_index = staged_index_snapshot;
10235 return Err(error);
10236 }
10237 self.deferred_build_as_first_load()
10238 } else {
10239 None
10240 };
10241 let built_sheets: Vec<String> = if first_build.is_some() {
10242 collected.iter().map(|(sheet, _)| sheet.clone()).collect()
10243 } else {
10244 Vec::new()
10245 };
10246
10247 let result = (|| {
10251 if !ordinary.is_empty() {
10252 self.ingest_formula_batches(ordinary)?;
10253 }
10254 if !compressed.is_empty() {
10255 self.ingest_compressed_formula_source_batches(compressed)?;
10256 }
10257 if !direct.is_empty() {
10258 self.finish_compressed_formula_sources(direct)?;
10259 }
10260 Ok(())
10261 })();
10262 if let Some(saved) = first_build {
10263 self.leave_deferred_first_load(saved, &built_sheets);
10264 }
10265 if let Err(error) = result {
10266 self.formula_parse_diagnostics.truncate(diagnostics_len);
10267 for (sheet, staged) in collected {
10268 self.restore_staged_sheet(sheet, staged);
10269 }
10270 self.staged_formula_index = staged_index_snapshot;
10271 return Err(error);
10272 }
10273 self.dedup_formula_parse_diagnostics_since(diagnostics_len);
10274 Ok(())
10275 }
10276
10277 fn deferred_build_can_be_first_load(&self) -> bool {
10283 !self.graph.first_load_assume_new()
10284 && !self.graph_admission_enabled()
10285 && self.graph.formula_vertex_count() == 0
10286 }
10287
10288 fn check_staged_formula_plans<'b>(
10293 &mut self,
10294 batches: impl Iterator<Item = &'b FormulaIngestBatch>,
10295 may_fail: &FxHashSet<crate::engine::arena::AstNodeId>,
10296 ) -> Result<(), ExcelError> {
10297 if may_fail.is_empty() {
10298 return Ok(());
10299 }
10300 let mut seen: FxHashSet<(SheetId, crate::engine::arena::AstNodeId)> = FxHashSet::default();
10301 for batch in batches {
10302 let sheet_id = self.graph.sheet_id(&batch.sheet_name).ok_or_else(|| {
10303 ExcelError::new(ExcelErrorKind::Ref)
10304 .with_message(format!("unknown ingest sheet: {}", batch.sheet_name))
10305 })?;
10306 let mut pipeline = self.ingest_pipeline();
10307 for record in &batch.formulas {
10308 if record.member_anchor.is_some()
10309 || !may_fail.contains(&record.ast_id)
10310 || !seen.insert((sheet_id, record.ast_id))
10311 {
10312 continue;
10313 }
10314 let placement = CellRef::new(
10315 sheet_id,
10316 Coord::from_excel(record.row, record.col, true, true),
10317 );
10318 pipeline.ingest_formula(
10319 FormulaAstInput::RawArena(record.ast_id),
10320 placement,
10321 None,
10322 )?;
10323 }
10324 }
10325 Ok(())
10326 }
10327
10328 fn deferred_build_as_first_load(&mut self) -> Option<(crate::engine::SheetIndexMode, usize)> {
10329 if !self.deferred_build_can_be_first_load() {
10330 return None;
10331 }
10332 let saved = (
10333 self.graph.get_config().sheet_index_mode,
10334 self.config.range_expansion_limit,
10335 );
10336 self.graph
10337 .set_sheet_index_mode(crate::engine::SheetIndexMode::Lazy);
10338 self.config.range_expansion_limit = 0;
10339 self.graph.set_first_load_assume_new(true);
10340 self.graph.reset_ensure_touched();
10341 Some(saved)
10342 }
10343
10344 fn leave_deferred_first_load(
10347 &mut self,
10348 (index_mode, range_limit): (crate::engine::SheetIndexMode, usize),
10349 sheets: &[String],
10350 ) {
10351 self.graph.set_first_load_assume_new(false);
10352 self.graph.reset_ensure_touched();
10353 self.graph.set_sheet_index_mode(index_mode);
10354 self.config.range_expansion_limit = range_limit;
10355 for sheet in sheets {
10356 self.graph.finalize_sheet_index(sheet);
10357 }
10358 }
10359
10360 fn prepare_staged_formula_batches(
10361 &mut self,
10362 collected: &mut StagedFormulaBatches,
10363 share_parse_cache_across_sheets: bool,
10364 ) -> Result<PreparedStagedFormulaBatches, formualizer_parse::ExcelError> {
10365 let mut ordinary = Vec::new();
10366 let mut compressed = Vec::new();
10367 let mut direct = Vec::new();
10368 let mut may_fail = FxHashSet::default();
10369 let mut cache: rustc_hash::FxHashMap<String, Option<crate::engine::arena::AstNodeId>> =
10370 rustc_hash::FxHashMap::default();
10371 cache.reserve(4096);
10372
10373 for (sheet, staged) in collected {
10374 if !share_parse_cache_across_sheets {
10375 cache.clear();
10376 }
10377 let mut grouper = crate::engine::FormulaFamilyGrouper::new();
10381 let mut replayed_records = Vec::new();
10385 let deferred_source = None;
10386 let mut deferred_fallback = None;
10387 if let Some(package) = staged.deferred_package.as_mut() {
10388 if package.sheet_name != *sheet {
10389 return Err(ExcelError::new(ExcelErrorKind::Value)
10390 .with_message("deferred formula package sheet mismatch"));
10391 }
10392 let eligible: Vec<_> = package
10393 .families
10394 .iter()
10395 .filter(|family| !package.invalidated.contains(&family.source_id))
10396 .cloned()
10397 .collect();
10398 let eligible_partitions: Vec<_> = package
10399 .partitioned_families
10400 .iter()
10401 .filter(|family| !package.invalidated.contains(&family.source_id))
10402 .cloned()
10403 .collect();
10404
10405 let mut replay_disposition = crate::engine::FormulaReplayDisposition::default();
10406 for partition in &eligible_partitions {
10407 replay_disposition
10408 .register_partition(partition, false)
10409 .map_err(|reason| {
10410 ExcelError::new(ExcelErrorKind::Value).with_message(reason)
10411 })?;
10412 }
10413 replay_disposition
10414 .extend_suppressed_excel_coords(package.suppressed.iter().copied());
10415 let replayed = package
10416 .replay
10417 .lock()
10418 .map_err(|_| {
10419 ExcelError::new(ExcelErrorKind::Value)
10420 .with_message("deferred formula spool lock poisoned")
10421 })?
10422 .replay(&replay_disposition)
10423 .map_err(|message| {
10424 ExcelError::new(ExcelErrorKind::Value).with_message(message)
10425 })?;
10426 replayed_records = replayed;
10427 let mut report = package.accounting_report();
10428 report.source_spool_replays = report.source_spool_replays.saturating_add(1);
10429 deferred_fallback = Some((report, package.families.clone(), eligible_partitions));
10430 }
10431
10432 let n_entries = staged.entries.len() + replayed_records.len();
10433 let entries = staged
10434 .entries
10435 .iter()
10436 .cloned()
10437 .map(|(row, col, text)| (row, col, text, None))
10438 .chain(replayed_records.into_iter().map(|record| {
10439 (
10440 record.row,
10441 record.col,
10442 record.text,
10443 Some((record.source_order, record.family, record.partition_owner)),
10444 )
10445 }));
10446 let mut formulas = Vec::with_capacity(n_entries);
10447 let staged_order_base = u64::MAX.saturating_sub(n_entries as u64);
10448 for (entry_index, (row, col, txt, source_proof)) in entries.enumerate() {
10449 let key = if txt.starts_with('=') {
10450 txt
10451 } else {
10452 format!("={txt}")
10453 };
10454 grouper.note_formula();
10455 let shared_key = source_proof
10457 .and_then(|(_, family, _)| family)
10458 .map(|family| u64::from(family.source_index));
10459 let staged_record = if let Some(cached) = cache.get(&key) {
10460 grouper.note_parse_cache_hit();
10461 cached.map(|ast_id| {
10462 self.note_staged_formula(&mut grouper, row, col, ast_id);
10463 FormulaIngestRecord::new(row, col, ast_id, Some(Arc::<str>::from(key)))
10464 })
10465 } else if let Some(record) =
10466 self.stage_relocated_text(&mut grouper, row, col, &key, shared_key)
10467 {
10468 Some(record)
10469 } else {
10470 grouper.note_parse(key.len());
10471 let parsed = formualizer_parse::parser::parse(&key);
10472 let parsed_text = parsed.is_ok();
10473 let parsed = match parsed {
10474 Ok(parsed) => Some(parsed),
10475 Err(error) => self.handle_formula_parse_error(
10476 sheet,
10477 row,
10478 col,
10479 &key,
10480 error.to_string(),
10481 )?,
10482 };
10483 match parsed {
10484 Some(ast) => {
10485 let record = self.stage_formula_ast(&mut grouper, row, col, &ast, None);
10486 if parsed_text {
10487 self.note_parsed_text(
10488 &mut grouper,
10489 row,
10490 col,
10491 &key,
10492 &ast,
10493 &record,
10494 shared_key,
10495 );
10496 }
10497 if !record.is_family_member() && self.formula_may_fail_planning(&ast) {
10498 may_fail.insert(record.ast_id);
10499 }
10500 if record.is_family_member() {
10503 Some(record)
10504 } else {
10505 let ast_id = record.ast_id;
10506 cache.insert(key.clone(), Some(ast_id));
10507 Some(FormulaIngestRecord::new(
10508 row,
10509 col,
10510 ast_id,
10511 Some(Arc::<str>::from(key)),
10512 ))
10513 }
10514 }
10515 None => {
10516 cache.insert(key, None);
10517 None
10518 }
10519 }
10520 };
10521
10522 if let Some(mut formula) = staged_record {
10523 if let Some((order, family, owner)) = source_proof {
10524 formula = formula.with_source_proof(order, family, owner);
10525 } else if deferred_source.is_some() {
10526 formula = formula.with_source_proof(
10527 crate::engine::SourceFormulaOrder::new(
10528 staged_order_base.saturating_add(entry_index as u64),
10529 ),
10530 None,
10531 None,
10532 );
10533 }
10534 formulas.push(formula);
10535 }
10536 }
10537
10538 self.finish_family_grouper(&mut grouper);
10539 let batch = FormulaIngestBatch::new(sheet.clone(), formulas);
10540 if let Some((report, preparation)) = deferred_source {
10541 direct.push((batch, report, preparation));
10542 } else if let Some((report, families, partitions)) = deferred_fallback {
10543 let source_batch = crate::engine::FormulaCompressedSourceBatch::with_proposals(
10544 batch.sheet_name.clone(),
10545 report,
10546 families,
10547 partitions,
10548 );
10549 compressed.push((batch, source_batch));
10550 } else if !batch.is_empty() {
10551 ordinary.push(batch);
10552 }
10553 }
10554 if std::env::var("FZ_DEBUG_LOAD")
10555 .ok()
10556 .is_some_and(|v| v != "0")
10557 {
10558 eprintln!(
10559 "[fz][families] deferred build: {}",
10560 self.source_family_counters.debug_line()
10561 );
10562 }
10563 Ok((ordinary, compressed, direct, may_fail))
10564 }
10565
10566 fn formula_may_fail_planning(&self, ast: &formualizer_parse::parser::ASTNode) -> bool {
10571 use formualizer_parse::parser::{ASTNodeType, ReferenceType};
10572 let sheet_missing = |sheet: &Option<String>| {
10573 sheet
10574 .as_deref()
10575 .is_some_and(|s| self.graph.sheet_id(s).is_none())
10576 };
10577 match &ast.node_type {
10578 ASTNodeType::Literal(_) | ASTNodeType::Omitted => false,
10579 ASTNodeType::Reference { reference, .. } => match reference {
10580 ReferenceType::Cell { sheet, .. } => sheet_missing(sheet),
10581 ReferenceType::Range {
10582 sheet,
10583 start_row,
10584 start_col,
10585 end_row,
10586 end_col,
10587 ..
10588 } => {
10589 sheet_missing(sheet)
10590 || matches!((start_row, end_row), (Some(a), Some(b)) if a > b)
10591 || matches!((start_col, end_col), (Some(a), Some(b)) if a > b)
10592 }
10593 ReferenceType::NamedRange(_) => false,
10594 _ => true,
10595 },
10596 ASTNodeType::UnaryOp { expr, .. } => self.formula_may_fail_planning(expr),
10597 ASTNodeType::BinaryOp { left, right, .. } => {
10598 self.formula_may_fail_planning(left) || self.formula_may_fail_planning(right)
10599 }
10600 ASTNodeType::Function { args, .. } => {
10601 args.iter().any(|a| self.formula_may_fail_planning(a))
10602 }
10603 ASTNodeType::Call { callee, args } => {
10604 self.formula_may_fail_planning(callee)
10605 || args.iter().any(|a| self.formula_may_fail_planning(a))
10606 }
10607 ASTNodeType::Array(rows) => rows
10608 .iter()
10609 .flatten()
10610 .any(|a| self.formula_may_fail_planning(a)),
10611 }
10612 }
10613
10614 pub fn begin_bulk_ingest_arrow(
10616 &mut self,
10617 ) -> crate::engine::arrow_ingest::ArrowBulkIngestBuilder<'_, R> {
10618 crate::engine::arrow_ingest::ArrowBulkIngestBuilder::new(self)
10619 }
10620
10621 pub fn begin_bulk_update_arrow(
10623 &mut self,
10624 ) -> crate::engine::arrow_ingest::ArrowBulkUpdateBuilder<'_, R> {
10625 crate::engine::arrow_ingest::ArrowBulkUpdateBuilder::new(self)
10626 }
10627
10628 fn ensure_known_sheet_id(&self, sheet: &str) -> Result<SheetId, crate::engine::EditorError> {
10629 self.graph.sheet_id(sheet).ok_or(
10630 crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
10631 name: sheet.to_string(),
10632 reason: "Unknown sheet".to_string(),
10633 },
10634 )
10635 }
10636
10637 fn normalize_row_1based(row_1based: u32) -> Result<u32, crate::engine::EditorError> {
10638 if row_1based == 0 {
10639 return Err(crate::engine::EditorError::OutOfBounds { row: 0, col: 0 });
10640 }
10641 Ok(row_1based - 1)
10642 }
10643
10644 fn normalize_row_range_1based(
10645 start_row_1based: u32,
10646 end_row_1based: u32,
10647 ) -> Result<(u32, u32), crate::engine::EditorError> {
10648 if start_row_1based == 0 || end_row_1based == 0 {
10649 return Err(crate::engine::EditorError::OutOfBounds { row: 0, col: 0 });
10650 }
10651 if start_row_1based > end_row_1based {
10652 return Err(crate::engine::EditorError::TransactionFailed {
10653 reason: "Row range start is greater than end".to_string(),
10654 });
10655 }
10656 Ok((start_row_1based - 1, end_row_1based - 1))
10657 }
10658
10659 fn invalidate_row_visibility_mask_cache(&self) {
10660 if let Ok(mut cache) = self.row_visibility_mask_cache.write() {
10661 cache.clear();
10662 }
10663 }
10664
10665 fn row_visibility_changed(&mut self) {
10668 self.invalidate_row_visibility_mask_cache();
10669 self.graph.mark_row_visibility_readers_dirty();
10670 }
10671
10672 fn set_row_hidden_by_sheet_id(
10673 &mut self,
10674 sheet_id: SheetId,
10675 row0: u32,
10676 hidden: bool,
10677 source: RowVisibilitySource,
10678 ) -> bool {
10679 let changed = {
10680 let state = self.row_visibility.entry(sheet_id).or_default();
10681 state.set_row_hidden(row0, hidden, source)
10682 };
10683
10684 let remove_entry = self
10685 .row_visibility
10686 .get(&sheet_id)
10687 .map(|state| state.is_empty())
10688 .unwrap_or(false);
10689 if remove_entry {
10690 self.row_visibility.remove(&sheet_id);
10691 }
10692
10693 if changed {
10694 self.row_visibility_changed();
10695 }
10696
10697 changed
10698 }
10699
10700 fn set_rows_hidden_by_sheet_id(
10701 &mut self,
10702 sheet_id: SheetId,
10703 start_row0: u32,
10704 end_row0: u32,
10705 hidden: bool,
10706 source: RowVisibilitySource,
10707 ) -> bool {
10708 let changed = {
10709 let state = self.row_visibility.entry(sheet_id).or_default();
10710 state.set_rows_hidden(start_row0, end_row0, hidden, source)
10711 };
10712
10713 let remove_entry = self
10714 .row_visibility
10715 .get(&sheet_id)
10716 .map(|state| state.is_empty())
10717 .unwrap_or(false);
10718 if remove_entry {
10719 self.row_visibility.remove(&sheet_id);
10720 }
10721
10722 if changed {
10723 self.row_visibility_changed();
10724 }
10725
10726 changed
10727 }
10728
10729 fn shift_row_visibility_insert(&mut self, sheet_id: SheetId, before0: u32, count: u32) {
10730 if count == 0 {
10731 return;
10732 }
10733 let mut changed = false;
10734 let remove_entry = if let Some(state) = self.row_visibility.get_mut(&sheet_id) {
10735 changed = state.insert_rows(before0, count);
10736 state.is_empty()
10737 } else {
10738 false
10739 };
10740 if remove_entry {
10741 self.row_visibility.remove(&sheet_id);
10742 }
10743 if changed {
10744 self.row_visibility_changed();
10745 }
10746 }
10747
10748 fn shift_row_visibility_delete(&mut self, sheet_id: SheetId, start0: u32, count: u32) {
10749 if count == 0 {
10750 return;
10751 }
10752 let mut changed = false;
10753 let remove_entry = if let Some(state) = self.row_visibility.get_mut(&sheet_id) {
10754 changed = state.delete_rows(start0, count);
10755 state.is_empty()
10756 } else {
10757 false
10758 };
10759 if remove_entry {
10760 self.row_visibility.remove(&sheet_id);
10761 }
10762 if changed {
10763 self.row_visibility_changed();
10764 }
10765 }
10766
10767 fn apply_inverse_row_visibility_event(&mut self, event: &crate::engine::ChangeEvent) {
10768 if let crate::engine::ChangeEvent::SetRowVisibility {
10769 sheet_id,
10770 row0,
10771 source,
10772 old_hidden,
10773 ..
10774 } = event
10775 {
10776 let _ = self.set_row_hidden_by_sheet_id(*sheet_id, *row0, *old_hidden, *source);
10777 }
10778 }
10779
10780 fn apply_forward_row_visibility_event(&mut self, event: &crate::engine::ChangeEvent) {
10781 if let crate::engine::ChangeEvent::SetRowVisibility {
10782 sheet_id,
10783 row0,
10784 source,
10785 new_hidden,
10786 ..
10787 } = event
10788 {
10789 let _ = self.set_row_hidden_by_sheet_id(*sheet_id, *row0, *new_hidden, *source);
10790 }
10791 }
10792
10793 fn apply_inverse_row_visibility_events(&mut self, events: &[crate::engine::ChangeEvent]) {
10794 for event in events.iter().rev() {
10795 self.apply_inverse_row_visibility_event(event);
10796 }
10797 }
10798
10799 fn apply_forward_row_visibility_events(&mut self, events: &[crate::engine::ChangeEvent]) {
10800 for event in events {
10801 self.apply_forward_row_visibility_event(event);
10802 }
10803 }
10804
10805 fn apply_inverse_staged_formula_event(&mut self, event: &crate::engine::ChangeEvent) {
10806 if let crate::engine::ChangeEvent::StagedFormulaCellChanged {
10807 sheet,
10808 row,
10809 col,
10810 old,
10811 ..
10812 } = event
10813 {
10814 self.apply_staged_formula_cell(sheet, *row, *col, old.as_deref());
10815 }
10816 }
10817
10818 fn apply_forward_staged_formula_event(&mut self, event: &crate::engine::ChangeEvent) {
10819 if let crate::engine::ChangeEvent::StagedFormulaCellChanged {
10820 sheet,
10821 row,
10822 col,
10823 new,
10824 ..
10825 } = event
10826 {
10827 self.apply_staged_formula_cell(sheet, *row, *col, new.as_deref());
10828 }
10829 }
10830
10831 fn apply_staged_formula_cell(&mut self, sheet: &str, row: u32, col: u32, target: Option<&str>) {
10834 match target {
10835 Some(text) => self.stage_formula_text(sheet, row, col, text.to_string()),
10836 None => {
10837 self.clear_staged_formula_text(sheet, row, col);
10838 }
10839 }
10840 }
10841
10842 pub fn set_row_hidden(
10843 &mut self,
10844 sheet: &str,
10845 row_1based: u32,
10846 hidden: bool,
10847 source: RowVisibilitySource,
10848 ) -> Result<(), crate::engine::EditorError> {
10849 self.observe_function_semantic_epoch()
10850 .map_err(crate::engine::EditorError::Excel)?;
10851 self.observe_function_semantic_epoch()
10852 .map_err(crate::engine::EditorError::Excel)?;
10853 let sheet_id = self.ensure_known_sheet_id(sheet)?;
10854 let row0 = Self::normalize_row_1based(row_1based)?;
10855 if self.set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source) {
10856 self.record_structural_change(StructuralScope::Region(Region::whole_row(
10857 sheet_id, row0,
10858 )));
10859 self.mark_data_edited();
10860 }
10861 Ok(())
10862 }
10863
10864 pub fn set_rows_hidden(
10865 &mut self,
10866 sheet: &str,
10867 start_row_1based: u32,
10868 end_row_1based: u32,
10869 hidden: bool,
10870 source: RowVisibilitySource,
10871 ) -> Result<(), crate::engine::EditorError> {
10872 let sheet_id = self.ensure_known_sheet_id(sheet)?;
10873 let (start_row0, end_row0) =
10874 Self::normalize_row_range_1based(start_row_1based, end_row_1based)?;
10875 if self.set_rows_hidden_by_sheet_id(sheet_id, start_row0, end_row0, hidden, source) {
10876 if start_row0 == end_row0 {
10877 self.record_structural_change(StructuralScope::Region(Region::whole_row(
10878 sheet_id, start_row0,
10879 )));
10880 } else {
10881 self.record_structural_change(StructuralScope::Sheet(sheet_id));
10882 }
10883 self.mark_data_edited();
10884 }
10885 Ok(())
10886 }
10887
10888 pub fn is_row_hidden(
10889 &self,
10890 sheet: &str,
10891 row_1based: u32,
10892 source: Option<RowVisibilitySource>,
10893 ) -> Option<bool> {
10894 let sheet_id = self.graph.sheet_id(sheet)?;
10895 let row0 = row_1based.checked_sub(1)?;
10896 Some(
10897 self.row_visibility
10898 .get(&sheet_id)
10899 .map(|state| state.is_row_hidden(row0, source))
10900 .unwrap_or(false),
10901 )
10902 }
10903
10904 pub fn row_visibility_version(&self, sheet: &str) -> Option<u64> {
10905 let sheet_id = self.graph.sheet_id(sheet)?;
10906 Some(
10907 self.row_visibility
10908 .get(&sheet_id)
10909 .map(|state| state.version())
10910 .unwrap_or(0),
10911 )
10912 }
10913
10914 fn nested_subtotal_cells_for_view(
10920 &self,
10921 view: &RangeView<'_>,
10922 include_aggregate: bool,
10923 ) -> Option<Vec<(usize, usize, usize)>> {
10924 if !view.is_sheet_backed() {
10925 return None;
10926 }
10927 let (rows, cols) = view.dims();
10928 if rows == 0 || cols == 0 {
10929 return Some(Vec::new());
10930 }
10931 let sheet_id = self.graph.sheet_id(view.sheet_name())?;
10932 let (r0, c0) = (view.start_row(), view.start_col());
10933 let mask = if include_aggregate {
10934 crate::engine::arena::SUBTOTAL_CALL | crate::engine::arena::AGGREGATE_CALL
10935 } else {
10936 crate::engine::arena::SUBTOTAL_CALL
10937 };
10938 let ds = self.graph.data_store();
10939 let mut found: Vec<(usize, usize, usize)> = self
10940 .graph
10941 .formula_intervals_in_region(
10942 sheet_id,
10943 r0 as u32,
10944 (r0 + rows - 1) as u32,
10945 c0 as u32,
10946 (c0 + cols - 1) as u32,
10947 |template| ds.ast_subtotal_calls(template) & mask != 0,
10948 )
10949 .into_iter()
10950 .map(|(col, lo, hi)| (col as usize - c0, lo as usize - r0, hi as usize - r0))
10951 .collect();
10952 found.sort_unstable();
10953 let mut out: Vec<(usize, usize, usize)> = Vec::with_capacity(found.len());
10956 for (col, lo, hi) in found {
10957 match out.last_mut() {
10958 Some(last) if last.0 == col && lo <= last.2 + 1 => last.2 = last.2.max(hi),
10959 _ => out.push((col, lo, hi)),
10960 }
10961 }
10962 Some(out)
10963 }
10964
10965 fn build_row_visibility_mask_for_view(
10966 &self,
10967 view: &RangeView<'_>,
10968 mode: VisibilityMaskMode,
10969 ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
10970 let sheet_rows = view.sheet().nrows as usize;
10971 if sheet_rows == 0 || view.start_row() >= sheet_rows {
10972 return Some(std::sync::Arc::new(arrow_array::BooleanArray::new_null(0)));
10973 }
10974
10975 let sheet_id = self.graph.sheet_id(view.sheet_name())?;
10976 let start_row0 = view.start_row() as u32;
10977 let end_row0 = view.end_row().min(sheet_rows.saturating_sub(1)) as u32;
10978 let version = self
10979 .row_visibility
10980 .get(&sheet_id)
10981 .map(|state| state.version())
10982 .unwrap_or(0);
10983 let key = VisibilityMaskCacheKey {
10984 sheet_id,
10985 start_row0,
10986 end_row0,
10987 mode,
10988 version,
10989 };
10990
10991 if let Ok(cache) = self.row_visibility_mask_cache.read()
10992 && let Some(mask) = cache.get(&key)
10993 {
10994 #[cfg(test)]
10995 visibility_mask_test_hooks::inc_hit();
10996 return Some(mask.clone());
10997 }
10998
10999 #[cfg(test)]
11000 visibility_mask_test_hooks::inc_miss();
11001
11002 let state = self.row_visibility.get(&sheet_id);
11003 let mut out = Vec::with_capacity((end_row0 - start_row0 + 1) as usize);
11004 for row0 in start_row0..=end_row0 {
11005 let manual_hidden = state
11006 .map(|s| s.is_row_hidden(row0, Some(RowVisibilitySource::Manual)))
11007 .unwrap_or(false);
11008 let filter_hidden = state
11009 .map(|s| s.is_row_hidden(row0, Some(RowVisibilitySource::Filter)))
11010 .unwrap_or(false);
11011
11012 let include = match mode {
11013 VisibilityMaskMode::IncludeAll => true,
11014 VisibilityMaskMode::ExcludeManualHidden => !manual_hidden,
11015 VisibilityMaskMode::ExcludeFilterHidden => !filter_hidden,
11016 VisibilityMaskMode::ExcludeManualOrFilterHidden => {
11017 !(manual_hidden || filter_hidden)
11018 }
11019 };
11020 out.push(include);
11021 }
11022
11023 let mask = std::sync::Arc::new(arrow_array::BooleanArray::from(out));
11024 if let Ok(mut cache) = self.row_visibility_mask_cache.write() {
11025 const MAX_CACHE_ENTRIES: usize = 4096;
11026 if cache.len() >= MAX_CACHE_ENTRIES {
11027 cache.clear();
11028 #[cfg(test)]
11029 visibility_mask_test_hooks::inc_eviction();
11030 }
11031 cache.insert(key, mask.clone());
11032 }
11033
11034 Some(mask)
11035 }
11036
11037 fn observe_function_semantic_epoch(&mut self) -> Result<bool, ExcelError> {
11038 let changes =
11039 crate::function_registry::semantic_changes_since(self.function_semantic_epoch_seen);
11040 let global_changed = changes.epoch != self.function_semantic_epoch_seen;
11041 let provider_revision = self.resolver.planning_semantic_revision();
11042 let provider_changed = provider_revision != self.function_provider_revision_seen;
11043 if !global_changed && !provider_changed {
11044 return Ok(false);
11045 }
11046
11047 let changed = !changes.keys.is_empty();
11048 if global_changed && changed || provider_changed {
11049 self.cached_static_schedule = None;
11050 self.recent_schedules.clear();
11051 self.base_schedule = None;
11052 }
11053 self.function_semantic_epoch_seen = changes.epoch;
11054 self.function_provider_revision_seen = provider_revision;
11055 Ok(false)
11056 }
11057
11058 pub(crate) fn ast_uses_changed_function(
11059 ast: &ASTNode,
11060 changed: &BTreeSet<(String, String)>,
11061 ) -> bool {
11062 match &ast.node_type {
11063 ASTNodeType::Function { name, args } => {
11064 let normalized = name.to_uppercase();
11065 let mut spellings = vec![(String::new(), normalized.clone())];
11066 let mut stripped = normalized.as_str();
11067 while let Some(rest) = ["_XLFN.", "_XLL.", "_XLWS."]
11068 .iter()
11069 .find_map(|prefix| stripped.strip_prefix(prefix))
11070 {
11071 stripped = rest;
11072 spellings.push((String::new(), stripped.to_string()));
11073 }
11074 let resolved = crate::function_registry::resolve("", name);
11075 let directly_changed = spellings.iter().any(|spelling| changed.contains(spelling))
11076 || resolved.as_ref().is_some_and(|resolved| {
11077 changed.contains(&(
11078 resolved.namespace.clone(),
11079 resolved.canonical_name.clone(),
11080 ))
11081 });
11082 directly_changed
11083 || resolved.is_none()
11084 || args
11085 .iter()
11086 .any(|arg| Self::ast_uses_changed_function(arg, changed))
11087 }
11088 ASTNodeType::Call { callee, args } => {
11089 Self::ast_uses_changed_function(callee, changed)
11090 || args
11091 .iter()
11092 .any(|arg| Self::ast_uses_changed_function(arg, changed))
11093 }
11094 ASTNodeType::UnaryOp { expr, .. } => Self::ast_uses_changed_function(expr, changed),
11095 ASTNodeType::BinaryOp { left, right, .. } => {
11096 Self::ast_uses_changed_function(left, changed)
11097 || Self::ast_uses_changed_function(right, changed)
11098 }
11099 ASTNodeType::Array(rows) => rows
11100 .iter()
11101 .flatten()
11102 .any(|node| Self::ast_uses_changed_function(node, changed)),
11103 _ => false,
11104 }
11105 }
11106
11107 fn ast_contains_function(ast: &ASTNode) -> bool {
11108 match &ast.node_type {
11109 ASTNodeType::Function { .. } => true,
11110 ASTNodeType::Call { callee, args } => {
11111 Self::ast_contains_function(callee) || args.iter().any(Self::ast_contains_function)
11112 }
11113 ASTNodeType::UnaryOp { expr, .. } => Self::ast_contains_function(expr),
11114 ASTNodeType::BinaryOp { left, right, .. } => {
11115 Self::ast_contains_function(left) || Self::ast_contains_function(right)
11116 }
11117 ASTNodeType::Array(rows) => rows.iter().flatten().any(Self::ast_contains_function),
11118 ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => false,
11119 }
11120 }
11121
11122 fn structural_row_region(sheet_id: SheetId, start_row0: u32) -> Region {
11123 Region::rows_from(sheet_id, start_row0)
11124 }
11125
11126 fn structural_col_region(sheet_id: SheetId, start_col0: u32) -> Region {
11127 Region::cols_from(sheet_id, start_col0)
11128 }
11129
11130 #[cfg(test)]
11131 pub(crate) fn force_non_cycle_schedule_fallback_for_test(&mut self) {
11132 self.force_non_cycle_schedule_fallback_for_test = true;
11133 }
11134
11135 fn materialize_deferred_sheet_before_structural_edit(
11136 &mut self,
11137 sheet: &str,
11138 ) -> Result<(), crate::engine::EditorError> {
11139 if self.staged_formulas.contains_key(sheet) {
11140 self.build_graph_for_sheets([sheet])?;
11141 }
11142 Ok(())
11143 }
11144
11145 fn check_insert_room(
11150 &self,
11151 sheet: &str,
11152 before0: u32,
11153 count: u32,
11154 rows: bool,
11155 ) -> Result<(), crate::engine::EditorError> {
11156 let limit = if rows {
11157 crate::engine::authority::geom::MAX_ROW
11158 } else {
11159 crate::engine::authority::geom::MAX_COL
11160 } + 1;
11161 let arrow = self.arrow_sheets.sheet(sheet).map_or(0, |a| {
11162 if rows {
11163 a.nrows
11164 } else {
11165 u32::try_from(a.columns.len()).unwrap_or(u32::MAX)
11166 }
11167 });
11168 let formulas = self.graph.sheet_id(sheet).map_or(0, |id| {
11169 self.graph
11170 .grid_vertices_in_sheet(id)
11171 .map(|(_, at)| if rows { at.row() } else { at.col() } + 1)
11172 .max()
11173 .unwrap_or(0)
11174 });
11175 let extent = arrow.max(formulas);
11176 let pushed_off = extent > before0 && extent.saturating_add(count) > limit;
11177 if count > limit || pushed_off {
11178 return Err(crate::engine::EditorError::Excel(
11179 ExcelError::new(ExcelErrorKind::Ref).with_message(format!(
11180 "inserting {count} {} would push non-empty cells off the sheet",
11181 if rows { "rows" } else { "columns" }
11182 )),
11183 ));
11184 }
11185 Ok(())
11186 }
11187
11188 fn structural_row_occupancy(
11189 &self,
11190 sheet: &str,
11191 sheet_id: SheetId,
11192 ) -> crate::engine::graph::StructuralOccupancy {
11193 if !self.graph.has_compressed_range_readers() {
11194 return crate::engine::graph::StructuralOccupancy::default();
11195 }
11196 let mut occupancy = self.graph.structural_occupancy(sheet_id);
11197 if let Some(arrow_sheet) = self.arrow_sheets.sheet(sheet) {
11198 occupancy.include_arrow_sheet(arrow_sheet);
11199 occupancy
11200 } else {
11201 crate::engine::graph::StructuralOccupancy::conservative()
11203 }
11204 }
11205
11206 fn structural_column_occupancy(&self) -> crate::engine::graph::StructuralOccupancy {
11207 crate::engine::graph::StructuralOccupancy::conservative()
11211 }
11212
11213 pub fn insert_rows(
11215 &mut self,
11216 sheet: &str,
11217 before: u32,
11218 count: u32,
11219 ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
11220 {
11221 if count == 0 {
11222 return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
11223 }
11224 self.observe_function_semantic_epoch()
11225 .map_err(crate::engine::EditorError::Excel)?;
11226 use crate::engine::graph::editor::vertex_editor::VertexEditor;
11227 self.materialize_deferred_sheet_before_structural_edit(sheet)?;
11228 let sheet_id = self.ensure_known_sheet_id(sheet)?;
11229 let before0 = before.saturating_sub(1);
11230 self.check_insert_room(sheet, before0, count, true)?;
11231 let affected_region = Self::structural_row_region(sheet_id, before0);
11232 let occupancy = self.structural_row_occupancy(sheet, sheet_id);
11233 let summary = {
11234 let mut editor =
11235 VertexEditor::new(&mut self.graph).with_structural_occupancy(occupancy);
11236 editor.insert_rows(sheet_id, before0, count)?
11237 };
11238 if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
11239 let before0 = before0 as usize;
11240 asheet.insert_rows(before0, count as usize);
11241 }
11242 self.purge_derived_formats_after_row(sheet_id, before0);
11243 self.mark_moved_formula_vertices_dirty(&summary);
11244 self.clear_computed_overlay_after_row(sheet, before0 as usize);
11245 self.shift_row_visibility_insert(sheet_id, before0, count);
11246 self.record_structural_change(StructuralScope::Region(affected_region));
11247 self.mark_topology_edited();
11248 Ok(summary)
11249 }
11250
11251 pub fn delete_rows(
11253 &mut self,
11254 sheet: &str,
11255 start: u32,
11256 count: u32,
11257 ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
11258 {
11259 if count == 0 {
11260 return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
11261 }
11262 self.observe_function_semantic_epoch()
11263 .map_err(crate::engine::EditorError::Excel)?;
11264 use crate::engine::graph::editor::vertex_editor::VertexEditor;
11265 self.materialize_deferred_sheet_before_structural_edit(sheet)?;
11266 let sheet_id = self.ensure_known_sheet_id(sheet)?;
11267 let start0 = start.saturating_sub(1);
11268 let affected_region = Self::structural_row_region(sheet_id, start0);
11269 let occupancy = self.structural_row_occupancy(sheet, sheet_id);
11270 let summary = {
11271 let mut editor =
11272 VertexEditor::new(&mut self.graph).with_structural_occupancy(occupancy);
11273 editor.delete_rows(sheet_id, start0, count)?
11274 };
11275 if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
11276 let start0 = start0 as usize;
11277 asheet.delete_rows(start0, count as usize);
11278 }
11279 self.purge_derived_formats_after_row(sheet_id, start0);
11280 self.mark_moved_formula_vertices_dirty(&summary);
11281 self.clear_computed_overlay_after_row(sheet, start0 as usize);
11282 self.shift_row_visibility_delete(sheet_id, start0, count);
11283 self.record_structural_change(StructuralScope::Region(affected_region));
11284 self.mark_topology_edited();
11285 Ok(summary)
11286 }
11287
11288 pub fn insert_columns(
11290 &mut self,
11291 sheet: &str,
11292 before: u32,
11293 count: u32,
11294 ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
11295 {
11296 if count == 0 {
11297 return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
11298 }
11299 self.observe_function_semantic_epoch()
11300 .map_err(crate::engine::EditorError::Excel)?;
11301 use crate::engine::graph::editor::vertex_editor::VertexEditor;
11302 self.materialize_deferred_sheet_before_structural_edit(sheet)?;
11303 let sheet_id = self.graph.sheet_id(sheet).ok_or(
11304 crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
11305 name: sheet.to_string(),
11306 reason: "Unknown sheet".to_string(),
11307 },
11308 )?;
11309 let before0 = before.saturating_sub(1);
11310 self.check_insert_room(sheet, before0, count, false)?;
11311 let affected_region = Self::structural_col_region(sheet_id, before0);
11312 let occupancy = self.structural_column_occupancy();
11313 let summary = {
11314 let mut editor =
11315 VertexEditor::new(&mut self.graph).with_structural_occupancy(occupancy);
11316 editor.insert_columns(sheet_id, before0, count)?
11317 };
11318 if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
11319 let before0 = before0 as usize;
11320 asheet.insert_columns(before0, count as usize);
11321 }
11322 self.purge_derived_formats_after_col(sheet_id, before0);
11323 self.mark_moved_formula_vertices_dirty(&summary);
11324 self.clear_computed_overlay_after_col(sheet, before0 as usize);
11325 self.record_structural_change(StructuralScope::Region(affected_region));
11326 self.mark_topology_edited();
11327 Ok(summary)
11328 }
11329
11330 pub fn delete_columns(
11332 &mut self,
11333 sheet: &str,
11334 start: u32,
11335 count: u32,
11336 ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
11337 {
11338 if count == 0 {
11339 return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
11340 }
11341 self.observe_function_semantic_epoch()
11342 .map_err(crate::engine::EditorError::Excel)?;
11343 use crate::engine::graph::editor::vertex_editor::VertexEditor;
11344 self.materialize_deferred_sheet_before_structural_edit(sheet)?;
11345 let sheet_id = self.graph.sheet_id(sheet).ok_or(
11346 crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
11347 name: sheet.to_string(),
11348 reason: "Unknown sheet".to_string(),
11349 },
11350 )?;
11351 let start0 = start.saturating_sub(1);
11352 let affected_region = Self::structural_col_region(sheet_id, start0);
11353 let occupancy = self.structural_column_occupancy();
11354 let summary = {
11355 let mut editor =
11356 VertexEditor::new(&mut self.graph).with_structural_occupancy(occupancy);
11357 editor.delete_columns(sheet_id, start0, count)?
11358 };
11359 if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
11360 let start0 = start0 as usize;
11361 asheet.delete_columns(start0, count as usize);
11362 }
11363 self.purge_derived_formats_after_col(sheet_id, start0);
11364 self.mark_moved_formula_vertices_dirty(&summary);
11365 self.clear_computed_overlay_after_col(sheet, start0 as usize);
11366 self.record_structural_change(StructuralScope::Region(affected_region));
11367 self.mark_topology_edited();
11368 Ok(summary)
11369 }
11370 fn arrow_used_row_bounds(
11372 &self,
11373 sheet: &str,
11374 start_col: u32,
11375 end_col: u32,
11376 ) -> Option<(u32, u32)> {
11377 let a = self.sheet_store().sheet(sheet)?;
11378 if a.columns.is_empty() {
11379 return None;
11380 }
11381 let sc0 = start_col.saturating_sub(1) as usize;
11382 let ec0 = end_col.saturating_sub(1) as usize;
11383 let col_hi = a.columns.len().saturating_sub(1);
11384 if sc0 > col_hi {
11385 return None;
11386 }
11387 let ec0 = ec0.min(col_hi);
11388 let snap = self.data_snapshot_id();
11390 let mut min_r0: Option<usize> = None;
11391 for ci in sc0..=ec0 {
11392 let sheet_id = self.graph.sheet_id(sheet)?;
11393 if let Some((Some(mv), _)) = self.row_bounds_cache.read().ok().and_then(|g| {
11394 g.as_ref()
11395 .and_then(|c| c.get_row_bounds(sheet_id, ci, snap))
11396 }) {
11397 let mv = mv as usize;
11398 min_r0 = Some(min_r0.map(|m| m.min(mv)).unwrap_or(mv));
11399 continue;
11400 }
11401 let (min_c, max_c) = Self::scan_column_used_bounds(a, ci);
11403 if let Ok(mut g) = self.row_bounds_cache.write() {
11404 g.get_or_insert_with(|| RowBoundsCache::new(snap))
11405 .put_row_bounds(sheet_id, ci, snap, (min_c, max_c));
11406 }
11407 if let Some(m) = min_c {
11408 min_r0 = Some(min_r0.map(|mm| mm.min(m as usize)).unwrap_or(m as usize));
11409 }
11410 }
11411 min_r0?;
11412 let mut max_r0: Option<usize> = None;
11413 for ci in sc0..=ec0 {
11414 let sheet_id = self.graph.sheet_id(sheet)?;
11415 if let Some((_, Some(mv))) = self.row_bounds_cache.read().ok().and_then(|g| {
11416 g.as_ref()
11417 .and_then(|c| c.get_row_bounds(sheet_id, ci, snap))
11418 }) {
11419 let mv = mv as usize;
11420 max_r0 = Some(max_r0.map(|m| m.max(mv)).unwrap_or(mv));
11421 continue;
11422 }
11423 let (_min_c, max_c) = Self::scan_column_used_bounds(a, ci);
11424 if let Ok(mut g) = self.row_bounds_cache.write() {
11425 g.get_or_insert_with(|| RowBoundsCache::new(snap))
11426 .put_row_bounds(sheet_id, ci, snap, (_min_c, max_c));
11427 }
11428 if let Some(m) = max_c {
11429 max_r0 = Some(max_r0.map(|mm| mm.max(m as usize)).unwrap_or(m as usize));
11430 }
11431 }
11432 match (min_r0, max_r0) {
11433 (Some(a0), Some(b0)) => Some(((a0 as u32) + 1, (b0 as u32) + 1)),
11434 _ => None,
11435 }
11436 }
11437
11438 fn scan_column_used_bounds(
11439 a: &crate::arrow_store::ArrowSheet,
11440 ci: usize,
11441 ) -> (Option<u32>, Option<u32>) {
11442 let col = &a.columns[ci];
11443
11444 let mut min_r0: Option<u32> = None;
11446 for (chunk_idx, chunk) in col.chunks.iter().enumerate() {
11447 let tags = chunk.type_tag.values();
11448 for (off, &t) in tags.iter().enumerate() {
11449 let overlay_non_empty = chunk
11450 .overlay
11451 .get(off)
11452 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11453 .unwrap_or(false)
11454 || chunk
11455 .computed_overlay
11456 .get(off)
11457 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11458 .unwrap_or(false);
11459 if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
11460 let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
11461 break;
11462 };
11463 let row0 = chunk_start + off;
11464 min_r0 = Some(row0 as u32);
11465 break;
11466 }
11467 }
11468 if min_r0.is_some() {
11469 break;
11470 }
11471 }
11472 if min_r0.is_none() && !col.sparse_chunks.is_empty() {
11473 let mut sparse_idxs: Vec<usize> = col.sparse_chunks.keys().copied().collect();
11474 sparse_idxs.sort_unstable();
11475 for chunk_idx in sparse_idxs {
11476 let Some(chunk) = col.sparse_chunks.get(&chunk_idx) else {
11477 continue;
11478 };
11479 let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
11480 continue;
11481 };
11482 let tags = chunk.type_tag.values();
11483 for (off, &t) in tags.iter().enumerate() {
11484 let overlay_non_empty = chunk
11485 .overlay
11486 .get(off)
11487 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11488 .unwrap_or(false)
11489 || chunk
11490 .computed_overlay
11491 .get(off)
11492 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11493 .unwrap_or(false);
11494 if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
11495 let row0 = chunk_start + off;
11496 min_r0 = Some(row0 as u32);
11497 break;
11498 }
11499 }
11500 if min_r0.is_some() {
11501 break;
11502 }
11503 }
11504 }
11505
11506 let mut max_r0: Option<u32> = None;
11508 if !col.sparse_chunks.is_empty() {
11509 let mut sparse_idxs: Vec<usize> = col.sparse_chunks.keys().copied().collect();
11510 sparse_idxs.sort_unstable_by(|a, b| b.cmp(a));
11511 for chunk_idx in sparse_idxs {
11512 let Some(chunk) = col.sparse_chunks.get(&chunk_idx) else {
11513 continue;
11514 };
11515 let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
11516 continue;
11517 };
11518 let tags = chunk.type_tag.values();
11519 for (rev_idx, &t) in tags.iter().enumerate().rev() {
11520 let overlay_non_empty = chunk
11521 .overlay
11522 .get(rev_idx)
11523 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11524 .unwrap_or(false)
11525 || chunk
11526 .computed_overlay
11527 .get(rev_idx)
11528 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11529 .unwrap_or(false);
11530 if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
11531 let row0 = chunk_start + rev_idx;
11532 max_r0 = Some(row0 as u32);
11533 break;
11534 }
11535 }
11536 if max_r0.is_some() {
11537 break;
11538 }
11539 }
11540 }
11541 if max_r0.is_none() {
11542 for (chunk_idx, chunk) in col.chunks.iter().enumerate().rev() {
11543 let tags = chunk.type_tag.values();
11544 for (rev_idx, &t) in tags.iter().enumerate().rev() {
11545 let overlay_non_empty = chunk
11546 .overlay
11547 .get(rev_idx)
11548 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11549 .unwrap_or(false)
11550 || chunk
11551 .computed_overlay
11552 .get(rev_idx)
11553 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11554 .unwrap_or(false);
11555 if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
11556 let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
11557 break;
11558 };
11559 let row0 = chunk_start + rev_idx;
11560 max_r0 = Some(row0 as u32);
11561 break;
11562 }
11563 }
11564 if max_r0.is_some() {
11565 break;
11566 }
11567 }
11568 }
11569
11570 (min_r0, max_r0)
11571 }
11572
11573 fn arrow_used_col_bounds(
11575 &self,
11576 sheet: &str,
11577 start_row: u32,
11578 end_row: u32,
11579 ) -> Option<(u32, u32)> {
11580 let a = self.sheet_store().sheet(sheet)?;
11581 if a.columns.is_empty() {
11582 return None;
11583 }
11584 let sr0 = start_row.saturating_sub(1) as usize;
11585 let er0 = end_row.saturating_sub(1) as usize;
11586 if sr0 > er0 {
11587 return None;
11588 }
11589 let mut min_c0: Option<usize> = None;
11592 let mut max_c0: Option<usize> = None;
11593 for (ci, col) in a.columns.iter().enumerate() {
11595 let mut any_in_range = false;
11596
11597 let scan_chunk = |chunk_idx: usize, chunk: &crate::arrow_store::ColumnChunk| -> bool {
11598 let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
11599 return false;
11600 };
11601 let chunk_len = chunk.type_tag.len();
11602 if chunk_len == 0 {
11603 return false;
11604 }
11605 let chunk_end = chunk_start + chunk_len.saturating_sub(1);
11606 if sr0 > chunk_end || er0 < chunk_start {
11608 return false;
11609 }
11610 let start_off = sr0.max(chunk_start) - chunk_start;
11611 let end_off = er0.min(chunk_end) - chunk_start;
11612 let tags = chunk.type_tag.values();
11613 for off in start_off..=end_off {
11614 let overlay_non_empty = chunk
11615 .overlay
11616 .get(off)
11617 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11618 .unwrap_or(false)
11619 || chunk
11620 .computed_overlay
11621 .get(off)
11622 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11623 .unwrap_or(false);
11624 if overlay_non_empty || tags[off] != crate::arrow_store::TypeTag::Empty as u8 {
11625 return true;
11626 }
11627 }
11628 false
11629 };
11630
11631 for (chunk_idx, chunk) in col.chunks.iter().enumerate() {
11632 if scan_chunk(chunk_idx, chunk) {
11633 any_in_range = true;
11634 break;
11635 }
11636 }
11637
11638 if !any_in_range && !col.sparse_chunks.is_empty() {
11639 for (&chunk_idx, chunk) in col.sparse_chunks.iter() {
11640 if scan_chunk(chunk_idx, chunk) {
11641 any_in_range = true;
11642 break;
11643 }
11644 }
11645 }
11646
11647 if any_in_range {
11648 min_c0 = Some(min_c0.map(|m| m.min(ci)).unwrap_or(ci));
11649 max_c0 = Some(max_c0.map(|m| m.max(ci)).unwrap_or(ci));
11650 }
11651 }
11652 match (min_c0, max_c0) {
11653 (Some(a0), Some(b0)) => Some(((a0 as u32) + 1, (b0 as u32) + 1)),
11654 _ => None,
11655 }
11656 }
11657
11658 fn formula_row_bounds_for_columns(
11659 &self,
11660 sheet: &str,
11661 start_col: u32,
11662 end_col: u32,
11663 ) -> Option<(u32, u32)> {
11664 let sheet_id = self.graph.sheet_id(sheet)?;
11665 let sc0 = start_col.saturating_sub(1);
11666 let ec0 = end_col.saturating_sub(1);
11667 let mut min_r0: Option<u32> = None;
11668 let mut max_r0: Option<u32> = None;
11669
11670 if self.graph.sheet_index(sheet_id).is_some() {
11671 for vid in self.graph.vertices_in_cols(sheet_id, sc0, ec0) {
11672 if !matches!(
11673 self.graph.get_vertex_kind(vid),
11674 VertexKind::FormulaScalar | VertexKind::FormulaArray
11675 ) {
11676 continue;
11677 }
11678 let Some(row0) = self.graph.vertex_grid_addr(vid).map(|addr| addr.row()) else {
11679 continue;
11680 };
11681 min_r0 = Some(min_r0.map(|m| m.min(row0)).unwrap_or(row0));
11682 max_r0 = Some(max_r0.map(|m| m.max(row0)).unwrap_or(row0));
11683 }
11684 } else {
11685 for (vid, coord) in self.graph.grid_vertices_in_sheet(sheet_id) {
11686 if !matches!(
11687 self.graph.get_vertex_kind(vid),
11688 VertexKind::FormulaScalar | VertexKind::FormulaArray
11689 ) {
11690 continue;
11691 }
11692 let col0 = coord.col();
11693 if col0 < sc0 || col0 > ec0 {
11694 continue;
11695 }
11696 let row0 = coord.row();
11697 min_r0 = Some(min_r0.map(|m| m.min(row0)).unwrap_or(row0));
11698 max_r0 = Some(max_r0.map(|m| m.max(row0)).unwrap_or(row0));
11699 }
11700 }
11701
11702 match (min_r0, max_r0) {
11703 (Some(a0), Some(b0)) => Some((a0 + 1, b0 + 1)),
11704 _ => None,
11705 }
11706 }
11707
11708 fn formula_col_bounds_for_rows(
11709 &self,
11710 sheet: &str,
11711 start_row: u32,
11712 end_row: u32,
11713 ) -> Option<(u32, u32)> {
11714 let sheet_id = self.graph.sheet_id(sheet)?;
11715 let sr0 = start_row.saturating_sub(1);
11716 let er0 = end_row.saturating_sub(1);
11717 let mut min_c0: Option<u32> = None;
11718 let mut max_c0: Option<u32> = None;
11719
11720 if self.graph.sheet_index(sheet_id).is_some() {
11721 for vid in self.graph.vertices_in_rows(sheet_id, sr0, er0) {
11722 if !matches!(
11723 self.graph.get_vertex_kind(vid),
11724 VertexKind::FormulaScalar | VertexKind::FormulaArray
11725 ) {
11726 continue;
11727 }
11728 let Some(col0) = self.graph.vertex_grid_addr(vid).map(|addr| addr.col()) else {
11729 continue;
11730 };
11731 min_c0 = Some(min_c0.map(|m| m.min(col0)).unwrap_or(col0));
11732 max_c0 = Some(max_c0.map(|m| m.max(col0)).unwrap_or(col0));
11733 }
11734 } else {
11735 for (vid, coord) in self.graph.grid_vertices_in_sheet(sheet_id) {
11736 if !matches!(
11737 self.graph.get_vertex_kind(vid),
11738 VertexKind::FormulaScalar | VertexKind::FormulaArray
11739 ) {
11740 continue;
11741 }
11742 let row0 = coord.row();
11743 if row0 < sr0 || row0 > er0 {
11744 continue;
11745 }
11746 let col0 = coord.col();
11747 min_c0 = Some(min_c0.map(|m| m.min(col0)).unwrap_or(col0));
11748 max_c0 = Some(max_c0.map(|m| m.max(col0)).unwrap_or(col0));
11749 }
11750 }
11751
11752 match (min_c0, max_c0) {
11753 (Some(a0), Some(b0)) => Some((a0 + 1, b0 + 1)),
11754 _ => None,
11755 }
11756 }
11757
11758 fn union_used_bounds(
11759 first: Option<(u32, u32)>,
11760 second: Option<(u32, u32)>,
11761 ) -> Option<(u32, u32)> {
11762 match (first, second) {
11763 (Some((a0, b0)), Some((a1, b1))) => Some((a0.min(a1), b0.max(b1))),
11764 (Some(bounds), None) | (None, Some(bounds)) => Some(bounds),
11765 (None, None) => None,
11766 }
11767 }
11768
11769 fn mirror_value_to_overlay(&mut self, sheet: &str, row: u32, col: u32, value: &LiteralValue) {
11772 if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
11773 return;
11774 }
11775 if self.arrow_sheets.sheet(sheet).is_none() {
11776 self.arrow_sheets
11777 .sheets
11778 .push(crate::arrow_store::ArrowSheet {
11779 name: std::sync::Arc::<str>::from(sheet),
11780 date_system: self.config.date_system,
11781 columns: Vec::new(),
11782 nrows: 0,
11783 chunk_starts: Vec::new(),
11784 chunk_rows: 32 * 1024,
11785 });
11786 }
11787
11788 let row0 = row.saturating_sub(1) as usize;
11789 let col0 = col.saturating_sub(1) as usize;
11790
11791 let asheet = self
11792 .arrow_sheets
11793 .sheet_mut(sheet)
11794 .expect("ArrowSheet must exist");
11795
11796 let cur_cols = asheet.columns.len();
11797 if col0 >= cur_cols {
11798 asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
11799 }
11800
11801 if row0 >= asheet.nrows as usize {
11802 if asheet.columns.is_empty() {
11803 asheet.insert_columns(0, 1);
11804 }
11805 asheet.ensure_row_capacity(row0 + 1);
11806 }
11807 if let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) {
11808 let ov =
11809 crate::arrow_store::OverlayValue::from_literal_value(value, asheet.date_system);
11810 let computed_delta = if let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) {
11811 let _ = ch.overlay.set(in_off, ov);
11812 let format = match value {
11813 LiteralValue::Date(_) => Some(crate::format::FormatId::DATE),
11814 LiteralValue::DateTime(_) => Some(crate::format::FormatId::DATETIME),
11815 LiteralValue::Time(_) => Some(crate::format::FormatId::TIME),
11816 LiteralValue::Duration(_) => Some(crate::format::FormatId::DURATION),
11817 _ => None,
11818 };
11819 ch.overlay.set_format(in_off, format);
11820 ch.computed_overlay.remove(in_off)
11825 } else {
11826 return;
11827 };
11828 let abs_threshold = 1024usize;
11830 let frac_den = 50usize;
11831 let freed = asheet.maybe_compact_chunk(col0, ch_idx, abs_threshold, frac_den);
11832 if freed > 0 {
11833 self.overlay_compactions = self.overlay_compactions.saturating_add(1);
11834 }
11835 self.adjust_computed_overlay_bytes(computed_delta);
11836 }
11837 }
11838
11839 fn clear_delta_overlay_cell(&mut self, sheet: &str, row: u32, col: u32) {
11844 if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
11845 return;
11846 }
11847 let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
11848 return;
11849 };
11850 let row0 = row.saturating_sub(1) as usize;
11851 let col0 = col.saturating_sub(1) as usize;
11852 if row0 >= asheet.nrows as usize {
11853 return;
11854 }
11855 if col0 >= asheet.columns.len() {
11856 return;
11857 }
11858 let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
11859 return;
11860 };
11861 if let Some(ch) = asheet.columns[col0].chunk_mut(ch_idx) {
11862 let _ = ch.overlay.remove(in_off);
11863 }
11864 }
11865
11866 fn clear_computed_overlay_after_row(&mut self, sheet: &str, start_row0: usize) {
11867 if !(self.config.arrow_storage_enabled && self.config.write_formula_overlay_enabled) {
11868 return;
11869 }
11870
11871 let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
11872 return;
11873 };
11874 if start_row0 >= asheet.nrows as usize {
11875 return;
11876 }
11877
11878 let starts = asheet.chunk_starts.clone();
11879 let nrows = asheet.nrows as usize;
11880 let mut delta = 0isize;
11881 for col in &mut asheet.columns {
11882 for (chunk_idx, ch) in col.chunks.iter_mut().enumerate() {
11883 let Some(&chunk_start) = starts.get(chunk_idx) else {
11884 continue;
11885 };
11886 let chunk_end = starts
11887 .get(chunk_idx + 1)
11888 .copied()
11889 .unwrap_or(nrows)
11890 .min(chunk_start.saturating_add(ch.len()));
11891 if chunk_end <= start_row0 {
11892 continue;
11893 }
11894 if chunk_start >= start_row0 {
11895 delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
11896 } else {
11897 let start_in_chunk = start_row0.saturating_sub(chunk_start).min(ch.len());
11898 delta = delta
11899 .saturating_add(ch.computed_overlay.remove_range(start_in_chunk..ch.len()));
11900 }
11901 }
11902
11903 for (chunk_idx, ch) in &mut col.sparse_chunks {
11904 let Some(&chunk_start) = starts.get(*chunk_idx) else {
11905 continue;
11906 };
11907 let chunk_end = starts
11908 .get(*chunk_idx + 1)
11909 .copied()
11910 .unwrap_or(nrows)
11911 .min(chunk_start.saturating_add(ch.len()));
11912 if chunk_end <= start_row0 {
11913 continue;
11914 }
11915 if chunk_start >= start_row0 {
11916 delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
11917 } else {
11918 let start_in_chunk = start_row0.saturating_sub(chunk_start).min(ch.len());
11919 delta = delta
11920 .saturating_add(ch.computed_overlay.remove_range(start_in_chunk..ch.len()));
11921 }
11922 }
11923 }
11924 self.adjust_computed_overlay_bytes(delta);
11925 }
11926
11927 fn clear_computed_overlay_after_col(&mut self, sheet: &str, start_col0: usize) {
11928 if !(self.config.arrow_storage_enabled && self.config.write_formula_overlay_enabled) {
11929 return;
11930 }
11931
11932 let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
11933 return;
11934 };
11935 if start_col0 >= asheet.columns.len() {
11936 return;
11937 }
11938
11939 let mut delta = 0isize;
11940 for col in asheet.columns.iter_mut().skip(start_col0) {
11941 for ch in &mut col.chunks {
11942 delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
11943 }
11944 for ch in col.sparse_chunks.values_mut() {
11945 delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
11946 }
11947 }
11948 self.adjust_computed_overlay_bytes(delta);
11949 }
11950
11951 #[inline]
11952 fn literal_to_overlay_value(
11953 value: &LiteralValue,
11954 date_system: crate::engine::DateSystem,
11955 ) -> crate::arrow_store::OverlayValue {
11956 crate::arrow_store::OverlayValue::from_literal_value(value, date_system)
11957 }
11958
11959 fn arrow_sheet_date_system(&self, sheet: &str) -> crate::engine::DateSystem {
11960 self.arrow_sheets
11961 .sheet(sheet)
11962 .map(|sheet| sheet.date_system)
11963 .unwrap_or(self.config.date_system)
11964 }
11965
11966 fn read_delta_overlay_cell(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
11969 if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
11970 return None;
11971 }
11972 let asheet = self.arrow_sheets.sheet(sheet)?;
11973 let row0 = row.saturating_sub(1) as usize;
11974 let col0 = col.saturating_sub(1) as usize;
11975 if row0 >= asheet.nrows as usize || col0 >= asheet.columns.len() {
11976 return None;
11977 }
11978 let (ch_idx, in_off) = asheet.chunk_of_row(row0)?;
11979 let ch = asheet.columns[col0].chunk(ch_idx)?;
11980 ch.overlay
11981 .get_scalar(in_off)
11982 .map(|ov| ov.to_literal_for(asheet.date_system))
11983 }
11984
11985 fn read_computed_overlay_cell(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
11988 if !(self.config.arrow_storage_enabled
11989 && self.config.delta_overlay_enabled
11990 && self.config.write_formula_overlay_enabled)
11991 {
11992 return None;
11993 }
11994 let asheet = self.arrow_sheets.sheet(sheet)?;
11995 let row0 = row.saturating_sub(1) as usize;
11996 let col0 = col.saturating_sub(1) as usize;
11997 if row0 >= asheet.nrows as usize || col0 >= asheet.columns.len() {
11998 return None;
11999 }
12000 let (ch_idx, in_off) = asheet.chunk_of_row(row0)?;
12001 let ch = asheet.columns[col0].chunk(ch_idx)?;
12002 ch.computed_overlay
12003 .get_scalar(in_off)
12004 .map(|ov| ov.to_literal_for(asheet.date_system))
12005 }
12006
12007 fn set_delta_overlay_cell_raw(
12008 &mut self,
12009 sheet: &str,
12010 row: u32,
12011 col: u32,
12012 value: Option<LiteralValue>,
12013 ) {
12014 if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
12015 return;
12016 }
12017
12018 self.ensure_arrow_sheet(sheet);
12019 let date_system = self.arrow_sheet_date_system(sheet);
12020 let ov_opt = value
12021 .as_ref()
12022 .map(|value| Self::literal_to_overlay_value(value, date_system));
12023 let row0 = row.saturating_sub(1) as usize;
12024 let col0 = col.saturating_sub(1) as usize;
12025 let asheet = self
12026 .arrow_sheets
12027 .sheet_mut(sheet)
12028 .expect("ArrowSheet must exist");
12029
12030 let cur_cols = asheet.columns.len();
12031 if col0 >= cur_cols {
12032 asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
12033 }
12034 if row0 >= asheet.nrows as usize {
12035 if asheet.columns.is_empty() {
12036 asheet.insert_columns(0, 1);
12037 }
12038 asheet.ensure_row_capacity(row0 + 1);
12039 }
12040
12041 let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
12042 return;
12043 };
12044 let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
12045 return;
12046 };
12047
12048 if let Some(ov) = ov_opt {
12049 let _ = ch.overlay.set(in_off, ov);
12050 } else {
12051 let _ = ch.overlay.remove(in_off);
12052 }
12053 }
12054
12055 fn set_computed_overlay_cell_raw(
12056 &mut self,
12057 sheet: &str,
12058 row: u32,
12059 col: u32,
12060 value: Option<LiteralValue>,
12061 ) {
12062 if !(self.config.arrow_storage_enabled
12063 && self.config.delta_overlay_enabled
12064 && self.config.write_formula_overlay_enabled)
12065 {
12066 return;
12067 }
12068
12069 self.ensure_arrow_sheet(sheet);
12070 let date_system = self.arrow_sheet_date_system(sheet);
12071 let ov_opt = value
12072 .as_ref()
12073 .map(|value| Self::literal_to_overlay_value(value, date_system));
12074 let row0 = row.saturating_sub(1) as usize;
12075 let col0 = col.saturating_sub(1) as usize;
12076 let asheet = self
12077 .arrow_sheets
12078 .sheet_mut(sheet)
12079 .expect("ArrowSheet must exist");
12080
12081 let cur_cols = asheet.columns.len();
12082 if col0 >= cur_cols {
12083 asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
12084 }
12085 if row0 >= asheet.nrows as usize {
12086 if asheet.columns.is_empty() {
12087 asheet.insert_columns(0, 1);
12088 }
12089 asheet.ensure_row_capacity(row0 + 1);
12090 }
12091
12092 let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
12093 return;
12094 };
12095 let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
12096 return;
12097 };
12098
12099 let delta = if let Some(ov) = ov_opt {
12100 ch.computed_overlay.set(in_off, ov)
12101 } else {
12102 ch.computed_overlay.remove(in_off)
12103 };
12104 self.adjust_computed_overlay_bytes(delta);
12105 }
12106
12107 fn apply_arrow_undo_batch(&mut self, batch: &crate::engine::ArrowUndoBatch, undo: bool) {
12108 use crate::engine::ArrowOp;
12109
12110 let iter: Box<dyn Iterator<Item = &ArrowOp>> = if undo {
12111 Box::new(batch.ops.iter().rev())
12112 } else {
12113 Box::new(batch.ops.iter())
12114 };
12115
12116 for op in iter {
12117 match op {
12118 ArrowOp::SetDeltaCell {
12119 sheet_id,
12120 row0,
12121 col0,
12122 old,
12123 new,
12124 } => {
12125 let sheet = self.graph.sheet_name(*sheet_id).to_string();
12126 let v = if undo { old.clone() } else { new.clone() };
12127 self.set_delta_overlay_cell_raw(&sheet, row0 + 1, col0 + 1, v);
12128 }
12129 ArrowOp::SetComputedCell {
12130 sheet_id,
12131 row0,
12132 col0,
12133 old,
12134 new,
12135 } => {
12136 let sheet = self.graph.sheet_name(*sheet_id).to_string();
12137 let v = if undo { old.clone() } else { new.clone() };
12138 self.set_computed_overlay_cell_raw(&sheet, row0 + 1, col0 + 1, v);
12139 }
12140 ArrowOp::RestoreComputedRect {
12141 sheet_id,
12142 sr0,
12143 sc0,
12144 er0,
12145 ec0,
12146 old,
12147 new,
12148 } => {
12149 let sheet = self.graph.sheet_name(*sheet_id).to_string();
12150 let vals = if undo { old } else { new };
12151 let height = (*er0).saturating_sub(*sr0) as usize + 1;
12152 let width = (*ec0).saturating_sub(*sc0) as usize + 1;
12153 for r in 0..height {
12154 for c in 0..width {
12155 let v = vals
12156 .get(r)
12157 .and_then(|row| row.get(c))
12158 .cloned()
12159 .unwrap_or(LiteralValue::Empty);
12160 self.set_computed_overlay_cell_raw(
12161 &sheet,
12162 *sr0 + 1 + r as u32,
12163 *sc0 + 1 + c as u32,
12164 Some(v),
12165 );
12166 }
12167 }
12168 }
12169 ArrowOp::InsertRows {
12170 sheet_id,
12171 before0,
12172 count,
12173 } => {
12174 let sheet = self.graph.sheet_name(*sheet_id).to_string();
12175 self.ensure_arrow_sheet(&sheet);
12176 if let Some(asheet) = self.arrow_sheets.sheet_mut(&sheet) {
12177 if undo {
12178 asheet.delete_rows(*before0 as usize, *count as usize);
12179 } else {
12180 asheet.insert_rows(*before0 as usize, *count as usize);
12181 }
12182 }
12183 self.purge_derived_formats_after_row(*sheet_id, *before0);
12184 }
12185 ArrowOp::InsertCols {
12186 sheet_id,
12187 before0,
12188 count,
12189 } => {
12190 let sheet = self.graph.sheet_name(*sheet_id).to_string();
12191 self.ensure_arrow_sheet(&sheet);
12192 if let Some(asheet) = self.arrow_sheets.sheet_mut(&sheet) {
12193 if undo {
12194 asheet.delete_columns(*before0 as usize, *count as usize);
12195 } else {
12196 asheet.insert_columns(*before0 as usize, *count as usize);
12197 }
12198 }
12199 self.purge_derived_formats_after_col(*sheet_id, *before0);
12200 }
12201 }
12202 }
12203 }
12204
12205 fn record_spill_ops_into_arrow_undo(
12206 &mut self,
12207 undo: &mut crate::engine::ArrowUndoBatch,
12208 events: &[crate::engine::ChangeEvent],
12209 ) {
12210 use crate::engine::ChangeEvent;
12211 use formualizer_common::LiteralValue;
12212
12213 #[allow(clippy::type_complexity)]
12214 let rect_from_snapshot =
12215 |snap: &crate::engine::graph::editor::change_log::SpillSnapshot|
12216 -> Option<(SheetId, u32, u32, u32, u32, Vec<Vec<LiteralValue>>)> {
12217 if snap.target_cells.is_empty() {
12218 return None;
12219 }
12220 let sheet_id = snap.target_cells[0].sheet_id;
12221 let sr0 = snap.target_cells[0].coord.row();
12222 let sc0 = snap.target_cells[0].coord.col();
12223 if snap.values.is_empty() || snap.values[0].is_empty() {
12224 return None;
12225 }
12226 let h = snap.values.len() as u32;
12227 let w = snap.values[0].len() as u32;
12228 let er0 = sr0.saturating_add(h.saturating_sub(1));
12229 let ec0 = sc0.saturating_add(w.saturating_sub(1));
12230 Some((sheet_id, sr0, sc0, er0, ec0, snap.values.clone()))
12231 };
12232
12233 for ev in events {
12234 match ev {
12235 ChangeEvent::SpillCommitted { old, new, .. } => {
12236 if let Some((sid, sr0, sc0, er0, ec0, new_vals)) = rect_from_snapshot(new) {
12237 let old_vals = if let Some(old_snap) = old {
12238 rect_from_snapshot(old_snap)
12239 .map(|(_, _, _, _, _, v)| v)
12240 .unwrap_or_else(|| {
12241 vec![
12242 vec![LiteralValue::Empty; new_vals[0].len()];
12243 new_vals.len()
12244 ]
12245 })
12246 } else {
12247 vec![vec![LiteralValue::Empty; new_vals[0].len()]; new_vals.len()]
12248 };
12249 undo.record_restore_computed_rect(
12250 sid, sr0, sc0, er0, ec0, old_vals, new_vals,
12251 );
12252 }
12253 }
12254 ChangeEvent::SpillCleared { old, .. } => {
12255 if let Some((sid, sr0, sc0, er0, ec0, old_vals)) = rect_from_snapshot(old) {
12256 let new_vals =
12257 vec![vec![LiteralValue::Empty; old_vals[0].len()]; old_vals.len()];
12258 undo.record_restore_computed_rect(
12259 sid, sr0, sc0, er0, ec0, old_vals, new_vals,
12260 );
12261 }
12262 }
12263 _ => {}
12264 }
12265 }
12266 }
12267
12268 fn mirror_value_to_computed_overlay(
12273 &mut self,
12274 sheet: &str,
12275 row: u32,
12276 col: u32,
12277 value: &LiteralValue,
12278 ) {
12279 if !(self.config.arrow_storage_enabled
12280 && self.config.delta_overlay_enabled
12281 && self.config.write_formula_overlay_enabled)
12282 {
12283 return;
12284 }
12285 if self.computed_overlay_mirroring_disabled {
12286 return;
12287 }
12288
12289 let date_system = self.arrow_sheet_date_system(sheet);
12290 let ov = Self::literal_to_overlay_value(value, date_system);
12291 self.write_computed_overlay_value_0based(
12292 sheet,
12293 row.saturating_sub(1),
12294 col.saturating_sub(1),
12295 ov,
12296 );
12297 }
12298
12299 fn record_derived_format(&self, vertex_id: VertexId, format: Option<crate::format::FormatId>) {
12300 if let Some(cell) = self.graph.get_cell_ref(vertex_id) {
12301 self.record_derived_format_at(cell, format);
12302 }
12303 }
12304
12305 fn record_derived_format_at(&self, cell: CellRef, format: Option<crate::format::FormatId>) {
12306 #[cfg(test)]
12307 self.derived_format_operations_for_test
12308 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
12309 let format = format.filter(|id| *id != crate::format::FormatId::GENERAL);
12310 self.derived_formats.set(cell, format);
12311 }
12312
12313 fn clear_cell_format_state(&mut self, sheet: &str, cell: CellRef) {
12314 self.derived_formats.set(cell, None);
12315 if let Some(arrow) = self.arrow_sheets.sheet_mut(sheet) {
12316 arrow.clear_format(cell.coord.row() as usize, cell.coord.col() as usize);
12317 }
12318 }
12319
12320 fn clear_logged_cell_format_states(&mut self, events: &[ChangeEvent]) {
12321 let cells = events
12322 .iter()
12323 .filter_map(|event| match event {
12324 ChangeEvent::SetValue { addr, .. } | ChangeEvent::SetFormula { addr, .. } => {
12325 Some(*addr)
12326 }
12327 _ => None,
12328 })
12329 .collect::<FxHashSet<_>>();
12330 for cell in cells {
12331 let sheet = self.graph.sheet_name(cell.sheet_id).to_string();
12332 self.clear_cell_format_state(&sheet, cell);
12333 }
12334 }
12335
12336 fn purge_derived_formats_after_row(&mut self, sheet_id: SheetId, start0: u32) {
12337 self.derived_formats
12338 .retain(|cell| cell.sheet_id != sheet_id || cell.coord.row() < start0);
12339 }
12340
12341 fn purge_derived_formats_after_col(&mut self, sheet_id: SheetId, start0: u32) {
12342 self.derived_formats
12343 .retain(|cell| cell.sheet_id != sheet_id || cell.coord.col() < start0);
12344 }
12345
12346 fn purge_derived_formats_for_sheet(&mut self, sheet_id: SheetId) {
12347 self.derived_formats
12348 .retain(|cell| cell.sheet_id != sheet_id);
12349 }
12350
12351 #[cfg(test)]
12352 pub(crate) fn debug_computed_overlay_format_0based(
12353 &self,
12354 sheet: &str,
12355 row0: u32,
12356 col0: u32,
12357 ) -> Option<crate::format::FormatId> {
12358 let sheet = self.arrow_sheets.sheet(sheet)?;
12359 let (chunk_idx, row_in_chunk) = sheet.chunk_of_row(row0 as usize)?;
12360 sheet
12361 .columns
12362 .get(col0 as usize)?
12363 .chunk(chunk_idx)?
12364 .computed_overlay
12365 .get_format(row_in_chunk)
12366 }
12367
12368 #[cfg(test)]
12369 pub(crate) fn debug_computed_overlay_chunk_has_formats_0based(
12370 &self,
12371 sheet: &str,
12372 row0: u32,
12373 col0: u32,
12374 ) -> bool {
12375 let Some(sheet) = self.arrow_sheets.sheet(sheet) else {
12376 return false;
12377 };
12378 let Some((chunk_idx, _)) = sheet.chunk_of_row(row0 as usize) else {
12379 return false;
12380 };
12381 sheet
12382 .columns
12383 .get(col0 as usize)
12384 .and_then(|column| column.chunk(chunk_idx))
12385 .is_some_and(|chunk| chunk.computed_overlay.has_formats())
12386 }
12387
12388 #[cfg(test)]
12389 pub(crate) fn debug_clear_derived_format_0based(&mut self, sheet: &str, row0: u32, col0: u32) {
12390 if let Some(sheet_id) = self.graph.sheet_id(sheet) {
12391 self.derived_formats
12392 .set(CellRef::new_absolute(sheet_id, row0, col0), None);
12393 }
12394 }
12395
12396 #[cfg(test)]
12397 pub(crate) fn debug_record_derived_format_0based(
12398 &self,
12399 sheet: &str,
12400 row0: u32,
12401 col0: u32,
12402 format: Option<crate::format::FormatId>,
12403 ) {
12404 if let Some(sheet_id) = self.graph.sheet_id(sheet) {
12405 self.record_derived_format_at(CellRef::new_absolute(sheet_id, row0, col0), format);
12406 }
12407 }
12408
12409 #[cfg(test)]
12410 pub(crate) fn debug_derived_format_0based(
12411 &self,
12412 sheet: &str,
12413 row0: u32,
12414 col0: u32,
12415 ) -> Option<crate::format::FormatId> {
12416 let sheet_id = self.graph.sheet_id(sheet)?;
12417 self.derived_formats
12418 .get(&CellRef::new_absolute(sheet_id, row0, col0))
12419 }
12420
12421 #[cfg(test)]
12422 pub(crate) fn debug_reset_format_write_operation_counts(&mut self) {
12423 self.derived_format_operations_for_test
12424 .store(0, std::sync::atomic::Ordering::Relaxed);
12425 self.computed_overlay_set_explicit_entry_operations_for_test = 0;
12426 self.computed_overlay_stale_clear_range_effects_for_test = 0;
12427 self.computed_overlay_stale_clear_offset_attempts_for_test = 0;
12428 self.computed_format_vector_allocations_for_test
12429 .store(0, std::sync::atomic::Ordering::Relaxed);
12430 }
12431
12432 #[cfg(test)]
12433 pub(crate) fn debug_format_write_operation_counts(&self) -> (u64, u64, u64, u64, u64) {
12434 (
12435 self.derived_format_operations_for_test
12436 .load(std::sync::atomic::Ordering::Relaxed),
12437 self.computed_overlay_set_explicit_entry_operations_for_test,
12438 self.computed_format_vector_allocations_for_test
12439 .load(std::sync::atomic::Ordering::Relaxed),
12440 self.computed_overlay_stale_clear_range_effects_for_test,
12441 self.computed_overlay_stale_clear_offset_attempts_for_test,
12442 )
12443 }
12444
12445 fn write_computed_overlay_format_0based(
12446 &mut self,
12447 sheet: &str,
12448 row0: u32,
12449 col0: u32,
12450 format: Option<crate::format::FormatId>,
12451 ) {
12452 self.ensure_arrow_sheet(sheet);
12453 let (row0, col0) = (row0 as usize, col0 as usize);
12454 let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
12455 return;
12456 };
12457 if col0 >= asheet.columns.len() {
12458 asheet.insert_columns(asheet.columns.len(), col0 + 1 - asheet.columns.len());
12459 }
12460 if row0 >= asheet.nrows as usize {
12461 asheet.ensure_row_capacity(row0 + 1);
12462 }
12463 let Some((chunk, offset)) = asheet.chunk_of_row(row0) else {
12464 return;
12465 };
12466 if let Some(chunk) = asheet.ensure_column_chunk_mut(col0, chunk) {
12467 chunk.computed_overlay.set_format(offset, format);
12468 }
12469 }
12470
12471 fn write_computed_cell_0based(
12476 &mut self,
12477 cell: CellRef,
12478 value: &LiteralValue,
12479 format: Option<crate::format::FormatId>,
12480 ) {
12481 if !(self.config.arrow_storage_enabled
12482 && self.config.delta_overlay_enabled
12483 && self.config.write_formula_overlay_enabled)
12484 || self.computed_overlay_mirroring_disabled
12485 {
12486 return;
12487 }
12488 if self.arrow_sheet_by_id(cell.sheet_id).is_none() {
12489 let sheet = self.graph.sheet_name(cell.sheet_id).to_string();
12490 self.ensure_arrow_sheet(&sheet);
12491 }
12492 let (row0, col0) = (cell.coord.row() as usize, cell.coord.col() as usize);
12493 let Some(asheet) = self.arrow_sheet_by_id_mut(cell.sheet_id) else {
12494 return;
12495 };
12496 let ov = Self::literal_to_overlay_value(value, asheet.date_system);
12497 let cur_cols = asheet.columns.len();
12498 if col0 >= cur_cols {
12499 asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
12500 }
12501 if row0 >= asheet.nrows as usize {
12502 asheet.ensure_row_capacity(row0 + 1);
12503 }
12504 let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
12505 return;
12506 };
12507 let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
12508 return;
12509 };
12510 let delta = ch.computed_overlay.set_scalar(in_off, ov);
12511 ch.computed_overlay.set_format(in_off, format);
12512 self.adjust_computed_overlay_bytes(delta);
12513 if let Some(cap) = self.config.max_overlay_memory_bytes
12514 && self.computed_overlay_bytes_estimate > cap
12515 {
12516 self.disable_computed_overlay_mirroring_due_to_budget(cap);
12517 }
12518 }
12519
12520 fn write_computed_overlay_value_0based(
12521 &mut self,
12522 sheet: &str,
12523 row0: u32,
12524 col0: u32,
12525 value: OverlayValue,
12526 ) {
12527 if !(self.config.arrow_storage_enabled
12528 && self.config.delta_overlay_enabled
12529 && self.config.write_formula_overlay_enabled)
12530 {
12531 return;
12532 }
12533 if self.computed_overlay_mirroring_disabled {
12534 return;
12535 }
12536
12537 self.ensure_arrow_sheet(sheet);
12538
12539 let row0 = row0 as usize;
12540 let col0 = col0 as usize;
12541 let asheet = self
12542 .arrow_sheets
12543 .sheet_mut(sheet)
12544 .expect("ArrowSheet must exist");
12545
12546 let cur_cols = asheet.columns.len();
12547 if col0 >= cur_cols {
12548 asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
12549 }
12550
12551 if row0 >= asheet.nrows as usize {
12552 if asheet.columns.is_empty() {
12553 asheet.insert_columns(0, 1);
12554 }
12555 asheet.ensure_row_capacity(row0 + 1);
12556 }
12557
12558 let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
12559 return;
12560 };
12561 let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
12562 return;
12563 };
12564
12565 let delta = ch.computed_overlay.set_scalar(in_off, value);
12566 self.adjust_computed_overlay_bytes(delta);
12567
12568 if let Some(cap) = self.config.max_overlay_memory_bytes
12569 && self.computed_overlay_bytes_estimate > cap
12570 {
12571 self.disable_computed_overlay_mirroring_due_to_budget(cap);
12572 }
12573 }
12574
12575 pub(crate) fn plan_computed_write_coalescing(
12576 &self,
12577 buffer: &ComputedWriteBuffer,
12578 ) -> ComputedWriteCoalescingPlan {
12579 self.plan_computed_write_coalescing_from_writes(
12580 buffer.writes().iter().cloned(),
12581 buffer.formats_present,
12582 )
12583 }
12584
12585 fn plan_owned_computed_write_coalescing(
12586 &self,
12587 writes: Vec<ComputedWrite>,
12588 formats_present: bool,
12589 ) -> ComputedWriteCoalescingPlan {
12590 self.plan_computed_write_coalescing_from_writes(writes, formats_present)
12591 }
12592
12593 fn plan_computed_write_coalescing_from_writes(
12594 &self,
12595 writes: impl IntoIterator<Item = ComputedWrite>,
12596 formats_present: bool,
12597 ) -> ComputedWriteCoalescingPlan {
12598 let mut groups: BTreeMap<ComputedWriteChunkKey, Vec<ComputedWriteChunkEntryPlan>> =
12599 BTreeMap::new();
12600 let mut input_cells = 0usize;
12601 let mut located: Option<(SheetId, Option<&crate::arrow_store::ArrowSheet>)> = None;
12603
12604 for write in writes {
12605 match write {
12606 ComputedWrite::Cell {
12607 seq,
12608 sheet_id,
12609 row0,
12610 col0,
12611 value,
12612 format_id,
12613 } => {
12614 input_cells = input_cells.saturating_add(1);
12615 let sheet = match located {
12616 Some((id, sheet)) if id == sheet_id => sheet,
12617 _ => {
12618 let sheet = self.arrow_sheets.sheet(self.graph.sheet_name(sheet_id));
12619 located = Some((sheet_id, sheet));
12620 sheet
12621 }
12622 };
12623 let (chunk_idx, chunk_start_row0, row_in_chunk) = match sheet {
12624 Some(sheet) => {
12625 Self::locate_row_in_sheet_for_computed_write_plan(sheet, row0 as usize)
12626 }
12627 None => Self::locate_row_in_empty_sheet_for_computed_write_plan(
12628 row0 as usize,
12629 32 * 1024,
12630 ),
12631 };
12632 groups
12633 .entry(ComputedWriteChunkKey {
12634 sheet_id,
12635 col0,
12636 chunk_idx,
12637 chunk_start_row0,
12638 })
12639 .or_default()
12640 .push(ComputedWriteChunkEntryPlan {
12641 row_in_chunk,
12642 seq,
12643 value,
12644 format_id,
12645 });
12646 }
12647 ComputedWrite::Run {
12648 seq,
12649 sheet_id,
12650 row0,
12651 col0,
12652 entries,
12653 } => {
12654 input_cells = input_cells.saturating_add(entries.len());
12655 let sheet = match located {
12656 Some((id, sheet)) if id == sheet_id => sheet,
12657 _ => {
12658 let sheet = self.arrow_sheets.sheet(self.graph.sheet_name(sheet_id));
12659 located = Some((sheet_id, sheet));
12660 sheet
12661 }
12662 };
12663 let mut segment: Vec<ComputedWriteChunkEntryPlan> = Vec::new();
12667 let mut segment_key: Option<ComputedWriteChunkKey> = None;
12668 for (k, (value, format_id)) in entries.into_iter().enumerate() {
12669 let row = row0.saturating_add(k as u32) as usize;
12670 let (chunk_idx, chunk_start_row0, row_in_chunk) = match sheet {
12671 Some(sheet) => {
12672 Self::locate_row_in_sheet_for_computed_write_plan(sheet, row)
12673 }
12674 None => Self::locate_row_in_empty_sheet_for_computed_write_plan(
12675 row,
12676 32 * 1024,
12677 ),
12678 };
12679 let key = ComputedWriteChunkKey {
12680 sheet_id,
12681 col0,
12682 chunk_idx,
12683 chunk_start_row0,
12684 };
12685 if segment_key != Some(key)
12686 && let Some(done) = segment_key.replace(key)
12687 {
12688 groups.entry(done).or_default().append(&mut segment);
12689 }
12690 segment.push(ComputedWriteChunkEntryPlan {
12691 row_in_chunk,
12692 seq,
12693 value,
12694 format_id,
12695 });
12696 }
12697 if let Some(done) = segment_key {
12698 groups.entry(done).or_default().append(&mut segment);
12699 }
12700 }
12701 ComputedWrite::Rect {
12702 seq,
12703 sheet_id,
12704 sr0,
12705 sc0,
12706 values,
12707 } => {
12708 for (r_off, row) in values.into_iter().enumerate() {
12709 for (c_off, value) in row.into_iter().enumerate() {
12710 input_cells = input_cells.saturating_add(1);
12711 self.push_computed_write_plan_entry(
12712 &mut groups,
12713 seq,
12714 sheet_id,
12715 sr0.saturating_add(r_off as u32),
12716 sc0.saturating_add(c_off as u32),
12717 value,
12718 None,
12719 );
12720 }
12721 }
12722 }
12723 }
12724 }
12725
12726 let mut plan = ComputedWriteCoalescingPlan {
12727 chunks: Vec::with_capacity(groups.len()),
12728 input_cells,
12729 coalesced_cells: 0,
12730 overwritten_cells: 0,
12731 };
12732 for (key, entries) in groups {
12733 let computed_lane_has_formats =
12734 self.computed_overlay_chunk_has_formats(key.sheet_id, key.col0, key.chunk_idx);
12735 let (chunk_plan, overwritten) = ComputedWriteChunkPlan::from_group(
12736 key,
12737 entries,
12738 formats_present,
12739 computed_lane_has_formats,
12740 );
12741 #[cfg(test)]
12742 if matches!(
12743 &chunk_plan.format_effect,
12744 ComputedWriteChunkFormatEffect::SetExplicit(_)
12745 ) {
12746 self.computed_format_vector_allocations_for_test
12747 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
12748 }
12749 plan.coalesced_cells = plan
12750 .coalesced_cells
12751 .saturating_add(chunk_plan.entries.len());
12752 plan.overwritten_cells = plan.overwritten_cells.saturating_add(overwritten);
12753 plan.chunks.push(chunk_plan);
12754 }
12755 debug_assert_eq!(
12756 plan.input_cells,
12757 plan.coalesced_cells.saturating_add(plan.overwritten_cells)
12758 );
12759 plan
12760 }
12761
12762 fn push_computed_write_plan_entry(
12763 &self,
12764 groups: &mut BTreeMap<ComputedWriteChunkKey, Vec<ComputedWriteChunkEntryPlan>>,
12765 seq: u64,
12766 sheet_id: SheetId,
12767 row0: u32,
12768 col0: u32,
12769 value: OverlayValue,
12770 format_id: Option<crate::format::FormatId>,
12771 ) {
12772 let (chunk_idx, chunk_start_row0, row_in_chunk) =
12773 self.locate_computed_write_chunk(sheet_id, row0);
12774 let key = ComputedWriteChunkKey {
12775 sheet_id,
12776 col0,
12777 chunk_idx,
12778 chunk_start_row0,
12779 };
12780 groups
12781 .entry(key)
12782 .or_default()
12783 .push(ComputedWriteChunkEntryPlan {
12784 row_in_chunk,
12785 seq,
12786 value,
12787 format_id,
12788 });
12789 }
12790
12791 fn computed_overlay_chunk_has_formats(
12792 &self,
12793 sheet_id: SheetId,
12794 col0: u32,
12795 chunk_idx: usize,
12796 ) -> bool {
12797 let sheet_name = self.graph.sheet_name(sheet_id);
12798 self.arrow_sheets
12799 .sheet(sheet_name)
12800 .and_then(|sheet| sheet.columns.get(col0 as usize))
12801 .and_then(|column| column.chunk(chunk_idx))
12802 .is_some_and(|chunk| chunk.computed_overlay.has_formats())
12803 }
12804
12805 fn locate_computed_write_chunk(&self, sheet_id: SheetId, row0: u32) -> (usize, u32, usize) {
12806 let sheet_name = self.graph.sheet_name(sheet_id);
12807 if let Some(sheet) = self.arrow_sheets.sheet(sheet_name) {
12808 return Self::locate_row_in_sheet_for_computed_write_plan(sheet, row0 as usize);
12809 }
12810 Self::locate_row_in_empty_sheet_for_computed_write_plan(row0 as usize, 32 * 1024)
12811 }
12812
12813 fn locate_row_in_sheet_for_computed_write_plan(
12814 sheet: &crate::arrow_store::ArrowSheet,
12815 row0: usize,
12816 ) -> (usize, u32, usize) {
12817 if row0 < sheet.nrows as usize
12818 && let Some((chunk_idx, row_in_chunk)) = sheet.chunk_of_row(row0)
12819 {
12820 let chunk_start = sheet.chunk_starts.get(chunk_idx).copied().unwrap_or(0);
12821 return (chunk_idx, chunk_start as u32, row_in_chunk);
12822 }
12823
12824 let chunk_rows = sheet.chunk_rows.max(1);
12825 if sheet.chunk_starts.is_empty() {
12826 return Self::locate_row_in_empty_sheet_for_computed_write_plan(row0, chunk_rows);
12827 }
12828
12829 let mut chunk_idx = sheet.chunk_starts.len().saturating_sub(1);
12830 let mut chunk_start = sheet.chunk_starts[chunk_idx];
12831 while chunk_start.saturating_add(chunk_rows) <= row0 {
12832 chunk_idx = chunk_idx.saturating_add(1);
12833 chunk_start = chunk_start.saturating_add(chunk_rows);
12834 }
12835 (
12836 chunk_idx,
12837 chunk_start as u32,
12838 row0.saturating_sub(chunk_start),
12839 )
12840 }
12841
12842 fn locate_row_in_empty_sheet_for_computed_write_plan(
12843 row0: usize,
12844 chunk_rows: usize,
12845 ) -> (usize, u32, usize) {
12846 let chunk_rows = chunk_rows.max(1);
12847 let chunk_idx = row0 / chunk_rows;
12848 let chunk_start = chunk_idx.saturating_mul(chunk_rows);
12849 (
12850 chunk_idx,
12851 chunk_start as u32,
12852 row0.saturating_sub(chunk_start),
12853 )
12854 }
12855
12856 #[cfg(test)]
12857 pub(crate) fn debug_plan_computed_write_coalescing(
12858 &self,
12859 buffer: &ComputedWriteBuffer,
12860 ) -> ComputedWriteCoalescingPlan {
12861 self.plan_computed_write_coalescing(buffer)
12862 }
12863
12864 pub(crate) fn flush_computed_write_buffer(
12865 &mut self,
12866 buffer: &mut ComputedWriteBuffer,
12867 ) -> Result<(), ExcelError> {
12868 if buffer.is_empty() {
12869 return Ok(());
12870 }
12871
12872 self.resource_checkpoint(0)?;
12877 let commit_started = self.preflight_evaluation_commit_window(buffer.len())?;
12878 let (writes, formats_present) = buffer.take_writes();
12879 let plan = self.plan_owned_computed_write_coalescing(writes, formats_present);
12880 self.flush_computed_write_plan(plan);
12881 self.observe_evaluation_commit_window(commit_started);
12882
12883 Ok(())
12884 }
12885
12886 fn flush_computed_write_plan(&mut self, plan: ComputedWriteCoalescingPlan) {
12887 for chunk in plan.chunks {
12888 self.flush_computed_write_chunk_plan(chunk);
12889 }
12890 }
12891
12892 fn flush_computed_write_chunk_plan(&mut self, chunk: ComputedWriteChunkPlan) {
12893 match &chunk.shape {
12894 ComputedWriteChunkPlanShape::Point => {
12895 self.flush_computed_write_chunk_plan_as_points(chunk);
12896 }
12897 ComputedWriteChunkPlanShape::SparseOffsets { .. } => {
12898 self.flush_computed_write_chunk_plan_as_sparse_fragment_or_points(chunk);
12899 }
12900 ComputedWriteChunkPlanShape::DenseRange { .. } => {
12901 self.flush_computed_write_chunk_plan_as_dense_fragment(chunk);
12902 }
12903 ComputedWriteChunkPlanShape::RunRange { len, runs, .. } => {
12904 if Self::should_emit_computed_run_fragment(*len, *runs) {
12905 self.flush_computed_write_chunk_plan_as_run_fragment(chunk);
12906 } else {
12907 self.flush_computed_write_chunk_plan_as_dense_fragment(chunk);
12908 }
12909 }
12910 }
12911 }
12912
12913 #[inline]
12914 fn should_emit_computed_run_fragment(len: usize, runs: usize) -> bool {
12915 runs <= len / 2
12916 }
12917
12918 fn flush_computed_write_chunk_plan_as_points(&mut self, chunk: ComputedWriteChunkPlan) {
12919 let sheet_name = self.graph.sheet_name(chunk.sheet_id).to_string();
12920 for entry in chunk.entries {
12921 let row0 = chunk
12922 .chunk_start_row0
12923 .saturating_add(entry.row_in_chunk as u32);
12924 self.write_computed_overlay_value_0based(&sheet_name, row0, chunk.col0, entry.value);
12925 }
12926 self.apply_computed_overlay_format_effect(
12927 chunk.sheet_id,
12928 chunk.col0,
12929 chunk.chunk_idx,
12930 chunk.format_effect,
12931 );
12932 }
12933
12934 fn flush_computed_write_chunk_plan_as_sparse_fragment_or_points(
12935 &mut self,
12936 chunk: ComputedWriteChunkPlan,
12937 ) {
12938 let point_estimate = Self::computed_write_chunk_plan_point_estimate(&chunk);
12939 let sheet_id = chunk.sheet_id;
12940 let col0 = chunk.col0;
12941 let chunk_idx = chunk.chunk_idx;
12942 let chunk_start_row0 = chunk.chunk_start_row0;
12943 let format_effect = chunk.format_effect;
12944 let items: Vec<(usize, OverlayValue)> = chunk
12945 .entries
12946 .into_iter()
12947 .map(|entry| (entry.row_in_chunk, entry.value))
12948 .collect();
12949 match OverlayFragment::sparse_offsets_if_estimated_smaller_than_points(
12950 items,
12951 point_estimate,
12952 ) {
12953 Some(Ok(fragment)) => {
12954 self.apply_computed_overlay_fragment(sheet_id, col0, chunk_idx, fragment);
12955 }
12956 Some(Err(cells)) => {
12957 self.flush_computed_overlay_cells_as_points(
12958 sheet_id,
12959 col0,
12960 chunk_start_row0,
12961 cells,
12962 );
12963 }
12964 None => {}
12965 }
12966 self.apply_computed_overlay_format_effect(sheet_id, col0, chunk_idx, format_effect);
12967 }
12968
12969 #[inline]
12970 fn computed_write_chunk_plan_point_estimate(chunk: &ComputedWriteChunkPlan) -> usize {
12971 chunk
12972 .entries
12973 .iter()
12974 .map(|entry| ComputedWriteBuffer::estimate_value_bytes(&entry.value))
12975 .fold(0usize, usize::saturating_add)
12976 }
12977
12978 fn flush_computed_overlay_cells_as_points(
12979 &mut self,
12980 sheet_id: SheetId,
12981 col0: u32,
12982 chunk_start_row0: u32,
12983 cells: Vec<(usize, OverlayValue)>,
12984 ) {
12985 let sheet_name = self.graph.sheet_name(sheet_id).to_string();
12986 for (row_in_chunk, value) in cells {
12987 let row0 = chunk_start_row0.saturating_add(row_in_chunk as u32);
12988 self.write_computed_overlay_value_0based(&sheet_name, row0, col0, value);
12989 }
12990 }
12991
12992 fn flush_computed_write_chunk_plan_as_dense_fragment(&mut self, chunk: ComputedWriteChunkPlan) {
12993 if chunk.entries.is_empty() {
12994 return;
12995 }
12996 let start = chunk.entries[0].row_in_chunk;
12997 let values: Vec<OverlayValue> =
12998 chunk.entries.into_iter().map(|entry| entry.value).collect();
12999 if let Some(fragment) = OverlayFragment::dense_range(start, values) {
13000 self.apply_computed_overlay_fragment(
13001 chunk.sheet_id,
13002 chunk.col0,
13003 chunk.chunk_idx,
13004 fragment,
13005 );
13006 }
13007 self.apply_computed_overlay_format_effect(
13008 chunk.sheet_id,
13009 chunk.col0,
13010 chunk.chunk_idx,
13011 chunk.format_effect,
13012 );
13013 }
13014
13015 fn flush_computed_write_chunk_plan_as_run_fragment(&mut self, chunk: ComputedWriteChunkPlan) {
13016 if chunk.entries.is_empty() {
13017 return;
13018 }
13019 let start = chunk.entries[0].row_in_chunk;
13020 let values: Vec<OverlayValue> =
13021 chunk.entries.into_iter().map(|entry| entry.value).collect();
13022 if let Some(fragment) = OverlayFragment::run_range(start, values) {
13023 self.apply_computed_overlay_fragment(
13024 chunk.sheet_id,
13025 chunk.col0,
13026 chunk.chunk_idx,
13027 fragment,
13028 );
13029 }
13030 self.apply_computed_overlay_format_effect(
13031 chunk.sheet_id,
13032 chunk.col0,
13033 chunk.chunk_idx,
13034 chunk.format_effect,
13035 );
13036 }
13037
13038 fn apply_computed_overlay_format_effect(
13039 &mut self,
13040 sheet_id: SheetId,
13041 col0: u32,
13042 chunk_idx: usize,
13043 effect: ComputedWriteChunkFormatEffect,
13044 ) {
13045 if !(self.config.arrow_storage_enabled
13046 && self.config.delta_overlay_enabled
13047 && self.config.write_formula_overlay_enabled)
13048 || self.computed_overlay_mirroring_disabled
13049 {
13050 return;
13051 }
13052
13053 let sheet_name = self.graph.sheet_name(sheet_id);
13054 let Some(sheet) = self.arrow_sheets.sheet_mut(sheet_name) else {
13055 return;
13056 };
13057 let Some(chunk) = sheet
13058 .columns
13059 .get_mut(col0 as usize)
13060 .and_then(|column| column.chunk_mut(chunk_idx))
13061 else {
13062 return;
13063 };
13064 match effect {
13065 ComputedWriteChunkFormatEffect::NoFormatWork => {}
13066 ComputedWriteChunkFormatEffect::ClearStale(ComputedWriteFormatClear::Range {
13067 start,
13068 end,
13069 }) => {
13070 #[cfg(test)]
13071 {
13072 self.computed_overlay_stale_clear_range_effects_for_test = self
13073 .computed_overlay_stale_clear_range_effects_for_test
13074 .saturating_add(1);
13075 }
13076 chunk.computed_overlay.clear_format_range(start, end);
13077 }
13078 ComputedWriteChunkFormatEffect::ClearStale(ComputedWriteFormatClear::Offsets(
13079 offsets,
13080 )) => {
13081 #[cfg(test)]
13082 {
13083 self.computed_overlay_stale_clear_offset_attempts_for_test = self
13084 .computed_overlay_stale_clear_offset_attempts_for_test
13085 .saturating_add(offsets.len() as u64);
13086 }
13087 chunk.computed_overlay.clear_format_offsets(&offsets);
13088 }
13089 ComputedWriteChunkFormatEffect::SetExplicit(formats) => {
13090 #[cfg(test)]
13091 {
13092 self.computed_overlay_set_explicit_entry_operations_for_test = self
13093 .computed_overlay_set_explicit_entry_operations_for_test
13094 .saturating_add(formats.len() as u64);
13095 }
13096 for (row_in_chunk, format_id) in formats {
13097 chunk.computed_overlay.set_format(row_in_chunk, format_id);
13098 }
13099 }
13100 }
13101 }
13102
13103 fn apply_computed_overlay_fragment(
13104 &mut self,
13105 sheet_id: SheetId,
13106 col0: u32,
13107 chunk_idx: usize,
13108 fragment: OverlayFragment,
13109 ) {
13110 if !(self.config.arrow_storage_enabled
13111 && self.config.delta_overlay_enabled
13112 && self.config.write_formula_overlay_enabled)
13113 {
13114 return;
13115 }
13116 if self.computed_overlay_mirroring_disabled {
13117 return;
13118 }
13119
13120 let sheet_name = self.graph.sheet_name(sheet_id).to_string();
13121 self.ensure_arrow_sheet(&sheet_name);
13122
13123 let col0 = col0 as usize;
13124 let asheet = self
13125 .arrow_sheets
13126 .sheet_mut(&sheet_name)
13127 .expect("ArrowSheet must exist");
13128
13129 let cur_cols = asheet.columns.len();
13130 if col0 >= cur_cols {
13131 asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
13132 }
13133
13134 let start_row0 = asheet
13135 .chunk_starts
13136 .get(chunk_idx)
13137 .copied()
13138 .unwrap_or_else(|| chunk_idx.saturating_mul(asheet.chunk_rows.max(1)));
13139 let required_rows =
13140 start_row0.saturating_add(fragment.max_covered_offset().saturating_add(1));
13141 if required_rows > asheet.nrows as usize {
13142 if asheet.columns.is_empty() {
13143 asheet.insert_columns(0, 1);
13144 }
13145 asheet.ensure_row_capacity(required_rows);
13146 }
13147
13148 let Some(ch) = asheet.ensure_column_chunk_mut(col0, chunk_idx) else {
13149 return;
13150 };
13151 let delta = ch.computed_overlay.apply_fragment(fragment);
13152 self.adjust_computed_overlay_bytes(delta);
13153
13154 if let Some(cap) = self.config.max_overlay_memory_bytes
13155 && self.computed_overlay_bytes_estimate > cap
13156 {
13157 self.disable_computed_overlay_mirroring_due_to_budget(cap);
13158 }
13159 }
13160
13161 #[inline]
13162 fn adjust_computed_overlay_bytes(&mut self, delta: isize) {
13163 if delta >= 0 {
13164 self.computed_overlay_bytes_estimate = self
13165 .computed_overlay_bytes_estimate
13166 .saturating_add(delta as usize);
13167 } else {
13168 self.computed_overlay_bytes_estimate = self
13169 .computed_overlay_bytes_estimate
13170 .saturating_sub((-delta) as usize);
13171 }
13172 }
13173
13174 fn clear_all_computed_overlays(&mut self) {
13175 let mut freed_total = 0usize;
13176 for sh in self.arrow_sheets.sheets.iter_mut() {
13177 for col in sh.columns.iter_mut() {
13178 for ch in col.chunks.iter_mut() {
13179 freed_total = freed_total.saturating_add(ch.computed_overlay.clear());
13180 }
13181 for ch in col.sparse_chunks.values_mut() {
13182 freed_total = freed_total.saturating_add(ch.computed_overlay.clear());
13183 }
13184 }
13185 }
13186 self.computed_overlay_bytes_estimate = self
13187 .computed_overlay_bytes_estimate
13188 .saturating_sub(freed_total);
13189 }
13190
13191 fn disable_computed_overlay_mirroring_due_to_budget(&mut self, _cap: usize) {
13192 self.compact_all_computed_overlays();
13195 }
13196
13197 fn compact_all_computed_overlays(&mut self) {
13200 let mut freed_total = 0usize;
13201 for sheet in self.arrow_sheets.sheets.iter_mut() {
13202 for col_idx in 0..sheet.columns.len() {
13203 let num_dense = sheet.columns[col_idx].chunks.len();
13205 for ch_idx in 0..num_dense {
13206 freed_total += sheet.compact_computed_overlay_chunk(col_idx, ch_idx);
13207 }
13208 let sparse_keys: Vec<usize> = sheet.columns[col_idx]
13210 .sparse_chunks
13211 .keys()
13212 .copied()
13213 .collect();
13214 for ch_idx in sparse_keys {
13215 freed_total += sheet.compact_computed_overlay_sparse_chunk(col_idx, ch_idx);
13216 }
13217 }
13218 }
13219 self.computed_overlay_bytes_estimate = self
13220 .computed_overlay_bytes_estimate
13221 .saturating_sub(freed_total);
13222 self.overlay_compactions = self.overlay_compactions.saturating_add(1);
13223 }
13224
13225 fn mirror_vertex_value_to_overlay(&mut self, vertex_id: VertexId, value: &LiteralValue) {
13226 let _ = self.record_vertex_value_to_overlay(vertex_id, value, None);
13227 }
13228
13229 fn record_vertex_value_to_overlay(
13230 &mut self,
13231 vertex_id: VertexId,
13232 value: &LiteralValue,
13233 computed_writes: Option<&mut ComputedWriteBuffer>,
13234 ) -> Result<(), ExcelError> {
13235 if !(self.config.arrow_storage_enabled
13236 && self.config.delta_overlay_enabled
13237 && self.config.write_formula_overlay_enabled)
13238 {
13239 return Ok(());
13240 }
13241 if self.computed_overlay_mirroring_disabled {
13242 return Ok(());
13243 }
13244 if !matches!(
13245 self.graph.get_vertex_kind(vertex_id),
13246 VertexKind::FormulaScalar | VertexKind::FormulaArray
13247 ) {
13248 return Ok(());
13249 }
13250 let Some(cell) = self.graph.get_cell_ref(vertex_id) else {
13251 return Ok(());
13252 };
13253 let Some(buffer) = computed_writes else {
13254 let format_id = self.derived_formats.get(&cell);
13256 self.write_computed_cell_0based(cell, value, format_id);
13257 return Ok(());
13258 };
13259 let date_system = self
13260 .arrow_sheet_by_id(cell.sheet_id)
13261 .map(|sheet| sheet.date_system)
13262 .unwrap_or(self.config.date_system);
13263 let ov = Self::literal_to_overlay_value(value, date_system);
13264 {
13265 let format_id = self.derived_formats.get(&cell);
13266 buffer.push_cell_with_format(
13267 cell.sheet_id,
13268 cell.coord.row(),
13269 cell.coord.col(),
13270 ov,
13271 format_id,
13272 );
13273 if self.should_flush_computed_write_buffer(buffer) {
13274 self.flush_computed_write_buffer(buffer)?;
13275 }
13276 }
13277 Ok(())
13278 }
13279
13280 #[inline]
13281 fn should_flush_computed_write_buffer(&self, buffer: &ComputedWriteBuffer) -> bool {
13282 self.config.max_overlay_memory_bytes.is_some_and(|cap| {
13283 if cap == 0 {
13284 return false;
13285 }
13286 self.computed_overlay_bytes_estimate
13287 .saturating_add(buffer.estimated_bytes())
13288 > cap
13289 })
13290 }
13291
13292 pub fn overlay_memory_usage(&self) -> usize {
13294 self.computed_overlay_bytes_estimate
13295 }
13296
13297 #[cfg(test)]
13298 pub(crate) fn debug_overlay_compactions(&self) -> u64 {
13299 self.overlay_compactions
13300 }
13301
13302 #[cfg(test)]
13303 pub(crate) fn debug_recompute_computed_overlay_bytes(&mut self) -> usize {
13304 let mut total = 0usize;
13305 for sheet in &self.arrow_sheets.sheets {
13306 for column in &sheet.columns {
13307 for chunk in &column.chunks {
13308 total = total.saturating_add(chunk.computed_overlay.estimated_bytes());
13309 }
13310 for chunk in column.sparse_chunks.values() {
13311 total = total.saturating_add(chunk.computed_overlay.estimated_bytes());
13312 }
13313 }
13314 }
13315 self.computed_overlay_bytes_estimate = total;
13316 total
13317 }
13318
13319 fn resolve_sheet_locator_for_write(
13320 &mut self,
13321 loc: formualizer_common::SheetLocator<'_>,
13322 current_sheet: &str,
13323 ) -> Result<SheetId, ExcelError> {
13324 Ok(match loc {
13325 formualizer_common::SheetLocator::Id(id) => id,
13326 formualizer_common::SheetLocator::Name(name) => self.graph.sheet_id_mut(name.as_ref()),
13327 formualizer_common::SheetLocator::Current => self.graph.sheet_id_mut(current_sheet),
13328 })
13329 }
13330
13331 fn resolve_sheet_locator_for_read(
13332 &self,
13333 loc: formualizer_common::SheetLocator<'_>,
13334 current_sheet: &str,
13335 ) -> Result<SheetId, ExcelError> {
13336 match loc {
13337 formualizer_common::SheetLocator::Id(id) => Ok(id),
13338 formualizer_common::SheetLocator::Name(name) => self
13339 .graph
13340 .sheet_id(name.as_ref())
13341 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref)),
13342 formualizer_common::SheetLocator::Current => self
13343 .graph
13344 .sheet_id(current_sheet)
13345 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref)),
13346 }
13347 }
13348
13349 pub fn set_cell_value(
13351 &mut self,
13352 sheet: &str,
13353 row: u32,
13354 col: u32,
13355 value: LiteralValue,
13356 ) -> Result<(), ExcelError> {
13357 self.observe_function_semantic_epoch()?;
13358 let sheet_existed = self.graph.sheet_id(sheet).is_some();
13359 let sheet_id = self.graph.sheet_id_mut(sheet);
13360 let cell_ref = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13361 let replaced = self
13362 .graph
13363 .get_vertex_id_for_address(&cell_ref)
13364 .filter(|&vertex| {
13365 matches!(
13366 self.graph.get_vertex_kind(vertex),
13367 VertexKind::FormulaScalar | VertexKind::FormulaArray
13368 )
13369 });
13370 let replaced_formula = replaced.is_some();
13371 if let Some(anchor) = replaced
13375 && self.graph.spill_registry_has_anchor(anchor)
13376 {
13377 self.clear_spill_projection_and_mirror(anchor, None);
13378 }
13379 self.graph.set_cell_value(sheet, row, col, value.clone())?;
13380 self.clear_cell_format_state(sheet, cell_ref);
13381 self.record_changed_cell(sheet, row, col);
13382 if !sheet_existed || replaced_formula {
13383 self.mark_topology_edited();
13384 }
13385 self.mirror_value_to_overlay(sheet, row, col, &value);
13387 self.mark_data_edited();
13389 Ok(())
13390 }
13391
13392 fn record_changed_cell(&mut self, sheet: &str, row: u32, col: u32) {
13394 let sheet_id = self.graph.sheet_id_mut(sheet);
13395 self.record_structural_change(StructuralScope::Cell {
13396 sheet: sheet_id,
13397 row: row.saturating_sub(1),
13398 col: col.saturating_sub(1),
13399 });
13400 }
13401
13402 fn record_change_for_event(&mut self, event: &ChangeEvent) {
13403 match event {
13404 ChangeEvent::SetValue { addr, .. } | ChangeEvent::SetFormula { addr, .. } => {
13405 self.record_structural_change(StructuralScope::Cell {
13406 sheet: addr.sheet_id,
13407 row: addr.coord.row(),
13408 col: addr.coord.col(),
13409 });
13410 }
13411 ChangeEvent::SpillCommitted { new, .. } => {
13412 if let Some(scope) = Self::structural_scope_from_cells(&new.target_cells) {
13413 self.record_structural_change(scope);
13414 }
13415 }
13416 ChangeEvent::SpillCleared { old, .. } => {
13417 if let Some(scope) = Self::structural_scope_from_cells(&old.target_cells) {
13418 self.record_structural_change(scope);
13419 }
13420 }
13421 ChangeEvent::DefineName { .. }
13422 | ChangeEvent::UpdateName { .. }
13423 | ChangeEvent::DeleteName { .. }
13424 | ChangeEvent::NamedRangeAdjusted { .. } => {
13425 self.record_structural_change(StructuralScope::AllSheets);
13429 }
13430 ChangeEvent::VertexMoved { .. } | ChangeEvent::FormulaAdjusted { .. } => {
13431 }
13434 ChangeEvent::SetRowVisibility { sheet_id, row0, .. } => {
13435 self.record_structural_change(StructuralScope::Region(Region::whole_row(
13436 *sheet_id, *row0,
13437 )));
13438 }
13439 ChangeEvent::AddVertex { .. }
13440 | ChangeEvent::RemoveVertex { .. }
13441 | ChangeEvent::EdgeAdded { .. }
13442 | ChangeEvent::EdgeRemoved { .. }
13443 | ChangeEvent::CompoundStart { .. }
13444 | ChangeEvent::CompoundEnd { .. }
13445 | ChangeEvent::StagedFormulaCellChanged { .. } => {}
13446 }
13447 }
13448
13449 fn record_structural_change(&mut self, scope: StructuralScope) {
13450 if let StructuralScope::Cell { sheet, row, col } = scope {
13451 self.wake_spill_owner_of_entered_formula(sheet, row, col);
13452 }
13453 self.wake_spill_blocker_waiters(scope);
13454 self.invalidate_pending_spills(scope);
13455 }
13456
13457 fn wake_spill_blocker_waiters(&mut self, scope: StructuralScope) {
13461 if self.spill_blocker_waiters.is_empty() {
13462 return;
13463 }
13464 let woken = self.spill_blocker_waiters.take_affected(scope);
13465 let woken: Vec<VertexId> = woken
13466 .into_iter()
13467 .filter(|&vertex| {
13468 self.graph.vertex_exists(vertex)
13469 && matches!(
13470 self.graph.get_vertex_kind(vertex),
13471 VertexKind::FormulaScalar | VertexKind::FormulaArray
13472 )
13473 })
13474 .collect();
13475 if !woken.is_empty() {
13476 self.graph.mark_dirty_many(&woken);
13477 }
13478 }
13479
13480 fn wake_spill_owner_of_entered_formula(&mut self, sheet: SheetId, row: u32, col: u32) {
13484 let cell = self.graph.make_cell_ref_internal(sheet, row, col);
13485 if let Some(owner) = self.graph.spill_registry_anchor_for_cell(cell)
13486 && self.graph.is_foreign_formula_cell(&cell, owner)
13487 {
13488 self.graph.mark_dirty_many(&[owner]);
13489 }
13490 }
13491
13492 fn structural_scope_from_cells(cells: &[CellRef]) -> Option<StructuralScope> {
13493 let first = cells.first()?;
13494 let sheet_id = first.sheet_id;
13495 if cells.iter().any(|cell| cell.sheet_id != sheet_id) {
13496 return Some(StructuralScope::OpaqueGlobal);
13497 }
13498 let mut row_start = first.coord.row();
13499 let mut row_end = row_start;
13500 let mut col_start = first.coord.col();
13501 let mut col_end = col_start;
13502 for cell in cells.iter().skip(1) {
13503 row_start = row_start.min(cell.coord.row());
13504 row_end = row_end.max(cell.coord.row());
13505 col_start = col_start.min(cell.coord.col());
13506 col_end = col_end.max(cell.coord.col());
13507 }
13508 Some(StructuralScope::Region(Region::rect(
13509 sheet_id, row_start, row_end, col_start, col_end,
13510 )))
13511 }
13512
13513 pub fn set_cell_value_ref(
13514 &mut self,
13515 cell: formualizer_common::SheetCellRef<'_>,
13516 current_sheet: &str,
13517 value: LiteralValue,
13518 ) -> Result<(), ExcelError> {
13519 let owned = cell.into_owned();
13520 let sheet_id = self.resolve_sheet_locator_for_write(owned.sheet, current_sheet)?;
13521 let sheet_name = self.graph.sheet_name(sheet_id).to_string();
13522 self.set_cell_value(
13523 &sheet_name,
13524 owned.coord.row() + 1,
13525 owned.coord.col() + 1,
13526 value,
13527 )
13528 }
13529
13530 pub fn set_cell_formula_ref(
13531 &mut self,
13532 cell: formualizer_common::SheetCellRef<'_>,
13533 current_sheet: &str,
13534 ast: ASTNode,
13535 ) -> Result<(), ExcelError> {
13536 let owned = cell.into_owned();
13537 let sheet_id = self.resolve_sheet_locator_for_write(owned.sheet, current_sheet)?;
13538 let sheet_name = self.graph.sheet_name(sheet_id).to_string();
13539 self.set_cell_formula(
13540 &sheet_name,
13541 owned.coord.row() + 1,
13542 owned.coord.col() + 1,
13543 ast,
13544 )
13545 }
13546
13547 pub fn get_cell_value_ref(
13548 &self,
13549 cell: formualizer_common::SheetCellRef<'_>,
13550 current_sheet: &str,
13551 ) -> Result<Option<LiteralValue>, ExcelError> {
13552 let owned = cell.into_owned();
13553 let sheet_id = self.resolve_sheet_locator_for_read(owned.sheet, current_sheet)?;
13554 let sheet_name = self.graph.sheet_name(sheet_id);
13555 Ok(self.get_cell_value(sheet_name, owned.coord.row() + 1, owned.coord.col() + 1))
13556 }
13557
13558 pub fn resolve_range_view_sheet_ref<'c>(
13559 &'c self,
13560 r: &formualizer_common::SheetRef<'_>,
13561 current_sheet: &str,
13562 ) -> Result<RangeView<'c>, ExcelError> {
13563 use formualizer_common::SheetLocator;
13564
13565 let sheet_to_opt_name = |loc: SheetLocator<'_>| -> Result<Option<String>, ExcelError> {
13566 match loc {
13567 SheetLocator::Current => Ok(None),
13568 SheetLocator::Name(name) => Ok(Some(name.as_ref().to_string())),
13569 SheetLocator::Id(id) => Ok(Some(self.graph.sheet_name(id).to_string())),
13570 }
13571 };
13572
13573 let rt = match r {
13574 formualizer_common::SheetRef::Cell(cell) => ReferenceType::Cell {
13575 sheet: sheet_to_opt_name(cell.sheet.clone())?,
13576 row: cell.coord.row() + 1,
13577 col: cell.coord.col() + 1,
13578 row_abs: cell.coord.row_abs(),
13579 col_abs: cell.coord.col_abs(),
13580 },
13581 formualizer_common::SheetRef::Range(range) => ReferenceType::Range {
13582 sheet: sheet_to_opt_name(range.sheet.clone())?,
13583 start_row: range.start_row.map(|b| b.index + 1),
13584 start_col: range.start_col.map(|b| b.index + 1),
13585 end_row: range.end_row.map(|b| b.index + 1),
13586 end_col: range.end_col.map(|b| b.index + 1),
13587 start_row_abs: range.start_row.map(|b| b.abs).unwrap_or(false),
13588 start_col_abs: range.start_col.map(|b| b.abs).unwrap_or(false),
13589 end_row_abs: range.end_row.map(|b| b.abs).unwrap_or(false),
13590 end_col_abs: range.end_col.map(|b| b.abs).unwrap_or(false),
13591 },
13592 };
13593
13594 crate::traits::EvaluationContext::resolve_range_view(self, &rt, current_sheet)
13595 }
13596
13597 pub fn set_cell_formula(
13599 &mut self,
13600 sheet: &str,
13601 row: u32,
13602 col: u32,
13603 ast: ASTNode,
13604 ) -> Result<(), ExcelError> {
13605 self.observe_function_semantic_epoch()?;
13606 let sheet_id = self.graph.sheet_id_mut(sheet);
13607 let placement = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13608 let ingested = {
13609 let mut pipeline = self.ingest_pipeline();
13610 pipeline.ingest_formula(FormulaAstInput::Tree(ast), placement, None)?
13611 };
13612 self.graph.set_cell_formula_with_plan(
13613 sheet,
13614 row,
13615 col,
13616 ingested.ast_id,
13617 &ingested.dep_plan,
13618 ingested.dep_plan.volatile,
13619 ingested.dep_plan.dynamic,
13620 )?;
13621 self.clear_cell_format_state(sheet, placement);
13622 self.record_changed_cell(sheet, row, col);
13623
13624 self.clear_delta_overlay_cell(sheet, row, col);
13629
13630 self.mark_topology_edited();
13632 Ok(())
13633 }
13634
13635 pub fn bulk_set_formulas<I>(&mut self, sheet: &str, items: I) -> Result<usize, ExcelError>
13637 where
13638 I: IntoIterator<Item = (u32, u32, ASTNode)>,
13639 {
13640 let collected: Vec<(u32, u32, ASTNode)> = items.into_iter().collect();
13641 let edited_cells: Vec<(u32, u32)> = collected.iter().map(|(r, c, _)| (*r, *c)).collect();
13642 let sheet_id = self.graph.sheet_id_mut(sheet);
13643 let ingested = {
13644 let mut pipeline = self.ingest_pipeline();
13645 let inputs = collected.into_iter().map(|(row, col, ast)| {
13646 let placement = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13647 (FormulaAstInput::Tree(ast), placement, None)
13648 });
13649 pipeline.ingest_batch(inputs)?
13650 };
13651 let planned: Vec<(u32, u32, AstNodeId, DependencyPlanRow)> = ingested
13652 .into_iter()
13653 .map(|formula| {
13654 (
13655 formula.placement.coord.row() + 1,
13656 formula.placement.coord.col() + 1,
13657 formula.ast_id,
13658 formula.dep_plan,
13659 )
13660 })
13661 .collect();
13662 let n = self.graph.bulk_set_formulas_with_plans(sheet, planned)?;
13663 for (row, col) in edited_cells {
13664 let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13665 self.clear_cell_format_state(sheet, cell);
13666 self.record_changed_cell(sheet, row, col);
13667 }
13668 if n > 0 {
13670 self.mark_topology_edited();
13671 }
13672 Ok(n)
13673 }
13674
13675 pub fn declare_dynamic_array_anchor(
13704 &mut self,
13705 sheet: &str,
13706 row: u32,
13707 col: u32,
13708 ) -> Result<(), ExcelError> {
13709 let not_a_formula = || {
13710 ExcelError::new(ExcelErrorKind::Ref)
13711 .with_message("Declared dynamic array anchor must be a formula cell")
13712 };
13713 if row == 0 || col == 0 {
13714 return Err(not_a_formula());
13715 }
13716 if self.get_staged_formula_text(sheet, row, col).is_some() {
13717 return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
13718 "Declared dynamic array anchor is still staged; build the graph first",
13719 ));
13720 }
13721 let sheet_id = self.graph.sheet_id(sheet).ok_or_else(not_a_formula)?;
13722 let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13723 let vertex = self
13724 .graph
13725 .cell_vertex_mut(&cell)
13726 .filter(|v| self.graph.vertex_has_formula(*v))
13727 .ok_or_else(not_a_formula)?;
13728 if self.graph.declare_dynamic_anchor(vertex, cell) {
13729 self.graph.mark_dirty_many(&[vertex]);
13730 self.mark_topology_edited();
13731 }
13732 Ok(())
13733 }
13734
13735 pub fn declare_fixed_array_formula(
13741 &mut self,
13742 sheet: &str,
13743 row: u32,
13744 col: u32,
13745 rows: u32,
13746 cols: u32,
13747 ) -> Result<(), ExcelError> {
13748 if self.config.family_execution {
13749 return Err(ExcelError::new(ExcelErrorKind::Ref)
13750 .with_message("Fixed arrays require family_execution = false"));
13751 }
13752 if row == 0
13753 || col == 0
13754 || rows == 0
13755 || cols == 0
13756 || row.checked_add(rows - 1).is_none_or(|r| r > 1_048_576)
13757 || col.checked_add(cols - 1).is_none_or(|c| c > 16_384)
13758 || u64::from(rows) * u64::from(cols) > u64::from(self.config.spill.max_spill_cells)
13759 {
13760 return Err(ExcelError::new(ExcelErrorKind::Ref)
13761 .with_message("Invalid or over-cap fixed array extent"));
13762 }
13763 if self.get_staged_formula_text(sheet, row, col).is_some() {
13764 return Err(ExcelError::new(ExcelErrorKind::Ref)
13765 .with_message("Fixed array formula is still staged; build the graph first"));
13766 }
13767 let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
13768 ExcelError::new(ExcelErrorKind::Ref).with_message("Unknown fixed array sheet")
13769 })?;
13770 let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13771 let vertex = self
13772 .graph
13773 .cell_vertex_mut(&cell)
13774 .filter(|v| self.graph.vertex_has_formula(*v))
13775 .ok_or_else(|| {
13776 ExcelError::new(ExcelErrorKind::Ref)
13777 .with_message("Fixed array anchor must be a formula cell")
13778 })?;
13779 if (rows == 1 && cols == 1 && self.graph.fixed_single_arrays.contains(&vertex))
13780 || self.graph.fixed_array_shapes.get(&vertex) == Some(&(rows, cols))
13781 {
13782 return Ok(());
13783 }
13784 if self.graph.fixed_single_arrays.contains(&vertex)
13785 || self.graph.spill_registry_has_anchor(vertex)
13786 || self.graph.is_current_declared_dynamic_anchor(vertex)
13787 {
13788 return Err(ExcelError::new(ExcelErrorKind::Ref)
13789 .with_message("Array declaration already exists; replace the formula first"));
13790 }
13791 if rows == 1 && cols == 1 {
13792 self.graph.fixed_single_arrays.insert(vertex);
13793 } else {
13794 let mut targets = Vec::with_capacity((rows * cols) as usize);
13795 for r in 0..rows {
13796 for c in 0..cols {
13797 targets.push(self.graph.make_cell_ref(sheet, row + r, col + c));
13798 }
13799 }
13800 self.graph.plan_spill_region(vertex, &targets)?;
13801 self.graph
13803 .commit_spill_region_atomic_with_fault(vertex, targets, vec![], None)?;
13804 self.graph.fixed_array_shapes.insert(vertex, (rows, cols));
13805 }
13806 self.graph.mark_dirty_many(&[vertex]);
13807 self.mark_topology_edited();
13808 Ok(())
13809 }
13810
13811 #[inline]
13812 fn normalize_public_cell_read(v: LiteralValue) -> Option<LiteralValue> {
13813 match v {
13814 LiteralValue::Empty => None,
13815 LiteralValue::Int(i) => Some(LiteralValue::Number(i as f64)),
13816 other => Some(other),
13817 }
13818 }
13819
13820 fn materialize_temporal_egress(
13821 value: LiteralValue,
13822 class: Option<&formualizer_common::numfmt::FormatClass>,
13823 policy: crate::engine::TemporalEgress,
13824 date_system: crate::engine::DateSystem,
13825 ) -> LiteralValue {
13826 use formualizer_common::numfmt::FormatClass;
13827 if policy == crate::engine::TemporalEgress::Serial {
13828 return value;
13829 }
13830 let LiteralValue::Number(serial) = value else {
13831 return value;
13832 };
13833 match class {
13834 Some(FormatClass::Date) if serial.fract() != 0.0 => {
13835 formualizer_common::try_serial_to_datetime_for(date_system, serial)
13836 .map(LiteralValue::DateTime)
13837 .unwrap_or(LiteralValue::Number(serial))
13838 }
13839 Some(FormatClass::Date) => {
13840 formualizer_common::try_serial_to_date_for(date_system, serial)
13841 .map(LiteralValue::Date)
13842 .unwrap_or(LiteralValue::Number(serial))
13843 }
13844 Some(FormatClass::DateTime) => {
13845 formualizer_common::try_serial_to_datetime_for(date_system, serial)
13846 .map(LiteralValue::DateTime)
13847 .unwrap_or(LiteralValue::Number(serial))
13848 }
13849 Some(FormatClass::Time) if (0.0..1.0).contains(&serial) => {
13850 let nanos = (serial * 86_400_000_000_000.0).round() as u64;
13851 let nanos = nanos.min(86_400_000_000_000 - 1);
13852 chrono::NaiveTime::from_num_seconds_from_midnight_opt(
13853 (nanos / 1_000_000_000) as u32,
13854 (nanos % 1_000_000_000) as u32,
13855 )
13856 .map(LiteralValue::Time)
13857 .unwrap_or(LiteralValue::Number(serial))
13858 }
13859 Some(FormatClass::Time | FormatClass::Duration) => {
13860 let nanos = (serial * 86_400.0 * 1_000_000_000.0).round();
13861 if nanos.is_finite() && nanos >= i64::MIN as f64 && nanos <= i64::MAX as f64 {
13862 LiteralValue::Duration(chrono::Duration::nanoseconds(nanos as i64))
13863 } else {
13864 LiteralValue::Number(serial)
13865 }
13866 }
13867 _ => LiteralValue::Number(serial),
13868 }
13869 }
13870
13871 pub(crate) fn effective_format_id(
13872 &self,
13873 sheet: &str,
13874 row: u32,
13875 col: u32,
13876 ) -> Option<crate::format::FormatId> {
13877 let arrow = self.arrow_sheets.sheet(sheet).and_then(|arrow| {
13878 arrow.format_id(
13879 row.saturating_sub(1) as usize,
13880 col.saturating_sub(1) as usize,
13881 )
13882 });
13883 arrow.or_else(|| {
13884 let sheet_id = self.graph.sheet_id(sheet)?;
13885 let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13886 self.derived_formats.get(&cell)
13887 })
13888 }
13889
13890 pub fn get_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
13892 let raw = self.read_cell_value(sheet, row, col)?;
13893 let format = self.effective_format_id(sheet, row, col);
13894 let class = format.and_then(|id| self.format_registry.class(id));
13895 Self::normalize_public_cell_read(Self::materialize_temporal_egress(
13896 raw,
13897 class,
13898 self.config.temporal_egress,
13899 self.config.date_system,
13900 ))
13901 }
13902
13903 pub fn get_range_values(
13905 &self,
13906 sheet: &str,
13907 sr: u32,
13908 sc: u32,
13909 er: u32,
13910 ec: u32,
13911 ) -> Vec<Vec<LiteralValue>> {
13912 let height = er.saturating_sub(sr).saturating_add(1) as usize;
13913 let width = ec.saturating_sub(sc).saturating_add(1) as usize;
13914 let Some(asheet) = self.sheet_store().sheet(sheet) else {
13915 return vec![vec![LiteralValue::Empty; width]; height];
13916 };
13917 let view = asheet.range_view(
13918 sr.saturating_sub(1) as usize,
13919 sc.saturating_sub(1) as usize,
13920 er.saturating_sub(1) as usize,
13921 ec.saturating_sub(1) as usize,
13922 );
13923 let sheet_id = self.graph.sheet_id(sheet);
13924 let derived_formats = &self.derived_formats;
13925 let has_derived_formats =
13926 sheet_id.is_some_and(|sheet_id| derived_formats.any(|cell| cell.sheet_id == sheet_id));
13927 let mut out = Vec::with_capacity(height);
13928 if !asheet.has_formats() && !has_derived_formats {
13929 for rr in 0..height {
13930 let mut row = Vec::with_capacity(width);
13931 for cc in 0..width {
13932 row.push(view.get_cell(rr, cc));
13933 }
13934 out.push(row);
13935 }
13936 return out;
13937 }
13938 let format_registry = &self.format_registry;
13939 for rr in 0..height {
13940 let mut row = Vec::with_capacity(width);
13941 for cc in 0..width {
13942 let raw = view.get_cell(rr, cc);
13943 let row0 = sr.saturating_sub(1).saturating_add(rr as u32);
13944 let col0 = sc.saturating_sub(1).saturating_add(cc as u32);
13945 let format = asheet.format_id(row0 as usize, col0 as usize).or_else(|| {
13946 let cell = CellRef::new(sheet_id?, Coord::new(row0, col0, true, true));
13947 derived_formats.get(&cell)
13948 });
13949 let class = format.and_then(|id| format_registry.class(id));
13950 row.push(Self::materialize_temporal_egress(
13951 raw,
13952 class,
13953 self.config.temporal_egress,
13954 self.config.date_system,
13955 ));
13956 }
13957 out.push(row);
13958 }
13959 out
13960 }
13961
13962 fn declared_anchor_holds_scalar(&self, anchor: CellRef) -> bool {
13968 let sheet = self.graph.sheet_name(anchor.sheet_id);
13969 if sheet.is_empty() {
13970 return false;
13971 }
13972 matches!(
13973 self.read_cell_value(sheet, anchor.coord.row() + 1, anchor.coord.col() + 1),
13974 Some(value) if !matches!(
13975 value,
13976 LiteralValue::Error(_)
13977 | LiteralValue::Array(_)
13978 | LiteralValue::Pending
13979 | LiteralValue::Empty
13980 )
13981 )
13982 }
13983
13984 pub(crate) fn read_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
13985 let asheet = self.sheet_store().sheet(sheet)?;
13986 let r0 = row.saturating_sub(1) as usize;
13987 let c0 = col.saturating_sub(1) as usize;
13988 let v = asheet.get_cell_value(r0, c0);
13989 if matches!(v, LiteralValue::Empty) {
13990 None
13991 } else {
13992 Some(v)
13993 }
13994 }
13995
13996 pub(crate) fn read_range_values(
13998 &self,
13999 sheet: &str,
14000 sr: u32,
14001 sc: u32,
14002 er: u32,
14003 ec: u32,
14004 ) -> RangeView<'_> {
14005 let Some(asheet) = self.sheet_store().sheet(sheet) else {
14006 return RangeView::from_owned_rows(Vec::new(), self.config.date_system);
14007 };
14008 if er < sr || ec < sc {
14009 return asheet.range_view(1, 1, 0, 0);
14010 }
14011 let sr0 = sr.saturating_sub(1) as usize;
14012 let sc0 = sc.saturating_sub(1) as usize;
14013 let er0 = er.saturating_sub(1) as usize;
14014 let ec0 = ec.saturating_sub(1) as usize;
14015 asheet.range_view(sr0, sc0, er0, ec0)
14016 }
14017
14018 pub fn get_cell(
14020 &self,
14021 sheet: &str,
14022 row: u32,
14023 col: u32,
14024 ) -> Option<(Option<formualizer_parse::ASTNode>, Option<LiteralValue>)> {
14025 let v = self.get_cell_value(sheet, row, col);
14026 let sheet_id = self.graph.sheet_id(sheet)?;
14027 let coord = Coord::from_excel(row, col, true, true);
14028 let cell = CellRef::new(sheet_id, coord);
14029 if let Some(vid) = self.graph.get_vertex_for_cell(&cell) {
14030 let ast = self.graph.get_formula(vid);
14031 Some((ast, v))
14032 } else if v.is_some() || self.graph.had_legacy_cell_vertex(&cell) {
14033 Some((None, v))
14037 } else {
14038 None
14039 }
14040 }
14041
14042 pub fn begin_batch(&mut self) {
14044 self.graph.begin_batch();
14045 }
14046
14047 pub fn end_batch(&mut self) {
14049 self.graph.end_batch();
14050 }
14051
14052 pub fn begin_deferred_dirty(&mut self) {
14061 self.graph.begin_deferred_dirty();
14062 }
14063
14064 pub fn end_deferred_dirty(&mut self) {
14067 let _ = self.graph.end_deferred_dirty();
14068 }
14069
14070 pub fn dirty_propagation_visits(&self) -> u64 {
14074 self.graph.dirty_propagation_visits()
14075 }
14076
14077 #[inline]
14080 fn record_cell_if_changed(
14081 delta: &mut DeltaCollector,
14082 cell: &CellRef,
14083 old: &LiteralValue,
14084 new: &LiteralValue,
14085 ) {
14086 if old != new {
14087 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
14088 }
14089 }
14090
14091 pub fn evaluate_vertex(&mut self, vertex_id: VertexId) -> Result<LiteralValue, ExcelError> {
14092 self.observe_evaluation_resource_request(EvaluationRequestKind::Vertex, |engine| {
14093 engine.observe_function_semantic_epoch()?;
14094 engine.resource_checkpoint(1)?;
14096 if !engine.graph.vertex_exists(vertex_id) {
14097 return engine.evaluate_vertex_impl(vertex_id, None);
14098 }
14099 let is_formula = matches!(
14100 engine.graph.get_vertex_kind(vertex_id),
14101 VertexKind::FormulaScalar | VertexKind::FormulaArray
14102 );
14103 if is_formula {
14104 engine.begin_evaluation_request();
14105 #[cfg(any(test, feature = "legacy_oracle"))]
14106 engine.graph.flush_pending_edge_deltas();
14107 let roots = [crate::engine::target_preparation::TargetProducer::Legacy(
14108 vertex_id,
14109 )];
14110 engine.evaluate_legacy_target_roots(&roots, None)?;
14111 }
14112 if !is_formula {
14113 engine.refresh_resolution_snapshots();
14114 }
14115 engine.evaluate_vertex_impl(vertex_id, None)
14116 })
14117 }
14118
14119 fn publish_oversized_spill(
14121 &mut self,
14122 vertex_id: VertexId,
14123 error: ExcelError,
14124 mut delta: Option<&mut DeltaCollector>,
14125 ) -> LiteralValue {
14126 self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
14127 let anchor = self
14128 .graph
14129 .get_cell_ref(vertex_id)
14130 .expect("cell ref for vertex");
14131 let spill_val = LiteralValue::Error(error);
14132 if let Some(d) = delta {
14133 let old = self
14134 .read_cell_value(
14135 self.graph.sheet_name(anchor.sheet_id),
14136 anchor.coord.row() + 1,
14137 anchor.coord.col() + 1,
14138 )
14139 .unwrap_or(LiteralValue::Empty);
14140 if old != spill_val {
14141 d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
14142 }
14143 }
14144 self.graph.update_vertex_value_ref(vertex_id, &spill_val);
14145 if self.config.arrow_storage_enabled
14146 && self.config.delta_overlay_enabled
14147 && self.config.write_formula_overlay_enabled
14148 {
14149 let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
14150 self.mirror_value_to_computed_overlay(
14151 &sheet_name,
14152 anchor.coord.row() + 1,
14153 anchor.coord.col() + 1,
14154 &spill_val,
14155 );
14156 }
14157 spill_val
14158 }
14159
14160 fn evaluate_vertex_impl(
14161 &mut self,
14162 vertex_id: VertexId,
14163 delta: Option<&mut DeltaCollector>,
14164 ) -> Result<LiteralValue, ExcelError> {
14165 if !self.graph.vertex_exists(vertex_id) {
14169 return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
14170 .with_message(format!("Vertex not found: {vertex_id:?}")));
14171 }
14172 if (self.active_resource_ledger.is_some()
14173 || self.graph.fixed_array_shapes.contains_key(&vertex_id))
14174 && matches!(
14175 self.graph.get_vertex_kind(vertex_id),
14176 VertexKind::FormulaScalar | VertexKind::FormulaArray
14177 )
14178 {
14179 let value = self
14180 .evaluate_vertex_immutable(vertex_id)
14181 .unwrap_or_else(LiteralValue::Error);
14182 let effects = self.plan_vertex_effects(vertex_id, value.clone())?;
14183 self.resource_checkpoint(0)?;
14185 let mut delta = delta;
14186 for effect in &effects {
14187 self.apply_effect_with_computed_writes(effect, delta.as_deref_mut(), None, None)?;
14188 }
14189 return Ok(value);
14190 }
14191
14192 let mut delta = delta;
14193
14194 let kind = self.graph.get_vertex_kind(vertex_id);
14196 let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
14197
14198 let view = match kind {
14199 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
14200 if let Some(view) = self.graph.formula_view(vertex_id) {
14201 view
14202 } else {
14203 return Ok(LiteralValue::Number(0.0));
14204 }
14205 }
14206 VertexKind::Empty | VertexKind::Cell => {
14207 if let Some(cell_ref) = self.graph.get_cell_ref(vertex_id) {
14208 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
14209 let row = cell_ref.coord.row() + 1;
14210 let col = cell_ref.coord.col() + 1;
14211 if let Some(v) = self.read_cell_value(sheet_name, row, col) {
14212 return Ok(v);
14213 }
14214 }
14215 return Ok(LiteralValue::Number(0.0));
14216 }
14217 VertexKind::NamedScalar => {
14218 let value = self.evaluate_named_scalar(vertex_id, sheet_id)?;
14219 return Ok(value);
14220 }
14221 VertexKind::NamedArray => {
14222 let value = self.evaluate_named_array(vertex_id, sheet_id)?;
14223 return Ok(value);
14224 }
14225 VertexKind::InfiniteRange
14226 | VertexKind::Range
14227 | VertexKind::External
14228 | VertexKind::Table => {
14229 return Ok(LiteralValue::Number(0.0));
14231 }
14232 };
14233
14234 let sheet_name = self.graph.sheet_name(sheet_id);
14236 let cell_ref = self
14237 .graph
14238 .get_cell_ref(vertex_id)
14239 .expect("cell ref for vertex");
14240 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref)
14241 .with_resolved_functions(&self.resolved_functions)
14242 .with_arena_cells(self);
14243
14244 let result = interpreter.evaluate_formula_view(
14245 view,
14246 self.graph.data_store(),
14247 self.graph.sheet_reg(),
14248 );
14249
14250 match result {
14252 Ok(cv) => {
14253 let derived_format = cv.format_id();
14254 self.record_derived_format(vertex_id, derived_format);
14255 let oversized_range = if matches!(cv, crate::traits::CalcValue::Range(_))
14256 && self.graph.fixed_single_arrays.contains(&vertex_id)
14257 {
14258 None
14259 } else {
14260 crate::engine::result_finalization::range_spill_error(
14261 &cv,
14262 self.config.spill.max_spill_cells,
14263 )
14264 };
14265 let is_oversized_range = oversized_range.is_some();
14266 let result_literal = if let Some(error) = oversized_range {
14267 drop(cv);
14268 LiteralValue::Error(error)
14269 } else {
14270 self.materialize_formula_result(vertex_id, cv)
14271 };
14272 let output_sheet_name = sheet_name.to_string();
14273 self.write_computed_overlay_format_0based(
14274 &output_sheet_name,
14275 cell_ref.coord.row(),
14276 cell_ref.coord.col(),
14277 derived_format,
14278 );
14279 if is_oversized_range {
14280 let LiteralValue::Error(error) = result_literal else {
14281 unreachable!()
14282 };
14283 self.graph.set_kind(vertex_id, VertexKind::FormulaArray);
14284 return Ok(self.publish_oversized_spill(
14285 vertex_id,
14286 error,
14287 delta.as_deref_mut(),
14288 ));
14289 }
14290 match result_literal {
14291 LiteralValue::Array(rows) => {
14292 self.graph
14294 .set_kind(vertex_id, crate::engine::vertex::VertexKind::FormulaArray);
14295 let anchor = self
14297 .graph
14298 .get_cell_ref(vertex_id)
14299 .expect("cell ref for vertex");
14300 let sheet_id = anchor.sheet_id;
14301 let h = rows.len() as u32;
14302 let w = rows.first().map(|r| r.len()).unwrap_or(0) as u32;
14303
14304 let spill_cells = (h as u64).saturating_mul(w as u64);
14306 if spill_cells > self.config.spill.max_spill_cells as u64 {
14307 let spill_err = ExcelError::new(ExcelErrorKind::Spill)
14308 .with_message("SpillTooLarge")
14309 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
14310 expected_rows: h,
14311 expected_cols: w,
14312 });
14313 return Ok(self.publish_oversized_spill(
14314 vertex_id,
14315 spill_err,
14316 delta.as_deref_mut(),
14317 ));
14318 }
14319 const PACKED_MAX_ROW: u32 = 1_048_575; const PACKED_MAX_COL: u32 = 16_383; let end_row = anchor.coord.row().saturating_add(h).saturating_sub(1);
14323 let end_col = anchor.coord.col().saturating_add(w).saturating_sub(1);
14324 if end_row > PACKED_MAX_ROW || end_col > PACKED_MAX_COL {
14325 self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
14326 let spill_err = ExcelError::new(ExcelErrorKind::Spill)
14327 .with_message("Spill exceeds sheet bounds")
14328 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
14329 expected_rows: h,
14330 expected_cols: w,
14331 });
14332 let spill_val = LiteralValue::Error(spill_err.clone());
14333 if let Some(d) = delta.as_deref_mut() {
14334 let old = self
14335 .read_cell_value(
14336 self.graph.sheet_name(anchor.sheet_id),
14337 anchor.coord.row() + 1,
14338 anchor.coord.col() + 1,
14339 )
14340 .unwrap_or(LiteralValue::Empty);
14341 if old != spill_val {
14342 d.record_cell(
14343 anchor.sheet_id,
14344 anchor.coord.row(),
14345 anchor.coord.col(),
14346 );
14347 }
14348 }
14349 self.graph.update_vertex_value_ref(vertex_id, &spill_val);
14350 if self.config.arrow_storage_enabled
14351 && self.config.delta_overlay_enabled
14352 && self.config.write_formula_overlay_enabled
14353 {
14354 let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
14355 self.mirror_value_to_computed_overlay(
14356 &sheet_name,
14357 anchor.coord.row() + 1,
14358 anchor.coord.col() + 1,
14359 &spill_val,
14360 );
14361 }
14362 return Ok(spill_val);
14363 }
14364 let mut targets = Vec::new();
14365 for r in 0..h {
14366 for c in 0..w {
14367 targets.push(self.graph.make_cell_ref_internal(
14368 sheet_id,
14369 anchor.coord.row() + r,
14370 anchor.coord.col() + c,
14371 ));
14372 }
14373 }
14374
14375 match self.spill_mgr.reserve(
14377 vertex_id,
14378 anchor,
14379 SpillShape { rows: h, cols: w },
14380 SpillMeta {
14381 epoch: self.recalc_epoch,
14382 config: self.config.spill,
14383 },
14384 ) {
14385 Ok(()) => {
14386 if let Err(e) = self.commit_spill_and_mirror(
14390 vertex_id,
14391 &targets,
14392 rows.clone(),
14393 delta.as_deref_mut(),
14394 ) {
14395 if e.kind != ExcelErrorKind::Spill {
14396 return Err(e);
14397 }
14398 self.clear_spill_projection_and_mirror(
14400 vertex_id,
14401 delta.as_deref_mut(),
14402 );
14403 if let Some(d) = delta.as_deref_mut() {
14404 let old = self
14405 .read_cell_value(
14406 self.graph.sheet_name(anchor.sheet_id),
14407 anchor.coord.row() + 1,
14408 anchor.coord.col() + 1,
14409 )
14410 .unwrap_or(LiteralValue::Empty);
14411 let new = LiteralValue::Error(e.clone());
14412 if old != new {
14413 d.record_cell(
14414 anchor.sheet_id,
14415 anchor.coord.row(),
14416 anchor.coord.col(),
14417 );
14418 }
14419 }
14420 let err_val = LiteralValue::Error(e.clone());
14421 self.graph.update_vertex_value_ref(vertex_id, &err_val);
14422 if self.config.arrow_storage_enabled
14423 && self.config.delta_overlay_enabled
14424 && self.config.write_formula_overlay_enabled
14425 {
14426 let sheet_name =
14427 self.graph.sheet_name(anchor.sheet_id).to_string();
14428 self.mirror_value_to_computed_overlay(
14429 &sheet_name,
14430 anchor.coord.row() + 1,
14431 anchor.coord.col() + 1,
14432 &err_val,
14433 );
14434 }
14435 return Ok(err_val);
14436 }
14437 let top_left = rows
14439 .first()
14440 .and_then(|r| r.first())
14441 .cloned()
14442 .unwrap_or(LiteralValue::Empty);
14443 self.graph.update_vertex_value_ref(vertex_id, &top_left);
14444 Ok(top_left)
14445 }
14446 Err(e) => {
14447 self.clear_spill_projection_and_mirror(
14448 vertex_id,
14449 delta.as_deref_mut(),
14450 );
14451 let spill_err = ExcelError::new(ExcelErrorKind::Spill)
14452 .with_message(e.message().unwrap_or("Spill blocked"))
14453 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
14454 expected_rows: h,
14455 expected_cols: w,
14456 });
14457 let spill_val = LiteralValue::Error(spill_err.clone());
14458 if let Some(d) = delta.as_deref_mut() {
14459 let old = self
14460 .read_cell_value(
14461 self.graph.sheet_name(anchor.sheet_id),
14462 anchor.coord.row() + 1,
14463 anchor.coord.col() + 1,
14464 )
14465 .unwrap_or(LiteralValue::Empty);
14466 if old != spill_val {
14467 d.record_cell(
14468 anchor.sheet_id,
14469 anchor.coord.row(),
14470 anchor.coord.col(),
14471 );
14472 }
14473 }
14474 self.graph.update_vertex_value_ref(vertex_id, &spill_val);
14475 if self.config.arrow_storage_enabled
14476 && self.config.delta_overlay_enabled
14477 && self.config.write_formula_overlay_enabled
14478 {
14479 let sheet_name =
14480 self.graph.sheet_name(anchor.sheet_id).to_string();
14481 self.mirror_value_to_computed_overlay(
14482 &sheet_name,
14483 anchor.coord.row() + 1,
14484 anchor.coord.col() + 1,
14485 &spill_val,
14486 );
14487 }
14488 Ok(spill_val)
14489 }
14490 }
14491 }
14492 other => {
14493 let spill_cells = self.graph.spill_cells_to_clear(vertex_id);
14495 if let Some(d) = delta.as_deref_mut()
14496 && let Some(anchor) = self.graph.get_cell_ref_for_vertex(vertex_id)
14497 {
14498 if spill_cells.is_empty() {
14499 let old = self
14500 .read_cell_value(
14501 self.graph.sheet_name(anchor.sheet_id),
14502 anchor.coord.row() + 1,
14503 anchor.coord.col() + 1,
14504 )
14505 .unwrap_or(LiteralValue::Empty);
14506 if old != other {
14507 d.record_cell(
14508 anchor.sheet_id,
14509 anchor.coord.row(),
14510 anchor.coord.col(),
14511 );
14512 }
14513 } else {
14514 for cell in spill_cells.iter() {
14515 let sheet_name = self.graph.sheet_name(cell.sheet_id);
14516 let old = self
14517 .read_cell_value(
14518 sheet_name,
14519 cell.coord.row() + 1,
14520 cell.coord.col() + 1,
14521 )
14522 .unwrap_or(LiteralValue::Empty);
14523 let new = if cell.sheet_id == anchor.sheet_id
14524 && cell.coord.row() == anchor.coord.row()
14525 && cell.coord.col() == anchor.coord.col()
14526 {
14527 other.clone()
14528 } else {
14529 LiteralValue::Empty
14530 };
14531 Self::record_cell_if_changed(d, cell, &old, &new);
14532 }
14533 }
14534 }
14535 self.graph.clear_spill_region(vertex_id);
14536 if let Some(scope) = Self::structural_scope_from_cells(&spill_cells) {
14537 self.record_structural_change(scope);
14538 }
14539 if self.config.arrow_storage_enabled
14540 && self.config.delta_overlay_enabled
14541 && self.config.write_formula_overlay_enabled
14542 {
14543 let empty = LiteralValue::Empty;
14544 for cell in spill_cells.iter() {
14545 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
14546 self.mirror_value_to_computed_overlay(
14547 &sheet_name,
14548 cell.coord.row() + 1,
14549 cell.coord.col() + 1,
14550 &empty,
14551 );
14552 }
14553 }
14554 self.graph.update_vertex_value_ref(vertex_id, &other);
14555 if self.config.arrow_storage_enabled
14557 && self.config.delta_overlay_enabled
14558 && self.config.write_formula_overlay_enabled
14559 {
14560 let anchor = self
14561 .graph
14562 .get_cell_ref(vertex_id)
14563 .expect("cell ref for vertex");
14564 let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
14565 self.mirror_value_to_computed_overlay(
14566 &sheet_name,
14567 anchor.coord.row() + 1,
14568 anchor.coord.col() + 1,
14569 &other,
14570 );
14571 }
14572 Ok(other)
14573 }
14574 }
14575 }
14576 Err(e) => {
14577 let spill_cells = self.graph.spill_cells_to_clear(vertex_id);
14580 let err_val = LiteralValue::Error(e.clone());
14581 if let Some(d) = delta
14582 && let Some(anchor) = self.graph.get_cell_ref_for_vertex(vertex_id)
14583 {
14584 if spill_cells.is_empty() {
14585 let old = self
14586 .read_cell_value(
14587 self.graph.sheet_name(anchor.sheet_id),
14588 anchor.coord.row() + 1,
14589 anchor.coord.col() + 1,
14590 )
14591 .unwrap_or(LiteralValue::Empty);
14592 if old != err_val {
14593 d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
14594 }
14595 } else {
14596 for cell in spill_cells.iter() {
14597 let sheet_name = self.graph.sheet_name(cell.sheet_id);
14598 let old = self
14599 .read_cell_value(
14600 sheet_name,
14601 cell.coord.row() + 1,
14602 cell.coord.col() + 1,
14603 )
14604 .unwrap_or(LiteralValue::Empty);
14605 let new = if cell.sheet_id == anchor.sheet_id
14606 && cell.coord.row() == anchor.coord.row()
14607 && cell.coord.col() == anchor.coord.col()
14608 {
14609 err_val.clone()
14610 } else {
14611 LiteralValue::Empty
14612 };
14613 Self::record_cell_if_changed(d, cell, &old, &new);
14614 }
14615 }
14616 }
14617 self.graph.clear_spill_region(vertex_id);
14618 if let Some(scope) = Self::structural_scope_from_cells(&spill_cells) {
14619 self.record_structural_change(scope);
14620 }
14621 if self.config.arrow_storage_enabled
14622 && self.config.delta_overlay_enabled
14623 && self.config.write_formula_overlay_enabled
14624 {
14625 let empty = LiteralValue::Empty;
14626 for cell in spill_cells.iter() {
14627 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
14628 self.mirror_value_to_computed_overlay(
14629 &sheet_name,
14630 cell.coord.row() + 1,
14631 cell.coord.col() + 1,
14632 &empty,
14633 );
14634 }
14635 }
14636 self.graph.update_vertex_value_ref(vertex_id, &err_val);
14637 if self.config.arrow_storage_enabled
14638 && self.config.delta_overlay_enabled
14639 && self.config.write_formula_overlay_enabled
14640 {
14641 let anchor = self
14642 .graph
14643 .get_cell_ref(vertex_id)
14644 .expect("cell ref for vertex");
14645 let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
14646 self.mirror_value_to_computed_overlay(
14647 &sheet_name,
14648 anchor.coord.row() + 1,
14649 anchor.coord.col() + 1,
14650 &err_val,
14651 );
14652 }
14653 Ok(err_val)
14654 }
14655 }
14656 }
14657
14658 fn evaluate_named_scalar(
14659 &mut self,
14660 vertex_id: VertexId,
14661 sheet_id: SheetId,
14662 ) -> Result<LiteralValue, ExcelError> {
14663 let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
14664 ExcelError::new(ExcelErrorKind::Name)
14665 .with_message("Named range metadata missing".to_string())
14666 })?;
14667
14668 match &named_range.definition {
14669 NamedDefinition::Cell(cell_ref) => {
14670 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
14671 let row = cell_ref.coord.row() + 1;
14672 let col = cell_ref.coord.col() + 1;
14673
14674 if let Some(dep_vertex) = self.graph.get_vertex_for_cell(cell_ref)
14675 && matches!(
14676 self.graph.get_vertex_kind(dep_vertex),
14677 VertexKind::FormulaScalar | VertexKind::FormulaArray
14678 )
14679 {
14680 let value = self.evaluate_vertex(dep_vertex)?;
14682 self.graph.update_vertex_value_ref(vertex_id, &value);
14683 Ok(value)
14684 } else {
14685 let value = self
14686 .read_cell_value(sheet_name, row, col)
14687 .unwrap_or(LiteralValue::Empty);
14688 self.graph.update_vertex_value_ref(vertex_id, &value);
14689 Ok(value)
14690 }
14691 }
14692 NamedDefinition::Literal(v) => {
14693 let out = v.clone();
14694 self.graph.update_vertex_value_ref(vertex_id, &out);
14695 Ok(out)
14696 }
14697 NamedDefinition::Formula { ast, .. } => {
14698 let context_sheet = match named_range.scope {
14699 NameScope::Sheet(id) => id,
14700 NameScope::Workbook => sheet_id,
14701 };
14702 let sheet_name = self.graph.sheet_name(context_sheet);
14703 let cell_ref = self
14704 .graph
14705 .get_cell_ref(vertex_id)
14706 .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
14707 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref)
14708 .with_resolved_functions(&self.resolved_functions)
14709 .with_arena_cells(self);
14710 match interpreter.evaluate_ast(ast) {
14711 Ok(cv) => {
14712 let value = cv.into_literal();
14713 match value {
14714 LiteralValue::Array(_) => {
14715 let err = ExcelError::new(ExcelErrorKind::NImpl)
14716 .with_message("Array result in scalar named range".to_string());
14717 let err_val = LiteralValue::Error(err.clone());
14718 self.graph.update_vertex_value_ref(vertex_id, &err_val);
14719 Ok(err_val)
14720 }
14721 other => {
14722 self.graph.update_vertex_value_ref(vertex_id, &other);
14723 Ok(other)
14724 }
14725 }
14726 }
14727 Err(err) => {
14728 let err_val = LiteralValue::Error(err.clone());
14729 self.graph.update_vertex_value_ref(vertex_id, &err_val);
14730 Ok(err_val)
14731 }
14732 }
14733 }
14734 NamedDefinition::Range(_) => Err(ExcelError::new(ExcelErrorKind::Value)
14735 .with_message("Range-valued name evaluated as scalar".to_string())),
14736 }
14737 }
14738
14739 fn evaluate_named_array(
14740 &mut self,
14741 vertex_id: VertexId,
14742 sheet_id: SheetId,
14743 ) -> Result<LiteralValue, ExcelError> {
14744 let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
14745 ExcelError::new(ExcelErrorKind::Name)
14746 .with_message("Named range metadata missing".to_string())
14747 })?;
14748
14749 let out = match &named_range.definition {
14750 NamedDefinition::Range(range_ref) => {
14751 if range_ref.start.sheet_id != range_ref.end.sheet_id {
14752 return Err(ExcelError::new(ExcelErrorKind::Ref)
14753 .with_message("Named range cannot span sheets".to_string()));
14754 }
14755
14756 let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
14757 let sr0 = range_ref.start.coord.row();
14758 let sc0 = range_ref.start.coord.col();
14759 let er0 = range_ref.end.coord.row();
14760 let ec0 = range_ref.end.coord.col();
14761 if sr0 > er0 || sc0 > ec0 {
14762 return Err(ExcelError::new(ExcelErrorKind::Ref)
14763 .with_message("Invalid named range bounds".to_string()));
14764 }
14765
14766 let h = (er0 - sr0 + 1) as usize;
14767 let w = (ec0 - sc0 + 1) as usize;
14768 let cell_count = (h as u64).saturating_mul(w as u64);
14769 if cell_count > self.config.spill.max_spill_cells as u64 {
14770 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
14771 "Named range too large to materialize as an array".to_string(),
14772 ));
14773 }
14774
14775 let mut rows = Vec::with_capacity(h);
14776 for r0 in sr0..=er0 {
14777 let mut row = Vec::with_capacity(w);
14778 for c0 in sc0..=ec0 {
14779 let v = self
14780 .read_cell_value(sheet_name, r0 + 1, c0 + 1)
14781 .unwrap_or(LiteralValue::Empty);
14782 row.push(v);
14783 }
14784 rows.push(row);
14785 }
14786 LiteralValue::Array(rows)
14787 }
14788 NamedDefinition::Cell(cell_ref) => {
14789 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
14790 let row = cell_ref.coord.row() + 1;
14791 let col = cell_ref.coord.col() + 1;
14792 let v = self
14793 .read_cell_value(sheet_name, row, col)
14794 .unwrap_or(LiteralValue::Empty);
14795 LiteralValue::Array(vec![vec![v]])
14796 }
14797 NamedDefinition::Literal(v) => LiteralValue::Array(vec![vec![v.clone()]]),
14798 NamedDefinition::Formula { ast, .. } => {
14799 let context_sheet = match named_range.scope {
14800 NameScope::Sheet(id) => id,
14801 NameScope::Workbook => sheet_id,
14802 };
14803 let sheet_name = self.graph.sheet_name(context_sheet);
14804 let cell_ref = self
14805 .graph
14806 .get_cell_ref(vertex_id)
14807 .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
14808 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref)
14809 .with_resolved_functions(&self.resolved_functions)
14810 .with_arena_cells(self);
14811 match interpreter.evaluate_ast(ast) {
14812 Ok(cv) => {
14813 let v = cv.into_literal();
14814 match v {
14815 LiteralValue::Array(_) => v,
14816 other => LiteralValue::Array(vec![vec![other]]),
14817 }
14818 }
14819 Err(err) => LiteralValue::Error(err),
14820 }
14821 }
14822 };
14823
14824 self.graph.update_vertex_value_ref(vertex_id, &out);
14825 Ok(out)
14826 }
14827
14828 fn replan_exhausted_error(&self, limit: usize, context: &str) -> ExcelError {
14829 crate::engine::ResourceLedgerError::Exhausted(
14830 formualizer_common::ResourceExhaustionDetail {
14831 reason: formualizer_common::ResourceExhaustionReason::WorkUnits,
14832 limit: limit as u64,
14833 observed: limit.saturating_add(1) as u64,
14834 request_id: self
14835 .active_evaluation_resource_request
14836 .as_ref()
14837 .map(|stats| stats.request_id),
14838 },
14839 )
14840 .into_excel_error()
14841 .with_message(format!("{context} did not converge after {limit} replans"))
14842 }
14843
14844 fn transient_target_preparation_stale(error: &ExcelError) -> bool {
14845 matches!(
14846 error.extra(),
14847 formualizer_common::ExcelErrorExtra::PreparationStale {
14848 reason: formualizer_common::PreparationStaleReason::Semantic
14849 | formualizer_common::PreparationStaleReason::Provider
14850 }
14851 )
14852 }
14853
14854 fn prepare_graph_for_routed_evaluation(
14855 &mut self,
14856 targets: &[crate::engine::EvaluationTarget],
14857 options: &crate::engine::TargetEvalOptions<'_>,
14858 ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
14859 const MAX_TRANSIENT_PREPARATION_RETRIES: usize = 2;
14860 let mut retries = 0usize;
14861 loop {
14862 match self.prepare_graph_for_targets_unobserved(targets, options) {
14863 Err(error)
14864 if Self::transient_target_preparation_stale(&error)
14865 && retries < MAX_TRANSIENT_PREPARATION_RETRIES =>
14866 {
14867 retries = retries.saturating_add(1);
14868 }
14869 result => return result,
14870 }
14871 }
14872 }
14873
14874 fn evaluate_mixed_targets(
14875 &mut self,
14876 targets: &[crate::engine::EvaluationTarget],
14877 delta: Option<&mut DeltaCollector>,
14878 ) -> Result<EvalResult, ExcelError> {
14879 let _source_cache = self.source_cache_session();
14880 let cancel = self.active_cancel_flag.clone();
14881 let options = crate::engine::TargetEvalOptions {
14882 request_id: self
14883 .active_evaluation_resource_request
14884 .as_ref()
14885 .map(|stats| stats.request_id),
14886 cancel,
14887 deadline: None,
14888 budgets: None,
14889 opaque_policy: crate::engine::OpaquePreparePolicy::Widen,
14890 };
14891 self.prepare_and_execute_target_recipe(targets, &options, delta)
14892 }
14893
14894 fn prepare_and_execute_target_recipe(
14895 &mut self,
14896 targets: &[crate::engine::EvaluationTarget],
14897 options: &crate::engine::TargetEvalOptions<'_>,
14898 delta: Option<&mut DeltaCollector>,
14899 ) -> Result<EvalResult, ExcelError> {
14900 let preparation = self.prepare_graph_for_routed_evaluation(targets, options)?;
14901 self.execute_prepared_target_recipe(targets, &preparation.widened_scope, delta)
14902 }
14903
14904 fn execute_prepared_target_recipe(
14905 &mut self,
14906 targets: &[crate::engine::EvaluationTarget],
14907 scope: &crate::engine::PrepareScope,
14908 delta: Option<&mut DeltaCollector>,
14909 ) -> Result<EvalResult, ExcelError> {
14910 self.require_unified_authority()?;
14911 if matches!(scope, crate::engine::PrepareScope::Workbook)
14912 && let Some(stats) = self.active_evaluation_resource_request.as_mut()
14913 {
14914 stats.workbook_exact_attempts = stats.workbook_exact_attempts.max(1);
14915 }
14916 let mut roots = self.resolve_target_producers(targets)?;
14917 if let crate::engine::PrepareScope::Sheets(sheets) = scope {
14918 let request_id = self
14919 .active_evaluation_resource_request
14920 .as_ref()
14921 .map(|request| request.request_id);
14922 let root_count = roots.len();
14923 let mut widened_roots =
14924 OrderedTargetProducers::from_ordered(std::mem::take(&mut roots))
14925 .map_err(|_| target_root_allocation_error(root_count, request_id))?;
14926 let sheet_ids = sheets
14927 .iter()
14928 .filter_map(|sheet| self.graph.sheet_id(sheet))
14929 .collect::<FxHashSet<_>>();
14930 for vertex in self.graph.formula_vertices() {
14931 if sheet_ids.contains(&self.graph.get_vertex_sheet_id(vertex)) {
14932 widened_roots
14933 .push(crate::engine::target_preparation::TargetProducer::Legacy(
14934 vertex,
14935 ))
14936 .map_err(|_| {
14937 target_root_allocation_error(widened_roots.len() + 1, request_id)
14938 })?;
14939 }
14940 }
14941 roots = widened_roots.into_vec();
14942 }
14943 self.begin_evaluation_request();
14944 #[cfg(any(test, feature = "legacy_oracle"))]
14945 self.graph.flush_pending_edge_deltas();
14946 let workbook_scope = matches!(scope, crate::engine::PrepareScope::Workbook);
14947 if workbook_scope {
14948 if let Some(delta) = delta {
14949 self.evaluate_all_with_delta_collector(delta)
14950 } else {
14951 self.evaluate_all_legacy_impl()
14952 }
14953 } else {
14954 self.evaluate_legacy_target_roots(&roots, delta)
14955 }
14956 }
14957
14958 fn legacy_coordinate_targets(
14959 &mut self,
14960 targets: &[(&str, u32, u32)],
14961 ) -> Vec<crate::engine::EvaluationTarget> {
14962 targets
14963 .iter()
14964 .map(|(sheet, row, col)| {
14965 self.graph.sheet_id_mut(sheet);
14968 crate::engine::EvaluationTarget::Cell {
14969 sheet: (*sheet).to_string(),
14970 row: *row,
14971 col: *col,
14972 }
14973 })
14974 .collect()
14975 }
14976
14977 pub fn evaluate_targets(
14979 &mut self,
14980 targets: &[crate::engine::EvaluationTarget],
14981 ) -> Result<EvalResult, ExcelError> {
14982 self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
14983 engine.observe_function_semantic_epoch()?;
14984 engine.validate_deterministic_mode()?;
14985 engine.evaluate_mixed_targets(targets, None)
14986 })
14987 }
14988
14989 pub fn evaluate_targets_with_options(
14991 &mut self,
14992 targets: &[crate::engine::EvaluationTarget],
14993 options: crate::engine::TargetEvalOptions<'_>,
14994 ) -> Result<EvalResult, ExcelError> {
14995 self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
14996 engine.active_cancel_flag = options.cancel.clone();
14997 engine.active_evaluation_deadline = options.deadline;
14998 let result = (|| {
14999 engine.cancellation_checkpoint("Evaluation cancelled before target preparation")?;
15000 engine.observe_function_semantic_epoch()?;
15001 engine.validate_deterministic_mode()?;
15002 let _source_cache = engine.source_cache_session();
15003 engine.prepare_and_execute_target_recipe(targets, &options, None)
15004 })();
15005 engine.active_cancel_flag = None;
15006 engine.active_evaluation_deadline = None;
15007 result
15008 })
15009 }
15010
15011 pub fn evaluate_targets_with_delta(
15013 &mut self,
15014 targets: &[crate::engine::EvaluationTarget],
15015 ) -> Result<(EvalResult, crate::engine::TargetEvalDelta), ExcelError> {
15016 self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
15017 engine.observe_function_semantic_epoch()?;
15018 engine.validate_deterministic_mode()?;
15019 let mut collector = DeltaCollector::new(DeltaMode::Cells);
15020 let result = engine.evaluate_mixed_targets(targets, Some(&mut collector))?;
15021 Ok((result, collector.finish_target()))
15022 })
15023 }
15024
15025 pub fn evaluate_until(
15027 &mut self,
15028 targets: &[(&str, u32, u32)],
15029 ) -> Result<EvalResult, ExcelError> {
15030 self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
15031 engine.evaluate_until_unobserved(targets)
15032 })
15033 }
15034
15035 fn evaluate_until_unobserved(
15036 &mut self,
15037 targets: &[(&str, u32, u32)],
15038 ) -> Result<EvalResult, ExcelError> {
15039 self.observe_function_semantic_epoch()?;
15040 let targets = self.legacy_coordinate_targets(targets);
15041 self.evaluate_mixed_targets(&targets, None)
15042 }
15043
15044 fn evaluate_until_with_delta_collector(
15045 &mut self,
15046 targets: &[(&str, u32, u32)],
15047 delta: &mut DeltaCollector,
15048 ) -> Result<EvalResult, ExcelError> {
15049 let targets = self.legacy_coordinate_targets(targets);
15050 self.evaluate_mixed_targets(&targets, Some(delta))
15051 }
15052
15053 fn evaluate_legacy_target_roots(
15054 &mut self,
15055 roots: &[crate::engine::target_preparation::TargetProducer],
15056 mut delta: Option<&mut DeltaCollector>,
15057 ) -> Result<EvalResult, ExcelError> {
15058 use crate::engine::target_preparation::TargetProducer;
15059 #[cfg(any(test, feature = "benchmark_internal"))]
15060 {
15061 self.recalc_reuse_probe
15062 .get_mut()
15063 .unwrap()
15064 .legacy_target_requests += 1;
15065 }
15066 let start = crate::instant::FzInstant::now();
15067 let root_vertices = roots
15068 .iter()
15069 .filter_map(|root| match root {
15070 TargetProducer::Legacy(vertex) | TargetProducer::Symbol(vertex) => Some(*vertex),
15071 TargetProducer::ValueOnly(_) => None,
15072 })
15073 .collect::<Vec<_>>();
15074 let mut computed_vertices = 0usize;
15075 let mut cycle_errors = 0usize;
15076 let mut replans = 0usize;
15077 const MAX_REPLAN: usize = 5;
15078 self.graph.authority_sync();
15079 loop {
15080 let (precedents_to_eval, old_vdeps) = self.demand_subgraph(&root_vertices)?;
15081 if precedents_to_eval.is_empty() {
15082 break;
15083 }
15084 #[cfg(any(test, feature = "benchmark_internal"))]
15085 {
15086 self.recalc_reuse_probe
15087 .get_mut()
15088 .unwrap()
15089 .target_schedule_builds += 1;
15090 }
15091 let schedule = {
15092 self.graph.authority_sync();
15093 let mut ledger = self.active_resource_ledger.take();
15094 let result = self.create_authority_schedule(
15095 &precedents_to_eval,
15096 &old_vdeps,
15097 ledger.as_mut(),
15098 );
15099 self.active_resource_ledger = ledger;
15100 result?
15101 };
15102 self.begin_pass(&schedule);
15103 for (unit_index, &unit) in schedule.units.iter().enumerate() {
15104 self.cancellation_checkpoint("Evaluation cancelled before target schedule unit")?;
15105 match unit {
15106 ScheduleUnit::Cycle(index) => {
15107 if self.handle_cycle_unit(
15108 schedule.unit_cycle(index),
15109 delta.as_deref_mut(),
15110 None,
15111 None,
15112 )? > 0
15113 {
15114 cycle_errors = cycle_errors.saturating_add(1);
15115 }
15116 }
15117 ScheduleUnit::Layer(index) => {
15118 let layer = schedule.unit_layer(index);
15119 let evaluated = if let Some(delta) = delta.as_deref_mut() {
15120 if self.thread_pool.is_some() && layer.vertices.len() > 1 {
15121 self.evaluate_layer_parallel_with_delta(layer, delta)?
15122 } else {
15123 self.evaluate_layer_sequential_with_delta(layer, delta)?
15124 }
15125 } else if self.thread_pool.is_some() && layer.vertices.len() > 1 {
15126 self.evaluate_layer_parallel(layer)?
15127 } else {
15128 self.evaluate_layer_sequential(layer)?
15129 };
15130 computed_vertices = computed_vertices.saturating_add(evaluated);
15131 }
15132 }
15133 if self.stop_after_unit(&schedule, unit_index) {
15134 break;
15135 }
15136 }
15137 let changed = self.changed_virtual_dep_vertices(&precedents_to_eval, &old_vdeps);
15138 self.resource_checkpoint(0)?;
15139 if !self.finish_target_pass_dirty(&precedents_to_eval, &changed) {
15140 break;
15141 }
15142 if replans >= MAX_REPLAN {
15143 return Err(self.replan_exhausted_error(
15144 MAX_REPLAN,
15145 "targeted legacy dynamic dependency evaluation",
15146 ));
15147 }
15148 replans = replans.saturating_add(1);
15149 }
15150 self.redirty_for_next_recalc();
15151 Ok(EvalResult {
15152 computed_vertices,
15153 cycle_errors,
15154 elapsed: start.elapsed(),
15155 })
15156 }
15157
15158 pub fn build_recalc_plan(&self) -> Result<RecalcPlan, ExcelError> {
15160 if self.has_staged_formulas() || self.staged_formula_index.has_packages() {
15161 return Err(
15162 Self::plan_stale(formualizer_common::PlanStaleReason::Staged).with_message(
15163 "compatibility recalculation plans require all staged formulas to be prepared",
15164 ),
15165 );
15166 }
15167 let key = self.recalc_plan_key();
15168 let mut vertices: Vec<VertexId> = self.graph.vertices_with_formulas().collect();
15169 vertices.sort_unstable();
15170 let has_dynamic_refs = vertices.iter().copied().any(|v| self.graph.is_dynamic(v));
15171 let schedule = if vertices.is_empty() {
15172 crate::engine::Schedule {
15173 units: Vec::new(),
15174 layers: Vec::new(),
15175 cycles: Vec::new(),
15176 }
15177 } else {
15178 self.create_evaluation_schedule_uncached(&vertices, None)?.0
15179 };
15180 self.validate_recalc_plan_key(&key)?;
15181 Ok(RecalcPlan {
15182 key,
15183 kind: RecalcPlanKind::CompatibilityFull {
15184 schedule,
15185 has_dynamic_refs,
15186 },
15187 })
15188 }
15189
15190 pub fn build_recalc_plan_for_targets(
15192 &mut self,
15193 targets: &[crate::engine::EvaluationTarget],
15194 ) -> Result<RecalcPlan, ExcelError> {
15195 self.build_recalc_plan_for_targets_with_options(
15196 targets,
15197 crate::engine::TargetEvalOptions::default(),
15198 )
15199 }
15200
15201 pub fn build_recalc_plan_for_targets_with_options(
15202 &mut self,
15203 targets: &[crate::engine::EvaluationTarget],
15204 options: crate::engine::TargetEvalOptions<'_>,
15205 ) -> Result<RecalcPlan, ExcelError> {
15206 self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
15207 engine.observe_function_semantic_epoch()?;
15208 engine.validate_deterministic_mode()?;
15209 let _source_cache = engine.source_cache_session();
15210 let preparation = engine.prepare_graph_for_routed_evaluation(targets, &options)?;
15211 #[cfg(any(test, feature = "legacy_oracle"))]
15212 engine.graph.flush_pending_edge_deltas();
15213 let topology = if matches!(
15214 preparation.widened_scope,
15215 crate::engine::PrepareScope::Workbook
15216 ) {
15217 RecalcTopology::Workbook
15218 } else {
15219 RecalcTopology::RunLocalRecipe
15220 };
15221 Ok(RecalcPlan {
15222 key: engine.recalc_plan_key(),
15223 kind: RecalcPlanKind::Target {
15224 targets: targets.to_vec(),
15225 scope: preparation.widened_scope,
15226 topology,
15227 dynamic_policy: DynamicPlanPolicy::BoundedTargetReplan,
15228 },
15229 })
15230 })
15231 }
15232
15233 pub fn evaluate_recalc_plan(&mut self, plan: &RecalcPlan) -> Result<EvalResult, ExcelError> {
15235 self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
15236 engine.evaluate_recalc_plan_unobserved(plan)
15237 })
15238 }
15239
15240 pub fn evaluate_recalc_plan_with_controls(
15241 &mut self,
15242 plan: &RecalcPlan,
15243 cancel: Option<crate::engine::CancelToken>,
15244 deadline: Option<Instant>,
15245 ) -> Result<EvalResult, ExcelError> {
15246 self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
15247 engine.active_cancel_flag = cancel.clone();
15248 engine.active_evaluation_deadline = deadline;
15249 let result = engine.evaluate_recalc_plan_unobserved(plan);
15250 engine.active_cancel_flag = None;
15251 engine.active_evaluation_deadline = None;
15252 result
15253 })
15254 }
15255
15256 fn evaluate_recalc_plan_unobserved(
15257 &mut self,
15258 plan: &RecalcPlan,
15259 ) -> Result<EvalResult, ExcelError> {
15260 #[cfg(any(test, feature = "legacy_oracle"))]
15261 self.graph.flush_pending_edge_deltas();
15262 self.validate_recalc_plan_key(&plan.key)?;
15263 self.cancellation_checkpoint("Evaluation cancelled before recalculation plan execution")?;
15264 self.validate_deterministic_mode()?;
15265
15266 match &plan.kind {
15267 RecalcPlanKind::Target {
15268 targets,
15269 scope,
15270 topology,
15271 dynamic_policy,
15272 } => {
15273 debug_assert_eq!(*dynamic_policy, DynamicPlanPolicy::BoundedTargetReplan);
15274 debug_assert_eq!(
15275 matches!(topology, RecalcTopology::Workbook),
15276 matches!(scope, crate::engine::PrepareScope::Workbook)
15277 );
15278 let _source_cache = self.source_cache_session();
15279 self.execute_prepared_target_recipe(targets, scope, None)
15280 }
15281 RecalcPlanKind::CompatibilityFull {
15282 schedule,
15283 has_dynamic_refs,
15284 } => {
15285 let _source_cache = self.source_cache_session();
15286 self.begin_evaluation_request();
15287 if *has_dynamic_refs {
15288 self.virtual_dep_fallback_activations =
15289 self.virtual_dep_fallback_activations.saturating_add(1);
15290 return self.evaluate_all_coordinator();
15291 }
15292
15293 let start = crate::instant::FzInstant::now();
15294 let dirty_vertices = self.graph.get_evaluation_vertices();
15295 if dirty_vertices.is_empty() {
15296 return Ok(EvalResult {
15297 computed_vertices: 0,
15298 cycle_errors: 0,
15299 elapsed: start.elapsed(),
15300 });
15301 }
15302
15303 let dirty_set: FxHashSet<VertexId> = dirty_vertices.iter().copied().collect();
15304 let mut computed_vertices = 0;
15305 let mut cycle_errors = 0;
15306 for &unit in &schedule.units {
15307 self.cancellation_checkpoint(
15308 "Evaluation cancelled before recalculation plan schedule unit",
15309 )?;
15310 match unit {
15311 ScheduleUnit::Cycle(i) => {
15312 let stamped = self.handle_cycle_unit(
15313 schedule.unit_cycle(i),
15314 None,
15315 Some(&dirty_set),
15316 None,
15317 )?;
15318 if stamped > 0 {
15319 cycle_errors += 1;
15320 }
15321 }
15322 ScheduleUnit::Layer(i) => {
15323 let work: Vec<VertexId> = schedule
15324 .unit_layer(i)
15325 .vertices
15326 .iter()
15327 .copied()
15328 .filter(|v| dirty_set.contains(v))
15329 .collect();
15330 if work.is_empty() {
15331 continue;
15332 }
15333 let temp_layer = crate::engine::scheduler::Layer::new(work);
15334 if self.thread_pool.is_some() && temp_layer.vertices.len() > 1 {
15335 computed_vertices += self.evaluate_layer_parallel(&temp_layer)?;
15336 } else {
15337 computed_vertices += self.evaluate_layer_sequential(&temp_layer)?;
15338 }
15339 }
15340 }
15341 }
15342
15343 self.resource_checkpoint(0)?;
15344 self.graph.clear_dirty_flags(&dirty_vertices);
15345 self.redirty_for_next_recalc();
15346 Ok(EvalResult {
15347 computed_vertices,
15348 cycle_errors,
15349 elapsed: start.elapsed(),
15350 })
15351 }
15352 }
15353 }
15354}
15355
15356impl<R> Engine<R>
15357where
15358 R: EvaluationContext,
15359{
15360 fn require_unified_authority(&mut self) -> Result<(), ExcelError> {
15365 self.graph
15366 .authority()
15367 .map(|_| ())
15368 .map_err(Self::authority_excel_error)
15369 }
15370
15371 fn authority_excel_error(error: crate::engine::authority::store::AuthorityError) -> ExcelError {
15372 use crate::engine::authority::store::AuthorityError;
15373 #[cfg(test)]
15377 if std::env::var_os("FZ_AUTHORITY_DEFERRED_TRACE").is_some() {
15378 eprintln!("M1B_AUTHORITY_ERROR {error:?}");
15379 }
15380 let kind = match error {
15381 AuthorityError::Unsupported { .. } => ExcelErrorKind::NImpl,
15382 _ => ExcelErrorKind::Error,
15383 };
15384 ExcelError::new(kind).with_message(format!("unified_authority: {error:?}"))
15385 }
15386
15387 pub fn evaluate_all_for_snapshot(
15401 &mut self,
15402 cancel: Option<crate::engine::CancelToken>,
15403 ) -> Result<EvalResult, ExcelError> {
15404 struct SnapshotScope<'e, R: EvaluationContext>(&'e mut Engine<R>);
15408 impl<R: EvaluationContext> Drop for SnapshotScope<'_, R> {
15409 fn drop(&mut self) {
15410 self.0.snapshot_evaluation_active = false;
15411 if std::thread::panicking() {
15412 self.0.snapshot_volatile_redirty_pending = true;
15413 }
15414 }
15415 }
15416 self.snapshot_evaluation_active = true;
15417 let result = {
15418 let scope = SnapshotScope(self);
15419 match cancel {
15420 Some(cancel) => scope.0.evaluate_all_cancellable(cancel),
15421 None => scope.0.evaluate_all(),
15422 }
15423 };
15424 if result.is_err() {
15425 self.snapshot_volatile_redirty_pending = false;
15426 self.graph.redirty_volatiles();
15427 }
15428 result
15429 }
15430
15431 pub fn evaluate_all(&mut self) -> Result<EvalResult, ExcelError> {
15433 self.observe_evaluation_resource_request(EvaluationRequestKind::Full, |engine| {
15435 engine.evaluate_all_unobserved()
15436 })
15437 }
15438
15439 fn evaluate_all_unobserved(&mut self) -> Result<EvalResult, ExcelError> {
15440 debug_assert!(
15441 !self.graph.deferred_dirty_active(),
15442 "deferred-dirty scope leaked into evaluate_all: a begin_deferred_dirty \
15443 was not balanced by end_deferred_dirty"
15444 );
15445 self.observe_function_semantic_epoch()?;
15446 self.lookup_index_cache.reset_counters();
15447 let _source_cache = self.source_cache_session();
15448 self.validate_deterministic_mode()?;
15449 if self.config.defer_graph_building {
15450 self.build_graph_all()?;
15452 }
15453 self.evaluate_all_coordinator()
15454 }
15455
15456 fn evaluate_all_coordinator(&mut self) -> Result<EvalResult, ExcelError> {
15459 self.require_unified_authority()?;
15460 self.begin_evaluation_request();
15461 self.evaluate_all_legacy_impl()
15462 }
15463
15464 fn legacy_pass_run_units(
15470 &mut self,
15471 schedule: &crate::engine::scheduler::Schedule,
15472 ) -> Result<(usize, usize), ExcelError> {
15473 let mut computed_vertices = 0;
15474 let mut cycle_count = 0;
15475 self.begin_pass(schedule);
15476 for (unit_index, &unit) in schedule.units.iter().enumerate() {
15477 match unit {
15478 ScheduleUnit::Cycle(i) => {
15479 if self.handle_cycle_unit(schedule.unit_cycle(i), None, None, None)? > 0 {
15480 cycle_count += 1;
15481 }
15482 }
15483 ScheduleUnit::Layer(i) => {
15484 let layer = schedule.unit_layer(i);
15485 if self.thread_pool.is_some() && layer.vertices.len() > 1 {
15486 computed_vertices += self.evaluate_layer_parallel(layer)?;
15487 } else {
15488 computed_vertices += self.evaluate_layer_sequential(layer)?;
15489 }
15490 }
15491 }
15492 if self.stop_after_unit(schedule, unit_index) {
15493 break;
15494 }
15495 }
15496 Ok((computed_vertices, cycle_count))
15497 }
15498
15499 fn evaluate_all_legacy_impl(&mut self) -> Result<EvalResult, ExcelError> {
15506 self.reset_virtual_dep_telemetry_if_disabled();
15507 let _span_eval =
15508 crate::engine::trace::fz_span!(tracing::Level::INFO, "evaluate", "evaluate.legacy");
15509 let start = crate::instant::FzInstant::now();
15510 let mut computed_vertices = 0;
15511 let mut cycle_errors = 0;
15512 let mut replan_iterations = 0;
15513 const MAX_REPLAN: usize = 5;
15514 let mut telemetry = self
15515 .config
15516 .enable_virtual_dep_telemetry
15517 .then(|| self.start_virtual_dep_telemetry());
15518
15519 loop {
15520 let to_evaluate = self.graph.get_evaluation_vertices();
15521 if to_evaluate.is_empty() {
15522 if let Some(t) = telemetry.as_mut()
15523 && t.bailout_reason.is_none()
15524 {
15525 t.bailout_reason = Some("no_work");
15526 }
15527 break;
15528 }
15529
15530 let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
15531 if let Some(t) = telemetry.as_mut() {
15532 Self::accumulate_schedule_meta(t, &meta);
15533 }
15534
15535 let (pass_computed, pass_cycles) = self.legacy_pass_run_units(&schedule)?;
15536 computed_vertices += pass_computed;
15537 cycle_errors += pass_cycles;
15538
15539 let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
15541 if let Some(t) = telemetry.as_mut() {
15542 t.changed_vdeps_total += changed_vertices.len();
15543 }
15544
15545 self.resource_checkpoint(0)?;
15546 if !self.finish_pass_dirty(&to_evaluate, &changed_vertices) {
15547 if let Some(t) = telemetry.as_mut() {
15548 t.bailout_reason = Some("converged");
15549 }
15550 break;
15551 }
15552 if replan_iterations >= MAX_REPLAN {
15553 if let Some(mut t) = telemetry.take() {
15554 t.bailout_reason = Some("max_replan");
15555 t.replan_iterations = replan_iterations;
15556 self.last_virtual_dep_telemetry = t;
15557 }
15558 return Err(
15559 self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
15560 );
15561 }
15562
15563 replan_iterations += 1;
15564 }
15565
15566 if let Some(mut t) = telemetry {
15567 t.replan_iterations = replan_iterations;
15568 self.last_virtual_dep_telemetry = t;
15569 }
15570
15571 self.redirty_for_next_recalc();
15573
15574 self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
15576
15577 Ok(EvalResult {
15578 computed_vertices,
15579 cycle_errors,
15580 elapsed: start.elapsed(),
15581 })
15582 }
15583
15584 pub fn evaluate_all_with_target_delta(
15585 &mut self,
15586 ) -> Result<(EvalResult, crate::engine::TargetEvalDelta), ExcelError> {
15587 self.observe_evaluation_resource_request(EvaluationRequestKind::FullWithDelta, |engine| {
15588 engine.observe_function_semantic_epoch()?;
15589 let mut collector = DeltaCollector::new(DeltaMode::Cells);
15590 let result = engine.evaluate_all_with_delta_collector(&mut collector)?;
15591 Ok((result, collector.finish_target()))
15592 })
15593 }
15594
15595 pub fn evaluate_all_with_delta(&mut self) -> Result<(EvalResult, EvalDelta), ExcelError> {
15596 self.evaluate_all_with_delta_policy(EvalDeltaCompatibilityPolicy::Unlimited)
15597 }
15598
15599 pub fn evaluate_all_with_delta_policy(
15600 &mut self,
15601 policy: EvalDeltaCompatibilityPolicy,
15602 ) -> Result<(EvalResult, EvalDelta), ExcelError> {
15603 self.observe_evaluation_resource_request(EvaluationRequestKind::FullWithDelta, |engine| {
15604 engine.observe_function_semantic_epoch()?;
15605 let mut collector = DeltaCollector::new(DeltaMode::Cells);
15606 let result = engine.evaluate_all_with_delta_collector(&mut collector)?;
15607 Ok((result, collector.finish_with_policy(policy)?))
15608 })
15609 }
15610
15611 fn evaluate_all_with_delta_collector(
15612 &mut self,
15613 delta: &mut DeltaCollector,
15614 ) -> Result<EvalResult, ExcelError> {
15615 let _source_cache = self.source_cache_session();
15616 if self.config.defer_graph_building {
15617 self.build_graph_all()?;
15618 }
15619 self.require_unified_authority()?;
15620 self.begin_evaluation_request();
15621 self.reset_virtual_dep_telemetry_if_disabled();
15622 let _span_eval = crate::engine::trace::fz_span!(
15623 tracing::Level::INFO,
15624 "evaluate",
15625 "evaluate.legacy_delta"
15626 );
15627 let start = crate::instant::FzInstant::now();
15628 let mut computed_vertices = 0;
15629 let mut cycle_errors = 0;
15630
15631 let mut replan_iterations = 0;
15632 const MAX_REPLAN: usize = 5;
15633 let mut telemetry = self
15634 .config
15635 .enable_virtual_dep_telemetry
15636 .then(|| self.start_virtual_dep_telemetry());
15637
15638 loop {
15639 let to_evaluate = self.graph.get_evaluation_vertices();
15640 if to_evaluate.is_empty() {
15641 if let Some(t) = telemetry.as_mut()
15642 && t.bailout_reason.is_none()
15643 {
15644 t.bailout_reason = Some("no_work");
15645 }
15646 break;
15647 }
15648
15649 let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
15650 if let Some(t) = telemetry.as_mut() {
15651 Self::accumulate_schedule_meta(t, &meta);
15652 }
15653
15654 self.begin_pass(&schedule);
15655 for (unit_index, &unit) in schedule.units.iter().enumerate() {
15656 match unit {
15657 ScheduleUnit::Cycle(i) => {
15658 if self.handle_cycle_unit(
15659 schedule.unit_cycle(i),
15660 Some(delta),
15661 None,
15662 None,
15663 )? > 0
15664 {
15665 cycle_errors += 1;
15666 }
15667 }
15668 ScheduleUnit::Layer(i) => {
15669 let layer = schedule.unit_layer(i);
15670 if self.thread_pool.is_some() && layer.vertices.len() > 1 {
15671 computed_vertices +=
15672 self.evaluate_layer_parallel_with_delta(layer, delta)?;
15673 } else {
15674 computed_vertices +=
15675 self.evaluate_layer_sequential_with_delta(layer, delta)?;
15676 }
15677 }
15678 }
15679 if self.stop_after_unit(&schedule, unit_index) {
15680 break;
15681 }
15682 }
15683
15684 let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
15685 if let Some(t) = telemetry.as_mut() {
15686 t.changed_vdeps_total += changed_vertices.len();
15687 }
15688 self.resource_checkpoint(0)?;
15689 if !self.finish_pass_dirty(&to_evaluate, &changed_vertices) {
15690 if let Some(t) = telemetry.as_mut() {
15691 t.bailout_reason = Some("converged");
15692 }
15693 break;
15694 }
15695 if replan_iterations >= MAX_REPLAN {
15696 if let Some(mut t) = telemetry.take() {
15697 t.bailout_reason = Some("max_replan");
15698 t.replan_iterations = replan_iterations;
15699 self.last_virtual_dep_telemetry = t;
15700 }
15701 return Err(
15702 self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
15703 );
15704 }
15705 replan_iterations += 1;
15706 }
15707
15708 if let Some(mut t) = telemetry {
15709 t.replan_iterations = replan_iterations;
15710 self.last_virtual_dep_telemetry = t;
15711 }
15712
15713 self.redirty_for_next_recalc();
15714 self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
15715
15716 Ok(EvalResult {
15717 computed_vertices,
15718 cycle_errors,
15719 elapsed: start.elapsed(),
15720 })
15721 }
15722
15723 pub fn evaluate_cell(
15733 &mut self,
15734 sheet: &str,
15735 row: u32,
15736 col: u32,
15737 ) -> Result<Option<LiteralValue>, ExcelError> {
15738 self.observe_evaluation_resource_request(EvaluationRequestKind::Cell, |engine| {
15739 engine.evaluate_cell_unobserved(sheet, row, col)
15740 })
15741 }
15742
15743 fn evaluate_cell_unobserved(
15744 &mut self,
15745 sheet: &str,
15746 row: u32,
15747 col: u32,
15748 ) -> Result<Option<LiteralValue>, ExcelError> {
15749 if row == 0 || col == 0 {
15750 return Err(ExcelError::new(ExcelErrorKind::Ref)
15751 .with_message("Row and column must be >= 1".to_string()));
15752 }
15753
15754 let result = self.evaluate_cells(&[(sheet, row, col)])?;
15755
15756 match result.len() {
15757 0 => Ok(None),
15758 1 => {
15759 let v = result.into_iter().next().unwrap();
15760 Ok(v)
15761 }
15762 _ => unreachable!("evaluate_cells returned unexpected length"),
15763 }
15764 }
15765
15766 pub fn evaluate_cells(
15773 &mut self,
15774 targets: &[(&str, u32, u32)],
15775 ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
15776 self.observe_evaluation_resource_request(EvaluationRequestKind::Cells, |engine| {
15777 engine.evaluate_cells_unobserved(targets)
15778 })
15779 }
15780
15781 fn evaluate_cells_unobserved(
15782 &mut self,
15783 targets: &[(&str, u32, u32)],
15784 ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
15785 self.observe_function_semantic_epoch()?;
15786 debug_assert!(
15787 !self.graph.deferred_dirty_active(),
15788 "deferred-dirty scope leaked into evaluate_cells: a begin_deferred_dirty \
15789 was not balanced by end_deferred_dirty"
15790 );
15791 self.validate_deterministic_mode()?;
15792 if targets.is_empty() {
15793 return Ok(Vec::new());
15794 }
15795 let typed_targets = self.legacy_coordinate_targets(targets);
15796 self.evaluate_mixed_targets(&typed_targets, None)?;
15797 Ok(targets
15798 .iter()
15799 .map(|(s, r, c)| self.get_cell_value(s, *r, *c))
15800 .collect())
15801 }
15802
15803 pub fn evaluate_cells_cancellable(
15804 &mut self,
15805 targets: &[(&str, u32, u32)],
15806 cancel: crate::engine::CancelToken,
15807 ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
15808 self.observe_evaluation_resource_request(
15809 EvaluationRequestKind::CellsCancellable,
15810 |engine| {
15811 engine.observe_function_semantic_epoch()?;
15812 engine.active_cancel_flag = Some(cancel.clone());
15813 let res = engine.evaluate_cells_cancellable_impl(targets, cancel.as_flag());
15814 engine.active_cancel_flag = None;
15815 res
15816 },
15817 )
15818 }
15819
15820 fn evaluate_cells_cancellable_impl(
15821 &mut self,
15822 targets: &[(&str, u32, u32)],
15823 cancel_flag: &AtomicBool,
15824 ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
15825 self.validate_deterministic_mode()?;
15826 if targets.is_empty() {
15827 return Ok(Vec::new());
15828 }
15829 if cancel_flag.load(Ordering::Relaxed) {
15830 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
15831 .with_message("Evaluation cancelled before target preparation"));
15832 }
15833 let typed_targets = self.legacy_coordinate_targets(targets);
15834 self.evaluate_mixed_targets(&typed_targets, None)?;
15835 Ok(targets
15836 .iter()
15837 .map(|(sheet, row, col)| self.get_cell_value(sheet, *row, *col))
15838 .collect())
15839 }
15840
15841 pub fn evaluate_cells_with_target_delta(
15842 &mut self,
15843 targets: &[(&str, u32, u32)],
15844 ) -> Result<(Vec<Option<LiteralValue>>, crate::engine::TargetEvalDelta), ExcelError> {
15845 self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
15846 engine.observe_function_semantic_epoch()?;
15847 engine.validate_deterministic_mode()?;
15848 if targets.is_empty() {
15849 return Ok((Vec::new(), crate::engine::TargetEvalDelta::default()));
15850 }
15851 let mut collector = DeltaCollector::new(DeltaMode::Cells);
15852 engine.evaluate_until_with_delta_collector(targets, &mut collector)?;
15853 let values = targets
15854 .iter()
15855 .map(|(sheet, row, col)| engine.get_cell_value(sheet, *row, *col))
15856 .collect();
15857 Ok((values, collector.finish_target()))
15858 })
15859 }
15860
15861 pub fn evaluate_cells_with_delta(
15862 &mut self,
15863 targets: &[(&str, u32, u32)],
15864 ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
15865 self.evaluate_cells_with_delta_policy(targets, EvalDeltaCompatibilityPolicy::Unlimited)
15866 }
15867
15868 pub fn evaluate_cells_with_delta_policy(
15869 &mut self,
15870 targets: &[(&str, u32, u32)],
15871 policy: EvalDeltaCompatibilityPolicy,
15872 ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
15873 self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
15874 engine.evaluate_cells_with_delta_unobserved(targets, policy)
15875 })
15876 }
15877
15878 fn evaluate_cells_with_delta_unobserved(
15879 &mut self,
15880 targets: &[(&str, u32, u32)],
15881 policy: EvalDeltaCompatibilityPolicy,
15882 ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
15883 self.observe_function_semantic_epoch()?;
15884 self.validate_deterministic_mode()?;
15885 if targets.is_empty() {
15886 return Ok((Vec::new(), EvalDelta::default()));
15887 }
15888 let mut collector = DeltaCollector::new(DeltaMode::Cells);
15889 self.evaluate_until_with_delta_collector(targets, &mut collector)?;
15890 let values = targets
15891 .iter()
15892 .map(|(s, r, c)| self.get_cell_value(s, *r, *c))
15893 .collect();
15894 Ok((values, collector.finish_with_policy(policy)?))
15895 }
15896
15897 pub fn get_eval_plan(&self, targets: &[(&str, u32, u32)]) -> Result<EvalPlan, ExcelError> {
15899 if targets.is_empty() {
15900 return Ok(EvalPlan {
15901 total_vertices_to_evaluate: 0,
15902 layers: Vec::new(),
15903 cycles_detected: 0,
15904 dirty_count: 0,
15905 volatile_count: 0,
15906 parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
15907 estimated_parallel_layers: 0,
15908 target_cells: Vec::new(),
15909 });
15910 }
15911 if self.config.defer_graph_building && self.has_staged_formulas() {
15912 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
15913 "Evaluation plan requested with deferred graph; build first or call evaluate_*",
15914 ));
15915 }
15916
15917 let addresses: Vec<String> = targets
15919 .iter()
15920 .map(|(s, r, c)| format!("{}!{}{}", s, Self::col_to_letters(*c), r))
15921 .collect();
15922
15923 let mut target_addrs = Vec::new();
15925 for (sheet, row, col) in targets {
15926 if let Some(sheet_id) = self.graph.sheet_id(sheet) {
15927 let coord = Coord::from_excel(*row, *col, true, true);
15928 target_addrs.push(CellRef::new(sheet_id, coord));
15929 }
15930 }
15931
15932 let mut target_vertex_ids = Vec::new();
15934 for addr in &target_addrs {
15935 if let Some(vertex_id) = self.graph.get_vertex_id_for_address(addr) {
15936 target_vertex_ids.push(vertex_id);
15937 }
15938 }
15939
15940 if target_vertex_ids.is_empty() {
15941 return Ok(EvalPlan {
15942 total_vertices_to_evaluate: 0,
15943 layers: Vec::new(),
15944 cycles_detected: 0,
15945 dirty_count: 0,
15946 volatile_count: 0,
15947 parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
15948 estimated_parallel_layers: 0,
15949 target_cells: addresses,
15950 });
15951 }
15952
15953 let (precedents_to_eval, vdeps) = self.demand_subgraph(&target_vertex_ids)?;
15955
15956 if precedents_to_eval.is_empty() {
15957 return Ok(EvalPlan {
15958 total_vertices_to_evaluate: 0,
15959 layers: Vec::new(),
15960 cycles_detected: 0,
15961 dirty_count: 0,
15962 volatile_count: 0,
15963 parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
15964 estimated_parallel_layers: 0,
15965 target_cells: addresses,
15966 });
15967 }
15968
15969 let mut dirty_count = 0;
15971 let mut volatile_count = 0;
15972 for &vertex_id in &precedents_to_eval {
15973 if self.graph.is_dirty(vertex_id) {
15974 dirty_count += 1;
15975 }
15976 if self.graph.is_volatile(vertex_id) {
15977 volatile_count += 1;
15978 }
15979 }
15980
15981 let schedule = self.create_authority_schedule(&precedents_to_eval, &vdeps, None)?;
15983
15984 let mut layers = Vec::new();
15986 let mut estimated_parallel_layers = 0;
15987 let parallel_enabled = self.config.enable_parallel && self.thread_pool.is_some();
15988
15989 for layer in &schedule.layers {
15990 let parallel_eligible = parallel_enabled && layer.vertices.len() > 1;
15991 if parallel_eligible {
15992 estimated_parallel_layers += 1;
15993 }
15994
15995 let sample_cells: Vec<String> = layer
15997 .vertices
15998 .iter()
15999 .take(5)
16000 .filter_map(|&vertex_id| {
16001 self.graph
16002 .get_cell_ref_for_vertex(vertex_id)
16003 .map(|cell_ref| {
16004 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
16005 format!(
16006 "{}!{}{}",
16007 sheet_name,
16008 Self::col_to_letters(cell_ref.coord.col().saturating_add(1)),
16009 cell_ref.coord.row() + 1
16010 )
16011 })
16012 })
16013 .collect();
16014
16015 layers.push(LayerInfo {
16016 vertex_count: layer.vertices.len(),
16017 parallel_eligible,
16018 sample_cells,
16019 });
16020 }
16021
16022 Ok(EvalPlan {
16023 total_vertices_to_evaluate: precedents_to_eval.len(),
16024 layers,
16025 cycles_detected: schedule.cycles.len(),
16026 dirty_count,
16027 volatile_count,
16028 parallel_enabled,
16029 estimated_parallel_layers,
16030 target_cells: addresses,
16031 })
16032 }
16033 fn create_evaluation_schedule(
16035 &mut self,
16036 to_evaluate: &[VertexId],
16037 ) -> Result<EvaluationScheduleBuildOutput, ExcelError> {
16038 #[cfg(any(test, feature = "benchmark_internal"))]
16039 {
16040 self.recalc_reuse_probe.get_mut().unwrap().schedule_requests += 1;
16041 }
16042 #[cfg(any(test, feature = "legacy_oracle"))]
16045 self.graph.flush_pending_edge_deltas();
16046 self.graph.authority_sync();
16048 if self.can_use_static_schedule_cache(to_evaluate) {
16049 let revision = self.schedule_cache_authority_revision();
16051 let current = |e: &CachedScheduleEntry| {
16052 e.topology_epoch == self.topology_epoch && e.authority_revision == revision
16053 };
16054 if !self
16055 .cached_static_schedule
16056 .as_ref()
16057 .is_some_and(|c| current(c) && c.candidate_vertices.equals(to_evaluate))
16058 && let Some(i) = self.recent_schedules.iter().position(|e| {
16059 current(e)
16060 && e.candidate_vertices.len() == to_evaluate.len()
16061 && e.candidate_vertices.equals(to_evaluate)
16062 })
16063 {
16064 let hit = self.recent_schedules.remove(i);
16065 if let Some(previous) = self.cached_static_schedule.replace(hit) {
16066 self.retain_recent_schedule(previous);
16067 }
16068 }
16069 if let Some(cached) = self.cached_static_schedule.as_ref()
16070 && cached.topology_epoch == self.topology_epoch
16071 && cached.authority_revision == self.schedule_cache_authority_revision()
16072 && cached.candidate_vertices.equals(to_evaluate)
16073 {
16074 let meta = ScheduleBuildMeta {
16075 candidate_vertices: to_evaluate.len(),
16076 vdeps_vertices: 0,
16077 vdeps_edges: 0,
16078 builder_elapsed_ms: 0,
16079 used_virtual_schedule: false,
16080 schedule_cache_hit: true,
16081 schedule_cache_eligible: true,
16082 };
16083 #[cfg(any(test, feature = "benchmark_internal"))]
16084 {
16085 let mut probe = self.recalc_reuse_probe.lock().unwrap();
16086 probe.schedule_cache_hits += 1;
16087 probe.schedule_shared_handles += 1;
16088 }
16089 return Ok((
16090 EvaluationSchedule::Shared(Arc::clone(&cached.schedule)),
16091 FxHashMap::default(),
16092 meta,
16093 ));
16094 }
16095
16096 let (schedule, vdeps, mut meta) = match self.schedule_from_base(to_evaluate)? {
16097 Some(schedule) => (
16098 schedule,
16099 FxHashMap::default(),
16100 ScheduleBuildMeta {
16101 candidate_vertices: to_evaluate.len(),
16102 vdeps_vertices: 0,
16103 vdeps_edges: 0,
16104 builder_elapsed_ms: 0,
16105 used_virtual_schedule: false,
16106 schedule_cache_hit: false,
16107 schedule_cache_eligible: true,
16108 },
16109 ),
16110 None => self.create_evaluation_schedule_active(to_evaluate)?,
16111 };
16112 meta.schedule_cache_hit = false;
16113 meta.schedule_cache_eligible = true;
16114 #[cfg(any(test, feature = "benchmark_internal"))]
16115 {
16116 self.recalc_reuse_probe
16117 .get_mut()
16118 .unwrap()
16119 .schedule_cache_misses += 1;
16120 }
16121 let schedule = if vdeps.is_empty() {
16122 let mut schedule = schedule;
16125 schedule.units.shrink_to_fit();
16126 for layer in &mut schedule.layers {
16127 layer.vertices.shrink_to_fit();
16128 }
16129 schedule.layers.shrink_to_fit();
16130 for cycle in &mut schedule.cycles {
16131 cycle.shrink_to_fit();
16132 }
16133 schedule.cycles.shrink_to_fit();
16134 let schedule = Arc::new(schedule);
16135 #[cfg(any(test, feature = "benchmark_internal"))]
16136 {
16137 self.recalc_reuse_probe
16138 .get_mut()
16139 .unwrap()
16140 .schedule_shared_handles += 1;
16141 }
16142 let entry = CachedScheduleEntry {
16143 topology_epoch: self.topology_epoch,
16144 authority_revision: self.schedule_cache_authority_revision(),
16145 candidate_vertices: VertexIdRuns::from_slice(to_evaluate),
16146 schedule: Arc::clone(&schedule),
16147 };
16148 if let Some(previous) = self.cached_static_schedule.replace(entry) {
16149 self.retain_recent_schedule(previous);
16150 }
16151 EvaluationSchedule::Shared(schedule)
16152 } else {
16153 EvaluationSchedule::Owned(schedule)
16154 };
16155 return Ok((schedule, vdeps, meta));
16156 }
16157
16158 let (schedule, vdeps, mut meta) = self.create_evaluation_schedule_active(to_evaluate)?;
16159 meta.schedule_cache_hit = false;
16160 meta.schedule_cache_eligible = false;
16161 #[cfg(any(test, feature = "benchmark_internal"))]
16162 {
16163 self.recalc_reuse_probe
16164 .get_mut()
16165 .unwrap()
16166 .schedule_cache_ineligible += 1;
16167 }
16168 Ok((EvaluationSchedule::Owned(schedule), vdeps, meta))
16169 }
16170
16171 fn schedule_from_base(
16177 &mut self,
16178 to_evaluate: &[VertexId],
16179 ) -> Result<Option<crate::engine::scheduler::Schedule>, ExcelError> {
16180 let revision = self.schedule_cache_authority_revision();
16181 let current = |e: &&CachedScheduleEntry| {
16182 e.topology_epoch == self.topology_epoch && e.authority_revision == revision
16183 };
16184 let Some(base) = [
16187 self.base_schedule.as_ref(),
16188 self.cached_static_schedule.as_ref(),
16189 ]
16190 .into_iter()
16191 .flatten()
16192 .filter(current)
16193 .max_by_key(|e| e.candidate_vertices.len()) else {
16194 return Ok(None);
16195 };
16196 let base_len = base.candidate_vertices.len();
16197 if base_len <= BASE_SCHEDULE_MIN_VERTICES
16198 || to_evaluate.len() < BASE_SCHEDULE_MIN_REQUEST
16199 || to_evaluate.len() > base_len
16200 || base_len > to_evaluate.len().saturating_mul(BASE_SCHEDULE_RATIO)
16201 {
16202 return Ok(None);
16203 }
16204 let mut keep = crate::engine::idset::DenseIdSet::default();
16205 keep.extend(to_evaluate.iter().copied());
16206 let Some((schedule, kept)) = base.schedule.restrict(&keep) else {
16207 return Ok(None);
16208 };
16209 if kept != keep.len() {
16211 return Ok(None);
16212 }
16213 if let Some(ledger) = self.active_resource_ledger.as_mut() {
16214 let layers: usize = schedule
16215 .layers
16216 .iter()
16217 .map(|l| {
16218 l.vertices.capacity() * std::mem::size_of::<VertexId>()
16219 + l.runs.capacity()
16220 * std::mem::size_of::<crate::engine::scheduler::LayerRun>()
16221 })
16222 .sum();
16223 let bytes = (keep.heap_bytes()
16224 + layers
16225 + schedule.layers.capacity()
16226 * std::mem::size_of::<crate::engine::scheduler::Layer>()
16227 + schedule.units.capacity()
16228 * std::mem::size_of::<crate::engine::scheduler::ScheduleUnit>()
16229 + schedule
16230 .cycles
16231 .iter()
16232 .map(|c| c.capacity() * 4)
16233 .sum::<usize>()) as u64;
16234 ledger
16235 .reserve_schedule_discovery(bytes)
16236 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16237 ledger
16238 .release_scratch(bytes)
16239 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16240 }
16241 #[cfg(debug_assertions)]
16242 self.debug_check_restricted_schedule(to_evaluate, &schedule);
16243 #[cfg(any(test, feature = "benchmark_internal"))]
16244 {
16245 self.recalc_reuse_probe
16246 .get_mut()
16247 .unwrap()
16248 .schedule_base_restrictions += 1;
16249 }
16250 Ok(Some(schedule))
16251 }
16252
16253 #[cfg(debug_assertions)]
16257 fn debug_check_restricted_schedule(
16258 &self,
16259 to_evaluate: &[VertexId],
16260 schedule: &crate::engine::scheduler::Schedule,
16261 ) {
16262 let mut position: FxHashMap<VertexId, usize> = FxHashMap::default();
16263 for (u, unit) in schedule.units.iter().enumerate() {
16264 let vertices: &[VertexId] = match *unit {
16265 crate::engine::scheduler::ScheduleUnit::Layer(i) => {
16266 &schedule.unit_layer(i).vertices
16267 }
16268 crate::engine::scheduler::ScheduleUnit::Cycle(i) => schedule.unit_cycle(i),
16269 };
16270 for &v in vertices {
16271 assert!(
16272 position.insert(v, u).is_none(),
16273 "vertex twice in a restricted schedule"
16274 );
16275 }
16276 }
16277 assert_eq!(position.len(), to_evaluate.len());
16278 if to_evaluate.len() > 512 {
16279 return;
16280 }
16281 let Ok(store) = self.graph.authority_plan_store() else {
16284 return;
16285 };
16286 let cells: Vec<(VertexId, (u16, u32, u32))> = to_evaluate
16287 .iter()
16288 .filter_map(|&v| self.graph.authority_cell_of_vertex(v).map(|c| (v, c)))
16289 .collect();
16290 for &(v, (sheet, row, col)) in &cells {
16291 let Some(owner) = store.owner_at((sheet, row, col)) else {
16292 continue;
16293 };
16294 let Ok(refined) = store.refine_owner_column(owner, col, row, row, None) else {
16295 continue;
16296 };
16297 for piece in &refined.pieces {
16298 for edge in &refined.edges[piece.edge_start..piece.edge_end] {
16299 let Some(image) = edge.proj.forward(&piece.domain) else {
16300 continue;
16301 };
16302 for &(d, (s2, r2, c2)) in &cells {
16303 if d == v
16304 || edge.proj.sheet != s2
16305 || !(image.r0 <= r2
16306 && r2 <= image.r1
16307 && image.c0 <= c2
16308 && c2 <= image.c1)
16309 {
16310 continue;
16311 }
16312 let (pd, pv) = (position[&d], position[&v]);
16313 let same_ok = pd == pv
16314 && match schedule.units[pv] {
16315 crate::engine::scheduler::ScheduleUnit::Layer(i) => {
16316 schedule.unit_layer(i).sequential
16317 }
16318 crate::engine::scheduler::ScheduleUnit::Cycle(_) => true,
16319 };
16320 assert!(
16321 pd < pv || same_ok,
16322 "restricted schedule orders {v:?} before its precedent {d:?}"
16323 );
16324 }
16325 }
16326 }
16327 }
16328 }
16329
16330 fn maybe_compress_formulas(&mut self) {
16334 if !self.config.formula_compression {
16335 return;
16336 }
16337 let builds = self.graph.authority_host().builds;
16338 if self.compressed_at_build == Some(builds) {
16339 return;
16340 }
16341 self.compressed_at_build = Some(builds);
16342 if self.has_staged_formulas() {
16343 self.graph.virtualize_family_members();
16346 return;
16347 }
16348 let pool = self.thread_pool.clone();
16349 let (_, garbage) = self.graph.compress_family_formulas(pool.as_deref());
16350 self.graph.virtualize_family_members();
16351 match pool {
16355 Some(pool) if garbage.len() >= 1024 => pool.spawn(move || drop(garbage)),
16356 _ => drop(garbage),
16357 }
16358 }
16359
16360 fn create_evaluation_schedule_active(
16361 &mut self,
16362 to_evaluate: &[VertexId],
16363 ) -> Result<ScheduleBuildOutput, ExcelError> {
16364 self.graph.authority_sync();
16365 self.maybe_compress_formulas();
16366 let mut ledger = self.active_resource_ledger.take();
16367 let result = self.create_evaluation_schedule_uncached(to_evaluate, ledger.as_mut());
16368 self.active_resource_ledger = ledger;
16369 result
16370 }
16371
16372 fn create_evaluation_schedule_uncached(
16373 &self,
16374 to_evaluate: &[VertexId],
16375 #[allow(unused_variables)] ledger: Option<&mut ResourceLedger>,
16376 ) -> Result<ScheduleBuildOutput, ExcelError> {
16377 #[cfg(any(test, feature = "benchmark_internal"))]
16378 {
16379 self.recalc_reuse_probe.lock().unwrap().schedule_builds += 1;
16380 }
16381 let builder = VirtualDepBuilder::new(self);
16382 #[allow(unused_mut)]
16383 let (mut vdeps, augmented, builder_elapsed_ms, vdeps_edges) =
16384 if self.config.enable_virtual_dep_telemetry {
16385 let build_started = crate::instant::FzInstant::now();
16386 let (vdeps, augmented) = builder.build(to_evaluate);
16387 let builder_elapsed_ms = build_started.elapsed().as_millis();
16388 let vdeps_edges = vdeps.values().map(|deps| deps.len()).sum::<usize>();
16389 (vdeps, augmented, builder_elapsed_ms, vdeps_edges)
16390 } else {
16391 let (vdeps, augmented) = builder.build(to_evaluate);
16392 (vdeps, augmented, 0, 0)
16393 };
16394
16395 {
16397 self.freshness_merge_hints(to_evaluate, &mut vdeps);
16398 self.freshness_extent_hints(to_evaluate, &mut vdeps);
16399 }
16400 let mut final_evaluate = to_evaluate.to_vec();
16401 if !augmented.is_empty() {
16402 final_evaluate.extend(augmented);
16403 final_evaluate.sort_unstable();
16404 final_evaluate.dedup();
16405 }
16406
16407 let use_virtual = !vdeps.is_empty();
16408
16409 let schedule = self.create_authority_schedule(&final_evaluate, &vdeps, ledger)?;
16410
16411 let meta = ScheduleBuildMeta {
16412 candidate_vertices: to_evaluate.len(),
16413 vdeps_vertices: vdeps.len(),
16414 vdeps_edges,
16415 builder_elapsed_ms,
16416 used_virtual_schedule: use_virtual,
16417 schedule_cache_hit: false,
16418 schedule_cache_eligible: false,
16419 };
16420
16421 Ok((schedule, vdeps, meta))
16422 }
16423
16424 fn create_authority_schedule(
16425 &self,
16426 candidates: &[VertexId],
16427 vdeps: &FxHashMap<VertexId, Vec<VertexId>>,
16428 mut ledger: Option<&mut ResourceLedger>,
16429 ) -> Result<crate::engine::scheduler::Schedule, ExcelError> {
16430 use crate::engine::authority::{
16431 geom::{Cover, Rect},
16432 plan_schedule, planner,
16433 proj::{AxisMap, RefProj},
16434 store::{EdgeKey, Tag},
16435 };
16436 let failure = |message: String| {
16437 ExcelError::new(ExcelErrorKind::Error)
16438 .with_message(format!("unified_authority planner: {message}"))
16439 };
16440 self.cancellation_checkpoint("Evaluation cancelled before authority planning")?;
16441 let store = self
16442 .graph
16443 .authority_plan_store()
16444 .map_err(Self::authority_excel_error)?;
16445 if let [only] = candidates
16448 && vdeps.is_empty()
16449 && self.graph.authority_host().observed(*only).is_none()
16450 && let Some(cell) = self.graph.authority_cell_of_vertex(*only)
16451 && cell.0 != crate::engine::authority::geom::SYMBOL_SHEET
16452 && let Some(single) = planner::plan_single(store, cell)
16453 {
16454 #[cfg(debug_assertions)]
16455 {
16456 let mut cover = Cover::new();
16457 cover.insert_rect(cell.0, &Rect::new(cell.1, cell.2, cell.1, cell.2));
16458 let general = planner::plan_with_hints(store, &cover, &[], None, None, None)
16459 .expect("general plan of one cell");
16460 assert_eq!(
16461 general.cells.as_slice(),
16462 &[single],
16463 "single-cell plan differs from the planner at {cell:?}"
16464 );
16465 }
16466 let adapted = plan_schedule::schedule(
16467 &[single],
16468 0,
16469 None,
16470 |cell| {
16471 self.graph
16472 .authority_vertex_of_formula(cell.id, (cell.sheet, cell.row, cell.col))
16473 .ok_or_else(|| failure("missing executor identity".to_owned()))
16474 },
16475 |_work| Ok(()),
16476 )
16477 .map_err(|error| match error {
16478 plan_schedule::ScheduleError::Runtime(error) => error,
16479 other => failure(format!("{other:?}")),
16480 })?;
16481 if let Some(ledger) = ledger {
16482 ledger
16483 .reserve_schedule_discovery(adapted.peak_heap_bytes)
16484 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16485 ledger
16486 .release_scratch(adapted.peak_heap_bytes)
16487 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16488 }
16489 return Ok(adapted.schedule);
16490 }
16491 if (2..=planner::SMALL_PLAN_MAX).contains(&candidates.len())
16494 && vdeps.is_empty()
16495 && candidates
16496 .iter()
16497 .all(|&v| self.graph.authority_host().observed(v).is_none())
16498 {
16499 let cells: Option<Vec<(u16, u32, u32)>> = candidates
16500 .iter()
16501 .map(|&v| self.graph.authority_cell_of_vertex(v))
16502 .collect();
16503 if let Some(small) = cells.as_deref().and_then(|c| planner::plan_small(store, c)) {
16504 #[cfg(debug_assertions)]
16505 {
16506 let mut cover = Cover::new();
16507 for c in cells.as_deref().unwrap_or_default() {
16508 cover.insert_rect(c.0, &Rect::new(c.1, c.2, c.1, c.2));
16509 }
16510 let general = planner::plan_with_hints(store, &cover, &[], None, None, None)
16511 .expect("general plan of a small request");
16512 let key = |c: &planner::OrderedCell| (c.sheet, c.row, c.col, c.id, c.owner);
16513 let mut a: Vec<_> = general.cells.iter().map(key).collect();
16514 let mut b: Vec<_> = small.iter().map(key).collect();
16515 a.sort_unstable();
16516 b.sort_unstable();
16517 assert_eq!(a, b, "small plan cells differ from the planner");
16518 assert!(
16519 general.cells.iter().all(|c| c.cycle.is_none()),
16520 "small plan of a cyclic request"
16521 );
16522 }
16523 let adapted = plan_schedule::schedule(
16524 &small,
16525 0,
16526 None,
16527 |cell| {
16528 self.graph
16529 .authority_vertex_of_formula(cell.id, (cell.sheet, cell.row, cell.col))
16530 .ok_or_else(|| failure("missing executor identity".to_owned()))
16531 },
16532 |_work| Ok(()),
16533 )
16534 .map_err(|error| match error {
16535 plan_schedule::ScheduleError::Runtime(error) => error,
16536 other => failure(format!("{other:?}")),
16537 })?;
16538 if let Some(ledger) = ledger {
16539 ledger
16540 .reserve_schedule_discovery(adapted.peak_heap_bytes)
16541 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16542 ledger
16543 .release_scratch(adapted.peak_heap_bytes)
16544 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16545 }
16546 return Ok(adapted.schedule);
16547 }
16548 }
16549 let mut cover = Cover::new();
16554 {
16555 let cell_of = |&id: &VertexId| {
16556 self.graph
16557 .authority_cell_of_vertex(id)
16558 .map(|(sheet, row, col)| (sheet, col, row))
16559 };
16560 let cells: Vec<(u16, u32, u32)> = match self.thread_pool.as_deref() {
16563 Some(pool) if candidates.len() >= PARALLEL_SCHEDULE_MIN_CANDIDATES => {
16564 use rayon::prelude::*;
16565 pool.install(|| {
16566 let mut cells: Vec<_> = candidates.par_iter().filter_map(cell_of).collect();
16567 cells.par_sort_unstable();
16568 cells
16569 })
16570 }
16571 _ => {
16572 let mut cells: Vec<_> = candidates.iter().filter_map(cell_of).collect();
16573 cells.sort_unstable();
16574 cells
16575 }
16576 };
16577 let mut i = 0;
16578 while i < cells.len() {
16579 let (sheet, col, r0) = cells[i];
16580 let mut r1 = r0;
16581 let mut j = i + 1;
16582 while j < cells.len() && cells[j].0 == sheet && cells[j].1 == col {
16583 if cells[j].2 > r1 + 1 {
16584 break;
16585 }
16586 r1 = r1.max(cells[j].2);
16587 j += 1;
16588 }
16589 cover.insert_rect(sheet, &Rect::new(r0, col, r1, col));
16590 i = j;
16591 }
16592 }
16593 let mut hints = Vec::new();
16594 for (&reader, deps) in vdeps {
16595 let Some(reader) = self.graph.authority_cell_of_vertex(reader) else {
16596 continue;
16597 };
16598 for &dependency in deps {
16599 let Some(dep) = self.graph.authority_cell_of_vertex(dependency) else {
16600 continue;
16601 };
16602 hints.push(planner::PlanHint {
16603 reader: (reader.0, reader.2, reader.1),
16604 edge: EdgeKey {
16605 dep_sheet: reader.0,
16606 tag: Tag::X,
16607 lk: u32::MAX,
16608 proj: RefProj {
16609 sheet: dep.0,
16610 rows: AxisMap::fixed(dep.1, dep.1),
16611 cols: AxisMap::fixed(dep.2, dep.2),
16612 },
16613 },
16614 });
16615 }
16616 }
16617 let host = self.graph.authority_host();
16619 for &id in candidates.iter().filter(|_| host.has_observed()) {
16620 let Some(reads) = host.observed(id) else {
16621 continue;
16622 };
16623 let Some(reader) = self.graph.authority_cell_of_vertex(id) else {
16624 continue;
16625 };
16626 for &(sheet, r0, c0, r1, c1) in reads {
16627 hints.push(planner::PlanHint {
16628 reader: (reader.0, reader.2, reader.1),
16629 edge: EdgeKey {
16630 dep_sheet: reader.0,
16631 tag: Tag::X,
16632 lk: u32::MAX,
16633 proj: RefProj {
16634 sheet,
16635 rows: AxisMap::fixed(r0, r1),
16636 cols: AxisMap::fixed(c0, c1),
16637 },
16638 },
16639 });
16640 }
16641 }
16642 hints.sort_unstable_by_key(|hint| hint.reader);
16643 let checkpoint = ledger
16644 .as_ref()
16645 .map_or(0, |ledger| ledger.scratch_checkpoint());
16646 let scratch_limit = ledger
16647 .as_ref()
16648 .and_then(|ledger| ledger.schedule_discovery_limit())
16649 .map(|limit| limit.saturating_sub(checkpoint));
16650 let ordered = planner::plan_with_hints(store, &cover, &hints, scratch_limit, None, None)
16651 .map_err(|error| failure(format!("{error:?}")))?;
16652 let adapted = plan_schedule::schedule(
16656 &ordered.cells,
16657 ordered.heap_bytes(),
16658 scratch_limit,
16659 |cell| {
16660 self.graph
16661 .authority_vertex_of_formula(cell.id, (cell.sheet, cell.row, cell.col))
16662 .ok_or_else(|| failure("missing executor identity".to_owned()))
16663 },
16664 |_work| {
16665 self.cancellation_checkpoint("Evaluation cancelled during authority planning")?;
16666 if let Some(ledger) = ledger.as_deref_mut() {
16667 ledger
16668 .checkpoint_deadline()
16669 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16670 }
16671 Ok(())
16672 },
16673 )
16674 .map_err(|error| match error {
16675 plan_schedule::ScheduleError::Runtime(error) => error,
16676 other => failure(format!("{other:?}")),
16677 })?;
16678 if let Some(ledger) = ledger {
16679 let peak = ordered.peak_heap_bytes.max(adapted.peak_heap_bytes);
16680 ledger
16681 .reserve_schedule_discovery(peak)
16682 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16683 ledger
16684 .release_scratch(peak)
16685 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16686 }
16687 match self.thread_pool.as_deref() {
16689 Some(pool) if ordered.cells.len() >= PARALLEL_SCHEDULE_MIN_CANDIDATES => {
16690 pool.spawn(move || drop(ordered))
16691 }
16692 _ => drop(ordered),
16693 }
16694 Ok(adapted.schedule)
16695 }
16696
16697 fn can_use_static_schedule_cache(&self, to_evaluate: &[VertexId]) -> bool {
16703 {
16704 let host = self.graph.authority_host();
16708 !to_evaluate.is_empty()
16709 && !self.freshness_has_hints()
16710 && to_evaluate
16711 .iter()
16712 .all(|&v| !self.graph.is_dynamic(v) || host.observed(v).is_some())
16713 }
16714 }
16715
16716 fn schedule_cache_authority_revision(&self) -> (u64, u64) {
16717 {
16718 let host = self.graph.authority_host();
16721 (host.revision(), host.rev_dyn())
16722 }
16723 }
16724
16725 fn start_virtual_dep_telemetry(&self) -> VirtualDepTelemetry {
16726 VirtualDepTelemetry {
16727 fallback_mode_activations: self.virtual_dep_fallback_activations,
16728 ..VirtualDepTelemetry::default()
16729 }
16730 }
16731
16732 fn accumulate_schedule_meta(telemetry: &mut VirtualDepTelemetry, meta: &ScheduleBuildMeta) {
16733 telemetry.candidate_vertices_total += meta.candidate_vertices;
16734 telemetry.vdeps_vertices_total += meta.vdeps_vertices;
16735 telemetry.vdeps_edges_total += meta.vdeps_edges;
16736 telemetry.builder_elapsed_ms_total += meta.builder_elapsed_ms;
16737 if meta.schedule_cache_eligible {
16738 if meta.schedule_cache_hit {
16739 telemetry.schedule_cache_hits += 1;
16740 telemetry.reused_schedule_vertices_total += meta.candidate_vertices;
16741 } else {
16742 telemetry.schedule_cache_misses += 1;
16743 }
16744 }
16745 if meta.used_virtual_schedule {
16746 telemetry.schedule_virtual_passes += 1;
16747 } else {
16748 telemetry.schedule_static_passes += 1;
16749 }
16750 }
16751
16752 fn finish_pass_dirty(&mut self, to_evaluate: &[VertexId], changed: &[VertexId]) -> bool {
16757 self.finish_pass_dirty_scoped(to_evaluate, changed, true)
16758 }
16759
16760 fn finish_target_pass_dirty(&mut self, to_evaluate: &[VertexId], changed: &[VertexId]) -> bool {
16763 self.finish_pass_dirty_scoped(to_evaluate, changed, false)
16764 }
16765
16766 fn finish_pass_dirty_scoped(
16767 &mut self,
16768 to_evaluate: &[VertexId],
16769 changed: &[VertexId],
16770 #[allow(unused_variables)] whole_workbook: bool,
16771 ) -> bool {
16772 self.freshness_finish_pass(to_evaluate, changed, whole_workbook)
16773 }
16774
16775 fn begin_pass(
16777 &mut self,
16778 #[allow(unused_variables)] schedule: &crate::engine::scheduler::Schedule,
16779 ) {
16780 self.freshness_begin_pass(schedule);
16781 }
16782
16783 fn stop_after_unit(
16785 &mut self,
16786 #[allow(unused_variables)] schedule: &crate::engine::scheduler::Schedule,
16787 #[allow(unused_variables)] index: usize,
16788 ) -> bool {
16789 self.freshness_stop_after_unit(schedule, index)
16790 }
16791
16792 fn changed_virtual_dep_vertices(
16793 &mut self,
16794 to_evaluate: &[VertexId],
16795 old_vdeps: &FxHashMap<VertexId, Vec<VertexId>>,
16796 ) -> Vec<VertexId> {
16797 #[cfg(test)]
16798 if self.force_virtual_dep_changes_remaining_for_test > 0
16799 && let Some(vertex) = to_evaluate.first().copied()
16800 {
16801 self.force_virtual_dep_changes_remaining_for_test -= 1;
16802 return vec![vertex];
16803 }
16804 if self.freshness_armed() {
16807 return Vec::new();
16808 }
16809 if !to_evaluate
16810 .iter()
16811 .copied()
16812 .any(|v| self.graph.is_dynamic(v))
16813 {
16814 return Vec::new();
16815 }
16816
16817 let builder = VirtualDepBuilder::new(self);
16818 let (new_vdeps, _) = builder.build(to_evaluate);
16819
16820 let mut candidates = FxHashSet::default();
16821 candidates.extend(old_vdeps.keys().copied());
16822 candidates.extend(new_vdeps.keys().copied());
16823
16824 let mut changed = Vec::new();
16825 for v in candidates {
16826 if old_vdeps.get(&v) != new_vdeps.get(&v) {
16827 changed.push(v);
16828 }
16829 }
16830 changed
16831 }
16832
16833 #[allow(clippy::type_complexity)]
16839 fn demand_subgraph(
16840 &self,
16841 targets: &[VertexId],
16842 ) -> Result<
16843 (
16844 Vec<VertexId>,
16845 rustc_hash::FxHashMap<VertexId, Vec<VertexId>>,
16846 ),
16847 ExcelError,
16848 > {
16849 self.authority_demand_subgraph(targets)
16850 }
16851
16852 #[allow(clippy::type_complexity)]
16858 fn authority_demand_subgraph(
16859 &self,
16860 targets: &[VertexId],
16861 ) -> Result<
16862 (
16863 Vec<VertexId>,
16864 rustc_hash::FxHashMap<VertexId, Vec<VertexId>>,
16865 ),
16866 ExcelError,
16867 > {
16868 use crate::engine::authority::geom::{Cell, Rect, SYMBOL_SHEET};
16869 use crate::engine::authority::store::TagFilter;
16870 use rustc_hash::{FxHashMap, FxHashSet};
16871 let store = self
16872 .graph
16873 .authority_plan_store()
16874 .map_err(Self::authority_excel_error)?;
16875 let ids = store.ids();
16876 let mut to_evaluate: FxHashSet<VertexId> = FxHashSet::default();
16877 let mut vdeps: FxHashMap<VertexId, Vec<VertexId>> = FxHashMap::default();
16878 let mut visited: FxHashSet<Cell> = FxHashSet::default();
16879 let mut stack: Vec<(Cell, u32)> = Vec::new();
16882 let push_formulas = |stack: &mut Vec<(Cell, u32)>,
16884 visited: &FxHashSet<Cell>,
16885 sheet: u16,
16886 rect: Rect| {
16887 for col in rect.c0..=rect.c1 {
16888 ids.visit_runs_in(sheet, col, rect.r0, rect.r1, &mut |h| {
16889 let run = ids.run(h);
16890 let r0 = run.row_start.max(rect.r0);
16891 let r1 = (run.row_start + run.len - 1).min(rect.r1);
16892 for row in r0..=r1 {
16893 if !visited.contains(&(sheet, row, col)) {
16894 stack.push(((sheet, row, col), run.first_id + (row - run.row_start)));
16895 }
16896 }
16897 });
16898 }
16899 };
16900 for &v in targets {
16901 if let Some(table) = self.graph.table_by_vertex(v) {
16902 let (s, e) = (table.range.start, table.range.end);
16906 push_formulas(
16907 &mut stack,
16908 &visited,
16909 s.sheet_id,
16910 Rect::new(s.coord.row(), s.coord.col(), e.coord.row(), e.coord.col()),
16911 );
16912 } else if let Some(cell) = self.graph.authority_cell_of_vertex(v) {
16913 stack.push((cell, crate::engine::authority::identity::NO_VID));
16914 }
16915 }
16916 let mut hits = Vec::new();
16917 #[cfg(any(test, feature = "benchmark_internal"))]
16918 let (mut probe_vertices, mut probe_clean_formulas, mut probe_edges, mut probe_dynamic) =
16919 (0, 0, 0, 0);
16920 while let Some((cell, id)) = stack.pop() {
16921 if !visited.insert(cell) {
16922 continue;
16923 }
16924 let Some(v) = self.graph.authority_vertex_of_formula(id, cell) else {
16925 continue;
16926 };
16927 if !self.graph.vertex_exists(v) {
16928 continue;
16929 }
16930 #[cfg(any(test, feature = "benchmark_internal"))]
16931 {
16932 probe_vertices += 1;
16933 }
16934 match self.graph.get_vertex_kind(v) {
16935 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
16936 if self.graph.is_dirty(v) || self.graph.is_volatile(v) {
16937 to_evaluate.insert(v);
16938 } else {
16939 #[cfg(any(test, feature = "benchmark_internal"))]
16940 {
16941 probe_clean_formulas += 1;
16942 }
16943 }
16944 }
16945 VertexKind::NamedScalar | VertexKind::NamedArray => {
16946 to_evaluate.insert(v);
16947 }
16948 _ => {}
16949 }
16950 hits.clear();
16951 store.direct_precedents(cell, TagFilter::All, &mut hits);
16952 #[cfg(any(test, feature = "benchmark_internal"))]
16953 {
16954 probe_edges += hits.len();
16955 }
16956 for &(_, sheet, rect) in &hits {
16957 if sheet != SYMBOL_SHEET {
16960 for anchor in self
16961 .graph
16962 .spill_anchors_in_region(sheet, rect.r0, rect.c0, rect.r1, rect.c1)
16963 {
16964 if anchor != v && self.graph.is_dirty(anchor) {
16965 vdeps.entry(v).or_default().push(anchor);
16966 if let Some(c) = self.graph.authority_cell_of_vertex(anchor) {
16967 stack.push((c, crate::engine::authority::identity::NO_VID));
16968 }
16969 }
16970 }
16971 }
16972 if sheet == SYMBOL_SHEET {
16973 stack.extend((rect.r0..=rect.r1).map(|slot| {
16974 (
16975 (SYMBOL_SHEET, slot, 0),
16976 crate::engine::authority::identity::NO_VID,
16977 )
16978 }));
16979 continue;
16980 }
16981 push_formulas(&mut stack, &visited, sheet, rect);
16982 }
16983 if self.graph.is_dynamic(v) {
16984 #[cfg(any(test, feature = "benchmark_internal"))]
16985 {
16986 probe_dynamic += 1;
16987 }
16988 if let Some(reads) = self.graph.authority_host().observed(v) {
16990 for &(sheet, r0, c0, r1, c1) in reads {
16991 push_formulas(&mut stack, &visited, sheet, Rect::new(r0, c0, r1, c1));
16992 }
16993 }
16994 let (vdeps_map, _) = VirtualDepBuilder::new(self).build(&[v]);
16995 let hinted = self.freshness_hints(v).unwrap_or(&[]);
16998 if let Some(deps) = vdeps_map
16999 .get(&v)
17000 .map(|deps| deps.iter().chain(hinted))
17001 .or(Some([].iter().chain(hinted)))
17002 {
17003 for &u in deps {
17004 vdeps.entry(v).or_default().push(u);
17005 if let Some(c) = self.graph.authority_cell_of_vertex(u) {
17006 stack.push((c, crate::engine::authority::identity::NO_VID));
17007 }
17008 }
17009 }
17010 }
17011 }
17012 let mut result: Vec<VertexId> = to_evaluate.into_iter().collect();
17013 result.sort_unstable();
17014 for deps in vdeps.values_mut() {
17015 deps.sort_unstable();
17016 deps.dedup();
17017 }
17018 #[cfg(any(test, feature = "benchmark_internal"))]
17019 {
17020 let mut probe = self.recalc_reuse_probe.lock().unwrap();
17021 probe.demand_builds += 1;
17022 probe.demand_vertices += probe_vertices;
17023 probe.demand_clean_formulas += probe_clean_formulas;
17024 probe.demand_explicit_edges += probe_edges;
17025 probe.demand_virtual_builder_calls += probe_dynamic;
17026 }
17027 Ok((result, vdeps))
17028 }
17029
17030 fn col_to_letters(col: u32) -> String {
17032 col_letters_from_1based(col).expect("column index must be >= 1")
17033 }
17034
17035 pub fn evaluate_all_cancellable(
17037 &mut self,
17038 cancel: crate::engine::CancelToken,
17039 ) -> Result<EvalResult, ExcelError> {
17040 self.observe_evaluation_resource_request(EvaluationRequestKind::FullCancellable, |engine| {
17041 engine.observe_function_semantic_epoch()?;
17042 engine.active_cancel_flag = Some(cancel.clone());
17043 let res = engine.evaluate_all_cancellable_impl(cancel.as_flag());
17044 engine.active_cancel_flag = None;
17045 res
17046 })
17047 }
17048
17049 fn evaluate_all_cancellable_impl(
17050 &mut self,
17051 cancel_flag: &AtomicBool,
17052 ) -> Result<EvalResult, ExcelError> {
17053 let _source_cache = self.source_cache_session();
17054 self.validate_deterministic_mode()?;
17055 if self.config.defer_graph_building {
17056 self.build_graph_all()?;
17057 }
17058 if cancel_flag.load(Ordering::Relaxed) {
17059 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
17060 .with_message("Evaluation cancelled before scheduling".to_string()));
17061 }
17062 self.require_unified_authority()?;
17063 self.begin_evaluation_request();
17064 self.reset_virtual_dep_telemetry_if_disabled();
17065 let start = crate::instant::FzInstant::now();
17066 let mut computed_vertices = 0;
17067 let mut cycle_errors = 0;
17068
17069 let mut replan_iterations = 0;
17070 const MAX_REPLAN: usize = 5;
17071 let mut telemetry = self
17072 .config
17073 .enable_virtual_dep_telemetry
17074 .then(|| self.start_virtual_dep_telemetry());
17075
17076 loop {
17077 if cancel_flag.load(Ordering::Relaxed) {
17078 if let Some(mut t) = telemetry {
17079 t.bailout_reason = Some("cancelled");
17080 t.replan_iterations = replan_iterations;
17081 self.last_virtual_dep_telemetry = t;
17082 }
17083 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
17084 .with_message("Evaluation cancelled before scheduling".to_string()));
17085 }
17086
17087 let to_evaluate = self.graph.get_evaluation_vertices();
17088 if to_evaluate.is_empty() {
17089 if let Some(t) = telemetry.as_mut()
17090 && t.bailout_reason.is_none()
17091 {
17092 t.bailout_reason = Some("no_work");
17093 }
17094 break;
17095 }
17096
17097 let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
17098 if let Some(t) = telemetry.as_mut() {
17099 Self::accumulate_schedule_meta(t, &meta);
17100 }
17101
17102 self.begin_pass(&schedule);
17105 for (unit_index, &unit) in schedule.units.iter().enumerate() {
17106 match unit {
17107 ScheduleUnit::Cycle(i) => {
17108 if cancel_flag.load(Ordering::Relaxed) {
17110 if let Some(mut t) = telemetry {
17111 t.bailout_reason = Some("cancelled");
17112 t.replan_iterations = replan_iterations;
17113 self.last_virtual_dep_telemetry = t;
17114 }
17115 return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(
17116 "Evaluation cancelled during cycle handling".to_string(),
17117 ));
17118 }
17119
17120 if self.handle_cycle_unit(
17121 schedule.unit_cycle(i),
17122 None,
17123 None,
17124 Some(cancel_flag),
17125 )? > 0
17126 {
17127 cycle_errors += 1;
17128 }
17129 }
17130 ScheduleUnit::Layer(i) => {
17131 let layer = schedule.unit_layer(i);
17132 if cancel_flag.load(Ordering::Relaxed) {
17134 if let Some(mut t) = telemetry {
17135 t.bailout_reason = Some("cancelled");
17136 t.replan_iterations = replan_iterations;
17137 self.last_virtual_dep_telemetry = t;
17138 }
17139 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
17140 .with_message("Evaluation cancelled between layers".to_string()));
17141 }
17142
17143 if self.thread_pool.is_some() && layer.vertices.len() > 1 {
17145 computed_vertices +=
17146 self.evaluate_layer_parallel_cancellable(layer, cancel_flag)?;
17147 } else {
17148 computed_vertices +=
17149 self.evaluate_layer_sequential_cancellable(layer, cancel_flag)?;
17150 }
17151 }
17152 }
17153 if self.stop_after_unit(&schedule, unit_index) {
17154 break;
17155 }
17156 }
17157
17158 let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
17159 if let Some(t) = telemetry.as_mut() {
17160 t.changed_vdeps_total += changed_vertices.len();
17161 }
17162 self.resource_checkpoint(0)?;
17163 if !self.finish_pass_dirty(&to_evaluate, &changed_vertices) {
17164 if let Some(t) = telemetry.as_mut() {
17165 t.bailout_reason = Some("converged");
17166 }
17167 break;
17168 }
17169 if replan_iterations >= MAX_REPLAN {
17170 if let Some(mut t) = telemetry.take() {
17171 t.bailout_reason = Some("max_replan");
17172 t.replan_iterations = replan_iterations;
17173 self.last_virtual_dep_telemetry = t;
17174 }
17175 return Err(
17176 self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
17177 );
17178 }
17179 replan_iterations += 1;
17180 }
17181
17182 if let Some(mut t) = telemetry {
17183 t.replan_iterations = replan_iterations;
17184 self.last_virtual_dep_telemetry = t;
17185 }
17186
17187 self.redirty_for_next_recalc();
17189 self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
17190
17191 Ok(EvalResult {
17192 computed_vertices,
17193 cycle_errors,
17194 elapsed: start.elapsed(),
17195 })
17196 }
17197
17198 pub fn evaluate_until_cancellable(
17200 &mut self,
17201 targets: &[&str],
17202 cancel: crate::engine::CancelToken,
17203 ) -> Result<EvalResult, ExcelError> {
17204 self.observe_evaluation_resource_request(
17205 EvaluationRequestKind::TargetedCancellable,
17206 |engine| {
17207 engine.observe_function_semantic_epoch()?;
17208 engine.active_cancel_flag = Some(cancel.clone());
17209 let res = engine.evaluate_until_cancellable_impl(targets, cancel.as_flag());
17210 engine.active_cancel_flag = None;
17211 res
17212 },
17213 )
17214 }
17215
17216 fn evaluate_until_cancellable_impl(
17217 &mut self,
17218 targets: &[&str],
17219 cancel_flag: &AtomicBool,
17220 ) -> Result<EvalResult, ExcelError> {
17221 if cancel_flag.load(Ordering::Relaxed) {
17222 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
17223 .with_message("Evaluation cancelled before target preparation"));
17224 }
17225 let mut typed_targets = Vec::with_capacity(targets.len());
17226 for target in targets {
17227 let (sheet, row, col) = self.parse_a1_notation(target)?;
17228 self.graph.sheet_id_mut(&sheet);
17229 typed_targets.push(crate::engine::EvaluationTarget::Cell { sheet, row, col });
17230 }
17231 self.evaluate_mixed_targets(&typed_targets, None)
17232 }
17233
17234 fn parse_a1_notation(&self, address: &str) -> Result<(String, u32, u32), ExcelError> {
17235 let mut quoted = false;
17236 let mut separator = None;
17237 let bytes = address.as_bytes();
17238 let mut index = 0usize;
17239 while index < bytes.len() {
17240 match bytes[index] {
17241 b'\'' => {
17242 if quoted && bytes.get(index + 1) == Some(&b'\'') {
17243 index = index.saturating_add(1);
17244 } else {
17245 quoted = !quoted;
17246 }
17247 }
17248 b'!' if !quoted => separator = Some(index),
17249 _ => {}
17250 }
17251 index = index.saturating_add(1);
17252 }
17253 if quoted {
17254 return Err(ExcelError::new(ExcelErrorKind::Ref)
17255 .with_message(format!("Invalid quoted sheet reference `{address}`")));
17256 }
17257 let (sheet, cell_part) = match separator {
17258 Some(separator) => {
17259 let raw_sheet = &address[..separator];
17260 let sheet = if raw_sheet.starts_with('\'') && raw_sheet.ends_with('\'') {
17261 raw_sheet[1..raw_sheet.len().saturating_sub(1)].replace("''", "'")
17262 } else {
17263 raw_sheet.to_string()
17264 };
17265 (sheet, &address[separator + 1..])
17266 }
17267 None => (self.default_sheet_name().to_string(), address),
17268 };
17269
17270 let (row, col, _, _) = parse_a1_1based(cell_part).map_err(|err| {
17271 ExcelError::new(ExcelErrorKind::Ref)
17272 .with_message(format!("Invalid cell reference `{cell_part}`: {err}"))
17273 })?;
17274
17275 Ok((sheet, row, col))
17276 }
17277
17278 fn is_ast_volatile_with_provider(&self, ast: &ASTNode) -> bool {
17280 use formualizer_parse::parser::ASTNodeType;
17281 match &ast.node_type {
17282 ASTNodeType::Function { name, args, .. } => {
17283 if let Some(func) = self
17284 .get_function("", name)
17285 .or_else(|| crate::function_registry::get("", name))
17286 && func.caps().contains(crate::function::FnCaps::VOLATILE)
17287 {
17288 return true;
17289 }
17290 args.iter()
17291 .any(|arg| self.is_ast_volatile_with_provider(arg))
17292 }
17293 ASTNodeType::BinaryOp { left, right, .. } => {
17294 self.is_ast_volatile_with_provider(left)
17295 || self.is_ast_volatile_with_provider(right)
17296 }
17297 ASTNodeType::UnaryOp { expr, .. } => self.is_ast_volatile_with_provider(expr),
17298 ASTNodeType::Array(rows) => rows.iter().any(|row| {
17299 row.iter()
17300 .any(|cell| self.is_ast_volatile_with_provider(cell))
17301 }),
17302 _ => false,
17303 }
17304 }
17305
17306 fn evaluate_layer_sequential(
17308 &mut self,
17309 layer: &super::scheduler::Layer,
17310 ) -> Result<usize, ExcelError> {
17311 self.resource_checkpoint(layer.vertices.len() as u64)?;
17312 self.evaluate_layer_sequential_effects(layer)
17313 }
17314
17315 fn update_vertex_value_with_delta(
17316 &mut self,
17317 vertex_id: VertexId,
17318 new_value: LiteralValue,
17319 delta: &mut DeltaCollector,
17320 ) {
17321 if delta.mode != DeltaMode::Off
17322 && let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id)
17323 {
17324 let sheet_name = self.graph.sheet_name(cell.sheet_id);
17325 let old = self
17326 .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
17327 .unwrap_or(LiteralValue::Empty);
17328 if old != new_value {
17329 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
17330 }
17331 }
17332 self.graph.update_vertex_value_ref(vertex_id, &new_value);
17333 self.mirror_vertex_value_to_overlay(vertex_id, &new_value);
17334 }
17335
17336 fn evaluate_layer_sequential_with_delta(
17337 &mut self,
17338 layer: &super::scheduler::Layer,
17339 delta: &mut DeltaCollector,
17340 ) -> Result<usize, ExcelError> {
17341 self.resource_checkpoint(layer.vertices.len() as u64)?;
17342 self.evaluate_layer_sequential_with_delta_effects(layer, delta)
17343 }
17344
17345 fn evaluate_layer_sequential_cancellable(
17347 &mut self,
17348 layer: &super::scheduler::Layer,
17349 cancel_flag: &AtomicBool,
17350 ) -> Result<usize, ExcelError> {
17351 self.resource_checkpoint(layer.vertices.len() as u64)?;
17352 self.evaluate_layer_sequential_cancellable_effects(layer, cancel_flag)
17353 }
17354
17355 fn evaluate_layer_sequential_cancellable_demand_driven(
17357 &mut self,
17358 layer: &super::scheduler::Layer,
17359 cancel_flag: &AtomicBool,
17360 ) -> Result<usize, ExcelError> {
17361 self.resource_checkpoint(layer.vertices.len() as u64)?;
17362 self.evaluate_layer_sequential_cancellable_demand_driven_effects(layer, cancel_flag)
17363 }
17364
17365 fn evaluate_layer_parallel(
17377 &mut self,
17378 layer: &super::scheduler::Layer,
17379 ) -> Result<usize, ExcelError> {
17380 if layer.sequential {
17381 return self.evaluate_layer_sequential(layer);
17382 }
17383 self.resource_checkpoint(layer.vertices.len() as u64)?;
17384 let len = layer.vertices.len();
17385 let buffered = buffer_layer_writes(layer);
17386 let (probe, worth) = (PARALLEL_LAYER_PROBE, PARALLEL_LAYER_WORTH);
17387 let start = crate::instant::FzInstant::now();
17388 let mut pos = 0usize;
17389 let mut step = 1usize;
17390 while pos < len {
17391 if pos > 0 {
17392 let elapsed = start.elapsed();
17393 let per_vertex = elapsed.as_nanos() / pos as u128 + 1;
17395 let rest_estimate = per_vertex * (len - pos) as u128;
17396 if len - pos >= 2 && (elapsed >= probe || rest_estimate >= worth.as_nanos()) {
17397 let rest = layer.sub_layer(pos, len);
17398 let min_chunk = if per_vertex >= EXPENSIVE_VERTEX_NS {
17401 1
17402 } else {
17403 8
17404 };
17405 return Ok(pos + self.evaluate_layer_parallel_effects(&rest, min_chunk)?);
17406 }
17407 let fit = (probe.saturating_sub(elapsed).as_nanos() / per_vertex) as usize + 1;
17410 step = step.saturating_mul(2).min(fit);
17411 }
17412 let end = (pos + step).min(len);
17413 let slice = layer.sub_layer(pos, end);
17414 pos += self.evaluate_layer_units_until(
17417 &slice,
17418 None,
17419 None,
17420 None,
17421 buffered,
17422 Some(start + probe),
17423 )?;
17424 }
17425 Ok(len)
17426 }
17427
17428 fn evaluate_layer_parallel_with_delta(
17429 &mut self,
17430 layer: &super::scheduler::Layer,
17431 delta: &mut DeltaCollector,
17432 ) -> Result<usize, ExcelError> {
17433 if layer.sequential {
17434 return self.evaluate_layer_sequential_with_delta(layer, delta);
17435 }
17436 self.resource_checkpoint(layer.vertices.len() as u64)?;
17437 self.evaluate_layer_parallel_with_delta_effects(layer, delta)
17438 }
17439
17440 fn evaluate_layer_parallel_cancellable(
17442 &mut self,
17443 layer: &super::scheduler::Layer,
17444 cancel_flag: &AtomicBool,
17445 ) -> Result<usize, ExcelError> {
17446 if layer.sequential {
17447 return self.evaluate_layer_sequential_cancellable(layer, cancel_flag);
17448 }
17449 self.resource_checkpoint(layer.vertices.len() as u64)?;
17450 self.evaluate_layer_parallel_cancellable_effects(layer, cancel_flag)
17451 }
17452
17453 fn evaluate_vertex_immutable(&self, vertex_id: VertexId) -> Result<LiteralValue, ExcelError> {
17455 if !self.graph.vertex_exists(vertex_id) {
17457 return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
17458 .with_message(format!("Vertex not found: {vertex_id:?}")));
17459 }
17460
17461 let kind = self.graph.get_vertex_kind(vertex_id);
17463 let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
17464
17465 let view = match kind {
17466 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
17467 if let Some(view) = self.graph.formula_view(vertex_id) {
17468 view
17469 } else {
17470 return Ok(LiteralValue::Number(0.0));
17471 }
17472 }
17473 VertexKind::Empty | VertexKind::Cell => {
17474 if let Some(cell_ref) = self.graph.get_cell_ref(vertex_id) {
17475 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
17476 let row = cell_ref.coord.row() + 1;
17477 let col = cell_ref.coord.col() + 1;
17478 if let Some(v) = self.read_cell_value(sheet_name, row, col) {
17479 return Ok(v);
17480 }
17481 }
17482 return Ok(LiteralValue::Number(0.0));
17483 }
17484 VertexKind::NamedScalar => {
17485 let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
17486 ExcelError::new(ExcelErrorKind::Name)
17487 .with_message("Named range metadata missing".to_string())
17488 })?;
17489
17490 return match &named_range.definition {
17491 NamedDefinition::Cell(cell_ref) => {
17492 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
17493 Ok(self
17494 .read_cell_value(
17495 sheet_name,
17496 cell_ref.coord.row() + 1,
17497 cell_ref.coord.col() + 1,
17498 )
17499 .unwrap_or(LiteralValue::Empty))
17500 }
17501 NamedDefinition::Literal(v) => Ok(v.clone()),
17502 NamedDefinition::Formula { ast, .. } => {
17503 let context_sheet = match named_range.scope {
17504 NameScope::Sheet(id) => id,
17505 NameScope::Workbook => sheet_id,
17506 };
17507 let sheet_name = self.graph.sheet_name(context_sheet);
17508 let cell_ref = self
17509 .graph
17510 .get_cell_ref(vertex_id)
17511 .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
17512 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref)
17513 .with_resolved_functions(&self.resolved_functions)
17514 .with_arena_cells(self);
17515 interpreter.evaluate_ast(ast).map(|cv| cv.into_literal())
17516 }
17517 NamedDefinition::Range(_) => Err(ExcelError::new(ExcelErrorKind::Value)
17518 .with_message("Range-valued name evaluated as scalar".to_string())),
17519 };
17520 }
17521 VertexKind::NamedArray => {
17522 let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
17523 ExcelError::new(ExcelErrorKind::Name)
17524 .with_message("Named range metadata missing".to_string())
17525 })?;
17526
17527 return match &named_range.definition {
17528 NamedDefinition::Range(range_ref) => {
17529 if range_ref.start.sheet_id != range_ref.end.sheet_id {
17530 return Err(ExcelError::new(ExcelErrorKind::Ref)
17531 .with_message("Named range cannot span sheets".to_string()));
17532 }
17533 let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
17534 let sr0 = range_ref.start.coord.row();
17535 let sc0 = range_ref.start.coord.col();
17536 let er0 = range_ref.end.coord.row();
17537 let ec0 = range_ref.end.coord.col();
17538 if sr0 > er0 || sc0 > ec0 {
17539 return Err(ExcelError::new(ExcelErrorKind::Ref)
17540 .with_message("Invalid named range bounds".to_string()));
17541 }
17542
17543 let h = (er0 - sr0 + 1) as usize;
17544 let w = (ec0 - sc0 + 1) as usize;
17545 let cell_count = (h as u64).saturating_mul(w as u64);
17546 if cell_count > self.config.spill.max_spill_cells as u64 {
17547 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
17548 "Named range too large to materialize as an array".to_string(),
17549 ));
17550 }
17551
17552 let sheet_id = range_ref.start.sheet_id;
17554 let asheet = self.arrow_sheets.sheet(sheet_name);
17555 let mut rows = Vec::with_capacity(h);
17556 for r0 in sr0..=er0 {
17557 let mut row = Vec::with_capacity(w);
17558 for c0 in sc0..=ec0 {
17559 row.push(
17560 self.read_cell_formatted_in(sheet_id, asheet, r0 + 1, c0 + 1)
17561 .0,
17562 );
17563 }
17564 rows.push(row);
17565 }
17566 Ok(LiteralValue::Array(rows))
17567 }
17568 NamedDefinition::Cell(cell_ref) => {
17569 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
17570 let row = cell_ref.coord.row() + 1;
17571 let col = cell_ref.coord.col() + 1;
17572 let v = self
17573 .read_cell_value(sheet_name, row, col)
17574 .unwrap_or(LiteralValue::Empty);
17575 Ok(LiteralValue::Array(vec![vec![v]]))
17576 }
17577 NamedDefinition::Literal(v) => Ok(LiteralValue::Array(vec![vec![v.clone()]])),
17578 NamedDefinition::Formula { ast, .. } => {
17579 let context_sheet = match named_range.scope {
17580 NameScope::Sheet(id) => id,
17581 NameScope::Workbook => sheet_id,
17582 };
17583 let sheet_name = self.graph.sheet_name(context_sheet);
17584 let cell_ref = self
17585 .graph
17586 .get_cell_ref(vertex_id)
17587 .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
17588 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref)
17589 .with_resolved_functions(&self.resolved_functions)
17590 .with_arena_cells(self);
17591 match interpreter.evaluate_ast(ast) {
17592 Ok(cv) => {
17593 let v = cv.into_literal();
17594 match v {
17595 LiteralValue::Array(_) => Ok(v),
17596 other => Ok(LiteralValue::Array(vec![vec![other]])),
17597 }
17598 }
17599 Err(err) => Ok(LiteralValue::Error(err)),
17600 }
17601 }
17602 };
17603 }
17604 VertexKind::InfiniteRange
17605 | VertexKind::Range
17606 | VertexKind::External
17607 | VertexKind::Table => {
17608 return Ok(LiteralValue::Number(0.0));
17610 }
17611 };
17612
17613 let sheet_name = self.graph.sheet_name(sheet_id);
17615 let cell_ref = self
17616 .graph
17617 .get_cell_ref(vertex_id)
17618 .expect("cell ref for vertex");
17619 if let Some(result) =
17620 self.freshness_evaluate_recorded(vertex_id, sheet_name, cell_ref, view)
17621 {
17622 return result;
17623 }
17624 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref)
17625 .with_resolved_functions(&self.resolved_functions)
17626 .with_arena_cells(self);
17627
17628 interpreter
17629 .evaluate_formula_view(view, self.graph.data_store(), self.graph.sheet_reg())
17630 .map(|cv| {
17631 let format = cv.format_id();
17632 self.record_derived_format(vertex_id, format);
17633 self.materialize_formula_result(vertex_id, cv)
17634 })
17635 .or_else(|error| self.fit_formula_error(vertex_id, error))
17636 }
17637
17638 pub fn thread_pool(&self) -> Option<&Arc<rayon::ThreadPool>> {
17640 self.thread_pool.as_ref()
17641 }
17642}
17643
17644#[derive(Default)]
17645struct RowBoundsCache {
17646 snapshot: u64,
17647 map: rustc_hash::FxHashMap<(u32, usize), (Option<u32>, Option<u32>)>,
17649}
17650
17651impl RowBoundsCache {
17652 fn new(snapshot: u64) -> Self {
17653 Self {
17654 snapshot,
17655 map: Default::default(),
17656 }
17657 }
17658 fn get_row_bounds(
17659 &self,
17660 sheet_id: SheetId,
17661 col_idx: usize,
17662 snapshot: u64,
17663 ) -> Option<(Option<u32>, Option<u32>)> {
17664 if self.snapshot != snapshot {
17665 return None;
17666 }
17667 self.map.get(&(sheet_id as u32, col_idx)).copied()
17668 }
17669 fn put_row_bounds(
17670 &mut self,
17671 sheet_id: SheetId,
17672 col_idx: usize,
17673 snapshot: u64,
17674 bounds: (Option<u32>, Option<u32>),
17675 ) {
17676 if self.snapshot != snapshot {
17677 self.snapshot = snapshot;
17678 self.map.clear();
17679 }
17680 self.map.insert((sheet_id as u32, col_idx), bounds);
17681 }
17682}
17683
17684struct UsedAxisBoundsCache {
17685 snapshot: u64,
17686 row_bounds_by_col_span: rustc_hash::FxHashMap<(SheetId, u32, u32), Option<(u32, u32)>>,
17687 col_bounds_by_row_span: rustc_hash::FxHashMap<(SheetId, u32, u32), Option<(u32, u32)>>,
17688 #[cfg(test)]
17689 row_hits: std::sync::atomic::AtomicUsize,
17690 #[cfg(test)]
17691 row_misses: std::sync::atomic::AtomicUsize,
17692 #[cfg(test)]
17693 col_hits: std::sync::atomic::AtomicUsize,
17694 #[cfg(test)]
17695 col_misses: std::sync::atomic::AtomicUsize,
17696}
17697
17698impl UsedAxisBoundsCache {
17699 fn new(snapshot: u64) -> Self {
17700 Self {
17701 snapshot,
17702 row_bounds_by_col_span: Default::default(),
17703 col_bounds_by_row_span: Default::default(),
17704 #[cfg(test)]
17705 row_hits: std::sync::atomic::AtomicUsize::new(0),
17706 #[cfg(test)]
17707 row_misses: std::sync::atomic::AtomicUsize::new(0),
17708 #[cfg(test)]
17709 col_hits: std::sync::atomic::AtomicUsize::new(0),
17710 #[cfg(test)]
17711 col_misses: std::sync::atomic::AtomicUsize::new(0),
17712 }
17713 }
17714
17715 fn reset_for_snapshot(&mut self, snapshot: u64) {
17716 if self.snapshot != snapshot {
17717 self.snapshot = snapshot;
17718 self.row_bounds_by_col_span.clear();
17719 self.col_bounds_by_row_span.clear();
17720 }
17721 }
17722
17723 fn get_row_bounds(
17724 &self,
17725 sheet_id: SheetId,
17726 start_col: u32,
17727 end_col: u32,
17728 snapshot: u64,
17729 ) -> Option<Option<(u32, u32)>> {
17730 if self.snapshot != snapshot {
17731 return None;
17732 }
17733 let cached = self
17734 .row_bounds_by_col_span
17735 .get(&(sheet_id, start_col, end_col))
17736 .copied();
17737 #[cfg(test)]
17738 if cached.is_some() {
17739 self.row_hits.fetch_add(1, Ordering::Relaxed);
17740 }
17741 cached
17742 }
17743
17744 fn put_row_bounds(
17745 &mut self,
17746 sheet_id: SheetId,
17747 start_col: u32,
17748 end_col: u32,
17749 snapshot: u64,
17750 bounds: Option<(u32, u32)>,
17751 ) {
17752 self.reset_for_snapshot(snapshot);
17753 self.row_bounds_by_col_span
17754 .insert((sheet_id, start_col, end_col), bounds);
17755 #[cfg(test)]
17756 self.row_misses.fetch_add(1, Ordering::Relaxed);
17757 }
17758
17759 fn get_col_bounds(
17760 &self,
17761 sheet_id: SheetId,
17762 start_row: u32,
17763 end_row: u32,
17764 snapshot: u64,
17765 ) -> Option<Option<(u32, u32)>> {
17766 if self.snapshot != snapshot {
17767 return None;
17768 }
17769 let cached = self
17770 .col_bounds_by_row_span
17771 .get(&(sheet_id, start_row, end_row))
17772 .copied();
17773 #[cfg(test)]
17774 if cached.is_some() {
17775 self.col_hits.fetch_add(1, Ordering::Relaxed);
17776 }
17777 cached
17778 }
17779
17780 fn put_col_bounds(
17781 &mut self,
17782 sheet_id: SheetId,
17783 start_row: u32,
17784 end_row: u32,
17785 snapshot: u64,
17786 bounds: Option<(u32, u32)>,
17787 ) {
17788 self.reset_for_snapshot(snapshot);
17789 self.col_bounds_by_row_span
17790 .insert((sheet_id, start_row, end_row), bounds);
17791 #[cfg(test)]
17792 self.col_misses.fetch_add(1, Ordering::Relaxed);
17793 }
17794}
17795
17796#[derive(Default)]
17798pub struct ShimSpillManager {
17799 region_locks: RegionLockManager,
17800 pub(crate) active_locks: rustc_hash::FxHashMap<VertexId, u64>,
17801}
17802
17803impl ShimSpillManager {
17804 pub(crate) fn reserve(
17805 &mut self,
17806 owner: VertexId,
17807 anchor_cell: CellRef,
17808 shape: SpillShape,
17809 _meta: SpillMeta,
17810 ) -> Result<(), ExcelError> {
17811 let region = crate::engine::spill::Region {
17813 sheet_id: anchor_cell.sheet_id as u32,
17814 row_start: anchor_cell.coord.row(),
17815 row_end: anchor_cell
17816 .coord
17817 .row()
17818 .saturating_add(shape.rows)
17819 .saturating_sub(1),
17820 col_start: anchor_cell.coord.col(),
17821 col_end: anchor_cell
17822 .coord
17823 .col()
17824 .saturating_add(shape.cols)
17825 .saturating_sub(1),
17826 };
17827 match self.region_locks.reserve(region, owner) {
17828 Ok(id) => {
17829 if id != 0 {
17830 self.active_locks.insert(owner, id);
17831 }
17832 Ok(())
17833 }
17834 Err(e) => Err(e),
17835 }
17836 }
17837
17838 pub(crate) fn release_owner(&mut self, owner: VertexId) {
17844 if let Some(id) = self.active_locks.remove(&owner) {
17845 self.region_locks.release(id);
17846 }
17847 }
17848
17849 pub(crate) fn commit_array_with_value_probe<F>(
17850 &mut self,
17851 graph: &mut DependencyGraph,
17852 anchor_vertex: VertexId,
17853 targets: &[CellRef],
17854 rows: Vec<Vec<LiteralValue>>,
17855 mut value_probe: F,
17856 ) -> Result<(), ExcelError>
17857 where
17858 F: FnMut(&DependencyGraph, &CellRef) -> Option<LiteralValue>,
17859 {
17860 use formualizer_common::{ExcelErrorExtra, ExcelErrorKind};
17861
17862 let plan_res = graph.plan_spill_region(anchor_vertex, targets);
17866 if let Err(e) = plan_res {
17867 if let Some(id) = self.active_locks.remove(&anchor_vertex) {
17868 self.region_locks.release(id);
17869 }
17870 return Err(e);
17871 }
17872
17873 if !graph.value_cache_enabled() {
17874 let (expected_rows, expected_cols) = if targets.is_empty() {
17876 (0u32, 0u32)
17877 } else {
17878 let mut min_r = u32::MAX;
17879 let mut max_r = 0u32;
17880 let mut min_c = u32::MAX;
17881 let mut max_c = 0u32;
17882 for cell in targets {
17883 let r = cell.coord.row();
17884 let c = cell.coord.col();
17885 min_r = min_r.min(r);
17886 max_r = max_r.max(r);
17887 min_c = min_c.min(c);
17888 max_c = max_c.max(c);
17889 }
17890 (
17891 max_r.saturating_sub(min_r).saturating_add(1),
17892 max_c.saturating_sub(min_c).saturating_add(1),
17893 )
17894 };
17895
17896 let anchor_cell = graph
17897 .get_cell_ref(anchor_vertex)
17898 .expect("anchor cell ref for spill commit");
17899
17900 for cell in targets {
17901 if *cell == anchor_cell {
17903 continue;
17904 }
17905 if graph.spill_registry_anchor_for_cell(*cell).is_some() {
17907 continue;
17908 }
17909 if let Some(vid) = graph.get_vertex_id_for_address(cell)
17911 && vid != anchor_vertex
17912 {
17913 match graph.get_vertex_kind(vid) {
17914 crate::engine::vertex::VertexKind::FormulaScalar
17915 | crate::engine::vertex::VertexKind::FormulaArray => {
17916 continue;
17918 }
17919 _ => {}
17920 }
17921 }
17922
17923 if let Some(v) = value_probe(graph, cell)
17924 && !matches!(v, LiteralValue::Empty)
17925 {
17926 if let Some(id) = self.active_locks.remove(&anchor_vertex) {
17927 self.region_locks.release(id);
17928 }
17929 return Err(ExcelError::new(ExcelErrorKind::Spill)
17930 .with_message("BlockedByValue")
17931 .with_extra(ExcelErrorExtra::Spill {
17932 expected_rows,
17933 expected_cols,
17934 }));
17935 }
17936 }
17937 }
17938
17939 let commit_res = graph.commit_spill_region_atomic_with_fault(
17940 anchor_vertex,
17941 targets.to_vec(),
17942 rows,
17943 None,
17944 );
17945 if let Some(id) = self.active_locks.remove(&anchor_vertex) {
17946 self.region_locks.release(id);
17947 }
17948 commit_res.map(|_| ())
17949 }
17950
17951 pub(crate) fn commit_array_with_overlay<R: EvaluationContext>(
17953 &mut self,
17954 engine: &mut Engine<R>,
17955 anchor_vertex: VertexId,
17956 targets: &[CellRef],
17957 rows: Vec<Vec<LiteralValue>>,
17958 ) -> Result<(), ExcelError> {
17959 if let Err(error) = engine.guard_pending_spill_commit(anchor_vertex, targets) {
17960 self.release_owner(anchor_vertex);
17961 return Err(error);
17962 }
17963 let plan_res = engine.graph.plan_spill_region(anchor_vertex, targets);
17965 if let Err(e) = plan_res {
17966 if let Some(id) = self.active_locks.remove(&anchor_vertex) {
17967 self.region_locks.release(id);
17968 }
17969 return Err(e);
17970 }
17971
17972 let commit_res = engine.graph.commit_spill_region_atomic_with_fault(
17973 anchor_vertex,
17974 targets.to_vec(),
17975 rows.clone(),
17976 None,
17977 );
17978 if let Some(id) = self.active_locks.remove(&anchor_vertex) {
17979 self.region_locks.release(id);
17980 }
17981 commit_res.map(|_| ())?;
17982 engine
17983 .blocked_pending_spills
17984 .retain(|entry| entry.0 != anchor_vertex);
17985 engine.spill_blocker_waiters.forget(anchor_vertex);
17986
17987 if engine.config.arrow_storage_enabled
17989 && engine.config.delta_overlay_enabled
17990 && engine.config.write_formula_overlay_enabled
17991 {
17992 for (idx, cell) in targets.iter().enumerate() {
17994 let (r_off, c_off) = {
17995 if rows.is_empty() || rows[0].is_empty() {
17996 (0usize, 0usize)
17997 } else {
17998 let width = rows[0].len();
17999 (idx / width, idx % width)
18000 }
18001 };
18002 let v = rows
18003 .get(r_off)
18004 .and_then(|r| r.get(c_off))
18005 .cloned()
18006 .unwrap_or(LiteralValue::Empty);
18007 let sheet_name = engine.graph.sheet_name(cell.sheet_id).to_string();
18008 engine.mirror_value_to_computed_overlay(
18009 &sheet_name,
18010 cell.coord.row() + 1,
18011 cell.coord.col() + 1,
18012 &v,
18013 );
18014 }
18015 }
18016 Ok(())
18017 }
18018}
18019
18020impl<R> Engine<R>
18021where
18022 R: EvaluationContext,
18023{
18024 fn resolve_shared_ref(
18025 &self,
18026 reference: &ReferenceType,
18027 current_sheet: &str,
18028 ) -> Result<formualizer_common::SheetRef<'static>, ExcelError> {
18029 use formualizer_common::{
18030 SheetCellRef as SharedCellRef, SheetLocator, SheetRangeRef as SharedRangeRef,
18031 SheetRef as SharedRef,
18032 };
18033
18034 let sr = match reference {
18036 ReferenceType::Cell {
18037 sheet,
18038 row,
18039 col,
18040 row_abs,
18041 col_abs,
18042 } => {
18043 let row0 = row
18044 .checked_sub(1)
18045 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18046 let col0 = col
18047 .checked_sub(1)
18048 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18049 let sheet_loc = match sheet.as_deref() {
18050 Some(name) => SheetLocator::from_name(name),
18051 None => SheetLocator::Current,
18052 };
18053 let coord = formualizer_common::RelativeCoord::new(row0, col0, *row_abs, *col_abs);
18054 SharedRef::Cell(SharedCellRef::new(sheet_loc, coord))
18055 }
18056 ReferenceType::Range {
18057 sheet,
18058 start_row,
18059 start_col,
18060 end_row,
18061 end_col,
18062 start_row_abs,
18063 start_col_abs,
18064 end_row_abs,
18065 end_col_abs,
18066 } => {
18067 let sheet_loc = match sheet.as_deref() {
18068 Some(name) => SheetLocator::from_name(name),
18069 None => SheetLocator::Current,
18070 };
18071 let sr = start_row
18072 .map(|r| {
18073 r.checked_sub(1)
18074 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
18075 })
18076 .transpose()?;
18077 let sc = start_col
18078 .map(|c| {
18079 c.checked_sub(1)
18080 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
18081 })
18082 .transpose()?;
18083 let er = end_row
18084 .map(|r| {
18085 r.checked_sub(1)
18086 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
18087 })
18088 .transpose()?;
18089 let ec = end_col
18090 .map(|c| {
18091 c.checked_sub(1)
18092 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
18093 })
18094 .transpose()?;
18095 let range = SharedRangeRef::from_parts(
18096 sheet_loc,
18097 sr.map(|idx| formualizer_common::AxisBound::new(idx, *start_row_abs)),
18098 sc.map(|idx| formualizer_common::AxisBound::new(idx, *start_col_abs)),
18099 er.map(|idx| formualizer_common::AxisBound::new(idx, *end_row_abs)),
18100 ec.map(|idx| formualizer_common::AxisBound::new(idx, *end_col_abs)),
18101 )
18102 .map_err(|_| ExcelError::new(ExcelErrorKind::Ref))?;
18103 SharedRef::Range(range)
18104 }
18105 _ => return Err(ExcelError::new(ExcelErrorKind::Ref)),
18106 };
18107
18108 let current_id = self
18109 .graph
18110 .sheet_id(current_sheet)
18111 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18112
18113 let resolve_loc = |loc: SheetLocator<'_>| -> Result<SheetLocator<'static>, ExcelError> {
18114 match loc {
18115 SheetLocator::Current => Ok(SheetLocator::Id(current_id)),
18116 SheetLocator::Id(id) => Ok(SheetLocator::Id(id)),
18117 SheetLocator::Name(name) => {
18118 let n = name.as_ref();
18119 self.graph
18120 .sheet_id(n)
18121 .map(SheetLocator::Id)
18122 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
18123 }
18124 }
18125 };
18126
18127 match sr {
18128 SharedRef::Cell(cell) => {
18129 let owned = cell.into_owned();
18130 let sheet = resolve_loc(owned.sheet)?;
18131 Ok(SharedRef::Cell(SharedCellRef::new(sheet, owned.coord)))
18132 }
18133 SharedRef::Range(range) => {
18134 let owned = range.into_owned();
18135 let sheet = resolve_loc(owned.sheet)?;
18136 Ok(SharedRef::Range(SharedRangeRef {
18137 sheet,
18138 start_row: owned.start_row,
18139 start_col: owned.start_col,
18140 end_row: owned.end_row,
18141 end_col: owned.end_col,
18142 }))
18143 }
18144 }
18145 }
18146}
18147
18148impl<R> crate::traits::ReferenceResolver for Engine<R>
18151where
18152 R: EvaluationContext,
18153{
18154 fn resolve_cell_reference(
18155 &self,
18156 sheet: Option<&str>,
18157 row: u32,
18158 col: u32,
18159 ) -> Result<LiteralValue, ExcelError> {
18160 let Some(sheet_name) = sheet else {
18171 return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
18172 "Unqualified cell reference resolved without sheet context".to_string(),
18173 ));
18174 };
18175 if let Some(v) = self.read_cell_value(sheet_name, row, col) {
18178 Ok(v)
18179 } else {
18180 Ok(LiteralValue::Number(0.0))
18182 }
18183 }
18184}
18185
18186impl<R> crate::traits::RangeResolver for Engine<R>
18187where
18188 R: EvaluationContext,
18189{
18190 fn resolve_range_reference(
18191 &self,
18192 sheet: Option<&str>,
18193 sr: Option<u32>,
18194 sc: Option<u32>,
18195 er: Option<u32>,
18196 ec: Option<u32>,
18197 ) -> Result<Box<dyn crate::traits::Range>, ExcelError> {
18198 self.resolver.resolve_range_reference(sheet, sr, sc, er, ec)
18201 }
18202}
18203
18204impl<R> crate::traits::NamedRangeResolver for Engine<R>
18205where
18206 R: EvaluationContext,
18207{
18208 fn resolve_named_range_reference(
18209 &self,
18210 name: &str,
18211 ) -> Result<Vec<Vec<LiteralValue>>, ExcelError> {
18212 self.resolver.resolve_named_range_reference(name)
18213 }
18214}
18215
18216impl<R> crate::traits::TableResolver for Engine<R>
18217where
18218 R: EvaluationContext,
18219{
18220 fn resolve_table_reference(
18221 &self,
18222 tref: &formualizer_parse::parser::TableReference,
18223 ) -> Result<Box<dyn crate::traits::Table>, ExcelError> {
18224 self.resolver.resolve_table_reference(tref)
18225 }
18226}
18227
18228impl<R> crate::traits::SourceResolver for Engine<R>
18229where
18230 R: EvaluationContext,
18231{
18232 fn source_scalar_version(&self, name: &str) -> Option<u64> {
18233 self.resolver.source_scalar_version(name)
18234 }
18235
18236 fn resolve_source_scalar(&self, name: &str) -> Result<LiteralValue, ExcelError> {
18237 self.resolver.resolve_source_scalar(name)
18238 }
18239
18240 fn source_table_version(&self, name: &str) -> Option<u64> {
18241 self.resolver.source_table_version(name)
18242 }
18243
18244 fn resolve_source_table(
18245 &self,
18246 name: &str,
18247 ) -> Result<Box<dyn crate::traits::Table>, ExcelError> {
18248 self.resolver.resolve_source_table(name)
18249 }
18250}
18251
18252impl<R> crate::traits::Resolver for Engine<R> where R: EvaluationContext {}
18254
18255impl<R: EvaluationContext> crate::interpreter::ArenaCellResolver for Engine<R> {
18257 fn resolve_arena_range(
18258 &self,
18259 reference: &crate::engine::arena::CompactRefType,
18260 current: SheetId,
18261 row_delta: i64,
18262 col_delta: i64,
18263 ) -> Result<RangeView<'_>, ExcelError> {
18264 use crate::engine::arena::{CompactRefType, SheetKey};
18265 let CompactRefType::Range {
18266 sheet,
18267 start_row,
18268 start_col,
18269 end_row,
18270 end_col,
18271 start_row_abs,
18272 start_col_abs,
18273 end_row_abs,
18274 end_col_abs,
18275 } = reference
18276 else {
18277 return Err(ExcelError::new(ExcelErrorKind::Ref));
18278 };
18279 let sheet_id = match sheet {
18280 None => current,
18281 Some(SheetKey::Id(id)) => *id,
18282 Some(SheetKey::Name(id)) => self
18283 .graph
18284 .sheet_id(self.graph.data_store().resolve_ast_string(*id))
18285 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?,
18286 };
18287 if self.graph.sheet_reg().name(sheet_id).is_empty() {
18288 return Err(ExcelError::new(ExcelErrorKind::Ref));
18289 }
18290 let axis = |value, sentinel, delta, absolute| {
18291 if value == sentinel {
18292 Ok(None)
18293 } else {
18294 crate::interpreter::shift_axis_for_offset(value, delta, absolute).map(Some)
18295 }
18296 };
18297 let sr = axis(*start_row, 0, row_delta, *start_row_abs)?;
18298 let sc = axis(*start_col, 0, col_delta, *start_col_abs)?;
18299 let er = axis(*end_row, u32::MAX, row_delta, *end_row_abs)?;
18300 let ec = axis(*end_col, u32::MAX, col_delta, *end_col_abs)?;
18301 if sr.zip(er).is_some_and(|(s, e)| s > e) || sc.zip(ec).is_some_and(|(s, e)| s > e) {
18302 return Err(ExcelError::new(ExcelErrorKind::Ref));
18303 }
18304 self.resolve_range_coordinates(sheet_id, sr, sc, er, ec)
18305 }
18306
18307 fn resolve_arena_cell(
18308 &self,
18309 sheet: Option<crate::engine::arena::SheetKey>,
18310 current: crate::SheetId,
18311 row: u32,
18312 col: u32,
18313 ) -> Result<(LiteralValue, Option<crate::format::FormatId>), ExcelError> {
18314 use crate::engine::arena::SheetKey;
18315 let (sheet_id, named_arrow) = match sheet {
18316 None => (current, None),
18317 Some(SheetKey::Id(id)) => {
18318 if self.graph.sheet_reg().name(id).is_empty() {
18319 return Err(ExcelError::new(ExcelErrorKind::Ref));
18320 }
18321 (id, None)
18322 }
18323 Some(SheetKey::Name(id)) => {
18324 let (sheet_id, arrow) = self
18325 .resolved_sheet_names
18326 .get(&id)
18327 .copied()
18328 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18329 (sheet_id, Some(arrow))
18330 }
18331 };
18332 let arrow = named_arrow.unwrap_or_else(|| {
18333 self.resolved_arrow_sheets
18334 .get(usize::from(sheet_id))
18335 .copied()
18336 .flatten()
18337 });
18338 let asheet = arrow
18339 .and_then(|index| self.arrow_sheets.sheets.get(index))
18340 .or_else(|| {
18341 let name = match sheet {
18345 Some(SheetKey::Name(id)) => self.graph.data_store().resolve_ast_string(id),
18346 _ => self.graph.sheet_name(sheet_id),
18347 };
18348 self.arrow_sheets.sheet(name)
18349 });
18350 Ok(self.read_cell_formatted_in(sheet_id, asheet, row, col))
18351 }
18352}
18353
18354impl<R> crate::traits::FunctionProvider for Engine<R>
18355where
18356 R: EvaluationContext,
18357{
18358 fn planning_semantic_revision(&self) -> Option<u64> {
18359 self.resolver.planning_semantic_revision()
18360 }
18361
18362 fn get_function(
18363 &self,
18364 prefix: &str,
18365 name: &str,
18366 ) -> Option<std::sync::Arc<dyn crate::function::Function>> {
18367 self.resolver.get_function(prefix, name)
18368 }
18369
18370 fn get_function_for_planning(
18371 &self,
18372 prefix: &str,
18373 name: &str,
18374 ) -> Option<std::sync::Arc<dyn crate::function::Function>> {
18375 self.resolver.get_function_for_planning(prefix, name)
18376 }
18377}
18378
18379impl<R> Engine<R>
18380where
18381 R: EvaluationContext,
18382{
18383 fn resolve_range_coordinates(
18384 &self,
18385 sheet_id: SheetId,
18386 sr: Option<u32>,
18387 sc: Option<u32>,
18388 er: Option<u32>,
18389 ec: Option<u32>,
18390 ) -> Result<RangeView<'_>, ExcelError> {
18391 let sheet_name = self.graph.sheet_name(sheet_id);
18392 let extent = resolve_used_extent_with_fallback(
18393 OpenRangeBounds {
18394 start_row: sr,
18395 start_column: sc,
18396 end_row: er,
18397 end_column: ec,
18398 },
18399 ExtentPolicy::EvaluationCompat {
18400 fallback_row: None,
18401 fallback_column: None,
18402 },
18403 || {
18404 self.sheet_bounds(sheet_name)
18405 .map(|_| self.config.max_open_ended_rows)
18406 },
18407 || {
18408 self.sheet_bounds(sheet_name)
18409 .map(|_| self.config.max_open_ended_cols)
18410 },
18411 |first, last| self.used_rows_for_columns(sheet_name, first, last),
18412 |first, last| self.used_cols_for_rows(sheet_name, first, last),
18413 );
18414 let (sr, sc, er, ec) = extent
18415 .map(|e| (e.start_row, e.start_column, e.end_row, e.end_column))
18416 .unwrap_or((1, 1, 0, 0));
18417 if self.force_materialize_range_views {
18418 if er < sr || ec < sc {
18419 return Ok(RangeView::from_owned_rows(
18420 Vec::new(),
18421 self.config.date_system,
18422 ));
18423 }
18424 let h = (er - sr + 1) as u64;
18425 let w = (ec - sc + 1) as u64;
18426 if h.saturating_mul(w) <= self.config.spill.max_spill_cells as u64 {
18427 let mut rows = Vec::with_capacity(h as usize);
18428 for r in sr..=er {
18429 let mut row = Vec::with_capacity(w as usize);
18430 for c in sc..=ec {
18431 row.push(
18432 self.read_cell_value(sheet_name, r, c)
18433 .unwrap_or(LiteralValue::Empty),
18434 );
18435 }
18436 rows.push(row);
18437 }
18438 return Ok(RangeView::from_owned_rows(rows, self.config.date_system));
18439 }
18440 }
18441 let Some(asheet) = self.arrow_sheet_by_id(sheet_id) else {
18442 return Ok(RangeView::from_owned_rows(
18443 Vec::new(),
18444 self.config.date_system,
18445 ));
18446 };
18447 Ok(if er < sr || ec < sc {
18448 asheet.range_view(1, 1, 0, 0)
18449 } else {
18450 asheet.range_view(
18451 sr.saturating_sub(1) as usize,
18452 sc.saturating_sub(1) as usize,
18453 er.saturating_sub(1) as usize,
18454 ec.saturating_sub(1) as usize,
18455 )
18456 })
18457 }
18458
18459 pub(crate) fn semantic_used_rows_for_columns(
18467 &self,
18468 sheet: &str,
18469 start_col: u32,
18470 end_col: u32,
18471 ) -> Option<(u32, u32)> {
18472 let arrow_bounds = self
18473 .sheet_store()
18474 .sheet(sheet)
18475 .and_then(|_| self.arrow_used_row_bounds(sheet, start_col, end_col));
18476 let formula_bounds = self.formula_row_bounds_for_columns(sheet, start_col, end_col);
18477 Self::union_used_bounds(arrow_bounds, formula_bounds)
18478 }
18479
18480 pub(crate) fn semantic_used_cols_for_rows(
18481 &self,
18482 sheet: &str,
18483 start_row: u32,
18484 end_row: u32,
18485 ) -> Option<(u32, u32)> {
18486 let arrow_bounds = self
18487 .sheet_store()
18488 .sheet(sheet)
18489 .and_then(|_| self.arrow_used_col_bounds(sheet, start_row, end_row));
18490 let formula_bounds = self.formula_col_bounds_for_rows(sheet, start_row, end_row);
18491 Self::union_used_bounds(arrow_bounds, formula_bounds)
18492 }
18493}
18494
18495impl<R> crate::traits::EvaluationContext for Engine<R>
18497where
18498 R: EvaluationContext,
18499{
18500 fn clock(&self) -> &dyn crate::timezone::ClockProvider {
18501 &self.clock
18502 }
18503
18504 fn thread_pool(&self) -> Option<&Arc<rayon::ThreadPool>> {
18505 self.thread_pool.as_ref()
18506 }
18507
18508 fn cancellation_token(&self) -> Option<crate::engine::CancelToken> {
18509 self.active_cancel_flag.clone()
18510 }
18511
18512 fn chunk_hint(&self) -> Option<usize> {
18513 let hint =
18515 (self.config.stripe_height as usize).saturating_mul(self.config.stripe_width as usize);
18516 Some(hint.clamp(1024, 1 << 20)) }
18518
18519 fn materialized_array_cells(&self) -> u64 {
18520 self.config
18521 .evaluation_budgets
18522 .scratch
18523 .materialized_array_cells
18524 .unwrap_or(crate::engine::resource_ledger::DEFAULT_MATERIALIZED_ARRAY_CELLS)
18525 }
18526
18527 fn volatile_level(&self) -> crate::traits::VolatileLevel {
18528 self.config.volatile_level
18529 }
18530
18531 fn workbook_seed(&self) -> u64 {
18532 self.config.workbook_seed
18533 }
18534
18535 fn recalc_epoch(&self) -> u64 {
18536 self.recalc_epoch
18537 }
18538
18539 fn workbook_sheet_count(&self) -> Option<usize> {
18540 Some(self.graph.sheet_reg().active_len())
18541 }
18542
18543 fn sheet_index_by_name(&self, sheet: &str) -> Option<usize> {
18544 self.graph.sheet_reg().active_position(sheet)
18545 }
18546
18547 fn current_sheet_index(&self, current_sheet: &str) -> Option<usize> {
18548 self.sheet_index_by_name(current_sheet)
18549 }
18550
18551 fn inspect_reference(
18552 &self,
18553 reference: &ReferenceType,
18554 current_sheet: &str,
18555 ) -> Result<Option<ReferenceInfo>, ExcelError> {
18556 let sheet_info = |sheet_name: &str| -> Result<(SheetId, usize), ExcelError> {
18557 let sheet_id = self
18558 .graph
18559 .sheet_id(sheet_name)
18560 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18561 let sheet_index = self
18562 .graph
18563 .sheet_reg()
18564 .active_position_by_id(sheet_id)
18565 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18566 Ok((sheet_id, sheet_index))
18567 };
18568
18569 let cell_info =
18570 |sheet_name: &str, row: u32, col: u32| -> Result<ReferenceInfo, ExcelError> {
18571 let (sheet_id, sheet_index) = sheet_info(sheet_name)?;
18572 let row0 = row
18573 .checked_sub(1)
18574 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18575 let col0 = col
18576 .checked_sub(1)
18577 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18578 Ok(ReferenceInfo {
18579 first_sheet_index: Some(sheet_index),
18580 sheet_count: Some(1),
18581 first_cell: Some(CellRef::new(sheet_id, Coord::new(row0, col0, true, true))),
18582 })
18583 };
18584
18585 let range_info = |sheet_name: &str,
18586 start_row: Option<u32>,
18587 start_col: Option<u32>|
18588 -> Result<ReferenceInfo, ExcelError> {
18589 let (sheet_id, sheet_index) = sheet_info(sheet_name)?;
18590 let row = start_row.unwrap_or(1);
18591 let col = start_col.unwrap_or(1);
18592 let row0 = row
18593 .checked_sub(1)
18594 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18595 let col0 = col
18596 .checked_sub(1)
18597 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18598 Ok(ReferenceInfo {
18599 first_sheet_index: Some(sheet_index),
18600 sheet_count: Some(1),
18601 first_cell: Some(CellRef::new(sheet_id, Coord::new(row0, col0, true, true))),
18602 })
18603 };
18604
18605 let info = match reference {
18606 ReferenceType::Cell {
18607 sheet, row, col, ..
18608 } => {
18609 let sheet_name = sheet.as_deref().unwrap_or(current_sheet);
18610 cell_info(sheet_name, *row, *col)?
18611 }
18612 ReferenceType::Range {
18613 sheet,
18614 start_row,
18615 start_col,
18616 ..
18617 } => {
18618 let sheet_name = sheet.as_deref().unwrap_or(current_sheet);
18619 range_info(sheet_name, *start_row, *start_col)?
18620 }
18621 ReferenceType::Cell3D {
18622 sheet_first,
18623 sheet_last,
18624 row,
18625 col,
18626 ..
18627 } => {
18628 let first = cell_info(sheet_first, *row, *col)?;
18629 ReferenceInfo {
18630 first_sheet_index: first.first_sheet_index,
18631 sheet_count: self
18632 .graph
18633 .sheet_reg()
18634 .active_span_len(sheet_first, sheet_last),
18635 first_cell: first.first_cell,
18636 }
18637 }
18638 ReferenceType::Range3D {
18639 sheet_first,
18640 sheet_last,
18641 start_row,
18642 start_col,
18643 ..
18644 } => {
18645 let first = range_info(sheet_first, *start_row, *start_col)?;
18646 ReferenceInfo {
18647 first_sheet_index: first.first_sheet_index,
18648 sheet_count: self
18649 .graph
18650 .sheet_reg()
18651 .active_span_len(sheet_first, sheet_last),
18652 first_cell: first.first_cell,
18653 }
18654 }
18655 ReferenceType::NamedRange(name) => {
18656 let current_id = self
18657 .graph
18658 .sheet_id(current_sheet)
18659 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18660 let named = self
18661 .graph
18662 .resolve_name_entry(name, current_id)
18663 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18664 match &named.definition {
18665 NamedDefinition::Cell(cell) => ReferenceInfo {
18666 first_sheet_index: self
18667 .graph
18668 .sheet_reg()
18669 .active_position_by_id(cell.sheet_id),
18670 sheet_count: Some(1),
18671 first_cell: Some(*cell),
18672 },
18673 NamedDefinition::Range(range) => ReferenceInfo {
18674 first_sheet_index: self
18675 .graph
18676 .sheet_reg()
18677 .active_position_by_id(range.start.sheet_id),
18678 sheet_count: Some(1),
18679 first_cell: Some(range.start),
18680 },
18681 NamedDefinition::Literal(_) | NamedDefinition::Formula { .. } => {
18682 ReferenceInfo {
18683 first_sheet_index: None,
18684 sheet_count: None,
18685 first_cell: None,
18686 }
18687 }
18688 }
18689 }
18690 ReferenceType::Table(tref) => {
18691 let table = self
18692 .graph
18693 .resolve_table_entry(&tref.name)
18694 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18695 ReferenceInfo {
18696 first_sheet_index: self
18697 .graph
18698 .sheet_reg()
18699 .active_position_by_id(table.range.start.sheet_id),
18700 sheet_count: Some(1),
18701 first_cell: Some(table.range.start),
18702 }
18703 }
18704 ReferenceType::External(_) => return Err(ExcelError::new(ExcelErrorKind::Ref)),
18705 };
18706
18707 Ok(Some(info))
18708 }
18709
18710 fn resolve_spill_reference(
18711 &self,
18712 anchor: &ReferenceType,
18713 current_sheet: &str,
18714 ) -> Result<ReferenceType, ExcelError> {
18715 let no_spill = || {
18716 ExcelError::new(ExcelErrorKind::Ref)
18717 .with_message("Spill reference anchor has no current spill")
18718 };
18719 let mut qualify = true;
18722 let anchor_cell = match anchor {
18723 ReferenceType::Cell {
18724 sheet, row, col, ..
18725 } => {
18726 qualify = sheet.is_some();
18727 let sheet_name = sheet.as_deref().unwrap_or(current_sheet);
18728 let sheet_id = self.graph.sheet_id(sheet_name).ok_or_else(no_spill)?;
18729 if *row == 0 || *col == 0 {
18730 return Err(no_spill());
18731 }
18732 CellRef::new(sheet_id, Coord::from_excel(*row, *col, true, true))
18733 }
18734 ReferenceType::NamedRange(name) => {
18735 let current_id = self.graph.sheet_id(current_sheet).ok_or_else(no_spill)?;
18736 let named = self
18737 .graph
18738 .resolve_name_entry(name, current_id)
18739 .ok_or_else(no_spill)?;
18740 match &named.definition {
18741 NamedDefinition::Cell(cell) => *cell,
18742 NamedDefinition::Range(range)
18743 if range.start.sheet_id == range.end.sheet_id
18744 && range.start.coord.row() == range.end.coord.row()
18745 && range.start.coord.col() == range.end.coord.col() =>
18746 {
18747 range.start
18748 }
18749 _ => {
18750 return Err(ExcelError::new(ExcelErrorKind::Ref)
18751 .with_message("Spill reference name must refer to a single cell"));
18752 }
18753 }
18754 }
18755 _ => {
18756 return Err(ExcelError::new(ExcelErrorKind::Ref)
18757 .with_message("Spill reference operand must be a single cell"));
18758 }
18759 };
18760 let vertex = self
18761 .graph
18762 .get_vertex_id_for_address(&anchor_cell)
18763 .ok_or_else(no_spill)?;
18764 if self.graph.fixed_single_arrays.contains(&vertex)
18767 || self.graph.fixed_array_shapes.contains_key(&vertex)
18768 {
18769 return Err(ExcelError::new(ExcelErrorKind::Ref)
18770 .with_message("Spill reference to a fixed array formula"));
18771 }
18772 let (first, last) = match self.graph.spill_extent_for_anchor(vertex) {
18773 Some(extent) => extent,
18774 None if !self.graph.spill_registry_has_anchor(vertex)
18778 && self.graph.is_current_declared_dynamic_anchor(vertex)
18779 && self.declared_anchor_holds_scalar(anchor_cell) =>
18780 {
18781 (anchor_cell, anchor_cell)
18782 }
18783 None => return Err(no_spill()),
18784 };
18785 let sheet_name = qualify.then(|| self.graph.sheet_name(first.sheet_id).to_string());
18786 let (sr, sc) = (first.coord.row() + 1, first.coord.col() + 1);
18787 let (er, ec) = (last.coord.row() + 1, last.coord.col() + 1);
18788 Ok(if sr == er && sc == ec {
18790 ReferenceType::cell_with_abs(sheet_name, sr, sc, true, true)
18791 } else {
18792 ReferenceType::Range {
18793 sheet: sheet_name,
18794 start_row: Some(sr),
18795 start_col: Some(sc),
18796 end_row: Some(er),
18797 end_col: Some(ec),
18798 start_row_abs: true,
18799 start_col_abs: true,
18800 end_row_abs: true,
18801 end_col_abs: true,
18802 }
18803 })
18804 }
18805
18806 fn formula_text_at_cell(&self, cell: CellRef) -> Result<Option<String>, ExcelError> {
18807 let sheet_name = self.graph.sheet_name(cell.sheet_id);
18808 if sheet_name.is_empty() {
18809 return Err(ExcelError::new(ExcelErrorKind::Ref));
18810 }
18811 let row = cell.coord.row() + 1;
18812 let col = cell.coord.col() + 1;
18813
18814 if let Some(entries) = self.staged_formulas.get(sheet_name)
18815 && let Some(text) = entries.get(row, col)
18816 {
18817 return Ok(Some(if text.starts_with('=') {
18818 text.to_owned()
18819 } else {
18820 format!("={text}")
18821 }));
18822 }
18823
18824 let Some((Some(ast), _)) = self.get_cell(sheet_name, row, col) else {
18825 return Ok(None);
18826 };
18827 Ok(Some(formualizer_parse::pretty::canonical_formula(&ast)))
18828 }
18829
18830 fn used_rows_for_columns(
18831 &self,
18832 sheet: &str,
18833 start_col: u32,
18834 end_col: u32,
18835 ) -> Option<(u32, u32)> {
18836 let sheet_id = self.graph.sheet_id(sheet)?;
18838 let snap = self.data_snapshot_id();
18839 if let Some(cached) = self.used_axis_bounds_cache.read().ok().and_then(|guard| {
18840 guard
18841 .as_ref()
18842 .and_then(|cache| cache.get_row_bounds(sheet_id, start_col, end_col, snap))
18843 }) {
18844 return cached;
18845 }
18846
18847 let arrow_bounds = self
18848 .sheet_store()
18849 .sheet(sheet)
18850 .and_then(|_| self.arrow_used_row_bounds(sheet, start_col, end_col));
18851 let formula_bounds = self.formula_row_bounds_for_columns(sheet, start_col, end_col);
18852 let computed = if let Some(bounds) = Self::union_used_bounds(arrow_bounds, formula_bounds) {
18853 Some(bounds)
18854 } else {
18855 let sc0 = start_col.saturating_sub(1);
18856 let ec0 = end_col.saturating_sub(1);
18857 self.graph
18858 .used_row_bounds_for_columns(sheet_id, sc0, ec0)
18859 .map(|(a0, b0)| (a0 + 1, b0 + 1))
18860 };
18861
18862 if let Ok(mut guard) = self.used_axis_bounds_cache.write() {
18863 guard
18864 .get_or_insert_with(|| UsedAxisBoundsCache::new(snap))
18865 .put_row_bounds(sheet_id, start_col, end_col, snap, computed);
18866 }
18867
18868 computed
18869 }
18870
18871 fn used_cols_for_rows(&self, sheet: &str, start_row: u32, end_row: u32) -> Option<(u32, u32)> {
18872 let sheet_id = self.graph.sheet_id(sheet)?;
18874 let snap = self.data_snapshot_id();
18875 if let Some(cached) = self.used_axis_bounds_cache.read().ok().and_then(|guard| {
18876 guard
18877 .as_ref()
18878 .and_then(|cache| cache.get_col_bounds(sheet_id, start_row, end_row, snap))
18879 }) {
18880 return cached;
18881 }
18882
18883 let arrow_bounds = self
18884 .sheet_store()
18885 .sheet(sheet)
18886 .and_then(|_| self.arrow_used_col_bounds(sheet, start_row, end_row));
18887 let formula_bounds = self.formula_col_bounds_for_rows(sheet, start_row, end_row);
18888 let computed = if let Some(bounds) = Self::union_used_bounds(arrow_bounds, formula_bounds) {
18889 Some(bounds)
18890 } else {
18891 let sr0 = start_row.saturating_sub(1);
18892 let er0 = end_row.saturating_sub(1);
18893 self.graph
18894 .used_col_bounds_for_rows(sheet_id, sr0, er0)
18895 .map(|(a0, b0)| (a0 + 1, b0 + 1))
18896 };
18897
18898 if let Ok(mut guard) = self.used_axis_bounds_cache.write() {
18899 guard
18900 .get_or_insert_with(|| UsedAxisBoundsCache::new(snap))
18901 .put_col_bounds(sheet_id, start_row, end_row, snap, computed);
18902 }
18903
18904 computed
18905 }
18906
18907 fn sheet_bounds(&self, sheet: &str) -> Option<(u32, u32)> {
18908 let _ = self.graph.sheet_id(sheet)?;
18909 Some((1_048_576, 16_384)) }
18913
18914 fn data_snapshot_id(&self) -> u64 {
18915 self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed)
18916 }
18917
18918 fn backend_caps(&self) -> crate::traits::BackendCaps {
18919 crate::traits::BackendCaps {
18920 streaming: true,
18921 used_region: true,
18922 write: false,
18923 tables: false,
18924 async_stream: false,
18925 }
18926 }
18927
18928 fn build_lookup_index(
18929 &self,
18930 view: &RangeView<'_>,
18931 axis: LookupAxis,
18932 ) -> Option<Arc<LookupIndex>> {
18933 self.build_lookup_index_impl(view, axis)
18934 }
18935
18936 fn date_system(&self) -> crate::engine::DateSystem {
18939 self.config.date_system
18940 }
18941 fn resolve_range_view<'c>(
18943 &'c self,
18944 reference: &ReferenceType,
18945 current_sheet: &str,
18946 ) -> Result<RangeView<'c>, ExcelError> {
18947 match reference {
18948 ReferenceType::External(ext) => {
18949 let name = ext.raw.as_str();
18950 match ext.kind {
18951 formualizer_parse::parser::ExternalRefKind::Cell { .. } => {
18952 let Some(source) = self.graph.resolve_source_scalar_entry(name) else {
18953 return Err(ExcelError::new(ExcelErrorKind::Name)
18954 .with_message(format!("Undefined name: {name}")));
18955 };
18956 let version = source
18957 .version
18958 .or_else(|| self.resolver.source_scalar_version(name));
18959 let v = self.resolve_source_scalar_cached(name, version)?;
18960 Ok(RangeView::from_owned_rows(
18961 vec![vec![v]],
18962 self.config.date_system,
18963 ))
18964 }
18965 formualizer_parse::parser::ExternalRefKind::Range { .. } => {
18966 let Some(source) = self.graph.resolve_source_table_entry(name) else {
18967 let kind =
18971 if crate::engine::refs::unbound_external_range_defers(&ext.kind) {
18972 ExcelErrorKind::Ref
18973 } else {
18974 ExcelErrorKind::Name
18975 };
18976 return Err(ExcelError::new(kind)
18977 .with_message(format!("Undefined table: {name}")));
18978 };
18979 let version = source
18980 .version
18981 .or_else(|| self.resolver.source_table_version(name));
18982 let key = (name.to_string(), version);
18983 if let Ok(g) = self.source_cache.read()
18984 && let Some(view) = g.external_views.get(&key)
18985 {
18986 return Ok(view.clone());
18987 }
18988 let table = self.resolve_source_table_cached(name, version)?;
18989 let spec = Some(formualizer_parse::parser::TableSpecifier::Data);
18990 let Ok(mut g) = self.source_cache.write() else {
18993 return self.source_table_to_range_view(table.as_ref(), &spec);
18994 };
18995 if let Some(view) = g.external_views.get(&key) {
18996 return Ok(view.clone());
18997 }
18998 let view = self.source_table_to_range_view(table.as_ref(), &spec)?;
18999 g.external_views.insert(key, view.clone());
19000 Ok(view)
19001 }
19002 }
19003 }
19004 ReferenceType::Range { .. } => {
19005 let shared = self.resolve_shared_ref(reference, current_sheet)?;
19006 let formualizer_common::SheetRef::Range(range) = shared else {
19007 return Err(ExcelError::new(ExcelErrorKind::Ref));
19008 };
19009 let sheet_id = match range.sheet {
19012 formualizer_common::SheetLocator::Id(id) => id,
19013 formualizer_common::SheetLocator::Current
19014 | formualizer_common::SheetLocator::Name(_) => {
19015 return Err(ExcelError::new(ExcelErrorKind::Ref));
19016 }
19017 };
19018 let bounded_range = if range.start_row.is_some()
19019 && range.start_col.is_some()
19020 && range.end_row.is_some()
19021 && range.end_col.is_some()
19022 {
19023 Some(RangeRef::try_from_shared(range.as_ref())?)
19024 } else {
19025 None
19026 };
19027
19028 let sr = bounded_range
19029 .as_ref()
19030 .map(|r| r.start.coord.row() + 1)
19031 .or_else(|| range.start_row.map(|b| b.index + 1));
19032 let sc = bounded_range
19033 .as_ref()
19034 .map(|r| r.start.coord.col() + 1)
19035 .or_else(|| range.start_col.map(|b| b.index + 1));
19036 let er = bounded_range
19037 .as_ref()
19038 .map(|r| r.end.coord.row() + 1)
19039 .or_else(|| range.end_row.map(|b| b.index + 1));
19040 let ec = bounded_range
19041 .as_ref()
19042 .map(|r| r.end.coord.col() + 1)
19043 .or_else(|| range.end_col.map(|b| b.index + 1));
19044
19045 self.resolve_range_coordinates(sheet_id, sr, sc, er, ec)
19046 }
19047 ReferenceType::Cell { .. } => {
19048 let shared = self.resolve_shared_ref(reference, current_sheet)?;
19049 let formualizer_common::SheetRef::Cell(cell) = shared else {
19050 return Err(ExcelError::new(ExcelErrorKind::Ref));
19051 };
19052 let addr = CellRef::try_from_shared(cell)?;
19053 let sheet_id = addr.sheet_id;
19054 let sheet_name = self.graph.sheet_name(sheet_id);
19055 let row = addr.coord.row() + 1;
19056 let col = addr.coord.col() + 1;
19057
19058 if self.force_materialize_range_views {
19059 let v = self
19060 .read_cell_value(sheet_name, row, col)
19061 .unwrap_or(LiteralValue::Empty);
19062 return Ok(RangeView::from_owned_rows(
19063 vec![vec![v]],
19064 self.config.date_system,
19065 ));
19066 }
19067
19068 if let Some(asheet) = self.sheet_store().sheet(sheet_name) {
19069 let r0 = row.saturating_sub(1) as usize;
19070 let c0 = col.saturating_sub(1) as usize;
19071 let rv = asheet.range_view(r0, c0, r0, c0);
19072 Ok(rv)
19073 } else {
19074 let v = self
19075 .read_cell_value(sheet_name, row, col)
19076 .unwrap_or(LiteralValue::Empty);
19077 Ok(RangeView::from_owned_rows(
19078 vec![vec![v]],
19079 self.config.date_system,
19080 ))
19081 }
19082 }
19083 ReferenceType::NamedRange(name) => {
19084 if let Some(current_id) = self.graph.sheet_id(current_sheet)
19085 && let Some(named) = self.graph.resolve_name_entry(name, current_id)
19086 {
19087 match &named.definition {
19088 NamedDefinition::Cell(cell_ref) => {
19089 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
19090 if self.force_materialize_range_views {
19091 let v = self
19092 .read_cell_value(
19093 sheet_name,
19094 cell_ref.coord.row() + 1,
19095 cell_ref.coord.col() + 1,
19096 )
19097 .unwrap_or(LiteralValue::Empty);
19098 return Ok(RangeView::from_owned_rows(
19099 vec![vec![v]],
19100 self.config.date_system,
19101 ));
19102 } else {
19103 let asheet = self
19104 .sheet_store()
19105 .sheet(sheet_name)
19106 .expect("Arrow sheet missing for named cell");
19107 let r0 = cell_ref.coord.row() as usize;
19108 let c0 = cell_ref.coord.col() as usize;
19109 let rv = asheet.range_view(r0, c0, r0, c0);
19110 return Ok(rv);
19111 }
19112 }
19113 NamedDefinition::Range(range_ref) => {
19114 let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
19115 let sr = range_ref.start.coord.row() + 1;
19116 let sc = range_ref.start.coord.col() + 1;
19117 let er = range_ref.end.coord.row() + 1;
19118 let ec = range_ref.end.coord.col() + 1;
19119 if self.force_materialize_range_views {
19120 let h = (er.saturating_sub(sr) + 1) as u64;
19121 let w = (ec.saturating_sub(sc) + 1) as u64;
19122 let cell_count = h.saturating_mul(w);
19123 if cell_count <= self.config.spill.max_spill_cells as u64 {
19124 let mut rows: Vec<Vec<LiteralValue>> =
19125 Vec::with_capacity(h as usize);
19126 for r in sr..=er {
19127 let mut rowv: Vec<LiteralValue> =
19128 Vec::with_capacity(w as usize);
19129 for c in sc..=ec {
19130 rowv.push(
19131 self.read_cell_value(sheet_name, r, c)
19132 .unwrap_or(LiteralValue::Empty),
19133 );
19134 }
19135 rows.push(rowv);
19136 }
19137 return Ok(RangeView::from_owned_rows(
19138 rows,
19139 self.config.date_system,
19140 ));
19141 }
19142 }
19143 let asheet = self
19144 .sheet_store()
19145 .sheet(sheet_name)
19146 .expect("Arrow sheet missing for named range");
19147 let sr0 = range_ref.start.coord.row() as usize;
19148 let sc0 = range_ref.start.coord.col() as usize;
19149 let er0 = range_ref.end.coord.row() as usize;
19150 let ec0 = range_ref.end.coord.col() as usize;
19151 let rv = asheet.range_view(sr0, sc0, er0, ec0);
19152 return Ok(rv);
19153 }
19154 NamedDefinition::Literal(v) => {
19155 return Ok(RangeView::from_owned_rows(
19156 vec![vec![v.clone()]],
19157 self.config.date_system,
19158 ));
19159 }
19160 NamedDefinition::Formula { ast, .. } => {
19161 if let ASTNodeType::Reference {
19164 reference: external @ ReferenceType::External(_),
19165 ..
19166 } = &ast.node_type
19167 {
19168 return self.resolve_range_view(external, current_sheet);
19169 }
19170 if let Some(value) = self.graph.get_value(named.vertex) {
19171 return Ok(RangeView::from_owned_rows(
19172 vec![vec![value]],
19173 self.config.date_system,
19174 ));
19175 }
19176 }
19177 }
19178 }
19179
19180 if let Some(source) = self.graph.resolve_source_scalar_entry(name) {
19181 let version = source
19182 .version
19183 .or_else(|| self.resolver.source_scalar_version(name));
19184 let v = self.resolve_source_scalar_cached(name, version)?;
19185 return Ok(RangeView::from_owned_rows(
19186 vec![vec![v]],
19187 self.config.date_system,
19188 ));
19189 }
19190
19191 let data = self.resolver.resolve_named_range_reference(name)?;
19192 Ok(RangeView::from_owned_rows(data, self.config.date_system))
19193 }
19194 ReferenceType::Table(tref) => {
19195 if let Some(table) = self.graph.resolve_table_entry(&tref.name) {
19196 let sheet_name = self.graph.sheet_name(table.range.start.sheet_id);
19197 let asheet = self
19198 .sheet_store()
19199 .sheet(sheet_name)
19200 .expect("Arrow sheet missing for table reference");
19201
19202 let sr0 = table.range.start.coord.row() as usize;
19203 let sc0 = table.range.start.coord.col() as usize;
19204 let er0 = table.range.end.coord.row() as usize;
19205 let ec0 = table.range.end.coord.col() as usize;
19206
19207 let has_totals = table.totals_row;
19208 let has_headers = table.header_row;
19209 let data_sr = if has_headers {
19210 sr0.saturating_add(1)
19211 } else {
19212 sr0
19213 };
19214 let data_er = if has_totals {
19215 er0.saturating_sub(1)
19216 } else {
19217 er0
19218 };
19219
19220 let select = |sr: usize, sc: usize, er: usize, ec: usize| {
19221 if sr > er || sc > ec {
19222 asheet.range_view(1, 1, 0, 0)
19223 } else {
19224 asheet.range_view(sr, sc, er, ec)
19225 }
19226 };
19227
19228 let av = match &tref.specifier {
19229 None => {
19230 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
19231 "Table reference without specifier is unsupported".to_string(),
19232 ));
19233 }
19234 Some(formualizer_parse::parser::TableSpecifier::Column(col)) => {
19235 let Some(idx) = table.col_index(col) else {
19236 return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
19237 "Column refers to unknown table column".to_string(),
19238 ));
19239 };
19240 let c0 = sc0 + idx;
19241 select(data_sr, c0, data_er, c0)
19242 }
19243 Some(formualizer_parse::parser::TableSpecifier::ColumnRange(
19244 start,
19245 end,
19246 )) => {
19247 let Some(si) = table.col_index(start) else {
19248 return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
19249 "Column range refers to unknown column(s)".to_string(),
19250 ));
19251 };
19252 let Some(ei) = table.col_index(end) else {
19253 return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
19254 "Column range refers to unknown column(s)".to_string(),
19255 ));
19256 };
19257 let (mut a, mut b) = (si, ei);
19258 if a > b {
19259 std::mem::swap(&mut a, &mut b);
19260 }
19261 let c_start = sc0 + a;
19262 let c_end = sc0 + b;
19263 select(data_sr, c_start, data_er, c_end)
19264 }
19265 Some(formualizer_parse::parser::TableSpecifier::All)
19266 | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
19267 formualizer_parse::parser::SpecialItem::All,
19268 )) => select(sr0, sc0, er0, ec0),
19269 Some(formualizer_parse::parser::TableSpecifier::Data)
19270 | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
19271 formualizer_parse::parser::SpecialItem::Data,
19272 )) => select(data_sr, sc0, data_er, ec0),
19273 Some(formualizer_parse::parser::TableSpecifier::Headers)
19274 | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
19275 formualizer_parse::parser::SpecialItem::Headers,
19276 )) => {
19277 if !has_headers {
19278 asheet.range_view(1, 1, 0, 0)
19279 } else {
19280 select(sr0, sc0, sr0, ec0)
19281 }
19282 }
19283 Some(formualizer_parse::parser::TableSpecifier::Totals)
19284 | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
19285 formualizer_parse::parser::SpecialItem::Totals,
19286 )) => {
19287 if !has_totals {
19288 asheet.range_view(1, 1, 0, 0)
19289 } else {
19290 select(er0, sc0, er0, ec0)
19291 }
19292 }
19293 Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
19294 formualizer_parse::parser::SpecialItem::ThisRow,
19295 )) => {
19296 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
19297 "@ (This Row) requires table-aware context; not yet supported"
19298 .to_string(),
19299 ));
19300 }
19301 Some(formualizer_parse::parser::TableSpecifier::Row(_))
19302 | Some(formualizer_parse::parser::TableSpecifier::Combination(_)) => {
19303 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
19304 "Complex structured references not yet supported".to_string(),
19305 ));
19306 }
19307 };
19308
19309 return Ok(av);
19310 }
19311
19312 if let Some(source) = self.graph.resolve_source_table_entry(&tref.name) {
19313 let version = source
19314 .version
19315 .or_else(|| self.resolver.source_table_version(&tref.name));
19316 let table = self.resolve_source_table_cached(&tref.name, version)?;
19317 return self.source_table_to_range_view(table.as_ref(), &tref.specifier);
19318 }
19319
19320 let boxed = self
19325 .resolve_range_like(&ReferenceType::Table(tref.clone()))
19326 .map_err(|e| {
19327 if e.kind == ExcelErrorKind::NImpl {
19328 ExcelError::new(ExcelErrorKind::Name)
19329 .with_message(format!("Unknown table: {}", tref.name))
19330 } else {
19331 e
19332 }
19333 })?;
19334 let owned = boxed.materialise().into_owned();
19335 Ok(RangeView::from_owned_rows(owned, self.config.date_system))
19336 }
19337 ReferenceType::Cell3D { .. } | ReferenceType::Range3D { .. } => {
19338 Err(ExcelError::new(ExcelErrorKind::NImpl)
19339 .with_message("3D references are not yet supported".to_string()))
19340 }
19341 }
19342 }
19343
19344 fn resolve_cell_format(
19345 &self,
19346 sheet: Option<&str>,
19347 row: u32,
19348 col: u32,
19349 current_sheet: &str,
19350 ) -> Option<crate::format::FormatId> {
19351 self.effective_format_id(sheet.unwrap_or(current_sheet), row, col)
19352 }
19353
19354 fn format_class(
19355 &self,
19356 format: crate::format::FormatId,
19357 ) -> Option<formualizer_common::numfmt::FormatClass> {
19358 self.format_registry.class(format).cloned()
19359 }
19360
19361 fn record_cell_derived_format(
19362 &self,
19363 sheet: &str,
19364 row: u32,
19365 col: u32,
19366 format: Option<crate::format::FormatId>,
19367 ) {
19368 if let Some(sheet_id) = self.graph.sheet_id(sheet) {
19369 let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
19370 self.record_derived_format_at(cell, format);
19371 }
19372 }
19373
19374 fn resolve_cell_reference_value(
19375 &self,
19376 sheet: Option<&str>,
19377 row: u32,
19378 col: u32,
19379 current_sheet: &str,
19380 ) -> Result<LiteralValue, ExcelError> {
19381 let sheet_name = sheet.unwrap_or(current_sheet);
19382 if self.graph.sheet_id(sheet_name).is_none() {
19383 return Err(ExcelError::new(ExcelErrorKind::Ref));
19384 }
19385 Ok(self
19386 .read_cell_value(sheet_name, row, col)
19387 .unwrap_or(LiteralValue::Empty))
19388 }
19389
19390 fn resolve_cell_reference_value_formatted(
19391 &self,
19392 sheet: Option<&str>,
19393 row: u32,
19394 col: u32,
19395 current_sheet: &str,
19396 ) -> Result<(LiteralValue, Option<crate::format::FormatId>), ExcelError> {
19397 let sheet_name = sheet.unwrap_or(current_sheet);
19400 let Some(sheet_id) = self.graph.sheet_id(sheet_name) else {
19401 return Err(ExcelError::new(ExcelErrorKind::Ref));
19402 };
19403 let asheet = self.arrow_sheets.sheet(sheet_name);
19404 Ok(self.read_cell_formatted_in(sheet_id, asheet, row, col))
19405 }
19406
19407 fn build_criteria_mask(
19408 &self,
19409 view: &RangeView<'_>,
19410 col_in_view: usize,
19411 pred: &crate::args::CriteriaPredicate,
19412 ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
19413 #[cfg(any(test, feature = "test-support"))]
19414 criteria_mask_test_hooks::note_mask(view.dims().0);
19415 if view.dims().1 == 0 {
19416 return None;
19417 }
19418 let sheet_rows = view.sheet().nrows as usize;
19421 if sheet_rows == 0 || view.start_row() >= sheet_rows {
19422 return Some(std::sync::Arc::new(arrow_array::BooleanArray::new_null(0)));
19423 }
19424 self.criteria_mask_cache.get_or_build(
19425 view,
19426 col_in_view,
19427 pred,
19428 (
19429 self.recalc_epoch,
19430 self.topology_epoch,
19431 self.graph.topology_revision(),
19432 ),
19433 || compute_criteria_mask(view, col_in_view, pred),
19434 )
19435 }
19436
19437 fn build_row_visibility_mask(
19438 &self,
19439 view: &RangeView<'_>,
19440 mode: VisibilityMaskMode,
19441 ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
19442 self.build_row_visibility_mask_for_view(view, mode)
19443 }
19444
19445 fn nested_subtotal_cells(
19446 &self,
19447 view: &RangeView<'_>,
19448 include_aggregate: bool,
19449 ) -> Option<Vec<(usize, usize, usize)>> {
19450 self.nested_subtotal_cells_for_view(view, include_aggregate)
19451 }
19452}
19453
19454impl<R> Engine<R>
19455where
19456 R: EvaluationContext,
19457{
19458 fn clear_spill_projection_and_mirror(
19459 &mut self,
19460 anchor_vertex: VertexId,
19461 delta: Option<&mut DeltaCollector>,
19462 ) {
19463 let spill_cells = self.graph.spill_cells_to_clear(anchor_vertex);
19464 if spill_cells.is_empty() {
19465 return;
19466 }
19467
19468 if let Some(delta) = delta
19469 && delta.mode != DeltaMode::Off
19470 {
19471 let empty = LiteralValue::Empty;
19472 for cell in spill_cells.iter() {
19473 let sheet_name = self.graph.sheet_name(cell.sheet_id);
19474 let old = self
19475 .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
19476 .unwrap_or(LiteralValue::Empty);
19477 if old != empty {
19478 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
19479 }
19480 }
19481 }
19482
19483 self.graph.clear_spill_region(anchor_vertex);
19484 if let Some(scope) = Self::structural_scope_from_cells(&spill_cells) {
19485 self.record_structural_change(scope);
19486 }
19487
19488 if self.config.arrow_storage_enabled
19489 && self.config.delta_overlay_enabled
19490 && self.config.write_formula_overlay_enabled
19491 {
19492 let empty = LiteralValue::Empty;
19493 for cell in spill_cells.iter() {
19494 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
19495 self.mirror_value_to_computed_overlay(
19496 &sheet_name,
19497 cell.coord.row() + 1,
19498 cell.coord.col() + 1,
19499 &empty,
19500 );
19501 }
19502 }
19503 }
19504
19505 fn apply_cycle_outcome(
19519 &mut self,
19520 cycle: &[VertexId],
19521 mut delta: Option<&mut DeltaCollector>,
19522 dirty_filter: Option<&FxHashSet<VertexId>>,
19523 ) -> usize {
19524 let circ_error = LiteralValue::Error(
19525 ExcelError::new(ExcelErrorKind::Circ)
19526 .with_message("Circular dependency detected".to_string()),
19527 );
19528 let mut stamped = 0usize;
19529 for &vertex_id in cycle {
19530 if let Some(filter) = dirty_filter
19531 && !filter.contains(&vertex_id)
19532 {
19533 continue;
19534 }
19535 self.stamp_cycle_error(vertex_id, &circ_error, delta.as_deref_mut());
19536 stamped += 1;
19537 }
19538 stamped
19539 }
19540
19541 fn stamp_cycle_error(
19556 &mut self,
19557 vertex_id: VertexId,
19558 circ_error: &LiteralValue,
19559 mut delta: Option<&mut DeltaCollector>,
19560 ) {
19561 if self.graph.spill_registry_has_anchor(vertex_id) {
19563 self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
19564 }
19565 self.spill_mgr.release_owner(vertex_id);
19568
19569 if let Some(d) = delta
19571 && d.mode != DeltaMode::Off
19572 && let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id)
19573 {
19574 let sheet_name = self.graph.sheet_name(cell.sheet_id);
19575 let old = self
19576 .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
19577 .unwrap_or(LiteralValue::Empty);
19578 if old != *circ_error {
19579 d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
19580 }
19581 }
19582
19583 self.graph.update_vertex_value_ref(vertex_id, circ_error);
19584 self.mirror_vertex_value_to_overlay(vertex_id, circ_error);
19585 }
19586
19587 fn handle_cycle_unit(
19600 &mut self,
19601 cycle: &[VertexId],
19602 mut delta: Option<&mut DeltaCollector>,
19603 dirty_filter: Option<&FxHashSet<VertexId>>,
19604 cancel_flag: Option<&AtomicBool>,
19605 ) -> Result<usize, ExcelError> {
19606 self.resource_checkpoint(cycle.len() as u64)?;
19607 match self.config.cycle.detection {
19608 CycleDetection::Static => {
19609 Ok(self.apply_cycle_outcome(cycle, delta.as_deref_mut(), dirty_filter))
19610 }
19611 CycleDetection::Runtime => {
19612 if let Some(filter) = dirty_filter
19613 && !cycle.iter().any(|v| filter.contains(v))
19614 {
19615 return Ok(0);
19616 }
19617 self.evaluate_scc_unit(cycle, delta, cancel_flag)
19621 }
19622 }
19623 }
19624
19625 pub(crate) fn evaluate_scc_unit(
19645 &mut self,
19646 cycle: &[VertexId],
19647 mut delta: Option<&mut DeltaCollector>,
19648 cancel_flag: Option<&AtomicBool>,
19649 ) -> Result<usize, ExcelError> {
19650 struct SccMember {
19651 vertex: VertexId,
19652 cell: Option<CellRef>,
19653 }
19654
19655 let task_start = crate::instant::FzInstant::now();
19656
19657 let mut cell_members: Vec<(VertexId, CellRef)> = Vec::new();
19662 let mut name_members: Vec<(VertexId, String)> = Vec::new();
19663 let mut other_members: Vec<VertexId> = Vec::new();
19664 for &v in cycle {
19665 match self.graph.get_vertex_kind(v) {
19666 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
19667 match self.graph.get_cell_ref(v) {
19668 Some(cell) => cell_members.push((v, cell)),
19669 None => other_members.push(v),
19670 }
19671 }
19672 VertexKind::NamedScalar | VertexKind::NamedArray => {
19673 match self.graph.name_key_for_vertex(v) {
19674 Some(key) => name_members.push((v, key)),
19675 None => other_members.push(v),
19676 }
19677 }
19678 _ => other_members.push(v),
19679 }
19680 }
19681 cell_members.sort_unstable_by_key(|(_, c)| (c.sheet_id, c.coord.row(), c.coord.col()));
19682 name_members.sort_unstable_by(|(av, ak), (bv, bk)| ak.cmp(bk).then(av.cmp(bv)));
19683 other_members.sort_unstable();
19684
19685 let cell_refs: Vec<CellRef> = cell_members.iter().map(|(_, c)| *c).collect();
19686 let name_keys: Vec<String> = name_members.iter().map(|(_, k)| k.clone()).collect();
19687 let mut members: Vec<SccMember> = Vec::with_capacity(cycle.len());
19688 for (v, c) in &cell_members {
19689 members.push(SccMember {
19690 vertex: *v,
19691 cell: Some(*c),
19692 });
19693 }
19694 for (v, _) in &name_members {
19695 members.push(SccMember {
19696 vertex: *v,
19697 cell: None,
19698 });
19699 }
19700 for v in &other_members {
19701 members.push(SccMember {
19702 vertex: *v,
19703 cell: None,
19704 });
19705 }
19706 let n = members.len();
19707 let recordable = cell_refs.len() + name_keys.len();
19710
19711 let circ_error = LiteralValue::Error(
19712 ExcelError::new(ExcelErrorKind::Circ)
19713 .with_message("Circular dependency detected".to_string()),
19714 );
19715
19716 if !self.iterative_state_values.is_empty() {
19727 let restore: Vec<(VertexId, LiteralValue)> = members
19728 .iter()
19729 .filter_map(|m| {
19730 let cell = m.cell?;
19731 let persisted = self.iterative_state_values.get(&m.vertex)?;
19732 let sheet_name = self.graph.sheet_name(cell.sheet_id);
19733 let overlay = self
19734 .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
19735 .unwrap_or(LiteralValue::Empty);
19736 if matches!(overlay, LiteralValue::Empty) {
19737 Some((m.vertex, persisted.clone()))
19738 } else {
19739 None
19740 }
19741 })
19742 .collect();
19743 for (vertex, value) in restore {
19744 self.mirror_vertex_value_to_overlay(vertex, &value);
19745 }
19746 }
19747
19748 let snapshot: Vec<LiteralValue> = members
19751 .iter()
19752 .map(|m| match m.cell {
19753 Some(cell) => {
19754 let sheet_name = self.graph.sheet_name(cell.sheet_id);
19755 self.read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
19756 .unwrap_or(LiteralValue::Empty)
19757 }
19758 None => self
19759 .graph
19760 .get_value(m.vertex)
19761 .unwrap_or(LiteralValue::Empty),
19762 })
19763 .collect();
19764
19765 let mut excluded = vec![false; n];
19770 let mut last_value = snapshot.clone();
19771 let mut stamped = 0usize;
19772 for (i, m) in members.iter().enumerate() {
19773 match self.graph.get_vertex_kind(m.vertex) {
19774 VertexKind::FormulaArray => {
19775 self.stamp_cycle_error(m.vertex, &circ_error, delta.as_deref_mut());
19779 excluded[i] = true;
19780 last_value[i] = circ_error.clone();
19781 stamped += 1;
19782 }
19783 VertexKind::FormulaScalar | VertexKind::NamedScalar | VertexKind::NamedArray => {}
19784 _ => excluded[i] = true,
19785 }
19786 }
19787
19788 let collector = LiveEdgeCollector::new_with_names(&cell_refs, &name_keys);
19789
19790 let mut out_edges: Vec<Vec<u32>> = vec![Vec::new(); n];
19792 let mut pos: Vec<i64> = vec![-1; n];
19794 let mut changed = vec![false; n];
19796
19797 macro_rules! run_member {
19800 ($i:expr) => {{
19801 let i: usize = $i;
19802 let m = &members[i];
19803 if i < recordable {
19804 collector.set_current(i as u32);
19805 }
19806 let value = {
19807 let ctx = RecordingContext::new(&*self, &collector);
19808 match self.evaluate_vertex_recorded(m.vertex, &ctx, &collector) {
19809 Ok(v) => v,
19810 Err(e) => LiteralValue::Error(e),
19811 }
19812 };
19813 let is_cell_formula = m.cell.is_some();
19814 if is_cell_formula && matches!(value, LiteralValue::Array(_)) {
19815 self.stamp_cycle_error(m.vertex, &circ_error, None);
19820 excluded[i] = true;
19821 stamped += 1;
19822 changed[i] = last_value[i] != circ_error;
19823 last_value[i] = circ_error.clone();
19824 } else {
19825 self.graph.update_vertex_value_ref(m.vertex, &value);
19826 self.mirror_vertex_value_to_overlay(m.vertex, &value);
19827 #[cfg(debug_assertions)]
19831 if let Some(cell) = m.cell {
19832 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
19833 debug_assert!(
19834 self.read_delta_overlay_cell(
19835 &sheet_name,
19836 cell.coord.row() + 1,
19837 cell.coord.col() + 1
19838 )
19839 .is_none(),
19840 "user overlay must never shadow a formula SCC member ({sheet_name}!r{}c{})",
19841 cell.coord.row() + 1,
19842 cell.coord.col() + 1
19843 );
19844 }
19845 changed[i] = last_value[i] != value;
19846 last_value[i] = value;
19847 }
19848 }};
19849 }
19850
19851 let check_cancel = |flag: Option<&AtomicBool>| -> Result<(), ExcelError> {
19852 if let Some(flag) = flag
19853 && flag.load(Ordering::Relaxed)
19854 {
19855 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
19856 .with_message("Evaluation cancelled during SCC evaluation".to_string()));
19857 }
19858 Ok(())
19859 };
19860
19861 check_cancel(cancel_flag)?;
19863 let mut passes = 1usize;
19864 {
19865 let mut p = 0i64;
19866 for i in 0..n {
19867 if excluded[i] {
19868 continue;
19869 }
19870 run_member!(i);
19871 pos[i] = p;
19872 p += 1;
19873 }
19874 }
19875
19876 let policy = self.config.cycle.policy;
19892 let cap = n + 2;
19893 let mut witnessed_cycles = 0usize;
19894 let mut capped = false;
19895 let mut iterating = false;
19897 let mut converged = false;
19898 let mut exact_fixed_point = false;
19899 let mut prev_pass: Option<Vec<LiteralValue>> = None;
19903 let mut iter_max_delta = 0f64;
19906 let mut iter_nan_converged = 0usize;
19907 let mut settle_passes = 0usize;
19911 loop {
19912 let drained = collector.take_edges();
19915 for i in 0..n {
19916 if pos[i] >= 0 {
19917 out_edges[i].clear();
19918 }
19919 }
19920 for (from, to) in drained {
19921 debug_assert!(
19922 pos[from as usize] >= 0,
19923 "edge from a member that did not run"
19924 );
19925 out_edges[from as usize].push(to);
19926 }
19927 let mut edges: Vec<(u32, u32)> = Vec::new();
19928 for (i, outs) in out_edges.iter().enumerate() {
19929 if excluded[i] {
19930 continue;
19931 }
19932 for &t in outs {
19933 edges.push((i as u32, t));
19934 }
19935 }
19936 edges.sort_unstable();
19937 edges.dedup();
19938
19939 let analysis = analyze_live_graph(n, &edges);
19940
19941 if analysis.cycle_count > 0 {
19942 witnessed_cycles = witnessed_cycles.max(analysis.cycle_count);
19946 match policy {
19947 CyclePolicy::Error => {
19948 for i in 0..n {
19954 if analysis.in_cycle[i] && !excluded[i] {
19955 self.stamp_cycle_error(members[i].vertex, &circ_error, None);
19956 excluded[i] = true;
19957 last_value[i] = circ_error.clone();
19958 stamped += 1;
19959 }
19960 }
19961 check_cancel(cancel_flag)?;
19962 let order: Vec<usize> = analysis
19963 .topo
19964 .iter()
19965 .map(|&i| i as usize)
19966 .filter(|&i| !excluded[i])
19967 .collect();
19968 if !order.is_empty() {
19969 passes += 1;
19970 for i in order {
19971 run_member!(i);
19972 }
19973 }
19974 break;
19975 }
19976 CyclePolicy::Iterate {
19977 max_iterations,
19978 max_change,
19979 } => {
19980 iterating = true;
19982
19983 if let Some(prev) = &prev_pass {
19989 let mut round_max_delta = 0f64;
19990 let mut round_nan = 0usize;
19991 let mut all_converged = true;
19992 let mut round_exact = true;
19993 for i in 0..n {
19994 if excluded[i] {
19995 continue;
19999 }
20000 let out = crate::engine::convergence::values_converged(
20001 &prev[i],
20002 &last_value[i],
20003 max_change,
20004 self.config.date_system,
20005 );
20006 if out.nan_converged {
20007 round_nan += 1;
20008 }
20009 if let Some(d) = out.abs_delta {
20010 round_max_delta = round_max_delta.max(d);
20011 if d != 0.0 {
20012 round_exact = false;
20013 }
20014 }
20015 if !out.converged {
20016 all_converged = false;
20017 }
20018 }
20019 iter_max_delta = round_max_delta;
20022 iter_nan_converged = round_nan;
20023 if all_converged {
20024 exact_fixed_point = round_exact;
20025 converged = true;
20026 break;
20027 }
20028 }
20029
20030 if passes >= max_iterations as usize {
20042 capped = true;
20043 break;
20044 }
20045
20046 check_cancel(cancel_flag)?;
20047 prev_pass = Some(last_value.clone());
20055 for x in pos.iter_mut() {
20056 *x = -1;
20057 }
20058 changed.fill(false);
20059 passes += 1;
20060 let mut p = 0i64;
20061 for i in 0..n {
20062 if excluded[i] {
20063 continue;
20064 }
20065 run_member!(i);
20066 pos[i] = p;
20067 p += 1;
20068 }
20069 continue;
20070 }
20071 }
20072 }
20073
20074 let mut stale: Vec<usize> = Vec::new();
20077 for i in 0..n {
20078 if excluded[i] {
20079 continue;
20080 }
20081 let is_stale = out_edges[i].iter().any(|&t| {
20082 let t = t as usize;
20083 changed[t] && (pos[i] < 0 || (pos[t] >= 0 && pos[i] < pos[t]))
20084 });
20085 if is_stale {
20086 stale.push(i);
20087 }
20088 }
20089 if stale.is_empty() {
20090 break; }
20092 if 1 + settle_passes >= cap {
20093 capped = true;
20095 for (i, m) in members.iter().enumerate() {
20096 if !excluded[i] {
20097 self.stamp_cycle_error(m.vertex, &circ_error, None);
20098 excluded[i] = true;
20099 last_value[i] = circ_error.clone();
20100 stamped += 1;
20101 }
20102 }
20103 break;
20104 }
20105
20106 check_cancel(cancel_flag)?;
20107 prev_pass = None;
20114 let topo_pos = analysis.topo_positions();
20115 stale.sort_unstable_by_key(|&i| topo_pos[i]);
20116 for x in pos.iter_mut() {
20117 *x = -1;
20118 }
20119 changed.fill(false);
20120 passes += 1;
20121 settle_passes += 1;
20122 for (p, i) in stale.into_iter().enumerate() {
20123 run_member!(i);
20124 pos[i] = p as i64;
20125 }
20126 }
20127
20128 self.live_cancellation_after_work(SCC_CANCELLED)?;
20134
20135 if iterating && !converged && !capped {
20140 converged = true;
20141 }
20142
20143 collector.clear_current();
20146 if let Some(d) = delta
20147 && d.mode != DeltaMode::Off
20148 {
20149 for (i, m) in members.iter().enumerate() {
20150 if let Some(cell) = m.cell
20151 && last_value[i] != snapshot[i]
20152 {
20153 d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
20154 }
20155 }
20156 }
20157
20158 if !self.retained_scc_members.is_empty() {
20176 for m in members.iter() {
20177 self.retained_scc_members.remove(&m.vertex);
20178 }
20179 }
20180 if iterating {
20181 let retain = converged
20182 && !capped
20183 && exact_fixed_point
20184 && iter_nan_converged == 0
20185 && members
20186 .iter()
20187 .all(|m| !self.graph.is_volatile(m.vertex) && !self.graph.is_dynamic(m.vertex));
20188 if retain {
20189 if self.retained_scc_members.is_empty() {
20190 self.retained_scc_config_fingerprint = self.retained_scc_config_fingerprint();
20191 self.retained_scc_function_epoch_seen =
20192 crate::function_registry::semantic_epoch();
20193 self.retained_scc_provider_revision_seen =
20194 self.resolver.planning_semantic_revision();
20195 }
20196 let scc_id = self.next_retained_scc_id;
20197 self.next_retained_scc_id = self.next_retained_scc_id.wrapping_add(1);
20198 for (i, m) in members.iter().enumerate() {
20199 self.retained_scc_members.insert(m.vertex, scc_id);
20200 if matches!(last_value[i], LiteralValue::Empty) {
20203 self.iterative_state_values.remove(&m.vertex);
20204 } else {
20205 self.iterative_state_values
20206 .insert(m.vertex, last_value[i].clone());
20207 }
20208 }
20209 } else {
20210 self.pending_iterative_redirty
20211 .extend(members.iter().map(|m| m.vertex));
20212 }
20213 } else if !self.iterative_state_values.is_empty() {
20214 for m in members.iter() {
20218 self.iterative_state_values.remove(&m.vertex);
20219 }
20220 }
20221
20222 {
20223 let t = &mut self.last_cycle_telemetry;
20224 t.static_sccs += 1;
20225 if witnessed_cycles == 0 && stamped == 0 && !capped {
20226 t.phantom_sccs += 1;
20227 }
20228 t.live_cycles_witnessed += witnessed_cycles;
20229 t.circ_cells_stamped += stamped;
20230 t.settle_passes_total += passes;
20231 t.max_passes_single_scc = t.max_passes_single_scc.max(passes);
20232 if iterating {
20233 t.iterated_sccs += 1;
20234 if converged {
20235 t.converged_sccs += 1;
20236 }
20237 t.max_abs_delta_at_stop = t.max_abs_delta_at_stop.max(iter_max_delta);
20238 t.nan_converged += iter_nan_converged;
20239 }
20240 if capped {
20241 t.capped_sccs += 1;
20242 }
20243 t.elapsed_ms += task_start.elapsed().as_millis();
20244 }
20245
20246 Ok(stamped)
20247 }
20248
20249 fn evaluate_vertex_recorded(
20257 &self,
20258 vertex_id: VertexId,
20259 ctx: &RecordingContext<'_, R>,
20260 collector: &LiveEdgeCollector,
20261 ) -> Result<LiteralValue, ExcelError> {
20262 if !self.graph.vertex_exists(vertex_id) {
20263 return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
20264 .with_message(format!("Vertex not found: {vertex_id:?}")));
20265 }
20266
20267 let kind = self.graph.get_vertex_kind(vertex_id);
20268 let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
20269
20270 match kind {
20271 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
20272 let Some(view) = self.graph.formula_view(vertex_id) else {
20273 return Ok(LiteralValue::Number(0.0)); };
20275 let sheet_name = self.graph.sheet_name(sheet_id);
20276 let cell_ref = self
20277 .graph
20278 .get_cell_ref(vertex_id)
20279 .expect("cell ref for vertex");
20280 let interpreter = Interpreter::new_with_cell(ctx, sheet_name, cell_ref)
20281 .with_resolved_functions(&self.resolved_functions);
20282 interpreter
20283 .evaluate_formula_view(view, self.graph.data_store(), self.graph.sheet_reg())
20284 .map(|cv| {
20285 let format = cv.format_id();
20286 self.record_derived_format(vertex_id, format);
20287 self.materialize_formula_result(vertex_id, cv)
20288 })
20289 .or_else(|error| self.fit_formula_error(vertex_id, error))
20290 }
20291 VertexKind::NamedScalar | VertexKind::NamedArray => {
20292 let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
20293 ExcelError::new(ExcelErrorKind::Name)
20294 .with_message("Named range metadata missing".to_string())
20295 })?;
20296
20297 match &named_range.definition {
20298 NamedDefinition::Formula { ast, .. } => {
20299 let context_sheet = match named_range.scope {
20300 NameScope::Sheet(id) => id,
20301 NameScope::Workbook => sheet_id,
20302 };
20303 let sheet_name = self.graph.sheet_name(context_sheet);
20304 let cell_ref = self
20305 .graph
20306 .get_cell_ref(vertex_id)
20307 .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
20308 let interpreter = Interpreter::new_with_cell(ctx, sheet_name, cell_ref)
20309 .with_resolved_functions(&self.resolved_functions);
20310 if kind == VertexKind::NamedScalar {
20311 interpreter.evaluate_ast(ast).map(|cv| cv.into_literal())
20312 } else {
20313 match interpreter.evaluate_ast(ast) {
20314 Ok(cv) => match cv.into_literal() {
20315 v @ LiteralValue::Array(_) => Ok(v),
20316 other => Ok(LiteralValue::Array(vec![vec![other]])),
20317 },
20318 Err(err) => Ok(LiteralValue::Error(err)),
20319 }
20320 }
20321 }
20322 NamedDefinition::Cell(cell_ref) => {
20323 collector.record_scalar(
20327 cell_ref.sheet_id,
20328 cell_ref.coord.row(),
20329 cell_ref.coord.col(),
20330 );
20331 self.evaluate_vertex_immutable(vertex_id)
20332 }
20333 NamedDefinition::Range(range_ref) => {
20334 if range_ref.start.sheet_id == range_ref.end.sheet_id {
20335 collector.record_rect(
20336 range_ref.start.sheet_id,
20337 range_ref.start.coord.row(),
20338 range_ref.start.coord.col(),
20339 range_ref.end.coord.row(),
20340 range_ref.end.coord.col(),
20341 );
20342 }
20343 self.evaluate_vertex_immutable(vertex_id)
20344 }
20345 NamedDefinition::Literal(_) => self.evaluate_vertex_immutable(vertex_id),
20346 }
20347 }
20348 _ => self.evaluate_vertex_immutable(vertex_id),
20349 }
20350 }
20351
20352 fn pending_spill_occupied(&self, anchor: CellRef, end_row: u32, end_col: u32) -> bool {
20354 let sheet = self.graph.sheet_name(anchor.sheet_id);
20355 let package = self
20356 .staged_formulas
20357 .get(sheet)
20358 .and_then(|staged| staged.deferred_package.as_ref());
20359 self.staged_formula_index.occupies_spill(
20360 sheet,
20361 (anchor.coord.row() + 1, anchor.coord.col() + 1),
20362 (end_row + 1, end_col + 1),
20363 |point| package.is_some_and(|package| package.suppressed.contains(&point)),
20364 )
20365 }
20366
20367 fn remember_pending_spill(
20368 &mut self,
20369 vertex: VertexId,
20370 anchor: CellRef,
20371 region: Region,
20372 ) -> Result<(), ExcelError> {
20373 self.cancellation_checkpoint("pending spill occupancy")?;
20374 self.resource_checkpoint(1)?;
20375 if let Some(entry) = self
20376 .blocked_pending_spills
20377 .iter_mut()
20378 .find(|entry| entry.0 == vertex)
20379 {
20380 *entry = (vertex, anchor, region);
20381 return Ok(());
20382 }
20383 if self.blocked_pending_spills.len() == self.blocked_pending_spills.capacity() {
20384 let additional = self.blocked_pending_spills.capacity().max(1);
20387 let bytes =
20388 (additional as u64)
20389 .saturating_mul(std::mem::size_of::<(VertexId, CellRef, Region)>() as u64);
20390 if let Some(ledger) = self.active_resource_ledger.as_mut() {
20391 ledger
20392 .reserve_retained(bytes)
20393 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
20394 self.source_cache_accounted = self.source_cache_accounted.saturating_add(bytes);
20395 }
20396 if self
20397 .blocked_pending_spills
20398 .try_reserve_exact(additional)
20399 .is_err()
20400 {
20401 if let Some(ledger) = self.active_resource_ledger.as_mut() {
20402 ledger
20403 .release_retained(bytes)
20404 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
20405 self.source_cache_accounted -= bytes;
20406 }
20407 return Err(crate::engine::ResourceLedgerError::Exhausted(
20408 formualizer_common::ResourceExhaustionDetail {
20409 reason: formualizer_common::ResourceExhaustionReason::RetainedMemory,
20410 limit: u64::MAX,
20411 observed: bytes,
20412 request_id: None,
20413 },
20414 )
20415 .into_excel_error());
20416 }
20417 }
20418 self.blocked_pending_spills.push((vertex, anchor, region));
20419 Ok(())
20420 }
20421
20422 fn invalidate_pending_spills(&mut self, scope: StructuralScope) {
20426 if let StructuralScope::RemovedSheet(sheet) = scope {
20427 self.blocked_pending_spills
20428 .retain(|entry| entry.1.sheet_id != sheet);
20429 return;
20430 }
20431 for &(vertex, anchor, region) in &self.blocked_pending_spills {
20432 let affected = match scope {
20433 StructuralScope::Cell { sheet, row, col } => {
20434 region.intersects(&Region::point(sheet, row, col))
20435 }
20436 StructuralScope::Region(changed) => region.intersects(&changed),
20437 StructuralScope::Sheet(sheet) | StructuralScope::RemovedSheet(sheet) => {
20438 region.sheet_id() == sheet
20439 }
20440 StructuralScope::OpaqueGlobal | StructuralScope::AllSheets => true,
20441 };
20442 if affected
20443 && self.graph.vertex_exists(vertex)
20444 && self.graph.get_cell_ref(vertex) == Some(anchor)
20445 && matches!(
20446 self.graph.get_vertex_kind(vertex),
20447 VertexKind::FormulaScalar | VertexKind::FormulaArray
20448 )
20449 {
20450 self.graph.mark_vertex_dirty(vertex);
20451 }
20452 }
20453 }
20454
20455 fn guard_pending_spill_commit(
20456 &mut self,
20457 anchor_vertex: VertexId,
20458 targets: &[CellRef],
20459 ) -> Result<(), ExcelError> {
20460 let Some(anchor) = self.graph.get_cell_ref(anchor_vertex) else {
20461 return Ok(());
20462 };
20463 let Some(last) = targets.last() else {
20464 return Ok(());
20465 };
20466 let occupied = self.pending_spill_occupied(anchor, last.coord.row(), last.coord.col())
20467 || self
20468 .graph
20469 .table_intersects_spill(anchor, last.coord.row(), last.coord.col());
20470 if (occupied
20471 || self
20472 .blocked_pending_spills
20473 .iter()
20474 .any(|entry| entry.0 == anchor_vertex))
20475 && let Err(error) = self.remember_pending_spill(
20476 anchor_vertex,
20477 anchor,
20478 Region::rect(
20479 anchor.sheet_id,
20480 anchor.coord.row(),
20481 last.coord.row(),
20482 anchor.coord.col(),
20483 last.coord.col(),
20484 ),
20485 )
20486 {
20487 self.spill_mgr.release_owner(anchor_vertex);
20488 return Err(error);
20489 }
20490 if occupied {
20491 self.spill_mgr.release_owner(anchor_vertex);
20492 return Err(ExcelError::new(ExcelErrorKind::Spill)
20493 .with_message("Spill blocked")
20494 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
20495 expected_rows: last.coord.row() - anchor.coord.row() + 1,
20496 expected_cols: last.coord.col() - anchor.coord.col() + 1,
20497 }));
20498 }
20499 Ok(())
20500 }
20501
20502 fn commit_spill_and_mirror(
20504 &mut self,
20505 anchor_vertex: VertexId,
20506 targets: &[CellRef],
20507 rows: Vec<Vec<LiteralValue>>,
20508 delta: Option<&mut DeltaCollector>,
20509 ) -> Result<(), ExcelError> {
20510 self.guard_pending_spill_commit(anchor_vertex, targets)?;
20511 let prev_spill_cells = self.graph.spill_cells_to_clear(anchor_vertex);
20512
20513 if let Some(delta) = delta
20514 && delta.mode != DeltaMode::Off
20515 {
20516 let target_set: std::collections::HashSet<CellRef, CoordBuildHasher> =
20517 targets.iter().copied().collect();
20518 let empty = LiteralValue::Empty;
20519
20520 for cell in prev_spill_cells.iter() {
20522 if target_set.contains(cell) {
20523 continue;
20524 }
20525 let sheet_name = self.graph.sheet_name(cell.sheet_id);
20526 let old = self
20527 .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
20528 .unwrap_or(LiteralValue::Empty);
20529 if old != empty {
20530 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
20531 }
20532 }
20533
20534 if !targets.is_empty() && !rows.is_empty() && !rows[0].is_empty() {
20536 let width = rows[0].len();
20537 for (idx, cell) in targets.iter().enumerate() {
20538 let r_off = idx / width;
20539 let c_off = idx % width;
20540 let new = rows
20541 .get(r_off)
20542 .and_then(|r| r.get(c_off))
20543 .cloned()
20544 .unwrap_or(LiteralValue::Empty);
20545 let sheet_name = self.graph.sheet_name(cell.sheet_id);
20546 let old = self
20547 .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
20548 .unwrap_or(LiteralValue::Empty);
20549 if old != new {
20550 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
20551 }
20552 }
20553 } else {
20554 for cell in targets.iter() {
20556 let sheet_name = self.graph.sheet_name(cell.sheet_id);
20557 let old = self
20558 .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
20559 .unwrap_or(LiteralValue::Empty);
20560 if !matches!(old, LiteralValue::Empty) {
20561 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
20562 }
20563 }
20564 }
20565 }
20566
20567 let arrow_sheets = &self.arrow_sheets;
20570 self.spill_mgr.commit_array_with_value_probe(
20571 &mut self.graph,
20572 anchor_vertex,
20573 targets,
20574 rows.clone(),
20575 |g, cell| {
20576 let sheet_name = g.sheet_name(cell.sheet_id);
20577 let asheet = arrow_sheets.sheet(sheet_name)?;
20578 let r0 = cell.coord.row() as usize;
20579 let c0 = cell.coord.col() as usize;
20580 let v = asheet.get_cell_value(r0, c0);
20581 if matches!(v, LiteralValue::Empty) {
20582 None
20583 } else {
20584 Some(v)
20585 }
20586 },
20587 )?;
20588
20589 self.blocked_pending_spills
20590 .retain(|entry| entry.0 != anchor_vertex);
20591 self.spill_blocker_waiters.forget(anchor_vertex);
20592 if let Some(scope) = Self::structural_scope_from_cells(&prev_spill_cells) {
20593 self.record_structural_change(scope);
20594 }
20595 if let Some(scope) = Self::structural_scope_from_cells(targets) {
20596 self.record_structural_change(scope);
20597 }
20598
20599 if self.config.arrow_storage_enabled
20600 && self.config.delta_overlay_enabled
20601 && self.config.write_formula_overlay_enabled
20602 {
20603 if !prev_spill_cells.is_empty() {
20604 let target_set: std::collections::HashSet<CellRef, CoordBuildHasher> =
20605 targets.iter().copied().collect();
20606 let empty = LiteralValue::Empty;
20607 for cell in prev_spill_cells.iter() {
20608 if !target_set.contains(cell) {
20609 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
20610 self.mirror_value_to_computed_overlay(
20611 &sheet_name,
20612 cell.coord.row() + 1,
20613 cell.coord.col() + 1,
20614 &empty,
20615 );
20616 }
20617 }
20618 }
20619
20620 for (idx, cell) in targets.iter().enumerate() {
20621 if rows.is_empty() || rows[0].is_empty() {
20622 break;
20623 }
20624 let width = rows[0].len();
20625 let r_off = idx / width;
20626 let c_off = idx % width;
20627 let v = rows[r_off][c_off].clone();
20628 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
20629 self.mirror_value_to_computed_overlay(
20630 &sheet_name,
20631 cell.coord.row() + 1,
20632 cell.coord.col() + 1,
20633 &v,
20634 );
20635 }
20636 }
20637 Ok(())
20638 }
20639}
20640
20641#[cfg(test)]
20642#[path = "tests/authority_schedule_execution.rs"]
20643mod authority_schedule_execution;
20644
20645#[cfg(test)]
20646#[path = "tests/scalar_commit.rs"]
20647mod scalar_commit_tests;
20648
20649#[cfg(test)]
20650#[path = "tests/pending_spill.rs"]
20651mod pending_spill_tests;
20652
20653#[cfg(test)]
20654#[path = "tests/spill_batch_abort_192.rs"]
20655mod spill_batch_abort_192;
20656
20657use crate::engine::effects::Effect;
20662
20663type PlannedEffects = smallvec::SmallVec<[Effect; 2]>;
20665use crate::engine::graph::editor::change_log::{ChangeEvent, ChangeLog, SpillSnapshot};
20666
20667impl<R> Engine<R>
20668where
20669 R: EvaluationContext,
20670{
20671 pub(crate) fn plan_vertex_effects(
20678 &mut self,
20679 vertex_id: VertexId,
20680 computed_value: LiteralValue,
20681 ) -> Result<PlannedEffects, ExcelError> {
20682 if self.freshness_armed() {
20685 if self.freshness_drop_stale(vertex_id) {
20686 return Ok(PlannedEffects::new());
20687 }
20688 let effects = self.plan_vertex_effects_unrecorded(vertex_id, computed_value)?;
20689 self.freshness_mark_committed(vertex_id);
20690 return Ok(effects);
20691 }
20692 self.plan_vertex_effects_unrecorded(vertex_id, computed_value)
20693 }
20694
20695 fn plan_vertex_effects_unrecorded(
20696 &mut self,
20697 vertex_id: VertexId,
20698 computed_value: LiteralValue,
20699 ) -> Result<PlannedEffects, ExcelError> {
20700 let kind = self.graph.get_vertex_kind(vertex_id);
20701 let is_formula = matches!(kind, VertexKind::FormulaScalar | VertexKind::FormulaArray);
20702
20703 if !is_formula {
20707 if let Some(cell) = self.graph.get_cell_ref(vertex_id)
20708 && let Some(owner) = self.graph.spill_registry_anchor_for_cell(cell)
20709 && owner != vertex_id
20710 {
20711 return Ok(PlannedEffects::new());
20712 }
20713 return Ok(smallvec::smallvec![Effect::WriteCell {
20715 vertex_id,
20716 value: computed_value,
20717 }]);
20718 }
20719
20720 match computed_value {
20721 LiteralValue::Array(rows) => self.plan_array_effects(vertex_id, rows),
20722 other => self.plan_scalar_effects(vertex_id, other),
20723 }
20724 }
20725
20726 fn plan_scalar_effects(
20728 &mut self,
20729 vertex_id: VertexId,
20730 value: LiteralValue,
20731 ) -> Result<PlannedEffects, ExcelError> {
20732 if let LiteralValue::Error(error) = &value
20735 && error.kind == ExcelErrorKind::Spill
20736 && error.message() == Some("SpillTooLarge")
20737 && let formualizer_common::ExcelErrorExtra::Spill {
20738 expected_rows,
20739 expected_cols,
20740 } = error.extra()
20741 && u64::from(*expected_rows).saturating_mul(u64::from(*expected_cols))
20742 > u64::from(self.config.spill.max_spill_cells)
20743 {
20744 self.graph.set_kind(vertex_id, VertexKind::FormulaArray);
20745 return self.plan_spill_error_effects(
20746 vertex_id,
20747 "SpillTooLarge",
20748 *expected_rows,
20749 *expected_cols,
20750 );
20751 }
20752 if !matches!(&value, LiteralValue::Error(e) if e.kind == ExcelErrorKind::Spill) {
20753 self.blocked_pending_spills
20754 .retain(|entry| entry.0 != vertex_id);
20755 self.spill_blocker_waiters.forget(vertex_id);
20756 }
20757 let has_spill = self
20758 .graph
20759 .spill_cells_for_anchor(vertex_id)
20760 .is_some_and(|c| !c.is_empty());
20761
20762 let mut effects = PlannedEffects::new();
20763 if has_spill {
20764 effects.push(Effect::SpillClear {
20765 anchor_vertex: vertex_id,
20766 });
20767 }
20768 effects.push(Effect::WriteCell { vertex_id, value });
20769 Ok(effects)
20770 }
20771
20772 fn plan_array_effects(
20774 &mut self,
20775 vertex_id: VertexId,
20776 rows: Vec<Vec<LiteralValue>>,
20777 ) -> Result<PlannedEffects, ExcelError> {
20778 self.graph.set_kind(vertex_id, VertexKind::FormulaArray);
20780
20781 let anchor = self
20782 .graph
20783 .get_cell_ref(vertex_id)
20784 .expect("cell ref for vertex");
20785 let sheet_id = anchor.sheet_id;
20786 let h = rows.len() as u32;
20787 let w = rows.first().map(|r| r.len()).unwrap_or(0) as u32;
20788
20789 let spill_cells = (h as u64).saturating_mul(w as u64);
20791 if spill_cells > self.config.spill.max_spill_cells as u64 {
20792 return self.plan_spill_error_effects(vertex_id, "SpillTooLarge", h, w);
20793 }
20794
20795 const PACKED_MAX_ROW: u32 = 1_048_575;
20797 const PACKED_MAX_COL: u32 = 16_383;
20798 let end_row = anchor.coord.row().saturating_add(h).saturating_sub(1);
20799 let end_col = anchor.coord.col().saturating_add(w).saturating_sub(1);
20800 if end_row > PACKED_MAX_ROW || end_col > PACKED_MAX_COL {
20801 return self.plan_spill_error_effects(vertex_id, "Spill exceeds sheet bounds", h, w);
20802 }
20803
20804 if h != 0 && w != 0 && self.graph.table_intersects_spill(anchor, end_row, end_col) {
20805 return self.plan_spill_error_effects(vertex_id, "Spill blocked by table", h, w);
20806 }
20807 let mut targets = Vec::new();
20808 for r in 0..h {
20809 for c in 0..w {
20810 targets.push(self.graph.make_cell_ref_internal(
20811 sheet_id,
20812 anchor.coord.row() + r,
20813 anchor.coord.col() + c,
20814 ));
20815 }
20816 }
20817
20818 if h != 0 && w != 0 {
20819 let occupied = self.pending_spill_occupied(anchor, end_row, end_col);
20820 if occupied
20821 || self
20822 .blocked_pending_spills
20823 .iter()
20824 .any(|entry| entry.0 == vertex_id)
20825 {
20826 self.spill_mgr.release_owner(vertex_id);
20827 self.remember_pending_spill(
20828 vertex_id,
20829 anchor,
20830 Region::rect(
20831 sheet_id,
20832 anchor.coord.row(),
20833 end_row,
20834 anchor.coord.col(),
20835 end_col,
20836 ),
20837 )?;
20838 }
20839 if occupied {
20840 return self.plan_spill_error_effects(vertex_id, "Spill blocked", h, w);
20841 }
20842 }
20843
20844 match self.spill_mgr.reserve(
20846 vertex_id,
20847 anchor,
20848 SpillShape { rows: h, cols: w },
20849 SpillMeta {
20850 epoch: self.recalc_epoch,
20851 config: self.config.spill,
20852 },
20853 ) {
20854 Ok(()) => {
20855 let mut yield_effects = PlannedEffects::new();
20857 if let Err((e, blocker)) = self
20858 .graph
20859 .plan_spill_region_with_blocker(vertex_id, &targets)
20860 {
20861 let yielding = if e.message() == Some("BlockedBySpill") {
20862 self.graph.spills_yielding_to(vertex_id, &targets)
20863 } else {
20864 None
20865 };
20866 if let Some(yielding) = yielding {
20867 for &other in &yielding {
20871 let (rows, cols) = self
20872 .graph
20873 .spill_extent_for_anchor(other)
20874 .map(|(first, last)| {
20875 (
20876 last.coord.row() - first.coord.row() + 1,
20877 last.coord.col() - first.coord.col() + 1,
20878 )
20879 })
20880 .unwrap_or((0, 0));
20881 yield_effects.push(Effect::SpillClear {
20882 anchor_vertex: other,
20883 });
20884 yield_effects.push(Effect::WriteCell {
20885 vertex_id: other,
20886 value: Self::spill_error_value("Spill blocked", rows, cols),
20887 });
20888 }
20889 self.graph.mark_dirty_many(&yielding);
20890 } else {
20891 if matches!(e.message(), Some("BlockedByFormula" | "BlockedBySpill")) {
20895 self.spill_blocker_waiters.wait(vertex_id, blocker);
20896 }
20897 return self.plan_spill_error_effects(vertex_id, "Spill blocked", h, w);
20898 }
20899 }
20900
20901 if !self.graph.value_cache_enabled() {
20905 let sheet_name = self.graph.sheet_name(sheet_id);
20906 if let Some(asheet) = self.sheet_store().sheet(sheet_name) {
20907 for cell in targets.iter() {
20908 if *cell == anchor {
20910 continue;
20911 }
20912 if self.graph.spill_registry_anchor_for_cell(*cell).is_some() {
20914 continue;
20915 }
20916 if let Some(vid) = self.graph.get_vertex_id_for_address(cell)
20918 && vid != vertex_id
20919 {
20920 match self.graph.get_vertex_kind(vid) {
20921 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
20922 continue;
20923 }
20924 _ => {}
20925 }
20926 }
20927
20928 let v = asheet.get_cell_value(
20929 cell.coord.row() as usize,
20930 cell.coord.col() as usize,
20931 );
20932 if !matches!(v, LiteralValue::Empty) {
20933 return self.plan_spill_error_effects(
20934 vertex_id,
20935 "BlockedByValue",
20936 h,
20937 w,
20938 );
20939 }
20940 }
20941 }
20942 }
20943
20944 let top_left = rows
20945 .first()
20946 .and_then(|r| r.first())
20947 .cloned()
20948 .unwrap_or(LiteralValue::Empty);
20949
20950 let mut effects = yield_effects;
20951 let has_prev = self
20953 .graph
20954 .spill_cells_for_anchor(vertex_id)
20955 .is_some_and(|c| !c.is_empty());
20956 if has_prev {
20957 effects.push(Effect::SpillClear {
20958 anchor_vertex: vertex_id,
20959 });
20960 }
20961 effects.push(Effect::SpillCommit {
20962 anchor_vertex: vertex_id,
20963 anchor_cell: anchor,
20964 target_cells: targets,
20965 values: rows,
20966 });
20967 effects.push(Effect::WriteCell {
20968 vertex_id,
20969 value: top_left,
20970 });
20971 Ok(effects)
20972 }
20973 Err(e) => {
20974 let msg = e.message().unwrap_or("Spill blocked");
20975 self.plan_spill_error_effects(vertex_id, msg, h, w)
20976 }
20977 }
20978 }
20979
20980 fn spill_error_value(message: &str, expected_rows: u32, expected_cols: u32) -> LiteralValue {
20982 LiteralValue::Error(
20983 ExcelError::new(ExcelErrorKind::Spill)
20984 .with_message(message)
20985 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
20986 expected_rows,
20987 expected_cols,
20988 }),
20989 )
20990 }
20991
20992 fn plan_spill_error_effects(
20994 &mut self,
20995 vertex_id: VertexId,
20996 message: &str,
20997 expected_rows: u32,
20998 expected_cols: u32,
20999 ) -> Result<PlannedEffects, ExcelError> {
21000 self.spill_mgr.release_owner(vertex_id);
21001 let spill_val = Self::spill_error_value(message, expected_rows, expected_cols);
21002
21003 let effects = smallvec::smallvec![
21004 Effect::SpillClear {
21005 anchor_vertex: vertex_id,
21006 },
21007 Effect::WriteCell {
21008 vertex_id,
21009 value: spill_val,
21010 },
21011 ];
21012 Ok(effects)
21013 }
21014
21015 pub(crate) fn apply_effect(
21017 &mut self,
21018 effect: &Effect,
21019 delta: Option<&mut DeltaCollector>,
21020 log: Option<&mut ChangeLog>,
21021 ) -> Result<(), ExcelError> {
21022 self.apply_effect_with_computed_writes(effect, delta, log, None)
21023 }
21024
21025 fn apply_effect_with_computed_writes(
21026 &mut self,
21027 effect: &Effect,
21028 delta: Option<&mut DeltaCollector>,
21029 log: Option<&mut ChangeLog>,
21030 computed_writes: Option<&mut ComputedWriteBuffer>,
21031 ) -> Result<(), ExcelError> {
21032 match effect {
21033 Effect::WriteCell { vertex_id, value } => {
21034 self.apply_write_cell(*vertex_id, value, delta, computed_writes)?;
21035 }
21036 Effect::SpillClear { anchor_vertex } => {
21037 self.apply_spill_clear(*anchor_vertex, delta, log, computed_writes)?;
21038 }
21039 Effect::SpillCommit {
21040 anchor_vertex,
21041 anchor_cell: _,
21042 target_cells,
21043 values,
21044 } => {
21045 self.apply_spill_commit(
21046 *anchor_vertex,
21047 target_cells,
21048 values.clone(),
21049 delta,
21050 log,
21051 computed_writes,
21052 )?;
21053 }
21054 }
21055 Ok(())
21056 }
21057
21058 fn apply_write_cell(
21060 &mut self,
21061 vertex_id: VertexId,
21062 value: &LiteralValue,
21063 delta: Option<&mut DeltaCollector>,
21064 mut computed_writes: Option<&mut ComputedWriteBuffer>,
21065 ) -> Result<(), ExcelError> {
21066 if let Some(d) = delta
21067 && d.mode != DeltaMode::Off
21068 {
21069 if let Some(buffer) = computed_writes.as_deref_mut() {
21070 self.flush_computed_write_buffer(buffer)?;
21071 }
21072 if let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id) {
21073 let sheet_name = self.graph.sheet_name(cell.sheet_id);
21074 let old = self
21075 .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
21076 .unwrap_or(LiteralValue::Empty);
21077 if old != *value {
21078 d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
21079 }
21080 }
21081 }
21082 self.graph.update_vertex_value_ref(vertex_id, value);
21083 self.record_vertex_value_to_overlay(vertex_id, value, computed_writes)?;
21084 Ok(())
21085 }
21086
21087 fn apply_spill_clear(
21089 &mut self,
21090 anchor_vertex: VertexId,
21091 delta: Option<&mut DeltaCollector>,
21092 log: Option<&mut ChangeLog>,
21093 computed_writes: Option<&mut ComputedWriteBuffer>,
21094 ) -> Result<(), ExcelError> {
21095 if let Some(buffer) = computed_writes {
21096 self.flush_computed_write_buffer(buffer)?;
21097 }
21098
21099 let spill_cells = self.graph.spill_cells_to_clear(anchor_vertex);
21100 if spill_cells.is_empty() {
21101 return Ok(());
21102 }
21103
21104 let snapshot = if log.is_some() {
21106 self.snapshot_spill_for_anchor(anchor_vertex)
21107 } else {
21108 None
21109 };
21110
21111 if let Some(d) = delta
21113 && d.mode != DeltaMode::Off
21114 {
21115 let empty = LiteralValue::Empty;
21116 for cell in spill_cells.iter() {
21117 let sheet_name = self.graph.sheet_name(cell.sheet_id);
21118 let old = self
21119 .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
21120 .unwrap_or(LiteralValue::Empty);
21121 if old != empty {
21122 d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
21123 }
21124 }
21125 }
21126
21127 self.graph.clear_spill_region(anchor_vertex);
21128 if let Some(scope) = Self::structural_scope_from_cells(&spill_cells) {
21129 self.record_structural_change(scope);
21130 }
21131
21132 if self.config.arrow_storage_enabled
21134 && self.config.delta_overlay_enabled
21135 && self.config.write_formula_overlay_enabled
21136 {
21137 let empty = LiteralValue::Empty;
21138 for cell in spill_cells.iter() {
21139 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
21140 self.mirror_value_to_computed_overlay(
21141 &sheet_name,
21142 cell.coord.row() + 1,
21143 cell.coord.col() + 1,
21144 &empty,
21145 );
21146 }
21147 }
21148
21149 if let Some(log) = log
21151 && let Some(old) = snapshot
21152 {
21153 log.record(ChangeEvent::SpillCleared {
21154 anchor: anchor_vertex,
21155 old,
21156 });
21157 }
21158 Ok(())
21159 }
21160
21161 fn apply_spill_commit(
21163 &mut self,
21164 anchor_vertex: VertexId,
21165 target_cells: &[CellRef],
21166 values: Vec<Vec<LiteralValue>>,
21167 delta: Option<&mut DeltaCollector>,
21168 log: Option<&mut ChangeLog>,
21169 computed_writes: Option<&mut ComputedWriteBuffer>,
21170 ) -> Result<(), ExcelError> {
21171 self.guard_pending_spill_commit(anchor_vertex, target_cells)?;
21172 if let Some(buffer) = computed_writes {
21173 self.flush_computed_write_buffer(buffer)?;
21174 }
21175
21176 let old_snapshot = if log.is_some() {
21178 self.snapshot_spill_for_anchor(anchor_vertex)
21179 } else {
21180 None
21181 };
21182
21183 self.commit_spill_and_mirror(anchor_vertex, target_cells, values.clone(), delta)?;
21185
21186 if let Some(log) = log {
21188 log.record(ChangeEvent::SpillCommitted {
21189 anchor: anchor_vertex,
21190 old: old_snapshot,
21191 new: SpillSnapshot {
21192 target_cells: target_cells.to_vec(),
21193 values,
21194 },
21195 });
21196 }
21197 Ok(())
21198 }
21199
21200 fn snapshot_spill_for_anchor(&self, anchor: VertexId) -> Option<SpillSnapshot> {
21205 let cells = self.graph.spill_cells_for_anchor(anchor)?.to_vec();
21206 if cells.is_empty() {
21207 return None;
21208 }
21209
21210 let max = self.config.spill.max_spill_cells as usize;
21211 let mut cells = cells;
21212 if cells.len() > max {
21213 cells.truncate(max);
21214 }
21215
21216 let first = *cells.first().expect("non-empty spill cells");
21217 let sheet_name = self.graph.sheet_name(first.sheet_id).to_string();
21218 let row0 = first.coord.row();
21219 let col0 = first.coord.col();
21220
21221 let mut max_row = row0;
21222 let mut max_col = col0;
21223 let mut by_coord: FxHashMap<(u32, u32), LiteralValue> = FxHashMap::default();
21224 for cell in &cells {
21225 max_row = max_row.max(cell.coord.row());
21226 max_col = max_col.max(cell.coord.col());
21227 let v = self
21228 .read_cell_value(&sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
21229 .unwrap_or(LiteralValue::Empty);
21230 by_coord.insert((cell.coord.row(), cell.coord.col()), v);
21231 }
21232
21233 let rows = (max_row - row0 + 1) as usize;
21234 let cols = (max_col - col0 + 1) as usize;
21235 let mut values: Vec<Vec<LiteralValue>> = Vec::with_capacity(rows);
21236 for r in 0..rows {
21237 let mut row: Vec<LiteralValue> = Vec::with_capacity(cols);
21238 for c in 0..cols {
21239 row.push(
21240 by_coord
21241 .get(&(row0 + r as u32, col0 + c as u32))
21242 .cloned()
21243 .unwrap_or(LiteralValue::Empty),
21244 );
21245 }
21246 values.push(row);
21247 }
21248
21249 Some(SpillSnapshot {
21250 target_cells: cells,
21251 values,
21252 })
21253 }
21254
21255 fn flush_before_range_dependent_vertex(
21256 &mut self,
21257 vertex_id: VertexId,
21258 computed_writes: &mut ComputedWriteBuffer,
21259 ) -> Result<(), ExcelError> {
21260 if self.graph.reads_compressed_range(vertex_id) {
21261 self.flush_computed_write_buffer(computed_writes)?;
21262 }
21263 Ok(())
21264 }
21265
21266 fn plan_vertex_effects_with_computed_flush(
21267 &mut self,
21268 vertex_id: VertexId,
21269 computed_value: LiteralValue,
21270 computed_writes: &mut ComputedWriteBuffer,
21271 ) -> Result<PlannedEffects, ExcelError> {
21272 if matches!(&computed_value, LiteralValue::Array(_)) {
21273 self.flush_computed_write_buffer(computed_writes)?;
21274 }
21275 self.plan_vertex_effects(vertex_id, computed_value)
21276 }
21277
21278 fn evaluate_small_layer_direct_effects(
21281 &mut self,
21282 layer: &super::scheduler::Layer,
21283 delta: Option<&mut DeltaCollector>,
21284 log: Option<&mut ChangeLog>,
21285 cancel_flag: Option<&AtomicBool>,
21286 cancel_check_every: usize,
21287 cancel_message: &'static str,
21288 ) -> Result<usize, ExcelError> {
21289 let cancel = cancel_flag.map(|flag| (flag, cancel_check_every, cancel_message));
21290 self.evaluate_layer_units(layer, delta, log, cancel, false)
21291 }
21292
21293 fn evaluate_layer_units(
21300 &mut self,
21301 layer: &super::scheduler::Layer,
21302 delta: Option<&mut DeltaCollector>,
21303 log: Option<&mut ChangeLog>,
21304 cancel: Option<(&AtomicBool, usize, &'static str)>,
21305 buffered: bool,
21306 ) -> Result<usize, ExcelError> {
21307 self.evaluate_layer_units_until(layer, delta, log, cancel, buffered, None)
21308 }
21309
21310 fn evaluate_layer_units_until(
21314 &mut self,
21315 layer: &super::scheduler::Layer,
21316 mut delta: Option<&mut DeltaCollector>,
21317 mut log: Option<&mut ChangeLog>,
21318 cancel: Option<(&AtomicBool, usize, &'static str)>,
21319 buffered: bool,
21320 stop_at: Option<crate::instant::FzInstant>,
21321 ) -> Result<usize, ExcelError> {
21322 let mut chain_values = None;
21325 if layer.sequential && !layer.runs.is_empty() {
21326 chain_values = match layer.runs.as_slice() {
21327 [run] if run.start == 0 && run.len as usize == layer.vertices.len() => {
21328 self.try_chain_lift(*run, &layer.vertices)
21329 }
21330 _ => None,
21331 };
21332 if chain_values.is_none() {
21333 let cells = super::scheduler::Layer {
21334 vertices: layer.vertices.clone(),
21335 runs: Vec::new(),
21336 sequential: true,
21337 };
21338 return self.evaluate_layer_units_until(&cells, delta, log, cancel, false, stop_at);
21339 }
21340 }
21341 let chained = chain_values.is_some();
21342 let buffered = buffered || chained;
21344 let track_unflushed = buffered
21351 && self.freshness_armed()
21352 && layer.vertices.iter().any(|&v| self.graph.is_dynamic(v));
21353 let mut committed_unit: &[VertexId] = &[];
21354 let mut computed_writes = ComputedWriteBuffer::default();
21355 let mut next_check = 0usize;
21356 let mut done = 0usize;
21357 for unit in layer_units(layer) {
21358 if done > 0
21359 && let Some(stop_at) = stop_at
21360 && crate::instant::FzInstant::now() >= stop_at
21361 {
21362 self.flush_computed_write_buffer(&mut computed_writes)?;
21363 if track_unflushed {
21364 self.freshness_flushed();
21365 }
21366 return Ok(done);
21367 }
21368 if let Some((flag, every, message)) = cancel
21369 && every > 0
21370 && done >= next_check
21371 {
21372 next_check = (done / every + 1) * every;
21373 if flag.load(Ordering::Relaxed) {
21374 if buffered {
21375 self.flush_computed_write_buffer(&mut computed_writes)?;
21376 }
21377 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
21378 .with_message(message.to_string()));
21379 }
21380 }
21381 if buffered && self.unit_reads_compressed_range(layer, unit) {
21382 self.flush_computed_write_buffer(&mut computed_writes)?;
21383 }
21384 if track_unflushed {
21388 if computed_writes.is_empty() {
21389 self.freshness_flushed();
21390 } else {
21391 self.freshness_note_unflushed(committed_unit);
21392 }
21393 committed_unit = unit_members(layer, unit);
21394 }
21395 let values = match (chain_values.take(), unit) {
21396 (Some(chain), LayerUnit::Run(run)) => {
21397 if let Err(e) = self.live_cancellation_after_work(UNIT_CANCELLED) {
21398 self.flush_computed_write_buffer(&mut computed_writes)?;
21399 return Err(e);
21400 }
21401 let members =
21402 &layer.vertices[run.start as usize..(run.start + run.len) as usize];
21403 let delta_active = delta.as_deref().is_some_and(|d| d.mode != DeltaMode::Off);
21404 match self.commit_run_numbers(
21405 run,
21406 members,
21407 &chain,
21408 delta_active,
21409 Some(&mut computed_writes),
21410 ) {
21411 Ok(true) => {
21412 done += chain.len();
21413 continue;
21414 }
21415 Ok(false) => {}
21416 Err(e) => {
21417 self.flush_computed_write_buffer(&mut computed_writes)?;
21418 return Err(e);
21419 }
21420 }
21421 members
21422 .iter()
21423 .copied()
21424 .zip(chain.into_iter().map(LiteralValue::Number))
21425 .collect()
21426 }
21427 (_, unit) => self.evaluate_unit_immutable(layer, unit),
21428 };
21429 if let Err(e) = self.live_cancellation_after_work(UNIT_CANCELLED) {
21432 self.flush_computed_write_buffer(&mut computed_writes)?;
21433 return Err(e);
21434 }
21435 done += values.len();
21436 if let LayerUnit::Run(run) = unit {
21437 let members = &layer.vertices[run.start as usize..(run.start + run.len) as usize];
21438 let delta_active = delta.as_deref().is_some_and(|d| d.mode != DeltaMode::Off);
21439 let committed = self.commit_run_scalars(
21440 run,
21441 members,
21442 &values,
21443 delta_active,
21444 buffered.then_some(&mut computed_writes),
21445 );
21446 match committed {
21447 Ok(true) => continue,
21448 Ok(false) => {}
21449 Err(e) => {
21450 self.flush_computed_write_buffer(&mut computed_writes)?;
21451 return Err(e);
21452 }
21453 }
21454 }
21455 let guarded = self.freshness_begin_batch_commit(&values);
21458 for (vertex_id, value) in values {
21459 let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21460 let effects = if buffered {
21461 self.plan_vertex_effects_with_computed_flush(
21462 vertex_id,
21463 value,
21464 &mut computed_writes,
21465 )
21466 } else {
21467 self.plan_vertex_effects(vertex_id, value)
21468 };
21469 let effects = match effects {
21470 Ok(effects) => effects,
21471 Err(e) => {
21472 self.flush_computed_write_buffer(&mut computed_writes)?;
21473 return Err(e);
21474 }
21475 };
21476 for effect in &effects {
21477 if let Err(e) = self.apply_effect_with_computed_writes(
21478 effect,
21479 delta.as_deref_mut(),
21480 log.as_deref_mut(),
21481 buffered.then_some(&mut computed_writes),
21482 ) {
21483 self.flush_computed_write_buffer(&mut computed_writes)?;
21484 return Err(e);
21485 }
21486 }
21487 self.freshness_keep_redirtied(redirtied, vertex_id);
21488 }
21489 }
21490 self.flush_computed_write_buffer(&mut computed_writes)?;
21491 if track_unflushed {
21492 self.freshness_flushed();
21493 }
21494 #[cfg(debug_assertions)]
21497 if chained {
21498 for &v in &layer.vertices {
21499 let cell = self.graph.get_cell_ref(v);
21500 let (sheet, row, col) = cell
21501 .map(|c| {
21502 (
21503 self.graph.sheet_name(c.sheet_id).to_string(),
21504 c.coord.row() + 1,
21505 c.coord.col() + 1,
21506 )
21507 })
21508 .expect("chain member cell");
21509 let written = self.read_cell_value(&sheet, row, col);
21510 let oracle = self
21511 .evaluate_vertex_immutable(v)
21512 .unwrap_or_else(LiteralValue::Error);
21513 assert!(
21514 written
21515 .as_ref()
21516 .is_some_and(|w| same_value_bits(w, &oracle)),
21517 "chain member {sheet}!R{row}C{col}: {written:?} vs per-cell {oracle:?}"
21518 );
21519 }
21520 }
21521 #[cfg(not(debug_assertions))]
21522 let _ = chained;
21523 Ok(layer.vertices.len())
21524 }
21525
21526 fn evaluate_layer_sequential_effects(
21528 &mut self,
21529 layer: &super::scheduler::Layer,
21530 ) -> Result<usize, ExcelError> {
21531 let buffered = buffer_layer_writes(layer);
21532 self.evaluate_layer_units(layer, None, None, None, buffered)
21533 }
21534
21535 fn evaluate_layer_sequential_with_delta_effects(
21537 &mut self,
21538 layer: &super::scheduler::Layer,
21539 delta: &mut DeltaCollector,
21540 ) -> Result<usize, ExcelError> {
21541 let buffered = buffer_layer_writes(layer);
21542 self.evaluate_layer_units(layer, Some(delta), None, None, buffered)
21543 }
21544
21545 fn evaluate_layer_sequential_cancellable_effects(
21547 &mut self,
21548 layer: &super::scheduler::Layer,
21549 cancel_flag: &AtomicBool,
21550 ) -> Result<usize, ExcelError> {
21551 let buffered = buffer_layer_writes(layer);
21552 let cancel = (cancel_flag, 256, "Evaluation cancelled within layer");
21553 self.evaluate_layer_units(layer, None, None, Some(cancel), buffered)
21554 }
21555
21556 fn evaluate_layer_sequential_cancellable_demand_driven_effects(
21558 &mut self,
21559 layer: &super::scheduler::Layer,
21560 cancel_flag: &AtomicBool,
21561 ) -> Result<usize, ExcelError> {
21562 let buffered = buffer_layer_writes(layer);
21563 let cancel = (
21564 cancel_flag,
21565 128,
21566 "Demand-driven evaluation cancelled within layer",
21567 );
21568 self.evaluate_layer_units(layer, None, None, Some(cancel), buffered)
21569 }
21570
21571 fn evaluate_layer_parallel_effects(
21573 &mut self,
21574 layer: &super::scheduler::Layer,
21575 min_chunk: u32,
21576 ) -> Result<usize, ExcelError> {
21577 let thread_pool = self.thread_pool.as_ref().unwrap().clone();
21578
21579 let phases = self.parallel_phases(layer);
21580 let mut applied = 0usize;
21581
21582 for (units, group) in &phases {
21583 let group = &group[..];
21584 if group.is_empty() {
21585 continue;
21586 }
21587 let mut computed_writes = ComputedWriteBuffer::default();
21588
21589 let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> =
21590 thread_pool.install(|| self.evaluate_units_parallel(layer, units, None, min_chunk));
21591
21592 let results = results.and_then(|results| {
21595 self.live_cancellation_after_work(GROUP_CANCELLED)
21596 .map(|()| results)
21597 });
21598 self.freshness_gate_group(group);
21601 match results {
21602 Ok(vertex_results) => {
21603 let (vertex_results, committed) = self.commit_parallel_runs(
21604 layer,
21605 units,
21606 vertex_results,
21607 false,
21608 &mut computed_writes,
21609 )?;
21610 applied = applied.saturating_add(committed);
21611 let guarded = self.freshness_begin_batch_commit(&vertex_results);
21616 let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
21617 let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
21618 for (vertex_id, result) in vertex_results {
21619 if matches!(result, LiteralValue::Array(_)) {
21620 arrays.push((vertex_id, result));
21621 } else {
21622 others.push((vertex_id, result));
21623 }
21624 }
21625 for (vertex_id, result) in arrays {
21626 let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21627 let effects = match self.plan_vertex_effects_with_computed_flush(
21628 vertex_id,
21629 result,
21630 &mut computed_writes,
21631 ) {
21632 Ok(effects) => effects,
21633 Err(e) => {
21634 self.flush_computed_write_buffer(&mut computed_writes)?;
21635 return Err(e);
21636 }
21637 };
21638 for effect in &effects {
21639 if let Err(e) = self.apply_effect_with_computed_writes(
21640 effect,
21641 None,
21642 None,
21643 Some(&mut computed_writes),
21644 ) {
21645 self.flush_computed_write_buffer(&mut computed_writes)?;
21646 return Err(e);
21647 }
21648 }
21649 self.freshness_keep_redirtied(redirtied, vertex_id);
21650 applied = applied.saturating_add(1);
21651 }
21652 self.flush_computed_write_buffer(&mut computed_writes)?;
21654 for (vertex_id, result) in others {
21655 let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21656 let effects = match self.plan_vertex_effects_with_computed_flush(
21657 vertex_id,
21658 result,
21659 &mut computed_writes,
21660 ) {
21661 Ok(effects) => effects,
21662 Err(e) => {
21663 self.flush_computed_write_buffer(&mut computed_writes)?;
21664 return Err(e);
21665 }
21666 };
21667 for effect in &effects {
21668 if let Err(e) = self.apply_effect_with_computed_writes(
21669 effect,
21670 None,
21671 None,
21672 Some(&mut computed_writes),
21673 ) {
21674 self.flush_computed_write_buffer(&mut computed_writes)?;
21675 return Err(e);
21676 }
21677 }
21678 self.freshness_keep_redirtied(redirtied, vertex_id);
21679 applied = applied.saturating_add(1);
21680 }
21681 self.flush_computed_write_buffer(&mut computed_writes)?;
21683 }
21684 Err(e) => {
21685 self.flush_computed_write_buffer(&mut computed_writes)?;
21686 return Err(e);
21687 }
21688 }
21689 }
21690
21691 Ok(applied)
21692 }
21693
21694 fn evaluate_layer_parallel_with_delta_effects(
21696 &mut self,
21697 layer: &super::scheduler::Layer,
21698 delta: &mut DeltaCollector,
21699 ) -> Result<usize, ExcelError> {
21700 let thread_pool = self.thread_pool.as_ref().unwrap().clone();
21701
21702 let phases = self.parallel_phases(layer);
21703 let mut applied = 0usize;
21704
21705 for (units, group) in &phases {
21706 let group = &group[..];
21707 if group.is_empty() {
21708 continue;
21709 }
21710 let mut computed_writes = ComputedWriteBuffer::default();
21711 let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> =
21712 thread_pool.install(|| self.evaluate_units_parallel(layer, units, None, 8));
21713
21714 let results = results.and_then(|results| {
21717 self.live_cancellation_after_work(GROUP_CANCELLED)
21718 .map(|()| results)
21719 });
21720 self.freshness_gate_group(group);
21723 match results {
21724 Ok(vertex_results) => {
21725 let (vertex_results, committed) = self.commit_parallel_runs(
21726 layer,
21727 units,
21728 vertex_results,
21729 delta.mode != DeltaMode::Off,
21730 &mut computed_writes,
21731 )?;
21732 applied = applied.saturating_add(committed);
21733 let guarded = self.freshness_begin_batch_commit(&vertex_results);
21736 let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
21737 let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
21738 for (vertex_id, result) in vertex_results {
21739 if matches!(result, LiteralValue::Array(_)) {
21740 arrays.push((vertex_id, result));
21741 } else {
21742 others.push((vertex_id, result));
21743 }
21744 }
21745 for (vertex_id, result) in arrays {
21746 let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21747 let effects = match self.plan_vertex_effects_with_computed_flush(
21748 vertex_id,
21749 result,
21750 &mut computed_writes,
21751 ) {
21752 Ok(effects) => effects,
21753 Err(e) => {
21754 self.flush_computed_write_buffer(&mut computed_writes)?;
21755 return Err(e);
21756 }
21757 };
21758 for effect in &effects {
21759 if let Err(e) = self.apply_effect_with_computed_writes(
21760 effect,
21761 Some(delta),
21762 None,
21763 Some(&mut computed_writes),
21764 ) {
21765 self.flush_computed_write_buffer(&mut computed_writes)?;
21766 return Err(e);
21767 }
21768 }
21769 self.freshness_keep_redirtied(redirtied, vertex_id);
21770 applied = applied.saturating_add(1);
21771 }
21772 self.flush_computed_write_buffer(&mut computed_writes)?;
21773 for (vertex_id, result) in others {
21774 let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21775 let effects = match self.plan_vertex_effects_with_computed_flush(
21776 vertex_id,
21777 result,
21778 &mut computed_writes,
21779 ) {
21780 Ok(effects) => effects,
21781 Err(e) => {
21782 self.flush_computed_write_buffer(&mut computed_writes)?;
21783 return Err(e);
21784 }
21785 };
21786 for effect in &effects {
21787 if let Err(e) = self.apply_effect_with_computed_writes(
21788 effect,
21789 Some(delta),
21790 None,
21791 Some(&mut computed_writes),
21792 ) {
21793 self.flush_computed_write_buffer(&mut computed_writes)?;
21794 return Err(e);
21795 }
21796 }
21797 self.freshness_keep_redirtied(redirtied, vertex_id);
21798 applied = applied.saturating_add(1);
21799 }
21800 self.flush_computed_write_buffer(&mut computed_writes)?;
21801 }
21802 Err(e) => {
21803 self.flush_computed_write_buffer(&mut computed_writes)?;
21804 return Err(e);
21805 }
21806 }
21807 }
21808
21809 Ok(applied)
21810 }
21811
21812 fn evaluate_layer_parallel_cancellable_effects(
21814 &mut self,
21815 layer: &super::scheduler::Layer,
21816 cancel_flag: &AtomicBool,
21817 ) -> Result<usize, ExcelError> {
21818 let thread_pool = self.thread_pool.as_ref().unwrap().clone();
21819
21820 if cancel_flag.load(Ordering::Relaxed) {
21821 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
21822 .with_message("Parallel evaluation cancelled before starting".to_string()));
21823 }
21824
21825 let phases = self.parallel_phases(layer);
21826 let mut applied = 0usize;
21827
21828 for (units, group) in &phases {
21829 let group = &group[..];
21830 if group.is_empty() {
21831 continue;
21832 }
21833 let mut computed_writes = ComputedWriteBuffer::default();
21834
21835 let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> = thread_pool
21836 .install(|| self.evaluate_units_parallel(layer, units, Some(cancel_flag), 8));
21837
21838 let results = results.and_then(|results| {
21841 self.live_cancellation_after_work(GROUP_CANCELLED)
21842 .map(|()| results)
21843 });
21844 self.freshness_gate_group(group);
21847 match results {
21848 Ok(vertex_results) => {
21849 let (vertex_results, committed) = self.commit_parallel_runs(
21850 layer,
21851 units,
21852 vertex_results,
21853 false,
21854 &mut computed_writes,
21855 )?;
21856 applied = applied.saturating_add(committed);
21857 let guarded = self.freshness_begin_batch_commit(&vertex_results);
21860 let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
21861 let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
21862 for (vertex_id, result) in vertex_results {
21863 if matches!(result, LiteralValue::Array(_)) {
21864 arrays.push((vertex_id, result));
21865 } else {
21866 others.push((vertex_id, result));
21867 }
21868 }
21869 for (vertex_id, result) in arrays {
21870 let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21871 let effects = match self.plan_vertex_effects_with_computed_flush(
21872 vertex_id,
21873 result,
21874 &mut computed_writes,
21875 ) {
21876 Ok(effects) => effects,
21877 Err(e) => {
21878 self.flush_computed_write_buffer(&mut computed_writes)?;
21879 return Err(e);
21880 }
21881 };
21882 for effect in &effects {
21883 if let Err(e) = self.apply_effect_with_computed_writes(
21884 effect,
21885 None,
21886 None,
21887 Some(&mut computed_writes),
21888 ) {
21889 self.flush_computed_write_buffer(&mut computed_writes)?;
21890 return Err(e);
21891 }
21892 }
21893 self.freshness_keep_redirtied(redirtied, vertex_id);
21894 applied = applied.saturating_add(1);
21895 }
21896 self.flush_computed_write_buffer(&mut computed_writes)?;
21897 for (vertex_id, result) in others {
21898 let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21899 let effects = match self.plan_vertex_effects_with_computed_flush(
21900 vertex_id,
21901 result,
21902 &mut computed_writes,
21903 ) {
21904 Ok(effects) => effects,
21905 Err(e) => {
21906 self.flush_computed_write_buffer(&mut computed_writes)?;
21907 return Err(e);
21908 }
21909 };
21910 for effect in &effects {
21911 if let Err(e) = self.apply_effect_with_computed_writes(
21912 effect,
21913 None,
21914 None,
21915 Some(&mut computed_writes),
21916 ) {
21917 self.flush_computed_write_buffer(&mut computed_writes)?;
21918 return Err(e);
21919 }
21920 }
21921 self.freshness_keep_redirtied(redirtied, vertex_id);
21922 applied = applied.saturating_add(1);
21923 }
21924 self.flush_computed_write_buffer(&mut computed_writes)?;
21925 }
21926 Err(e) => {
21927 self.flush_computed_write_buffer(&mut computed_writes)?;
21928 return Err(e);
21929 }
21930 }
21931 }
21932
21933 Ok(applied)
21934 }
21935
21936 pub fn evaluate_all_logged(&mut self, log: &mut ChangeLog) -> Result<EvalResult, ExcelError> {
21943 self.observe_evaluation_resource_request(EvaluationRequestKind::FullLogged, |engine| {
21944 engine.evaluate_all_logged_unobserved(log)
21945 })
21946 }
21947
21948 fn evaluate_all_logged_unobserved(
21949 &mut self,
21950 log: &mut ChangeLog,
21951 ) -> Result<EvalResult, ExcelError> {
21952 self.observe_function_semantic_epoch()?;
21953 let _source_cache = self.source_cache_session();
21954 self.validate_deterministic_mode()?;
21955 if self.config.defer_graph_building {
21956 self.build_graph_all()?;
21957 }
21958 self.require_unified_authority()?;
21959 self.begin_evaluation_request();
21960 self.reset_virtual_dep_telemetry_if_disabled();
21961 let start = crate::instant::FzInstant::now();
21962 let mut computed_vertices = 0;
21963 let mut cycle_errors = 0;
21964
21965 let mut replan_iterations = 0;
21966 const MAX_REPLAN: usize = 5;
21967 let mut telemetry = self
21968 .config
21969 .enable_virtual_dep_telemetry
21970 .then(|| self.start_virtual_dep_telemetry());
21971
21972 log.begin_compound(format!("evaluate_all(epoch={})", self.recalc_epoch));
21973
21974 let result = (|| -> Result<EvalResult, ExcelError> {
21975 loop {
21976 let to_evaluate = self.graph.get_evaluation_vertices();
21977 if to_evaluate.is_empty() {
21978 if let Some(t) = telemetry.as_mut()
21979 && t.bailout_reason.is_none()
21980 {
21981 t.bailout_reason = Some("no_work");
21982 }
21983 break;
21984 }
21985
21986 let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
21987 if let Some(t) = telemetry.as_mut() {
21988 Self::accumulate_schedule_meta(t, &meta);
21989 }
21990
21991 self.begin_pass(&schedule);
21994 for (unit_index, &unit) in schedule.units.iter().enumerate() {
21995 match unit {
21996 ScheduleUnit::Cycle(i) => {
21997 if self.handle_cycle_unit(schedule.unit_cycle(i), None, None, None)? > 0
22008 {
22009 cycle_errors += 1;
22010 }
22011 }
22012 ScheduleUnit::Layer(i) => {
22013 computed_vertices +=
22014 self.evaluate_layer_logged(schedule.unit_layer(i), log)?;
22015 }
22016 }
22017 if self.stop_after_unit(&schedule, unit_index) {
22018 break;
22019 }
22020 }
22021
22022 let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
22023 if let Some(t) = telemetry.as_mut() {
22024 t.changed_vdeps_total += changed_vertices.len();
22025 }
22026 self.resource_checkpoint(0)?;
22027 if !self.finish_pass_dirty(&to_evaluate, &changed_vertices) {
22028 if let Some(t) = telemetry.as_mut() {
22029 t.bailout_reason = Some("converged");
22030 }
22031 break;
22032 }
22033 if replan_iterations >= MAX_REPLAN {
22034 if let Some(mut t) = telemetry.take() {
22035 t.bailout_reason = Some("max_replan");
22036 t.replan_iterations = replan_iterations;
22037 self.last_virtual_dep_telemetry = t;
22038 }
22039 return Err(
22040 self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
22041 );
22042 }
22043 replan_iterations += 1;
22044 }
22045
22046 if let Some(mut t) = telemetry {
22047 t.replan_iterations = replan_iterations;
22048 self.last_virtual_dep_telemetry = t;
22049 }
22050
22051 self.redirty_for_next_recalc();
22052 self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
22053
22054 Ok(EvalResult {
22055 computed_vertices,
22056 cycle_errors,
22057 elapsed: start.elapsed(),
22058 })
22059 })();
22060 log.end_compound();
22061 result
22062 }
22063
22064 fn evaluate_layer_logged(
22066 &mut self,
22067 layer: &super::scheduler::Layer,
22068 log: &mut ChangeLog,
22069 ) -> Result<usize, ExcelError> {
22070 self.resource_checkpoint(layer.vertices.len() as u64)?;
22071 self.evaluate_layer_units(layer, None, Some(log), None, true)
22072 }
22073}